Kill a use of %n format specifier

This format has been used as a security exploit vector for decades
now.  Some operating systems (OpenBSD, Android, MSVC). It is targeted
for removal in C2X standard:
http://www.open-std.org/jtc1/sc22/wg14/www/docs/n2834.htm

This requires extending the debug message function to return the
number of bytes written (like printf(3)), to permit %n format
specifier in one in one invocation of statsPrintf() in
report_summary().

Implemented by Matthias Kilian (kili<AT>outback.escape.de)

rts/RtsMessages.c

@@ -36,7 +36,7 @@
 
 // Default to the stdio implementation of these hooks.
 RtsMsgFunction *fatalInternalErrorFn = rtsFatalInternalErrorFn;
-RtsMsgFunction *debugMsgFn           = rtsDebugMsgFn;
+RtsMsgFunctionRetLen *debugMsgFn           = rtsDebugMsgFn;
 RtsMsgFunction *errorMsgFn           = rtsErrorMsgFn;
 RtsMsgFunction *sysErrorMsgFn        = rtsSysErrorMsgFn;
 
@@ -102,10 +102,10 @@ debugBelch(const char*s, ...)
   va_end(ap);
 }
 
-void
+int
 vdebugBelch(const char*s, va_list ap)
 {
-  (*debugMsgFn)(s,ap);
+  return (*debugMsgFn)(s,ap);
 }
 
 /* -----------------------------------------------------------------------------
@@ -285,16 +285,16 @@ rtsSysErrorMsgFn(const char *s, va_list ap)
 #endif
 }
 
-void
+int
 rtsDebugMsgFn(const char *s, va_list ap)
 {
+  int r;
 #if defined(mingw32_HOST_OS)
   /* Ensure we're in text mode so newlines get encoded properly.  */
   int mode = _setmode (_fileno(stderr), _O_TEXT);
   if (isGUIApp())
   {
      char buf[BUFSIZE];
-         int r;
 
          r = vsnprintf(buf, BUFSIZE, s, ap);
          if (r > 0 && r < BUFSIZE) {
@@ -305,12 +305,13 @@ rtsDebugMsgFn(const char *s, va_list ap)
 #endif
   {
      /* don't fflush(stdout); WORKAROUND bug in Linux glibc */
-     vfprintf(stderr, s, ap);
+     r = vfprintf(stderr, s, ap);
      fflush(stderr);
   }
 #if defined(mingw32_HOST_OS)
   _setmode (_fileno(stderr), mode);
 #endif
+  return r;
 }
 
 

rts/Stats.c

@@ -69,7 +69,7 @@ static Time *GC_coll_cpu = NULL;
 static Time *GC_coll_elapsed = NULL;
 static Time *GC_coll_max_pause = NULL;
 
-static void statsPrintf( char *s, ... ) GNUC3_ATTRIBUTE(format (PRINTF, 1, 2));
+static int statsPrintf( char *s, ... ) GNUC3_ATTRIBUTE(format (PRINTF, 1, 2));
 static void statsFlush( void );
 static void statsClose( void );
 
@@ -995,8 +995,10 @@ static void report_summary(const RTSSummaryStats* sum)
 
         for (g = 0; g < RtsFlags.GcFlags.generations; g++) {
             int prefix_length = 0;
-            statsPrintf("%*s" "gen[%" FMT_Word32 "%n",
-                        col_width[0], "", g, &prefix_length);
+            prefix_length = statsPrintf("%*s" "gen[%" FMT_Word32,
+                        col_width[0], "", g);
+            if (prefix_length < 0)
+                prefix_length = 0;
             prefix_length -= col_width[0];
             int suffix_length = col_width[1] + prefix_length;
             suffix_length =
@@ -1716,19 +1718,21 @@ void getRTSStats( RTSStats *s )
    Dumping stuff in the stats file, or via the debug message interface
    -------------------------------------------------------------------------- */
 
-void
+int
 statsPrintf( char *s, ... )
 {
+    int ret = 0;
     FILE *sf = RtsFlags.GcFlags.statsFile;
     va_list ap;
 
     va_start(ap,s);
     if (sf == NULL) {
-        vdebugBelch(s,ap);
+        ret = vdebugBelch(s,ap);
     } else {
-        vfprintf(sf, s, ap);
+        ret = vfprintf(sf, s, ap);
     }
     va_end(ap);
+    return ret;
 }
 
 static void

rts/include/rts/Messages.h

@@ -85,20 +85,21 @@ void vsysErrorBelch(const char *s, va_list ap);
 void debugBelch(const char *s, ...)
    GNUC3_ATTRIBUTE(format (PRINTF, 1, 2));
 
-void vdebugBelch(const char *s, va_list ap);
+int vdebugBelch(const char *s, va_list ap);
 
 
 /* Hooks for redirecting message generation: */
 
 typedef void RtsMsgFunction(const char *, va_list);
+typedef int RtsMsgFunctionRetLen(const char *, va_list);
 
 extern RtsMsgFunction *fatalInternalErrorFn;
-extern RtsMsgFunction *debugMsgFn;
+extern RtsMsgFunctionRetLen *debugMsgFn;
 extern RtsMsgFunction *errorMsgFn;
 
 /* Default stdio implementation of the message hooks: */
 
 extern RtsMsgFunction rtsFatalInternalErrorFn;
-extern RtsMsgFunction rtsDebugMsgFn;
+extern RtsMsgFunctionRetLen rtsDebugMsgFn;
 extern RtsMsgFunction rtsErrorMsgFn;
 extern RtsMsgFunction rtsSysErrorMsgFn;