msvcrt: [2/3] estimate required buffer size better in pf_vsnprintf

[Jesse Allen]

We do not check whether the precision specified would require a larger buffer
for conversion than we normally provide. Also, sign output, alternate formats,
etc is not always accounted for either. There is no easy way to check whether
a sign byte is required when the field length is not big enough. Exponents
for floating point numbers is difficult to check too as the conversion is
currently provided by libc. Alternate formats are just outputted during
conversion. So rather than have lots of extra checking to save a few bytes,
I included an extra 10 bytes in a size estimation. The extra 10 bytes is for
a worst case scenario.

pf_integer_conv is updated too to create a larger temporary buffer when a
larger one is required.

Comments to document the estimation rational and to clear up the conversion
process.

Test cases follow that show buffer overruns without this patch. Windows allows
very large conversions like this too without crashing.
-------------- next part --------------
diff --git a/dlls/msvcrt/wcs.c b/dlls/msvcrt/wcs.c
index 591f28f..800d3b5 100644
--- a/dlls/msvcrt/wcs.c
+++ b/dlls/msvcrt/wcs.c
@@ -281,6 +281,7 @@ static inline int pf_output_stringA( pf_
     return -1;
 }
 
+/* pf_fill: takes care of signs, alignment, zero and field padding */
 static inline int pf_fill( pf_output *out, int len, pf_flags *flags, char left )
 {
     int i, r = 0;
@@ -426,15 +427,19 @@ static void pf_rebuild_format_string( ch
     *p++ = 0;
 }
 
-/* pf_integer_conv:  prints x to buf, including alternate formats,
-   but not the sign */
-static void pf_integer_conv( char *buf, pf_flags *flags, LONGLONG x )
+/* pf_integer_conv:  prints x to buf, including alternate formats and
+   additional precision digits, but not field characters or the sign */
+static void pf_integer_conv( char *buf, int buf_len, pf_flags *flags,
+                             LONGLONG x )
 {
     unsigned int base;
     const char *digits;
 
     int i, j, k;
-    char tmp[40];
+    char number[40], *tmp = number;
+
+    if( buf_len > sizeof number )
+        tmp = HeapAlloc( GetProcessHeap(), 0, buf_len );
 
     base = 10;
     if( flags->Format == 'o' )
@@ -487,6 +492,9 @@ static void pf_integer_conv( char *buf, 
     /* Adjust precision so pf_fill won't truncate the number later */
     flags->Precision = strlen( buf );
 
+    if( tmp != number )
+        HeapFree( GetProcessHeap(), 0, tmp );
+
     return;
 }
 
@@ -660,10 +668,18 @@ static int pf_vsnprintf( pf_output *out,
         {
             char number[40], *x = number;
 
-            if( flags.FieldLength >= sizeof number )
-                x = HeapAlloc( GetProcessHeap(), 0, flags.FieldLength+1 );
+            /* Estimate largest possible required buffer size:
+               * Chooses the larger of the field or precision
+               * Includes extra bytes: 1 byte for null, 1 byte for sign,
+                 4 bytes for exponent, 2 bytes for alternate formats, 1 byte 
+                 for a decimal, and 1 byte for an additional float digit. */
+            int x_len = ((flags.FieldLength > flags.Precision) ? 
+                        flags.FieldLength : flags.Precision) + 10;
 
-            pf_integer_conv( x, &flags, va_arg(valist, LONGLONG) );
+            if( x_len >= sizeof number)
+                x = HeapAlloc( GetProcessHeap(), 0, x_len );
+
+            pf_integer_conv( x, x_len, &flags, va_arg(valist, LONGLONG) );
 
             r = pf_output_format_A( out, x, -1, &flags );
             if( x != number )
@@ -675,8 +691,16 @@ static int pf_vsnprintf( pf_output *out,
         {
             char fmt[20], number[40], *x = number;
 
-            if( flags.FieldLength >= sizeof number )
-                x = HeapAlloc( GetProcessHeap(), 0, flags.FieldLength+1 );
+            /* Estimate largest possible required buffer size:
+               * Chooses the larger of the field or precision
+               * Includes extra bytes: 1 byte for null, 1 byte for sign,
+                 4 bytes for exponent, 2 bytes for alternate formats, 1 byte 
+                 for a decimal, and 1 byte for an additional float digit. */
+            int x_len = ((flags.FieldLength > flags.Precision) ? 
+                        flags.FieldLength : flags.Precision) + 10;
+
+            if( x_len >= sizeof number)
+                x = HeapAlloc( GetProcessHeap(), 0, x_len );
 
             pf_rebuild_format_string( fmt, &flags );
 
-- 
1.4.2.1