Nikolay Sivov : msvcrt: Handle overflow in calloc().

[Alexandre Julliard]

Module: wine
Branch: master
Commit: 50dd4b892825c75db35cd1f378291b51fa782f3e
URL:    http://source.winehq.org/git/wine.git/?a=commit;h=50dd4b892825c75db35cd1f378291b51fa782f3e

Author: Nikolay Sivov <nsivov at codeweavers.com>
Date:   Wed Apr 27 11:11:49 2016 +0300

msvcrt: Handle overflow in calloc().

Signed-off-by: Nikolay Sivov <nsivov at codeweavers.com>
Signed-off-by: Piotr Caban <piotr at codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard at winehq.org>

---

 dlls/msvcrt/heap.c       | 10 +++++++++-
 dlls/msvcrt/tests/heap.c | 23 +++++++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/dlls/msvcrt/heap.c b/dlls/msvcrt/heap.c
index 52a5287..3fe0227 100644
--- a/dlls/msvcrt/heap.c
+++ b/dlls/msvcrt/heap.c
@@ -394,7 +394,15 @@ size_t CDECL _aligned_msize(void *p, MSVCRT_size_t alignment, MSVCRT_size_t offs
  */
 void* CDECL MSVCRT_calloc(MSVCRT_size_t count, MSVCRT_size_t size)
 {
-  return msvcrt_heap_alloc(HEAP_ZERO_MEMORY, count*size);
+  MSVCRT_size_t bytes = count*size;
+
+  if (size && bytes / size != count)
+  {
+    *MSVCRT__errno() = MSVCRT_ENOMEM;
+    return NULL;
+  }
+
+  return msvcrt_heap_alloc(HEAP_ZERO_MEMORY, bytes);
 }
 
 /*********************************************************************
diff --git a/dlls/msvcrt/tests/heap.c b/dlls/msvcrt/tests/heap.c
index 2aaf48d..82a5961 100644
--- a/dlls/msvcrt/tests/heap.c
+++ b/dlls/msvcrt/tests/heap.c
@@ -456,6 +456,28 @@ static void test_sbheap(void)
     free(mem);
 }
 
+static void test_calloc(void)
+{
+    void *ptr;
+
+    ptr = calloc(1, 0);
+    ok(ptr != NULL, "got %p\n", ptr);
+    free(ptr);
+
+    ptr = calloc(0, 0);
+    ok(ptr != NULL, "got %p\n", ptr);
+    free(ptr);
+
+    ptr = calloc(0, 1);
+    ok(ptr != NULL, "got %p\n", ptr);
+    free(ptr);
+
+    errno = 0;
+    ptr = calloc(~(size_t)0 / 2, ~(size_t)0 / 2);
+    ok(ptr == NULL, "got %p\n", ptr);
+    ok(errno == ENOMEM || broken(errno == 0) /* winxp, win2k3 */, "got errno %d\n", errno);
+}
+
 START_TEST(heap)
 {
     void *mem;
@@ -480,4 +502,5 @@ START_TEST(heap)
 
     test_aligned();
     test_sbheap();
+    test_calloc();
 }