[PATCH v3 10/10] ntdll: Workaround NtAllocateVirtualMemory with zero_bits set to 1

[Rémi Bernon]

Implement the correct zero_bits behavior for this single case:
* Limit the search in reserved areas to the lower 2G range,
* Pass the MAP_32BIT flag to mmap as a fallback.

LuaJIT <= v2.0.5 for example, when running in 64bit, allocates its
memory in the lower 2GB memory region by using the zero_bits parameter.

This will fix this particular scenario, while trying to minimize the
changes on all the other cases.

Signed-off-by: Rémi Bernon <rbernon at codeweavers.com>
---
 dlls/ntdll/tests/virtual.c |  2 --
 dlls/ntdll/virtual.c       | 22 +++++++++++++++++++++-
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/dlls/ntdll/tests/virtual.c b/dlls/ntdll/tests/virtual.c
index c892e4aa4e0..0b80d840f52 100644
--- a/dlls/ntdll/tests/virtual.c
+++ b/dlls/ntdll/tests/virtual.c
@@ -120,7 +120,6 @@ static void test_AllocateVirtualMemory(void)
        "NtAllocateVirtualMemory returned %08x\n", status);
     if (status == STATUS_SUCCESS)
     {
-        todo_wine_if(is_win64)
         ok(((UINT_PTR)addr2 >> (32 - zero_bits)) == 0,
            "NtAllocateVirtualMemory returned address: %p\n", addr2);
 
@@ -266,7 +265,6 @@ static void test_MapViewOfSection(void)
        "NtMapViewOfSection returned %08x\n", status);
     if (status == STATUS_SUCCESS)
     {
-        todo_wine_if(is_win64)
         ok(((UINT_PTR)ptr2 >> (32 - zero_bits)) == 0,
            "NtMapViewOfSection returned address: %p\n", ptr2);
 
diff --git a/dlls/ntdll/virtual.c b/dlls/ntdll/virtual.c
index 88ea958ccf5..7af1a1cff60 100644
--- a/dlls/ntdll/virtual.c
+++ b/dlls/ntdll/virtual.c
@@ -1101,14 +1101,34 @@ static NTSTATUS map_view_aligned( struct file_view **view_ret, void *base, size_
     {
         size_t view_size = size + mask + 1;
         struct alloc_area alloc;
+        int flags = 0;
 
+#if defined(__x86_64__) && !defined(MAP_32BIT)
         if (zero_bits)
+#else
+        if (zero_bits > 1)
+#endif
+        {
             FIXME("Unimplemented zero_bits parameter value\n");
+        }
 
         alloc.size = size;
         alloc.mask = mask;
         alloc.top_down = top_down;
         alloc.limit = user_space_limit;
+
+#if defined(__x86_64__) && defined(MAP_32BIT)
+        /* HACK: only works for zero_bits == 1, this is a simple workaround
+         * for some 64bit code that tries to allocate memory in the lower
+         * 2GB segment using zero_bits parameter.
+         */
+        if (zero_bits == 1)
+        {
+            alloc.limit = (void*)(((~(UINT_PTR)0) >> (32 + zero_bits)) & ~0xffff);
+            flags = MAP_32BIT;
+        }
+#endif
+
         if (wine_mmap_enum_reserved_areas( alloc_reserved_area_callback, &alloc, top_down ))
         {
             ptr = alloc.result;
@@ -1120,7 +1140,7 @@ static NTSTATUS map_view_aligned( struct file_view **view_ret, void *base, size_
 
         for (;;)
         {
-            if ((ptr = wine_anon_mmap( NULL, view_size, VIRTUAL_GetUnixProt(vprot), 0 )) == (void *)-1)
+            if ((ptr = wine_anon_mmap( NULL, view_size, VIRTUAL_GetUnixProt(vprot), flags )) == (void *)-1)
             {
                 if (errno == ENOMEM) return STATUS_NO_MEMORY;
                 return STATUS_INVALID_PARAMETER;
-- 
2.20.1