[PATCH v12 2/2] ntdll: Workaround NtAllocateVirtualMemory with zero_bits set to 1

[Rémi Bernon]

Implement the correct zero_bits behavior for this single case:
* Limit the search in reserved areas to the lower 2G range,
* Pass the MAP_32BIT flag to mmap as a fallback.

LuaJIT <= v2.0.5 for example, when running in 64bit, allocates its
memory in the lower 2GB memory region by using the zero_bits parameter.

This will fix this particular scenario, while trying to minimize the
changes on all the other cases.

Signed-off-by: Rémi Bernon <rbernon at codeweavers.com>
---
 dlls/ntdll/virtual.c | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/dlls/ntdll/virtual.c b/dlls/ntdll/virtual.c
index 7a6e461ffaf..3f9ec0e5658 100644
--- a/dlls/ntdll/virtual.c
+++ b/dlls/ntdll/virtual.c
@@ -1100,14 +1100,34 @@ static NTSTATUS map_view( struct file_view **view_ret, void *base, size_t size,
     {
         size_t view_size = size + mask + 1;
         struct alloc_area alloc;
+        int flags = 0;
 
+#if defined(__x86_64__) && !defined(MAP_32BIT)
         if (zero_bits)
+#else
+        if (zero_bits > 1)
+#endif
+        {
             FIXME("Unimplemented zero_bits parameter value\n");
+        }
 
         alloc.size = size;
         alloc.mask = mask;
         alloc.top_down = top_down;
         alloc.limit = user_space_limit;
+
+#if defined(__x86_64__) && defined(MAP_32BIT)
+        /* HACK: only works for zero_bits == 1, this is a simple workaround
+         * for some 64bit code that tries to allocate memory in the lower
+         * 2GB segment using zero_bits parameter.
+         */
+        if (zero_bits == 1)
+        {
+            alloc.limit = (void*)(((~(UINT_PTR)0) >> (32 + zero_bits)) & ~0xffff);
+            flags = MAP_32BIT;
+        }
+#endif
+
         if (wine_mmap_enum_reserved_areas( alloc_reserved_area_callback, &alloc, top_down ))
         {
             ptr = alloc.result;
@@ -1119,7 +1139,7 @@ static NTSTATUS map_view( struct file_view **view_ret, void *base, size_t size,
 
         for (;;)
         {
-            if ((ptr = wine_anon_mmap( NULL, view_size, VIRTUAL_GetUnixProt(vprot), 0 )) == (void *)-1)
+            if ((ptr = wine_anon_mmap( NULL, view_size, VIRTUAL_GetUnixProt(vprot), flags )) == (void *)-1)
             {
                 if (errno == ENOMEM) return STATUS_NO_MEMORY;
                 return STATUS_INVALID_PARAMETER;
-- 
2.20.1