ntdll: improve error handling in NtLockVirtualMemory (try 2)

[Austin English]

For https://bugs.winehq.org/show_bug.cgi?id=14275

-- 
-Austin
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://www.winehq.org/pipermail/wine-patches/attachments/20140411/13b3ed09/attachment.html>
-------------- next part --------------
diff --git a/dlls/ntdll/virtual.c b/dlls/ntdll/virtual.c
index e93ce2e..0e470dc 100644
--- a/dlls/ntdll/virtual.c
+++ b/dlls/ntdll/virtual.c
@@ -57,6 +57,7 @@
 
 WINE_DEFAULT_DEBUG_CHANNEL(virtual);
 WINE_DECLARE_DEBUG_CHANNEL(module);
+WINE_DECLARE_DEBUG_CHANNEL(winediag);
 
 #ifndef MS_SYNC
 #define MS_SYNC 0
@@ -2350,7 +2351,16 @@ NTSTATUS WINAPI NtLockVirtualMemory( HANDLE process, PVOID *addr, SIZE_T *size,
     *size = ROUND_SIZE( *addr, *size );
     *addr = ROUND_ADDR( *addr, page_mask );
 
-    if (mlock( *addr, *size )) status = STATUS_ACCESS_DENIED;
+    if (mlock( *addr, *size ))
+    {
+        if (errno == ENOMEM)
+        {
+            static int once;
+            if (!once++) ERR_(winediag)( "Cannot allocate memory, ulimit -l may need to be increased.\n");
+            status = STATUS_NO_MEMORY;
+        }
+        else status = STATUS_ACCESS_DENIED;
+    }
     return status;
 }