[PATCH] libwine: Use the true maximum open FDs to set the limit on Mac.

[Chip Davis]

The actual upper limit is stored in the sysctl(3)
"kern.maxfilesperproc". This value may be higher than the value of the
OPEN_MAX constant.

Using sysconf(3) with _SC_OPEN_MAX just returns the current value of
RLIMIT_NOFILE, so that won't work here.

Signed-off-by: Chip Davis <cdavis5x at gmail.com>
---
 libs/wine/loader.c | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/libs/wine/loader.c b/libs/wine/loader.c
index ad72ec50904e..5999c9313fe1 100644
--- a/libs/wine/loader.c
+++ b/libs/wine/loader.c
@@ -32,6 +32,9 @@
 #ifdef HAVE_SYS_RESOURCE_H
 # include <sys/resource.h>
 #endif
+#ifdef HAVE_SYS_SYSCTL_H
+# include <sys/sysctl.h>
+#endif
 #include <unistd.h>
 #include <dlfcn.h>
 
@@ -800,13 +803,29 @@ static void set_max_limit( int limit )
         rlimit.rlim_cur = rlimit.rlim_max;
         if (setrlimit( limit, &rlimit ) != 0)
         {
-#if defined(__APPLE__) && defined(RLIMIT_NOFILE) && defined(OPEN_MAX)
-            /* On Leopard, setrlimit(RLIMIT_NOFILE, ...) fails on attempts to set
-             * rlim_cur above OPEN_MAX (even if rlim_max > OPEN_MAX). */
-            if (limit == RLIMIT_NOFILE && rlimit.rlim_cur > OPEN_MAX)
+#if defined(__APPLE__) && defined(RLIMIT_NOFILE) && (defined(OPEN_MAX) || defined(KERN_MAXFILESPERPROC))
+            if (limit == RLIMIT_NOFILE)
             {
-                rlimit.rlim_cur = OPEN_MAX;
-                setrlimit( limit, &rlimit );
+                /* On Leopard, setrlimit(RLIMIT_NOFILE, ...) fails on attempts to set
+                 * rlim_cur above OPEN_MAX (even if rlim_max > OPEN_MAX). */
+                int open_max = INT_MAX;
+#ifdef KERN_MAXFILESPERPROC
+                /* This is the true OPEN_MAX; it may be greater than the constant.
+                 * sysconf(_SC_OPEN_MAX) won't work here: it just returns getrlimit(RLIMIT_NOFILE). */
+                static int maxfilesperproc_oid[] = { CTL_KERN, KERN_MAXFILESPERPROC };
+                size_t len = sizeof(open_max);
+                if (sysctl(maxfilesperproc_oid, ARRAY_SIZE(maxfilesperproc_oid), &open_max, &len, NULL, 0) != 0)
+                    open_max = INT_MAX;
+#endif
+#ifdef OPEN_MAX
+                if (open_max == INT_MAX)
+                    open_max = OPEN_MAX;
+#endif
+                if (rlimit.rlim_cur > open_max)
+                {
+                    rlimit.rlim_cur = open_max;
+                    setrlimit( limit, &rlimit );
+                }
             }
 #endif
         }
-- 
2.34.1