[PATCH] winemac: Cope with multiple seemingly-identical display modes, only some of which work, by trying them in sequence.

[Ken Thomases]

OS X 10.11 (El Capitan) has strange behavior with respect to display modes on some
hardware.  First, it lies when describing display modes.  The CGDisplayModeCopyPixelEncoding()
function claims modes are X8R8G8B8 when they are actually X2R10G10B10.  Since there are
also modes which are actually X8R8G8B8, this results in multiple modes which are
indistinguishable from each other in terms of how the API describes them while being
different in fact.

Second, on the affected hardware, attempts to change the mode to one of the X8R8G8B8
modes fails with no meaningful error code or explanation.  Only the X2R10G10B10 modes can
actually be used, although there's no way to tell which is which without trying them.

Which leads to the workaround: when trying to change the display mode, try all of the
modes which seem to match the desired mode until one works.

Signed-off-by: Ken Thomases <ken at codeweavers.com>
---
 dlls/winemac.drv/cocoa_app.m | 38 +++++++++++++++++++++++++-------------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/dlls/winemac.drv/cocoa_app.m b/dlls/winemac.drv/cocoa_app.m
index 7f6ca23..c0581b8 100644
--- a/dlls/winemac.drv/cocoa_app.m
+++ b/dlls/winemac.drv/cocoa_app.m
@@ -699,9 +699,9 @@ - (BOOL) mode:(CGDisplayModeRef)mode1 matchesMode:(CGDisplayModeRef)mode2
         return TRUE;
     }
 
-    - (CGDisplayModeRef)modeMatchingMode:(CGDisplayModeRef)mode forDisplay:(CGDirectDisplayID)displayID
+    - (NSArray*)modesMatchingMode:(CGDisplayModeRef)mode forDisplay:(CGDirectDisplayID)displayID
     {
-        CGDisplayModeRef ret = NULL;
+        NSMutableArray* ret = [NSMutableArray array];
         NSDictionary* options = nil;
 
 #if defined(MAC_OS_X_VERSION_10_8) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_8
@@ -715,10 +715,7 @@ - (CGDisplayModeRef)modeMatchingMode:(CGDisplayModeRef)mode forDisplay:(CGDirect
         {
             CGDisplayModeRef candidateMode = (CGDisplayModeRef)candidateModeObject;
             if ([self mode:candidateMode matchesMode:mode])
-            {
-                ret = candidateMode;
-                break;
-            }
+                [ret addObject:candidateModeObject];
         }
         return ret;
     }
@@ -743,11 +740,15 @@ - (BOOL) setMode:(CGDisplayModeRef)mode forDisplay:(CGDirectDisplayID)displayID
             }
             else // ... otherwise, try to restore just the one display
             {
-                mode = [self modeMatchingMode:mode forDisplay:displayID];
-                if (mode && CGDisplaySetDisplayMode(displayID, mode, NULL) == CGDisplayNoErr)
+                for (id modeObject in [self modesMatchingMode:mode forDisplay:displayID])
                 {
-                    [originalDisplayModes removeObjectForKey:displayIDKey];
-                    ret = TRUE;
+                    mode = (CGDisplayModeRef)modeObject;
+                    if (CGDisplaySetDisplayMode(displayID, mode, NULL) == CGDisplayNoErr)
+                    {
+                        [originalDisplayModes removeObjectForKey:displayIDKey];
+                        ret = TRUE;
+                        break;
+                    }
                 }
             }
         }
@@ -755,6 +756,7 @@ - (BOOL) setMode:(CGDisplayModeRef)mode forDisplay:(CGDirectDisplayID)displayID
         {
             BOOL active = [NSApp isActive];
             CGDisplayModeRef currentMode;
+            NSArray* modes;
 
             currentMode = CGDisplayModeRetain((CGDisplayModeRef)[latentDisplayModes objectForKey:displayIDKey]);
             if (!currentMode)
@@ -771,8 +773,8 @@ - (BOOL) setMode:(CGDisplayModeRef)mode forDisplay:(CGDirectDisplayID)displayID
             CGDisplayModeRelease(currentMode);
             currentMode = NULL;
 
-            mode = [self modeMatchingMode:mode forDisplay:displayID];
-            if (!mode)
+            modes = [self modesMatchingMode:mode forDisplay:displayID];
+            if (!modes.count)
                 return FALSE;
 
             if ([originalDisplayModes count] || displaysCapturedForFullscreen ||
@@ -789,7 +791,17 @@ - (BOOL) setMode:(CGDisplayModeRef)mode forDisplay:(CGDirectDisplayID)displayID
                         originalMode = currentMode = CGDisplayCopyDisplayMode(displayID);
 
                     if (originalMode)
-                        ret = (CGDisplaySetDisplayMode(displayID, mode, NULL) == CGDisplayNoErr);
+                    {
+                        for (id modeObject in modes)
+                        {
+                            mode = (CGDisplayModeRef)modeObject;
+                            if (CGDisplaySetDisplayMode(displayID, mode, NULL) == CGDisplayNoErr)
+                            {
+                                ret = TRUE;
+                                break;
+                            }
+                        }
+                    }
                     if (ret && !(currentMode && [self mode:mode matchesMode:currentMode]))
                         [originalDisplayModes setObject:(id)originalMode forKey:displayIDKey];
                     else if (![originalDisplayModes count])
-- 
2.6.0