[05/10] d3d: Fix the surface locking rectangle validation

[H. Verbeet]

Validation on the rectangle passed to LockRect/Lock is inconsistent
across ddraw/d3d8/d3d9. D3d9 doesn't do any validation at all, d3d8
doesn't allow empty, flipped, mirrored, or out of bounds rectangles,
while ddraw does allow empty rectangles, but not flipped, mirrored or
out of bounds ones. Tests for this are in the next few patches in this
set.

The original issue that prompted this patch (d3d9 allows empty
rectangles to be passed to LockRect) was found by Rudi Mayerhofer in
Bug 7970. This patch should also fix that bug.

Changelog:
  - Fix the surface locking rectangle validation
-------------- next part --------------
---

 dlls/d3d8/surface.c        |   16 ++++++++++++++++
 dlls/ddraw/surface.c       |   14 ++++++++++++++
 dlls/wined3d/surface.c     |   11 -----------
 dlls/wined3d/surface_gdi.c |   11 -----------
 4 files changed, 30 insertions(+), 22 deletions(-)

diff --git a/dlls/d3d8/surface.c b/dlls/d3d8/surface.c
index 93ea2f5..4d2f9b4 100644
--- a/dlls/d3d8/surface.c
+++ b/dlls/d3d8/surface.c
@@ -147,6 +147,22 @@ static HRESULT WINAPI IDirect3DSurface8Impl_LockRect(LPDIRECT3DSURFACE8 iface, D
     IDirect3DSurface8Impl *This = (IDirect3DSurface8Impl *)iface;
     TRACE("(%p) Relay\n", This);
     TRACE("(%p) calling IWineD3DSurface_LockRect %p %p %p %d\n", This, This->wineD3DSurface, pLockedRect, pRect, Flags);
+
+    if (pRect) {
+        D3DSURFACE_DESC desc;
+        IDirect3DSurface8_GetDesc(iface, &desc);
+
+        if ((pRect->left < 0)
+                || (pRect->top < 0)
+                || (pRect->left >= pRect->right)
+                || (pRect->top >= pRect->bottom)
+                || (pRect->right > desc.Width)
+                || (pRect->bottom > desc.Height)) {
+            WARN("Trying to lock an invalid rectangle, returning D3DERR_INVALIDCALL\n");
+            return D3DERR_INVALIDCALL;
+        }
+    }
+
     return IWineD3DSurface_LockRect(This->wineD3DSurface, (WINED3DLOCKED_RECT *) pLockedRect, pRect, Flags);
 }
 
diff --git a/dlls/ddraw/surface.c b/dlls/ddraw/surface.c
index 00b2820..e99c2b8 100644
--- a/dlls/ddraw/surface.c
+++ b/dlls/ddraw/surface.c
@@ -549,6 +549,20 @@ IDirectDrawSurfaceImpl_Lock(IDirectDrawSurface7 *iface,
     HRESULT hr;
     TRACE("(%p)->(%p,%p,%x,%p)\n", This, Rect, DDSD, Flags, h);
 
+    if (Rect)
+    {
+        if ((Rect->left < 0)
+                || (Rect->top < 0)
+                || (Rect->left > Rect->right)
+                || (Rect->top > Rect->bottom)
+                || (Rect->right > This->surface_desc.dwWidth)
+                || (Rect->bottom > This->surface_desc.dwHeight))
+        {
+            WARN("Trying to lock an invalid rectangle, returning DDERR_INVALIDPARAMS\n");
+            return DDERR_INVALIDPARAMS;
+        }
+    }
+
     if(!DDSD)
         return DDERR_INVALIDPARAMS;
 
diff --git a/dlls/wined3d/surface.c b/dlls/wined3d/surface.c
index 909ee84..65a14d3 100644
--- a/dlls/wined3d/surface.c
+++ b/dlls/wined3d/surface.c
@@ -720,17 +720,6 @@ static HRESULT WINAPI IWineD3DSurfaceImpl_LockRect(IWineD3DSurface *iface, WINED
     } else {
         TRACE("Lock Rect (%p) = l %d, t %d, r %d, b %d\n", pRect, pRect->left, pRect->top, pRect->right, pRect->bottom);
 
-        if ((pRect->top < 0) ||
-             (pRect->left < 0) ||
-             (pRect->left >= pRect->right) ||
-             (pRect->top >= pRect->bottom) ||
-             (pRect->right > This->currentDesc.Width) ||
-             (pRect->bottom > This->currentDesc.Height))
-        {
-            WARN(" Invalid values in pRect !!!\n");
-            return WINED3DERR_INVALIDCALL;
-        }
-
         /* DXTn textures are based on compressed blocks of 4x4 pixels, each
          * 16 bytes large (8 bytes in case of DXT1). Because of that Pitch has
          * slightly different meaning compared to regular textures. For DXTn
diff --git a/dlls/wined3d/surface_gdi.c b/dlls/wined3d/surface_gdi.c
index 4731608..ce37680 100644
--- a/dlls/wined3d/surface_gdi.c
+++ b/dlls/wined3d/surface_gdi.c
@@ -209,17 +209,6 @@ IWineGDISurfaceImpl_LockRect(IWineD3DSurface *iface,
         TRACE("Lock Rect (%p) = l %d, t %d, r %d, b %d\n",
               pRect, pRect->left, pRect->top, pRect->right, pRect->bottom);
 
-        if ((pRect->top < 0) ||
-             (pRect->left < 0) ||
-             (pRect->left >= pRect->right) ||
-             (pRect->top >= pRect->bottom) ||
-             (pRect->right > This->currentDesc.Width) ||
-             (pRect->bottom > This->currentDesc.Height))
-        {
-            WARN(" Invalid values in pRect !!!\n");
-            return WINED3DERR_INVALIDCALL;
-        }
-
         if (This->resource.format == WINED3DFMT_DXT1)
         {
             /* DXT1 is half byte per pixel */