Fabian Maurer : winmm: Don' t crash in waveOutOpen when nSamplesPerSec is 0 and add tests.

[Alexandre Julliard]

Module: wine
Branch: stable
Commit: f7d1fa464cd2a99cac83dea093991336517efd3a
URL:    https://source.winehq.org/git/wine.git/?a=commit;h=f7d1fa464cd2a99cac83dea093991336517efd3a

Author: Fabian Maurer <dark.shadow4 at web.de>
Date:   Fri Aug  3 07:55:03 2018 -0500

winmm: Don't crash in waveOutOpen when nSamplesPerSec is 0 and add tests.

Wine-Bug: https://bugs.winehq.org/show_bug.cgi?id=45530
Signed-off-by: Fabian Maurer <dark.shadow4 at web.de>
Signed-off-by: Andrew Eikum <aeikum at codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard at winehq.org>
(cherry picked from commit 7000af554740db447aa891efc25644f4be825299)
Signed-off-by: Michael Stefaniuc <mstefani at winehq.org>

---

 dlls/winmm/tests/wave.c | 22 ++++++++++++++++++++++
 dlls/winmm/waveform.c   |  7 +++++++
 2 files changed, 29 insertions(+)

diff --git a/dlls/winmm/tests/wave.c b/dlls/winmm/tests/wave.c
index 284b624..2d52891 100644
--- a/dlls/winmm/tests/wave.c
+++ b/dlls/winmm/tests/wave.c
@@ -1412,6 +1412,28 @@ static void wave_out_test_device(UINT_PTR device)
     } else
         trace("waveOutOpen(%s): 32 bit float samples not supported\n",
               dev_name(device));
+
+    /* Test invalid parameters */
+
+    format.wFormatTag = WAVE_FORMAT_PCM;
+    format.nChannels = 1;
+    format.nSamplesPerSec = 11025;
+    format.nBlockAlign = 1;
+    format.nAvgBytesPerSec = 11025 * 1;
+    format.wBitsPerSample = 8;
+    format.cbSize = 0;
+
+    format.nAvgBytesPerSec = 0;
+    rc = waveOutOpen(&wout, device, &format, 0, 0, 0);
+    ok(rc == MMSYSERR_NOERROR,
+       "waveOutOpen(%s): returned %s\n",dev_name(device),wave_out_error(rc));
+    waveOutClose(wout);
+    format.nAvgBytesPerSec = 11025 * 1;
+
+    format.nSamplesPerSec = 0;
+    rc = waveOutOpen(&wout, device, &format, 0, 0, 0);
+    ok(rc == MMSYSERR_INVALPARAM || rc == WAVERR_BADFORMAT, /* XP and lower return WAVERR_BADFORMAT */
+       "waveOutOpen(%s): returned %s\n",dev_name(device),wave_out_error(rc));
 }
 
 static void wave_out_tests(void)
diff --git a/dlls/winmm/waveform.c b/dlls/winmm/waveform.c
index 045bf4a..0a259c0 100644
--- a/dlls/winmm/waveform.c
+++ b/dlls/winmm/waveform.c
@@ -1088,6 +1088,13 @@ static LRESULT WINMM_OpenDevice(WINMM_Device *device, WINMM_OpenInfo *info,
     }
 
     if(info->format->wFormatTag == WAVE_FORMAT_PCM){
+
+        if (info->format->nSamplesPerSec == 0)
+        {
+            ret = MMSYSERR_INVALPARAM;
+            goto error;
+        }
+
         /* we aren't guaranteed that the struct in lpFormat is a full
          * WAVEFORMATEX struct, which IAC::IsFormatSupported requires */
         device->orig_fmt = HeapAlloc(GetProcessHeap(), 0, sizeof(WAVEFORMATEX));