l3codeca.acm: Check input format in MPEG3_StreamOpen.

Signed-off-by: Vijay Kiran Kamuju <infyquest@gmail.com>
Signed-off-by: Andrew Eikum <aeikum@codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard@winehq.org>

dlls/l3codeca.acm/mpegl3.c

@@ -212,6 +212,7 @@ static void MPEG3_Reset(PACMDRVSTREAMINSTANCE adsi, AcmMpeg3Data* aad)
  */
 static	LRESULT	MPEG3_StreamOpen(PACMDRVSTREAMINSTANCE adsi)
 {
+    LRESULT error = MMSYSERR_NOTSUPPORTED;
     AcmMpeg3Data*	aad;
     int err;
 
@@ -235,6 +236,18 @@ static	LRESULT	MPEG3_StreamOpen(PACMDRVSTREAMINSTANCE adsi)
               adsi->pwfxSrc->wFormatTag == WAVE_FORMAT_MPEG) &&
              adsi->pwfxDst->wFormatTag == WAVE_FORMAT_PCM)
     {
+        if (adsi->pwfxSrc->wFormatTag == WAVE_FORMAT_MPEGLAYER3)
+        {
+            MPEGLAYER3WAVEFORMAT *formatmp3 = (MPEGLAYER3WAVEFORMAT *)adsi->pwfxSrc;
+
+            if (adsi->pwfxSrc->cbSize < MPEGLAYER3_WFX_EXTRA_BYTES ||
+                formatmp3->wID != MPEGLAYER3_ID_MPEG)
+            {
+                error = ACMERR_NOTPOSSIBLE;
+                goto theEnd;
+            }
+        }
+
 	/* resampling or mono <=> stereo not available
          * MPEG3 algo only define 16 bit per sample output
          */
@@ -270,7 +283,7 @@ static	LRESULT	MPEG3_StreamOpen(PACMDRVSTREAMINSTANCE adsi)
  theEnd:
     HeapFree(GetProcessHeap(), 0, aad);
     adsi->dwDriver = 0L;
-    return MMSYSERR_NOTSUPPORTED;
+    return error;
 }
 
 /***********************************************************************

dlls/msacm32/tests/msacm.c

@@ -1276,6 +1276,88 @@ static void test_acmFormatChoose(void)
     HeapFree(GetProcessHeap(), 0, pwfx);
 }
 
+static void test_mp3(void)
+{
+    MPEGLAYER3WAVEFORMAT src;
+    WAVEFORMATEX dst;
+    HACMSTREAM has;
+    DWORD output;
+    MMRESULT mr;
+
+    src.wfx.wFormatTag = WAVE_FORMAT_MPEGLAYER3;
+    src.wfx.nSamplesPerSec = 11025;
+    src.wfx.wBitsPerSample = 0;
+    src.wfx.nChannels = 1;
+    src.wfx.nBlockAlign = 576;
+    src.wfx.nAvgBytesPerSec = 2000;
+
+    src.wID = MPEGLAYER3_ID_MPEG;
+    src.fdwFlags = 0;
+    src.nBlockSize = 576;
+    src.nFramesPerBlock = 1;
+    src.nCodecDelay = 0;
+
+    dst.cbSize = 0;
+    dst.wFormatTag = WAVE_FORMAT_PCM;
+    dst.nSamplesPerSec = 11025;
+    dst.wBitsPerSample = 16;
+    dst.nChannels = 1;
+    dst.nBlockAlign = dst.wBitsPerSample * dst.nChannels / 8;
+    dst.nAvgBytesPerSec = dst.nSamplesPerSec * dst.nBlockAlign;
+
+    src.wfx.cbSize = 0;
+
+    mr = acmStreamOpen(&has, NULL, (WAVEFORMATEX*)&src, &dst, NULL, 0, 0, 0);
+    ok(mr == ACMERR_NOTPOSSIBLE, "expected error ACMERR_NOTPOSSIBLE, got 0x%x\n", mr);
+    if (mr == MMSYSERR_NOERROR) acmStreamClose(has, 0);
+
+    src.wfx.cbSize = MPEGLAYER3_WFX_EXTRA_BYTES;
+    src.wID = 0;
+
+    mr = acmStreamOpen(&has, NULL, (WAVEFORMATEX*)&src, &dst, NULL, 0, 0, 0);
+    ok(mr == ACMERR_NOTPOSSIBLE, "expected error ACMERR_NOTPOSSIBLE, got 0x%x\n", mr);
+    if (mr == MMSYSERR_NOERROR) acmStreamClose(has, 0);
+
+    src.wID = MPEGLAYER3_ID_MPEG;
+    src.nBlockSize = 0;
+
+    mr = acmStreamOpen(&has, NULL, (WAVEFORMATEX*)&src, &dst, NULL, 0, 0, 0);
+    ok(mr == MMSYSERR_NOERROR || broken(mr == ACMERR_NOTPOSSIBLE) /* Win 2008 */,
+       "failed with error 0x%x\n", mr);
+    if (mr == MMSYSERR_NOERROR)
+    {
+       mr = acmStreamClose(has, 0);
+       ok(mr == MMSYSERR_NOERROR, "failed with error 0x%x\n", mr);
+    }
+    src.nBlockSize = 576;
+    src.wfx.nAvgBytesPerSec = 0;
+
+    mr = acmStreamOpen(&has, NULL, (WAVEFORMATEX*)&src, &dst, NULL, 0, 0, 0);
+    ok(mr == MMSYSERR_NOERROR || broken(mr == ACMERR_NOTPOSSIBLE) /* Win 2008 */,
+       "failed with error 0x%x\n", mr);
+    if (mr == MMSYSERR_NOERROR)
+    {
+       /* causes a division by zero exception in XP, Vista,
+          but throws ACMERR_NOTPOSSIBLE on others */
+       if (0) acmStreamSize(has, 4000, &output, ACM_STREAMSIZEF_SOURCE);
+       mr = acmStreamClose(has, 0);
+       ok(mr == MMSYSERR_NOERROR, "failed with error 0x%x\n", mr);
+    }
+
+    src.wfx.nAvgBytesPerSec = 2000;
+
+    mr = acmStreamOpen(&has, NULL, (WAVEFORMATEX*)&src, &dst, NULL, 0, 0, 0);
+    ok(mr == MMSYSERR_NOERROR || broken(mr == ACMERR_NOTPOSSIBLE) /* Win 2008 */,
+       "failed with error 0x%x\n", mr);
+    if (mr == MMSYSERR_NOERROR)
+    {
+       mr = acmStreamSize(has, 4000, &output, ACM_STREAMSIZEF_SOURCE);
+       ok(mr == MMSYSERR_NOERROR, "failed with error 0x%x\n", mr);
+       mr = acmStreamClose(has, 0);
+       ok(mr == MMSYSERR_NOERROR, "failed with error 0x%x\n", mr);
+    }
+}
+
 static struct
 {
     struct
@@ -1446,6 +1528,7 @@ START_TEST(msacm)
     test_acmFormatSuggest();
     test_acmFormatTagDetails();
     test_acmFormatChoose();
+    test_mp3();
     /* Test acmDriverAdd in the end as it may conflict
      * with other tests due to codec lookup order */
     test_acmDriverAdd();