Cryp{En|De}codeObjectEx should return success if the input buffer is

NULL, some MS sample programs depend on that.  Fix functions and
tests.

dlls/crypt32/encode.c

@@ -317,17 +317,17 @@ BOOL WINAPI CryptEncodeObject(DWORD dwCertEncodingType, LPCSTR lpszStructType,
     return ret;
 }
 
+/* Helper function to check *pcbEncoded, set it to the required size, and
+ * optionally to allocate memory.  Assumes pbEncoded is not NULL.
+ * If CRYPT_ENCODE_ALLOC_FLAG is set in dwFlags, *pbEncoded will be set to a
+ * pointer to the newly allocated memory.
+ */
 static BOOL CRYPT_EncodeEnsureSpace(DWORD dwFlags,
  PCRYPT_ENCODE_PARA pEncodePara, BYTE *pbEncoded, DWORD *pcbEncoded,
  DWORD bytesNeeded)
 {
-    if (!pbEncoded || (!(dwFlags & CRYPT_ENCODE_ALLOC_FLAG) &&
+    BOOL ret = TRUE;
-     bytesNeeded > *pcbEncoded))
+
-    {
-        *pcbEncoded = bytesNeeded;
-        SetLastError(ERROR_MORE_DATA);
-        return FALSE;
-    }
     if (dwFlags & CRYPT_ENCODE_ALLOC_FLAG)
     {
         if (pEncodePara && pEncodePara->pfnAlloc)
@@ -335,9 +335,15 @@ static BOOL CRYPT_EncodeEnsureSpace(DWORD dwFlags,
         else
             *(BYTE **)pbEncoded = LocalAlloc(0, bytesNeeded);
         if (!*(BYTE **)pbEncoded)
-            return FALSE;
+            ret = FALSE;
     }
-    return TRUE;
+    else if (bytesNeeded > *pcbEncoded)
+    {
+        *pcbEncoded = bytesNeeded;
+        SetLastError(ERROR_MORE_DATA);
+        ret = FALSE;
+    }
+    return ret;
 }
 
 static BOOL WINAPI CRYPT_AsnEncodeInt(DWORD dwCertEncodingType,
@@ -383,6 +389,11 @@ static BOOL WINAPI CRYPT_AsnEncodeInt(DWORD dwCertEncodingType,
     bytesNeeded = 2 + significantBytes;
     if (pad)
         bytesNeeded++;
+    if (!pbEncoded)
+    {
+        *pcbEncoded = bytesNeeded;
+        return TRUE;
+    }
     if (!CRYPT_EncodeEnsureSpace(dwFlags, pEncodePara, pbEncoded, pcbEncoded,
      bytesNeeded))
         return FALSE;
@@ -425,6 +436,11 @@ static BOOL WINAPI CRYPT_AsnEncodeUtcTime(DWORD dwCertEncodingType,
         SetLastError(CRYPT_E_BAD_ENCODE);
         return FALSE;
     }
+    if (!pbEncoded)
+    {
+        *pcbEncoded = bytesNeeded;
+        return TRUE;
+    }
     if (!CRYPT_EncodeEnsureSpace(dwFlags, pEncodePara, pbEncoded, pcbEncoded,
      bytesNeeded))
         return FALSE;
@@ -440,7 +456,7 @@ static BOOL WINAPI CRYPT_AsnEncodeUtcTime(DWORD dwCertEncodingType,
     return TRUE;
 }
 
-static BOOL WINAPI CRYPT_AsnEncodeChoiceOfTime(DWORD dwCertEncodingType,
+static BOOL WINAPI CRYPT_AsnEncodeGeneralizedTime(DWORD dwCertEncodingType,
  LPCSTR lpszStructType, const void *pvStructInfo, DWORD dwFlags,
  PCRYPT_ENCODE_PARA pEncodePara, BYTE *pbEncoded, DWORD *pcbEncoded)
 {
@@ -456,12 +472,13 @@ static BOOL WINAPI CRYPT_AsnEncodeChoiceOfTime(DWORD dwCertEncodingType,
         SetLastError(ERROR_INVALID_PARAMETER);
         return FALSE;
     }
-    /* Check the year, if it's in the UTCTime range call that encode func */
+    if (!pbEncoded)
+    {
+        *pcbEncoded = bytesNeeded;
+        return TRUE;
+    }
     if (!FileTimeToSystemTime((const FILETIME *)pvStructInfo, &sysTime))
         return FALSE;
-    if (sysTime.wYear >= 1950 && sysTime.wYear <= 2050)
-        return CRYPT_AsnEncodeUtcTime(dwCertEncodingType, lpszStructType,
-         pvStructInfo, dwFlags, pEncodePara, pbEncoded, pcbEncoded);
     if (!CRYPT_EncodeEnsureSpace(dwFlags, pEncodePara, pbEncoded, pcbEncoded,
      bytesNeeded))
         return FALSE;
@@ -476,6 +493,31 @@ static BOOL WINAPI CRYPT_AsnEncodeChoiceOfTime(DWORD dwCertEncodingType,
     return TRUE;
 }
 
+static BOOL WINAPI CRYPT_AsnEncodeChoiceOfTime(DWORD dwCertEncodingType,
+ LPCSTR lpszStructType, const void *pvStructInfo, DWORD dwFlags,
+ PCRYPT_ENCODE_PARA pEncodePara, BYTE *pbEncoded, DWORD *pcbEncoded)
+{
+    SYSTEMTIME sysTime;
+    BOOL ret;
+
+    if (!pvStructInfo)
+    {
+        SetLastError(ERROR_INVALID_PARAMETER);
+        return FALSE;
+    }
+    /* Check the year, if it's in the UTCTime range call that encode func */
+    if (!FileTimeToSystemTime((const FILETIME *)pvStructInfo, &sysTime))
+        return FALSE;
+    if (sysTime.wYear >= 1950 && sysTime.wYear <= 2050)
+        ret = CRYPT_AsnEncodeUtcTime(dwCertEncodingType, lpszStructType,
+         pvStructInfo, dwFlags, pEncodePara, pbEncoded, pcbEncoded);
+    else
+        ret = CRYPT_AsnEncodeGeneralizedTime(dwCertEncodingType,
+         lpszStructType, pvStructInfo, dwFlags, pEncodePara, pbEncoded,
+         pcbEncoded);
+    return ret;
+}
+
 typedef BOOL (WINAPI *CryptEncodeObjectExFunc)(DWORD, LPCSTR, const void *,
  DWORD, PCRYPT_ENCODE_PARA, BYTE *, DWORD *);
 
@@ -504,6 +546,7 @@ BOOL WINAPI CryptEncodeObjectEx(DWORD dwCertEncodingType, LPCSTR lpszStructType,
         return FALSE;
     }
 
+    SetLastError(NOERROR);
     if (!HIWORD(lpszStructType))
     {
         switch (LOWORD(lpszStructType))
@@ -577,10 +620,17 @@ BOOL WINAPI CryptDecodeObject(DWORD dwCertEncodingType, LPCSTR lpszStructType,
     return ret;
 }
 
+/* Helper function to check *pcbStructInfo, set it to the required size, and
+ * optionally to allocate memory.  Assumes pvStructInfo is not NULL.
+ * If CRYPT_DECODE_ALLOC_FLAG is set in dwFlags, *pvStructInfo will be set to a
+ * pointer to the newly allocated memory.
+ */
 static BOOL CRYPT_DecodeEnsureSpace(DWORD dwFlags,
  PCRYPT_DECODE_PARA pDecodePara, void *pvStructInfo, DWORD *pcbStructInfo,
  DWORD bytesNeeded)
 {
+    BOOL ret = TRUE;
+
     if (dwFlags & CRYPT_DECODE_ALLOC_FLAG)
     {
         if (pDecodePara && pDecodePara->pfnAlloc)
@@ -588,15 +638,15 @@ static BOOL CRYPT_DecodeEnsureSpace(DWORD dwFlags,
         else
             *(BYTE **)pvStructInfo = LocalAlloc(0, bytesNeeded);
         if (!*(BYTE **)pvStructInfo)
-            return FALSE;
+            ret = FALSE;
     }
     else if (*pcbStructInfo < bytesNeeded)
     {
         *pcbStructInfo = bytesNeeded;
         SetLastError(ERROR_MORE_DATA);
-        return FALSE;
+        ret = FALSE;
     }
-    return TRUE;
+    return ret;
 }
 
 static BOOL WINAPI CRYPT_AsnDecodeInt(DWORD dwCertEncodingType,
@@ -613,8 +663,7 @@ static BOOL WINAPI CRYPT_AsnDecodeInt(DWORD dwCertEncodingType,
     if (!pvStructInfo)
     {
         *pcbStructInfo = sizeof(int);
-        SetLastError(ERROR_MORE_DATA);
+        return TRUE;
-        return FALSE;
     }
     if (pbEncoded[0] != ASN_INTEGER)
     {
@@ -685,8 +734,7 @@ static BOOL WINAPI CRYPT_AsnDecodeUtcTime(DWORD dwCertEncodingType,
     if (!pvStructInfo)
     {
         *pcbStructInfo = sizeof(FILETIME);
-        SetLastError(ERROR_MORE_DATA);
+        return TRUE;
-        return FALSE;
     }
     if (pbEncoded[0] != ASN_UTCTIME)
     {
@@ -731,7 +779,7 @@ static BOOL WINAPI CRYPT_AsnDecodeUtcTime(DWORD dwCertEncodingType,
     return SystemTimeToFileTime(&sysTime, (FILETIME *)pvStructInfo);
 }
 
-static BOOL WINAPI CRYPT_AsnDecodeChoiceOfTime(DWORD dwCertEncodingType,
+static BOOL WINAPI CRYPT_AsnDecodeGeneralizedTime(DWORD dwCertEncodingType,
  LPCSTR lpszStructType, const BYTE *pbEncoded, DWORD cbEncoded, DWORD dwFlags,
  PCRYPT_DECODE_PARA pDecodePara, void *pvStructInfo, DWORD *pcbStructInfo)
 {
@@ -746,13 +794,8 @@ static BOOL WINAPI CRYPT_AsnDecodeChoiceOfTime(DWORD dwCertEncodingType,
     if (!pvStructInfo)
     {
         *pcbStructInfo = sizeof(FILETIME);
-        SetLastError(ERROR_MORE_DATA);
+        return TRUE;
-        return FALSE;
     }
-    if (pbEncoded[0] == ASN_UTCTIME)
-        return CRYPT_AsnDecodeUtcTime(dwCertEncodingType, lpszStructType,
-         pbEncoded, cbEncoded, dwFlags, pDecodePara, pvStructInfo,
-         pcbStructInfo);
     if (pbEncoded[0] != ASN_GENERALTIME)
     {
         SetLastError(CRYPT_E_ASN1_BADTAG);
@@ -831,6 +874,39 @@ static BOOL WINAPI CRYPT_AsnDecodeChoiceOfTime(DWORD dwCertEncodingType,
     return SystemTimeToFileTime(&sysTime, (FILETIME *)pvStructInfo);
 }
 
+static BOOL WINAPI CRYPT_AsnDecodeChoiceOfTime(DWORD dwCertEncodingType,
+ LPCSTR lpszStructType, const BYTE *pbEncoded, DWORD cbEncoded, DWORD dwFlags,
+ PCRYPT_DECODE_PARA pDecodePara, void *pvStructInfo, DWORD *pcbStructInfo)
+{
+    BOOL ret;
+
+    if (!pbEncoded || !cbEncoded)
+    {
+        SetLastError(ERROR_INVALID_PARAMETER);
+        return FALSE;
+    }
+    if (!pvStructInfo)
+    {
+        *pcbStructInfo = sizeof(FILETIME);
+        return TRUE;
+    }
+
+    if (pbEncoded[0] == ASN_UTCTIME)
+        ret = CRYPT_AsnDecodeUtcTime(dwCertEncodingType, lpszStructType,
+         pbEncoded, cbEncoded, dwFlags, pDecodePara, pvStructInfo,
+         pcbStructInfo);
+    else if (pbEncoded[0] == ASN_GENERALTIME)
+        ret = CRYPT_AsnDecodeGeneralizedTime(dwCertEncodingType,
+         lpszStructType, pbEncoded, cbEncoded, dwFlags, pDecodePara,
+         pvStructInfo, pcbStructInfo);
+    else
+    {
+        SetLastError(CRYPT_E_ASN1_BADTAG);
+        ret = FALSE;
+    }
+    return ret;
+}
+
 typedef BOOL (WINAPI *CryptDecodeObjectExFunc)(DWORD, LPCSTR, const BYTE *,
  DWORD, DWORD, PCRYPT_DECODE_PARA, void *, DWORD *);
 
@@ -859,6 +935,7 @@ BOOL WINAPI CryptDecodeObjectEx(DWORD dwCertEncodingType, LPCSTR lpszStructType,
         return FALSE;
     }
 
+    SetLastError(NOERROR);
     if (!HIWORD(lpszStructType))
     {
         switch (LOWORD(lpszStructType))

dlls/crypt32/tests/encode.c

@@ -70,8 +70,7 @@ static void test_encodeint(void)
 
         ret = CryptEncodeObjectEx(X509_ASN_ENCODING | PKCS_7_ASN_ENCODING,
          X509_INTEGER, &ints[i].val, 0, NULL, NULL, &bufSize);
-        ok(ret || GetLastError() == ERROR_MORE_DATA,
+        ok(ret, "Expected success, got %ld\n", GetLastError());
-         "Expected success or ERROR_MORE_DATA, got %ld\n", GetLastError());
         ret = CryptEncodeObjectEx(X509_ASN_ENCODING | PKCS_7_ASN_ENCODING,
          X509_INTEGER, &ints[i].val, CRYPT_ENCODE_ALLOC_FLAG, NULL,
          (BYTE *)&buf, &bufSize);
@@ -122,12 +121,13 @@ static void test_decodeint(void)
      "Expected CRYPT_E_ASN1_BADTAG, got %ld\n", GetLastError());
     for (i = 0; i < sizeof(ints) / sizeof(ints[0]); i++)
     {
-        /* WinXP succeeds rather than failing with ERROR_MORE_DATA */
+        /* When the output buffer is NULL, this always succeeds */
+        SetLastError(0xdeadbeef);
         ret = CryptDecodeObjectEx(X509_ASN_ENCODING, X509_INTEGER,
          (BYTE *)&ints[i].encoded, ints[i].encoded[1] + 2, 0, NULL, NULL,
          &bufSize);
-        ok(ret || GetLastError() == ERROR_MORE_DATA,
+        ok(ret && GetLastError() == NOERROR,
-         "Expected success or ERROR_MORE_DATA, got %ld\n", GetLastError());
+         "Expected success and NOERROR, got %ld\n", GetLastError());
         ret = CryptDecodeObjectEx(X509_ASN_ENCODING | PKCS_7_ASN_ENCODING,
          X509_INTEGER, (BYTE *)&ints[i].encoded, ints[i].encoded[1] + 2,
          CRYPT_ENCODE_ALLOC_FLAG, NULL, (BYTE *)&buf, &bufSize);
@@ -262,8 +262,19 @@ static void test_decodeFiletime(void)
      "\x18" "\x04" "2145",
      "\x18" "\x08" "21450606",
     };
-    DWORD i;
+    DWORD i, size;
+    FILETIME ft1 = { 0 }, ft2 = { 0 };
+    BOOL ret;
 
+    /* Check bogus length with non-NULL buffer */
+    ret = SystemTimeToFileTime(&times[0].sysTime, &ft1);
+    ok(ret, "SystemTimeToFileTime failed: %ld\n", GetLastError());
+    size = 1;
+    ret = CryptDecodeObjectEx(X509_ASN_ENCODING, X509_CHOICE_OF_TIME,
+     times[0].encodedTime, times[0].encodedTime[1] + 2, 0, NULL, &ft2, &size);
+    ok(!ret && GetLastError() == ERROR_MORE_DATA,
+     "Expected ERROR_MORE_DATA, got %ld\n", GetLastError());
+    /* Normal tests */
     for (i = 0; i < sizeof(times) / sizeof(times[0]); i++)
     {
         testTimeDecoding(X509_CHOICE_OF_TIME, &times[i]);
@@ -278,19 +289,9 @@ static void test_decodeFiletime(void)
     }
     for (i = 0; i < sizeof(bogusTimes) / sizeof(bogusTimes[0]); i++)
     {
-        FILETIME ft;
+        size = sizeof(ft1);
-        SYSTEMTIME sysTime;
+        ret = CryptDecodeObjectEx(X509_ASN_ENCODING, X509_CHOICE_OF_TIME,
-        DWORD size = sizeof(ft);
+         bogusTimes[i], bogusTimes[i][1] + 2, 0, NULL, &ft1, &size);
-        BOOL ret = CryptDecodeObjectEx(X509_ASN_ENCODING, X509_CHOICE_OF_TIME,
-         bogusTimes[i], bogusTimes[i][1] + 2, 0, NULL, &ft, &size);
-
-        if (ret)
-        {
-            ret = FileTimeToSystemTime(&ft, &sysTime);
-            printf("%02d %02d %04d %02d:%02d.%02d\n", sysTime.wMonth,
-             sysTime.wDay, sysTime.wYear, sysTime.wHour, sysTime.wMinute,
-             sysTime.wSecond);
-        }
         ok(!ret && GetLastError() == CRYPT_E_ASN1_CORRUPT,
          "Expected CRYPT_E_ASN1_CORRUPT, got %08lx\n", GetLastError());
     }