ole32: Performing a COM call from within the processing of a sent message during a wait for completion of another COM call is not allowed.

Add a test for the behaviour where RPC_E_CANTCALLOUT_ININPUTSYNCCALL is returned.

dlls/ole32/compobj.c

@@ -3190,10 +3190,12 @@ HRESULT WINAPI CoWaitForMultipleHandles(DWORD dwFlags, DWORD dwTimeout,
 
                 if (COM_CurrentApt()->filter)
                 {
+                    PENDINGTYPE pendingtype =
+                        COM_CurrentInfo()->pending_call_count_server ?
+                            PENDINGTYPE_NESTED : PENDINGTYPE_TOPLEVEL;
                     DWORD be_handled = IMessageFilter_MessagePending(
                         COM_CurrentApt()->filter, 0 /* FIXME */,
-                        now - start_time,
+                        now - start_time, pendingtype);
-                        COM_CurrentInfo()->pending_call_count ? PENDINGTYPE_NESTED : PENDINGTYPE_TOPLEVEL);
                     TRACE("IMessageFilter_MessagePending returned %d\n", be_handled);
                     switch (be_handled)
                     {

dlls/ole32/compobj_private.h

@@ -169,7 +169,8 @@ struct oletls
     IUnknown         *state;       /* see CoSetState */
     DWORD            inits;        /* number of times CoInitializeEx called */
     GUID             causality_id; /* unique identifier for each COM call */
-    LONG             pending_call_count; /* number of calls pending */
+    LONG             pending_call_count_client; /* number of client calls pending */
+    LONG             pending_call_count_server; /* number of server calls pending */
 };
 
 

dlls/ole32/rpc.c

@@ -566,6 +566,18 @@ static HRESULT WINAPI ClientRpcChannelBuffer_SendReceive(LPRPCCHANNELBUFFER ifac
             wine_dbgstr_longlong(This->oxid));
         return RPC_E_WRONG_THREAD;
     }
+    /* this situation should be impossible in multi-threaded apartments,
+     * because the calling thread isn't re-entrable.
+     * Note: doing a COM call during the processing of a sent message is
+     * only disallowed if a client call is already being waited for
+     * completion */
+    if (!COM_CurrentApt()->multi_threaded &&
+        COM_CurrentInfo()->pending_call_count_client &&
+        InSendMessage())
+    {
+        ERR("can't make an outgoing COM call in response to a sent message\n");
+        return RPC_E_CANTCALLOUT_ININPUTSYNCCALL;
+    }
 
     params = HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, sizeof(*params));
     if (!params) return E_OUTOFMEMORY;
@@ -632,7 +644,11 @@ static HRESULT WINAPI ClientRpcChannelBuffer_SendReceive(LPRPCCHANNELBUFFER ifac
     if (hr == S_OK)
     {
         if (WaitForSingleObject(params->handle, 0))
+        {
+            COM_CurrentInfo()->pending_call_count_client++;
             hr = CoWaitForMultipleHandles(0, INFINITE, 1, &params->handle, &index);
+            COM_CurrentInfo()->pending_call_count_client--;
+        }
     }
     ClientRpcChannelBuffer_ReleaseEventHandle(This, params->handle);
 
@@ -986,7 +1002,7 @@ void RPC_ExecuteCall(struct dispatch_params *params)
 
         if (IsEqualGUID(&orpcthis.cid, &COM_CurrentInfo()->causality_id))
             calltype = CALLTYPE_NESTED;
-        else if (COM_CurrentInfo()->pending_call_count == 0)
+        else if (COM_CurrentInfo()->pending_call_count_server == 0)
             calltype = CALLTYPE_TOPLEVEL;
         else
             calltype = CALLTYPE_TOPLEVEL_CALLPENDING;
@@ -1023,9 +1039,9 @@ void RPC_ExecuteCall(struct dispatch_params *params)
      * this call - this should be checked with what Windows does */
     old_causality_id = COM_CurrentInfo()->causality_id;
     COM_CurrentInfo()->causality_id = orpcthis.cid;
-    COM_CurrentInfo()->pending_call_count++;
+    COM_CurrentInfo()->pending_call_count_server++;
     params->hr = IRpcStubBuffer_Invoke(params->stub, params->msg, params->chan);
-    COM_CurrentInfo()->pending_call_count--;
+    COM_CurrentInfo()->pending_call_count_server--;
     COM_CurrentInfo()->causality_id = old_causality_id;
 
     message_state = (struct message_state *)msg->Handle;

dlls/ole32/tests/marshal.c

@@ -1614,6 +1614,38 @@ static LRESULT CALLBACK window_proc(HWND hwnd, UINT msg, WPARAM wparam, LPARAM l
 
         return 0;
     }
+    case WM_USER+2:
+    {
+        HRESULT hr;
+        IStream *pStream = NULL;
+        IClassFactory *proxy = NULL;
+        IUnknown *object;
+        DWORD tid;
+        HANDLE thread;
+
+        hr = CreateStreamOnHGlobal(NULL, TRUE, &pStream);
+        ok_ole_success(hr, CreateStreamOnHGlobal);
+        tid = start_host_object(pStream, &IID_IClassFactory, (IUnknown*)&Test_ClassFactory, MSHLFLAGS_NORMAL, &thread);
+
+        IStream_Seek(pStream, ullZero, STREAM_SEEK_SET, NULL);
+        hr = CoUnmarshalInterface(pStream, &IID_IClassFactory, (void **)&proxy);
+        ok_ole_success(hr, CoReleaseMarshalData);
+        IStream_Release(pStream);
+
+        /* shows that COM calls executed during the processing of sent
+         * messages should fail */
+        hr = IClassFactory_CreateInstance(proxy, NULL, &IID_IUnknown, (void **)&object);
+        ok(hr == RPC_E_CANTCALLOUT_ININPUTSYNCCALL,
+           "COM call during processing of sent message should return RPC_E_CANTCALLOUT_ININPUTSYNCCALL instead of 0x%08x\n", hr);
+
+        IClassFactory_Release(proxy);
+
+        end_host_object(tid, thread);
+
+        PostQuitMessage(0);
+
+        return 0;
+    }
     default:
         return DefWindowProc(hwnd, msg, wparam, lparam);
     }
@@ -1642,6 +1674,73 @@ static void test_message_reentrancy(void)
     }
 }
 
+static HRESULT WINAPI TestMsg_IClassFactory_CreateInstance(
+    LPCLASSFACTORY iface,
+    LPUNKNOWN pUnkOuter,
+    REFIID riid,
+    LPVOID *ppvObj)
+{
+    *ppvObj = NULL;
+    SendMessage(hwnd_app, WM_USER+2, 0, 0);
+    return S_OK;
+}
+
+static IClassFactoryVtbl TestMsgClassFactory_Vtbl =
+{
+    Test_IClassFactory_QueryInterface,
+    Test_IClassFactory_AddRef,
+    Test_IClassFactory_Release,
+    TestMsg_IClassFactory_CreateInstance,
+    Test_IClassFactory_LockServer
+};
+
+IClassFactory TestMsg_ClassFactory = { &TestMsgClassFactory_Vtbl };
+
+static void test_call_from_message(void)
+{
+    MSG msg;
+    IStream *pStream;
+    HRESULT hr;
+    IClassFactory *proxy;
+    DWORD tid;
+    HANDLE thread;
+    IUnknown *object;
+
+    hwnd_app = CreateWindow("WineCOMTest", NULL, 0, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, NULL, NULL, NULL, 0);
+    ok(hwnd_app != NULL, "Window creation failed\n");
+
+    hr = CreateStreamOnHGlobal(NULL, TRUE, &pStream);
+    ok_ole_success(hr, CreateStreamOnHGlobal);
+    tid = start_host_object(pStream, &IID_IClassFactory, (IUnknown*)&TestMsg_ClassFactory, MSHLFLAGS_NORMAL, &thread);
+
+    ok_more_than_one_lock();
+
+    IStream_Seek(pStream, ullZero, STREAM_SEEK_SET, NULL);
+    hr = CoUnmarshalInterface(pStream, &IID_IClassFactory, (void **)&proxy);
+    ok_ole_success(hr, CoReleaseMarshalData);
+    IStream_Release(pStream);
+
+    ok_more_than_one_lock();
+
+    hr = CoRegisterMessageFilter(&MessageFilter, NULL);
+
+    /* start message re-entrancy test */
+    hr = IClassFactory_CreateInstance(proxy, NULL, &IID_IUnknown, (void **)&object);
+    ok_ole_success(hr, IClassFactory_CreateInstance);
+
+    IClassFactory_Release(proxy);
+
+    ok_no_locks();
+
+    end_host_object(tid, thread);
+
+    while (GetMessage(&msg, NULL, 0, 0))
+    {
+        TranslateMessage(&msg);
+        DispatchMessage(&msg);
+    }
+}
+
 static void test_WM_QUIT_handling(void)
 {
     MSG msg;
@@ -2131,6 +2230,7 @@ START_TEST(marshal)
     test_stubbuffer(&IID_IClassFactory);
     test_proxybuffer(&IID_IClassFactory);
     test_message_reentrancy();
+    test_call_from_message();
     test_WM_QUIT_handling();
     test_freethreadedmarshaler();