By-pass the RPC runtime if possible when calling an STA by posting a

message directly to the apartment window for it to process. Fixes a
deadlock in InstallShield caused by having to create a thread when
freeing an object that comes from an STA apartment. Added tests that
fail without this fix.

dlls/ole32/rpc.c

@@ -212,10 +212,14 @@ static DWORD WINAPI rpc_sendreceive_thread(LPVOID param)
 static HRESULT WINAPI RpcChannelBuffer_SendReceive(LPRPCCHANNELBUFFER iface, RPCOLEMESSAGE *olemsg, ULONG *pstatus)
 {
     HRESULT hr = S_OK;
+    RPC_MESSAGE *msg = (RPC_MESSAGE *)olemsg;
     RPC_STATUS status;
     DWORD index;
     struct dispatch_params *params;
     DWORD tid;
+    IRpcStubBuffer *stub;
+    APARTMENT *apt;
+    IPID ipid;
 
     TRACE("(%p) iMethod=%ld\n", olemsg, olemsg->iMethod);
 
@@ -225,6 +229,29 @@ static HRESULT WINAPI RpcChannelBuffer_SendReceive(LPRPCCHANNELBUFFER iface, RPC
     params->msg = olemsg;
     params->status = RPC_S_OK;
 
+    /* Note: this is an optimization in the Microsoft OLE runtime that we need
+     * to copy, as shown by the test_no_couninitialize_client test. without
+     * short-circuiting the RPC runtime in the case below, the test will
+     * deadlock on the loader lock due to the RPC runtime needing to create
+     * a thread to process the RPC when this function is called indirectly
+     * from DllMain */
+
+    RpcBindingInqObject(msg->Handle, &ipid);
+    stub = ipid_to_apt_and_stubbuffer(&ipid, &apt);
+    if (apt && (apt->model & COINIT_APARTMENTTHREADED))
+    {
+        params->stub = stub;
+        params->chan = NULL; /* FIXME: pass server channel */
+        params->handle = CreateEventW(NULL, FALSE, FALSE, NULL);
+
+        TRACE("Calling apartment thread 0x%08lx...\n", apt->tid);
+
+        PostMessageW(apt->win, DM_EXECUTERPC, 0, (LPARAM)params);
+    }
+    else
+    {
+        if (stub) IRpcStubBuffer_Release(stub);
+
         /* we use a separate thread here because we need to be able to
          * pump the message loop in the application thread: if we do not,
          * any windows created by this thread will hang and RPCs that try
@@ -237,6 +264,8 @@ static HRESULT WINAPI RpcChannelBuffer_SendReceive(LPRPCCHANNELBUFFER iface, RPC
             ERR("Could not create RpcSendReceive thread, error %lx\n", GetLastError());
             hr = E_UNEXPECTED;
         }
+    }
+    if (apt) apartment_release(apt);
 
     if (hr == S_OK)
         hr = CoWaitForMultipleHandles(0, INFINITE, 1, &params->handle, &index);

dlls/ole32/tests/marshal.c

@@ -534,8 +534,8 @@ struct ncu_params
     HANDLE unmarshal_event;
 };
 
-/* helper for test_proxy_used_in_wrong_thread */
+/* helper for test_no_couninitialize_server */
-static DWORD CALLBACK no_couninitialize_proc(LPVOID p)
+static DWORD CALLBACK no_couninitialize_server_proc(LPVOID p)
 {
     struct ncu_params *ncu_params = (struct ncu_params *)p;
     HRESULT hr;
@@ -554,8 +554,9 @@ static DWORD CALLBACK no_couninitialize_proc(LPVOID p)
     return 0;
 }
 
-/* tests apartment that an apartment is released if the owning thread exits */
+/* tests apartment that an apartment with a stub is released without deadlock
-static void test_no_couninitialize(void)
+ * if the owning thread exits */
+static void test_no_couninitialize_server()
 {
     HRESULT hr;
     IStream *pStream = NULL;
@@ -573,7 +574,7 @@ static void test_no_couninitialize(void)
     ok_ole_success(hr, CreateStreamOnHGlobal);
     ncu_params.stream = pStream;
 
-    thread = CreateThread(NULL, 0, no_couninitialize_proc, &ncu_params, 0, &tid);
+    thread = CreateThread(NULL, 0, no_couninitialize_server_proc, &ncu_params, 0, &tid);
 
     WaitForSingleObject(ncu_params.marshal_event, INFINITE);
     ok_more_than_one_lock();
@@ -599,6 +600,59 @@ static void test_no_couninitialize(void)
     ok_no_locks();
 }
 
+/* STA -> STA call during DLL_THREAD_DETACH */
+static DWORD CALLBACK no_couninitialize_client_proc(LPVOID p)
+{
+    struct ncu_params *ncu_params = (struct ncu_params *)p;
+    HRESULT hr;
+    IUnknown *pProxy = NULL;
+
+    pCoInitializeEx(NULL, COINIT_APARTMENTTHREADED);
+
+    hr = CoUnmarshalInterface(ncu_params->stream, &IID_IClassFactory, (void **)&pProxy);
+    ok_ole_success(hr, CoUnmarshalInterface);
+
+    ok_more_than_one_lock();
+
+    /* die without calling CoUninitialize */
+
+    return 0;
+}
+
+/* tests STA -> STA call during DLL_THREAD_DETACH doesn't deadlock */
+static void test_no_couninitialize_client()
+{
+    HRESULT hr;
+    IStream *pStream = NULL;
+    DWORD tid;
+    DWORD host_tid;
+    HANDLE thread;
+    HANDLE host_thread;
+    struct ncu_params ncu_params;
+
+    cLocks = 0;
+
+    hr = CreateStreamOnHGlobal(NULL, TRUE, &pStream);
+    ok_ole_success(hr, CreateStreamOnHGlobal);
+    ncu_params.stream = pStream;
+
+    /* NOTE: assumes start_host_object uses an STA to host the object, as MTAs
+     * always deadlock when called from within DllMain */
+    host_tid = start_host_object(pStream, &IID_IClassFactory, (IUnknown *)&Test_ClassFactory, MSHLFLAGS_NORMAL, &host_thread);
+    IStream_Seek(pStream, ullZero, STREAM_SEEK_SET, NULL);
+
+    ok_more_than_one_lock();
+
+    thread = CreateThread(NULL, 0, no_couninitialize_client_proc, &ncu_params, 0, &tid);
+
+    WaitForSingleObject(thread, INFINITE);
+    CloseHandle(thread);
+
+    ok_no_locks();
+
+    end_host_object(host_tid, host_thread);
+}
+
 /* tests success case of a same-thread table-weak marshal, unmarshal, unmarshal */
 static void test_tableweak_marshal_and_unmarshal_twice(void)
 {
@@ -1588,7 +1642,8 @@ START_TEST(marshal)
     test_marshal_stub_apartment_shutdown();
     test_marshal_proxy_apartment_shutdown();
     test_marshal_proxy_mta_apartment_shutdown();
-    test_no_couninitialize();
+    test_no_couninitialize_server();
+    test_no_couninitialize_client();
     test_tableweak_marshal_and_unmarshal_twice();
     test_tableweak_marshal_releasedata1();
     test_tableweak_marshal_releasedata2();