Out-of-process plugin host via threaded callback pump (alternative to #40120)

.pipelines/build-stage.yml

@@ -32,8 +32,8 @@ parameters:
   - name: targets
     type: object
     default:
-      - target: "wsl;libwsl;wslg;wslservice;wslhost;wslrelay;wslinstaller;wslinstall;initramfs;wslserviceproxystub;wslsettings;wslinstallerproxystub;testplugin;wslcsession;wslc;wsltests;wslcsdk"
-        pattern: "wsl.exe,libwsl.dll,wslg.exe,wslservice.exe,wslhost.exe,wslrelay.exe,wslinstaller.exe,wslinstall.dll,wslserviceproxystub.dll,wslsettings/wslsettings.dll,wslsettings/wslsettings.exe,wslinstallerproxystub.dll,WSLDVCPlugin.dll,testplugin.dll,wsldeps.dll,wslcsession.exe,wslc.exe,wslcsdk.dll"
+      - target: "wsl;libwsl;wslg;wslservice;wslhost;wslrelay;wslpluginhost;wslinstaller;wslinstall;initramfs;wslserviceproxystub;wslsettings;wslinstallerproxystub;testplugin;wslcsession;wslc;wsltests;wslcsdk"
+        pattern: "wsl.exe,libwsl.dll,wslg.exe,wslservice.exe,wslhost.exe,wslrelay.exe,wslpluginhost.exe,wslinstaller.exe,wslinstall.dll,wslserviceproxystub.dll,wslsettings/wslsettings.dll,wslsettings/wslsettings.exe,wslinstallerproxystub.dll,WSLDVCPlugin.dll,testplugin.dll,wsldeps.dll,wslcsession.exe,wslc.exe,wslcsdk.dll"
       - target: "msixgluepackage"
         pattern: "gluepackage.msix"
       - target: "msipackage;wslcsdkcs"

CMakeLists.txt

@@ -545,6 +545,7 @@ add_subdirectory(src/windows/wsl)
 add_subdirectory(src/windows/wslg)
 add_subdirectory(src/windows/wslhost)
 add_subdirectory(src/windows/wslrelay)
+add_subdirectory(src/windows/wslpluginhost)
 add_subdirectory(src/windows/wslinstall)
 add_subdirectory(src/windows/wslc)
 add_subdirectory(src/windows/WslcSDK)

msipackage/CMakeLists.txt

@@ -17,7 +17,7 @@ set(OUTPUT_PACKAGE ${BIN}/wsl.msi)
 set(PACKAGE_WIX_IN ${CMAKE_CURRENT_LIST_DIR}/package.wix.in)
 set(PACKAGE_WIX ${BIN}/package.wix)
 set(CAB_CACHE ${BIN}/cab)
-set(WINDOWS_BINARIES wsl.exe;wslg.exe;wslhost.exe;wslrelay.exe;wslservice.exe;wslserviceproxystub.dll;wslinstall.dll;wslc.exe;wslcsession.exe)
+set(WINDOWS_BINARIES wsl.exe;wslg.exe;wslhost.exe;wslrelay.exe;wslpluginhost.exe;wslservice.exe;wslserviceproxystub.dll;wslinstall.dll;wslc.exe;wslcsession.exe)
 if (WSL_BUILD_WSL_SETTINGS)
     list(APPEND WINDOWS_BINARIES "wslsettings/wslsettings.dll;wslsettings/wslsettings.exe;libwsl.dll")
 endif()
@@ -57,7 +57,7 @@ add_custom_command(
 
 add_custom_target(msipackage DEPENDS ${OUTPUT_PACKAGE})
 set_target_properties(msipackage PROPERTIES EXCLUDE_FROM_ALL FALSE SOURCES ${PACKAGE_WIX_IN})
-add_dependencies(msipackage wsl wslg wslservice wslhost wslrelay wslserviceproxystub init initramfs wslinstall msixgluepackage wslc wslcsession)
+add_dependencies(msipackage wsl wslg wslservice wslhost wslrelay wslpluginhost wslserviceproxystub init initramfs wslinstall msixgluepackage wslc wslcsession)
 
 if (WSL_BUILD_WSL_SETTINGS)
     add_dependencies(msipackage wslsettings libwsl)

msipackage/package.wix.in

@@ -31,6 +31,7 @@
                 <File Id="container.exe" Name="container.exe" Source="${PACKAGE_INPUT_DIR}/wslc.exe" />
                 <File Id="wslhost.exe" Name="wslhost.exe" Source="${PACKAGE_INPUT_DIR}/wslhost.exe" />
                 <File Id="wslrelay.exe" Name="wslrelay.exe" Source="${PACKAGE_INPUT_DIR}/wslrelay.exe" />
+                <File Id="wslpluginhost.exe" Name="wslpluginhost.exe" Source="${PACKAGE_INPUT_DIR}/wslpluginhost.exe" />
                 <File Id="wslserviceproxystub.dll" Name="wslserviceproxystub.dll" Source="${PACKAGE_INPUT_DIR}/wslserviceproxystub.dll" />
                 <File Id="wsldeps.dll" Name="wsldeps.dll" Source="${PACKAGE_INPUT_DIR}/wsldeps.dll" />
 
@@ -177,6 +178,35 @@
                     <RegistryValue Value='"[INSTALLDIR]wslhost.exe"' Type="string" />
                 </RegistryKey>
 
+                <!-- WslPluginHost - out-of-process plugin isolation (process spawned by service) -->
+                <!-- WslPluginHost AppID — SYSTEM-only activation -->
+                <!-- O:SYG:SYD:(A;;CCDCSW;;;SY) -->
+                <RegistryKey Root="HKCR" Key="AppID\{7a1d2c3e-4b5f-6a7d-8e9f-0a1b2c3d4e5f}">
+                    <RegistryValue Name="AccessPermission" Value="010004801400000020000000000000002C00000001010000000000051200000001010000000000051200000002001C0001000000000014000B000000010100000000000512000000" Type="binary" />
+                    <RegistryValue Name="LaunchPermission" Value="010004801400000020000000000000002C00000001010000000000051200000001010000000000051200000002001C0001000000000014000B000000010100000000000512000000" Type="binary" />
+                </RegistryKey>
+                <RegistryKey Root="HKCR" Key="CLSID\{7a1d2c3e-4b5f-6a7d-8e9f-0a1b2c3d4e5f}">
+                    <RegistryValue Name="AppId" Value="{7a1d2c3e-4b5f-6a7d-8e9f-0a1b2c3d4e5f}" Type="string" />
+                    <RegistryValue Value="WslPluginHost" Type="string" />
+                </RegistryKey>
+                <RegistryKey Root="HKCR" Key="CLSID\{7a1d2c3e-4b5f-6a7d-8e9f-0a1b2c3d4e5f}\LocalServer32">
+                    <RegistryValue Value='"[INSTALLDIR]wslpluginhost.exe"' Type="string" />
+                </RegistryKey>
+
+                <!-- WslPluginHost interfaces — use the shared wslserviceproxystub.dll -->
+                <RegistryKey Root="HKCR" Key="Interface\{A2B3C4D5-E6F7-4890-AB12-CD34EF56A789}">
+                    <RegistryValue Value="IWslPluginHostCallback" Type="string" />
+                    <RegistryKey Key="ProxyStubClsid32">
+                        <RegistryValue Value="{4EA0C6DD-E9FF-48E7-994E-13A31D10DC60}" Type="string" />
+                    </RegistryKey>
+                </RegistryKey>
+                <RegistryKey Root="HKCR" Key="Interface\{B3C4D5E6-F7A8-4901-BC23-DE45FA67B890}">
+                    <RegistryValue Value="IWslPluginHost" Type="string" />
+                    <RegistryKey Key="ProxyStubClsid32">
+                        <RegistryValue Value="{4EA0C6DD-E9FF-48E7-994E-13A31D10DC60}" Type="string" />
+                    </RegistryKey>
+                </RegistryKey>
+
                 <!-- wsldevicehost.dll -->
                 <RegistryKey Root="HKCR" Key="AppID\{17696EAC-9568-4CF5-BB8C-82515AAD6C09}">
                     <RegistryValue Name="DllSurrogate" Value="" Type="string" />

src/windows/common/precomp.h

@@ -84,6 +84,7 @@ Module Name:
 #include <optional>
 #include <filesystem>
 #include <mutex>
+#include <shared_mutex>
 #include <chrono>
 #include <codecvt>
 #include <random>

src/windows/service/exe/CMakeLists.txt

@@ -7,6 +7,7 @@ set(SOURCES
     LxssHttpProxy.cpp
     LxssInstance.cpp
     PluginManager.cpp
+    PluginCallPump.cpp
     ServiceMain.cpp
     BridgedNetworking.cpp
     GnsRpcServer.cpp
@@ -37,6 +38,7 @@ set(HEADERS
     LxssIptables.h
     LxssHttpProxy.h
     PluginManager.h
+    PluginCallPump.h
     LxssInstance.h
     BridgedNetworking.h
     GnsRpcServer.h
@@ -58,7 +60,7 @@ set(HEADERS
     WSLCPluginNotifier.h)
 
 add_executable(wslservice ${SOURCES} ${HEADERS})
-add_dependencies(wslservice wslserviceidl wslservicemc)
+add_dependencies(wslservice wslserviceidl wslservicemc wslpluginhostidl)
 add_compile_definitions(__WRL_CLASSIC_COM__)
 add_compile_definitions(__WRL_DISABLE_STATIC_INITIALIZE__)
 add_compile_definitions(USE_COM_CONTEXT_DEF=1)

src/windows/service/exe/LxssUserSession.cpp

@@ -3653,6 +3653,18 @@ HRESULT LxssUserSessionImpl::MountRootNamespaceFolder(_In_ LPCWSTR HostPath, _In
     return S_OK;
 }
 
+bool LxssUserSessionImpl::TryInvokeUnderInstanceLock(std::chrono::milliseconds Timeout, const std::function<HRESULT()>& Work, _Out_ HRESULT& Result)
+{
+    std::unique_lock<std::recursive_timed_mutex> lock(m_instanceLock, std::defer_lock);
+    if (!lock.try_lock_for(Timeout))
+    {
+        return false;
+    }
+
+    Result = Work();
+    return true;
+}
+
 HRESULT LxssUserSessionImpl::CreateLinuxProcess(_In_opt_ const GUID* Distro, _In_ LPCSTR Path, _In_ LPCSTR* Arguments, _Out_ SOCKET* Socket)
 {
     std::lock_guard lock(m_instanceLock);

src/windows/service/exe/LxssUserSession.h

@@ -445,6 +445,18 @@ class LxssUserSessionImpl
 
     HRESULT MountRootNamespaceFolder(_In_ LPCWSTR HostPath, _In_ LPCWSTR GuestPath, _In_ bool ReadOnly, _In_ LPCWSTR Name);
 
+    /// <summary>
+    /// Attempts to run Work while holding m_instanceLock, acquired with the
+    /// given timeout. Returns true and sets Result to Work()'s HRESULT if the
+    /// lock was acquired and Work ran; returns false (without running Work) if
+    /// the lock could not be acquired within Timeout. Used by the plugin
+    /// callback pump to run an out-of-hook callback directly without blocking
+    /// indefinitely on the instance lock — a notification thread may hold it in
+    /// its pre-notification phase and later wait for this very callback, which
+    /// would deadlock if we blocked on the lock unconditionally.
+    /// </summary>
+    bool TryInvokeUnderInstanceLock(std::chrono::milliseconds Timeout, const std::function<HRESULT()>& Work, _Out_ HRESULT& Result);
+
     /// <summary>
     /// Registers a distribution.
     /// </summary>

src/windows/service/exe/PluginCallPump.cpp

@@ -0,0 +1,147 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+
+#include "precomp.h"
+#include "PluginCallPump.h"
+#include <thread>
+
+using wsl::windows::service::PluginCallPump;
+
+PluginCallPump::PluginCallPump() = default;
+
+void PluginCallPump::DrainQueue()
+{
+    for (;;)
+    {
+        Call* call = nullptr;
+        {
+            auto lock = m_lock.lock_exclusive();
+            if (m_queue.empty())
+            {
+                break;
+            }
+            call = m_queue.front();
+            m_queue.pop_front();
+        }
+
+        // Run the plugin's service-side work on the pump (notification) thread,
+        // which still holds the notifying lock — recursive locks re-enter here.
+        // The closures already translate exceptions to HRESULTs via CATCH_RETURN,
+        // but guard defensively so a throw can never skip done.SetEvent() and
+        // strand the waiting RPC thread.
+        try
+        {
+            call->result = call->work ? call->work() : E_UNEXPECTED;
+        }
+        catch (...)
+        {
+            call->result = wil::ResultFromCaughtException();
+        }
+
+        call->done.SetEvent();
+    }
+}
+
+HRESULT PluginCallPump::Run(const std::function<HRESULT()>& Notification)
+try
+{
+    HRESULT notificationResult = E_FAIL;
+
+    // The worker makes the outbound cross-process notification call. It runs on
+    // its own thread so THIS thread is free to pump the plugin's callbacks while
+    // the notification is in flight. (Thread creation can throw std::system_error
+    // under resource exhaustion; the surrounding try/CATCH_RETURN converts that to
+    // an HRESULT so it never escapes into a non-throwing teardown notification.)
+    wil::unique_event workerDone(wil::EventOptions::ManualReset);
+    std::thread worker([&]() {
+        // Guard defensively: a throw escaping the thread's top-level function
+        // would call std::terminate() and crash the service, and would skip
+        // workerDone.SetEvent() — stranding the pumping thread. Translate to an
+        // HRESULT and always signal completion (mirrors DrainQueue).
+        try
+        {
+            notificationResult = Notification();
+        }
+        catch (...)
+        {
+            notificationResult = wil::ResultFromCaughtException();
+        }
+
+        workerDone.SetEvent();
+    });
+
+    auto join = wil::scope_exit([&]() {
+        if (worker.joinable())
+        {
+            worker.join();
+        }
+    });
+
+    const HANDLE waits[] = {m_callAvailable.get(), workerDone.get()};
+    for (;;)
+    {
+        const DWORD wait = ::WaitForMultipleObjects(ARRAYSIZE(waits), waits, FALSE, INFINITE);
+
+        // A kernel wait failure must never spin this thread. Stop the pump and
+        // fail any queued/future calls so their RPC threads aren't stranded.
+        FAIL_FAST_LAST_ERROR_IF(wait == WAIT_FAILED);
+
+        // Drain regardless of which handle woke us: the auto-reset event
+        // coalesces multiple enqueues into a single signal, and a final call may
+        // race in just as the worker completes.
+        DrainQueue();
+
+        // Check worker completion independently of the wait result.
+        // WaitForMultipleObjects reports the LOWEST signaled index, so a steady
+        // stream of callbacks on m_callAvailable (index 0) would otherwise starve
+        // the workerDone (index 1) branch and keep this thread pumping (and the
+        // notifying lock held) forever. workerDone is manual-reset, so this is a
+        // cheap non-consuming poll.
+        if (::WaitForSingleObject(workerDone.get(), 0) == WAIT_OBJECT_0)
+        {
+            // Worker (notification) finished. Stop accepting further work and
+            // fail any call that raced in after this point so its RPC thread is
+            // never stranded, then drain whatever was already queued.
+            {
+                auto lock = m_lock.lock_exclusive();
+                m_stopped = true;
+            }
+            DrainQueue();
+            break;
+        }
+    }
+
+    return notificationResult;
+}
+CATCH_RETURN()
+
+bool PluginCallPump::Invoke(std::function<HRESULT()> Work, _Out_ HRESULT& Result)
+{
+    Call call;
+    call.work = std::move(Work);
+
+    const HRESULT createHr = call.done.create();
+    if (FAILED(createHr))
+    {
+        // Could not create the completion event (resource exhaustion). Surface
+        // the failure as the executed result rather than reporting "not run":
+        // retrying on the direct path would not help and risks double-execution.
+        Result = createHr;
+        return true;
+    }
+
+    {
+        auto lock = m_lock.lock_exclusive();
+        if (m_stopped)
+        {
+            // The notification already returned; there is no pump thread left to
+            // run this. Report "not run" so the caller executes it directly.
+            return false;
+        }
+        m_queue.push_back(&call);
+    }
+
+    m_callAvailable.SetEvent();
+    call.done.wait();
+    Result = call.result;
+    return true;
+}

src/windows/service/exe/PluginCallPump.h

@@ -0,0 +1,85 @@
+// Copyright (C) Microsoft Corporation. All rights reserved.
+
+#pragma once
+
+#include <wil/resource.h>
+#include <atomic>
+#include <deque>
+#include <functional>
+
+namespace wsl::windows::service {
+
+//
+// PluginCallPump implements the threaded-callback model for out-of-process
+// plugin notifications.
+//
+// Problem: a plugin lifecycle notification (OnVMStarted, OnDistributionStarted,
+// ...) is an outbound cross-process COM call. While the service thread is
+// blocked inside that call, the plugin may call back into the service
+// (MountFolder, ExecuteBinary, ...). That callback arrives on a *different* COM
+// RPC thread, so it cannot re-enter the locks held by the notifying thread
+// (m_instanceLock etc.) without a second, parallel locking scheme.
+//
+// Solution (matches the in-process model): make the outbound notification on a
+// worker thread and "pump" the plugin's service-side API calls back onto the
+// ORIGINAL notifying thread. Because the work then runs on the lock-holding
+// thread, recursive locks (std::recursive_timed_mutex m_instanceLock) re-enter
+// exactly as they did when plugins were loaded in-process — no second lock
+// (m_callbackLock) and no out-of-band session registry are needed.
+//
+//   Notifying thread:  pump.Run([&]{ return host->OnVMStarted(...); });
+//   RPC callback:      return pump.Invoke([&]{ return session->Mount...(); });
+//
+// A single pump instance services one outbound notification at a time. The
+// pump thread is the single consumer; any number of RPC threads may Invoke().
+//
+class PluginCallPump
+{
+public:
+    PluginCallPump();
+    ~PluginCallPump() = default;
+
+    PluginCallPump(const PluginCallPump&) = delete;
+    PluginCallPump& operator=(const PluginCallPump&) = delete;
+    PluginCallPump(PluginCallPump&&) = delete;
+    PluginCallPump& operator=(PluginCallPump&&) = delete;
+
+    // Runs `Notification` (the outbound host->On... COM call) on a dedicated
+    // worker thread and pumps queued Invoke() calls on the CALLING thread until
+    // the worker completes. The worker performs its own COM initialization, so
+    // `Notification` should acquire the apartment-local host proxy itself.
+    //
+    // Returns the HRESULT returned by `Notification`. Any service-side work
+    // requested by the plugin runs on the calling (lock-holding) thread, so
+    // recursive locks behave exactly as in the in-process model.
+    HRESULT Run(const std::function<HRESULT()>& Notification);
+
+    // Called from a COM RPC callback thread. Marshals `Work` to the pump thread,
+    // blocks until it has executed there, and reports its HRESULT via `Result`.
+    // Returns true if `Work` was executed (`Result` is set); returns false if the
+    // pump is no longer running (the notification already returned), in which case
+    // `Work` was NOT run and the caller must run it itself. Reporting "not run"
+    // out-of-band (rather than via a sentinel HRESULT) lets `Work` legitimately
+    // return any HRESULT, including RPC_E_DISCONNECTED, without ambiguity.
+    bool Invoke(std::function<HRESULT()> Work, _Out_ HRESULT& Result);
+
+private:
+    struct Call
+    {
+        std::function<HRESULT()> work;
+        HRESULT result{E_FAIL};
+        wil::unique_event_nothrow done;
+    };
+
+    // Runs every currently-queued call on the calling (pump) thread.
+    void DrainQueue();
+
+    wil::srwlock m_lock;
+    _Guarded_by_(m_lock) std::deque<Call*> m_queue;
+    _Guarded_by_(m_lock) bool m_stopped { false };
+
+    // Auto-reset event signaled whenever a call is enqueued.
+    wil::unique_event m_callAvailable{wil::EventOptions::None};
+};
+
+} // namespace wsl::windows::service