Isolate plugins in out-of-process COM host

Plugin DLLs are now loaded in isolated wslpluginhost.exe processes
instead of directly in wslservice.exe via LoadLibrary. This prevents
a buggy or malicious plugin from crashing the WSL service.

Architecture:
- New IWslPluginHost/IWslPluginHostCallback COM interfaces
  (WslPluginHost.idl) for cross-process plugin lifecycle management
- New wslpluginhost.exe: COM local server (REGCLS_SINGLEUSE), one
  per plugin, loads the plugin DLL and dispatches notifications
- Refactored PluginManager: CoCreateInstance replaces LoadLibrary,
  PluginError returned via [out] parameter, crash recovery via
  IsHostCrash() detecting RPC_E_DISCONNECTED/SERVER_DIED

Callback safety:
- Plugin callbacks (MountFolder, ExecuteBinary) arrive on a different
  COM RPC thread and use std::shared_lock(m_callbackLock) instead of
  m_instanceLock to avoid re-entrancy deadlocks
- _VmTerminate takes exclusive m_callbackLock before destroying the
  VM, blocking until in-flight callbacks complete
- Lock ordering: m_instanceLock -> m_callbackLock (never reverse)
- All writes to m_runningInstances take m_callbackLock exclusive to
  prevent data races with concurrent callback reads

Security:
- COM AppID with SYSTEM-only launch/access permissions
- Plugin signature validation (ValidateFileSignature) keeps the file
  handle open until after LoadLibrary to prevent TOCTOU attacks
- Plugin host processes use minimal access rights for handles

Process lifecycle:
- Plugin hosts added to a job object with KILL_ON_JOB_CLOSE for
  automatic cleanup if wslservice exits
- g_pluginHost is process-wide (REGCLS_SINGLEUSE guarantees one
  plugin per process), nulled on destruction to prevent UAF
- std::call_once for thread-safe initialization and job creation

Packaging:
- WslPluginHost.idl compiled into existing wslserviceproxystub.dll
- MSI: COM class/interface registration, AppID security, proxy/stub
- wslpluginhost.exe added to build/signing pipeline, WER crash dump
  list, LSP registration, and test validation

Plugins are not loaded for WSL1-only sessions since all plugin hooks
require a WSL2 VM. WslPluginApi.h is unchanged - existing plugin DLLs
work unmodified.

Author: Ben Hillis

.pipelines/build-stage.yml

@@ -27,8 +27,8 @@ parameters:
   - name: targets
     type: object
     default:
-      - target: "wsl;libwsl;wslg;wslservice;wslhost;wslrelay;wslinstaller;wslinstall;initramfs;wslserviceproxystub;wslsettings;wslinstallerproxystub;testplugin"
-        pattern: "wsl.exe,libwsl.dll,wslg.exe,wslservice.exe,wslhost.exe,wslrelay.exe,wslinstaller.exe,wslinstall.dll,wslserviceproxystub.dll,wslsettings/wslsettings.dll,wslsettings/wslsettings.exe,wslinstallerproxystub.dll,WSLDVCPlugin.dll,testplugin.dll,wsldeps.dll"
+      - target: "wsl;libwsl;wslg;wslservice;wslhost;wslrelay;wslpluginhost;wslinstaller;wslinstall;initramfs;wslserviceproxystub;wslsettings;wslinstallerproxystub;testplugin"
+        pattern: "wsl.exe,libwsl.dll,wslg.exe,wslservice.exe,wslhost.exe,wslrelay.exe,wslpluginhost.exe,wslinstaller.exe,wslinstall.dll,wslserviceproxystub.dll,wslsettings/wslsettings.dll,wslsettings/wslsettings.exe,wslinstallerproxystub.dll,WSLDVCPlugin.dll,testplugin.dll,wsldeps.dll"
       - target: "msixgluepackage"
         pattern: "gluepackage.msix"
       - target: "msipackage"

CMakeLists.txt

@@ -486,6 +486,7 @@ add_subdirectory(src/windows/wsl)
 add_subdirectory(src/windows/wslg)
 add_subdirectory(src/windows/wslhost)
 add_subdirectory(src/windows/wslrelay)
+add_subdirectory(src/windows/wslpluginhost)
 add_subdirectory(src/windows/wslinstall)
 
 if (WSL_BUILD_WSL_SETTINGS)

msipackage/CMakeLists.txt

@@ -12,7 +12,7 @@ set(OUTPUT_PACKAGE ${BIN}/wsl.msi)
 set(PACKAGE_WIX_IN ${CMAKE_CURRENT_LIST_DIR}/package.wix.in)
 set(PACKAGE_WIX ${BIN}/package.wix)
 set(CAB_CACHE ${BIN}/cab)
-set(WINDOWS_BINARIES wsl.exe;wslg.exe;wslhost.exe;wslrelay.exe;wslservice.exe;wslserviceproxystub.dll;wslinstall.dll)
+set(WINDOWS_BINARIES wsl.exe;wslg.exe;wslhost.exe;wslrelay.exe;wslpluginhost.exe;wslservice.exe;wslserviceproxystub.dll;wslinstall.dll)
 if (WSL_BUILD_WSL_SETTINGS)
     list(APPEND WINDOWS_BINARIES "wslsettings/wslsettings.dll;wslsettings/wslsettings.exe;libwsl.dll")
 endif()
@@ -52,7 +52,7 @@ add_custom_command(
 
 add_custom_target(msipackage DEPENDS ${OUTPUT_PACKAGE})
 set_target_properties(msipackage PROPERTIES EXCLUDE_FROM_ALL FALSE SOURCES ${PACKAGE_WIX_IN})
-add_dependencies(msipackage wsl wslg wslservice wslhost wslrelay wslserviceproxystub init initramfs wslinstall msixgluepackage)
+add_dependencies(msipackage wsl wslg wslservice wslhost wslrelay wslpluginhost wslserviceproxystub init initramfs wslinstall msixgluepackage)
 
 if (WSL_BUILD_WSL_SETTINGS)
     add_dependencies(msipackage wslsettings libwsl)

msipackage/package.wix.in

@@ -29,6 +29,7 @@
                 <File Id="wslg.exe" Name="wslg.exe" Source="${PACKAGE_INPUT_DIR}/wslg.exe" />
                 <File Id="wslhost.exe" Name="wslhost.exe" Source="${PACKAGE_INPUT_DIR}/wslhost.exe" />
                 <File Id="wslrelay.exe" Name="wslrelay.exe" Source="${PACKAGE_INPUT_DIR}/wslrelay.exe" />
+                <File Id="wslpluginhost.exe" Name="wslpluginhost.exe" Source="${PACKAGE_INPUT_DIR}/wslpluginhost.exe" />
                 <File Id="wslserviceproxystub.dll" Name="wslserviceproxystub.dll" Source="${PACKAGE_INPUT_DIR}/wslserviceproxystub.dll" />
                 <File Id="wsldeps.dll" Name="wsldeps.dll" Source="${PACKAGE_INPUT_DIR}/wsldeps.dll" />
 
@@ -159,6 +160,35 @@
                     <RegistryValue Value='"[INSTALLDIR]wslhost.exe"' Type="string" />
                 </RegistryKey>
 
+                <!-- WslPluginHost - out-of-process plugin isolation (process spawned by service) -->
+                <!-- WslPluginHost AppID — SYSTEM-only activation -->
+                <!-- O:SYG:SYD:(A;;CCDCSW;;;SY) -->
+                <RegistryKey Root="HKCR" Key="AppID\{7a1d2c3e-4b5f-6a7d-8e9f-0a1b2c3d4e5f}">
+                    <RegistryValue Name="AccessPermission" Value="010004801400000020000000000000002C00000001010000000000051200000001010000000000051200000002001C0001000000000014000B000000010100000000000512000000" Type="binary" />
+                    <RegistryValue Name="LaunchPermission" Value="010004801400000020000000000000002C00000001010000000000051200000001010000000000051200000002001C0001000000000014000B000000010100000000000512000000" Type="binary" />
+                </RegistryKey>
+                <RegistryKey Root="HKCR" Key="CLSID\{7a1d2c3e-4b5f-6a7d-8e9f-0a1b2c3d4e5f}">
+                    <RegistryValue Name="AppId" Value="{7a1d2c3e-4b5f-6a7d-8e9f-0a1b2c3d4e5f}" Type="string" />
+                    <RegistryValue Value="WslPluginHost" Type="string" />
+                </RegistryKey>
+                <RegistryKey Root="HKCR" Key="CLSID\{7a1d2c3e-4b5f-6a7d-8e9f-0a1b2c3d4e5f}\LocalServer32">
+                    <RegistryValue Value='"[INSTALLDIR]wslpluginhost.exe"' Type="string" />
+                </RegistryKey>
+
+                <!-- WslPluginHost interfaces — use the shared wslserviceproxystub.dll -->
+                <RegistryKey Root="HKCR" Key="Interface\{A2B3C4D5-E6F7-4890-AB12-CD34EF56A789}">
+                    <RegistryValue Value="IWslPluginHostCallback" Type="string" />
+                    <RegistryKey Key="ProxyStubClsid32">
+                        <RegistryValue Value="{4EA0C6DD-E9FF-48E7-994E-13A31D10DC60}" Type="string" />
+                    </RegistryKey>
+                </RegistryKey>
+                <RegistryKey Root="HKCR" Key="Interface\{B3C4D5E6-F7A8-4901-BC23-DE45FA67B890}">
+                    <RegistryValue Value="IWslPluginHost" Type="string" />
+                    <RegistryKey Key="ProxyStubClsid32">
+                        <RegistryValue Value="{4EA0C6DD-E9FF-48E7-994E-13A31D10DC60}" Type="string" />
+                    </RegistryKey>
+                </RegistryKey>
+
                 <!-- wsldevicehost.dll -->
                 <RegistryKey Root="HKCR" Key="AppID\{17696EAC-9568-4CF5-BB8C-82515AAD6C09}">
                     <RegistryValue Name="DllSurrogate" Value="" Type="string" />

src/windows/common/precomp.h

@@ -83,6 +83,7 @@ Module Name:
 #include <optional>
 #include <filesystem>
 #include <mutex>
+#include <shared_mutex>
 #include <chrono>
 #include <codecvt>
 #include <random>

src/windows/service/exe/CMakeLists.txt

@@ -50,7 +50,7 @@ set(HEADERS
     WslCoreVm.h)
 
 add_executable(wslservice ${SOURCES} ${HEADERS})
-add_dependencies(wslservice wslserviceidl wslservicemc)
+add_dependencies(wslservice wslserviceidl wslservicemc wslpluginhostidl)
 add_compile_definitions(__WRL_CLASSIC_COM__)
 add_compile_definitions(__WRL_DISABLE_STATIC_INITIALIZE__)
 add_compile_definitions(USE_COM_CONTEXT_DEF=1)

src/windows/service/exe/LxssUserSession.cpp

@@ -2604,13 +2604,18 @@ std::shared_ptr<LxssRunningInstance> LxssUserSessionImpl::_CreateInstance(_In_op
                     registration.Write(Property::OsVersion, distributionInfo->Version);
                 }
 
-                // This needs to be done before plugins are notifed because they might try to run a command inside the distribution.
-                m_runningInstances[registration.Id()] = instance;
+                // This needs to be done before plugins are notified because they might try to run a command inside the distribution.
+                {
+                    std::unique_lock callbackLock(m_callbackLock);
+                    m_runningInstances[registration.Id()] = instance;
+                }
 
                 if (version == LXSS_WSL_VERSION_2)
                 {
-                    auto cleanupOnFailure =
-                        wil::scope_exit_log(WI_DIAGNOSTICS_INFO, [&]() { m_runningInstances.erase(registration.Id()); });
+                    auto cleanupOnFailure = wil::scope_exit_log(WI_DIAGNOSTICS_INFO, [&]() {
+                        std::unique_lock callbackLock(m_callbackLock);
+                        m_runningInstances.erase(registration.Id());
+                    });
                     m_pluginManager.OnDistributionStarted(&m_session, instance->DistributionInformation());
                     cleanupOnFailure.release();
                 }
@@ -3580,14 +3585,18 @@ bool LxssUserSessionImpl::_TerminateInstanceInternal(_In_ LPCGUID DistroGuid, _I
                 instance->second->Stop();
 
                 const auto clientId = instance->second->GetClientId();
-                {
-                    auto lock = m_terminatedInstanceLock.lock_exclusive();
-                    m_terminatedInstances.push_back(std::move(instance->second));
-                }
 
                 m_lifetimeManager.RemoveCallback(clientKey);
 
-                m_runningInstances.erase(instance);
+                {
+                    std::unique_lock callbackLock(m_callbackLock);
+                    {
+                        auto lock = m_terminatedInstanceLock.lock_exclusive();
+                        m_terminatedInstances.push_back(std::move(instance->second));
+                    }
+
+                    m_runningInstances.erase(instance);
+                }
 
                 // If the instance that was terminated was a WSL2 instance,
                 // check if the VM is now idle.
@@ -3615,7 +3624,10 @@ void LxssUserSessionImpl::_UpdateInit(_In_ const LXSS_DISTRO_CONFIGURATION& Conf
 
 HRESULT LxssUserSessionImpl::MountRootNamespaceFolder(_In_ LPCWSTR HostPath, _In_ LPCWSTR GuestPath, _In_ bool ReadOnly, _In_ LPCWSTR Name)
 {
-    std::lock_guard lock(m_instanceLock);
+    // Shared lock prevents _VmTerminate from destroying the VM while we use it.
+    // Do NOT acquire m_instanceLock — callbacks arrive on a different COM thread
+    // from the notification thread that holds m_instanceLock.
+    std::shared_lock lock(m_callbackLock);
     RETURN_HR_IF(E_NOT_VALID_STATE, !m_utilityVm);
 
     m_utilityVm->MountRootNamespaceFolder(HostPath, GuestPath, ReadOnly, Name);
@@ -3624,7 +3636,9 @@ HRESULT LxssUserSessionImpl::MountRootNamespaceFolder(_In_ LPCWSTR HostPath, _In
 
 HRESULT LxssUserSessionImpl::CreateLinuxProcess(_In_opt_ const GUID* Distro, _In_ LPCSTR Path, _In_ LPCSTR* Arguments, _Out_ SOCKET* Socket)
 {
-    std::lock_guard lock(m_instanceLock);
+    // Shared lock prevents _VmTerminate from destroying the VM or instances
+    // while we use them. See MountRootNamespaceFolder for rationale.
+    std::shared_lock lock(m_callbackLock);
     RETURN_HR_IF(E_NOT_VALID_STATE, !m_utilityVm);
 
     if (Distro == nullptr)
@@ -3871,7 +3885,12 @@ void LxssUserSessionImpl::_VmTerminate()
         m_telemetryThread.join();
     }
 
-    m_utilityVm.reset();
+    // Acquire exclusive callback lock to wait for any in-flight plugin callbacks
+    // (MountRootNamespaceFolder, CreateLinuxProcess) to complete before destroying the VM.
+    {
+        std::unique_lock callbackLock(m_callbackLock);
+        m_utilityVm.reset();
+    }
     m_vmId.store(GUID_NULL);
 
     // Reset the user's token since its lifetime is tied to the VM.

src/windows/service/exe/LxssUserSession.h

@@ -310,6 +310,10 @@ class DECLSPEC_UUID("a9b7a1b9-0671-405c-95f1-e0612cb4ce7e") LxssUserSession
 /// </summary>
 class LxssUserSessionImpl
 {
+    // Plugin callbacks arrive on a different COM RPC thread and use m_callbackLock
+    // (shared) instead of m_instanceLock to access m_utilityVm and m_runningInstances.
+    friend class wsl::windows::service::PluginHostCallbackImpl;
+
 public:
     LxssUserSessionImpl(_In_ PSID userSid, _In_ DWORD sessionId, _Inout_ wsl::windows::service::PluginManager& pluginManager);
     virtual ~LxssUserSessionImpl();
@@ -363,11 +367,6 @@ class LxssUserSessionImpl
     /// </summary>
     void ClearDiskStateInRegistry(_In_opt_ LPCWSTR Disk);
 
-    /// <summary>
-    /// Start a process in the root namespace or in a user distribution.
-    /// </summary>
-    HRESULT CreateLinuxProcess(_In_opt_ const GUID* Distro, _In_ LPCSTR Path, _In_ LPCSTR* Arguments, _Out_ SOCKET* socket);
-
     /// <summary>
     /// Enumerates registered distributions, optionally including ones that are
     /// currently being registered, unregistered, or converted.
@@ -443,8 +442,6 @@ class LxssUserSessionImpl
 
     HRESULT MoveDistribution(_In_ LPCGUID DistroGuid, _In_ LPCWSTR Location);
 
-    HRESULT MountRootNamespaceFolder(_In_ LPCWSTR HostPath, _In_ LPCWSTR GuestPath, _In_ bool ReadOnly, _In_ LPCWSTR Name);
-
     /// <summary>
     /// Registers a distribution.
     /// </summary>
@@ -533,6 +530,18 @@ class LxssUserSessionImpl
     static CreateLxProcessContext s_GetCreateProcessContext(_In_ const GUID& DistroGuid, _In_ bool SystemDistro);
 
 private:
+    /// <summary>
+    /// Plugin callback methods — called from PluginHostCallbackImpl on a COM RPC
+    /// thread during plugin notifications. These acquire m_callbackLock (shared)
+    /// instead of m_instanceLock, preventing _VmTerminate from destroying the VM
+    /// while a callback is in-flight. Access is restricted via friend declaration.
+    /// </summary>
+    _Requires_lock_not_held_(m_instanceLock)
+    HRESULT MountRootNamespaceFolder(_In_ LPCWSTR HostPath, _In_ LPCWSTR GuestPath, _In_ bool ReadOnly, _In_ LPCWSTR Name);
+
+    _Requires_lock_not_held_(m_instanceLock)
+    HRESULT CreateLinuxProcess(_In_opt_ const GUID* Distro, _In_ LPCSTR Path, _In_ LPCSTR* Arguments, _Out_ SOCKET* socket);
+
     /// <summary>
     /// Adds a distro to the list of converting distros.
     /// </summary>
@@ -794,7 +803,9 @@ class LxssUserSessionImpl
     std::recursive_timed_mutex m_instanceLock;
 
     /// <summary>
-    /// Contains the currently running utility VM's.
+    /// Contains the currently running instances.
+    /// Reads guarded by m_instanceLock OR m_callbackLock (shared).
+    /// Mutations require BOTH m_instanceLock AND m_callbackLock (exclusive).
     /// </summary>
     _Guarded_by_(m_instanceLock) std::map<GUID, std::shared_ptr<LxssRunningInstance>, wsl::windows::common::helpers::GuidLess> m_runningInstances;
 
@@ -811,9 +822,24 @@ class LxssUserSessionImpl
 
     /// <summary>
     /// The running utility vm for WSL2 distributions.
-    ///
+    /// Reads guarded by m_instanceLock OR m_callbackLock (shared).
+    /// Mutations require BOTH m_instanceLock AND m_callbackLock (exclusive).
+    /// </summary>
     _Guarded_by_(m_instanceLock) std::unique_ptr<WslCoreVm> m_utilityVm;
 
+    /// <summary>
+    /// Reader-writer lock protecting m_utilityVm and m_runningInstances for
+    /// plugin callbacks. Callbacks take a shared (read) lock; _VmTerminate and
+    /// instance mutations take an exclusive (write) lock.
+    ///
+    /// Mutations of m_runningInstances/m_utilityVm require BOTH m_instanceLock
+    /// AND m_callbackLock (exclusive). Reads are safe under either lock alone.
+    ///
+    /// Lock ordering: m_instanceLock → m_callbackLock (never reverse).
+    /// Callbacks must NEVER acquire m_instanceLock (deadlock with notification thread).
+    /// </summary>
+    std::shared_mutex m_callbackLock;
+
     std::atomic<GUID> m_vmId{GUID_NULL};
 
     /// <summary>