fix(test-adapter): run settings parsing, launch options copy, launch options for remote browsers

src/Playwright.MSTest/BrowserService.cs

@@ -53,12 +53,15 @@ private static async Task<IBrowser> CreateBrowser(IBrowserType browserType, (str
         {
             var options = new BrowserTypeConnectOptions(connectOptions?.Options ?? new());
             var headers = options.Headers?.ToDictionary(kvp => kvp.Key, kvp => kvp.Value) ?? [];
-            headers.Add("x-playwright-launch-options", JsonSerializer.Serialize(PlaywrightSettingsProvider.LaunchOptions, new JsonSerializerOptions() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }));
+            if (!headers.Keys.Any(name => string.Equals(name, "x-playwright-launch-options", StringComparison.OrdinalIgnoreCase)))
+            {
+                headers["x-playwright-launch-options"] = JsonSerializer.Serialize(launchOptions ?? PlaywrightSettingsProvider.LaunchOptions, new JsonSerializerOptions() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull });
+            }
             options.Headers = headers;
             return await browserType.ConnectAsync(connectOptions!.Value.WSEndpoint, options).ConfigureAwait(false);
         }
 
-        var legacyBrowser = await ConnectBasedOnEnv(browserType);
+        var legacyBrowser = await ConnectBasedOnEnv(browserType, launchOptions);
         if (legacyBrowser != null)
         {
             return legacyBrowser;
@@ -67,7 +70,7 @@ private static async Task<IBrowser> CreateBrowser(IBrowserType browserType, (str
     }
 
     // TODO: Remove at some point
-    private static async Task<IBrowser?> ConnectBasedOnEnv(IBrowserType browserType)
+    private static async Task<IBrowser?> ConnectBasedOnEnv(IBrowserType browserType, BrowserTypeLaunchOptions? launchOptions)
     {
         var accessToken = Environment.GetEnvironmentVariable("PLAYWRIGHT_SERVICE_ACCESS_TOKEN");
         var serviceUrl = Environment.GetEnvironmentVariable("PLAYWRIGHT_SERVICE_URL");
@@ -90,7 +93,7 @@ private static async Task<IBrowser> CreateBrowser(IBrowserType browserType, (str
             Headers = new Dictionary<string, string>
             {
                 ["Authorization"] = $"Bearer {accessToken}",
-                ["x-playwright-launch-options"] = JsonSerializer.Serialize(PlaywrightSettingsProvider.LaunchOptions, new JsonSerializerOptions() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull })
+                ["x-playwright-launch-options"] = JsonSerializer.Serialize(launchOptions ?? PlaywrightSettingsProvider.LaunchOptions, new JsonSerializerOptions() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull })
             }
         }).ConfigureAwait(false);
     }

src/Playwright.NUnit/BrowserService.cs

@@ -53,12 +53,15 @@ private static async Task<IBrowser> CreateBrowser(IBrowserType browserType, (str
         {
             var options = new BrowserTypeConnectOptions(connectOptions?.Options ?? new());
             var headers = options.Headers?.ToDictionary(kvp => kvp.Key, kvp => kvp.Value) ?? [];
-            headers.Add("x-playwright-launch-options", JsonSerializer.Serialize(PlaywrightSettingsProvider.LaunchOptions, new JsonSerializerOptions() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }));
+            if (!headers.Keys.Any(name => string.Equals(name, "x-playwright-launch-options", StringComparison.OrdinalIgnoreCase)))
+            {
+                headers["x-playwright-launch-options"] = JsonSerializer.Serialize(launchOptions ?? PlaywrightSettingsProvider.LaunchOptions, new JsonSerializerOptions() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull });
+            }
             options.Headers = headers;
             return await browserType.ConnectAsync(connectOptions!.Value.WSEndpoint, options).ConfigureAwait(false);
         }
 
-        var legacyBrowser = await ConnectBasedOnEnv(browserType);
+        var legacyBrowser = await ConnectBasedOnEnv(browserType, launchOptions);
         if (legacyBrowser != null)
         {
             return legacyBrowser;
@@ -67,7 +70,7 @@ private static async Task<IBrowser> CreateBrowser(IBrowserType browserType, (str
     }
 
     // TODO: Remove at some point
-    private static async Task<IBrowser?> ConnectBasedOnEnv(IBrowserType browserType)
+    private static async Task<IBrowser?> ConnectBasedOnEnv(IBrowserType browserType, BrowserTypeLaunchOptions? launchOptions)
     {
         var accessToken = Environment.GetEnvironmentVariable("PLAYWRIGHT_SERVICE_ACCESS_TOKEN");
         var serviceUrl = Environment.GetEnvironmentVariable("PLAYWRIGHT_SERVICE_URL");
@@ -90,7 +93,7 @@ private static async Task<IBrowser> CreateBrowser(IBrowserType browserType, (str
             Headers = new Dictionary<string, string>
             {
                 ["Authorization"] = $"Bearer {accessToken}",
-                ["x-playwright-launch-options"] = JsonSerializer.Serialize(PlaywrightSettingsProvider.LaunchOptions, new JsonSerializerOptions() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull })
+                ["x-playwright-launch-options"] = JsonSerializer.Serialize(launchOptions ?? PlaywrightSettingsProvider.LaunchOptions, new JsonSerializerOptions() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull })
             }
         }).ConfigureAwait(false);
     }

src/Playwright.TestAdapter/CompatibilitySuppressions.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- https://learn.microsoft.com/dotnet/fundamentals/package-validation/diagnostic-ids -->
+<Suppressions xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Microsoft.Playwright.TestAdapter.PlaywrightSettingsXml.get_Retries</Target>
+    <Left>lib/netstandard2.0/Microsoft.Playwright.TestAdapter.dll</Left>
+    <Right>lib/netstandard2.0/Microsoft.Playwright.TestAdapter.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
+</Suppressions>

src/Playwright.TestAdapter/PlaywrightSettingsProvider.cs

@@ -116,15 +116,17 @@ public static BrowserTypeLaunchOptions LaunchOptions
     {
         get
         {
-            var launchOptions = _settings?.LaunchOptions ?? new BrowserTypeLaunchOptions();
+            var launchOptions = new BrowserTypeLaunchOptions(_settings?.LaunchOptions ?? new());
             if (Environment.GetEnvironmentVariable("HEADED") == "1")
             {
                 launchOptions.Headless = false;
             }
+#pragma warning disable CS0618 // Type or member is obsolete
             else if (_settings != null && _settings.Headless.HasValue)
             {
                 launchOptions.Headless = _settings.Headless.Value;
             }
+#pragma warning restore CS0618 // Type or member is obsolete
             return launchOptions;
         }
     }

src/Playwright.TestAdapter/PlaywrightSettingsXml.cs

@@ -41,8 +41,9 @@ public PlaywrightSettingsXml()
 
     public PlaywrightSettingsXml(XmlReader reader)
     {
-        // Skip Playwright root Element
-        reader.Read();
+        // Go to the Playwright root element, past any whitespace or declaration before it.
+        // The loop below then reads its children.
+        reader.MoveToContent();
         while (reader.Read())
         {
             if (reader.NodeType != XmlNodeType.Element)
@@ -79,11 +80,30 @@ public PlaywrightSettingsXml(XmlReader reader)
                     break;
                 default:
                     Console.WriteLine($"Playwright RunSettings Parsing Error: Playwright>{reader.Name} is not implemented");
+                    SkipElement(reader);
                     break;
             }
         }
     }
 
+    // Moves to the end tag of the element that the reader is on, past everything inside it, so
+    // that the children of an element that is not supported are not read as settings.
+    private static void SkipElement(XmlReader reader)
+    {
+        if (reader.IsEmptyElement)
+        {
+            return;
+        }
+        var depth = reader.Depth;
+        while (reader.Read())
+        {
+            if (reader.NodeType == XmlNodeType.EndElement && reader.Depth == depth)
+            {
+                break;
+            }
+        }
+    }
+
     private static object ParseXmlIntoClass(Type classType, XmlReader reader)
     {
         var endTag = reader.Name;
@@ -101,6 +121,7 @@ private static object ParseXmlIntoClass(Type classType, XmlReader reader)
                 if (property == null)
                 {
                     Console.WriteLine($"Playwright RunSettings Parsing Error: Playwright>{endTag}>{key} is not supported");
+                    SkipElement(reader);
                     continue;
                 }
                 var type = Nullable.GetUnderlyingType(property.PropertyType) ?? property.PropertyType;
@@ -145,11 +166,12 @@ private static void ApplyParameter(string key, string value, object options)
                 break;
             case Type t when t?.IsEnum == true:
                 {
+                    // The value as the docs spell it ("no-preference"), or the C# name in any case ("NoPreference").
                     var enumValue = Enum.GetNames(t).Where(name =>
                     {
                         var field = t.GetField(name);
                         return field.GetCustomAttribute<EnumMemberAttribute>()?.Value == value;
-                    }).FirstOrDefault();
+                    }).FirstOrDefault() ?? Enum.GetNames(t).FirstOrDefault(name => string.Equals(name, value, StringComparison.OrdinalIgnoreCase));
                     if (enumValue == null)
                     {
                         throw new ArgumentException($"Invalid value '{value}' for enum {t.Name}");
@@ -175,17 +197,28 @@ private static void ApplyParameter(string key, string value, object options)
 
     private static object ParseAsJson(string value, Type type)
     {
-        return JsonSerializer.Deserialize(value.Replace('\'', '"'), type)!;
+        // JSON as it is keeps the apostrophes in its strings. The shorthand with single quotes,
+        // ['--a'], is not JSON: its quotes are swapped, as before.
+        try
+        {
+            return JsonSerializer.Deserialize(value, type)!;
+        }
+        catch (JsonException)
+        {
+            return JsonSerializer.Deserialize(value.Replace('\'', '"'), type)!;
+        }
     }
 
     public BrowserTypeLaunchOptions? LaunchOptions { get; set; }
     public BrowserNewContextOptions? ContextOptions { get; set; }
     public string? BrowserName { get; set; }
+
+    [Obsolete("Not read from the run settings. Use LaunchOptions.Headless instead.")]
     public bool? Headless { get; set; }
+
     public float? ExpectTimeout { get; set; }
     public float? ActionTimeout { get; set; }
     public float? NavigationTimeout { get; set; }
     public string? TestIdAttribute { get; set; }
-    public int? Retries { get; set; }
 }
 

src/Playwright.TestingHarnessTest/tests/baseTest.ts

@@ -22,6 +22,7 @@ export const test = base.extend<{
   testMode: 'nunit' | 'mstest' | 'mstest.v4' | 'xunit' | 'xunit.v3';
   runTest: (files: Record<string, string>, command: string, env?: NodeJS.ProcessEnv) => Promise<RunResult>;
   launchServer: (options: { port: number }) => Promise<void>;
+  runServer: (options: { port: number }) => Promise<void>;
   server: SimpleServer;
 }>({
   proxyServer: async ({}, use) => {
@@ -49,6 +50,26 @@ export const test = base.extend<{
     for (const server of servers)
       await server.close();
   },
+  runServer: async ({}, use) => {
+    // Unlike launchServer, which serves one browser that it has launched, `run-server` launches
+    // a browser for each client, with the launch options that the client sends in a header.
+    // --unsafe lets the client pass browser args.
+    const cli = path.join(__dirname, '..', '..', 'Playwright', '.drivers', 'package', 'cli.js');
+    const servers: childProcess.ChildProcess[] = [];
+    await use(async ({port}: {port: number}) => {
+      const server = childProcess.spawn(process.execPath, [cli, 'run-server', '--host', '127.0.0.1', '--port', String(port), '--unsafe'], { stdio: ['ignore', 'pipe', 'inherit'] });
+      servers.push(server);
+      await new Promise<void>((resolve, reject) => {
+        server.stdout!.on('data', data => {
+          if (data.toString().includes('Listening on'))
+            resolve();
+        });
+        server.on('exit', code => reject(new Error(`run-server exited with code ${code}`)));
+      });
+    });
+    for (const server of servers)
+      server.kill();
+  },
   runTest: async ({ testMode }, use, testInfo) => {
     const testResults: RunResult[] = [];
     await use(async (files, command, env) => {

src/Playwright.TestingHarnessTest/tests/nunit.spec.ts

@@ -304,6 +304,49 @@ test('should be able to parse LaunchOptions.Args from runsettings', async ({ run
   expect(result.stdout).toContain("User-Agent: hello")
 });
 
+// See https://github.com/microsoft/playwright-dotnet/issues/3425
+test('should hand out a copy of the launch options from runsettings', async ({ runTest }) => {
+  const result = await runTest({
+    'ExampleTests.cs': `
+      using System;
+      using System.Linq;
+      using NUnit.Framework;
+      using Microsoft.Playwright.NUnit;
+      using Microsoft.Playwright.TestAdapter;
+
+      namespace Playwright.TestingHarnessTest.NUnit;
+
+      public class <class-name> : PageTest
+      {
+          [Test]
+          public void Test()
+          {
+              var first = PlaywrightSettingsProvider.LaunchOptions;
+              first.Args = first.Args.Append("--added-by-a-test");
+              first.Channel = "changed-by-a-test";
+              var second = PlaywrightSettingsProvider.LaunchOptions;
+              Console.WriteLine("Args: " + string.Join(" ", second.Args) + ";");
+              Console.WriteLine("Channel: " + (second.Channel ?? "not set") + ";");
+          }
+      }`,
+      '.runsettings': `
+        <?xml version="1.0" encoding="utf-8"?>
+        <RunSettings>
+            <Playwright>
+              <LaunchOptions>
+                <Args>['--user-agent=hello']</Args>
+              </LaunchOptions>
+            </Playwright>
+        </RunSettings>
+      `,
+  }, 'dotnet test --settings=.runsettings');
+  expect(result.passed).toBe(1);
+  expect(result.failed).toBe(0);
+  expect(result.total).toBe(1);
+  expect(result.stdout).toContain("Args: --user-agent=hello;")
+  expect(result.stdout).toContain("Channel: not set;")
+});
+
 test('should be able to override context options', async ({ runTest, server }) => {
   const result = await runTest({
     'ExampleTests.cs': `
@@ -708,6 +751,89 @@ test.describe('ConnectOptions', () => {
     expect(result.total).toBe(1);
     expect(fs.existsSync(testInfo.outputPath('browser-closed.txt'))).toBe(true);
   });
+
+  // See https://github.com/microsoft/playwright-dotnet/issues/3424
+  test('should send the launch options of the test class to the server', async ({ runTest, runServer }) => {
+    await runServer({ port: 1234 });
+    const result = await runTest({
+      'ExampleTests.cs': `
+      using System;
+      using System.Threading.Tasks;
+      using Microsoft.Playwright;
+      using Microsoft.Playwright.NUnit;
+      using NUnit.Framework;
+
+      namespace Playwright.TestingHarnessTest.NUnit;
+
+      public class <class-name> : PageTest
+      {
+          [Test]
+          public async Task Test()
+          {
+              await Page.GotoAsync("about:blank");
+              Console.WriteLine("User-Agent: " + await Page.EvaluateAsync<string>("() => navigator.userAgent"));
+          }
+
+          public override async Task<(string, BrowserTypeConnectOptions)?> ConnectOptionsAsync()
+          {
+              return ("ws://127.0.0.1:1234", null);
+          }
+
+          public override Task<BrowserTypeLaunchOptions?> LaunchOptionsAsync()
+          {
+              return Task.FromResult<BrowserTypeLaunchOptions?>(new BrowserTypeLaunchOptions
+              {
+                  Args = new[] { "--user-agent=hello" },
+              });
+          }
+      }`,
+    }, 'dotnet test');
+    expect(result.passed).toBe(1);
+    expect(result.failed).toBe(0);
+    expect(result.total).toBe(1);
+    expect(result.stdout).toContain("User-Agent: hello");
+  });
+
+  // See https://github.com/microsoft/playwright-dotnet/issues/3424
+  test('should keep the launch options header that the test class sets itself', async ({ runTest, runServer }) => {
+    await runServer({ port: 1234 });
+    const result = await runTest({
+      'ExampleTests.cs': `
+      using System;
+      using System.Collections.Generic;
+      using System.Threading.Tasks;
+      using Microsoft.Playwright;
+      using Microsoft.Playwright.NUnit;
+      using NUnit.Framework;
+
+      namespace Playwright.TestingHarnessTest.NUnit;
+
+      public class <class-name> : PageTest
+      {
+          [Test]
+          public async Task Test()
+          {
+              await Page.GotoAsync("about:blank");
+              Console.WriteLine("User-Agent: " + await Page.EvaluateAsync<string>("() => navigator.userAgent"));
+          }
+
+          public override async Task<(string, BrowserTypeConnectOptions)?> ConnectOptionsAsync()
+          {
+              return ("ws://127.0.0.1:1234", new BrowserTypeConnectOptions
+              {
+                  Headers = new Dictionary<string, string>
+                  {
+                      ["x-playwright-launch-options"] = "{\\"args\\":[\\"--user-agent=from-the-header\\"]}",
+                  },
+              });
+          }
+      }`,
+    }, 'dotnet test');
+    expect(result.passed).toBe(1);
+    expect(result.failed).toBe(0);
+    expect(result.total).toBe(1);
+    expect(result.stdout).toContain("User-Agent: from-the-header");
+  });
 });
 
 test.describe('LaunchOptions', () => {