chore(tests): Added test variant flag (#18458)

I have been running a lot of Next.js tests and needed to run specific
variants against my changes, so I made these changes to our test runner
script and thought to share it with everyone.

```
yarn test:run <app-name> --variant <variant-name>
```

Closes #18459 (added automatically)

Author: logaretm

dev-packages/e2e-tests/README.md

@@ -25,6 +25,37 @@ Or run only a single E2E test app:
 yarn test:run <app-name>
 ```
 
+Or you can run a single E2E test app with a specific variant:
+
+```bash
+yarn test:run <app-name> --variant <variant-name>
+```
+
+Variant name matching is case-insensitive and partial. For example, `--variant 13` will match `nextjs-pages-dir (next@13)` if a matching variant is present in the test app's `package.json`.
+
+For example, if you have the following variants in your test app's `package.json`:
+
+```json
+"sentryTest": {
+  "variants": [
+    {
+      "build-command": "pnpm test:build-13",
+      "label": "nextjs-pages-dir (next@13)"
+    },
+    {
+      "build-command": "pnpm test:build-13-canary",
+      "label": "nextjs-pages-dir (next@13-canary)"
+    },
+    {
+      "build-command": "pnpm test:build-15",
+      "label": "nextjs-pages-dir (next@15)"
+    }
+  ]
+}
+```
+
+If you run `yarn test:run nextjs-pages-dir --variant 13`, it will match against the very first matching variant, which is `nextjs-pages-dir (next@13)`. If you need to target the second variant in the example, you need to be more specific and use `--variant 13-canary`.
+
 ## How they work
 
 Before running any tests we launch a fake test registry (in our case [Verdaccio](https://verdaccio.org/docs/e2e/)), we

dev-packages/e2e-tests/run.ts

@@ -1,13 +1,26 @@
 /* eslint-disable no-console */
 import { spawn } from 'child_process';
 import * as dotenv from 'dotenv';
-import { mkdtemp, rm } from 'fs/promises';
+import { mkdtemp, readFile, rm } from 'fs/promises';
 import { sync as globSync } from 'glob';
 import { tmpdir } from 'os';
 import { join, resolve } from 'path';
 import { copyToTemp } from './lib/copyToTemp';
 import { registrySetup } from './registrySetup';
 
+interface SentryTestVariant {
+  'build-command': string;
+  'assert-command'?: string;
+  label?: string;
+}
+
+interface PackageJson {
+  sentryTest?: {
+    variants?: SentryTestVariant[];
+    optionalVariants?: SentryTestVariant[];
+  };
+}
+
 const DEFAULT_DSN = 'https://username@domain/123';
 const DEFAULT_SENTRY_ORG_SLUG = 'sentry-javascript-sdks';
 const DEFAULT_SENTRY_PROJECT = 'sentry-javascript-e2e-tests';
@@ -58,14 +71,100 @@ function asyncExec(
   });
 }
 
+function findMatchingVariant(variants: SentryTestVariant[], variantLabel: string): SentryTestVariant | undefined {
+  const variantLabelLower = variantLabel.toLowerCase();
+
+  return variants.find(variant => variant.label?.toLowerCase().includes(variantLabelLower));
+}
+
+async function getVariantBuildCommand(
+  packageJsonPath: string,
+  variantLabel: string,
+  testAppPath: string,
+): Promise<{ buildCommand: string; assertCommand: string; testLabel: string; matchedVariantLabel?: string }> {
+  try {
+    const packageJsonContent = await readFile(packageJsonPath, 'utf-8');
+    const packageJson: PackageJson = JSON.parse(packageJsonContent);
+
+    const allVariants = [
+      ...(packageJson.sentryTest?.variants || []),
+      ...(packageJson.sentryTest?.optionalVariants || []),
+    ];
+
+    const matchingVariant = findMatchingVariant(allVariants, variantLabel);
+
+    if (matchingVariant) {
+      return {
+        buildCommand: matchingVariant['build-command'] || 'pnpm test:build',
+        assertCommand: matchingVariant['assert-command'] || 'pnpm test:assert',
+        testLabel: matchingVariant.label || testAppPath,
+        matchedVariantLabel: matchingVariant.label,
+      };
+    }
+
+    console.log(`No matching variant found for "${variantLabel}" in ${testAppPath}, using default build`);
+  } catch {
+    console.log(`Could not read variants from package.json for ${testAppPath}, using default build`);
+  }
+
+  return {
+    buildCommand: 'pnpm test:build',
+    assertCommand: 'pnpm test:assert',
+    testLabel: testAppPath,
+  };
+}
+
 async function run(): Promise<void> {
   // Load environment variables from .env file locally
   dotenv.config();
 
   // Allow to run a single app only via `yarn test:run <app-name>`
   const appName = process.argv[2] || '';
   // Forward any additional flags to the test command
-  const testFlags = process.argv.slice(3);
+  const allTestFlags = process.argv.slice(3);
+
+  // Check for --variant flag
+  let variantLabel: string | undefined;
+  let skipNextFlag = false;
+
+  const testFlags = allTestFlags.filter((flag, index) => {
+    // Skip this flag if it was marked to skip (variant value after --variant)
+    if (skipNextFlag) {
+      skipNextFlag = false;
+      return false;
+    }
+
+    // Handle --variant=<value> format
+    if (flag.startsWith('--variant=')) {
+      const value = flag.slice('--variant='.length);
+      const trimmedValue = value?.trim();
+      if (trimmedValue) {
+        variantLabel = trimmedValue;
+      } else {
+        console.warn('Warning: --variant= specified but no value provided. Ignoring variant flag.');
+      }
+      return false; // Remove this flag from testFlags
+    }
+
+    // Handle --variant <value> format
+    if (flag === '--variant') {
+      if (index + 1 < allTestFlags.length) {
+        const value = allTestFlags[index + 1];
+        const trimmedValue = value?.trim();
+        if (trimmedValue) {
+          variantLabel = trimmedValue;
+          skipNextFlag = true; // Mark next flag to be skipped
+        } else {
+          console.warn('Warning: --variant specified but no value provided. Ignoring variant flag.');
+        }
+      } else {
+        console.warn('Warning: --variant specified but no value provided. Ignoring variant flag.');
+      }
+      return false;
+    }
+
+    return true;
+  });
 
   const dsn = process.env.E2E_TEST_DSN || DEFAULT_DSN;
 
@@ -107,13 +206,42 @@ async function run(): Promise<void> {
 
       await copyToTemp(originalPath, tmpDirPath);
       const cwd = tmpDirPath;
+      // Resolve variant if needed
+      const { buildCommand, assertCommand, testLabel, matchedVariantLabel } = variantLabel
+        ? await getVariantBuildCommand(join(tmpDirPath, 'package.json'), variantLabel, testAppPath)
+        : {
+            buildCommand: 'pnpm test:build',
+            assertCommand: 'pnpm test:assert',
+            testLabel: testAppPath,
+          };
+
+      // Print which variant we're using if found
+      if (matchedVariantLabel) {
+        console.log(`\n\nUsing variant: "${matchedVariantLabel}"\n\n`);
+      }
 
-      console.log(`Building ${testAppPath} in ${tmpDirPath}...`);
-      await asyncExec('volta run pnpm test:build', { env, cwd });
+      console.log(`Building ${testLabel} in ${tmpDirPath}...`);
+      await asyncExec(`volta run ${buildCommand}`, { env, cwd });
 
-      console.log(`Testing ${testAppPath}...`);
-      // Pass command and arguments as an array to prevent command injection
-      const testCommand = ['volta', 'run', 'pnpm', 'test:assert', ...testFlags];
+      console.log(`Testing ${testLabel}...`);
+      // Pass command as a string to support shell features (env vars, operators like &&)
+      // This matches how buildCommand is handled for consistency
+      // Properly quote test flags to preserve spaces and special characters
+      const quotedTestFlags = testFlags.map(flag => {
+        // If flag contains spaces or special shell characters, quote it
+        if (
+          flag.includes(' ') ||
+          flag.includes('"') ||
+          flag.includes("'") ||
+          flag.includes('$') ||
+          flag.includes('`')
+        ) {
+          // Escape single quotes and wrap in single quotes (safest for shell)
+          return `'${flag.replace(/'/g, "'\\''")}'`;
+        }
+        return flag;
+      });
+      const testCommand = `volta run ${assertCommand}${quotedTestFlags.length > 0 ? ` ${quotedTestFlags.join(' ')}` : ''}`;
       await asyncExec(testCommand, { env, cwd });
 
       // clean up (although this is tmp, still nice to do)