Switch registry at CI builds explicitly (#2075)

Dataform uses trusted registry for CI builds. It does that by relying on nodejs's npm install interaction with yarn.lock. This interaction is known to be error-prone (for ex. npm/cli#5126), yarn recommends to only use yarn CLI when working with it's ecosystem. In particular, it blocks update of the Node, forcing us to use the version that is long out of EOL support (v16). Newer version breaks yarn.lock on npm install. Apart from that we don't verify that yarn.lock updates only registry.

Addressing that by reimplementing registry substitution explicitly and switching to yarn install --freeze-lockfile instead of npm install. This will ensure that:

We only change the registry, all other parts of the lock file like versions or hashsums remain the same.
We don't change resolved versions by accident if there is a conflict between package.json spec and lockfile.
Long-term solution would be migration to a supported NodeJS rules (most likely Aspect's), where we would be able to leverage PNPM, whose lockfile doesn't fix registry URL at all - only versions and hashsums.

Author: ikholopov-omni

scripts/publish

@@ -16,7 +16,8 @@ fi
 
 # Run all the tests.
 bazel run @nodejs//:yarn config set registry https://us-npm.pkg.dev/artifact-foundry-prod/ah-3p-staging-npm/
-bazel run @nodejs//:npm install
+bazel run //tools/registry-tools:switch_registry -- $(pwd)/yarn.lock https://us-npm.pkg.dev/artifact-foundry-prod/npm-3p-trusted/
+bazel run @nodejs//:yarn install -- --frozen-lockfile
 bazel test //... --build_tests_only
 
 # After the code is build with Airlock dependencies, we change the registry to public npmjs

scripts/run_tests_on_cloudbuild

@@ -2,7 +2,9 @@
 set -e
 
 bazel run @nodejs//:yarn config set registry https://us-npm.pkg.dev/artifact-foundry-prod/npm-3p-trusted/
-bazel run @nodejs//:npm install
+bazel run //tools/registry-tools:switch_registry -- $(pwd)/yarn.lock https://us-npm.pkg.dev/artifact-foundry-prod/npm-3p-trusted/
+# Install from trusted registry, prohibit lockfile changes.
+bazel run @nodejs//:yarn install -- --frozen-lockfile
 
 # Run unit tests
 ./scripts/run_tests

tools/registry-tools/BUILD

@@ -0,0 +1,6 @@
+load("@build_bazel_rules_nodejs//:index.bzl", "nodejs_binary")
+
+nodejs_binary(
+    name = "switch_registry",
+    entry_point = "switch_registry.js",
+)

tools/registry-tools/switch_registry.js

@@ -0,0 +1,38 @@
+// Replace Registry in yarn.lock with the specified one.
+// This script is executed before any node packages are fetched,
+// keep depencency free.
+
+const fs = require('fs');
+
+const args = process.argv.slice(2);
+const lockfilePath = args[0];
+let newRegistry = args[1];
+
+if (!newRegistry) {
+  console.error('Usage: node switch_registry.js /path/to/yarn.lock https://new.registry.com');
+  process.exit(1);
+}
+
+if (!newRegistry.match('https://.*')) {
+  console.error(`Not a valid registry URL: ${newRegistry}`);
+  process.exit(1);
+}
+
+if (newRegistry.endsWith('/')) {
+    newRegistry = newRegistry.slice(0, -1);
+}
+
+try {
+  const content = fs.readFileSync(lockfilePath, 'utf8');
+  const registryRegex = /^(\s+resolved ")(https?:\/\/[^\/]+)(.*)"$/gm;
+
+  const updatedContent = content.replace(registryRegex, (match, prefix, oldOrigin, path) => {
+    return `${prefix}${newRegistry}${path}"`;
+  });
+
+  fs.writeFileSync(lockfilePath, updatedContent, 'utf8');
+
+  console.log(`yarn lockfile ${lockfilePath} updated successfully to: ${newRegistry}`);
+} catch (error) {
+  console.error('Error processing yarn.lock:', error.message);
+}