Preserve symlinks when staging, and link node_modules into web builds

millbun/integration/resources/scalajs-web/build.mill

@@ -16,6 +16,7 @@ object app extends BunScalaJSWebModule {
   def scalaVersion = "3.8.2"
   def scalaJSVersion = "1.22.0"
   def mvnDeps = Seq(mvn"org.scala-js::scalajs-dom::2.8.1")
+  override def npmDeps = Task { Seq("lodash@4.17.21") }
   override def mainClass = Some("Main")
   override def moduleKind = Task { ModuleKind.ESModule }
 }

millbun/integration/resources/scalajs-web/src/Main.scala

@@ -1,5 +1,13 @@
 import org.scalajs.dom.document
+import scala.scalajs.js
+import scala.scalajs.js.annotation.JSImport
+
+/** Forces the linker to emit a real npm import, so the staged web build must resolve it. */
+@js.native
+@JSImport("lodash", JSImport.Namespace)
+object Lodash extends js.Object:
+  def capitalize(value: String): String = js.native
 
 object Main:
   def main(args: Array[String]): Unit =
-    document.getElementById("app").textContent = "Hello from Scala.js web"
+    document.getElementById("app").textContent = Lodash.capitalize("hello from scala.js web")

millbun/integration/resources/typescript-test-deps/build.mill

@@ -11,11 +11,16 @@ import mill.javascriptlib.bun.*
 object app extends BunTypeScriptModule {
   override def moduleDir = build.moduleDir
 
+  // Pinned in the fixture rather than relying on the suite's env default, so the strict
+  // lockfile path is exercised regardless of how the suite is invoked.
+  override def bunRequireLockfile = Task { true }
+
   // Production dependency
   override def npmDeps = Task { Seq("is-even@1.0.0") }
 
   object test extends BunTypeScriptTests {
-    // Test-only dependency — should land in devDependencies, not dependencies
+    // Test-only dependency — should land in devDependencies, not dependencies. Because it is a
+    // dependency the outer module does not have, this test module needs its own bun.lock.
     override def npmDeps = Task { Seq("is-odd@3.0.1") }
   }
 }

millbun/integration/src/mill/bun/BunScalaJSIntegrationTests.scala

@@ -83,6 +83,27 @@ object BunScalaJSIntegrationTests extends TestSuite {
       assert(files.exists(_.ext == "js"))
     }
 
+    test("web stage resolves npm dependencies") {
+      // The fixture imports lodash via @JSImport, so the linked output carries a real npm
+      // import that `bun build` has to resolve out of the staged directory. Staging used to
+      // flatten the install's node_modules symlink into an empty directory, which made every
+      // npm import unresolvable.
+      val tester = this.tester("scalajs-web")
+      assert(tester.eval("app.bundle").isSuccess)
+
+      val stage = tester.workspacePath / "out" / "app" / "webProductionStage.dest"
+      assert(os.isLink(stage / "node_modules"))
+      assert(os.exists(stage / "node_modules" / "lodash" / "package.json"))
+
+      // lodash must be inlined into the bundle, not left as a bare import.
+      val bundled = os.walk(outputPath(tester, "app.bundle"))
+        .filter(p => os.isFile(p) && p.ext == "js")
+        .map(os.read)
+        .mkString
+      assert(!bundled.contains("""from"lodash""""))
+      assert(!bundled.contains("""from "lodash""""))
+    }
+
     test("compileExecutable") {
       val tester = this.tester("scalajs-bundle")
       val res = tester.eval("app.compileExecutable")

millbun/integration/src/mill/bun/BunTypeScriptIntegrationTests.scala

@@ -178,6 +178,9 @@ object BunTypeScriptIntegrationTests extends TestSuite {
 
     test("test deps are devDependencies") {
       val tester = this.tester("typescript-test-deps")
+      // The fixture sets bunRequireLockfile, so both modules need a lock before installing.
+      assert(tester.eval("app.bunLock").isSuccess)
+      assert(tester.eval("app.test.bunLock").isSuccess)
 
       // Outer module should have is-even in dependencies
       val outerRes = tester.eval("app.npmInstall")
@@ -201,6 +204,47 @@ object BunTypeScriptIntegrationTests extends TestSuite {
       assert(runRes.isSuccess)
     }
 
+    test("test modules with extra deps own their lockfile") {
+      // A test module installs a strict superset of the outer package.json. Reusing the outer
+      // module's lock under --frozen-lockfile fails with "lockfile had changes, but lockfile is
+      // frozen", so the test module needs its own lock and its own bunLock command.
+      val tester = this.tester("typescript-test-deps")
+
+      // Without any lock, the install refuses and names the test module's own path.
+      val unlocked = tester.eval("app.test.npmInstall")
+      assert(!unlocked.isSuccess)
+
+      assert(tester.eval("app.bunLock").isSuccess)
+      assert(tester.eval("app.test.bunLock").isSuccess)
+
+      val outerLock = tester.workspacePath / "bun.lock"
+      val testLock = tester.workspacePath / "test" / "bun.lock"
+      assert(os.exists(outerLock))
+      assert(os.exists(testLock))
+      assert(os.read(outerLock) != os.read(testLock))
+
+      // Compare the `workspaces` block, which records the root package's *declared* deps.
+      // is-odd also arrives transitively through is-even, so a whole-file substring match
+      // would not distinguish the two locks.
+      def declaredDeps(lock: os.Path): String =
+        val text = os.read(lock)
+        text.slice(text.indexOf("\"workspaces\""), text.indexOf("\"packages\""))
+
+      assert(!declaredDeps(outerLock).contains("is-odd"))
+      assert(declaredDeps(testLock).contains("is-odd"))
+
+      // The frozen install now succeeds against the test module's own lock.
+      assert(tester.eval("app.test.npmInstall").isSuccess)
+      assert(tester.eval("app.test.test").isSuccess)
+    }
+
+    test("test modules adding nothing reuse the outer install") {
+      // A bare test module must not demand a second lockfile.
+      val tester = this.tester("typescript-tests")
+      assert(tester.eval("app.test.npmInstall").isSuccess)
+      assert(!os.exists(tester.workspacePath / "test" / "bun.lock"))
+    }
+
     test("bunEnv") {
       val tester = this.tester("typescript-env")
       val res = tester.eval("app.bundle")

millbun/src/mill/bun/BunToolchainModule.scala

@@ -299,12 +299,42 @@ object BunToolchainModule {
       .find(path => os.isFile(path) && java.nio.file.Files.isExecutable(path.toNIO))
   }
 
-  /** Copy a generated workspace into a fresh task destination, preserving layout. */
-  def copyWorkspace(source: os.Path, dest: os.Path): Unit = {
-    os.walk(source)
-      .foreach(path => os.copy.over(path, dest / path.relativeTo(source), createFolders = true))
+  /**
+   * Copy a directory tree, recreating symlinks instead of resolving them.
+   *
+   * `os.walk` does not follow links, but `os.isDir` and `os.copy` do. Branching on one while
+   * copying with the other means a `node_modules` symlink is either deep-copied (turning an O(1)
+   * link into an O(node_modules) copy) or flattened into an empty directory, and a broken link
+   * anywhere in the tree — routine for the `.bin` shims of skipped optional dependencies — aborts
+   * the whole copy. Recreating the link is both correct and cheap.
+   *
+   * @param exclude top-level entry names to skip entirely
+   */
+  def copyTree(source: os.Path, dest: os.Path, exclude: Set[String] = Set.empty): Unit = {
+    if (!os.exists(source, followLinks = false)) return
+
+    val skip = (path: os.Path) => {
+      val relative = path.relativeTo(source)
+      relative.segments.headOption.exists(exclude.contains)
+    }
+
+    os.walk(source, skip = skip, followLinks = false).foreach { path =>
+      val target = dest / path.relativeTo(source)
+      if (os.isLink(path)) {
+        os.makeDir.all(target / os.up)
+        if (os.exists(target, followLinks = false)) os.remove(target)
+        os.symlink(target, os.readLink.absolute(path))
+      } else if (os.isDir(path, followLinks = false)) {
+        os.makeDir.all(target)
+      } else {
+        os.copy.over(path, target, createFolders = true)
+      }
+    }
   }
 
+  /** Copy a generated workspace into a fresh task destination, preserving layout and symlinks. */
+  def copyWorkspace(source: os.Path, dest: os.Path): Unit = copyTree(source, dest)
+
   /**
    * Copy files or directories into a Bun workspace while preserving their relative path
    * beneath the nearest matching source root when possible.
@@ -322,13 +352,8 @@ object BunToolchainModule {
           .map(root => destRoot / source.relativeTo(root))
           .getOrElse(destRoot / source.last)
 
-      if (os.isDir(source)) {
-        os.walk(source).foreach { path =>
-          os.copy.over(path, target / path.relativeTo(source), createFolders = true)
-        }
-      } else {
-        os.copy.over(source, target, createFolders = true)
-      }
+      if (os.isDir(source)) copyTree(source, target)
+      else os.copy.over(source, target, createFolders = true)
     }
   }
 }
@@ -455,14 +480,19 @@ trait BunToolchainModule extends Module {
        else Seq.empty)
   }
 
+  /**
+   * @param lockfilePath where the missing lockfile is expected. Nested modules must pass their own
+   *                     path, or the error points a user at the wrong file.
+   */
   protected def requireBunLockfile(
       hasInstallInputs: Boolean,
       lockfile: Option[PathRef],
-      required: Boolean
+      required: Boolean,
+      lockfilePath: os.Path = moduleDir / "bun.lock"
   ): Unit = {
     if (hasInstallInputs && required && lockfile.isEmpty) {
       throw new RuntimeException(
-        s"Missing ${moduleDir / "bun.lock"}. Run this module's bunLock command and commit the generated lockfile."
+        s"Missing $lockfilePath. Run this module's bunLock command and commit the generated lockfile."
       )
     }
   }

millbun/src/mill/bun/BunWebSupport.scala

@@ -13,32 +13,29 @@ private[mill] object BunWebSupport:
       copyPath(source, target)
     }
 
-  def copyContents(source: os.Path, destination: os.Path): Unit =
+  def copyContents(
+      source: os.Path,
+      destination: os.Path,
+      exclude: Set[String] = Set.empty
+  ): Unit =
     if os.exists(source) then
-      if os.isDir(source) then
-        os.walk(source).foreach { path =>
-          val relative = path.relativeTo(source)
-          if relative.segments.nonEmpty then
-            val target = destination / relative
-            if os.isDir(path) then os.makeDir.all(target)
-            else os.copy.over(path, target, createFolders = true)
-        }
+      if os.isDir(source) then BunToolchainModule.copyTree(source, destination, exclude)
       else copyPath(source, destination / source.last)
 
   private def copyPath(source: os.Path, target: os.Path): Unit =
-    if os.isDir(source) then
-      os.walk(source).foreach { path =>
-        val destination = target / path.relativeTo(source)
-        if os.isDir(path) then os.makeDir.all(destination)
-        else os.copy.over(path, destination, createFolders = true)
-      }
+    if os.isDir(source) then BunToolchainModule.copyTree(source, target)
     else os.copy.over(source, target, createFolders = true)
 
+  /**
+   * Resolve the HTML entrypoints a staged web build will use. Pure — writes nothing.
+   *
+   * Kept separate from [[materializeHtmlEntries]] so `dev()` and `bundle` can ask which entries
+   * exist without writing into the staging task's already-cached output directory.
+   */
   def htmlEntries(
       configured: Seq[PathRef],
       moduleDir: os.Path,
-      destination: os.Path,
-      generatedScript: String
+      destination: os.Path
   ): Seq[os.Path] =
     val copied = configured
       .map(_.path)
@@ -48,11 +45,20 @@ private[mill] object BunWebSupport:
         else destination / path.last
       )
 
-    if copied.nonEmpty then copied
-    else
-      val index = destination / "index.html"
+    if copied.nonEmpty then copied else Seq(destination / "index.html")
+
+  /** Resolve entrypoints, generating a minimal `index.html` when the module supplies none. */
+  def materializeHtmlEntries(
+      configured: Seq[PathRef],
+      moduleDir: os.Path,
+      destination: os.Path,
+      generatedScript: String
+  ): Seq[os.Path] =
+    val entries = htmlEntries(configured, moduleDir, destination)
+    val hasConfigured = configured.exists(ref => os.exists(ref.path) && os.isFile(ref.path))
+    if !hasConfigured then
       os.write.over(
-        index,
+        entries.head,
         s"""<!doctype html>
            |<html>
            |  <head><meta charset="utf-8"><meta name="viewport" content="width=device-width"></head>
@@ -61,7 +67,7 @@ private[mill] object BunWebSupport:
            |""".stripMargin,
         createFolders = true
       )
-      Seq(index)
+    entries
 
   def syncRoots(roots: Seq[(os.Path, os.Path)]): Unit =
     roots.foreach { case (source, target) =>

millbun/src/mill/bun/BunWorkersModule.scala

@@ -59,6 +59,9 @@ trait BunWorkersModule extends BunToolchainModule { this: BunTypeScriptModule =>
   def bundleWorkers: T[PathRef] = Task {
     val workspace = Task.dest / "workspace"
     val outDir = Task.dest / "workers"
+    // Declared explicitly: the staged tree carries a node_modules symlink into this install,
+    // and Mill's filesystem checker only permits reading a dest we depend on.
+    npmInstall()
     BunToolchainModule.copyWorkspace(compile().path, workspace)
     os.makeDir.all(outDir)