bindeps failing build fix

crates/bin/docs_rs_builder/src/docbuilder/rustwide_builder.rs

@@ -106,9 +106,11 @@ fn load_metadata_from_rustwide(
     workspace: &Workspace,
     toolchain: &Toolchain,
     source_dir: &Path,
+    host_unstable_flags: &[String],
 ) -> Result<CargoMetadata> {
     let res = Command::new(workspace, toolchain.cargo())
         .args(["metadata", "--format-version", "1"])
+        .args(host_unstable_flags)
         .current_directory(source_dir)
         .log_output(false)
         .run_capture()?;
@@ -495,10 +497,17 @@ impl RustwideBuilder {
     }
 
     pub fn build_local_package(&mut self, path: &Path) -> Result<BuildPackageSummary> {
-        let metadata = load_metadata_from_rustwide(&self.workspace, &self.toolchain, path)
-            .map_err(|err| {
-                err.context(format!("failed to load local package {}", path.display()))
-            })?;
+        let host_unstable_flags = Metadata::from_crate_root(path)
+            .map(|metadata| metadata.unstable_cargo_flags())
+            .unwrap_or_default();
+
+        let metadata = load_metadata_from_rustwide(
+            &self.workspace,
+            &self.toolchain,
+            path,
+            &host_unstable_flags,
+        )
+        .map_err(|err| err.context(format!("failed to load local package {}", path.display())))?;
         let package = metadata.root();
         self.build_package(
             &package
@@ -639,13 +648,19 @@ impl RustwideBuilder {
         let local_storage = tempfile::tempdir_in(&self.config.temp_dir)?;
 
         let mut algs = HashSet::new();
-        let source_stats = {
+        let (source_stats, host_unstable_flags) = {
             let _span = info_span!("adding sources into database").entered();
             debug!("adding sources into database");
             let temp_dir = tempfile::tempdir_in(&self.config.temp_dir)?;
 
             krate.copy_source_to(&self.workspace, temp_dir.path())?;
 
+            // Read from this copy: rustwide's prepare phase needs the flags before the build
+            // closure below gets access to the sources.
+            let host_unstable_flags = Metadata::from_crate_root(temp_dir.path())
+                .map(|metadata| metadata.unstable_cargo_flags())
+                .unwrap_or_default();
+
             let stats = self.runtime.block_on(
                 self.storage
                     .store_all_in_archive(&source_archive_path(name, version), &temp_dir),
@@ -654,11 +669,12 @@ impl RustwideBuilder {
             fs::remove_dir_all(temp_dir.path())?;
 
             algs.insert(stats.alg);
-            stats
+            (stats, host_unstable_flags)
         };
 
         let successful = build_dir
             .build(&self.toolchain, &krate, self.prepare_sandbox(&limits))
+            .extra_cargo_args(&host_unstable_flags)
             .run(|build| {
                 // NOTE: rustwide will run `copy_source_to` again when preparing the call to this
                 // closure.
@@ -674,7 +690,25 @@ impl RustwideBuilder {
                 {
                     let _span = info_span!("fetch_build_std_dependencies").entered();
                     // Fetch this before we enter the sandbox, so networking isn't blocked.
-                    build.fetch_build_std_dependencies(&targets)?;
+                    if host_unstable_flags.is_empty() {
+                        build.fetch_build_std_dependencies(&targets)?;
+                    } else {
+                        // `Build` doesn't carry the `extra_cargo_args` given to `BuildBuilder`,
+                        // so rustwide would run this fetch without them and fail to parse the
+                        // manifest. Run the equivalent command ourselves until rustwide can
+                        // forward them here too.
+                        self.toolchain.add_component(&self.workspace, "rust-src")?;
+                        let mut cmd = Command::new(&self.workspace, self.toolchain.cargo())
+                            .args(["fetch", "--manifest-path", "Cargo.toml"])
+                            .args(&host_unstable_flags)
+                            .args(["-Zbuild-std"])
+                            .env("RUSTC_BOOTSTRAP", "1")
+                            .current_directory(build.host_source_dir());
+                        for target in &targets {
+                            cmd = cmd.args(["--target", target]);
+                        }
+                        cmd.run()?;
+                    }
                 }
 
 
@@ -693,14 +727,16 @@ impl RustwideBuilder {
                         let _span = info_span!("cargo_generate_lockfile").entered();
                         Command::new(&self.workspace, self.toolchain.cargo())
                             .current_directory(build.host_source_dir())
-                            .arg("generate-lockfile")
+                            .args(["generate-lockfile"])
+                            .args(&host_unstable_flags)
                             .run_capture()?;
                     }
                     {
                         let _span = info_span!("cargo fetch --locked").entered();
                         Command::new(&self.workspace, self.toolchain.cargo())
                             .current_directory(build.host_source_dir())
                             .args(["fetch", "--locked"])
+                            .args(&host_unstable_flags)
                             .run_capture()?;
                     }
                     res =
@@ -1145,6 +1181,7 @@ impl RustwideBuilder {
             &self.workspace,
             &self.toolchain,
             &build.host_source_dir(),
+            &metadata.unstable_cargo_flags(),
         )?;
 
         let mut rustdoc_flags = vec![
@@ -2359,6 +2396,52 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    #[ignore]
+    fn test_bindeps_metadata_with_unstable_flags() -> Result<()> {
+        let env = TestEnvironment::new()?;
+        let mut builder = env.build_builder()?;
+        builder.update_toolchain()?;
+        let crate_path = Path::new("tests/crates/bindeps-test");
+        let metadata = Metadata::from_crate_root(crate_path)?;
+        let unstable_flags = metadata.unstable_cargo_flags();
+
+        assert!(
+            load_metadata_from_rustwide(&builder.workspace, &builder.toolchain, crate_path, &[])
+                .is_err(),
+            "cargo metadata should fail without -Zbindeps",
+        );
+
+        assert!(
+            load_metadata_from_rustwide(
+                &builder.workspace,
+                &builder.toolchain,
+                crate_path,
+                &unstable_flags,
+            )
+            .is_ok(),
+            "cargo metadata should succeed with -Zbindeps",
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    #[ignore]
+    fn test_bindeps_crate_full_build() -> Result<()> {
+        let env = TestEnvironment::new()?;
+        let mut builder = env.build_builder()?;
+        builder.update_toolchain()?;
+
+        assert!(
+            builder
+                .build_local_package(Path::new("tests/crates/bindeps-test"))?
+                .successful
+        );
+
+        Ok(())
+    }
+
     #[test]
     #[ignore]
     fn test_build_with_cpu_limit() -> Result<()> {

crates/bin/docs_rs_builder/tests/crates/bindeps-test/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "bindeps-test"
+version = "0.1.0"
+edition = "2021"
+
+[package.metadata.docs.rs]
+cargo-args = ["-Zbindeps"]
+
+[build-dependencies]
+bindeps-helper = { path = "bindeps-helper", artifact = "bin" }
+
+[lib]
+name = "bindeps_test"
+path = "src/lib.rs"

crates/bin/docs_rs_builder/tests/crates/bindeps-test/bindeps-helper/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "bindeps-helper"
+version = "0.1.0"
+edition = "2021"
+
+[[bin]]
+name = "bindeps-helper"
+path = "src/main.rs"

crates/bin/docs_rs_builder/tests/crates/bindeps-test/bindeps-helper/src/main.rs

@@ -0,0 +1 @@
+fn main() {}

crates/bin/docs_rs_builder/tests/crates/bindeps-test/src/lib.rs

@@ -0,0 +1,3 @@
+pub fn hello() -> &'static str {
+    "Hello from bindeps-test!"
+}

crates/bin/docs_rs_web/templates/core/Cargo.toml.example

@@ -48,4 +48,5 @@ rustdoc-args = ["--example-rustdoc-arg"]
 # List of command line arguments for `cargo`.
 #
 # These cannot be a subcommand, they may only be options.
+# Of the unstable flags, only `bindeps` also reaches host-side cargo commands.
 cargo-args = ["-Z", "build-std"]

crates/bin/docs_rs_web/templates/core/about/metadata.html

@@ -17,6 +17,13 @@ <h1>Metadata for custom builds</h1>
 	{% filter highlight("toml") %}
 		{%- include "core/Cargo.toml.example" -%}
 	{% endfilter %}
+
+	<p>
+		For security reasons, docs.rs only forwards a whitelist of unstable cargo flags to
+		host-side cargo commands (<code>cargo metadata</code>, <code>cargo fetch</code>,
+		<code>cargo generate-lockfile</code>). Currently, only <code>bindeps</code> is
+		supported (as <code>-Zbindeps</code> or <code>-Z bindeps</code>).
+	</p>
 	</div>
 	</div>
 {%- endblock body %}

crates/lib/metadata/lib.rs

@@ -146,6 +146,30 @@ pub struct Metadata {
     additional_targets: Vec<String>,
 }
 
+impl Metadata {
+    /// Unstable flags from `cargo_args` that may also be passed to cargo commands we run
+    /// outside the sandbox. These run with our privileges, so this is a whitelist.
+    pub fn unstable_cargo_flags(&self) -> Vec<String> {
+        let mut flags = Vec::new();
+        let mut iter = self.cargo_args.iter();
+        while let Some(arg) = iter.next() {
+            if arg == "-Z" {
+                if let Some(value) = iter.next()
+                    && value == "bindeps"
+                {
+                    flags.push("-Z".to_string());
+                    flags.push(value.clone());
+                }
+            } else if let Some(value) = arg.strip_prefix("-Z")
+                && value == "bindeps"
+            {
+                flags.push(arg.clone());
+            }
+        }
+        flags
+    }
+}
+
 /// The targets that should be built for a crate.
 ///
 /// The `default_target` is the target to be used as the home page for that crate.
@@ -553,6 +577,48 @@ mod test_parsing {
         let metadata = Metadata::from_str(manifest).unwrap();
         assert!(!metadata.proc_macro);
     }
+
+    #[test]
+    fn test_unstable_cargo_flags() {
+        let manifest = r#"
+            [package]
+            name = "test"
+            [package.metadata.docs.rs]
+            cargo-args = ["-Zbindeps", "--some-other-arg"]
+        "#;
+        let metadata = Metadata::from_str(manifest).unwrap();
+        assert_eq!(metadata.unstable_cargo_flags(), vec!["-Zbindeps"]);
+
+        let manifest = r#"
+            [package]
+            name = "test"
+            [package.metadata.docs.rs]
+            cargo-args = ["-Z", "bindeps", "--other"]
+        "#;
+        let metadata = Metadata::from_str(manifest).unwrap();
+        assert_eq!(metadata.unstable_cargo_flags(), vec!["-Z", "bindeps"]);
+
+        let manifest = r#"
+            [package]
+            name = "test"
+            [package.metadata.docs.rs]
+            cargo-args = ["-Zbindeps", "-Z", "build-std", "--offline"]
+        "#;
+        let metadata = Metadata::from_str(manifest).unwrap();
+        assert_eq!(metadata.unstable_cargo_flags(), vec!["-Zbindeps"]);
+
+        let manifest = r#"
+            [package]
+            name = "test"
+            [package.metadata.docs.rs]
+            cargo-args = ["--offline", "--locked"]
+        "#;
+        let metadata = Metadata::from_str(manifest).unwrap();
+        assert!(metadata.unstable_cargo_flags().is_empty());
+
+        let metadata = Metadata::default();
+        assert!(metadata.unstable_cargo_flags().is_empty());
+    }
 }
 
 #[cfg(test)]