Use bundle install --target-rbconfig to install gems for Wasm target

Author: kateinoigakukun

Rakefile

@@ -28,20 +28,20 @@ NPM_PACKAGES = [
   {
     name: "ruby-head-wasm-wasi",
     ruby_version: "head",
-    gemfile: "packages/npm-packages/ruby-wasm-wasi/Gemfile",
+    gemfile: "packages/npm-packages/ruby-head-wasm-wasi/Gemfile",
     target: "wasm32-unknown-wasip1",
     enable_component_model: true,
   },
   {
     name: "ruby-3.3-wasm-wasi",
     ruby_version: "3.3",
-    gemfile: "packages/npm-packages/ruby-wasm-wasi/Gemfile",
+    gemfile: "packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile",
     target: "wasm32-unknown-wasip1"
   },
   {
     name: "ruby-3.2-wasm-wasi",
     ruby_version: "3.2",
-    gemfile: "packages/npm-packages/ruby-wasm-wasi/Gemfile",
+    gemfile: "packages/npm-packages/ruby-3.2-wasm-wasi/Gemfile",
     target: "wasm32-unknown-wasip1"
   },
   { name: "ruby-wasm-wasi", target: "wasm32-unknown-wasip1" }

bin/setup

@@ -6,7 +6,9 @@ set -vx
 root="$(cd "$(dirname "$0")/.." && pwd)"
 
 env BUNDLE_GEMFILE="$root/Gemfile" bundle install
-env BUNDLE_GEMFILE="$root/packages/npm-packages/ruby-wasm-wasi/Gemfile" bundle install
+for gemfile in $root/packages/npm-packages/*/Gemfile; do
+  env BUNDLE_GEMFILE="$gemfile" bundle install
+done
 
 # Build vendored jco if Rust toolchain is available and submodule is checked out
 if command -v rustc && [ -f vendor/jco/package.json ]; then

ext/ruby_wasm/src/lib.rs

@@ -252,6 +252,9 @@ impl WasiVirt {
             // Disable sockets for now since `sockets/ip-name-lookup` is not
             // supported by @bytecodealliance/preview2-shim yet
             virt.sockets(false);
+            // Disable http for now since `http` is not supported by
+            // wasmtime yet
+            virt.http(false);
             Ok(())
         })
     }

lib/ruby_wasm/build/product/crossruby.rb

@@ -71,7 +71,7 @@ def do_extconf(executor, crossruby)
         return
       end
       objdir = product_build_dir crossruby
-      rbconfig_rb = Dir.glob(File.join(crossruby.dest_dir, "usr/local/lib/ruby/*/wasm32-wasi/rbconfig.rb")).first
+      rbconfig_rb = crossruby.rbconfig_rb
       raise "rbconfig.rb not found" unless rbconfig_rb
       extconf_args = [
         "-C", objdir,
@@ -301,6 +301,10 @@ def extinit_c_erb
       File.expand_path("../crossruby/extinit.c.erb", __FILE__)
     end
 
+    def rbconfig_rb
+      Dir.glob(File.join(dest_dir, "usr/local/lib/ruby/*/wasm32-wasi/rbconfig.rb")).first
+    end
+
     def baseruby_path
       File.join(@baseruby.install_dir, "bin/ruby")
     end

lib/ruby_wasm/cli.rb

@@ -321,7 +321,7 @@ def derive_packager(options)
           Bundler.ui.level = old_level
         end
       end
-      RubyWasm.logger.info "Using Gemfile: #{definition.gemfiles}" if definition
+      RubyWasm.logger.info "Using Gemfile: #{definition.gemfiles.map(&:to_s).join(", ")}" if definition
       RubyWasm::Packager.new(
         root, build_config(options), definition,
         features: RubyWasm::FeatureSet.derive_from_env

lib/ruby_wasm/packager.rb

@@ -34,7 +34,7 @@ def package(executor, dest_dir, options)
 
     ruby_core.build_gem_exts(executor, fs.bundle_dir)
 
-    fs.package_gems
+    fs.package_gems unless features.support_component_model?
     fs.remove_non_runtime_files(executor)
     if options[:stdlib]
       options[:without_stdlib_components].each do |component|

lib/ruby_wasm/packager/core.rb

@@ -187,31 +187,41 @@ def _build_gem_exts(executor, build, gem_home)
         baseruby.build(executor)
       end
 
-      exts = specs_with_extensions.flat_map do |spec, exts|
-        exts.map do |ext|
-          ext_feature = File.dirname(ext) # e.g. "ext/cgi/escape"
-          ext_srcdir = File.join(spec.full_gem_path, ext_feature)
-          ext_relative_path = File.join(spec.full_name, ext_feature)
-          prod = RubyWasm::CrossRubyExtProduct.new(
-            ext_srcdir,
-            build.toolchain,
-            features: @packager.features,
-            ext_relative_path: ext_relative_path
-          )
-          [prod, spec]
-        end
-      end
+      crossruby = build.crossruby
+      rbconfig_rb = crossruby.rbconfig_rb
 
-      exts.each do |prod, spec|
-        libdir = File.join(gem_home, "gems", spec.full_name, spec.raw_require_paths.first)
-        extra_mkargs = [
-          "sitearchdir=#{libdir}",
-          "sitelibdir=#{libdir}",
-        ]
-        executor.begin_section prod.class, prod.name, "Building"
-        prod.build(executor, build.crossruby, extra_mkargs)
-        executor.end_section prod.class, prod.name
-      end
+      options = @packager.full_build_options
+      target_triplet = options[:target]
+
+      local_path = File.join("bundle", target_triplet)
+      env = {
+        "BUNDLE_APP_CONFIG" => File.join(".bundle", target_triplet),
+        "BUNDLE_PATH" => local_path,
+        "BUNDLE_WITHOUT" => "build",
+        # FIXME: BUNDLE_PATH is set as a installation destination here, but
+        # it is also used as a source of gems to be loaded by RubyGems itself.
+        # RubyGems loads "psych" gem and if Gemfile includes "psych" gem,
+        # RubyGems tries to load "psych" gem from BUNDLE_PATH at the second
+        # time of "bundle install" command. But the extension of "psych" gem
+        # under BUNDLE_PATH is built for Wasm target, not for host platform,
+        # so it fails to load the extension.
+        #
+        # Thus we preload psych from the default LOAD_PATH here to avoid
+        # loading Wasm version of psych.so via `Kernel#require` patched by
+        # RubyGems.
+        "RUBYOPT" => "-rpsych",
+      }
+
+      args = [
+        File.join(baseruby.install_dir, "bin", "bundle"),
+        "install",
+        "--standalone",
+        "--target-rbconfig",
+        rbconfig_rb,
+      ]
+
+      executor.system(*args, env: env)
+      executor.cp_r(local_path, gem_home)
     end
 
     def cache_key(digest)

lib/ruby_wasm/packager/file_system.rb

@@ -44,7 +44,6 @@ def remove_stdlib_component(executor, component)
     when "enc"
       # Remove all encodings except for encdb.so and transdb.so
       enc_dir = File.join(@ruby_root, "lib", "ruby", ruby_version, "wasm32-wasi", "enc")
-      puts File.join(enc_dir, "**/*.so")
       Dir.glob(File.join(enc_dir, "**/*.so")).each do |entry|
         next if entry.end_with?("encdb.so", "transdb.so")
         RubyWasm.logger.debug "Removing stdlib encoding: #{entry}"

packages/npm-packages/ruby-3.2-wasm-wasi/.gitignore

@@ -1 +1,2 @@
 *.tgz
+/bundle