fix: Non-whitespace IFS splitting drops empty fields

SHELL_FEATURES.md

@@ -149,7 +149,7 @@ or privileged-helper protocol change is needed.
 ## Variables
 
 - ✅ Assignment: `VAR=value`
-- ✅ Expansion: `$VAR`, `${VAR}`
+- ✅ Expansion: `$VAR`, `${VAR}` — unquoted values split on `IFS`, preserving empty fields from leading or adjacent non-whitespace separators; quoted values remain intact
 - ✅ `$?` — last exit code (the only supported special variable)
 - ✅ Inline assignment: `VAR=value command` (scoped to that command)
 - ✅ Command substitution: `$(cmd)`, `` `cmd` `` — captures stdout; trailing newlines stripped; `$(<file)` shortcut reads file directly (gated on `cat` being in the command allowlist); output capped at 1 MiB

analysis/symbols_interp.go

@@ -33,6 +33,7 @@ var interpAllowedSymbols = []string{
 	"fmt.Fprintf",                 // 🟠 formatted write to an io.Writer; delegates to Write, no filesystem access.
 	"fmt.Fprintln",                // 🟠 writes to an io.Writer with newline; delegates to Write, no filesystem access.
 	"fmt.Sprintf",                 // 🟢 string formatting; pure function, no I/O.
+	"iter.Seq2",                   // 🟢 lazy field iterator type; pure type, no I/O.
 	"io.Closer",                   // 🟢 interface type for closing; no side effects.
 	"io.Copy",                     // 🟠 copies from Reader to Writer; no filesystem access, delegates to Read/Write.
 	"io.Discard",                  // 🟢 write sink that discards all data; no side effects.
@@ -93,12 +94,15 @@ var interpAllowedSymbols = []string{
 	"unicode.IsSpace",             // 🟢 reports whether a rune is a Unicode whitespace character; pure function, no I/O.
 	"unicode/utf8.ValidString",    // 🟢 validates configured exact systemd selectors; pure string inspection.
 
+	"unicode/utf8.DecodeRuneInString", // 🟢 decodes IFS characters while preserving byte boundaries; pure function, no I/O.
+
 	// --- github.com/DataDog/datadog-agent/pkg/fleet/installer/telemetry --- (lightweight span tracer used by the Agent Installer)
 
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/telemetry.StartSpanFromContext", // 🟠 starts a span from a parent carried on ctx; registers with the package-global tracer. No I/O here — flushing happens only if the embedding process has also called NewTelemetry.
 
 	// --- mvdan.cc/sh/v3/expand --- (shell word expansion library)
 
+	"mvdan.cc/sh/v3/expand.BracesSeq",              // 🟢 bounded lazy brace expansion; pure function, no I/O.
 	"mvdan.cc/sh/v3/expand.Config",                 // 🟢 configuration for word expansion; pure type.
 	"mvdan.cc/sh/v3/expand.Document",               // 🟢 expands a here-document; pure function.
 	"mvdan.cc/sh/v3/expand.Environ",                // 🟢 interface for environment variable access; pure interface.
@@ -202,6 +206,7 @@ var interpPerModeSymbols = map[string][]string{
 		"fmt.Fprintf",                 // 🟠 formatted write to an io.Writer; delegates to Write, no filesystem access.
 		"fmt.Fprintln",                // 🟠 writes to an io.Writer with newline; delegates to Write, no filesystem access.
 		"fmt.Sprintf",                 // 🟢 string formatting; pure function, no I/O.
+		"iter.Seq2",                   // 🟢 lazy field iterator type; pure type, no I/O.
 		"io.Closer",                   // 🟢 interface type for closing; no side effects.
 		"io.Copy",                     // 🟠 copies from Reader to Writer; no filesystem access, delegates to Read/Write.
 		"io.Discard",                  // 🟢 write sink that discards all data; no side effects.
@@ -255,12 +260,15 @@ var interpPerModeSymbols = map[string][]string{
 		"unicode.IsSpace",             // 🟢 reports whether a rune is a Unicode whitespace character; pure function, no I/O.
 		"unicode/utf8.ValidString",    // 🟢 validates configured exact systemd selectors; pure string inspection.
 
+		"unicode/utf8.DecodeRuneInString", // 🟢 decodes IFS characters while preserving byte boundaries; pure function, no I/O.
+
 		// --- github.com/DataDog/datadog-agent/pkg/fleet/installer/telemetry ---
 
 		"github.com/DataDog/datadog-agent/pkg/fleet/installer/telemetry.StartSpanFromContext", // 🟠 starts a span from a parent carried on ctx; registers with the package-global tracer. No I/O here — flushing happens only if the embedding process has also called NewTelemetry.
 
 		// --- mvdan.cc/sh/v3/expand ---
 
+		"mvdan.cc/sh/v3/expand.BracesSeq",              // 🟢 bounded lazy brace expansion; pure function, no I/O.
 		"mvdan.cc/sh/v3/expand.Config",                 // 🟢 configuration for word expansion; pure type.
 		"mvdan.cc/sh/v3/expand.Document",               // 🟢 expands a here-document; pure function.
 		"mvdan.cc/sh/v3/expand.Environ",                // 🟢 interface for environment variable access; pure interface.

interp/expansion_limits_test.go

@@ -113,16 +113,37 @@ func TestExpandedArgumentCountLimitAllowsBoundary(t *testing.T) {
 	}
 }
 
+func TestNonWhitespaceIFSEmptyFieldsCountTowardLimit(t *testing.T) {
+	for _, count := range []int{MaxExpandedArgumentsPerCommand, MaxExpandedArgumentsPerCommand + 1} {
+		stdout, stderr, err := runExpansionScript(t, "IFS=:; true $X\necho after",
+			Env("X="+strings.Repeat(":", count)),
+			AllowedCommands([]string{"rshell:true", "rshell:echo"}),
+		)
+		if count == MaxExpandedArgumentsPerCommand {
+			if err != nil || stdout != "after\n" || stderr != "" {
+				t.Fatalf("at limit: stdout=%q, stderr=%q, err=%v", stdout, stderr, err)
+			}
+		} else {
+			requireExitStatus(t, err, 1)
+			if stdout != "" || !strings.Contains(stderr, "expansion exceeds maximum field count") {
+				t.Fatalf("over limit: stdout=%q, stderr=%q", stdout, stderr)
+			}
+		}
+	}
+}
+
 func TestExpandedArgumentByteLimit(t *testing.T) {
 	value := strings.Repeat("x", 1<<20)
 	script := "true " + strings.TrimSpace(strings.Repeat("$X ", 11))
-	_, stderr, err := runExpansionScript(t, script,
-		Env("X="+value),
-		AllowedCommands([]string{"rshell:true"}),
-	)
-	requireExitStatus(t, err, 1)
-	if !strings.Contains(stderr, "expansion exceeds maximum command size") {
-		t.Fatalf("stderr = %q, want command expansion byte limit", stderr)
+	for _, prefix := range []string{"", "IFS=:; "} {
+		_, stderr, err := runExpansionScript(t, prefix+script,
+			Env("X="+value),
+			AllowedCommands([]string{"rshell:true"}),
+		)
+		requireExitStatus(t, err, 1)
+		if !strings.Contains(stderr, "expansion exceeds maximum command size") {
+			t.Fatalf("stderr = %q, want command expansion byte limit", stderr)
+		}
 	}
 }
 

interp/runner_expand.go

@@ -417,7 +417,7 @@ func (c *fieldCollector) add(words ...*syntax.Word) (ok bool) {
 			return false
 		}
 
-		for field, err := range expand.FieldsSeq(c.r.ecfg, word) {
+		for field, err := range c.fieldsSeq(word) {
 			if err != nil {
 				if strings.HasPrefix(err.Error(), braceExpansionLimitErrorPrefix) {
 					err = &expansionLimitError{message: err.Error()}

interp/runner_fields.go

@@ -0,0 +1,161 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2026-present Datadog, Inc.
+
+package interp
+
+import (
+	"fmt"
+	"iter"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+
+	"mvdan.cc/sh/v3/expand"
+	"mvdan.cc/sh/v3/syntax"
+)
+
+// fieldsSeq works around mvdan.cc/sh/v3's coalescing of all IFS characters.
+// Non-whitespace separators must preserve leading and adjacent empty fields.
+func (c *fieldCollector) fieldsSeq(word *syntax.Word) iter.Seq2[string, error] {
+	ifs := c.r.ecfg.Env.Get("IFS").String()
+	if strings.TrimLeft(ifs, " \t\n\r\v\f") == "" {
+		return expand.FieldsSeq(c.r.ecfg, word)
+	}
+	return func(yield func(string, error) bool) {
+		word := *word
+		syntax.SplitBraces(&word)
+		for w, err := range expand.BracesSeq(c.r.ecfg, &word) {
+			if err != nil {
+				yield("", err)
+				return
+			}
+			fields, cfg, err := c.splitWordFields(w, ifs)
+			if err != nil {
+				yield("", err)
+				return
+			}
+			for _, field := range fields {
+				for value, err := range expand.FieldsSeq(cfg, field) {
+					if !yield(value, err) || err != nil {
+						return
+					}
+				}
+			}
+		}
+	}
+}
+
+// splitFieldEnv holds already-expanded, unquoted fragments. Passing them back
+// as parameters with IFS disabled preserves globbing without interpreting their
+// backslashes, braces, or tildes as shell source. It never changes the runner's env.
+type splitFieldEnv struct {
+	expand.Environ
+	values map[string]expand.Variable
+}
+
+func (e splitFieldEnv) Get(name string) expand.Variable {
+	if name == "IFS" {
+		return expand.Variable{Set: true, Kind: expand.String}
+	}
+	if value, ok := e.values[name]; ok {
+		return value
+	}
+	return e.Environ.Get(name)
+}
+
+func (c *fieldCollector) splitWordFields(word *syntax.Word, ifs string) ([]*syntax.Word, *expand.Config, error) {
+	cfg := *c.r.ecfg
+	env := splitFieldEnv{Environ: cfg.Env, values: make(map[string]expand.Variable)}
+	cfg.Env = env
+	var fields []*syntax.Word
+	var parts []syntax.WordPart
+	// Whitespace immediately before a non-whitespace separator belongs to the
+	// same delimiter. Keep this state across adjacent expansions, including empty ones.
+	afterWhitespace := false
+	flush := func() error {
+		if len(fields) >= c.maxFields-len(c.fields) {
+			return &expansionLimitError{message: fmt.Sprintf(
+				"expansion exceeds maximum field count (%d)", c.maxFields)}
+		}
+		if len(parts) == 0 {
+			parts = []syntax.WordPart{&syntax.SglQuoted{}}
+		}
+		fields = append(fields, &syntax.Word{Parts: parts})
+		parts = nil
+		return nil
+	}
+	addPart := func(part syntax.WordPart) {
+		if len(parts) == 0 {
+			// Only the original word's first literal may undergo tilde expansion.
+			parts = append(parts, &syntax.SglQuoted{})
+		}
+		parts = append(parts, part)
+		afterWhitespace = false
+	}
+	addValue := func(value string) {
+		if value == "" {
+			return
+		}
+		name := "_rshell_field_" + strconv.Itoa(len(env.values))
+		env.values[name] = expand.Variable{Set: true, Kind: expand.String, Str: value}
+		addPart(&syntax.ParamExp{Param: &syntax.Lit{Value: name}})
+	}
+	for i, part := range word.Parts {
+		switch part.(type) {
+		case *syntax.Lit:
+			if i == 0 {
+				parts = append(parts, part)
+			} else {
+				addPart(part)
+			}
+		case *syntax.SglQuoted, *syntax.DblQuoted:
+			value, err := expand.Literal(c.r.ecfg, &syntax.Word{Parts: []syntax.WordPart{part}})
+			if err != nil {
+				return nil, nil, err
+			}
+			addPart(&syntax.SglQuoted{Value: value})
+		case *syntax.ParamExp, *syntax.CmdSubst:
+			value, err := expand.Literal(c.r.ecfg, &syntax.Word{Parts: []syntax.WordPart{part}})
+			if err != nil {
+				return nil, nil, err
+			}
+			start := 0
+			for pos := 0; pos < len(value); {
+				r, size := utf8.DecodeRuneInString(value[pos:])
+				if !strings.ContainsRune(ifs, r) {
+					pos += size
+					continue
+				}
+				addValue(value[start:pos])
+				if strings.ContainsRune(" \t\n\r\v\f", r) {
+					if len(parts) > 0 {
+						if err := flush(); err != nil {
+							return nil, nil, err
+						}
+						afterWhitespace = true
+					}
+				} else {
+					if !afterWhitespace {
+						if err := flush(); err != nil {
+							return nil, nil, err
+						}
+					}
+					afterWhitespace = false
+				}
+				pos += size
+				start = pos
+			}
+			addValue(value[start:])
+		default:
+			return nil, nil, fmt.Errorf("unsupported word part in field splitting: %T", part)
+		}
+	}
+	if len(parts) > 0 {
+		if err := flush(); err != nil {
+			return nil, nil, err
+		}
+	}
+	return fields, &cfg, nil
+}

tests/scenarios/shell/field_splitting/consecutive_nonwhitespace_ifs.yaml

@@ -1,18 +1,13 @@
-description: >
-  Consecutive non-whitespace IFS delimiters should produce empty fields.
-  In bash, "a::b" with IFS=: produces three fields: "a", "", "b".
-  The upstream mvdan.cc/sh library does not produce the empty middle field.
-  skip_assert_against_bash because of this known upstream limitation.
-# skip: known upstream library limitation with consecutive non-whitespace IFS delimiters
-skip_assert_against_bash: true
+description: Consecutive non-whitespace IFS delimiters produce empty fields.
 input:
   script: |+
     IFS=:
     A="a::b"
     for w in $A; do echo "[$w]"; done
 expect:
-  # bash produces: [a]\n[]\n[b]
-  # rshell currently produces: [a]\n[b] (upstream library limitation)
-  stdout: "[a]\n[b]\n"
-  stderr: ""
+  stdout: |+
+    [a]
+    []
+    [b]
+  stderr: |+
   exit_code: 0

tests/scenarios/shell/field_splitting/mixed_ws_nonws_ifs.yaml

@@ -1,9 +1,25 @@
 description: "Whitespace IFS characters coalesce, non-whitespace splits"
 input:
-  script: |
-    IFS=" -"
-    A="a  b-c"
+  script: |+
+    IFS=' :'
+    A='  : a  : : b : '
     for w in $A; do echo "[$w]"; done
+    LEFT='a '
+    RIGHT=' :b::'
+    for w in $LEFT$RIGHT; do echo "[$w]"; done
+    for w in $LEFT""$RIGHT; do echo "[$w]"; done
 expect:
-  stdout: "[a]\n[b]\n[c]\n"
+  stdout: |+
+    []
+    [a]
+    []
+    [b]
+    [a]
+    [b]
+    []
+    [a]
+    []
+    [b]
+    []
+  stderr: |+
   exit_code: 0

tests/scenarios/shell/field_splitting/nonws_ifs_ambiguous_redirect.yaml

@@ -0,0 +1,19 @@
+description: A leading empty IFS field makes an input redirect ambiguous instead of reading the remaining filename.
+setup:
+  files:
+    - path: input.txt
+      content: |+
+        must not be read
+input:
+  allowed_paths: ["$DIR"]
+  script: |+
+    IFS=:
+    target=':input.txt'
+    cat < $target
+    echo $?
+expect:
+  stdout: |+
+    1
+  stderr_contains:
+    - ambiguous redirect
+  exit_code: 0

tests/scenarios/shell/field_splitting/nonws_ifs_argument_boundaries.yaml

@@ -0,0 +1,20 @@
+description: Leading and adjacent non-whitespace IFS delimiters preserve empty arguments, with quotes and adjoining literals.
+input:
+  script: |+
+    IFS=:
+    x=':a::b:'
+    printf '<%s>\n' $x "$x" pre${x}post
+expect:
+  stdout: |+
+    <>
+    <a>
+    <>
+    <b>
+    <:a::b:>
+    <pre>
+    <a>
+    <>
+    <b>
+    <post>
+  stderr: |+
+  exit_code: 0

tests/scenarios/shell/field_splitting/nonws_ifs_delimiter_edges.yaml

@@ -0,0 +1,29 @@
+description: Delimiter-only and trailing IFS separators preserve the Bash argument boundaries.
+input:
+  script: |+
+    IFS=:
+    for x in '' ':' '::' 'a:' 'a::'; do
+      echo next
+      for w in $x; do printf '<%s>\n' "$w"; done
+    done
+    IFS=$(printf '\r\v\f:')
+    x=$(printf '\r:\v:\fa:\r')
+    printf '<%s>\n' $x
+expect:
+  stdout: |+
+    next
+    next
+    <>
+    next
+    <>
+    <>
+    next
+    <a>
+    next
+    <a>
+    <>
+    <>
+    <>
+    <a>
+  stderr: |+
+  exit_code: 0