From 28c7308778ce4463aa6a295b2123be2ae3153f57 Mon Sep 17 00:00:00 2001 From: datadog-bits-staging <264369727+datadog-bits-staging@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:44:01 +0000 Subject: [PATCH 1/3] Preserve empty fields from non-whitespace IFS Co-authored-by: matt-dz <91019033+matt-dz@users.noreply.github.com> --- SHELL_FEATURES.md | 2 +- analysis/symbols_interp.go | 8 + interp/expansion_limits_test.go | 35 +++- interp/runner_expand.go | 2 +- interp/runner_fields.go | 161 ++++++++++++++++++ .../consecutive_nonwhitespace_ifs.yaml | 17 +- .../field_splitting/mixed_ws_nonws_ifs.yaml | 24 ++- .../nonws_ifs_ambiguous_redirect.yaml | 19 +++ .../nonws_ifs_argument_boundaries.yaml | 20 +++ .../nonws_ifs_delimiter_edges.yaml | 36 ++++ .../nonws_ifs_expansion_order.yaml | 38 +++++ 11 files changed, 338 insertions(+), 24 deletions(-) create mode 100644 interp/runner_fields.go create mode 100644 tests/scenarios/shell/field_splitting/nonws_ifs_ambiguous_redirect.yaml create mode 100644 tests/scenarios/shell/field_splitting/nonws_ifs_argument_boundaries.yaml create mode 100644 tests/scenarios/shell/field_splitting/nonws_ifs_delimiter_edges.yaml create mode 100644 tests/scenarios/shell/field_splitting/nonws_ifs_expansion_order.yaml diff --git a/SHELL_FEATURES.md b/SHELL_FEATURES.md index d69857074..4e2e17557 100644 --- a/SHELL_FEATURES.md +++ b/SHELL_FEATURES.md @@ -149,7 +149,7 @@ or privileged-helper protocol change is needed. ## Variables - ✅ Assignment: `VAR=value` -- ✅ Expansion: `$VAR`, `${VAR}` +- ✅ Expansion: `$VAR`, `${VAR}` — unquoted values split on `IFS`, preserving empty fields from leading or adjacent non-whitespace separators; quoted values remain intact - ✅ `$?` — last exit code (the only supported special variable) - ✅ Inline assignment: `VAR=value command` (scoped to that command) - ✅ Command substitution: `$(cmd)`, `` `cmd` `` — captures stdout; trailing newlines stripped; `$(= c.maxFields-len(c.fields) { + return &expansionLimitError{message: fmt.Sprintf( + "expansion exceeds maximum field count (%d)", c.maxFields)} + } + if len(parts) == 0 { + parts = []syntax.WordPart{&syntax.SglQuoted{}} + } + fields = append(fields, &syntax.Word{Parts: parts}) + parts = nil + return nil + } + addPart := func(part syntax.WordPart) { + if len(parts) == 0 { + // Only the original word's first literal may undergo tilde expansion. + parts = append(parts, &syntax.SglQuoted{}) + } + parts = append(parts, part) + afterWhitespace = false + } + addValue := func(value string) { + if value == "" { + return + } + name := "_rshell_field_" + strconv.Itoa(len(env.values)) + env.values[name] = expand.Variable{Set: true, Kind: expand.String, Str: value} + addPart(&syntax.ParamExp{Param: &syntax.Lit{Value: name}}) + } + for i, part := range word.Parts { + switch part.(type) { + case *syntax.Lit: + if i == 0 { + parts = append(parts, part) + } else { + addPart(part) + } + case *syntax.SglQuoted, *syntax.DblQuoted: + value, err := expand.Literal(c.r.ecfg, &syntax.Word{Parts: []syntax.WordPart{part}}) + if err != nil { + return nil, nil, err + } + addPart(&syntax.SglQuoted{Value: value}) + case *syntax.ParamExp, *syntax.CmdSubst: + value, err := expand.Literal(c.r.ecfg, &syntax.Word{Parts: []syntax.WordPart{part}}) + if err != nil { + return nil, nil, err + } + start := 0 + for pos := 0; pos < len(value); { + r, size := utf8.DecodeRuneInString(value[pos:]) + if !strings.ContainsRune(ifs, r) { + pos += size + continue + } + addValue(value[start:pos]) + if strings.ContainsRune(" \t\n\r\v\f", r) { + if len(parts) > 0 { + if err := flush(); err != nil { + return nil, nil, err + } + afterWhitespace = true + } + } else { + if !afterWhitespace { + if err := flush(); err != nil { + return nil, nil, err + } + } + afterWhitespace = false + } + pos += size + start = pos + } + addValue(value[start:]) + default: + return nil, nil, fmt.Errorf("unsupported word part in field splitting: %T", part) + } + } + if len(parts) > 0 { + if err := flush(); err != nil { + return nil, nil, err + } + } + return fields, &cfg, nil +} diff --git a/tests/scenarios/shell/field_splitting/consecutive_nonwhitespace_ifs.yaml b/tests/scenarios/shell/field_splitting/consecutive_nonwhitespace_ifs.yaml index d2671a6d8..3846d4ca5 100644 --- a/tests/scenarios/shell/field_splitting/consecutive_nonwhitespace_ifs.yaml +++ b/tests/scenarios/shell/field_splitting/consecutive_nonwhitespace_ifs.yaml @@ -1,18 +1,13 @@ -description: > - Consecutive non-whitespace IFS delimiters should produce empty fields. - In bash, "a::b" with IFS=: produces three fields: "a", "", "b". - The upstream mvdan.cc/sh library does not produce the empty middle field. - skip_assert_against_bash because of this known upstream limitation. -# skip: known upstream library limitation with consecutive non-whitespace IFS delimiters -skip_assert_against_bash: true +description: Consecutive non-whitespace IFS delimiters produce empty fields. input: script: |+ IFS=: A="a::b" for w in $A; do echo "[$w]"; done expect: - # bash produces: [a]\n[]\n[b] - # rshell currently produces: [a]\n[b] (upstream library limitation) - stdout: "[a]\n[b]\n" - stderr: "" + stdout: |+ + [a] + [] + [b] + stderr: |+ exit_code: 0 diff --git a/tests/scenarios/shell/field_splitting/mixed_ws_nonws_ifs.yaml b/tests/scenarios/shell/field_splitting/mixed_ws_nonws_ifs.yaml index 15f9a7fc5..bc5839a86 100644 --- a/tests/scenarios/shell/field_splitting/mixed_ws_nonws_ifs.yaml +++ b/tests/scenarios/shell/field_splitting/mixed_ws_nonws_ifs.yaml @@ -1,9 +1,25 @@ description: "Whitespace IFS characters coalesce, non-whitespace splits" input: - script: | - IFS=" -" - A="a b-c" + script: |+ + IFS=' :' + A=' : a : : b : ' for w in $A; do echo "[$w]"; done + LEFT='a ' + RIGHT=' :b::' + for w in $LEFT$RIGHT; do echo "[$w]"; done + for w in $LEFT""$RIGHT; do echo "[$w]"; done expect: - stdout: "[a]\n[b]\n[c]\n" + stdout: |+ + [] + [a] + [] + [b] + [a] + [b] + [] + [a] + [] + [b] + [] + stderr: |+ exit_code: 0 diff --git a/tests/scenarios/shell/field_splitting/nonws_ifs_ambiguous_redirect.yaml b/tests/scenarios/shell/field_splitting/nonws_ifs_ambiguous_redirect.yaml new file mode 100644 index 000000000..5f145be64 --- /dev/null +++ b/tests/scenarios/shell/field_splitting/nonws_ifs_ambiguous_redirect.yaml @@ -0,0 +1,19 @@ +description: A leading empty IFS field makes an input redirect ambiguous instead of reading the remaining filename. +setup: + files: + - path: input.txt + content: |+ + must not be read +input: + allowed_paths: ["$DIR"] + script: |+ + IFS=: + target=':input.txt' + cat < $target + echo $? +expect: + stdout: |+ + 1 + stderr_contains: + - ambiguous redirect + exit_code: 0 diff --git a/tests/scenarios/shell/field_splitting/nonws_ifs_argument_boundaries.yaml b/tests/scenarios/shell/field_splitting/nonws_ifs_argument_boundaries.yaml new file mode 100644 index 000000000..f2638c0fa --- /dev/null +++ b/tests/scenarios/shell/field_splitting/nonws_ifs_argument_boundaries.yaml @@ -0,0 +1,20 @@ +description: Leading and adjacent non-whitespace IFS delimiters preserve empty arguments, with quotes and adjoining literals. +input: + script: |+ + IFS=: + x=':a::b:' + printf '<%s>\n' $x "$x" pre${x}post +expect: + stdout: |+ + <> + + <> + + <:a::b:> +
+    
+    <>
+    
+    
+  stderr: |+
+  exit_code: 0
diff --git a/tests/scenarios/shell/field_splitting/nonws_ifs_delimiter_edges.yaml b/tests/scenarios/shell/field_splitting/nonws_ifs_delimiter_edges.yaml
new file mode 100644
index 000000000..a51951da0
--- /dev/null
+++ b/tests/scenarios/shell/field_splitting/nonws_ifs_delimiter_edges.yaml
@@ -0,0 +1,36 @@
+description: Delimiter-only, trailing, and Unicode IFS separators preserve the Bash argument boundaries.
+input:
+  script: |+
+    IFS=:
+    for x in '' ':' '::' 'a:' 'a::'; do
+      echo next
+      for w in $x; do printf '<%s>\n' "$w"; done
+    done
+    IFS=é
+    x='éaééb é'
+    printf '<%s>\n' $x
+    IFS=$(printf '\r\v\f:')
+    x=$(printf '\r:\v:\fa:\r')
+    printf '<%s>\n' $x
+expect:
+  stdout: |+
+    next
+    next
+    <>
+    next
+    <>
+    <>
+    next
+    
+    next
+    
+    <>
+    <>
+    
+    <>
+    
+    <>
+    <>
+    
+  stderr: |+
+  exit_code: 0
diff --git a/tests/scenarios/shell/field_splitting/nonws_ifs_expansion_order.yaml b/tests/scenarios/shell/field_splitting/nonws_ifs_expansion_order.yaml
new file mode 100644
index 000000000..4bc43b06c
--- /dev/null
+++ b/tests/scenarios/shell/field_splitting/nonws_ifs_expansion_order.yaml
@@ -0,0 +1,38 @@
+description: Splitting preserves globbing and evaluates command substitutions once without reinterpreting their data as shell source.
+setup:
+  files:
+    - path: a.txt
+      content: |+
+    - path: b.txt
+      content: |+
+input:
+  allowed_paths: ["$DIR"]
+  script: |+
+    IFS=:
+    for w in $(echo once >&2; printf ':*.txt::'); do printf '<%s>\n' "$w"; done
+    x=':{a,b}:~user:\*:'
+    printf '<%s>\n' $x
+    x='::'
+    printf '<%s>\n' {pre,post}${x}end
+    x='a b'
+    printf '<%s>\n' $x
+expect:
+  stdout: |+
+    <>
+    
+    
+    <>
+    <>
+    <{a,b}>
+    <~user>
+    <\*>
+    
+    <>
+    
+    
+    <>
+    
+    
+  stderr: |+
+    once
+  exit_code: 0

From 2bfd4679c97c9cb07fb7d2e0db3705e56f1188eb Mon Sep 17 00:00:00 2001
From: "matthew.deguzman" 
Date: Wed, 28 Oct 2026 18:08:42 -0400
Subject: [PATCH 2/3] test: split Unicode IFS case out of the bash-compared
 scenario
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The Debian bash oracle runs in the POSIX locale and splits on each byte
of a multibyte IFS character, while rshell always uses UTF-8. Move the
IFS=é case into its own scenario with skip_assert_against_bash, matching
the existing printf/wc Unicode precedents, so the remaining delimiter
edge cases stay compared against bash.

Environment: Datadog workspace

Co-Authored-By: Claude Code Sonnet 5.5 
---
 .../nonws_ifs_delimiter_edges.yaml                |  9 +--------
 .../nonws_ifs_unicode_separator.yaml              | 15 +++++++++++++++
 2 files changed, 16 insertions(+), 8 deletions(-)
 create mode 100644 tests/scenarios/shell/field_splitting/nonws_ifs_unicode_separator.yaml

diff --git a/tests/scenarios/shell/field_splitting/nonws_ifs_delimiter_edges.yaml b/tests/scenarios/shell/field_splitting/nonws_ifs_delimiter_edges.yaml
index a51951da0..39021460c 100644
--- a/tests/scenarios/shell/field_splitting/nonws_ifs_delimiter_edges.yaml
+++ b/tests/scenarios/shell/field_splitting/nonws_ifs_delimiter_edges.yaml
@@ -1,4 +1,4 @@
-description: Delimiter-only, trailing, and Unicode IFS separators preserve the Bash argument boundaries.
+description: Delimiter-only and trailing IFS separators preserve the Bash argument boundaries.
 input:
   script: |+
     IFS=:
@@ -6,9 +6,6 @@ input:
       echo next
       for w in $x; do printf '<%s>\n' "$w"; done
     done
-    IFS=é
-    x='éaééb é'
-    printf '<%s>\n' $x
     IFS=$(printf '\r\v\f:')
     x=$(printf '\r:\v:\fa:\r')
     printf '<%s>\n' $x
@@ -26,10 +23,6 @@ expect:
     
     <>
     <>
-    
-    <>
-    
-    <>
     <>
     
   stderr: |+
diff --git a/tests/scenarios/shell/field_splitting/nonws_ifs_unicode_separator.yaml b/tests/scenarios/shell/field_splitting/nonws_ifs_unicode_separator.yaml
new file mode 100644
index 000000000..009d45a28
--- /dev/null
+++ b/tests/scenarios/shell/field_splitting/nonws_ifs_unicode_separator.yaml
@@ -0,0 +1,15 @@
+description: A multibyte non-whitespace IFS character splits on the whole character and preserves empty fields.
+skip_assert_against_bash: true  # bash oracle runs in the POSIX locale and splits on each byte of é; rshell always uses UTF-8
+input:
+  script: |+
+    IFS=é
+    x='éaééb é'
+    printf '<%s>\n' $x
+expect:
+  stdout: |+
+    <>
+    
+    <>
+    
+  stderr: |+
+  exit_code: 0

From 683bdfbd6ee368b4868a73be5d188bac7e72b0bf Mon Sep 17 00:00:00 2001
From: "matthew.deguzman" 
Date: Wed, 28 Oct 2026 18:22:09 -0400
Subject: [PATCH 3/3] ci: trigger checks after retargeting PR to main

Environment: Datadog workspace

Co-Authored-By: Claude Code Opus 5.5