Add auto-extract mode for spath command (#5140)

* Add auto extraction mode in spath command

Signed-off-by: Chen Dai <daichen@amazon.com>

* Change json_extract_all to return map<string,string> and fix null perserve issue

Signed-off-by: Chen Dai <daichen@amazon.com>

* Refactor all unit test and integration tests

Signed-off-by: Chen Dai <daichen@amazon.com>

* Refactor json_extract_all and fix stringify issue

Signed-off-by: Chen Dai <daichen@amazon.com>

* Fix broken IT and doctest

Signed-off-by: Chen Dai <daichen@amazon.com>

* Address PR comments

Signed-off-by: Chen Dai <daichen@amazon.com>

* Mark auto extract mode as experimental

Signed-off-by: Chen Dai <daichen@amazon.com>

---------

Signed-off-by: Chen Dai <daichen@amazon.com>

Author: dai-chen

core/src/main/java/org/opensearch/sql/ast/tree/SPath.java

@@ -30,7 +30,7 @@ public class SPath extends UnresolvedPlan {
 
   @Nullable private final String outField;
 
-  private final String path;
+  @Nullable private final String path;
 
   @Override
   public UnresolvedPlan attach(UnresolvedPlan child) {
@@ -48,7 +48,20 @@ public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
     return nodeVisitor.visitSpath(this, context);
   }
 
+  /**
+   * Rewrites this spath node to an equivalent {@link Eval} node.
+   *
+   * <p>In path mode, rewrites to {@code eval output = json_extract(input, path)}. In auto-extract
+   * mode (path is null), rewrites to {@code eval output = json_extract_all(input)}.
+   */
   public Eval rewriteAsEval() {
+    if (path != null) {
+      return rewritePathMode();
+    }
+    return rewriteAutoExtractMode();
+  }
+
+  private Eval rewritePathMode() {
     String outField = this.outField;
     String unquotedPath = unquoteText(this.path);
     if (outField == null) {
@@ -62,4 +75,12 @@ public Eval rewriteAsEval() {
             AstDSL.function(
                 "json_extract", AstDSL.field(inField), AstDSL.stringLiteral(unquotedPath))));
   }
+
+  private Eval rewriteAutoExtractMode() {
+    String output = (outField != null) ? outField : inField;
+    return AstDSL.eval(
+        child,
+        AstDSL.let(
+            AstDSL.field(output), AstDSL.function("json_extract_all", AstDSL.field(inField))));
+  }
 }

core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractAllFunctionImpl.java

@@ -5,6 +5,7 @@
 
 package org.opensearch.sql.expression.function.jsonUDF;
 
+import static java.util.stream.Collectors.toMap;
 import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY;
 
 import com.fasterxml.jackson.core.JsonFactory;
@@ -51,7 +52,7 @@ public SqlReturnTypeInference getReturnTypeInference() {
     return ReturnTypes.explicit(
         TYPE_FACTORY.createMapType(
             TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR),
-            TYPE_FACTORY.createSqlType(SqlTypeName.ANY),
+            TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR),
             true));
   }
 
@@ -72,6 +73,11 @@ public Expression implement(
     }
   }
 
+  /**
+   * Evaluate the JSON extract-all function. Returns a {@code Map<String, String>} where keys are
+   * dot-separated JSON paths (with {@code {}} suffix for arrays) and all values are strings. Merged
+   * array values use {@code [a, b, c]} format.
+   */
   public static Object eval(Object... args) {
     if (args.length < 1) {
       return null;
@@ -82,7 +88,18 @@ public static Object eval(Object... args) {
       return null;
     }
 
-    return parseJson(jsonStr);
+    Map<String, Object> parsed = parseJson(jsonStr);
+    return parsed == null ? null : stringifyMap(parsed);
+  }
+
+  // TODO: JSON parsing dominates cost; consider stringify scalars in place during parsing
+  //  to avoid this extra pass.
+  private static Map<String, String> stringifyMap(Map<String, Object> map) {
+    return map.entrySet().stream()
+        .collect(
+            toMap(
+                Map.Entry::getKey,
+                e -> String.valueOf(e.getValue()))); // relies on List.toString() for [a, b, c]
   }
 
   private static Map<String, Object> parseJson(String jsonStr) {
@@ -150,7 +167,7 @@ private static Map<String, Object> parseJson(String jsonStr) {
   @SuppressWarnings("unchecked")
   private static void appendValue(Map<String, Object> resultMap, String path, Object value) {
     Object existingValue = resultMap.get(path);
-    if (existingValue == null) {
+    if (existingValue == null && !resultMap.containsKey(path)) { // key absent, not null value
       resultMap.put(path, value);
     } else if (existingValue instanceof List) {
       ((List<Object>) existingValue).add(value);