Correct spath's implementation and multiple explain results

Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>

Author: yuancu

core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

@@ -747,8 +747,8 @@ public RelNode visitTranspose(
             .map(
                 f ->
                     Map.entry(
-                        ImmutableList.of(rx.makeLiteral(f)),
-                        ImmutableList.of((RexNode) rx.makeCast(varchar, b.field(f), true))))
+                        ImmutableList.of((RexLiteral) rx.makeLiteral(f, varchar, true)),
+                        ImmutableList.of(rx.makeCast(varchar, b.field(f), true, true))))
             .collect(Collectors.toList()));
 
     // Step 3: Trim spaces from columnName column before pivot

core/src/main/java/org/opensearch/sql/calcite/DynamicFieldsHelper.java

@@ -290,13 +290,28 @@ private static RexNode getFieldsAsMap(
     List<String> keys = excludeMetaFields(existingFields);
     keys.removeAll(excluded);
     Collections.sort(keys);
+    // Handle empty keys case - Calcite ARRAY[] requires at least 1 element
+    if (keys.isEmpty()) {
+      return createEmptyMap(context);
+    }
     RexNode keysArray = getStringLiteralArray(keys, context);
     List<RexNode> values =
-        keys.stream().map(key -> context.relBuilder.field(key)).collect(Collectors.toList());
+        keys.stream()
+            .map(key -> context.relBuilder.cast(context.relBuilder.field(key), SqlTypeName.VARCHAR))
+            .collect(Collectors.toList());
     RexNode valuesArray = makeArray(values, context);
     return context.rexBuilder.makeCall(BuiltinFunctionName.MAP_FROM_ARRAYS, keysArray, valuesArray);
   }
 
+  /** Create an empty map by casting NULL to MAP type */
+  private static RexNode createEmptyMap(CalcitePlanContext context) {
+    RelDataType varcharType =
+        context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR);
+    RelDataType mapType =
+        context.rexBuilder.getTypeFactory().createMapType(varcharType, varcharType);
+    return context.rexBuilder.makeNullLiteral(mapType);
+  }
+
   /** Create string literal array from collection of strings */
   private static RexNode getStringLiteralArray(
       Collection<String> keys, CalcitePlanContext context) {

core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java

@@ -516,6 +516,8 @@ public SqlNode rewriteCall(SqlValidator validator, SqlCall call) {
   // it will choose none of them in the end.) Therefore, SPARK functions used are explicitly
   // declared here for lookup.
   public static final SqlFunction REGEXP = SqlLibraryOperators.REGEXP;
+  public static final SqlFunction MAP_FROM_ARRAYS = SqlLibraryOperators.MAP_FROM_ARRAYS;
+  public static final SqlFunction MAP_CONCAT = SqlLibraryOperators.MAP_CONCAT;
 
   /**
    * Returns the PPL specific operator table, creating it if necessary.

core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java

@@ -806,9 +806,9 @@ void populate() {
       registerOperator(MVZIP, PPLBuiltinOperators.MVZIP);
       registerOperator(MVMAP, PPLBuiltinOperators.TRANSFORM);
       registerOperator(MAP_APPEND, PPLBuiltinOperators.MAP_APPEND);
-      registerOperator(MAP_CONCAT, SqlLibraryOperators.MAP_CONCAT);
+      registerOperator(MAP_CONCAT, PPLBuiltinOperators.MAP_CONCAT);
       registerOperator(MAP_REMOVE, PPLBuiltinOperators.MAP_REMOVE);
-      registerOperator(MAP_FROM_ARRAYS, SqlLibraryOperators.MAP_FROM_ARRAYS);
+      registerOperator(MAP_FROM_ARRAYS, PPLBuiltinOperators.MAP_FROM_ARRAYS);
       registerOperator(ARRAY_LENGTH, SqlLibraryOperators.ARRAY_LENGTH);
       registerOperator(ARRAY_SLICE, SqlLibraryOperators.ARRAY_SLICE);
       registerOperator(FORALL, PPLBuiltinOperators.FORALL);

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLSpathCommandIT.java

@@ -5,9 +5,11 @@
 
 package org.opensearch.sql.calcite.remote;
 
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_LOGS;
 import static org.opensearch.sql.util.MatcherUtils.rows;
 import static org.opensearch.sql.util.MatcherUtils.schema;
 import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
+import static org.opensearch.sql.util.MatcherUtils.verifyNumOfRows;
 import static org.opensearch.sql.util.MatcherUtils.verifySchema;
 
 import java.io.IOException;
@@ -19,6 +21,7 @@ public class CalcitePPLSpathCommandIT extends CalcitePPLSpathTestBase {
   public void init() throws Exception {
     super.init();
     enableCalcite();
+    loadIndex(Index.LOGS);
 
     putJsonItem(1, "simple", sj("{'a': 1, 'b': 2, 'c': 3}"));
     putJsonItem(2, "simple", sj("{'a': 1, 'b': 2, 'c': 3}"));
@@ -154,6 +157,14 @@ public void testSpathWithAbsentField() throws IOException {
     verifyDataRows(result, rows("1", null));
   }
 
+  @Test
+  public void testSpathWithDynamicFields() throws IOException {
+    JSONObject result =
+        executeQuery(source(TEST_INDEX_LOGS, "spath input=message | where status = '200'"));
+    verifySchema(result, schema("status", "string"));
+    verifyNumOfRows(result, 0);
+  }
+
   @Test
   public void testOverwrap() throws IOException {
     JSONObject result =

integ-test/src/test/resources/expectedOutput/calcite/explain_mvcombine.yaml

@@ -1,10 +1,9 @@
 calcite:
   logical: |
     LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])
-      LogicalProject(state=[$0], city=[$1], age=[$2])
-        LogicalAggregate(group=[{0, 1}], age=[ARRAY_AGG($2) FILTER $3])
-          LogicalProject(state=[$7], city=[$5], age=[$8], $f3=[IS NOT NULL($8)])
-            CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])
+      LogicalAggregate(group=[{0, 1}], age=[ARRAY_AGG($2) FILTER $3])
+        LogicalProject(state=[$7], city=[$5], age=[$8], $f3=[IS NOT NULL($8)])
+          CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])
   physical: |
     EnumerableLimit(fetch=[10000])
       EnumerableAggregate(group=[{0, 1}], age=[ARRAY_AGG($2) FILTER $3])

integ-test/src/test/resources/expectedOutput/calcite/explain_spath_with_dynamic_fields.yaml

@@ -1,9 +1,10 @@
 calcite:
   logical: |
     LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])
-      LogicalFilter(condition=[=($0, '200')])
-        LogicalProject(status=[CAST(ITEM(JSON_EXTRACT_ALL($3), 'status')):VARCHAR NOT NULL], _MAP=[map_append(MAP_FROM_ARRAYS(ARRAY('@timestamp', 'created_at', 'level', 'message', 'server'), ARRAY($2, $0, $4, $3, $1)), map_remove(JSON_EXTRACT_ALL($3), ARRAY('status')))])
-          CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])
+      LogicalProject(status=[$0], _MAP=[$1])
+        LogicalFilter(condition=[=($0, '200')])
+          LogicalProject(status=[CAST(ITEM(JSON_EXTRACT_ALL($3), 'status')):VARCHAR], _MAP=[map_append(MAP_FROM_ARRAYS(ARRAY('@timestamp':VARCHAR, 'created_at':VARCHAR, 'level':VARCHAR, 'message':VARCHAR, 'server':VARCHAR), ARRAY(CAST($2):VARCHAR, CAST($0):VARCHAR, $4, $3, $1)), map_remove(JSON_EXTRACT_ALL($3), ARRAY('status':VARCHAR)))])
+            CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])
   physical: |
-    EnumerableCalc(expr#0..4=[{inputs}], expr#5=[JSON_EXTRACT_ALL($t0)], expr#6=['status'], expr#7=[ITEM($t5, $t6)], expr#8=[CAST($t7):VARCHAR NOT NULL], expr#9=['@timestamp'], expr#10=['created_at'], expr#11=['level'], expr#12=['message'], expr#13=['server'], expr#14=[ARRAY($t9, $t10, $t11, $t12, $t13)], expr#15=[ARRAY($t1, $t2, $t3, $t0, $t4)], expr#16=[MAP_FROM_ARRAYS($t14, $t15)], expr#17=[ARRAY($t6)], expr#18=[map_remove($t5, $t17)], expr#19=[map_append($t16, $t18)], status=[$t8], _MAP=[$t19])
-      CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[PROJECT->[message, @timestamp, created_at, level, server], SCRIPT->=(CAST(ITEM(JSON_EXTRACT_ALL($0), 'status')):VARCHAR NOT NULL, '200'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQHkXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICJraW5kIjogIkNBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiSVRFTSIsCiAgICAgICAgICAgICJraW5kIjogIklURU0iLAogICAgICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgICAgICB9LAogICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgIm9wIjogewogICAgICAgICAgICAgICAgIm5hbWUiOiAiSlNPTl9FWFRSQUNUX0FMTCIsCiAgICAgICAgICAgICAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAgICAgICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiTUFQIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICAgICAia2V5IjogewogICAgICAgICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgICJ2YWx1ZSI6IHsKICAgICAgICAgICAgICAgICAgInR5cGUiOiAiQU5ZIiwKICAgICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgICAgICAgICAgICAgInNjYWxlIjogLTIxNDc0ODM2NDgKICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICB9LAogICAgICAgICAgICAgICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAgICAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgICAgIH0KICAgICAgICAgICAgfQogICAgICAgICAgXQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["message.keyword","status","200"]}},"boost":1.0}},"_source":{"includes":["message","@timestamp","created_at","level","server"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])
+    EnumerableCalc(expr#0..4=[{inputs}], expr#5=[JSON_EXTRACT_ALL($t0)], expr#6=['status'], expr#7=[ITEM($t5, $t6)], expr#8=[CAST($t7):VARCHAR], expr#9=['@timestamp':VARCHAR], expr#10=['created_at':VARCHAR], expr#11=['level':VARCHAR], expr#12=['message':VARCHAR], expr#13=['server':VARCHAR], expr#14=[ARRAY($t9, $t10, $t11, $t12, $t13)], expr#15=[CAST($t1):VARCHAR], expr#16=[CAST($t2):VARCHAR], expr#17=[ARRAY($t15, $t16, $t3, $t0, $t4)], expr#18=[MAP_FROM_ARRAYS($t14, $t17)], expr#19=['status':VARCHAR], expr#20=[ARRAY($t19)], expr#21=[map_remove($t5, $t20)], expr#22=[map_append($t18, $t21)], status=[$t8], _MAP=[$t22])
+      CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[PROJECT->[message, @timestamp, created_at, level, server], SCRIPT->=(CAST(ITEM(JSON_EXTRACT_ALL($0), 'status')):VARCHAR, '200'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQHkHsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICJraW5kIjogIkNBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiSVRFTSIsCiAgICAgICAgICAgICJraW5kIjogIklURU0iLAogICAgICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgICAgICB9LAogICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgIm9wIjogewogICAgICAgICAgICAgICAgIm5hbWUiOiAiSlNPTl9FWFRSQUNUX0FMTCIsCiAgICAgICAgICAgICAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAgICAgICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiTUFQIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICAgICAia2V5IjogewogICAgICAgICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgICJ2YWx1ZSI6IHsKICAgICAgICAgICAgICAgICAgInR5cGUiOiAiQU5ZIiwKICAgICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgICAgICAgICAgICAgInNjYWxlIjogLTIxNDc0ODM2NDgKICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICB9LAogICAgICAgICAgICAgICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAgICAgICAgICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgICAgIH0KICAgICAgICAgICAgfQogICAgICAgICAgXQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["message.keyword","status","200"]}},"boost":1.0}},"_source":{"includes":["message","@timestamp","created_at","level","server"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])