Skip to content

Commit cd3a8e7

Browse files
ritvibhattritvibhatt
authored
Add time conversion functions for convert command (#5210)
* add time conversion functions Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * add integ tests Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * fix number of arguments Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * fix timeformat parsing in integ tests Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * update comments Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * apply spotless Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * update convert command doc Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * sql cli test fix Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * update utils Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * fix integ tests Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * add explain tests Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * fix test Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * apply spotless Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * fix null timeformat Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * fix tests Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * empty Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * update docs Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * empty Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * fix parsing Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * update docs table format Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * update doc example Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> * fix ctime fractional seconds Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com> --------- Signed-off-by: Ritvi Bhatt <ribhatt@amazon.com>
1 parent 3f740fb commit cd3a8e7

28 files changed

Lines changed: 1104 additions & 18 deletions

‎core/src/main/java/org/opensearch/sql/ast/tree/Convert.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
@RequiredArgsConstructor
2424
public class Convert extends UnresolvedPlan {
2525
private final List<Let> conversions;
26+
private final String timeFormat;
2627
private UnresolvedPlan child;
2728

2829
@Override

‎core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1170,7 +1170,7 @@ public RelNode visitConvert(Convert node, CalcitePlanContext context) {
11701170
ConversionState state = new ConversionState();
11711171

11721172
for (Let conversion : node.getConversions()) {
1173-
processConversion(conversion, state, context);
1173+
processConversion(conversion, node.getTimeFormat(), state, context);
11741174
}
11751175

11761176
return buildConversionProjection(state, context);
@@ -1183,14 +1183,14 @@ private static class ConversionState {
11831183
}
11841184

11851185
private void processConversion(
1186-
Let conversion, ConversionState state, CalcitePlanContext context) {
1186+
Let conversion, String timeFormat, ConversionState state, CalcitePlanContext context) {
11871187
String target = conversion.getVar().getField().toString();
11881188
UnresolvedExpression expression = conversion.getExpression();
11891189

11901190
if (expression instanceof Field) {
11911191
processFieldCopyConversion(target, (Field) expression, state, context);
11921192
} else if (expression instanceof Function) {
1193-
processFunctionConversion(target, (Function) expression, state, context);
1193+
processFunctionConversion(target, (Function) expression, timeFormat, state, context);
11941194
} else {
11951195
throw new SemanticCheckException("Convert command requires function call expressions");
11961196
}
@@ -1213,7 +1213,11 @@ private void processFieldCopyConversion(
12131213
}
12141214

12151215
private void processFunctionConversion(
1216-
String target, Function function, ConversionState state, CalcitePlanContext context) {
1216+
String target,
1217+
Function function,
1218+
String timeFormat,
1219+
ConversionState state,
1220+
CalcitePlanContext context) {
12171221
String functionName = function.getFuncName();
12181222
List<UnresolvedExpression> args = function.getFuncArgs();
12191223

@@ -1230,8 +1234,7 @@ private void processFunctionConversion(
12301234
state.seenFields.add(source);
12311235

12321236
RexNode sourceField = context.relBuilder.field(source);
1233-
RexNode convertCall =
1234-
PPLFuncImpTable.INSTANCE.resolve(context.rexBuilder, functionName, sourceField);
1237+
RexNode convertCall = resolveConvertFunction(functionName, sourceField, timeFormat, context);
12351238

12361239
if (!target.equals(source)) {
12371240
state.additions.add(Pair.of(target, context.relBuilder.alias(convertCall, target)));
@@ -1240,6 +1243,23 @@ private void processFunctionConversion(
12401243
}
12411244
}
12421245

1246+
private RexNode resolveConvertFunction(
1247+
String functionName, RexNode sourceField, String timeFormat, CalcitePlanContext context) {
1248+
1249+
// Time functions that support timeformat parameter
1250+
Set<String> timeFunctions = Set.of("ctime", "mktime");
1251+
1252+
if (timeFunctions.contains(functionName.toLowerCase()) && timeFormat != null) {
1253+
// For time functions with custom timeformat, pass the format as a second parameter
1254+
RexNode timeFormatLiteral = context.rexBuilder.makeLiteral(timeFormat);
1255+
return PPLFuncImpTable.INSTANCE.resolve(
1256+
context.rexBuilder, functionName, sourceField, timeFormatLiteral);
1257+
} else {
1258+
// Regular conversion functions or time functions without custom format
1259+
return PPLFuncImpTable.INSTANCE.resolve(context.rexBuilder, functionName, sourceField);
1260+
}
1261+
}
1262+
12431263
private RelNode buildConversionProjection(ConversionState state, CalcitePlanContext context) {
12441264
List<String> originalFields = context.relBuilder.peek().getRowType().getFieldNames();
12451265
List<RexNode> projectList = new ArrayList<>();

‎core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,10 @@ private PPLOperandTypes() {}
8484
UDFOperandMetadata.wrap(
8585
(CompositeOperandTypeChecker)
8686
OperandTypes.ANY.or(OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.INTEGER)));
87+
public static final UDFOperandMetadata ANY_OPTIONAL_STRING =
88+
UDFOperandMetadata.wrap(
89+
(CompositeOperandTypeChecker)
90+
OperandTypes.ANY.or(OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.CHARACTER)));
8791
public static final UDFOperandMetadata ANY_OPTIONAL_TIMESTAMP =
8892
UDFOperandMetadata.wrap(
8993
(CompositeOperandTypeChecker)

‎core/src/main/java/org/opensearch/sql/expression/datetime/StrftimeFormatterUtil.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,4 +249,40 @@ private static long extractFirstNDigits(double value, int digits) {
249249

250250
return isNegative ? -result : result;
251251
}
252+
253+
/** Mapping from strftime specifiers to Java DateTimeFormatter patterns for parsing. */
254+
private static final Map<String, String> STRFTIME_TO_JAVA_PARSE =
255+
ImmutableMap.<String, String>builder()
256+
.put("%Y", "yyyy")
257+
.put("%y", "yy")
258+
.put("%m", "MM")
259+
.put("%B", "MMMM")
260+
.put("%b", "MMM")
261+
.put("%d", "dd")
262+
.put("%H", "HH")
263+
.put("%I", "hh")
264+
.put("%M", "mm")
265+
.put("%S", "ss")
266+
.put("%p", "a")
267+
.put("%T", "HH:mm:ss")
268+
.put("%F", "yyyy-MM-dd")
269+
.put("%%", "'%'")
270+
.build();
271+
272+
/**
273+
* Convert a strftime format string to a Java DateTimeFormatter pattern suitable for parsing.
274+
*
275+
* @param strftimeFormat the strftime-style format string (e.g. {@code %Y-%m-%d %H:%M:%S})
276+
* @return a Java DateTimeFormatter pattern (e.g. {@code yyyy-MM-dd HH:mm:ss})
277+
*/
278+
public static String toJavaPattern(String strftimeFormat) {
279+
Matcher m = Pattern.compile("%[A-Za-z%]").matcher(strftimeFormat);
280+
StringBuilder sb = new StringBuilder();
281+
while (m.find()) {
282+
String replacement = STRFTIME_TO_JAVA_PARSE.getOrDefault(m.group(), m.group());
283+
m.appendReplacement(sb, Matcher.quoteReplacement(replacement));
284+
}
285+
m.appendTail(sb);
286+
return sb.toString();
287+
}
252288
}

‎core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,12 @@
6363
import org.opensearch.sql.expression.function.jsonUDF.JsonKeysFunctionImpl;
6464
import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl;
6565
import org.opensearch.sql.expression.function.udf.AutoConvertFunction;
66+
import org.opensearch.sql.expression.function.udf.CTimeConvertFunction;
6667
import org.opensearch.sql.expression.function.udf.CryptographicFunction;
68+
import org.opensearch.sql.expression.function.udf.Dur2SecConvertFunction;
6769
import org.opensearch.sql.expression.function.udf.MemkConvertFunction;
70+
import org.opensearch.sql.expression.function.udf.MkTimeConvertFunction;
71+
import org.opensearch.sql.expression.function.udf.MsTimeConvertFunction;
6872
import org.opensearch.sql.expression.function.udf.NumConvertFunction;
6973
import org.opensearch.sql.expression.function.udf.ParseFunction;
7074
import org.opensearch.sql.expression.function.udf.RelevanceQueryFunction;
@@ -431,6 +435,10 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable {
431435
public static final SqlOperator RMCOMMA = new RmcommaConvertFunction().toUDF("RMCOMMA");
432436
public static final SqlOperator RMUNIT = new RmunitConvertFunction().toUDF("RMUNIT");
433437
public static final SqlOperator MEMK = new MemkConvertFunction().toUDF("MEMK");
438+
public static final SqlOperator CTIME = new CTimeConvertFunction().toUDF("CTIME");
439+
public static final SqlOperator MKTIME = new MkTimeConvertFunction().toUDF("MKTIME");
440+
public static final SqlOperator MSTIME = new MsTimeConvertFunction().toUDF("MSTIME");
441+
public static final SqlOperator DUR2SEC = new Dur2SecConvertFunction().toUDF("DUR2SEC");
434442

435443
public static final SqlOperator WIDTH_BUCKET =
436444
new org.opensearch.sql.expression.function.udf.binning.WidthBucketFunction()

‎core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import static org.opensearch.sql.expression.function.BuiltinFunctionName.COT;
4040
import static org.opensearch.sql.expression.function.BuiltinFunctionName.COUNT;
4141
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CRC32;
42+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CTIME;
4243
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURDATE;
4344
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURRENT_DATE;
4445
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURRENT_TIME;
@@ -61,6 +62,7 @@
6162
import static org.opensearch.sql.expression.function.BuiltinFunctionName.DEGREES;
6263
import static org.opensearch.sql.expression.function.BuiltinFunctionName.DIVIDE;
6364
import static org.opensearch.sql.expression.function.BuiltinFunctionName.DIVIDEFUNCTION;
65+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.DUR2SEC;
6466
import static org.opensearch.sql.expression.function.BuiltinFunctionName.E;
6567
import static org.opensearch.sql.expression.function.BuiltinFunctionName.EARLIEST;
6668
import static org.opensearch.sql.expression.function.BuiltinFunctionName.EQUAL;
@@ -144,12 +146,14 @@
144146
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINUTE;
145147
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINUTE_OF_DAY;
146148
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINUTE_OF_HOUR;
149+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MKTIME;
147150
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MOD;
148151
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MODULUS;
149152
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MODULUSFUNCTION;
150153
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTH;
151154
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTHNAME;
152155
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTH_OF_YEAR;
156+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MSTIME;
153157
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLY;
154158
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLYFUNCTION;
155159
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTI_MATCH;
@@ -991,6 +995,10 @@ void populate() {
991995
registerOperator(RMCOMMA, PPLBuiltinOperators.RMCOMMA);
992996
registerOperator(RMUNIT, PPLBuiltinOperators.RMUNIT);
993997
registerOperator(MEMK, PPLBuiltinOperators.MEMK);
998+
registerOperator(CTIME, PPLBuiltinOperators.CTIME);
999+
registerOperator(MKTIME, PPLBuiltinOperators.MKTIME);
1000+
registerOperator(MSTIME, PPLBuiltinOperators.MSTIME);
1001+
registerOperator(DUR2SEC, PPLBuiltinOperators.DUR2SEC);
9941002

9951003
register(
9961004
TOSTRING,

‎core/src/main/java/org/opensearch/sql/expression/function/udf/CTimeConvertFunction.java‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.expression.function.udf;
7+
8+
import java.time.Instant;
9+
import java.time.ZoneId;
10+
import java.time.ZonedDateTime;
11+
import java.util.List;
12+
import org.apache.calcite.adapter.enumerable.NotNullImplementor;
13+
import org.apache.calcite.adapter.enumerable.NullPolicy;
14+
import org.apache.calcite.adapter.enumerable.RexToLixTranslator;
15+
import org.apache.calcite.linq4j.tree.Expression;
16+
import org.apache.calcite.linq4j.tree.Expressions;
17+
import org.apache.calcite.rex.RexCall;
18+
import org.apache.calcite.sql.type.SqlReturnTypeInference;
19+
import org.opensearch.sql.calcite.utils.PPLOperandTypes;
20+
import org.opensearch.sql.calcite.utils.PPLReturnTypes;
21+
import org.opensearch.sql.expression.datetime.StrftimeFormatterUtil;
22+
import org.opensearch.sql.expression.function.ImplementorUDF;
23+
import org.opensearch.sql.expression.function.UDFOperandMetadata;
24+
25+
/**
26+
* PPL ctime() conversion function. Converts UNIX epoch timestamps to human-readable time strings
27+
* using strftime format specifiers. Default format: {@code %m/%d/%Y %H:%M:%S}.
28+
*/
29+
public class CTimeConvertFunction extends ImplementorUDF {
30+
31+
private static final String DEFAULT_FORMAT = "%m/%d/%Y %H:%M:%S";
32+
33+
public CTimeConvertFunction() {
34+
super(new CTimeImplementor(), NullPolicy.ANY);
35+
}
36+
37+
@Override
38+
public SqlReturnTypeInference getReturnTypeInference() {
39+
return PPLReturnTypes.STRING_FORCE_NULLABLE;
40+
}
41+
42+
@Override
43+
public UDFOperandMetadata getOperandMetadata() {
44+
return PPLOperandTypes.ANY_OPTIONAL_STRING;
45+
}
46+
47+
public static class CTimeImplementor implements NotNullImplementor {
48+
@Override
49+
public Expression implement(
50+
RexToLixTranslator translator, RexCall call, List<Expression> translatedOperands) {
51+
if (translatedOperands.isEmpty()) {
52+
return Expressions.constant(null, String.class);
53+
}
54+
Expression fieldValue = Expressions.box(translatedOperands.get(0));
55+
if (translatedOperands.size() == 1) {
56+
return Expressions.call(CTimeConvertFunction.class, "convert", fieldValue);
57+
}
58+
Expression timeFormat = Expressions.box(translatedOperands.get(1));
59+
return Expressions.call(
60+
CTimeConvertFunction.class, "convertWithFormat", fieldValue, timeFormat);
61+
}
62+
}
63+
64+
public static String convert(Object value) {
65+
return convertWithFormat(value, null);
66+
}
67+
68+
public static String convertWithFormat(Object value, Object timeFormatObj) {
69+
Double timestamp = toEpochSeconds(value);
70+
if (timestamp == null) {
71+
return null;
72+
}
73+
String format = (timeFormatObj != null) ? timeFormatObj.toString().trim() : DEFAULT_FORMAT;
74+
if (format.isEmpty()) {
75+
return null;
76+
}
77+
try {
78+
long seconds = timestamp.longValue();
79+
int nanos = (int) ((timestamp - seconds) * 1_000_000_000);
80+
Instant instant = Instant.ofEpochSecond(seconds, nanos);
81+
ZonedDateTime zdt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC"));
82+
return StrftimeFormatterUtil.formatZonedDateTime(zdt, format).stringValue();
83+
} catch (Exception e) {
84+
return null;
85+
}
86+
}
87+
88+
public static Double toEpochSeconds(Object value) {
89+
if (value == null) {
90+
return null;
91+
}
92+
if (value instanceof Number) {
93+
return ((Number) value).doubleValue();
94+
}
95+
String str = value.toString().trim();
96+
if (str.isEmpty()) {
97+
return null;
98+
}
99+
try {
100+
return Double.parseDouble(str);
101+
} catch (NumberFormatException e) {
102+
return null;
103+
}
104+
}
105+
}

‎core/src/main/java/org/opensearch/sql/expression/function/udf/Dur2SecConvertFunction.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.expression.function.udf;
7+
8+
import java.util.regex.Matcher;
9+
import java.util.regex.Pattern;
10+
11+
/** PPL dur2sec() conversion function. Converts duration format {@code [D+]HH:MM:SS} to seconds */
12+
public class Dur2SecConvertFunction extends BaseConversionUDF {
13+
14+
public static final Dur2SecConvertFunction INSTANCE = new Dur2SecConvertFunction();
15+
16+
// Matches [D+]HH:MM:SS — optional days prefix with + separator
17+
private static final Pattern DURATION_PATTERN =
18+
Pattern.compile("^(?:(\\d+)\\+)?(\\d{1,2}):(\\d{1,2}):(\\d{1,2})$");
19+
20+
public Dur2SecConvertFunction() {
21+
super(Dur2SecConvertFunction.class);
22+
}
23+
24+
public static Object convert(Object value) {
25+
return INSTANCE.convertValue(value);
26+
}
27+
28+
@Override
29+
protected Object applyConversion(String preprocessedValue) {
30+
Double existingSeconds = tryParseDouble(preprocessedValue);
31+
if (existingSeconds != null) {
32+
return existingSeconds;
33+
}
34+
35+
Matcher matcher = DURATION_PATTERN.matcher(preprocessedValue);
36+
if (!matcher.matches()) {
37+
return null;
38+
}
39+
40+
try {
41+
int days = matcher.group(1) != null ? Integer.parseInt(matcher.group(1)) : 0;
42+
int hours = Integer.parseInt(matcher.group(2));
43+
int minutes = Integer.parseInt(matcher.group(3));
44+
int seconds = Integer.parseInt(matcher.group(4));
45+
46+
if (hours >= 24 || minutes >= 60 || seconds >= 60) {
47+
return null;
48+
}
49+
50+
return (double) (days * 86400 + hours * 3600 + minutes * 60 + seconds);
51+
} catch (NumberFormatException e) {
52+
return null;
53+
}
54+
}
55+
}

0 commit comments

Comments
 (0)