From b122f7846a21da3e64b72e00525a6246bc22da20 Mon Sep 17 00:00:00 2001 From: currantw Date: Thu, 9 Jan 2025 13:59:28 -0800 Subject: [PATCH 1/6] Review comments: add more documentation, add ignored tests for earliest and latest. Signed-off-by: currantw --- ...arkPPLBuiltInDateTimeFunctionITSuite.scala | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala index d9d5099cd..f37e5d712 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala @@ -432,6 +432,32 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite assertSameRows(Seq(Row(false), Row(true), Row(true)), frame) } + // TODO #957: Support earliest + ignore("test EARLIEST") { + var frame = sql(s""" + | source = $testTable + | | eval earliest_hour_before = earliest(now(), "-1h") + | | eval earliest_now = earliest(now(), "now") + | | eval earliest_hour_after = earliest(now(), "+1h") + | | fields earliest_hour_before, earliest_now, earliest_hour_after + | | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(true), Row(true), Row(false)), frame) + } + + // TODO #957: Support latest + ignore("test LATEST") { + var frame = sql(s""" + | source = $testTable + | | eval latest_hour_before = latest(now(), "-1h") + | | eval latest_now = latest(now(), "now") + | | eval latest_hour_after = latest(now(), "+1h") + | | fields latest_hour_before, latest_now, latest_hour_after + | | head 1 + | """.stripMargin) + assertSameRows(Seq(Row(false), Row(true), Row(true)), frame) + } + test("test CURRENT_TIME is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" | source = $testTable From d7ce2544ffb760192eee550de984830a3a8832fd Mon Sep 17 00:00:00 2001 From: currantw Date: Tue, 14 Jan 2025 10:53:07 -0800 Subject: [PATCH 2/6] Initial implementation of earliest/latest Signed-off-by: currantw --- ...arkPPLBuiltInDateTimeFunctionITSuite.scala | 26 ++++++++--------- .../src/main/antlr4/OpenSearchPPLLexer.g4 | 10 +++---- .../src/main/antlr4/OpenSearchPPLParser.g4 | 12 ++++---- .../function/BuiltinFunctionName.java | 4 ++- .../ppl/utils/BuiltinFunctionTransformer.java | 28 +++++++++++++++++-- 5 files changed, 53 insertions(+), 27 deletions(-) diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala index f37e5d712..fb2f64207 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala @@ -406,30 +406,28 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite assertSameRows(Seq(Row(true)), frame) } - // TODO #957: Support earliest - ignore("test EARLIEST") { + test("test EARLIEST") { var frame = sql(s""" | source=$testTable - | | eval earliest_hour_before = earliest(now(), "-1h") - | | eval earliest_now = earliest(now(), "now") - | | eval earliest_hour_after = earliest(now(), "+1h") - | | fields earliest_hour_before, earliest_now, earliest_hour_after + | | eval earliest_second_before = earliest("-1s", now()) + | | eval earliest_now = earliest("now", now()) + | | eval earliest_second_after = earliest("+1s", now()) + | | fields earliest_second_before, earliest_now, earliest_second_after | | head 1 | """.stripMargin) - assertSameRows(Seq(Row(true), Row(true), Row(false)), frame) + assertSameRows(Seq(Row(true, true, false)), frame) } - // TODO #957: Support latest - ignore("test LATEST") { + test("test LATEST") { var frame = sql(s""" | source=$testTable - | | eval latest_hour_before = latest(now(), "-1h") - | | eval latest_now = latest(now(), "now") - | | eval latest_hour_after = latest(now(), "+1h") - | | fields latest_hour_before, latest_now, latest_hour_after + | | eval latest_second_before = latest("-1s", now()) + | | eval latest_now = latest("now", now()) + | | eval latest_second_after = latest("+1s", now()) + | | fields latest_second_before, latest_now, latest_second_after | | head 1 | """.stripMargin) - assertSameRows(Seq(Row(false), Row(true), Row(true)), frame) + assertSameRows(Seq(Row(false, true, true)), frame) } // TODO #957: Support earliest diff --git a/ppl-spark-integration/src/main/antlr4/OpenSearchPPLLexer.g4 b/ppl-spark-integration/src/main/antlr4/OpenSearchPPLLexer.g4 index d80f988d3..1bb4b49d1 100644 --- a/ppl-spark-integration/src/main/antlr4/OpenSearchPPLLexer.g4 +++ b/ppl-spark-integration/src/main/antlr4/OpenSearchPPLLexer.g4 @@ -247,10 +247,6 @@ FIRST: 'FIRST'; LAST: 'LAST'; LIST: 'LIST'; VALUES: 'VALUES'; -EARLIEST: 'EARLIEST'; -EARLIEST_TIME: 'EARLIEST_TIME'; -LATEST: 'LATEST'; -LATEST_TIME: 'LATEST_TIME'; PER_DAY: 'PER_DAY'; PER_HOUR: 'PER_HOUR'; PER_MINUTE: 'PER_MINUTE'; @@ -338,7 +334,6 @@ MONTHNAME: 'MONTHNAME'; NOW: 'NOW'; PERIOD_ADD: 'PERIOD_ADD'; PERIOD_DIFF: 'PERIOD_DIFF'; -RELATIVE_TIMESTAMP: 'RELATIVE_TIMESTAMP'; SEC_TO_TIME: 'SEC_TO_TIME'; STR_TO_DATE: 'STR_TO_DATE'; SUBDATE: 'SUBDATE'; @@ -360,6 +355,11 @@ UTC_TIMESTAMP: 'UTC_TIMESTAMP'; WEEKDAY: 'WEEKDAY'; YEARWEEK: 'YEARWEEK'; +// RELATIVE TIME FUNCTIONS +RELATIVE_TIMESTAMP: 'RELATIVE_TIMESTAMP'; +EARLIEST: 'EARLIEST'; +LATEST: 'LATEST'; + // TEXT FUNCTIONS SUBSTR: 'SUBSTR'; SUBSTRING: 'SUBSTRING'; diff --git a/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4 b/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4 index 10ffe1977..dc5d44c57 100644 --- a/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4 +++ b/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4 @@ -755,7 +755,6 @@ dateTimeFunctionName | NOW | PERIOD_ADD | PERIOD_DIFF - | RELATIVE_TIMESTAMP | QUARTER | SECOND | SECOND_OF_MINUTE @@ -780,6 +779,13 @@ dateTimeFunctionName | WEEK_OF_YEAR | YEAR | YEARWEEK + | relativeTimeFunctionName + ; + +relativeTimeFunctionName + : RELATIVE_TIMESTAMP + | EARLIEST + | LATEST ; getFormatFunction @@ -1171,10 +1177,6 @@ keywordsCanBeId | LAST | LIST | VALUES - | EARLIEST - | EARLIEST_TIME - | LATEST - | LATEST_TIME | PER_DAY | PER_HOUR | PER_MINUTE diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 411a9c5ea..57a6372ca 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -133,8 +133,10 @@ public enum BuiltinFunctionName { LOCALTIMESTAMP(FunctionName.of("localtimestamp")), SYSDATE(FunctionName.of("sysdate")), - // Relative timestamp functions + // Relative time functions RELATIVE_TIMESTAMP(FunctionName.of("relative_timestamp")), + EARLIEST(FunctionName.of("earliest")), + LATEST(FunctionName.of("latest")), /** Text Functions. */ TOSTRING(FunctionName.of("tostring")), diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTransformer.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTransformer.java index 01987757f..1211b857b 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTransformer.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTransformer.java @@ -12,6 +12,8 @@ import org.apache.spark.sql.catalyst.expressions.CurrentTimestamp$; import org.apache.spark.sql.catalyst.expressions.DateAddInterval$; import org.apache.spark.sql.catalyst.expressions.Expression; +import org.apache.spark.sql.catalyst.expressions.GreaterThanOrEqual$; +import org.apache.spark.sql.catalyst.expressions.LessThanOrEqual$; import org.apache.spark.sql.catalyst.expressions.Literal$; import org.apache.spark.sql.catalyst.expressions.ScalaUDF; import org.apache.spark.sql.catalyst.expressions.TimestampAdd$; @@ -37,6 +39,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATE_SUB; import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAY_OF_MONTH; import static org.opensearch.sql.expression.function.BuiltinFunctionName.COALESCE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.EARLIEST; import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON; import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_ARRAY; import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_ARRAY_LENGTH; @@ -44,6 +47,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_KEYS; import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_OBJECT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_VALID; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LATEST; import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBTRACT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLY; import static org.opensearch.sql.expression.function.BuiltinFunctionName.DIVIDE; @@ -174,9 +178,23 @@ public interface BuiltinFunctionTransformer { args -> { return ToUTCTimestamp$.MODULE$.apply(CurrentTimestamp$.MODULE$.apply(), CurrentTimeZone$.MODULE$.apply()); }) + + // Relative time functions + .put( + RELATIVE_TIMESTAMP, + BuiltinFunctionTransformer::buildRelativeTimestampExpression) + .put( + EARLIEST, + args -> + LessThanOrEqual$.MODULE$.apply( + buildRelativeTimestampExpression(List.of(args.get(0))), + args.get(1))) .put( - RELATIVE_TIMESTAMP, - args -> SerializableUdf.visit("relative_timestamp", List.of(args.get(0), CurrentTimestamp$.MODULE$.apply(), CurrentTimeZone$.MODULE$.apply()))) + LATEST, + args -> + GreaterThanOrEqual$.MODULE$.apply( + buildRelativeTimestampExpression(List.of(args.get(0))), + args.get(1))) .build(); static Expression builtinFunction(org.opensearch.sql.ast.expression.Function function, List args) { @@ -218,4 +236,10 @@ static Expression[] createIntervalArgs(IntervalUnit unit, Expression value) { } return args; } + + private static Expression buildRelativeTimestampExpression(List args) { + return SerializableUdf.visit( + RELATIVE_TIMESTAMP.getName().getFunctionName(), + List.of(args.get(0), CurrentTimestamp$.MODULE$.apply(), CurrentTimeZone$.MODULE$.apply())); + } } From 99459533a7f7ea1e063d170af5d4add678370b35 Mon Sep 17 00:00:00 2001 From: currantw Date: Tue, 14 Jan 2025 16:39:40 -0800 Subject: [PATCH 3/6] Update/add documentation Signed-off-by: currantw --- docs/ppl-lang/PPL-Example-Commands.md | 16 +++- docs/ppl-lang/functions/ppl-datetime.md | 87 ++++++++++++++++++- .../expression/function/SerializableUdf.java | 6 -- 3 files changed, 98 insertions(+), 11 deletions(-) diff --git a/docs/ppl-lang/PPL-Example-Commands.md b/docs/ppl-lang/PPL-Example-Commands.md index 6e3fc9786..d9d9d8a7b 100644 --- a/docs/ppl-lang/PPL-Example-Commands.md +++ b/docs/ppl-lang/PPL-Example-Commands.md @@ -499,9 +499,23 @@ _- **Limitation: another command usage of (relation) subquery is in `appendcols` - `source = table | eval cdate = CAST('2012-08-07' as date), ctime = cast('2012-08-07T08:07:06' as timestamp) | fields cdate, ctime` - `source = table | eval chained_cast = cast(cast("true" as boolean) as integer) | fields chained_cast` +### **Relative Time Functions** + #### **relative_timestamp** -[See additional function details](functions/ppl-datetime#RELATIVE_TIMESTAMP) +[See additional function details](functions/ppl-datetime#relative_timestamp) - `source = table | eval one_hour_ago = relative_timestamp("-1h") | where timestamp < one_hour_ago` - `source = table | eval start_of_today = relative_timestamp("@d") | where timestamp > start_of_today` - `source = table | eval last_saturday = relative_timestamp("-1d@w6") | where timestamp >= last_saturday` + +#### **earliest** +[See additional function details](functions/ppl-datetime#earliest) +- `source = table | where earliest("-1wk", timestamp)` +- `source = table | where earliest("@qtr", timestamp)` +- `source = table | where earliest("-2y@q", timestamp)` + +#### **latest** +[See additional function details](functions/ppl-datetime#latest) +- `source = table | where latest("-60m", timestamp)` +- `source = table | where latest("@year", timestamp)` +- `source = table | where latest("-day@w1", timestamp)` --- diff --git a/docs/ppl-lang/functions/ppl-datetime.md b/docs/ppl-lang/functions/ppl-datetime.md index de345b581..4264e00b9 100644 --- a/docs/ppl-lang/functions/ppl-datetime.md +++ b/docs/ppl-lang/functions/ppl-datetime.md @@ -397,6 +397,44 @@ Example: +-------------------------------+ +### `EARLIEST` + +**Description:** + +**Usage:** earliest(string, timestamp) returns whether the timestamp defined by the given relative string is earlier +than or at the same time as the specified timestamp. + +Argument type: STRING, TIMESTAMP + +Return type: BOOLEAN + +Example: + + os> source=people | eval earliest = earliest("-1s", now()) | fields earliest | head 1 + fetched rows / total rows = 1/1 + +----------+ + | earliest | + |----------| + | True | + +----------+ + + os> source=people | eval earliest = earliest("now", now()) | fields earliest | head 1 + fetched rows / total rows = 1/1 + +----------+ + | earliest | + |----------| + | True | + +----------+ + + os> source=people | eval earliest = earliest("+1s", now()) | fields earliest | head 1 + fetched rows / total rows = 1/1 + +----------+ + | earliest | + |----------| + | False | + +----------+ + + ### `FROM_UNIXTIME` **Description:** @@ -507,6 +545,44 @@ Example: +--------------------------+ +### `LATEST` + +**Description:** + +**Usage:** latest(string, timestamp) returns whether the timestamp defined by the given relative string is later +than or at the same time as the specified timestamp. See [relative_timestamp](#relative_timestamp) for more details. + +Argument type: STRING, TIMESTAMP + +Return type: BOOLEAN + +Example: + + os> source=people | eval latest = latest("-1s", now()) | fields latest | head 1 + fetched rows / total rows = 1/1 + +--------+ + | latest | + |--------| + | False | + +--------+ + + os> source=people | eval latest = latest("now", now()) | fields latest | head 1 + fetched rows / total rows = 1/1 + +--------+ + | latest | + |--------| + | True | + +--------+ + + os> source=people | eval latest = latest("+1s", now()) | fields latest | head 1 + fetched rows / total rows = 1/1 + +--------+ + | latest | + |--------| + | True | + +--------+ + + ### `LOCALTIMESTAMP` **Description:** @@ -738,7 +814,7 @@ Example: **Description:** -**Usage:** relative_timestamp(str) returns a relative timestamp corresponding to the given relative string and the +**Usage:** relative_timestamp(string) returns a relative timestamp corresponding to the given relative string and the current timestamp at the time of query execution. The relative timestamp string has syntax `[+|-]@`, and is @@ -750,9 +826,12 @@ made up of two optional components. specified), and rounds the time down to the start of the specified time unit. For example, `@wk` is the start of the current week (Sunday is considered to be the first day of the week). -The special relative timestamp string `now`, corresponding to the current timestamp, is also supported. The current -timestamp is determined once at the start of query execution, and is used for all relative timestamp calculations for -that query. +The special relative timestamp string `now`, corresponding to the current timestamp, is also supported. + +The current timestamp is determined once at the start of query execution, and is used for all relative timestamp +calculations for that query. The Spark session time zone (`spark.sql.session.timeZone`) is used for determining +relative timestamps, and accounts for changes in the time zone offset (e.g. daylight savings time); as a result, adding +one day (`+1d`) is not the same as adding twenty-four hours (`+24h`). The relative timestamp string is case-insensitive. diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/SerializableUdf.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/SerializableUdf.java index c559fb7ad..14cc1d0bf 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/SerializableUdf.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/SerializableUdf.java @@ -227,13 +227,7 @@ public Instant apply(String relativeString, Object currentTimestamp, String zone ? ((Timestamp) currentTimestamp).toInstant() : (Instant) currentTimestamp; - /// The Spark session time zone (`spark.sql.session.timeZone`) - /// is used, which may be different from the system time zone. ZoneId zoneId = ZoneId.of(zoneIdString); - - /// Relative time calculations are performed using [ZonedDateTime] because offsets (e.g. one hour ago) - /// need to account for changes in the time zone offset (e.g. daylight savings time), while snaps (e.g. - /// start of previous Wednesday) need to account for the local date time. ZonedDateTime currentDateTime = ZonedDateTime.ofInstant(currentInstant, zoneId); ZonedDateTime relativeDateTime = TimeUtils.getRelativeZonedDateTime(relativeString, currentDateTime); From e718b51d5773d68cac4b8e829895712991f5898a Mon Sep 17 00:00:00 2001 From: currantw Date: Thu, 16 Jan 2025 09:29:47 -0800 Subject: [PATCH 4/6] Add examples to `ppl-where-command.md` Signed-off-by: currantw --- docs/ppl-lang/ppl-where-command.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/ppl-lang/ppl-where-command.md b/docs/ppl-lang/ppl-where-command.md index d6c68cdaf..528100201 100644 --- a/docs/ppl-lang/ppl-where-command.md +++ b/docs/ppl-lang/ppl-where-command.md @@ -62,3 +62,4 @@ PPL query: | where case(factor = 2, 'even', factor = 4, 'even', factor = 6, 'even', factor = 8, 'even' else 'odd') = 'even' | stats count() by factor` - `source = table | where timestamp >= relative_timestamp("-1d@w6")` +- `source = table | where earliest("-1d@w0", timestamp) and latest("now")` From 282f3f6f2099370644034fc1cbc3eae574e151f8 Mon Sep 17 00:00:00 2001 From: currantw Date: Thu, 16 Jan 2025 10:15:22 -0800 Subject: [PATCH 5/6] Minor cleanup Signed-off-by: currantw --- docs/ppl-lang/functions/ppl-datetime.md | 6 ++-- docs/ppl-lang/ppl-where-command.md | 2 +- ...arkPPLBuiltInDateTimeFunctionITSuite.scala | 28 +------------------ 3 files changed, 6 insertions(+), 30 deletions(-) diff --git a/docs/ppl-lang/functions/ppl-datetime.md b/docs/ppl-lang/functions/ppl-datetime.md index 4264e00b9..a56946951 100644 --- a/docs/ppl-lang/functions/ppl-datetime.md +++ b/docs/ppl-lang/functions/ppl-datetime.md @@ -402,7 +402,8 @@ Example: **Description:** **Usage:** earliest(string, timestamp) returns whether the timestamp defined by the given relative string is earlier -than or at the same time as the specified timestamp. +than or at the same time as the given timestamp. See [RELATIVE_TIMESTAMP](#relative_timestamp) +for more details on relative timestamp strings. Argument type: STRING, TIMESTAMP @@ -550,7 +551,8 @@ Example: **Description:** **Usage:** latest(string, timestamp) returns whether the timestamp defined by the given relative string is later -than or at the same time as the specified timestamp. See [relative_timestamp](#relative_timestamp) for more details. +than or at the same time as the given timestamp. See [RELATIVE_TIMESTAMP](#relative_timestamp) +for more details on relative timestamp strings. Argument type: STRING, TIMESTAMP diff --git a/docs/ppl-lang/ppl-where-command.md b/docs/ppl-lang/ppl-where-command.md index 528100201..b24907f3d 100644 --- a/docs/ppl-lang/ppl-where-command.md +++ b/docs/ppl-lang/ppl-where-command.md @@ -62,4 +62,4 @@ PPL query: | where case(factor = 2, 'even', factor = 4, 'even', factor = 6, 'even', factor = 8, 'even' else 'odd') = 'even' | stats count() by factor` - `source = table | where timestamp >= relative_timestamp("-1d@w6")` -- `source = table | where earliest("-1d@w0", timestamp) and latest("now")` +- `source = table | where earliest("-1d@w0", timestamp) and latest("now", timestamp)` diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala index fb2f64207..bb8544ce6 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLBuiltInDateTimeFunctionITSuite.scala @@ -386,7 +386,7 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite assertSameRows(Seq(Row(23)), frame) frame = sql(s""" - | source =$testTable + | source=$testTable | | eval day = day_of_week(relative_timestamp("@w0")) | | fields day | | head 1 @@ -430,32 +430,6 @@ class FlintSparkPPLBuiltInDateTimeFunctionITSuite assertSameRows(Seq(Row(false, true, true)), frame) } - // TODO #957: Support earliest - ignore("test EARLIEST") { - var frame = sql(s""" - | source = $testTable - | | eval earliest_hour_before = earliest(now(), "-1h") - | | eval earliest_now = earliest(now(), "now") - | | eval earliest_hour_after = earliest(now(), "+1h") - | | fields earliest_hour_before, earliest_now, earliest_hour_after - | | head 1 - | """.stripMargin) - assertSameRows(Seq(Row(true), Row(true), Row(false)), frame) - } - - // TODO #957: Support latest - ignore("test LATEST") { - var frame = sql(s""" - | source = $testTable - | | eval latest_hour_before = latest(now(), "-1h") - | | eval latest_now = latest(now(), "now") - | | eval latest_hour_after = latest(now(), "+1h") - | | fields latest_hour_before, latest_now, latest_hour_after - | | head 1 - | """.stripMargin) - assertSameRows(Seq(Row(false), Row(true), Row(true)), frame) - } - test("test CURRENT_TIME is not supported") { val ex = intercept[UnsupportedOperationException](sql(s""" | source = $testTable From 4fef581e4e13a8d3cddeb35de486b905bc0ec8a8 Mon Sep 17 00:00:00 2001 From: currantw Date: Thu, 16 Jan 2025 15:43:20 -0800 Subject: [PATCH 6/6] Address review comments Signed-off-by: currantw --- docs/ppl-lang/functions/ppl-datetime.md | 9 ++++--- .../ppl/utils/BuiltinFunctionTransformer.java | 24 ++++++++++--------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/docs/ppl-lang/functions/ppl-datetime.md b/docs/ppl-lang/functions/ppl-datetime.md index a56946951..ba73c4d0d 100644 --- a/docs/ppl-lang/functions/ppl-datetime.md +++ b/docs/ppl-lang/functions/ppl-datetime.md @@ -831,9 +831,12 @@ made up of two optional components. The special relative timestamp string `now`, corresponding to the current timestamp, is also supported. The current timestamp is determined once at the start of query execution, and is used for all relative timestamp -calculations for that query. The Spark session time zone (`spark.sql.session.timeZone`) is used for determining -relative timestamps, and accounts for changes in the time zone offset (e.g. daylight savings time); as a result, adding -one day (`+1d`) is not the same as adding twenty-four hours (`+24h`). +calculations for that query. The Spark session time zone (`spark.sql.session.timeZone`) is used for determining relative +timestamps. Offsets using time units (seconds, minutes, or hours) represent a fixed time period; adding twenty-four +hours (`+24h`) will yield a timestamp that is exactly twenty-four hours later, but which may not have the same local +time (because of daylight savings, for example). Conversely, offsets using date units (days, weeks, months, quarters, or +years) do not represent a fixed time period; adding one day (`+1d`) will yield a timestamp with the same local time, +but which may not be exactly twenty-four hours later. The relative timestamp string is case-insensitive. diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTransformer.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTransformer.java index 1211b857b..45a92bf22 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTransformer.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTransformer.java @@ -182,19 +182,21 @@ public interface BuiltinFunctionTransformer { // Relative time functions .put( RELATIVE_TIMESTAMP, - BuiltinFunctionTransformer::buildRelativeTimestampExpression) + args -> buildRelativeTimestamp(args.get(0))) .put( EARLIEST, - args -> - LessThanOrEqual$.MODULE$.apply( - buildRelativeTimestampExpression(List.of(args.get(0))), - args.get(1))) + args -> { + Expression relativeTimestamp = buildRelativeTimestamp(args.get(0)); + Expression timestamp = args.get(1); + return LessThanOrEqual$.MODULE$.apply(relativeTimestamp, timestamp); + }) .put( LATEST, - args -> - GreaterThanOrEqual$.MODULE$.apply( - buildRelativeTimestampExpression(List.of(args.get(0))), - args.get(1))) + args -> { + Expression relativeTimestamp = buildRelativeTimestamp(args.get(0)); + Expression timestamp = args.get(1); + return GreaterThanOrEqual$.MODULE$.apply(relativeTimestamp, timestamp); + }) .build(); static Expression builtinFunction(org.opensearch.sql.ast.expression.Function function, List args) { @@ -237,9 +239,9 @@ static Expression[] createIntervalArgs(IntervalUnit unit, Expression value) { return args; } - private static Expression buildRelativeTimestampExpression(List args) { + private static Expression buildRelativeTimestamp(Expression relativeStringExpression) { return SerializableUdf.visit( RELATIVE_TIMESTAMP.getName().getFunctionName(), - List.of(args.get(0), CurrentTimestamp$.MODULE$.apply(), CurrentTimeZone$.MODULE$.apply())); + List.of(relativeStringExpression, CurrentTimestamp$.MODULE$.apply(), CurrentTimeZone$.MODULE$.apply())); } }