diff --git a/build.sbt b/build.sbt index 28ad6dd5b..365b88aa3 100644 --- a/build.sbt +++ b/build.sbt @@ -194,7 +194,6 @@ lazy val pplSparkIntegration = (project in file("ppl-spark-integration")) "com.github.sbt" % "junit-interface" % "0.13.3" % "test", "org.projectlombok" % "lombok" % "1.18.30", "com.github.seancfoley" % "ipaddress" % "5.5.1", - "org.mockito" % "mockito-inline" % "4.6.0" % "test", ), libraryDependencies ++= deps(sparkVersion), // ANTLR settings diff --git a/docs/ppl-lang/functions/ppl-datetime.md b/docs/ppl-lang/functions/ppl-datetime.md index 0b62e9550..49b9866ed 100644 --- a/docs/ppl-lang/functions/ppl-datetime.md +++ b/docs/ppl-lang/functions/ppl-datetime.md @@ -738,17 +738,17 @@ Example: **Description:** -**Usage:** relative_timestamp(str) returns a timestamp corresponding to the give relative string and the current +**Usage:** relative_timestamp(str) returns a relative timestamp from given relative string and the current timestamp at the time of query execution. -The relative time string has syntax `[+|-]@`, and is made up of -two optional components: -* An offset from the current timestamp at the start of query execution, which is composed of a sign (`+` or `-`), an - optional time integer, and a time unit. If the time integer is not specified, it defaults to one. For example, `+2hr` - corresponds to two hours after the current timestamp, while `-mon` corresponds to one month ago. +The relative timestamp string has syntax `[+|-]@`, and is made up +of two optional components: +* An offset from the current timestamp, which is composed of a sign (`+` or `-`), an optional time integer, and a time + unit. If the time integer is not specified, it defaults to one. For example, `+2hr` is two hours after the current + timestamp, while `-mon` is one month ago. * A snap-to time using the `@` symbol followed by a time unit. The snap-to time is applied after the offset (if specified), and rounds the time down to the start of the specified time unit (i.e. backwards in time). For - example, `@wk` corresponds to the start of the current week (Sunday is considered to be the first day of the week). + example, `@wk` is the start of the current week (Sunday is considered to be the first day of the week). The following offset time units are supported: @@ -774,7 +774,7 @@ The snap-to time supports all the time units above, as well as the following day | Friday | `w5` | | Saturday | `w6` | -The special relative time string `now` for the current timestamp is also supported. +The special relative timestamp string `now`, corresponding to the current timestamp, is also supported. For example, if the current timestamp is Monday, January 03, 2000 at 01:01:01 am: diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/SerializableUdf.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/SerializableUdf.java index 1119c1601..d25ab7b9e 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/SerializableUdf.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/SerializableUdf.java @@ -204,14 +204,14 @@ public BigInteger apply(String ipAddress) { } /** - * Returns the {@link Timestamp} corresponding to the given relative string, current timestamp, and time zone identifier. - * Throws {@link RuntimeException} if the relative timestamp string is not supported. + * Returns the {@link Timestamp} corresponding to the given relative string and current timestamp. + * Throws {@link RuntimeException} if the relative string is not supported. */ Function2 relativeTimestampFunction = new SerializableAbstractFunction2() { @Override - public Timestamp apply(String relativeDateTimeString, Timestamp currentTimestamp) { + public Timestamp apply(String relativeString, Timestamp currentTimestamp) { LocalDateTime currentLocalDateTime = currentTimestamp.toLocalDateTime(); - LocalDateTime relativeLocalDateTime = TimeUtils.getRelativeLocalDateTime(relativeDateTimeString, currentLocalDateTime); + LocalDateTime relativeLocalDateTime = TimeUtils.getRelativeLocalDateTime(relativeString, currentLocalDateTime); return Timestamp.valueOf(relativeLocalDateTime); } }; diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/TimeUtils.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/TimeUtils.java index 1f415d9b6..ddf659560 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/TimeUtils.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/expression/function/TimeUtils.java @@ -5,13 +5,12 @@ package org.opensearch.sql.expression.function; +import com.google.common.collect.ImmutableMap; import lombok.experimental.UtilityClass; -import java.time.DayOfWeek; -import java.time.Duration; -import java.time.LocalDateTime; -import java.time.Period; +import java.time.*; import java.time.temporal.ChronoUnit; +import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -43,62 +42,26 @@ public class TimeUtils { private static final Set YEAR_UNITS_SET = Set.of("y", "yr", "yrs", "year", "years"); // Map from time unit to the corresponding duration. - private static final Duration DURATION_SECOND = Duration.ofSeconds(1); - private static final Duration DURATION_MINUTE = Duration.ofMinutes(1); - private static final Duration DURATION_HOUR = Duration.ofHours(1); - - private static final Map DURATION_FOR_TIME_UNIT_MAP = Map.ofEntries( - Map.entry("s", DURATION_SECOND), - Map.entry("sec", DURATION_SECOND), - Map.entry("secs", DURATION_SECOND), - Map.entry("second", DURATION_SECOND), - Map.entry("seconds", DURATION_SECOND), - - Map.entry("m", DURATION_MINUTE), - Map.entry("min", DURATION_MINUTE), - Map.entry("mins", DURATION_MINUTE), - Map.entry("minute", DURATION_MINUTE), - Map.entry("minutes", DURATION_MINUTE), - - Map.entry("h", DURATION_HOUR), - Map.entry("hr", DURATION_HOUR), - Map.entry("hrs", DURATION_HOUR), - Map.entry("hour", DURATION_HOUR), - Map.entry("hours", DURATION_HOUR)); + private static final Map DURATION_FOR_TIME_UNIT_MAP; + static { + Map durationMap = new HashMap<>(); + SECOND_UNITS_SET.forEach(u -> durationMap.put(u, Duration.ofSeconds(1))); + MINUTE_UNITS_SET.forEach(u -> durationMap.put(u, Duration.ofMinutes(1))); + HOUR_UNITS_SET.forEach(u -> durationMap.put(u, Duration.ofHours(1))); + DURATION_FOR_TIME_UNIT_MAP = ImmutableMap.copyOf(durationMap); + } // Map from time unit to the corresponding period. - private static final Period PERIOD_DAY = Period.ofDays(1); - private static final Period PERIOD_WEEK = Period.ofWeeks(1); - private static final Period PERIOD_MONTH = Period.ofMonths(1); - private static final Period PERIOD_QUARTER = Period.ofMonths(3); - private static final Period PERIOD_YEAR = Period.ofYears(1); - - private static final Map PERIOD_FOR_TIME_UNIT_MAP = Map.ofEntries( - Map.entry("d", PERIOD_DAY), - Map.entry("day", PERIOD_DAY), - Map.entry("days", PERIOD_DAY), - - Map.entry("w", PERIOD_WEEK), - Map.entry("wk", PERIOD_WEEK), - Map.entry("wks", PERIOD_WEEK), - Map.entry("week", PERIOD_WEEK), - Map.entry("weeks", PERIOD_WEEK), - - Map.entry("mon", PERIOD_MONTH), - Map.entry("month", PERIOD_MONTH), - Map.entry("months", PERIOD_MONTH), - - Map.entry("q", PERIOD_QUARTER), - Map.entry("qtr", PERIOD_QUARTER), - Map.entry("qtrs", PERIOD_QUARTER), - Map.entry("quarter", PERIOD_QUARTER), - Map.entry("quarters", PERIOD_QUARTER), - - Map.entry("y", PERIOD_YEAR), - Map.entry("yr", PERIOD_YEAR), - Map.entry("yrs", PERIOD_YEAR), - Map.entry("year", PERIOD_YEAR), - Map.entry("years", PERIOD_YEAR)); + private static final Map PERIOD_FOR_TIME_UNIT_MAP; + static { + Map periodMap = new HashMap<>(); + DAY_UNITS_SET.forEach(u -> periodMap.put(u, Period.ofDays(1))); + WEEK_UNITS_SET.forEach(u -> periodMap.put(u, Period.ofWeeks(1))); + MONTH_UNITS_SET.forEach(u -> periodMap.put(u, Period.ofMonths(1))); + QUARTER_UNITS_SET.forEach(u -> periodMap.put(u, Period.ofMonths(3))); + YEAR_UNITS_SET.forEach(u -> periodMap.put(u, Period.ofYears(1))); + PERIOD_FOR_TIME_UNIT_MAP = ImmutableMap.copyOf(periodMap); + } // Map from snap unit to the corresponding day of the week. private static final Map DAY_OF_THE_WEEK_FOR_SNAP_UNIT_MAP = Map.ofEntries( @@ -111,11 +74,8 @@ public class TimeUtils { Map.entry("w5", DayOfWeek.FRIDAY), Map.entry("w6", DayOfWeek.SATURDAY)); - static final int DAYS_PER_WEEK = 7; - static final int MONTHS_PER_QUARTER = 3; - /** - * Returns the {@link LocalDateTime} corresponding to the given relative string and local date time. + * Returns the relative {@link LocalDateTime} corresponding to the given relative string and local date time. *

* The relative time string has syntax {@code [+|-]@}, and * is made up of two optional components: @@ -339,8 +299,8 @@ private LocalDateTime applySnap(LocalDateTime localDateTime, String snapUnit) { } else if (MONTH_UNITS_SET.contains(snapUnitLowerCase)) { return localDateTime.truncatedTo(ChronoUnit.DAYS).withDayOfMonth(1); } else if (QUARTER_UNITS_SET.contains(snapUnitLowerCase)) { - int monthsToSnap = (localDateTime.getMonthValue() - 1) % MONTHS_PER_QUARTER; - return localDateTime.truncatedTo(ChronoUnit.DAYS).withDayOfMonth(1).minusMonths(monthsToSnap); + Month snapMonth = localDateTime.getMonth().firstMonthOfQuarter(); + return localDateTime.truncatedTo(ChronoUnit.DAYS).withDayOfMonth(1).withMonth(snapMonth.getValue()); } else if (YEAR_UNITS_SET.contains(snapUnitLowerCase)) { return localDateTime.truncatedTo(ChronoUnit.DAYS).withDayOfYear(1); } else if (DAY_OF_THE_WEEK_FOR_SNAP_UNIT_MAP.containsKey(snapUnitLowerCase)) { @@ -363,7 +323,7 @@ private LocalDateTime applySnapToDayOfWeek(LocalDateTime dateTime, DayOfWeek sna return snappedDateTime; } - int daysToSnap = DAYS_PER_WEEK - snapDayOfWeek.getValue() + dayOfWeek.getValue(); + int daysToSnap = DayOfWeek.values().length - snapDayOfWeek.getValue() + dayOfWeek.getValue(); return snappedDateTime.minusDays(daysToSnap); } } diff --git a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTransformer.java b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTransformer.java index 476f756bc..56d8b5d83 100644 --- a/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTransformer.java +++ b/ppl-spark-integration/src/main/java/org/opensearch/sql/ppl/utils/BuiltinFunctionTransformer.java @@ -8,7 +8,16 @@ import com.google.common.collect.ImmutableMap; import org.apache.spark.sql.catalyst.analysis.UnresolvedFunction; import org.apache.spark.sql.catalyst.analysis.UnresolvedFunction$; -import org.apache.spark.sql.catalyst.expressions.*; +import org.apache.spark.sql.catalyst.expressions.CurrentTimeZone$; +import org.apache.spark.sql.catalyst.expressions.CurrentTimestamp$; +import org.apache.spark.sql.catalyst.expressions.DateAddInterval$; +import org.apache.spark.sql.catalyst.expressions.Expression; +import org.apache.spark.sql.catalyst.expressions.Literal$; +import org.apache.spark.sql.catalyst.expressions.ScalaUDF; +import org.apache.spark.sql.catalyst.expressions.TimestampAdd$; +import org.apache.spark.sql.catalyst.expressions.TimestampDiff$; +import org.apache.spark.sql.catalyst.expressions.ToUTCTimestamp$; +import org.apache.spark.sql.catalyst.expressions.UnaryMinus$; import org.opensearch.sql.ast.expression.IntervalUnit; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.SerializableUdf;