diff --git a/docs/user/dql/aggregations.rst b/docs/user/dql/aggregations.rst index 4b8ca57a32d..e828441b708 100644 --- a/docs/user/dql/aggregations.rst +++ b/docs/user/dql/aggregations.rst @@ -74,6 +74,40 @@ The group by expression could be expression:: +---------------------+----------+ +Bucket Function +--------------- + +The group by expression could be a bucket function, which splits a field into +fixed-width buckets. ``date_histogram`` takes a time interval, ``histogram`` a +numeric width, and the field is given first. The interval parameter is one of +``interval``, ``fixed_interval`` or ``calendar_interval``. A bucket has to be +projected in a subquery before it can be grouped on:: + + os> SELECT b, count(*) FROM (SELECT date_histogram(field=timestamp, interval='1w') AS b FROM nyc_taxi) sub GROUP BY b ORDER BY b; + fetched rows / total rows = 4/4 + +---------------------+----------+ + | b | count(*) | + |---------------------+----------| + | 2014-06-30 00:00:00 | 288 | + | 2014-07-07 00:00:00 | 336 | + | 2014-07-14 00:00:00 | 336 | + | 2014-07-21 00:00:00 | 13 | + +---------------------+----------+ + +The time units are millisecond (``ms``), second (``s``), minute (``m``), hour +(``h``), day (``d``), week (``w``), month (``M``), quarter (``q``) and year +(``y``). A numeric field is bucketed the same way, with the width as a number:: + + os> SELECT b, count(*) FROM (SELECT histogram(field=age, interval=10) AS b FROM accounts) sub GROUP BY b ORDER BY b; + fetched rows / total rows = 2/2 + +----+----------+ + | b | count(*) | + |----+----------| + | 20 | 1 | + | 30 | 3 | + +----+----------+ + + Aggregation =========== diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index c8a61e417d3..54820a298b9 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -15,6 +15,8 @@ There is support for a wide variety of functions shared by SQL/PPL. We are inten Most of the specifications can be self explained just as a regular function with data type as argument. The only notation that needs elaboration is generic type ``T`` which binds to an actual type and can be used as return type. For example, ``ABS(NUMBER T) -> T`` means function ``ABS`` accepts an numerical argument of type ``T`` which could be any sub-type of ``NUMBER`` type and returns the actual type of ``T`` as return type. The actual type binds to generic type at runtime dynamically. +The bucket functions ``date_histogram`` and ``histogram`` are not listed here because they are only valid as a grouping key, please see also: `Aggregations `_ + Type Conversion =============== diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index fc15c908c63..2bd1f98a5ee 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -994,7 +994,12 @@ public enum Index { "timewrap_test", "timewrap_test", "{\"mappings\":{\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"host\":{\"type\":\"keyword\"},\"requests\":{\"type\":\"integer\"},\"errors\":{\"type\":\"integer\"}}}}", - "src/test/resources/timewrap_test.json"); + "src/test/resources/timewrap_test.json"), + DATE_HISTOGRAM_TEST( + "date_histogram_test", + "date_histogram_test", + getMappingFile("date_histogram_test_index_mapping.json"), + "src/test/resources/date_histogram_test.json"); private final String name; private final String type; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateHistogramBucketFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateHistogramBucketFunctionIT.java new file mode 100644 index 00000000000..939e79f71f0 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateHistogramBucketFunctionIT.java @@ -0,0 +1,199 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.sql; + +import static org.junit.Assert.assertThrows; +import static org.opensearch.sql.util.Capability.LEGACY_ENGINE_FALLBACK; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRowsInOrder; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; + +/** + * Execution coverage for {@code date_histogram} and {@code histogram}. The expander unit tests + * assert the AST that gets built; these assert what comes back after analysis, planning and + * pushdown, against 72 documents on fixed timestamps: + * + * 
+ *   00:00 x5 alpha   00:30 x7 beta    01:00 x11 alpha
+ *   01:45 x13 gamma  02:00 x17 beta   03:00 x19 alpha
+ *
+ * + * so hourly grouping must yield 12/24/17/19 and half-hourly 5/7/11/13/17/19. + */ +public class DateHistogramBucketFunctionIT extends SQLIntegTestCase { + + private static final String IDX = "date_histogram_test"; + + @Override + protected void init() throws Exception { + super.init(); + loadIndex(Index.DATE_HISTOGRAM_TEST); + } + + /** + * The bucket has to be projected in a derived table before it can be grouped on; see {@link + * #groupingOnTheBucketWithoutADerivedTableIsRejected}. This is also the shape Dashboards emits. + */ + private static String bucketed(String bucketExpr) { + return "SELECT b, COUNT(*) FROM (SELECT " + + bucketExpr + + " AS b FROM " + + IDX + + ") sub GROUP BY b ORDER BY b"; + } + + @Test + public void hourlyBucketsCarryKeysAndCounts() throws IOException { + JSONObject response = executeQuery(bucketed("date_histogram(field=ts, interval='1h')")); + + verifyDataRowsInOrder( + response, + rows("2026-01-01 00:00:00", 12), + rows("2026-01-01 01:00:00", 24), + rows("2026-01-01 02:00:00", 17), + rows("2026-01-01 03:00:00", 19)); + } + + /** A sub-hour interval must split 00:00/00:30 and 01:00/01:45 rather than merge them. */ + @Test + public void halfHourlyBucketsSplitWithinTheHour() throws IOException { + JSONObject response = executeQuery(bucketed("date_histogram(field=ts, interval='30m')")); + + verifyDataRowsInOrder( + response, + rows("2026-01-01 00:00:00", 5), + rows("2026-01-01 00:30:00", 7), + rows("2026-01-01 01:00:00", 11), + rows("2026-01-01 01:30:00", 13), + rows("2026-01-01 02:00:00", 17), + rows("2026-01-01 03:00:00", 19)); + } + + @Test + public void dailyIntervalCollapsesEverythingIntoOneBucket() throws IOException { + JSONObject response = executeQuery(bucketed("date_histogram(field=ts, interval='1d')")); + + verifyDataRows(response, rows("2026-01-01 00:00:00", 72)); + } + + /** {@code fixed_interval} and {@code calendar_interval} are accepted as synonyms of interval. */ + @Test + public void intervalSynonymsProduceTheSameBuckets() throws IOException { + JSONObject viaFixed = executeQuery(bucketed("date_histogram(field=ts, fixed_interval='1h')")); + JSONObject viaCalendar = + executeQuery(bucketed("date_histogram(field=ts, calendar_interval='1h')")); + + for (JSONObject response : new JSONObject[] {viaFixed, viaCalendar}) { + verifyDataRowsInOrder( + response, + rows("2026-01-01 00:00:00", 12), + rows("2026-01-01 01:00:00", 24), + rows("2026-01-01 02:00:00", 17), + rows("2026-01-01 03:00:00", 19)); + } + } + + /** A second grouping key needs the scan in a derived table of its own as well. */ + @Test + public void bucketsCombineWithAnAdditionalGroupingKey() throws IOException { + JSONObject response = + executeQuery( + "SELECT b, c, COUNT(*) FROM (SELECT date_histogram(field=ts, interval='1h') AS b," + + " category AS c FROM (SELECT * FROM " + + IDX + + ") inner_scan) sub GROUP BY b, c ORDER BY b, c"); + + verifyDataRowsInOrder( + response, + rows("2026-01-01 00:00:00", "alpha", 5), + rows("2026-01-01 00:00:00", "beta", 7), + rows("2026-01-01 01:00:00", "alpha", 11), + rows("2026-01-01 01:00:00", "gamma", 13), + rows("2026-01-01 02:00:00", "beta", 17), + rows("2026-01-01 03:00:00", "alpha", 19)); + } + + @Test + public void bucketsRespectAWhereClause() throws IOException { + JSONObject response = + executeQuery( + "SELECT b, COUNT(*) FROM (SELECT date_histogram(field=ts, interval='1h') AS b FROM " + + IDX + + " WHERE category = 'alpha') sub GROUP BY b ORDER BY b"); + + verifyDataRowsInOrder( + response, + rows("2026-01-01 00:00:00", 5), + rows("2026-01-01 01:00:00", 11), + rows("2026-01-01 03:00:00", 19)); + } + + @Test + public void numericHistogramBucketsByInterval() throws IOException { + JSONObject response = executeQuery(bucketed("histogram(field=value, interval=20)")); + + // value runs 1..72, so the 20-wide buckets hold 19, 20, 20 and 13 documents. + verifyDataRowsInOrder(response, rows(0, 19), rows(20, 20), rows(40, 20), rows(60, 13)); + } + + /** + * The legacy engine implements alias, format, time_zone, min_doc_count and order through the + * native date_histogram aggregation; this lowering has no equivalent, so those queries still have + * to reach it. CsvFormatResponseIT.dateHistogramTest has asserted this shape for years. + */ + @Test + @RequiresCapability(LEGACY_ENGINE_FALLBACK) + public void callWithAliasParameterReturnsHourlyBuckets() throws IOException { + JSONObject response = + executeQuery( + "SELECT COUNT(*) FROM " + + IDX + + " GROUP BY date_histogram(field='ts',fixed_interval='1h','alias'='hours')"); + + verifyDataRows(response, rows(12), rows(24), rows(17), rows(19)); + } + + @Test + public void unquotedArgumentNamesReturnNumericBuckets() throws IOException { + JSONObject response = executeQuery(bucketed("histogram(field=value, interval=20)")); + + verifyDataRowsInOrder(response, rows(0, 19), rows(20, 20), rows(40, 20), rows(60, 13)); + } + + /** + * A span over a bare table scan cannot resolve its field, with or without a select alias, so the + * bucket always has to be projected in a derived table first. Both routes reject this; only the + * message differs, so the assertion is on the rejection alone. + */ + @Test + public void groupingOnTheBucketWithoutADerivedTableIsRejected() { + assertThrows( + ResponseException.class, + () -> + executeQuery( + "SELECT date_histogram(field=ts, interval='1h') AS b, COUNT(*) FROM " + + IDX + + " GROUP BY date_histogram(field=ts, interval='1h')")); + } + + /** Calendar units: Dashboards emits 1M and 1y at the wider zoom levels. */ + @Test + public void calendarIntervalsBucketByMonthAndYear() throws IOException { + verifyDataRows( + executeQuery(bucketed("date_histogram(field=ts, interval='1M')")), + rows("2026-01-01 00:00:00", 72)); + verifyDataRows( + executeQuery(bucketed("date_histogram(field=ts, interval='1y')")), + rows("2026-01-01 00:00:00", 72)); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java index 1d4067f1414..bb99dc85115 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java @@ -537,6 +537,17 @@ public enum Capability { PREPARED_STATEMENT( "Prepared statements are unsupported on the analytics-engine route (Calcite path)."), + /** + * FRONTEND: the legacy V1 engine answers call shapes the V2 grammar declines, but only on the + * default route. Requests reach it when RestSQLQueryAction catches a SyntaxCheckException; the + * analytics-engine route enters through RestUnifiedQueryAction, which has no such fallback. + */ + LEGACY_ENGINE_FALLBACK( + "A call shape only the legacy V1 engine understands (e.g. a date_histogram `alias`" + + " parameter) can't be answered on the analytics-engine route: reaching that engine" + + " depends on RestSQLQueryAction's SyntaxCheckException fallback, and the analytics" + + " route does not go through it."), + /** * FRONTEND: legacy method-query syntax (regexp_query/wildcard_query) is not in the Calcite * grammar. diff --git a/integ-test/src/test/resources/date_histogram_test.json b/integ-test/src/test/resources/date_histogram_test.json new file mode 100644 index 00000000000..2d43eca9da3 --- /dev/null +++ b/integ-test/src/test/resources/date_histogram_test.json @@ -0,0 +1,144 @@ +{"index":{}} +{"ts":"2026-01-01 00:00:00","category":"alpha","value":1} +{"index":{}} +{"ts":"2026-01-01 00:00:00","category":"alpha","value":2} +{"index":{}} +{"ts":"2026-01-01 00:00:00","category":"alpha","value":3} +{"index":{}} +{"ts":"2026-01-01 00:00:00","category":"alpha","value":4} +{"index":{}} +{"ts":"2026-01-01 00:00:00","category":"alpha","value":5} +{"index":{}} +{"ts":"2026-01-01 00:30:00","category":"beta","value":6} +{"index":{}} +{"ts":"2026-01-01 00:30:00","category":"beta","value":7} +{"index":{}} +{"ts":"2026-01-01 00:30:00","category":"beta","value":8} +{"index":{}} +{"ts":"2026-01-01 00:30:00","category":"beta","value":9} +{"index":{}} +{"ts":"2026-01-01 00:30:00","category":"beta","value":10} +{"index":{}} +{"ts":"2026-01-01 00:30:00","category":"beta","value":11} +{"index":{}} +{"ts":"2026-01-01 00:30:00","category":"beta","value":12} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":13} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":14} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":15} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":16} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":17} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":18} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":19} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":20} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":21} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":22} +{"index":{}} +{"ts":"2026-01-01 01:00:00","category":"alpha","value":23} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":24} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":25} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":26} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":27} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":28} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":29} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":30} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":31} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":32} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":33} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":34} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":35} +{"index":{}} +{"ts":"2026-01-01 01:45:00","category":"gamma","value":36} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":37} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":38} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":39} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":40} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":41} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":42} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":43} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":44} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":45} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":46} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":47} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":48} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":49} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":50} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":51} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":52} +{"index":{}} +{"ts":"2026-01-01 02:00:00","category":"beta","value":53} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":54} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":55} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":56} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":57} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":58} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":59} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":60} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":61} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":62} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":63} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":64} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":65} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":66} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":67} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":68} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":69} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":70} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":71} +{"index":{}} +{"ts":"2026-01-01 03:00:00","category":"alpha","value":72} diff --git a/integ-test/src/test/resources/indexDefinitions/date_histogram_test_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/date_histogram_test_index_mapping.json new file mode 100644 index 00000000000..d8538083641 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/date_histogram_test_index_mapping.json @@ -0,0 +1,16 @@ +{ + "mappings": { + "properties": { + "ts": { + "type": "date", + "format": "yyyy-MM-dd HH:mm:ss" + }, + "category": { + "type": "keyword" + }, + "value": { + "type": "integer" + } + } + } +} diff --git a/language-grammar/src/main/antlr4/OpenSearchSQLLexer.g4 b/language-grammar/src/main/antlr4/OpenSearchSQLLexer.g4 index ba7c5be85ab..9960908869f 100644 --- a/language-grammar/src/main/antlr4/OpenSearchSQLLexer.g4 +++ b/language-grammar/src/main/antlr4/OpenSearchSQLLexer.g4 @@ -143,6 +143,8 @@ OFFSET: 'OFFSET'; // INTERVAL AND UNIT KEYWORDS INTERVAL: 'INTERVAL'; +FIXED_INTERVAL: 'FIXED_INTERVAL'; +CALENDAR_INTERVAL: 'CALENDAR_INTERVAL'; MICROSECOND: 'MICROSECOND'; SECOND: 'SECOND'; MINUTE: 'MINUTE'; diff --git a/language-grammar/src/main/antlr4/OpenSearchSQLParser.g4 b/language-grammar/src/main/antlr4/OpenSearchSQLParser.g4 index 5f7361160b3..947bcb3bf64 100644 --- a/language-grammar/src/main/antlr4/OpenSearchSQLParser.g4 +++ b/language-grammar/src/main/antlr4/OpenSearchSQLParser.g4 @@ -335,6 +335,7 @@ functionCall | extractFunction # extractFunctionCall | getFormatFunction # getFormatFunctionCall | timestampFunction # timestampFunctionCall + | bucketFunction # bucketFunctionCall ; timestampFunction @@ -396,6 +397,17 @@ highlightFunction : HIGHLIGHT LR_BRACKET relevanceField (COMMA highlightArg)* RR_BRACKET ; +bucketFunction + : bucketFunctionName LR_BRACKET FIELD EQUAL_SYMBOL field = bucketArgValue COMMA + intervalArgName EQUAL_SYMBOL interval = constant RR_BRACKET + ; + +intervalArgName + : INTERVAL + | FIXED_INTERVAL + | CALENDAR_INTERVAL + ; + positionFunction : POSITION LR_BRACKET functionArg IN functionArg RR_BRACKET ; @@ -413,6 +425,11 @@ scalarFunctionName | nestedFunctionName ; +bucketFunctionName + : HISTOGRAM + | DATE_HISTOGRAM + ; + specificFunction : CASE expression caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall | CASE caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall @@ -780,6 +797,11 @@ relevanceArgValue | constant ; +bucketArgValue + : constant + | qualifiedName + ; + highlightArgValue : stringLiteral ; @@ -830,6 +852,8 @@ ident keywordsCanBeId : FULL | FIELD + | FIXED_INTERVAL + | CALENDAR_INTERVAL | D | T | TS // OD SQL and ODBC special diff --git a/sql/src/main/antlr/OpenSearchSQLLexer.g4 b/sql/src/main/antlr/OpenSearchSQLLexer.g4 index ba7c5be85ab..9960908869f 100644 --- a/sql/src/main/antlr/OpenSearchSQLLexer.g4 +++ b/sql/src/main/antlr/OpenSearchSQLLexer.g4 @@ -143,6 +143,8 @@ OFFSET: 'OFFSET'; // INTERVAL AND UNIT KEYWORDS INTERVAL: 'INTERVAL'; +FIXED_INTERVAL: 'FIXED_INTERVAL'; +CALENDAR_INTERVAL: 'CALENDAR_INTERVAL'; MICROSECOND: 'MICROSECOND'; SECOND: 'SECOND'; MINUTE: 'MINUTE'; diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 5b52b9d3387..e372382805c 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -368,6 +368,7 @@ functionCall | extractFunction # extractFunctionCall | getFormatFunction # getFormatFunctionCall | timestampFunction # timestampFunctionCall + | bucketFunction # bucketFunctionCall ; timestampFunction @@ -429,6 +430,17 @@ highlightFunction : HIGHLIGHT LR_BRACKET relevanceField (COMMA highlightArg)* RR_BRACKET ; +bucketFunction + : bucketFunctionName LR_BRACKET FIELD EQUAL_SYMBOL field = bucketArgValue COMMA + intervalArgName EQUAL_SYMBOL interval = constant RR_BRACKET + ; + +intervalArgName + : INTERVAL + | FIXED_INTERVAL + | CALENDAR_INTERVAL + ; + positionFunction : POSITION LR_BRACKET functionArg IN functionArg RR_BRACKET ; @@ -446,6 +458,11 @@ scalarFunctionName | nestedFunctionName ; +bucketFunctionName + : HISTOGRAM + | DATE_HISTOGRAM + ; + specificFunction : CASE expression caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall | CASE caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall @@ -813,6 +830,11 @@ relevanceArgValue | constant ; +bucketArgValue + : constant + | qualifiedName + ; + highlightArgValue : stringLiteral ; @@ -863,6 +885,8 @@ ident keywordsCanBeId : FULL | FIELD + | FIXED_INTERVAL + | CALENDAR_INTERVAL | D | T | TS // OD SQL and ODBC special diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index e7510f31b7a..d371961d11a 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -20,6 +20,7 @@ import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.BetweenPredicateContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.BinaryComparisonPredicateContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.BooleanContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.BucketFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.CaseFuncAlternativeContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.CaseFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ColumnFilterContext; @@ -165,6 +166,26 @@ public UnresolvedExpression visitScalarFunctionCall(ScalarFunctionCallContext ct return buildFunction(ctx.scalarFunctionName().getText(), ctx.functionArgs().functionArg()); } + /** + * Lowers {@code histogram} and {@code date_histogram} to a {@link Span}, the same node PPL's + * {@code span()} produces. Anything the grammar does not admit here is a syntax error, which + * RestSQLQueryAction hands to the legacy engine. + */ + @Override + public UnresolvedExpression visitBucketFunctionCall(BucketFunctionCallContext ctx) { + OpenSearchSQLParser.BucketFunctionContext bucket = ctx.bucketFunction(); + return AstDSL.spanFromSpanLengthLiteral( + normalizeField(visit(bucket.field)), (Literal) visit(bucket.interval)); + } + + /** A string literal naming a column is coerced so downstream sees a column reference. */ + private static UnresolvedExpression normalizeField(UnresolvedExpression field) { + if (field instanceof Literal literal && literal.getType() == DataType.STRING) { + return AstDSL.qualifiedName(literal.getValue().toString()); + } + return field; + } + @Override public UnresolvedExpression visitGetFormatFunctionCall(GetFormatFunctionCallContext ctx) { return new Function( diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java index aba8023b07e..1902283a458 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -47,12 +47,15 @@ import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.RelevanceFieldList; +import org.opensearch.sql.ast.expression.Span; +import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.expression.WindowFrame; import org.opensearch.sql.ast.expression.WindowFunction; import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; import org.opensearch.sql.common.antlr.SyntaxAnalysisErrorListener; +import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLLexer; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser; @@ -860,6 +863,103 @@ private static String nest(int depth, String base, UnaryOperator wrap) { return expr; } + @Test + public void canBuildDateHistogramAsSpan() { + assertEquals( + new Span(qualifiedName("ts"), intLiteral(1), SpanUnit.H), + buildExprAst("date_histogram(field=ts, interval='1h')")); + } + + /** Bare argument names are the spelling the legacy engine accepts; both forms lower alike. */ + @Test + public void canBuildBucketFunctionWithUnquotedArgumentNames() { + assertEquals( + new Span(qualifiedName("ts"), intLiteral(1), SpanUnit.H), + buildExprAst("date_histogram(field=ts, interval='1h')")); + assertEquals( + new Span(qualifiedName("age"), intLiteral(10), SpanUnit.NONE), + buildExprAst("histogram(field=age, interval=10)")); + } + + @Test + public void canBuildDateHistogramWithIntervalSynonyms() { + Span expected = new Span(qualifiedName("ts"), intLiteral(1), SpanUnit.D); + assertEquals(expected, buildExprAst("date_histogram(field=ts, fixed_interval='1d')")); + assertEquals(expected, buildExprAst("date_histogram(field=ts, calendar_interval='1d')")); + } + + @Test + public void canBuildDateHistogramWithStringFieldName() { + assertEquals( + new Span(qualifiedName("ts"), intLiteral(30), SpanUnit.m), + buildExprAst("date_histogram(field='ts', interval='30m')")); + } + + /** A numeric literal field is left alone rather than coerced to a column reference. */ + @Test + public void bucketFieldGivenNonStringLiteralIsPassedThrough() { + assertEquals( + new Span(intLiteral(1), intLiteral(10), SpanUnit.NONE), + buildExprAst("histogram(field=1, interval=10)")); + } + + @Test + public void canBuildNumericHistogramAsSpan() { + assertEquals( + new Span(qualifiedName("age"), intLiteral(10), SpanUnit.NONE), + buildExprAst("histogram(field=age, interval=10)")); + } + + /** + * A parameter with no lowering here has to raise SyntaxCheckException -- the one type + * RestSQLQueryAction falls back on -- because the legacy engine implements alias, min_doc_count + * and order, and has answered queries using them for years. + */ + @Test + public void unsupportedBucketParameterDefersToLegacyEngine() { + assertThrows( + SyntaxCheckException.class, + () -> buildExprAst("date_histogram(field=ts, interval='1d', 'alias'='days')")); + assertThrows( + SyntaxCheckException.class, + () -> buildExprAst("histogram(field=age, interval=10, 'min_doc_count'=1)")); + assertThrows( + SyntaxCheckException.class, + () -> buildExprAst("date_histogram(field=ts, interval='1d', 'format'='yyyy-MM-dd')")); + assertThrows( + SyntaxCheckException.class, + () -> buildExprAst("date_histogram(field=ts, interval='1h', 'time_zone'='+05:30')")); + for (String name : List.of("children", "extended_bounds", "nested", "reverse_nested")) { + assertThrows( + SyntaxCheckException.class, + () -> + buildExprAst( + String.format("date_histogram(field=ts, interval='1d', '%s'='x')", name))); + } + } + + /** A shape the grammar does not admit is a syntax error, which routes to the legacy engine. */ + @Test + public void badBucketArgumentIsASyntaxError() { + for (String call : + List.of( + "date_histogram(interval='1d')", + "date_histogram(field=ts)", + "date_histogram(field=ts, interval='1d', fixed_interval='2d')", + "date_histogram(field=ts, interval=ts)", + "date_histogram(field=ts, interval='1d', interval='2d')")) { + assertThrows(SyntaxCheckException.class, () -> buildExprAst(call)); + } + } + + /** A name the grammar does not list is a parse error, which routes to the legacy engine. */ + @Test + public void unknownBucketParameterIsASyntaxError() { + assertThrows( + SyntaxCheckException.class, + () -> buildExprAst("date_histogram(field=ts, interval='1d', 'feild'='ts')")); + } + private Node buildExprAst(String expr) { return buildExprAst(expr, astExprBuilder); }