Add SQL histogram and date_histogram bucket functions

integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java

@@ -994,7 +994,13 @@ public enum Index {
         "timewrap_test",
         "timewrap_test",
         "{\"mappings\":{\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"host\":{\"type\":\"keyword\"},\"requests\":{\"type\":\"integer\"},\"errors\":{\"type\":\"integer\"}}}}",
-        "src/test/resources/timewrap_test.json");
+        "src/test/resources/timewrap_test.json"),
+    DATE_HISTOGRAM_TEST(
+        "date_histogram_test",
+        "date_histogram_test",
+        "{\"mappings\":{\"properties\":{\"ts\":{\"type\":\"date\",\"format\":\"yyyy-MM-dd"
+            + " HH:mm:ss\"},\"category\":{\"type\":\"keyword\"},\"value\":{\"type\":\"integer\"}}}}",
+        "src/test/resources/date_histogram_test.json");
 
     private final String name;
     private final String type;

integ-test/src/test/java/org/opensearch/sql/sql/DateHistogramBucketFunctionIT.java

@@ -0,0 +1,247 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.sql;
+
+import static org.junit.Assert.assertThrows;
+import static org.opensearch.sql.util.Capability.LEGACY_ENGINE_FALLBACK;
+import static org.opensearch.sql.util.MatcherUtils.rows;
+import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
+import static org.opensearch.sql.util.MatcherUtils.verifyDataRowsInOrder;
+
+import java.io.IOException;
+import org.json.JSONObject;
+import org.junit.Test;
+import org.opensearch.client.ResponseException;
+import org.opensearch.sql.legacy.SQLIntegTestCase;
+import org.opensearch.sql.util.RequiresCapability;
+
+/**
+ * Execution coverage for {@code date_histogram} and {@code histogram}. The expander unit tests
+ * assert the AST that gets built; these assert what comes back after analysis, planning and
+ * pushdown, against 72 documents on fixed timestamps:
+ *
+ * <pre>
+ *   00:00 x5 alpha   00:30 x7 beta    01:00 x11 alpha
+ *   01:45 x13 gamma  02:00 x17 beta   03:00 x19 alpha
+ * </pre>
+ *
+ * so hourly grouping must yield 12/24/17/19 and half-hourly 5/7/11/13/17/19.
+ */
+public class DateHistogramBucketFunctionIT extends SQLIntegTestCase {
+
+  private static final String IDX = "date_histogram_test";
+
+  @Override
+  protected void init() throws Exception {
+    super.init();
+    loadIndex(Index.DATE_HISTOGRAM_TEST);
+  }
+
+  /**
+   * The bucket has to be projected in a derived table before it can be grouped on; see {@link
+   * #groupingOnTheBucketWithoutADerivedTableIsRejected}. This is also the shape Dashboards emits.
+   */
+  private static String bucketed(String bucketExpr) {
+    return "SELECT b, COUNT(*) FROM (SELECT "
+        + bucketExpr
+        + " AS b FROM "
+        + IDX
+        + ") sub GROUP BY b ORDER BY b";
+  }
+
+  @Test
+  public void hourlyBucketsCarryKeysAndCounts() throws IOException {
+    JSONObject response = executeQuery(bucketed("date_histogram('field'=ts, 'interval'='1h')"));
+
+    verifyDataRowsInOrder(
+        response,
+        rows("2026-01-01 00:00:00", 12),
+        rows("2026-01-01 01:00:00", 24),
+        rows("2026-01-01 02:00:00", 17),
+        rows("2026-01-01 03:00:00", 19));
+  }
+
+  /** A sub-hour interval must split 00:00/00:30 and 01:00/01:45 rather than merge them. */
+  @Test
+  public void halfHourlyBucketsSplitWithinTheHour() throws IOException {
+    JSONObject response = executeQuery(bucketed("date_histogram('field'=ts, 'interval'='30m')"));
+
+    verifyDataRowsInOrder(
+        response,
+        rows("2026-01-01 00:00:00", 5),
+        rows("2026-01-01 00:30:00", 7),
+        rows("2026-01-01 01:00:00", 11),
+        rows("2026-01-01 01:30:00", 13),
+        rows("2026-01-01 02:00:00", 17),
+        rows("2026-01-01 03:00:00", 19));
+  }
+
+  @Test
+  public void dailyIntervalCollapsesEverythingIntoOneBucket() throws IOException {
+    JSONObject response = executeQuery(bucketed("date_histogram('field'=ts, 'interval'='1d')"));
+
+    verifyDataRows(response, rows("2026-01-01 00:00:00", 72));
+  }
+
+  /** {@code fixed_interval} and {@code calendar_interval} are accepted as synonyms of interval. */
+  @Test
+  public void intervalSynonymsProduceTheSameBuckets() throws IOException {
+    JSONObject viaFixed =
+        executeQuery(bucketed("date_histogram('field'=ts, 'fixed_interval'='1h')"));
+    JSONObject viaCalendar =
+        executeQuery(bucketed("date_histogram('field'=ts, 'calendar_interval'='1h')"));
+
+    for (JSONObject response : new JSONObject[] {viaFixed, viaCalendar}) {
+      verifyDataRowsInOrder(
+          response,
+          rows("2026-01-01 00:00:00", 12),
+          rows("2026-01-01 01:00:00", 24),
+          rows("2026-01-01 02:00:00", 17),
+          rows("2026-01-01 03:00:00", 19));
+    }
+  }
+
+  /** A second grouping key needs the scan in a derived table of its own as well. */
+  @Test
+  public void bucketsCombineWithAnAdditionalGroupingKey() throws IOException {
+    JSONObject response =
+        executeQuery(
+            "SELECT b, c, COUNT(*) FROM (SELECT date_histogram('field'=ts, 'interval'='1h') AS b,"
+                + " category AS c FROM (SELECT * FROM "
+                + IDX
+                + ") inner_scan) sub GROUP BY b, c ORDER BY b, c");
+
+    verifyDataRowsInOrder(
+        response,
+        rows("2026-01-01 00:00:00", "alpha", 5),
+        rows("2026-01-01 00:00:00", "beta", 7),
+        rows("2026-01-01 01:00:00", "alpha", 11),
+        rows("2026-01-01 01:00:00", "gamma", 13),
+        rows("2026-01-01 02:00:00", "beta", 17),
+        rows("2026-01-01 03:00:00", "alpha", 19));
+  }
+
+  @Test
+  public void bucketsRespectAWhereClause() throws IOException {
+    JSONObject response =
+        executeQuery(
+            "SELECT b, COUNT(*) FROM (SELECT date_histogram('field'=ts, 'interval'='1h') AS b FROM "
+                + IDX
+                + " WHERE category = 'alpha') sub GROUP BY b ORDER BY b");
+
+    verifyDataRowsInOrder(
+        response,
+        rows("2026-01-01 00:00:00", 5),
+        rows("2026-01-01 01:00:00", 11),
+        rows("2026-01-01 03:00:00", 19));
+  }
+
+  /**
+   * `missing` substitutes a value for a null field before bucketing. Asserted end to end because
+   * the AST alone cannot show whether the substitution function is one the engine evaluates —
+   * `coalesce` is a registered name with no V2 implementation, `ifnull` is the one that runs.
+   *
+   * <p>Uses the numeric field: substituting into a date needs a timestamp-typed replacement, and
+   * the grammar admits only literals here, so a date `missing` reaches IFNULL as TIMESTAMP against
+   * STRING. V2 accepts that pair, the analytics engine rejects it, and a test asserting either
+   * result would disagree with the other route.
+   */
+  @Test
+  public void missingParameterSubstitutesBeforeBucketing() throws IOException {
+    JSONObject response =
+        executeQuery(
+            "SELECT b, COUNT(*) FROM (SELECT histogram('field'=value, 'interval'=20, 'missing'=0)"
+                + " AS b FROM "
+                + IDX
+                + ") sub GROUP BY b ORDER BY b");
+
+    verifyDataRowsInOrder(response, rows(0, 19), rows(20, 20), rows(40, 20), rows(60, 13));
+  }
+
+  @Test
+  public void numericHistogramBucketsByInterval() throws IOException {
+    JSONObject response =
+        executeQuery(
+            "SELECT b, COUNT(*) FROM (SELECT histogram('field'=value, 'interval'=20) AS b FROM "
+                + IDX
+                + ") sub GROUP BY b ORDER BY b");
+
+    // value runs 1..72, so the 20-wide buckets hold 19, 20, 20 and 13 documents.
+    verifyDataRowsInOrder(response, rows(0, 19), rows(20, 20), rows(40, 20), rows(60, 13));
+  }
+
+  /** Argument names may be written bare, the spelling the legacy engine has always accepted. */
+  @Test
+  public void unquotedArgumentNamesReturnHourlyBuckets() throws IOException {
+    JSONObject response =
+        executeQuery(
+            "SELECT b, COUNT(*) FROM (SELECT date_histogram(field=ts, interval='1h') AS b FROM "
+                + IDX
+                + ") sub GROUP BY b ORDER BY b");
+
+    verifyDataRowsInOrder(
+        response,
+        rows("2026-01-01 00:00:00", 12),
+        rows("2026-01-01 01:00:00", 24),
+        rows("2026-01-01 02:00:00", 17),
+        rows("2026-01-01 03:00:00", 19));
+  }
+
+  /**
+   * The legacy engine implements alias, format, time_zone, min_doc_count and order through the
+   * native date_histogram aggregation; this lowering has no equivalent, so those queries still have
+   * to reach it. CsvFormatResponseIT.dateHistogramTest has asserted this shape for years.
+   */
+  @Test
+  @RequiresCapability(LEGACY_ENGINE_FALLBACK)
+  public void callWithAliasParameterReturnsHourlyBuckets() throws IOException {
+    JSONObject response =
+        executeQuery(
+            "SELECT COUNT(*) FROM "
+                + IDX
+                + " GROUP BY date_histogram('field'='ts','fixed_interval'='1h','alias'='hours')");
+
+    verifyDataRows(response, rows(12), rows(24), rows(17), rows(19));
+  }
+
+  @Test
+  public void unquotedArgumentNamesReturnNumericBuckets() throws IOException {
+    JSONObject response =
+        executeQuery(
+            "SELECT b, COUNT(*) FROM (SELECT histogram(field=value, interval=20) AS b FROM "
+                + IDX
+                + ") sub GROUP BY b ORDER BY b");
+
+    verifyDataRowsInOrder(response, rows(0, 19), rows(20, 20), rows(40, 20), rows(60, 13));
+  }
+
+  /**
+   * A span over a bare table scan cannot resolve its field, with or without a select alias, so the
+   * bucket always has to be projected in a derived table first. Both routes reject this; only the
+   * message differs, so the assertion is on the rejection alone.
+   */
+  @Test
+  public void groupingOnTheBucketWithoutADerivedTableIsRejected() {
+    assertThrows(
+        ResponseException.class,
+        () ->
+            executeQuery(
+                "SELECT date_histogram('field'=ts, 'interval'='1h') AS b, COUNT(*) FROM "
+                    + IDX
+                    + " GROUP BY date_histogram('field'=ts, 'interval'='1h')"));
+  }
+
+  /** Calendar units: Dashboards emits 1M and 1y at the wider zoom levels. */
+  @Test
+  public void calendarIntervalsBucketByMonthAndYear() throws IOException {
+    verifyDataRows(
+        executeQuery(bucketed("date_histogram('field'=ts, 'interval'='1M')")),
+        rows("2026-01-01 00:00:00", 72));
+    verifyDataRows(
+        executeQuery(bucketed("date_histogram('field'=ts, 'interval'='1y')")),
+        rows("2026-01-01 00:00:00", 72));
+  }
+}

integ-test/src/test/java/org/opensearch/sql/util/Capability.java

@@ -537,6 +537,17 @@ public enum Capability {
   PREPARED_STATEMENT(
       "Prepared statements are unsupported on the analytics-engine route (Calcite path)."),
 
+  /**
+   * FRONTEND: the legacy V1 engine answers call shapes the V2 grammar declines, but only on the
+   * default route. Requests reach it when RestSQLQueryAction catches a SyntaxCheckException; the
+   * analytics-engine route enters through RestUnifiedQueryAction, which has no such fallback.
+   */
+  LEGACY_ENGINE_FALLBACK(
+      "A call shape only the legacy V1 engine understands (e.g. a date_histogram `alias`"
+          + " parameter) can't be answered on the analytics-engine route: reaching that engine"
+          + " depends on RestSQLQueryAction's SyntaxCheckException fallback, and the analytics"
+          + " route does not go through it."),
+
   /**
    * FRONTEND: legacy method-query syntax (regexp_query/wildcard_query) is not in the Calcite
    * grammar.