diff --git a/docs/configuration/index.md b/docs/configuration/index.md index f4c40ba15b77..556643bdd973 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -1896,6 +1896,7 @@ The Druid SQL server is configured through the following properties on the Broke |`druid.sql.planner.metadataSegmentCacheEnable`|Whether to keep a cache of published segments on Broker that can be used to serve queries against `sys.segments`. If true, broker polls coordinator in background to get segments from metadata store and maintains a local cache. If false, coordinator's REST API will be invoked when broker needs published segments info.|true| |`druid.sql.planner.metadataSegmentPollPeriod`|How often to poll coordinator for published segments list if `druid.sql.planner.metadataSegmentCacheEnable` is set to true. Poll period is in milliseconds. |60000| |`druid.sql.planner.authorizeSystemTablesDirectly`|If true, Druid authorizes queries against any of the system schema tables (`sys` in SQL) as `SYSTEM_TABLE` resources which require `READ` access, in addition to permissions based content filtering.|false| +|`druid.sql.planner.enableCatalogDdl`|If true, `CREATE TABLE` and `ALTER TABLE` statements may be used to define [catalog](../development/extensions-core/catalog.md) tables. These statements require `WRITE` permission on the datasource, the same permission the catalog API requires, so enabling this lets anyone who can ingest into a datasource also change its catalog definition. Requires the `druid-catalog` extension. Cannot be overridden per query.|false| |`druid.sql.planner.useNativeQueryExplain`|If true, `EXPLAIN PLAN FOR` will return the explain plan as a JSON representation of equivalent native query(s), else it will return the original version of explain plan generated by Calcite. It can be overridden per query with `useNativeQueryExplain` context key.|true| |`druid.sql.planner.maxNumericInFilters`|Max limit for the amount of numeric values that can be compared for a string type dimension when the entire SQL WHERE clause of a query translates to an [OR](../querying/filters.md#or) of [Bound filter](../querying/filters.md#bound-filter). By default, Druid does not restrict the amount of numeric Bound Filters on String columns, although this situation may block other queries from running. Set this property to a smaller value to prevent Druid from running queries that have prohibitively long segment processing times. The optimal limit requires some trial and error; we recommend starting with 100. Users who submit a query that exceeds the limit of `maxNumericInFilters` should instead rewrite their queries to use strings in the `WHERE` clause instead of numbers. For example, `WHERE someString IN (‘123’, ‘456’)`. If this value is disabled, `maxNumericInFilters` set through query context is ignored.|`-1` (disabled)| |`druid.sql.approxCountDistinct.function`|Implementation to use for the [`APPROX_COUNT_DISTINCT` function](../querying/sql-aggregations.md). Without extensions loaded, the only valid value is `APPROX_COUNT_DISTINCT_BUILTIN` (a HyperLogLog, or HLL, based implementation). If the [DataSketches extension](../development/extensions-core/datasketches-extension.md) is loaded, this can also be `APPROX_COUNT_DISTINCT_DS_HLL` (alternative HLL implementation) or `APPROX_COUNT_DISTINCT_DS_THETA`.

Theta sketches use significantly more memory than HLL sketches, so you should prefer one of the two HLL implementations.|`APPROX_COUNT_DISTINCT_BUILTIN`| diff --git a/docs/development/extensions-core/catalog.md b/docs/development/extensions-core/catalog.md index 1825d56ef706..aef6f2cc9556 100644 --- a/docs/development/extensions-core/catalog.md +++ b/docs/development/extensions-core/catalog.md @@ -43,6 +43,165 @@ allowing queries to be more concise, and simpler to write. This also allows the written into a defined column of the table is consistent with that columns definition, minimizing errors where unexpected data is written into a particular column of the table. +### SQL DDL + +Tables can be defined with SQL instead of by posting a table specification. `CREATE TABLE` and `ALTER TABLE` are +submitted to the Broker like any other SQL statement, and write the same catalog metadata the REST API does. They +return no rows. + +These statements change catalog metadata only. They never create, modify, or delete segments: defining a table does +not ingest anything, and altering a column does not rewrite existing data. Column changes take effect for subsequent +ingestion. + +These statements are disabled by default. Set `druid.sql.planner.enableCatalogDdl` to `true` on the Broker to enable +them. They require `WRITE` permission on the datasource, the same permission the catalog API requires, so enabling +them lets anyone who can ingest into a datasource also change its catalog definition; leave them disabled if you +manage catalog entries with your own tooling. The setting cannot be overridden per query. + +The `druid-catalog` extension must be loaded on both the Broker and the Coordinator; without it, these statements +report that the extension is not available. + +```sql +CREATE [OR REPLACE] TABLE [IF NOT EXISTS] + [ ( { | PROJECTION AS (
ADD COLUMN +ALTER TABLE
DROP COLUMN +ALTER TABLE
ALTER COLUMN SET DATA TYPE +ALTER TABLE
ADD [IF NOT EXISTS] PROJECTION AS (
DROP PROJECTION [IF EXISTS] +ALTER TABLE
SET PROPERTIES ( = [, ...] ) +``` + +#### Projections + +A table may declare [projections](../../querying/projections.md), which are pre-aggregated views stored inside each +segment. A projection is written as a `SELECT` over the table's own columns, with no `FROM` clause: + +```sql +CREATE TABLE "druid"."visits" ( + __time TIMESTAMP, + user_id VARCHAR, + user_agent VARCHAR, + pages_visited BIGINT, + PROJECTION daily_by_agent AS ( + SELECT TIME_FLOOR(__time, 'P1D'), user_agent, SUM(pages_visited) AS total_pages + WHERE user_agent IS NOT NULL + GROUP BY 1, 2 + ) +) +PARTITIONED BY DAY +``` + +The body is planned exactly as the equivalent query would be, so a projection matches the queries it was written to +serve. Every aggregate needs an alias, which becomes the name of the stored column. Time granularity is expressed +with `TIME_FLOOR`, as it would be in a query. + +A projection body accepts a select list, an optional `WHERE` and an optional `GROUP BY`. It cannot use `ORDER BY`, +`LIMIT` or `HAVING`: a projection's ordering follows its grouping columns and is not something you choose. It also +cannot use joins, subqueries, or expressions computed over aggregates. Store the aggregates instead: `SUM(x)` and +`COUNT(x)` rather than `AVG(x)`. + +Projections may also be added to and removed from an existing table: + +```sql +ALTER TABLE "druid"."visits" ADD [IF NOT EXISTS] PROJECTION by_agent AS ( + SELECT user_agent, SUM(pages_visited) AS total_pages GROUP BY user_agent +) +ALTER TABLE "druid"."visits" DROP PROJECTION [IF EXISTS] by_agent +``` + +Both take effect for subsequent ingestion. Segments already built keep whatever projections they were built with, so +dropping a projection does not rewrite data. + +#### The base table + +The reserved projection name `__base` describes the table's own physical layout rather than an additional +pre-aggregation. Defining it makes the table a 'clustered' table: rows of segments are stored grouped by the clustering +columns. + +Its body lists the columns in the order segments store them, so it must name every declared column, in declared +order. An item written as ` AS ` makes that column computed at ingest time, from the columns it reads: + +```sql +CREATE TABLE "druid"."events" ( + tenant VARCHAR, + bucket BIGINT, + __time TIMESTAMP, + user_id BIGINT, + payload TYPE('COMPLEX'), + PROJECTION __base AS ( + SELECT tenant, ABS(user_id) % 128 AS bucket, __time, user_id, payload + CLUSTERED BY tenant, bucket + ) +) +PARTITIONED BY DAY +SEALED +``` + +The clustering columns must be the leading columns of the table, because the declared order is the physical order. +`SEALED` is required: the declared columns define the physical segment schema, so a column that is not declared +cannot be stored. + +A computed column is written by the expression, not by the ingestion query, so an `INSERT` must supply the +expression's inputs and leave the computed column out. Above, that means supplying `user_id` and letting `bucket` +be derived. + +Unlike an aggregate projection, a `__base` body cannot filter or group: the base table stores every ingested row. +It is the only projection that chooses a clustering. + +`ALTER TABLE ... ADD PROJECTION __base AS ( ... )` gives an existing table a layout, and +`ALTER TABLE ... DROP PROJECTION __base` removes it. Both affect future segments only. + +Every other name beginning with `__` remains reserved. + +#### Setting table properties + +`SET PROPERTIES` merges the given [table properties](#table-properties) into the table. A value of `NULL` removes a +property. Values must be literals: + +```sql +ALTER TABLE "druid"."visits" SET PROPERTIES (targetSegmentRows = 3000000, sealed = TRUE) +ALTER TABLE "druid"."visits" SET PROPERTIES (sealed = NULL) +``` + +Both statements require the `WRITE` permission on the datasource, the same permission the REST API checks. Table +names may be unqualified or qualified with the `druid` schema; other schemas are rejected. Names are case-sensitive. + +There is no `DROP TABLE`. Deleting a table's catalog entry without deleting its data would be a surprising meaning +for the statement, so removing a specification is left to the [delete API](#delete-a-table) until the semantics are +settled. + ### API Objects #### TableSpec @@ -90,10 +249,11 @@ The endpoint supports a set of optional query parameters to enforce optimistic l is meant to update a table rather than create a new one. In the default case, with no query parameters set, this request will return an error if a table of the same name already exists in the schema specified. -| Parameter | Type | Description | -|-------------|---------|-------------------------------------------------------------------------------------------------------------------------------| -| `version` | Long | the expected version of an existing table. The version must match. If not (or if the table does not exist), returns an error. | -| `overwrite` | boolean | if true, then overwrites any existing table. Otherwise, the operation fails if the table already exists. | +| Parameter | Type | Description | +|---------------|---------|-------------------------------------------------------------------------------------------------------------------------------| +| `version` | Long | the expected version of an existing table. The version must match. If not (or if the table does not exist), returns an error. | +| `overwrite` | boolean | if true, then overwrites any existing table. Otherwise, the operation fails if the table already exists. | +| `ifNotExists` | boolean | if true, then leaves an existing table unchanged and reports a version of 0. Otherwise, the operation fails if the table already exists. | ##### Responses diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java new file mode 100644 index 000000000000..1e51490c8d06 --- /dev/null +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java @@ -0,0 +1,465 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testing.embedded.catalog; + +import com.fasterxml.jackson.core.type.TypeReference; +import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.DatasourceProjectionMetadata; +import org.apache.druid.catalog.model.TableId; +import org.apache.druid.catalog.model.TableMetadata; +import org.apache.druid.catalog.model.table.ClusterKeySpec; +import org.apache.druid.catalog.model.table.DatasourceDefn; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.query.http.SqlTaskStatus; +import org.apache.druid.segment.TestHelper; +import org.apache.druid.server.metrics.LatchableEmitter; +import org.apache.druid.testing.embedded.EmbeddedDruidCluster; +import org.apache.druid.testing.embedded.msq.EmbeddedMSQApis; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * End-to-end coverage of catalog DDL: define a table with SQL, ingest into it, and query it. This is what proves the + * Broker-to-Coordinator write path and the cache refresh actually work against a running cluster, rather than against + * a recording stub. + */ +public class CatalogDdlAndIngestTest extends CatalogTestBase +{ + private TestCatalogClient client; + private EmbeddedMSQApis msqApis; + + @Override + protected EmbeddedDruidCluster createCluster() + { + // Catalog DDL is opt-in. + broker.addProperty("druid.sql.planner.enableCatalogDdl", "true"); + return super.createCluster(); + } + + @BeforeAll + public void initializeClient() + { + client = new TestCatalogClient(cluster); + msqApis = new EmbeddedMSQApis(cluster, overlord); + } + + @Test + public void testCreateTableThenIngestAndQuery() + { + final String tableName = dataSource; + + cluster.callApi().runSql( + "CREATE TABLE \"%s\" (\n" + + " __time TIMESTAMP,\n" + + " varchar_col1 VARCHAR,\n" + + " double_col1 DOUBLE\n" + + ")\n" + + "PARTITIONED BY DAY\n" + + "CLUSTERED BY varchar_col1", + tableName + ); + + // The spec reached the Coordinator, in the declared column order. + final TableMetadata table = client.readTable(TableId.datasource(tableName)); + assertEquals( + List.of("__time", "varchar_col1", "double_col1"), + columnNames(table) + ); + assertEquals( + List.of("TIMESTAMP", "VARCHAR", "DOUBLE"), + table.spec().columns().stream().map(ColumnSpec::dataType).collect(Collectors.toList()) + ); + assertEquals("P1D", table.spec().properties().get(DatasourceDefn.SEGMENT_GRANULARITY_PROPERTY)); + + // An INSERT that omits PARTITIONED BY picks it up from the catalog entry the DDL just wrote, which only works + // if this Broker's catalog cache saw the write. + ingest( + "INSERT INTO \"%s\"\n" + + "SELECT\n" + + " TIME_PARSE(a) AS __time,\n" + + " b AS varchar_col1,\n" + + " c AS double_col1\n" + + "FROM TABLE(\n" + + " EXTERN(\n" + + " '{\"type\":\"inline\",\"data\":\"2022-12-26T12:34:56,foo,10\\n2022-12-26T12:34:56,bar,20\"}',\n" + + " '{\"type\":\"csv\",\"findColumnsFromHeader\":false,\"columns\":[\"a\",\"b\",\"c\"]}'\n" + + " )\n" + + ") EXTEND (a VARCHAR, b VARCHAR, c BIGINT)\n", + tableName + ); + + // The BIGINT source column is coerced to the DOUBLE declared by the DDL, and rows come back in clustered order. + cluster.callApi().verifySqlQuery( + "SELECT * FROM %s", + tableName, + "2022-12-26T12:34:56.000Z,bar,20.0\n" + + "2022-12-26T12:34:56.000Z,foo,10.0" + ); + } + + @Test + public void testAlterTableAddColumnThenIngest() + { + final String tableName = dataSource; + + cluster.callApi().runSql("CREATE TABLE \"%s\" (__time TIMESTAMP, a VARCHAR) PARTITIONED BY DAY", tableName); + cluster.callApi().runSql("ALTER TABLE \"%s\" ADD COLUMN b BIGINT", tableName); + + final TableMetadata table = client.readTable(TableId.datasource(tableName)); + assertEquals(List.of("__time", "a", "b"), columnNames(table)); + + ingest( + "INSERT INTO \"%s\"\n" + + "SELECT TIME_PARSE(x) AS __time, y AS a, z AS b\n" + + "FROM TABLE(\n" + + " EXTERN(\n" + + " '{\"type\":\"inline\",\"data\":\"2022-12-26T12:34:56,hello,7\"}',\n" + + " '{\"type\":\"csv\",\"findColumnsFromHeader\":false,\"columns\":[\"x\",\"y\",\"z\"]}'\n" + + " )\n" + + ") EXTEND (x VARCHAR, y VARCHAR, z BIGINT)\n", + tableName + ); + + cluster.callApi().verifySqlQuery("SELECT * FROM %s", tableName, "2022-12-26T12:34:56.000Z,hello,7"); + } + + @Test + public void testAlterTableColumnAndProperties() + { + final String tableName = dataSource; + + cluster.callApi().runSql("CREATE TABLE \"%s\" (__time TIMESTAMP, a VARCHAR, b VARCHAR)", tableName); + + cluster.callApi().runSql("ALTER TABLE \"%s\" ALTER COLUMN b SET DATA TYPE BIGINT", tableName); + cluster.callApi().runSql("ALTER TABLE \"%s\" DROP COLUMN a", tableName); + cluster.callApi().runSql( + "ALTER TABLE \"%s\" SET PROPERTIES (segmentGranularity = 'P1D', sealed = TRUE)", + tableName + ); + + final TableMetadata table = client.readTable(TableId.datasource(tableName)); + assertEquals(List.of("__time", "b"), columnNames(table)); + assertEquals("BIGINT", table.spec().columns().get(1).dataType()); + assertEquals("P1D", table.spec().properties().get(DatasourceDefn.SEGMENT_GRANULARITY_PROPERTY)); + assertEquals(true, table.spec().properties().get(DatasourceDefn.SEALED_PROPERTY)); + + // A null value removes a property. + cluster.callApi().runSql("ALTER TABLE \"%s\" SET PROPERTIES (sealed = NULL)", tableName); + assertNull( + client.readTable(TableId.datasource(tableName)).spec().properties().get(DatasourceDefn.SEALED_PROPERTY) + ); + } + + @Test + public void testCreateOrReplaceAndIfNotExists() + { + final String tableName = dataSource; + + cluster.callApi().runSql("CREATE TABLE \"%s\" (__time TIMESTAMP, a VARCHAR) CLUSTERED BY a", tableName); + + // IF NOT EXISTS leaves the original definition alone. + cluster.callApi().runSql("CREATE TABLE IF NOT EXISTS \"%s\" (__time TIMESTAMP, zzz VARCHAR)", tableName); + assertEquals(List.of("__time", "a"), columnNames(client.readTable(TableId.datasource(tableName)))); + + // OR REPLACE swaps the whole spec, including dropping the clustering the first statement set. + cluster.callApi().runSql("CREATE OR REPLACE TABLE \"%s\" (__time TIMESTAMP, b BIGINT)", tableName); + final TableMetadata replaced = client.readTable(TableId.datasource(tableName)); + assertEquals(List.of("__time", "b"), columnNames(replaced)); + assertNull(replaced.spec().properties().get(DatasourceDefn.CLUSTER_KEYS_PROPERTY)); + } + + @Test + public void testCreateTableClusterKeys() + { + final String tableName = dataSource; + cluster.callApi().runSql("CREATE TABLE \"%s\" (__time TIMESTAMP, a VARCHAR, b BIGINT) CLUSTERED BY a, b", tableName); + + // Catalog properties round trip as untyped JSON, so decode before asserting. + final TableMetadata table = client.readTable(TableId.datasource(tableName)); + final List keys = TestHelper.JSON_MAPPER.convertValue( + table.spec().properties().get(DatasourceDefn.CLUSTER_KEYS_PROPERTY), + ClusterKeySpec.CLUSTER_KEY_LIST_TYPE_REF + ); + assertEquals(List.of("a", "b"), keys.stream().map(ClusterKeySpec::expr).collect(Collectors.toList())); + assertTrue(keys.stream().noneMatch(ClusterKeySpec::desc)); + } + + /** + * A rejection from the Coordinator's own validation must reach the SQL user as the Coordinator worded it, not as + * a generic remote-call failure. Segment granularity is validated only on the Coordinator, so it exercises the + * whole round trip. + */ + @Test + public void testCoordinatorValidationErrorSurfacesToSqlUser() + { + final String tableName = dataSource; + cluster.callApi().runSql("CREATE TABLE \"%s\" (__time TIMESTAMP, a VARCHAR)", tableName); + + final Exception e = assertThrows( + Exception.class, + () -> cluster.callApi().runSql( + "ALTER TABLE \"%s\" SET PROPERTIES (segmentGranularity = 'not_a_granularity')", + tableName + ) + ); + assertTrue(e.getMessage().contains("granularity"), e.getMessage()); + } + + @Test + public void testCreateTableAlreadyExistsFails() + { + final String tableName = dataSource; + cluster.callApi().runSql("CREATE TABLE \"%s\" (__time TIMESTAMP, a VARCHAR)", tableName); + + final Exception e = assertThrows( + Exception.class, + () -> cluster.callApi().runSql("CREATE TABLE \"%s\" (__time TIMESTAMP, a VARCHAR)", tableName) + ); + assertTrue(e.getMessage().contains("duplicate table"), e.getMessage()); + } + + /** + * A projection defined in SQL must actually be used at query time, which is the whole point of storing one: the + * specification the translator produces has to match what the planner generates for the equivalent query. + */ + @Test + public void testCreateTableWithProjectionThenQuery() + { + final String tableName = dataSource; + + cluster.callApi().runSql( + "CREATE TABLE \"%s\" (\n" + + " __time TIMESTAMP,\n" + + " varchar_col1 VARCHAR,\n" + + " bigint_col1 BIGINT,\n" + + " PROJECTION by_varchar AS (\n" + + " SELECT varchar_col1, SUM(bigint_col1) AS sum_bigint_col1\n" + + " GROUP BY varchar_col1\n" + + " )\n" + + ")\n" + + "PARTITIONED BY DAY", + tableName + ); + + ingest( + "INSERT INTO \"%s\"\n" + + "SELECT TIME_PARSE(a) AS __time, b AS varchar_col1, c AS bigint_col1\n" + + "FROM TABLE(\n" + + " EXTERN(\n" + + " '{\"type\":\"inline\",\"data\":\"2022-12-26T12:34:56,foo,10\\n2022-12-26T12:34:56,foo,9" + + "\\n2022-12-26T12:34:56,bar,8\"}',\n" + + " '{\"type\":\"csv\",\"findColumnsFromHeader\":false,\"columns\":[\"a\",\"b\",\"c\"]}'\n" + + " )\n" + + ") EXTEND (a VARCHAR, b VARCHAR, c BIGINT)\n", + tableName + ); + + final LatchableEmitter emitter = historical.latchableEmitter(); + emitter.flush(); + + cluster.callApi().verifySqlQuery( + "SELECT varchar_col1, SUM(bigint_col1) FROM %s GROUP BY 1 ORDER BY 1", + tableName, + "bar,8\nfoo,19" + ); + + // The segment-scan metrics name the projection that served the query. + emitter.waitForEvent( + event -> event.hasMetricName("query/segment/time").hasDimension("projection", "by_varchar") + ); + } + + @Test + public void testAlterTableAddAndDropProjection() + { + final String tableName = dataSource; + + cluster.callApi().runSql( + "CREATE TABLE \"%s\" (__time TIMESTAMP, a VARCHAR, b BIGINT) PARTITIONED BY DAY", + tableName + ); + cluster.callApi().runSql( + "ALTER TABLE \"%s\" ADD PROJECTION p AS (SELECT a, SUM(b) AS sum_b GROUP BY a)", + tableName + ); + assertEquals(1, projectionsOf(tableName).size()); + assertEquals("p", projectionsOf(tableName).get(0).getSpec().getName()); + + // Adding it again is an error, but IF NOT EXISTS tolerates it. + assertThrows( + Exception.class, + () -> cluster.callApi().runSql( + "ALTER TABLE \"%s\" ADD PROJECTION p AS (SELECT a, SUM(b) AS sum_b GROUP BY a)", + tableName + ) + ); + cluster.callApi().runSql( + "ALTER TABLE \"%s\" ADD IF NOT EXISTS PROJECTION p AS (SELECT a, SUM(b) AS sum_b GROUP BY a)", + tableName + ); + assertEquals(1, projectionsOf(tableName).size()); + + cluster.callApi().runSql("ALTER TABLE \"%s\" DROP PROJECTION p", tableName); + assertNull( + client.readTable(TableId.datasource(tableName)) + .spec().properties().get(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY) + ); + cluster.callApi().runSql("ALTER TABLE \"%s\" DROP PROJECTION IF EXISTS p", tableName); + } + + private List projectionsOf(String tableName) + { + return TestHelper.JSON_MAPPER.convertValue( + client.readTable(TableId.datasource(tableName)) + .spec().properties().get(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY), + new TypeReference>() {} + ); + } + + /** + * A clustered table defined entirely in SQL. The declared column order is the physical segment order, so rows come + * back grouped by the clustering column, and a computed column is materialized at ingest time from the expression + * the __base projection gives it. + */ + @Test + public void testCreateClusteredBaseTableThenIngestAndQuery() + { + final String tableName = dataSource; + + cluster.callApi().runSql( + "CREATE TABLE \"%s\" (\n" + + " varchar_col2 VARCHAR,\n" + + " __time TIMESTAMP,\n" + + " varchar_col1 VARCHAR,\n" + + " bigint_col1 BIGINT,\n" + + " doubled BIGINT,\n" + + " PROJECTION __base AS (\n" + + " SELECT varchar_col2, __time, varchar_col1, bigint_col1, bigint_col1 * 2 AS doubled\n" + + " CLUSTERED BY varchar_col2\n" + + " )\n" + + ")\n" + + "PARTITIONED BY DAY SEALED", + tableName + ); + + final TableMetadata table = client.readTable(TableId.datasource(tableName)); + assertEquals(true, table.spec().properties().get(DatasourceDefn.SEALED_PROPERTY)); + assertNotNull(table.spec().properties().get(DatasourceDefn.BASE_TABLE_PROPERTY)); + + // 'doubled' is computed by the base table, so the INSERT supplies only its input column. + ingest( + "INSERT INTO \"%s\"\n" + + "SELECT TIME_PARSE(a) AS __time, b AS varchar_col1, c AS bigint_col1, f AS varchar_col2\n" + + "FROM TABLE(\n" + + " EXTERN(\n" + + " '{\"type\":\"inline\",\"data\":\"2022-12-26T12:34:56,extra,10,foo" + + "\\n2022-12-26T12:34:56,extra,9,foo\\n2022-12-26T12:34:56,extra,8,foq" + + "\\n2022-12-26T12:34:56,extra,8,fop\"}',\n" + + " '{\"type\":\"csv\",\"findColumnsFromHeader\":false,\"columns\":[\"a\",\"b\",\"c\",\"f\"]}'\n" + + " )\n" + + ") EXTEND (a VARCHAR, b VARCHAR, c BIGINT, f VARCHAR)\n", + tableName + ); + + // Columns come back in declared order, rows in clustering-value order, and 'doubled' was materialized at ingest. + cluster.callApi().verifySqlQuery( + "SELECT * FROM %s", + tableName, + "foo,2022-12-26T12:34:56.000Z,extra,9,18\n" + + "foo,2022-12-26T12:34:56.000Z,extra,10,20\n" + + "fop,2022-12-26T12:34:56.000Z,extra,8,16\n" + + "foq,2022-12-26T12:34:56.000Z,extra,8,16" + ); + } + + /** + * A column the base table computes cannot be written directly: the catalog rejects the INSERT and says to supply + * the expression's inputs instead. + */ + @Test + public void testInsertIntoComputedColumnIsRejected() + { + final String tableName = dataSource; + + cluster.callApi().runSql( + "CREATE TABLE \"%s\" (t VARCHAR, __time TIMESTAMP, v BIGINT, doubled BIGINT," + + " PROJECTION __base AS (SELECT t, __time, v, v * 2 AS doubled CLUSTERED BY t))" + + " PARTITIONED BY DAY SEALED", + tableName + ); + + verifySubmitSqlTaskFailsWith400BadRequest( + StringUtils.format( + "INSERT INTO \"%s\" SELECT TIME_PARSE(a) AS __time, b AS t, 1 AS v, 2 AS doubled" + + " FROM TABLE(EXTERN('{\"type\":\"inline\",\"data\":\"2022-12-26T12:34:56,x\"}'," + + " '{\"type\":\"csv\",\"findColumnsFromHeader\":false,\"columns\":[\"a\",\"b\"]}'))" + + " EXTEND (a VARCHAR, b VARCHAR) PARTITIONED BY DAY", + tableName + ), + "computed by a virtual column" + ); + } + + @Test + public void testAlterTableAddAndDropBaseProjection() + { + final String tableName = dataSource; + + cluster.callApi().runSql( + "CREATE TABLE \"%s\" (t VARCHAR, __time TIMESTAMP, v BIGINT) PARTITIONED BY DAY SEALED", + tableName + ); + cluster.callApi().runSql( + "ALTER TABLE \"%s\" ADD PROJECTION __base AS (SELECT t, __time, v CLUSTERED BY t)", + tableName + ); + assertNotNull( + client.readTable(TableId.datasource(tableName)) + .spec().properties().get(DatasourceDefn.BASE_TABLE_PROPERTY) + ); + + cluster.callApi().runSql("ALTER TABLE \"%s\" DROP PROJECTION __base", tableName); + assertNull( + client.readTable(TableId.datasource(tableName)) + .spec().properties().get(DatasourceDefn.BASE_TABLE_PROPERTY) + ); + } + + private void ingest(String sqlPattern, String tableName) + { + final SqlTaskStatus status = msqApis.submitTaskSql(StringUtils.format(sqlPattern, tableName)); + cluster.callApi().waitForTaskToSucceed(status.getTaskId(), overlord); + cluster.callApi().waitForAllSegmentsToBeAvailable(tableName, coordinator, broker); + } + + private static List columnNames(TableMetadata table) + { + return table.spec().columns().stream().map(ColumnSpec::name).collect(Collectors.toList()); + } +} diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/guice/CatalogClientModule.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/guice/CatalogClientModule.java index a9ca664b2be0..8c44c87a487e 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/guice/CatalogClientModule.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/guice/CatalogClientModule.java @@ -29,6 +29,7 @@ import org.apache.druid.catalog.sync.CatalogClient; import org.apache.druid.catalog.sync.CatalogClientConfig; import org.apache.druid.catalog.sync.CatalogSource; +import org.apache.druid.catalog.sync.CatalogSqlTableWriter; import org.apache.druid.catalog.sync.CatalogUpdateListener; import org.apache.druid.catalog.sync.CatalogUpdateReceiver; import org.apache.druid.discovery.NodeRole; @@ -41,6 +42,7 @@ import org.apache.druid.guice.annotations.LoadScope; import org.apache.druid.initialization.DruidModule; import org.apache.druid.sql.calcite.planner.CatalogResolver; +import org.apache.druid.sql.calcite.planner.CatalogTableWriter; /** * Configures the metadata catalog on the Broker and Overlord to use a cache @@ -101,6 +103,12 @@ public void configure(Binder binder) .to(LiveCatalogResolver.class) .in(LazySingleton.class); + // Catalog writer for DDL statements. This will override the base binding. + binder + .bind(CatalogTableWriter.class) + .to(CatalogSqlTableWriter.class) + .in(LazySingleton.class); + // The listener resource sends to the catalog listener (the cached catalog.) Jerseys.addResource(binder, CatalogListenerResource.class); } diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/CatalogResource.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/CatalogResource.java index f96f042e7223..3d51cc179ebb 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/CatalogResource.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/CatalogResource.java @@ -129,6 +129,7 @@ public Response postTable( TableSpec spec, @QueryParam("version") long version, @QueryParam("overwrite") boolean overwrite, + @QueryParam("ifNotExists") boolean ifNotExists, @Context final HttpServletRequest req ) { @@ -159,6 +160,9 @@ public Response postTable( if (overwrite) { // User wants to overwrite, so do so. newVersion = catalog.tables().replace(table); + } else if (ifNotExists) { + // User asked to leave an existing table alone. Report no change, as the edit operations do. + newVersion = 0; } else { throw e; } diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditRequest.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditRequest.java index 79136d4a0595..428f982c7848 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditRequest.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditRequest.java @@ -25,9 +25,9 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.DatasourceProjectionMetadata; import javax.annotation.Nullable; - import java.util.List; import java.util.Map; @@ -51,6 +51,8 @@ @Type(name = "updateProperties", value = TableEditRequest.UpdateProperties.class), @Type(name = "updateColumns", value = TableEditRequest.UpdateColumns.class), @Type(name = "moveColumn", value = TableEditRequest.MoveColumn.class), + @Type(name = "addProjection", value = TableEditRequest.AddProjection.class), + @Type(name = "dropProjection", value = TableEditRequest.DropProjection.class), }) public class TableEditRequest { @@ -169,4 +171,49 @@ public MoveColumn( this.anchor = anchor; } } + + /** + * Append a projection to the list of projections. Fails if a projection of the same name already exists, unless + * {@code ifNotExists} is set, in which case the existing projection is left alone. + */ + public static class AddProjection extends TableEditRequest + { + @JsonProperty("projection") + public final DatasourceProjectionMetadata projection; + @JsonProperty("ifNotExists") + public final boolean ifNotExists; + + @JsonCreator + public AddProjection( + @JsonProperty("projection") DatasourceProjectionMetadata projection, + @JsonProperty("ifNotExists") boolean ifNotExists + ) + { + this.projection = projection; + this.ifNotExists = ifNotExists; + } + } + + /** + * Remove the named projection from the list of projections. Fails if no such projection exists, unless + * {@code ifExists} is set. Segments already built keep whatever projections they were built with. + */ + public static class DropProjection extends TableEditRequest + { + @JsonProperty("projection") + public final String projection; + @JsonProperty("ifExists") + public final boolean ifExists; + + @JsonCreator + public DropProjection( + @JsonProperty("projection") String projection, + @JsonProperty("ifExists") boolean ifExists + ) + { + this.projection = projection; + this.ifExists = ifExists; + } + } + } diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java index b3a762b05686..b69cb48acf6b 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/TableEditor.java @@ -21,7 +21,9 @@ import com.google.common.base.Strings; import org.apache.druid.catalog.CatalogException; +import org.apache.druid.catalog.http.TableEditRequest.AddProjection; import org.apache.druid.catalog.http.TableEditRequest.DropColumns; +import org.apache.druid.catalog.http.TableEditRequest.DropProjection; import org.apache.druid.catalog.http.TableEditRequest.HideColumns; import org.apache.druid.catalog.http.TableEditRequest.MoveColumn; import org.apache.druid.catalog.http.TableEditRequest.MoveColumn.Position; @@ -30,6 +32,8 @@ import org.apache.druid.catalog.http.TableEditRequest.UpdateProperties; import org.apache.druid.catalog.model.CatalogUtils; import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.DatasourceProjectionMetadata; +import org.apache.druid.catalog.model.ResolvedTable; import org.apache.druid.catalog.model.TableDefn; import org.apache.druid.catalog.model.TableId; import org.apache.druid.catalog.model.TableMetadata; @@ -41,7 +45,6 @@ import org.apache.druid.utils.CollectionUtils; import javax.ws.rs.core.Response; - import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -99,6 +102,12 @@ public long go() throws CatalogException return updateProperties(((UpdateProperties) editRequest).properties); } else if (editRequest instanceof UpdateColumns) { return updateColumns(((UpdateColumns) editRequest).columns); + } else if (editRequest instanceof AddProjection) { + final AddProjection addProjection = (AddProjection) editRequest; + return addProjection(addProjection.projection, addProjection.ifNotExists); + } else if (editRequest instanceof DropProjection) { + final DropProjection dropProjection = (DropProjection) editRequest; + return dropProjection(dropProjection.projection, dropProjection.ifExists); } else if (editRequest instanceof MoveColumn) { return moveColumn(((MoveColumn) editRequest)); } else { @@ -379,4 +388,90 @@ private TableSpec applyMoveColumn( return existingSpec.withColumns(revised); } + + /** + * Projections are stored as a table property, so adding one is a read-modify-write of that property performed + * inside the update transaction rather than by the caller. + */ + private long addProjection(DatasourceProjectionMetadata projection, boolean ifNotExists) throws CatalogException + { + if (projection == null || projection.getSpec() == null) { + throw CatalogException.badRequest("A projection is required"); + } + final String name = projection.getSpec().getName(); + return catalog.tables().updateProperties(id, table -> { + final List existing = projectionsOf(table); + for (DatasourceProjectionMetadata current : existing) { + if (name.equals(current.getSpec().getName())) { + if (ifNotExists) { + return null; + } + throw CatalogException.badRequest( + "Projection [%s] already exists in table %s", + name, + id.sqlName() + ); + } + } + final List revised = new ArrayList<>(existing); + revised.add(projection); + return withProjections(table, revised); + }); + } + + private long dropProjection(String name, boolean ifExists) throws CatalogException + { + if (Strings.isNullOrEmpty(name)) { + throw CatalogException.badRequest("A projection name is required"); + } + return catalog.tables().updateProperties(id, table -> { + final List existing = projectionsOf(table); + final List revised = new ArrayList<>(existing.size()); + for (DatasourceProjectionMetadata current : existing) { + if (!name.equals(current.getSpec().getName())) { + revised.add(current); + } + } + if (revised.size() == existing.size()) { + if (ifExists) { + return null; + } + throw CatalogException.badRequest( + "Projection [%s] does not exist in table %s", + name, + id.sqlName() + ); + } + return withProjections(table, revised); + }); + } + + private List projectionsOf(TableMetadata table) + { + final ResolvedTable resolved = catalog.tableRegistry().resolve(table.spec()); + final List projections = + resolved.decodeProperty(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY); + return projections == null ? Collections.emptyList() : projections; + } + + private TableSpec withProjections(TableMetadata table, List projections) + throws CatalogException + { + final TableSpec existingSpec = table.spec(); + final Map revised = new HashMap<>(existingSpec.properties()); + if (projections.isEmpty()) { + revised.remove(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY); + } else { + revised.put(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY, projections); + } + final TableSpec revisedSpec = existingSpec.withProperties(revised); + try { + catalog.tableRegistry().resolve(revisedSpec).validate(); + } + catch (IAE | DruidException e) { + throw CatalogException.badRequest(e.getMessage()); + } + return revisedSpec; + } + } diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogClient.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogClient.java index 1c3a2be487b7..7a0bf20ee783 100644 --- a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogClient.java +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogClient.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.util.concurrent.ListenableFuture; import org.apache.druid.catalog.http.CatalogResource; +import org.apache.druid.catalog.http.TableEditRequest; import org.apache.druid.catalog.model.ResolvedTable; import org.apache.druid.catalog.model.TableDefnRegistry; import org.apache.druid.catalog.model.TableId; @@ -59,6 +60,7 @@ public class CatalogClient implements CatalogSource public static final String SCHEMA_SYNC_PATH = CatalogResource.ROOT_PATH + CatalogResource.SCHEMA_SYNC; public static final String TABLE_SYNC_PATH = CatalogResource.ROOT_PATH + CatalogResource.TABLE_SYNC; private static final String TABLE_CREATE_PATH = CatalogResource.ROOT_PATH + "/schemas/{schema}/tables/{name}"; + private static final String TABLE_EDIT_PATH = TABLE_CREATE_PATH + "/edit"; private final ServiceClient serviceClient; private final ObjectMapper jsonMapper; @@ -105,12 +107,38 @@ public ResolvedTable resolveTable(TableId id) /** * Creates a table for the given {@link TableId} and {@link TableSpec}. * If a table already exists for this id, it is overwritten. - *

- * This method is currently used only in tests. */ public void createTable(TableId tableId, TableSpec tableSpec) { - getResult(postCreateTable(tableId, tableSpec)); + getResult(postCreateTable(tableId, tableSpec, false, true)); + } + + /** + * Creates a table for the given {@link TableId} and {@link TableSpec}. + * + * @param ifNotExists leave an existing table alone rather than failing + * @param overwrite replace the spec of an existing table + */ + public void createTable(TableId tableId, TableSpec tableSpec, boolean ifNotExists, boolean overwrite) + { + FutureUtils.getUnchecked(postCreateTable(tableId, tableSpec, ifNotExists, overwrite), true); + } + + /** + * Applies an edit to an existing table's catalog entry. + *

+ * API: {@code POST /druid/coordinator/v1/catalog/schemas/{schema}/tables/{name}/edit} + */ + public void editTable(TableId tableId, TableEditRequest editRequest) + { + String path = tablePath(TABLE_EDIT_PATH, tableId); + FutureUtils.getUnchecked( + serviceClient.asyncRequest( + new RequestBuilder(HttpMethod.POST, path).jsonContent(jsonMapper, editRequest), + IgnoreHttpResponseHandler.INSTANCE + ), + true + ); } /** @@ -118,10 +146,19 @@ public void createTable(TableId tableId, TableSpec tableSpec) *

* API: {@code POST /druid/coordinator/v1/catalog/schemas/{schema}/tables/{name}} */ - private ListenableFuture postCreateTable(TableId tableId, TableSpec tableSpec) + private ListenableFuture postCreateTable( + TableId tableId, + TableSpec tableSpec, + boolean ifNotExists, + boolean overwrite + ) { - String path = StringUtils.replace(TABLE_CREATE_PATH, "{schema}", StringUtils.urlEncode(tableId.schema())); - path = StringUtils.replace(path, "{name}", StringUtils.urlEncode(tableId.name())); + final String path = StringUtils.format( + "%s?overwrite=%s&ifNotExists=%s", + tablePath(TABLE_CREATE_PATH, tableId), + overwrite, + ifNotExists + ); return serviceClient.asyncRequest( new RequestBuilder(HttpMethod.POST, path) @@ -130,6 +167,12 @@ private ListenableFuture postCreateTable(TableId tableId, TableSpec tableS ); } + private static String tablePath(String template, TableId tableId) + { + String path = StringUtils.replace(template, "{schema}", StringUtils.urlEncode(tableId.schema())); + return StringUtils.replace(path, "{name}", StringUtils.urlEncode(tableId.name())); + } + /** * Fetches the metadata of a table from the Coordinator. *

diff --git a/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogSqlTableWriter.java b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogSqlTableWriter.java new file mode 100644 index 000000000000..f75b2bd27eec --- /dev/null +++ b/extensions-core/druid-catalog/src/main/java/org/apache/druid/catalog/sync/CatalogSqlTableWriter.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.catalog.sync; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Throwables; +import org.apache.druid.catalog.CatalogException; +import org.apache.druid.catalog.http.TableEditRequest; +import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.DatasourceProjectionMetadata; +import org.apache.druid.catalog.model.TableId; +import org.apache.druid.catalog.model.TableMetadata; +import org.apache.druid.catalog.model.TableSpec; +import org.apache.druid.error.DruidException; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.rpc.HttpResponseException; +import org.apache.druid.sql.calcite.planner.CatalogTableWriter; + +import javax.annotation.Nullable; +import javax.inject.Inject; +import java.util.List; +import java.util.Map; + +/** + * Applies catalog DDL by calling the Coordinator, which owns catalog metadata. + *

+ * The Coordinator is authoritative for validation, so the errors it reports are unwrapped and presented as the + * statement's own error rather than as a failed HTTP call. + */ +public class CatalogSqlTableWriter implements CatalogTableWriter +{ + private static final Logger LOG = new Logger(CatalogSqlTableWriter.class); + + private final CatalogClient client; + private final CachedMetadataCatalog cache; + private final ObjectMapper jsonMapper; + + @Inject + public CatalogSqlTableWriter( + final CatalogClient client, + final CachedMetadataCatalog cache, + final ObjectMapper jsonMapper + ) + { + this.client = client; + this.cache = cache; + this.jsonMapper = jsonMapper; + } + + @Override + public void createTable(TableId tableId, TableSpec spec, boolean ifNotExists, boolean replace) + { + execute(tableId, () -> client.createTable(tableId, spec, ifNotExists, replace)); + } + + @Override + public void updateColumns(TableId tableId, List columns) + { + execute(tableId, () -> client.editTable(tableId, new TableEditRequest.UpdateColumns(columns))); + } + + @Override + public void dropColumns(TableId tableId, List columns) + { + execute(tableId, () -> client.editTable(tableId, new TableEditRequest.DropColumns(columns))); + } + + @Override + public void updateProperties(TableId tableId, Map properties) + { + execute(tableId, () -> client.editTable(tableId, new TableEditRequest.UpdateProperties(properties))); + } + + @Override + public void addProjection(TableId tableId, DatasourceProjectionMetadata projection, boolean ifNotExists) + { + execute(tableId, () -> client.editTable(tableId, new TableEditRequest.AddProjection(projection, ifNotExists))); + } + + @Override + public void dropProjection(TableId tableId, String projectionName, boolean ifExists) + { + execute(tableId, () -> client.editTable(tableId, new TableEditRequest.DropProjection(projectionName, ifExists))); + } + + @Nullable + @Override + public TableMetadata readTable(TableId tableId) + { + return client.table(tableId); + } + + /** + * Run a catalog write, translate any failure into a statement error, then refresh this Broker's cache. + *

+ * The refresh matters because the Coordinator's update notification is asynchronous: without it, a CREATE TABLE + * followed immediately by an INSERT on the same connection could plan against the pre-DDL schema. Other Brokers + * still converge through the normal notification and polling path. + */ + private void execute(TableId tableId, Runnable operation) + { + try { + operation.run(); + } + catch (Exception e) { + throw translateError(tableId, e); + } + refreshCache(tableId); + } + + private void refreshCache(TableId tableId) + { + try { + final TableMetadata table = client.table(tableId); + cache.updated( + new UpdateEvent( + table == null ? UpdateEvent.EventType.DELETE : UpdateEvent.EventType.UPDATE, + table == null ? TableMetadata.empty(tableId) : table + ) + ); + } + catch (Exception e) { + // The write succeeded, so failing the statement here would be misleading. The cache converges on the next + // notification or poll; the only cost is that this Broker may briefly plan against the older spec. + LOG.warn(e, "Could not refresh the catalog cache for table[%s] after a DDL statement", tableId); + } + } + + /** + * Surface the Coordinator's own error message. Catalog validation lives there, so its wording is what explains + * why the statement was rejected; wrapping it in "server error" would bury it. + */ + private DruidException translateError(TableId tableId, Exception e) + { + final HttpResponseException httpError = findHttpResponseException(e); + if (httpError != null) { + final String message = errorMessage(httpError); + if (message != null) { + return DruidException.forPersona(DruidException.Persona.USER) + .ofCategory(DruidException.Category.INVALID_INPUT) + .build("%s", message); + } + } + if (e instanceof DruidException) { + return (DruidException) e; + } + return DruidException.forPersona(DruidException.Persona.USER) + .ofCategory(DruidException.Category.RUNTIME_FAILURE) + .build(e, "Could not update the catalog entry for table[%s]", tableId.name()); + } + + @Nullable + private String errorMessage(HttpResponseException e) + { + final String content = e.getResponse().getContent(); + if (content == null || content.isEmpty()) { + return null; + } + try { + final Map payload = jsonMapper.readValue(content, Map.class); + final Object message = payload.get(CatalogException.ERR_MSG_KEY); + return message == null ? null : message.toString(); + } + catch (Exception ignored) { + // Not a catalog error payload; fall back to the generic message. + return null; + } + } + + @Nullable + private static HttpResponseException findHttpResponseException(Throwable t) + { + for (Throwable cause : Throwables.getCausalChain(t)) { + if (cause instanceof HttpResponseException) { + return (HttpResponseException) cause; + } + } + return null; + } +} diff --git a/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/CatalogResourceTest.java b/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/CatalogResourceTest.java index 8f99eec43f56..623e53caeebd 100644 --- a/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/CatalogResourceTest.java +++ b/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/CatalogResourceTest.java @@ -102,48 +102,48 @@ public void testCreate() TableSpec dsSpec = TableBuilder.datasource(tableName, "P1D").buildSpec(); // Blank schema name: infer the schema. - Response resp = resource.postTable("", tableName, dsSpec, 0, false, postBy(CatalogTests.SUPER_USER)); + Response resp = resource.postTable("", tableName, dsSpec, 0, false, false, postBy(CatalogTests.SUPER_USER)); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp.getStatus()); // Missing table name - resp = resource.postTable(TableId.DRUID_SCHEMA, "", dsSpec, 0, false, postBy(CatalogTests.SUPER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, "", dsSpec, 0, false, false, postBy(CatalogTests.SUPER_USER)); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp.getStatus()); // Invalid table name - resp = resource.postTable(TableId.DRUID_SCHEMA, " bogus ", dsSpec, 0, false, postBy(CatalogTests.SUPER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, " bogus ", dsSpec, 0, false, false, postBy(CatalogTests.SUPER_USER)); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp.getStatus()); // Unknown schema - resp = resource.postTable("bogus", tableName, dsSpec, 0, false, postBy(CatalogTests.SUPER_USER)); + resp = resource.postTable("bogus", tableName, dsSpec, 0, false, false, postBy(CatalogTests.SUPER_USER)); assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp.getStatus()); // Immutable schema - resp = resource.postTable(TableId.CATALOG_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.SUPER_USER)); + resp = resource.postTable(TableId.CATALOG_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.SUPER_USER)); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp.getStatus()); // Wrong definition type. - resp = resource.postTable(TableId.EXTERNAL_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.SUPER_USER)); + resp = resource.postTable(TableId.EXTERNAL_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.SUPER_USER)); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp.getStatus()); // No permissions assertThrows( ForbiddenException.class, - () -> resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.DENY_USER)) + () -> resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.DENY_USER)) ); // Read permission assertThrows( ForbiddenException.class, - () -> resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.READER_USER)) + () -> resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.READER_USER)) ); // Write permission - resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); assertTrue(getVersion(resp) > 0); // Duplicate - resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp.getStatus()); // Invalid column type: table-level validation failures (which raise DruidException rather than IAE) must also @@ -151,7 +151,7 @@ public void testCreate() TableSpec badTypeSpec = TableBuilder.datasource("badType", "P1D") .column("foo", "FOO") .buildSpec(); - resp = resource.postTable(TableId.DRUID_SCHEMA, "badType", badTypeSpec, 0, false, postBy(CatalogTests.SUPER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, "badType", badTypeSpec, 0, false, false, postBy(CatalogTests.SUPER_USER)); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp.getStatus()); // Inline input source @@ -162,11 +162,11 @@ public void testCreate() .column("b", Columns.STRING) .column("c", Columns.LONG) .buildSpec(); - resp = resource.postTable(TableId.EXTERNAL_SCHEMA, "inline", inputSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.EXTERNAL_SCHEMA, "inline", inputSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); // Wrong spec type - resp = resource.postTable(TableId.DRUID_SCHEMA, "invalid", inputSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, "invalid", inputSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp.getStatus()); } @@ -187,36 +187,57 @@ public void testUpdate() TableSpec dsSpec = TableBuilder.datasource(tableName, "P1D").buildSpec(); // Does not exist - Response resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 10, false, postBy(CatalogTests.SUPER_USER)); + Response resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 10, false, false, postBy(CatalogTests.SUPER_USER)); assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp.getStatus()); // Create the table - resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); long version = getVersion(resp); // No update permission assertThrows( ForbiddenException.class, - () -> resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.READER_USER)) + () -> resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.READER_USER)) ); // Out-of-date version - resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 10, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 10, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp.getStatus()); // Valid version - resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, version, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, version, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); assertTrue(getVersion(resp) > version); version = getVersion(resp); // Overwrite - resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, true, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, true, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); assertTrue(getVersion(resp) > version); } + @Test + public void testCreateIfNotExists() + { + final String tableName = "ifNotExists"; + TableSpec dsSpec = TableBuilder.datasource(tableName, "P1D").buildSpec(); + + Response resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, true, postBy(CatalogTests.WRITER_USER)); + assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + final long version = getVersion(resp); + assertTrue(version > 0); + + // Creating again reports no change rather than failing, and leaves the existing spec in place. + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, true, postBy(CatalogTests.WRITER_USER)); + assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + assertEquals(0, getVersion(resp)); + + // Without the flag, a duplicate is still an error. + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); + assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp.getStatus()); + } + @Test public void testRead() { @@ -240,7 +261,7 @@ public void testRead() assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp.getStatus()); // Create the table - resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); long version = getVersion(resp); @@ -323,7 +344,7 @@ public void testGetSchemas() // Create a table final String tableName = "list"; TableSpec dsSpec = TableBuilder.datasource(tableName, "P1D").buildSpec(); - resp = resource.postTable(TableId.DRUID_SCHEMA, "list", dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, "list", dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); // Table paths - no read access @@ -370,7 +391,7 @@ public void testGetSchemaTables() // Create a table final String tableName = "list"; TableSpec dsSpec = TableBuilder.datasource(tableName, "P1D").buildSpec(); - resp = resource.postTable(TableId.DRUID_SCHEMA, "list", dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, "list", dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); // No read access - name @@ -416,7 +437,7 @@ public void testSync() { final String tableName = "sync"; TableSpec dsSpec = TableBuilder.datasource(tableName, "P1D").buildSpec(); - Response resp = resource.postTable(TableId.DRUID_SCHEMA, "list", dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + Response resp = resource.postTable(TableId.DRUID_SCHEMA, "list", dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); // Internal sync schema API @@ -456,7 +477,7 @@ public void testDelete() // Create the table TableSpec dsSpec = TableBuilder.datasource(tableName, "P1D").buildSpec(); - resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); // No write permission @@ -479,7 +500,7 @@ public void testLifecycle() // Operations for one table - create String table1Name = "lifecycle1"; TableSpec dsSpec = TableBuilder.datasource(table1Name, "P1D").buildSpec(); - Response resp = resource.postTable(TableId.DRUID_SCHEMA, table1Name, dsSpec, 0, true, postBy(CatalogTests.WRITER_USER)); + Response resp = resource.postTable(TableId.DRUID_SCHEMA, table1Name, dsSpec, 0, true, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); long version = getVersion(resp); @@ -507,7 +528,7 @@ public void testLifecycle() // update TableSpec table2Spec = TableBuilder.datasource(table1Name, "PT1H").buildSpec(); - resp = resource.postTable(TableId.DRUID_SCHEMA, table1Name, table2Spec, version, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, table1Name, table2Spec, version, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); assertTrue(getVersion(resp) > version); version = getVersion(resp); @@ -522,7 +543,7 @@ public void testLifecycle() // add second table String table2Name = "lifecycle2"; - resp = resource.postTable(TableId.DRUID_SCHEMA, table2Name, dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + resp = resource.postTable(TableId.DRUID_SCHEMA, table2Name, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); TableId id2 = TableId.of(TableId.DRUID_SCHEMA, table2Name); @@ -568,7 +589,7 @@ public void testMoveColumn() .column("b", "BIGINT") .column("c", "FLOAT") .buildSpec(); - Response resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + Response resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); long version = getVersion(resp); @@ -612,7 +633,7 @@ public void testHideColumns() String tableName = "hide"; TableSpec dsSpec = TableBuilder.datasource(tableName, "P1D") .buildSpec(); - Response resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + Response resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); long version = getVersion(resp); @@ -673,7 +694,7 @@ public void testDropColumns() .column("c", "FLOAT") .buildSpec(); - Response resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, postBy(CatalogTests.WRITER_USER)); + Response resp = resource.postTable(TableId.DRUID_SCHEMA, tableName, dsSpec, 0, false, false, postBy(CatalogTests.WRITER_USER)); assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); long version = getVersion(resp); diff --git a/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java b/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java index 107cedd565a9..18491161803d 100644 --- a/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java +++ b/extensions-core/druid-catalog/src/test/java/org/apache/druid/server/http/catalog/EditorTest.java @@ -24,7 +24,9 @@ import com.google.common.collect.ImmutableMap; import org.apache.druid.catalog.CatalogException; import org.apache.druid.catalog.http.TableEditRequest; +import org.apache.druid.catalog.http.TableEditRequest.AddProjection; import org.apache.druid.catalog.http.TableEditRequest.DropColumns; +import org.apache.druid.catalog.http.TableEditRequest.DropProjection; import org.apache.druid.catalog.http.TableEditRequest.HideColumns; import org.apache.druid.catalog.http.TableEditRequest.MoveColumn; import org.apache.druid.catalog.http.TableEditRequest.UnhideColumns; @@ -34,6 +36,7 @@ import org.apache.druid.catalog.model.CatalogUtils; import org.apache.druid.catalog.model.ColumnSpec; import org.apache.druid.catalog.model.Columns; +import org.apache.druid.catalog.model.DatasourceProjectionMetadata; import org.apache.druid.catalog.model.TableId; import org.apache.druid.catalog.model.TableMetadata; import org.apache.druid.catalog.model.table.ClusterKeySpec; @@ -41,8 +44,11 @@ import org.apache.druid.catalog.model.table.TableBuilder; import org.apache.druid.catalog.storage.CatalogStorage; import org.apache.druid.catalog.storage.CatalogTests; +import org.apache.druid.data.input.impl.AggregateProjectionSpec; +import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.metadata.TestDerbyConnector; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -57,6 +63,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; public class EditorTest { @@ -486,4 +493,55 @@ public void testUpdateColumns() throws CatalogException CatalogUtils.columnNames(revised.spec().columns()) ); } + + @Test + public void testAddAndDropProjection() throws CatalogException + { + final String tableName = "projections"; + final TableMetadata table = TableBuilder.datasource(tableName, "P1D") + .timeColumn() + .column("dim", "VARCHAR") + .column("met", "BIGINT") + .build(); + catalog.tables().create(table); + + final DatasourceProjectionMetadata daily = new DatasourceProjectionMetadata( + AggregateProjectionSpec.builder("daily") + .groupingColumns(new StringDimensionSchema("dim")) + .aggregators(new LongSumAggregatorFactory("sum_met", "met")) + .build() + ); + + assertTrue(new TableEditor(catalog, table.id(), new AddProjection(daily, false)).go() > 0); + assertEquals(List.of(daily), projectionsOf(tableName)); + + // Adding the same name again is an error, unless the caller said to leave it alone. + assertThrows( + CatalogException.class, + () -> new TableEditor(catalog, table.id(), new AddProjection(daily, false)).go() + ); + assertEquals(0, new TableEditor(catalog, table.id(), new AddProjection(daily, true)).go()); + assertEquals(List.of(daily), projectionsOf(tableName)); + + // Dropping a projection that is not there is likewise an error unless tolerated. + assertThrows( + CatalogException.class, + () -> new TableEditor(catalog, table.id(), new DropProjection("nope", false)).go() + ); + assertEquals(0, new TableEditor(catalog, table.id(), new DropProjection("nope", true)).go()); + + assertTrue(new TableEditor(catalog, table.id(), new DropProjection("daily", false)).go() > 0); + assertNull( + catalog.tables().read(TableId.datasource(tableName)) + .spec().properties().get(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY) + ); + } + + private List projectionsOf(String tableName) throws CatalogException + { + return catalog.tableRegistry() + .resolve(catalog.tables().read(TableId.datasource(tableName)).spec()) + .decodeProperty(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY); + } + } diff --git a/server/src/main/java/org/apache/druid/catalog/model/ColumnSpec.java b/server/src/main/java/org/apache/druid/catalog/model/ColumnSpec.java index cfc9fc55cf43..23f333b658da 100644 --- a/server/src/main/java/org/apache/druid/catalog/model/ColumnSpec.java +++ b/server/src/main/java/org/apache/druid/catalog/model/ColumnSpec.java @@ -27,6 +27,7 @@ import org.apache.druid.catalog.model.ModelProperties.PropertyDefn; import org.apache.druid.guice.annotations.UnstableApi; import org.apache.druid.java.util.common.IAE; +import org.apache.druid.segment.column.ColumnType; import javax.annotation.Nullable; @@ -104,11 +105,15 @@ public void validate() throw new IAE("Column name is required"); } if (Columns.isTimeColumn(name)) { - if (dataType != null && !Columns.LONG.equalsIgnoreCase(dataType)) { + // Any spelling that resolves to a LONG is fine: the time column is written as TIMESTAMP in SQL and as LONG + // natively, and both mean the same stored type. + if (dataType != null && !ColumnType.LONG.equals(Columns.druidTypeFromString(dataType))) { throw new IAE( - "[%s] column must have type [%s] or no type. Found [%s]", + "[%s] column must have a type that resolves to [%s], such as [%s] or [%s], or no type. Found [%s]", name, Columns.LONG, + Columns.SQL_TIMESTAMP, + Columns.SQL_BIGINT, dataType ); } diff --git a/server/src/main/java/org/apache/druid/catalog/model/Columns.java b/server/src/main/java/org/apache/druid/catalog/model/Columns.java index 9d64a4ed22e9..9be0d4ab3122 100644 --- a/server/src/main/java/org/apache/druid/catalog/model/Columns.java +++ b/server/src/main/java/org/apache/druid/catalog/model/Columns.java @@ -25,6 +25,7 @@ import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.ValueType; +import javax.annotation.Nullable; import java.util.List; import java.util.Map; @@ -76,12 +77,25 @@ private Columns() { } + /** + * The Druid type of a column, which for {@code __time} is always {@link ColumnType#LONG} whatever was declared. + * Use {@link #druidTypeFromString} to resolve a declared type on its own terms. + */ public static ColumnType druidType(ColumnSpec spec) { if (isTimeColumn(spec.name())) { return ColumnType.LONG; } - String dataType = spec.dataType(); + return druidTypeFromString(spec.dataType()); + } + + /** + * Resolve a declared type string to its Druid type: a SQL type name such as {@code BIGINT} or {@code VARCHAR ARRAY}, + * or a native type string such as {@code COMPLEX}. Returns null if the string is null or does not name a type. + */ + @Nullable + public static ColumnType druidTypeFromString(@Nullable String dataType) + { if (dataType == null) { return null; } diff --git a/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java b/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java index 89982a94883f..ae1b7a75f6e6 100644 --- a/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java +++ b/server/src/main/java/org/apache/druid/catalog/model/table/DatasourceDefn.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.catalog.model.CatalogUtils; import org.apache.druid.catalog.model.Columns; import org.apache.druid.catalog.model.DatasourceBaseTableMetadata; import org.apache.druid.catalog.model.DatasourceProjectionMetadata; @@ -30,12 +31,20 @@ import org.apache.druid.catalog.model.ResolvedTable; import org.apache.druid.catalog.model.TableDefn; import org.apache.druid.catalog.model.TableSpec; +import org.apache.druid.data.input.impl.AggregateProjectionSpec; +import org.apache.druid.data.input.impl.DimensionSchema; import org.apache.druid.error.InvalidInput; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.segment.VirtualColumn; +import org.apache.druid.segment.indexing.DataSchema; +import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; public class DatasourceDefn extends TableDefn { @@ -123,6 +132,75 @@ public void validate(ResolvedTable table) // fail fast instead of surfacing layout problems at ingest time. baseTable.createSpec(table.spec().columns()); } + validateProjections(table); + } + + /** + * Cross-validate the declared projections. Names must be unique, and a projection must not be coarser than the + * segments it lives in. For a sealed table the declared columns are the whole schema, so a projection that reads a + * column the table does not declare can never be built and is rejected; for a non-sealed table ingestion may add + * columns the catalog has not seen, so only the projections' internal consistency is checked. + */ + private void validateProjections(ResolvedTable table) + { + final List projections = table.decodeProperty(PROJECTIONS_KEYS_PROPERTY); + if (projections == null || projections.isEmpty()) { + return; + } + + final List specs = new ArrayList<>(projections.size()); + for (DatasourceProjectionMetadata projection : projections) { + if (projection == null || projection.getSpec() == null) { + throw InvalidInput.exception("Projections must each have a [spec]"); + } + specs.add(projection.getSpec()); + } + + final String granularity = table.stringProperty(SEGMENT_GRANULARITY_PROPERTY); + DataSchema.validateProjections( + specs, + granularity == null ? null : CatalogUtils.asDruidGranularity(granularity) + ); + + if (!table.booleanProperty(SEALED_PROPERTY) || table.spec().columns() == null) { + return; + } + final Set declared = new HashSet<>(CatalogUtils.columnNames(table.spec().columns())); + declared.add(Columns.TIME_COLUMN); + for (AggregateProjectionSpec spec : specs) { + final Set available = new HashSet<>(declared); + for (VirtualColumn virtualColumn : spec.getVirtualColumns().getVirtualColumns()) { + available.add(virtualColumn.getOutputName()); + } + for (String required : requiredColumns(spec)) { + if (!available.contains(required)) { + throw InvalidInput.exception( + "Projection [%s] references column [%s], which table [%s] does not declare", + spec.getName(), + required, + table.spec().type() + ); + } + } + } + } + + private static Set requiredColumns(AggregateProjectionSpec spec) + { + final Set required = new HashSet<>(); + for (VirtualColumn virtualColumn : spec.getVirtualColumns().getVirtualColumns()) { + required.addAll(virtualColumn.requiredColumns()); + } + for (DimensionSchema groupingColumn : spec.getGroupingColumns()) { + required.add(groupingColumn.getName()); + } + for (AggregatorFactory aggregator : spec.getAggregators()) { + required.addAll(aggregator.requiredFields()); + } + if (spec.getFilter() != null) { + required.addAll(spec.getFilter().getRequiredColumns()); + } + return required; } /** diff --git a/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java b/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java index ba32f61de7fc..3d409723785a 100644 --- a/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java +++ b/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java @@ -29,17 +29,22 @@ import org.apache.druid.catalog.model.ColumnSpec; import org.apache.druid.catalog.model.Columns; import org.apache.druid.catalog.model.DatasourceBaseTableMetadata; +import org.apache.druid.catalog.model.DatasourceProjectionMetadata; import org.apache.druid.catalog.model.ResolvedTable; import org.apache.druid.catalog.model.TableDefn; import org.apache.druid.catalog.model.TableDefnRegistry; import org.apache.druid.catalog.model.TableSpec; import org.apache.druid.catalog.model.facade.DatasourceFacade; import org.apache.druid.catalog.model.facade.DatasourceFacade.ColumnFacade; +import org.apache.druid.data.input.impl.AggregateProjectionSpec; +import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.error.DruidException; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; @@ -483,6 +488,73 @@ public void testColumns() } } + /** + * Projections hold polymorphic aggregators and dimension schemas, so validating them needs a mapper that knows + * those subtypes; the shared mapper in this class does not register them. + */ + @Test + public void testProjectionValidation() + { + final TableDefnRegistry projectionRegistry = new TableDefnRegistry(TestHelper.makeJsonMapper()); + final TableBuilder builder = TableBuilder.datasource("foo", "P1D") + .timeColumn() + .column("dim", Columns.SQL_VARCHAR) + .column("met", Columns.SQL_BIGINT); + + final AggregateProjectionSpec good = AggregateProjectionSpec + .builder("daily") + .groupingColumns(new StringDimensionSchema("dim")) + .aggregators(new LongSumAggregatorFactory("sum_met", "met")) + .build(); + + projectionRegistry.resolve( + builder.copy() + .property(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY, List.of(new DatasourceProjectionMetadata(good))) + .buildSpec() + ).validate(); + + // Two projections cannot share a name. + final TableSpec duplicateNames = + builder.copy() + .property( + DatasourceDefn.PROJECTIONS_KEYS_PROPERTY, + List.of(new DatasourceProjectionMetadata(good), new DatasourceProjectionMetadata(good)) + ) + .buildSpec(); + assertThrows(DruidException.class, () -> projectionRegistry.resolve(duplicateNames).validate()); + + // A sealed table declares its whole schema, so a projection over an undeclared column can never be built. + final AggregateProjectionSpec undeclared = AggregateProjectionSpec + .builder("bad") + .groupingColumns(new StringDimensionSchema("nope")) + .aggregators(new LongSumAggregatorFactory("sum_met", "met")) + .build(); + + final TableSpec sealedWithUndeclared = + builder.copy() + .sealed(true) + .property( + DatasourceDefn.PROJECTIONS_KEYS_PROPERTY, + List.of(new DatasourceProjectionMetadata(undeclared)) + ) + .buildSpec(); + final DruidException e = assertThrows( + DruidException.class, + () -> projectionRegistry.resolve(sealedWithUndeclared).validate() + ); + assertTrue(e.getMessage().contains("references column [nope]")); + + // Ingestion may add columns to a table that is not sealed, so the same projection is allowed there. + projectionRegistry.resolve( + builder.copy() + .property( + DatasourceDefn.PROJECTIONS_KEYS_PROPERTY, + List.of(new DatasourceProjectionMetadata(undeclared)) + ) + .buildSpec() + ).validate(); + } + @Test public void testTimeColumn() { @@ -521,9 +593,19 @@ public void testTimeColumn() assertSame(ColumnType.LONG, col.druidType()); } - { + // Any spelling that resolves to a LONG is accepted: TIMESTAMP is how SQL names the time column, BIGINT is its + // SQL storage type, and LONG is the native name. + for (String timeType : new String[]{Columns.SQL_TIMESTAMP, Columns.SQL_BIGINT, Columns.LONG, "long"}) { + TableSpec spec = builder.copy() + .column(Columns.TIME_COLUMN, timeType) + .buildSpec(); + registry.resolve(spec).validate(); + } + + // Types that do not resolve to a LONG are rejected, as are types that do not resolve at all. + for (String badType : new String[]{Columns.STRING, Columns.SQL_VARCHAR, Columns.SQL_DOUBLE, "NOT_A_TYPE"}) { TableSpec spec = builder.copy() - .column(Columns.TIME_COLUMN, Columns.STRING) + .column(Columns.TIME_COLUMN, badType) .buildSpec(); expectValidationFails(spec); } diff --git a/sql/src/main/codegen/config.fmpp b/sql/src/main/codegen/config.fmpp index 837c04f42a63..4f09afab3fb4 100644 --- a/sql/src/main/codegen/config.fmpp +++ b/sql/src/main/codegen/config.fmpp @@ -50,16 +50,25 @@ data: { imports: [ "java.util.List" "org.apache.calcite.sql.SqlNode" + "org.apache.calcite.sql.SqlCreate" "org.apache.calcite.sql.SqlInsert" "org.apache.calcite.sql.SqlNodeList" + "org.apache.calcite.sql.SqlSelect" + "org.apache.calcite.sql.SqlLiteral" + "org.apache.calcite.sql.parser.SqlParserPos" "org.apache.calcite.sql.SqlBasicCall" "org.apache.druid.java.util.common.granularity.Granularity" "org.apache.druid.java.util.common.granularity.GranularityType" "org.apache.druid.java.util.common.granularity.Granularities" + "org.apache.druid.sql.calcite.parser.DruidSqlAlterTable" + "org.apache.druid.sql.calcite.parser.DruidSqlColumnDeclaration" + "org.apache.druid.sql.calcite.parser.DruidSqlCreateTable" "org.apache.druid.sql.calcite.parser.DruidSqlInsert" "org.apache.druid.sql.calcite.parser.DruidSqlParserUtils" + "org.apache.druid.sql.calcite.parser.DruidSqlPropertyAssignment" "org.apache.druid.sql.calcite.external.ExtendOperator" "org.apache.druid.sql.calcite.external.ParameterizeOperator" + "org.apache.druid.sql.calcite.parser.SqlProjectionSpec" "org.apache.druid.sql.calcite.parser.ExternalDestinationSqlIdentifier" "java.util.HashMap" ] @@ -71,11 +80,19 @@ data: { "OVERWRITE" "PARTITIONED" "EXTERN" + "IF" + "PROPERTIES" + "PROJECTION" + "SEALED" ] nonReservedKeywordsToAdd: [ "OVERWRITE" "EXTERN" + "IF" + "PROPERTIES" + "PROJECTION" + "SEALED" ] # List of methods for parsing custom SQL statements. @@ -85,6 +102,14 @@ data: { "DruidSqlInsertEof()" "DruidSqlExplain()" "DruidSqlReplaceEof()" + "DruidSqlAlterTable()" + ] + + # CREATE reaches DruidSqlCreateTable through Calcite's stock SqlCreate() production, which supplies the + # (Span, boolean replace) arguments. DROP TABLE is deliberately absent: catalog-only deletion would be a + # surprising meaning for the statement, so it is left unclaimed until the semantics are settled. + createStatementParserMethods: [ + "DruidSqlCreateTable" ] # List of methods for parsing custom data types. @@ -100,6 +125,7 @@ data: { # "dataTypeParserMethods". implementationFiles: [ "common.ftl" + "ddl.ftl" "explain.ftl" "replace.ftl" ] diff --git a/sql/src/main/codegen/includes/ddl.ftl b/sql/src/main/codegen/includes/ddl.ftl new file mode 100644 index 000000000000..b941f53060af --- /dev/null +++ b/sql/src/main/codegen/includes/ddl.ftl @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Druid catalog DDL. These statements write catalog metadata only; they do not create or delete data. +// +// CREATE TABLE is reached through the standard Calcite SqlCreate() production, which has already consumed +// CREATE [OR REPLACE], so this production must not consume : the enclosing SqlStmtList() handles statement +// separators. ALTER TABLE is a top-level statement production instead, because Calcite's stock SqlAlter() mandates +// a SYSTEM or SESSION scope that does not apply here. + +SqlCreate DruidSqlCreateTable(Span s, boolean replace) : +{ + boolean ifNotExists = false; + final SqlIdentifier id; + final List columns = new ArrayList(); + final List projections = new ArrayList(); + Span elementSpan = null; + SqlGranularityLiteral partitionedBy = null; + SqlNodeList clusteredBy = null; + boolean sealed = false; +} +{ +

+ [ { ifNotExists = true; } ] + id = CompoundTableIdentifier() + [ + { elementSpan = span(); } + AddDruidTableElement(columns, projections) + ( + AddDruidTableElement(columns, projections) + )* + + ] + [ + + partitionedBy = PartitionGranularity() + ] + [ + clusteredBy = ClusteredBy() + ] + [ + { sealed = true; } + ] + { + final SqlParserPos elementPos = elementSpan == null ? s.pos() : elementSpan.end(this); + return new DruidSqlCreateTable( + s.end(this), + replace, + ifNotExists, + id, + new SqlNodeList(columns, elementPos), + new SqlNodeList(projections, elementPos), + partitionedBy, + clusteredBy, + sealed + ); + } +} + +// A table element is either a column declaration or a projection definition. A column may legitimately be named +// "projection" (the keyword is non-reserved) and may have a bare-identifier type, so two tokens are not enough to +// tell the two apart: a projection definition is distinguished by its third token, which is always '(' or AS. +void AddDruidTableElement(List columns, List projections) : +{ + final DruidSqlColumnDeclaration column; + final SqlProjectionSpec projection; +} +{ + LOOKAHEAD(3) + projection = DruidProjectionDefinition() + { + projections.add(projection); + } +| + column = DruidColumnDeclaration() + { + columns.add(column); + } +} + +// The body is a SELECT with no FROM: the table the projection belongs to is implicit. Only a select list, WHERE and +// GROUP BY are admitted; a projection has no way to express ordering, limits or having. +SqlProjectionSpec DruidProjectionDefinition() : +{ + final Span s; + final Span bodySpan; + final SqlIdentifier name; + final List keywords = new ArrayList(); + final SqlNodeList keywordList; + final List selectList = new ArrayList(); + SqlLiteral keyword = null; + final SqlNode where; + final SqlNodeList groupBy; + SqlNodeList clusteredBy = null; +} +{ + { s = span(); } + name = SimpleIdentifier() + [ ] + +
id = CompoundTableIdentifier() + ( + + ( + column = DruidColumnDeclaration() + { + return new DruidSqlAlterTable.AddColumn(s.end(this), id, column); + } + | + [ { ifNotExists = true; } ] projection = DruidProjectionDefinition() + { + return new DruidSqlAlterTable.AddProjection(s.end(this), id, projection, ifNotExists); + } + ) + | + + ( + columnName = SimpleIdentifier() + { + return new DruidSqlAlterTable.DropColumn(s.end(this), id, columnName); + } + | + [ { ifExists = true; } ] projectionName = SimpleIdentifier() + { + return new DruidSqlAlterTable.DropProjection(s.end(this), id, projectionName, ifExists); + } + ) + | + columnName = SimpleIdentifier() dataType = DataType() + { + return new DruidSqlAlterTable.AlterColumn( + s.end(this), + id, + new DruidSqlColumnDeclaration( + columnName.getParserPosition().plus(dataType.getParserPosition()), + columnName, + dataType + ) + ); + } + | + properties = DruidPropertyList() + { + return new DruidSqlAlterTable.SetProperties(s.end(this), id, properties); + } + ) +} + +SqlNodeList DruidPropertyList() : +{ + final Span s; + final List list = new ArrayList(); +} +{ + { s = span(); } + AddDruidPropertyAssignment(list) + ( + AddDruidPropertyAssignment(list) + )* + + { + return new SqlNodeList(list, s.end(this)); + } +} + +void AddDruidPropertyAssignment(List list) : +{ + final SqlIdentifier key; + final SqlNode value; +} +{ + key = SimpleIdentifier() value = Literal() + { + list.add( + new DruidSqlPropertyAssignment( + key.getParserPosition().plus(value.getParserPosition()), + key, + value + ) + ); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/external/Externals.java b/sql/src/main/java/org/apache/druid/sql/calcite/external/Externals.java index 53908dbc4705..dd29e5e0d376 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/external/Externals.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/external/Externals.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; -import org.apache.calcite.avatica.SqlType; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.schema.FunctionParameter; @@ -32,10 +31,8 @@ import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlOperandCountRange; import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.SqlTypeNameSpec; import org.apache.calcite.sql.type.SqlOperandCountRanges; import org.apache.calcite.sql.type.SqlOperandMetadata; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.catalog.model.ColumnSpec; import org.apache.druid.catalog.model.table.ExternalTableSpec; import org.apache.druid.catalog.model.table.TableFunction; @@ -48,6 +45,7 @@ import org.apache.druid.server.security.Resource; import org.apache.druid.server.security.ResourceAction; import org.apache.druid.server.security.ResourceType; +import org.apache.druid.sql.calcite.planner.CatalogColumnTypes; import org.apache.druid.sql.calcite.planner.DruidTypeSystem; import org.apache.druid.sql.calcite.table.ExternalTable; @@ -232,7 +230,7 @@ public static List convertColumns(SqlNodeList schema) final List columns = new ArrayList<>(); for (int i = 0; i < schema.size(); i += 2) { final String name = convertName((SqlIdentifier) schema.get(i)); - final String sqlType = convertType(name, (SqlDataTypeSpec) schema.get(i + 1)); + final String sqlType = CatalogColumnTypes.forExternalColumn(name, (SqlDataTypeSpec) schema.get(i + 1)); columns.add(new ColumnSpec(name, sqlType, null)); } return columns; @@ -254,66 +252,6 @@ private static String convertName(SqlIdentifier ident) return ident.getSimple(); } - /** - * Define the SQL input column type from a type provided in the - * EXTEND clause. Calcite allows any form of type. But, Druid - * requires only the Druid supported types (and their aliases.) - *

- * Druid has its own rules for nullability. We ignore any nullability - * clause in the EXTEND list. - */ - private static String convertType(String name, SqlDataTypeSpec dataType) - { - SqlTypeNameSpec spec = dataType.getTypeNameSpec(); - if (spec == null) { - throw unsupportedType(name, dataType); - } - SqlIdentifier typeNameIdentifier = spec.getTypeName(); - if (typeNameIdentifier == null || !typeNameIdentifier.isSimple()) { - throw unsupportedType(name, dataType); - } - String simpleName = typeNameIdentifier.getSimple(); - if (StringUtils.toLowerCase(simpleName).startsWith("complex<")) { - // Parse and validate rather than passing the raw string downstream, where a malformed type string would - // silently resolve to a different type; return the canonical form. - final ColumnType complexType = ColumnType.fromString(simpleName); - if (complexType == null) { - throw unsupportedType(name, dataType); - } - return complexType.asTypeString(); - } - SqlTypeName type = SqlTypeName.get(simpleName); - if (type == null) { - throw unsupportedType(name, dataType); - } - if (SqlTypeName.CHAR_TYPES.contains(type)) { - return SqlTypeName.VARCHAR.name(); - } - if (SqlTypeName.INT_TYPES.contains(type)) { - return SqlTypeName.BIGINT.name(); - } - switch (type) { - case DOUBLE: - return SqlType.DOUBLE.name(); - case FLOAT: - case REAL: - return SqlType.FLOAT.name(); - case ARRAY: - return convertType(name, dataType.getComponentTypeSpec()) + " " + SqlType.ARRAY.name(); - default: - throw unsupportedType(name, dataType); - } - } - - private static RuntimeException unsupportedType(String name, SqlDataTypeSpec dataType) - { - return new IAE(StringUtils.format( - "Column [%s] has an unsupported type: [%s]", - name, - dataType - )); - } - /** * Create an MSQ ExternalTable given an external table spec. Enforces type restructions * (which should be revisited.) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlAlterTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlAlterTable.java new file mode 100644 index 000000000000..d9cadd2d4c12 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlAlterTable.java @@ -0,0 +1,466 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.parser; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlSpecialOperator; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.util.ImmutableNullableList; + +import javax.annotation.Nonnull; +import java.util.List; + +/** + * {@code ALTER TABLE }, which edits the catalog metadata of an existing table. + *

+ * Each concrete subclass corresponds to exactly one catalog edit operation, so that a single statement is always a + * single atomic change on the Coordinator. See {@link org.apache.druid.sql.calcite.planner.CatalogDdlHandler}. + */ +public abstract class DruidSqlAlterTable extends SqlCall +{ + private final SqlIdentifier name; + + protected DruidSqlAlterTable(SqlParserPos pos, SqlIdentifier name) + { + super(pos); + this.name = name; + } + + public SqlIdentifier getName() + { + return name; + } + + @Override + public void unparse(SqlWriter writer, int leftPrec, int rightPrec) + { + writer.keyword("ALTER TABLE"); + name.unparse(writer, leftPrec, rightPrec); + unparseOperation(writer, leftPrec, rightPrec); + } + + /** + * Unparse the portion of the statement following the table name. + */ + protected abstract void unparseOperation(SqlWriter writer, int leftPrec, int rightPrec); + + /** + * {@code ALTER TABLE ADD COLUMN }. + */ + public static class AddColumn extends DruidSqlAlterTable + { + public static final SqlOperator OPERATOR = new Operator("ALTER TABLE ADD COLUMN"); + + private final DruidSqlColumnDeclaration column; + + public AddColumn(SqlParserPos pos, SqlIdentifier name, DruidSqlColumnDeclaration column) + { + super(pos, name); + this.column = column; + } + + public DruidSqlColumnDeclaration getColumn() + { + return column; + } + + @Nonnull + @Override + public SqlOperator getOperator() + { + return OPERATOR; + } + + @Nonnull + @Override + public List getOperandList() + { + return ImmutableNullableList.of(getName(), column); + } + + @Override + protected void unparseOperation(SqlWriter writer, int leftPrec, int rightPrec) + { + writer.keyword("ADD COLUMN"); + column.unparse(writer, 0, 0); + } + + private static class Operator extends SqlSpecialOperator + { + Operator(String name) + { + super(name, SqlKind.ALTER_TABLE); + } + + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) + { + return new AddColumn(pos, (SqlIdentifier) operands[0], (DruidSqlColumnDeclaration) operands[1]); + } + } + } + + /** + * {@code ALTER TABLE DROP COLUMN }. Removes the column from the catalog spec; existing segments are + * unaffected. + */ + public static class DropColumn extends DruidSqlAlterTable + { + public static final SqlOperator OPERATOR = new Operator("ALTER TABLE DROP COLUMN"); + + private final SqlIdentifier column; + + public DropColumn(SqlParserPos pos, SqlIdentifier name, SqlIdentifier column) + { + super(pos, name); + this.column = column; + } + + public SqlIdentifier getColumn() + { + return column; + } + + @Nonnull + @Override + public SqlOperator getOperator() + { + return OPERATOR; + } + + @Nonnull + @Override + public List getOperandList() + { + return ImmutableNullableList.of(getName(), column); + } + + @Override + protected void unparseOperation(SqlWriter writer, int leftPrec, int rightPrec) + { + writer.keyword("DROP COLUMN"); + column.unparse(writer, 0, 0); + } + + private static class Operator extends SqlSpecialOperator + { + Operator(String name) + { + super(name, SqlKind.ALTER_TABLE); + } + + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) + { + return new DropColumn(pos, (SqlIdentifier) operands[0], (SqlIdentifier) operands[1]); + } + } + } + + /** + * {@code ALTER TABLE ALTER COLUMN SET DATA TYPE }. + */ + public static class AlterColumn extends DruidSqlAlterTable + { + public static final SqlOperator OPERATOR = new Operator("ALTER TABLE ALTER COLUMN"); + + private final DruidSqlColumnDeclaration column; + + public AlterColumn(SqlParserPos pos, SqlIdentifier name, DruidSqlColumnDeclaration column) + { + super(pos, name); + this.column = column; + } + + public DruidSqlColumnDeclaration getColumn() + { + return column; + } + + @Nonnull + @Override + public SqlOperator getOperator() + { + return OPERATOR; + } + + @Nonnull + @Override + public List getOperandList() + { + return ImmutableNullableList.of(getName(), column); + } + + @Override + protected void unparseOperation(SqlWriter writer, int leftPrec, int rightPrec) + { + writer.keyword("ALTER COLUMN"); + column.getName().unparse(writer, 0, 0); + writer.keyword("SET DATA TYPE"); + DruidSqlColumnDeclaration.unparseDataType(writer, column.getDataType()); + } + + private static class Operator extends SqlSpecialOperator + { + Operator(String name) + { + super(name, SqlKind.ALTER_TABLE); + } + + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) + { + return new AlterColumn(pos, (SqlIdentifier) operands[0], (DruidSqlColumnDeclaration) operands[1]); + } + } + } + + /** + * {@code ALTER TABLE ADD [IF NOT EXISTS] PROJECTION AS ( ... )}. + */ + public static class AddProjection extends DruidSqlAlterTable + { + public static final SqlOperator OPERATOR = new Operator("ALTER TABLE ADD PROJECTION"); + + private final SqlProjectionSpec projection; + private final boolean ifNotExists; + + public AddProjection( + SqlParserPos pos, + SqlIdentifier name, + SqlProjectionSpec projection, + boolean ifNotExists + ) + { + super(pos, name); + this.projection = projection; + this.ifNotExists = ifNotExists; + } + + public SqlProjectionSpec getProjection() + { + return projection; + } + + public boolean isIfNotExists() + { + return ifNotExists; + } + + @Nonnull + @Override + public SqlOperator getOperator() + { + return OPERATOR; + } + + @Nonnull + @Override + public List getOperandList() + { + return ImmutableNullableList.of( + getName(), + projection, + SqlLiteral.createBoolean(ifNotExists, SqlParserPos.ZERO) + ); + } + + @Override + protected void unparseOperation(SqlWriter writer, int leftPrec, int rightPrec) + { + writer.keyword("ADD"); + if (ifNotExists) { + writer.keyword("IF NOT EXISTS"); + } + projection.unparse(writer, 0, 0); + } + + private static class Operator extends SqlSpecialOperator + { + Operator(String name) + { + super(name, SqlKind.ALTER_TABLE); + } + + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) + { + return new AddProjection( + pos, + (SqlIdentifier) operands[0], + (SqlProjectionSpec) operands[1], + ((SqlLiteral) operands[2]).booleanValue() + ); + } + } + } + + /** + * {@code ALTER TABLE DROP PROJECTION [IF EXISTS] }. Existing segments keep whatever projections they + * were built with; this only stops future ingestion from building it. + */ + public static class DropProjection extends DruidSqlAlterTable + { + public static final SqlOperator OPERATOR = new Operator("ALTER TABLE DROP PROJECTION"); + + private final SqlIdentifier projectionName; + private final boolean ifExists; + + public DropProjection( + SqlParserPos pos, + SqlIdentifier name, + SqlIdentifier projectionName, + boolean ifExists + ) + { + super(pos, name); + this.projectionName = projectionName; + this.ifExists = ifExists; + } + + public SqlIdentifier getProjectionName() + { + return projectionName; + } + + public boolean isIfExists() + { + return ifExists; + } + + @Nonnull + @Override + public SqlOperator getOperator() + { + return OPERATOR; + } + + @Nonnull + @Override + public List getOperandList() + { + return ImmutableNullableList.of( + getName(), + projectionName, + SqlLiteral.createBoolean(ifExists, SqlParserPos.ZERO) + ); + } + + @Override + protected void unparseOperation(SqlWriter writer, int leftPrec, int rightPrec) + { + writer.keyword("DROP PROJECTION"); + if (ifExists) { + writer.keyword("IF EXISTS"); + } + projectionName.unparse(writer, 0, 0); + } + + private static class Operator extends SqlSpecialOperator + { + Operator(String name) + { + super(name, SqlKind.ALTER_TABLE); + } + + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) + { + return new DropProjection( + pos, + (SqlIdentifier) operands[0], + (SqlIdentifier) operands[1], + ((SqlLiteral) operands[2]).booleanValue() + ); + } + } + } + + /** + * {@code ALTER TABLE SET PROPERTIES ( = , ...)}. A {@code NULL} value removes the property. + */ + public static class SetProperties extends DruidSqlAlterTable + { + public static final SqlOperator OPERATOR = new Operator("ALTER TABLE SET PROPERTIES"); + + private final SqlNodeList properties; + + public SetProperties(SqlParserPos pos, SqlIdentifier name, SqlNodeList properties) + { + super(pos, name); + this.properties = properties; + } + + /** + * The property assignments, each a {@link DruidSqlPropertyAssignment}. + */ + public SqlNodeList getProperties() + { + return properties; + } + + @Nonnull + @Override + public SqlOperator getOperator() + { + return OPERATOR; + } + + @Nonnull + @Override + public List getOperandList() + { + return ImmutableNullableList.of(getName(), properties); + } + + @Override + protected void unparseOperation(SqlWriter writer, int leftPrec, int rightPrec) + { + writer.keyword("SET PROPERTIES"); + final SqlWriter.Frame frame = writer.startList("(", ")"); + for (SqlNode property : properties) { + writer.sep(","); + property.unparse(writer, 0, 0); + } + writer.endList(frame); + } + + private static class Operator extends SqlSpecialOperator + { + Operator(String name) + { + super(name, SqlKind.ALTER_TABLE); + } + + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) + { + return new SetProperties(pos, (SqlIdentifier) operands[0], (SqlNodeList) operands[1]); + } + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlColumnDeclaration.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlColumnDeclaration.java new file mode 100644 index 000000000000..de700e8d66e3 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlColumnDeclaration.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.parser; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlSpecialOperator; +import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.util.ImmutableNullableList; + +import javax.annotation.Nonnull; +import java.util.List; + +/** + * A single {@code } column declaration within a Druid DDL statement, such as the column list of + * {@code CREATE TABLE} or the target of {@code ALTER TABLE ... ADD COLUMN}. + */ +public class DruidSqlColumnDeclaration extends SqlCall +{ + public static final SqlOperator OPERATOR = new DruidSqlColumnDeclarationOperator(); + + /** + * The {@code TYPE('...')} escape hatch used to name Druid native types that have no SQL spelling, defined by the + * {@code DruidType()} production in {@code common.ftl}. + */ + public static final String COMPLEX_TYPE_FUNCTION = "TYPE"; + + private final SqlIdentifier name; + private final SqlDataTypeSpec dataType; + + public DruidSqlColumnDeclaration( + SqlParserPos pos, + SqlIdentifier name, + SqlDataTypeSpec dataType + ) + { + super(pos); + this.name = name; + this.dataType = dataType; + } + + public SqlIdentifier getName() + { + return name; + } + + public SqlDataTypeSpec getDataType() + { + return dataType; + } + + @Nonnull + @Override + public SqlOperator getOperator() + { + return OPERATOR; + } + + @Nonnull + @Override + public List getOperandList() + { + return ImmutableNullableList.of(name, dataType); + } + + @Override + public void unparse(SqlWriter writer, int leftPrec, int rightPrec) + { + name.unparse(writer, 0, 0); + unparseDataType(writer, dataType); + } + + /** + * Unparse a column type. Types named by {@link #COMPLEX_TYPE_FUNCTION} (the {@code TYPE('...')} escape hatch used + * for Druid native types such as {@code COMPLEX}) are written back in that form, since + * {@link SqlUserDefinedTypeNameSpec} would otherwise emit the bare name, which does not parse. + */ + public static void unparseDataType(SqlWriter writer, SqlDataTypeSpec dataType) + { + if (dataType.getTypeNameSpec() instanceof SqlUserDefinedTypeNameSpec) { + writer.keyword(COMPLEX_TYPE_FUNCTION); + final SqlWriter.Frame frame = writer.startList(SqlWriter.FrameTypeEnum.FUN_CALL, "(", ")"); + SqlLiteral.createCharString(dataType.getTypeName().toString(), dataType.getParserPosition()) + .unparse(writer, 0, 0); + writer.endList(frame); + } else { + dataType.unparse(writer, 0, 0); + } + } + + private static class DruidSqlColumnDeclarationOperator extends SqlSpecialOperator + { + public DruidSqlColumnDeclarationOperator() + { + super("COLUMN_DECL", SqlKind.COLUMN_DECL); + } + + @Override + public SqlCall createCall( + SqlLiteral functionQualifier, + SqlParserPos pos, + SqlNode... operands + ) + { + return new DruidSqlColumnDeclaration(pos, (SqlIdentifier) operands[0], (SqlDataTypeSpec) operands[1]); + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java new file mode 100644 index 000000000000..a9c9e6c77c80 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlCreateTable.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.parser; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlCreate; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlSpecialOperator; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.util.ImmutableNullableList; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; + +/** + * {@code CREATE [OR REPLACE] TABLE [IF NOT EXISTS] () [PARTITIONED BY ] + * [CLUSTERED BY ]}, which defines a table in the Druid catalog. + *

+ * This statement writes catalog metadata only; it neither creates segments nor otherwise touches data. See + * {@link org.apache.druid.sql.calcite.planner.CatalogDdlHandler} for the execution side. + */ +public class DruidSqlCreateTable extends SqlCreate +{ + public static final SqlOperator OPERATOR = new DruidSqlCreateTableOperator(); + + private final SqlIdentifier name; + private final SqlNodeList columnList; + private final SqlNodeList projectionList; + @Nullable + private final SqlGranularityLiteral partitionedBy; + @Nullable + private final SqlNodeList clusteredBy; + private final boolean sealed; + + public DruidSqlCreateTable( + SqlParserPos pos, + boolean replace, + boolean ifNotExists, + SqlIdentifier name, + SqlNodeList columnList, + SqlNodeList projectionList, + @Nullable SqlGranularityLiteral partitionedBy, + @Nullable SqlNodeList clusteredBy, + boolean sealed + ) + { + super(OPERATOR, pos, replace, ifNotExists); + this.sealed = sealed; + this.name = name; + this.columnList = columnList; + this.projectionList = projectionList; + this.partitionedBy = partitionedBy; + this.clusteredBy = clusteredBy; + } + + public SqlIdentifier getName() + { + return name; + } + + /** + * The declared columns, each a {@link DruidSqlColumnDeclaration}. Order is significant: it is the order columns are + * recorded in the catalog table spec. + */ + public SqlNodeList getColumnList() + { + return columnList; + } + + /** + * The declared projections, each a {@link SqlProjectionSpec}. + */ + public SqlNodeList getProjectionList() + { + return projectionList; + } + + @Nullable + public SqlGranularityLiteral getPartitionedBy() + { + return partitionedBy; + } + + @Nullable + public SqlNodeList getClusteredBy() + { + return clusteredBy; + } + + public boolean isIfNotExists() + { + return ifNotExists; + } + + /** + * Whether the statement declared SEALED, which requires every ingested column to be declared. + */ + public boolean isSealed() + { + return sealed; + } + + @Nonnull + @Override + public List getOperandList() + { + // The replace and ifNotExists flags travel as operands so that createCall() can rebuild an equivalent node. + return ImmutableNullableList.of( + name, + columnList, + projectionList, + partitionedBy, + clusteredBy, + SqlLiteral.createBoolean(getReplace(), SqlParserPos.ZERO), + SqlLiteral.createBoolean(ifNotExists, SqlParserPos.ZERO), + SqlLiteral.createBoolean(sealed, SqlParserPos.ZERO) + ); + } + + @Override + public void unparse(SqlWriter writer, int leftPrec, int rightPrec) + { + writer.keyword("CREATE"); + if (getReplace()) { + writer.keyword("OR REPLACE"); + } + writer.keyword("TABLE"); + if (ifNotExists) { + writer.keyword("IF NOT EXISTS"); + } + name.unparse(writer, leftPrec, rightPrec); + + final SqlWriter.Frame frame = writer.startList("(", ")"); + for (SqlNode column : columnList) { + writer.sep(","); + column.unparse(writer, 0, 0); + } + for (SqlNode projection : projectionList) { + writer.sep(","); + projection.unparse(writer, 0, 0); + } + writer.endList(frame); + + if (partitionedBy != null) { + writer.keyword("PARTITIONED BY"); + partitionedBy.unparse(writer, 0, 0); + } + + if (clusteredBy != null) { + writer.keyword("CLUSTERED BY"); + final SqlWriter.Frame clusterFrame = writer.startList("", ""); + for (SqlNode clusterByOpts : clusteredBy.getList()) { + writer.sep(","); + clusterByOpts.unparse(writer, leftPrec, rightPrec); + } + writer.endList(clusterFrame); + } + + if (sealed) { + writer.keyword("SEALED"); + } + } + + private static class DruidSqlCreateTableOperator extends SqlSpecialOperator + { + public DruidSqlCreateTableOperator() + { + super("CREATE TABLE", SqlKind.CREATE_TABLE); + } + + @Override + public SqlCall createCall( + SqlLiteral functionQualifier, + SqlParserPos pos, + SqlNode... operands + ) + { + return new DruidSqlCreateTable( + pos, + ((SqlLiteral) operands[5]).booleanValue(), + ((SqlLiteral) operands[6]).booleanValue(), + (SqlIdentifier) operands[0], + (SqlNodeList) operands[1], + (SqlNodeList) operands[2], + (SqlGranularityLiteral) operands[3], + (SqlNodeList) operands[4], + ((SqlLiteral) operands[7]).booleanValue() + ); + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlParser.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlParser.java index ab9e9e1878b9..da2fccb62d41 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlParser.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlParser.java @@ -149,6 +149,17 @@ private static StatementAndSetContext processStatementList( */ @Nullable static Object sqlLiteralToContextValue(final SqlLiteral literal) + { + return sqlLiteralToJavaValue(literal, "SET"); + } + + /** + * Coerces a SQL literal to a plain Java value, as used for query context entries and catalog table properties. + * + * @param what the clause the literal came from, named in the error message if its type has no Java equivalent + */ + @Nullable + public static Object sqlLiteralToJavaValue(final SqlLiteral literal, final String what) { if (SqlUtil.isNullLiteral(literal, false)) { return null; @@ -172,7 +183,7 @@ static Object sqlLiteralToContextValue(final SqlLiteral literal) } else if (literal.getTypeName() == SqlTypeName.TIMESTAMP) { return Calcites.CALCITE_TIMESTAMP_PARSER.parse(literal.getValue().toString()).toString(); } else { - throw InvalidSqlInput.exception("Unsupported type for SET[%s]", literal.getTypeName()); + throw InvalidSqlInput.exception("Unsupported type for %s[%s]", what, literal.getTypeName()); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlPropertyAssignment.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlPropertyAssignment.java new file mode 100644 index 000000000000..e24f069a2904 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/DruidSqlPropertyAssignment.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.parser; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlSpecialOperator; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.util.ImmutableNullableList; + +import javax.annotation.Nonnull; +import java.util.List; + +/** + * A single {@code = } pair within {@code ALTER TABLE ... SET PROPERTIES (...)}. The value is a literal; + * a {@code NULL} literal means "remove this property". + */ +public class DruidSqlPropertyAssignment extends SqlCall +{ + public static final SqlOperator OPERATOR = new DruidSqlPropertyAssignmentOperator(); + + private final SqlIdentifier key; + private final SqlNode value; + + public DruidSqlPropertyAssignment(SqlParserPos pos, SqlIdentifier key, SqlNode value) + { + super(pos); + this.key = key; + this.value = value; + } + + public SqlIdentifier getKey() + { + return key; + } + + /** + * The assigned value. Normally a {@link SqlLiteral}, but a multi-line string literal parses as a concatenation + * call, so callers must handle a non-literal node rather than assume the cast succeeds. + */ + public SqlNode getValue() + { + return value; + } + + @Nonnull + @Override + public SqlOperator getOperator() + { + return OPERATOR; + } + + @Nonnull + @Override + public List getOperandList() + { + return ImmutableNullableList.of(key, value); + } + + @Override + public void unparse(SqlWriter writer, int leftPrec, int rightPrec) + { + key.unparse(writer, 0, 0); + writer.keyword("="); + value.unparse(writer, 0, 0); + } + + private static class DruidSqlPropertyAssignmentOperator extends SqlSpecialOperator + { + public DruidSqlPropertyAssignmentOperator() + { + super("PROPERTY_ASSIGNMENT", SqlKind.OTHER); + } + + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) + { + return new DruidSqlPropertyAssignment(pos, (SqlIdentifier) operands[0], operands[1]); + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/parser/SqlProjectionSpec.java b/sql/src/main/java/org/apache/druid/sql/calcite/parser/SqlProjectionSpec.java new file mode 100644 index 000000000000..8cfa53cc733b --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/parser/SqlProjectionSpec.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.parser; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.SqlSpecialOperator; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.util.ImmutableNullableList; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; + +/** + * A {@code PROJECTION AS ( SELECT ... )} clause, which defines a projection of the table it appears in. + *

+ * The body is a {@link SqlSelect} with no FROM clause: the table is implicit, and the grammar admits only a select + * list, an optional WHERE and an optional GROUP BY. ORDER BY, LIMIT, HAVING, joins and set operations are + * structurally excluded rather than validated away, because a projection cannot express them: its ordering is + * derived from its grouping columns. + */ +public class SqlProjectionSpec extends SqlCall +{ + public static final SqlOperator OPERATOR = new SqlProjectionSpecOperator(); + + private final SqlIdentifier name; + @Nullable + private final SqlNodeList clusteredBy; + private final SqlSelect body; + + public SqlProjectionSpec( + SqlParserPos pos, + SqlIdentifier name, + @Nullable SqlNodeList clusteredBy, + SqlSelect body + ) + { + super(pos); + this.name = name; + this.clusteredBy = clusteredBy; + this.body = body; + } + + /** + * The columns segments are clustered on, meaningful only for the reserved base-table projection: an aggregate + * projection is ordered by its grouping columns and has nothing to choose. + */ + @Nullable + public SqlNodeList getClusteredBy() + { + return clusteredBy; + } + + public SqlIdentifier getName() + { + return name; + } + + public SqlSelect getBody() + { + return body; + } + + @Nonnull + @Override + public SqlOperator getOperator() + { + return OPERATOR; + } + + @Nonnull + @Override + public List getOperandList() + { + return ImmutableNullableList.of(name, clusteredBy, body); + } + + @Override + public void unparse(SqlWriter writer, int leftPrec, int rightPrec) + { + writer.keyword("PROJECTION"); + name.unparse(writer, 0, 0); + writer.keyword("AS"); + final SqlWriter.Frame frame = writer.startList("(", ")"); + body.unparse(writer, 0, 0); + if (clusteredBy != null) { + writer.keyword("CLUSTERED BY"); + final SqlWriter.Frame clusterFrame = writer.startList("", ""); + for (SqlNode column : clusteredBy) { + writer.sep(","); + column.unparse(writer, 0, 0); + } + writer.endList(clusterFrame); + } + writer.endList(frame); + } + + private static class SqlProjectionSpecOperator extends SqlSpecialOperator + { + public SqlProjectionSpecOperator() + { + super("PROJECTION", SqlKind.OTHER); + } + + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) + { + return new SqlProjectionSpec( + pos, + (SqlIdentifier) operands[0], + (SqlNodeList) operands[1], + (SqlSelect) operands[2] + ); + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogColumnTypes.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogColumnTypes.java new file mode 100644 index 000000000000..e7f3fa3e4bcd --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogColumnTypes.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.planner; + +import org.apache.calcite.avatica.SqlType; +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlTypeNameSpec; +import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.druid.catalog.model.Columns; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.segment.column.ColumnType; + +/** + * Converts a SQL type as written in a statement into the type string stored in the catalog. Calcite accepts any + * type spelling; Druid accepts only its own supported types and their aliases, plus the {@code TYPE('...')} escape + * hatch for native type strings that have no SQL spelling, such as {@code COMPLEX}. + *

+ * Druid has its own rules for nullability, so any nullability clause is ignored. + */ +public class CatalogColumnTypes +{ + /** + * Which statement the type was written in. The two differ in what they accept, so they are named rather than + * flagged. + */ + private enum Target + { + /** + * The {@code EXTEND} clause, describing columns read from an external input source. Those are read as their + * underlying storage type, so {@code TIMESTAMP} is not among them, and the escape hatch is limited to complex + * types. + */ + EXTERNAL, + + /** + * A catalog DDL statement, which additionally accepts {@code TIMESTAMP} (how {@code __time} is spelled in SQL) + * and any native type string through the escape hatch. + */ + CATALOG + } + + private CatalogColumnTypes() + { + // No instantiation. + } + + public static String forExternalColumn(String name, SqlDataTypeSpec dataType) + { + return convert(name, dataType, Target.EXTERNAL); + } + + public static String forCatalogColumn(String name, SqlDataTypeSpec dataType) + { + final String typeString = convert(name, dataType, Target.CATALOG); + // the catalog rejects unparseable types at write time, but catching it here attributes the error to the statement + // rather than to a Coordinator round trip. + if (Columns.druidTypeFromString(typeString) == null) { + throw unsupportedType(name, dataType); + } + return typeString; + } + + private static String convert(String name, SqlDataTypeSpec dataType, Target target) + { + final SqlTypeNameSpec spec = dataType.getTypeNameSpec(); + if (spec == null) { + throw unsupportedType(name, dataType); + } + final SqlIdentifier typeNameIdentifier = spec.getTypeName(); + if (typeNameIdentifier == null || !typeNameIdentifier.isSimple()) { + throw unsupportedType(name, dataType); + } + final String simpleName = typeNameIdentifier.getSimple(); + + if (spec instanceof SqlUserDefinedTypeNameSpec) { + // The TYPE('...') escape hatch names a Druid native type. Parse and validate rather than passing the raw + // string downstream, where a malformed type string would silently resolve to a different type, and return the + // canonical form. + if (target == Target.EXTERNAL && !StringUtils.toLowerCase(simpleName).startsWith("complex<")) { + throw unsupportedType(name, dataType); + } + final ColumnType nativeType = ColumnType.fromString(simpleName); + if (nativeType == null) { + throw unsupportedType(name, dataType); + } + return nativeType.asTypeString(); + } + + final SqlTypeName type = SqlTypeName.get(simpleName); + if (type == null) { + throw unsupportedType(name, dataType); + } + if (SqlTypeName.CHAR_TYPES.contains(type)) { + return SqlTypeName.VARCHAR.name(); + } + if (SqlTypeName.INT_TYPES.contains(type)) { + return SqlTypeName.BIGINT.name(); + } + switch (type) { + case DOUBLE: + return SqlType.DOUBLE.name(); + case FLOAT: + case REAL: + return SqlType.FLOAT.name(); + case ARRAY: + return convert(name, dataType.getComponentTypeSpec(), target) + " " + SqlType.ARRAY.name(); + case TIMESTAMP: + if (target == Target.CATALOG) { + return Columns.SQL_TIMESTAMP; + } + throw unsupportedType(name, dataType); + default: + throw unsupportedType(name, dataType); + } + } + + private static RuntimeException unsupportedType(String name, SqlDataTypeSpec dataType) + { + return new IAE(StringUtils.format( + "Column [%s] has an unsupported type: [%s]", + name, + dataType + )); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java new file mode 100644 index 000000000000..c12cc826ae40 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java @@ -0,0 +1,736 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.planner; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import org.apache.calcite.jdbc.CalciteSchema; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.druid.catalog.model.ClusteredValueGroupsBaseTableMetadata; +import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.Columns; +import org.apache.druid.catalog.model.DatasourceProjectionMetadata; +import org.apache.druid.catalog.model.TableId; +import org.apache.druid.catalog.model.TableMetadata; +import org.apache.druid.catalog.model.TableSpec; +import org.apache.druid.catalog.model.table.ClusterKeySpec; +import org.apache.druid.catalog.model.table.DatasourceDefn; +import org.apache.druid.common.utils.IdUtils; +import org.apache.druid.error.DruidException; +import org.apache.druid.error.InvalidSqlInput; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.granularity.Granularity; +import org.apache.druid.java.util.common.granularity.PeriodGranularity; +import org.apache.druid.java.util.common.guava.Sequences; +import org.apache.druid.query.explain.ExplainAttributes; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.projections.Projections; +import org.apache.druid.server.QueryResponse; +import org.apache.druid.server.security.Action; +import org.apache.druid.server.security.Resource; +import org.apache.druid.server.security.ResourceAction; +import org.apache.druid.server.security.ResourceType; +import org.apache.druid.sql.calcite.parser.DruidSqlAlterTable; +import org.apache.druid.sql.calcite.parser.DruidSqlColumnDeclaration; +import org.apache.druid.sql.calcite.parser.DruidSqlCreateTable; +import org.apache.druid.sql.calcite.parser.DruidSqlParser; +import org.apache.druid.sql.calcite.parser.DruidSqlPropertyAssignment; +import org.apache.druid.sql.calcite.parser.SqlGranularityLiteral; +import org.apache.druid.sql.calcite.parser.SqlProjectionSpec; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Handles the catalog DDL statements: {@code CREATE TABLE} and {@code ALTER TABLE}. + *

+ * These statements are metadata operations, not queries. They are validated here, converted to a catalog + * {@link TableSpec} or column/property edit, and applied through {@link CatalogTableWriter}, which forwards them to + * the Coordinator. No Calcite validation or query planning takes place, and no rows are returned. + *

+ * Validation is deliberately split. This class checks what it can attribute to a position in the statement (type + * spellings, duplicate columns, granularity, clustering) so that the error names the offending SQL. The Coordinator + * remains authoritative: {@code DatasourceDefn.validate} runs on write, and its message is surfaced verbatim. + */ +public abstract class CatalogDdlHandler extends SqlStatementHandler.BaseStatementHandler +{ + /** + * DDL produces no rows. A single-column type still has to be declared, because JDBC clients ask for a result set + * signature when preparing the statement. + */ + private static final RelDataType RESULT_TYPE = resultType(); + + protected final SqlIdentifier tableIdentifier; + protected TableId tableId; + + protected CatalogDdlHandler(SqlStatementHandler.HandlerContext handlerContext, SqlIdentifier tableIdentifier) + { + super(handlerContext); + this.tableIdentifier = tableIdentifier; + } + + /** + * The runtime property that gates these statements. Read from {@link PlannerConfig} rather than the query context + * so that a user cannot turn the feature on for their own statement. + */ + public static final String ENABLE_CATALOG_DDL_PROPERTY = "druid.sql.planner.enableCatalogDdl"; + + /** + * The reserved name of the base-table projection, which describes the physical layout of the table itself. Handled + * as a separate catalog property, not as one of the aggregate projections. + */ + public static final String BASE_PROJECTION_NAME = "__base"; + + @Override + public void validate() + { + if (!handlerContext.plannerContext().getPlannerConfig().isEnableCatalogDdl()) { + throw DruidException.forPersona(DruidException.Persona.ADMIN) + .ofCategory(DruidException.Category.UNSUPPORTED) + .build( + "Catalog DDL statements are disabled. Set [%s] to true on the Broker to enable [%s].", + ENABLE_CATALOG_DDL_PROPERTY, + operationName() + ); + } + if (!handlerContext.plannerContext().getParameters().isEmpty()) { + throw InvalidSqlInput.exception("Dynamic parameters are not supported for [%s]", operationName()); + } + tableId = TableId.datasource(resolveTableName()); + resourceActions = Collections.singleton( + new ResourceAction(new Resource(tableId.name(), ResourceType.DATASOURCE), Action.WRITE) + ); + validateStatement(); + } + + /** + * Statement-specific validation, which also prepares whatever {@link #execute} will apply. + */ + protected abstract void validateStatement(); + + protected abstract void execute(CatalogTableWriter writer); + + protected abstract String operationName(); + + @Override + public void prepare() + { + // Nothing to prepare: there is no query to plan. + } + + @Override + public PrepareResult prepareResult() + { + return new PrepareResult(RESULT_TYPE, RESULT_TYPE, DruidTypeSystem.TYPE_FACTORY.createStructType( + Collections.emptyList(), + Collections.emptyList() + )); + } + + @Override + public PlannerResult plan() + { + execute(handlerContext.plannerContext().getPlannerToolbox().catalogTableWriter()); + final Supplier> resultsSupplier = Suppliers.ofInstance( + QueryResponse.withEmptyContext(Sequences.empty()) + ); + return new PlannerResult(resultsSupplier, RESULT_TYPE); + } + + @Override + public ExplainAttributes explainAttributes() + { + throw InvalidSqlInput.exception("EXPLAIN is not supported for [%s]", operationName()); + } + + /** + * Resolve the table name, which may be unqualified or qualified by the Druid schema. Other schemas are rejected: + * only datasources have catalog specs that DDL can write. + */ + private String resolveTableName() + { + final String tableName; + if (tableIdentifier.names.size() == 1) { + tableName = tableIdentifier.names.get(0); + } else if (tableIdentifier.names.size() == 2) { + final String defaultSchemaName = + Iterables.getOnlyElement(CalciteSchema.from(handlerContext.defaultSchema()).path(null)); + if (!defaultSchemaName.equals(tableIdentifier.names.get(0))) { + throw InvalidSqlInput.exception( + "Table [%s] does not support operation [%s] because it is not a Druid datasource", + tableIdentifier, + operationName() + ); + } + tableName = tableIdentifier.names.get(1); + } else { + throw InvalidSqlInput.exception( + "Table name [%s] is not valid for operation [%s]", + tableIdentifier, + operationName() + ); + } + IdUtils.validateId("table", tableName); + return tableName; + } + + /** + * Convert a parsed column declaration into its catalog form, checking that the type is one Druid can store. + */ + protected static ColumnSpec toColumnSpec(DruidSqlColumnDeclaration declaration) + { + final String name = simpleName(declaration.getName(), "Column"); + final String type; + try { + type = CatalogColumnTypes.forCatalogColumn(name, declaration.getDataType()); + } + catch (IAE e) { + throw InvalidSqlInput.exception(e, "%s", e.getMessage()); + } + if (Columns.isTimeColumn(name) && !ColumnType.LONG.equals(Columns.druidTypeFromString(type))) { + throw InvalidSqlInput.exception( + "Column [%s] must have type [%s] or [%s], but was [%s]", + Columns.TIME_COLUMN, + Columns.SQL_TIMESTAMP, + Columns.SQL_BIGINT, + type + ); + } + return new ColumnSpec(name, type, null); + } + + /** + * Translate one projection definition into the catalog form. + *

+ * {@code __base} is reserved for the base-table projection, which is a different catalog entity: it describes the + * physical layout of the table itself rather than an additional aggregate. It is rejected here rather than being + * translated as an ordinary projection. + */ + protected static DatasourceProjectionMetadata translateProjection( + final SqlStatementHandler.HandlerContext handlerContext, + final String tableName, + final List columns, + final SqlProjectionSpec projection + ) + { + final String name = simpleName(projection.getName(), "Projection"); + try { + Projections.validateProjectionName(name); + } + catch (DruidException e) { + throw InvalidSqlInput.exception(e, "%s", e.getMessage()); + } + if (projection.getClusteredBy() != null) { + throw InvalidSqlInput.exception( + "Projection [%s] cannot use CLUSTERED BY: an aggregate projection is ordered by its grouping columns." + + " Only the [%s] projection, which describes the table's own layout, chooses a clustering", + name, + BASE_PROJECTION_NAME + ); + } + return new DatasourceProjectionMetadata( + new ProjectionSpecTranslator(handlerContext.plannerFactory()) + .translate(tableName, columns, name, projection.getBody()) + ); + } + + /** + * Translate the reserved {@code __base} projection, which describes the physical layout of the table rather than an + * additional aggregate, and so becomes the {@code baseTable} property instead of one of the projections. + */ + protected static ClusteredValueGroupsBaseTableMetadata translateBaseTable( + final SqlStatementHandler.HandlerContext handlerContext, + final String tableName, + final List columns, + final SqlProjectionSpec projection + ) + { + return new ProjectionSpecTranslator(handlerContext.plannerFactory()) + .translateBaseTable(tableName, columns, projection.getBody(), projection.getClusteredBy()); + } + + /** + * A base table layout derives the physical segment schema from the declared columns, so a column that is not + * declared cannot be stored. The catalog enforces this too, but saying it here names the clause that is missing. + */ + protected static void requireSealed(boolean sealed) + { + if (!sealed) { + throw InvalidSqlInput.exception( + "A table with a [%s] projection must be declared SEALED: its columns define the physical segment schema," + + " so columns that are not declared cannot be ingested", + BASE_PROJECTION_NAME + ); + } + } + + protected static String simpleName(SqlIdentifier identifier, String what) + { + if (!identifier.isSimple()) { + throw InvalidSqlInput.exception("%s name [%s] must be a simple name", what, identifier); + } + return identifier.getSimple(); + } + + /** + * The catalog stores a segment granularity as either {@code ALL} or an ISO period string. + */ + protected static String toGranularityString(SqlGranularityLiteral partitionedBy) + { + final Granularity granularity = partitionedBy.getGranularity(); + if (Granularities.ALL.equals(granularity)) { + return DatasourceDefn.ALL_GRANULARITY; + } + if (granularity instanceof PeriodGranularity) { + return ((PeriodGranularity) granularity).getPeriod().toString(); + } + throw InvalidSqlInput.exception("Granularity [%s] is not supported by the catalog", partitionedBy); + } + + /** + * The catalog's clustering keys are plain ascending column references. Expressions, ordinals and DESC have no + * catalog representation, so they are rejected here rather than silently dropped. + */ + protected static List toClusterKeys(SqlNodeList clusteredBy) + { + final List keys = new ArrayList<>(clusteredBy.size()); + for (SqlNode node : clusteredBy) { + if (!(node instanceof SqlIdentifier) || !((SqlIdentifier) node).isSimple()) { + throw InvalidSqlInput.exception( + "CLUSTERED BY column [%s] must be a column name; expressions, ordinals and DESC are not supported when" + + " defining a table", + node + ); + } + keys.add(new ClusterKeySpec(((SqlIdentifier) node).getSimple(), false)); + } + return keys; + } + + private static RelDataType resultType() + { + final RelDataTypeFactory typeFactory = DruidTypeSystem.TYPE_FACTORY; + return typeFactory.createStructType( + ImmutableList.of(Calcites.createSqlType(typeFactory, SqlTypeName.VARCHAR)), + ImmutableList.of("RESULT") + ); + } + + /** + * {@code CREATE [OR REPLACE] TABLE [IF NOT EXISTS] ...}. + */ + public static class CreateTableHandler extends CatalogDdlHandler + { + private final DruidSqlCreateTable createTable; + private TableSpec tableSpec; + + public CreateTableHandler(SqlStatementHandler.HandlerContext handlerContext, DruidSqlCreateTable createTable) + { + super(handlerContext, createTable.getName()); + this.createTable = createTable; + } + + @Override + protected void validateStatement() + { + if (createTable.getReplace() && createTable.isIfNotExists()) { + throw InvalidSqlInput.exception("Cannot specify both OR REPLACE and IF NOT EXISTS"); + } + + final List columns = new ArrayList<>(createTable.getColumnList().size()); + final Set seen = new HashSet<>(); + for (SqlNode node : createTable.getColumnList()) { + final ColumnSpec column = toColumnSpec((DruidSqlColumnDeclaration) node); + if (!seen.add(column.name())) { + throw InvalidSqlInput.exception("Column [%s] is declared more than once", column.name()); + } + columns.add(column); + } + + final Map properties = new LinkedHashMap<>(); + if (createTable.getPartitionedBy() != null) { + properties.put( + DatasourceDefn.SEGMENT_GRANULARITY_PROPERTY, + toGranularityString(createTable.getPartitionedBy()) + ); + } + if (createTable.getClusteredBy() != null) { + properties.put(DatasourceDefn.CLUSTER_KEYS_PROPERTY, toClusterKeys(createTable.getClusteredBy())); + } + if (createTable.isSealed()) { + properties.put(DatasourceDefn.SEALED_PROPERTY, true); + } + if (!createTable.getProjectionList().isEmpty()) { + final List projections = + new ArrayList<>(createTable.getProjectionList().size()); + final Set seenProjections = new HashSet<>(); + for (SqlNode node : createTable.getProjectionList()) { + final SqlProjectionSpec projection = (SqlProjectionSpec) node; + final String name = simpleName(projection.getName(), "Projection"); + if (!seenProjections.add(name)) { + throw InvalidSqlInput.exception("Projection [%s] is declared more than once", name); + } + if (BASE_PROJECTION_NAME.equals(name)) { + requireSealed(createTable.isSealed()); + properties.put( + DatasourceDefn.BASE_TABLE_PROPERTY, + translateBaseTable(handlerContext, tableId.name(), columns, projection) + ); + } else { + projections.add(translateProjection(handlerContext, tableId.name(), columns, projection)); + } + } + if (!projections.isEmpty()) { + properties.put(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY, projections); + } + } + + tableSpec = new TableSpec(DatasourceDefn.TABLE_TYPE, properties, columns); + } + + @Override + protected void execute(CatalogTableWriter writer) + { + writer.createTable(tableId, tableSpec, createTable.isIfNotExists(), createTable.getReplace()); + } + + @Override + protected String operationName() + { + return "CREATE TABLE"; + } + } + + /** + * {@code ALTER TABLE ... ADD COLUMN}. The Coordinator merges columns by name, so an existing column would be + * silently updated; this checks first so that {@code ADD} means add. + */ + public static class AddColumnHandler extends CatalogDdlHandler + { + private final DruidSqlAlterTable.AddColumn alterTable; + private ColumnSpec column; + + public AddColumnHandler(SqlStatementHandler.HandlerContext handlerContext, DruidSqlAlterTable.AddColumn alterTable) + { + super(handlerContext, alterTable.getName()); + this.alterTable = alterTable; + } + + @Override + protected void validateStatement() + { + column = toColumnSpec(alterTable.getColumn()); + } + + @Override + protected void execute(CatalogTableWriter writer) + { + final TableMetadata existing = writer.readTable(tableId); + if (existing == null) { + throw InvalidSqlInput.exception("Table [%s] does not have a catalog entry", tableId.name()); + } + if (existing.spec().columns() != null + && existing.spec().columns().stream().anyMatch(c -> column.name().equals(c.name()))) { + throw InvalidSqlInput.exception( + "Column [%s] already exists in table [%s]; use ALTER COLUMN to change its type", + column.name(), + tableId.name() + ); + } + writer.updateColumns(tableId, Collections.singletonList(column)); + } + + @Override + protected String operationName() + { + return "ALTER TABLE ADD COLUMN"; + } + } + + /** + * {@code ALTER TABLE ... DROP COLUMN}. + */ + public static class DropColumnHandler extends CatalogDdlHandler + { + private final DruidSqlAlterTable.DropColumn alterTable; + private String column; + + public DropColumnHandler( + SqlStatementHandler.HandlerContext handlerContext, + DruidSqlAlterTable.DropColumn alterTable + ) + { + super(handlerContext, alterTable.getName()); + this.alterTable = alterTable; + } + + @Override + protected void validateStatement() + { + column = simpleName(alterTable.getColumn(), "Column"); + } + + @Override + protected void execute(CatalogTableWriter writer) + { + writer.dropColumns(tableId, Collections.singletonList(column)); + } + + @Override + protected String operationName() + { + return "ALTER TABLE DROP COLUMN"; + } + } + + /** + * {@code ALTER TABLE ... ALTER COLUMN ... SET DATA TYPE}. Merging a column by name is exactly what changing its + * type requires, so this reuses the same update as ADD COLUMN without the existence check. + */ + public static class AlterColumnHandler extends CatalogDdlHandler + { + private final DruidSqlAlterTable.AlterColumn alterTable; + private ColumnSpec column; + + public AlterColumnHandler( + SqlStatementHandler.HandlerContext handlerContext, + DruidSqlAlterTable.AlterColumn alterTable + ) + { + super(handlerContext, alterTable.getName()); + this.alterTable = alterTable; + } + + @Override + protected void validateStatement() + { + column = toColumnSpec(alterTable.getColumn()); + } + + @Override + protected void execute(CatalogTableWriter writer) + { + writer.updateColumns(tableId, Collections.singletonList(column)); + } + + @Override + protected String operationName() + { + return "ALTER TABLE ALTER COLUMN"; + } + } + + /** + * {@code ALTER TABLE ... ADD PROJECTION}. The body is translated against the table's current declared columns, so + * the table must already have a catalog entry. + */ + public static class AddProjectionHandler extends CatalogDdlHandler + { + private final DruidSqlAlterTable.AddProjection alterTable; + private String projectionName; + + public AddProjectionHandler( + SqlStatementHandler.HandlerContext handlerContext, + DruidSqlAlterTable.AddProjection alterTable + ) + { + super(handlerContext, alterTable.getName()); + this.alterTable = alterTable; + } + + @Override + protected void validateStatement() + { + projectionName = simpleName(alterTable.getProjection().getName(), "Projection"); + } + + @Override + protected void execute(CatalogTableWriter writer) + { + final TableMetadata existing = writer.readTable(tableId); + if (existing == null) { + throw InvalidSqlInput.exception("Table [%s] does not have a catalog entry", tableId.name()); + } + final List columns = + existing.spec().columns() == null ? Collections.emptyList() : existing.spec().columns(); + + if (BASE_PROJECTION_NAME.equals(projectionName)) { + // The base table is a property of the table, not one of its projections, so it is set rather than appended. + if (existing.spec().properties().get(DatasourceDefn.BASE_TABLE_PROPERTY) != null) { + if (alterTable.isIfNotExists()) { + return; + } + throw InvalidSqlInput.exception( + "Table [%s] already has a [%s] projection; drop it before defining another", + tableId.name(), + BASE_PROJECTION_NAME + ); + } + requireSealed(Boolean.TRUE.equals(existing.spec().properties().get(DatasourceDefn.SEALED_PROPERTY))); + writer.updateProperties( + tableId, + Collections.singletonMap( + DatasourceDefn.BASE_TABLE_PROPERTY, + translateBaseTable(handlerContext, tableId.name(), columns, alterTable.getProjection()) + ) + ); + return; + } + + writer.addProjection( + tableId, + translateProjection(handlerContext, tableId.name(), columns, alterTable.getProjection()), + alterTable.isIfNotExists() + ); + } + + @Override + protected String operationName() + { + return "ALTER TABLE ADD PROJECTION"; + } + } + + /** + * {@code ALTER TABLE ... DROP PROJECTION}. Segments already built keep whatever projections they were built with; + * this only stops future ingestion from building it. + */ + public static class DropProjectionHandler extends CatalogDdlHandler + { + private final DruidSqlAlterTable.DropProjection alterTable; + private String projectionName; + + public DropProjectionHandler( + SqlStatementHandler.HandlerContext handlerContext, + DruidSqlAlterTable.DropProjection alterTable + ) + { + super(handlerContext, alterTable.getName()); + this.alterTable = alterTable; + } + + @Override + protected void validateStatement() + { + projectionName = simpleName(alterTable.getProjectionName(), "Projection"); + } + + @Override + protected void execute(CatalogTableWriter writer) + { + if (BASE_PROJECTION_NAME.equals(projectionName)) { + // Removing the layout leaves the declared columns alone; only future segments are affected. + final TableMetadata existing = writer.readTable(tableId); + final boolean present = existing != null + && existing.spec().properties().get(DatasourceDefn.BASE_TABLE_PROPERTY) != null; + if (!present) { + if (alterTable.isIfExists()) { + return; + } + throw InvalidSqlInput.exception( + "Table [%s] does not have a [%s] projection", + tableId.name(), + BASE_PROJECTION_NAME + ); + } + writer.updateProperties( + tableId, + Collections.singletonMap(DatasourceDefn.BASE_TABLE_PROPERTY, null) + ); + return; + } + writer.dropProjection(tableId, projectionName, alterTable.isIfExists()); + } + + @Override + protected String operationName() + { + return "ALTER TABLE DROP PROJECTION"; + } + } + + /** + * {@code ALTER TABLE ... SET PROPERTIES}. A NULL value removes the property. The set of legal keys is not checked + * here: the Coordinator's table definition registry is what knows them. + */ + public static class SetPropertiesHandler extends CatalogDdlHandler + { + private final DruidSqlAlterTable.SetProperties alterTable; + private Map properties; + + public SetPropertiesHandler( + SqlStatementHandler.HandlerContext handlerContext, + DruidSqlAlterTable.SetProperties alterTable + ) + { + super(handlerContext, alterTable.getName()); + this.alterTable = alterTable; + } + + @Override + protected void validateStatement() + { + properties = new LinkedHashMap<>(); + for (SqlNode node : alterTable.getProperties()) { + final DruidSqlPropertyAssignment assignment = (DruidSqlPropertyAssignment) node; + final String key = simpleName(assignment.getKey(), "Property"); + if (properties.containsKey(key)) { + throw InvalidSqlInput.exception("Property [%s] is assigned more than once", key); + } + properties.put(key, propertyValue(key, assignment.getValue())); + } + } + + private static Object propertyValue(String key, SqlNode value) + { + if (!(value instanceof SqlLiteral)) { + throw InvalidSqlInput.exception("Value for property [%s] must be a literal", key); + } + // A NULL literal coerces to null, which the catalog treats as "remove this property". + return DruidSqlParser.sqlLiteralToJavaValue((SqlLiteral) value, "property " + key); + } + + @Override + protected void execute(CatalogTableWriter writer) + { + writer.updateProperties(tableId, properties); + } + + @Override + protected String operationName() + { + return "ALTER TABLE SET PROPERTIES"; + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java new file mode 100644 index 000000000000..20fab1cbf41d --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogTableWriter.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.planner; + +import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.DatasourceProjectionMetadata; +import org.apache.druid.catalog.model.TableId; +import org.apache.druid.catalog.model.TableMetadata; +import org.apache.druid.catalog.model.TableSpec; +import org.apache.druid.error.DruidException; + +import javax.annotation.Nullable; +import java.util.List; +import java.util.Map; + +/** + * Write side of the catalog, the counterpart to the read-only {@link CatalogResolver}. Catalog metadata is owned by + * the Coordinator, so an implementation of this interface makes a remote call; it is bound by the {@code druid-catalog} + * extension and defaults to {@link #NOT_AVAILABLE} when that extension is absent. + *

+ * The methods are deliberately semantic rather than a generic "apply this edit request", so that the extension's edit + * request types need not be visible here. Each corresponds to exactly one atomic Coordinator operation, which is why + * every DDL statement maps to a single call: the catalog stores properties and columns as separate blobs with separate + * update paths, so a statement that changed both would not be atomic. + */ +public interface CatalogTableWriter +{ + CatalogTableWriter NOT_AVAILABLE = new UnavailableCatalogTableWriter(); + + /** + * Create a table. + * + * @param ifNotExists if the table already exists, do nothing rather than failing + * @param replace overwrite any existing spec for this table + */ + void createTable(TableId tableId, TableSpec spec, boolean ifNotExists, boolean replace); + + /** + * Merge the given columns into the table's column list, matching on name. Columns that do not yet exist are + * appended; existing columns are updated in place. + */ + void updateColumns(TableId tableId, List columns); + + /** + * Remove the named columns from the table's column list. Segments are unaffected. + */ + void dropColumns(TableId tableId, List columns); + + /** + * Merge the given properties into the table's properties. A null value removes the property. + */ + void updateProperties(TableId tableId, Map properties); + + /** + * Append a projection to the table's projections. + * + * @param ifNotExists if a projection of the same name exists, do nothing rather than failing + */ + void addProjection(TableId tableId, DatasourceProjectionMetadata projection, boolean ifNotExists); + + /** + * Remove the named projection from the table's projections. + * + * @param ifExists if no projection of that name exists, do nothing rather than failing + */ + void dropProjection(TableId tableId, String projectionName, boolean ifExists); + + /** + * Read a table's current metadata directly from the Coordinator, bypassing any local cache, or null if the table + * has no catalog entry. Used for pre-checks that must not race against a stale cache. + */ + @Nullable + TableMetadata readTable(TableId tableId); + + /** + * Stand-in used when the {@code druid-catalog} extension is not loaded. Every operation fails with an explanation + * rather than silently doing nothing. + */ + class UnavailableCatalogTableWriter implements CatalogTableWriter + { + @Override + public void createTable(TableId tableId, TableSpec spec, boolean ifNotExists, boolean replace) + { + throw notAvailable(); + } + + @Override + public void updateColumns(TableId tableId, List columns) + { + throw notAvailable(); + } + + @Override + public void dropColumns(TableId tableId, List columns) + { + throw notAvailable(); + } + + @Override + public void updateProperties(TableId tableId, Map properties) + { + throw notAvailable(); + } + + @Override + public void addProjection(TableId tableId, DatasourceProjectionMetadata projection, boolean ifNotExists) + { + throw notAvailable(); + } + + @Override + public void dropProjection(TableId tableId, String projectionName, boolean ifExists) + { + throw notAvailable(); + } + + @Nullable + @Override + public TableMetadata readTable(TableId tableId) + { + throw notAvailable(); + } + + private static DruidException notAvailable() + { + return DruidException.forPersona(DruidException.Persona.USER) + .ofCategory(DruidException.Category.UNSUPPORTED) + .build( + "Catalog DDL statements require the [druid-catalog] extension, which is not loaded" + ); + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java index f361e43797b4..614764d66881 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java @@ -35,6 +35,8 @@ import org.apache.druid.server.security.AuthorizationResult; import org.apache.druid.server.security.Resource; import org.apache.druid.server.security.ResourceAction; +import org.apache.druid.sql.calcite.parser.DruidSqlAlterTable; +import org.apache.druid.sql.calcite.parser.DruidSqlCreateTable; import org.apache.druid.sql.calcite.parser.DruidSqlInsert; import org.apache.druid.sql.calcite.parser.DruidSqlReplace; import org.apache.druid.sql.calcite.run.SqlEngine; @@ -97,6 +99,7 @@ public AuthResult( private final PlannerContext plannerContext; private final SqlEngine engine; private final PlannerHook hook; + private final PlannerFactory plannerFactory; private State state = State.START; private SqlStatementHandler handler; private boolean authorized; @@ -105,9 +108,11 @@ public AuthResult( final FrameworkConfig frameworkConfig, final PlannerContext plannerContext, final SqlEngine engine, - final PlannerHook hook + final PlannerHook hook, + final PlannerFactory plannerFactory ) { + this.plannerFactory = plannerFactory; this.frameworkConfig = frameworkConfig; this.planner = new CalcitePlanner(frameworkConfig); this.plannerContext = plannerContext; @@ -147,6 +152,16 @@ private SqlStatementHandler createHandler(final SqlNode node) } SqlStatementHandler.HandlerContext handlerContext = new HandlerContextImpl(); + + if (query instanceof DruidSqlCreateTable || query instanceof DruidSqlAlterTable) { + // The grammar does not admit EXPLAIN of a DDL statement; this guards the case anyway, since a DDL statement + // has no query to explain. + if (explain != null) { + throw InvalidSqlInput.exception("EXPLAIN is not supported for [%s]", query.getKind()); + } + return createDdlHandler(handlerContext, query); + } + if (query.getKind() == SqlKind.INSERT) { if (query instanceof DruidSqlInsert) { return new IngestHandler.InsertHandler(handlerContext, (DruidSqlInsert) query, explain); @@ -161,6 +176,35 @@ private SqlStatementHandler createHandler(final SqlNode node) throw InvalidSqlInput.exception("Unsupported SQL statement [%s]", node.getKind()); } + private static SqlStatementHandler createDdlHandler( + final SqlStatementHandler.HandlerContext handlerContext, + final SqlNode query + ) + { + if (query instanceof DruidSqlCreateTable) { + return new CatalogDdlHandler.CreateTableHandler(handlerContext, (DruidSqlCreateTable) query); + } + if (query instanceof DruidSqlAlterTable.AddColumn) { + return new CatalogDdlHandler.AddColumnHandler(handlerContext, (DruidSqlAlterTable.AddColumn) query); + } + if (query instanceof DruidSqlAlterTable.DropColumn) { + return new CatalogDdlHandler.DropColumnHandler(handlerContext, (DruidSqlAlterTable.DropColumn) query); + } + if (query instanceof DruidSqlAlterTable.AlterColumn) { + return new CatalogDdlHandler.AlterColumnHandler(handlerContext, (DruidSqlAlterTable.AlterColumn) query); + } + if (query instanceof DruidSqlAlterTable.AddProjection) { + return new CatalogDdlHandler.AddProjectionHandler(handlerContext, (DruidSqlAlterTable.AddProjection) query); + } + if (query instanceof DruidSqlAlterTable.DropProjection) { + return new CatalogDdlHandler.DropProjectionHandler(handlerContext, (DruidSqlAlterTable.DropProjection) query); + } + if (query instanceof DruidSqlAlterTable.SetProperties) { + return new CatalogDdlHandler.SetPropertiesHandler(handlerContext, (DruidSqlAlterTable.SetProperties) query); + } + throw DruidException.defensive("Unhandled catalog DDL statement [%s]", query.getClass().getSimpleName()); + } + /** * Uses {@link SqlParameterizerShuttle} to rewrite {@link SqlNode} to swap out any * {@link org.apache.calcite.sql.SqlDynamicParam} early for their {@link org.apache.calcite.sql.SqlLiteral} @@ -319,6 +363,12 @@ public PlannerHook hook() { return hook; } + + @Override + public PlannerFactory plannerFactory() + { + return plannerFactory; + } } public static DruidException translateException(Exception e) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java index d8c77c695894..f66928f967f0 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java @@ -83,6 +83,9 @@ public class PlannerConfig @JsonProperty private boolean enableSysQueriesTable = false; + @JsonProperty + private boolean enableCatalogDdl = false; + public int getMaxNumericInFilters() { return maxNumericInFilters; @@ -160,6 +163,16 @@ public boolean isEnableSysQueriesTable() return enableSysQueriesTable; } + /** + * Whether catalog DDL statements (CREATE TABLE, ALTER TABLE) may be executed. Off by default: the Broker's SQL + * endpoint is typically reachable by far more people than the Coordinator's catalog API, so enabling this widens + * what an existing datasource WRITE permission allows. Deliberately not overridable from the query context. + */ + public boolean isEnableCatalogDdl() + { + return enableCatalogDdl; + } + public PlannerConfig withOverrides(final Map queryContext) { if (queryContext.isEmpty()) { @@ -189,6 +202,7 @@ public boolean equals(Object o) && forceExpressionVirtualColumns == that.forceExpressionVirtualColumns && maxNumericInFilters == that.maxNumericInFilters && enableSysQueriesTable == that.enableSysQueriesTable + && enableCatalogDdl == that.enableCatalogDdl && Objects.equals(sqlTimeZone, that.sqlTimeZone) && Objects.equals(nativeQuerySqlPlanningMode, that.nativeQuerySqlPlanningMode); } @@ -210,7 +224,8 @@ public int hashCode() forceExpressionVirtualColumns, maxNumericInFilters, nativeQuerySqlPlanningMode, - enableSysQueriesTable + enableSysQueriesTable, + enableCatalogDdl ); } @@ -227,6 +242,7 @@ public String toString() ", useNativeQueryExplain=" + useNativeQueryExplain + ", nativeQuerySqlPlanningMode=" + nativeQuerySqlPlanningMode + ", enableSysQueriesTable=" + enableSysQueriesTable + + ", enableCatalogDdl=" + enableCatalogDdl + '}'; } @@ -262,6 +278,7 @@ public static class Builder private int maxNumericInFilters; private String nativeQuerySqlPlanningMode; private boolean enableSysQueriesTable; + private boolean enableCatalogDdl; public Builder(PlannerConfig base) { @@ -282,6 +299,7 @@ public Builder(PlannerConfig base) maxNumericInFilters = base.getMaxNumericInFilters(); nativeQuerySqlPlanningMode = base.getNativeQuerySqlPlanningMode(); enableSysQueriesTable = base.isEnableSysQueriesTable(); + enableCatalogDdl = base.isEnableCatalogDdl(); } public Builder requireTimeCondition(boolean option) @@ -362,6 +380,12 @@ public Builder enableSysQueriesTable(boolean option) return this; } + public Builder enableCatalogDdl(boolean option) + { + this.enableCatalogDdl = option; + return this; + } + public Builder withOverrides(final Map queryContext) { useApproximateCountDistinct = QueryContexts.parseBoolean( @@ -459,6 +483,7 @@ public PlannerConfig build() config.forceExpressionVirtualColumns = forceExpressionVirtualColumns; config.nativeQuerySqlPlanningMode = nativeQuerySqlPlanningMode; config.enableSysQueriesTable = enableSysQueriesTable; + config.enableCatalogDdl = enableCatalogDdl; return config; } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerFactory.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerFactory.java index 32ca488e923a..0bcd1fbf36f0 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerFactory.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerFactory.java @@ -21,15 +21,20 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import org.apache.calcite.config.CalciteConnectionConfig; import org.apache.calcite.config.CalciteConnectionConfigImpl; import org.apache.calcite.config.CalciteConnectionProperty; +import org.apache.calcite.jdbc.CalciteSchema; import org.apache.calcite.plan.Context; import org.apache.calcite.plan.ConventionTraitDef; import org.apache.calcite.plan.volcano.DruidVolcanoCost; import org.apache.calcite.rel.RelCollationTraitDef; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.Table; +import org.apache.calcite.schema.impl.AbstractSchema; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.validate.SqlConformance; import org.apache.calcite.sql2rel.SqlToRelConverter; @@ -50,15 +55,54 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.schema.DruidSchemaName; +import org.apache.druid.sql.calcite.table.DruidTable; import org.apache.druid.sql.hook.DruidHook; import org.apache.druid.sql.hook.DruidHookDispatcher; +import java.util.Collections; import java.util.Map; import java.util.Properties; import java.util.Set; public class PlannerFactory extends PlannerToolbox { + /** + * Convenience for callers that never execute catalog DDL, such as tests and benchmarks. + */ + public PlannerFactory( + final DruidSchemaCatalog rootSchema, + final DruidOperatorTable operatorTable, + final ExprMacroTable macroTable, + final PlannerConfig plannerConfig, + final AuthorizerMapper authorizerMapper, + final @Json ObjectMapper jsonMapper, + final @DruidSchemaName String druidSchemaName, + final CalciteRulesManager calciteRuleManager, + final JoinableFactoryWrapper joinableFactoryWrapper, + final CatalogResolver catalog, + final AuthConfig authConfig, + final PolicyEnforcer policyEnforcer, + final DruidHookDispatcher hookDispatcher + ) + { + this( + rootSchema, + operatorTable, + macroTable, + plannerConfig, + authorizerMapper, + jsonMapper, + druidSchemaName, + calciteRuleManager, + joinableFactoryWrapper, + catalog, + CatalogTableWriter.NOT_AVAILABLE, + authConfig, + policyEnforcer, + hookDispatcher + ); + } + @Inject public PlannerFactory( final DruidSchemaCatalog rootSchema, @@ -71,6 +115,7 @@ public PlannerFactory( final CalciteRulesManager calciteRuleManager, final JoinableFactoryWrapper joinableFactoryWrapper, final CatalogResolver catalog, + final CatalogTableWriter catalogTableWriter, final AuthConfig authConfig, final PolicyEnforcer policyEnforcer, final DruidHookDispatcher hookDispatcher @@ -84,6 +129,7 @@ public PlannerFactory( rootSchema, joinableFactoryWrapper, catalog, + catalogTableWriter, druidSchemaName, calciteRuleManager, authorizerMapper, @@ -127,7 +173,7 @@ public DruidPlanner createPlanner( ); context.dispatchHook(DruidHook.SQL, sql); - return new DruidPlanner(buildFrameworkConfig(context), context, engine, hook); + return new DruidPlanner(buildFrameworkConfig(context), context, engine, hook, this); } /** @@ -159,12 +205,70 @@ public DruidPlanner createPlannerForTesting( return thePlanner; } + /** + * Create a planner for a statement that refers to a table which may not be in the schema, or which should be seen + * with a different definition than the one the schema holds. Used to plan the body of a projection definition + * against the columns its {@code CREATE TABLE} or {@code ALTER TABLE} statement declares, before those columns + * have been written to the catalog. + *

+ * The schema holds only that table. Resource typing still consults the real schema, so authorization is + * unaffected; callers are expected to have authorized the enclosing statement already. + */ + public DruidPlanner createPlannerForTable( + final SqlEngine engine, + final String sql, + final SqlNode sqlNode, + final Map queryContext, + final String tableName, + final DruidTable table + ) + { + final PlannerContext context = PlannerContext.create( + this, + sql, + sqlNode, + engine, + Collections.emptySet(), + queryContext, + null + ); + final SchemaPlus defaultSchema = CalciteSchema.createRootSchema(false, false) + .plus() + .add(druidSchemaName, new SingleTableSchema(tableName, table)); + return new DruidPlanner(buildFrameworkConfig(context, defaultSchema), context, engine, null, this); + } + + /** + * A schema holding exactly the table being defined. The body of a projection has no FROM clause, so the table it + * belongs to is the only one it can name; anything else is a mistake worth reporting as an unknown table. + */ + private static class SingleTableSchema extends AbstractSchema + { + private final Map tables; + + SingleTableSchema(String tableName, Table table) + { + this.tables = ImmutableMap.of(tableName, table); + } + + @Override + protected Map getTableMap() + { + return tables; + } + } + public AuthorizerMapper getAuthorizerMapper() { return authorizerMapper; } private FrameworkConfig buildFrameworkConfig(PlannerContext plannerContext) + { + return buildFrameworkConfig(plannerContext, rootSchema.getSubSchema(druidSchemaName)); + } + + private FrameworkConfig buildFrameworkConfig(PlannerContext plannerContext, SchemaPlus defaultSchema) { final SqlToRelConverter.Config sqlToRelConverterConfig = SqlToRelConverter .config() @@ -184,7 +288,7 @@ private FrameworkConfig buildFrameworkConfig(PlannerContext plannerContext) .programs(calciteRuleManager.programs(plannerContext)) .executor(new DruidRexExecutor(plannerContext)) .typeSystem(DruidTypeSystem.INSTANCE) - .defaultSchema(rootSchema.getSubSchema(druidSchemaName)) + .defaultSchema(defaultSchema) .sqlToRelConverterConfig(sqlToRelConverterConfig) .context(new Context() { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerToolbox.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerToolbox.java index 17887afd06a5..6fc4ffdf2663 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerToolbox.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerToolbox.java @@ -38,6 +38,7 @@ public class PlannerToolbox protected final PlannerConfig plannerConfig; protected final DruidSchemaCatalog rootSchema; protected final CatalogResolver catalog; + protected final CatalogTableWriter catalogTableWriter; protected final String druidSchemaName; protected final CalciteRulesManager calciteRuleManager; protected final AuthorizerMapper authorizerMapper; @@ -45,6 +46,9 @@ public class PlannerToolbox protected final PolicyEnforcer policyEnforcer; protected final DruidHookDispatcher hookDispatcher; + /** + * Convenience for callers that never execute catalog DDL, such as tests and benchmarks. + */ public PlannerToolbox( final DruidOperatorTable operatorTable, final ExprMacroTable macroTable, @@ -60,6 +64,41 @@ public PlannerToolbox( final PolicyEnforcer policyEnforcer, final DruidHookDispatcher hookDispatcher ) + { + this( + operatorTable, + macroTable, + jsonMapper, + plannerConfig, + rootSchema, + joinableFactoryWrapper, + catalog, + CatalogTableWriter.NOT_AVAILABLE, + druidSchemaName, + calciteRuleManager, + authorizerMapper, + authConfig, + policyEnforcer, + hookDispatcher + ); + } + + public PlannerToolbox( + final DruidOperatorTable operatorTable, + final ExprMacroTable macroTable, + final ObjectMapper jsonMapper, + final PlannerConfig plannerConfig, + final DruidSchemaCatalog rootSchema, + final JoinableFactoryWrapper joinableFactoryWrapper, + final CatalogResolver catalog, + final CatalogTableWriter catalogTableWriter, + final String druidSchemaName, + final CalciteRulesManager calciteRuleManager, + final AuthorizerMapper authorizerMapper, + final AuthConfig authConfig, + final PolicyEnforcer policyEnforcer, + final DruidHookDispatcher hookDispatcher + ) { this.operatorTable = operatorTable; this.macroTable = macroTable; @@ -68,6 +107,7 @@ public PlannerToolbox( this.rootSchema = rootSchema; this.joinableFactoryWrapper = joinableFactoryWrapper; this.catalog = catalog; + this.catalogTableWriter = catalogTableWriter; this.druidSchemaName = druidSchemaName; this.calciteRuleManager = calciteRuleManager; this.authorizerMapper = authorizerMapper; @@ -106,6 +146,11 @@ public CatalogResolver catalogResolver() return catalog; } + public CatalogTableWriter catalogTableWriter() + { + return catalogTableWriter; + } + public String druidSchemaName() { return druidSchemaName; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java new file mode 100644 index 000000000000..1efe8e0c2504 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java @@ -0,0 +1,546 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.planner; + +import com.google.common.collect.ImmutableMap; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.util.SqlBasicVisitor; +import org.apache.druid.catalog.model.ClusteredValueGroupsBaseTableMetadata; +import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.Columns; +import org.apache.druid.data.input.impl.AggregateProjectionSpec; +import org.apache.druid.data.input.impl.DimensionSchema; +import org.apache.druid.error.DruidException; +import org.apache.druid.error.InvalidSqlInput; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.DataSource; +import org.apache.druid.query.Query; +import org.apache.druid.query.QueryContexts; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.dimension.DefaultDimensionSpec; +import org.apache.druid.query.dimension.DimensionSpec; +import org.apache.druid.query.filter.AndDimFilter; +import org.apache.druid.query.filter.DimFilter; +import org.apache.druid.query.filter.RangeFilter; +import org.apache.druid.query.groupby.GroupByQuery; +import org.apache.druid.query.scan.ScanQuery; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.segment.VirtualColumn; +import org.apache.druid.segment.VirtualColumns; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.virtual.ExpressionVirtualColumn; +import org.apache.druid.server.security.AuthorizationResult; +import org.apache.druid.server.security.NoopEscalator; +import org.apache.druid.sql.calcite.rel.DruidQuery; +import org.apache.druid.sql.calcite.rel.Grouping; +import org.apache.druid.sql.calcite.table.DatasourceTable; +import org.apache.druid.sql.calcite.table.DatasourceTable.PhysicalDatasourceMetadata; +import org.apache.druid.sql.calcite.table.DruidTable; +import org.joda.time.Interval; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Turns the SQL body of a projection definition into the {@link AggregateProjectionSpec} the catalog stores. + *

+ * The body is planned through the normal pipeline against the columns the enclosing statement declares, and the + * specification is lifted out of the resulting native query. Going through the planner is the point: a projection is + * only useful if it matches the queries the planner generates at query time, and that agreement is guaranteed when + * the same machinery produces both. It also means aggregators contributed by extensions work without a second + * registry. + */ +public class ProjectionSpecTranslator +{ + /** + * The reserved projection name that describes the table's own physical layout. + */ + public static final String BASE_PROJECTION_NAME = "__base"; + + /** + * Planning is deterministic in the shapes the lift understands: no timeseries or topN rewrite to hide the grouping + * columns, and no approximation choices that depend on unrelated configuration. + */ + private static final Map CONTEXT = ImmutableMap.of( + PlannerContext.CTX_SQL_USE_GRANULARITY, false, + QueryContexts.TIME_BOUNDARY_PLANNING_KEY, false, + PlannerConfig.CTX_KEY_USE_APPROXIMATE_TOPN, false + ); + + private final PlannerFactory plannerFactory; + + public ProjectionSpecTranslator(PlannerFactory plannerFactory) + { + this.plannerFactory = plannerFactory; + } + + /** + * Translate one projection definition. + * + * @param tableName the table the projection belongs to + * @param columns the table's declared columns, which are the only ones the body may reference + */ + public AggregateProjectionSpec translate( + final String tableName, + final List columns, + final String projectionName, + final SqlSelect body + ) + { + rejectSubqueries(projectionName, body); + + final DruidQuery druidQuery = planBody(tableName, columns, projectionName, body); + return lift(projectionName, tableName, druidQuery); + } + + /** + * Translate the reserved base-table projection, which describes the physical layout of the table itself rather + * than an additional aggregate. + *

+ * The body enumerates the table's columns in the order segments store them, so it must name every declared column, + * in declared order. An item written as {@code AS } makes that column computed at ingest time: the + * expression becomes a virtual column materializing the declared column, which is why the declared type has to + * match what the expression produces. + * + * @param clusteredBy the columns segments are clustered on, which must be the leading prefix of the column list + */ + public ClusteredValueGroupsBaseTableMetadata translateBaseTable( + final String tableName, + final List columns, + final SqlSelect body, + @Nullable final SqlNodeList clusteredBy + ) + { + if (body.getWhere() != null || body.getGroup() != null) { + throw invalid( + BASE_PROJECTION_NAME, + "its body filters or groups. The base table stores every ingested row, so it can do neither" + ); + } + rejectSubqueries(BASE_PROJECTION_NAME, body); + + final DruidQuery druidQuery = planBody(tableName, columns, BASE_PROJECTION_NAME, body); + final ClusteredValueGroupsBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata( + clusteringColumns(clusteredBy), + liftComputedColumns(columns, druidQuery), + null + ); + + // Derive the physical spec now. The catalog does this too when the write lands, but doing it here attributes + // layout problems to the statement that caused them rather than to a Coordinator round trip. + try { + metadata.createSpec(columns); + } + catch (DruidException e) { + throw contextualize(BASE_PROJECTION_NAME, e); + } + return metadata; + } + + private static List clusteringColumns(@Nullable final SqlNodeList clusteredBy) + { + if (clusteredBy == null) { + return Collections.emptyList(); + } + final List names = new ArrayList<>(clusteredBy.size()); + for (SqlNode node : clusteredBy) { + if (!(node instanceof SqlIdentifier) || !((SqlIdentifier) node).isSimple()) { + throw invalid( + BASE_PROJECTION_NAME, + "its CLUSTERED BY names [" + node + "], which is not a column. Segments are clustered on stored columns;" + + " to cluster on a computed value, declare it as a column of the table" + ); + } + names.add(((SqlIdentifier) node).getSimple()); + } + return names; + } + + /** + * Pair the planned output with the declared columns and lift the virtual columns behind the computed ones. + *

+ * The planner names its virtual columns {@code v0}, {@code v1}, ...; each is renamed to the declared column it + * fills, which is what makes it a materialized column rather than an anonymous intermediate. + */ + private static VirtualColumns liftComputedColumns( + final List columns, + final DruidQuery druidQuery + ) + { + final Query query = druidQuery.getQuery(); + if (!(query instanceof ScanQuery)) { + throw invalid( + BASE_PROJECTION_NAME, + "its body does not select rows directly. The base table stores every ingested row as it arrives" + ); + } + final List selected = ((ScanQuery) query).getColumns(); + final List outputNames = druidQuery.getOutputRowType().getFieldNames(); + + if (outputNames.size() != columns.size()) { + throw invalid( + BASE_PROJECTION_NAME, + StringUtils.format( + "it selects %d column(s) but the table declares %d. The body lists the columns in the order segments" + + " store them, so it must name every declared column", + outputNames.size(), + columns.size() + ) + ); + } + + final VirtualColumns planned = ((ScanQuery) query).getVirtualColumns(); + final List materialized = new ArrayList<>(); + for (int i = 0; i < columns.size(); i++) { + final String declared = columns.get(i).name(); + if (!declared.equals(outputNames.get(i))) { + throw invalid( + BASE_PROJECTION_NAME, + StringUtils.format( + "its column %d is [%s] but the table declares [%s] there. The body lists the columns in the order" + + " segments store them", + i + 1, + outputNames.get(i), + declared + ) + ); + } + final VirtualColumn virtualColumn = planned.getVirtualColumn(selected.get(i)); + if (virtualColumn == null) { + // A plain reference: the column is ingested as it arrives. + continue; + } + if (!(virtualColumn instanceof ExpressionVirtualColumn)) { + throw invalid( + BASE_PROJECTION_NAME, + "column [" + declared + "] is computed by an expression the base table cannot store" + ); + } + final ExpressionVirtualColumn expression = (ExpressionVirtualColumn) virtualColumn; + materialized.add( + new ExpressionVirtualColumn( + declared, + expression.getExpression(), + expression.getOutputType(), + ExprMacroTable.nil() + ) + ); + } + return VirtualColumns.create(materialized); + } + + /** + * Plan the body against a table built from the declared columns. The table is synthesized rather than looked up + * because for {@code CREATE TABLE} it does not exist yet, and for {@code ALTER TABLE} the statement's own columns + * are what the projection must agree with, not whatever a possibly stale cache holds. + */ + private DruidQuery planBody( + final String tableName, + final List columns, + final String projectionName, + final SqlSelect body + ) + { + final SqlSelect query = (SqlSelect) body.clone(body.getParserPosition()); + query.setFrom(new SqlIdentifier(tableName, SqlParserPos.ZERO)); + + final ProjectionSqlEngine engine = new ProjectionSqlEngine(); + final String sql = query.toString(); + try (DruidPlanner planner = plannerFactory.createPlannerForTable( + engine, + sql, + query, + CONTEXT, + tableName, + tableFor(tableName, columns) + )) { + planner.getPlannerContext() + .setAuthenticationResult(NoopEscalator.getInstance().createEscalatedAuthenticationResult()); + planner.validate(); + planner.authorize(ra -> AuthorizationResult.ALLOW_NO_RESTRICTION, Collections.emptySet()); + planner.plan().run(); + } + catch (DruidException e) { + throw contextualize(projectionName, e); + } + return engine.captured(); + } + + /** + * Build the table the body is planned against. Mirrors how a catalog-only table is presented to the planner: + * declared columns in declared order, with {@code __time} supplied if the statement did not declare it. + */ + private static DruidTable tableFor(final String tableName, final List columns) + { + RowSignature.Builder builder = RowSignature.builder(); + boolean hasTime = false; + for (ColumnSpec column : columns) { + ColumnType type = Columns.druidType(column); + if (type == null) { + type = ColumnType.STRING; + } + if (Columns.isTimeColumn(column.name())) { + hasTime = true; + } + builder.add(column.name(), type); + } + if (!hasTime) { + builder = RowSignature.builder() + .add(Columns.TIME_COLUMN, ColumnType.LONG) + .addAll(builder.build()); + } + final RowSignature signature = builder.build(); + return new DatasourceTable( + signature, + new PhysicalDatasourceMetadata(new TableDataSource(tableName), signature, false, false), + DatasourceTable.EffectiveMetadata.of(signature) + ); + } + + /** + * Lift the projection specification out of the planned query. + */ + private static AggregateProjectionSpec lift( + final String projectionName, + final String tableName, + final DruidQuery druidQuery + ) + { + final DataSource dataSource = druidQuery.getDataSource(); + if (!(dataSource instanceof TableDataSource) || !tableName.equals(((TableDataSource) dataSource).getName())) { + throw invalid( + projectionName, + "its body requires more than one pass over the data. Rewrite it as a single aggregation, for example by" + + " using APPROX_COUNT_DISTINCT instead of COUNT(DISTINCT ...)" + ); + } + + final Grouping grouping = druidQuery.getGrouping(); + if (grouping == null) { + throw invalid(projectionName, "its body does not aggregate. Add a GROUP BY clause, or use SELECT DISTINCT"); + } + if (!grouping.getPostAggregators().isEmpty()) { + throw invalid( + projectionName, + "its body computes an expression over aggregates, which a projection cannot store. Store the aggregates" + + " themselves instead, for example SUM(x) and COUNT(x) rather than AVG(x)" + ); + } + if (grouping.getHavingFilter() != null) { + throw invalid(projectionName, "its body has a HAVING clause, which a projection cannot store"); + } + + final Query query = druidQuery.getQuery(); + final List dimensions; + final List aggregators; + final VirtualColumnsAndFilter virtualColumnsAndFilter; + if (query instanceof GroupByQuery) { + final GroupByQuery groupBy = (GroupByQuery) query; + dimensions = groupBy.getDimensions(); + aggregators = groupBy.getAggregatorSpecs(); + virtualColumnsAndFilter = new VirtualColumnsAndFilter( + groupBy.getVirtualColumns(), + groupBy.getDimFilter(), + groupBy.getIntervals() + ); + } else if (query instanceof TimeseriesQuery) { + // GROUP BY () plans to a timeseries over all time; it has no grouping columns. + final TimeseriesQuery timeseries = (TimeseriesQuery) query; + dimensions = Collections.emptyList(); + aggregators = List.of(timeseries.getAggregatorSpecs().toArray(new AggregatorFactory[0])); + virtualColumnsAndFilter = new VirtualColumnsAndFilter( + timeseries.getVirtualColumns(), + timeseries.getFilter(), + timeseries.getIntervals() + ); + } else { + throw invalid( + projectionName, + "its body did not plan to an aggregation. Add a GROUP BY clause, or use SELECT DISTINCT" + ); + } + + return AggregateProjectionSpec + .builder(projectionName) + .virtualColumns(virtualColumnsAndFilter.virtualColumns) + .filter(virtualColumnsAndFilter.filter(projectionName)) + .groupingColumns(groupingColumns(projectionName, dimensions)) + .aggregators(renameToOutputNames(projectionName, druidQuery, aggregators)) + .build(); + } + + private static List groupingColumns( + final String projectionName, + final List dimensions + ) + { + final List groupingColumns = new ArrayList<>(dimensions.size()); + for (DimensionSpec dimension : dimensions) { + if (!(dimension instanceof DefaultDimensionSpec)) { + throw invalid( + projectionName, + "its grouping column [" + dimension.getOutputName() + "] is not a plain column reference" + ); + } + final ColumnType type = dimension.getOutputType(); + if (type == null || (!type.isPrimitive() && !type.isArray())) { + throw invalid( + projectionName, + "its grouping column [" + dimension.getDimension() + "] has type [" + type + + "], which a projection cannot group on" + ); + } + // The stored name is the physical column or virtual column the planner grouped on, not the SELECT alias: + // projections are matched structurally against a query's grouping columns, not by output name. + groupingColumns.add(DimensionSchema.getDefaultSchemaForBuiltInType(dimension.getDimension(), type)); + } + return groupingColumns; + } + + /** + * Aggregators are stored under the name the projection's own column will have, which is the SELECT alias. The + * planner names them {@code a0}, {@code a1}, ... internally, so an explicit alias is required. + */ + private static AggregatorFactory[] renameToOutputNames( + final String projectionName, + final DruidQuery druidQuery, + final List aggregators + ) + { + final RowSignature internal = druidQuery.getOutputRowSignature(); + final List outputNames = druidQuery.getOutputRowType().getFieldNames(); + final AggregatorFactory[] renamed = new AggregatorFactory[aggregators.size()]; + for (int i = 0; i < aggregators.size(); i++) { + final AggregatorFactory aggregator = aggregators.get(i); + final int position = internal.indexOf(aggregator.getName()); + final String outputName = position < 0 || position >= outputNames.size() + ? aggregator.getName() + : outputNames.get(position); + if (isPlannerGeneratedName(outputName)) { + throw invalid( + projectionName, + "one of its aggregate expressions has no name. Give every aggregate an alias, for example" + + " SUM(x) AS sum_x" + ); + } + renamed[i] = aggregator.withName(outputName); + } + return renamed; + } + + private static boolean isPlannerGeneratedName(String name) + { + return name.startsWith("EXPR$"); + } + + /** + * The virtual columns, filter and intervals of the planned query. Kept together because a time filter written in + * the body is moved out of the filter and into the query's intervals during planning, and has to be put back: + * a projection has nowhere to store an interval. + */ + private static class VirtualColumnsAndFilter + { + private final VirtualColumns virtualColumns; + @Nullable + private final DimFilter dimFilter; + private final List intervals; + + VirtualColumnsAndFilter( + VirtualColumns virtualColumns, + @Nullable DimFilter dimFilter, + List intervals + ) + { + this.virtualColumns = virtualColumns; + this.dimFilter = dimFilter; + this.intervals = intervals; + } + + @Nullable + DimFilter filter(String projectionName) + { + if (intervals.size() == 1 && Intervals.ETERNITY.equals(intervals.get(0))) { + return dimFilter; + } + if (intervals.size() != 1) { + throw invalid( + projectionName, + "its WHERE clause selects more than one time range, which a projection cannot store" + ); + } + final Interval interval = intervals.get(0); + final DimFilter timeFilter = new RangeFilter( + Columns.TIME_COLUMN, + ColumnType.LONG, + interval.getStartMillis(), + interval.getEndMillis(), + false, + true, + null + ); + return dimFilter == null ? timeFilter : new AndDimFilter(timeFilter, dimFilter); + } + } + + /** + * Subqueries would be planned as a second pass over the data, which a projection cannot represent. The grammar + * cannot exclude them, because they appear inside expressions. + */ + private static void rejectSubqueries(final String projectionName, final SqlSelect body) + { + body.accept(new SqlBasicVisitor() + { + @Override + public Void visit(SqlCall call) + { + if (call instanceof SqlSelect && call != body) { + throw invalid(projectionName, "its body contains a subquery, which a projection cannot store"); + } + return super.visit(call); + } + }); + } + + private static DruidException contextualize(final String projectionName, final DruidException e) + { + if (e.getTargetPersona() == DruidException.Persona.USER) { + return InvalidSqlInput.exception(e, "Cannot define projection [%s]: %s", projectionName, e.getMessage()); + } + return e; + } + + private static DruidException invalid(final String projectionName, final String reason) + { + return InvalidSqlInput.exception("Cannot define projection [%s] because %s", projectionName, reason); + } + +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSqlEngine.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSqlEngine.java new file mode 100644 index 000000000000..e0fc7db7e1df --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSqlEngine.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.planner; + +import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.druid.error.DruidException; +import org.apache.druid.java.util.common.guava.Sequences; +import org.apache.druid.server.QueryResponse; +import org.apache.druid.sql.SqlStatementFactory; +import org.apache.druid.sql.calcite.rel.DruidQuery; +import org.apache.druid.sql.calcite.run.EngineFeature; +import org.apache.druid.sql.calcite.run.QueryMaker; +import org.apache.druid.sql.calcite.run.SqlEngine; +import org.apache.druid.sql.calcite.run.SqlEngines; +import org.apache.druid.sql.destination.IngestDestination; + +import java.util.Map; + +/** + * Engine used to plan the body of a projection definition without running it. The planned {@link DruidQuery} is + * captured so the projection specification can be lifted out of it. + *

+ * A projection is defined by SQL but stored as a native specification, and it is only useful if it matches the + * queries the planner generates at query time. Planning the body through the normal pipeline is what makes the two + * agree: the same aggregator factories, virtual column expressions and filters come out either way. + *

+ * Not a singleton, unlike {@link org.apache.druid.sql.calcite.view.ViewSqlEngine}: each instance captures one query. + */ +public class ProjectionSqlEngine implements SqlEngine +{ + private static final String NAME = "projection"; + + private DruidQuery captured; + + /** + * The query planned for the projection body, available after the statement has been planned and run. + */ + public DruidQuery captured() + { + if (captured == null) { + throw DruidException.defensive("Projection body was not planned into a native query"); + } + return captured; + } + + @Override + public String name() + { + return NAME; + } + + @Override + public boolean featureAvailable(EngineFeature feature) + { + switch (feature) { + case CAN_SELECT: + case GROUPING_SETS: + return true; + + // A projection stores grouping columns and aggregators, so the body must plan to a group-by rather than to a + // specialized query shape that would hide them. + case TIMESERIES_QUERY: + case TOPN_QUERY: + case TIME_BOUNDARY_QUERY: + case GROUPBY_IMPLICITLY_SORTS: + case ALLOW_BINDABLE_PLAN: + return false; + + // The body has no FROM clause, so it can only read the table it belongs to. + case READ_EXTERNAL_DATA: + case WRITE_EXTERNAL_DATA: + case CAN_INSERT: + case CAN_REPLACE: + case SCAN_ORDER_BY_NON_TIME: + case WINDOW_FUNCTIONS: + case WINDOW_LEAF_OPERATOR: + case UNNEST: + case ALLOW_BROADCAST_RIGHTY_JOIN: + case ALLOW_TOP_LEVEL_UNION_ALL: + return false; + + default: + throw SqlEngines.generateUnrecognizedFeatureException(ProjectionSqlEngine.class.getSimpleName(), feature); + } + } + + @Override + public void validateContext(Map queryContext) + { + // The context is supplied by the translator, not by the user. + } + + @Override + public RelDataType resultTypeForSelect( + RelDataTypeFactory typeFactory, + RelDataType validatedRowType, + Map queryContext + ) + { + return validatedRowType; + } + + @Override + public RelDataType resultTypeForInsert( + RelDataTypeFactory typeFactory, + RelDataType validatedRowType, + Map queryContext + ) + { + throw new UnsupportedOperationException(); + } + + @Override + public QueryMaker buildQueryMakerForSelect(RelRoot relRoot, PlannerContext plannerContext) + { + return druidQuery -> { + captured = druidQuery; + return QueryResponse.withEmptyContext(Sequences.empty()); + }; + } + + @Override + public QueryMaker buildQueryMakerForInsert( + IngestDestination destination, + RelRoot relRoot, + PlannerContext plannerContext + ) + { + throw new UnsupportedOperationException(); + } + + @Override + public SqlStatementFactory getSqlStatementFactory() + { + throw new UnsupportedOperationException(); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/SqlStatementHandler.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/SqlStatementHandler.java index 393363587f7d..7d490c85b975 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/SqlStatementHandler.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/SqlStatementHandler.java @@ -57,6 +57,11 @@ interface HandlerContext ObjectMapper jsonMapper(); DateTimeZone timeZone(); PlannerHook hook(); + + /** + * The factory that created the enclosing planner, for statements that need to plan a nested query of their own. + */ + PlannerFactory plannerFactory(); } abstract class BaseStatementHandler implements SqlStatementHandler diff --git a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java index 1565527f601b..8fee6e2a2f98 100644 --- a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java +++ b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java @@ -41,6 +41,7 @@ import org.apache.druid.sql.calcite.expression.builtin.QueryLookupOperatorConversion; import org.apache.druid.sql.calcite.planner.CalcitePlannerModule; import org.apache.druid.sql.calcite.planner.CatalogResolver; +import org.apache.druid.sql.calcite.planner.CatalogTableWriter; import org.apache.druid.sql.calcite.planner.PlannerFactory; import org.apache.druid.sql.calcite.run.NativeSqlEngine; import org.apache.druid.sql.calcite.run.SqlEngine; @@ -125,6 +126,9 @@ public void configure(Binder binder) // Default do-nothing catalog resolver binder.bind(CatalogResolver.class).toInstance(CatalogResolver.NULL_RESOLVER); + // Default catalog writer, which reports that catalog DDL needs the druid-catalog extension + binder.bind(CatalogTableWriter.class).toInstance(CatalogTableWriter.NOT_AVAILABLE); + // Bind the engine Multibinder.newSetBinder(binder, SqlEngine.class) .addBinding() diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java index 20c46f3a1082..7be5e70a288b 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java @@ -84,6 +84,7 @@ import org.apache.druid.sql.calcite.planner.CalciteRulesManager; import org.apache.druid.sql.calcite.planner.Calcites; import org.apache.druid.sql.calcite.planner.CatalogResolver; +import org.apache.druid.sql.calcite.planner.CatalogTableWriter; import org.apache.druid.sql.calcite.planner.DruidOperatorTable; import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerFactory; @@ -318,6 +319,7 @@ public void setUp() throws Exception binder.bind(CalciteRulesManager.class).toInstance(new CalciteRulesManager(ImmutableSet.of())); binder.bind(JoinableFactoryWrapper.class).toInstance(CalciteTests.createJoinableFactoryWrapper()); binder.bind(CatalogResolver.class).toInstance(CatalogResolver.NULL_RESOLVER); + binder.bind(CatalogTableWriter.class).toInstance(CatalogTableWriter.NOT_AVAILABLE); } ) .build(); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java new file mode 100644 index 000000000000..240af0d5c40a --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java @@ -0,0 +1,888 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.druid.catalog.model.ClusteredValueGroupsBaseTableMetadata; +import org.apache.druid.catalog.model.ColumnSpec; +import org.apache.druid.catalog.model.DatasourceProjectionMetadata; +import org.apache.druid.catalog.model.TableId; +import org.apache.druid.catalog.model.TableMetadata; +import org.apache.druid.catalog.model.TableSpec; +import org.apache.druid.catalog.model.table.ClusterKeySpec; +import org.apache.druid.catalog.model.table.DatasourceDefn; +import org.apache.druid.data.input.impl.AggregateProjectionSpec; +import org.apache.druid.error.DruidException; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.server.security.Action; +import org.apache.druid.server.security.AuthConfig; +import org.apache.druid.server.security.Resource; +import org.apache.druid.server.security.ResourceAction; +import org.apache.druid.server.security.ResourceType; +import org.apache.druid.sql.DirectStatement; +import org.apache.druid.sql.SqlQueryPlus; +import org.apache.druid.sql.calcite.CalciteCatalogDdlTest.CatalogDdlComponentSupplier; +import org.apache.druid.sql.calcite.planner.CatalogTableWriter; +import org.apache.druid.sql.calcite.planner.PlannerConfig; +import org.apache.druid.sql.calcite.util.CalciteTests; +import org.apache.druid.sql.calcite.util.SqlTestFramework.StandardComponentSupplier; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests that catalog DDL statements plan into the catalog operations they claim to, using a writer that records + * calls instead of contacting a Coordinator. + */ +@SqlTestFrameworkConfig.ComponentSupplier(CatalogDdlComponentSupplier.class) +public class CalciteCatalogDdlTest extends BaseCalciteQueryTest +{ + private static final RecordingCatalogTableWriter WRITER = new RecordingCatalogTableWriter(); + + public static class CatalogDdlComponentSupplier extends StandardComponentSupplier + { + public CatalogDdlComponentSupplier(TempDirProducer tempFolderProducer) + { + super(tempFolderProducer); + } + + @Override + public CatalogTableWriter createCatalogTableWriter() + { + return WRITER; + } + } + + @BeforeEach + public void resetWriter() + { + WRITER.reset(); + } + + @Test + public void testCreateTable() + { + execute("CREATE TABLE tbl (__time TIMESTAMP, page VARCHAR, cnt BIGINT)"); + + assertEquals(1, WRITER.calls.size()); + final RecordingCatalogTableWriter.Call call = WRITER.calls.get(0); + assertEquals("createTable", call.operation); + assertEquals(TableId.datasource("tbl"), call.tableId); + assertEquals(DatasourceDefn.TABLE_TYPE, call.spec.type()); + assertEquals(ImmutableMap.of(), call.spec.properties()); + assertEquals( + ImmutableList.of( + new ColumnSpec("__time", "TIMESTAMP", null), + new ColumnSpec("page", "VARCHAR", null), + new ColumnSpec("cnt", "BIGINT", null) + ), + call.spec.columns() + ); + assertFalse(call.ifNotExists); + assertFalse(call.replace); + } + + @Test + public void testCreateTableWithPartitioningAndClustering() + { + execute("CREATE TABLE tbl (page VARCHAR, cnt BIGINT) PARTITIONED BY DAY CLUSTERED BY page, cnt"); + + final TableSpec spec = WRITER.calls.get(0).spec; + assertEquals("P1D", spec.properties().get(DatasourceDefn.SEGMENT_GRANULARITY_PROPERTY)); + assertEquals( + ImmutableList.of(new ClusterKeySpec("page", false), new ClusterKeySpec("cnt", false)), + spec.properties().get(DatasourceDefn.CLUSTER_KEYS_PROPERTY) + ); + } + + @Test + public void testCreateTablePartitionedByAll() + { + execute("CREATE TABLE tbl (page VARCHAR) PARTITIONED BY ALL TIME"); + assertEquals("ALL", WRITER.calls.get(0).spec.properties().get(DatasourceDefn.SEGMENT_GRANULARITY_PROPERTY)); + } + + @Test + public void testCreateTableTypeCanonicalization() + { + execute( + "CREATE TABLE tbl (a CHAR, b INTEGER, c REAL, d DOUBLE, e VARCHAR ARRAY, f TYPE('complex'))" + ); + assertEquals( + ImmutableList.of( + new ColumnSpec("a", "VARCHAR", null), + new ColumnSpec("b", "BIGINT", null), + new ColumnSpec("c", "FLOAT", null), + new ColumnSpec("d", "DOUBLE", null), + new ColumnSpec("e", "VARCHAR ARRAY", null), + new ColumnSpec("f", "COMPLEX", null) + ), + WRITER.calls.get(0).spec.columns() + ); + } + + @Test + public void testCreateTableFlags() + { + execute("CREATE OR REPLACE TABLE tbl (a VARCHAR)"); + assertTrue(WRITER.calls.get(0).replace); + + WRITER.reset(); + execute("CREATE TABLE IF NOT EXISTS tbl (a VARCHAR)"); + assertTrue(WRITER.calls.get(0).ifNotExists); + } + + @Test + public void testCreateTableInDruidSchema() + { + execute("CREATE TABLE druid.tbl (a VARCHAR)"); + assertEquals(TableId.datasource("tbl"), WRITER.calls.get(0).tableId); + } + + @Test + public void testResourceActionIsDatasourceWrite() + { + final DirectStatement stmt = statement("CREATE TABLE tbl (a VARCHAR)"); + stmt.execute(); + assertEquals( + Collections.singleton(new ResourceAction(new Resource("tbl", ResourceType.DATASOURCE), Action.WRITE)), + stmt.resources() + ); + } + + @Test + public void testDdlReturnsNoRows() + { + final DirectStatement stmt = statement("CREATE TABLE tbl (a VARCHAR)"); + final List results = stmt.execute().getResults().toList(); + assertEquals(ImmutableList.of(), results); + } + + @Test + public void testAlterTableAddColumn() + { + WRITER.existing.put(TableId.datasource("tbl"), tableWithColumns("a")); + execute("ALTER TABLE tbl ADD COLUMN b BIGINT"); + + final RecordingCatalogTableWriter.Call call = WRITER.lastCall("updateColumns"); + assertEquals(ImmutableList.of(new ColumnSpec("b", "BIGINT", null)), call.columns); + } + + @Test + public void testAlterTableAddExistingColumnFails() + { + WRITER.existing.put(TableId.datasource("tbl"), tableWithColumns("a")); + final DruidException e = assertThrows( + DruidException.class, + () -> execute("ALTER TABLE tbl ADD COLUMN a BIGINT") + ); + assertTrue(e.getMessage().contains("Column [a] already exists")); + } + + @Test + public void testAlterTableAddColumnToMissingTableFails() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("ALTER TABLE tbl ADD COLUMN a BIGINT") + ); + assertTrue(e.getMessage().contains("does not have a catalog entry")); + } + + @Test + public void testAlterTableDropColumn() + { + execute("ALTER TABLE tbl DROP COLUMN gone"); + assertEquals(ImmutableList.of("gone"), WRITER.lastCall("dropColumns").droppedColumns); + } + + @Test + public void testAlterTableAlterColumn() + { + // Unlike ADD COLUMN, changing a type does not require the column to be absent, so no read is needed. + execute("ALTER TABLE tbl ALTER COLUMN cnt SET DATA TYPE DOUBLE"); + assertEquals( + ImmutableList.of(new ColumnSpec("cnt", "DOUBLE", null)), + WRITER.lastCall("updateColumns").columns + ); + } + + @Test + public void testAlterTableSetProperties() + { + execute("ALTER TABLE tbl SET PROPERTIES (targetSegmentRows = 3000000, sealed = TRUE, description = 'hi')"); + + final Map properties = WRITER.lastCall("updateProperties").properties; + assertEquals(3000000L, properties.get("targetSegmentRows")); + assertEquals(true, properties.get("sealed")); + assertEquals("hi", properties.get("description")); + } + + @Test + public void testAlterTableSetPropertyToNullRemovesIt() + { + execute("ALTER TABLE tbl SET PROPERTIES (description = NULL)"); + final Map properties = WRITER.lastCall("updateProperties").properties; + assertTrue(properties.containsKey("description")); + assertNull(properties.get("description")); + } + + @Test + public void testCreateTableRejectsDuplicateColumn() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE tbl (a VARCHAR, a BIGINT)") + ); + assertTrue(e.getMessage().contains("Column [a] is declared more than once")); + } + + @Test + public void testCreateTableRejectsUnsupportedType() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE tbl (a TYPE('NOT_A_TYPE'))") + ); + assertTrue(e.getMessage().contains("unsupported type")); + } + + /** + * Any spelling that resolves to a LONG is accepted for the time column, and is stored as written. + */ + @Test + public void testCreateTableTimeColumnSpellings() + { + execute("CREATE TABLE tbl (__time BIGINT)"); + assertEquals(ImmutableList.of(new ColumnSpec("__time", "BIGINT", null)), WRITER.calls.get(0).spec.columns()); + + WRITER.reset(); + execute("CREATE TABLE tbl (__time TYPE('LONG'))"); + assertEquals(ImmutableList.of(new ColumnSpec("__time", "LONG", null)), WRITER.calls.get(0).spec.columns()); + } + + @Test + public void testCreateTableRejectsNonLongTimeColumn() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE tbl (__time VARCHAR)") + ); + assertTrue(e.getMessage().contains("Column [__time] must have type")); + } + + @Test + public void testCreateTableRejectsNonDruidSchema() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE lookup.tbl (a VARCHAR)") + ); + assertTrue(e.getMessage().contains("is not a Druid datasource")); + } + + @Test + public void testCreateTableRejectsBothReplaceAndIfNotExists() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE OR REPLACE TABLE IF NOT EXISTS tbl (a VARCHAR)") + ); + assertTrue(e.getMessage().contains("Cannot specify both OR REPLACE and IF NOT EXISTS")); + } + + @Test + public void testCreateTableRejectsClusteringExpression() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE tbl (a VARCHAR) CLUSTERED BY a DESC") + ); + assertTrue(e.getMessage().contains("must be a column name")); + } + + /** + * The feature is off unless an operator turns it on, so that upgrading a cluster does not silently widen what a + * datasource WRITE permission allows. + */ + @Test + public void testDdlIsDisabledByDefault() + { + final DirectStatement stmt = getSqlStatementFactory(PlannerConfig.builder().build(), new AuthConfig()) + .directStatement( + SqlQueryPlus.builder("CREATE TABLE tbl (a VARCHAR)") + .auth(CalciteTests.SUPER_USER_AUTH_RESULT) + .build() + ); + final DruidException e = assertThrows(DruidException.class, stmt::execute); + assertTrue(e.getMessage().contains("druid.sql.planner.enableCatalogDdl"), e.getMessage()); + assertEquals(ImmutableList.of(), WRITER.calls); + } + + /** + * The stored specification must be the one the planner would produce for the equivalent query, since that is what + * makes a projection match at query time. Pinned as JSON so a change in planner output is visible here. + */ + @Test + public void testCreateTableWithProjection() throws Exception + { + execute( + "CREATE TABLE tbl (__time TIMESTAMP, page VARCHAR, cnt BIGINT," + + " PROJECTION daily AS (SELECT TIME_FLOOR(__time, 'P1D'), page, SUM(cnt) AS total GROUP BY 1, 2))" + ); + + assertEquals( + "[{\"spec\":{\"type\":\"aggregate\",\"name\":\"daily\"," + + "\"virtualColumns\":[{\"type\":\"expression\",\"name\":\"v0\"," + + "\"expression\":\"timestamp_floor(\\\"__time\\\",'P1D',null,'UTC')\",\"outputType\":\"LONG\"}]," + + "\"groupingColumns\":[{\"type\":\"long\",\"name\":\"v0\",\"multiValueHandling\":\"SORTED_ARRAY\"," + + "\"createBitmapIndex\":false},{\"type\":\"string\",\"name\":\"page\"," + + "\"multiValueHandling\":\"SORTED_ARRAY\",\"createBitmapIndex\":true}]," + + "\"aggregators\":[{\"type\":\"longSum\",\"name\":\"total\",\"fieldName\":\"cnt\"}]," + + "\"ordering\":[{\"columnName\":\"v0\",\"order\":\"ascending\"}," + + "{\"columnName\":\"page\",\"order\":\"ascending\"}]}}]", + projectionsJson() + ); + } + + /** + * A projection defined with TIME_FLOOR must carry a granularity the segment layer can recover, which is how the + * projection gets matched to time-grouped queries. + */ + @Test + public void testProjectionGranularityIsRecoverable() + { + execute( + "CREATE TABLE tbl (__time TIMESTAMP, page VARCHAR, cnt BIGINT," + + " PROJECTION hourly AS (SELECT TIME_FLOOR(__time, 'PT1H'), page, SUM(cnt) AS total GROUP BY 1, 2))" + ); + + final AggregateProjectionSpec spec = projection(0).getSpec(); + final String timeColumn = spec.toMetadataSchema().getTimeColumnName(); + assertEquals("v0", timeColumn); + assertEquals( + Granularities.HOUR, + Granularities.fromVirtualColumn(spec.getVirtualColumns().getVirtualColumn(timeColumn)) + ); + } + + @Test + public void testProjectionWithFilter() + { + execute( + "CREATE TABLE tbl (__time TIMESTAMP, page VARCHAR, cnt BIGINT," + + " PROJECTION filtered AS (SELECT page, SUM(cnt) AS total WHERE page <> 'skip' GROUP BY page))" + ); + assertEquals("!page = skip", projection(0).getSpec().getFilter().toString()); + } + + /** + * A time bound written in the body is moved into the query's intervals during planning, and has to be put back: + * a projection stores a filter, not an interval. + */ + @Test + public void testProjectionWithTimeFilter() + { + execute( + "CREATE TABLE tbl (__time TIMESTAMP, page VARCHAR, cnt BIGINT," + + " PROJECTION recent AS (SELECT page, SUM(cnt) AS total" + + " WHERE __time >= TIMESTAMP '2020-01-01 00:00:00' GROUP BY page))" + ); + assertNotNull(projection(0).getSpec().getFilter(), "time filter must survive as a filter"); + assertTrue(projection(0).getSpec().getFilter().getRequiredColumns().contains("__time")); + } + + @Test + public void testProjectionSelectDistinct() + { + execute("CREATE TABLE tbl (a VARCHAR, PROJECTION d AS (SELECT DISTINCT a))"); + final AggregateProjectionSpec spec = projection(0).getSpec(); + assertEquals(1, spec.getGroupingColumns().size()); + assertEquals("a", spec.getGroupingColumns().get(0).getName()); + assertEquals(0, spec.getAggregators().length); + } + + @Test + public void testMultipleProjections() + { + execute( + "CREATE TABLE tbl (a VARCHAR, b BIGINT," + + " PROJECTION p1 AS (SELECT a, SUM(b) AS s GROUP BY a)," + + " PROJECTION p2 AS (SELECT b, COUNT(*) AS c GROUP BY b))" + ); + assertEquals(List.of("p1", "p2"), List.of(projection(0).getSpec().getName(), projection(1).getSpec().getName())); + } + + @Test + public void testAlterTableAddProjection() + { + WRITER.existing.put(TableId.datasource("tbl"), tableWithColumns("a")); + execute("ALTER TABLE tbl ADD PROJECTION p AS (SELECT a, COUNT(*) AS c GROUP BY a)"); + + final RecordingCatalogTableWriter.Call call = WRITER.lastCall("addProjection"); + assertEquals("p", call.projection.getSpec().getName()); + assertFalse(call.ifNotExists); + } + + @Test + public void testAlterTableAddProjectionIfNotExists() + { + WRITER.existing.put(TableId.datasource("tbl"), tableWithColumns("a")); + execute("ALTER TABLE tbl ADD IF NOT EXISTS PROJECTION p AS (SELECT a GROUP BY a)"); + assertTrue(WRITER.lastCall("addProjection").ifNotExists); + } + + @Test + public void testAlterTableDropProjection() + { + execute("ALTER TABLE tbl DROP PROJECTION p"); + final RecordingCatalogTableWriter.Call call = WRITER.lastCall("dropProjection"); + assertEquals("p", call.projectionName); + assertFalse(call.ifExists); + + WRITER.reset(); + execute("ALTER TABLE tbl DROP PROJECTION IF EXISTS p"); + assertTrue(WRITER.lastCall("dropProjection").ifExists); + } + + @Test + public void testProjectionRejectsUnaliasedAggregate() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE tbl (a VARCHAR, b BIGINT, PROJECTION p AS (SELECT a, SUM(b) GROUP BY a))") + ); + assertTrue(e.getMessage().contains("no name"), e.getMessage()); + } + + @Test + public void testProjectionRejectsPostAggregation() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE tbl (a VARCHAR, b BIGINT, PROJECTION p AS (SELECT a, AVG(b) AS m GROUP BY a))") + ); + assertTrue(e.getMessage().contains("expression over aggregates"), e.getMessage()); + } + + @Test + public void testProjectionRejectsUnknownColumn() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE tbl (a VARCHAR, PROJECTION p AS (SELECT nope GROUP BY nope))") + ); + assertTrue(e.getMessage().contains("nope"), e.getMessage()); + } + + @Test + public void testProjectionRejectsNonAggregatingBody() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE tbl (a VARCHAR, PROJECTION p AS (SELECT a))") + ); + assertTrue(e.getMessage().contains("does not aggregate"), e.getMessage()); + } + + /** + * {@code __base} names the table's own layout and is handled separately; every other name beginning with the + * reserved prefix stays unavailable. + */ + @Test + public void testProjectionRejectsOtherReservedNames() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute("CREATE TABLE tbl (a VARCHAR, PROJECTION __other AS (SELECT a GROUP BY a))") + ); + assertTrue(e.getMessage().contains("reserved name"), e.getMessage()); + } + + @Test + public void testProjectionRejectsDuplicateName() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute( + "CREATE TABLE tbl (a VARCHAR, PROJECTION p AS (SELECT a GROUP BY a)," + + " PROJECTION p AS (SELECT a GROUP BY a))" + ) + ); + assertTrue(e.getMessage().contains("declared more than once"), e.getMessage()); + } + + @SuppressWarnings("unchecked") + private DatasourceProjectionMetadata projection(int index) + { + return ((List) WRITER.calls.get(0).spec.properties().get("projections")).get(index); + } + + private String projectionsJson() throws Exception + { + return queryFramework().queryJsonMapper() + .writeValueAsString(WRITER.calls.get(0).spec.properties().get("projections")); + } + + /** + * The reserved {@code __base} projection describes the table's own layout, so it becomes the baseTable property + * rather than one of the projections. A computed column becomes a virtual column materializing the declared column + * it fills. + */ + @Test + public void testCreateTableWithBaseProjection() throws Exception + { + execute( + "CREATE TABLE tbl (" + + " tenant VARCHAR," + + " bucket BIGINT," + + " __time TIMESTAMP," + + " user_id BIGINT," + + " PROJECTION __base AS (" + + " SELECT tenant, ABS(user_id) AS bucket, __time, user_id" + + " CLUSTERED BY tenant, bucket" + + " )" + + ") PARTITIONED BY DAY SEALED" + ); + + final TableSpec spec = WRITER.calls.get(0).spec; + assertEquals(true, spec.properties().get(DatasourceDefn.SEALED_PROPERTY)); + assertNull(spec.properties().get(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY)); + assertEquals( + "{\"clusteringColumns\":[\"tenant\",\"bucket\"]," + + "\"virtualColumns\":[{\"type\":\"expression\",\"name\":\"bucket\"," + + "\"expression\":\"abs(\\\"user_id\\\")\",\"outputType\":\"LONG\"}]," + + "\"type\":\"clusteredValueGroups\"}", + queryFramework().queryJsonMapper() + .writeValueAsString(spec.properties().get(DatasourceDefn.BASE_TABLE_PROPERTY)) + ); + } + + @Test + public void testBaseProjectionWithoutComputedColumns() + { + execute( + "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, v BIGINT," + + " PROJECTION __base AS (SELECT tenant, __time, v CLUSTERED BY tenant)) SEALED" + ); + final ClusteredValueGroupsBaseTableMetadata baseTable = baseTable(); + assertEquals(List.of("tenant"), baseTable.getClusteringColumns()); + assertEquals(0, baseTable.getVirtualColumns().getVirtualColumns().length); + } + + /** + * A base table and aggregate projections are different catalog entities and may coexist. + */ + @Test + public void testBaseProjectionAlongsideAggregateProjection() + { + execute( + "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, v BIGINT," + + " PROJECTION __base AS (SELECT tenant, __time, v CLUSTERED BY tenant)," + + " PROJECTION by_tenant AS (SELECT tenant, SUM(v) AS sum_v GROUP BY tenant)) SEALED" + ); + final TableSpec spec = WRITER.calls.get(0).spec; + assertNotNull(spec.properties().get(DatasourceDefn.BASE_TABLE_PROPERTY)); + assertEquals(1, ((List) spec.properties().get(DatasourceDefn.PROJECTIONS_KEYS_PROPERTY)).size()); + } + + @Test + public void testBaseProjectionRequiresSealed() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute( + "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP," + + " PROJECTION __base AS (SELECT tenant, __time CLUSTERED BY tenant))" + ) + ); + assertTrue(e.getMessage().contains("must be declared SEALED"), e.getMessage()); + } + + /** + * The body lists the columns in the order segments store them, so it must match the declaration exactly. + */ + @Test + public void testBaseProjectionColumnOrderMustMatch() + { + final DruidException wrongOrder = assertThrows( + DruidException.class, + () -> execute( + "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, v BIGINT," + + " PROJECTION __base AS (SELECT __time, tenant, v CLUSTERED BY tenant)) SEALED" + ) + ); + assertTrue(wrongOrder.getMessage().contains("the table declares"), wrongOrder.getMessage()); + + final DruidException missing = assertThrows( + DruidException.class, + () -> execute( + "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, v BIGINT," + + " PROJECTION __base AS (SELECT tenant, __time CLUSTERED BY tenant)) SEALED" + ) + ); + assertTrue(missing.getMessage().contains("must name every declared column"), missing.getMessage()); + } + + /** + * Clustering columns must lead the declared column list, since the declared order is the physical order. The + * catalog enforces this on write; catching it here names the statement that caused it. + */ + @Test + public void testBaseProjectionClusteringMustBeLeadingPrefix() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute( + "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, v BIGINT," + + " PROJECTION __base AS (SELECT tenant, __time, v CLUSTERED BY v)) SEALED" + ) + ); + assertTrue(e.getMessage().contains("__base"), e.getMessage()); + } + + @Test + public void testBaseProjectionRejectsFilterOrGrouping() + { + for (String body : new String[]{ + "SELECT tenant, __time WHERE tenant <> 'x'", + "SELECT tenant, __time GROUP BY tenant, __time" + }) { + final DruidException e = assertThrows( + DruidException.class, + () -> execute( + "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, PROJECTION __base AS (" + body + ")) SEALED" + ), + body + ); + assertTrue(e.getMessage().contains("filters or groups"), e.getMessage()); + } + } + + /** + * Only the base projection chooses a clustering; an aggregate projection is ordered by its grouping columns. + */ + @Test + public void testAggregateProjectionRejectsClusteredBy() + { + final DruidException e = assertThrows( + DruidException.class, + () -> execute( + "CREATE TABLE tbl (a VARCHAR, b BIGINT," + + " PROJECTION p AS (SELECT a, SUM(b) AS s GROUP BY a CLUSTERED BY a))" + ) + ); + assertTrue(e.getMessage().contains("cannot use CLUSTERED BY"), e.getMessage()); + } + + @Test + public void testAlterTableAddBaseProjection() + { + WRITER.existing.put( + TableId.datasource("tbl"), + TableMetadata.newTable( + TableId.datasource("tbl"), + new TableSpec( + DatasourceDefn.TABLE_TYPE, + ImmutableMap.of(DatasourceDefn.SEALED_PROPERTY, true), + List.of( + new ColumnSpec("tenant", "VARCHAR", null), + new ColumnSpec("__time", "TIMESTAMP", null) + ) + ) + ) + ); + execute("ALTER TABLE tbl ADD PROJECTION __base AS (SELECT tenant, __time CLUSTERED BY tenant)"); + + final RecordingCatalogTableWriter.Call call = WRITER.lastCall("updateProperties"); + assertNotNull(call.properties.get(DatasourceDefn.BASE_TABLE_PROPERTY)); + } + + @Test + public void testAlterTableDropBaseProjection() + { + WRITER.existing.put( + TableId.datasource("tbl"), + TableMetadata.newTable( + TableId.datasource("tbl"), + new TableSpec( + DatasourceDefn.TABLE_TYPE, + ImmutableMap.of(DatasourceDefn.BASE_TABLE_PROPERTY, ImmutableMap.of("type", "clusteredValueGroups")), + List.of(new ColumnSpec("tenant", "VARCHAR", null)) + ) + ) + ); + execute("ALTER TABLE tbl DROP PROJECTION __base"); + + final RecordingCatalogTableWriter.Call call = WRITER.lastCall("updateProperties"); + assertTrue(call.properties.containsKey(DatasourceDefn.BASE_TABLE_PROPERTY)); + assertNull(call.properties.get(DatasourceDefn.BASE_TABLE_PROPERTY)); + + // Dropping one that is not there is an error unless tolerated. + WRITER.reset(); + assertThrows(DruidException.class, () -> execute("ALTER TABLE tbl DROP PROJECTION __base")); + execute("ALTER TABLE tbl DROP PROJECTION IF EXISTS __base"); + } + + private ClusteredValueGroupsBaseTableMetadata baseTable() + { + return (ClusteredValueGroupsBaseTableMetadata) + WRITER.calls.get(0).spec.properties().get(DatasourceDefn.BASE_TABLE_PROPERTY); + } + + private void execute(String sql) + { + statement(sql).execute(); + } + + private DirectStatement statement(String sql) + { + return getSqlStatementFactory(PlannerConfig.builder().enableCatalogDdl(true).build(), new AuthConfig()) + .directStatement( + SqlQueryPlus.builder(sql).auth(CalciteTests.SUPER_USER_AUTH_RESULT).build() + ); + } + + private static TableMetadata tableWithColumns(String... names) + { + final List columns = new ArrayList<>(); + for (String name : names) { + columns.add(new ColumnSpec(name, "VARCHAR", null)); + } + return TableMetadata.newTable( + TableId.datasource("tbl"), + new TableSpec(DatasourceDefn.TABLE_TYPE, ImmutableMap.of(), columns) + ); + } + + /** + * Records what a DDL statement asked the catalog to do, so tests can assert on the resulting operation rather + * than on a Coordinator round trip. + */ + private static class RecordingCatalogTableWriter implements CatalogTableWriter + { + static class Call + { + String operation; + TableId tableId; + TableSpec spec; + boolean ifNotExists; + boolean replace; + List columns; + List droppedColumns; + Map properties; + DatasourceProjectionMetadata projection; + String projectionName; + boolean ifExists; + } + + final List calls = new ArrayList<>(); + final Map existing = new HashMap<>(); + + void reset() + { + calls.clear(); + existing.clear(); + } + + Call lastCall(String operation) + { + for (int i = calls.size() - 1; i >= 0; i--) { + if (operation.equals(calls.get(i).operation)) { + return calls.get(i); + } + } + throw new AssertionError("No call to [" + operation + "] in " + calls); + } + + @Override + public void createTable(TableId tableId, TableSpec spec, boolean ifNotExists, boolean replace) + { + final Call call = record("createTable", tableId); + call.spec = spec; + call.ifNotExists = ifNotExists; + call.replace = replace; + } + + @Override + public void updateColumns(TableId tableId, List columns) + { + record("updateColumns", tableId).columns = columns; + } + + @Override + public void dropColumns(TableId tableId, List columns) + { + record("dropColumns", tableId).droppedColumns = columns; + } + + @Override + public void updateProperties(TableId tableId, Map properties) + { + record("updateProperties", tableId).properties = properties; + } + + @Override + public void addProjection(TableId tableId, DatasourceProjectionMetadata projection, boolean ifNotExists) + { + final Call call = record("addProjection", tableId); + call.projection = projection; + call.ifNotExists = ifNotExists; + } + + @Override + public void dropProjection(TableId tableId, String projectionName, boolean ifExists) + { + final Call call = record("dropProjection", tableId); + call.projectionName = projectionName; + call.ifExists = ifExists; + } + + @Nullable + @Override + public TableMetadata readTable(TableId tableId) + { + return existing.get(tableId); + } + + private Call record(String operation, TableId tableId) + { + final Call call = new Call(); + call.operation = operation; + call.tableId = tableId; + calls.add(call); + return call; + } + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java b/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java index 25972ae556ca..4a65bcecefa7 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java @@ -94,7 +94,9 @@ enum Modes AGGREGATION_NOT_SUPPORT_TYPE(Scope.WINDOWING, DruidException.class, "Aggregation \\[(MIN|MAX)\\] does not support type \\[STRING\\]"), ALLDATA_CSV(Scope.WINDOWING, DruidException.class, "allData.csv"), BIGINT_TIME_COMPARE(Scope.WINDOWING, DruidException.class, "Cannot apply '.' to arguments of type"), - VIEWS_NOT_SUPPORTED(Scope.WINDOWING, DruidException.class, "Incorrect syntax near the keyword 'CREATE'"), + // CREATE starts a catalog DDL statement, so the parser now gets as far as the object being created before + // failing, rather than rejecting the CREATE keyword outright. + VIEWS_NOT_SUPPORTED(Scope.WINDOWING, DruidException.class, "Received an unexpected token \\[VIEW\\]"), RESULT_MISMATCH(Scope.WINDOWING, AssertionError.class, "(assertResulEquals|AssertionError: column content mismatch)"), LONG_CASTING(Scope.WINDOWING, AssertionError.class, "expected: java.lang.Long"), UNSUPPORTED_NULL_ORDERING(Scope.WINDOWING, DruidException.class, "(A|DE)SCENDING ordering with NULLS (LAST|FIRST)"), diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java b/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java index f950f933a7bd..3fdb4b313234 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java @@ -385,6 +385,7 @@ public static class ConfigurationInstance SqlTestFramework.Builder builder = new SqlTestFramework.Builder(testHost) .withConfig(config) .catalogResolver(testHost.createCatalogResolver()) + .catalogTableWriter(testHost.createCatalogTableWriter()) .mergeBufferCount(config.numMergeBuffers) .withOverrideModule(config.resultCache.makeModule()); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java new file mode 100644 index 000000000000..f88440deea30 --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java @@ -0,0 +1,529 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.parser; + +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlSetOption; +import org.apache.calcite.sql.dialect.CalciteSqlDialect; +import org.apache.calcite.sql.parser.SqlParseException; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.druid.error.DruidException; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.junit.jupiter.api.Test; + +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Parser coverage for the catalog DDL statements: {@code CREATE TABLE} and {@code ALTER TABLE}. + */ +public class DruidSqlDdlParserTest +{ + @Test + public void testCreateTableMinimal() + { + final DruidSqlCreateTable create = parseCreate("CREATE TABLE tbl (a VARCHAR, b BIGINT)"); + + assertEquals("tbl", create.getName().toString()); + assertFalse(create.getReplace()); + assertFalse(create.isIfNotExists()); + assertNull(create.getPartitionedBy()); + assertNull(create.getClusteredBy()); + assertEquals("a VARCHAR, b BIGINT", columnsOf(create)); + } + + @Test + public void testCreateTableWithSchemaQualifiedName() + { + final DruidSqlCreateTable create = parseCreate("CREATE TABLE \"druid\".tbl (a VARCHAR)"); + assertEquals("druid.tbl", create.getName().toString()); + } + + @Test + public void testCreateTableWithoutColumns() + { + // A catalog entry that only carries properties is legal. + final DruidSqlCreateTable create = parseCreate("CREATE TABLE tbl PARTITIONED BY DAY"); + assertEquals(0, create.getColumnList().size()); + assertEquals(Granularities.DAY, create.getPartitionedBy().getGranularity()); + } + + @Test + public void testCreateTableAllClauses() + { + final DruidSqlCreateTable create = parseCreate( + "CREATE OR REPLACE TABLE \"druid\".sales (\n" + + " __time TIMESTAMP,\n" + + " page VARCHAR NOT NULL,\n" + + " cnt BIGINT,\n" + + " vals DOUBLE ARRAY,\n" + + " usr TYPE('COMPLEX')\n" + + ")\n" + + "PARTITIONED BY HOUR\n" + + "CLUSTERED BY page, cnt" + ); + + assertTrue(create.getReplace()); + assertFalse(create.isIfNotExists()); + // Calcite renders a user-defined type name as a quoted identifier; TYPE('...') round-trips correctly through + // unparse, which is what actually matters (see testUnparseRoundTrip). + assertEquals( + "__time TIMESTAMP, page VARCHAR, cnt BIGINT, vals DOUBLE ARRAY, usr `COMPLEX`", + columnsOf(create) + ); + assertEquals(Granularities.HOUR, create.getPartitionedBy().getGranularity()); + assertEquals("`page`, `cnt`", create.getClusteredBy().toString()); + } + + @Test + public void testCreateTableIfNotExists() + { + final DruidSqlCreateTable create = parseCreate("CREATE TABLE IF NOT EXISTS tbl (a VARCHAR)"); + assertTrue(create.isIfNotExists()); + assertFalse(create.getReplace()); + } + + @Test + public void testCreateTableExpressionGranularity() + { + final DruidSqlCreateTable create = parseCreate("CREATE TABLE tbl (a VARCHAR) PARTITIONED BY FLOOR(__time TO HOUR)"); + assertEquals(Granularities.HOUR, create.getPartitionedBy().getGranularity()); + } + + @Test + public void testAlterTableAddColumn() + { + final DruidSqlAlterTable.AddColumn alter = parseAlter( + "ALTER TABLE tbl ADD COLUMN added DOUBLE", + DruidSqlAlterTable.AddColumn.class + ); + assertEquals("tbl", alter.getName().toString()); + assertEquals("added", alter.getColumn().getName().toString()); + assertEquals("DOUBLE", alter.getColumn().getDataType().toString()); + } + + @Test + public void testAlterTableDropColumn() + { + final DruidSqlAlterTable.DropColumn alter = parseAlter( + "ALTER TABLE tbl DROP COLUMN gone", + DruidSqlAlterTable.DropColumn.class + ); + assertEquals("gone", alter.getColumn().toString()); + } + + @Test + public void testAlterTableAlterColumn() + { + final DruidSqlAlterTable.AlterColumn alter = parseAlter( + "ALTER TABLE tbl ALTER COLUMN cnt SET DATA TYPE DOUBLE", + DruidSqlAlterTable.AlterColumn.class + ); + assertEquals("cnt", alter.getColumn().getName().toString()); + assertEquals("DOUBLE", alter.getColumn().getDataType().toString()); + } + + @Test + public void testAlterTableAlterColumnToComplexType() + { + final DruidSqlAlterTable.AlterColumn alter = parseAlter( + "ALTER TABLE tbl ALTER COLUMN payload SET DATA TYPE TYPE('COMPLEX')", + DruidSqlAlterTable.AlterColumn.class + ); + assertEquals("COMPLEX", alter.getColumn().getDataType().getTypeName().toString()); + } + + @Test + public void testAlterTableSetProperties() + { + final DruidSqlAlterTable.SetProperties alter = parseAlter( + "ALTER TABLE tbl SET PROPERTIES (targetSegmentRows = 3000000, sealed = TRUE, description = NULL)", + DruidSqlAlterTable.SetProperties.class + ); + assertEquals(3, alter.getProperties().size()); + assertEquals( + "targetSegmentRows = 3000000, sealed = TRUE, description = NULL", + alter.getProperties() + .stream() + .map(p -> { + final DruidSqlPropertyAssignment assignment = (DruidSqlPropertyAssignment) p; + return assignment.getKey() + " = " + assignment.getValue(); + }) + .collect(Collectors.joining(", ")) + ); + } + + @Test + public void testCreateTableWithProjection() + { + final DruidSqlCreateTable create = parseCreate( + "CREATE TABLE events (\n" + + " __time TIMESTAMP,\n" + + " user_id VARCHAR,\n" + + " pages_visited BIGINT,\n" + + " PROJECTION daily_visits AS (\n" + + " SELECT TIME_FLOOR(__time, 'P1D'), user_id, SUM(pages_visited) AS total\n" + + " WHERE user_id IS NOT NULL\n" + + " GROUP BY 1, 2\n" + + " )\n" + + ")" + ); + + assertEquals("__time TIMESTAMP, user_id VARCHAR, pages_visited BIGINT", columnsOf(create)); + assertEquals(1, create.getProjectionList().size()); + + final SqlProjectionSpec projection = (SqlProjectionSpec) create.getProjectionList().get(0); + assertEquals("daily_visits", projection.getName().toString()); + assertNull(projection.getBody().getFrom(), "projection body must have no FROM clause"); + assertEquals(3, projection.getBody().getSelectList().size()); + assertNotNull(projection.getBody().getWhere()); + assertEquals(2, projection.getBody().getGroup().size()); + } + + @Test + public void testCreateTableProjectionWithoutAs() + { + // ClickHouse spells this without AS; both are accepted. + final DruidSqlCreateTable create = parseCreate( + "CREATE TABLE t (a VARCHAR, PROJECTION p (SELECT a, COUNT(*) AS c GROUP BY a))" + ); + assertEquals(1, create.getProjectionList().size()); + assertEquals("a VARCHAR", columnsOf(create)); + } + + @Test + public void testCreateTableMultipleProjections() + { + final DruidSqlCreateTable create = parseCreate( + "CREATE TABLE t (a VARCHAR, b BIGINT," + + " PROJECTION p1 AS (SELECT a, SUM(b) AS s GROUP BY a)," + + " PROJECTION p2 AS (SELECT b, COUNT(*) AS c GROUP BY b))" + ); + assertEquals(2, create.getProjectionList().size()); + assertEquals("a VARCHAR, b BIGINT", columnsOf(create)); + } + + /** + * A column may be named "projection": the keyword is non-reserved, and a projection definition is told apart by + * its third token, which is always '(' or AS. + */ + @Test + public void testColumnNamedProjection() + { + assertEquals("projection VARCHAR", columnsOf(parseCreate("CREATE TABLE t (projection VARCHAR)"))); + // A bare-identifier type is the case two tokens of lookahead could not resolve. Calcite renders such a type as + // a quoted identifier. + assertEquals("projection `LONG`", columnsOf(parseCreate("CREATE TABLE t (projection LONG)"))); + assertEquals( + "a VARCHAR, projection `LONG`", + columnsOf(parseCreate("CREATE TABLE t (a VARCHAR, projection LONG)")) + ); + + final DruidSqlCreateTable both = parseCreate( + "CREATE TABLE t (projection LONG, PROJECTION projection AS (SELECT projection GROUP BY projection))" + ); + assertEquals("projection `LONG`", columnsOf(both)); + assertEquals(1, both.getProjectionList().size()); + } + + @Test + public void testAlterTableAddProjection() + { + final DruidSqlAlterTable.AddProjection alter = parseAlter( + "ALTER TABLE t ADD PROJECTION p AS (SELECT a, SUM(b) AS s GROUP BY a)", + DruidSqlAlterTable.AddProjection.class + ); + assertEquals("t", alter.getName().toString()); + assertEquals("p", alter.getProjection().getName().toString()); + assertFalse(alter.isIfNotExists()); + } + + @Test + public void testAlterTableAddProjectionIfNotExists() + { + final DruidSqlAlterTable.AddProjection alter = parseAlter( + "ALTER TABLE t ADD IF NOT EXISTS PROJECTION p AS (SELECT a GROUP BY a)", + DruidSqlAlterTable.AddProjection.class + ); + assertTrue(alter.isIfNotExists()); + } + + @Test + public void testAlterTableDropProjection() + { + final DruidSqlAlterTable.DropProjection alter = parseAlter( + "ALTER TABLE t DROP PROJECTION p", + DruidSqlAlterTable.DropProjection.class + ); + assertEquals("p", alter.getProjectionName().toString()); + assertFalse(alter.isIfExists()); + + assertTrue( + parseAlter("ALTER TABLE t DROP PROJECTION IF EXISTS p", DruidSqlAlterTable.DropProjection.class) + .isIfExists() + ); + } + + /** + * A projection has no way to express ordering or limits, so the grammar excludes them rather than validating them + * away later. + */ + @Test + public void testProjectionBodyRejectsUnsupportedClauses() + { + for (String body : new String[]{ + "SELECT a GROUP BY a ORDER BY a", + "SELECT a GROUP BY a LIMIT 10", + "SELECT a GROUP BY a HAVING COUNT(*) > 1", + "SELECT a FROM other GROUP BY a", + "SELECT a GROUP BY a UNION ALL SELECT b GROUP BY b" + }) { + assertThrows( + DruidException.class, + () -> parse("CREATE TABLE t (a VARCHAR, PROJECTION p AS (" + body + "))"), + body + ); + } + } + + @Test + public void testCreateTableWithBaseProjectionAndSealed() + { + final DruidSqlCreateTable create = parseCreate( + "CREATE TABLE t (\n" + + " tenant VARCHAR,\n" + + " bucket BIGINT,\n" + + " __time TIMESTAMP,\n" + + " PROJECTION __base AS (\n" + + " SELECT tenant, ABS(user_id) AS bucket, __time\n" + + " CLUSTERED BY tenant, bucket\n" + + " )\n" + + ") PARTITIONED BY DAY SEALED" + ); + + assertTrue(create.isSealed()); + assertEquals(1, create.getProjectionList().size()); + + final SqlProjectionSpec base = (SqlProjectionSpec) create.getProjectionList().get(0); + assertEquals("__base", base.getName().toString()); + assertEquals("`tenant`, `bucket`", base.getClusteredBy().toString()); + assertNull(base.getBody().getGroup()); + } + + @Test + public void testSealedWithoutProjection() + { + assertTrue(parseCreate("CREATE TABLE t (a VARCHAR) SEALED").isSealed()); + assertFalse(parseCreate("CREATE TABLE t (a VARCHAR)").isSealed()); + } + + /** + * SEALED is a non-reserved keyword, so it remains usable as an identifier. + */ + @Test + public void testSealedUsableAsIdentifier() + { + assertEquals("sealed VARCHAR", columnsOf(parseCreate("CREATE TABLE t (sealed VARCHAR)"))); + assertTrue(parseCreate("CREATE TABLE sealed (a VARCHAR) SEALED").isSealed()); + } + + @Test + public void testAlterTableAddBaseProjection() + { + final DruidSqlAlterTable.AddProjection alter = parseAlter( + "ALTER TABLE t ADD PROJECTION __base AS (SELECT a, __time CLUSTERED BY a)", + DruidSqlAlterTable.AddProjection.class + ); + assertEquals("__base", alter.getProjection().getName().toString()); + assertEquals("`a`", alter.getProjection().getClusteredBy().toString()); + } + + /** + * DDL nodes must round-trip through {@link SqlNode#unparse}, which is what makes them safe to log and re-print. + */ + @Test + public void testUnparseRoundTrip() + { + assertUnparseRoundTrips("CREATE TABLE \"tbl\" (\"a\" VARCHAR, \"b\" BIGINT)"); + assertUnparseRoundTrips("CREATE OR REPLACE TABLE \"tbl\" (\"a\" VARCHAR)"); + assertUnparseRoundTrips("CREATE TABLE IF NOT EXISTS \"tbl\" (\"a\" VARCHAR)"); + assertUnparseRoundTrips("CREATE TABLE \"tbl\" (\"a\" VARCHAR) PARTITIONED BY DAY"); + assertUnparseRoundTrips("CREATE TABLE \"tbl\" (\"a\" VARCHAR) PARTITIONED BY DAY CLUSTERED BY \"a\""); + assertUnparseRoundTrips("CREATE TABLE \"tbl\" (\"p\" TYPE('COMPLEX'))"); + assertUnparseRoundTrips("ALTER TABLE \"tbl\" ADD COLUMN \"a\" DOUBLE"); + assertUnparseRoundTrips("ALTER TABLE \"tbl\" DROP COLUMN \"a\""); + assertUnparseRoundTrips("ALTER TABLE \"tbl\" ALTER COLUMN \"a\" SET DATA TYPE BIGINT"); + assertUnparseRoundTrips("ALTER TABLE \"tbl\" SET PROPERTIES (\"sealed\" = TRUE)"); + assertUnparseRoundTrips("CREATE TABLE \"tbl\" (\"a\" VARCHAR) SEALED"); + } + + /** + * Calcite clones a node by asking its operator to rebuild it from its operand list, which is how shuttles rewrite + * a statement. The operand order is hand-written per node, so a round trip is what proves it is right. + */ + @Test + public void testCloneRoundTrip() + { + for (String sql : new String[]{ + "CREATE TABLE \"tbl\" (\"a\" VARCHAR, \"b\" BIGINT)", + "CREATE OR REPLACE TABLE \"tbl\" (\"a\" VARCHAR)", + "CREATE TABLE IF NOT EXISTS \"tbl\" (\"a\" VARCHAR)", + "CREATE TABLE \"tbl\" (\"a\" VARCHAR) PARTITIONED BY DAY CLUSTERED BY \"a\"", + "CREATE TABLE \"tbl\" (\"a\" VARCHAR) SEALED", + "CREATE TABLE \"tbl\" (\"a\" VARCHAR, PROJECTION \"p\" AS (SELECT \"a\" GROUP BY \"a\"))", + "CREATE TABLE \"tbl\" (\"a\" VARCHAR, PROJECTION \"__base\" AS (SELECT \"a\" CLUSTERED BY \"a\")) SEALED", + "ALTER TABLE \"tbl\" ADD COLUMN \"a\" DOUBLE", + "ALTER TABLE \"tbl\" DROP COLUMN \"a\"", + "ALTER TABLE \"tbl\" ALTER COLUMN \"a\" SET DATA TYPE BIGINT", + "ALTER TABLE \"tbl\" ADD PROJECTION \"p\" AS (SELECT \"a\" GROUP BY \"a\")", + "ALTER TABLE \"tbl\" ADD IF NOT EXISTS PROJECTION \"p\" AS (SELECT \"a\" GROUP BY \"a\")", + "ALTER TABLE \"tbl\" DROP PROJECTION \"p\"", + "ALTER TABLE \"tbl\" DROP PROJECTION IF EXISTS \"p\"", + "ALTER TABLE \"tbl\" SET PROPERTIES (\"sealed\" = TRUE)" + }) { + final SqlNode node = parse(sql); + final SqlNode clone = node.clone(node.getParserPosition()); + assertEquals( + node.toSqlString(CalciteSqlDialect.DEFAULT).getSql(), + clone.toSqlString(CalciteSqlDialect.DEFAULT).getSql(), + sql + ); + } + } + + @Test + public void testDdlAfterSetStatement() + { + final SqlNode node = parse("SET sqlQueryId = 'abc'; CREATE TABLE tbl (a VARCHAR)"); + assertInstanceOf(DruidSqlCreateTable.class, node); + } + + @Test + public void testDdlWithTrailingSemicolon() + { + assertInstanceOf(DruidSqlCreateTable.class, parse("CREATE TABLE tbl (a VARCHAR);")); + assertInstanceOf(DruidSqlAlterTable.AddColumn.class, parse("ALTER TABLE tbl ADD COLUMN a VARCHAR;")); + } + + @Test + public void testDdlBeforeAnotherStatementIsRejected() + { + final DruidException e = assertThrows( + DruidException.class, + () -> parse("CREATE TABLE tbl (a VARCHAR); SELECT 1") + ); + assertTrue(e.getMessage().contains("Only SET statements can appear before the final statement")); + } + + /** + * {@code ALTER SYSTEM}/{@code ALTER SESSION} must keep working: {@code ALTER TABLE} is dispatched by a two-token + * lookahead ahead of Calcite's stock {@code SqlAlter()} production. + */ + @Test + public void testAlterSystemStillParses() throws SqlParseException + { + // Parsed directly rather than through DruidSqlParser.parse, which folds SET options into the query context and + // then requires a non-SET statement to execute. + assertInstanceOf(SqlSetOption.class, parseStatementList("ALTER SYSTEM SET \"a\" = 1").get(0)); + assertInstanceOf(SqlSetOption.class, parseStatementList("ALTER SESSION SET \"a\" = 1").get(0)); + } + + /** + * {@code IF} and {@code PROPERTIES} are added as non-reserved keywords, so they must remain usable as identifiers. + */ + @Test + public void testNewKeywordsRemainUsableAsIdentifiers() + { + final DruidSqlCreateTable create = parseCreate("CREATE TABLE properties (if VARCHAR, properties BIGINT)"); + assertEquals("properties", create.getName().toString()); + assertEquals("if VARCHAR, properties BIGINT", columnsOf(create)); + } + + @Test + public void testExplainOfDdlIsRejected() + { + assertThrows(DruidException.class, () -> parse("EXPLAIN PLAN FOR CREATE TABLE tbl (a VARCHAR)")); + } + + @Test + public void testCreateTableWithoutTypeIsRejected() + { + assertThrows(DruidException.class, () -> parse("CREATE TABLE tbl (a)")); + } + + @Test + public void testAlterTableWithoutOperationIsRejected() + { + assertThrows(DruidException.class, () -> parse("ALTER TABLE tbl")); + } + + @Test + public void testDropTableIsNotSupported() + { + // DROP TABLE is deliberately unclaimed; it must not silently parse as something else. + assertThrows(DruidException.class, () -> parse("DROP TABLE tbl")); + } + + private static void assertUnparseRoundTrips(String sql) + { + final SqlNode node = parse(sql); + assertEquals(sql, StringUtils.replace(node.toSqlString(CalciteSqlDialect.DEFAULT).getSql(), "\n", " ")); + } + + private static String columnsOf(DruidSqlCreateTable create) + { + return create.getColumnList() + .stream() + .map(c -> { + final DruidSqlColumnDeclaration column = (DruidSqlColumnDeclaration) c; + return column.getName() + " " + column.getDataType(); + }) + .collect(Collectors.joining(", ")); + } + + private static SqlNode parse(String sql) + { + return DruidSqlParser.parse(sql, true).getMainStatement(); + } + + private static SqlNodeList parseStatementList(String sql) throws SqlParseException + { + return (SqlNodeList) SqlParser.create(sql, DruidSqlParser.PARSER_CONFIG).parseStmtList(); + } + + private static DruidSqlCreateTable parseCreate(String sql) + { + return assertInstanceOf(DruidSqlCreateTable.class, parse(sql)); + } + + private static T parseAlter(String sql, Class clazz) + { + return assertInstanceOf(clazz, parse(sql)); + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModuleTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModuleTest.java index c283a364d1af..0df3eb6aa533 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModuleTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModuleTest.java @@ -140,6 +140,7 @@ public void onMatch(RelOptRuleCall call) binder.bind(DruidSchemaCatalog.class).toInstance(rootSchema); binder.bind(JoinableFactoryWrapper.class).toInstance(joinableFactoryWrapper); binder.bind(CatalogResolver.class).toInstance(CatalogResolver.NULL_RESOLVER); + binder.bind(CatalogTableWriter.class).toInstance(CatalogTableWriter.NOT_AVAILABLE); }, target, binder -> { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 487c50570f3b..70b5b3662f99 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -104,6 +104,7 @@ import org.apache.druid.sql.calcite.TempDirProducer; import org.apache.druid.sql.calcite.planner.CalciteRulesManager; import org.apache.druid.sql.calcite.planner.CatalogResolver; +import org.apache.druid.sql.calcite.planner.CatalogTableWriter; import org.apache.druid.sql.calcite.planner.DruidOperatorTable; import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerFactory; @@ -220,6 +221,11 @@ default CatalogResolver createCatalogResolver() return CatalogResolver.NULL_RESOLVER; } + default CatalogTableWriter createCatalogTableWriter() + { + return CatalogTableWriter.NOT_AVAILABLE; + } + /** * Configure the JSON mapper. */ @@ -751,6 +757,7 @@ public static class Builder private final QueryComponentSupplier componentSupplier; private int mergeBufferCount; private CatalogResolver catalogResolver = CatalogResolver.NULL_RESOLVER; + private CatalogTableWriter catalogTableWriter = CatalogTableWriter.NOT_AVAILABLE; private List overrideModules = new ArrayList<>(); private SqlTestFrameworkConfig config; private Closer resourceCloser = Closer.create(); @@ -772,6 +779,12 @@ public Builder catalogResolver(CatalogResolver catalogResolver) return this; } + public Builder catalogTableWriter(CatalogTableWriter catalogTableWriter) + { + this.catalogTableWriter = catalogTableWriter; + return this; + } + public Builder withOverrideModule(Module m) { this.overrideModules.add(m); @@ -799,6 +812,11 @@ public CatalogResolver getCatalogResolver() return catalogResolver; } + public CatalogTableWriter getCatalogTableWriter() + { + return catalogTableWriter; + } + public Closer getResourceCloser() { return resourceCloser; @@ -848,6 +866,7 @@ public PlannerFixture( new CalciteRulesManager(componentSupplier.extensionCalciteRules()), framework.injector.getInstance(JoinableFactoryWrapper.class), framework.builder.catalogResolver, + framework.builder.catalogTableWriter, authConfig != null ? authConfig : new AuthConfig(), NoopPolicyEnforcer.instance(), new DruidHookDispatcher()