diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c1e1680c0..c0a704fb1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,6 +39,11 @@ For bugs, please include: or any user-facing SQL behavior; running `make test` regenerates the diff. The `QuidemTestBase` class in `hoptimator-jdbc` is the shared harness. + + Behavior that isn't SQL — e.g. the direct `TableService.create(...)` + API — is covered by plain JUnit integration tests tagged + `@Tag("integration")` (see the `*TableServiceIntegrationTest` classes), + which drive the new APIs directly instead of routing through Quidem. 3. **Cover your changes.** New code needs tests; don't ship behavior that isn't exercised by either a unit or an integration test. Generate a coverage report with: diff --git a/deploy/docker/venice/docker-compose-single-dc-setup.yaml b/deploy/docker/venice/docker-compose-single-dc-setup.yaml index 9c979b68e..ea30a41ab 100644 --- a/deploy/docker/venice/docker-compose-single-dc-setup.yaml +++ b/deploy/docker/venice/docker-compose-single-dc-setup.yaml @@ -32,6 +32,14 @@ services: image: venicedb/venice-controller:0.4.858 container_name: venice-controller hostname: venice-controller + # Local-only override: disable Venice's default 6h store-recreation cooldown + # (controller.store.recreation.after.deletion.time.window.seconds, default 21600) so + # integration tests that create and delete Venice stores are re-runnable without waiting. + # Appends the override to the image's baked-in controller.properties, then launches normally. + command: + - sh + - -c + - echo "controller.store.recreation.after.deletion.time.window.seconds=0" >> configs/single-dc/controller.properties && java -jar bin/venice-controller-all.jar configs/single-dc/cluster.properties configs/single-dc/controller.properties depends_on: kafka: condition: service_healthy diff --git a/docs/extending/config-providers.md b/docs/extending/config-providers.md index 09de1527b..b4ebb2979 100644 --- a/docs/extending/config-providers.md +++ b/docs/extending/config-providers.md @@ -30,7 +30,7 @@ don't need a custom provider. ```java public interface ConfigProvider { - Properties loadConfig(Connection connection) throws Exception; + Properties loadConfig(DeploymentContext context) throws Exception; } ``` @@ -38,9 +38,9 @@ One method. Return a `Properties` populated however you like; throw if loading fails — Hoptimator will log it and continue with whatever the other providers produced. -The `Connection` argument is the active Hoptimator JDBC connection. -`HoptimatorConnection.connectionProperties()` and the bundled -`K8sContext.create(connection)` are useful when you want to scope your +The `DeploymentContext` argument is a Calcite-free handle onto the caller's +environment. `context.properties()` and the bundled +`K8sContext.create(context)` are useful when you want to scope your loading by namespace, kubeconfig, or anything else the caller already configured. @@ -64,9 +64,8 @@ order. public class VaultConfigProvider implements ConfigProvider { @Override - public Properties loadConfig(Connection connection) throws Exception { - HoptimatorConnection conn = (HoptimatorConnection) connection; - String namespace = K8sContext.create(conn).namespace(); + public Properties loadConfig(DeploymentContext context) throws Exception { + String namespace = K8sContext.create(context).namespace(); // Pull namespace-scoped values from Vault. Whatever you return becomes // available as {{key}} in any template — and as a connection property diff --git a/docs/extending/deployers.md b/docs/extending/deployers.md index 3eb6fa34b..f3c394dbd 100644 --- a/docs/extending/deployers.md +++ b/docs/extending/deployers.md @@ -72,7 +72,7 @@ A `Deployer` doesn't get loaded directly. Instead, you ship a ```java public interface DeployerProvider { - Collection deployers(T obj, Connection connection); + Collection deployers(T obj, DeploymentContext context); int priority(); } ``` @@ -82,6 +82,17 @@ provider returns the deployers that apply to it. Return an empty collection when the deployable isn't yours — the runtime will skip you and move on to the next provider. +The `DeploymentContext` is a Calcite-free handle onto everything a deployer +needs: connection-level `properties()` (namespace, hints, cluster config) and +per-`Database` connection config via `databaseProperties(catalog, schema, +urlPrefix)`. The SQL path supplies a Calcite-backed implementation +(`CalciteDeploymentContext`); the direct table-create API supplies a +`DirectDeploymentContext` that carries the caller's Avro schema. A deployer that +needs the row type calls `HoptimatorDriver.rowType(source, context)`, which +resolves it from the Calcite catalog (SQL path) or derives it from the carried +Avro schema (direct path) — so a deployer never touches `java.sql.Connection` or +a Calcite `SchemaPlus` directly. + `priority()` controls ordering: providers with **lower priority numbers run first**. If two providers can both deploy the same object, the lower- priority one wins. The default Kubernetes provider is priority `2`; ship @@ -103,9 +114,9 @@ com.example.hoptimator.mysystem.MySystemDeployerProvider The bundled Kafka path is a good shape to copy: - [`KafkaDeployerProvider`](https://github.com/linkedin/Hoptimator/blob/main/hoptimator-kafka/src/main/java/com/linkedin/hoptimator/kafka/KafkaDeployerProvider.java) - — type-checks the `Deployable`, extracts per-schema connection - properties from the Calcite schema (i.e. the JDBC URL the `Database` - CRD points at), and constructs the deployer. + — type-checks the `Deployable`, resolves the `Database`'s connection + config via `context.databaseProperties(catalog, schema, "jdbc:kafka://")` + (the JDBC URL the `Database` CRD points at), and constructs the deployer. - [`KafkaDeployer`](https://github.com/linkedin/Hoptimator/blob/main/hoptimator-kafka/src/main/java/com/linkedin/hoptimator/kafka/KafkaDeployer.java) — `create()` calls Kafka's AdminClient API to create the topic; `restore()` walks back and deletes any topic the current operation @@ -115,7 +126,7 @@ The shape any provider should follow: - Type-check the `Deployable` and return an empty collection if it's not what you handle. -- Extract any per-schema configuration from the connection. +- Extract any per-schema configuration from the context. - Construct one or more deployer instances and return them. ## Validation @@ -126,7 +137,7 @@ Deployers can opt into pre-deploy validation by also implementing ```java public class MyDeployer implements Deployer, Validated { @Override - public void validate(Validator.Issues issues) { + public void validate(Validator.Issues issues, DeploymentContext context) { // emit warnings or errors before any side effects } } @@ -143,7 +154,7 @@ The validate-then-specify path is the main test surface: ```java Source source = new Source("my-database", List.of("MYSYS", "foo"), Map.of()); DeployerProvider provider = new MyDeployerProvider(); -Collection deployers = provider.deployers(source, mockConnection); +Collection deployers = provider.deployers(source, context); assertThat(deployers).hasSize(1); List specs = deployers.iterator().next().specify(); diff --git a/docs/extending/validators.md b/docs/extending/validators.md index d3e966a84..a0b28ca2d 100644 --- a/docs/extending/validators.md +++ b/docs/extending/validators.md @@ -26,7 +26,7 @@ the operator's status field for CRD-driven changes). ```java public interface Validated { - void validate(Validator.Issues issues); + void validate(Validator.Issues issues, DeploymentContext context); } public interface Validator extends Validated { @@ -35,10 +35,16 @@ public interface Validator extends Validated { } public interface ValidatorProvider { - Collection validators(T obj); + Collection validators(T obj, DeploymentContext context); } ``` +The `DeploymentContext` (Calcite-free) exposes connection-level +`properties()` and per-`Database` config via `databaseProperties(...)` — +everything a validator needs to run lookups against external systems without +touching `java.sql.Connection`. A validator that needs the deployable's row +type calls `HoptimatorDriver.rowType(source, context)`. + Two ways to participate: - **Make your own type `Validated`.** If you've authored a new @@ -58,7 +64,7 @@ nested contexts let the final error message read like a structured report. ```java public class NamingPolicyValidator implements Validator { @Override - public void validate(Validator.Issues issues) { + public void validate(Validator.Issues issues, DeploymentContext context) { if (tableName.contains("_")) { issues.error("Table names must not contain underscores"); } @@ -112,7 +118,7 @@ input: ```java public class MyPolicyValidatorProvider implements ValidatorProvider { @Override - public Collection validators(T obj) { + public Collection validators(T obj, DeploymentContext context) { if (obj instanceof Source) { return List.of(new NamingPolicyValidator((Source) obj)); } @@ -182,7 +188,7 @@ recognize. ```java Issues issues = new Issues("test"); -new MyValidator(testSource).validate(issues); +new MyValidator(testSource).validate(issues, context); issues.close(); assertThat(issues.valid()).isTrue(); // or assertThat(issues.toString()).contains("ERROR: ...") ``` diff --git a/docs/getting-started/architecture.md b/docs/getting-started/architecture.md index 3efed3df5..6b37fb4fb 100644 --- a/docs/getting-started/architecture.md +++ b/docs/getting-started/architecture.md @@ -138,6 +138,37 @@ Triggers (`TableTrigger`, `CronJob`) plug in here too — they let upstream events or schedules drive downstream side-effects without modeling them inside the pipeline. +## Creating a table without SQL + +A single table (a `Source`) doesn't need a query to describe it — just a row +type. For that case there's a SQL-free, **connection-free** entrypoint, +`TableService.create(...)` (and `TableService.delete(...)`) in +`hoptimator-jdbc`, which takes connection-level properties plus a table path and +an **Avro schema** and runs the *same* validation and deployment as +`CREATE TABLE` — the row type is derived from the Avro schema (via +`AvroConverter`) instead of from SQL column declarations. Both paths converge on +`HoptimatorDdlUtils.deployTableInternal`, so validators, deployers, and rollback +behave identically regardless of where the schema came from. + +This is what lets a caller (e.g. a gRPC service) create a table by handing over +a name and a schema, with no SQL parsing, no planning, and no JDBC connection. A +`dryRun` flag mirrors `!specify` / the MCP `plan` tool: it validates and renders +the specs without deploying anything. + +This works because the deploy/validate SPI is decoupled from Calcite. Providers +(`Deployer`, `Validator`, `Connector`, `Config`) receive a neutral +`DeploymentContext` — not a `java.sql.Connection` — that exposes only what they +need: connection-level properties and per-`Database` config. The row type is +resolved on demand via `HoptimatorDriver.rowType(source, context)`. The SQL path +supplies a Calcite-backed `DeploymentContext` (row type read from the catalog); +the direct path supplies a `DirectDeploymentContext` that carries the caller's +Avro schema (deriving the row type on demand) and resolves `Database` config +registry-natively via a `DatabaseConfigResolver` +(the K8s implementation reads `Database` CRDs directly — see +`DatabaseConfigResolvers`). The direct path opens no connection and touches no +Calcite catalog; only the SQL engine's read/plan path still uses the JDBC +driver layer. + ## Module map The repo is split into focused modules. The ones you'll touch most often: diff --git a/hoptimator-api/build.gradle b/hoptimator-api/build.gradle index d73eca132..9c564be02 100644 --- a/hoptimator-api/build.gradle +++ b/hoptimator-api/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'java' + id 'java-library' id 'maven-publish' } diff --git a/hoptimator-api/src/main/java/com/linkedin/hoptimator/ConfigProvider.java b/hoptimator-api/src/main/java/com/linkedin/hoptimator/ConfigProvider.java index 451270fb9..135fc4512 100644 --- a/hoptimator-api/src/main/java/com/linkedin/hoptimator/ConfigProvider.java +++ b/hoptimator-api/src/main/java/com/linkedin/hoptimator/ConfigProvider.java @@ -1,9 +1,8 @@ package com.linkedin.hoptimator; -import java.sql.Connection; import java.util.Properties; public interface ConfigProvider { - Properties loadConfig(Connection connection) throws Exception; + Properties loadConfig(DeploymentContext context) throws Exception; } diff --git a/hoptimator-api/src/main/java/com/linkedin/hoptimator/ConnectorProvider.java b/hoptimator-api/src/main/java/com/linkedin/hoptimator/ConnectorProvider.java index 1fa8a84a8..11d879984 100644 --- a/hoptimator-api/src/main/java/com/linkedin/hoptimator/ConnectorProvider.java +++ b/hoptimator-api/src/main/java/com/linkedin/hoptimator/ConnectorProvider.java @@ -1,11 +1,10 @@ package com.linkedin.hoptimator; -import java.sql.Connection; import java.util.Collection; public interface ConnectorProvider { /** Find connectors capable of configuring data plane connectors for the obj. */ - Collection connectors(T obj, Connection connection); + Collection connectors(T obj, DeploymentContext context); } diff --git a/hoptimator-api/src/main/java/com/linkedin/hoptimator/Deployer.java b/hoptimator-api/src/main/java/com/linkedin/hoptimator/Deployer.java index 2dc0bd6d7..bfd596bd3 100644 --- a/hoptimator-api/src/main/java/com/linkedin/hoptimator/Deployer.java +++ b/hoptimator-api/src/main/java/com/linkedin/hoptimator/Deployer.java @@ -13,6 +13,17 @@ public interface Deployer { void update() throws SQLException; + /** + * Whether the backing resource this deployer manages already exists. Used by the connection-free + * direct path to enforce {@code CREATE} (not {@code OR REPLACE}) semantics — the SQL/DDL path + * enforces the same thing via the Calcite catalog before deployers run, so this is not consulted + * there. Defaults to {@code false} (unknown/assume-absent) so deployers that do not implement it + * keep their existing behavior. + */ + default boolean exists() throws SQLException { + return false; + } + /** Render a list of specs, usually YAML. */ List specify() throws SQLException; diff --git a/hoptimator-api/src/main/java/com/linkedin/hoptimator/DeployerProvider.java b/hoptimator-api/src/main/java/com/linkedin/hoptimator/DeployerProvider.java index 0aa82fb6e..56dde9eb4 100644 --- a/hoptimator-api/src/main/java/com/linkedin/hoptimator/DeployerProvider.java +++ b/hoptimator-api/src/main/java/com/linkedin/hoptimator/DeployerProvider.java @@ -1,13 +1,12 @@ package com.linkedin.hoptimator; -import java.sql.Connection; import java.util.Collection; public interface DeployerProvider { /** Find deployers capable of deploying the obj. */ - Collection deployers(T obj, Connection connection); + Collection deployers(T obj, DeploymentContext context); /** A DeployerProvider with lower priority will execute first */ int priority(); diff --git a/hoptimator-api/src/main/java/com/linkedin/hoptimator/DeploymentContext.java b/hoptimator-api/src/main/java/com/linkedin/hoptimator/DeploymentContext.java new file mode 100644 index 000000000..4d4f1bf0f --- /dev/null +++ b/hoptimator-api/src/main/java/com/linkedin/hoptimator/DeploymentContext.java @@ -0,0 +1,48 @@ +package com.linkedin.hoptimator; + +import java.util.Properties; + +import javax.annotation.Nullable; + + +/** + * The ambient information a {@link Deployer}, {@link Connector}, or {@link Validator} needs in + * order to act on a {@link Deployable}, independent of how it was produced. + * + *

This replaces passing a raw {@code java.sql.Connection} through the SPI. The SQL path + * supplies a Calcite-backed implementation; a direct (non-SQL) caller supplies an + * implementation backed by its own control plane. Neither the deploy nor validate machinery + * needs to know which producer it came from. + * + *

A context exposes only two things: + *

    + *
  • connection-level {@link #properties()} (namespace, hints, cluster config); + *
  • per-{@code Database} connection config via {@link #databaseProperties}. + *
+ * + *

A table's row type is not exposed here (that would couple this API to a schema + * representation). It is resolved by the producer-specific machinery: the SQL path reads it from + * the Calcite catalog; the direct path carries it on its own context implementation. + */ +public interface DeploymentContext { + + /** Connection-level properties and hints (e.g. namespace, {@code k8s.*}, hints, mode). */ + Properties properties(); + + /** + * Returns the parsed connection properties for a database, extracted from its connection URL + * after stripping {@code connectionPrefix}, or {@code null} when the database is unknown or its + * URL does not start with {@code connectionPrefix}. + * + *

The {@code Database} it identifies is keyed by a catalog and/or a schema: catalog-style + * databases (e.g. MySQL) are matched by catalog, while schema-style databases (e.g. Kafka, + * Venice) are matched by schema. Both are individually optional — mirroring the {@code Database} + * CRD, where {@code catalog} and {@code schema} are optional — but at least one must be provided. + * + * @param catalog the catalog name, or {@code null} + * @param schema the schema name, or {@code null} + * @param connectionPrefix the expected URL scheme prefix (e.g. {@code "jdbc:kafka://"}) + */ + @Nullable Properties databaseProperties(@Nullable String catalog, @Nullable String schema, + String connectionPrefix); +} diff --git a/hoptimator-api/src/main/java/com/linkedin/hoptimator/Validated.java b/hoptimator-api/src/main/java/com/linkedin/hoptimator/Validated.java index 7c32c1480..9a7e902a3 100644 --- a/hoptimator-api/src/main/java/com/linkedin/hoptimator/Validated.java +++ b/hoptimator-api/src/main/java/com/linkedin/hoptimator/Validated.java @@ -1,14 +1,12 @@ package com.linkedin.hoptimator; -import java.sql.Connection; - public interface Validated { /** - * Validates {@code this}, recording any problems in {@code issues}. The connection is always + * Validates {@code this}, recording any problems in {@code issues}. The context is always * supplied so validators can run lookups against external systems (e.g. pre-delete dependency * checks). */ - void validate(Validator.Issues issues, Connection connection); + void validate(Validator.Issues issues, DeploymentContext context); } diff --git a/hoptimator-api/src/main/java/com/linkedin/hoptimator/Validator.java b/hoptimator-api/src/main/java/com/linkedin/hoptimator/Validator.java index 9db6a279d..ea44663cd 100644 --- a/hoptimator-api/src/main/java/com/linkedin/hoptimator/Validator.java +++ b/hoptimator-api/src/main/java/com/linkedin/hoptimator/Validator.java @@ -1,9 +1,8 @@ package com.linkedin.hoptimator; -import java.sql.Connection; +import java.util.Collections; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -53,8 +52,8 @@ public DefaultValidator(T t) { } @Override - public void validate(Issues issues, Connection connection) { - t.validate(issues.child(t.getClass().getSimpleName()), connection); + public void validate(Issues issues, DeploymentContext context) { + t.validate(issues.child(t.getClass().getSimpleName()), context); } } diff --git a/hoptimator-api/src/main/java/com/linkedin/hoptimator/ValidatorProvider.java b/hoptimator-api/src/main/java/com/linkedin/hoptimator/ValidatorProvider.java index d4729e4aa..7042b0777 100644 --- a/hoptimator-api/src/main/java/com/linkedin/hoptimator/ValidatorProvider.java +++ b/hoptimator-api/src/main/java/com/linkedin/hoptimator/ValidatorProvider.java @@ -1,6 +1,5 @@ package com.linkedin.hoptimator; -import java.sql.Connection; import java.util.Collection; @@ -9,5 +8,5 @@ public interface ValidatorProvider { /** * Returns validators that should be applied to {@code obj}. */ - Collection validators(T obj, Connection connection); + Collection validators(T obj, DeploymentContext context); } diff --git a/hoptimator-avro/src/main/java/com/linkedin/hoptimator/avro/AvroConverter.java b/hoptimator-avro/src/main/java/com/linkedin/hoptimator/avro/AvroConverter.java index 0870d4db1..9c3f3f6f0 100644 --- a/hoptimator-avro/src/main/java/com/linkedin/hoptimator/avro/AvroConverter.java +++ b/hoptimator-avro/src/main/java/com/linkedin/hoptimator/avro/AvroConverter.java @@ -13,6 +13,7 @@ import org.apache.calcite.util.Pair; import java.util.AbstractMap; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; @@ -148,6 +149,98 @@ public static Pair avroKeyPayloadSchema(String namespace, String return new Pair<>(keySchema, payloadSchema); } + /** + * Lossless twin of {@link #avroKeyPayloadSchema(String, String, String, RelDataType, Map)} that + * splits an already-Avro merged record (e.g. the schema a direct-API caller supplied) into its + * key and payload schemas, instead of synthesizing them from a {@link RelDataType}. Field types + * are cloned by reference via {@link AvroSchemas#cloneField}, so nested record identities, + * namespaces, unions, enums/fixed, and defaults survive intact — the whole point of preferring + * the caller's Avro over the lossy round-trip. + * + *

Key selection mirrors the RelDataType twin: {@code key.fields} (semicolon-separated) names + * the key fields and {@code key.fields-prefix} is stripped from each. A single key field whose + * stripped name is {@link AvroSchemas#PRIMITIVE_KEY_NAME} yields a primitive key (that field's own + * schema). The payload record keeps the merged record's own identity (name, namespace, doc, + * aliases, props) so a value schema registered elsewhere still matches; the key record — whose + * identity the merge did not preserve — is named {@code keySchemaName} in the merged namespace. + * + * @return {@code Pair}; either side may be {@code null} (no key, or an + * all-key record with no payload). + */ + public static Pair avroKeyPayloadSchema(String keySchemaName, Schema mergedAvro, + Map keyOptions) { + String keys = keyOptions.get(KEY_OPTION); + String keyPrefix = keyOptions.getOrDefault(KEY_PREFIX_OPTION, ""); + + if (keys == null || keys.isEmpty() || mergedAvro.getType() != Schema.Type.RECORD) { + // No key configured (or not a record): the caller's schema is the payload verbatim. + return new Pair<>(null, mergedAvro); + } + + List keyNames = List.of(keys.split(";")); + List keyFields = new ArrayList<>(); + List payloadFields = new ArrayList<>(); + Schema primitiveKeySchema = null; + for (Schema.Field field : mergedAvro.getFields()) { + if (keyNames.contains(field.name())) { + String keyName = field.name().substring(keyPrefix.length()); + if (keyNames.size() == 1 && keyName.equals(AvroSchemas.PRIMITIVE_KEY_NAME)) { + primitiveKeySchema = field.schema(); + } else { + keyFields.add(AvroSchemas.cloneField(keyName, field)); + } + } else { + payloadFields.add(AvroSchemas.cloneField(field.name(), field)); + } + } + + Schema payloadSchema = payloadFields.isEmpty() ? null : recordLike(mergedAvro, payloadFields); + if (primitiveKeySchema != null) { + return new Pair<>(primitiveKeySchema, payloadSchema); + } + Schema keySchema = keyFields.isEmpty() ? null + : record(sanitize(keySchemaName), null, mergedAvro.getNamespace(), keyFields); + return new Pair<>(keySchema, payloadSchema); + } + + /** + * Extracts the value (payload) portion of a merged key+value Avro record, treating fields whose + * names start with {@code keyPrefix} as keys and dropping them. Thin convenience wrapper over + * {@link #avroKeyPayloadSchema(String, Schema, Map)} that derives its {@code key.fields} from the + * prefix (rather than an explicit list) and keeps only the payload side, so the two share one + * splitting implementation. Like that method it is lossless (nested field schemas are cloned by + * reference, preserving namespaces, nested record names, unions, enums/fixed, defaults). A + * non-record, or a record with no key-prefixed fields, is returned unchanged; a record whose + * fields are all keys yields {@code null} (no payload). + */ + public static Schema valueSchemaOf(Schema mergedAvro, String keyPrefix) { + if (mergedAvro.getType() != Schema.Type.RECORD) { + return mergedAvro; + } + String keys = mergedAvro.getFields().stream() + .map(Schema.Field::name) + .filter(n -> n.startsWith(keyPrefix)) + .collect(Collectors.joining(";")); + Map keyOptions = keys.isEmpty() ? Map.of() + : Map.of(KEY_OPTION, keys, KEY_PREFIX_OPTION, keyPrefix); + return avroKeyPayloadSchema(mergedAvro.getName() + "_Key", mergedAvro, keyOptions).getValue(); + } + + /** Builds a record that inherits {@code template}'s identity (name/namespace/doc/aliases/props). */ + private static Schema recordLike(Schema template, List fields) { + Schema schema = record(template.getName(), template.getDoc(), template.getNamespace(), fields); + template.getAliases().forEach(schema::addAlias); + template.getObjectProps().forEach(schema::addProp); + return schema; + } + + /** Builds a record with the given name, doc, and namespace holding {@code fields}. */ + private static Schema record(String name, String doc, String namespace, List fields) { + Schema schema = Schema.createRecord(name, doc, namespace, false); + schema.setFields(fields); + return schema; + } + private static Schema createAvroSchemaWithNullability(Schema schema, boolean nullable) { if (nullable) { return Schema.createUnion(Schema.create(Schema.Type.NULL), schema); diff --git a/hoptimator-avro/src/main/java/com/linkedin/hoptimator/avro/AvroTableValidator.java b/hoptimator-avro/src/main/java/com/linkedin/hoptimator/avro/AvroTableValidator.java index 57d35fed2..ec2061dc9 100644 --- a/hoptimator-avro/src/main/java/com/linkedin/hoptimator/avro/AvroTableValidator.java +++ b/hoptimator-avro/src/main/java/com/linkedin/hoptimator/avro/AvroTableValidator.java @@ -1,5 +1,6 @@ package com.linkedin.hoptimator.avro; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Validator; import org.apache.avro.Schema; import org.apache.avro.file.DataFileWriter; @@ -17,7 +18,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.sql.Connection; /** Validates that tables follow Avro schema evolution rules. */ @@ -30,7 +30,7 @@ class AvroTableValidator implements Validator { } @Override - public void validate(Issues issues, Connection connection) { + public void validate(Issues issues, DeploymentContext context) { try { CalciteSchema originalSchema = schema.unwrap(CalciteSchema.class); if (originalSchema == null || originalSchema.schema == null) { diff --git a/hoptimator-avro/src/main/java/com/linkedin/hoptimator/avro/AvroValidatorProvider.java b/hoptimator-avro/src/main/java/com/linkedin/hoptimator/avro/AvroValidatorProvider.java index de5a986b0..9f625a27a 100644 --- a/hoptimator-avro/src/main/java/com/linkedin/hoptimator/avro/AvroValidatorProvider.java +++ b/hoptimator-avro/src/main/java/com/linkedin/hoptimator/avro/AvroValidatorProvider.java @@ -1,10 +1,10 @@ package com.linkedin.hoptimator.avro; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Validator; import com.linkedin.hoptimator.ValidatorProvider; import org.apache.calcite.schema.SchemaPlus; -import java.sql.Connection; import java.util.Collection; import java.util.Collections; @@ -13,7 +13,7 @@ public class AvroValidatorProvider implements ValidatorProvider { @Override - public Collection validators(T obj, Connection connection) { + public Collection validators(T obj, DeploymentContext context) { if (obj instanceof SchemaPlus) { return Collections.singletonList(new AvroTableValidator((SchemaPlus) obj)); } else { diff --git a/hoptimator-avro/src/test/java/com/linkedin/hoptimator/avro/AvroConverterTest.java b/hoptimator-avro/src/test/java/com/linkedin/hoptimator/avro/AvroConverterTest.java index 9ae3fb038..533d15ca0 100644 --- a/hoptimator-avro/src/test/java/com/linkedin/hoptimator/avro/AvroConverterTest.java +++ b/hoptimator-avro/src/test/java/com/linkedin/hoptimator/avro/AvroConverterTest.java @@ -772,4 +772,102 @@ void testAvroFromTimestampUsesLogicalType() { assertEquals(SqlTypeName.TIMESTAMP, Objects.requireNonNull(relDataTypeAgain.getField("timestampField", false, false)).getType().getSqlTypeName()); } + + @Test + public void avroKeyPayloadSchemaFromAvroPreservesValueNamespacesLosslessly() { + // A merged record whose payload field is a nested record in its OWN namespace, plus a struct + // key field. The RelDataType round-trip would flatten these namespaces away; the Avro overload + // must preserve them. + Schema merged = new Schema.Parser().parse("{" + + "\"type\":\"record\",\"name\":\"UserEvent\",\"namespace\":\"com.example.events\",\"fields\":[" + + "{\"name\":\"KEY_id\",\"type\":\"long\"}," + + "{\"name\":\"widget\",\"type\":{\"type\":\"record\",\"name\":\"Widget\"," + + "\"namespace\":\"com.example.custom\",\"fields\":[{\"name\":\"w\",\"type\":\"string\"}]}}" + + "]}"); + Map keyOptions = Map.of("key.fields", "KEY_id", "key.fields-prefix", "KEY_"); + + Pair result = AvroConverter.avroKeyPayloadSchema("UserEvent_Key", merged, keyOptions); + + Schema payload = result.getValue(); + assertNotNull(payload); + // The payload keeps the merged record's own identity, not a synthesized name/namespace. + assertEquals("UserEvent", payload.getName()); + assertEquals("com.example.events", payload.getNamespace()); + assertNull(payload.getField("KEY_id")); + Schema nested = payload.getField("widget").schema(); + assertEquals(Schema.Type.RECORD, nested.getType()); + // The nested record's distinct namespace + name survive — the whole point of losslessness. + assertEquals("com.example.custom", nested.getNamespace()); + assertEquals("Widget", nested.getName()); + + Schema key = result.getKey(); + assertNotNull(key); + assertEquals(1, key.getFields().size()); + assertEquals("id", key.getFields().get(0).name()); // KEY_ prefix stripped + assertEquals(Schema.Type.LONG, key.getFields().get(0).schema().getType()); + } + + @Test + public void avroKeyPayloadSchemaFromAvroExtractsPrimitiveKey() { + Schema merged = new Schema.Parser().parse("{" + + "\"type\":\"record\",\"name\":\"R\",\"namespace\":\"com.example\",\"fields\":[" + + "{\"name\":\"KEY\",\"type\":\"long\"}," + + "{\"name\":\"v\",\"type\":\"string\"}]}"); + Map keyOptions = Map.of("key.fields", "KEY"); + + Pair result = AvroConverter.avroKeyPayloadSchema("R_Key", merged, keyOptions); + + // Primitive key: the key's own Avro type, not a wrapper record. + assertEquals(Schema.Type.LONG, result.getKey().getType()); + Schema payload = result.getValue(); + assertEquals("com.example", payload.getNamespace()); + assertNull(payload.getField("KEY")); + assertNotNull(payload.getField("v")); + } + + @Test + public void avroKeyPayloadSchemaFromAvroWithNoKeyReturnsWholeRecordAsPayload() { + Schema merged = new Schema.Parser().parse("{" + + "\"type\":\"record\",\"name\":\"R\",\"namespace\":\"com.example\",\"fields\":[" + + "{\"name\":\"a\",\"type\":\"int\"}]}"); + + Pair result = AvroConverter.avroKeyPayloadSchema("R_Key", merged, Map.of()); + + assertNull(result.getKey()); + assertEquals(merged, result.getValue()); // caller's schema verbatim + } + + @Test + public void valueSchemaOfDropsKeyPrefixedFieldsLosslessly() { + Schema merged = new Schema.Parser().parse("{" + + "\"type\":\"record\",\"name\":\"StoreValue\",\"namespace\":\"com.example\",\"fields\":[" + + "{\"name\":\"KEY_id\",\"type\":\"int\"}," + + "{\"name\":\"widget\",\"type\":{\"type\":\"record\",\"name\":\"Widget\"," + + "\"namespace\":\"com.example.custom\",\"fields\":[{\"name\":\"w\",\"type\":\"string\"}]}}" + + "]}"); + + Schema value = AvroConverter.valueSchemaOf(merged, "KEY_"); + + assertNull(value.getField("KEY_id"), "key field dropped"); + assertNotNull(value.getField("widget"), "value field kept"); + assertEquals("com.example", value.getNamespace(), "merged record identity preserved"); + assertEquals("com.example.custom", value.getField("widget").schema().getNamespace(), + "nested namespace preserved"); + } + + @Test + public void valueSchemaOfReturnsRecordUnchangedWhenNoKeyFields() { + Schema merged = new Schema.Parser().parse("{" + + "\"type\":\"record\",\"name\":\"R\",\"namespace\":\"com.example\",\"fields\":[" + + "{\"name\":\"a\",\"type\":\"int\"}]}"); + + assertEquals(merged, AvroConverter.valueSchemaOf(merged, "KEY_")); + } + + @Test + public void valueSchemaOfReturnsNonRecordUnchanged() { + Schema primitive = Schema.create(Schema.Type.STRING); + + assertEquals(primitive, AvroConverter.valueSchemaOf(primitive, "KEY_")); + } } diff --git a/hoptimator-cli/src/main/java/sqlline/HoptimatorAppConfig.java b/hoptimator-cli/src/main/java/sqlline/HoptimatorAppConfig.java index 1b1105d50..3684dcfb1 100644 --- a/hoptimator-cli/src/main/java/sqlline/HoptimatorAppConfig.java +++ b/hoptimator-cli/src/main/java/sqlline/HoptimatorAppConfig.java @@ -136,7 +136,7 @@ public void execute(String line, DispatchCallback dispatchCallback) { schemaSnapshot = HoptimatorDdlUtils.snapshotAndSetSinkSchema(conn.createPrepareContext(), new HoptimatorDriver.Prepare(conn), plan, create, querySql); } - sqlline.output(plan.sql(conn).apply(SqlDialect.ANSI)); + sqlline.output(plan.sql(conn.deploymentContext()).apply(SqlDialect.ANSI)); } catch (Exception e) { sqlline.error(e); dispatchCallback.setToFailure(); diff --git a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/CalciteDeploymentContext.java b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/CalciteDeploymentContext.java new file mode 100644 index 000000000..fc0c6fcf0 --- /dev/null +++ b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/CalciteDeploymentContext.java @@ -0,0 +1,43 @@ +package com.linkedin.hoptimator.jdbc; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; +import java.util.Properties; + +import com.linkedin.hoptimator.DeploymentContext; + + +/** + * A {@link DeploymentContext} backed by a Calcite {@link HoptimatorConnection}. This is the + * context the SQL path supplies: {@link #databaseProperties} reads each {@code Database}'s JDBC + * URL from the Calcite catalog, and the table's row type is resolved from the Calcite catalog + * (see {@link HoptimatorDriver#rowType}). + */ +public final class CalciteDeploymentContext implements DeploymentContext { + + private static final Logger LOG = LoggerFactory.getLogger(CalciteDeploymentContext.class); + + private final HoptimatorConnection connection; + + public CalciteDeploymentContext(HoptimatorConnection connection) { + this.connection = connection; + } + + /** The underlying connection. Retained for the Calcite-only planning path (not the deploy SPI). */ + public HoptimatorConnection connection() { + return connection; + } + + @Override + public Properties properties() { + return connection.connectionProperties(); + } + + @Override + public @Nullable Properties databaseProperties(@Nullable String catalog, @Nullable String schema, + String connectionPrefix) { + return DeployerUtils.extractPropertiesFromJdbcSchema(catalog, schema, connection, connectionPrefix); + } +} diff --git a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/CompatibilityValidatorBase.java b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/CompatibilityValidatorBase.java index c6a8b12f8..2e4d11b54 100644 --- a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/CompatibilityValidatorBase.java +++ b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/CompatibilityValidatorBase.java @@ -1,13 +1,12 @@ package com.linkedin.hoptimator.jdbc; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Validator; import org.apache.calcite.jdbc.CalciteSchema; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; import org.apache.calcite.schema.lookup.LikePattern; -import java.sql.Connection; - /** Base class for shared schema evolution validators. */ abstract class CompatibilityValidatorBase implements Validator { @@ -19,7 +18,7 @@ abstract class CompatibilityValidatorBase implements Validator { } @Override - public void validate(Issues issues, Connection connection) { + public void validate(Issues issues, DeploymentContext context) { try { CalciteSchema originalSchema = schema.unwrap(CalciteSchema.class); if (originalSchema == null || originalSchema.schema == null) { diff --git a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/CompatibilityValidatorProvider.java b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/CompatibilityValidatorProvider.java index 325887821..3cebc7ffb 100644 --- a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/CompatibilityValidatorProvider.java +++ b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/CompatibilityValidatorProvider.java @@ -1,10 +1,10 @@ package com.linkedin.hoptimator.jdbc; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Validator; import com.linkedin.hoptimator.ValidatorProvider; import org.apache.calcite.schema.SchemaPlus; -import java.sql.Connection; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -14,7 +14,7 @@ public class CompatibilityValidatorProvider implements ValidatorProvider { @Override - public Collection validators(T obj, Connection connection) { + public Collection validators(T obj, DeploymentContext context) { if (obj instanceof SchemaPlus) { return Arrays.asList(new Validator[]{new BackwardCompatibilityValidator((SchemaPlus) obj), new ForwardCompatibilityValidator((SchemaPlus) obj)}); diff --git a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/DefaultValidatorProvider.java b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/DefaultValidatorProvider.java index fb21346ed..cf87364e5 100644 --- a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/DefaultValidatorProvider.java +++ b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/DefaultValidatorProvider.java @@ -1,10 +1,10 @@ package com.linkedin.hoptimator.jdbc; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Validated; import com.linkedin.hoptimator.Validator; import com.linkedin.hoptimator.ValidatorProvider; -import java.sql.Connection; import java.util.Collection; import java.util.Collections; @@ -13,7 +13,7 @@ public class DefaultValidatorProvider implements ValidatorProvider { @Override - public Collection validators(T obj, Connection connection) { + public Collection validators(T obj, DeploymentContext context) { if (obj instanceof Validated) { return Collections.singletonList(new Validator.DefaultValidator<>((Validated) obj)); } else { diff --git a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/DeployerUtils.java b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/DeployerUtils.java index ed3e4446a..3bf2cf301 100644 --- a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/DeployerUtils.java +++ b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/DeployerUtils.java @@ -122,11 +122,10 @@ public static Double parseDoubleOption(Map options, String key, * @param schemaName the name of the schema to look up * @param connection the connection to search in * @param connectionPrefix the JDBC connection prefix to strip (e.g., "jdbc:kafka://") - * @param logger optional logger for debug messages * @return Properties extracted from the JDBC URL, or null if schema not found or not JDBC-backed */ - public static Properties extractPropertiesFromJdbcSchema(@Nullable String catalogName, String schemaName, - Connection connection, String connectionPrefix, @Nullable Logger logger) { + public static Properties extractPropertiesFromJdbcSchema(@Nullable String catalogName, @Nullable String schemaName, + Connection connection, String connectionPrefix) { if (schemaName == null) { return null; @@ -167,9 +166,7 @@ public static Properties extractPropertiesFromJdbcSchema(@Nullable String catalo properties.putAll(ConnectStringParser.parse(jdbcUrl.substring(connectionPrefix.length()))); return properties; } catch (Exception e) { - if (logger != null) { - logger.debug("Could not extract properties from schema '{}': {}", schemaName, e.getMessage()); - } + log.warn("Could not extract properties from schema '{}': {}", schemaName, e.getMessage()); } return null; } diff --git a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/DualLogger.java b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/DualLogger.java new file mode 100644 index 000000000..66ad8311d --- /dev/null +++ b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/DualLogger.java @@ -0,0 +1,37 @@ +package com.linkedin.hoptimator.jdbc; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.helpers.MessageFormatter; + +import java.util.List; +import java.util.function.Consumer; + + +/** + * A logger that fans out each message to both an SLF4J logger and a list of log hooks. Used by both + * the SQL DDL path (via {@link HoptimatorConnection#getLogger}) and the connection-free direct path + * (constructed directly from log hooks); it depends only on a class — for the SLF4J logger and the + * message prefix — and the hook list, never on a JDBC connection. + */ +final class DualLogger { + private final String className; + private final Logger slf4jLogger; + private final List> hooks; + + DualLogger(Class clazz, List> hooks) { + this.className = clazz.getSimpleName(); + this.slf4jLogger = LoggerFactory.getLogger(clazz); + this.hooks = hooks; + } + + /** + * Log a message with slf4j format at the INFO level. + */ + public void info(String format, Object... arguments) { + slf4jLogger.info(format, arguments); + String msg = MessageFormatter.arrayFormat(format, arguments).getMessage(); + String msgWithClassName = String.format("[%s] %s", className, msg); + hooks.forEach(hook -> hook.accept(msgWithClassName)); + } +} diff --git a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorConnection.java b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorConnection.java index 143c48f66..56b1f30aa 100644 --- a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorConnection.java +++ b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorConnection.java @@ -1,6 +1,7 @@ package com.linkedin.hoptimator.jdbc; import com.linkedin.hoptimator.Database; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Sink; import com.linkedin.hoptimator.Source; import com.linkedin.hoptimator.avro.AvroConverter; @@ -17,9 +18,6 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; import org.apache.calcite.util.Util; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.helpers.MessageFormatter; import java.sql.DatabaseMetaData; import java.sql.PreparedStatement; @@ -42,6 +40,7 @@ public class HoptimatorConnection extends DelegatingConnection { private final List materializations = new ArrayList<>(); private final List> logHooks = new ArrayList<>(); + private DeploymentContext deploymentContext; public HoptimatorConnection(CalciteConnection connection, Properties connectionProperties) { super(connection); @@ -49,6 +48,18 @@ public HoptimatorConnection(CalciteConnection connection, Properties connectionP this.connectionProperties = connectionProperties; } + /** + * The Calcite-backed {@link DeploymentContext} for this connection, created once and reused. + * This is the neutral handle the deploy/validate SPI operates on; callers should pass this + * rather than constructing a fresh {@code CalciteDeploymentContext} per operation. + */ + public DeploymentContext deploymentContext() { + if (deploymentContext == null) { + deploymentContext = new CalciteDeploymentContext(this); + } + return deploymentContext; + } + public ResolvedTable resolve(List tablePath, Map hints) throws SQLException { try { Schema avroSchema = avroSchema(tablePath); @@ -68,8 +79,9 @@ public ResolvedTable resolve(List tablePath, Map hints) String database = databaseName(this.createPrepareContext(), tablePath); Source source = new Source(database, tablePath, hints); Sink sink = new Sink(database, tablePath, hints); - return new ResolvedTable(tablePath, avroSchema, ConnectionService.configure(source, this), - ConnectionService.configure(sink, this)); + DeploymentContext context = this.deploymentContext(); + return new ResolvedTable(tablePath, avroSchema, ConnectionService.configure(source, context), + ConnectionService.configure(sink, context)); } catch (Exception e) { throw new SQLException("Failed to resolve " + String.join(".", tablePath) + ": " + e.getMessage(), e); } @@ -147,8 +159,8 @@ public void registerMaterialization(List viewPath, String querySql) { /** * Returns a logger for a client of this connection. The logger logs to both SLF4J and hooks. */ - HoptimatorConnectionDualLogger getLogger(Class clazz) { - return new HoptimatorConnectionDualLogger(clazz, logHooks); + DualLogger getLogger(Class clazz) { + return new DualLogger(clazz, logHooks); } /** @@ -159,6 +171,11 @@ public void addLogHook(Consumer hook) { logHooks.add(hook); } + /** The connection's log hooks, so deploy machinery can log to them without holding the connection. */ + public List> logHooks() { + return logHooks; + } + private void registerMaterialization(List viewPath, RelNode tableRel, RelNode queryRel) { materializations.add(new RelOptMaterialization(tableRel, queryRel, null, viewPath)); } @@ -178,29 +195,4 @@ private static String databaseName(CalcitePrepare.Context context, List } return ((Database) schema.schema).databaseName(); } - - /** - * A logger that logs to both SLF4J logger and registered hooks. - */ - static class HoptimatorConnectionDualLogger { - private final String className; - private final Logger slf4jLogger; - private final List> hooks; - - HoptimatorConnectionDualLogger(Class clazz, List> hooks) { - this.className = clazz.getSimpleName(); - this.slf4jLogger = LoggerFactory.getLogger(clazz); - this.hooks = hooks; - } - - /** - * Log a message with slf4j format at the INFO level. - */ - public void info(String format, Object... arguments) { - slf4jLogger.info(format, arguments); - String msg = MessageFormatter.arrayFormat(format, arguments).getMessage(); - String msgWithClassName = String.format("[%s] %s", className, msg); - hooks.forEach(hook -> hook.accept(msgWithClassName)); - } - } } diff --git a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlExecutor.java b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlExecutor.java index 1c2d871da..3707a9ba0 100644 --- a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlExecutor.java +++ b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlExecutor.java @@ -74,7 +74,7 @@ public final class HoptimatorDdlExecutor extends ServerDdlExecutor { private final HoptimatorConnection connection; - private final HoptimatorConnection.HoptimatorConnectionDualLogger logger; + private final DualLogger logger; public HoptimatorDdlExecutor(HoptimatorConnection connection) { try { @@ -109,7 +109,7 @@ public DdlExecutor getDdlExecutor() { public void execute(SqlCreateView create, CalcitePrepare.Context context) { logger.info("Validating statement: {}", create); try { - ValidationService.validateOrThrow(create, connection); + ValidationService.validateOrThrow(create, connection.deploymentContext()); } catch (SQLException e) { throw new DdlException(create, e.getMessage(), e); } @@ -123,7 +123,7 @@ public void execute(SqlCreateView create, CalcitePrepare.Context context) { throw new DdlException(create, "Cannot overwrite physical table " + pair.right + " with a view."); } - HoptimatorDdlUtils.DdlMode mode = HoptimatorDdlUtils.effectiveMode(create.getReplace(), connection); + HoptimatorDdlUtils.DdlMode mode = HoptimatorDdlUtils.effectiveMode(create.getReplace(), connection.deploymentContext()); for (Function function : schemaPlus.getFunctions(pair.right)) { if (function.getParameters().isEmpty()) { if (mode == HoptimatorDdlUtils.DdlMode.CREATE) { @@ -148,9 +148,9 @@ public void execute(SqlCreateView create, CalcitePrepare.Context context) { Collection deployers = null; try { logger.info("Validating deployable resources for view {}", viewName); - ValidationService.validateOrThrow(viewTable, connection); - deployers = DeploymentService.deployers(view, connection); - ValidationService.validateOrThrow(deployers, connection); + ValidationService.validateOrThrow(viewTable, connection.deploymentContext()); + deployers = DeploymentService.deployers(view, connection.deploymentContext()); + ValidationService.validateOrThrow(deployers, connection.deploymentContext()); logger.info("Validated view {}", viewName); if (mode == HoptimatorDdlUtils.DdlMode.UPDATE) { logger.info("Deploying update view {}", viewName); @@ -178,7 +178,7 @@ public void execute(SqlCreateView create, CalcitePrepare.Context context) { /** Executes a {@code CREATE MATERIALIZED VIEW} command. */ public void execute(SqlCreateMaterializedView create, CalcitePrepare.Context context) { logger.info("Validating statement: {}", create); - HoptimatorDdlUtils.DdlMode mode = HoptimatorDdlUtils.effectiveMode(create.getReplace(), connection); + HoptimatorDdlUtils.DdlMode mode = HoptimatorDdlUtils.effectiveMode(create.getReplace(), connection.deploymentContext()); try { HoptimatorDdlUtils.processCreateMaterializedView( context, @@ -197,7 +197,7 @@ public void execute(SqlCreateMaterializedView create, CalcitePrepare.Context con public void execute(SqlCreateTrigger create, CalcitePrepare.Context context) { logger.info("Validating statement: {}", create); try { - ValidationService.validateOrThrow(create, connection); + ValidationService.validateOrThrow(create, connection.deploymentContext()); } catch (SQLException e) { throw new DdlException(create, e.getMessage(), e); } @@ -236,11 +236,11 @@ public void execute(SqlCreateTrigger create, CalcitePrepare.Context context) { Collection deployers = null; try { logger.info("Validating trigger {} with deployers", name); - ValidationService.validateOrThrow(trigger, connection); - deployers = DeploymentService.deployers(trigger, connection); - ValidationService.validateOrThrow(deployers, connection); + ValidationService.validateOrThrow(trigger, connection.deploymentContext()); + deployers = DeploymentService.deployers(trigger, connection.deploymentContext()); + ValidationService.validateOrThrow(deployers, connection.deploymentContext()); logger.info("Validated trigger {}", name); - HoptimatorDdlUtils.DdlMode mode = HoptimatorDdlUtils.effectiveMode(create.getReplace(), connection); + HoptimatorDdlUtils.DdlMode mode = HoptimatorDdlUtils.effectiveMode(create.getReplace(), connection.deploymentContext()); if (mode == HoptimatorDdlUtils.DdlMode.UPDATE) { logger.info("Updating trigger {}", name); DeploymentService.update(deployers); @@ -279,7 +279,7 @@ private static String databaseOf(Table target) { /** Executes a {@code CREATE TABLE} command. */ public void execute(SqlCreateTable create, CalcitePrepare.Context context) { - HoptimatorDdlUtils.DdlMode mode = HoptimatorDdlUtils.effectiveMode(create.getReplace(), connection); + HoptimatorDdlUtils.DdlMode mode = HoptimatorDdlUtils.effectiveMode(create.getReplace(), connection.deploymentContext()); try { HoptimatorDdlUtils.processCreateTable(context, connection, create, mode); } catch (SQLException | RuntimeException e) { @@ -291,7 +291,7 @@ public void execute(SqlCreateTable create, CalcitePrepare.Context context) { /** Executes a {@code CREATE DATABASE} command. */ public void execute(SqlCreateDatabase create, CalcitePrepare.Context context) { - HoptimatorDdlUtils.DdlMode mode = HoptimatorDdlUtils.effectiveMode(create.getReplace(), connection); + HoptimatorDdlUtils.DdlMode mode = HoptimatorDdlUtils.effectiveMode(create.getReplace(), connection.deploymentContext()); try { HoptimatorDdlUtils.processCreateDatabase(connection, create, mode); } catch (SQLException | RuntimeException e) { @@ -318,7 +318,7 @@ public void execute(SqlResumeTrigger resume, CalcitePrepare.Context context) { public void execute(SqlFireTrigger fire, CalcitePrepare.Context context) { logger.info("Validating statement: {}", fire); try { - ValidationService.validateOrThrow(fire, connection); + ValidationService.validateOrThrow(fire, connection.deploymentContext()); } catch (SQLException e) { throw new DdlException(fire, e.getMessage(), e); } @@ -335,7 +335,7 @@ public void execute(SqlFireTrigger fire, CalcitePrepare.Context context) { Collection deployers = null; try { logger.info("Firing trigger {} with {} option(s)", name, options.size() - 1); - deployers = DeploymentService.deployers(trigger, connection); + deployers = DeploymentService.deployers(trigger, connection.deploymentContext()); DeploymentService.update(deployers); logger.info("FIRE TRIGGER {} completed", name); } catch (Exception e) { @@ -350,7 +350,7 @@ public void execute(SqlFireTrigger fire, CalcitePrepare.Context context) { public void execute(SqlDropTrigger drop, CalcitePrepare.Context context) { logger.info("Validating statement: {}", drop); try { - ValidationService.validateOrThrow(drop, connection); + ValidationService.validateOrThrow(drop, connection.deploymentContext()); } catch (SQLException e) { throw new DdlException(drop, e.getMessage(), e); } @@ -365,7 +365,7 @@ public void execute(SqlDropTrigger drop, CalcitePrepare.Context context) { Collection deployers = null; try { logger.info("Deleting trigger {}", name); - deployers = DeploymentService.deployers(trigger, connection); + deployers = DeploymentService.deployers(trigger, connection.deploymentContext()); DeploymentService.delete(deployers); logger.info("Deleted trigger {}", name); logger.info("DROP TRIGGER {} completed", name); @@ -385,7 +385,7 @@ public void execute(SqlDropTrigger drop, CalcitePrepare.Context context) { private void updateTriggerPausedState(SqlNode sqlNode, SqlIdentifier triggerName, boolean paused) { logger.info("Validating statement: {}", sqlNode); try { - ValidationService.validateOrThrow(sqlNode, connection); + ValidationService.validateOrThrow(sqlNode, connection.deploymentContext()); } catch (SQLException e) { throw new DdlException(sqlNode, e.getMessage(), e); } @@ -402,7 +402,7 @@ private void updateTriggerPausedState(SqlNode sqlNode, SqlIdentifier triggerName Collection deployers = null; try { logger.info("Updating trigger {} with paused state: {}", name, paused); - deployers = DeploymentService.deployers(trigger, connection); + deployers = DeploymentService.deployers(trigger, connection.deploymentContext()); DeploymentService.update(deployers); logger.info("Successfully updated trigger {} with paused state: {}", name, paused); } catch (Exception e) { @@ -420,7 +420,7 @@ private void updateTriggerPausedState(SqlNode sqlNode, SqlIdentifier triggerName public void execute(SqlDropObject drop, CalcitePrepare.Context context) { logger.info("Validating statement: {}", drop); try { - ValidationService.validateOrThrow(drop, connection); + ValidationService.validateOrThrow(drop, connection.deploymentContext()); } catch (SQLException e) { throw new DdlException(drop, e.getMessage(), e); } @@ -458,7 +458,7 @@ public void execute(SqlDropObject drop, CalcitePrepare.Context context) { } MaterializedViewTable materializedViewTable = (MaterializedViewTable) table; View view = new View(tablePath, materializedViewTable.viewSql()); - deployers = DeploymentService.deployers(view, connection); + deployers = DeploymentService.deployers(view, connection.deploymentContext()); logger.info("Deleting materialized view {}", tableName); DeploymentService.delete(deployers); schemaPlus.removeTable(tableName); @@ -470,7 +470,7 @@ public void execute(SqlDropObject drop, CalcitePrepare.Context context) { } ViewTable viewTable = (ViewTable) table; View view = new View(tablePath, viewTable.getViewSql()); - deployers = DeploymentService.deployers(view, connection); + deployers = DeploymentService.deployers(view, connection.deploymentContext()); logger.info("Deleting view {}", tableName); DeploymentService.delete(deployers); schemaPlus.removeTable(tableName); @@ -492,8 +492,8 @@ public void execute(SqlDropObject drop, CalcitePrepare.Context context) { // Pre-delete dependency guard. PendingDelete is the explicit "delete intent" signal // — only validators that key off it (the K8s dep checker) fire here. The check throws // before any deployer-level state change. - ValidationService.validateOrThrow(new PendingDelete<>(source), connection); - deployers = DeploymentService.deployers(source, connection); + ValidationService.validateOrThrow(new PendingDelete<>(source), connection.deploymentContext()); + deployers = DeploymentService.deployers(source, connection.deploymentContext()); logger.info("Deleting table {}", tableName); DeploymentService.delete(deployers); schemaPlus.removeTable(tableName); diff --git a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlUtils.java b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlUtils.java index 7ef9bd1b0..492f23831 100644 --- a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlUtils.java +++ b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlUtils.java @@ -23,6 +23,7 @@ import com.linkedin.hoptimator.Database; import com.linkedin.hoptimator.DatabaseDeployable; import com.linkedin.hoptimator.Deployer; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.MaterializedView; import com.linkedin.hoptimator.Pipeline; import com.linkedin.hoptimator.Source; @@ -70,9 +71,9 @@ import org.apache.calcite.util.Util; import javax.annotation.Nullable; -import java.sql.Connection; import java.sql.SQLException; import java.util.ArrayList; +import java.util.function.Consumer; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -119,16 +120,15 @@ private HoptimatorDdlUtils() { * {@code CREATE OR REPLACE} → {@link DdlMode#UPDATE}. In {@code apply} mode: both forms * resolve to {@link DdlMode#UPDATE}, making CREATE idempotent. */ - static DdlMode effectiveMode(boolean orReplace, HoptimatorConnection conn) { - if (isApplyMode(conn)) { + static DdlMode effectiveMode(boolean orReplace, DeploymentContext context) { + if (isApplyMode(context.properties())) { return DdlMode.UPDATE; } return orReplace ? DdlMode.UPDATE : DdlMode.CREATE; } /** Whether the connection is configured for apply-mode DDL. */ - static boolean isApplyMode(HoptimatorConnection conn) { - Properties props = conn.connectionProperties(); + static boolean isApplyMode(Properties props) { if (props == null) { return false; } @@ -152,7 +152,7 @@ public static final class SpecifyResult { /** Fully-qualified path of the sink (catalog + schema + table). */ public final List viewPath; - SpecifyResult(List specs, RelDataType sinkRowType, List viewPath) { + public SpecifyResult(List specs, RelDataType sinkRowType, List viewPath) { this.specs = Collections.unmodifiableList(specs); this.sinkRowType = sinkRowType; this.viewPath = Collections.unmodifiableList(viewPath); @@ -166,7 +166,7 @@ public static final class SpecifyResult { enum DdlMode { CREATE { @Override - List executeDeployers(Collection deployers, Connection conn) throws SQLException { + List executeDeployers(Collection deployers) throws SQLException { DeploymentService.create(deployers); return Collections.emptyList(); } @@ -178,7 +178,7 @@ boolean mutable() { }, UPDATE { @Override - List executeDeployers(Collection deployers, Connection conn) throws SQLException { + List executeDeployers(Collection deployers) throws SQLException { DeploymentService.update(deployers); return Collections.emptyList(); } @@ -190,7 +190,7 @@ boolean mutable() { }, SPECIFY { @Override - List executeDeployers(Collection deployers, Connection conn) throws SQLException { + List executeDeployers(Collection deployers) throws SQLException { List specs = new ArrayList<>(); for (Deployer deployer : deployers) { specs.addAll(deployer.specify()); @@ -204,7 +204,7 @@ boolean mutable() { } }; - abstract List executeDeployers(Collection deployers, Connection conn) throws SQLException; + abstract List executeDeployers(Collection deployers) throws SQLException; abstract boolean mutable(); } @@ -340,10 +340,10 @@ public static Pair snapshotAndSetSinkSchema(CalcitePrepare.Co static SpecifyResult processCreateMaterializedView(CalcitePrepare.Context ctx, HoptimatorDriver.Prepare prepare, HoptimatorConnection conn, SqlCreateMaterializedView create, DdlMode mode) throws SQLException { - HoptimatorConnection.HoptimatorConnectionDualLogger logger = conn.getLogger(HoptimatorDdlUtils.class); + DualLogger logger = conn.getLogger(HoptimatorDdlUtils.class); // Validate the DDL statement. logger.info("Validating statement: {}", create); - ValidationService.validateOrThrow(create, conn); + ValidationService.validateOrThrow(create, conn.deploymentContext()); // Extract query SQL (rename columns if a column list was provided) and plan the query. // This is done first — before schema/conflict checks — so that: @@ -431,15 +431,15 @@ static SpecifyResult processCreateMaterializedView(CalcitePrepare.Context ctx, boolean success = false; try { // Build the pipeline and create the MaterializedView hook. - Pipeline pipeline = plan.pipeline(viewName, conn); + Pipeline pipeline = plan.pipeline(viewName, conn.deploymentContext()); MaterializedView hook = new MaterializedView(database, viewPath, sql, pipeline.job().sql(), pipeline); // Validate the hook and its deployers. logger.info("Validating materialized view {}", viewName); - ValidationService.validateOrThrow(hook, conn); - deployers = DeploymentService.deployers(hook, conn); + ValidationService.validateOrThrow(hook, conn.deploymentContext()); + deployers = DeploymentService.deployers(hook, conn.deploymentContext()); logger.info("Validating deployable resources for materialized view {}", viewName); - ValidationService.validateOrThrow(deployers, conn); + ValidationService.validateOrThrow(deployers, conn.deploymentContext()); logger.info("Validated materialized view {}", viewName); // Execute (create/update) or collect specs (specify). @@ -450,7 +450,7 @@ static SpecifyResult processCreateMaterializedView(CalcitePrepare.Context ctx, } else { logger.info("Specifying materialized view {}", viewName); } - List specs = mode.executeDeployers(deployers, conn); + List specs = mode.executeDeployers(deployers); if (mode.mutable()) { logger.info("Deployed materialized view {}", viewName); } else { @@ -501,10 +501,10 @@ static SpecifyResult processCreateMaterializedView(CalcitePrepare.Context ctx, */ static SpecifyResult processCreateTable(CalcitePrepare.Context ctx, HoptimatorConnection conn, SqlCreateTable create, DdlMode mode) throws SQLException { - HoptimatorConnection.HoptimatorConnectionDualLogger logger = conn.getLogger(HoptimatorDdlUtils.class); + DualLogger logger = conn.getLogger(HoptimatorDdlUtils.class); logger.info("Validating statement: {}", create); - ValidationService.validateOrThrow(create, conn); + ValidationService.validateOrThrow(create, conn.deploymentContext()); // TODO: Add support for populating new tables from a query as a one-time operation. if (create.query != null) { @@ -514,49 +514,7 @@ static SpecifyResult processCreateTable(CalcitePrepare.Context ctx, HoptimatorCo throw new SQLException("No columns provided."); } - boolean isNewSchema = false; - Pair pair = schema(ctx, mode.mutable(), create.name); - if (pair.left == null) { - // If the schema is not found, it might be because it's a 3-level path (CATALOG.SCHEMA.TABLE) - if (create.name.names.size() > 2) { - pair = catalog(ctx, mode.mutable(), create.name); - isNewSchema = true; - if (pair.left == null) { - throw new SQLException("Catalog for " + create.name + " not found."); - } - } else { - throw new SQLException("Schema for " + create.name + " not found."); - } - } - - final SchemaPlus schemaPlus = pair.left.plus(); - String database = null; - String tableName; - if (isNewSchema) { - int idx = pair.right.indexOf("."); - database = pair.right.substring(0, idx); - tableName = pair.right.substring(idx + 1); - } else { - tableName = pair.right; - } - - if (!isNewSchema && schemaPlus.tables().get(tableName) != null) { - // Strict CREATE without IF NOT EXISTS is the only path that errors. UPDATE - // (apply mode or explicit OR REPLACE) targets the existing table; SPECIFY - // (dry-run) preserves its syntax-driven semantics. - boolean wouldFail; - if (mode == DdlMode.UPDATE) { - wouldFail = false; - } else if (mode == DdlMode.CREATE) { - wouldFail = !create.ifNotExists; - } else { // SPECIFY - wouldFail = !create.ifNotExists && !create.getReplace(); - } - if (wouldFail) { - throw new SQLException( - "Table " + tableName + " already exists. Use CREATE OR REPLACE to update."); - } - } + CreateTarget target = resolveCreateTarget(ctx, conn, mode.mutable(), create.name); // Build row type and column definitions. final JavaTypeFactory typeFactory = ctx.getTypeFactory(); @@ -599,6 +557,72 @@ public RexNode newColumnDefaultValue(RelOptTable table, int iColumn, } }; + Map tableOptions = options(create.options); + CalciteSchemaTarget calcite = new CalciteSchemaTarget(target.pair, target.isNewSchema, ief, rowType); + return deployTableInternal(conn.logHooks(), conn.deploymentContext(), calcite, target.tablePath(), + target.database, target.tableName, tableOptions, + create.ifNotExists, create.getReplace(), mode); + } + + /** Resolved location for a table to be created: its schema/catalog node, database, and name. */ + static final class CreateTarget { + final Pair pair; + final boolean isNewSchema; + final String database; + final String tableName; + + CreateTarget(Pair pair, boolean isNewSchema, String database, String tableName) { + this.pair = pair; + this.isNewSchema = isNewSchema; + this.database = database; + this.tableName = tableName; + } + + /** The fully-qualified table path (schema path + optional new database segment + table name). */ + List tablePath() { + List path = new ArrayList<>(pair.left.path(null)); + if (isNewSchema) { + path.add(database); + } + path.add(tableName); + return path; + } + } + + /** + * Resolves the schema/catalog node, database, and table name for a to-be-created table from a + * (possibly multi-level) identifier by walking the Calcite catalog. This is the Calcite/SQL-path + * resolution, used by the DDL {@code CREATE TABLE} path ({@link #processCreateTable}). The + * connection-free direct path does not use this; it resolves the database identifier + * registry-natively via {@link DatabaseConfigResolver#databaseName} instead. + */ + static CreateTarget resolveCreateTarget(CalcitePrepare.Context ctx, HoptimatorConnection conn, + boolean mutable, SqlIdentifier name) throws SQLException { + boolean isNewSchema = false; + Pair pair = schema(ctx, mutable, name); + if (pair.left == null) { + // If the schema is not found, it might be because it's a 3-level path (CATALOG.SCHEMA.TABLE) + if (name.names.size() > 2) { + pair = catalog(ctx, mutable, name); + isNewSchema = true; + if (pair.left == null) { + throw new SQLException("Catalog for " + name + " not found."); + } + } else { + throw new SQLException("Schema for " + name + " not found."); + } + } + + String database = null; + String tableName; + if (isNewSchema) { + int idx = pair.right.indexOf("."); + database = pair.right.substring(0, idx); + tableName = pair.right.substring(idx + 1); + } else { + tableName = pair.right; + } + if (database == null) { if (pair.left.schema instanceof Database) { database = ((Database) pair.left.schema).databaseName(); @@ -606,49 +630,128 @@ public RexNode newColumnDefaultValue(RelOptTable table, int iColumn, database = conn.getSchema(); } } + return new CreateTarget(pair, isNewSchema, database, tableName); + } + + /** + * Bundles the Calcite-catalog parameters used only by the SQL DDL path. Present ({@code non-null}) + * on the SQL path ({@link #processCreateTable}) and absent ({@code null}) on the connection-free + * direct path ({@code TableService}). + * + *

Grouping these fields into one nullable holder makes the two paths structurally distinct: a + * caller either supplies the whole Calcite handle or none of it. + */ + static final class CalciteSchemaTarget { + // The Calcite schema/catalog node and the (compound) table name + final Pair pair; + // Whether a brand-new JDBC-backed sub-schema must be registered (3-level CATALOG.SCHEMA.TABLE) + final boolean isNewSchema; + // Default-value/generation strategy for the temporary table + final InitializerExpressionFactory ief; + // Row type to register on the temporary table (derived from the SQL column declarations) + final RelDataType rowType; + + CalciteSchemaTarget(Pair pair, boolean isNewSchema, InitializerExpressionFactory ief, + RelDataType rowType) { + this.pair = requireNonNull(pair, "pair"); + requireNonNull(pair.left, "pair.left (Calcite schema node)"); + this.isNewSchema = isNewSchema; + this.ief = ief; + this.rowType = requireNonNull(rowType, "rowType"); + } + + SchemaPlus schemaPlus() { + return pair.left.plus(); + } + } + + /** + * Shared core that deploys (or, in SPECIFY mode, dry-run specifies) a single table given an + * already-resolved schema target and row type. Used by both the DDL {@code CREATE TABLE} path + * ({@link #processCreateTable}) and the SQL-free direct path ({@code TableService}), so that + * existence checks, temporary-table registration, validation, deployment, and rollback behave + * identically whether the row type came from Calcite column declarations or an Avro schema. + * + *

The SQL path passes a non-null {@code calcite} holder. The direct path passes + * {@code null} and touches no catalog. + */ + static SpecifyResult deployTableInternal(List> logHooks, DeploymentContext context, + @Nullable CalciteSchemaTarget calcite, List tablePath, String database, String tableName, + Map options, boolean ifNotExists, boolean orReplace, DdlMode mode) + throws SQLException { + DualLogger logger = new DualLogger(HoptimatorDdlUtils.class, logHooks); + + // Only the SQL path mutates the Calcite catalog: it registers a temporary table (so + // subsequent statements and Calcite-backed deployers can resolve the row type by name) and, + // for a new schema, a JDBC-backed sub-schema. The direct API path carries the row type on its + // DirectDeploymentContext and resolves Database config registry-natively, so it neither reads + // nor mutates the catalog here — existence is enforced store-natively by the deployers. + final boolean manageCalciteSchema = calcite != null; + final SchemaPlus schemaPlus = manageCalciteSchema ? calcite.schemaPlus() : null; + final boolean isNewSchema = manageCalciteSchema && calcite.isNewSchema; + + if (manageCalciteSchema && !isNewSchema && schemaPlus.tables().get(tableName) != null) { + // Strict CREATE without IF NOT EXISTS is the only path that errors. UPDATE + // (apply mode or explicit OR REPLACE) targets the existing table; SPECIFY + // (dry-run) preserves its syntax-driven semantics. + boolean wouldFail; + if (mode == DdlMode.UPDATE) { + wouldFail = false; + } else if (mode == DdlMode.CREATE) { + wouldFail = !ifNotExists; + } else { // SPECIFY + wouldFail = !ifNotExists && !orReplace; + } + if (wouldFail) { + throw new SQLException( + "Table " + tableName + " already exists. Use CREATE OR REPLACE to update."); + } + } - // Snapshot current state for rollback (only meaningful when the schema already exists). + // Snapshot current state for rollback (only meaningful when we mutate the Calcite schema). Pair schemaSnapshot = null; - if (!isNewSchema) { + if (manageCalciteSchema && !isNewSchema) { Table currentTable = schemaPlus.tables().get(tableName); schemaSnapshot = Pair.of(schemaPlus, currentTable); } - // Table does not exist. Create it. - // Add a temporary table with the correct row type so deployers can resolve the schema - // TODO: This may cause problems if we reuse connections, only the next connection will load this as a HoptimatorJdbcTable. - if (isNewSchema) { - HoptimatorJdbcCatalogSchema catalogSchema = schemaPlus.unwrap(HoptimatorJdbcCatalogSchema.class); - if (catalogSchema == null) { - throw new SQLException("Catalog for " + schemaPlus.getName() + " not found."); + if (manageCalciteSchema) { + // For a brand-new schema, register the JDBC-backed sub-schema, then the temporary table. + // TODO: This may cause problems if we reuse connections, only the next connection will load this as a HoptimatorJdbcTable. + SchemaPlus databaseSchema = schemaPlus; + if (isNewSchema) { + HoptimatorJdbcCatalogSchema catalogSchema = schemaPlus.unwrap(HoptimatorJdbcCatalogSchema.class); + if (catalogSchema == null) { + throw new SQLException("Catalog for " + schemaPlus.getName() + " not found."); + } + databaseSchema = schemaPlus.add(database, catalogSchema.createSchema(database)); + logger.info("Added schema {} to catalog {}", database, schemaPlus.getName()); } - SchemaPlus databaseSchema = schemaPlus.add(database, catalogSchema.createSchema(database)); - logger.info("Added schema {} to catalog {}", database, schemaPlus.getName()); - databaseSchema.add(tableName, new TemporaryTable(rowType, database, ief)); + databaseSchema.add(tableName, new TemporaryTable(calcite.rowType, database, calcite.ief)); logger.info("Added table {} to schema {}", tableName, databaseSchema.getName()); - } else { - schemaPlus.add(tableName, new TemporaryTable(rowType, database, ief)); - logger.info("Added table {} to schema {}", tableName, schemaPlus.getName()); } - final List schemaPath = pair.left.path(null); - List tablePath = new ArrayList<>(schemaPath); - if (isNewSchema) { - tablePath.add(database); - } - tablePath.add(tableName); - - Map tableOptions = options(create.options); - Source source = new Source(database, tablePath, tableOptions); + Source source = new Source(database, tablePath, options); Collection deployers = null; boolean success = false; try { logger.info("Validating new table {}", source); - ValidationService.validateOrThrow(source, conn); - deployers = DeploymentService.deployers(source, conn); + ValidationService.validateOrThrow(source, context); + deployers = DeploymentService.deployers(source, context); + // Enforce CREATE (not OR REPLACE) semantics on the connection-free direct path, mirroring the + // Calcite existence check the SQL path ran above. Gated on !manageCalciteSchema so the SQL/DDL + // path (which already checked existence + OR REPLACE) never double-checks or calls exists(). + if (!manageCalciteSchema && mode == DdlMode.CREATE) { + for (Deployer deployer : deployers) { + if (deployer.exists()) { + throw new SQLException("Table " + tableName + + " already exists. Set updateIfExists=true to update it."); + } + } + } logger.info("Validating deployable resources for table {}", tableName); - ValidationService.validateOrThrow(deployers, conn); + ValidationService.validateOrThrow(deployers, context); if (mode == DdlMode.UPDATE) { logger.info("Deploying update table {}", source); @@ -657,7 +760,7 @@ public RexNode newColumnDefaultValue(RelOptTable table, int iColumn, } else { logger.info("Specifying table {}", source); } - List specs = mode.executeDeployers(deployers, conn); + List specs = mode.executeDeployers(deployers); if (mode.mutable()) { logger.info("Deployed table {}", source); } else { @@ -665,7 +768,11 @@ public RexNode newColumnDefaultValue(RelOptTable table, int iColumn, DeploymentService.restore(deployers); } success = true; - return new SpecifyResult(specs, rowType, tablePath); + // Resolve the sink row type only now, for the result: the SQL path already has it on the + // Calcite target; the direct path derives it from the schema carried on its context. Neither + // deployment nor validation above needs it (deployers resolve the row type themselves). + RelDataType sinkRowType = calcite == null ? HoptimatorDriver.rowType(source, context) : calcite.rowType; + return new SpecifyResult(specs, sinkRowType, tablePath); } catch (SQLException | RuntimeException e) { logger.info("Failed to deploy table {}", tableName); if (deployers != null) { @@ -674,11 +781,17 @@ public RexNode newColumnDefaultValue(RelOptTable table, int iColumn, } throw e; } finally { - // For SPECIFY (dry-run): always restore schema. - // For CREATE/UPDATE on success: do NOT restore. - // For CREATE/UPDATE on failure: restore. - if (!success || !mode.mutable()) { - if (schemaSnapshot != null) { + // Undo any Calcite-catalog mutations we made when we are not keeping them (SQL path only; + // the direct path made none): + // - SPECIFY (dry-run): always undo. + // - CREATE/UPDATE on success: keep. + // - CREATE/UPDATE on failure: undo. + if (manageCalciteSchema && (!success || !mode.mutable())) { + if (isNewSchema) { + calcite.pair.left.removeSubSchema(database); + logger.info("Removed schema {} from catalog", database); + } else if (schemaSnapshot != null) { + // Restore the temporary table's prior state. if (schemaSnapshot.right == null) { schemaSnapshot.left.removeTable(tableName); logger.info("Removed schema for table {}", tableName); @@ -686,10 +799,6 @@ public RexNode newColumnDefaultValue(RelOptTable table, int iColumn, schemaPlus.add(tableName, schemaSnapshot.right); logger.info("Restored schema for table {}", tableName); } - } else { - // isNewSchema case on failure: remove the newly created sub-schema. - pair.left.removeSubSchema(database); - logger.info("Removed schema {} from catalog", database); } } } @@ -707,10 +816,10 @@ public RexNode newColumnDefaultValue(RelOptTable table, int iColumn, */ static SpecifyResult processCreateDatabase(HoptimatorConnection conn, SqlCreateDatabase create, DdlMode mode) throws SQLException { - HoptimatorConnection.HoptimatorConnectionDualLogger logger = conn.getLogger(HoptimatorDdlUtils.class); + DualLogger logger = conn.getLogger(HoptimatorDdlUtils.class); logger.info("Validating statement: {}", create); - ValidationService.validateOrThrow(create, conn); + ValidationService.validateOrThrow(create, conn.deploymentContext()); if (create.name.names.size() > 1) { throw new SQLException("Database names cannot be compound identifiers."); @@ -723,11 +832,11 @@ static SpecifyResult processCreateDatabase(HoptimatorConnection conn, Collection deployers = null; try { logger.info("Validating database {}", name); - ValidationService.validateOrThrow(database, conn); - deployers = DeploymentService.deployers(database, conn); - ValidationService.validateOrThrow(deployers, conn); + ValidationService.validateOrThrow(database, conn.deploymentContext()); + deployers = DeploymentService.deployers(database, conn.deploymentContext()); + ValidationService.validateOrThrow(deployers, conn.deploymentContext()); - List specs = mode.executeDeployers(deployers, conn); + List specs = mode.executeDeployers(deployers); if (mode.mutable()) { logger.info("Deployed database {}", name); } else { @@ -824,13 +933,13 @@ public static SpecifyResult specifyFromSql(String sql, HoptimatorConnection conn } try { - Pipeline pipeline = plan.pipeline(viewName, conn); + Pipeline pipeline = plan.pipeline(viewName, conn.deploymentContext()); List specs = new ArrayList<>(); for (Source source : pipeline.sources()) { - specs.addAll(DeploymentService.specify(source, conn)); + specs.addAll(DeploymentService.specify(source, conn.deploymentContext())); } - specs.addAll(DeploymentService.specify(pipeline.sink(), conn)); - specs.addAll(DeploymentService.specify(pipeline.job(), conn)); + specs.addAll(DeploymentService.specify(pipeline.sink(), conn.deploymentContext())); + specs.addAll(DeploymentService.specify(pipeline.job(), conn.deploymentContext())); return new SpecifyResult(specs, sinkRowType, viewPath); } finally { // Restore the schema — the virtual sink must not persist after this call. diff --git a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDriver.java b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDriver.java index a58c4f97e..c52edf11e 100644 --- a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDriver.java +++ b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/HoptimatorDriver.java @@ -1,7 +1,10 @@ package com.linkedin.hoptimator.jdbc; import com.linkedin.hoptimator.Catalog; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Source; +import com.linkedin.hoptimator.avro.AvroSchemaSource; +import org.apache.avro.Schema; import org.apache.calcite.avatica.ConnectStringParser; import org.apache.calcite.jdbc.CalciteConnection; import org.apache.calcite.jdbc.CalcitePrepare; @@ -29,7 +32,6 @@ import java.sql.SQLNonTransientException; import java.sql.SQLTransientConnectionException; import java.sql.SQLTransientException; -import java.util.List; import java.util.Objects; import java.util.Properties; import java.util.logging.LogManager; @@ -162,19 +164,49 @@ public Connection connect(String url, Properties props) throws SQLException { } } - public static RelDataType rowType(Source source, HoptimatorConnection connection) throws SQLException { - final List path = Util.skipLast(source.path()); - String name = source.table(); - SchemaPlus schema = Objects.requireNonNull(connection.calciteConnection().getRootSchema()); - for (String p : path) { - schema = Objects.requireNonNull(schema.subSchemas().get(p)); + public static RelDataType rowType(Source source, DeploymentContext context) + throws SQLException { + if (context instanceof CalciteDeploymentContext) { + HoptimatorConnection connection = ((CalciteDeploymentContext) context).connection(); + SchemaPlus schema = Objects.requireNonNull(connection.calciteConnection().getRootSchema()); + for (String p : Util.skipLast(source.path())) { + schema = Objects.requireNonNull(schema.subSchemas().get(p)); + } + RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); + Table table = schema.tables().get(source.table()); + if (table == null) { + throw new SQLException("Table " + source.table() + " not found in schema " + schema.getName() + "."); + } + return table.getRowType(typeFactory); } - RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); - Table table = schema.tables().get(name); - if (table == null) { - throw new SQLException("Table " + name + " not found in schema " + schema.getName() + "."); + throw new SQLException("Cannot resolve row type for " + source + " from a " + + context.getClass().getSimpleName() + "."); + } + + /** + * Returns the native value (payload) Avro schema for a table when it is backed by an + * {@link AvroSchemaSource} resolved through the Calcite catalog (e.g. a Venice store), or + * {@code null} otherwise (a SQL source with no native Avro, such as a MySQL table or a computed + * view). Callers rendering {@code {{avroValueSchema}}} then synthesize a value schema from the row + * type. + */ + public static Schema valueSchema(Source source, DeploymentContext context) { + if (!(context instanceof CalciteDeploymentContext)) { + return null; + } + HoptimatorConnection connection = ((CalciteDeploymentContext) context).connection(); + if (connection == null) { + return null; + } + SchemaPlus schema = connection.calciteConnection().getRootSchema(); + for (String part : Util.skipLast(source.path())) { + if (schema == null) { + return null; + } + schema = schema.subSchemas().get(part); } - return table.getRowType(typeFactory); + Table table = schema == null ? null : schema.tables().get(source.table()); + return table instanceof AvroSchemaSource ? ((AvroSchemaSource) table).valueSchema() : null; } private static final class ConnectionHolder { diff --git a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/SystemPropertiesConfigProvider.java b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/SystemPropertiesConfigProvider.java index 08bb26017..215fa5c40 100644 --- a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/SystemPropertiesConfigProvider.java +++ b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/SystemPropertiesConfigProvider.java @@ -1,13 +1,13 @@ package com.linkedin.hoptimator.jdbc; import com.linkedin.hoptimator.ConfigProvider; +import com.linkedin.hoptimator.DeploymentContext; -import java.sql.Connection; import java.util.Properties; public class SystemPropertiesConfigProvider implements ConfigProvider { - public Properties loadConfig(Connection connection) { + public Properties loadConfig(DeploymentContext context) { return System.getProperties(); } } diff --git a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/ValidationService.java b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/ValidationService.java index ccd6ceb5d..f8b26493d 100644 --- a/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/ValidationService.java +++ b/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/ValidationService.java @@ -1,13 +1,9 @@ package com.linkedin.hoptimator.jdbc; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Validator; import com.linkedin.hoptimator.ValidatorProvider; -import org.apache.calcite.jdbc.CalciteConnection; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.schema.Table; -import org.apache.calcite.schema.lookup.LikePattern; -import java.sql.Connection; import java.sql.SQLDataException; import java.sql.SQLException; import java.util.ArrayList; @@ -22,46 +18,22 @@ public final class ValidationService { private ValidationService() { } - public static Validator.Issues validate(Connection connection) { - if (!(connection instanceof CalciteConnection)) { - throw new IllegalArgumentException("This connection is unsupported."); - } - CalciteConnection conn = (CalciteConnection) connection; - Validator.Issues issues = new Validator.Issues(""); - walk(conn.getRootSchema(), issues, connection); - return issues; - } - - private static void walk(SchemaPlus schema, Validator.Issues issues, Connection connection) { - validate(schema, issues, connection); - for (String x : schema.subSchemas().getNames(LikePattern.any())) { - walk(schema.subSchemas().get(x), issues.child(x), connection); - } - for (String x : schema.tables().getNames(LikePattern.any())) { - walk(schema.tables().get(x), issues.child(x), connection); - } - } - - private static void walk(Table table, Validator.Issues issues, Connection connection) { - validate(table, issues, connection); - } - - public static void validate(T obj, Validator.Issues issues, Connection connection) { - validators(obj, connection).forEach(x -> x.validate(issues, connection)); + public static void validate(T obj, Validator.Issues issues, DeploymentContext context) { + validators(obj, context).forEach(x -> x.validate(issues, context)); } - public static void validateOrThrow(T obj, Connection connection) throws SQLException { + public static void validateOrThrow(T obj, DeploymentContext context) throws SQLException { Validator.Issues issues = new Validator.Issues(""); - validate(obj, issues, connection); + validate(obj, issues, context); if (!issues.valid()) { throw new SQLDataException("Failed validation:\n" + issues); } } - public static void validateOrThrow(Collection objs, Connection connection) throws SQLException { + public static void validateOrThrow(Collection objs, DeploymentContext context) throws SQLException { Validator.Issues issues = new Validator.Issues(""); for (T obj : objs) { - validate(obj, issues, connection); + validate(obj, issues, context); if (!issues.valid()) { throw new SQLDataException("Failed validation:\n" + issues); } @@ -75,7 +47,7 @@ public static Collection providers() { return providers; } - public static Collection validators(T obj, Connection connection) { - return providers().stream().flatMap(x -> x.validators(obj, connection).stream()).collect(Collectors.toList()); + public static Collection validators(T obj, DeploymentContext context) { + return providers().stream().flatMap(x -> x.validators(obj, context).stream()).collect(Collectors.toList()); } } diff --git a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/DeployerUtilsTest.java b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/DeployerUtilsTest.java index 461ae9ee4..338f81242 100644 --- a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/DeployerUtilsTest.java +++ b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/DeployerUtilsTest.java @@ -4,8 +4,6 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.sql.Connection; import java.sql.SQLException; @@ -156,7 +154,7 @@ void testParseLongOptionKeyPresentValueTenReturns10L() { @Test void testExtractPropertiesReturnsNullForNullSchemaName() { Properties result = DeployerUtils.extractPropertiesFromJdbcSchema(null, null, - mockNonHoptimatorConnection, "jdbc:test://", null); + mockNonHoptimatorConnection, "jdbc:test://"); assertNull(result); } @@ -164,7 +162,7 @@ void testExtractPropertiesReturnsNullForNullSchemaName() { @Test void testExtractPropertiesReturnsNullForNonHoptimatorConnection() { Properties result = DeployerUtils.extractPropertiesFromJdbcSchema(null, "mySchema", - mockNonHoptimatorConnection, "jdbc:test://", null); + mockNonHoptimatorConnection, "jdbc:test://"); assertNull(result); } @@ -176,7 +174,7 @@ void testExtractPropertiesWithHoptimatorConnectionAndMissingSchema() throws SQLE HoptimatorConnection hoptimatorConnection = (HoptimatorConnection) driver.connect("jdbc:hoptimator://", props); Properties result = DeployerUtils.extractPropertiesFromJdbcSchema(null, "nonexistent-schema", - hoptimatorConnection, "jdbc:test://", null); + hoptimatorConnection, "jdbc:test://"); assertNull(result); hoptimatorConnection.close(); @@ -189,7 +187,7 @@ void testExtractPropertiesWithHoptimatorConnectionAndCatalog() throws SQLExcepti HoptimatorConnection hoptimatorConnection = (HoptimatorConnection) driver.connect("jdbc:hoptimator://", props); Properties result = DeployerUtils.extractPropertiesFromJdbcSchema("nonexistent-catalog", "schema", - hoptimatorConnection, "jdbc:test://", null); + hoptimatorConnection, "jdbc:test://"); assertNull(result); hoptimatorConnection.close(); @@ -203,21 +201,7 @@ void testExtractPropertiesWithNonUnwrappableSchema() throws SQLException { // "util" schema exists but is not a HoptimatorJdbcSchema, so unwrap returns null Properties result = DeployerUtils.extractPropertiesFromJdbcSchema(null, "util", - hoptimatorConnection, "jdbc:test://", null); - - assertNull(result); - hoptimatorConnection.close(); - } - - @Test - void testExtractPropertiesWithLoggerOnException() throws SQLException { - HoptimatorDriver driver = new HoptimatorDriver(); - Properties props = new Properties(); - HoptimatorConnection hoptimatorConnection = (HoptimatorConnection) driver.connect("jdbc:hoptimator://catalogs=util", props); - - Logger testLogger = LoggerFactory.getLogger(DeployerUtilsTest.class); - Properties result = DeployerUtils.extractPropertiesFromJdbcSchema(null, "util", - hoptimatorConnection, "jdbc:test://", testLogger); + hoptimatorConnection, "jdbc:test://"); assertNull(result); hoptimatorConnection.close(); diff --git a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/DualLoggerTest.java b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/DualLoggerTest.java new file mode 100644 index 000000000..2503fe0e6 --- /dev/null +++ b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/DualLoggerTest.java @@ -0,0 +1,46 @@ +package com.linkedin.hoptimator.jdbc; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; + + +/** + * Unit tests for {@link DualLogger}. + */ +class DualLoggerTest { + + @Test + void infoFansOutToHookWithClassPrefixAndFormatting() { + List logged = new ArrayList<>(); + DualLogger logger = new DualLogger(DualLoggerTest.class, List.of(logged::add)); + + logger.info("created {} in {}", "table", "schema"); + + assertThat(logged).containsExactly("[DualLoggerTest] created table in schema"); + } + + @Test + void infoInvokesEveryHook() { + List first = new ArrayList<>(); + List second = new ArrayList<>(); + DualLogger logger = new DualLogger(DualLoggerTest.class, List.of(first::add, second::add)); + + logger.info("hello"); + + assertThat(first).containsExactly("[DualLoggerTest] hello"); + assertThat(second).containsExactly("[DualLoggerTest] hello"); + } + + @Test + void infoWithNoHooksDoesNotThrow() { + DualLogger logger = new DualLogger(DualLoggerTest.class, List.of()); + + // slf4j still receives the message; with no hooks there is nothing to assert but it must not throw. + assertThatCode(() -> logger.info("no hooks {}", "here")).doesNotThrowAnyException(); + } +} diff --git a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorConnectionTest.java b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorConnectionTest.java index 169249522..c6a1eafda 100644 --- a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorConnectionTest.java +++ b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorConnectionTest.java @@ -158,7 +158,7 @@ void testAddLogHookIsInvoked() { Consumer hook = logged::add; connection.addLogHook(hook); - HoptimatorConnection.HoptimatorConnectionDualLogger logger = connection.getLogger(HoptimatorConnectionTest.class); + DualLogger logger = connection.getLogger(HoptimatorConnectionTest.class); logger.info("test message {}", "arg1"); assertEquals(1, logged.size()); @@ -168,7 +168,7 @@ void testAddLogHookIsInvoked() { @Test void testGetLoggerReturnsNonNull() { - HoptimatorConnection.HoptimatorConnectionDualLogger logger = connection.getLogger(String.class); + DualLogger logger = connection.getLogger(String.class); assertNotNull(logger); } @@ -206,7 +206,7 @@ void resolveReturnsNonNullTypeForExistingTwoPartPath() throws SQLException { RelDataType result = HoptimatorDriver.rowType( new Source("UTIL", Arrays.asList("UTIL", "PRINT"), Collections.emptyMap()), - conn); + new CalciteDeploymentContext(conn)); assertNotNull(result, "resolve() must find the PRINT table"); assertTrue(result.getFieldCount() > 0, "resolve() must return a non-empty row type"); // "OUTPUT" is a known field in UTIL.PRINT — ensures correct path math, not empty Optional @@ -315,18 +315,4 @@ public RelDataType getRowType(RelDataTypeFactory factory) { throw new UnsupportedOperationException(); } } - - @Test - void testMultipleLogHooksAllInvoked() { - List hook1Messages = new ArrayList<>(); - List hook2Messages = new ArrayList<>(); - connection.addLogHook(hook1Messages::add); - connection.addLogHook(hook2Messages::add); - - HoptimatorConnection.HoptimatorConnectionDualLogger logger = connection.getLogger(HoptimatorConnectionTest.class); - logger.info("hello"); - - assertEquals(1, hook1Messages.size()); - assertEquals(1, hook2Messages.size()); - } -} +} \ No newline at end of file diff --git a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlUtilsTest.java b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlUtilsTest.java index a0c51dc3e..670d37036 100644 --- a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlUtilsTest.java +++ b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorDdlUtilsTest.java @@ -2,6 +2,7 @@ import com.linkedin.hoptimator.Database; import com.linkedin.hoptimator.Deployer; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Job; import com.linkedin.hoptimator.Pipeline; import com.linkedin.hoptimator.Sink; @@ -859,7 +860,7 @@ void ddlModeSpecifyExecuteDeployersCollectsSpecsFromAllDeployers() throws SQLExc when(deployer2.specify()).thenReturn(List.of("spec2")); List result = HoptimatorDdlUtils.DdlMode.SPECIFY.executeDeployers( - List.of(deployer1, deployer2), null); + List.of(deployer1, deployer2)); assertEquals(List.of("spec1a", "spec1b", "spec2"), result); } @@ -867,7 +868,7 @@ void ddlModeSpecifyExecuteDeployersCollectsSpecsFromAllDeployers() throws SQLExc @Test void ddlModeSpecifyExecuteDeployersWithNoDeployersReturnsEmptyList() throws SQLException { List result = HoptimatorDdlUtils.DdlMode.SPECIFY.executeDeployers( - Collections.emptyList(), null); + Collections.emptyList()); assertTrue(result.isEmpty()); } @@ -1699,89 +1700,88 @@ void removeTableFromSchemaIsNoOpWhenEntriesMissing() throws SQLException { } } - /** Helper: a HoptimatorConnection mock that returns the given Properties. */ - private HoptimatorConnection connectionWith(Properties props) { - HoptimatorConnection conn = mock(HoptimatorConnection.class); - lenient().when(conn.connectionProperties()).thenReturn(props); - return conn; + /** Helper: a DeploymentContext that exposes the given Properties. */ + private DeploymentContext contextWith(Properties props) { + DeploymentContext context = mock(DeploymentContext.class); + lenient().when(context.properties()).thenReturn(props); + return context; } @Test void testEffectiveModeDefaultsToStrictCreate() { - HoptimatorConnection conn = connectionWith(new Properties()); + DeploymentContext context = contextWith(new Properties()); assertEquals(HoptimatorDdlUtils.DdlMode.CREATE, - HoptimatorDdlUtils.effectiveMode(false, conn)); + HoptimatorDdlUtils.effectiveMode(false, context)); assertEquals(HoptimatorDdlUtils.DdlMode.UPDATE, - HoptimatorDdlUtils.effectiveMode(true, conn)); + HoptimatorDdlUtils.effectiveMode(true, context)); } @Test void testEffectiveModeExplicitCreateMatchesDefault() { Properties props = new Properties(); props.setProperty(HoptimatorDdlUtils.MODE_PROPERTY, HoptimatorDdlUtils.MODE_CREATE); - HoptimatorConnection conn = connectionWith(props); + DeploymentContext context = contextWith(props); assertEquals(HoptimatorDdlUtils.DdlMode.CREATE, - HoptimatorDdlUtils.effectiveMode(false, conn)); + HoptimatorDdlUtils.effectiveMode(false, context)); assertEquals(HoptimatorDdlUtils.DdlMode.UPDATE, - HoptimatorDdlUtils.effectiveMode(true, conn)); + HoptimatorDdlUtils.effectiveMode(true, context)); } @Test void testEffectiveModeApplyMapsBothCreateFormsToUpdate() { Properties props = new Properties(); props.setProperty(HoptimatorDdlUtils.MODE_PROPERTY, HoptimatorDdlUtils.MODE_APPLY); - HoptimatorConnection conn = connectionWith(props); + DeploymentContext context = contextWith(props); // The whole point of apply mode: plain CREATE becomes idempotent (UPDATE). assertEquals(HoptimatorDdlUtils.DdlMode.UPDATE, - HoptimatorDdlUtils.effectiveMode(false, conn)); + HoptimatorDdlUtils.effectiveMode(false, context)); // CREATE OR REPLACE keeps converging behavior in apply mode. assertEquals(HoptimatorDdlUtils.DdlMode.UPDATE, - HoptimatorDdlUtils.effectiveMode(true, conn)); + HoptimatorDdlUtils.effectiveMode(true, context)); } @Test void testEffectiveModeApplyIsCaseInsensitive() { Properties props = new Properties(); props.setProperty(HoptimatorDdlUtils.MODE_PROPERTY, "APPLY"); - HoptimatorConnection conn = connectionWith(props); + DeploymentContext context = contextWith(props); assertEquals(HoptimatorDdlUtils.DdlMode.UPDATE, - HoptimatorDdlUtils.effectiveMode(false, conn)); + HoptimatorDdlUtils.effectiveMode(false, context)); } @Test void testEffectiveModeUnknownValueFallsBackToStrictCreate() { Properties props = new Properties(); props.setProperty(HoptimatorDdlUtils.MODE_PROPERTY, "nonsense"); - HoptimatorConnection conn = connectionWith(props); + DeploymentContext context = contextWith(props); // Unknown values must not silently promote to apply — typos shouldn't change behavior. assertEquals(HoptimatorDdlUtils.DdlMode.CREATE, - HoptimatorDdlUtils.effectiveMode(false, conn)); + HoptimatorDdlUtils.effectiveMode(false, context)); } @Test void testEffectiveModeTolerantOfNullProperties() { - HoptimatorConnection conn = mock(HoptimatorConnection.class); - lenient().when(conn.connectionProperties()).thenReturn(null); + DeploymentContext context = contextWith(null); assertEquals(HoptimatorDdlUtils.DdlMode.CREATE, - HoptimatorDdlUtils.effectiveMode(false, conn)); + HoptimatorDdlUtils.effectiveMode(false, context)); } @Test void testIsApplyMode() { Properties applyProps = new Properties(); applyProps.setProperty(HoptimatorDdlUtils.MODE_PROPERTY, HoptimatorDdlUtils.MODE_APPLY); - assertTrue(HoptimatorDdlUtils.isApplyMode(connectionWith(applyProps))); + assertTrue(HoptimatorDdlUtils.isApplyMode(applyProps)); Properties createProps = new Properties(); createProps.setProperty(HoptimatorDdlUtils.MODE_PROPERTY, HoptimatorDdlUtils.MODE_CREATE); - assertFalse(HoptimatorDdlUtils.isApplyMode(connectionWith(createProps))); + assertFalse(HoptimatorDdlUtils.isApplyMode(createProps)); - assertFalse(HoptimatorDdlUtils.isApplyMode(connectionWith(new Properties()))); + assertFalse(HoptimatorDdlUtils.isApplyMode(new Properties())); } } diff --git a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorDriverTest.java b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorDriverTest.java index 6225b7027..0df50e4c0 100644 --- a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorDriverTest.java +++ b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/HoptimatorDriverTest.java @@ -1,8 +1,8 @@ package com.linkedin.hoptimator.jdbc; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Source; -import com.linkedin.hoptimator.Validator; -import org.apache.calcite.jdbc.CalciteConnection; +import org.apache.avro.Schema; import org.apache.calcite.jdbc.CalcitePrepare; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlNode; @@ -121,7 +121,7 @@ void testRowTypeResolvesTableFromSchema() throws SQLException { (HoptimatorConnection) driver.connect("jdbc:hoptimator://catalogs=util", new Properties())) { Source source = new Source("UTIL", Arrays.asList("UTIL", "PRINT"), Collections.emptyMap()); - RelDataType rowType = HoptimatorDriver.rowType(source, connection); + RelDataType rowType = HoptimatorDriver.rowType(source, new CalciteDeploymentContext(connection)); assertNotNull(rowType); assertEquals(1, rowType.getFieldCount()); @@ -135,7 +135,38 @@ void testRowTypeThrowsForMissingTable() throws SQLException { (HoptimatorConnection) driver.connect("jdbc:hoptimator://catalogs=util", new Properties())) { Source source = new Source("UTIL", Arrays.asList("UTIL", "NONEXISTENT"), Collections.emptyMap()); - assertThrows(SQLException.class, () -> HoptimatorDriver.rowType(source, connection)); + assertThrows(SQLException.class, () -> HoptimatorDriver.rowType(source, new CalciteDeploymentContext(connection))); + } + } + + @Test + void testRowTypeThrowsForUnknownContextType() { + Source source = new Source("KAFKA", Arrays.asList("KAFKA", "my_topic"), Collections.emptyMap()); + DeploymentContext unknown = new DeploymentContext() { + @Override + public Properties properties() { + return new Properties(); + } + + @Override + public Properties databaseProperties(String catalog, String schema, String connectionPrefix) { + return null; + } + }; + + assertThrows(SQLException.class, () -> HoptimatorDriver.rowType(source, unknown)); + } + + @Test + void testValueSchemaReturnsNullForTableWithoutNativeAvro() throws SQLException { + try (HoptimatorConnection connection = + (HoptimatorConnection) driver.connect("jdbc:hoptimator://catalogs=util", new Properties())) { + // UTIL.PRINT is a real catalog table but not an AvroSchemaSource, so no native value schema. + Source source = new Source("UTIL", Arrays.asList("UTIL", "PRINT"), Collections.emptyMap()); + + Schema schema = HoptimatorDriver.valueSchema(source, new CalciteDeploymentContext(connection)); + + assertNull(schema); } } @@ -222,17 +253,6 @@ void testPrepareStatementWorks() throws SQLException { } } - @Test - void testValidationServiceWithRealConnection() throws SQLException { - try (HoptimatorConnection connection = - (HoptimatorConnection) driver.connect("jdbc:hoptimator://catalogs=util", new Properties())) { - CalciteConnection calciteConn = connection.calciteConnection(); - Validator.Issues issues = ValidationService.validate(calciteConn); - - assertNotNull(issues); - } - } - @Test void testCreatePrepareContextReturnsContext() throws SQLException { try (HoptimatorConnection connection = diff --git a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/TestSqlScripts.java b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/TestSqlScripts.java index d735543d2..53712b717 100644 --- a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/TestSqlScripts.java +++ b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/TestSqlScripts.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.jdbc; +import com.linkedin.hoptimator.DeploymentContext; + import com.linkedin.hoptimator.Validator; import com.linkedin.hoptimator.ValidatorProvider; import com.linkedin.hoptimator.jdbc.ddl.SqlCreateMaterializedView; @@ -13,7 +15,6 @@ import java.net.URLClassLoader; import java.nio.file.Files; import java.nio.file.Path; -import java.sql.Connection; import java.util.Collection; import java.util.List; @@ -77,7 +78,7 @@ private void useTestValidators() throws IOException { public static class CreateViewValidatorProvider implements ValidatorProvider { @Override - public Collection validators(T obj, Connection connection) { + public Collection validators(T obj, DeploymentContext context) { if (obj instanceof SqlCreateView || obj instanceof SqlCreateMaterializedView) { return List.of(new SqlCreateViewValidator()); } @@ -89,7 +90,7 @@ static class SqlCreateViewValidator implements Validator { static final String ERROR_MESSAGE = "Create view is not allowed in this test."; @Override - public void validate(Issues issues, Connection connection) { + public void validate(Issues issues, DeploymentContext context) { issues.error(ERROR_MESSAGE); } } diff --git a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/ValidationServiceTest.java b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/ValidationServiceTest.java index af2bead6a..981da39e9 100644 --- a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/ValidationServiceTest.java +++ b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/ValidationServiceTest.java @@ -10,13 +10,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.sql.Connection; -import java.sql.DriverManager; import java.sql.SQLException; -import java.sql.Statement; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -30,9 +26,6 @@ @ExtendWith(MockitoExtension.class) class ValidationServiceTest { - @Mock - private Connection mockConnection; - @BeforeEach void setUp() { ValidatorProviderTest.reset(); @@ -43,57 +36,6 @@ void tearDown() { ValidatorProviderTest.reset(); } - @Test - void testValidateWithNonCalciteConnectionThrows() { - assertThrows(IllegalArgumentException.class, - () -> ValidationService.validate(mockConnection)); - } - - /** - *

We create the table while errors are off so DDL succeeds, then enable errors - * and call validate(). If walk() or the walk-subschema calls are removed, the provider - * is never called and issues stays valid — the assertion fails. - */ - @Test - void testWalkVisitsTablesInSchema() throws SQLException { - try (HoptimatorConnection conn = - (HoptimatorConnection) DriverManager.getConnection("jdbc:hoptimator://")) { - // Create the table while errors are disabled so the DDL itself does not fail - try (Statement stmt = conn.createStatement()) { - stmt.executeUpdate("CREATE TABLE WALK_VST (X VARCHAR)"); - } - ValidatorProviderTest.enableErrors(); - // Use the underlying CalciteConnection — ValidationService requires CalciteConnection - Validator.Issues issues = ValidationService.validate(conn.calciteConnection()); - assertFalse(issues.valid(), - "walk() must visit tables so that ValidatorProviderTest can record errors"); - } - } - - // util catalog has sub-schemas; if walk() skips recursion, no errors fire. - @Test - void testWalkVisitsSubSchemas() throws SQLException { - try (HoptimatorConnection conn = - (HoptimatorConnection) DriverManager.getConnection("jdbc:hoptimator://catalogs=util")) { - ValidatorProviderTest.enableErrors(); - Validator.Issues issues = ValidationService.validate(conn.calciteConnection()); - assertFalse(issues.valid(), - "walk() must recurse into sub-schemas so that errors in children are propagated"); - } - } - - // Uses the util catalog (always has schemas) to ensure traversal fires. - @Test - void testValidateConnectionCallsWalk() throws SQLException { - try (HoptimatorConnection conn = - (HoptimatorConnection) DriverManager.getConnection("jdbc:hoptimator://catalogs=util")) { - ValidatorProviderTest.enableErrors(); - Validator.Issues issues = ValidationService.validate(conn.calciteConnection()); - assertFalse(issues.valid(), - "validate(connection) must call walk() so that provider errors are propagated"); - } - } - /** * Removes the forEach in validate(obj, issues). * When ValidatorProviderTest is in error mode, calling validate(obj, issues) must diff --git a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/ValidatorProviderTest.java b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/ValidatorProviderTest.java index 9537c4fb8..7fd43c3c2 100644 --- a/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/ValidatorProviderTest.java +++ b/hoptimator-jdbc/src/test/java/com/linkedin/hoptimator/jdbc/ValidatorProviderTest.java @@ -1,9 +1,10 @@ package com.linkedin.hoptimator.jdbc; +import com.linkedin.hoptimator.DeploymentContext; + import com.linkedin.hoptimator.Validator; import com.linkedin.hoptimator.ValidatorProvider; -import java.sql.Connection; import java.util.Collection; import java.util.Collections; import java.util.concurrent.atomic.AtomicBoolean; @@ -40,7 +41,7 @@ static Object lastSeen() { } @Override - public Collection validators(T obj, Connection connection) { + public Collection validators(T obj, DeploymentContext context) { LAST_SEEN.set(obj); if (SHOULD_ERROR.get()) { return Collections.singletonList((issues, conn) -> issues.error("ValidatorProviderTest injected error")); diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sCatalog.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sCatalog.java index bb4c181f8..156e5c3dd 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sCatalog.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sCatalog.java @@ -29,7 +29,7 @@ public String description() { public void register(Wrapper wrapper) throws SQLException { SchemaPlus schemaPlus = wrapper.unwrap(SchemaPlus.class); HoptimatorConnection conn = wrapper.unwrap(HoptimatorConnection.class); - K8sContext context = K8sContext.create(conn); + K8sContext context = K8sContext.create(conn.deploymentContext()); log.info("Using K8s context {}", context); K8sMetadata metadata = createMetadata(conn, context); schemaPlus.add("k8s", metadata); diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sConfigProvider.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sConfigProvider.java index b183dd663..6d8912f45 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sConfigProvider.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sConfigProvider.java @@ -1,10 +1,10 @@ package com.linkedin.hoptimator.k8s; import com.linkedin.hoptimator.ConfigProvider; +import com.linkedin.hoptimator.DeploymentContext; import io.kubernetes.client.openapi.models.V1ConfigMap; import io.kubernetes.client.openapi.models.V1ConfigMapList; -import java.sql.Connection; import java.sql.SQLException; import java.util.Map; import java.util.Properties; @@ -14,8 +14,8 @@ public class K8sConfigProvider implements ConfigProvider { public static final String HOPTIMATOR_CONFIG_MAP = "hoptimator-configmap"; - public Properties loadConfig(Connection connection) throws SQLException { - Map topLevelConfigs = loadTopLevelConfig(HOPTIMATOR_CONFIG_MAP, connection); + public Properties loadConfig(DeploymentContext deploymentContext) throws SQLException { + Map topLevelConfigs = loadTopLevelConfig(HOPTIMATOR_CONFIG_MAP, deploymentContext); Properties p = new Properties(); p.putAll(topLevelConfigs); return p; @@ -26,8 +26,9 @@ K8sApi createConfigMapApi(K8sContext context) { return new K8sApi<>(context, K8sApiEndpoints.CONFIG_MAPS); } - private Map loadTopLevelConfig(String configMapName, Connection connection) throws SQLException { - K8sContext context = K8sContext.create(connection); + private Map loadTopLevelConfig(String configMapName, DeploymentContext deploymentContext) + throws SQLException { + K8sContext context = K8sContext.create(deploymentContext); K8sApi configMapApi = createConfigMapApi(context); String namespace = context.namespace(); if (namespace == null || namespace.isEmpty()) { diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sConnector.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sConnector.java index fe5145a5d..87af1e341 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sConnector.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sConnector.java @@ -5,7 +5,6 @@ import com.linkedin.hoptimator.Sink; import com.linkedin.hoptimator.Source; import com.linkedin.hoptimator.avro.AvroConverter; -import com.linkedin.hoptimator.avro.AvroSchemaSource; import com.linkedin.hoptimator.avro.AvroSchemas; import com.linkedin.hoptimator.jdbc.HoptimatorDriver; import com.linkedin.hoptimator.k8s.models.V1alpha1TableTemplate; @@ -15,9 +14,6 @@ import org.apache.avro.Schema; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.schema.Table; -import org.apache.calcite.util.Util; import java.io.IOException; import java.io.StringReader; @@ -55,7 +51,7 @@ K8sApi createTableTemplateApi( @Override public Map configure() throws SQLException { - RelDataType sourceRowType = HoptimatorDriver.rowType(source, context.connection()); + RelDataType sourceRowType = HoptimatorDriver.rowType(source, context.deploymentContext()); Map options = addKeysAsOption(source.options(), sourceRowType); Template.Environment env = @@ -115,34 +111,21 @@ private Map getConnectorHints(Map options, Strin } /** - * Renders the value Avro schema for the {@code {{avroValueSchema}}} template variable. Prefers - * the upstream table's native value schema when it implements {@link AvroSchemaSource} — keys - * aren't included, because the connector handles them separately via {@code key.fields}. Falls - * back to synthesizing from the flat row type, which loses source-level namespaces and nested - * record identities. + * Renders the value Avro schema for the {@code {{avroValueSchema}}} template variable. Prefers the + * schema {@link HoptimatorDriver#valueSchema} resolves — the caller's carried value schema on the + * direct path, or the upstream table's native value schema on the SQL path — both value-only, with + * keys handled separately via {@code key.fields}. Falls back to synthesizing from the flat row type + * only when neither exists (a SQL source with no native Avro, e.g. a MySQL table or a view), which + * loses source-level namespaces and nested record identities. */ private Schema avroValueSchema(Source source, RelDataType sourceRowType) { - Table table = lookupTable(source); - if (table instanceof AvroSchemaSource) { - Schema provided = ((AvroSchemaSource) table).valueSchema(); - if (provided != null) { - return provided; - } + Schema provided = HoptimatorDriver.valueSchema(source, context.deploymentContext()); + if (provided != null) { + return provided; } return AvroConverter.avro("com.linkedin.hoptimator", source.table(), sourceRowType); } - private Table lookupTable(Source source) { - SchemaPlus schema = context.connection().calciteConnection().getRootSchema(); - for (String part : Util.skipLast(source.path())) { - if (schema == null) { - return null; - } - schema = schema.subSchemas().get(part); - } - return schema == null ? null : schema.tables().get(source.table()); - } - @VisibleForTesting static Map addKeysAsOption(Map options, RelDataType rowType) { Map newOptions = new LinkedHashMap<>(options); diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sConnectorProvider.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sConnectorProvider.java index f36b11fc6..0e183e426 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sConnectorProvider.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sConnectorProvider.java @@ -2,9 +2,9 @@ import com.linkedin.hoptimator.Connector; import com.linkedin.hoptimator.ConnectorProvider; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Source; -import java.sql.Connection; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -13,8 +13,8 @@ public class K8sConnectorProvider implements ConnectorProvider { @Override - public Collection connectors(T obj, Connection connection) { - K8sContext context = K8sContext.create(connection); + public Collection connectors(T obj, DeploymentContext deploymentContext) { + K8sContext context = K8sContext.create(deploymentContext); List list = new ArrayList<>(); if (obj instanceof Source) { list.add(new K8sConnector((Source) obj, context)); diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sContext.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sContext.java index 387e1dd2d..17d946a09 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sContext.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sContext.java @@ -1,6 +1,6 @@ package com.linkedin.hoptimator.k8s; -import com.linkedin.hoptimator.jdbc.HoptimatorConnection; +import com.linkedin.hoptimator.DeploymentContext; import io.kubernetes.client.apimachinery.GroupVersion; import io.kubernetes.client.common.KubernetesListObject; import io.kubernetes.client.common.KubernetesObject; @@ -20,7 +20,6 @@ import java.io.Reader; import java.nio.file.Files; import java.nio.file.Paths; -import java.sql.Connection; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; @@ -51,11 +50,11 @@ public final class K8sContext { private final SharedInformerFactory informerFactory; private final V1OwnerReference ownerReference; private final Map labels; - private final HoptimatorConnection connection; + private final DeploymentContext deploymentContext; K8sContext(String namespace, String watchNamespace, String clientInfo, ApiClient apiClient, SharedInformerFactory informerFactory, V1OwnerReference ownerReference, Map labels, - HoptimatorConnection connection) { + DeploymentContext deploymentContext) { this.namespace = namespace; this.watchNamespace = watchNamespace; this.clientInfo = clientInfo; @@ -63,16 +62,27 @@ public final class K8sContext { this.informerFactory = informerFactory; this.ownerReference = ownerReference; this.labels = labels; - this.connection = connection; + this.deploymentContext = deploymentContext; } - public static K8sContext create(Connection connection) { + public static K8sContext create(DeploymentContext deploymentContext) { + return create(deploymentContext.properties(), deploymentContext); + } + + /** + * Builds a context from raw connection-level properties, with no backing {@code DeploymentContext}. + * Used by connection-free callers (e.g. a K8s-native {@code DatabaseConfigResolver}) that only + * need K8s API access, not a Calcite catalog. + */ + public static K8sContext create(Properties connectionProperties) { + return create(connectionProperties, null); + } + + private static K8sContext create(Properties connectionProperties, DeploymentContext deploymentContext) { String namespace; ApiClient apiClient; String info; - HoptimatorConnection hoptimatorConnection = (HoptimatorConnection) connection; - Properties connectionProperties = hoptimatorConnection.connectionProperties(); if (connectionProperties.getProperty(NAMESPACE_KEY) != null) { namespace = connectionProperties.getProperty(NAMESPACE_KEY); } else { @@ -151,19 +161,19 @@ public static K8sContext create(Connection connection) { } return new K8sContext(namespace, watchNamespace, info, apiClient, new SharedInformerFactory(apiClient), - null, Collections.emptyMap(), hoptimatorConnection); + null, Collections.emptyMap(), deploymentContext); } public K8sContext withOwner(V1OwnerReference owner) { return new K8sContext(namespace, watchNamespace, clientInfo + " Owner is " + owner.getName() + ".", apiClient, - informerFactory, owner, labels, connection); + informerFactory, owner, labels, deploymentContext); } public K8sContext withLabel(String key, String value) { Map newLabels = new HashMap<>(labels); newLabels.put(key, value); return new K8sContext(namespace, watchNamespace, clientInfo + " Label " + key + "=" + value + ".", apiClient, - informerFactory, ownerReference, newLabels, connection); + informerFactory, ownerReference, newLabels, deploymentContext); } public ApiClient apiClient() { @@ -237,8 +247,8 @@ public void own(KubernetesObject obj) { } } - public HoptimatorConnection connection() { - return connection; + public DeploymentContext deploymentContext() { + return deploymentContext; } @Override diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDatabaseTable.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDatabaseTable.java index a3bd11500..0986e3691 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDatabaseTable.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDatabaseTable.java @@ -1,5 +1,6 @@ package com.linkedin.hoptimator.k8s; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.jdbc.HoptimatorConnection; import com.linkedin.hoptimator.k8s.models.V1alpha1Database; import com.linkedin.hoptimator.k8s.models.V1alpha1DatabaseList; @@ -54,16 +55,17 @@ public K8sDatabaseTable(K8sContext context, K8sEngineTable engines) { } public void addDatabases(SchemaPlus parentSchema, Connection connection) { + DeploymentContext context = ((HoptimatorConnection) connection).deploymentContext(); for (Row row : rows()) { if (row.CATALOG != null) { Schema catalogSchema = HoptimatorJdbcCatalogSchema.create(row.NAME, row.CATALOG, row.SCHEMA, dataSource(row, ((HoptimatorConnection) connection).connectionProperties()), parentSchema, - dialect(row), engines.forDatabase(row.NAME), connection); + dialect(row), engines.forDatabase(row.NAME), context); parentSchema.add(row.CATALOG.toUpperCase(Locale.ROOT), catalogSchema); } else { Schema schema = HoptimatorJdbcSchema.create(row.NAME, row.CATALOG, row.SCHEMA, dataSource(row, ((HoptimatorConnection) connection).connectionProperties()), parentSchema, - dialect(row), engines.forDatabase(row.NAME), connection); + dialect(row), engines.forDatabase(row.NAME), context); parentSchema.add(schemaName(row), schema); } } @@ -71,6 +73,11 @@ public void addDatabases(SchemaPlus parentSchema, Connection connection) { @Override public Row toRow(V1alpha1Database obj) { + return rowOf(obj); + } + + /** Builds a {@link Row} from a Database CRD, usable without a {@link K8sDatabaseTable} instance. */ + static Row rowOf(V1alpha1Database obj) { return new Row(Objects.requireNonNull(obj.getMetadata()).getName(), Objects.requireNonNull(obj.getSpec()).getUrl(), obj.getSpec().getCatalog(), obj.getSpec().getSchema(), Optional.ofNullable(obj.getSpec().getDialect()).map(V1alpha1DatabaseSpec.DialectEnum::toString).orElse(null), @@ -101,15 +108,27 @@ static String schemaName(Row row) { static DataSource dataSource(Row row, Properties connectionProperties) { String user = "nouser"; String pass = "nopass"; - StringJoiner joiner = new StringJoiner(";"); for (String key : connectionProperties.stringPropertyNames()) { if ("user".equals(key)) { user = connectionProperties.getProperty(key); } else if ("password".equals(key)) { pass = connectionProperties.getProperty(key); - } else { - String value = connectionProperties.getProperty(key); - joiner.add(key + "=" + value); + } + } + return JdbcSchema.dataSource(joinedUrl(row, connectionProperties), row.DRIVER, user, pass); + } + + /** + * Builds the effective JDBC URL for a Database: its CRD {@code url} with the connection-level + * properties (except {@code user}/{@code password}) and the CRD name appended as + * {@code database=}. This is the URL a {@code DatabaseConfigResolver} parses to recover a + * database's connection properties, kept here so it stays in lockstep with {@link #dataSource}. + */ + static String joinedUrl(Row row, Properties connectionProperties) { + StringJoiner joiner = new StringJoiner(";"); + for (String key : connectionProperties.stringPropertyNames()) { + if (!"user".equals(key) && !"password".equals(key)) { + joiner.add(key + "=" + connectionProperties.getProperty(key)); } } // Inject the Database CRD name so drivers can identify which CRD they are backing. @@ -117,14 +136,11 @@ static DataSource dataSource(Row row, Properties connectionProperties) { if (row.NAME != null && !row.NAME.isEmpty()) { joiner.add("database=" + row.NAME); } - String joinedUrl = row.URL; // Handles case where there are no properties already in the URL if (row.URL.endsWith("//")) { - joinedUrl = joinedUrl + joiner; - } else { - joinedUrl = joinedUrl + ";" + joiner; + return row.URL + joiner; } - return JdbcSchema.dataSource(joinedUrl, row.DRIVER, user, pass); + return row.URL + ";" + joiner; } static SqlDialect dialect(Row row) { diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDependencyValidator.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDependencyValidator.java index 5a7462062..4214784a0 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDependencyValidator.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDependencyValidator.java @@ -1,9 +1,9 @@ package com.linkedin.hoptimator.k8s; -import java.sql.Connection; import java.sql.SQLException; import javax.annotation.Nullable; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Source; import com.linkedin.hoptimator.Validator; @@ -27,10 +27,10 @@ final class K8sDependencyValidator implements Validator { } @Override - public void validate(Issues issues, Connection connection) { + public void validate(Issues issues, DeploymentContext context) { try { DependencyChecker.assertNoExternalDependents( - K8sContext.create(connection), source.database(), source.path(), + K8sContext.create(context), source.database(), source.path(), selfOwnerKind, selfOwnerName); } catch (SQLException e) { issues.error(e.getMessage()); diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDeployer.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDeployer.java index 84a787b4e..ad3d566a1 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDeployer.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDeployer.java @@ -46,6 +46,17 @@ private void create(T obj) throws SQLException { api.create(obj); } + @Override + public boolean exists() throws SQLException { + T obj = toK8sObject(); + String namespace = obj.getMetadata().getNamespace(); + String name = obj.getMetadata().getName(); + if (namespace != null) { + return api.getIfExists(namespace, name) != null; + } + return api.getIfExists(name) != null; + } + @Override public void delete() throws SQLException { T obj = toK8sObject(); diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDeployerProvider.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDeployerProvider.java index 2464ff73a..23479eba2 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDeployerProvider.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDeployerProvider.java @@ -4,13 +4,13 @@ import com.linkedin.hoptimator.Deployable; import com.linkedin.hoptimator.Deployer; import com.linkedin.hoptimator.DeployerProvider; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Job; import com.linkedin.hoptimator.MaterializedView; import com.linkedin.hoptimator.Source; import com.linkedin.hoptimator.Trigger; import com.linkedin.hoptimator.View; -import java.sql.Connection; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -18,9 +18,9 @@ public class K8sDeployerProvider implements DeployerProvider { @Override - public Collection deployers(T obj, Connection connection) { + public Collection deployers(T obj, DeploymentContext deploymentContext) { List list = new ArrayList<>(); - K8sContext context = K8sContext.create(connection); + K8sContext context = K8sContext.create(deploymentContext); if (obj instanceof MaterializedView) { // K8sMaterializedViewDeployer also deploys a View. list.add(new K8sMaterializedViewDeployer((MaterializedView) obj, context)); diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sGraphProvider.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sGraphProvider.java index 3e666d760..284eb5528 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sGraphProvider.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sGraphProvider.java @@ -6,6 +6,7 @@ import com.linkedin.hoptimator.graph.GraphProvider; import com.linkedin.hoptimator.graph.GraphTarget; import com.linkedin.hoptimator.graph.PipelineGraph; +import com.linkedin.hoptimator.jdbc.HoptimatorConnection; /** @@ -35,7 +36,7 @@ public PipelineGraph forTarget(GraphTarget target, int depth, Connection connect throw new SQLException("K8sGraphProvider.forTarget requires a non-null JDBC connection; " + "K8sContext can't be derived from null."); } - K8sContext context = K8sContext.create(connection); + K8sContext context = K8sContext.create(((HoptimatorConnection) connection).deploymentContext()); PipelineGraphBuilder builder = createBuilder(context); if (target instanceof GraphTarget.View) { GraphTarget.View v = (GraphTarget.View) target; diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sJobDeployer.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sJobDeployer.java index 53f0abd42..5f8fb19d0 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sJobDeployer.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sJobDeployer.java @@ -40,7 +40,7 @@ K8sApi createJobTemplateApi(K8sCon @Override public List specify() throws SQLException { - Properties properties = ConfigService.config(context.connection(), false, FLINK_CONFIG); + Properties properties = ConfigService.config(context.deploymentContext(), false, FLINK_CONFIG); properties.putAll(job.sink().options()); ThrowingFunction sql = job.sql(); ThrowingFunction fieldMap = job.fieldMap(); diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sMaterializedViewDeployer.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sMaterializedViewDeployer.java index 76967411c..ad88b2e37 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sMaterializedViewDeployer.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sMaterializedViewDeployer.java @@ -91,10 +91,10 @@ public void restore() { List pipelineSpecs() throws SQLException { List specs = new ArrayList<>(); for (Source source : view.pipeline().sources()) { - specs.addAll(DeploymentService.specify(source, context.connection())); + specs.addAll(DeploymentService.specify(source, context.deploymentContext())); } - specs.addAll(DeploymentService.specify(view.pipeline().sink(), context.connection())); - specs.addAll(DeploymentService.specify(view.pipeline().job(), context.connection())); + specs.addAll(DeploymentService.specify(view.pipeline().sink(), context.deploymentContext())); + specs.addAll(DeploymentService.specify(view.pipeline().job(), context.deploymentContext())); return specs; } diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sSourceDeployer.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sSourceDeployer.java index 3565c454f..6edcf3d63 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sSourceDeployer.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sSourceDeployer.java @@ -1,7 +1,6 @@ package com.linkedin.hoptimator.k8s; import com.linkedin.hoptimator.Source; -import com.linkedin.hoptimator.jdbc.HoptimatorConnection; import com.linkedin.hoptimator.k8s.models.V1alpha1TableTemplate; import com.linkedin.hoptimator.k8s.models.V1alpha1TableTemplateList; import com.linkedin.hoptimator.k8s.models.V1alpha1TableTemplateSpec; @@ -36,7 +35,6 @@ K8sApi createTableTemplateApi( @Override public List specify() throws SQLException { String name = K8sUtils.canonicalizeName(source.database(), source.table()); - HoptimatorConnection connection = context.connection(); Template.Environment env = new Template.SimpleEnvironment() .with("name", name) @@ -47,7 +45,7 @@ public List specify() throws SQLException { .with("path", source.pathString()) .with(source.options()) .with(JOB_PROPERTIES_PREFIX, getJobPropertiesFromOptions(source.options())) - .with(DeploymentService.parseHints(connection.connectionProperties())); + .with(DeploymentService.parseHints(context.deploymentContext().properties())); List templates = tableTemplateApi.list() .stream() diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sValidatorProvider.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sValidatorProvider.java index 7cf14d20c..7be098768 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sValidatorProvider.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sValidatorProvider.java @@ -1,9 +1,9 @@ package com.linkedin.hoptimator.k8s; -import java.sql.Connection; import java.util.Collection; import java.util.Collections; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.PendingDelete; import com.linkedin.hoptimator.Source; import com.linkedin.hoptimator.Validator; @@ -20,7 +20,7 @@ public class K8sValidatorProvider implements ValidatorProvider { @Override - public Collection validators(T obj, Connection connection) { + public Collection validators(T obj, DeploymentContext context) { if (obj instanceof PendingDelete) { PendingDelete pd = (PendingDelete) obj; Object target = pd.target(); diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sViewTable.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sViewTable.java index 19a55611d..686e0db87 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sViewTable.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sViewTable.java @@ -2,6 +2,7 @@ import com.linkedin.hoptimator.Validated; import com.linkedin.hoptimator.Validator; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.jdbc.HoptimatorConnection; import com.linkedin.hoptimator.jdbc.HoptimatorDriver; import com.linkedin.hoptimator.jdbc.MaterializedViewTable; @@ -16,7 +17,6 @@ import org.apache.calcite.schema.impl.AbstractSchema; import org.apache.calcite.schema.impl.ViewTable; -import java.sql.Connection; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -158,7 +158,7 @@ public Schema.TableType getJdbcTableType() { } @Override - public void validate(Validator.Issues issues, Connection connection) { + public void validate(Validator.Issues issues, DeploymentContext context) { for (Row row : rows()) { Validator.Issues issues2 = issues.child(row.toString()); Validator.validateSubdomainName(row.NAME, issues2.child("NAME")); diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sYamlDeployer.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sYamlDeployer.java index 47bda97cd..8b7e1ba90 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sYamlDeployer.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sYamlDeployer.java @@ -33,6 +33,16 @@ public void create() throws SQLException { } } + @Override + public boolean exists() throws SQLException { + for (String spec : specify()) { + if (api.getIfExists(spec) != null) { + return true; + } + } + return false; + } + @Override public void delete() throws SQLException { for (String spec : specify()) { diff --git a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/DependencyLabelsTest.java b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/DependencyLabelsTest.java index 6b1027c86..2eda1a471 100644 --- a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/DependencyLabelsTest.java +++ b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/DependencyLabelsTest.java @@ -2,6 +2,7 @@ import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; @@ -67,7 +68,7 @@ void labelKeyFitsKubernetesNameLimit() { "name portion must match K8s label-name regex, got: " + namePortion); } - private static V1ObjectMeta stamp(java.util.List sources, Sink sink) { + private static V1ObjectMeta stamp(List sources, Sink sink) { V1ObjectMeta meta = new V1ObjectMeta(); DependencyLabels.stamp(meta, sources, sink == null ? Collections.emptyList() : Collections.singletonList(sink)); diff --git a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sCatalogTest.java b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sCatalogTest.java index 00b8c4dad..c07e44658 100644 --- a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sCatalogTest.java +++ b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sCatalogTest.java @@ -1,6 +1,7 @@ package com.linkedin.hoptimator.k8s; import com.linkedin.hoptimator.jdbc.HoptimatorConnection; +import com.linkedin.hoptimator.DeploymentContext; import org.apache.calcite.jdbc.CalciteSchema; import org.apache.calcite.schema.SchemaPlus; import org.junit.jupiter.api.Test; @@ -59,7 +60,7 @@ void registerAddsMetadataAndDatabases() throws SQLException { SchemaPlus schemaPlus = CalciteSchema.createRootSchema(true).plus(); doReturn(schemaPlus).when(wrapper).unwrap(SchemaPlus.class); doReturn(connection).when(wrapper).unwrap(HoptimatorConnection.class); - mockedK8sContext.when(() -> K8sContext.create(any())).thenReturn(mockContext); + mockedK8sContext.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(mockContext); // Use a subclass that returns our test metadata with mocked tables K8sCatalog catalog = new K8sCatalog() { @@ -90,7 +91,7 @@ void registerAddsViewsToSchema() throws SQLException { SchemaPlus schemaPlus = CalciteSchema.createRootSchema(true).plus(); doReturn(schemaPlus).when(wrapper).unwrap(SchemaPlus.class); doReturn(connection).when(wrapper).unwrap(HoptimatorConnection.class); - mockedK8sContext.when(() -> K8sContext.create(any())).thenReturn(mockContext); + mockedK8sContext.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(mockContext); K8sCatalog catalog = new K8sCatalog() { @Override @@ -122,7 +123,7 @@ void registerAddsDatabasesViaAddDatabases() throws SQLException { SchemaPlus schemaPlus = CalciteSchema.createRootSchema(true).plus(); doReturn(schemaPlus).when(wrapper).unwrap(SchemaPlus.class); doReturn(connection).when(wrapper).unwrap(HoptimatorConnection.class); - mockedK8sContext.when(() -> K8sContext.create(any())).thenReturn(mockContext); + mockedK8sContext.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(mockContext); K8sCatalog catalog = new K8sCatalog() { @Override diff --git a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sConfigProviderTest.java b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sConfigProviderTest.java index 12e4edfed..1902a704e 100644 --- a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sConfigProviderTest.java +++ b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sConfigProviderTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.k8s; +import com.linkedin.hoptimator.DeploymentContext; + import io.kubernetes.client.openapi.models.V1ConfigMap; import io.kubernetes.client.openapi.models.V1ConfigMapList; import io.kubernetes.client.openapi.models.V1ObjectMeta; @@ -9,7 +11,6 @@ import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoExtension; -import java.sql.Connection; import java.sql.SQLException; import java.util.HashMap; import java.util.Map; @@ -28,7 +29,7 @@ class K8sConfigProviderTest { @Mock - private Connection connection; + private DeploymentContext connection; @Mock private MockedStatic k8sContextStatic; @@ -61,7 +62,7 @@ void loadConfigReturnsProperties() throws SQLException { K8sApi mockApi = mock(K8sApi.class); when(mockContext.namespace()).thenReturn("test-ns"); when(mockApi.get("test-ns", "hoptimator-configmap")).thenReturn(configMap); - k8sContextStatic.when(() -> K8sContext.create(any(Connection.class))).thenReturn(mockContext); + k8sContextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(mockContext); K8sConfigProvider provider = new K8sConfigProvider() { @Override @@ -90,7 +91,7 @@ void loadConfigWithEmptyNamespaceUsesNameOnly() throws SQLException { K8sApi mockApi = mock(K8sApi.class); when(mockContext.namespace()).thenReturn(""); when(mockApi.get("hoptimator-configmap")).thenReturn(configMap); - k8sContextStatic.when(() -> K8sContext.create(any(Connection.class))).thenReturn(mockContext); + k8sContextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(mockContext); K8sConfigProvider provider = new K8sConfigProvider() { @Override @@ -110,7 +111,7 @@ void loadConfigThrowsWhenConfigMapNotFound() throws SQLException { K8sApi mockApi = mock(K8sApi.class); when(mockContext.namespace()).thenReturn("test-ns"); when(mockApi.get("test-ns", "hoptimator-configmap")).thenThrow(new SQLException("Not found")); - k8sContextStatic.when(() -> K8sContext.create(any(Connection.class))).thenReturn(mockContext); + k8sContextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(mockContext); K8sConfigProvider provider = new K8sConfigProvider() { @Override @@ -132,7 +133,7 @@ void loadConfigReturnsEmptyPropertiesWhenDataEmpty() throws SQLException { K8sApi mockApi = mock(K8sApi.class); when(mockContext.namespace()).thenReturn("test-ns"); when(mockApi.get("test-ns", "hoptimator-configmap")).thenReturn(configMap); - k8sContextStatic.when(() -> K8sContext.create(any(Connection.class))).thenReturn(mockContext); + k8sContextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(mockContext); K8sConfigProvider provider = new K8sConfigProvider() { @Override diff --git a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sConnectorProviderTest.java b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sConnectorProviderTest.java index 20b324c8e..f683bd226 100644 --- a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sConnectorProviderTest.java +++ b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sConnectorProviderTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.k8s; +import com.linkedin.hoptimator.DeploymentContext; + import com.linkedin.hoptimator.Connector; import com.linkedin.hoptimator.Source; import org.junit.jupiter.api.Test; @@ -8,7 +10,6 @@ import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoExtension; -import java.sql.Connection; import java.util.Collection; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -22,7 +23,7 @@ class K8sConnectorProviderTest { @Mock - private Connection connection; + private DeploymentContext connection; @Mock private K8sContext context; @@ -32,7 +33,7 @@ class K8sConnectorProviderTest { @Test void connectorsForSourceReturnsK8sConnector() { - contextStatic.when(() -> K8sContext.create(any(Connection.class))).thenReturn(context); + contextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(context); K8sConnectorProvider provider = new K8sConnectorProvider(); Source source = mock(Source.class); @@ -44,7 +45,7 @@ void connectorsForSourceReturnsK8sConnector() { @Test void connectorsForNonSourceReturnsEmpty() { - contextStatic.when(() -> K8sContext.create(any(Connection.class))).thenReturn(context); + contextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(context); K8sConnectorProvider provider = new K8sConnectorProvider(); Collection connectors = provider.connectors("not a source", connection); diff --git a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sConnectorTest.java b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sConnectorTest.java index 82e892391..4603a6f7c 100644 --- a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sConnectorTest.java +++ b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sConnectorTest.java @@ -1,9 +1,8 @@ package com.linkedin.hoptimator.k8s; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Sink; import com.linkedin.hoptimator.Source; -import com.linkedin.hoptimator.avro.AvroSchemaSource; -import com.linkedin.hoptimator.jdbc.HoptimatorConnection; import com.linkedin.hoptimator.jdbc.HoptimatorDriver; import com.linkedin.hoptimator.k8s.models.V1alpha1TableTemplate; import com.linkedin.hoptimator.k8s.models.V1alpha1TableTemplateList; @@ -11,15 +10,9 @@ import io.kubernetes.client.openapi.models.V1ObjectMeta; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; -import org.apache.calcite.jdbc.CalciteConnection; -import org.apache.calcite.jdbc.CalciteSchema; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeSystem; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.schema.Table; -import org.apache.calcite.schema.impl.AbstractSchema; -import org.apache.calcite.schema.impl.AbstractTable; import org.apache.calcite.sql.type.SqlTypeFactoryImpl; import org.apache.calcite.sql.type.SqlTypeName; import org.junit.jupiter.api.Test; @@ -43,8 +36,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import static org.mockito.ArgumentMatchers.nullable; @ExtendWith(MockitoExtension.class) @@ -53,9 +45,6 @@ class K8sConnectorTest { @Mock private MockedStatic hoptimatorDriverMock; - @Mock - private HoptimatorConnection connection; - @Mock private K8sContext mockContext; @@ -158,7 +147,7 @@ void configureWithNoTemplatesReturnsEmpty() throws SQLException { builder.add("value", SqlTypeName.VARCHAR); RelDataType rowType = builder.build(); - hoptimatorDriverMock.when(() -> HoptimatorDriver.rowType(any(Source.class), any())) + hoptimatorDriverMock.when(() -> HoptimatorDriver.rowType(any(Source.class), nullable(DeploymentContext.class))) .thenReturn(rowType); List templates = new ArrayList<>(); @@ -181,7 +170,7 @@ void configureRendersMatchingTemplate() throws SQLException { builder.add("value", SqlTypeName.VARCHAR); RelDataType rowType = builder.build(); - hoptimatorDriverMock.when(() -> HoptimatorDriver.rowType(any(Source.class), any())) + hoptimatorDriverMock.when(() -> HoptimatorDriver.rowType(any(Source.class), nullable(DeploymentContext.class))) .thenReturn(rowType); List templates = new ArrayList<>(); @@ -210,7 +199,7 @@ void configureFiltersOutNonMatchingDatabases() throws SQLException { builder.add("value", SqlTypeName.VARCHAR); RelDataType rowType = builder.build(); - hoptimatorDriverMock.when(() -> HoptimatorDriver.rowType(any(Source.class), any())) + hoptimatorDriverMock.when(() -> HoptimatorDriver.rowType(any(Source.class), nullable(DeploymentContext.class))) .thenReturn(rowType); List templates = new ArrayList<>(); @@ -238,7 +227,7 @@ void configureWithConnectorHints() throws SQLException { builder.add("value", SqlTypeName.VARCHAR); RelDataType rowType = builder.build(); - hoptimatorDriverMock.when(() -> HoptimatorDriver.rowType(any(Source.class), any())) + hoptimatorDriverMock.when(() -> HoptimatorDriver.rowType(any(Source.class), nullable(DeploymentContext.class))) .thenReturn(rowType); List templates = new ArrayList<>(); @@ -268,7 +257,7 @@ void configureHintsExactKeyAfterStrippingPrefix() throws SQLException { builder.add("value", SqlTypeName.VARCHAR); RelDataType rowType = builder.build(); - hoptimatorDriverMock.when(() -> HoptimatorDriver.rowType(any(Source.class), any())) + hoptimatorDriverMock.when(() -> HoptimatorDriver.rowType(any(Source.class), nullable(DeploymentContext.class))) .thenReturn(rowType); List templates = new ArrayList<>(); @@ -299,7 +288,7 @@ void configureOnlyIncludesMatchingHints() throws SQLException { builder.add("value", SqlTypeName.VARCHAR); RelDataType rowType = builder.build(); - hoptimatorDriverMock.when(() -> HoptimatorDriver.rowType(any(Source.class), any())) + hoptimatorDriverMock.when(() -> HoptimatorDriver.rowType(any(Source.class), nullable(DeploymentContext.class))) .thenReturn(rowType); List templates = new ArrayList<>(); @@ -330,7 +319,7 @@ void configureWithSink() throws SQLException { builder.add("value", SqlTypeName.VARCHAR); RelDataType rowType = builder.build(); - hoptimatorDriverMock.when(() -> HoptimatorDriver.rowType(any(Source.class), any())) + hoptimatorDriverMock.when(() -> HoptimatorDriver.rowType(any(Source.class), nullable(DeploymentContext.class))) .thenReturn(rowType); List templates = new ArrayList<>(); @@ -359,21 +348,22 @@ K8sApi createTableTemplateApi( @Test void configureAvroValueSchemaUsesSourceWhenAvailable() throws SQLException { - // When the resolved table implements AvroSchemaSource, its native Avro schema is rendered - // into the template as-is — no round-trip through RelDataType. + // When a table has a native Avro schema (resolved via the context), it is rendered into the + // template as-is — no round-trip through RelDataType. RelDataType rowType = new RelDataTypeFactory.Builder(typeFactory) .add("KEY_id", SqlTypeName.VARCHAR) .add("name", SqlTypeName.VARCHAR).build(); - hoptimatorDriverMock.when(() -> HoptimatorDriver.rowType(any(Source.class), any())) + hoptimatorDriverMock.when(() -> HoptimatorDriver.rowType(any(Source.class), nullable(DeploymentContext.class))) .thenReturn(rowType); Schema avroSchema = SchemaBuilder.record("User").namespace("com.linkedin.foo").fields() .requiredString("KEY_id") .requiredString("name") .endRecord(); + hoptimatorDriverMock.when(() -> HoptimatorDriver.valueSchema(any(Source.class), nullable(DeploymentContext.class))) + .thenReturn(avroSchema); Source source = new Source("testdb", Arrays.asList("schema", "table"), Collections.emptyMap()); - installRootSchemaWithTable(source, new SourceTable(avroSchema)); FakeK8sApi templateApi = new FakeK8sApi<>( List.of(new V1alpha1TableTemplate() @@ -395,15 +385,15 @@ void configureAvroValueSchemaUsesSourceWhenAvailable() throws SQLException { @Test void configureAvroValueSchemaFallsBackToRowTypeSynthesisWhenNoSource() throws SQLException { - // No AvroSchemaSource on the resolved table → synthesize from the row type using AvroConverter.avro. + // No value schema resolvable from the context (valueSchema returns null: a SQL source with no + // native Avro, e.g. a MySQL table or a view) → synthesize from the row type using AvroConverter.avro. RelDataType rowType = new RelDataTypeFactory.Builder(typeFactory) .add("name", SqlTypeName.VARCHAR).build(); - hoptimatorDriverMock.when(() -> HoptimatorDriver.rowType(any(Source.class), any())) + hoptimatorDriverMock.when(() -> HoptimatorDriver.rowType(any(Source.class), nullable(DeploymentContext.class))) .thenReturn(rowType); Source source = new Source("testdb", Arrays.asList("schema", "table"), Collections.emptyMap()); - installRootSchemaWithTable(source, new PlainTable()); FakeK8sApi templateApi = new FakeK8sApi<>( List.of(new V1alpha1TableTemplate() @@ -419,42 +409,5 @@ void configureAvroValueSchemaFallsBackToRowTypeSynthesisWhenNoSource() throws SQ "synthesized fallback produces the legacy namespace pattern; got " + config.get("avroValueSchema")); } - private void installRootSchemaWithTable(Source source, Table table) { - SchemaPlus root = CalciteSchema.createRootSchema(false).plus(); - SchemaPlus parent = root; - for (String part : source.path().subList(0, source.path().size() - 1)) { - parent = parent.add(part, new AbstractSchema()); - } - parent.add(source.table(), table); - - CalciteConnection calciteConn = mock(CalciteConnection.class); - when(calciteConn.getRootSchema()).thenReturn(root); - when(connection.calciteConnection()).thenReturn(calciteConn); - when(mockContext.connection()).thenReturn(connection); - } - - private static final class PlainTable extends AbstractTable { - @Override - public RelDataType getRowType(RelDataTypeFactory factory) { - throw new UnsupportedOperationException(); - } - } - - private static final class SourceTable extends AbstractTable implements AvroSchemaSource { - private final Schema value; - - SourceTable(Schema value) { - this.value = value; - } - - @Override - public Schema valueSchema() { - return value; - } - - @Override - public RelDataType getRowType(RelDataTypeFactory factory) { - throw new UnsupportedOperationException(); - } - } } + diff --git a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sContextTest.java b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sContextTest.java index 4082b4365..cb6759486 100644 --- a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sContextTest.java +++ b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sContextTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.k8s; +import com.linkedin.hoptimator.jdbc.CalciteDeploymentContext; + import com.linkedin.hoptimator.jdbc.HoptimatorConnection; import com.linkedin.hoptimator.k8s.models.V1alpha1View; import io.kubernetes.client.informer.SharedInformerFactory; @@ -37,15 +39,12 @@ class K8sContextTest { @Mock private SharedInformerFactory informerFactory; - @Mock - private HoptimatorConnection connection; - private K8sContext context; @BeforeEach void setUp() { context = new K8sContext("test-ns", "", "test context", apiClient, informerFactory, null, - Collections.emptyMap(), connection); + Collections.emptyMap(), null); } @Test @@ -68,11 +67,6 @@ void informerFactoryReturnsConfiguredFactory() { assertSame(informerFactory, context.informerFactory()); } - @Test - void connectionReturnsConfiguredConnection() { - assertSame(connection, context.connection()); - } - @Test void toStringReturnsClientInfo() { assertEquals("test context", context.toString()); @@ -258,7 +252,7 @@ void createWithPasswordAuthentication() { props.setProperty(K8sContext.PASSWORD_KEY, "secret"); when(mockConn.connectionProperties()).thenReturn(props); - K8sContext ctx = K8sContext.create(mockConn); + K8sContext ctx = K8sContext.create(new CalciteDeploymentContext(mockConn)); assertNotNull(ctx); assertEquals("custom-ns", ctx.namespace()); @@ -274,7 +268,7 @@ void createWithTokenAuthentication() { props.setProperty(K8sContext.TOKEN_KEY, "my-token"); when(mockConn.connectionProperties()).thenReturn(props); - K8sContext ctx = K8sContext.create(mockConn); + K8sContext ctx = K8sContext.create(new CalciteDeploymentContext(mockConn)); assertNotNull(ctx); assertEquals("token-ns", ctx.namespace()); @@ -293,7 +287,7 @@ void createWithImpersonation() { props.setProperty(K8sContext.IMPERSONATE_GROUPS_KEY, "group1,group2"); when(mockConn.connectionProperties()).thenReturn(props); - K8sContext ctx = K8sContext.create(mockConn); + K8sContext ctx = K8sContext.create(new CalciteDeploymentContext(mockConn)); assertNotNull(ctx); assertTrue(ctx.toString().contains("impuser")); @@ -311,7 +305,7 @@ void createWithWatchNamespace() { props.setProperty(K8sContext.TOKEN_KEY, "token"); when(mockConn.connectionProperties()).thenReturn(props); - K8sContext ctx = K8sContext.create(mockConn); + K8sContext ctx = K8sContext.create(new CalciteDeploymentContext(mockConn)); assertEquals("watch-ns", ctx.watchNamespace()); } @@ -325,7 +319,7 @@ void createWithNullWatchNamespaceDefaultsToEmpty() { props.setProperty(K8sContext.TOKEN_KEY, "token"); when(mockConn.connectionProperties()).thenReturn(props); - K8sContext ctx = K8sContext.create(mockConn); + K8sContext ctx = K8sContext.create(new CalciteDeploymentContext(mockConn)); assertEquals("", ctx.watchNamespace()); } @@ -344,7 +338,7 @@ void getPodNamespaceReturnsSelfPodNamespaceSystemProperty() { props.setProperty(K8sContext.TOKEN_KEY, "token"); when(mockConn.connectionProperties()).thenReturn(props); - K8sContext ctx = K8sContext.create(mockConn); + K8sContext ctx = K8sContext.create(new CalciteDeploymentContext(mockConn)); assertEquals("my-pod-namespace", ctx.namespace()); } finally { if (original == null) { @@ -368,7 +362,7 @@ void getPodNamespaceReturnsDefaultWhenNeitherEnvNorPropertySet() { props.setProperty(K8sContext.TOKEN_KEY, "token"); when(mockConn.connectionProperties()).thenReturn(props); - K8sContext ctx = K8sContext.create(mockConn); + K8sContext ctx = K8sContext.create(new CalciteDeploymentContext(mockConn)); // Should use DEFAULT_NAMESPACE when no env var or property is set assertEquals(K8sContext.DEFAULT_NAMESPACE, ctx.namespace()); } finally { @@ -378,6 +372,35 @@ void getPodNamespaceReturnsDefaultWhenNeitherEnvNorPropertySet() { } } + @Test + void createBuildsContextWithoutDeploymentContext() { + Properties props = new Properties(); + props.setProperty(K8sContext.NAMESPACE_KEY, "direct-ns"); + props.setProperty(K8sContext.SERVER_KEY, "https://k8s.example.com"); + props.setProperty(K8sContext.TOKEN_KEY, "token"); + + K8sContext ctx = K8sContext.create(props); + + assertNotNull(ctx); + assertEquals("direct-ns", ctx.namespace()); + assertNull(ctx.deploymentContext()); + } + + @Test + void deploymentContextReturnsBackingContext() { + HoptimatorConnection mockConn = mock(HoptimatorConnection.class); + Properties props = new Properties(); + props.setProperty(K8sContext.NAMESPACE_KEY, "ns"); + props.setProperty(K8sContext.SERVER_KEY, "https://k8s.example.com"); + props.setProperty(K8sContext.TOKEN_KEY, "token"); + when(mockConn.connectionProperties()).thenReturn(props); + CalciteDeploymentContext backing = new CalciteDeploymentContext(mockConn); + + K8sContext ctx = K8sContext.create(backing); + + assertSame(backing, ctx.deploymentContext()); + } + @Test void constantsAreDefined() { assertEquals("k8s.namespace", K8sContext.NAMESPACE_KEY); diff --git a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sDependencyValidatorTest.java b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sDependencyValidatorTest.java index f5e4ab926..9bb6ee9c6 100644 --- a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sDependencyValidatorTest.java +++ b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sDependencyValidatorTest.java @@ -1,6 +1,7 @@ package com.linkedin.hoptimator.k8s; -import java.sql.Connection; +import com.linkedin.hoptimator.DeploymentContext; + import java.sql.SQLException; import java.util.List; import java.util.Map; @@ -40,7 +41,7 @@ class K8sDependencyValidatorTest { private MockedStatic checkerStatic; @Mock - private Connection connection; + private DeploymentContext connection; @Mock private K8sContext context; diff --git a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sDeployerProviderTest.java b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sDeployerProviderTest.java index 4f893758a..7d24dd8e7 100644 --- a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sDeployerProviderTest.java +++ b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sDeployerProviderTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.k8s; +import com.linkedin.hoptimator.DeploymentContext; + import com.linkedin.hoptimator.DatabaseDeployable; import com.linkedin.hoptimator.Deployable; import com.linkedin.hoptimator.Deployer; @@ -14,7 +16,6 @@ import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoExtension; -import java.sql.Connection; import java.util.Collection; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -28,7 +29,7 @@ class K8sDeployerProviderTest { @Mock - private Connection connection; + private DeploymentContext connection; @Mock private K8sContext context; @@ -44,7 +45,7 @@ void priorityReturnsOne() { @Test void deployersForMaterializedViewReturnsMaterializedViewDeployer() { - contextStatic.when(() -> K8sContext.create(any(Connection.class))).thenReturn(context); + contextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(context); K8sDeployerProvider provider = new K8sDeployerProvider(); MaterializedView mv = mock(MaterializedView.class); @@ -56,7 +57,7 @@ void deployersForMaterializedViewReturnsMaterializedViewDeployer() { @Test void deployersForViewReturnsViewDeployer() { - contextStatic.when(() -> K8sContext.create(any(Connection.class))).thenReturn(context); + contextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(context); K8sDeployerProvider provider = new K8sDeployerProvider(); View view = mock(View.class); @@ -68,7 +69,7 @@ void deployersForViewReturnsViewDeployer() { @Test void deployersForJobReturnsJobDeployer() { - contextStatic.when(() -> K8sContext.create(any(Connection.class))).thenReturn(context); + contextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(context); K8sDeployerProvider provider = new K8sDeployerProvider(); Job job = mock(Job.class); @@ -80,7 +81,7 @@ void deployersForJobReturnsJobDeployer() { @Test void deployersForSourceReturnsSourceDeployer() { - contextStatic.when(() -> K8sContext.create(any(Connection.class))).thenReturn(context); + contextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(context); K8sDeployerProvider provider = new K8sDeployerProvider(); Source source = mock(Source.class); @@ -92,7 +93,7 @@ void deployersForSourceReturnsSourceDeployer() { @Test void deployersForTriggerReturnsTriggerDeployer() { - contextStatic.when(() -> K8sContext.create(any(Connection.class))).thenReturn(context); + contextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(context); K8sDeployerProvider provider = new K8sDeployerProvider(); Trigger trigger = mock(Trigger.class); @@ -104,7 +105,7 @@ void deployersForTriggerReturnsTriggerDeployer() { @Test void deployersForDatabaseDeployableReturnsDatabaseDeployer() { - contextStatic.when(() -> K8sContext.create(any(Connection.class))).thenReturn(context); + contextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(context); K8sDeployerProvider provider = new K8sDeployerProvider(); DatabaseDeployable database = mock(DatabaseDeployable.class); @@ -116,7 +117,7 @@ void deployersForDatabaseDeployableReturnsDatabaseDeployer() { @Test void deployersForUnknownTypeReturnsEmpty() { - contextStatic.when(() -> K8sContext.create(any(Connection.class))).thenReturn(context); + contextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(context); K8sDeployerProvider provider = new K8sDeployerProvider(); Deployable unknown = mock(Deployable.class); diff --git a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sDeployerTest.java b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sDeployerTest.java index eac5ed159..588a330c2 100644 --- a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sDeployerTest.java +++ b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sDeployerTest.java @@ -78,6 +78,17 @@ void createAddsObjectToApi() throws SQLException { assertEquals("test-pipeline", objects.get(0).getMetadata().getName()); } + @Test + void existsReflectsWhetherObjectIsPresent() throws SQLException { + K8sDeployer deployer = makeDeployer(fakeApi, snapshot); + + assertFalse(deployer.exists()); + + deployer.create(); + + assertTrue(deployer.exists()); + } + @Test void deleteRemovesObjectFromApi() throws SQLException { V1alpha1Pipeline pipeline = createTestPipeline(); diff --git a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sGraphProviderTest.java b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sGraphProviderTest.java index ca8cbaaad..4e88dce7a 100644 --- a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sGraphProviderTest.java +++ b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sGraphProviderTest.java @@ -1,6 +1,11 @@ package com.linkedin.hoptimator.k8s; -import java.sql.Connection; +import static org.mockito.ArgumentMatchers.any; + +import com.linkedin.hoptimator.jdbc.HoptimatorConnection; + +import com.linkedin.hoptimator.DeploymentContext; + import java.sql.SQLException; import java.util.List; @@ -31,7 +36,7 @@ class K8sGraphProviderTest { @Mock private MockedStatic contextStatic; @Mock - private Connection connection; + private HoptimatorConnection connection; @Mock private K8sContext context; @@ -56,7 +61,7 @@ void supportsAcceptsAllThreeGraphTargetSubtypes() { @Test void forTargetRoutesViewToBuilderForView() throws SQLException { PipelineGraphBuilder builder = mock(PipelineGraphBuilder.class); - contextStatic.when(() -> K8sContext.create(connection)).thenReturn(context); + contextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(context); providerWith(builder).forTarget(new GraphTarget.View("audience"), 3, connection); @@ -66,7 +71,7 @@ void forTargetRoutesViewToBuilderForView() throws SQLException { @Test void forTargetRoutesLogicalTableToBuilderForLogicalTable() throws SQLException { PipelineGraphBuilder builder = mock(PipelineGraphBuilder.class); - contextStatic.when(() -> K8sContext.create(connection)).thenReturn(context); + contextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(context); providerWith(builder).forTarget(new GraphTarget.LogicalTable("members"), 2, connection); @@ -76,7 +81,7 @@ void forTargetRoutesLogicalTableToBuilderForLogicalTable() throws SQLException { @Test void forTargetRoutesResourceToBuilderForResource() throws SQLException { PipelineGraphBuilder builder = mock(PipelineGraphBuilder.class); - contextStatic.when(() -> K8sContext.create(connection)).thenReturn(context); + contextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(context); providerWith(builder).forTarget( new GraphTarget.Resource("ads-database", List.of("ADS", "AD_CLICKS")), 1, connection); diff --git a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sJobDeployerTest.java b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sJobDeployerTest.java index c288aaba9..3024530bc 100644 --- a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sJobDeployerTest.java +++ b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sJobDeployerTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.k8s; +import com.linkedin.hoptimator.DeploymentContext; + import com.linkedin.hoptimator.Job; import com.linkedin.hoptimator.Sink; import com.linkedin.hoptimator.Source; @@ -45,6 +47,9 @@ class K8sJobDeployerTest { @Mock private HoptimatorConnection connection; + @Mock + private DeploymentContext deploymentContext; + @Mock private K8sContext mockContext; @@ -65,8 +70,8 @@ K8sYamlApi createYamlApi(K8sContext context) { return fakeYamlApi; } }; - when(mockContext.connection()).thenReturn(connection); - contextStatic.when(() -> K8sContext.create(any())).thenReturn(mockContext); + when(mockContext.deploymentContext()).thenReturn(deploymentContext); + contextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))).thenReturn(mockContext); } private Job createTestJob(Sink sink) { diff --git a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sSourceDeployerTest.java b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sSourceDeployerTest.java index 9bc945604..2829717dc 100644 --- a/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sSourceDeployerTest.java +++ b/hoptimator-k8s/src/test/java/com/linkedin/hoptimator/k8s/K8sSourceDeployerTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.k8s; +import com.linkedin.hoptimator.DeploymentContext; + import com.linkedin.hoptimator.Source; import com.linkedin.hoptimator.jdbc.HoptimatorConnection; import com.linkedin.hoptimator.k8s.models.V1alpha1TableTemplate; @@ -34,6 +36,9 @@ class K8sSourceDeployerTest { @Mock private HoptimatorConnection connection; + @Mock + private DeploymentContext deploymentContext; + @Mock private K8sContext mockContext; @@ -56,10 +61,10 @@ K8sYamlApi createYamlApi(K8sContext context) { }; } - /** Wires up {@code mockContext.connection()} for tests that exercise specify()'s template path. */ + /** Wires up {@code mockContext.deploymentContext()} for tests that exercise specify()'s template path. */ private void stubConnection() { - when(mockContext.connection()).thenReturn(connection); - when(connection.connectionProperties()).thenReturn(new Properties()); + when(mockContext.deploymentContext()).thenReturn(deploymentContext); + when(deploymentContext.properties()).thenReturn(new Properties()); } private K8sSourceDeployer makeDeployer(Source source) { diff --git a/hoptimator-kafka/build.gradle b/hoptimator-kafka/build.gradle index 7cd1c11a7..00d31c9fe 100644 --- a/hoptimator-kafka/build.gradle +++ b/hoptimator-kafka/build.gradle @@ -10,6 +10,7 @@ dependencies { implementation project(':hoptimator-util') implementation libs.calcite.core implementation libs.kafka.clients + testImplementation libs.avro compileOnly libs.findbugs testImplementation libs.assertj diff --git a/hoptimator-kafka/src/main/java/com/linkedin/hoptimator/kafka/KafkaDeployer.java b/hoptimator-kafka/src/main/java/com/linkedin/hoptimator/kafka/KafkaDeployer.java index 3a269d1e5..421c3403d 100644 --- a/hoptimator-kafka/src/main/java/com/linkedin/hoptimator/kafka/KafkaDeployer.java +++ b/hoptimator-kafka/src/main/java/com/linkedin/hoptimator/kafka/KafkaDeployer.java @@ -3,6 +3,7 @@ import com.linkedin.hoptimator.Deployer; import com.linkedin.hoptimator.Source; import com.linkedin.hoptimator.Validated; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Validator; import com.linkedin.hoptimator.jdbc.DeployerUtils; import org.apache.kafka.clients.admin.AdminClient; @@ -19,7 +20,6 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nullable; -import java.sql.Connection; import java.sql.SQLException; import java.time.Duration; import java.util.Collections; @@ -54,7 +54,7 @@ public KafkaDeployer(Source source, Properties properties) { } @Override - public void validate(Validator.Issues issues, Connection connection) { + public void validate(Validator.Issues issues, DeploymentContext context) { String topicName = source.table(); // null default = option was not specified by user, skip validation for that option Integer partitions = DeployerUtils.parseIntOption(source.options(), "partitions", null); @@ -109,6 +109,13 @@ public void create() throws SQLException { } } + @Override + public boolean exists() throws SQLException { + try (AdminClient admin = AdminClient.create(properties)) { + return topicExists(admin, source.table()); + } + } + @Override public void delete() throws SQLException { String topicName = source.table(); diff --git a/hoptimator-kafka/src/main/java/com/linkedin/hoptimator/kafka/KafkaDeployerProvider.java b/hoptimator-kafka/src/main/java/com/linkedin/hoptimator/kafka/KafkaDeployerProvider.java index af7b4ddb9..37b8fe19e 100644 --- a/hoptimator-kafka/src/main/java/com/linkedin/hoptimator/kafka/KafkaDeployerProvider.java +++ b/hoptimator-kafka/src/main/java/com/linkedin/hoptimator/kafka/KafkaDeployerProvider.java @@ -3,13 +3,9 @@ import com.linkedin.hoptimator.Deployable; import com.linkedin.hoptimator.Deployer; import com.linkedin.hoptimator.DeployerProvider; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Source; -import com.linkedin.hoptimator.jdbc.DeployerUtils; -import com.linkedin.hoptimator.util.planner.HoptimatorJdbcSchema; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.sql.Connection; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -19,26 +15,22 @@ /** * Provides {@link KafkaDeployer} instances for Kafka-backed tables. * - *

Detection works by looking up the source's schema in the Calcite connection, - * checking if it is a {@link HoptimatorJdbcSchema} backed by a {@code jdbc:kafka://} URL. - * The Kafka config (bootstrap.servers) is read from the JDBC URL properties stored on the schema. + *

Detection uses {@link DeploymentContext#databaseProperties} to resolve the source's + * {@code Database} config and checks whether its connection URL starts with {@code jdbc:kafka://}. + * This is Calcite-free: it works identically whether the config comes from the Calcite catalog + * (SQL path) or from a {@code Database} CRD (direct path). The Kafka config (bootstrap.servers) is + * parsed from that URL. */ public class KafkaDeployerProvider implements DeployerProvider { - private static final Logger log = LoggerFactory.getLogger(KafkaDeployerProvider.class); - @Override - public Collection deployers(T obj, Connection connection) { + public Collection deployers(T obj, DeploymentContext context) { List deployers = new ArrayList<>(); if (obj instanceof Source) { Source source = (Source) obj; - Properties properties = DeployerUtils.extractPropertiesFromJdbcSchema( - source.catalog(), - source.schema(), - connection, - KafkaDriver.CONNECTION_PREFIX, - log); + Properties properties = context.databaseProperties( + source.catalog(), source.schema(), KafkaDriver.CONNECTION_PREFIX); if (properties == null) { return deployers; diff --git a/hoptimator-kafka/src/test/java/com/linkedin/hoptimator/kafka/KafkaDeployerProviderTest.java b/hoptimator-kafka/src/test/java/com/linkedin/hoptimator/kafka/KafkaDeployerProviderTest.java index 0dacfe291..631d52dfe 100644 --- a/hoptimator-kafka/src/test/java/com/linkedin/hoptimator/kafka/KafkaDeployerProviderTest.java +++ b/hoptimator-kafka/src/test/java/com/linkedin/hoptimator/kafka/KafkaDeployerProviderTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.kafka; +import com.linkedin.hoptimator.jdbc.CalciteDeploymentContext; + import com.linkedin.hoptimator.Deployer; import com.linkedin.hoptimator.MaterializedView; import com.linkedin.hoptimator.Source; @@ -73,7 +75,7 @@ void testReturnsDeployerForKafkaSchema() { when(topicSubSchema.unwrap(HoptimatorJdbcSchema.class)).thenReturn(jdbcSchema); when(jdbcSchema.getDataSource()).thenReturn(dataSource); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertEquals(1, deployers.size()); assertInstanceOf(KafkaDeployer.class, deployers.iterator().next()); } @@ -83,7 +85,7 @@ void testReturnsEmptyForNonKafkaDatabase() { // Database name "test" doesn't start with "kafka" — short-circuits before schema lookup Source source = new Source("test", List.of("TEST", "MyStore"), Collections.emptyMap()); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty()); } @@ -97,7 +99,7 @@ void testReturnsEmptyForNonKafkaDatabaseEvenWhenSchemaWouldMatch() { Source source = new Source("not-kafka", List.of("KAFKA", "MyTopic"), Collections.emptyMap()); // No mocking of connection — the guard should prevent any downstream call - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty(), "Database not starting with 'kafka' should return empty regardless of schema name"); } @@ -111,14 +113,14 @@ void testReturnsEmptyWhenSchemaNotFound() { doReturn(subSchemaLookup).when(rootSchema).subSchemas(); when(subSchemaLookup.get("UNKNOWN")).thenReturn(null); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty()); } @Test void testReturnsEmptyForNonSourceDeployable() { MaterializedView view = mock(MaterializedView.class); - Collection deployers = provider.deployers(view, connection); + Collection deployers = provider.deployers(view, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty()); } @@ -126,14 +128,14 @@ void testReturnsEmptyForNonSourceDeployable() { void testReturnsEmptyWhenSchemaNameIsNull() { // Source with only a table name (single-element path) — schema() returns null Source source = new Source("kafka-database", List.of("MyTopic"), Collections.emptyMap()); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty()); } @Test void testReturnsEmptyWhenDatabaseIsNull() { Source source = new Source(null, List.of("KAFKA", "MyTopic"), Collections.emptyMap()); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty()); } @@ -147,7 +149,7 @@ void testReturnsEmptyWhenUnwrapThrowsException() { when(subSchemaLookup.get("KAFKA")).thenReturn(topicSubSchema); when(topicSubSchema.unwrap(HoptimatorJdbcSchema.class)).thenThrow(new RuntimeException("unwrap failed")); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty()); } } diff --git a/hoptimator-kafka/src/test/java/com/linkedin/hoptimator/kafka/KafkaDeployerTest.java b/hoptimator-kafka/src/test/java/com/linkedin/hoptimator/kafka/KafkaDeployerTest.java index 985ca0f76..a8ea76902 100644 --- a/hoptimator-kafka/src/test/java/com/linkedin/hoptimator/kafka/KafkaDeployerTest.java +++ b/hoptimator-kafka/src/test/java/com/linkedin/hoptimator/kafka/KafkaDeployerTest.java @@ -92,6 +92,33 @@ void testCreateNewTopic() throws Exception { verify(mockAdmin).close(); } + @Test + void testExistsReturnsFalseWhenTopicMissing() throws Exception { + Source source = new Source("db", List.of("KAFKA", "NewTopic"), Collections.emptyMap()); + + DescribeTopicsResult describeResult = mock(DescribeTopicsResult.class); + KafkaFuture failedFuture = mock(KafkaFuture.class); + doThrow(new ExecutionException(new UnknownTopicOrPartitionException("not found"))).when(failedFuture).get(); + doReturn(Map.of("NewTopic", failedFuture)).when(describeResult).topicNameValues(); + when(mockAdmin.describeTopics(anyList())).thenReturn(describeResult); + + assertFalse(createDeployer(source).exists()); + verify(mockAdmin).close(); + } + + @Test + void testExistsReturnsTrueWhenTopicPresent() throws Exception { + Source source = new Source("db", List.of("KAFKA", "ExistingTopic"), Collections.emptyMap()); + + DescribeTopicsResult describeResult = mock(DescribeTopicsResult.class); + KafkaFuture future = KafkaFuture.completedFuture(mockTopicWithPartitions(3)); + when(describeResult.topicNameValues()).thenReturn(Map.of("ExistingTopic", future)); + when(mockAdmin.describeTopics(anyList())).thenReturn(describeResult); + + assertTrue(createDeployer(source).exists()); + verify(mockAdmin).close(); + } + @Test void testUpdateExistingTopicSkipsCreation() throws Exception { Source source = new Source("db", List.of("KAFKA", "ExistingTopic"), Collections.emptyMap()); diff --git a/hoptimator-logical/build.gradle b/hoptimator-logical/build.gradle index 008ed83af..52700d69d 100644 --- a/hoptimator-logical/build.gradle +++ b/hoptimator-logical/build.gradle @@ -15,6 +15,7 @@ dependencies { // They are discovered at runtime via the DeployerProvider SPI (ServiceLoader). // Add them as testRuntimeOnly in your application build if needed for integration tests. + testImplementation libs.avro testImplementation libs.commons.dbcp2 testImplementation(testFixtures(project(':hoptimator-k8s'))) testImplementation(testFixtures(project(':hoptimator-jdbc'))) diff --git a/hoptimator-logical/src/main/java/com/linkedin/hoptimator/logical/LogicalTableDeployer.java b/hoptimator-logical/src/main/java/com/linkedin/hoptimator/logical/LogicalTableDeployer.java index 4326c92c6..624088034 100644 --- a/hoptimator-logical/src/main/java/com/linkedin/hoptimator/logical/LogicalTableDeployer.java +++ b/hoptimator-logical/src/main/java/com/linkedin/hoptimator/logical/LogicalTableDeployer.java @@ -1,6 +1,5 @@ package com.linkedin.hoptimator.logical; -import java.sql.Connection; import java.sql.SQLException; import java.sql.SQLNonTransientException; import java.util.ArrayList; @@ -19,14 +18,17 @@ import com.linkedin.hoptimator.k8s.models.V1alpha1TableTrigger; import com.linkedin.hoptimator.k8s.models.V1alpha1TableTriggerList; import com.linkedin.hoptimator.util.planner.PipelineRel; +import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelRoot; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.runtime.ImmutablePairList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.kubernetes.client.openapi.models.V1OwnerReference; import com.linkedin.hoptimator.Deployer; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.PendingDelete; import com.linkedin.hoptimator.Sink; import com.linkedin.hoptimator.Trigger; @@ -47,6 +49,7 @@ import com.linkedin.hoptimator.k8s.models.V1alpha1DatabaseList; import com.linkedin.hoptimator.util.DeploymentService; +import com.linkedin.hoptimator.jdbc.CalciteDeploymentContext; import com.linkedin.hoptimator.jdbc.HoptimatorConnection; import com.linkedin.hoptimator.jdbc.HoptimatorDriver; import com.linkedin.hoptimator.jdbc.HoptimatorDdlUtils; @@ -119,7 +122,7 @@ K8sLogicalTableDeployer createLogicalTableDeployer( *

Called by {@link com.linkedin.hoptimator.jdbc.ValidationService} before deployment. */ @Override - public void validate(Validator.Issues issues, Connection connection) { + public void validate(Validator.Issues issues, DeploymentContext deploymentContext) { try { // Pre-register the row type in tier schemas so deployers (e.g. VeniceDeployer) can // call HoptimatorDriver.rowType() during their own validate() calls. @@ -127,10 +130,10 @@ public void validate(Validator.Issues issues, Connection connection) { // These methods are self-caching; subsequent calls return the cached result. for (Map.Entry entry : buildTierSources().entrySet()) { Source tierSource = entry.getValue(); - Collection deployers = DeploymentService.deployers(tierSource, context.connection()); + Collection deployers = DeploymentService.deployers(tierSource, context.deploymentContext()); for (Deployer deployer : deployers) { if (deployer instanceof Validated) { - ((Validated) deployer).validate(issues, connection); + ((Validated) deployer).validate(issues, deploymentContext); } } } @@ -150,11 +153,13 @@ private void ensureTierRowTypesRegistered() throws SQLException { if (!schemaRollbacks.isEmpty()) { return; // Already registered (e.g. validate() was called before create/update). } - HoptimatorConnection conn = context.connection(); - if (conn == null) { + // Only the SQL path registers Calcite TemporaryTables so deployers can resolve tier row types; + // the direct path carries the row type on its context and needs no catalog registration. + if (!(context.deploymentContext() instanceof CalciteDeploymentContext)) { return; } - RelDataType rowType = HoptimatorDriver.rowType(source, conn); + HoptimatorConnection conn = ((CalciteDeploymentContext) context.deploymentContext()).connection(); + RelDataType rowType = HoptimatorDriver.rowType(source, context.deploymentContext()); for (Source tierSource : buildTierSources().values()) { schemaRollbacks.add(HoptimatorDdlUtils.registerTemporaryTableInSchema( conn, tierSource.catalog(), tierSource.schema(), @@ -233,13 +238,12 @@ private void deployAll(boolean update) throws SQLException { @Override public void delete() throws SQLException { Map tierSources = buildTierSources(); - HoptimatorConnection conn = context.connection(); String selfName = K8sUtils.canonicalizeName(source.path()); // 1. Per-tier pre-flight dep check. for (Source tierSource : tierSources.values()) { ValidationService.validateOrThrow( - new PendingDelete<>(tierSource, "LogicalTable", selfName), conn); + new PendingDelete<>(tierSource, "LogicalTable", selfName), context.deploymentContext()); } // 2. Delete the LogicalTable CRD (cascades owned pipelines/triggers). @@ -249,7 +253,7 @@ public void delete() throws SQLException { // physical delete succeeded; failed tiers keep their entries so the user can retry. for (Source tierSource : tierSources.values()) { boolean tierSucceeded = true; - for (Deployer deployer : DeploymentService.deployers(tierSource, conn)) { + for (Deployer deployer : DeploymentService.deployers(tierSource, context.deploymentContext())) { try { deployer.delete(); } catch (Exception e) { @@ -258,7 +262,9 @@ public void delete() throws SQLException { tierSource.pathString(), e.getMessage(), e); } } - if (tierSucceeded) { + if (tierSucceeded && context.deploymentContext() instanceof CalciteDeploymentContext) { + // Only the SQL path registered a tier TemporaryTable to deregister; the direct path did not. + HoptimatorConnection conn = ((CalciteDeploymentContext) context.deploymentContext()).connection(); HoptimatorDdlUtils.removeTableFromSchema(conn, tierSource.catalog(), tierSource.schema(), tierSource.table()); } @@ -315,7 +321,7 @@ public List specify() throws SQLException { // Step 1: tier resource specs (mirrors deployTierResources) for (Source tierSource : tierSources.values()) { - specs.addAll(DeploymentService.specify(tierSource, context.connection())); + specs.addAll(DeploymentService.specify(tierSource, context.deploymentContext())); } // Step 2: pipeline job specs (mirrors deployImplicitPipelines) @@ -347,16 +353,21 @@ public List specify() throws SQLException { *

Shared by {@link #deployPipelineBundle} and {@link #specifyPipelineJob}. */ private Pipeline planPipeline(Source fromSource, Source toSource, String pipelineName) throws Exception { - HoptimatorConnection conn = context.connection(); - Properties props = conn.connectionProperties(); + DeploymentContext deploymentContext = context.deploymentContext(); + // Plan SELECT * FROM against the Calcite catalog. + HoptimatorConnection conn = ((CalciteDeploymentContext) deploymentContext).connection(); + Properties props = deploymentContext.properties(); props.setProperty(DeploymentService.PIPELINE_OPTION, pipelineName); RelRoot root = HoptimatorDriver.convert(conn, buildSelectSql(fromSource)).root; - PipelineRel.Implementor plan = new PipelineRel.Implementor( - root.fields, DeploymentService.parseHints(props)); - plan.addSource(fromSource.database(), fromSource.path(), root.rel.getRowType(), Collections.emptyMap()); - plan.setSink(toSource.database(), toSource.path(), root.rel.getRowType(), Collections.emptyMap()); - plan.setQuery(root.rel); - return plan.pipeline(pipelineName, conn); + final RelNode query = root.rel; + final RelDataType rowType = root.rel.getRowType(); + final ImmutablePairList targetFields = root.fields; + final Map hints = DeploymentService.parseHints(props); + PipelineRel.Implementor plan = new PipelineRel.Implementor(targetFields, hints); + plan.addSource(fromSource.database(), fromSource.path(), rowType, Collections.emptyMap()); + plan.setSink(toSource.database(), toSource.path(), rowType, Collections.emptyMap()); + plan.setQuery(query); + return plan.pipeline(pipelineName, context.deploymentContext()); } /** Plans the pipeline and returns only the job artifact specs (sources/sink already in step 1). */ @@ -364,7 +375,7 @@ private List specifyPipelineJob(Source fromSource, Source toSource, Stri throws SQLException { try { Pipeline pipeline = planPipeline(fromSource, toSource, pipelineName); - return DeploymentService.specify(pipeline.job(), context.connection()); + return DeploymentService.specify(pipeline.job(), context.deploymentContext()); } catch (Exception e) { String message = String.format("Pipeline spec generation failed for %s on table %s", pipelineName, source.table()); @@ -434,7 +445,7 @@ private void deployTierResources(Map tierSources) throws SQLExce Source tierSource = entry.getValue(); log.info("Deploying tier {} (database CRD: {}) for table {}", tierName, tierSource.database(), source.table()); - Collection deployers = DeploymentService.deployers(tierSource, context.connection()); + Collection deployers = DeploymentService.deployers(tierSource, context.deploymentContext()); for (Deployer deployer : deployers) { // Always use update() — it is create-or-update, making deployment idempotent // if tier resources already exist (e.g. from a previous partial run). @@ -483,14 +494,13 @@ void deployPipelineBundle(String fromTier, String toTier, Map ti throw new SQLNonTransientException(message, e); } - HoptimatorConnection conn = context.connection(); String pipelineSql = pipeline.job().sql().apply(SqlDialect.ANSI); List pipelineSpecs = new ArrayList<>(); for (Source src : pipeline.sources()) { - pipelineSpecs.addAll(DeploymentService.specify(src, conn)); + pipelineSpecs.addAll(DeploymentService.specify(src, context.deploymentContext())); } - pipelineSpecs.addAll(DeploymentService.specify(pipeline.sink(), conn)); - pipelineSpecs.addAll(DeploymentService.specify(pipeline.job(), conn)); + pipelineSpecs.addAll(DeploymentService.specify(pipeline.sink(), context.deploymentContext())); + pipelineSpecs.addAll(DeploymentService.specify(pipeline.job(), context.deploymentContext())); K8sPipelineBundle bundle = new K8sPipelineBundle(pipelineName, pipelineSpecs, pipelineSql, pipeline.sources(), pipeline.sink(), ownerContext); diff --git a/hoptimator-logical/src/main/java/com/linkedin/hoptimator/logical/LogicalTableDeployerProvider.java b/hoptimator-logical/src/main/java/com/linkedin/hoptimator/logical/LogicalTableDeployerProvider.java index f17162f8a..5c0b19bbc 100644 --- a/hoptimator-logical/src/main/java/com/linkedin/hoptimator/logical/LogicalTableDeployerProvider.java +++ b/hoptimator-logical/src/main/java/com/linkedin/hoptimator/logical/LogicalTableDeployerProvider.java @@ -1,6 +1,5 @@ package com.linkedin.hoptimator.logical; -import java.sql.Connection; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -12,16 +11,16 @@ import com.linkedin.hoptimator.Deployable; import com.linkedin.hoptimator.Deployer; import com.linkedin.hoptimator.DeployerProvider; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Source; -import com.linkedin.hoptimator.jdbc.DeployerUtils; import com.linkedin.hoptimator.k8s.K8sContext; /** * Activates {@link LogicalTableDeployer} for sources backed by a logical Database CRD. * - *

Detection uses {@link DeployerUtils#extractPropertiesFromJdbcSchema} to look up the - * schema by name in the connection and check if its underlying JDBC URL starts with + *

Detection uses {@link DeploymentContext#databaseProperties} to look up the + * schema by name and check if its underlying JDBC URL starts with * {@link LogicalTableDriver#CONNECT_STRING_PREFIX}. No K8s API calls needed for activation. * The returned Properties contain the tier params (e.g. nearline=kafka-database, online=venice). */ @@ -30,23 +29,22 @@ public class LogicalTableDeployerProvider implements DeployerProvider { private static final Logger log = LoggerFactory.getLogger(LogicalTableDeployerProvider.class); @Override - public Collection deployers(T obj, Connection connection) { + public Collection deployers(T obj, DeploymentContext context) { if (!(obj instanceof Source)) { return Collections.emptyList(); } Source source = (Source) obj; - Properties tierProps = DeployerUtils.extractPropertiesFromJdbcSchema( - source.catalog(), source.schema(), connection, - LogicalTableDriver.CONNECT_STRING_PREFIX, log); + Properties tierProps = context.databaseProperties( + source.catalog(), source.schema(), LogicalTableDriver.CONNECT_STRING_PREFIX); if (tierProps == null) { return Collections.emptyList(); } log.debug("LogicalTableDeployerProvider activating for source {}", source); - K8sContext context = K8sContext.create(connection); - return List.of(new LogicalTableDeployer(source, tierProps, context)); + K8sContext k8sContext = K8sContext.create(context); + return List.of(new LogicalTableDeployer(source, tierProps, k8sContext)); } @Override diff --git a/hoptimator-logical/src/main/java/com/linkedin/hoptimator/logical/LogicalTableDriver.java b/hoptimator-logical/src/main/java/com/linkedin/hoptimator/logical/LogicalTableDriver.java index a92692c9e..72a227a23 100644 --- a/hoptimator-logical/src/main/java/com/linkedin/hoptimator/logical/LogicalTableDriver.java +++ b/hoptimator-logical/src/main/java/com/linkedin/hoptimator/logical/LogicalTableDriver.java @@ -90,7 +90,7 @@ public Connection connect(String url, Properties props) throws SQLException { CalciteConnection calciteConnection = (CalciteConnection) connection; SchemaPlus rootSchema = calciteConnection.getRootSchema(); - K8sContext context = K8sContext.create(new HoptimatorConnection(calciteConnection, properties)); + K8sContext context = K8sContext.create(new HoptimatorConnection(calciteConnection, properties).deploymentContext()); LogicalTableSchema logicalSchema = new LogicalTableSchema(properties, context, databaseName); rootSchema.add(databaseName.toUpperCase(), logicalSchema); diff --git a/hoptimator-logical/src/test/java/com/linkedin/hoptimator/logical/LogicalTableDeployerProviderTest.java b/hoptimator-logical/src/test/java/com/linkedin/hoptimator/logical/LogicalTableDeployerProviderTest.java index 2408af3e8..c829ba73d 100644 --- a/hoptimator-logical/src/test/java/com/linkedin/hoptimator/logical/LogicalTableDeployerProviderTest.java +++ b/hoptimator-logical/src/test/java/com/linkedin/hoptimator/logical/LogicalTableDeployerProviderTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.logical; +import com.linkedin.hoptimator.DeploymentContext; + import java.util.Collection; import java.util.List; import java.util.Map; @@ -31,10 +33,11 @@ public void deployersReturnsEmptyWhenInputIsNotSource() { } @Test - public void deployersReturnsEmptyWhenConnectionIsNull() { - // When connection is null, extractPropertiesFromJdbcSchema returns null + public void deployersReturnsEmptyWhenDatabaseUnresolvable() { + // A context that can't resolve the database (databaseProperties returns null) yields no deployers. Source source = new Source("mydb", List.of("mydb", "myschema", "mytable"), Map.of()); - Collection deployers = provider.deployers(source, null); + DeploymentContext context = org.mockito.Mockito.mock(DeploymentContext.class); + Collection deployers = provider.deployers(source, context); assertTrue(deployers.isEmpty()); } } diff --git a/hoptimator-logical/src/test/java/com/linkedin/hoptimator/logical/LogicalTableDeployerTest.java b/hoptimator-logical/src/test/java/com/linkedin/hoptimator/logical/LogicalTableDeployerTest.java index 04688cd21..befda9b6f 100644 --- a/hoptimator-logical/src/test/java/com/linkedin/hoptimator/logical/LogicalTableDeployerTest.java +++ b/hoptimator-logical/src/test/java/com/linkedin/hoptimator/logical/LogicalTableDeployerTest.java @@ -23,11 +23,13 @@ import io.kubernetes.client.openapi.models.V1OwnerReference; import com.linkedin.hoptimator.Deployer; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Source; import com.linkedin.hoptimator.Trigger; import com.linkedin.hoptimator.Validated; import com.linkedin.hoptimator.Validator; import com.linkedin.hoptimator.jdbc.DeployerUtils; +import com.linkedin.hoptimator.jdbc.CalciteDeploymentContext; import com.linkedin.hoptimator.jdbc.HoptimatorConnection; import com.linkedin.hoptimator.jdbc.HoptimatorDdlUtils; import com.linkedin.hoptimator.jdbc.HoptimatorDriver; @@ -127,6 +129,16 @@ private static K8sContext mockContext() { return ctx; } + /** A context representing the SQL path — it carries a connection, so tier TemporaryTables are + * registered/removed in the Calcite catalog. (The connection-free direct path is covered by + * {@code LogicalTableServiceIntegrationTest}.) */ + private static K8sContext mockContextWithConnection() { + K8sContext ctx = mockContext(); + lenient().when(ctx.deploymentContext()) + .thenReturn(new CalciteDeploymentContext(mock(HoptimatorConnection.class))); + return ctx; + } + private static Source testSource() { return new Source("logical", Arrays.asList("logical", "testevent"), Collections.emptyMap()); } @@ -243,8 +255,12 @@ void pipelineNameNearlineToOnline() { /** Builds a 2-tier deployer with mocked CRD deployer and a pre-populated fake Database API. */ private LogicalTableDeployer deployerWithApis(Properties props, List dbs) { + return deployerWithApis(props, dbs, mockContext()); + } + + private LogicalTableDeployer deployerWithApis(Properties props, List dbs, K8sContext ctx) { FakeK8sApi dbApi = new FakeK8sApi<>(new ArrayList<>(dbs)); - return new LogicalTableDeployer(testSource(), props, mockContext(), dbApi) { + return new LogicalTableDeployer(testSource(), props, ctx, dbApi) { @Override K8sLogicalTableDeployer createLogicalTableDeployer( String crdName, String databaseLabel, Map tierMap) { @@ -311,7 +327,7 @@ private void withMockedDdlUtils(Runnable body, Consumer DeploymentService.deployers(any(), any())) @@ -339,7 +355,7 @@ void deleteKeepsSchemaEntryForTierWhoseDeleteFailed() throws SQLException { // The failed tier's schema entry must NOT be removed; the succeeded tier's must be. LogicalTableDeployer deployer = deployerWithApis( twoTierProps("kafka-db", "venice-db"), - Arrays.asList(makeDb("kafka-db", "KAFKA"), makeDb("venice-db", "VENICE"))); + Arrays.asList(makeDb("kafka-db", "KAFKA"), makeDb("venice-db", "VENICE")), mockContextWithConnection()); Deployer failingTier = mock(Deployer.class); Deployer succeedingTier = mock(Deployer.class); @@ -701,17 +717,18 @@ void deployerProviderReturnsDeployerWhenLogicalSchemaFound() { Properties tierProps = new Properties(); tierProps.setProperty(LogicalTier.NEARLINE.tierName(), "nearline-db"); deployerUtilsMock.when(() -> DeployerUtils.extractPropertiesFromJdbcSchema( - any(), any(), any(), anyString(), any())) + any(), any(), any(), anyString())) .thenReturn(tierProps); K8sContext mockCtx = mock(K8sContext.class); - k8sContextMock.when(() -> K8sContext.create(any())) + k8sContextMock.when(() -> K8sContext.create(any(DeploymentContext.class))) .thenReturn(mockCtx); HoptimatorConnection mockConn = mock(HoptimatorConnection.class); LogicalTableDeployerProvider provider = new LogicalTableDeployerProvider(); - Collection deployers = provider.deployers(makeSource("logical", "testevent"), mockConn); + Collection deployers = provider.deployers(makeSource("logical", "testevent"), + new CalciteDeploymentContext(mockConn)); assertFalse(deployers.isEmpty()); assertEquals(1, deployers.size()); @@ -722,10 +739,10 @@ void deployerProviderReturnsDeployerWhenLogicalSchemaFound() { void ensureTierRowTypesRegisteredWithConnectionRecordsRowTypeError() throws Exception { HoptimatorConnection mockConn = mock(HoptimatorConnection.class); K8sContext ctx = mock(K8sContext.class); - when(ctx.connection()).thenReturn(mockConn); + when(ctx.deploymentContext()).thenReturn(new CalciteDeploymentContext(mockConn)); hoptimatorDriverMock - .when(() -> HoptimatorDriver.rowType(any(Source.class), any(HoptimatorConnection.class))) + .when(() -> HoptimatorDriver.rowType(any(Source.class), any(DeploymentContext.class))) .thenThrow(new SQLException("schema not found")); Properties oneTierProps = new Properties(); diff --git a/hoptimator-logical/src/test/java/com/linkedin/hoptimator/logical/LogicalTableDriverTest.java b/hoptimator-logical/src/test/java/com/linkedin/hoptimator/logical/LogicalTableDriverTest.java index c62dbc215..f78d37451 100644 --- a/hoptimator-logical/src/test/java/com/linkedin/hoptimator/logical/LogicalTableDriverTest.java +++ b/hoptimator-logical/src/test/java/com/linkedin/hoptimator/logical/LogicalTableDriverTest.java @@ -6,6 +6,7 @@ import java.util.Properties; import com.linkedin.hoptimator.k8s.K8sContext; +import com.linkedin.hoptimator.DeploymentContext; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -103,7 +104,7 @@ public void connectThrowsWhenDatabasePropertyInUrl() throws Exception { @Test public void connectThrowsNonTransientWhenK8sContextCreationFails() throws Exception { // All validation passes (2 tiers + database property set) but K8sContext.create() fails. - k8sContextStatic.when(() -> K8sContext.create(any())) + k8sContextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))) .thenThrow(new RuntimeException("simulated K8sContext failure")); String url = "jdbc:logical://nearline=kafka-database;online=venice"; @@ -122,7 +123,7 @@ void connectPreservesCauseWhenK8sContextCreationFails() throws Exception { // The catch(Exception e) branch must wrap the underlying failure as the cause rather than // swallowing it. Stub K8sContext.create() to throw a known exception and assert it is preserved. RuntimeException boom = new RuntimeException("boom"); - k8sContextStatic.when(() -> K8sContext.create(any())).thenThrow(boom); + k8sContextStatic.when(() -> K8sContext.create(any(DeploymentContext.class))).thenThrow(boom); Properties props = new Properties(); props.setProperty("database", "logical"); diff --git a/hoptimator-mcp-server/src/main/java/com/linkedin/hoptimator/mcp/server/HoptimatorMcpServer.java b/hoptimator-mcp-server/src/main/java/com/linkedin/hoptimator/mcp/server/HoptimatorMcpServer.java index 14390336b..ba87659cd 100644 --- a/hoptimator-mcp-server/src/main/java/com/linkedin/hoptimator/mcp/server/HoptimatorMcpServer.java +++ b/hoptimator-mcp-server/src/main/java/com/linkedin/hoptimator/mcp/server/HoptimatorMcpServer.java @@ -331,13 +331,13 @@ static CallToolResult handlePlan(HoptimatorConnection conn, Gson gson, ObjectMap PipelineRel.Implementor sqlPlan = DeploymentService.plan(root, conn.materializations(), connectionProperties); schemaSnapshot = HoptimatorDdlUtils.snapshotAndSetSinkSchema(conn.createPrepareContext(), new HoptimatorDriver.Prepare(conn), sqlPlan, create, querySql); - Pipeline pipeline = sqlPlan.pipeline(viewName, conn); + Pipeline pipeline = sqlPlan.pipeline(viewName, conn.deploymentContext()); List specs = new ArrayList<>(); for (Source source : pipeline.sources()) { - specs.addAll(DeploymentService.specify(source, conn)); + specs.addAll(DeploymentService.specify(source, conn.deploymentContext())); } - specs.addAll(DeploymentService.specify(pipeline.sink(), conn)); - specs.addAll(DeploymentService.specify(pipeline.job(), conn)); + specs.addAll(DeploymentService.specify(pipeline.sink(), conn.deploymentContext())); + specs.addAll(DeploymentService.specify(pipeline.job(), conn.deploymentContext())); List mappedObjs = new ArrayList<>(); for (String spec : specs) { mappedObjs.add(yamlMapper.readValue(spec, Object.class)); diff --git a/hoptimator-mysql/build.gradle b/hoptimator-mysql/build.gradle index fa3cf35b7..11448132b 100644 --- a/hoptimator-mysql/build.gradle +++ b/hoptimator-mysql/build.gradle @@ -11,6 +11,7 @@ dependencies { implementation libs.mysql.connector compileOnly libs.findbugs + testImplementation libs.avro testImplementation libs.assertj testImplementation libs.commons.dbcp2 testCompileOnly libs.findbugs.annotations diff --git a/hoptimator-mysql/src/main/java/com/linkedin/hoptimator/mysql/MySqlDeployer.java b/hoptimator-mysql/src/main/java/com/linkedin/hoptimator/mysql/MySqlDeployer.java index 227600bba..41ff82e95 100644 --- a/hoptimator-mysql/src/main/java/com/linkedin/hoptimator/mysql/MySqlDeployer.java +++ b/hoptimator-mysql/src/main/java/com/linkedin/hoptimator/mysql/MySqlDeployer.java @@ -5,7 +5,7 @@ import com.linkedin.hoptimator.Validated; import com.linkedin.hoptimator.Validator; import com.linkedin.hoptimator.avro.AvroSchemas; -import com.linkedin.hoptimator.jdbc.HoptimatorConnection; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.jdbc.HoptimatorDriver; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; @@ -40,13 +40,13 @@ public class MySqlDeployer implements Deployer, Validated { private final Source source; private final Properties properties; - private final HoptimatorConnection hoptimatorConnection; + private final DeploymentContext deploymentContext; private boolean created = false; - public MySqlDeployer(Source source, Properties properties, HoptimatorConnection connection) { + public MySqlDeployer(Source source, Properties properties, DeploymentContext context) { this.source = source; this.properties = properties; - this.hoptimatorConnection = connection; + this.deploymentContext = context; } /** @@ -99,7 +99,7 @@ private String toMySqlType(RelDataTypeField field) { } @Override - public void validate(Validator.Issues issues, Connection connection) { + public void validate(Validator.Issues issues, DeploymentContext context) { String tableName = source.table(); String database = source.schema(); @@ -122,7 +122,7 @@ public void validate(Validator.Issues issues, Connection connection) { RelDataType newRowType; List newKeyFields; try { - newRowType = HoptimatorDriver.rowType(source, hoptimatorConnection); + newRowType = HoptimatorDriver.rowType(source, deploymentContext); newKeyFields = parseKeyFields(newRowType); if (newKeyFields.isEmpty()) { @@ -217,6 +217,17 @@ public void create() throws SQLException { } } + @Override + public boolean exists() throws SQLException { + String database = source.schema(); + if (database == null) { + return false; + } + try (Connection conn = getConnection()) { + return tableExists(conn, database, source.table()); + } + } + @Override public void delete() throws SQLException { String tableName = source.table(); @@ -380,7 +391,7 @@ private void ensureDatabaseExists(Connection conn, String database) throws SQLEx * Alters an existing table to match the desired schema. */ private void alterTable(Connection conn, String database, String tableName) throws SQLException { - RelDataType newRowType = HoptimatorDriver.rowType(source, hoptimatorConnection); + RelDataType newRowType = HoptimatorDriver.rowType(source, deploymentContext); // Get existing schema Map existingColumns = getExistingColumns(conn, database, tableName); @@ -539,7 +550,7 @@ private String buildCreateTableSql(String database, String tableName) throws SQL throw new SQLException("Invalid table name: " + tableName); } - RelDataType rowType = HoptimatorDriver.rowType(source, hoptimatorConnection); + RelDataType rowType = HoptimatorDriver.rowType(source, deploymentContext); List keyFields = parseKeyFields(rowType); if (keyFields.isEmpty()) { diff --git a/hoptimator-mysql/src/main/java/com/linkedin/hoptimator/mysql/MySqlDeployerProvider.java b/hoptimator-mysql/src/main/java/com/linkedin/hoptimator/mysql/MySqlDeployerProvider.java index 8a460c5c0..fc0f66072 100644 --- a/hoptimator-mysql/src/main/java/com/linkedin/hoptimator/mysql/MySqlDeployerProvider.java +++ b/hoptimator-mysql/src/main/java/com/linkedin/hoptimator/mysql/MySqlDeployerProvider.java @@ -3,13 +3,9 @@ import com.linkedin.hoptimator.Deployable; import com.linkedin.hoptimator.Deployer; import com.linkedin.hoptimator.DeployerProvider; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Source; -import com.linkedin.hoptimator.jdbc.DeployerUtils; -import com.linkedin.hoptimator.jdbc.HoptimatorConnection; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.sql.Connection; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -18,36 +14,27 @@ /** * Provides {@link MySqlDeployer} instances for MySQL-backed tables. * - *

Detection works by looking up the source's schema in the Calcite connection, - * checking if it is a MySQL schema (TableSchema). + *

Detection uses {@link DeploymentContext#databaseProperties} to resolve the source's + * {@code Database} config and checks whether its connection URL starts with + * {@code jdbc:mysql-hoptimator://}. This is Calcite-free: it works identically whether the config + * comes from the Calcite catalog (SQL path) or from a {@code Database} CRD (direct path). */ public class MySqlDeployerProvider implements DeployerProvider { - private static final Logger log = LoggerFactory.getLogger(MySqlDeployerProvider.class); - @Override - public Collection deployers(T obj, Connection connection) { + public Collection deployers(T obj, DeploymentContext context) { List deployers = new ArrayList<>(); if (obj instanceof Source) { Source source = (Source) obj; - Properties properties = DeployerUtils.extractPropertiesFromJdbcSchema( - source.catalog(), - source.schema(), - connection, - MySqlDriver.CONNECTION_PREFIX, - log); + Properties properties = context.databaseProperties( + source.catalog(), source.schema(), MySqlDriver.CONNECTION_PREFIX); if (properties == null) { return deployers; } - if (!(connection instanceof HoptimatorConnection)) { - log.error("Connection is not a HoptimatorConnection, cannot create MySqlDeployer"); - return deployers; - } - - deployers.add(new MySqlDeployer(source, properties, (HoptimatorConnection) connection)); + deployers.add(new MySqlDeployer(source, properties, context)); } return deployers; } diff --git a/hoptimator-mysql/src/test/java/com/linkedin/hoptimator/mysql/MySqlDeployerProviderTest.java b/hoptimator-mysql/src/test/java/com/linkedin/hoptimator/mysql/MySqlDeployerProviderTest.java index 33b3e97d6..a585817f7 100644 --- a/hoptimator-mysql/src/test/java/com/linkedin/hoptimator/mysql/MySqlDeployerProviderTest.java +++ b/hoptimator-mysql/src/test/java/com/linkedin/hoptimator/mysql/MySqlDeployerProviderTest.java @@ -1,5 +1,8 @@ package com.linkedin.hoptimator.mysql; +import com.linkedin.hoptimator.jdbc.CalciteDeploymentContext; +import com.linkedin.hoptimator.DeploymentContext; + import com.linkedin.hoptimator.Deployer; import com.linkedin.hoptimator.MaterializedView; import com.linkedin.hoptimator.Source; @@ -15,7 +18,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.sql.Connection; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -82,7 +84,7 @@ void testReturnsDeployerForMySqlSchema() { when(testdbSchema.unwrap(HoptimatorJdbcSchema.class)).thenReturn(jdbcSchema); when(jdbcSchema.getDataSource()).thenReturn(dataSource); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertEquals(1, deployers.size()); assertInstanceOf(MySqlDeployer.class, deployers.iterator().next()); } @@ -92,7 +94,7 @@ void testReturnsEmptyForNonMySqlCatalog() { // Catalog name "KAFKA" doesn't match "MYSQL" — short-circuits before schema lookup Source source = new Source("kafka", List.of("KAFKA", "MyTopic"), Collections.emptyMap()); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty()); } @@ -101,14 +103,14 @@ void testReturnsEmptyWhenCatalogNotFound() { Source source = new Source("mysql", List.of("UNKNOWN", "testdb", "users"), Collections.emptyMap()); // No mocking needed - catalog name doesn't match, short-circuits early - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty()); } @Test void testReturnsEmptyForNonSourceDeployable() { MaterializedView view = mock(MaterializedView.class); - Collection deployers = provider.deployers(view, connection); + Collection deployers = provider.deployers(view, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty()); } @@ -116,7 +118,7 @@ void testReturnsEmptyForNonSourceDeployable() { void testReturnsEmptyWhenCatalogIsNull() { // Source with only schema and table (2-level path) — catalog() returns null Source source = new Source("mysql", List.of("testdb", "users"), Collections.emptyMap()); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty()); } @@ -131,7 +133,7 @@ void testReturnsEmptyWhenSchemaNotFoundInCatalog() { doReturn(catalogSubSchemaLookup).when(mysqlCatalogSchema).subSchemas(); when(catalogSubSchemaLookup.get("nonexistent")).thenReturn(null); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty()); } @@ -147,28 +149,21 @@ void testReturnsEmptyWhenUnwrapThrowsException() { when(catalogSubSchemaLookup.get("testdb")).thenReturn(testdbSchema); when(testdbSchema.unwrap(HoptimatorJdbcSchema.class)).thenThrow(new RuntimeException("unwrap failed")); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty()); } // --- deployers() with non-HoptimatorConnection returns empty --- @Test - void testReturnsEmptyWhenConnectionIsNotHoptimatorConnection() { + void testReturnsEmptyWhenDatabaseUnresolvable() { Source source = new Source("mysql", List.of("MYSQL", "testdb", "users"), Collections.emptyMap()); - // Use a Connection (not HoptimatorConnection) as the connection parameter - Connection rawConnection = mock(Connection.class); - - // The provider uses a Connection param (not HoptimatorConnection typed directly here) - // But DeployerUtils.extractPropertiesFromJdbcSchema requires HoptimatorConnection - // We need to feed a non-HoptimatorConnection to trigger the branch - // MySqlDeployerProvider.deployers() takes Connection, not HoptimatorConnection - Collection deployers = provider.deployers(source, rawConnection); - // Without the isinstance check, this would try to cast and fail or create a broken deployer - // With the check, it should return empty + // A context that can't resolve the database (databaseProperties returns null) yields no deployers. + DeploymentContext unresolvable = mock(DeploymentContext.class); + Collection deployers = provider.deployers(source, unresolvable); assertTrue(deployers.isEmpty(), - "Expected empty deployers when connection is not a HoptimatorConnection"); + "Expected empty deployers when the database cannot be resolved"); } // --- catalog check: catalog name comparison is case-insensitive --- @@ -179,7 +174,7 @@ void testDeployersCatalogMatchIsCaseInsensitive() { // but non-matching catalog returns empty Source source = new Source("mysql", List.of("OTHER", "testdb", "users"), Collections.emptyMap()); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty(), "Expected empty deployers for non-MYSQL catalog name"); } diff --git a/hoptimator-mysql/src/test/java/com/linkedin/hoptimator/mysql/MySqlDeployerTest.java b/hoptimator-mysql/src/test/java/com/linkedin/hoptimator/mysql/MySqlDeployerTest.java index 08f352cf6..353217322 100644 --- a/hoptimator-mysql/src/test/java/com/linkedin/hoptimator/mysql/MySqlDeployerTest.java +++ b/hoptimator-mysql/src/test/java/com/linkedin/hoptimator/mysql/MySqlDeployerTest.java @@ -1,5 +1,8 @@ package com.linkedin.hoptimator.mysql; +import com.linkedin.hoptimator.DeploymentContext; +import com.linkedin.hoptimator.jdbc.CalciteDeploymentContext; + import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import com.linkedin.hoptimator.Source; import com.linkedin.hoptimator.Validator; @@ -106,12 +109,12 @@ void setUp() throws SQLException { builder.add("name", typeFactory.createSqlType(SqlTypeName.VARCHAR, 255)); RelDataType rowType = builder.build(); - hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(HoptimatorConnection.class))) + hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(DeploymentContext.class))) .thenReturn(rowType); } private MySqlDeployer createDeployer(Source source) { - return new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + return new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); } private Validator.Issues collectIssues(MySqlDeployer deployer) { @@ -140,7 +143,7 @@ private void stubDefaultRowType() { builder.add("KEY_id", typeFactory.createSqlType(SqlTypeName.INTEGER)); builder.add("name", typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR, 255), true)); RelDataType rowType = builder.build(); - hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(HoptimatorConnection.class))) + hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(DeploymentContext.class))) .thenReturn(rowType); } @@ -179,6 +182,30 @@ void testCreateExistingTableSkipsCreation() throws Exception { verify(mockConnection).close(); } + @Test + void testExistsReturnsTrueWhenTableExists() throws Exception { + Source source = new Source("db", List.of("MYSQL", DATABASE, "ExistingTable"), Collections.emptyMap()); + + ResultSet existingRs = mock(ResultSet.class); + when(existingRs.next()).thenReturn(true); + when(mockMetaData.getTables(eq(DATABASE), any(), eq("ExistingTable"), any())).thenReturn(existingRs); + + assertTrue(createDeployer(source).exists()); + verify(mockConnection).close(); + } + + @Test + void testExistsReturnsFalseWhenTableMissing() throws Exception { + Source source = new Source("db", List.of("MYSQL", DATABASE, "NewTable"), Collections.emptyMap()); + + ResultSet emptyRs = mock(ResultSet.class); + when(emptyRs.next()).thenReturn(false); + when(mockMetaData.getTables(eq(DATABASE), any(), eq("NewTable"), any())).thenReturn(emptyRs); + + assertFalse(createDeployer(source).exists()); + verify(mockConnection).close(); + } + @Test void testCreatePropagatesException() throws Exception { Source source = new Source("db", List.of("MYSQL", DATABASE, "ErrorTable"), Collections.emptyMap()); @@ -198,7 +225,7 @@ void testCreatePropagatesException() throws Exception { void testCreateFailsWithMissingUrl() throws Exception { Source source = new Source("db", List.of("MYSQL", DATABASE, "TestTable"), Collections.emptyMap()); Properties emptyProps = new Properties(); - MySqlDeployer deployer = new MySqlDeployer(source, emptyProps, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, emptyProps, new CalciteDeploymentContext(mockHoptimatorConnection)); SQLException exception = assertThrows(SQLException.class, deployer::create); assertTrue(exception.getMessage().contains("Failed to create table TestTable") @@ -208,7 +235,7 @@ void testCreateFailsWithMissingUrl() throws Exception { @Test void testCreateFailsWithNullDatabase() { Source source = new Source("db", List.of("TestTable"), Collections.emptyMap()); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); assertThrows(SQLException.class, deployer::create); } @@ -290,7 +317,7 @@ void testDeleteNonExistentTableSkipsDeletion() throws Exception { when(emptyRs.next()).thenReturn(false); when(mockMetaData.getTables(eq("test_db"), any(), eq("GhostTable"), any())).thenReturn(emptyRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); deployer.delete(); verify(mockStatement, never()).executeUpdate(anyString()); @@ -358,7 +385,7 @@ void testValidateFailsWhenKeyFieldTypeChanges() throws Exception { RelDataType newRowType = builder.build(); // Set up the mock BEFORE creating the deployer - hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(HoptimatorConnection.class))) + hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(DeploymentContext.class))) .thenReturn(newRowType); Source source = new Source("db", List.of("MYSQL", DATABASE, "ExistingTable"), Collections.emptyMap()); @@ -404,7 +431,7 @@ void testValidateFailsWhenKeyFieldTypeChanges() throws Exception { void testValidateFailsWithNullDatabase() { Source source = new Source("db", List.of("TestTable"), Collections.emptyMap()); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); Validator.Issues issues = new Validator.Issues("test"); deployer.validate(issues, null); @@ -416,7 +443,7 @@ void testValidateFailsWithNullDatabase() { void testValidateFailsWithInvalidDatabaseName() { Source source = new Source("db", List.of("MYSQL", "invalid-db", "TestTable"), Collections.emptyMap()); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); Validator.Issues issues = new Validator.Issues("test"); deployer.validate(issues, null); @@ -428,7 +455,7 @@ void testValidateFailsWithInvalidDatabaseName() { void testValidateFailsWithInvalidTableName() { Source source = new Source("db", List.of("MYSQL", "test_db", "invalid table"), Collections.emptyMap()); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); Validator.Issues issues = new Validator.Issues("test"); deployer.validate(issues, null); @@ -440,7 +467,7 @@ void testValidateFailsWithInvalidTableName() { void testValidateFailsWithEmptyDatabaseName() { Source source = new Source("db", List.of("MYSQL", "", "TestTable"), Collections.emptyMap()); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); Validator.Issues issues = new Validator.Issues("test"); deployer.validate(issues, null); @@ -458,7 +485,7 @@ void testValidateFailsNoKeyFields() throws SQLException { builder.add("age", typeFactory.createSqlType(SqlTypeName.INTEGER)); RelDataType rowType = builder.build(); - hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(HoptimatorConnection.class))) + hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(DeploymentContext.class))) .thenReturn(rowType); Source source = new Source("db", List.of("MYSQL", "test_db", "TestTable"), Collections.emptyMap()); @@ -467,7 +494,7 @@ void testValidateFailsNoKeyFields() throws SQLException { when(emptyRs.next()).thenReturn(false); when(mockMetaData.getTables(eq("test_db"), any(), eq("TestTable"), any())).thenReturn(emptyRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); Validator.Issues issues = new Validator.Issues("test"); deployer.validate(issues, null); @@ -502,7 +529,7 @@ void testValidateFailsWhenPrimaryKeysChange() throws SQLException { when(columnsRs.getInt("NULLABLE")).thenReturn(DatabaseMetaData.columnNoNulls, DatabaseMetaData.columnNullable); when(mockMetaData.getColumns(eq("test_db"), any(), eq("ExistingTable"), any())).thenReturn(columnsRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); Validator.Issues issues = new Validator.Issues("test"); deployer.validate(issues, null); @@ -520,7 +547,7 @@ void testValidateFailsWithInvalidColumnName() throws SQLException { builder.add("invalid column", typeFactory.createSqlType(SqlTypeName.VARCHAR, 255)); RelDataType rowType = builder.build(); - hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(HoptimatorConnection.class))) + hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(DeploymentContext.class))) .thenReturn(rowType); Source source = new Source("db", List.of("MYSQL", "test_db", "BadColTable"), Collections.emptyMap()); @@ -529,7 +556,7 @@ void testValidateFailsWithInvalidColumnName() throws SQLException { when(emptyRs.next()).thenReturn(false); when(mockMetaData.getTables(eq("test_db"), any(), eq("BadColTable"), any())).thenReturn(emptyRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); Validator.Issues issues = new Validator.Issues("test"); deployer.validate(issues, null); @@ -540,12 +567,12 @@ void testValidateFailsWithInvalidColumnName() throws SQLException { @Test void testValidateFailsWhenRowTypeThrowsException() throws SQLException { SQLException schemaError = new SQLException("schema error"); - hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(HoptimatorConnection.class))) + hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(DeploymentContext.class))) .thenThrow(schemaError); Source source = new Source("db", List.of("MYSQL", "test_db", "SchemaErrorTable"), Collections.emptyMap()); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); Validator.Issues issues = new Validator.Issues("test"); deployer.validate(issues, null); @@ -566,7 +593,7 @@ void testValidatePassesWithMaxLength64Identifier() throws SQLException { when(emptyRs.next()).thenReturn(false); when(mockMetaData.getTables(eq("test_db"), any(), eq(longName), any())).thenReturn(emptyRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); Validator.Issues issues = new Validator.Issues("test"); deployer.validate(issues, null); @@ -579,7 +606,7 @@ void testValidateFailsWithTooLongIdentifier() { String tooLong = "a".repeat(65); Source source = new Source("db", List.of("MYSQL", "test_db", tooLong), Collections.emptyMap()); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); Validator.Issues issues = new Validator.Issues("test"); deployer.validate(issues, null); @@ -599,7 +626,7 @@ void testValidatePassesWhenAllConditionsGood() throws SQLException { when(emptyRs.next()).thenReturn(false); when(mockMetaData.getTables(eq("test_db"), any(), eq("GoodTable"), any())).thenReturn(emptyRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); Validator.Issues issues = new Validator.Issues("test"); deployer.validate(issues, null); @@ -628,7 +655,7 @@ void testUpdateCallsCreate() throws Exception { @Test void testUpdateFailsWithNullDatabase() { Source source = new Source("db", List.of("TestTable"), Collections.emptyMap()); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); assertThrows(SQLException.class, deployer::update); } @@ -645,7 +672,7 @@ void testUpdateAltersExistingTableAddsColumn() throws Exception { builder.add("email", typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR, 255), true)); RelDataType rowType = builder.build(); - hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(HoptimatorConnection.class))) + hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(DeploymentContext.class))) .thenReturn(rowType); Source source = new Source("db", List.of("MYSQL", "test_db", "MyTable"), Collections.emptyMap()); @@ -669,7 +696,7 @@ void testUpdateAltersExistingTableAddsColumn() throws Exception { when(columnsRs.getInt("NULLABLE")).thenReturn(DatabaseMetaData.columnNoNulls, DatabaseMetaData.columnNullable); when(mockMetaData.getColumns(eq("test_db"), any(), eq("MyTable"), any())).thenReturn(columnsRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); deployer.update(); // CREATE DATABASE + ALTER TABLE ADD COLUMN email @@ -703,7 +730,7 @@ void testUpdateAltersExistingTableNoChanges() throws Exception { when(columnsRs.getInt("NULLABLE")).thenReturn(DatabaseMetaData.columnNoNulls, DatabaseMetaData.columnNullable); when(mockMetaData.getColumns(eq("test_db"), any(), eq("MyTable"), any())).thenReturn(columnsRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); deployer.update(); // Only CREATE DATABASE, no ALTER TABLE @@ -721,7 +748,7 @@ void testUpdateAltersExistingTableModifiesColumn() throws Exception { builder.add("name", typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR, 500), true)); RelDataType rowType = builder.build(); - hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(HoptimatorConnection.class))) + hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(DeploymentContext.class))) .thenReturn(rowType); Source source = new Source("db", List.of("MYSQL", "test_db", "ModTable"), Collections.emptyMap()); @@ -746,7 +773,7 @@ void testUpdateAltersExistingTableModifiesColumn() throws Exception { when(columnsRs.getInt("NULLABLE")).thenReturn(DatabaseMetaData.columnNoNulls, DatabaseMetaData.columnNullable); when(mockMetaData.getColumns(eq("test_db"), any(), eq("ModTable"), any())).thenReturn(columnsRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); deployer.update(); // CREATE DATABASE + ALTER TABLE MODIFY COLUMN @@ -780,7 +807,7 @@ void testUpdateAltersExistingTableDropsColumn() throws Exception { when(columnsRs.getInt("NULLABLE")).thenReturn(DatabaseMetaData.columnNoNulls, DatabaseMetaData.columnNullable, DatabaseMetaData.columnNullable); when(mockMetaData.getColumns(eq("test_db"), any(), eq("DropTable"), any())).thenReturn(columnsRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); deployer.update(); // CREATE DATABASE + ALTER TABLE DROP COLUMN old_col @@ -797,7 +824,7 @@ void testBuildDesiredColumnsInvalidColumnNameThrowsSqlException() throws Excepti builder.add("invalid-col", typeFactory.createSqlType(SqlTypeName.VARCHAR, 100)); RelDataType rowType = builder.build(); - hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(HoptimatorConnection.class))) + hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(DeploymentContext.class))) .thenReturn(rowType); Source source = new Source("db", List.of("MYSQL", "test_db", "BadColTable2"), Collections.emptyMap()); @@ -818,7 +845,7 @@ void testBuildDesiredColumnsInvalidColumnNameThrowsSqlException() throws Excepti when(columnsRs.next()).thenReturn(false); when(mockMetaData.getColumns(eq("test_db"), any(), eq("BadColTable2"), any())).thenReturn(columnsRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); // update() -> alterTable() -> buildDesiredColumns() should throw on invalid col name assertThrows(SQLException.class, deployer::update, "Expected SQLException for invalid column name in buildDesiredColumns"); @@ -899,7 +926,7 @@ void testToMySqlTypeExactSqlString(String label, SqlTypeName sqlType, int precis } RelDataType rowType = builder.build(); - hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(HoptimatorConnection.class))) + hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(DeploymentContext.class))) .thenReturn(rowType); Source source = new Source("db", List.of("MYSQL", "test_db", "TypeExact"), Collections.emptyMap()); @@ -908,7 +935,7 @@ void testToMySqlTypeExactSqlString(String label, SqlTypeName sqlType, int precis when(emptyRs.next()).thenReturn(false); when(mockMetaData.getTables(eq("test_db"), any(), eq("TypeExact"), any())).thenReturn(emptyRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); deployer.create(); @@ -937,7 +964,7 @@ void testToMySqlTypeVarcharWithPrecisionGivesVarcharN() throws Exception { builder.add("col", typeFactory.createSqlType(SqlTypeName.VARCHAR, 1)); // precision == 1 (> 0 boundary) RelDataType rowType = builder.build(); - hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(HoptimatorConnection.class))) + hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(DeploymentContext.class))) .thenReturn(rowType); Source source = new Source("db", List.of("MYSQL", "test_db", "VarcharBound"), Collections.emptyMap()); @@ -946,7 +973,7 @@ void testToMySqlTypeVarcharWithPrecisionGivesVarcharN() throws Exception { when(emptyRs.next()).thenReturn(false); when(mockMetaData.getTables(eq("test_db"), any(), eq("VarcharBound"), any())).thenReturn(emptyRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); deployer.create(); verify(mockStatement, times(2)).executeUpdate(sqlCaptor.capture()); @@ -971,7 +998,7 @@ void testBuildCreateTableSqlNonNullableColumnContainsNotNull() throws Exception typeFactory.createSqlType(SqlTypeName.VARCHAR, 255), false)); RelDataType rowType = builder.build(); - hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(HoptimatorConnection.class))) + hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(DeploymentContext.class))) .thenReturn(rowType); Source source = new Source("db", List.of("MYSQL", "test_db", "NonNullTable"), Collections.emptyMap()); @@ -980,7 +1007,7 @@ void testBuildCreateTableSqlNonNullableColumnContainsNotNull() throws Exception when(emptyRs.next()).thenReturn(false); when(mockMetaData.getTables(eq("test_db"), any(), eq("NonNullTable"), any())).thenReturn(emptyRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); deployer.create(); verify(mockStatement, times(2)).executeUpdate(sqlCaptor.capture()); @@ -1003,7 +1030,7 @@ void testBuildCreateTableSqlNullableColumnDoesNotContainNotNull() throws Excepti typeFactory.createSqlType(SqlTypeName.VARCHAR, 255), true)); RelDataType rowType = builder.build(); - hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(HoptimatorConnection.class))) + hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(DeploymentContext.class))) .thenReturn(rowType); Source source = new Source("db", List.of("MYSQL", "test_db", "NullableTable"), Collections.emptyMap()); @@ -1012,7 +1039,7 @@ void testBuildCreateTableSqlNullableColumnDoesNotContainNotNull() throws Excepti when(emptyRs.next()).thenReturn(false); when(mockMetaData.getTables(eq("test_db"), any(), eq("NullableTable"), any())).thenReturn(emptyRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); deployer.create(); verify(mockStatement, times(2)).executeUpdate(sqlCaptor.capture()); @@ -1039,7 +1066,7 @@ void testBuildCreateTableSqlContainsPrimaryKey() throws Exception { typeFactory.createSqlType(SqlTypeName.VARCHAR, 100), true)); RelDataType rowType = builder.build(); - hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(HoptimatorConnection.class))) + hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(DeploymentContext.class))) .thenReturn(rowType); Source source = new Source("db", List.of("MYSQL", "test_db", "PKTable"), Collections.emptyMap()); @@ -1048,7 +1075,7 @@ void testBuildCreateTableSqlContainsPrimaryKey() throws Exception { when(emptyRs.next()).thenReturn(false); when(mockMetaData.getTables(eq("test_db"), any(), eq("PKTable"), any())).thenReturn(emptyRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); deployer.create(); verify(mockStatement, times(2)).executeUpdate(sqlCaptor.capture()); @@ -1072,7 +1099,7 @@ void testBuildCreateTableSqlVarcharWithLengthInDdl() throws Exception { typeFactory.createSqlType(SqlTypeName.VARCHAR, 512), true)); RelDataType rowType = builder.build(); - hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(HoptimatorConnection.class))) + hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(DeploymentContext.class))) .thenReturn(rowType); Source source = new Source("db", List.of("MYSQL", "test_db", "VarLenTable"), Collections.emptyMap()); @@ -1081,7 +1108,7 @@ void testBuildCreateTableSqlVarcharWithLengthInDdl() throws Exception { when(emptyRs.next()).thenReturn(false); when(mockMetaData.getTables(eq("test_db"), any(), eq("VarLenTable"), any())).thenReturn(emptyRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); deployer.create(); verify(mockStatement, times(2)).executeUpdate(sqlCaptor.capture()); @@ -1108,7 +1135,7 @@ void testAlterTableAddColumnSqlContainsAddColumn() throws Exception { typeFactory.createSqlType(SqlTypeName.VARCHAR, 200), true)); RelDataType rowType = builder.build(); - hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(HoptimatorConnection.class))) + hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(DeploymentContext.class))) .thenReturn(rowType); Source source = new Source("db", List.of("MYSQL", "test_db", "AddColTable"), Collections.emptyMap()); @@ -1134,7 +1161,7 @@ void testAlterTableAddColumnSqlContainsAddColumn() throws Exception { DatabaseMetaData.columnNoNulls, DatabaseMetaData.columnNullable); when(mockMetaData.getColumns(eq("test_db"), any(), eq("AddColTable"), any())).thenReturn(columnsRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); deployer.update(); verify(mockStatement, times(2)).executeUpdate(sqlCaptor.capture()); @@ -1164,7 +1191,7 @@ void testAlterTableModifyColumnSqlContainsModifyColumn() throws Exception { typeFactory.createSqlType(SqlTypeName.VARCHAR, 500), true)); RelDataType rowType = builder.build(); - hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(HoptimatorConnection.class))) + hoptimatorDriverStatic.when(() -> HoptimatorDriver.rowType(any(Source.class), any(DeploymentContext.class))) .thenReturn(rowType); Source source = new Source("db", List.of("MYSQL", "test_db", "ModColTable"), Collections.emptyMap()); @@ -1189,7 +1216,7 @@ void testAlterTableModifyColumnSqlContainsModifyColumn() throws Exception { DatabaseMetaData.columnNoNulls, DatabaseMetaData.columnNullable); when(mockMetaData.getColumns(eq("test_db"), any(), eq("ModColTable"), any())).thenReturn(columnsRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); deployer.update(); verify(mockStatement, times(2)).executeUpdate(sqlCaptor.capture()); @@ -1235,7 +1262,7 @@ void testAlterTableDropColumnSqlContainsDropColumn() throws Exception { DatabaseMetaData.columnNoNulls, DatabaseMetaData.columnNullable, DatabaseMetaData.columnNullable); when(mockMetaData.getColumns(eq("test_db"), any(), eq("DropColTable"), any())).thenReturn(columnsRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); deployer.update(); verify(mockStatement, times(2)).executeUpdate(sqlCaptor.capture()); @@ -1269,7 +1296,7 @@ void testEscapeIdentifierViaDeleteSqlContainsBacktickedName() throws Exception { when(dbNotEmptyRs.next()).thenReturn(true); when(mockMetaData.getTables(eq("test_db"), any(), isNull(), any())).thenReturn(dbNotEmptyRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); deployer.delete(); verify(mockStatement).executeUpdate(sqlCaptor.capture()); @@ -1296,7 +1323,7 @@ void testEnsureDatabaseExistsSqlContainsCreateDatabase() throws Exception { when(emptyRs.next()).thenReturn(false); when(mockMetaData.getTables(eq("test_db"), any(), eq("SomeTable"), any())).thenReturn(emptyRs); - MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, mockHoptimatorConnection); + MySqlDeployer deployer = new MySqlDeployer(source, PROPERTIES, new CalciteDeploymentContext(mockHoptimatorConnection)); ArgumentCaptor sqlCaptor = ArgumentCaptor.forClass(String.class); deployer.create(); verify(mockStatement, times(2)).executeUpdate(sqlCaptor.capture()); diff --git a/hoptimator-operator/src/main/java/com/linkedin/hoptimator/operator/PipelineOperatorApp.java b/hoptimator-operator/src/main/java/com/linkedin/hoptimator/operator/PipelineOperatorApp.java index 9641e65ff..eaf92c81e 100644 --- a/hoptimator-operator/src/main/java/com/linkedin/hoptimator/operator/PipelineOperatorApp.java +++ b/hoptimator-operator/src/main/java/com/linkedin/hoptimator/operator/PipelineOperatorApp.java @@ -1,7 +1,6 @@ package com.linkedin.hoptimator.operator; import com.google.common.annotations.VisibleForTesting; -import com.linkedin.hoptimator.jdbc.HoptimatorConnection; import com.linkedin.hoptimator.k8s.K8sApiEndpoints; import com.linkedin.hoptimator.k8s.K8sContext; import com.linkedin.hoptimator.operator.pipeline.PipelineReconciler; @@ -94,7 +93,7 @@ public static void main(String[] args) throws Exception { String watchNamespaceInput = cmd.getOptionValue("watch", ""); Properties connectionProperties = new Properties(); connectionProperties.put("k8s.watch.namespace", watchNamespaceInput); - K8sContext context = K8sContext.create(new HoptimatorConnection(null, connectionProperties)); + K8sContext context = K8sContext.create(connectionProperties); PipelineOperatorApp app = new PipelineOperatorApp(context); app.installShutdownHook(Duration.ofMinutes(1)); app.start(Collections.emptyList()); diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/ConfigService.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/ConfigService.java index 09b5aff12..7ed340614 100644 --- a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/ConfigService.java +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/ConfigService.java @@ -1,11 +1,11 @@ package com.linkedin.hoptimator.util; import com.linkedin.hoptimator.ConfigProvider; +import com.linkedin.hoptimator.DeploymentContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.StringReader; -import java.sql.Connection; import java.util.Properties; import java.util.ServiceLoader; @@ -21,12 +21,12 @@ private ConfigService() { // Ex: // log.properties: | // level=INFO - public static Properties config(Connection connection, boolean loadTopLevelConfigs, String... expansionFields) { + public static Properties config(DeploymentContext context, boolean loadTopLevelConfigs, String... expansionFields) { ServiceLoader loader = ServiceLoader.load(ConfigProvider.class); Properties properties = new Properties(); for (ConfigProvider provider : loader) { try { - Properties loadedProperties = provider.loadConfig(connection); + Properties loadedProperties = provider.loadConfig(context); if (loadTopLevelConfigs) { log.debug("Loaded properties={} from provider={}", loadedProperties, provider); properties.putAll(loadedProperties); @@ -45,7 +45,7 @@ public static Properties config(Connection connection, boolean loadTopLevelConfi return properties; } - public static Properties config(Connection connection, String... expansionFields) { - return config(connection, true, expansionFields); + public static Properties config(DeploymentContext context, String... expansionFields) { + return config(context, true, expansionFields); } } diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/ConnectionService.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/ConnectionService.java index f11228229..02334bb9c 100644 --- a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/ConnectionService.java +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/ConnectionService.java @@ -2,8 +2,8 @@ import com.linkedin.hoptimator.Connector; import com.linkedin.hoptimator.ConnectorProvider; +import com.linkedin.hoptimator.DeploymentContext; -import java.sql.Connection; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; @@ -19,10 +19,10 @@ public final class ConnectionService { private ConnectionService() { } - public static Map configure(T obj, Connection connection) + public static Map configure(T obj, DeploymentContext context) throws SQLException { Map configs = new LinkedHashMap<>(); - for (Connector connector : connectors(obj, connection)) { + for (Connector connector : connectors(obj, context)) { configs.putAll(connector.configure()); } return configs; @@ -35,9 +35,9 @@ public static Collection providers() { return providers; } - public static Collection connectors(T obj, Connection connection) { + public static Collection connectors(T obj, DeploymentContext context) { return providers().stream() - .flatMap(x -> x.connectors(obj, connection).stream()) + .flatMap(x -> x.connectors(obj, context).stream()) .collect(Collectors.toList()); } } diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/DeploymentService.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/DeploymentService.java index 6e8c8b22d..382f0b32c 100644 --- a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/DeploymentService.java +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/DeploymentService.java @@ -4,6 +4,7 @@ import com.linkedin.hoptimator.Deployable; import com.linkedin.hoptimator.Deployer; import com.linkedin.hoptimator.DeployerProvider; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.util.planner.PipelineRel; import com.linkedin.hoptimator.util.planner.PipelineRules; import org.apache.calcite.plan.RelOptMaterialization; @@ -16,7 +17,6 @@ import java.net.URLDecoder; import java.nio.charset.StandardCharsets; -import java.sql.Connection; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; @@ -62,10 +62,10 @@ public static void update(Collection deployers) } // Since nothing about specify needs to be stateful, the deployers can be fetched on demand - public static List specify(T obj, Connection connection) + public static List specify(T obj, DeploymentContext context) throws SQLException { List specs = new ArrayList<>(); - for (Deployer deployer : deployers(obj, connection)) { + for (Deployer deployer : deployers(obj, context)) { specs.addAll(deployer.specify()); } return specs; @@ -85,11 +85,11 @@ public static Collection providers() { return providers; } - public static Collection deployers(T obj, Connection connection) { - return deployers(obj, connection, providers()); + public static Collection deployers(T obj, DeploymentContext context) { + return deployers(obj, context, providers()); } - static Collection deployers(T obj, Connection connection, + static Collection deployers(T obj, DeploymentContext context, Collection providers) { // Filter out base classes when subclasses exist Set filteredProviders = new HashSet<>(); @@ -109,7 +109,7 @@ static Collection deployers(T obj, Connection c // Now collect deployers from filtered providers return filteredProviders.stream() - .flatMap(x -> x.deployers(obj, connection).stream()) + .flatMap(x -> x.deployers(obj, context).stream()) .collect(Collectors.toList()); } diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/SimpleDeploymentContext.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/SimpleDeploymentContext.java new file mode 100644 index 000000000..f45e42f33 --- /dev/null +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/SimpleDeploymentContext.java @@ -0,0 +1,39 @@ +package com.linkedin.hoptimator.util; + +import com.linkedin.hoptimator.DeploymentContext; + +import javax.annotation.Nullable; +import java.util.Properties; + + +/** + * A minimal {@link DeploymentContext} that carries only connection-level {@link #properties()} and + * resolves no per-{@code Database} config. Intended for control-plane callers — operators, + * reconcilers, event processors — that only need to run {@link ConfigService} / {@code K8sContext} + * off a plain {@link Properties} bag (a namespace, K8s access config, hints), without a Calcite + * connection or a {@code Database} registry. + * + *

{@link #databaseProperties} always returns {@code null}: these callers never resolve database + * connection URLs, so there is no need to wrap a dummy {@code HoptimatorConnection} in a + * {@code CalciteDeploymentContext} just to shuttle properties. Use {@code CalciteDeploymentContext} + * (SQL path) or {@code DirectDeploymentContext} (direct API path) when database config is required. + */ +public final class SimpleDeploymentContext implements DeploymentContext { + + private final Properties properties; + + public SimpleDeploymentContext(Properties properties) { + this.properties = properties; + } + + @Override + public Properties properties() { + return properties; + } + + @Override + public @Nullable Properties databaseProperties(@Nullable String catalog, @Nullable String schema, + String connectionPrefix) { + return null; + } +} diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/EngineRules.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/EngineRules.java index a9e7aa910..cb90ff033 100644 --- a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/EngineRules.java +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/EngineRules.java @@ -1,5 +1,6 @@ package com.linkedin.hoptimator.util.planner; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Engine; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptCost; @@ -18,7 +19,6 @@ import org.apache.calcite.sql.dialect.AnsiSqlDialect; import org.apache.calcite.sql.dialect.MysqlSqlDialect; -import java.sql.Connection; import java.util.Collections; import java.util.Objects; @@ -33,9 +33,9 @@ public EngineRules(Engine engine) { } public void register(HoptimatorJdbcConvention inTrait, RelOptPlanner planner, - Connection connection) { + DeploymentContext context) { RemoteConvention remote = inTrait.remoteConventionForEngine(engine); - planner.addRule(RemoteToEnumerableConverterRule.create(remote, connection)); + planner.addRule(RemoteToEnumerableConverterRule.create(remote, context)); planner.addRule(RemoteJoinRule.Config.INSTANCE .withConversion(PipelineRules.PipelineJoin.class, PipelineRel.CONVENTION, remote, "RemoteJoinRule") .withRuleFactory(RemoteJoinRule::new) diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcCatalogSchema.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcCatalogSchema.java index 71d529ea2..65e24dbf4 100644 --- a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcCatalogSchema.java +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcCatalogSchema.java @@ -2,6 +2,7 @@ import com.google.common.collect.ImmutableSet; import com.linkedin.hoptimator.Database; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Engine; import org.apache.calcite.adapter.jdbc.JdbcCatalogSchema; import org.apache.calcite.adapter.jdbc.JdbcSchema; @@ -36,9 +37,9 @@ public class HoptimatorJdbcCatalogSchema extends JdbcCatalogSchema implements Da private final HoptimatorJdbcConvention convention; public static HoptimatorJdbcCatalogSchema create(String database, String catalog, String schema, DataSource dataSource, - SchemaPlus parentSchema, SqlDialect dialect, List engines, Connection connection) { + SchemaPlus parentSchema, SqlDialect dialect, List engines, DeploymentContext context) { Expression expression = Schemas.subSchemaExpression(parentSchema, schema, HoptimatorJdbcCatalogSchema.class); - HoptimatorJdbcConvention convention = new HoptimatorJdbcConvention(dialect, expression, database, engines, connection); + HoptimatorJdbcConvention convention = new HoptimatorJdbcConvention(dialect, expression, database, engines, context); return new HoptimatorJdbcCatalogSchema(database, catalog, dataSource, dialect, convention, engines); } diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcConvention.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcConvention.java index 55e7f7dd0..b31bc04cd 100644 --- a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcConvention.java +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcConvention.java @@ -1,12 +1,12 @@ package com.linkedin.hoptimator.util.planner; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Engine; import org.apache.calcite.adapter.jdbc.JdbcConvention; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.sql.SqlDialect; -import java.sql.Connection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -16,15 +16,15 @@ public class HoptimatorJdbcConvention extends JdbcConvention { private final String database; private final List engines; - private final Connection connection; + private final DeploymentContext context; private final Map remoteConventions = new HashMap<>(); public HoptimatorJdbcConvention(SqlDialect dialect, Expression expression, String name, - List engines, Connection connection) { + List engines, DeploymentContext context) { super(dialect, expression, name); this.database = name; this.engines = engines; - this.connection = connection; + this.context = context; } public String database() { @@ -46,6 +46,6 @@ public void register(RelOptPlanner planner) { planner.addRule(PipelineRules.PipelineTableScanRule.create(this)); planner.addRule(PipelineRules.PipelineTableModifyRule.create(this)); PipelineRules.rules().forEach(planner::addRule); - engines().forEach(x -> new EngineRules(x).register(this, planner, connection)); + engines().forEach(x -> new EngineRules(x).register(this, planner, context)); } } diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcSchema.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcSchema.java index 9e7c67a43..7f87b7b02 100644 --- a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcSchema.java +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcSchema.java @@ -1,6 +1,7 @@ package com.linkedin.hoptimator.util.planner; import com.linkedin.hoptimator.Database; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Engine; import org.apache.calcite.adapter.jdbc.JdbcSchema; import org.apache.calcite.adapter.jdbc.JdbcTable; @@ -41,9 +42,9 @@ public class HoptimatorJdbcSchema extends JdbcSchema implements Database { private volatile Boolean cachedLogical; public static HoptimatorJdbcSchema create(String database, String catalog, String schema, DataSource dataSource, - SchemaPlus parentSchema, SqlDialect dialect, List engines, Connection connection) { + SchemaPlus parentSchema, SqlDialect dialect, List engines, DeploymentContext context) { Expression expression = Schemas.subSchemaExpression(parentSchema, schema, HoptimatorJdbcSchema.class); - HoptimatorJdbcConvention convention = new HoptimatorJdbcConvention(dialect, expression, database, engines, connection); + HoptimatorJdbcConvention convention = new HoptimatorJdbcConvention(dialect, expression, database, engines, context); return new HoptimatorJdbcSchema(database, catalog, schema, dataSource, dialect, convention, engines); } diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/PipelineRel.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/PipelineRel.java index d5fe73c12..6c9c4ca79 100644 --- a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/PipelineRel.java +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/PipelineRel.java @@ -1,6 +1,7 @@ package com.linkedin.hoptimator.util.planner; import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Job; import com.linkedin.hoptimator.Pipeline; import com.linkedin.hoptimator.Sink; @@ -23,7 +24,6 @@ import org.apache.calcite.sql.dialect.AnsiSqlDialect; import org.apache.calcite.sql.fun.SqlItemOperator; -import java.sql.Connection; import java.sql.SQLException; import java.sql.SQLNonTransientException; import java.util.HashMap; @@ -107,17 +107,17 @@ public void setQuery(RelNode query) { } /** Combine deployables into a Pipeline */ - public Pipeline pipeline(String name, Connection connection) throws SQLException { + public Pipeline pipeline(String name, DeploymentContext context) throws SQLException { Map> templateEvals = new HashMap<>(); - templateEvals.put("sql", sql(connection)); - templateEvals.put("query", query(connection)); + templateEvals.put("sql", sql(context)); + templateEvals.put("query", query(context)); templateEvals.put("fieldMap", fieldMap()); Job job = new Job(name, sources.keySet(), sink, templateEvals); return new Pipeline(sources.keySet(), sink, job); } - private ScriptImplementor script(Connection connection) throws SQLException { + private ScriptImplementor script(DeploymentContext context) throws SQLException { ScriptImplementor script = ScriptImplementor.empty(); // Check if we need to add suffixes to avoid table name collisions boolean needsSuffixes = hasTableNameCollision(); @@ -125,7 +125,7 @@ private ScriptImplementor script(Connection connection) throws SQLException { for (Map.Entry source : sources.entrySet()) { script = script.catalog(source.getKey().catalog()); script = script.database(source.getKey().catalog(), source.getKey().schema()); - Map configs = ConnectionService.configure(source.getKey(), connection); + Map configs = ConnectionService.configure(source.getKey(), context); String suffix = needsSuffixes ? "_source" : null; script = script.connector(source.getKey().catalog(), source.getKey().schema(), source.getKey().table(), suffix, source.getValue(), configs); } @@ -151,16 +151,16 @@ private boolean hasTableNameCollision() { } /** SQL script ending in an INSERT INTO */ - public ThrowingFunction sql(Connection connection) throws SQLException { + public ThrowingFunction sql(DeploymentContext context) throws SQLException { return wrap(x -> { - ScriptImplementor script = script(connection); + ScriptImplementor script = script(context); RelDataType targetRowType = sinkRowType; if (targetRowType == null) { targetRowType = query.getRowType(); } else { validateFieldMapping(targetRowType); } - Map sinkConfigs = ConnectionService.configure(sink, connection); + Map sinkConfigs = ConnectionService.configure(sink, context); script = script.catalog(sink.catalog()); script = script.database(sink.catalog(), sink.schema()); // Check if we need to add suffixes to avoid table name collisions @@ -184,8 +184,8 @@ public ThrowingFunction sql(Connection connection) throws SQ } /** SQL script ending in a SELECT */ - public ThrowingFunction query(Connection connection) throws SQLException { - return wrap(x -> script(connection).query(query).sql(x)); + public ThrowingFunction query(DeploymentContext context) throws SQLException { + return wrap(x -> script(context).query(query).sql(x)); } public ThrowingFunction fieldMap() { diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverter.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverter.java index 0a958f2f6..1408ef8c0 100644 --- a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverter.java +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverter.java @@ -19,6 +19,7 @@ package com.linkedin.hoptimator.util.planner; import com.google.common.collect.ImmutableList; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.util.DelegatingDataSource; import com.linkedin.hoptimator.util.DeploymentService; import org.apache.calcite.DataContext; @@ -59,7 +60,6 @@ import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.sql.Array; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; @@ -80,15 +80,15 @@ public class RemoteToEnumerableConverter extends ConverterImpl implements EnumerableRel { - private final Connection connection; + private final DeploymentContext context; protected RemoteToEnumerableConverter( RelOptCluster cluster, RelTraitSet traits, RelNode input, - Connection connection) { + DeploymentContext context) { super(cluster, ConventionTraitDef.INSTANCE, traits, input); - this.connection = connection; + this.context = context; } /** This method modified from upstream */ @@ -96,7 +96,7 @@ private SqlString generateSql(SqlDialect dialect) { RelRoot root = RelRoot.of(getInput(), SqlKind.SELECT); try { PipelineRel.Implementor plan = DeploymentService.plan(root, Collections.emptyList(), new Properties()); - return new SqlString(AnsiSqlDialect.DEFAULT, plan.query(connection) + return new SqlString(AnsiSqlDialect.DEFAULT, plan.query(context) .apply(com.linkedin.hoptimator.SqlDialect.FLINK)); // TODO dialect } catch (SQLException e) { throw new RuntimeException(e); @@ -105,7 +105,7 @@ private SqlString generateSql(SqlDialect dialect) { @Override public RelNode copy(RelTraitSet traitSet, List inputs) { return new RemoteToEnumerableConverter( - getCluster(), traitSet, sole(inputs), connection); + getCluster(), traitSet, sole(inputs), context); } @Override public @Nullable RelOptCost computeSelfCost(RelOptPlanner planner, diff --git a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverterRule.java b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverterRule.java index 95d1918ab..5bb6853b0 100644 --- a/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverterRule.java +++ b/hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverterRule.java @@ -18,6 +18,7 @@ */ package com.linkedin.hoptimator.util.planner; +import com.linkedin.hoptimator.DeploymentContext; import org.apache.calcite.adapter.enumerable.EnumerableConvention; import org.apache.calcite.adapter.jdbc.JdbcConvention; import org.apache.calcite.plan.RelTraitSet; @@ -25,7 +26,6 @@ import org.apache.calcite.rel.convert.ConverterRule; import javax.annotation.Nullable; -import java.sql.Connection; /** * Rule to convert a relational expression from @@ -34,25 +34,25 @@ */ public class RemoteToEnumerableConverterRule extends ConverterRule { - private final Connection connection; + private final DeploymentContext context; /** Creates a RemoteToEnumerableConverterRule. */ - public static RemoteToEnumerableConverterRule create(RemoteConvention inTrait, Connection connection) { + public static RemoteToEnumerableConverterRule create(RemoteConvention inTrait, DeploymentContext context) { return Config.INSTANCE .withConversion(RelNode.class, inTrait, EnumerableConvention.INSTANCE, "RemoteToEnumerableConverterRule") - .withRuleFactory(x -> new RemoteToEnumerableConverterRule(x, connection)) + .withRuleFactory(x -> new RemoteToEnumerableConverterRule(x, context)) .toRule(RemoteToEnumerableConverterRule.class); } /** Called from the Config. */ - protected RemoteToEnumerableConverterRule(Config config, Connection connection) { + protected RemoteToEnumerableConverterRule(Config config, DeploymentContext context) { super(config); - this.connection = connection; + this.context = context; } @Override public @Nullable RelNode convert(RelNode rel) { RelTraitSet newTraitSet = rel.getTraitSet().replace(getOutTrait()); - return new RemoteToEnumerableConverter(rel.getCluster(), newTraitSet, rel, connection); + return new RemoteToEnumerableConverter(rel.getCluster(), newTraitSet, rel, context); } } diff --git a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/ConfigServiceTest.java b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/ConfigServiceTest.java index 37373e243..52649dd69 100644 --- a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/ConfigServiceTest.java +++ b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/ConfigServiceTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.util; +import com.linkedin.hoptimator.DeploymentContext; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -7,7 +9,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.sql.Connection; import java.util.Properties; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -21,7 +22,7 @@ class ConfigServiceTest { @Mock - private Connection mockConnection; + private DeploymentContext mockConnection; @BeforeEach void setUp() { diff --git a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/ConnectionServiceTest.java b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/ConnectionServiceTest.java index c323cea46..a5963f2d5 100644 --- a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/ConnectionServiceTest.java +++ b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/ConnectionServiceTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.util; +import com.linkedin.hoptimator.DeploymentContext; + import com.linkedin.hoptimator.Connector; import com.linkedin.hoptimator.ConnectorProvider; import org.junit.jupiter.api.Test; @@ -9,7 +11,6 @@ import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoExtension; -import java.sql.Connection; import java.sql.SQLException; import java.util.Collection; import java.util.Collections; @@ -27,7 +28,7 @@ class ConnectionServiceTest { @Mock - private Connection mockConnection; + private DeploymentContext mockConnection; @Mock(answer = Answers.CALLS_REAL_METHODS) private MockedStatic mockedConnectionService; @@ -67,7 +68,7 @@ void testConfigureCollectsConfigsFromConnectors() throws SQLException { ConnectorProvider provider = new ConnectorProvider() { @Override - public Collection connectors(T obj, Connection conn) { + public Collection connectors(T obj, DeploymentContext conn) { return Collections.singletonList(mockConnector); } }; @@ -92,7 +93,7 @@ void testProvidersViaStaticMockReturnsNonEmptyProviderList() throws SQLException ConnectorProvider provider = new ConnectorProvider() { @Override - public Collection connectors(T obj, Connection conn) { + public Collection connectors(T obj, DeploymentContext conn) { return Collections.singletonList(mockConnector); } }; diff --git a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/DeploymentServiceTest.java b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/DeploymentServiceTest.java index 06c9e9cf3..f0dca6491 100644 --- a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/DeploymentServiceTest.java +++ b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/DeploymentServiceTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.util; +import com.linkedin.hoptimator.DeploymentContext; + import com.linkedin.hoptimator.Deployable; import com.linkedin.hoptimator.Deployer; import com.linkedin.hoptimator.DeployerProvider; @@ -26,7 +28,6 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; -import java.sql.Connection; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; @@ -54,7 +55,7 @@ class DeploymentServiceTest { private MockedStatic mockedDeploymentService; @Mock - private Connection mockConnection; + private DeploymentContext mockConnection; @Mock private Deployer mockDeployer1; @@ -389,7 +390,7 @@ void testDeployersFilteringWithSingleProvider() { Deployable deployable = new Deployable() { }; DeployerProvider provider = new DeployerProvider() { @Override - public Collection deployers(T obj, Connection conn) { + public Collection deployers(T obj, DeploymentContext conn) { return Collections.singletonList(mockDeployer1); } @Override @@ -410,7 +411,7 @@ void testDeployersFilteringRemovesBaseClassProvider() { DeployerProvider baseProvider = new DeployerProvider() { @Override - public Collection deployers(T obj, Connection conn) { + public Collection deployers(T obj, DeploymentContext conn) { return Collections.singletonList(mockDeployer1); } @Override @@ -422,7 +423,7 @@ public int priority() { // Create a subclass of the base provider DeployerProvider subProvider = new DeployerProvider() { @Override - public Collection deployers(T obj, Connection conn) { + public Collection deployers(T obj, DeploymentContext conn) { return Collections.singletonList(mockDeployer2); } @Override @@ -486,7 +487,7 @@ private static class BaseTestProvider implements DeployerProvider { this.deployer = deployer; } @Override - public Collection deployers(T obj, Connection conn) { + public Collection deployers(T obj, DeploymentContext conn) { return Collections.singletonList(deployer); } @Override @@ -536,7 +537,7 @@ private static class AnotherTestProvider implements DeployerProvider { this.deployer = deployer; } @Override - public Collection deployers(T obj, Connection conn) { + public Collection deployers(T obj, DeploymentContext conn) { return Collections.singletonList(deployer); } @Override diff --git a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/SimpleDeploymentContextTest.java b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/SimpleDeploymentContextTest.java new file mode 100644 index 000000000..8c43c0be7 --- /dev/null +++ b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/SimpleDeploymentContextTest.java @@ -0,0 +1,28 @@ +package com.linkedin.hoptimator.util; + +import org.junit.jupiter.api.Test; + +import java.util.Properties; + +import static org.assertj.core.api.Assertions.assertThat; + + +class SimpleDeploymentContextTest { + + @Test + void propertiesReturnsSuppliedBag() { + Properties props = new Properties(); + props.setProperty("k8s.watch.namespace", "ns"); + SimpleDeploymentContext context = new SimpleDeploymentContext(props); + + assertThat(context.properties()).isSameAs(props); + } + + @Test + void databasePropertiesAlwaysReturnsNull() { + SimpleDeploymentContext context = new SimpleDeploymentContext(new Properties()); + + assertThat(context.databaseProperties("CAT", "SCHEMA", "jdbc:kafka://")).isNull(); + assertThat(context.databaseProperties(null, null, "jdbc:venice://")).isNull(); + } +} diff --git a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/TestConfigProvider.java b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/TestConfigProvider.java index 05dca928a..5d1e29254 100644 --- a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/TestConfigProvider.java +++ b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/TestConfigProvider.java @@ -2,7 +2,7 @@ import com.linkedin.hoptimator.ConfigProvider; -import java.sql.Connection; +import com.linkedin.hoptimator.DeploymentContext; import java.util.Properties; @@ -30,7 +30,7 @@ public static void put(String key, String value) { } @Override - public Properties loadConfig(Connection connection) { + public Properties loadConfig(DeploymentContext context) { Properties copy = new Properties(); copy.putAll(PROPERTIES); return copy; diff --git a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/TestDeploymentContext.java b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/TestDeploymentContext.java new file mode 100644 index 000000000..fe146b786 --- /dev/null +++ b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/TestDeploymentContext.java @@ -0,0 +1,33 @@ +package com.linkedin.hoptimator.util; + +import com.linkedin.hoptimator.DeploymentContext; + +import java.util.Properties; + +import javax.annotation.Nullable; + + +/** Minimal no-op {@link DeploymentContext} for unit tests that don't exercise config resolution. */ +public class TestDeploymentContext implements DeploymentContext { + + private final Properties properties; + + public TestDeploymentContext() { + this(new Properties()); + } + + public TestDeploymentContext(Properties properties) { + this.properties = properties; + } + + @Override + public Properties properties() { + return properties; + } + + @Override + public Properties databaseProperties(@Nullable String catalog, @Nullable String schema, + String connectionPrefix) { + return null; + } +} diff --git a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/EngineRulesTest.java b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/EngineRulesTest.java index 184b21122..51c3f7b33 100644 --- a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/EngineRulesTest.java +++ b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/EngineRulesTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.util.planner; +import com.linkedin.hoptimator.DeploymentContext; + import com.linkedin.hoptimator.Engine; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.plan.RelOptPlanner; @@ -11,7 +13,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.sql.Connection; import java.util.Collections; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -32,7 +33,7 @@ class EngineRulesTest { private RelOptPlanner mockPlanner; @Mock - private Connection mockConnection; + private DeploymentContext mockConnection; @Mock private Expression mockExpression; diff --git a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcCatalogSchemaTest.java b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcCatalogSchemaTest.java index 6773e14bb..68d0f2402 100644 --- a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcCatalogSchemaTest.java +++ b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcCatalogSchemaTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.util.planner; +import com.linkedin.hoptimator.DeploymentContext; + import com.linkedin.hoptimator.Engine; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.schema.Schema; @@ -41,7 +43,7 @@ class HoptimatorJdbcCatalogSchemaTest { private DataSource mockDataSource; @Mock - private Connection mockConnection; + private DeploymentContext mockConnection; @Mock private Expression mockExpression; diff --git a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcConventionTest.java b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcConventionTest.java index 79b7b8f48..f3aca05a4 100644 --- a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcConventionTest.java +++ b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcConventionTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.util.planner; +import com.linkedin.hoptimator.DeploymentContext; + import com.linkedin.hoptimator.Engine; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.plan.RelOptPlanner; @@ -9,7 +11,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.sql.Connection; import java.util.Collections; import java.util.List; @@ -30,7 +31,7 @@ class HoptimatorJdbcConventionTest { private Expression mockExpression; @Mock - private Connection mockConnection; + private DeploymentContext mockConnection; @Mock private Engine mockEngine; diff --git a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcSchemaTest.java b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcSchemaTest.java index d0ea3a18b..bda85965c 100644 --- a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcSchemaTest.java +++ b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcSchemaTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.util.planner; +import com.linkedin.hoptimator.DeploymentContext; + import com.linkedin.hoptimator.Engine; import org.apache.calcite.jdbc.CalciteConnection; import org.apache.calcite.linq4j.tree.Expression; @@ -44,7 +46,7 @@ class HoptimatorJdbcSchemaTest { private DataSource mockDataSource; @Mock - private Connection mockConnection; + private DeploymentContext mockConnection; @Mock private Expression mockExpression; diff --git a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcTableScanTest.java b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcTableScanTest.java index 271f2b060..c18b68954 100644 --- a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcTableScanTest.java +++ b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcTableScanTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.util.planner; +import com.linkedin.hoptimator.DeploymentContext; + import org.apache.calcite.adapter.jdbc.JdbcTable; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; @@ -17,7 +19,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.sql.Connection; import java.util.Collections; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -31,7 +32,7 @@ class HoptimatorJdbcTableScanTest { private JdbcTable mockJdbcTable; @Mock - private Connection mockConnection; + private DeploymentContext mockConnection; @Mock private RelOptTable mockRelOptTable; diff --git a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcTableTest.java b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcTableTest.java index 8d11a59dd..62c531acc 100644 --- a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcTableTest.java +++ b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/HoptimatorJdbcTableTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.util.planner; +import com.linkedin.hoptimator.DeploymentContext; + import com.linkedin.hoptimator.avro.AvroSchemaSource; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; @@ -27,7 +29,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.sql.Connection; import java.util.Collections; import static org.junit.jupiter.api.Assertions.assertNull; @@ -46,7 +47,7 @@ class HoptimatorJdbcTableTest { private JdbcTable mockJdbcTable; @Mock - private Connection mockConnection; + private DeploymentContext mockConnection; @Mock private Expression mockExpression; diff --git a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/PipelineRelImplementorTest.java b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/PipelineRelImplementorTest.java index 895ade800..93e19b79f 100644 --- a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/PipelineRelImplementorTest.java +++ b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/PipelineRelImplementorTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.util.planner; +import com.linkedin.hoptimator.DeploymentContext; + import com.linkedin.hoptimator.Pipeline; import com.linkedin.hoptimator.SqlDialect; import com.linkedin.hoptimator.ThrowingFunction; @@ -30,7 +32,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.sql.Connection; import java.sql.SQLException; import java.sql.SQLNonTransientException; import java.util.AbstractMap; @@ -51,7 +52,7 @@ class PipelineRelImplementorTest { @Mock - private Connection mockConnection; + private DeploymentContext mockConnection; @Test void testWrapAnsiDialect() throws SQLException { diff --git a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/PipelineRelTest.java b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/PipelineRelTest.java index f1712ac1b..3783700ca 100644 --- a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/PipelineRelTest.java +++ b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/PipelineRelTest.java @@ -1,5 +1,7 @@ package com.linkedin.hoptimator.util.planner; +import com.linkedin.hoptimator.DeploymentContext; + import com.linkedin.hoptimator.Job; import com.linkedin.hoptimator.Pipeline; import com.linkedin.hoptimator.SqlDialect; @@ -26,7 +28,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.sql.Connection; import java.sql.SQLException; import java.sql.SQLNonTransientException; import java.util.AbstractMap; @@ -48,7 +49,7 @@ public class PipelineRelTest { @Mock - private Connection mockConnection; + private DeploymentContext mockConnection; private RelDataTypeFactory typeFactory; private PipelineRel.Implementor implementor; diff --git a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverterRuleTest.java b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverterRuleTest.java index cfa37cf9f..2cbe7c6fa 100644 --- a/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverterRuleTest.java +++ b/hoptimator-util/src/test/java/com/linkedin/hoptimator/util/planner/RemoteToEnumerableConverterRuleTest.java @@ -1,13 +1,21 @@ package com.linkedin.hoptimator.util.planner; +import com.linkedin.hoptimator.DeploymentContext; + import com.linkedin.hoptimator.Engine; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataTypeSystem; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.tools.FrameworkConfig; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.RelBuilder; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.sql.Connection; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -18,7 +26,7 @@ class RemoteToEnumerableConverterRuleTest { private Engine mockEngine; @Mock - private Connection mockConnection; + private DeploymentContext mockConnection; // If create() returns null (NullReturnVals), assertNotNull fails. @Test @@ -38,4 +46,25 @@ void testCreatedRuleHasCorrectDescription() { assertNotNull(rule.toString(), "created rule must have a non-null description"); } + + @Test + void testConvertProducesRemoteToEnumerableConverter() { + RemoteConvention convention = new RemoteConvention("test-remote", mockEngine); + RemoteToEnumerableConverterRule rule = + RemoteToEnumerableConverterRule.create(convention, mockConnection); + + // A trivial VALUES node gives convert() a real cluster + trait set to re-home. + SchemaPlus root = Frameworks.createRootSchema(false); + FrameworkConfig config = Frameworks.newConfigBuilder() + .defaultSchema(root) + .typeSystem(RelDataTypeSystem.DEFAULT) + .build(); + RelBuilder builder = RelBuilder.create(config); + RelNode values = builder.values(new String[] {"C"}, 1).build(); + + RelNode converted = rule.convert(values); + + assertNotNull(converted, "convert() must return a converter node"); + assertInstanceOf(RemoteToEnumerableConverter.class, converted); + } } diff --git a/hoptimator-venice/build.gradle b/hoptimator-venice/build.gradle index cf0d382bc..ed59cd715 100644 --- a/hoptimator-venice/build.gradle +++ b/hoptimator-venice/build.gradle @@ -6,6 +6,7 @@ plugins { dependencies { implementation project(':hoptimator-api') implementation project(':hoptimator-avro') + implementation libs.avro implementation project(':hoptimator-jdbc') implementation project(':hoptimator-util') implementation libs.calcite.core diff --git a/hoptimator-venice/src/main/java/com/linkedin/hoptimator/venice/VeniceDeployer.java b/hoptimator-venice/src/main/java/com/linkedin/hoptimator/venice/VeniceDeployer.java index ee61a1b3f..e258dd64b 100644 --- a/hoptimator-venice/src/main/java/com/linkedin/hoptimator/venice/VeniceDeployer.java +++ b/hoptimator-venice/src/main/java/com/linkedin/hoptimator/venice/VeniceDeployer.java @@ -5,7 +5,7 @@ import com.linkedin.hoptimator.Validated; import com.linkedin.hoptimator.Validator; import com.linkedin.hoptimator.avro.AvroConverter; -import com.linkedin.hoptimator.jdbc.HoptimatorConnection; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.jdbc.HoptimatorDriver; import com.linkedin.hoptimator.util.ConnectionService; import com.linkedin.venice.client.schema.StoreSchemaFetcher; @@ -25,12 +25,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.Connection; import java.sql.SQLException; import java.sql.SQLNonTransientException; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Map; import java.util.Properties; @@ -47,16 +47,16 @@ public class VeniceDeployer implements Deployer, Validated { protected final Source source; protected final Properties properties; - protected final HoptimatorConnection connection; + protected final DeploymentContext context; - public VeniceDeployer(Source source, Properties properties, HoptimatorConnection connection) { + public VeniceDeployer(Source source, Properties properties, DeploymentContext context) { this.source = source; this.properties = properties; - this.connection = connection; + this.context = context; } @Override - public void validate(Validator.Issues issues, Connection connection) { + public void validate(Validator.Issues issues, DeploymentContext context) { String storeName = source.table(); // Validate Venice configuration @@ -137,6 +137,15 @@ public void create() throws SQLException { } } + @Override + public boolean exists() throws SQLException { + try (ControllerClient controllerClient = createControllerClient()) { + return checkStoreExists(controllerClient); + } catch (RuntimeException e) { + throw new SQLException("Failed to check whether Venice store exists: " + source.table(), e); + } + } + @Override public void delete() throws SQLException { try (ControllerClient controllerClient = createControllerClient()) { @@ -189,11 +198,21 @@ public void restore() { } protected Pair getKeyPayloadSchema() throws SQLException { + Map keyOptions = resolveKeyOptions(); return AvroConverter.avroKeyPayloadSchema("com.linkedin.hoptimator", source.table() + "_Key", source.table() + "_Value", - HoptimatorDriver.rowType(source, connection), - ConnectionService.configure(source, connection)); + HoptimatorDriver.rowType(source, context), + keyOptions); + } + + /** + * Resolves connector-supplied options (e.g. the {@code keys} option that drives the key/payload + * split) for this store. Extracted so unit tests can supply options directly instead of resolving + * them through the live {@link ConnectionService} connector chain, which would reach out to K8s. + */ + protected Map resolveKeyOptions() throws SQLException { + return ConnectionService.configure(source, context); } private boolean checkStoreExists(ControllerClient controllerClient) { diff --git a/hoptimator-venice/src/main/java/com/linkedin/hoptimator/venice/VeniceDeployerProvider.java b/hoptimator-venice/src/main/java/com/linkedin/hoptimator/venice/VeniceDeployerProvider.java index 079e9f11a..d122fc8ba 100644 --- a/hoptimator-venice/src/main/java/com/linkedin/hoptimator/venice/VeniceDeployerProvider.java +++ b/hoptimator-venice/src/main/java/com/linkedin/hoptimator/venice/VeniceDeployerProvider.java @@ -3,13 +3,9 @@ import com.linkedin.hoptimator.Deployable; import com.linkedin.hoptimator.Deployer; import com.linkedin.hoptimator.DeployerProvider; +import com.linkedin.hoptimator.DeploymentContext; import com.linkedin.hoptimator.Source; -import com.linkedin.hoptimator.jdbc.DeployerUtils; -import com.linkedin.hoptimator.jdbc.HoptimatorConnection; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.sql.Connection; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -18,26 +14,20 @@ public class VeniceDeployerProvider implements DeployerProvider { - private static final Logger log = LoggerFactory.getLogger(VeniceDeployerProvider.class); - @Override - public Collection deployers(T obj, Connection connection) { + public Collection deployers(T obj, DeploymentContext context) { List deployers = new ArrayList<>(); - if (obj instanceof Source && connection instanceof HoptimatorConnection) { + if (obj instanceof Source) { Source source = (Source) obj; - Properties properties = DeployerUtils.extractPropertiesFromJdbcSchema( - source.catalog(), - source.schema(), - connection, - VeniceDriver.CONNECTION_PREFIX, - log); + Properties properties = context.databaseProperties( + source.catalog(), source.schema(), VeniceDriver.CONNECTION_PREFIX); if (properties == null) { return deployers; } - deployers.add(new VeniceDeployer(source, properties, (HoptimatorConnection) connection)); + deployers.add(new VeniceDeployer(source, properties, context)); } return deployers; diff --git a/hoptimator-venice/src/test/java/com/linkedin/hoptimator/venice/VeniceDeployerProviderTest.java b/hoptimator-venice/src/test/java/com/linkedin/hoptimator/venice/VeniceDeployerProviderTest.java index b24e0e5e6..7422194d7 100644 --- a/hoptimator-venice/src/test/java/com/linkedin/hoptimator/venice/VeniceDeployerProviderTest.java +++ b/hoptimator-venice/src/test/java/com/linkedin/hoptimator/venice/VeniceDeployerProviderTest.java @@ -1,5 +1,8 @@ package com.linkedin.hoptimator.venice; +import com.linkedin.hoptimator.jdbc.CalciteDeploymentContext; +import com.linkedin.hoptimator.DeploymentContext; + import com.linkedin.hoptimator.Deployer; import com.linkedin.hoptimator.MaterializedView; import com.linkedin.hoptimator.Source; @@ -15,7 +18,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.sql.Connection; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -74,7 +76,7 @@ void testReturnsDeployerForVeniceSchema() { when(veniceSubSchema.unwrap(HoptimatorJdbcSchema.class)).thenReturn(jdbcSchema); when(jdbcSchema.getDataSource()).thenReturn(dataSource); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertEquals(1, deployers.size()); assertInstanceOf(VeniceDeployer.class, deployers.iterator().next()); } @@ -84,7 +86,7 @@ void testReturnsEmptyForNonVeniceDatabase() { // Database name "test" doesn't match "venice" — short-circuits before schema lookup Source source = new Source("test", List.of("TEST", "MyStore"), Collections.emptyMap()); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty()); } @@ -97,14 +99,14 @@ void testReturnsEmptyWhenSchemaNotFound() { doReturn(subSchemaLookup).when(rootSchema).subSchemas(); when(subSchemaLookup.get("UNKNOWN")).thenReturn(null); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty()); } @Test void testReturnsEmptyForNonSourceDeployable() { MaterializedView view = mock(MaterializedView.class); - Collection deployers = provider.deployers(view, connection); + Collection deployers = provider.deployers(view, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty()); } @@ -112,23 +114,23 @@ void testReturnsEmptyForNonSourceDeployable() { void testReturnsEmptyWhenSchemaNameIsNull() { // Source with only a table name (single-element path) — schema() returns null Source source = new Source("venice", List.of("MyStore"), Collections.emptyMap()); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty()); } @Test void testReturnsEmptyWhenDatabaseIsNull() { Source source = new Source(null, List.of("VENICE", "MyStore"), Collections.emptyMap()); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty()); } @Test - void testReturnsEmptyWhenConnectionIsNotHoptimatorConnection() { - // Use a plain Connection mock (not HoptimatorConnection) → should return empty + void testReturnsEmptyWhenDatabaseUnresolvable() { + // A context that can't resolve the database (databaseProperties returns null) yields no deployers. Source source = new Source("venice", List.of("VENICE", "MyStore"), Collections.emptyMap()); - Connection plainConnection = mock(Connection.class); - Collection deployers = provider.deployers(source, plainConnection); + DeploymentContext unresolvable = mock(DeploymentContext.class); + Collection deployers = provider.deployers(source, unresolvable); assertTrue(deployers.isEmpty()); } @@ -148,7 +150,7 @@ void testReturnsDeployerForCaseInsensitiveVeniceDatabase() { when(veniceSubSchema.unwrap(HoptimatorJdbcSchema.class)).thenReturn(jdbcSchema); when(jdbcSchema.getDataSource()).thenReturn(dataSource); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); // Source database is "venice" (lowercase) which equalsIgnoreCase "VENICE" assertEquals(1, deployers.size()); assertInstanceOf(VeniceDeployer.class, deployers.iterator().next()); @@ -164,7 +166,7 @@ void testReturnsEmptyWhenUnwrapThrowsException() { when(subSchemaLookup.get("VENICE")).thenReturn(veniceSubSchema); when(veniceSubSchema.unwrap(HoptimatorJdbcSchema.class)).thenThrow(new RuntimeException("unwrap failed")); - Collection deployers = provider.deployers(source, connection); + Collection deployers = provider.deployers(source, new CalciteDeploymentContext(connection)); assertTrue(deployers.isEmpty()); } } diff --git a/hoptimator-venice/src/test/java/com/linkedin/hoptimator/venice/VeniceDeployerTest.java b/hoptimator-venice/src/test/java/com/linkedin/hoptimator/venice/VeniceDeployerTest.java index 9a16cea9c..e8eb5e1cf 100644 --- a/hoptimator-venice/src/test/java/com/linkedin/hoptimator/venice/VeniceDeployerTest.java +++ b/hoptimator-venice/src/test/java/com/linkedin/hoptimator/venice/VeniceDeployerTest.java @@ -2,6 +2,7 @@ import com.linkedin.hoptimator.Source; import com.linkedin.hoptimator.Validator; +import com.linkedin.hoptimator.jdbc.CalciteDeploymentContext; import com.linkedin.hoptimator.jdbc.HoptimatorConnection; import com.linkedin.venice.client.schema.StoreSchemaFetcher; import com.linkedin.venice.controllerapi.ControllerClient; @@ -11,6 +12,16 @@ import com.linkedin.venice.controllerapi.StoreResponse; import com.linkedin.venice.meta.StoreInfo; import org.apache.avro.Schema; +import org.apache.calcite.jdbc.CalciteConnection; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeSystem; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.impl.AbstractSchema; +import org.apache.calcite.schema.impl.AbstractTable; +import org.apache.calcite.sql.type.SqlTypeFactoryImpl; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.tools.Frameworks; import org.apache.calcite.util.Pair; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -22,9 +33,11 @@ import java.sql.SQLNonTransientException; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Properties; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyString; @@ -75,7 +88,7 @@ private class TestableVeniceDeployer extends VeniceDeployer { TestableVeniceDeployer(Source source, Properties properties, HoptimatorConnection connection, ControllerClient mockClient) { - super(source, properties, connection); + super(source, properties, new CalciteDeploymentContext(connection)); this.mockClient = mockClient; } @@ -290,6 +303,67 @@ public void testSpecifyReturnsEmptyList() throws Exception { assertTrue(deployer.specify().isEmpty()); } + @Test + public void testExistsReturnsTrueWhenStorePresent() throws Exception { + Source source = new Source("venice", List.of("VENICE", TEST_STORE), Collections.emptyMap()); + StoreResponse storeResponse = mock(StoreResponse.class); + when(storeResponse.getStore()).thenReturn(mock(StoreInfo.class)); + when(mockControllerClient.getStore(TEST_STORE)).thenReturn(storeResponse); + + assertTrue(createDeployer(source).exists()); + } + + @Test + public void testExistsReturnsFalseWhenStoreAbsent() throws Exception { + Source source = new Source("venice", List.of("VENICE", TEST_STORE), Collections.emptyMap()); + StoreResponse storeResponse = mock(StoreResponse.class); + when(storeResponse.getStore()).thenReturn(null); + when(mockControllerClient.getStore(TEST_STORE)).thenReturn(storeResponse); + + assertFalse(createDeployer(source).exists()); + } + + @Test + public void testGetKeyPayloadSchemaProducesPayloadFromRowType() throws Exception { + Source source = new Source("venice", List.of("VENICE", TEST_STORE), Collections.emptyMap()); + // On the SQL/Calcite path no Avro is carried; the payload is synthesized from the row type + // resolved through the Calcite catalog. Without a resolved "keys" option (there is no connector + // on this unit-test classpath to supply one), avroKeyPayloadSchema treats the whole row type as + // the payload; the real key/payload split is covered by the Venice integration test. + RelDataTypeFactory factory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); + RelDataType rowType = factory.builder() + .add("id", factory.createSqlType(SqlTypeName.INTEGER)) + .add("name", factory.createSqlType(SqlTypeName.VARCHAR)) + .build(); + SchemaPlus rootSchema = Frameworks.createRootSchema(true); + SchemaPlus veniceSchema = rootSchema.add("VENICE", new AbstractSchema()); + veniceSchema.add(TEST_STORE, new AbstractTable() { + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return rowType; + } + }); + CalciteConnection calciteConnection = mock(CalciteConnection.class); + when(mockConnection.calciteConnection()).thenReturn(calciteConnection); + when(calciteConnection.getRootSchema()).thenReturn(rootSchema); + VeniceDeployer deployer = + new VeniceDeployer(source, properties, new CalciteDeploymentContext(mockConnection)) { + @Override + protected Map resolveKeyOptions() { + // Unit-test seam: no connector on this classpath supplies a "keys" option, and resolving + // through the live ConnectionService would reach out to K8s. Return none so the whole + // row type is treated as the payload. + return Collections.emptyMap(); + } + }; + + Pair keyPayload = deployer.getKeyPayloadSchema(); + + assertNotNull(keyPayload.right, "payload schema"); + assertNotNull(keyPayload.right.getField("id"), "payload field id"); + assertNotNull(keyPayload.right.getField("name"), "payload field name"); + } + // --- validate() tests --- @Test