diff --git a/build.gradle b/build.gradle index 1f4cbbb0b..85eb88b4f 100644 --- a/build.gradle +++ b/build.gradle @@ -153,19 +153,22 @@ allprojects { } afterEvaluate { - for (def task in it.tasks) { - if (task != rootProject.tasks.CopyGitHooksTask) { + it.tasks + .findAll { task -> task != rootProject.tasks.CopyGitHooksTask } + .each { task -> task.dependsOn rootProject.tasks.CopyGitHooksTask } - } } } -// Local Git Hooks cannot be shared, as .git directory is gitignore'd. +def gitHooksDirectory = providers.exec { + commandLine 'git', 'rev-parse', '--git-path', 'hooks' +}.standardOutput.asText.map { it.trim() } + tasks.register('CopyGitHooksTask', Copy) { - println 'Make the git hook available in .git/hooks directory.' + println 'Make the Git hooks available in the repository hooks directory.' from file('scripts/git-hooks') - into file('.git/hooks/') + into gitHooksDirectory } // ============================================================================= diff --git a/integrations/spark/delta-harness/build.gradle b/integrations/spark/delta-harness/build.gradle new file mode 100644 index 000000000..ba6609d4e --- /dev/null +++ b/integrations/spark/delta-harness/build.gradle @@ -0,0 +1,116 @@ +plugins { + id 'openhouse.java-minimal-conventions' + id 'openhouse.maven-publish' + id 'scala' +} + +// The delta-harness behavioral matrix is published as a portable Scala library. This repository runs +// it against the embedded catalog, and the LinkedIn acceptance tests supply a remote environment. +// +// Only the portable scenario/framework sources are published. Env.scala boots the embedded OpenHouse +// server and pulls in its test fixtures, and LocalRunner.scala carries the harness.Main launch class +// that drives it, so both are excluded from the published library and compiled in the local source +// set. + +ext { + icebergVersion = rootProject.ext.iceberg_1_5_version + sparkVersion = '3.5.2' + scalaLibVersion = '2.12.18' +} + +// The embedded boot wiring and the local run loop. They are excluded from the portable library and +// are the whole of the local source set. +def embeddedOnlySources = ['harness/openhouse/Env.scala', 'harness/openhouse/LocalRunner.scala'] + +sourceSets { + main { + scala { + srcDirs = ['src/main/scala'] + exclude embeddedOnlySources + } + } + local { + scala { + srcDirs = ['src/main/scala'] + include embeddedOnlySources + } + compileClasspath += sourceSets.main.output + runtimeClasspath += sourceSets.main.output + } +} + +dependencies { + implementation "org.scala-lang:scala-library:${scalaLibVersion}" + + // The consumer provides the Spark, Iceberg, and OpenHouse runtime. The library jar carries only the + // harness classes. + compileOnly("org.apache.spark:spark-sql_2.12:${sparkVersion}") { + exclude group: 'io.netty' + } + compileOnly("com.linkedin.iceberg:iceberg-spark-runtime-3.5_2.12:${icebergVersion}") { + exclude group: 'io.netty' + } + // Provides com.linkedin.openhouse.javaclient.* (WebClientResponseWithMessageException, etc.). + compileOnly(project(path: ':integrations:spark:spark-3.5:openhouse-spark-3.5-runtime_2.12', configuration: 'shadow')) + + // The catalog regression test builds Plan.cases without starting Spark. Its JVM still loads the + // harness signatures, so the compile-only harness dependencies must be present on the test classpath. + testImplementation("org.apache.spark:spark-sql_2.12:${sparkVersion}") { + exclude group: 'io.netty' + } + testImplementation("com.linkedin.iceberg:iceberg-spark-runtime-3.5_2.12:${icebergVersion}") { + exclude group: 'io.netty' + } + testImplementation( + project(path: ':integrations:spark:spark-3.5:openhouse-spark-3.5-runtime_2.12', + configuration: 'shadow')) + + localImplementation sourceSets.main.output + localImplementation "org.scala-lang:scala-library:${scalaLibVersion}" + localImplementation("org.apache.spark:spark-sql_2.12:${sparkVersion}") { + exclude group: 'io.netty' + } + localImplementation( + project(path: ':integrations:spark:spark-3.5:openhouse-spark-3.5-runtime_2.12', + configuration: 'shadow')) { + exclude group: 'org.apache.commons', module: 'commons-lang3' + } + localImplementation project(':tables-test-fixtures:tables-test-fixtures-iceberg-1.5_2.12') +} + +configurations.localRuntimeClasspath { + exclude group: 'com.linkedin.iceberg', module: 'iceberg-core' + exclude group: 'com.linkedin.iceberg', module: 'iceberg-api' + exclude group: 'com.linkedin.iceberg', module: 'iceberg-common' + exclude group: 'com.linkedin.iceberg', module: 'iceberg-data' + resolutionStrategy.force( + 'com.fasterxml.jackson.core:jackson-annotations:2.15.2', + 'com.fasterxml.jackson.core:jackson-core:2.15.2', + 'com.fasterxml.jackson.core:jackson-databind:2.15.2', + 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.15.2', + 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.15.2', + 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.15.2', + 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.2', + 'com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.15.2', + 'com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.15.2', + 'com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.15.2', + 'com.fasterxml.jackson.module:jackson-module-parameter-names:2.15.2', + 'com.fasterxml.jackson.module:jackson-module-scala_2.12:2.15.2') +} + +tasks.register('runOpenHouse', JavaExec) { + group = 'verification' + description = 'Runs the delta harness against an embedded OpenHouse catalog.' + dependsOn localClasses + classpath = sourceSets.local.runtimeClasspath + mainClass = 'harness.Main' + if (JavaVersion.current() >= JavaVersion.VERSION_1_9) { + jvmArgs( + '--add-opens=java.base/java.nio=ALL-UNNAMED', + '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED', + '--add-opens=java.base/sun.util.calendar=ALL-UNNAMED', + '--add-exports=java.base/sun.util.calendar=ALL-UNNAMED') + } +} + +jar.enabled = true diff --git a/integrations/spark/delta-harness/run-openhouse.sh b/integrations/spark/delta-harness/run-openhouse.sh new file mode 100755 index 000000000..6cda85599 --- /dev/null +++ b/integrations/spark/delta-harness/run-openhouse.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +JAVA_HOME="${JAVA17_HOME:-${JAVA_HOME:-}}" +if [[ -z "$JAVA_HOME" ]]; then + echo "Set JAVA17_HOME or JAVA_HOME to a JDK 17 installation." >&2 + exit 2 +fi +export JAVA_HOME + +JAVA_VERSION="$("$JAVA_HOME/bin/java" -version 2>&1 | sed -n '1s/.*version "\([0-9][0-9]*\).*/\1/p')" +if [[ "$JAVA_VERSION" != "17" ]]; then + echo "The delta harness requires JDK 17; $JAVA_HOME reports Java $JAVA_VERSION." >&2 + exit 2 +fi + +cd "$REPO_ROOT" +if (( $# == 0 )); then + exec ./gradlew --no-daemon \ + :integrations:spark:openhouse-spark-delta-harness_2.12:runOpenHouse +fi + +printf -v FILTERS ' %q' "$@" +exec ./gradlew --no-daemon \ + :integrations:spark:openhouse-spark-delta-harness_2.12:runOpenHouse \ + --args="${FILTERS# }" diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/Env.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/Env.scala new file mode 100644 index 000000000..d75f81de0 --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/Env.scala @@ -0,0 +1,72 @@ +package harness + +import org.apache.spark.sql.SparkSession +import scala.io.Source + +/** + * Embedded environment wiring: boots the in-process OpenHouse server and hands back a SparkSession pointed at its + * catalog. This file is compiled into the `local` source set only, because it pulls in the server test fixtures that + * the published portable library leaves out. + */ +object OpenHouseEnv { + import com.linkedin.openhouse.tablestest.OpenHouseLocalServer + + private def authToken(): String = + Option(getClass.getClassLoader.getResourceAsStream("dummy.token")) + .map(tokenStream => Source.fromInputStream(tokenStream, "UTF-8").mkString.trim) + .getOrElse("default-token") + + private def wireCatalog( + builder: SparkSession.Builder, + name: String, + uri: String, + token: String): SparkSession.Builder = + builder + .config(s"spark.sql.catalog.$name", "org.apache.iceberg.spark.SparkCatalog") + .config( + s"spark.sql.catalog.$name.catalog-impl", + "com.linkedin.openhouse.spark.OpenHouseCatalog") + .config(s"spark.sql.catalog.$name.uri", uri) + .config(s"spark.sql.catalog.$name.cluster", "local-cluster") + .config(s"spark.sql.catalog.$name.auth-token", token) + + def start(): (OpenHouseLocalServer, SparkSession, String, String) = { + // The embedded server uses Hibernate to create its H2 schema. Hibernate owns initialization for this process, so + // classpath SQL initialization stays disabled. + System.setProperty("spring.sql.init.mode", "never") + System.setProperty("spring.jpa.hibernate.ddl-auto", "create-drop") + + val server = new OpenHouseLocalServer() + server.start() + try { + val uri = s"http://localhost:${server.getPort}" + val token = authToken() + + val base = SparkSession.builder() + .appName("delta-harness-openhouse") + .master("local[2]") + .config("spark.sql.extensions", + "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions," + + "com.linkedin.openhouse.spark.extensions.OpenhouseSparkSessionExtensions") + .config("spark.hadoop.fs.defaultFS", "file:///") + .config("spark.sql.session.timeZone", "UTC") + .config("spark.sql.autoBroadcastJoinThreshold", "-1") + .config("spark.driver.bindAddress", "127.0.0.1") + .config("spark.ui.enabled", "false") + + val wired = + Seq("openhouse", "default_iceberg") + .foldLeft(base)(wireCatalog(_, _, uri, token)) + (server, wired.getOrCreate(), uri, token) + } catch { + case startupFailure: Throwable => + try { + server.stop() + } catch { + case cleanupFailure: Throwable => + startupFailure.addSuppressed(cleanupFailure) + } + throw startupFailure + } + } +} diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/Framework.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/Framework.scala new file mode 100644 index 000000000..3d4022e1a --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/Framework.scala @@ -0,0 +1,457 @@ +package harness + +import org.apache.spark.sql.{Row, SparkSession} +import java.math.{BigDecimal => JavaBigDecimal} +import java.net.{ConnectException, SocketException, SocketTimeoutException} +import java.sql.{Date, Timestamp} +import java.time.LocalDate +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter +import java.util.UUID +import java.util.concurrent.atomic.AtomicInteger +import scala.annotation.tailrec +import scala.reflect.{ClassTag, classTag} +import scala.util.control.NonFatal + +// The harness defines typed, reusable table preparations and localized TestCase bodies. Each case gets a fresh table, +// executes its preparation, runs its action and assertions, and drops the table during teardown. + +/** + * One catalog case: the ID that names it, the body that runs it, and its two skip policies. + * `knownBugReason` marks a case the catalog under test is known to fail; `embeddedSkipReason` marks a case the + * embedded local catalog cannot run at all. A runner reports either policy as a skip. + */ +final case class TestCase( + id: String, + run: Ctx => Unit, + knownBugReason: Option[String] = None, + embeddedSkipReason: Option[String] = None +) { + /** The skip reason a known bug produces, phrased so a run log explains the skip. */ + def bugReason: Option[String] = knownBugReason.map(reason => s"bug: $reason") +} + +final case class Ctx(spark: SparkSession, namespace: String, restUri: String = "", restToken: String = "") + +// Minimal REST client to the embedded OpenHouse server (control-plane ops with no SQL surface: lock/unlock). Uses JDK +// 17's java.net.http; auth is the same Bearer token the Spark catalog uses. +object Rest { + import java.net.http.{HttpClient, HttpRequest, HttpResponse} + import java.net.URI + private lazy val client = HttpClient.newHttpClient() + private def base(ctx: Ctx, path: String): HttpRequest.Builder = + HttpRequest.newBuilder(URI.create(ctx.restUri + path)) + .header("Authorization", s"Bearer ${ctx.restToken}") + .header("Content-Type", "application/json") + def post(ctx: Ctx, path: String, body: String): (Int, String) = { + val response = client.send( + base(ctx, path).POST(HttpRequest.BodyPublishers.ofString(body)).build(), + HttpResponse.BodyHandlers.ofString()) + (response.statusCode(), response.body()) + } + def delete(ctx: Ctx, path: String): (Int, String) = { + val response = client.send(base(ctx, path).DELETE().build(), HttpResponse.BodyHandlers.ofString()) + (response.statusCode(), response.body()) + } + def put(ctx: Ctx, path: String, body: String): (Int, String) = { + val response = client.send( + base(ctx, path).PUT(HttpRequest.BodyPublishers.ofString(body)).build(), + HttpResponse.BodyHandlers.ofString()) + (response.statusCode(), response.body()) + } + def get(ctx: Ctx, path: String): (Int, String) = { + val response = client.send(base(ctx, path).GET().build(), HttpResponse.BodyHandlers.ofString()) + (response.statusCode(), response.body()) + } +} + +sealed trait Outcome { def label: String } +object Outcome { + case object Passed extends Outcome { val label = "PASS" } + final case class Failed(cause: Throwable) extends Outcome { + val label = "FAIL" + def retryable: Boolean = Exceptions.isTransient(cause) + def reason: String = s"${Exceptions.root(cause).getClass.getSimpleName}: ${cause.getMessage}" + } + final case class Skipped(reason: String) extends Outcome { val label = "SKIP" } +} + +object Exceptions { + def causeChain(throwable: Throwable): List[Throwable] = { + @tailrec + def collect( + current: Option[Throwable], + seen: Set[Throwable], + collected: List[Throwable] + ): List[Throwable] = + current match { + case Some(cause) if !seen.contains(cause) => + collect(Option(cause.getCause), seen + cause, cause :: collected) + case _ => + collected.reverse + } + + collect(Some(throwable), Set.empty, Nil) + } + + def root(throwable: Throwable): Throwable = causeChain(throwable).last + + /** + * Retries errors positively identified as transient. Other failures remain terminal so data, permission, and + * assertion failures surface on their first attempt. + */ + def isTransient(throwable: Throwable): Boolean = causeChain(throwable).exists { + case _: SocketTimeoutException => true + case _: ConnectException => true + case socketFailure: SocketException => + Option(socketFailure.getMessage).exists(_.toLowerCase.contains("reset")) + case _ => false + } +} + +// Tests assert with plain `assert`; a failed assertion throws AssertionError, which is NonFatal and so is caught at the +// Runner edge and reported as a (terminal) failure. +object Check { + /** Requires `operation` to throw `E` and returns the exception for message assertions. */ + def intercept[E <: Throwable: ClassTag](operation: => Unit): E = { + val expected = classTag[E].runtimeClass + val caught: Option[Throwable] = + try { + operation + None + } catch { + case NonFatal(throwable) => Some(throwable) + } + caught match { + case Some(throwable) if expected.isInstance(throwable) => + throwable.asInstanceOf[E] + case Some(throwable) => + throw new AssertionError( + s"expected ${expected.getName} but got ${throwable.getClass.getName}: " + + throwable.getMessage, + throwable) + case None => + throw new AssertionError( + s"expected ${expected.getName} to be thrown, but nothing was") + } + } +} + +// `Column[T]` carries the Scala type the column reads back as, so typed row access (`row.get(CoreTable.long0): Long`) +// is compiler-checked. `literalAt(rowIndex)` is a pure function of the row index, so generated data is reproducible. +// Value generation lives on the column, which keeps RowGenerator a plain iteration with no knowledge of types. +final case class Column[T](columnName: String, sqlType: String, literalAt: Int => String) + +sealed trait Schema { + def tableColumns: Seq[Column[_]] + def columnNames: Seq[String] = tableColumns.map(_.columnName) +} + +/** Typed row access, keyed by the column's name: `row.get(CoreTable.long0)` returns a `Long`. */ +object Rows { + implicit class TypedRow(val row: Row) extends AnyVal { + def get[T](column: Column[T]): T = row.getAs[T](column.columnName) + } +} + +// A representative core table with one column per common data type and a string-encoded date. Tests reference columns +// through these handles, so a column rename propagates to every caller. +object CoreTable extends Schema { + val long0: Column[Long] = Column("foo_col_long", "bigint", rowIndex => rowIndex.toString) + val int0: Column[Int] = Column("foo_col_int", "int", rowIndex => rowIndex.toString) + val string0: Column[String] = Column("foo_col_string", "string", rowIndex => s"'row-$rowIndex'") + val double0: Column[Double] = Column("foo_col_double", "double", rowIndex => s"$rowIndex.5") + val boolean0: Column[Boolean] = + Column("foo_col_boolean", "boolean", rowIndex => if (rowIndex % 2 == 0) "true" else "false") + val date0: Column[String] = + Column("foo_col_date", "string", rowIndex => s"'${CoreTable.dateLiteral(rowIndex)}'") + def tableColumns: Seq[Column[_]] = Seq(long0, int0, string0, double0, boolean0, date0) + + private val DateFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH") + private val DateEpoch = LocalDateTime.of(2024, 1, 1, 0, 0) + + /** Deterministic YYYY-MM-DD-HH date value (one hour per row), formatted via java.time. */ + def dateLiteral(rowIndex: Int): String = + DateEpoch.plusHours((rowIndex - 1).toLong).format(DateFormat) +} + +// A schema exercising complex/nested types: a struct, an array, a map, and a struct-in-struct. Struct/array read back +// as Row/Seq; map as a Map. `id` is first so it is the ordering key. +object NestedTable extends Schema { + val id: Column[Long] = + Column("id", "bigint", rowIndex => rowIndex.toString) + val s: Column[Row] = + Column( + "s", + "struct", + rowIndex => s"named_struct('x', $rowIndex, 'y', 'row-$rowIndex')") + val arr: Column[Seq[Int]] = + Column("arr", "array", rowIndex => s"array($rowIndex, ${rowIndex + 1})") + val m: Column[Map[String, Int]] = + Column("m", "map", rowIndex => s"map('k', $rowIndex)") + val nested: Column[Row] = + Column( + "nested", + "struct>", + rowIndex => s"named_struct('inner', named_struct('z', $rowIndex))") + def tableColumns: Seq[Column[_]] = Seq(id, s, arr, m, nested) + + val columnDefinitions: String = + "id bigint, s struct, arr array, m map, nested struct>" +} + +// A schema for type-edge coverage: the common scalar types, exercised with nulls, special float values, boundary +// values, and unicode/empty strings. +object TypesTable extends Schema { + val id: Column[Long] = + Column("id", "bigint", rowIndex => rowIndex.toString) + val n: Column[Int] = + Column("n", "int", rowIndex => rowIndex.toString) + val x: Column[Double] = + Column("x", "double", rowIndex => s"$rowIndex.5") + val dec: Column[JavaBigDecimal] = + Column("dec", "decimal(10,2)", rowIndex => s"CAST($rowIndex.50 AS decimal(10,2))") + val str: Column[String] = + Column("str", "string", rowIndex => s"'row-$rowIndex'") + val bin: Column[Array[Byte]] = + Column("bin", "binary", rowIndex => s"CAST('bin-$rowIndex' AS binary)") + val dt: Column[Date] = + Column( + "dt", + "date", + rowIndex => s"DATE '${DateEpoch.plusDays((rowIndex - 1).toLong)}'") + val ts: Column[Timestamp] = + Column( + "ts", + "timestamp", + rowIndex => + s"TIMESTAMP '${TimestampEpoch.plusHours((rowIndex - 1).toLong).format(TimestampFormat)}'") + val tsntz: Column[LocalDateTime] = + Column( + "tsntz", + "timestamp_ntz", + rowIndex => + s"TIMESTAMP_NTZ '${TimestampEpoch.plusHours((rowIndex - 1).toLong).format(TimestampFormat)}'") + def tableColumns: Seq[Column[_]] = Seq(id, n, x, dec, str, bin, dt, ts, tsntz) + + val columnDefinitions: String = + "id bigint, n int, x double, dec decimal(10,2), str string, bin binary, dt date, ts timestamp, " + + "tsntz timestamp_ntz" + + private val DateEpoch = LocalDate.of(2024, 1, 1) + private val TimestampEpoch = LocalDateTime.of(2024, 1, 1, 0, 0) + private val TimestampFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") +} + +object RowGenerator { + /** VALUES clause for `numberOfRows` deterministic rows, one literal per column. */ + def valuesClause(schema: Schema, numberOfRows: Int): String = + (1 to numberOfRows).map { rowIndex => + schema.tableColumns.map(column => column.literalAt(rowIndex)).mkString("(", ", ", ")") + }.mkString("VALUES ", ", ", "") +} + +/** + * What a step's validation thunk sees: the live table, its rows before and after the step, and the table's snapshot + * (commit) count before and after, so a test can assert the delta in both data and commits (e.g. "a no-match UPDATE + * still commits exactly one snapshot"). + */ +final case class StepView[S <: Schema]( + spark: SparkSession, + table: String, + schema: S, + before: Seq[Row], + after: Seq[Row], + snapshotsBefore: Long, + snapshotsAfter: Long +) + +final case class TableState(rows: Seq[Row], snapshotCount: Long) + +/** A fresh table after its reusable preparation has completed. */ +final case class PreparedTable[S <: Schema]( + spark: SparkSession, + name: String, + schema: S, + preparedRows: Seq[Row], + preparedSnapshotCount: Long +) { + def rows: Seq[Row] = PreparedTable.currentRows(spark, name, schema) + def snapshotCount: Long = PreparedTable.snapshotCount(spark, name) + def state: TableState = TableState(rows, snapshotCount) +} + +object PreparedTable { + private[harness] def currentRows[S <: Schema](spark: SparkSession, table: String, schema: S): Seq[Row] = { + val columns = schema.columnNames.mkString(", ") + spark.sql(s"SELECT $columns FROM $table ORDER BY ${schema.columnNames.head}").collect().toSeq + } + + private[harness] def snapshotCount(spark: SparkSession, table: String): Long = + spark.sql(s"SELECT count(*) FROM $table.snapshots").collect()(0).getLong(0) +} + +/** One preparation step and its validation. */ +final case class Step[S <: Schema]( + label: String, + execute: (SparkSession, String, S) => Unit, + validate: StepView[S] => Unit +) + +/** An immutable, typed sequence of table-preparation steps. */ +final class TableTest[S <: Schema] private (val schema: S, val steps: Vector[Step[S]]) { + private def add(step: Step[S]): TableTest[S] = new TableTest(schema, steps :+ step) + + // The default validator asserts the seed actually appended `numberOfRows` rows. This defends the localized assertions + // from a vacuous pass on an empty or short baseline. + def insert(numberOfRows: Int)( + validate: StepView[S] => Unit = view => assert( + view.after.size == view.before.size + numberOfRows, + s"seed insert($numberOfRows) expected ${view.before.size + numberOfRows} rows, got ${view.after.size}") + ): TableTest[S] = + add(Step(s"insert($numberOfRows)", (spark, table, schema) => + spark.sql(s"INSERT INTO $table ${RowGenerator.valuesClause(schema, numberOfRows)}"), validate)) + + /** Run an arbitrary preparation step, then validate its result. */ + def step(label: String)(mutate: (SparkSession, String) => Unit) + (validate: StepView[S] => Unit = _ => ()): TableTest[S] = + add(Step(label, (spark, table, _) => mutate(spark, table), validate)) + + /** Run one preparation SQL statement, then validate its result. */ + def sql(label: String)(statement: String => String) + (validate: StepView[S] => Unit = _ => ()): TableTest[S] = + step(label)((spark, table) => spark.sql(statement(table)))(validate) + + /** + * Execute these steps as a reusable preparation, then hand the prepared table to one localized test body. The + * fresh-table lifecycle covers both the preparation and the test body. + */ + def prepare(ctx: Ctx)(use: PreparedTable[S] => Unit): Unit = + withTable(ctx) { (table, markTableCreated) => + val (preparedRows, preparedSnapshotCount) = + steps.zipWithIndex.foldLeft((Seq.empty[Row], 0L)) { + case ((beforeRows, beforeSnapshots), (step, stepIndex)) => + step.execute(ctx.spark, table, schema) + if (stepIndex == 0) { + markTableCreated() + } + val afterRows = PreparedTable.currentRows(ctx.spark, table, schema) + val afterSnapshots = PreparedTable.snapshotCount(ctx.spark, table) + step.validate( + StepView( + ctx.spark, + table, + schema, + beforeRows, + afterRows, + beforeSnapshots, + afterSnapshots)) + (afterRows, afterSnapshots) + } + use(PreparedTable(ctx.spark, table, schema, preparedRows, preparedSnapshotCount)) + } + + // Gives the preparation a unique table name and drops that table after the test. Cleanup starts only after the first + // preparation step creates the table, so a name conflict preserves the pre-existing table. A test failure stays + // primary, and a cleanup failure is attached to it as a suppressed exception. + private def withTable(ctx: Ctx)(use: (String, () => Unit) => Unit): Unit = { + val table = TableTest.nextQualifiedTableName(ctx.namespace) + OwnedTableLifecycle.withOwnership( + ctx.spark.sql(s"DROP TABLE IF EXISTS $table"))( + markTableCreated => use(table, markTableCreated)) + } + +} + +private[harness] object OwnedTableLifecycle { + /** + * Runs `use`, then runs `cleanUp` on every outcome. A failure from `use` is the failure the caller sees, with a + * cleanup failure attached to it as a suppressed exception. When `use` returns normally a cleanup failure is the + * failure the caller sees, so cleanup that silently fails cannot pass for a clean run. + */ + def withCleanup(cleanUp: => Unit)(use: => Unit): Unit = { + var primaryFailure: Option[Throwable] = None + try { + use + } catch { + case failure: Throwable => + primaryFailure = Some(failure) + throw failure + } finally { + try { + cleanUp + } catch { + case cleanupFailure: Throwable => + primaryFailure match { + case Some(failure) => failure.addSuppressed(cleanupFailure) + case None => throw cleanupFailure + } + } + } + } + + /** + * Runs `use` with a mark it calls once the table exists. `dropOwnedTable` runs only when that mark was set, so a + * create that fails leaves whatever already answered to the name untouched. + */ + def withOwnership(dropOwnedTable: => Unit)(use: (() => Unit) => Unit): Unit = { + var tableCreated = false + withCleanup(if (tableCreated) dropOwnedTable)(use(() => tableCreated = true)) + } +} + +object TableTest { + private val counter = new AtomicInteger(0) + + def apply[S <: Schema](schema: S): TableTest[S] = new TableTest(schema, Vector.empty) + def seedCounter(value: Int): Unit = counter.set(value) + + private[harness] def nextQualifiedTableName(namespace: String): String = + s"$namespace.t_${UUID.randomUUID().toString.replace("-", "")}_${counter.incrementAndGet()}" +} + +/** An immutable recipe that prepares one fresh table for each localized test case. */ +final case class TablePreparation[S <: Schema]( + label: String, + preparation: TableTest[S], + casePrefix: String = "", + afterTest: PreparedTable[S] => Unit = (_: PreparedTable[S]) => () +) { + /** + * Build the case that runs `body` against one freshly prepared table. The case ID combines the preparation's prefix + * and label with `caseName`, so one test body yields a separate case on every preparation it runs on. + */ + def test(caseName: String)(body: PreparedTable[S] => Unit): TestCase = + TestCase( + s"$casePrefix$caseName @ $label", + context => preparation.prepare(context) { table => + var testFailure: Option[Throwable] = None + try body(table) + catch { + case failure: Throwable => + testFailure = Some(failure) + throw failure + } finally { + try afterTest(table) + catch { + case afterTestFailure: Throwable => + testFailure match { + case Some(failure) => failure.addSuppressed(afterTestFailure) + case None => throw afterTestFailure + } + } + } + }) +} + +final case class DmlTestCase[S <: Schema]( + id: String, + run: PreparedTable[S] => Unit, + knownBugReason: Option[String] = None +) { + /** Build the case that runs this operation against a table `preparation` produces. */ + def runOn(preparation: TablePreparation[S]): TestCase = + preparation + .test(id)(run) + .copy(knownBugReason = knownBugReason) +} diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/LocalRunner.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/LocalRunner.scala new file mode 100644 index 000000000..c5c5c7db1 --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/LocalRunner.scala @@ -0,0 +1,143 @@ +package harness + +import java.util.concurrent.{Callable, Executors, TimeUnit} +import scala.annotation.tailrec +import scala.util.control.NonFatal + +/** + * The local runner: the `harness.Main` launch class the `runOpenHouse` task starts, plus the retry policy it executes + * each case under. This file is compiled into the `local` source set only, so the published portable library carries + * the catalog and the framework without a run loop of its own. + */ +object Runner { + val MaxAttempts = 3 + + /** Runs a case, retrying only a transient-infrastructure failure. */ + def execute(testCase: TestCase, context: Ctx): (Outcome, Int) = { + @tailrec def attempt(attemptIndex: Int): (Outcome, Int) = { + val outcome = + try { + testCase.run(context.copy(spark = context.spark.newSession())) + Outcome.Passed + } + catch { case NonFatal(throwable) => Outcome.Failed(throwable) } + outcome match { + case failure: Outcome.Failed + if failure.retryable && attemptIndex + 1 < MaxAttempts => + attempt(attemptIndex + 1) + case terminal => + (terminal, attemptIndex + 1) + } + } + attempt(0) + } +} + +object Main { + def main(args: Array[String]): Unit = { + val (server, spark, restUri, restToken) = OpenHouseEnv.start() + var runFailure: Option[Throwable] = None + try { + spark.sparkContext.setLogLevel("ERROR") + val ctx = Ctx(spark, "openhouse.dbMatrix", restUri, restToken) + + // Each command-line argument is an include substring. A case runs when its ID contains every provided substring. + // An empty argument list selects the full catalog. + val filters = args.toList + val cases = ScenarioCatalog.cases.filter(testCase => + filters.forall(testCase.id.contains)) + + val header = + if (filters.isEmpty) { + "all cases" + } else { + s"filter ${filters.mkString(", ")} -> ${cases.size} cases" + } + println(s"\n=== delta-harness :: localized cases @ OpenHouse catalog ($header) ===\n") + + // Each case owns a fresh table. Worker tasks use separate Spark sessions over the shared Spark context, and + // results are printed in catalog order. + val parallelism = sys.env.get("HARNESS_PARALLELISM").map(_.toInt) + .getOrElse(math.max(1, Runtime.getRuntime.availableProcessors())) + println(s"parallelism: $parallelism worker sessions\n") + + def runOne(testCase: TestCase): (TestCase, (Outcome, Int)) = + testCase.embeddedSkipReason + .map(reason => s"embedded limitation: $reason") + .orElse(testCase.bugReason) match { + case Some(reason) => + (testCase, (Outcome.Skipped(reason): Outcome, 0)) + case None => + (testCase, Runner.execute(testCase, ctx)) + } + + val results = + if (parallelism <= 1) { + cases.map(runOne) + } else { + val pool = Executors.newFixedThreadPool(parallelism) + try { + val futures = cases.map(testCase => + pool.submit( + new Callable[(TestCase, (Outcome, Int))] { + def call(): (TestCase, (Outcome, Int)) = runOne(testCase) + })) + futures.map(_.get(60, TimeUnit.MINUTES)) + } finally { + pool.shutdownNow() + } + } + + results.foreach { case (testCase, (outcome, attempts)) => + val note = outcome match { + case failure: Outcome.Failed => + s" (${failure.reason}${if (failure.retryable) " [retryable]" else ""})" + case Outcome.Skipped(reason) => + s" ($reason)" + case Outcome.Passed => + "" + } + println(f"${outcome.label}%-4s ${testCase.id}%-52s try=$attempts$note") + } + + val failed = + results.count { case (_, (outcome, _)) => outcome.isInstanceOf[Outcome.Failed] } + val skipped = + results.count { case (_, (outcome, _)) => outcome.isInstanceOf[Outcome.Skipped] } + val passed = results.size - failed - skipped + println(f"\n$passed passed, $skipped skipped, $failed failed (${results.size} cases)") + + if (failed > 0 || passed == 0) { + throw new AssertionError( + s"delta harness finished with $passed passed, $skipped skipped, and $failed failed cases") + } + } catch { + case failure: Throwable => + runFailure = Some(failure) + throw failure + } finally { + val cleanupFailures = + List[() => Unit]( + () => spark.stop(), + () => server.stop()) + .flatMap { cleanup => + try { + cleanup() + None + } catch { + case failure: Throwable => Some(failure) + } + } + + runFailure match { + case Some(failure) => + cleanupFailures.foreach(failure.addSuppressed) + case None => + cleanupFailures.headOption.foreach { failure => + cleanupFailures.drop(1).foreach(failure.addSuppressed) + throw failure + } + } + } + } +} diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ChangelogSupport.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ChangelogSupport.scala new file mode 100644 index 000000000..eb26ec953 --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ChangelogSupport.scala @@ -0,0 +1,114 @@ +package harness + +/** + * One changelog operation: the name its case carries, the statement it runs against the prepared table, and the + * change-type histogram the changelog view reports for the snapshot range that statement opened. + */ +final case class ChangelogOperation( + name: String, + statement: String => String, + expectedChangeCounts: Map[String, Long] +) + +/** + * Reusable changelog support for capability layers. It contributes zero catalog cases while holding the row-level + * operations whose change feed `create_changelog_view` reports and the factory that turns those operations into cases + * on preparations a caller supplies. + * + * A feature layer that needs changelog signal mixes this trait in and crosses `changelogOperations` with its own + * preparations. The replace-table layer uses it to require rejection when a changelog range crosses a table + * replacement. The follow-up standard changelog scenario builds on the same operation definitions. + */ +trait ChangelogSupport extends ScenarioKit { + + /** + * The five row-level operations whose change feed the catalog reports: an append, an INSERT OVERWRITE that drops one + * row, a row-level DELETE, an UPDATE, and a MERGE that updates one row and inserts another. Every one starts from + * the standard three-row seed, so its expected histogram holds on any preparation that seeds those rows. + */ + lazy val changelogOperations: List[ChangelogOperation] = + List( + ChangelogOperation( + "changelog.append", + table => + s"INSERT INTO $table VALUES " + + "(CAST(6 AS BIGINT), 6, 'row-6', 6.5, true, '2024-01-06-05')", + Map("INSERT" -> 1L)), + ChangelogOperation( + "changelog.overwrite", + table => + s"INSERT OVERWRITE $table SELECT * FROM $table " + + s"WHERE ${Core.long0.columnName} <= 2", + Map("DELETE" -> 1L)), + ChangelogOperation( + "changelog.delete", + table => s"DELETE FROM $table WHERE ${Core.long0.columnName} = 1", + Map("DELETE" -> 1L)), + ChangelogOperation( + "changelog.update", + table => + s"UPDATE $table SET ${Core.string0.columnName} = 'upd' " + + s"WHERE ${Core.long0.columnName} = 2", + Map("DELETE" -> 1L, "INSERT" -> 1L)), + ChangelogOperation( + "changelog.merge", + table => + s"MERGE INTO $table target " + + "USING (SELECT CAST(2 AS BIGINT) key " + + "UNION ALL SELECT CAST(9 AS BIGINT)) source " + + s"ON target.${Core.long0.columnName} = source.key " + + s"WHEN MATCHED THEN UPDATE SET ${Core.string0.columnName} = 'm' " + + "WHEN NOT MATCHED THEN INSERT " + + s"(${Core.long0.columnName}, ${Core.int0.columnName}, " + + s"${Core.string0.columnName}, ${Core.double0.columnName}, " + + s"${Core.boolean0.columnName}, ${Core.date0.columnName}) " + + "VALUES (source.key, 9, 'row-9', 9.5, true, '2024-01-09-01')", + Map("DELETE" -> 1L, "INSERT" -> 2L))) + + /** The changelog cases for every operation on every preparation given, one preparation at a time. */ + def changelogOperationCasesFor( + preparations: List[TablePreparation[CoreTable.type]] + ): List[TestCase] = + preparations.flatMap(preparation => + changelogOperations.map(operation => changelogOperationCase(preparation, operation))) + + /** The change-type histogram the named changelog view reports. */ + def changeCounts(table: PreparedTable[CoreTable.type], view: String): Map[String, Long] = + table.spark + .sql(s"SELECT _change_type, count(*) FROM $view GROUP BY _change_type") + .collect() + .map(row => row.getString(0) -> row.getLong(1)) + .toMap + + /** The name of a changelog view over `table`, opened at `startSnapshotId`. */ + def changelogViewFrom( + table: PreparedTable[CoreTable.type], + startSnapshotId: Long): String = + table.spark + .sql( + "CALL openhouse.system.create_changelog_view(" + + s"table => '${catalogRelative(table.name)}', " + + s"options => map('start-snapshot-id', '$startSnapshotId'))") + .collect()(0) + .getString(0) + + // --- the case body the surface above composes --- + + /** + * Running the operation against a seeded table and opening a changelog view at the seed snapshot reports exactly the + * change types and counts that operation is defined to produce. + */ + private def changelogOperationCase( + preparation: TablePreparation[CoreTable.type], + operation: ChangelogOperation): TestCase = + preparation.test(operation.name) { table => + val seedSnapshotId = snapshotIds(table.spark, table.name).head + table.spark.sql(operation.statement(table.name)) + val actualChangeCounts = changeCounts(table, changelogViewFrom(table, seedSnapshotId)) + + assert( + actualChangeCounts == operation.expectedChangeCounts, + s"${operation.name} reported $actualChangeCounts, expected ${operation.expectedChangeCounts}") + } + +} diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ConcurrencySupport.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ConcurrencySupport.scala new file mode 100644 index 000000000..4c975e36f --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ConcurrencySupport.scala @@ -0,0 +1,75 @@ +package harness + +import java.util.concurrent.{ConcurrentLinkedQueue, CountDownLatch, TimeUnit} + +/** + * Reusable concurrency support for racing-writer cases. It contributes zero catalog cases while exposing two + * primitives: concurrent function execution and explicit classification of typed commit conflicts. + * + * Both primitives are feature neutral and free of table state, so a feature layer reuses them for its own table mode. + * The replace-table layer uses them to prove that a replacement racing an append either commits or fails with a typed + * commit conflict. The general standard concurrency cases live in a follow-up scenario. + */ +object ConcurrencySupport { + + /** How long `runConcurrently` waits for every thread before it reports the stragglers as failures. */ + val completionTimeoutMinutes: Long = 3 + + /** + * Runs every function on its own daemon thread, releases them together, and waits up to + * `completionTimeoutMinutes` for all of them. Returns the throwables the threads raised, plus one for each thread + * still running at the deadline. A caller that expects conflicts catches them inside its own function, so a + * non-empty result always means a thread failed outside the operation under test. + */ + def runConcurrently(functions: Seq[() => Unit]): Seq[Throwable] = { + val errors = new ConcurrentLinkedQueue[Throwable]() + val start = new CountDownLatch(1) + val threads = functions.zipWithIndex.map { case (function, index) => + val thread = new Thread( + () => + try { + start.await() + function() + } catch { + case interrupted: InterruptedException => + Thread.currentThread().interrupt() + errors.add(interrupted) + case throwable: Throwable => + errors.add(throwable) + }, + s"delta-harness-concurrent-$index") + thread.setDaemon(true) + thread + } + threads.foreach(_.start()) + start.countDown() + + val deadline = System.nanoTime() + TimeUnit.MINUTES.toNanos(completionTimeoutMinutes) + threads.foreach { thread => + val remainingNanos = deadline - System.nanoTime() + if (remainingNanos > 0) { + TimeUnit.NANOSECONDS.timedJoin(thread, remainingNanos) + } + } + + threads.filter(_.isAlive).foreach { thread => + errors.add( + new AssertionError( + s"${thread.getName} did not complete within $completionTimeoutMinutes minutes")) + thread.interrupt() + } + errors.toArray(Array.empty[Throwable]).toSeq + } + + /** A commit conflict the catalog reports through one of its typed commit, validation or transport exceptions. */ + def isTypedCommitConflict(throwable: Throwable): Boolean = + Exceptions.causeChain(throwable).exists { cause => + val className = cause.getClass.getName + className.contains("CommitFailed") || + className.contains("CommitStateUnknown") || + className.contains("Validation") || + className.contains("BadRequest") || + className.contains("WebClientResponse") + } + +} diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioCatalog.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioCatalog.scala new file mode 100644 index 000000000..1a69bd8a0 --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioCatalog.scala @@ -0,0 +1,108 @@ +package harness + +/** + * The single integration file: the object that mixes every scenario in, the ordered catalog built from it, and the + * source entry points earlier consumers were written against. + * + * Adding a scenario is two lines here: one mixin on `Scenarios` and one named entry in + * `ScenarioCatalog.extensionContributions`. Explicit registration makes this file the complete catalog definition. + * + * Every capability trait and every support trait, mixed into one object. + * + * Mixing them here puts ScenarioKit first in the linearization, so its vals initialize before any capability's. This + * object is what a scenario body, a preparation list and the harness configuration are read from, so it exposes the + * shared kit surface (`dataSource`, `fileFormats`, the layout and preparation lists) alongside each capability's case + * list. + * + * Support traits expose reusable operations to later feature layers while the standard matrix includes only the + * selected scenario contributions. + */ +object Scenarios + extends ScenarioDataType + with ScenarioDml + with ScenarioDmlValidation + with ScenarioFileFormat + with ScenarioNestedType + with ScenarioPartitionEvolution + with ScenarioSchemaEvolution + with ScenarioTableProperty + with ScenarioRtas + with ScenarioMergeOnRead + with ChangelogSupport + +/** + * The ordered catalog of scenario-owned test cases. + * + * The catalog is built from two explicit lists. `foundationContributions` is the reusable DDL and DML base this + * branch froze; `extensionContributions` is where a later layer names the capabilities it adds. `contributions` + * merges the two and sorts by contribution name, giving every layer a deterministic order independent of list + * placement. + * + * A layer adds a capability through two append points: one mixin on `Scenarios` and one entry in + * `extensionContributions`. It keeps its behavior and assertions in its own scenario source while the framework and + * shared kit remain stable. + * + * Composition is all this object does: a scenario body, a preparation and a case ID all belong to the capability that + * owns them. + */ +object ScenarioCatalog { + + /** The reusable DDL and DML capabilities in the foundation, named once in alphabetical order. */ + def foundationContributions: List[(String, List[TestCase])] = + List( + "dataTypeCases" -> Scenarios.dataTypeCases, + "dmlCases" -> Scenarios.dmlCases, + "dmlValidationCases" -> Scenarios.dmlValidationCases, + "fileFormatCases" -> Scenarios.fileFormatCases, + "nestedTypeCases" -> Scenarios.nestedTypeCases, + "partitionEvolutionCases" -> Scenarios.partitionEvolutionCases, + "schemaEvolutionCases" -> Scenarios.schemaEvolutionCases, + "tablePropertyCases" -> Scenarios.tablePropertyCases) + + /** + * The capabilities this layer adds on top of the foundation, named the same way. This branch adds the replace-table + * capability, so a sibling layer appends its own entry here and leaves the foundation list and every entry below + * this one untouched. + */ + def extensionContributions: List[(String, List[TestCase])] = + List( + "mergeOnReadCases" -> Scenarios.mergeOnReadCases, + "rtasCases" -> Scenarios.rtasCases) + + /** Every capability contribution, named once, in the order the catalog integrates them. */ + def contributions: List[(String, List[TestCase])] = + (foundationContributions ++ extensionContributions).sortBy { case (name, _) => name } + + /** The deterministic ordered case catalog. */ + def cases: List[TestCase] = contributions.flatMap { case (_, contribution) => contribution } + + /** The case IDs in catalog order. Reading them is a Spark-free catalog operation. */ + def caseIds: List[String] = cases.map(_.id) + +} + +/** + * The entry point earlier consumers were written against. It preserves the source contract for `Plan.Case`, + * `Plan.cases`, `Plan.caseIds` and `Plan.bugReason`. + * + * This stateless facade forwards every member to `ScenarioCatalog` or to the case itself, keeping one catalog state. + * New code inside the harness reads `ScenarioCatalog` and `TestCase` directly. + */ +object Plan { + + /** The case type, which the harness now declares as `TestCase`. */ + type Case = TestCase + + /** The case constructor and extractor, so `Plan.Case(...)` still builds and matches a case. */ + val Case: TestCase.type = TestCase + + /** The deterministic ordered case catalog. */ + def cases: List[TestCase] = ScenarioCatalog.cases + + /** The case IDs in catalog order. */ + def caseIds: List[String] = ScenarioCatalog.caseIds + + /** The skip reason a known bug produces, phrased so a run log explains the skip. */ + def bugReason(testCase: TestCase): Option[String] = testCase.bugReason + +} diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioDataType.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioDataType.scala new file mode 100644 index 000000000..453d50196 --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioDataType.scala @@ -0,0 +1,161 @@ +package harness + +import java.math.BigDecimal + +/** + * Scalar data types: how the long, int, double, decimal, string, binary, date, timestamp and timestamp_ntz columns + * round trip, and how the catalog stores the edge values of each one. + * + * Operations: a round trip of the seeded long, int, double, decimal and string values; an INSERT of an all-null row; + * an INSERT of the special double values NaN and Infinity; an INSERT at the long, int and decimal boundaries; and an + * INSERT of a unicode string and an empty string. + * + * Preparation axes: one unpartitioned TypesTable layout per file format, each seeded with three rows covering every + * scalar column. + * + * Case families: five families over two layouts, contributing 10 cases. + */ +trait ScenarioDataType extends ScenarioKit { + + /** Every scalar-type case, one layout at a time. */ + lazy val dataTypeCases: List[TestCase] = + preparedTypesTables.flatMap(preparation => + List( + roundtripCase(preparation), + nullsCase(preparation), + specialFloatsCase(preparation), + boundariesCase(preparation), + unicodeAndEmptyCase(preparation))) + + /** One unpartitioned scalar-type table per file format. */ + lazy val typesLayouts: List[Layout] = + fileFormats.map(format => + Layout( + s"types-unpartitioned/$format", + table => + s"CREATE TABLE $table (${TypesTable.columnDefinitions}) USING $dataSource " + + s"TBLPROPERTIES ('write.format.default'='$format')")) + + /** One preparation per scalar-type layout: the table is created, then seeded with three fully valued rows. */ + lazy val preparedTypesTables: List[TablePreparation[TypesTable.type]] = + typesLayouts.map(layout => + TablePreparation( + layout.label, + TableTest(TypesTable).sql("create")(layout.create)().insert(standardSeedRowCount)())) + + // --- the preparations, shared helpers and case bodies the surface above composes --- + + // A fully valued TypesTable row with the given id; each case supplies the columns it is about. + private def typesRow(id: Long, n: String, x: String, dec: String, str: String): String = + s"(CAST($id AS BIGINT), $n, $x, $dec, $str, CAST('b' AS binary), DATE '2024-01-01', " + + s"TIMESTAMP '2024-01-01 00:00:00', TIMESTAMP_NTZ '2024-01-01 00:00:00')" + + /** + * Selecting id, n, x, dec and str for the first seeded row reads back the exact long, int, double, decimal and + * string values that were seeded. + */ + private def roundtripCase(preparation: TablePreparation[TypesTable.type]): TestCase = + preparation.test("types.roundtrip") { table => + val row = table.spark + .sql( + s"SELECT id, n, x, dec, str FROM ${table.name} WHERE id = 1") + .collect()(0) + + assert( + row.getLong(0) == 1L && + row.getInt(1) == 1 && + row.getDouble(2) == 1.5) + assert( + row.getDecimal(3).compareTo(new BigDecimal("1.50")) == 0) + assert(row.getString(4) == "row-1") + } + + /** + * Inserting a row with every non-key column NULL reads back as null for the int, double, string, timestamp and + * timestamp_ntz columns. + */ + private def nullsCase(preparation: TablePreparation[TypesTable.type]): TestCase = + preparation.test("types.nulls") { table => + table.spark.sql( + s"INSERT INTO ${table.name} VALUES (" + + "CAST(10 AS BIGINT), NULL, NULL, NULL, NULL, " + + "NULL, NULL, NULL, NULL)") + + val row = table.spark + .sql( + s"SELECT n, x, str, ts, tsntz FROM ${table.name} WHERE id = 10") + .collect()(0) + + assert((0 to 4).forall(row.isNullAt)) + } + + /** Inserting rows with double('NaN') and double('Infinity') reads back as NaN and positive infinity respectively. */ + private def specialFloatsCase(preparation: TablePreparation[TypesTable.type]): TestCase = + preparation.test("types.specialFloats") { table => + table.spark.sql( + s"INSERT INTO ${table.name} VALUES " + + s"${typesRow(11, "0", "double('NaN')", "CAST(0 AS decimal(10,2))", "'x'")}, " + + s"${typesRow(12, "0", "double('Infinity')", "CAST(0 AS decimal(10,2))", "'y'")}") + + assert( + table.spark + .sql(s"SELECT x FROM ${table.name} WHERE id = 11") + .collect()(0) + .getDouble(0) + .isNaN) + assert( + table.spark + .sql(s"SELECT x FROM ${table.name} WHERE id = 12") + .collect()(0) + .getDouble(0) + .isInfinite) + } + + /** + * Inserting a row at Long.MaxValue, Int.MaxValue and a max-precision decimal reads those boundary values back + * unchanged. + */ + private def boundariesCase(preparation: TablePreparation[TypesTable.type]): TestCase = + preparation.test("types.boundaries") { table => + table.spark.sql( + s"INSERT INTO ${table.name} VALUES " + + typesRow( + Long.MaxValue, + Int.MaxValue.toString, + "0.0", + "CAST(99999999.99 AS decimal(10,2))", + "'max'")) + + val row = table.spark + .sql( + s"SELECT id, n, dec FROM ${table.name} WHERE str = 'max'") + .collect()(0) + + assert( + row.getLong(0) == Long.MaxValue && + row.getInt(1) == Int.MaxValue) + assert( + row.getDecimal(2).compareTo(new BigDecimal("99999999.99")) == 0) + } + + /** Inserting rows with a unicode string and an empty string reads each back unchanged. */ + private def unicodeAndEmptyCase(preparation: TablePreparation[TypesTable.type]): TestCase = + preparation.test("types.unicodeAndEmpty") { table => + table.spark.sql( + s"INSERT INTO ${table.name} VALUES " + + s"${typesRow(13, "0", "0.0", "CAST(0 AS decimal(10,2))", "'\u65e5\u672c\u8a9e \uD83C\uDF89'")}, " + + s"${typesRow(14, "0", "0.0", "CAST(0 AS decimal(10,2))", "''")}") + + assert( + table.spark + .sql(s"SELECT str FROM ${table.name} WHERE id = 13") + .collect()(0) + .getString(0) == "\u65e5\u672c\u8a9e \uD83C\uDF89") + assert( + table.spark + .sql(s"SELECT str FROM ${table.name} WHERE id = 14") + .collect()(0) + .getString(0) == "") + } + +} diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioDml.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioDml.scala new file mode 100644 index 000000000..f11b574a5 --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioDml.scala @@ -0,0 +1,1570 @@ +package harness + +import org.apache.spark.sql.Row +import org.apache.spark.sql.functions.lit + +/** + * Data manipulation on the core table: the reads, deletes, updates, merges, inserts and overwrites the catalog + * supports, and the row and snapshot change each one commits. + * + * Operations: 54 reusable DML operations over the six CoreTable columns (bigint, int, string, double, boolean, and a + * string-encoded date), made up of 2 reads, 14 deletes, 13 updates, 16 merges, 6 inserts or overwrites, 1 null-string + * delete, and 2 partition-scoped overwrites. Each operation covers a distinct SQL or DataFrame form, or a distinct + * observable state change within its family. Every operation is defined once here, so a feature layer covers its own + * table mode by crossing these same definitions with its own preparations. + * + * Preparation axes: ScenarioKit supplies the starting states. Four core layouts cross the two columnar formats with + * partitioned and unpartitioned tables. Two date-partitioned layouts receive the partition-scoped writes. Four + * write-ordered layouts exercise the same catalog under a sort order. Four evolved layouts receive the 29 operations + * that address columns by name. Null-string variants isolate the one operation that requires a null value. + * + * Case families: 536 cases in four families, `coreDmlCases` (208), `partitionedDmlCases` (4), `orderedDmlCases` (208) + * and `evolvedDmlCases` (116). + */ +trait ScenarioDml extends ScenarioKit { + import Rows._ + + /** Every DML case, in preparation order: core, partition-scoped, write-ordered, then evolved. */ + lazy val dmlCases: List[TestCase] = + coreDmlCases ++ partitionedDmlCases ++ orderedDmlCases ++ evolvedDmlCases + + /** + * The reads. They select columns by name and write nothing, so they run on any preparation that starts from the three + * seed rows, including one whose column list has grown past that shape. + */ + lazy val readTestCases: List[DmlTestCase[CoreTable.type]] = List( + readProjection, + readFilter) + + /** + * The DELETE that selects a null string. It applies to a preparation that already holds a row whose string column is + * null, and it removes exactly that row. + */ + lazy val nullStringRowTestCases: List[DmlTestCase[CoreTable.type]] = List( + deleteByNullCondition) + + /** + * The partition-scoped writes. They replace whole partitions, so they apply to a preparation that partitions the + * table, and they cross with the partitioned preparations alone. + */ + lazy val partitionedTableTestCases: List[DmlTestCase[CoreTable.type]] = List( + insertDynamicOverwrite, + overwritePartitions) + + // --- which cases a preparation is compatible with --- + // Compatibility is a property of the starting state, so each list names the states it fits. + + /** Every DML case. Runs on any preparation that starts from three seed rows of the seed shape. */ + lazy val allDmlTestCases: List[DmlTestCase[CoreTable.type]] = + readTestCases ++ + deleteTestCases ++ + updateTestCases ++ + mergeTestCases ++ + insertAndOverwriteTestCases + + /** The row-mutating cases: every DELETE, UPDATE and MERGE. */ + lazy val rowMutationTestCases: List[DmlTestCase[CoreTable.type]] = + deleteTestCases ++ updateTestCases ++ mergeTestCases + + /** + * The cases that address columns by name and never write a whole seed-shaped row, so they run on a preparation whose + * column list has grown beyond the seed rows. + */ + lazy val testCasesCompatibleWithAnAddedColumn: List[DmlTestCase[CoreTable.type]] = + readTestCases ++ deleteTestCases ++ updateTestCases + + /** + * Every DML case, with the partition-predicate DELETE marked as a known bug: the Spark and Iceberg rewrite crashes on + * it when the table carries a write order. + */ + lazy val orderedDmlTestCases: List[DmlTestCase[CoreTable.type]] = + allDmlTestCases.map { + case testCase if testCase == deleteByPartitionPredicate => + testCase.copy(knownBugReason = Some( + "DELETE by partition predicate crashes in the Spark and Iceberg rewrite when the " + + "table has a write order.")) + case testCase => + testCase + } + + // --- standard preparations crossed with the cases they are compatible with --- + + /** + * Every DML case on the core preparations, plus the null-string DELETE on the same preparations extended with a + * null-string row. + */ + lazy val coreDmlCases: List[TestCase] = + preparedCoreTables.flatMap(preparation => allDmlTestCases.map(_.runOn(preparation))) ++ + preparedNullStringCoreTables.flatMap(preparation => + nullStringRowTestCases.map(_.runOn(preparation))) + + /** The partition-scoped writes on the partitioned preparations. */ + lazy val partitionedDmlCases: List[TestCase] = + preparedPartitionedCoreTables.flatMap(preparation => + partitionedTableTestCases.map(_.runOn(preparation))) + + /** Every DML case on the write-ordered preparations, plus the null-string DELETE on their null-string form. */ + lazy val orderedDmlCases: List[TestCase] = + preparedOrderedCoreTables.flatMap(preparation => orderedDmlTestCases.map(_.runOn(preparation))) ++ + preparedNullStringOrderedCoreTables.flatMap(preparation => + nullStringRowTestCases.map(_.runOn(preparation))) + + /** The cases that address columns by name, on the preparations that added a column. */ + lazy val evolvedDmlCases: List[TestCase] = + preparedEvolvedCoreTables.flatMap(preparation => + testCasesCompatibleWithAnAddedColumn.map(_.runOn(preparation))) + + // --- the operations the surface above composes --- + // Each case captures the table state, runs one operation, captures the state again, and asserts the row change and + // the snapshot delta that operation caused. Deltas are relative, so a case holds on any preparation regardless of how + // many snapshots the preparation itself committed. + + /** + * SELECT of foo_col_string alone returns that column for every prepared row in key order and leaves the table state + * unchanged. + */ + private val readProjection: DmlTestCase[CoreTable.type] = + DmlTestCase( + "read.projection", + table => { + val before = table.state + val projected = table.spark + .sql( + s"SELECT ${Core.string0.columnName} FROM ${table.name} " + + s"ORDER BY ${Core.long0.columnName}") + .collect() + .toSeq + .map(_.get(Core.string0)) + val after = table.state + + assert( + projected == before.rows.sortBy(_.get(Core.long0)).map(_.get(Core.string0)), + s"projection returned $projected") + assert(after == before, "a read leaves the rows and the snapshot count unchanged") + }) + + /** + * SELECT with a foo_col_long >= 2 predicate returns exactly the prepared rows whose key is 2 or greater and leaves + * the table state unchanged. + */ + private val readFilter: DmlTestCase[CoreTable.type] = + DmlTestCase( + "read.filter", + table => { + val before = table.state + val selected = table.spark + .sql( + s"SELECT ${Core.long0.columnName} FROM ${table.name} " + + s"WHERE ${Core.long0.columnName} >= 2 ORDER BY ${Core.long0.columnName}") + .collect() + .toSeq + .map(_.get(Core.long0)) + val after = table.state + + assert( + selected == before.rows.map(_.get(Core.long0)).filter(_ >= 2).sorted, + s"filtered read returned $selected") + assert(after == before, "a read leaves the rows and the snapshot count unchanged") + }) + + /** + * DELETE WHERE foo_col_string IS NULL removes exactly the prepared row whose string is null, leaves every other row + * unchanged, and commits one snapshot. + */ + private val deleteByNullCondition: DmlTestCase[CoreTable.type] = + DmlTestCase( + "delete.byNullCondition", + table => { + val before = table.state + + table.spark.sql( + s"DELETE FROM ${table.name} WHERE ${Core.string0.columnName} IS NULL") + val after = table.state + + assert( + after.rows == before.rows.filter(row => Option(row.get(Core.string0)).nonEmpty), + s"rows after the null-condition DELETE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "DELETE by a null condition commits one snapshot") + }) + + /** + * DELETE WHERE foo_col_date = '2024-01-01-00' removes the rows with that date, keeps the rest, and commits one + * snapshot. + */ + private val deleteByPartitionPredicate: DmlTestCase[CoreTable.type] = + DmlTestCase( + "delete.byPartitionPredicate", + table => { + val before = table.state + + table.spark.sql( + s"DELETE FROM ${table.name} WHERE " + + s"${Core.date0.columnName} = '2024-01-01-00'") + val after = table.state + + assert( + after.rows == before.rows.filterNot(_.get(Core.date0) == "2024-01-01-00"), + s"rows after the DELETE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "DELETE by a partition predicate commits one snapshot") + }) + + /** + * DELETE WHERE foo_col_long < 2 removes the rows below key 2, leaves every other row unchanged, and commits one + * snapshot. + */ + private val deleteByPredicate: DmlTestCase[CoreTable.type] = + DmlTestCase( + "delete.byPredicate", + table => { + val before = table.state + + table.spark.sql( + s"DELETE FROM ${table.name} WHERE ${Core.long0.columnName} < 2") + val after = table.state + + assert( + after.rows == before.rows.filterNot(_.get(Core.long0) < 2), + s"rows after the DELETE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "DELETE by a predicate commits one snapshot") + }) + + /** + * DELETE WHERE foo_col_long IN (1, 3) removes keys 1 and 3, leaves every other row exactly as prepared, and commits + * one snapshot. + */ + private val deleteByInList: DmlTestCase[CoreTable.type] = + DmlTestCase( + "delete.byInList", + table => { + val before = table.state + + table.spark.sql( + s"DELETE FROM ${table.name} WHERE ${Core.long0.columnName} IN (1, 3)") + val after = table.state + + assert( + after.rows == before.rows.filterNot(row => Set(1L, 3L)(row.get(Core.long0))), + s"rows after the DELETE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "DELETE by an IN list commits one snapshot") + }) + + /** + * DELETE WHERE foo_col_long IN (subquery yielding 2) removes key 2, leaves every other row unchanged, and commits one + * snapshot. + */ + private val deleteByInSubquery: DmlTestCase[CoreTable.type] = + DmlTestCase( + "delete.byInSubquery", + table => { + val before = table.state + + table.spark.sql( + s"DELETE FROM ${table.name} WHERE ${Core.long0.columnName} IN (" + + "SELECT col1 FROM VALUES (CAST(2 AS BIGINT)) AS s(col1))") + val after = table.state + + assert( + after.rows == before.rows.filterNot(_.get(Core.long0) == 2L), + s"rows after the DELETE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "DELETE by an IN subquery commits one snapshot") + }) + + /** + * DELETE WHERE foo_col_long NOT IN (subquery yielding 2) removes every key other than 2, leaves the row for key 2 + * unchanged, and commits one snapshot. + */ + private val deleteByNotInSubquery: DmlTestCase[CoreTable.type] = + DmlTestCase( + "delete.byNotInSubquery", + table => { + val before = table.state + + table.spark.sql( + s"DELETE FROM ${table.name} WHERE ${Core.long0.columnName} NOT IN (" + + "SELECT col1 FROM VALUES (CAST(2 AS BIGINT)) AS s(col1))") + val after = table.state + + assert( + after.rows == before.rows.filter(_.get(Core.long0) == 2L), + s"rows after the DELETE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "DELETE by a NOT IN subquery commits one snapshot") + }) + + /** + * DELETE WHERE EXISTS (correlated subquery matching foo_col_long = 2) removes key 2, leaves every other row + * unchanged, and commits one snapshot. + */ + private val deleteByExistsSubquery: DmlTestCase[CoreTable.type] = + DmlTestCase( + "delete.byExistsSubquery", + table => { + val before = table.state + + table.spark.sql( + s"DELETE FROM ${table.name} WHERE EXISTS (" + + "SELECT 1 FROM VALUES (CAST(2 AS BIGINT)) AS s(x) " + + s"WHERE s.x = ${Core.long0.columnName})") + val after = table.state + + assert( + after.rows == before.rows.filterNot(_.get(Core.long0) == 2L), + s"rows after the DELETE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "DELETE by an EXISTS subquery commits one snapshot") + }) + + /** + * DELETE WHERE NOT EXISTS (correlated subquery matching foo_col_long = 2) removes every key other than 2, leaves the + * row for key 2 unchanged, and commits one snapshot. + */ + private val deleteByNotExistsSubquery: DmlTestCase[CoreTable.type] = + DmlTestCase( + "delete.byNotExistsSubquery", + table => { + val before = table.state + + table.spark.sql( + s"DELETE FROM ${table.name} WHERE NOT EXISTS (" + + "SELECT 1 FROM VALUES (CAST(2 AS BIGINT)) AS s(x) " + + s"WHERE s.x = ${Core.long0.columnName})") + val after = table.state + + assert( + after.rows == before.rows.filter(_.get(Core.long0) == 2L), + s"rows after the DELETE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "DELETE by a NOT EXISTS subquery commits one snapshot") + }) + + /** + * DELETE WHERE foo_col_long = (scalar subquery yielding 2) removes key 2, leaves every other row unchanged, and + * commits one snapshot. + */ + private val deleteByScalarSubquery: DmlTestCase[CoreTable.type] = + DmlTestCase( + "delete.byScalarSubquery", + table => { + val before = table.state + + table.spark.sql( + s"DELETE FROM ${table.name} WHERE ${Core.long0.columnName} = (" + + "SELECT max(col1) FROM VALUES (CAST(2 AS BIGINT)) AS s(col1))") + val after = table.state + + assert( + after.rows == before.rows.filterNot(_.get(Core.long0) == 2L), + s"rows after the DELETE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "DELETE by a scalar subquery commits one snapshot") + }) + + /** DELETE FROM without a predicate empties the table and commits one snapshot. */ + private val deleteAll: DmlTestCase[CoreTable.type] = + DmlTestCase( + "delete.all", + table => { + val before = table.state + + table.spark.sql(s"DELETE FROM ${table.name}") + val after = table.state + + assert(after.rows.isEmpty, s"rows survived the unconditional DELETE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "an unconditional DELETE commits one snapshot") + }) + + /** DELETE WHERE foo_col_long = 999 matches no row, leaves every row unchanged, and still commits one snapshot. */ + private val deleteNone: DmlTestCase[CoreTable.type] = + DmlTestCase( + "delete.none", + table => { + val before = table.state + + table.spark.sql( + s"DELETE FROM ${table.name} WHERE ${Core.long0.columnName} = 999") + val after = table.state + + assert(after.rows == before.rows, s"a no-match DELETE changed the rows: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a no-match DELETE with a real predicate still commits one snapshot") + }) + + /** + * DELETE FROM AS x WHERE x.foo_col_long < 2 resolves the alias, removes the rows below key 2, and commits one + * snapshot. + */ + private val deleteWithAlias: DmlTestCase[CoreTable.type] = + DmlTestCase( + "delete.withAlias", + table => { + val before = table.state + + table.spark.sql( + s"DELETE FROM ${table.name} AS x WHERE x.${Core.long0.columnName} < 2") + val after = table.state + + assert( + after.rows == before.rows.filterNot(_.get(Core.long0) < 2L), + s"rows after the DELETE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "DELETE through an alias commits one snapshot") + }) + + /** DELETE WHERE false is optimized away: the rows stay as they are and no snapshot is committed. */ + private val deleteWhereFalseNoSnapshot: DmlTestCase[CoreTable.type] = + DmlTestCase( + "delete.whereFalse.noSnapshot", + table => { + val before = table.state + + table.spark.sql(s"DELETE FROM ${table.name} WHERE false") + val after = table.state + + assert(after.rows == before.rows, s"DELETE WHERE false changed the rows: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount, + "DELETE WHERE false must not commit a snapshot") + }) + + /** TRUNCATE TABLE empties the table and commits one snapshot. */ + private val deleteTruncate: DmlTestCase[CoreTable.type] = + DmlTestCase( + "delete.truncate", + table => { + val before = table.state + + table.spark.sql(s"TRUNCATE TABLE ${table.name}") + val after = table.state + + assert(after.rows.isEmpty, s"rows survived TRUNCATE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "TRUNCATE commits one snapshot") + }) + + /** + * DELETE against a snapshot-pinned identifier is rejected with an IllegalArgumentException naming that snapshot, and + * the rows and the snapshot count stay unchanged. + */ + private val deleteAtSnapshotRejected: DmlTestCase[CoreTable.type] = + DmlTestCase( + "delete.atSnapshot.rejected", + table => { + val before = table.state + val snapshotId = table.spark + .sql( + s"SELECT snapshot_id FROM ${table.name}.snapshots " + + "ORDER BY committed_at DESC LIMIT 1") + .collect()(0) + .getLong(0) + + val exception = Check.intercept[IllegalArgumentException]( + table.spark.sql( + s"DELETE FROM ${table.name}.snapshot_id_$snapshotId " + + s"WHERE ${Core.long0.columnName} < 4")) + val after = table.state + + assert( + exception.getMessage == + s"Cannot delete from table at a specific snapshot: $snapshotId", + s"unexpected rejection message: ${exception.getMessage}") + assert(after == before, "a rejected DELETE leaves the rows and the snapshot count unchanged") + }) + + /** + * The DELETE operations. They select rows by column name and write no new row, so they run on any preparation that + * starts from the three seed rows, including one whose column list has grown past that shape. + */ + private val deleteTestCases: List[DmlTestCase[CoreTable.type]] = List( + deleteByPredicate, + deleteByInList, + deleteByInSubquery, + deleteByNotInSubquery, + deleteByExistsSubquery, + deleteByNotExistsSubquery, + deleteByScalarSubquery, + deleteAll, + deleteNone, + deleteByPartitionPredicate, + deleteWithAlias, + deleteWhereFalseNoSnapshot, + deleteTruncate, + deleteAtSnapshotRejected) + + /** + * UPDATE SET foo_col_string = 'X' WHERE foo_col_long = 2 rewrites that column for key 2 only, leaves every other row + * unchanged, and commits one snapshot. + */ + private val updateByPredicate: DmlTestCase[CoreTable.type] = + DmlTestCase( + "update.byPredicate", + table => { + val before = table.state + + table.spark.sql( + s"UPDATE ${table.name} SET ${Core.string0.columnName} = 'X' " + + s"WHERE ${Core.long0.columnName} = 2") + val after = table.state + + assert( + after.rows == before.rows.map(row => + if (row.get(Core.long0) == 2L) withColumnValue(row, Core.string0, "X") else row), + s"rows after the UPDATE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "UPDATE by a predicate commits one snapshot") + }) + + /** + * UPDATE SET foo_col_string = 'Z' without a WHERE clause rewrites that column for every row and commits one snapshot. + */ + private val updateWithoutCondition: DmlTestCase[CoreTable.type] = + DmlTestCase( + "update.withoutCondition", + table => { + val before = table.state + + table.spark.sql( + s"UPDATE ${table.name} SET ${Core.string0.columnName} = 'Z'") + val after = table.state + + assert( + after.rows == before.rows.map(row => withColumnValue(row, Core.string0, "Z")), + s"rows after the UPDATE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "an unconditional UPDATE commits one snapshot") + }) + + /** UPDATE ... WHERE foo_col_long = 99 matches no row, leaves every row unchanged, and still commits one snapshot. */ + private val updateNoMatch: DmlTestCase[CoreTable.type] = + DmlTestCase( + "update.noMatch", + table => { + val before = table.state + + table.spark.sql( + s"UPDATE ${table.name} SET ${Core.string0.columnName} = 'Y' " + + s"WHERE ${Core.long0.columnName} = 99") + val after = table.state + + assert( + after.rows == before.rows, + s"a no-match UPDATE changed the rows: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a no-match UPDATE still commits one snapshot") + }) + + /** UPDATE ... WHERE foo_col_long IN (subquery yielding 2) rewrites key 2 only and commits one snapshot. */ + private val updateByInSubquery: DmlTestCase[CoreTable.type] = + DmlTestCase( + "update.byInSubquery", + table => { + val before = table.state + + table.spark.sql( + s"UPDATE ${table.name} SET ${Core.string0.columnName} = 'X' " + + s"WHERE ${Core.long0.columnName} IN (" + + "SELECT col1 FROM VALUES (CAST(2 AS BIGINT)) AS s(col1))") + val after = table.state + + assert( + after.rows == before.rows.map(row => + if (row.get(Core.long0) == 2L) withColumnValue(row, Core.string0, "X") else row), + s"rows after the UPDATE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "UPDATE by an IN subquery commits one snapshot") + }) + + /** + * UPDATE ... WHERE foo_col_long NOT IN (subquery yielding 2) rewrites every key other than 2 and commits one + * snapshot. + */ + private val updateByNotInSubquery: DmlTestCase[CoreTable.type] = + DmlTestCase( + "update.byNotInSubquery", + table => { + val before = table.state + + table.spark.sql( + s"UPDATE ${table.name} SET ${Core.string0.columnName} = 'X' " + + s"WHERE ${Core.long0.columnName} NOT IN (" + + "SELECT col1 FROM VALUES (CAST(2 AS BIGINT)) AS s(col1))") + val after = table.state + + assert( + after.rows == before.rows.map(row => + if (row.get(Core.long0) == 2L) row else withColumnValue(row, Core.string0, "X")), + s"rows after the UPDATE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "UPDATE by a NOT IN subquery commits one snapshot") + }) + + /** + * UPDATE ... WHERE EXISTS (correlated subquery matching foo_col_long = 2) rewrites key 2 only and commits one + * snapshot. + */ + private val updateByExistsSubquery: DmlTestCase[CoreTable.type] = + DmlTestCase( + "update.byExistsSubquery", + table => { + val before = table.state + + table.spark.sql( + s"UPDATE ${table.name} SET ${Core.string0.columnName} = 'X' " + + "WHERE EXISTS (SELECT 1 FROM VALUES (CAST(2 AS BIGINT)) AS s(x) " + + s"WHERE s.x = ${Core.long0.columnName})") + val after = table.state + + assert( + after.rows == before.rows.map(row => + if (row.get(Core.long0) == 2L) withColumnValue(row, Core.string0, "X") else row), + s"rows after the UPDATE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "UPDATE by an EXISTS subquery commits one snapshot") + }) + + /** + * UPDATE ... WHERE NOT EXISTS (correlated subquery matching foo_col_long = 2) rewrites every key other than 2 and + * commits one snapshot. + */ + private val updateByNotExistsSubquery: DmlTestCase[CoreTable.type] = + DmlTestCase( + "update.byNotExistsSubquery", + table => { + val before = table.state + + table.spark.sql( + s"UPDATE ${table.name} SET ${Core.string0.columnName} = 'X' " + + "WHERE NOT EXISTS (SELECT 1 FROM VALUES (CAST(2 AS BIGINT)) AS s(x) " + + s"WHERE s.x = ${Core.long0.columnName})") + val after = table.state + + assert( + after.rows == before.rows.map(row => + if (row.get(Core.long0) == 2L) row else withColumnValue(row, Core.string0, "X")), + s"rows after the UPDATE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "UPDATE by a NOT EXISTS subquery commits one snapshot") + }) + + /** UPDATE ... WHERE foo_col_long = (scalar subquery yielding 2) rewrites key 2 only and commits one snapshot. */ + private val updateByScalarSubquery: DmlTestCase[CoreTable.type] = + DmlTestCase( + "update.byScalarSubquery", + table => { + val before = table.state + + table.spark.sql( + s"UPDATE ${table.name} SET ${Core.string0.columnName} = 'X' " + + s"WHERE ${Core.long0.columnName} = (" + + "SELECT max(col1) FROM VALUES (CAST(2 AS BIGINT)) AS s(col1))") + val after = table.state + + assert( + after.rows == before.rows.map(row => + if (row.get(Core.long0) == 2L) withColumnValue(row, Core.string0, "X") else row), + s"rows after the UPDATE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "UPDATE by a scalar subquery commits one snapshot") + }) + + /** + * UPDATE
AS x SET x.foo_col_string ... WHERE x.foo_col_long = 2 resolves the alias on both sides, rewrites + * key 2 only, and commits one snapshot. + */ + private val updateWithAlias: DmlTestCase[CoreTable.type] = + DmlTestCase( + "update.withAlias", + table => { + val before = table.state + + table.spark.sql( + s"UPDATE ${table.name} AS x SET x.${Core.string0.columnName} = 'X' " + + s"WHERE x.${Core.long0.columnName} = 2") + val after = table.state + + assert( + after.rows == before.rows.map(row => + if (row.get(Core.long0) == 2L) withColumnValue(row, Core.string0, "X") else row), + s"rows after the UPDATE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "UPDATE through an alias commits one snapshot") + }) + + /** + * UPDATE SET foo_col_string = 'X', foo_col_int = 99 WHERE foo_col_long = 2 rewrites both columns of key 2 in one + * statement and commits one snapshot. + */ + private val updateMultipleColumns: DmlTestCase[CoreTable.type] = + DmlTestCase( + "update.multipleColumns", + table => { + val before = table.state + + table.spark.sql( + s"UPDATE ${table.name} SET ${Core.string0.columnName} = 'X', " + + s"${Core.int0.columnName} = 99 WHERE ${Core.long0.columnName} = 2") + val after = table.state + + assert( + after.rows == before.rows.map(row => + if (row.get(Core.long0) == 2L) { + withColumnValue(withColumnValue(row, Core.string0, "X"), Core.int0, 99) + } else row), + s"rows after the UPDATE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a multi-column UPDATE commits one snapshot") + }) + + /** + * UPDATE SET foo_col_long = foo_col_long + 10 WHERE foo_col_long = 2 moves key 2 to key 12, leaves every other row + * unchanged, and commits one snapshot. + */ + private val updateByExpression: DmlTestCase[CoreTable.type] = + DmlTestCase( + "update.byExpression", + table => { + val before = table.state + + table.spark.sql( + s"UPDATE ${table.name} SET " + + s"${Core.long0.columnName} = ${Core.long0.columnName} + 10 " + + s"WHERE ${Core.long0.columnName} = 2") + val after = table.state + + assert( + after.rows == inKeyOrder(before.rows.map(row => + if (row.get(Core.long0) == 2L) withColumnValue(row, Core.long0, 12L) else row)), + s"rows after the UPDATE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "UPDATE by an expression commits one snapshot") + }) + + /** + * UPDATE SET foo_col_date = '2099-12-31-23' WHERE foo_col_long = 2 moves key 2 to another date partition value, + * leaves every other row unchanged, and commits one snapshot. + */ + private val updateMovePartition: DmlTestCase[CoreTable.type] = + DmlTestCase( + "update.movePartition", + table => { + val before = table.state + + table.spark.sql( + s"UPDATE ${table.name} SET " + + s"${Core.date0.columnName} = '2099-12-31-23' " + + s"WHERE ${Core.long0.columnName} = 2") + val after = table.state + + assert( + after.rows == before.rows.map(row => + if (row.get(Core.long0) == 2L) { + withColumnValue(row, Core.date0, "2099-12-31-23") + } else row), + s"rows after the UPDATE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a partition-moving UPDATE commits one snapshot") + }) + + /** + * UPDATE SET foo_col_string = NULL WHERE foo_col_long = 2 stores a null in that column for key 2 only and commits one + * snapshot. + */ + private val updateNullAssignment: DmlTestCase[CoreTable.type] = + DmlTestCase( + "update.nullAssignment", + table => { + val before = table.state + + table.spark.sql( + s"UPDATE ${table.name} SET ${Core.string0.columnName} = NULL " + + s"WHERE ${Core.long0.columnName} = 2") + val after = table.state + + assert( + after.rows == before.rows.map(row => + if (row.get(Core.long0) == 2L) withColumnValue(row, Core.string0, null) else row), + s"rows after the UPDATE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "an UPDATE assigning null commits one snapshot") + }) + + /** + * The UPDATE operations. They assign columns by name, so they run on any preparation that starts from the three seed + * rows, including one whose column list has grown past that shape. + */ + private val updateTestCases: List[DmlTestCase[CoreTable.type]] = List( + updateByPredicate, + updateWithoutCondition, + updateNoMatch, + updateByInSubquery, + updateByNotInSubquery, + updateByExistsSubquery, + updateByNotExistsSubquery, + updateByScalarSubquery, + updateWithAlias, + updateMultipleColumns, + updateByExpression, + updateMovePartition, + updateNullAssignment) + + /** + * MERGE with only a WHEN NOT MATCHED THEN INSERT * clause appends the two source rows (keys 4 and 5) with every + * source column value, leaves the prepared rows unchanged, and commits one snapshot. + */ + private val mergeInsertNotMatched: DmlTestCase[CoreTable.type] = + DmlTestCase( + "merge.insertNotMatched", + table => { + val before = table.state + + table.spark.sql( + s"""MERGE INTO ${table.name} t USING ( + SELECT * FROM VALUES + (CAST(4 AS BIGINT), 4, 'row-4', 4.5, true, '2024-01-04-03'), + (CAST(5 AS BIGINT), 5, 'row-5', 5.5, false, '2024-01-05-04') + AS s($columnNameList) + ) s ON t.${Core.long0.columnName} = s.${Core.long0.columnName} + WHEN NOT MATCHED THEN INSERT *""") + val after = table.state + + assert( + after.rows == inKeyOrder(before.rows ++ Seq( + Row(4L, 4, "row-4", 4.5, true, "2024-01-04-03"), + Row(5L, 5, "row-5", 5.5, false, "2024-01-05-04"))), + s"rows after the MERGE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a MERGE that inserts commits one snapshot") + }) + + /** + * MERGE with only a WHEN MATCHED THEN UPDATE clause rewrites the matched key 2, leaves the unmatched rows unchanged, + * and commits one snapshot. + */ + private val mergeUpdateMatched: DmlTestCase[CoreTable.type] = + DmlTestCase( + "merge.updateMatched", + table => { + val before = table.state + + table.spark.sql( + s"""MERGE INTO ${table.name} t USING ( + SELECT * FROM VALUES (CAST(2 AS BIGINT), 'M') + AS s(${Core.long0.columnName}, ${Core.string0.columnName}) + ) s ON t.${Core.long0.columnName} = s.${Core.long0.columnName} + WHEN MATCHED THEN UPDATE + SET t.${Core.string0.columnName} = s.${Core.string0.columnName}""") + val after = table.state + + assert( + after.rows == before.rows.map(row => + if (row.get(Core.long0) == 2L) withColumnValue(row, Core.string0, "M") else row), + s"rows after the MERGE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a MERGE that updates commits one snapshot") + }) + + /** + * MERGE with only a WHEN MATCHED THEN DELETE clause removes the matched keys 1 and 3, keeps the unmatched rows, and + * commits one snapshot. + */ + private val mergeDeleteMatched: DmlTestCase[CoreTable.type] = + DmlTestCase( + "merge.deleteMatched", + table => { + val before = table.state + + table.spark.sql( + s"""MERGE INTO ${table.name} t USING ( + SELECT * FROM VALUES (CAST(1 AS BIGINT)), (CAST(3 AS BIGINT)) + AS s(${Core.long0.columnName}) + ) s ON t.${Core.long0.columnName} = s.${Core.long0.columnName} + WHEN MATCHED THEN DELETE""") + val after = table.state + + assert( + after.rows == before.rows.filterNot(row => Set(1L, 3L)(row.get(Core.long0))), + s"rows after the MERGE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a MERGE that deletes commits one snapshot") + }) + + /** + * MERGE with both an UPDATE clause and an INSERT clause rewrites the matched key 2 and appends the unmatched key 7 in + * a single statement, and commits one snapshot. + */ + private val mergeUpsert: DmlTestCase[CoreTable.type] = + DmlTestCase( + "merge.upsert", + table => { + val before = table.state + + table.spark.sql( + s"""MERGE INTO ${table.name} t USING ( + SELECT * FROM VALUES + (CAST(2 AS BIGINT), 2, 'U', 2.5, true, '2024-01-02-01'), + (CAST(7 AS BIGINT), 7, 'g', 7.5, false, '2024-01-07-06') + AS s($columnNameList) + ) s ON t.${Core.long0.columnName} = s.${Core.long0.columnName} + WHEN MATCHED THEN UPDATE + SET t.${Core.string0.columnName} = s.${Core.string0.columnName} + WHEN NOT MATCHED THEN INSERT *""") + val after = table.state + + assert( + after.rows == inKeyOrder( + before.rows.map(row => + if (row.get(Core.long0) == 2L) withColumnValue(row, Core.string0, "U") else row) :+ + Row(7L, 7, "g", 7.5, false, "2024-01-07-06")), + s"rows after the MERGE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "an upsert MERGE commits one snapshot") + }) + + /** + * MERGE with a WHEN NOT MATCHED BY SOURCE THEN DELETE clause removes every row the source does not carry, keeps the + * matched key 2, and commits one snapshot. + */ + private val mergeDeleteNotMatchedBySource: DmlTestCase[CoreTable.type] = + DmlTestCase( + "merge.deleteNotMatchedBySource", + table => { + val before = table.state + + table.spark.sql( + s"""MERGE INTO ${table.name} t USING ( + SELECT * FROM VALUES (CAST(2 AS BIGINT)) + AS s(${Core.long0.columnName}) + ) s ON t.${Core.long0.columnName} = s.${Core.long0.columnName} + WHEN NOT MATCHED BY SOURCE THEN DELETE""") + val after = table.state + + assert( + after.rows == before.rows.filter(_.get(Core.long0) == 2L), + s"rows after the MERGE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a not-matched-by-source MERGE commits one snapshot") + }) + + /** + * MERGE with a WHEN MATCHED AND THEN UPDATE clause rewrites only the matched row that also satisfies the + * condition (key 2), leaves matched key 3 unchanged, and commits one snapshot. + */ + private val mergeConditionalUpdate: DmlTestCase[CoreTable.type] = + DmlTestCase( + "merge.conditionalUpdate", + table => { + val before = table.state + + table.spark.sql( + s"""MERGE INTO ${table.name} t USING ( + SELECT * FROM VALUES (CAST(2 AS BIGINT), 'U2'), + (CAST(3 AS BIGINT), 'U3') + AS s(${Core.long0.columnName}, ${Core.string0.columnName}) + ) s ON t.${Core.long0.columnName} = s.${Core.long0.columnName} + WHEN MATCHED AND s.${Core.long0.columnName} = 2 THEN UPDATE + SET t.${Core.string0.columnName} = s.${Core.string0.columnName}""") + val after = table.state + + assert( + after.rows == before.rows.map(row => + if (row.get(Core.long0) == 2L) withColumnValue(row, Core.string0, "U2") else row), + s"rows after the MERGE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a conditional-update MERGE commits one snapshot") + }) + + /** + * MERGE with two MATCHED clauses applies the first matching clause per row: key 2 is updated by the conditional + * clause and key 3 falls through to the DELETE clause, in one snapshot. + */ + private val mergeMultipleMatchedClauses: DmlTestCase[CoreTable.type] = + DmlTestCase( + "merge.multipleMatchedClauses", + table => { + val before = table.state + + table.spark.sql( + s"""MERGE INTO ${table.name} t USING ( + SELECT * FROM VALUES (CAST(2 AS BIGINT), 'U'), + (CAST(3 AS BIGINT), 'x') + AS s(${Core.long0.columnName}, ${Core.string0.columnName}) + ) s ON t.${Core.long0.columnName} = s.${Core.long0.columnName} + WHEN MATCHED AND s.${Core.long0.columnName} = 2 THEN UPDATE + SET t.${Core.string0.columnName} = s.${Core.string0.columnName} + WHEN MATCHED THEN DELETE""") + val after = table.state + + assert( + after.rows == before.rows + .filterNot(_.get(Core.long0) == 3L) + .map(row => + if (row.get(Core.long0) == 2L) withColumnValue(row, Core.string0, "U") else row), + s"rows after the MERGE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a multi-clause MERGE commits one snapshot") + }) + + /** + * MERGE with a WHEN NOT MATCHED AND THEN INSERT clause appends only the source row that satisfies the + * condition (key 4), skips key 5, and commits one snapshot. + */ + private val mergeConditionalInsert: DmlTestCase[CoreTable.type] = + DmlTestCase( + "merge.conditionalInsert", + table => { + val before = table.state + + table.spark.sql( + s"""MERGE INTO ${table.name} t USING ( + SELECT * FROM VALUES + (CAST(4 AS BIGINT), 4, 'row-4', 4.5, true, '2024-01-04-03'), + (CAST(5 AS BIGINT), 5, 'row-5', 5.5, false, '2024-01-05-04') + AS s($columnNameList) + ) s ON t.${Core.long0.columnName} = s.${Core.long0.columnName} + WHEN NOT MATCHED AND s.${Core.long0.columnName} = 4 THEN INSERT *""") + val after = table.state + + assert( + after.rows == inKeyOrder(before.rows :+ Row(4L, 4, "row-4", 4.5, true, "2024-01-04-03")), + s"rows after the MERGE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a conditional-insert MERGE commits one snapshot") + }) + + /** + * MERGE carrying UPDATE, INSERT, and NOT MATCHED BY SOURCE DELETE clauses updates key 2, inserts key 4, deletes the + * rows the source omits, and commits one snapshot. + */ + private val mergeAllClauses: DmlTestCase[CoreTable.type] = + DmlTestCase( + "merge.allClauses", + table => { + val before = table.state + + table.spark.sql( + s"""MERGE INTO ${table.name} t USING ( + SELECT * FROM VALUES + (CAST(2 AS BIGINT), 2, 'M2', 2.5, true, '2024-01-02-01'), + (CAST(4 AS BIGINT), 4, 'row-4', 4.5, false, '2024-01-04-03') + AS s($columnNameList) + ) s ON t.${Core.long0.columnName} = s.${Core.long0.columnName} + WHEN MATCHED THEN UPDATE + SET t.${Core.string0.columnName} = s.${Core.string0.columnName} + WHEN NOT MATCHED THEN INSERT * + WHEN NOT MATCHED BY SOURCE THEN DELETE""") + val after = table.state + + assert( + after.rows == inKeyOrder( + before.rows + .filter(_.get(Core.long0) == 2L) + .map(row => withColumnValue(row, Core.string0, "M2")) :+ + Row(4L, 4, "row-4", 4.5, false, "2024-01-04-03")), + s"rows after the MERGE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a MERGE with every clause commits one snapshot") + }) + + /** + * MERGE with WHEN MATCHED THEN UPDATE SET * copies every source column onto the matched key 2, leaves the unmatched + * rows unchanged, and commits one snapshot. + */ + private val mergeUpdateStar: DmlTestCase[CoreTable.type] = + DmlTestCase( + "merge.updateStar", + table => { + val before = table.state + + table.spark.sql( + s"""MERGE INTO ${table.name} t USING ( + SELECT * FROM VALUES + (CAST(2 AS BIGINT), 22, 'S2', 22.5, true, '2024-06-06-06') + AS s($columnNameList) + ) s ON t.${Core.long0.columnName} = s.${Core.long0.columnName} + WHEN MATCHED THEN UPDATE SET *""") + val after = table.state + + assert( + after.rows == before.rows.map(row => + if (row.get(Core.long0) == 2L) { + Row(2L, 22, "S2", 22.5, true, "2024-06-06-06") + } else row), + s"rows after the MERGE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "an UPDATE SET * MERGE commits one snapshot") + }) + + /** + * MERGE whose INSERT clause names a column subset appends key 7 with the named values, leaves the unnamed columns + * null, and commits one snapshot. + */ + private val mergeInsertExplicitColumns: DmlTestCase[CoreTable.type] = + DmlTestCase( + "merge.insertExplicitColumns", + table => { + val before = table.state + + table.spark.sql( + s"""MERGE INTO ${table.name} t USING ( + SELECT * FROM VALUES (CAST(7 AS BIGINT), 'g') + AS s(${Core.long0.columnName}, ${Core.string0.columnName}) + ) s ON t.${Core.long0.columnName} = s.${Core.long0.columnName} + WHEN NOT MATCHED THEN + INSERT (${Core.long0.columnName}, ${Core.string0.columnName}) + VALUES (s.${Core.long0.columnName}, s.${Core.string0.columnName})""") + val after = table.state + + assert( + after.rows == inKeyOrder(before.rows :+ Row(7L, null, "g", null, null, null)), + s"rows after the MERGE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "an explicit-column MERGE insert commits one snapshot") + }) + + /** + * MERGE whose source is a common table expression appends the key 8 that CTE yields, with null in every column the + * CTE does not supply, and commits one snapshot. + */ + private val mergeSourceCTE: DmlTestCase[CoreTable.type] = + DmlTestCase( + "merge.sourceCTE", + table => { + val before = table.state + + table.spark.sql( + s"""MERGE INTO ${table.name} t USING ( + WITH src AS ( + SELECT CAST(8 AS BIGINT) AS ${Core.long0.columnName} + ) + SELECT * FROM src + ) s ON t.${Core.long0.columnName} = s.${Core.long0.columnName} + WHEN NOT MATCHED THEN + INSERT (${Core.long0.columnName}) VALUES (s.${Core.long0.columnName})""") + val after = table.state + + assert( + after.rows == inKeyOrder(before.rows :+ Row(8L, null, null, null, null, null)), + s"rows after the MERGE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a MERGE from a CTE source commits one snapshot") + }) + + /** + * MERGE whose source is a UNION ALL appends both keys the set operation yields (8 and 9), with null in every column + * the source does not supply, and commits one snapshot. + */ + private val mergeSourceSetOp: DmlTestCase[CoreTable.type] = + DmlTestCase( + "merge.sourceSetOp", + table => { + val before = table.state + + table.spark.sql( + s"""MERGE INTO ${table.name} t USING ( + SELECT CAST(8 AS BIGINT) AS ${Core.long0.columnName} + UNION ALL + SELECT CAST(9 AS BIGINT) + ) s ON t.${Core.long0.columnName} = s.${Core.long0.columnName} + WHEN NOT MATCHED THEN + INSERT (${Core.long0.columnName}) VALUES (s.${Core.long0.columnName})""") + val after = table.state + + assert( + after.rows == inKeyOrder(before.rows ++ Seq( + Row(8L, null, null, null, null, null), + Row(9L, null, null, null, null, null))), + s"rows after the MERGE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a MERGE from a set-operation source commits one snapshot") + }) + + /** + * After the table is emptied, MERGE with a NOT MATCHED INSERT clause inserts both source rows (keys 4 and 5) into the + * empty target and commits one snapshot. + */ + private val mergeIntoEmptyTarget: DmlTestCase[CoreTable.type] = + DmlTestCase( + "merge.intoEmptyTarget", + table => { + table.spark.sql(s"DELETE FROM ${table.name}") + val before = table.state + + assert(before.rows.isEmpty, s"precondition: the target is empty, got ${before.rows}") + + table.spark.sql( + s"""MERGE INTO ${table.name} t USING ( + SELECT * FROM VALUES + (CAST(4 AS BIGINT), 4, 'row-4', 4.5, true, '2024-01-04-03'), + (CAST(5 AS BIGINT), 5, 'row-5', 5.5, false, '2024-01-05-04') + AS s($columnNameList) + ) s ON t.${Core.long0.columnName} = s.${Core.long0.columnName} + WHEN NOT MATCHED THEN INSERT *""") + val after = table.state + + assert( + after.rows == Seq( + Row(4L, 4, "row-4", 4.5, true, "2024-01-04-03"), + Row(5L, 5, "row-5", 5.5, false, "2024-01-05-04")), + s"rows after the MERGE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a MERGE into an empty target commits one snapshot") + }) + + /** + * MERGE whose source carries a null join key matches no target row on that key: only the non-null key 2 is updated, + * no row is added or removed, and one snapshot is committed. + */ + private val mergeNullJoinKey: DmlTestCase[CoreTable.type] = + DmlTestCase( + "merge.nullJoinKey", + table => { + val before = table.state + + table.spark.sql( + s"""MERGE INTO ${table.name} t USING ( + SELECT * FROM VALUES (CAST(NULL AS BIGINT), 'n'), + (CAST(2 AS BIGINT), 'M') + AS s(${Core.long0.columnName}, ${Core.string0.columnName}) + ) s ON t.${Core.long0.columnName} = s.${Core.long0.columnName} + WHEN MATCHED THEN UPDATE + SET t.${Core.string0.columnName} = s.${Core.string0.columnName}""") + val after = table.state + + assert( + after.rows == before.rows.map(row => + if (row.get(Core.long0) == 2L) withColumnValue(row, Core.string0, "M") else row), + s"rows after the MERGE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a MERGE with a null join key commits one snapshot") + }) + + /** + * MERGE with INSERT * resolves the source columns by name: key 7 lands with every source value in its named column + * when the source lists its columns in another order, and one snapshot is committed. + */ + private val mergeResolveByName: DmlTestCase[CoreTable.type] = + DmlTestCase( + "merge.resolveByName", + table => { + val before = table.state + + table.spark.sql( + s"""MERGE INTO ${table.name} t USING ( + SELECT * FROM VALUES + ('g', CAST(7 AS BIGINT), 7, 7.5, false, '2024-07-07-07') + AS s( + ${Core.string0.columnName}, + ${Core.long0.columnName}, + ${Core.int0.columnName}, + ${Core.double0.columnName}, + ${Core.boolean0.columnName}, + ${Core.date0.columnName}) + ) s ON t.${Core.long0.columnName} = s.${Core.long0.columnName} + WHEN NOT MATCHED THEN INSERT *""") + val after = table.state + + assert( + after.rows == inKeyOrder(before.rows :+ Row(7L, 7, "g", 7.5, false, "2024-07-07-07")), + s"rows after the MERGE: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a name-resolved MERGE insert commits one snapshot") + }) + + /** + * The MERGE operations. Their INSERT clauses write a whole seed-shaped row, so they run on a preparation whose column + * list is still the seed shape. + */ + private val mergeTestCases: List[DmlTestCase[CoreTable.type]] = List( + mergeInsertNotMatched, + mergeUpdateMatched, + mergeDeleteMatched, + mergeUpsert, + mergeDeleteNotMatchedBySource, + mergeConditionalUpdate, + mergeMultipleMatchedClauses, + mergeConditionalInsert, + mergeAllClauses, + mergeUpdateStar, + mergeInsertExplicitColumns, + mergeSourceCTE, + mergeSourceSetOp, + mergeIntoEmptyTarget, + mergeNullJoinKey, + mergeResolveByName) + + /** + * INSERT INTO ... VALUES appends the two literal rows (keys 4 and 5), leaves the prepared rows unchanged, and commits + * one snapshot. + */ + private val insertInto: DmlTestCase[CoreTable.type] = + DmlTestCase( + "insert.into", + table => { + val before = table.state + + table.spark.sql( + s"""INSERT INTO ${table.name} VALUES + (CAST(4 AS BIGINT), 4, 'row-4', 4.5, true, '2024-01-04-03'), + (CAST(5 AS BIGINT), 5, 'row-5', 5.5, false, '2024-01-05-04')""") + val after = table.state + + assert( + after.rows == inKeyOrder(before.rows ++ Seq( + Row(4L, 4, "row-4", 4.5, true, "2024-01-04-03"), + Row(5L, 5, "row-5", 5.5, false, "2024-01-05-04"))), + s"rows after the INSERT: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "INSERT INTO commits one snapshot") + }) + + /** + * INSERT INTO naming a subset of the columns is rejected by the engine with a message naming the omitted data, and + * the rows and the snapshot count stay unchanged. + */ + private val insertExplicitColumns: DmlTestCase[CoreTable.type] = + DmlTestCase( + "insert.explicitColumns", + table => { + val before = table.state + + val exception = Check.intercept[Exception]( + table.spark.sql( + s"INSERT INTO ${table.name} " + + s"(${Core.long0.columnName}, ${Core.string0.columnName}) " + + "VALUES (CAST(4 AS BIGINT), 'd'), (CAST(5 AS BIGINT), 'e')")) + val after = table.state + val exceptionMessage = Option(exception.getMessage).getOrElse("") + + assert( + exceptionMessage.toUpperCase.contains("CANNOT_FIND_DATA") || + exceptionMessage.toUpperCase.contains("CANNOT FIND DATA") || + exceptionMessage.toUpperCase.contains("INCOMPATIBLE_DATA"), + "expected a partial-INSERT rejection naming the omitted column " + + s"(engine limitation), got: ${exceptionMessage.take(200)}") + assert(after == before, "a rejected INSERT leaves the rows and the snapshot count unchanged") + }) + + /** + * INSERT INTO ... SELECT appends the row the SELECT produces (key 6), leaves the prepared rows unchanged, and commits + * one snapshot. + */ + private val insertIntoSelect: DmlTestCase[CoreTable.type] = + DmlTestCase( + "insert.intoSelect", + table => { + val before = table.state + + table.spark.sql( + s"INSERT INTO ${table.name} SELECT * FROM VALUES " + + s"(CAST(6 AS BIGINT), 6, 'row-6', 6.5, true, '2024-01-06-05') " + + s"AS s($columnNameList)") + val after = table.state + + assert( + after.rows == inKeyOrder(before.rows :+ Row(6L, 6, "row-6", 6.5, true, "2024-01-06-05")), + s"rows after the INSERT: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "INSERT INTO ... SELECT commits one snapshot") + }) + + /** + * The DataFrame writeTo(...).append() path appends the frame's row (key 6), keeps the prepared rows, and commits one + * snapshot. + */ + private val appendDataFrame: DmlTestCase[CoreTable.type] = + DmlTestCase( + "append.dataFrame", + table => { + val before = table.state + + table.spark + .sql( + s"SELECT * FROM VALUES " + + s"(CAST(6 AS BIGINT), 6, 'row-6', 6.5, true, '2024-01-06-05') " + + s"AS s($columnNameList)") + .writeTo(table.name) + .append() + val after = table.state + + assert( + after.rows == inKeyOrder(before.rows :+ Row(6L, 6, "row-6", 6.5, true, "2024-01-06-05")), + s"rows after the append: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a DataFrame append commits one snapshot") + }) + + /** + * INSERT OVERWRITE ... VALUES replaces the table contents with the two literal rows (keys 1 and 2) and commits one + * snapshot. + */ + private val insertOverwrite: DmlTestCase[CoreTable.type] = + DmlTestCase( + "insert.overwrite", + table => { + val before = table.state + + table.spark.sql( + s"""INSERT OVERWRITE ${table.name} VALUES + (CAST(1 AS BIGINT), 1, 'p', 1.5, false, '2024-01-01-00'), + (CAST(2 AS BIGINT), 2, 'q', 2.5, true, '2024-01-02-01')""") + val after = table.state + + assert( + after.rows == Seq( + Row(1L, 1, "p", 1.5, false, "2024-01-01-00"), + Row(2L, 2, "q", 2.5, true, "2024-01-02-01")), + s"rows after the overwrite: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "INSERT OVERWRITE commits one snapshot") + }) + + /** + * The DataFrame writeTo(...).overwrite(lit(true)) path replaces every row with the frame's row (key 8) and commits + * one snapshot. + */ + private val overwriteDataFrame: DmlTestCase[CoreTable.type] = + DmlTestCase( + "overwrite.dataFrame", + table => { + val before = table.state + + table.spark + .sql( + s"SELECT * FROM VALUES " + + s"(CAST(8 AS BIGINT), 8, 'h', 8.5, false, '2024-01-08-07') " + + s"AS s($columnNameList)") + .writeTo(table.name) + .overwrite(lit(true)) + val after = table.state + + assert( + after.rows == Seq(Row(8L, 8, "h", 8.5, false, "2024-01-08-07")), + s"rows after the overwrite: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a DataFrame overwrite commits one snapshot") + }) + + /** + * The appends and the overwrites. They write whole seed-shaped rows, so they run on a preparation whose column list + * is still the seed shape. + */ + private val insertAndOverwriteTestCases: List[DmlTestCase[CoreTable.type]] = List( + insertInto, + insertExplicitColumns, + insertIntoSelect, + appendDataFrame, + insertOverwrite, + overwriteDataFrame) + + /** + * Under partitionOverwriteMode=dynamic, INSERT OVERWRITE with one row replaces only that row's partition + * (2024-01-01-00), leaves the rows of every other partition unchanged, and commits one snapshot. + */ + private val insertDynamicOverwrite: DmlTestCase[CoreTable.type] = + DmlTestCase( + "insert.dynamicOverwrite", + table => { + val before = table.state + + table.spark.conf.set("spark.sql.sources.partitionOverwriteMode", "dynamic") + try { + table.spark.sql( + s"INSERT OVERWRITE ${table.name} VALUES " + + "(CAST(10 AS BIGINT), 10, 'p', 10.5, true, '2024-01-01-00')") + } finally { + table.spark.conf.set("spark.sql.sources.partitionOverwriteMode", "static") + } + val after = table.state + + assert( + after.rows == inKeyOrder( + before.rows.filterNot(_.get(Core.date0) == "2024-01-01-00") :+ + Row(10L, 10, "p", 10.5, true, "2024-01-01-00")), + s"rows after the dynamic overwrite: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a dynamic partition overwrite commits one snapshot") + }) + + /** + * The DataFrame writeTo(...).overwritePartitions() path replaces only the partitions the frame carries + * (2024-01-01-00), leaves the rows of every other partition unchanged, and commits one snapshot. + */ + private val overwritePartitions: DmlTestCase[CoreTable.type] = + DmlTestCase( + "overwrite.partitions", + table => { + val before = table.state + + table.spark + .sql( + s"SELECT * FROM VALUES " + + "(CAST(10 AS BIGINT), 10, 'p', 10.5, true, '2024-01-01-00') " + + s"AS s($columnNameList)") + .writeTo(table.name) + .overwritePartitions() + val after = table.state + + assert( + after.rows == inKeyOrder( + before.rows.filterNot(_.get(Core.date0) == "2024-01-01-00") :+ + Row(10L, 10, "p", 10.5, true, "2024-01-01-00")), + s"rows after the partition overwrite: ${after.rows}") + assert( + after.snapshotCount == before.snapshotCount + 1, + "a partition overwrite commits one snapshot") + }) + +} diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioDmlValidation.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioDmlValidation.scala new file mode 100644 index 000000000..0f18ab83e --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioDmlValidation.scala @@ -0,0 +1,131 @@ +package harness + +import org.apache.spark.sql.AnalysisException + +/** + * DML validation: the DML statements the analyzer and the row-level rewrite refuse, and the message each rejection + * carries. + * + * Operations: DELETE on a column the table does not declare, DELETE and UPDATE with a nondeterministic predicate, + * INSERT INTO with fewer values than the table has columns, a MERGE whose UPDATE SET assigns one target column twice, + * and a MERGE whose source matches one target row twice. + * + * Preparation axes: the standard seeded core table in each columnar format. + * + * Case families: six families contributing 12 cases. + */ +trait ScenarioDmlValidation extends ScenarioKit { + + /** Every DML-validation case, one file format at a time. */ + lazy val dmlValidationCases: List[TestCase] = + preparedCoreFormats.flatMap { preparation => + List( + nonExistentColumnCase(preparation), + nonDeterministicDeleteCase(preparation), + nonDeterministicUpdateCase(preparation), + insertArityCase(preparation), + mergeConflictingUpdatesCase(preparation), + mergeCardinalityViolationCase(preparation)) + } + + // --- the preparations, shared helpers and case bodies the surface above composes --- + + /** DELETE with a WHERE clause on a nonexistent column is rejected with an AnalysisException naming that column. */ + private def nonExistentColumnCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("dmlValidation.nonExistentColumn") { table => + val exception = Check.intercept[AnalysisException]( + table.spark.sql( + s"DELETE FROM ${table.name} WHERE no_such_column = 1")) + + assert(exception.getMessage.contains("no_such_column")) + } + + /** + * DELETE with a nondeterministic WHERE clause (rand() < 0.5) is rejected with an AnalysisException about + * determinism. + */ + private def nonDeterministicDeleteCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("dmlValidation.nonDeterministicDelete") { table => + val exception = Check.intercept[AnalysisException]( + table.spark.sql( + s"DELETE FROM ${table.name} WHERE rand() < 0.5")) + + assert(exception.getMessage.toLowerCase.contains("deterministic")) + } + + /** + * UPDATE with a nondeterministic WHERE clause (rand() < 0.5) is rejected with an AnalysisException about + * determinism. + */ + private def nonDeterministicUpdateCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("dmlValidation.nonDeterministicUpdate") { table => + val exception = Check.intercept[AnalysisException]( + table.spark.sql( + s"UPDATE ${table.name} SET ${Core.string0.columnName} = 'x' WHERE rand() < 0.5")) + + assert(exception.getMessage.toLowerCase.contains("deterministic")) + } + + /** + * INSERT INTO with too few values for the table's columns is rejected with an AnalysisException about the missing + * data columns. + */ + private def insertArityCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("dmlValidation.insertArity") { table => + val exception = Check.intercept[AnalysisException]( + table.spark.sql( + s"INSERT INTO ${table.name} VALUES (CAST(1 AS BIGINT), 1)")) + + assert(exception.getMessage.toLowerCase.contains("not enough data columns")) + } + + /** + * A MERGE whose UPDATE SET assigns the same target column twice is rejected with an AnalysisException about multiple + * assignments. + */ + private def mergeConflictingUpdatesCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("dmlValidation.mergeConflictingUpdates") { table => + val keyColumn = Core.long0.columnName + val stringColumn = Core.string0.columnName + val exception = Check.intercept[AnalysisException]( + table.spark.sql( + s"""MERGE INTO ${table.name} target USING ( + SELECT * FROM VALUES (CAST(2 AS BIGINT)) AS source($keyColumn) + ) source + ON target.$keyColumn = source.$keyColumn + WHEN MATCHED THEN UPDATE + SET target.$stringColumn = 'a', target.$stringColumn = 'b'""")) + + assert(exception.getMessage.contains("Multiple assignments")) + } + + /** + * A MERGE whose source has two rows matching the same target row fails with a cardinality-violation error naming the + * multi-row match. + */ + private def mergeCardinalityViolationCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("dmlValidation.mergeCardinalityViolation") { table => + val keyColumn = Core.long0.columnName + val stringColumn = Core.string0.columnName + val exception = Check.intercept[Exception]( + table.spark.sql( + s"""MERGE INTO ${table.name} target USING ( + SELECT * FROM VALUES + (CAST(2 AS BIGINT), 'a'), + (CAST(2 AS BIGINT), 'b') + AS source($keyColumn, $stringColumn) + ) source + ON target.$keyColumn = source.$keyColumn + WHEN MATCHED THEN UPDATE SET target.$stringColumn = source.$stringColumn""")) + + assert( + Exceptions.causeChain(exception).exists { cause => + Option(cause.getMessage).exists( + _.contains("matched a single row from the target table")) + }, + s"expected a MERGE cardinality-violation message, got: ${exception.getMessage}") + } + +} diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioFileFormat.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioFileFormat.scala new file mode 100644 index 000000000..e60541677 --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioFileFormat.scala @@ -0,0 +1,55 @@ +package harness + +/** + * File format materialization: the write.format.default a table declares is the format its data files are actually + * written in. + * + * Operations: read the declared write.format.default from the table properties, then list the data files the + * preparation wrote and compare their extensions against it. + * + * Preparation axes: the eight standard preparations that leave data files behind, which are the four core layouts + * (Parquet and ORC crossed with unpartitioned and date-partitioned) and the same four carrying a write sort order. A + * feature layer covers its own table mode by passing its own preparations to `layoutFormatCasesFor`. + * + * Case families: one family, `format.materialization`, contributing 8 cases. + */ +trait ScenarioFileFormat extends ScenarioKit { + + /** The format-materialization case on every standard preparation that writes data files. */ + lazy val fileFormatCases: List[TestCase] = layoutFormatCasesFor(layoutFormatPreparations) + + /** + * The format-materialization case for each preparation given: every data file the preparation wrote carries the + * extension of the table's declared write.format.default, and listing the files leaves the rows and the snapshot + * count unchanged. It applies to any preparation that leaves data files behind, so each feature layer passes the + * list its own preparations produce. + */ + def layoutFormatCasesFor( + preparations: List[TablePreparation[CoreTable.type]] + ): List[TestCase] = + preparations.map { preparation => + preparation.test("format.materialization") { table => + val before = table.state + val declaredFormat = table.spark + .sql(s"SHOW TBLPROPERTIES ${table.name} ('write.format.default')") + .collect()(0) + .getString(1) + val filePaths = table.spark + .sql(s"SELECT file_path FROM ${table.name}.files") + .collect() + .toSeq + .map(_.getString(0)) + val after = table.state + + assert( + filePaths.nonEmpty && filePaths.forall(_.toLowerCase.endsWith(s".$declaredFormat")), + s"data files are not all .$declaredFormat: $filePaths") + assert(after == before, "listing files leaves the rows and the snapshot count unchanged") + } + } + + /** The standard preparations that leave data files behind: the core and the write-ordered ones. */ + lazy val layoutFormatPreparations: List[TablePreparation[CoreTable.type]] = + preparedCoreTables ++ preparedOrderedCoreTables + +} diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioKit.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioKit.scala new file mode 100644 index 000000000..8d8878717 --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioKit.scala @@ -0,0 +1,343 @@ +package harness + +import org.apache.spark.sql.{Row, SparkSession} +import java.util.concurrent.TimeUnit + +/** + * The shared starting-state kit every capability trait builds on: the core table shape, the layout cross-product, the + * standard seed, the standard preparations, and the small query helpers a case needs. + * + * Every capability trait extends this kit, so mixing them into `object Scenarios` puts ScenarioKit first in the + * linearization and its vals initialize before any capability's. It holds copy-on-write layouts and preparations only; + * each feature layer carries its own kit that extends this one. `protected` members are the shared kit; `public` ones + * are also consumed by `object ScenarioCatalog`, `object Plan`, and downstream runners. + */ +trait ScenarioKit { + + protected val Core = CoreTable // brevity in the typed column references below + protected val columnNameList = Core.columnNames.mkString(", ") // source column list, so renames propagate + + // The rows a case reads back are ordered by the long column and carry exactly the core columns in their declared + // order, so an expected row set is written as the rows the case started from, filtered, mapped through + // `withColumnValue`, extended with literal rows, and re-sorted. Both helpers address columns by position so they also + // work on the literal rows a case writes out. + private def columnPosition(column: Column[_]): Int = Core.columnNames.indexOf(column.columnName) + + protected def withColumnValue[T](row: Row, column: Column[T], value: T): Row = + Row.fromSeq(row.toSeq.updated(columnPosition(column), value)) + + protected def inKeyOrder(rows: Seq[Row]): Seq[Row] = + rows.sortBy(_.getLong(columnPosition(Core.long0))) + + // --- layouts: one file format and one partitioning per starting table shape --- + // A layout is one starting table shape. Each layout is a plain literal CREATE statement: the column list is one + // shared literal `columnDefinitions`, and format and partitioning are literal fragments. The schema-creation case + // cross-checks the literal against CoreTable's declared columns, so the two stay in step. A layout belongs to the + // preparation, so one test case is written once and runs on every layout. + protected val columnDefinitions = + "foo_col_long bigint, foo_col_int int, foo_col_string string, foo_col_double double, " + + "foo_col_boolean boolean, foo_col_date string" + + /** One starting table shape: the label that names it in a case ID and the CREATE statement that builds it. */ + final case class Layout(label: String, create: String => String) + + /** One partitioning choice: the label that names it in a case ID and the CREATE clause that applies it. */ + final case class Partitioning(label: String, clause: String) + + /** The empty partitioning clause: the table keeps all its rows in one unpartitioned file set. */ + protected val unpartitioned = Partitioning("unpartitioned", "") + + /** Partitions the table by its date column, so each distinct date value owns one partition. */ + protected val partitionedByDate = + Partitioning("partitioned", s"PARTITIONED BY (${Core.date0.columnName})") + + protected val partitionings: List[Partitioning] = List(unpartitioned, partitionedByDate) + + /** + * Every file format the standard matrix runs on. This is the single source for a format list anywhere in the + * harness, so every format-crossed family covers both columnar formats. A format beyond these two is proven by the + * file-format extension layer, which supplies its own list. + */ + val fileFormats: List[String] = List("parquet", "orc") + + /** One copy-on-write table in `format`, shaped by `partitioning`, labelled for its case IDs. */ + private def coreLayout(partitioning: Partitioning, format: String): Layout = + Layout( + s"${partitioning.label}/$format", + table => + s"CREATE TABLE $table ($columnDefinitions) USING $dataSource ${partitioning.clause} " + + s"TBLPROPERTIES ('write.format.default'='$format')") + + /** Every core layout: each file format crossed with each partitioning. */ + val layouts: List[Layout] = + for { + format <- fileFormats + partitioning <- partitionings + } yield coreLayout(partitioning, format) + + /** The core layouts partitioned by the date column, one per file format. */ + val partitionedLayouts: List[Layout] = + fileFormats.map(format => coreLayout(partitionedByDate, format)) + + /** + * The standard seed writes three deterministic rows with keys 1, 2 and 3. Row `n` holds key `n` in the long column, + * `n` in the int column, `row-n` in the string column, `n.5` in the double column, `n % 2 == 0` in the boolean + * column, and the date value `n - 1` hours after `2024-01-01-00`. `RowGenerator` builds those literals from + * `CoreTable`, so the seed follows a column rename. Every preparation that seeds a core table writes exactly these + * rows, so a case that starts from a seeded table knows its three starting keys. + */ + val standardSeedRowCount: Int = 3 + + /** Creates the table under `layout` and leaves it empty. The caller adds the seed step it wants. */ + def create(layout: Layout): TableTest[CoreTable.type] = + TableTest(Core).sql("create")(layout.create)() + + /** One preparation per core layout: the table is created, then seeded with the standard rows. */ + val preparedCoreTables: List[TablePreparation[CoreTable.type]] = + layouts.map(layout => + TablePreparation( + layout.label, + create(layout).insert(standardSeedRowCount)())) + + /** + * One preparation per date-partitioned core layout: the table is created, then seeded with the standard rows, whose + * date values put one row in each of three partitions. + */ + val preparedPartitionedCoreTables: List[TablePreparation[CoreTable.type]] = + partitionedLayouts.map(layout => + TablePreparation( + layout.label, + create(layout).insert(standardSeedRowCount)())) + + /** + * One preparation per core layout: the table is created, seeded, then given a write sort order on the long key by + * ALTER TABLE WRITE ORDERED BY. The column list stays as seeded, so every DML case runs on the result. + */ + val preparedOrderedCoreTables: List[TablePreparation[CoreTable.type]] = + layouts.map(layout => + TablePreparation( + layout.label, + create(layout) + .insert(standardSeedRowCount)() + .sql("writeOrderedByLongKey")(table => + s"ALTER TABLE $table WRITE ORDERED BY ${Core.long0.columnName}")(), + "prep.ordered:")) + + /** + * One preparation per core layout: the table is created, seeded, then given a prep_extra int column by ALTER TABLE + * ADD COLUMN. The column list grows past the seed row shape and the seeded rows read null for the new column, so the + * cases that address columns by name run on the result: the reads, the deletes and the updates. + */ + val preparedEvolvedCoreTables: List[TablePreparation[CoreTable.type]] = + layouts.map(layout => + TablePreparation( + layout.label, + create(layout) + .insert(standardSeedRowCount)() + .sql("addPrepExtraColumn")(table => s"ALTER TABLE $table ADD COLUMN prep_extra int")(), + "prep.evolved:")) + + /** One preparation per core layout: the table is created and left unseeded, so it holds no rows. */ + val preparedEmptyCoreTables: List[TablePreparation[CoreTable.type]] = + layouts.map(layout => TablePreparation(layout.label, create(layout))) + + /** + * The CREATE statement for an unpartitioned core table in `format`. This generic substrate contributes zero cases + * and gives later capability layers a stable shared starting point. + */ + protected def coreCreate(table: String, format: String): String = + coreLayout(unpartitioned, format).create(table) + + /** An unseeded, unpartitioned core table in `format`, so the case owns every row the table holds. */ + protected def preparedEmptyStandardTable(format: String): TablePreparation[CoreTable.type] = + TablePreparation(format, create(coreLayout(unpartitioned, format))) + + /** + * An unpartitioned core table in `format`, created and then seeded with the standard rows. This is the plainest + * starting state in the harness, so most capability families build on it. + */ + protected def preparedStandardTable(format: String): TablePreparation[CoreTable.type] = + TablePreparation( + format, + create(coreLayout(unpartitioned, format)).insert(standardSeedRowCount)()) + + /** The standard seeded table in each file format. */ + val preparedCoreFormats: List[TablePreparation[CoreTable.type]] = + fileFormats.map(preparedStandardTable) + + /** + * An unpartitioned core table in `format` holding five rows across two snapshots: the standard seed, then rows 4 and + * 5. The step between the two commits holds until the wall clock passes the seed commit's timestamp, so the two + * snapshots carry distinct commit times and a timestamp-bounded read resolves to exactly one of them. + * + * Every family that reads history needs this shape, so the shared kit owns it. + */ + protected def preparedTwoSnapshotTable(format: String): TablePreparation[CoreTable.type] = + TablePreparation( + format, + create(coreLayout(unpartitioned, format)) + .insert(standardSeedRowCount)() + .step("waitForNextSnapshotTimestamp")(waitForNextSnapshotTimestamp)() + .sql("insertRowsFourAndFive")(table => + s"INSERT INTO $table VALUES " + + "(CAST(4 AS BIGINT), 4, 'row-4', 4.5, true, '2024-01-04-03'), " + + "(CAST(5 AS BIGINT), 5, 'row-5', 5.5, false, '2024-01-05-04')")()) + + /** + * The same starting state with a fourth row whose key is 99 and whose string column is null, so exactly one row of + * the table reads null for that column. + */ + protected def withNullStringRow( + basePreparation: TablePreparation[CoreTable.type] + ): TablePreparation[CoreTable.type] = + basePreparation.copy( + preparation = basePreparation.preparation.sql("prep.nullStringRow")(table => + s"INSERT INTO $table VALUES (CAST(99 AS BIGINT), 99, NULL, 99.5, false, '2024-01-01-00')")()) + + /** The core preparations, each carrying one row whose string column is null. */ + val preparedNullStringCoreTables: List[TablePreparation[CoreTable.type]] = + preparedCoreTables.map(withNullStringRow) + + /** The write-ordered preparations, each carrying one row whose string column is null. */ + val preparedNullStringOrderedCoreTables: List[TablePreparation[CoreTable.type]] = + preparedOrderedCoreTables.map(withNullStringRow) + + // Waits until the wall clock passes the newest snapshot's commit timestamp, so the next commit lands on a strictly + // later millisecond and a timestamp-bounded read separates the two snapshots. + private def waitForNextSnapshotTimestamp(spark: SparkSession, table: String): Unit = { + val previousTimestamp = spark + .sql( + s"SELECT committed_at FROM $table.snapshots " + + "ORDER BY committed_at DESC LIMIT 1") + .collect()(0) + .getTimestamp(0) + .getTime + val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5) + + while ( + System.currentTimeMillis() <= previousTimestamp && + System.nanoTime() < deadline) { + Thread.sleep(1L) + } + + assert( + System.currentTimeMillis() > previousTimestamp, + s"clock did not advance beyond snapshot timestamp $previousTimestamp") + } + + // --- table, rename and lock lifecycle boundaries used by cases that build artifacts for themselves --- + // Each boundary takes the catalog statement executor, so a test drives the same code with a recorder in place of + // Spark. Call sites pass `spark.sql(_)`. + + /** + * Runs `use` against `table`, a table the case builds for itself. `create` issues the CREATE; ownership starts the + * moment it returns, so a name that is already taken leaves the pre-existing table intact and the drop afterwards + * removes only the table this call created. A failure in `create` or `use` stays the + * primary failure, and a cleanup failure is attached to it as a suppressed exception. Callers name the table with + * `TableTest.nextQualifiedTableName` or by extending the generated name of a prepared table, so two runs of the same + * case can never address the same table. + */ + private[harness] def withOwnedTable(runStatement: String => Unit, table: String)( + create: => Unit)(use: => Unit): Unit = + OwnedTableLifecycle.withOwnership(runStatement(s"DROP TABLE IF EXISTS $table")) { + markTableCreated => + create + markTableCreated() + use + } + + /** + * Runs `use`, then runs `cleanupStatement` on every outcome. A case uses this for an artifact whose creation is the + * rejection under test: the statement that would create it is expected to fail, so there is no successful create to + * take ownership of, yet a rejection that partly applied, threw the wrong type, or unexpectedly succeeded must still + * leave nothing behind. The failure `use` raises stays primary and a cleanup failure rides along suppressed. + */ + private[harness] def withCleanupStatement(runStatement: String => Unit, cleanupStatement: String)( + use: => Unit): Unit = + OwnedTableLifecycle.withCleanup(runStatement(cleanupStatement))(use) + + /** + * Runs `use` while tracking which name a table answers to. `use` receives a rename function that issues one ALTER + * TABLE RENAME TO and records the new name only once the catalog accepted it, so the boundary always knows the live + * name. If `use` leaves the table under any name other than `originalTable`, the boundary drops that live name, so + * a failed assertion or a failed rename back still ends with the table gone. A failure in `use` stays primary and a + * cleanup failure rides along suppressed. + */ + private[harness] def withTrackedRename(runStatement: String => Unit, originalTable: String)( + use: (String => Unit) => Unit): Unit = { + var liveTable = originalTable + OwnedTableLifecycle.withCleanup( + if (liveTable != originalTable) runStatement(s"DROP TABLE IF EXISTS $liveTable")) { + use { newTable => + runStatement(s"ALTER TABLE $liveTable RENAME TO $newTable") + liveTable = newTable + } + } + } + + /** + * Runs `use` while the case holds a table lock. `lock` is taken first and its response is checked; `use` receives a + * release function so a case that reads behavior after the lock is gone releases it itself. The boundary releases + * the lock afterwards only while the case still holds it, so exactly one release is attempted. Every release checks + * its response, so a rejected release fails the case, and a release failure that follows a failure inside `use` + * rides along as a suppressed exception. + */ + private[harness] def withTableLock( + lock: () => (Int, String), + unlock: () => (Int, String))(use: (() => Unit) => Unit): Unit = { + val (lockStatus, lockBody) = lock() + assert(lockStatus >= 200 && lockStatus < 300, s"lock request failed: $lockStatus $lockBody") + + var lockHeld = true + def releaseLock(): Unit = { + val (unlockStatus, unlockBody) = unlock() + lockHeld = false + assert( + unlockStatus >= 200 && unlockStatus < 300, + s"unlock request failed: $unlockStatus $unlockBody") + } + + OwnedTableLifecycle.withCleanup(if (lockHeld) releaseLock())(use(() => releaseLock())) + } + + // --- shared query helpers used across capability traits --- + + // Snapshots in ancestry order (root first), following the parent_id chain. The chain orders commits deterministically + // even when two of them share a committed_at millisecond. + protected def snapshotIds(spark: SparkSession, table: String): Seq[Long] = { + val rows = spark.sql(s"SELECT snapshot_id, parent_id FROM $table.snapshots").collect().toSeq + val snapshotIdSet = rows.map(_.getLong(0)).toSet + val childByParent = rows.collect { + case row if !row.isNullAt(1) => row.getLong(1) -> row.getLong(0) + }.toMap + val root = rows.collectFirst { + case row if row.isNullAt(1) || !snapshotIdSet.contains(row.getLong(1)) => row.getLong(0) + }.get + + Iterator + .iterate(Option(root))(parent => parent.flatMap(childByParent.get)) + .takeWhile(_.isDefined) + .flatten + .toList + } + + protected def catalogRelative(table: String): String = table.stripPrefix("openhouse.") + + /** One core row in the seed shape, keyed by `long` and tagged in the string column. */ + protected def coreRow(long: Long, tag: String): String = + s"(CAST($long AS BIGINT), ${long.toInt}, '$tag', ${long}.5, false, '2024-01-01-00')" + + // The Spark data source used by CREATE TABLE statements. The LinkedIn adapter overrides this before building + // ScenarioCatalog.cases. Catalog procedure calls still use the catalog name "openhouse". + var dataSource: String = "iceberg" + + protected def tableProps(spark: SparkSession, table: String): Map[String, String] = + spark.sql(s"SHOW TBLPROPERTIES $table").collect().toSeq.map(r => r.getString(0) -> r.getString(1)).toMap + + protected val extraColInsert9 = "(CAST(9 AS BIGINT), 9, 'row-9', 9.5, true, '2024-01-09-01', 42)" + protected val extraColInsert10 = "(CAST(10 AS BIGINT), 10, 'row-10', 10.5, true, '2024-01-10-01', 43)" + + protected def countOf(spark: SparkSession, sql: String): String = + spark.sql(sql).collect()(0).getLong(0).toString + +} diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioMergeOnRead.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioMergeOnRead.scala new file mode 100644 index 000000000..a80a0869d --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioMergeOnRead.scala @@ -0,0 +1,487 @@ +package harness + +/** + * Merge-on-read: what changes when a mutation records a position-delete file beside the data file it matched and + * leaves that data file in place. + * + * The reusable DML contract has to hold identically on both write paths, because a caller writes the same SQL either + * way. Everything else in this file is behavior only a merge-on-read table has: the physical delete file itself, the + * surface a table reaches once data files and a live delete file sit side by side, the metadata that exposes the + * delete, the changelog's ability to decode it, the history that spans it, and the maintenance procedures that fold + * or carry it. Maintenance is large enough to review on its own, so it lives in ScenarioMergeOnReadMaintenance and + * joins the one contribution this layer names. + * + * Operations, DML: the row-mutating operations `ScenarioDml` defines, its null-string DELETE and its reads, reused as + * data. A merge-on-read table runs the same statements and the same row and snapshot delta assertions as a + * copy-on-write one, so this file holds one definition of each preparation and none of each operation. + * + * Operations, merge-on-read contract: 26 focused families. Nineteen live here and cover the physical delete file + * against its copy-on-write counterpart, a mode change applied partway through a table's life, the six operations + * that run once a delete file is live, the position_deletes metadata table, the three changelog operations a scan + * decodes and the two it reports as unsupported, format materialization with a delete file present, the delete-file + * replication property, and time travel and rollback across the delete. Seven more live in + * ScenarioMergeOnReadMaintenance. + * + * Preparation axes: the write mode is the axis this layer adds. Four merge-on-read layouts cross the two columnar + * formats with unpartitioned and date-partitioned tables; two replace-lineage merge-on-read layouts put the same + * mutations on a table that also went through a replace, which is this layer's one dependency on its parent; and two + * verify layouts per write mode seed into a single data file so a strict-subset delete is a partial-file match and + * the physical outcome is deterministic. + * + * Case families: 320 cases. The DML axis contributes 268 in three families, and the merge-on-read contract + * contributes 52 in 26 families: 38 in the 19 families here and 14 in the 7 maintenance families. + */ +trait ScenarioMergeOnRead extends ScenarioMergeOnReadMaintenance { + this: ScenarioDml with ScenarioFileFormat with ChangelogSupport => + + /** Every merge-on-read case: the reusable DML operations on merge-on-read tables, then the write-mode contract. */ + lazy val mergeOnReadCases: List[TestCase] = + mergeOnReadDmlCases ++ mergeOnReadContractCases ++ mergeOnReadMaintenanceCases + + /** + * The reusable DML operations on merge-on-read tables: every row-mutating operation on the four merge-on-read + * preparations and the two replace-lineage ones, the null-string DELETE on their null-string forms, and the reads + * on the preparations that already carry a live position-delete file. + */ + lazy val mergeOnReadDmlCases: List[TestCase] = + mergeOnReadCoreDmlCases ++ replacedMergeOnReadDmlCases ++ deletedMergeOnReadDmlCases + + /** Every row-mutating operation on the merge-on-read preparations, plus the null-string DELETE on their null form. */ + lazy val mergeOnReadCoreDmlCases: List[TestCase] = + preparedMergeOnReadCoreTables.flatMap(preparation => + rowMutationTestCases.map(_.runOn(preparation))) ++ + preparedNullStringMergeOnReadCoreTables.flatMap(preparation => + nullStringRowTestCases.map(_.runOn(preparation))) + + /** The same operations on the replace-lineage merge-on-read preparations, so both paths apply at once. */ + lazy val replacedMergeOnReadDmlCases: List[TestCase] = + preparedReplacedMergeOnReadCoreTables.flatMap(preparation => + rowMutationTestCases.map(_.runOn(preparation))) ++ + preparedNullStringReplacedMergeOnReadCoreTables.flatMap(preparation => + nullStringRowTestCases.map(_.runOn(preparation))) + + /** The reads on preparations carrying a live position-delete file, so each read applies one at scan time. */ + lazy val deletedMergeOnReadDmlCases: List[TestCase] = + preparedDeletedMergeOnReadTables.flatMap(preparation => + readTestCases.map(_.runOn(preparation))) + + /** Every merge-on-read contract case outside maintenance, in the order this file introduces them. */ + lazy val mergeOnReadContractCases: List[TestCase] = + deleteFileCases ++ + deleteModeCases ++ + deleteFileCoexistenceCases ++ + mergeOnReadMetadataCases ++ + mergeOnReadChangelogCases ++ + mergeOnReadFileFormatCases ++ + mergeOnReadFileReplicationCases ++ + mergeOnReadHistoryCases + + // --- 1. the physical delete file, and the copy-on-write outcome it is defined against --- + + /** + * A strict-subset DELETE against a single data file, run once on each write mode. Merge-on-read records the removal + * in a position-delete file and keeps the data file; copy-on-write rewrites the data file and leaves no delete + * file. Both remove the same row and commit one snapshot, so the write mode is the only difference. + */ + lazy val deleteFileCases: List[TestCase] = + mergeOnReadVerifyLayouts.map(layout => + TablePreparation(layout.label, singleFileSeed(layout)) + .test("mergeOnRead.deleteFile.writesDeleteFile")(table => + assertSubsetDeleteOutcome(table, expectedDeleteFileCount = 1))) ++ + copyOnWriteVerifyLayouts.map(layout => + TablePreparation(layout.label, singleFileSeed(layout)) + .test("mergeOnRead.deleteFile.copyOnWriteRewritesDataFile")(table => + assertSubsetDeleteOutcome(table, expectedDeleteFileCount = 0))) + + /** + * Runs the strict-subset DELETE and asserts the outcome both write modes share, namely that the matching row is + * gone and exactly one snapshot was committed, together with the delete-file count the mode under test produces. + */ + private def assertSubsetDeleteOutcome( + table: PreparedTable[CoreTable.type], + expectedDeleteFileCount: Long): Unit = { + val before = table.state + + table.spark.sql(s"DELETE FROM ${table.name} WHERE ${Core.long0.columnName} < 2") + val after = table.state + + assert( + liveKeys(table.spark, table.name) == Seq(2L, 3L), + s"the strict-subset delete leaves keys 2 and 3, found ${liveKeys(table.spark, table.name)}") + assert( + currentDeleteFileCount(table.spark, table.name) == expectedDeleteFileCount, + s"the delete leaves $expectedDeleteFileCount delete files, found " + + s"${currentDeleteFileCount(table.spark, table.name)}") + assert( + after.snapshotCount == before.snapshotCount + 1, + s"the delete commits one snapshot, went from ${before.snapshotCount} to ${after.snapshotCount}") + } + + // --- 2. choosing the write mode partway through a table's life --- + + /** + * Switching a copy-on-write table's delete mode to merge-on-read makes the next partial-file DELETE write a + * position-delete file and keep the untouched rows in the data file, so the mode a table carries at commit time is + * the one that decides how the delete is written. + */ + lazy val deleteModeCases: List[TestCase] = + fileFormats.map(format => + preparedSingleFileCopyOnWriteTable(format) + .test("mergeOnRead.deleteMode.alterToMergeOnRead") { table => + table.spark.sql( + s"ALTER TABLE ${table.name} SET TBLPROPERTIES ('write.delete.mode'='merge-on-read')") + table.spark.sql(s"DELETE FROM ${table.name} WHERE ${Core.long0.columnName} = 1") + + assert( + currentDeleteFileCount(table.spark, table.name) == 1, + s"the mode change makes the delete write one delete file, found " + + s"${currentDeleteFileCount(table.spark, table.name)}") + assert( + liveKeys(table.spark, table.name) == Seq(2L, 3L), + s"the delete after the mode change leaves keys 2 and 3, found " + + s"${liveKeys(table.spark, table.name)}") + }) + + // --- 3. the surface a table reaches once a delete file is live beside its data --- + + /** + * The six operations that behave differently once data files and a live position-delete file sit side by side. A + * read or an insert on a delete-free merge-on-read table is identical to copy-on-write, so every family here starts + * from the state where a delete file is already live. + */ + lazy val deleteFileCoexistenceCases: List[TestCase] = + preparedDeletedMergeOnReadTables.flatMap { preparation => + List( + appendOverDeleteFileCase(preparation), + secondDeleteOverDeleteFileCase(preparation), + updateOverDeleteFileCase(preparation), + filteredReadOverDeleteFileCase(preparation), + compactDeletesOverDeleteFileCase(preparation), + mergeOverDeleteFileCase(preparation)) + } + + /** + * Asserts the table persists `propertyName` as merge-on-read before the mutation under test runs. The row + * assertions hold on either write path, so this guard is what ties the case to the merge-on-read path it claims to + * cover. + */ + private def assertConfiguredMergeOnRead( + table: PreparedTable[CoreTable.type], + propertyName: String): Unit = { + val configuredMode = persistedProperty(table.spark, table.name, propertyName) + + assert( + configuredMode.contains("merge-on-read"), + s"the table persists $propertyName as merge-on-read, found $configuredMode") + } + + /** An INSERT over a live position-delete file adds its row and keeps the deleted key out of the live rows. */ + private def appendOverDeleteFileCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("mergeOnRead.coexistence.append") { table => + table.spark.sql(s"INSERT INTO ${table.name} VALUES ${coreRow(6L, "row-6")}") + + assert( + liveKeys(table.spark, table.name) == Seq(2L, 3L, 6L), + s"the append lands beside the live delete, found ${liveKeys(table.spark, table.name)}") + } + + /** A second DELETE over a live position-delete file removes its row and the table still carries delete files. */ + private def secondDeleteOverDeleteFileCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("mergeOnRead.coexistence.secondDelete") { table => + table.spark.sql(s"DELETE FROM ${table.name} WHERE ${Core.long0.columnName} = 2") + + assert( + liveKeys(table.spark, table.name) == Seq(3L), + s"the second delete leaves key 3, found ${liveKeys(table.spark, table.name)}") + assert( + currentDeleteFileCount(table.spark, table.name) >= 1, + s"the second delete keeps delete files live, found " + + s"${currentDeleteFileCount(table.spark, table.name)}") + } + + /** An UPDATE over a live position-delete file changes its row's value and keeps the live key set. */ + private def updateOverDeleteFileCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("mergeOnRead.coexistence.update") { table => + assertConfiguredMergeOnRead(table, "write.update.mode") + table.spark.sql( + s"UPDATE ${table.name} SET ${Core.string0.columnName} = 'cx' " + + s"WHERE ${Core.long0.columnName} = 3") + val updatedValue = table.spark + .sql( + s"SELECT ${Core.string0.columnName} FROM ${table.name} " + + s"WHERE ${Core.long0.columnName} = 3") + .collect()(0) + .getString(0) + + assert(updatedValue == "cx", s"the update over a live delete sets the value, found $updatedValue") + assert( + liveKeys(table.spark, table.name) == Seq(2L, 3L), + s"the update keeps the live key set, found ${liveKeys(table.spark, table.name)}") + } + + /** A filtered read over a live position-delete file returns the live rows the filter selects. */ + private def filteredReadOverDeleteFileCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("mergeOnRead.coexistence.filteredRead") { table => + val selectedKeys = table.spark + .sql( + s"SELECT ${Core.long0.columnName} FROM ${table.name} " + + s"WHERE ${Core.long0.columnName} <= 2 ORDER BY ${Core.long0.columnName}") + .collect() + .toSeq + .map(_.getLong(0)) + + assert( + selectedKeys == Seq(2L), + s"the filter applies the position delete, found $selectedKeys") + } + + /** Compacting the position deletes over a live delete file keeps the live rows. */ + private def compactDeletesOverDeleteFileCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("mergeOnRead.coexistence.compactDeletes") { table => + table.spark.sql( + "CALL openhouse.system.rewrite_position_delete_files(" + + s"table => '${catalogRelative(table.name)}', " + + "options => map('rewrite-all', 'true'))") + + assert( + liveKeys(table.spark, table.name) == Seq(2L, 3L), + s"compacting the deletes keeps the live rows, found ${liveKeys(table.spark, table.name)}") + } + + /** A MERGE over a live position-delete file updates its matched row and keeps the live key set. */ + private def mergeOverDeleteFileCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("mergeOnRead.coexistence.merge") { table => + assertConfiguredMergeOnRead(table, "write.merge.mode") + table.spark.sql( + s"MERGE INTO ${table.name} target " + + "USING (SELECT CAST(3 AS BIGINT) key) source " + + s"ON target.${Core.long0.columnName} = source.key " + + s"WHEN MATCHED THEN UPDATE SET ${Core.string0.columnName} = 'mg'") + val mergedValue = table.spark + .sql( + s"SELECT ${Core.string0.columnName} FROM ${table.name} " + + s"WHERE ${Core.long0.columnName} = 3") + .collect()(0) + .getString(0) + + assert(mergedValue == "mg", s"the merge over a live delete sets the value, found $mergedValue") + assert( + liveKeys(table.spark, table.name) == Seq(2L, 3L), + s"the merge keeps the live key set, found ${liveKeys(table.spark, table.name)}") + } + + // --- 4. the metadata that exposes what the reader will apply --- + + /** + * After a merge-on-read DELETE, the position_deletes metadata table reports exactly the one delete entry the + * mutation created, so what the reader applies at scan time is visible to a caller reading metadata. + */ + lazy val mergeOnReadMetadataCases: List[TestCase] = + fileFormats.map(format => + preparedSingleFileMergeOnReadTable(format) + .test("mergeOnRead.metadata.positionDeletes") { table => + table.spark.sql(s"DELETE FROM ${table.name} WHERE ${Core.long0.columnName} = 1") + val positionDeleteCount = table.spark + .sql(s"SELECT count(*) FROM ${table.name}.position_deletes") + .collect()(0) + .getLong(0) + + assert( + positionDeleteCount == 1, + s"position_deletes exposes the one position delete, found $positionDeleteCount") + }) + + // --- 5. what a changelog scan makes of a merge-on-read history --- + + /** + * The changelog on a merge-on-read table. The append, the INSERT OVERWRITE and the row-level DELETE leave the + * change feed decodable and report exactly the rows they changed, so those three are pinned row by row. The + * UPDATE and the MERGE leave position-delete files that a changelog scan reports as unsupported, so each of those + * is pinned as a rejection. + */ + lazy val mergeOnReadChangelogCases: List[TestCase] = + fileFormats.flatMap { format => + changelogOperations + .filter(operation => decodableChangelogOperationNames.contains(operation.name)) + .map(operation => + decodableChangelogCase(preparedMergeOnReadTable(format), operation)) ++ + changelogOperations + .filterNot(operation => decodableChangelogOperationNames.contains(operation.name)) + .map(operation => + rejectedChangelogCase(preparedMergeOnReadTable(format), operation)) + } + + /** The operations whose merge-on-read change feed a changelog scan decodes, because they leave no delete file. */ + private val decodableChangelogOperationNames = + Set("changelog.append", "changelog.overwrite", "changelog.delete") + + /** The message a changelog scan reports when the range it was asked for spans position-delete files. */ + private val changelogDeleteFileRejectionMessage = "Delete files are currently not supported" + + /** + * The exact change rows each decodable operation reports on a merge-on-read table, as change type followed by the + * core columns in their declared order. Asserting the whole row pins which row the feed attributes each change + * to, so an operation that reported the right number of changes against the wrong row fails here. + */ + private val expectedChangeRowsByOperation: Map[String, List[List[Any]]] = Map( + "changelog.append" -> + List(List("INSERT", 6L, 6, "row-6", 6.5d, true, "2024-01-06-05")), + "changelog.overwrite" -> + List(List("DELETE", 3L, 3, "row-3", 3.5d, false, "2024-01-01-02")), + "changelog.delete" -> + List(List("DELETE", 1L, 1, "row-1", 1.5d, false, "2024-01-01-00"))) + + /** + * On a merge-on-read table, the operation's change feed reports exactly the rows it changed, so the write mode + * leaves the decodable part of the changelog contract as it is. + */ + private def decodableChangelogCase( + preparation: TablePreparation[CoreTable.type], + operation: ChangelogOperation): TestCase = + preparation.test(s"mergeOnRead.${operation.name}") { table => + val expectedChangeRows = expectedChangeRowsByOperation + .getOrElse( + operation.name, + throw new AssertionError(s"${operation.name} declares the change rows it reports")) + val seedSnapshotId = snapshotIds(table.spark, table.name).head + table.spark.sql(operation.statement(table.name)) + val changelogView = changelogViewFrom(table, seedSnapshotId) + val actualChangeRows = table.spark + .sql( + s"SELECT _change_type, $columnNameList FROM $changelogView " + + s"ORDER BY _change_type, ${Core.long0.columnName}") + .collect() + .toSeq + .map(_.toSeq.toList) + + assert( + actualChangeRows == expectedChangeRows, + s"${operation.name} reports $expectedChangeRows on a merge-on-read table, " + + s"found $actualChangeRows") + assert( + changeCounts(table, changelogView) == operation.expectedChangeCounts, + s"${operation.name} agrees with the shared histogram " + + s"${operation.expectedChangeCounts}, found ${changeCounts(table, changelogView)}") + } + + /** + * On a merge-on-read table, reading the operation's change feed reports that delete files are unsupported, so a + * caller learns the range is undecodable and can fall back to a range the scan does decode. + */ + private def rejectedChangelogCase( + preparation: TablePreparation[CoreTable.type], + operation: ChangelogOperation): TestCase = + preparation.test(s"mergeOnRead.${operation.name}.rejected") { table => + val seedSnapshotId = snapshotIds(table.spark, table.name).head + table.spark.sql(operation.statement(table.name)) + val rejection = Check.intercept[UnsupportedOperationException] { + val view = changelogViewFrom(table, seedSnapshotId) + table.spark.sql(s"SELECT * FROM $view").collect() + } + + assert( + Exceptions + .causeChain(rejection) + .exists(cause => + Option(cause.getMessage).exists(_.contains(changelogDeleteFileRejectionMessage))), + s"the rejection names delete files as unsupported, found: ${rejection.getMessage.take(200)}") + } + + // --- 6. the properties a merge-on-read write path owns --- + + /** + * Format materialization on a table that already carries a live position-delete file: the data files still carry + * the extension of the declared write.format.default, so a delete file present alongside them leaves the format + * contract as it is. The case body is the foundation's, reused as data. + */ + lazy val mergeOnReadFileFormatCases: List[TestCase] = + layoutFormatCasesFor(preparedDeletedMergeOnReadTables) + + /** + * write.delete-file-replication is the property the delete-file writer resolves into a block replication factor, so + * it applies exactly where a mutation writes a position-delete file. The property round-trips through the catalog, + * survives the DELETE that uses it, and the DELETE physically writes the delete file the property describes. The + * local catalog asserts the property and the delete file; HDFS verifies block replication in its own environment. + */ + lazy val mergeOnReadFileReplicationCases: List[TestCase] = + fileFormats.map(format => + preparedSingleFileMergeOnReadTable(format) + .test("mergeOnRead.fileReplication.deleteFileProperty") { table => + table.spark.sql( + s"ALTER TABLE ${table.name} SET TBLPROPERTIES ('write.delete-file-replication'='2')") + + assert( + tableProps(table.spark, table.name).get("write.delete-file-replication").contains("2"), + s"the delete-file replication property round-trips, found " + + s"${tableProps(table.spark, table.name).get("write.delete-file-replication")}") + + table.spark.sql(s"DELETE FROM ${table.name} WHERE ${Core.long0.columnName} = 1") + + assert( + currentDeleteFileCount(table.spark, table.name) == 1, + s"the delete writes the position-delete file the property describes, found " + + s"${currentDeleteFileCount(table.spark, table.name)}") + assert( + liveKeys(table.spark, table.name) == Seq(2L, 3L), + s"the delete leaves keys 2 and 3, found ${liveKeys(table.spark, table.name)}") + assert( + tableProps(table.spark, table.name).get("write.delete-file-replication").contains("2"), + "the delete-file replication property survives the delete that used it") + }) + + // --- 7. reading the history a position delete sits in --- + + /** + * Snapshot history over a live position-delete file. The delete is a commit like any other, so the snapshot before + * it still reads the removed row, and a rollback to that snapshot brings the row back into the live set. + */ + lazy val mergeOnReadHistoryCases: List[TestCase] = + preparedDeletedMergeOnReadTables.flatMap { preparation => + List( + timeTravelBeforeDeleteCase(preparation), + rollbackUndoesDeleteCase(preparation)) + } + + /** The current read applies the delete, while the snapshot before it still reads the removed row. */ + private def timeTravelBeforeDeleteCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("mergeOnRead.history.timeTravelBeforeDelete") { table => + val seedSnapshotId = snapshotIds(table.spark, table.name).head + val preDeleteKeys = table.spark + .sql( + s"SELECT ${Core.long0.columnName} FROM ${table.name} VERSION AS OF $seedSnapshotId " + + s"ORDER BY ${Core.long0.columnName}") + .collect() + .toSeq + .map(_.getLong(0)) + + assert( + liveKeys(table.spark, table.name) == Seq(2L, 3L), + s"the current read applies the delete, found ${liveKeys(table.spark, table.name)}") + assert( + preDeleteKeys == Seq(1L, 2L, 3L), + s"the snapshot before the delete reads the removed row, found $preDeleteKeys") + } + + /** A rollback to the snapshot before the delete brings the removed row back into the live set. */ + private def rollbackUndoesDeleteCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("mergeOnRead.history.rollbackUndoesDelete") { table => + val seedSnapshotId = snapshotIds(table.spark, table.name).head + table.spark.sql( + "CALL openhouse.system.rollback_to_snapshot(" + + s"table => '${catalogRelative(table.name)}', " + + s"snapshot_id => ${seedSnapshotId}L)") + + assert( + liveKeys(table.spark, table.name) == Seq(1L, 2L, 3L), + s"the rollback restores the position-deleted row, found ${liveKeys(table.spark, table.name)}") + } + +} diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioMergeOnReadKit.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioMergeOnReadKit.scala new file mode 100644 index 000000000..fdee1636a --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioMergeOnReadKit.scala @@ -0,0 +1,359 @@ +package harness + +/** + * The merge-on-read starting states. + * + * A merge-on-read table is format version 2 whose delete, update and merge modes are merge-on-read, so a mutation + * records position-delete files beside the data files it matched and leaves those data files in place. Copy-on-write + * rewrites the matched data file instead. That physical difference is the whole subject of this layer, so this kit + * supplies the starting states that put a table on one write path or the other and leaves the operations to the + * foundation. + * + * Several families need a table whose delete is a partial-file match, because a delete aligned with a whole data file + * is satisfied by dropping that file on either write path and the two modes become indistinguishable. The verify + * layouts seed through a single write task so all three rows land in one data file, which makes a strict-subset + * delete a partial-file match and the physical outcome deterministic in both formats. + * + * The members are lazy so they initialize on first read, after every trait mixed into `object Scenarios` has been + * constructed. + */ +trait ScenarioMergeOnReadKit extends ScenarioKit { + + /** Every merge-on-read layout: each file format crossed with each partitioning. */ + lazy val mergeOnReadLayouts: List[Layout] = + for { + format <- fileFormats + partitioning <- partitionings + } yield mergeOnReadLayout(partitioning, format) + + /** + * One merge-on-read layout per file format that pins how a mutation is written physically. It carries all three + * merge-on-read modes, so an UPDATE and a MERGE take the same write path a DELETE does, and it sets + * write.distribution-mode to none while staying unpartitioned, so a single seed INSERT lands every row in one data + * file and a strict-subset mutation is a partial-file match that Iceberg satisfies with a position delete. + */ + lazy val mergeOnReadVerifyLayouts: List[Layout] = + fileFormats.map(format => + Layout( + s"mor-verify/$format", + table => + s"CREATE TABLE $table ($columnDefinitions) USING $dataSource TBLPROPERTIES (" + + s"${mergeOnReadProperties(format)}, 'write.distribution-mode'='none')")) + + /** + * The copy-on-write counterpart of `mergeOnReadVerifyLayouts`, identical except that all three modes are + * copy-on-write, so the pair isolates the write mode as the only difference between the two physical outcomes. + */ + lazy val copyOnWriteVerifyLayouts: List[Layout] = + fileFormats.map(format => + Layout( + s"cow-verify/$format", + table => + s"CREATE TABLE $table ($columnDefinitions) USING $dataSource TBLPROPERTIES (" + + s"${copyOnWriteProperties(format)}, 'write.distribution-mode'='none')")) + + /** One preparation per merge-on-read layout: created, then seeded with the standard rows. */ + lazy val preparedMergeOnReadCoreTables: List[TablePreparation[CoreTable.type]] = + mergeOnReadLayouts.map(layout => + TablePreparation(layout.label, create(layout).insert(standardSeedRowCount)(), mergeOnReadCasePrefix)) + + /** The merge-on-read core preparations, each carrying one row whose string column is null. */ + lazy val preparedNullStringMergeOnReadCoreTables: List[TablePreparation[CoreTable.type]] = + preparedMergeOnReadCoreTables.map(withNullStringRow) + + /** + * One replace-lineage merge-on-read preparation per file format: the standard seed in an unpartitioned table, + * re-specified in place by CREATE OR REPLACE TABLE AS SELECT that restates the merge-on-read modes, then refreshed. + * A mutation on the result runs on replace lineage and the merge-on-read write path at once, which is the one + * direct dependency this layer has on its parent. + */ + lazy val preparedReplacedMergeOnReadCoreTables: List[TablePreparation[CoreTable.type]] = + fileFormats.map(format => + TablePreparation( + s"mor-${unpartitioned.label}/$format", + replaceLineageMergeOnRead(unpartitioned, format), + replacedMergeOnReadCasePrefix)) + + /** The replace-lineage merge-on-read preparations, each carrying one row whose string column is null. */ + lazy val preparedNullStringReplacedMergeOnReadCoreTables: List[TablePreparation[CoreTable.type]] = + preparedReplacedMergeOnReadCoreTables.map(withNullStringRow) + + /** + * One preparation per merge-on-read verify layout: three seed rows in one data file, then the row with key 1 + * deleted merge-on-read, so keys 2 and 3 remain behind a live position-delete file the reader applies at scan time. + */ + lazy val preparedDeletedMergeOnReadTables: List[TablePreparation[CoreTable.type]] = + preparedDeletedTables(mergeOnReadVerifyLayouts, deletedMergeOnReadCasePrefix) + + /** The prefix that marks a case ID as running on a merge-on-read table. */ + val mergeOnReadCasePrefix: String = "prep.mor:" + + /** The prefix that marks a case ID as running on a merge-on-read table reached through a replace. */ + val replacedMergeOnReadCasePrefix: String = "prep.rtasMor:" + + /** The prefix that marks a case ID as running on a table that already carries a live position-delete file. */ + val deletedMergeOnReadCasePrefix: String = "prep.morRead:" + + // --- the layouts, seeds and starting states the merge-on-read families build on --- + + /** + * One merge-on-read layout: a format-version 2 table whose delete, update and merge modes are merge-on-read, so a + * mutation records its change in position-delete files and leaves the untouched data files in place. + */ + private def mergeOnReadLayout(partitioning: Partitioning, format: String): Layout = + Layout( + s"mor-${partitioning.label}/$format", + table => + s"CREATE TABLE $table ($columnDefinitions) USING $dataSource ${partitioning.clause} " + + s"TBLPROPERTIES (${mergeOnReadProperties(format)})") + + /** + * The merge-on-read table property fragment for `format`: format-version 2 with the delete, update and merge modes + * all set to merge-on-read. + */ + protected def mergeOnReadProperties(format: String): String = + s"'write.format.default'='$format', 'format-version'='2', " + + "'write.delete.mode'='merge-on-read', 'write.update.mode'='merge-on-read', " + + "'write.merge.mode'='merge-on-read'" + + /** + * The copy-on-write table property fragment for `format`: format-version 2 with the delete, update and merge modes + * all set to copy-on-write, so a mutation rewrites the data file it matched. + */ + protected def copyOnWriteProperties(format: String): String = + s"'write.format.default'='$format', 'format-version'='2', " + + "'write.delete.mode'='copy-on-write', 'write.update.mode'='copy-on-write', " + + "'write.merge.mode'='copy-on-write'" + + /** The three properties that decide which write path a mutation takes. */ + val writeModePropertyNames: List[String] = + List("write.delete.mode", "write.update.mode", "write.merge.mode") + + /** + * Creates the table under `layout`, then seeds the standard rows through a single write task so they land in one + * data file. The COALESCE(1) hint is what forces the single file, which keeps a strict-subset delete a partial-file + * match: merge-on-read writes a position delete for it, and copy-on-write rewrites the data file. + */ + protected def singleFileSeed(layout: Layout): TableTest[CoreTable.type] = + create(layout) + .sql(s"seed($standardSeedRowCount, one-file)")(table => + s"INSERT INTO $table SELECT /*+ COALESCE(1) */ * FROM " + + s"(${RowGenerator.valuesClause(Core, standardSeedRowCount)}) AS seed")(view => + assert( + view.after.size == standardSeedRowCount, + s"the single-file seed lands $standardSeedRowCount rows, found ${view.after.size}")) + + /** + * Seeds the standard rows into one data file, then deletes the row with key 1. The table holds keys 2 and 3 behind + * a live position-delete file, which is the state every coexistence family starts from and the one a delete file + * makes reachable. + */ + protected def deletedMergeOnReadLineage(layout: Layout): TableTest[CoreTable.type] = + singleFileSeed(layout) + .step("prep.morDelete")((spark, table) => + spark.sql(s"DELETE FROM $table WHERE ${Core.long0.columnName} = 1"))(view => { + assert( + view.after.size == standardSeedRowCount - 1, + s"the preparation delete leaves ${standardSeedRowCount - 1} rows, found ${view.after.size}") + assert( + currentDeleteFileCount(view.spark, view.table) == 1, + s"the preparation leaves one live position-delete file, found " + + s"${currentDeleteFileCount(view.spark, view.table)}") + }) + + /** One preparation per layout given: three seed rows in one data file, with key 1 deleted merge-on-read. */ + protected def preparedDeletedTables( + layouts: List[Layout], + casePrefix: String): List[TablePreparation[CoreTable.type]] = + layouts.map(layout => + TablePreparation(layout.label, deletedMergeOnReadLineage(layout), casePrefix)) + + /** + * The number of delete files the table's current snapshot references, which is what a reader applies at scan time. + * Every assertion about the table as it stands now reads this. + */ + protected def currentDeleteFileCount( + spark: org.apache.spark.sql.SparkSession, + table: String): Long = + spark.sql(s"SELECT count(*) FROM $table.delete_files").collect()(0).getLong(0) + + /** + * The snapshot the table's main branch currently reads from, read from the refs metadata table, which names exactly + * one snapshot per branch. + */ + protected def currentSnapshotId( + spark: org.apache.spark.sql.SparkSession, + table: String): Long = + spark + .sql(s"SELECT snapshot_id FROM $table.refs WHERE name = 'main'") + .collect() + .toSeq + .map(_.getLong(0)) match { + case Seq(snapshotId) => snapshotId + case mainSnapshotIds => + throw new AssertionError(s"main names one snapshot, found $mainSnapshotIds") + } + + /** The snapshot IDs the table still retains. */ + protected def retainedSnapshotIds( + spark: org.apache.spark.sql.SparkSession, + table: String): Seq[Long] = + spark + .sql(s"SELECT snapshot_id FROM $table.snapshots") + .collect() + .toSeq + .map(_.getLong(0)) + + /** The manifest paths the table's current snapshot references, for the given manifest content code. */ + protected def currentManifestPaths( + spark: org.apache.spark.sql.SparkSession, + table: String, + manifestContent: Int): Set[String] = + spark + .sql(s"SELECT path FROM $table.manifests WHERE content = $manifestContent") + .collect() + .toSeq + .map(_.getString(0)) + .toSet + + /** The manifest content code for the manifests that list data files. */ + protected val dataManifestContent: Int = 0 + + /** The manifest content code for the manifests that list delete files. */ + protected val deleteManifestContent: Int = 1 + + /** + * The data-file paths the table's current snapshot references. The `files` metadata table lists delete files + * alongside data files, so the content code selects the data files on their own. + */ + protected def currentDataFilePaths( + spark: org.apache.spark.sql.SparkSession, + table: String): Set[String] = + spark + .sql(s"SELECT file_path FROM $table.files WHERE content = $dataFileContent") + .collect() + .toSeq + .map(_.getString(0)) + .toSet + + /** The number of data files the table's current snapshot references. */ + protected def currentDataFileCount( + spark: org.apache.spark.sql.SparkSession, + table: String): Long = + currentDataFilePaths(spark, table).size.toLong + + /** The file content code for a data file, as the `files` metadata table reports it. */ + protected val dataFileContent: Int = 0 + + /** The persisted value of `propertyName`, which is what the table is actually configured with. */ + protected def persistedProperty( + spark: org.apache.spark.sql.SparkSession, + table: String, + propertyName: String): Option[String] = + tableProps(spark, table).get(propertyName) + + /** The live keys the table reads back, in key order, with every position delete applied. */ + protected def liveKeys(spark: org.apache.spark.sql.SparkSession, table: String): Seq[Long] = + spark + .sql(s"SELECT ${Core.long0.columnName} FROM $table ORDER BY ${Core.long0.columnName}") + .collect() + .toSeq + .map(_.getLong(0)) + + /** The standard seed written as one data file in a merge-on-read table in `format`. */ + protected def preparedSingleFileMergeOnReadTable( + format: String): TablePreparation[CoreTable.type] = + TablePreparation( + format, + singleFileSeed( + Layout( + format, + table => + s"CREATE TABLE $table ($columnDefinitions) USING $dataSource TBLPROPERTIES (" + + s"${mergeOnReadProperties(format)})"))) + + /** The standard seed written as one data file in a copy-on-write table in `format`. */ + protected def preparedSingleFileCopyOnWriteTable( + format: String): TablePreparation[CoreTable.type] = + TablePreparation( + format, + singleFileSeed( + Layout( + format, + table => + s"CREATE TABLE $table ($columnDefinitions) USING $dataSource TBLPROPERTIES (" + + s"${copyOnWriteProperties(format)})"))) + + /** The standard seed in a merge-on-read table in `format`, labelled so its IDs name the write mode they ran on. */ + protected def preparedMergeOnReadTable(format: String): TablePreparation[CoreTable.type] = + TablePreparation( + s"mor/$format", + TableTest(Core) + .sql("create")(table => + s"CREATE TABLE $table ($columnDefinitions) USING $dataSource " + + s"TBLPROPERTIES (${mergeOnReadProperties(format)})")() + .insert(standardSeedRowCount)()) + + /** + * Creates a replace-lineage merge-on-read table: the standard seed, re-specified in place by CREATE OR REPLACE + * TABLE AS SELECT restating the merge-on-read modes, then refreshed so the Spark session reads the committed + * metadata pointer. Each step validates the state it leaves, so a mutation case that runs on the result starts from + * a known baseline. + */ + private def replaceLineageMergeOnRead( + partitioning: Partitioning, + format: String): TableTest[CoreTable.type] = + TableTest(Core) + .sql("create")(table => + s"CREATE TABLE $table ($columnDefinitions) USING $dataSource ${partitioning.clause} " + + s"TBLPROPERTIES (${mergeOnReadProperties(format)}, 'replace.enabled'='true')")() + .insert(standardSeedRowCount)() + .sql("prep.rtasMor")(table => + s"CREATE OR REPLACE TABLE $table USING $dataSource ${partitioning.clause} " + + s"TBLPROPERTIES (${mergeOnReadProperties(format)}) AS SELECT * FROM $table")(view => { + assertSeededMergeOnReadShape(view, "prep.rtasMor") + assert( + view.snapshotsAfter == view.snapshotsBefore + 1, + s"prep.rtasMor commits one snapshot, went from ${view.snapshotsBefore} to " + + s"${view.snapshotsAfter}") + }) + .sql("prep.rtasMor.refresh")(table => s"REFRESH TABLE $table")(view => { + assertSeededMergeOnReadShape(view, "prep.rtasMor.refresh") + assert( + view.snapshotsAfter == view.snapshotsBefore, + s"prep.rtasMor.refresh reads committed metadata and commits nothing, went from " + + s"${view.snapshotsBefore} to ${view.snapshotsAfter} snapshots") + }) + + /** + * The state both replace-lineage steps leave behind: the standard seed rows in key order, unchanged by the step, + * under exactly the core columns, on a table still configured merge-on-read. Asserting it here means a mutation + * case always compares against a known baseline. + */ + private def assertSeededMergeOnReadShape( + view: StepView[CoreTable.type], + stepLabel: String): Unit = { + val schemaColumnNames = view.spark.table(view.table).schema.fieldNames.toSeq + val configuredWriteModes = writeModePropertyNames.map(propertyName => + propertyName -> persistedProperty(view.spark, view.table, propertyName)) + + assert( + schemaColumnNames == Core.columnNames, + s"$stepLabel presents the core schema, found $schemaColumnNames") + assert( + view.after == view.before, + s"$stepLabel keeps every row it started from, went from ${view.before} to ${view.after}") + assert( + view.after.size == standardSeedRowCount, + s"$stepLabel holds the $standardSeedRowCount standard seed rows, found ${view.after.size}") + assert( + view.after.map(row => Rows.TypedRow(row).get(Core.long0)) == + (1L to standardSeedRowCount.toLong).toList, + s"$stepLabel holds the standard seed keys, found " + + s"${view.after.map(row => Rows.TypedRow(row).get(Core.long0))}") + assert( + configuredWriteModes == writeModePropertyNames.map(_ -> Some("merge-on-read")), + s"$stepLabel keeps every write mode on the merge-on-read path, found $configuredWriteModes") + } + +} diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioMergeOnReadMaintenance.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioMergeOnReadMaintenance.scala new file mode 100644 index 000000000..9821a0162 --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioMergeOnReadMaintenance.scala @@ -0,0 +1,444 @@ +package harness + +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter +import java.util.concurrent.TimeUnit + +import org.apache.hadoop.fs.{FileSystem, Path} + +/** + * Maintenance over a live position-delete file. + * + * Each maintenance procedure has to decide whether to fold the delete into the data it rewrites, carry it forward, or + * leave it in place. All of them keep the row the delete removed out of the live row set, so a table that accumulates + * delete files stays maintainable and a maintenance run stays safe to schedule. + * + * Every family reads the procedure's own report of what it rewrote, expired or removed, so each case proves the + * effect it is named for and not only that the rows survived it. + * + * Operations: rewrite_data_files, rewrite_position_delete_files, expire_snapshots, rewrite_manifests, + * remove_orphan_files against an orphan the case plants and backdates itself, and a compaction followed by an + * expiration whose two effects are proven separately. + * + * Preparation axes: the two merge-on-read verify layouts, each seeded into one data file with key 1 deleted, for the + * six families that start from a live delete; and the single-file merge-on-read table in each format for the family + * that writes its own delete first. + * + * Case families: seven families contributing 14 cases, 12 on the two deleted preparations and 2 on the self-deleting + * one. + */ +trait ScenarioMergeOnReadMaintenance extends ScenarioMergeOnReadKit { + + /** Every merge-on-read maintenance case, one deleted preparation at a time, then the self-deleting family. */ + lazy val mergeOnReadMaintenanceCases: List[TestCase] = + preparedDeletedMergeOnReadTables.flatMap { preparation => + List( + rewriteDataFilesLeavesDanglingDeleteCase(preparation), + rewritePositionDeleteFilesFoldsDanglingDeleteCase(preparation), + expireSnapshotsKeepsDeleteCase(preparation), + rewriteManifestsKeepsDeleteCase(preparation), + removeOrphanFilesKeepsDeleteCase(preparation), + compactThenExpireKeepsDeleteCase(preparation)) + } ++ fileFormats.map(format => + rewritePositionDeleteFilesCompactsCase(preparedSingleFileMergeOnReadTable(format))) + + // --- the case bodies the surface above composes --- + + /** + * rewrite_data_files folds the live delete into the data it compacts, so the deleted key stays gone and the two live + * rows read back, while the position-delete file it superseded stays referenced until a later procedure clears it. + */ + private def rewriteDataFilesLeavesDanglingDeleteCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("mergeOnRead.maintenance.rewriteDataFilesFoldsDelete") { table => + assert( + currentDeleteFileCount(table.spark, table.name) == 1, + s"the preparation leaves the delete this compaction folds, found " + + s"${currentDeleteFileCount(table.spark, table.name)}") + + val dataFilePathsBefore = currentDataFilePaths(table.spark, table.name) + val rewriteReport = table.spark + .sql( + "CALL openhouse.system.rewrite_data_files(" + + s"table => '${catalogRelative(table.name)}', " + + "options => map('rewrite-all', 'true'))") + .collect()(0) + val dataFilePathsAfter = currentDataFilePaths(table.spark, table.name) + + assert( + rewriteReport.getInt(0) == dataFilePathsBefore.size, + s"the compaction rewrites the ${dataFilePathsBefore.size} data files it started from, " + + s"rewrote ${rewriteReport.getInt(0)}") + assert( + rewriteReport.getInt(1) == dataFilePathsAfter.size, + s"the compaction adds the ${dataFilePathsAfter.size} data files it left behind, added " + + s"${rewriteReport.getInt(1)}") + assert( + dataFilePathsAfter.intersect(dataFilePathsBefore).isEmpty, + s"every data file the compaction rewrote leaves the current set, " + + s"${dataFilePathsAfter.intersect(dataFilePathsBefore)} stayed") + assert( + liveKeys(table.spark, table.name) == Seq(2L, 3L), + s"compaction folds the delete and keeps the live rows, found " + + s"${liveKeys(table.spark, table.name)}") + } + + /** + * rewrite_position_delete_files after a compaction clears the position-delete file the compaction superseded, and + * the live row set stays as it was, so the two procedures together return the table to a delete-free state. + */ + private def rewritePositionDeleteFilesFoldsDanglingDeleteCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("mergeOnRead.maintenance.rewritePositionDeletesClearsDangling") { table => + table.spark.sql( + "CALL openhouse.system.rewrite_data_files(" + + s"table => '${catalogRelative(table.name)}', " + + "options => map('rewrite-all', 'true'))") + val danglingDeleteFileCount = currentDeleteFileCount(table.spark, table.name) + val rewriteReport = table.spark + .sql( + "CALL openhouse.system.rewrite_position_delete_files(" + + s"table => '${catalogRelative(table.name)}', " + + "options => map('rewrite-all', 'true'))") + .collect()(0) + + assert( + danglingDeleteFileCount >= 1, + s"the compaction leaves the delete file this call clears, found $danglingDeleteFileCount") + assert( + rewriteReport.getInt(0) == danglingDeleteFileCount, + s"the call rewrites the $danglingDeleteFileCount delete files it found, rewrote " + + s"${rewriteReport.getInt(0)}") + assert( + rewriteReport.getInt(1) == currentDeleteFileCount(table.spark, table.name), + s"the call adds the ${currentDeleteFileCount(table.spark, table.name)} delete files it " + + s"left behind, added ${rewriteReport.getInt(1)}") + assert( + currentDeleteFileCount(table.spark, table.name) == 0, + s"the folded delete is cleared, found " + + s"${currentDeleteFileCount(table.spark, table.name)} delete files") + assert( + liveKeys(table.spark, table.name) == Seq(2L, 3L), + s"clearing the folded delete keeps the live rows, found " + + s"${liveKeys(table.spark, table.name)}") + } + + /** + * expire_snapshots drops every snapshot the table no longer needs to retain and keeps the one it currently reads + * from, so the history shrinks to the retained snapshot while the live rows and the delete file the reader applies + * stay exactly as they were. + */ + private def expireSnapshotsKeepsDeleteCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("mergeOnRead.maintenance.expireSnapshotsKeepsDelete") { table => + val snapshotIdsBefore = retainedSnapshotIds(table.spark, table.name) + val currentSnapshotIdBefore = currentSnapshotId(table.spark, table.name) + val deleteFileCountBefore = currentDeleteFileCount(table.spark, table.name) + + assert( + snapshotIdsBefore.size >= 2, + s"the preparation leaves history for the expiration to drop, found $snapshotIdsBefore") + + table.spark.sql( + "CALL openhouse.system.expire_snapshots(" + + s"table => '${catalogRelative(table.name)}', " + + s"older_than => TIMESTAMP '$expirationCutoff', " + + "retain_last => 1)") + val snapshotIdsAfter = retainedSnapshotIds(table.spark, table.name) + + assert( + snapshotIdsAfter == Seq(currentSnapshotIdBefore), + s"the expiration retains the snapshot the table reads from and drops the rest, " + + s"went from $snapshotIdsBefore to $snapshotIdsAfter") + assert( + snapshotIdsBefore.filterNot(_ == currentSnapshotIdBefore).forall(expiredSnapshotId => + !snapshotIdsAfter.contains(expiredSnapshotId)), + s"every superseded snapshot is gone, found $snapshotIdsAfter") + assert( + currentSnapshotId(table.spark, table.name) == currentSnapshotIdBefore, + "the expiration leaves the table reading from the snapshot it was already on") + assert( + currentDeleteFileCount(table.spark, table.name) == deleteFileCountBefore, + s"the expiration keeps the $deleteFileCountBefore delete files the reader applies, found " + + s"${currentDeleteFileCount(table.spark, table.name)}") + assert( + liveKeys(table.spark, table.name) == Seq(2L, 3L), + s"expiration keeps the delete applied, found ${liveKeys(table.spark, table.name)}") + } + + /** + * rewrite_manifests replaces the manifests the table references with the ones it wrote, and the data files, delete + * files and live rows the manifests point at stay exactly as they were. + */ + private def rewriteManifestsKeepsDeleteCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("mergeOnRead.maintenance.rewriteManifestsKeepsDelete") { table => + // Each append commits its own data manifest, so the rewrite has several to merge into one. + val appendedKeys = List(4L, 5L, 6L) + appendedKeys.foreach(key => + table.spark.sql(s"INSERT INTO ${table.name} VALUES ${coreRow(key, s"row-$key")}")) + + // rewrite_manifests rewrites the data manifests, so the data manifests present beforehand are the eligible set + // and the delete manifests are the part it is expected to leave alone. + val dataManifestPathsBefore = + currentManifestPaths(table.spark, table.name, dataManifestContent) + val deleteManifestPathsBefore = + currentManifestPaths(table.spark, table.name, deleteManifestContent) + val dataFileCountBefore = currentDataFileCount(table.spark, table.name) + val deleteFileCountBefore = currentDeleteFileCount(table.spark, table.name) + val rewriteReport = table.spark + .sql( + "CALL openhouse.system.rewrite_manifests(" + + s"table => '${catalogRelative(table.name)}')") + .collect()(0) + val dataManifestPathsAfter = + currentManifestPaths(table.spark, table.name, dataManifestContent) + + assert( + dataManifestPathsBefore.size >= 2, + s"the appends leave several data manifests for the rewrite to merge, found " + + s"${dataManifestPathsBefore.size}") + assert( + rewriteReport.getInt(0) == dataManifestPathsBefore.size, + s"the call rewrites the ${dataManifestPathsBefore.size} data manifests it started from, " + + s"rewrote ${rewriteReport.getInt(0)}") + assert( + rewriteReport.getInt(1) == dataManifestPathsAfter.size, + s"the call adds the ${dataManifestPathsAfter.size} data manifests it left behind, added " + + s"${rewriteReport.getInt(1)}") + assert( + dataManifestPathsAfter.intersect(dataManifestPathsBefore).isEmpty, + s"every data manifest the rewrite merged leaves the current set, " + + s"${dataManifestPathsAfter.intersect(dataManifestPathsBefore)} stayed") + assert( + currentManifestPaths(table.spark, table.name, deleteManifestContent) == + deleteManifestPathsBefore, + s"the rewrite leaves the delete manifests as they were, found " + + s"${currentManifestPaths(table.spark, table.name, deleteManifestContent)}") + assert( + currentDataFileCount(table.spark, table.name) == dataFileCountBefore, + s"manifest rewriting keeps the $dataFileCountBefore data files, found " + + s"${currentDataFileCount(table.spark, table.name)}") + assert( + currentDeleteFileCount(table.spark, table.name) == deleteFileCountBefore, + s"manifest rewriting keeps the $deleteFileCountBefore delete files, found " + + s"${currentDeleteFileCount(table.spark, table.name)}") + assert( + liveKeys(table.spark, table.name) == Seq(2L, 3L) ++ appendedKeys, + s"manifest rewriting keeps the delete applied, found ${liveKeys(table.spark, table.name)}") + } + + /** + * remove_orphan_files removes exactly the unreferenced file the case plants under the table's data directory and + * leaves every referenced data file, delete file and row in place. + * + * The case owns the orphan end to end: it writes the file itself, backdates its modification time behind the + * cutoff through the Hadoop FileSystem the table's own path resolves to, asserts the procedure reports that one + * location, and deletes the orphan on the way out if the procedure left it. The table's referenced files were + * written moments ago, so they sit ahead of the cutoff and are outside the removal window, which is what makes + * "exactly the orphan" a real assertion. + * + * Locations are compared as fully qualified paths resolved through the same filesystem, so the scheme and + * authority the procedure reports line up with the planted path on local storage and on HDFS alike. + */ + private def removeOrphanFilesKeepsDeleteCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("mergeOnRead.maintenance.removeOrphanFilesRemovesTheOrphan") { table => + val dataFileCountBefore = currentDataFileCount(table.spark, table.name) + val deleteFileCountBefore = currentDeleteFileCount(table.spark, table.name) + val fileSystem = tableFileSystem(table) + val orphanPath = plantBackdatedOrphanFile(table, fileSystem) + + OwnedTableLifecycle.withCleanup( + if (fileSystem.exists(orphanPath)) { + assert( + fileSystem.delete(orphanPath, false), + s"the case removes the orphan it planted at $orphanPath") + }) { + val removedPaths = table.spark + .sql( + "CALL openhouse.system.remove_orphan_files(" + + s"table => '${catalogRelative(table.name)}', " + + s"older_than => TIMESTAMP '${orphanRemovalCutoffTimestamp()}')") + .collect() + .toSeq + .map(row => qualified(fileSystem, new Path(row.getString(0)))) + + assert( + removedPaths == Seq(orphanPath), + s"the call removes exactly the planted orphan $orphanPath, removed $removedPaths") + assert( + !fileSystem.exists(orphanPath), + s"the removed orphan is gone from storage, $orphanPath is still there") + assert( + currentDataFileCount(table.spark, table.name) == dataFileCountBefore, + s"orphan removal keeps the $dataFileCountBefore referenced data files, found " + + s"${currentDataFileCount(table.spark, table.name)}") + assert( + currentDeleteFileCount(table.spark, table.name) == deleteFileCountBefore, + s"orphan removal keeps the $deleteFileCountBefore referenced delete files, found " + + s"${currentDeleteFileCount(table.spark, table.name)}") + assert( + liveKeys(table.spark, table.name) == Seq(2L, 3L), + s"orphan removal keeps the delete applied, found ${liveKeys(table.spark, table.name)}") + } + } + + /** The filesystem the table's own data files resolve through, which is the one the procedure reports against. */ + private def tableFileSystem(table: PreparedTable[CoreTable.type]): FileSystem = + referencedDataFilePath(table).getFileSystem(table.spark.sessionState.newHadoopConf()) + + /** One data-file path the table's current snapshot references. */ + private def referencedDataFilePath(table: PreparedTable[CoreTable.type]): Path = + new Path( + table.spark + .sql(s"SELECT file_path FROM ${table.name}.files LIMIT 1") + .collect()(0) + .getString(0)) + + /** + * The fully qualified form of `path` on `fileSystem`, carrying its scheme and authority. Comparing qualified paths + * keeps the assertion correct whether the procedure reports a bare path, a `file:` URI or an `hdfs:` URI. + */ + private def qualified(fileSystem: FileSystem, path: Path): Path = + path.makeQualified(fileSystem.getUri, fileSystem.getWorkingDirectory) + + /** + * Writes an unreferenced file beside the table's data files and backdates it well behind the removal cutoff, then + * returns its qualified path. Placing it beside a referenced data file puts it inside the directory tree the + * procedure scans, and it is unreferenced because no manifest names it. + */ + private def plantBackdatedOrphanFile( + table: PreparedTable[CoreTable.type], + fileSystem: FileSystem): Path = { + val orphanPath = qualified( + fileSystem, + new Path(referencedDataFilePath(table).getParent, "harness-planted-orphan.parquet")) + val backdatedModificationTime = + System.currentTimeMillis() - TimeUnit.DAYS.toMillis(orphanAgeDays) + + fileSystem.create(orphanPath, true).close() + fileSystem.setTimes(orphanPath, backdatedModificationTime, -1L) + + assert( + fileSystem.getFileStatus(orphanPath).getModificationTime == backdatedModificationTime, + s"the planted orphan carries the backdated modification time the cutoff is measured " + + s"against, found ${fileSystem.getFileStatus(orphanPath).getModificationTime} for " + + s"$backdatedModificationTime") + orphanPath + } + + /** How far behind the present the planted orphan's modification time sits. */ + private val orphanAgeDays = 30L + + /** How far behind the present the removal cutoff sits, which the procedure requires to exceed 24 hours. */ + private val orphanRemovalCutoffDays = 7L + + /** + * The older_than cutoff for orphan removal: far enough back that the procedure accepts it and the table's + * just-written files sit ahead of it, and recent enough that the planted orphan sits behind it. + */ + private def orphanRemovalCutoffTimestamp(): String = + LocalDateTime + .now() + .minusDays(orphanRemovalCutoffDays) + .format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")) + + /** + * The pair a scheduled maintenance run issues together, with each half proven on its own: the compaction rewrites + * the delete files it found, then the expiration drops the history that compaction superseded and leaves the table + * reading from one snapshot. The live rows are the same at the end as at the start. + */ + private def compactThenExpireKeepsDeleteCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("mergeOnRead.maintenance.compactThenExpireKeepsDelete") { table => + val deleteFileCountBefore = currentDeleteFileCount(table.spark, table.name) + val compactionReport = table.spark + .sql( + "CALL openhouse.system.rewrite_position_delete_files(" + + s"table => '${catalogRelative(table.name)}', " + + "options => map('rewrite-all', 'true'))") + .collect()(0) + + assert( + deleteFileCountBefore >= 1, + s"the preparation leaves the delete files this compaction rewrites, found $deleteFileCountBefore") + assert( + compactionReport.getInt(0) == deleteFileCountBefore, + s"the compaction rewrites the $deleteFileCountBefore delete files it found, rewrote " + + s"${compactionReport.getInt(0)}") + assert( + compactionReport.getInt(1) == currentDeleteFileCount(table.spark, table.name), + s"the compaction adds the ${currentDeleteFileCount(table.spark, table.name)} delete " + + s"files it left behind, added ${compactionReport.getInt(1)}") + assert( + liveKeys(table.spark, table.name) == Seq(2L, 3L), + s"the compaction keeps the delete applied, found ${liveKeys(table.spark, table.name)}") + + val snapshotIdsBeforeExpiration = retainedSnapshotIds(table.spark, table.name) + val currentSnapshotIdBeforeExpiration = currentSnapshotId(table.spark, table.name) + table.spark.sql( + "CALL openhouse.system.expire_snapshots(" + + s"table => '${catalogRelative(table.name)}', " + + s"older_than => TIMESTAMP '$expirationCutoff', " + + "retain_last => 1)") + val snapshotIdsAfterExpiration = retainedSnapshotIds(table.spark, table.name) + + assert( + snapshotIdsBeforeExpiration.size >= 2, + s"the compaction leaves history for the expiration to drop, found $snapshotIdsBeforeExpiration") + assert( + snapshotIdsAfterExpiration == Seq(currentSnapshotIdBeforeExpiration), + s"the expiration retains the compacted snapshot and drops the rest, went from " + + s"$snapshotIdsBeforeExpiration to $snapshotIdsAfterExpiration") + assert( + liveKeys(table.spark, table.name) == Seq(2L, 3L), + s"compaction followed by expiration keeps the delete applied, found " + + s"${liveKeys(table.spark, table.name)}") + } + + /** + * The older_than cutoff for snapshot expiration. It is far ahead of any snapshot the harness commits, so every + * snapshot outside the retained one is inside the expiration window and the call has real work to do. + */ + private val expirationCutoff = "2999-01-01 00:00:00" + + /** + * A merge-on-read DELETE writes one position-delete file, and rewrite_position_delete_files compacts it while the + * two surviving rows stay readable, so the procedure is available to a table that accumulates delete files. + */ + private def rewritePositionDeleteFilesCompactsCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("mergeOnRead.maintenance.rewritePositionDeleteFiles") { table => + assert( + persistedProperty(table.spark, table.name, "write.delete.mode").contains("merge-on-read"), + "the table persists write.delete.mode as merge-on-read before the delete under test") + + table.spark.sql(s"DELETE FROM ${table.name} WHERE ${Core.long0.columnName} = 1") + + assert( + currentDeleteFileCount(table.spark, table.name) == 1, + s"the merge-on-read delete writes one position-delete file, found " + + s"${currentDeleteFileCount(table.spark, table.name)}") + + val rewriteReport = table.spark + .sql( + "CALL openhouse.system.rewrite_position_delete_files(" + + s"table => '${catalogRelative(table.name)}', " + + "options => map('rewrite-all', 'true'))") + .collect()(0) + + assert( + rewriteReport.getInt(0) == 1, + s"the call rewrites the one delete file it found, rewrote ${rewriteReport.getInt(0)}") + assert( + rewriteReport.getInt(1) == currentDeleteFileCount(table.spark, table.name), + s"the call adds the ${currentDeleteFileCount(table.spark, table.name)} delete files it " + + s"left behind, added ${rewriteReport.getInt(1)}") + assert( + liveKeys(table.spark, table.name) == Seq(2L, 3L), + s"compacting the position deletes keeps the live rows, found " + + s"${liveKeys(table.spark, table.name)}") + } + +} diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioNestedType.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioNestedType.scala new file mode 100644 index 000000000..f4455e71a --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioNestedType.scala @@ -0,0 +1,260 @@ +package harness + +/** + * Nested and complex types: struct, array, map and struct-in-struct columns, the reads and writes that address their + * fields, and the schema changes the catalog allows inside a struct. + * + * Operations: a full round trip of every nested column, projection of a struct field, a filter on a struct field, an + * UPDATE of a struct field, a MERGE that inserts a fully nested row, a DELETE filtered on a struct field, an INSERT of + * null and empty nested values, ADD COLUMN of a new struct field, and the rejected DROP COLUMN of an existing struct + * field. + * + * Preparation axes: one unpartitioned NestedTable layout per file format, each seeded with three rows carrying struct, + * array, map and doubly-nested struct values; plus the standard seeded core table in Parquet and ORC for the two + * struct-evolution families, which build and drop their own side table. + * + * Case families: nine families contributing 18 cases, 14 on the nested layouts and 4 on the core formats. + */ +trait ScenarioNestedType extends ScenarioKit { + + /** Every nested-type case: the reads and writes on the nested layouts, then the struct-evolution cases. */ + lazy val nestedTypeCases: List[TestCase] = + preparedNestedTables.flatMap(preparation => + List( + roundtripCase(preparation), + projectFieldCase(preparation), + filterNestedFieldCase(preparation), + updateStructFieldCase(preparation), + mergeInsertCase(preparation), + deleteByNestedFieldCase(preparation), + nullValuesCase(preparation))) ++ + preparedCoreFormats.flatMap(preparation => + List( + addStructFieldCase(preparation), + dropStructFieldRejectedCase(preparation))) + + /** One unpartitioned nested-column table per file format. */ + lazy val nestedLayouts: List[Layout] = + fileFormats.map(format => + Layout( + s"nested-unpartitioned/$format", + table => + s"CREATE TABLE $table (${NestedTable.columnDefinitions}) USING $dataSource " + + s"TBLPROPERTIES ('write.format.default'='$format')")) + + /** One preparation per nested layout: the table is created, then seeded with three nested rows. */ + lazy val preparedNestedTables: List[TablePreparation[NestedTable.type]] = + nestedLayouts.map(layout => + TablePreparation( + layout.label, + TableTest(NestedTable).sql("create")(layout.create)().insert(standardSeedRowCount)())) + + // --- the preparations, shared helpers and case bodies the surface above composes --- + + /** + * Selecting the top-level id alongside struct, array, map and nested-struct fields reads back exactly the seeded + * values for all 3 rows. + */ + private def roundtripCase(preparation: TablePreparation[NestedTable.type]): TestCase = + preparation.test("nested.roundtrip") { table => + val actual = table.spark + .sql( + s"SELECT id, s.x, s.y, arr, m['k'], nested.inner.z " + + s"FROM ${table.name} ORDER BY id") + .collect() + .toSeq + .map(row => + ( + row.getLong(0), + row.getInt(1), + row.getString(2), + row.getSeq[Int](3), + row.getInt(4), + row.getInt(5))) + val expected = (1 to standardSeedRowCount).map { value => + ( + value.toLong, + value, + s"row-$value", + Seq(value, value + 1), + value, + value) + } + + assert(actual == expected) + } + + /** Selecting only a nested struct field (s.x) returns just that field's values for all 3 rows, in id order. */ + private def projectFieldCase(preparation: TablePreparation[NestedTable.type]): TestCase = + preparation.test("nested.projectField") { table => + val actual = table.spark + .sql(s"SELECT s.x FROM ${table.name} ORDER BY id") + .collect() + .toSeq + .map(_.getInt(0)) + + assert(actual == Seq(1, 2, 3)) + } + + /** Filtering WHERE s.x = 2 on a nested struct field returns only the matching row's id. */ + private def filterNestedFieldCase(preparation: TablePreparation[NestedTable.type]): TestCase = + preparation.test("nested.filterNestedField") { table => + val actual = table.spark + .sql(s"SELECT id FROM ${table.name} WHERE s.x = 2 ORDER BY id") + .collect() + .toSeq + .map(_.getLong(0)) + + assert(actual == Seq(2L)) + } + + /** UPDATE SET s.x = 99 WHERE id = 2 changes only that row's nested field and leaves every other row unchanged. */ + private def updateStructFieldCase(preparation: TablePreparation[NestedTable.type]): TestCase = + preparation.test("nested.updateStructField") { table => + table.spark.sql( + s"UPDATE ${table.name} SET s.x = 99 WHERE id = 2") + + assert( + table.spark + .sql(s"SELECT s.x FROM ${table.name} WHERE id = 2") + .collect()(0) + .getInt(0) == 99) + assert( + table.spark + .sql(s"SELECT s.x FROM ${table.name} WHERE id = 1") + .collect()(0) + .getInt(0) == 1) + } + + /** + * MERGE WHEN NOT MATCHED THEN INSERT with a fully nested source row adds a 4th row whose nested struct field reads + * back as inserted. + */ + private def mergeInsertCase(preparation: TablePreparation[NestedTable.type]): TestCase = + preparation.test("nested.mergeInsert") { table => + table.spark.sql( + s"""MERGE INTO ${table.name} target USING ( + SELECT * FROM VALUES + ( + CAST(4 AS BIGINT), + named_struct('x', 4, 'y', 'row-4'), + array(4, 5), + map('k', 4), + named_struct('inner', named_struct('z', 4))) + AS source(id, s, arr, m, nested) + ) source ON target.id = source.id + WHEN NOT MATCHED THEN INSERT *""") + + val ids = table.spark + .sql(s"SELECT id FROM ${table.name} ORDER BY id") + .collect() + .toSeq + .map(_.getLong(0)) + + assert(ids == Seq(1L, 2L, 3L, 4L)) + assert( + table.spark + .sql(s"SELECT s.x FROM ${table.name} WHERE id = 4") + .collect()(0) + .getInt(0) == 4) + } + + /** DELETE WHERE s.x = 2 filtering on a nested struct field removes only the matching row, leaving ids 1 and 3. */ + private def deleteByNestedFieldCase(preparation: TablePreparation[NestedTable.type]): TestCase = + preparation + .test("nested.deleteByNestedField") { table => + table.spark.sql( + s"DELETE FROM ${table.name} WHERE s.x = 2") + + val ids = table.spark + .sql(s"SELECT id FROM ${table.name} ORDER BY id") + .collect() + .toSeq + .map(_.getLong(0)) + + assert(ids == Seq(1L, 3L)) + } + .copy(knownBugReason = Some( + "DELETE on a nested struct field crashes in the Spark and Iceberg row-level " + + "rewrite.")) + + /** + * Inserting a row with NULL struct, empty array and empty map reads back a null struct and an empty array for that + * row. + */ + private def nullValuesCase(preparation: TablePreparation[NestedTable.type]): TestCase = + preparation.test("nested.nullValues") { table => + table.spark.sql( + s"INSERT INTO ${table.name} VALUES (" + + "CAST(4 AS BIGINT), " + + "CAST(NULL AS struct), " + + "CAST(array() AS array), " + + "CAST(map() AS map), " + + "CAST(NULL AS struct>))") + + val insertedRow = table.spark + .sql(s"SELECT id, s, arr FROM ${table.name} WHERE id = 4") + .collect()(0) + + assert(insertedRow.isNullAt(1)) + assert(insertedRow.getSeq[Int](2).isEmpty) + } + + /** + * On a side table, ADD COLUMN of a new nested struct field null-fills it for the existing row and accepts a new row + * that sets the field. + */ + private def addStructFieldCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("nested.addStructField") { table => + val sideTable = s"${table.name}_nst" + withOwnedTable(table.spark.sql(_), sideTable)( + table.spark.sql( + s"CREATE TABLE $sideTable " + + s"(id BIGINT, s STRUCT) USING $dataSource")) { + table.spark.sql( + s"INSERT INTO $sideTable VALUES " + + "(CAST(1 AS BIGINT), named_struct('x', 1, 'y', 'a'))") + table.spark.sql( + s"ALTER TABLE $sideTable ADD COLUMN s.w INT") + assert( + countOf(table.spark, s"SELECT count(*) FROM $sideTable WHERE s.w IS NULL") == "1", + "new nested field should null-fill the existing row") + + table.spark.sql( + s"INSERT INTO $sideTable VALUES " + + "(CAST(2 AS BIGINT), " + + "named_struct('x', 2, 'y', 'b', 'w', 9))") + assert( + countOf(table.spark, s"SELECT count(*) FROM $sideTable WHERE s.w = 9") == "1", + "new nested field should be writable") + } + } + + /** + * On a side table, ALTER TABLE DROP COLUMN of a nested struct field is rejected with an exception, and the field + * remains readable afterward. + */ + private def dropStructFieldRejectedCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("nested.dropStructField.rejected") { table => + val sideTable = s"${table.name}_nsd" + withOwnedTable(table.spark.sql(_), sideTable)( + table.spark.sql( + s"CREATE TABLE $sideTable " + + s"(id BIGINT, s STRUCT) USING $dataSource")) { + table.spark.sql( + s"INSERT INTO $sideTable VALUES " + + "(CAST(1 AS BIGINT), named_struct('x', 1, 'y', 'a'))") + Check.intercept[Exception]( + table.spark.sql( + s"ALTER TABLE $sideTable DROP COLUMN s.x")) + + assert( + table.spark + .sql(s"SELECT s.x FROM $sideTable") + .collect()(0) + .getInt(0) == 1, + "rejected nested drop should leave the field readable") + } + } + +} diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioPartitionEvolution.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioPartitionEvolution.scala new file mode 100644 index 000000000..bbb8cfa00 --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioPartitionEvolution.scala @@ -0,0 +1,60 @@ +package harness + +/** + * Partition evolution: changing the partition specification of an existing table. + * + * Operations: ALTER TABLE ADD PARTITION FIELD on an unpartitioned table and ALTER TABLE DROP PARTITION FIELD on a + * date-partitioned table. The catalog rejects both, so recreating the table is the way to change its partitioning. + * + * Preparation axes: in each columnar format, the standard seeded core table for the add case and a + * date-partitioned core table seeded with the standard rows for the drop case. + * + * Case families: two families contributing 4 cases. + */ +trait ScenarioPartitionEvolution extends ScenarioKit { + + /** The rejected partition-evolution statements, one file format at a time. */ + lazy val partitionEvolutionCases: List[TestCase] = + fileFormats.flatMap { format => + List( + addPartitionFieldRejectedCase(format), + dropPartitionFieldRejectedCase(format)) + } + + // --- the preparations, shared helpers and case bodies the surface above composes --- + + /** + * ALTER TABLE ADD PARTITION FIELD on an unpartitioned table is rejected with an exception stating that evolution of + * table partitioning is unsupported. + */ + private def addPartitionFieldRejectedCase(format: String): TestCase = + preparedStandardTable(format).test("partitionEvolution.add.rejected") { table => + val exception = Check.intercept[Exception]( + table.spark.sql( + s"ALTER TABLE ${table.name} ADD PARTITION FIELD ${Core.date0.columnName}")) + + assert(exception.getMessage.contains("Evolution of table partitioning")) + } + + /** + * ALTER TABLE DROP PARTITION FIELD on a date-partitioned table is rejected with an exception stating that evolution + * of table partitioning is unsupported. + */ + private def dropPartitionFieldRejectedCase(format: String): TestCase = + TablePreparation( + format, + TableTest(Core) + .sql("create")(table => + s"CREATE TABLE $table ($columnDefinitions) USING $dataSource " + + s"PARTITIONED BY (${Core.date0.columnName}) " + + s"TBLPROPERTIES ('write.format.default'='$format')")() + .insert(standardSeedRowCount)()) + .test("partitionEvolution.drop.rejected") { table => + val exception = Check.intercept[Exception]( + table.spark.sql( + s"ALTER TABLE ${table.name} DROP PARTITION FIELD ${Core.date0.columnName}")) + + assert(exception.getMessage.contains("Evolution of table partitioning")) + } + +} diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioRtas.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioRtas.scala new file mode 100644 index 000000000..623e452b7 --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioRtas.scala @@ -0,0 +1,1002 @@ +package harness + +import java.util.concurrent.ConcurrentHashMap + +import org.apache.iceberg.exceptions.{BadRequestException, ValidationException} +import org.apache.spark.sql.{AnalysisException, SparkSession} +import scala.util.control.NonFatal + +/** + * Replace table as select: what CREATE OR REPLACE TABLE AS SELECT is allowed to do to a table, and what survives it. + * + * A replace re-specifies a table in place and starts a new snapshot lineage under the same catalog identity. That + * makes it the one statement in the harness that can change a table's shape, its partitioning and its content at + * once, so this file owns two separate obligations. The first is that the reusable DML contract still holds on a + * table that reached its starting state through a replace. The second is that everything the catalog governs, and + * everything a reader can ask about history, behaves the way a new lineage requires. + * + * Operations, DML: every reusable operation `ScenarioDml` defines, reused as data. A replaced table runs the same + * statements and the same assertions as a freshly created one, so this file holds one definition of each operation. + * The 51 operations that + * fit any seeded table cross all four replace preparations, the null-string DELETE crosses the four replace + * preparations that carry a null row, and the two partition-scoped writes cross the two date-partitioned replace + * preparations. That is all 54 reusable operations on the preparations each one applies to. + * + * Operations, replace contract: 26 focused families covering the enablement gates, same-shape replacement and the + * write that follows it, the four schema discontinuities, partition replacement and the second replace that is the + * only legal way to repartition afterwards, property override and preservation, retention-policy and column-tag + * preservation, pre-replace time travel with rollback rejection and set-current-snapshot recovery, changelog and + * incremental-read rejection across the replacement boundary, replace crossed with rename in both orders, sort-order + * change and removal after a replace, creator-identity preservation, and a replace racing an append. + * + * Preparation axes: replace lineage is the axis this layer adds. Four replace preparations cross the two columnar + * formats with unpartitioned and date-partitioned tables; each creates a replace-enabled table, seeds the standard + * three rows, re-specifies the same shape through CREATE OR REPLACE TABLE AS SELECT, and refreshes, so the rows a + * case starts from arrived through the replace path. The contract families start from a plain replace-enabled table + * in each format, or from the property, retention or tag table each one needs, and drive the replace themselves so + * they can read the state on both sides of it. + * + * Case families: 264 cases. The DML axis contributes 212 in three families, and the replace contract contributes 52 + * in 26 families, each family running in both columnar formats. + */ +trait ScenarioRtas extends ScenarioKit { this: ScenarioDml with ChangelogSupport => + + /** Every replace case: the reusable DML operations on replaced tables first, then the replace contract. */ + lazy val rtasCases: List[TestCase] = rtasDmlCases ++ rtasContractCases + + /** + * The reusable DML operations on replaced tables, in preparation order: every operation on the four replace + * preparations, the null-string DELETE on their null-string form, then the partition-scoped writes on the two + * date-partitioned replace preparations. + */ + lazy val rtasDmlCases: List[TestCase] = + rtasCoreDmlCases ++ rtasNullStringDmlCases ++ rtasPartitionedDmlCases + + /** Every operation that fits any seeded table, on each of the four replace preparations. */ + lazy val rtasCoreDmlCases: List[TestCase] = + preparedRtasCoreTables.flatMap(preparation => allDmlTestCases.map(_.runOn(preparation))) + + /** The DELETE that selects a null string, on the replace preparations that carry a null row. */ + lazy val rtasNullStringDmlCases: List[TestCase] = + preparedNullStringRtasCoreTables.flatMap(preparation => + nullStringRowTestCases.map(_.runOn(preparation))) + + /** The partition-scoped writes, on the two date-partitioned replace preparations. */ + lazy val rtasPartitionedDmlCases: List[TestCase] = + preparedRtasPartitionedCoreTables.flatMap(preparation => + partitionedTableTestCases.map(_.runOn(preparation))) + + /** Every replace-contract case, one file format at a time. */ + lazy val rtasContractCases: List[TestCase] = + fileFormats.flatMap { format => + List( + enablementGateCase(preparedStandardTable(format)), + disabledGateRejectedCase(preparedStandardTable(format)), + replicationGateRejectedCase(preparedStandardTable(format)), + sameShapeReplacementCase(preparedReplaceEnabledTable(format)), + writeAfterReplaceCase(preparedReplaceEnabledTable(format)), + schemaAddColumnCase(preparedReplaceEnabledTable(format)), + schemaDropColumnCase(preparedReplaceEnabledTable(format)), + schemaWidenColumnCase(preparedReplaceEnabledTable(format)), + schemaIncompatibleTypeRejectedCase(preparedReplaceEnabledTable(format)), + partitionSpecReplacedCase(preparedReplaceEnabledTable(format)), + partitionChangeAfterReplaceCase(preparedReplaceEnabledPartitionedTable(format)), + userPropertyPreservedCase(preparedUserPropertyTable(format)), + statementOverridesPropertyCase(preparedUserPropertyTable(format)), + retentionPolicyPreservedCase(preparedRetentionPolicyTable(format)), + columnTagPreservedCase(preparedTaggedTable(format)), + preReplaceTimeTravelCase(preparedReplaceEnabledTable(format)), + rollbackAcrossLineageRejectedCase(preparedReplaceEnabledTable(format)), + setCurrentSnapshotRecoversCase(preparedReplaceEnabledTable(format)), + changelogAcrossBoundaryCase(preparedReplaceEnabledTable(format)), + incrementalReadAcrossBoundaryCase(preparedReplaceEnabledTable(format)), + replaceThenRenameCase(preparedReplaceEnabledTable(format)), + renameThenReplaceCase(preparedReplaceEnabledTable(format)), + sortOrderChangedAfterReplaceCase(preparedReplaceEnabledTable(format)), + sortOrderRemovedAfterReplaceCase(preparedReplaceEnabledTable(format)), + creatorIdentityPreservedCase(preparedReplaceEnabledTable(format)), + replaceVersusAppendCase(preparedReplaceEnabledTable(format))) + } + + // --- the replace preparations the DML axis runs on --- + + /** + * One replace preparation per columnar format and partitioning: the table is created replace-enabled, seeded with + * the standard rows, re-specified in place by a same-shape CREATE OR REPLACE TABLE AS SELECT, then refreshed. The + * result holds the standard seed reached through the replace path, so every reusable DML operation that holds on a + * freshly seeded table must also hold here. + */ + lazy val preparedRtasCoreTables: List[TablePreparation[CoreTable.type]] = + for { + format <- fileFormats + partitioning <- partitionings + } yield TablePreparation( + s"${partitioning.label}/$format", + replaceLineage(partitioning, format), + rtasCasePrefix) + + /** + * One replace preparation per date-partitioned layout, so the partition-scoped writes replace whole partitions of a + * table that reached those partitions through the replace path. + */ + lazy val preparedRtasPartitionedCoreTables: List[TablePreparation[CoreTable.type]] = + fileFormats.map(format => + TablePreparation( + s"${partitionedByDate.label}/$format", + replaceLineage(partitionedByDate, format), + rtasCasePrefix)) + + /** The replace preparations, each carrying one row whose string column is null. */ + lazy val preparedNullStringRtasCoreTables: List[TablePreparation[CoreTable.type]] = + preparedRtasCoreTables.map(withNullStringRow) + + /** The prefix that marks a case ID as running on a table that reached its starting state through a replace. */ + val rtasCasePrefix: String = "prep.rtas:" + + // --- the starting states, shared helpers and case bodies the surface above composes --- + + /** The layout of a table the catalog will let a case replace: the core shape, shaped by `partitioning`. */ + private def replaceEnabledLayout(partitioning: Partitioning, format: String): Layout = + Layout( + s"${partitioning.label}/$format", + table => + s"CREATE TABLE $table ($columnDefinitions) USING $dataSource ${partitioning.clause} " + + s"TBLPROPERTIES ('write.format.default'='$format', 'replace.enabled'='true')") + + /** + * Creates a replace-enabled table, seeds the standard rows, re-specifies the same shape through CREATE OR REPLACE + * TABLE AS SELECT, and refreshes it. The REFRESH is required: the Spark session holds the table state it read + * before the replace, and REFRESH re-reads the committed metadata pointer so later statements in the session + * address the replaced table. + */ + private def replaceLineage( + partitioning: Partitioning, + format: String): TableTest[CoreTable.type] = + create(replaceEnabledLayout(partitioning, format)) + .insert(standardSeedRowCount)() + .sql("prep.rtas")(table => + s"CREATE OR REPLACE TABLE $table USING $dataSource ${partitioning.clause} " + + s"TBLPROPERTIES ('write.format.default'='$format') AS SELECT * FROM $table")(view => { + assertSeededCoreShape(view, "prep.rtas") + assert( + view.snapshotsAfter == view.snapshotsBefore + 1, + s"prep.rtas commits one snapshot, went from ${view.snapshotsBefore} to " + + s"${view.snapshotsAfter}") + }) + .step("prep.rtas.refresh")((spark, table) => { + val currentSnapshotBefore = currentSnapshotId(spark, table) + val snapshotCountBefore = PreparedTable.snapshotCount(spark, table) + spark.sql(s"REFRESH TABLE $table") + assert( + currentSnapshotId(spark, table) == currentSnapshotBefore, + s"prep.rtas.refresh keeps main on snapshot $currentSnapshotBefore, moved it to " + + s"${currentSnapshotId(spark, table)}") + assert( + PreparedTable.snapshotCount(spark, table) == snapshotCountBefore, + s"prep.rtas.refresh keeps the snapshot count at $snapshotCountBefore") + })(view => { + assertSeededCoreShape(view, "prep.rtas.refresh") + assert( + view.snapshotsAfter == view.snapshotsBefore, + s"prep.rtas.refresh reads committed metadata and commits nothing, went from " + + s"${view.snapshotsBefore} to ${view.snapshotsAfter} snapshots") + }) + + /** + * The state both replace-preparation steps leave behind: the standard seed rows in key order, unchanged by the step, + * under exactly the core columns in their declared order. Both steps assert it, so a replace that loses a row, + * reorders the schema or drops a column fails during preparation, so the DML cases always compare against a known + * baseline. + */ + private def assertSeededCoreShape(view: StepView[CoreTable.type], stepLabel: String): Unit = { + val schemaColumnNames = view.spark.table(view.table).schema.fieldNames.toSeq + + assert( + schemaColumnNames == Core.columnNames, + s"$stepLabel presents the core schema, found $schemaColumnNames") + assert( + view.after == view.before, + s"$stepLabel keeps every row it started from, went from ${view.before} to ${view.after}") + assert( + view.after.size == standardSeedRowCount, + s"$stepLabel holds the $standardSeedRowCount standard seed rows, found ${view.after.size}") + assert( + inKeyOrder(view.after) == view.after, + s"$stepLabel returns the seed rows in key order, found ${view.after}") + assert( + view.after.map(row => Rows.TypedRow(row).get(Core.long0)) == + (1L to standardSeedRowCount.toLong).toList, + s"$stepLabel holds the standard seed keys, found " + + s"${view.after.map(row => Rows.TypedRow(row).get(Core.long0))}") + } + + /** + * The snapshot the table's main branch currently points at, read from the refs metadata table, which names exactly + * one snapshot per branch. A replace starts a second root in the snapshots metadata table, so main is the one + * source that identifies the live snapshot after a replace. + */ + private def currentSnapshotId(spark: SparkSession, table: String): Long = + spark + .sql(s"SELECT snapshot_id FROM $table.refs WHERE name = 'main'") + .collect() + .toSeq + .map(_.getLong(0)) match { + case Seq(snapshotId) => snapshotId + case mainSnapshotIds => + throw new AssertionError(s"main names one snapshot, found $mainSnapshotIds") + } + + /** + * The standard seed in an unpartitioned replace-enabled table in `format`, so a contract case starts from a table + * the catalog will let it replace and drives the replace itself. + */ + private def preparedReplaceEnabledTable(format: String): TablePreparation[CoreTable.type] = + TablePreparation( + format, + create(replaceEnabledLayout(unpartitioned, format)).insert(standardSeedRowCount)()) + + /** The same starting state partitioned by the date column, for the cases that repartition after a replace. */ + private def preparedReplaceEnabledPartitionedTable( + format: String): TablePreparation[CoreTable.type] = + TablePreparation( + format, + create(replaceEnabledLayout(partitionedByDate, format)).insert(standardSeedRowCount)()) + + /** + * A replace-enabled table in `format` carrying the user property user.key=v1, so a case reads back what a replace + * does to a property the user set. + */ + private def preparedUserPropertyTable(format: String): TablePreparation[CoreTable.type] = + TablePreparation( + format, + TableTest(Core) + .sql("create")(table => + s"CREATE TABLE $table ($columnDefinitions) USING $dataSource TBLPROPERTIES (" + + s"'write.format.default'='$format', 'replace.enabled'='true', 'user.key'='v1')")() + .insert(standardSeedRowCount)()) + + /** + * A date-partitioned replace-enabled table in `format` carrying a 30-day retention policy on the date column, so a + * case reads back what a replace does to a policy the catalog stores. + */ + private def preparedRetentionPolicyTable(format: String): TablePreparation[CoreTable.type] = + TablePreparation( + format, + create(replaceEnabledLayout(partitionedByDate, format)) + .insert(standardSeedRowCount)() + .sql("setRetentionPolicy")(table => + s"ALTER TABLE $table SET POLICY " + + s"(RETENTION = 30d ON COLUMN ${Core.date0.columnName} " + + "WHERE pattern = 'yyyy-MM-dd-HH')")()) + + /** + * A replace-enabled table in `format` whose string column carries the PII tag, so a case reads back what a replace + * does to a column classification. + */ + private def preparedTaggedTable(format: String): TablePreparation[CoreTable.type] = + TablePreparation( + format, + create(replaceEnabledLayout(unpartitioned, format)) + .insert(standardSeedRowCount)() + .sql("tagStringColumnAsPii")(table => + s"ALTER TABLE $table MODIFY COLUMN ${Core.string0.columnName} SET TAG = (PII)")()) + + /** The statement that replaces `table` in place with the rows whose key is at most `keyLimit`. */ + private def replaceWithKeysUpTo(table: String, keyLimit: Int): String = + s"CREATE OR REPLACE TABLE $table USING $dataSource " + + s"AS SELECT * FROM $table WHERE ${Core.long0.columnName} <= $keyLimit" + + /** + * The message Iceberg raises when a requested snapshot range starts outside the lineage the table currently follows. + * A replace starts a new lineage, so both the changelog view and the incremental scan report a pre-replace start + * snapshot this way. + */ + private val crossLineageRejectionMessage = "is not a parent ancestor of end snapshot" + + /** The two outcomes a racing writer records: its statement committed, or it hit a typed commit conflict. */ + private val committedOutcome = "committed" + private val conflictedOutcome = "conflicted" + + /** + * The markers a rejection carries when the catalog refuses a column type change as incompatible. The in-place ALTER + * COLUMN TYPE path answers with the Spark analyzer marker, and the catalog answers with a message naming the + * change it refused. + */ + private val incompatibleTypeRejectionMarkers = + List("NOT_SUPPORTED_CHANGE_COLUMN", "incompatible", "cannot be cast", "narrow") + + /** The statement that replaces `table` in place with `projection` selected from it. */ + private def replaceWithProjection(table: String, projection: String): String = + s"CREATE OR REPLACE TABLE $table USING $dataSource AS SELECT $projection FROM $table" + + /** The reserved properties that identify a table to the catalog and must outlive a replace. */ + private val identityPropertyNames = List( + "openhouse.tableUUID", + "openhouse.tableId", + "openhouse.databaseId", + "openhouse.tableCreator") + + /** The values `table` currently reports for the reserved identity properties. */ + private def identityProperties( + table: PreparedTable[CoreTable.type]): Map[String, String] = { + val properties = tableProps(table.spark, table.name) + identityPropertyNames.flatMap(name => properties.get(name).map(name -> _)).toMap + } + + /** The column names the table reports, in the order it reports them. */ + private def columnNamesOf(table: PreparedTable[CoreTable.type], name: String): Seq[String] = + table.spark.sql(s"SELECT * FROM $name LIMIT 0").columns.toSeq + + // --- 1. the gates that decide whether a replace is allowed at all --- + + /** + * With replace.enabled=true, CREATE OR REPLACE TABLE AS SELECT replaces the table's content, leaving exactly the two + * rows the replacement query selected. + */ + private def enablementGateCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.gate.enabled") { table => + table.spark.sql(s"ALTER TABLE ${table.name} SET TBLPROPERTIES ('replace.enabled'='true')") + table.spark.sql(replaceWithKeysUpTo(table.name, 2)) + + assert( + countOf(table.spark, s"SELECT count(*) FROM ${table.name}") == "2", + "an enabled replace should leave only the rows its query selected") + } + + /** + * On a table that has left replace.enabled unset, CREATE OR REPLACE TABLE AS SELECT is rejected with a + * BadRequestException naming the disabled feature, and the table keeps the rows it had, so a table opts in before + * anything rewrites it. + */ + private def disabledGateRejectedCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.gate.disabled.rejected") { table => + val stateBefore = table.state + val exception = Check.intercept[BadRequestException]( + table.spark.sql(replaceWithKeysUpTo(table.name, 2))) + + assert( + exception.getMessage.contains("REPLACE TABLE AS SELECT is not enabled"), + s"unexpected message: ${exception.getMessage.take(160)}") + assert(table.state == stateBefore, "a rejected replace should leave the table as it was") + } + + /** + * With replace.enabled=true but a replication policy also set, CREATE OR REPLACE TABLE AS SELECT is rejected with a + * BadRequestException naming replication, so a replicated table keeps the lineage its replicas follow. + */ + private def replicationGateRejectedCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.gate.replicationConflict.rejected") { table => + table.spark.sql(s"ALTER TABLE ${table.name} SET TBLPROPERTIES ('replace.enabled'='true')") + table.spark.sql( + s"ALTER TABLE ${table.name} SET POLICY (REPLICATION = ({destination:'WAR'}))") + val stateBefore = table.state + val exception = Check.intercept[BadRequestException]( + table.spark.sql(replaceWithKeysUpTo(table.name, 2))) + + assert( + exception.getMessage.contains("while replication is enabled"), + s"unexpected message: ${exception.getMessage.take(160)}") + assert(table.state == stateBefore, "a rejected replace should leave the table as it was") + } + + // --- 2. the plainest replace, and the write that follows it --- + + /** + * A same-shape CREATE OR REPLACE TABLE AS SELECT keeps every column in its declared order and every row it selected, + * and commits exactly one snapshot, so replacing a table with itself changes nothing a reader can see except the + * lineage. + */ + private def sameShapeReplacementCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.sameShapeReplacement") { table => + val rowsBefore = table.rows + val snapshotsBefore = table.snapshotCount + table.spark.sql(replaceWithProjection(table.name, columnNameList)) + + assert( + columnNamesOf(table, table.name) == Core.columnNames, + "a same-shape replace should keep the declared columns in order") + assert( + inKeyOrder(table.rows) == inKeyOrder(rowsBefore), + "a same-shape replace should keep every row it selected") + assert( + table.snapshotCount == snapshotsBefore + 1, + s"a replace should commit one snapshot, went from $snapshotsBefore to ${table.snapshotCount}") + } + + /** + * A replaced table accepts an INSERT immediately afterwards and the row lands, so a writer that follows a replace in + * the same session addresses the replaced table, which is the lineage the replace made current. + */ + private def writeAfterReplaceCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.writeAfterReplace") { table => + table.spark.sql(replaceWithKeysUpTo(table.name, 2)) + table.spark.sql(s"INSERT INTO ${table.name} VALUES ${coreRow(6L, "row-6")}") + + assert( + countOf(table.spark, s"SELECT count(*) FROM ${table.name}") == "3", + "the replaced table should hold the two selected rows plus the inserted one") + assert( + countOf( + table.spark, + s"SELECT count(*) FROM ${table.name} WHERE ${Core.long0.columnName} = 6") == "1", + "the row inserted after the replace should be readable") + } + + // --- 3. the four schema discontinuities a replace can introduce --- + + /** + * A replace whose projection adds a computed column widens the schema to that column and every row carries its + * value, so a replace is how a caller adds a column with data already in it. + */ + private def schemaAddColumnCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.schema.addColumn") { table => + table.spark.sql( + replaceWithProjection(table.name, s"$columnNameList, CAST(7 AS INT) AS added_col")) + + assert( + columnNamesOf(table, table.name) == Core.columnNames :+ "added_col", + "a replace that projects a new column should add it after the existing ones") + assert( + countOf( + table.spark, + s"SELECT count(*) FROM ${table.name} WHERE added_col = 7") == + standardSeedRowCount.toString, + "every row should carry the value the projection computed") + } + + /** + * A replace whose projection names fewer columns drops the rest while preserving every row, so a replace is how a + * caller removes a column the catalog refuses to drop in place. + */ + private def schemaDropColumnCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.schema.dropColumn") { table => + table.spark.sql( + replaceWithProjection( + table.name, + s"${Core.long0.columnName}, ${Core.string0.columnName}")) + + assert( + columnNamesOf(table, table.name) == + Seq(Core.long0.columnName, Core.string0.columnName), + "a replace that projects two columns should leave exactly those two") + assert( + countOf(table.spark, s"SELECT count(*) FROM ${table.name}") == + standardSeedRowCount.toString, + "dropping a column through a replace should preserve every row") + } + + /** + * A replace that casts the int column to bigint widens it and every value reads back unchanged, so a replace carries + * a widening type change the same way an in-place ALTER COLUMN TYPE does. + */ + private def schemaWidenColumnCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.schema.widenColumn") { table => + val intValuesBefore = table.spark + .sql(s"SELECT ${Core.int0.columnName} FROM ${table.name} ORDER BY ${Core.int0.columnName}") + .collect() + .toSeq + .map(_.getInt(0).toLong) + table.spark.sql( + replaceWithProjection( + table.name, + s"${Core.long0.columnName}, " + + s"CAST(${Core.int0.columnName} AS BIGINT) AS ${Core.int0.columnName}")) + val widenedType = table.spark + .table(table.name) + .schema + .fields + .toList + .collectFirst { case field if field.name == Core.int0.columnName => field.dataType.simpleString } + + assert( + widenedType.contains("bigint"), + s"the replace should widen the int column to bigint, got $widenedType") + assert( + table.spark + .sql(s"SELECT ${Core.int0.columnName} FROM ${table.name} ORDER BY ${Core.int0.columnName}") + .collect() + .toSeq + .map(_.getLong(0)) == intValuesBefore, + "widening through a replace should preserve every value") + } + + /** + * A replace narrows the bigint key column to int on a table holding a key wider than an int. A replace defines a + * fresh schema, so it accepts narrowing where the in-place ALTER COLUMN TYPE path refuses it. The contract is that + * the value survives either way: the replace is rejected for the type incompatibility and leaves the bigint column + * and its key intact, or it is accepted and the key still reads back as the key that was written. + * + * The current product accepts the replace, reports success and returns -1294967296 for the key 3000000000. This case + * keeps value preservation as the assertion so the gap stays visible, and is skipped until the product either + * rejects the narrowing or preserves the value. + */ + private def schemaIncompatibleTypeRejectedCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation + .test("rtas.schema.incompatibleType.notSilentlyLossy") { table => + val outOfRangeKey = 3000000000L + table.spark.sql( + s"INSERT INTO ${table.name} VALUES " + + s"(CAST($outOfRangeKey AS BIGINT), 4, 'row-big', 4.5, true, '2024-01-04-03')") + + val rejectionMessage = + try { + table.spark.sql( + replaceWithProjection( + table.name, + s"CAST(${Core.long0.columnName} AS INT) AS ${Core.long0.columnName}, " + + s"${Core.string0.columnName}")) + None + } catch { + case rejection: AnalysisException => Some(rejection.getMessage) + case rejection: BadRequestException => Some(rejection.getMessage) + case rejection: ValidationException => Some(rejection.getMessage) + } + val keyColumnType = table.spark + .table(table.name) + .schema + .fields + .toList + .collectFirst { + case field if field.name == Core.long0.columnName => field.dataType.simpleString + } + val storedKeys = table.spark + .sql( + s"SELECT ${Core.long0.columnName} FROM ${table.name} " + + s"WHERE ${Core.string0.columnName} = 'row-big'") + .collect() + .toSeq + .map(row => row.get(0).asInstanceOf[Number].longValue) + + rejectionMessage match { + case Some(message) => + assert( + incompatibleTypeRejectionMarkers.exists(message.contains), + s"the rejection identifies the type incompatibility, found: ${message.take(200)}") + assert( + keyColumnType.contains("bigint"), + s"a rejected narrowing leaves the key column bigint, found $keyColumnType") + assert( + storedKeys == List(outOfRangeKey), + s"a rejected narrowing leaves the key at $outOfRangeKey, found $storedKeys") + case None => + assert( + storedKeys == List(outOfRangeKey), + s"an accepted narrowing preserves the key $outOfRangeKey, " + + s"found $storedKeys under type $keyColumnType") + } + } + .copy(knownBugReason = Some( + "A replace that narrows bigint to int is accepted and wraps an out-of-range key around " + + "while the contract requires rejection or value preservation. The key 3000000000 reads back as " + + "-1294967296 after the product reports success.")) + + // --- 4. partition discontinuities, and evolving the partitioning the replace installed --- + + /** + * A replace with a new PARTITIONED BY clause installs that partition specification and preserves every row, so a + * replace is the supported repartitioning path after the catalog rejects in-place partition evolution. + */ + private def partitionSpecReplacedCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.partition.specReplaced") { table => + table.spark.sql( + s"CREATE OR REPLACE TABLE ${table.name} USING $dataSource " + + s"PARTITIONED BY (${Core.date0.columnName}) AS SELECT * FROM ${table.name}") + val description = table.spark.sql(s"DESCRIBE TABLE ${table.name}").collect().toSeq + + assert( + description.exists(_.getString(0) == "# Partition Information") && + description.count(_.getString(0) == Core.date0.columnName) == 2, + "the replace should install the partition specification it named") + assert( + countOf(table.spark, s"SELECT count(*) FROM ${table.name}") == + standardSeedRowCount.toString, + "repartitioning through a replace should preserve every row") + } + + /** + * After a replace installs a date partition specification, ALTER TABLE DROP PARTITION FIELD is still rejected, and a + * second replace with a different PARTITIONED BY clause is what changes the partitioning. In-place partition + * evolution stays rejected across a replace, so replacing the table again is the one legal way to repartition it, + * which is what the catalog's own rejection message tells a caller to do. + */ + private def partitionChangeAfterReplaceCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.partition.changeAfterReplace") { table => + table.spark.sql( + s"CREATE OR REPLACE TABLE ${table.name} USING $dataSource " + + s"PARTITIONED BY (${Core.date0.columnName}) AS SELECT * FROM ${table.name}") + val inPlaceEvolution = Check.intercept[Exception]( + table.spark.sql( + s"ALTER TABLE ${table.name} DROP PARTITION FIELD ${Core.date0.columnName}")) + + assert( + inPlaceEvolution.getMessage.contains("Evolution of table partitioning"), + s"unexpected message: ${inPlaceEvolution.getMessage.take(160)}") + + table.spark.sql(replaceWithProjection(table.name, columnNameList)) + val description = table.spark.sql(s"DESCRIBE TABLE ${table.name}").collect().toSeq + + assert( + !description.exists(_.getString(0) == "# Partition Information"), + "the second replace should leave the table unpartitioned") + assert( + countOf(table.spark, s"SELECT count(*) FROM ${table.name}") == + standardSeedRowCount.toString, + "repartitioning through a second replace should preserve every row") + table.spark.sql(s"INSERT INTO ${table.name} VALUES ${coreRow(6L, "row-6")}") + assert( + countOf(table.spark, s"SELECT count(*) FROM ${table.name}") == "4", + "the repartitioned table should stay writable") + } + + // --- 5. what a replace does to the properties a user set --- + + /** + * A replace that omits TBLPROPERTIES preserves the user property and the enablement flag the table carried, keeping + * the existing configuration. + */ + private def userPropertyPreservedCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.property.userPropertyPreserved") { table => + assert( + tableProps(table.spark, table.name).get("user.key").contains("v1"), + "the preparation should set the user property the replace is asked to preserve") + + table.spark.sql(replaceWithKeysUpTo(table.name, 2)) + val properties = tableProps(table.spark, table.name) + + assert( + properties.get("user.key").contains("v1"), + s"user.key should survive the replace, got ${properties.get("user.key")}") + assert( + properties.get("replace.enabled").contains("true"), + "replace.enabled should survive the replace") + } + + /** + * A replace whose TBLPROPERTIES clause names an existing property overrides that one and preserves every omitted + * property, so the statement decides exactly what it mentions. + */ + private def statementOverridesPropertyCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.property.statementOverridesProperty") { table => + table.spark.sql( + s"CREATE OR REPLACE TABLE ${table.name} USING $dataSource " + + "TBLPROPERTIES ('user.key'='v2') " + + s"AS SELECT * FROM ${table.name} WHERE ${Core.long0.columnName} <= 2") + val properties = tableProps(table.spark, table.name) + + assert( + properties.get("user.key").contains("v2"), + s"the property the statement named should win, got ${properties.get("user.key")}") + assert( + properties.get("replace.enabled").contains("true"), + "a property the statement omits should survive the replace") + } + + // --- 6. what a replace does to the governance the catalog stores --- + + /** + * A replace that also installs a new partition specification preserves the retention policy the catalog stored, so + * replacing a table's content keeps the rule that ages its data out. + */ + private def retentionPolicyPreservedCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.policy.retentionPreserved") { table => + val policiesBefore = tableProps(table.spark, table.name).getOrElse("policies", "") + assert( + policiesBefore.toLowerCase.contains("retention"), + s"the preparation should store the retention policy the replace must preserve: $policiesBefore") + + table.spark.sql( + s"CREATE OR REPLACE TABLE ${table.name} USING $dataSource " + + s"PARTITIONED BY (${Core.date0.columnName}) " + + s"AS SELECT * FROM ${table.name} WHERE ${Core.long0.columnName} <= 2") + + assert( + tableProps(table.spark, table.name).getOrElse("policies", "") == policiesBefore, + "the replace should preserve the retention policy") + } + + /** + * A replace preserves the PII tag the string column carried, so replacing a table's content keeps a column's + * classification. + */ + private def columnTagPreservedCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.policy.columnTagPreserved") { table => + val policiesBefore = tableProps(table.spark, table.name).getOrElse("policies", "") + assert( + policiesBefore.toLowerCase.contains("pii"), + s"the preparation should store the PII tag the replace must preserve: $policiesBefore") + + table.spark.sql(replaceWithKeysUpTo(table.name, 2)) + + assert( + tableProps(table.spark, table.name).getOrElse("policies", "") == policiesBefore, + "the replace should preserve the PII column tag") + } + + // --- 7. reading the history a replace retired --- + + /** + * A replace keeps the pre-replace snapshot in history and that snapshot still reads its three rows, so the content a + * replace overwrote stays reachable by time travel. + */ + private def preReplaceTimeTravelCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.history.preReplaceTimeTravel") { table => + val preReplaceSnapshotId = currentSnapshotId(table.spark, table.name) + table.spark.sql(replaceWithKeysUpTo(table.name, 2)) + + assert( + countOf(table.spark, s"SELECT count(*) FROM ${table.name}.snapshots") == "2", + "the replace appends to the history it found, leaving the pre-replace snapshot in place") + assert( + countOf( + table.spark, + s"SELECT count(*) FROM ${table.name} VERSION AS OF $preReplaceSnapshotId") == + standardSeedRowCount.toString, + "the pre-replace snapshot should still read the rows it held") + } + + /** + * Rolling back to a snapshot from before the replace is rejected because the replace started a new lineage and the + * earlier snapshot lies outside the current ancestry. + */ + private def rollbackAcrossLineageRejectedCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.history.rollbackRejected") { table => + val preReplaceSnapshotId = currentSnapshotId(table.spark, table.name) + table.spark.sql(replaceWithKeysUpTo(table.name, 2)) + + val exception = Check.intercept[ValidationException]( + table.spark.sql( + "CALL openhouse.system.rollback_to_snapshot(" + + s"'${catalogRelative(table.name)}', $preReplaceSnapshotId)")) + + assert( + exception.getMessage.contains("not an ancestor"), + s"unexpected message: ${exception.getMessage.take(160)}") + } + + /** + * set_current_snapshot to a pre-replace snapshot recovers the rows the replace overwrote, so the snapshot a rollback + * refuses is still the way back to the content that was there before. + */ + private def setCurrentSnapshotRecoversCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.history.setCurrentSnapshotRecovers") { table => + val preReplaceSnapshotId = currentSnapshotId(table.spark, table.name) + table.spark.sql(replaceWithKeysUpTo(table.name, 2)) + table.spark.sql( + "CALL openhouse.system.set_current_snapshot(" + + s"'${catalogRelative(table.name)}', $preReplaceSnapshotId)") + + assert( + countOf(table.spark, s"SELECT count(*) FROM ${table.name}") == + standardSeedRowCount.toString, + "set_current_snapshot should recover the pre-replace rows") + } + + // --- 8 and 9. asking for a range of changes that crosses the replacement boundary --- + + /** + * A changelog view whose start snapshot sits before the replace is rejected with an IllegalArgumentException naming + * the start snapshot as outside the current lineage, so a reader asking to span the replacement boundary is told + * the range is unanswerable and reads the new lineage's changes only through a range inside it. The append that + * follows the replace comes from ChangelogSupport, so this case and the general changelog cases agree on what the + * operation does. + */ + private def changelogAcrossBoundaryCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.changelog.acrossBoundaryRejected") { table => + val appendOperation = changelogOperations + .find(_.name == "changelog.append") + .getOrElse(throw new AssertionError("ChangelogSupport defines the changelog.append operation")) + val preReplaceSnapshotId = currentSnapshotId(table.spark, table.name) + table.spark.sql(replaceWithKeysUpTo(table.name, 2)) + table.spark.sql(appendOperation.statement(table.name)) + val rejection = Check.intercept[IllegalArgumentException]( + changeCounts(table, changelogViewFrom(table, preReplaceSnapshotId))) + + assert( + rejection.getMessage.contains(crossLineageRejectionMessage), + s"the rejection identifies the start snapshot as outside the current lineage, " + + s"found: ${rejection.getMessage.take(200)}") + } + + /** + * An incremental read bounded by a snapshot from before the replace and the snapshot the append after it made + * current is rejected with an IllegalArgumentException naming the start snapshot as outside the current lineage, so + * a scan spans one lineage at a time. + */ + private def incrementalReadAcrossBoundaryCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.incrementalRead.acrossBoundaryRejected") { table => + val preReplaceSnapshotId = currentSnapshotId(table.spark, table.name) + table.spark.sql(replaceWithKeysUpTo(table.name, 2)) + table.spark.sql(s"INSERT INTO ${table.name} VALUES ${coreRow(6L, "row-6")}") + val postAppendSnapshotId = currentSnapshotId(table.spark, table.name) + + val rejection = Check.intercept[IllegalArgumentException]( + table.spark.read + .format("iceberg") + .option("start-snapshot-id", preReplaceSnapshotId) + .option("end-snapshot-id", postAppendSnapshotId) + .load(table.name) + .count()) + + assert( + rejection.getMessage.contains(crossLineageRejectionMessage), + s"the rejection identifies the start snapshot as outside the current lineage, " + + s"found: ${rejection.getMessage.take(200)}") + } + + // --- 10. a replace crossed with a rename, in both orders --- + + /** + * A table replaced and then renamed keeps the replaced content under the new name, so a replace leaves a table + * to the name it was replaced under. The rename boundary records the live name after each accepted rename, so a + * failure between the two renames drops the table under the name it currently answers to. + */ + private def replaceThenRenameCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.rename.replaceThenRename") { table => + val renamedTable = s"${table.name}_replaced_then_renamed" + table.spark.sql(replaceWithKeysUpTo(table.name, 2)) + + withTrackedRename(table.spark.sql(_), table.name) { renameTo => + renameTo(renamedTable) + + assert( + countOf(table.spark, s"SELECT count(*) FROM $renamedTable") == "2", + "the renamed table should hold the rows the replace left") + renameTo(table.name) + } + } + + /** + * A table renamed and then replaced under its new name accepts the replace and holds the replaced content, so a + * rename keeps a table on the replace path. + */ + private def renameThenReplaceCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.rename.renameThenReplace") { table => + val renamedTable = s"${table.name}_renamed_then_replaced" + + withTrackedRename(table.spark.sql(_), table.name) { renameTo => + renameTo(renamedTable) + table.spark.sql(replaceWithKeysUpTo(renamedTable, 2)) + + assert( + countOf(table.spark, s"SELECT count(*) FROM $renamedTable") == "2", + "the table renamed before the replace should hold the rows the replace left") + renameTo(table.name) + } + } + + // --- 11. evolving the sort order the replaced table starts with --- + + /** + * A replaced table accepts ALTER TABLE WRITE ORDERED BY afterwards, which sets range distribution and leaves the + * table writable, so the write order stays settable after a replace. + */ + private def sortOrderChangedAfterReplaceCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.sortOrder.changedAfterReplace") { table => + table.spark.sql(replaceWithKeysUpTo(table.name, 2)) + table.spark.sql(s"ALTER TABLE ${table.name} WRITE ORDERED BY ${Core.long0.columnName}") + val distributionMode = tableProps(table.spark, table.name).get("write.distribution-mode") + + assert( + distributionMode.contains("range"), + s"a write sort order after a replace should set range distribution, got $distributionMode") + table.spark.sql(s"INSERT INTO ${table.name} VALUES ${coreRow(6L, "row-6")}") + assert( + countOf(table.spark, s"SELECT count(*) FROM ${table.name}") == "3", + "the ordered replaced table should stay writable") + } + + /** + * A replaced table that was given a write sort order accepts ALTER TABLE WRITE UNORDERED afterwards, which drops the + * range distribution and leaves the table writable, so a sort order applied after a replace is still removable. + */ + private def sortOrderRemovedAfterReplaceCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.sortOrder.removedAfterReplace") { table => + table.spark.sql(replaceWithKeysUpTo(table.name, 2)) + table.spark.sql(s"ALTER TABLE ${table.name} WRITE ORDERED BY ${Core.long0.columnName}") + table.spark.sql(s"ALTER TABLE ${table.name} WRITE UNORDERED") + val distributionMode = tableProps(table.spark, table.name).get("write.distribution-mode") + + assert( + !distributionMode.contains("range"), + s"dropping the sort order should drop range distribution, got $distributionMode") + table.spark.sql(s"INSERT INTO ${table.name} VALUES ${coreRow(6L, "row-6")}") + assert( + countOf(table.spark, s"SELECT count(*) FROM ${table.name}") == "3", + "the unordered replaced table should stay writable") + } + + // --- 12. the identity the catalog governs the table by --- + + /** + * A replace preserves every reserved property that identifies the table, including the creator the catalog recorded, + * so the table the catalog governs after a replace is the same table it governed before, which keeps a replace from + * being a way to take over a table's identity. + */ + private def creatorIdentityPreservedCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("rtas.identity.creatorPreserved") { table => + val identityBefore = identityProperties(table) + assert( + identityBefore.contains("openhouse.tableCreator"), + s"the catalog should record a creator before the replace: ${identityBefore.keys.toList.sorted}") + + table.spark.sql(replaceWithKeysUpTo(table.name, 2)) + + assert( + identityProperties(table) == identityBefore, + s"the replace changed the table's identity from $identityBefore to ${identityProperties(table)}") + } + + // --- 13. a replace racing another writer --- + + /** + * A serializable replace and INSERT race settles at either the two rows the replace selected, where the replace + * committed last, or three rows, where the append landed on the replaced table. Whichever writer loses fails with a + * typed commit conflict, so a caller recognizes every valid way this race ends. + */ + private def replaceVersusAppendCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation + .test("rtas.concurrency.replaceVersusAppend") { table => + val outcomeByWriter = new ConcurrentHashMap[String, String]() + def writer(writerName: String, statement: String): () => Unit = () => + try { + table.spark.sql(statement) + outcomeByWriter.put(writerName, committedOutcome) + } catch { + case NonFatal(conflict) if ConcurrencySupport.isTypedCommitConflict(conflict) => + outcomeByWriter.put(writerName, conflictedOutcome) + } + + val threadErrors = ConcurrencySupport.runConcurrently( + Seq( + writer("replace", replaceWithKeysUpTo(table.name, 2)), + writer("append", s"INSERT INTO ${table.name} VALUES ${coreRow(30L, "row-30")}"))) + assert( + threadErrors.isEmpty, + s"both writers either commit or hit a typed commit conflict, found: $threadErrors") + + table.spark.sql(s"REFRESH TABLE ${table.name}") + val settledKeys = table.spark + .sql(s"SELECT ${Core.long0.columnName} FROM ${table.name}") + .collect() + .toSeq + .map(_.getLong(0)) + .toSet + val raceOutcome = + (outcomeByWriter.get("replace"), outcomeByWriter.get("append")) + + println(s"DIAG rtas.concurrency.replaceVersusAppend: $raceOutcome settled at $settledKeys") + raceOutcome match { + case (`committedOutcome`, `conflictedOutcome`) => + assert( + settledKeys == Set(1L, 2L), + s"a winning replace leaves the keys it selected, found $settledKeys") + case (`conflictedOutcome`, `committedOutcome`) => + assert( + settledKeys == Set(1L, 2L, 3L, 30L), + s"a winning append leaves the seed plus its row, found $settledKeys") + case (`committedOutcome`, `committedOutcome`) => + assert( + settledKeys == Set(1L, 2L) || settledKeys == Set(1L, 2L, 30L), + s"two commits leave the replaced rows, with the append included when it landed " + + s"on the replaced table, found $settledKeys") + case recordedOutcome => + throw new AssertionError( + s"one writer commits when a replace races an append, recorded $recordedOutcome") + } + } + .copy(knownBugReason = Some( + "A replace and append can both report successful commits while the append's snapshot wins and loses the " + + "replace, leaving the seed rows plus the appended row.")) + +} diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioSchemaEvolution.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioSchemaEvolution.scala new file mode 100644 index 000000000..d37c9f447 --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioSchemaEvolution.scala @@ -0,0 +1,324 @@ +package harness + +import org.apache.spark.sql.AnalysisException +import org.apache.iceberg.exceptions.BadRequestException + +/** + * Schema evolution on the core table: the shape a CREATE TABLE statement produces, and the column additions, type + * changes, reorderings and nullability changes the catalog accepts or rejects afterwards. + * + * Operations: read the created schema, ADD COLUMN in its single, multiple, commented and positioned forms, ALTER + * COLUMN TYPE to widen an int and to widen a decimal, ALTER COLUMN FIRST to reorder, ALTER COLUMN DROP NOT NULL, + * RENAME COLUMN, and the rejected forms DROP COLUMN, DROP COLUMN over written data, ALTER COLUMN TYPE to a narrower + * type and ALTER COLUMN SET NOT NULL. + * + * Preparation axes: the four unseeded core layouts for the created-schema family; the four seeded core layouts for + * the evolution families; the standard seeded table in Parquet and ORC for the rejection families and for the + * families that build their own side table. + * + * Case families: 14 families contributing 42 cases, 4 created-schema, 24 evolution, and 14 rejection or side-table + * cases. + */ +trait ScenarioSchemaEvolution extends ScenarioKit { + + /** Every schema-evolution case: the created schema, then the accepted changes, then the boundaries. */ + lazy val schemaEvolutionCases: List[TestCase] = + createdSchemaCases ++ schemaChangeCases ++ schemaBoundaryCases + + // --- the preparations, shared helpers and case bodies the surface above composes --- + + /** + * The created table's schema is exactly CoreTable's columns, in declaration order and with their declared types, and + * the table holds no rows. + */ + private def createdSchemaCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("schema.create") { table => + val actual = table.spark + .table(table.name) + .schema + .fields + .toList + .map(field => field.name -> field.dataType.simpleString) + val expected = Core.tableColumns.toList.map(column => (column.columnName, column.sqlType)) + + assert(actual == expected, s"schema is $actual") + assert(table.rows.isEmpty, "a table that was never seeded holds no rows") + } + + /** ADD COLUMN adds the column to the schema, the existing rows read null for it, and the row count is unchanged. */ + private def addColumnSingleCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("schema.addColumn.single") { table => + table.spark.sql(s"ALTER TABLE ${table.name} ADD COLUMN added_int int") + + val columnNames = table.spark.table(table.name).schema.fields.toSeq.map(_.name) + val nullCount = table.spark + .sql(s"SELECT count(*) FROM ${table.name} WHERE added_int IS NULL") + .collect()(0) + .getLong(0) + + assert(columnNames.contains("added_int"), s"added_int missing: $columnNames") + assert( + nullCount == table.preparedRows.size, + s"existing rows should read null for added_int: $nullCount != ${table.preparedRows.size}") + assert(table.rows.size == table.preparedRows.size, "ADD COLUMN changed the row count") + } + + /** ADD COLUMNS with two columns in one statement adds both to the schema and leaves the row count unchanged. */ + private def addColumnMultipleCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("schema.addColumn.multiple") { table => + table.spark.sql(s"ALTER TABLE ${table.name} ADD COLUMNS (added_a int, added_b string)") + + val columnNames = table.spark.table(table.name).schema.fields.toSeq.map(_.name) + + assert( + columnNames.contains("added_a") && columnNames.contains("added_b"), + s"added columns missing: $columnNames") + assert(table.rows.size == table.preparedRows.size, "ADD COLUMNS changed the row count") + } + + /** ADD COLUMN ... COMMENT stores the comment on the added column and the reader sees it. */ + private def addColumnCommentCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("schema.addColumn.comment") { table => + table.spark.sql(s"ALTER TABLE ${table.name} ADD COLUMN added_c int COMMENT 'a note'") + + val addedColumn = table.spark + .table(table.name) + .schema + .fields + .find(_.name == "added_c") + .getOrElse(throw new AssertionError("added_c missing")) + + assert( + addedColumn.getComment().contains("a note"), + s"comment not stored: ${addedColumn.getComment()}") + } + + /** ADD COLUMN ... AFTER foo_col_long places the added column directly after that column in the schema. */ + private def addColumnPositionCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("schema.addColumn.position") { table => + table.spark.sql( + s"ALTER TABLE ${table.name} ADD COLUMN added_after int AFTER ${Core.long0.columnName}") + + val columnNames = table.spark.table(table.name).schema.fields.toSeq.map(_.name) + + assert( + columnNames.indexOf("added_after") == columnNames.indexOf(Core.long0.columnName) + 1, + s"added_after not after long0: $columnNames") + } + + /** + * ALTER COLUMN foo_col_int TYPE bigint widens the column in the schema and the already-written values read back + * unchanged. + */ + private def alterColumnTypeWidenCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("schema.alterColumn.typeWiden") { table => + table.spark.sql( + s"ALTER TABLE ${table.name} ALTER COLUMN ${Core.int0.columnName} TYPE bigint") + + val liveColumns = table.spark.table(table.name).schema.fields.toSeq + .map(field => field.name -> field.dataType.simpleString) + .toMap + val values = table.spark + .sql( + s"SELECT ${Core.int0.columnName} FROM ${table.name} ORDER BY ${Core.long0.columnName}") + .collect() + .toSeq + .map(_.getLong(0)) + + assert( + liveColumns.get(Core.int0.columnName).contains("bigint"), + s"int0 not widened: ${liveColumns.get(Core.int0.columnName)}") + assert(values == Seq(1L, 2L, 3L), s"values not preserved after widening: $values") + } + + /** + * RENAME COLUMN renames the column in the schema: the new name is present, the old name is gone, and the row count + * is unchanged. + */ + private def renameColumnCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation + .test("schema.renameColumn") { table => + table.spark.sql(s"ALTER TABLE ${table.name} ADD COLUMN to_rename int") + table.spark.sql(s"ALTER TABLE ${table.name} RENAME COLUMN to_rename TO renamed_col") + + val columnNames = table.spark.table(table.name).schema.fields.toSeq.map(_.name) + + assert( + columnNames.contains("renamed_col") && !columnNames.contains("to_rename"), + s"RENAME COLUMN silently no-oped: $columnNames") + assert(table.rows.size == table.preparedRows.size, "RENAME COLUMN changed the row count") + } + .copy(knownBugReason = Some( + "RENAME COLUMN is a silent no-op because server-side schema casing normalization " + + "restores the old name.")) + + /** ALTER TABLE DROP COLUMN is rejected with a BadRequestException naming the column that would be dropped. */ + private def dropColumnRejectedCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("schema.dropColumn.rejected") { table => + val exception = Check.intercept[BadRequestException]( + table.spark.sql( + s"ALTER TABLE ${table.name} DROP COLUMN ${Core.int0.columnName}")) + + assert( + exception.getMessage.contains("not found in newSchema"), + s"unexpected message: ${exception.getMessage.take(160)}") + assert( + exception.getMessage.contains(Core.int0.columnName), + s"message should name the dropped column: ${exception.getMessage.take(160)}") + } + + /** + * DROP COLUMN on a column that holds data is rejected, the column's data remains readable, and the table remains + * writable. + */ + private def dropColumnWithDataRejectedCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("schema.dropColumn.withData.rejected") { table => + table.spark.sql( + s"ALTER TABLE ${table.name} ADD COLUMN extra_col INT") + table.spark.sql( + s"INSERT INTO ${table.name} VALUES $extraColInsert9") + val exception = Check.intercept[BadRequestException]( + table.spark.sql( + s"ALTER TABLE ${table.name} DROP COLUMN extra_col")) + + assert( + exception.getMessage.contains("not found in newSchema"), + s"drop rejection message changed: ${exception.getMessage.take(200)}") + assert( + countOf( + table.spark, + s"SELECT count(*) FROM ${table.name} WHERE extra_col = 42") == "1", + "rejected drop should leave the column data readable") + + table.spark.sql( + s"INSERT INTO ${table.name} VALUES $extraColInsert10") + assert( + countOf(table.spark, s"SELECT count(*) FROM ${table.name}") == "5", + "rejected drop should leave the table writable") + } + + /** + * ALTER TABLE ALTER COLUMN to a narrower type (bigint to int) is rejected with an AnalysisException about the + * unsupported column change. + */ + private def alterColumnNarrowTypeRejectedCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("schema.alterColumn.narrowType.rejected") { table => + val exception = Check.intercept[AnalysisException]( + table.spark.sql( + s"ALTER TABLE ${table.name} ALTER COLUMN ${Core.long0.columnName} TYPE int")) + + assert( + exception.getMessage.contains("NOT_SUPPORTED_CHANGE_COLUMN"), + s"unexpected message: ${exception.getMessage.take(160)}") + } + + /** + * ALTER TABLE ALTER COLUMN SET NOT NULL on a nullable column is rejected with an AnalysisException about the + * nullable-to-non-nullable change. + */ + private def alterColumnSetNotNullRejectedCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("schema.alterColumn.setNotNull.rejected") { table => + val exception = Check.intercept[AnalysisException]( + table.spark.sql( + s"ALTER TABLE ${table.name} ALTER COLUMN ${Core.string0.columnName} SET NOT NULL")) + + assert( + exception.getMessage.contains("Cannot change nullable column to non-nullable"), + s"unexpected message: ${exception.getMessage.take(160)}") + } + + /** On a side table, dropping NOT NULL from a column allows a subsequent insert of a null value for that column. */ + private def alterColumnDropNotNullCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("schema.alterColumn.dropNotNull") { table => + val sideTable = s"${table.name}_nn" + withOwnedTable(table.spark.sql(_), sideTable)( + table.spark.sql( + s"CREATE TABLE $sideTable " + + s"(id BIGINT, req INT NOT NULL) USING $dataSource")) { + table.spark.sql( + s"ALTER TABLE $sideTable ALTER COLUMN req DROP NOT NULL") + table.spark.sql( + s"INSERT INTO $sideTable VALUES (CAST(1 AS BIGINT), NULL)") + assert( + countOf(table.spark, s"SELECT count(*) FROM $sideTable WHERE req IS NULL") == "1", + "relaxing NOT NULL should allow a null write") + } + } + + /** + * On a side table, widening a decimal column's precision preserves the original row and accepts a new row whose + * value only fits the wider precision. + */ + private def alterColumnDecimalWidenCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("schema.alterColumn.decimalWiden") { table => + val sideTable = s"${table.name}_dec" + withOwnedTable(table.spark.sql(_), sideTable)( + table.spark.sql( + s"CREATE TABLE $sideTable " + + s"(id BIGINT, dec DECIMAL(10,2)) USING $dataSource")) { + table.spark.sql( + s"INSERT INTO $sideTable VALUES " + + "(CAST(1 AS BIGINT), CAST(12345678.99 AS DECIMAL(10,2)))") + table.spark.sql( + s"ALTER TABLE $sideTable ALTER COLUMN dec TYPE DECIMAL(12,2)") + table.spark.sql( + s"INSERT INTO $sideTable VALUES " + + "(CAST(2 AS BIGINT), CAST(1234567890.99 AS DECIMAL(12,2)))") + assert( + countOf(table.spark, s"SELECT count(*) FROM $sideTable") == "2", + "decimal widening should preserve old and new values") + } + } + + /** ALTER TABLE ALTER COLUMN ... FIRST moves that column to the front of the schema while preserving all 3 rows. */ + private def alterColumnReorderFirstCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("schema.alterColumn.reorderFirst") { table => + table.spark.sql( + s"ALTER TABLE ${table.name} " + + s"ALTER COLUMN ${Core.string0.columnName} FIRST") + val columns = table.spark + .sql(s"SELECT * FROM ${table.name} LIMIT 1") + .columns + .toSeq + + assert( + columns.head == Core.string0.columnName, + s"FIRST should move the column to the front: $columns") + assert( + countOf(table.spark, s"SELECT count(*) FROM ${table.name}") == "3", + "column reorder should preserve the rows") + } + + /** The created-schema case on every unseeded core layout. */ + private val createdSchemaCases: List[TestCase] = + preparedEmptyCoreTables.map(createdSchemaCase) + + /** The accepted schema changes on every seeded core layout. */ + private val schemaChangeCases: List[TestCase] = + preparedCoreTables.flatMap { preparation => + List( + addColumnSingleCase(preparation), + addColumnMultipleCase(preparation), + addColumnCommentCase(preparation), + addColumnPositionCase(preparation), + alterColumnTypeWidenCase(preparation), + renameColumnCase(preparation)) + } + + /** The rejected schema changes and the side-table schema changes, in each columnar format. */ + private val schemaBoundaryCases: List[TestCase] = + preparedCoreFormats.flatMap { preparation => + List( + dropColumnRejectedCase(preparation), + dropColumnWithDataRejectedCase(preparation), + alterColumnNarrowTypeRejectedCase(preparation), + alterColumnSetNotNullRejectedCase(preparation), + alterColumnDropNotNullCase(preparation), + alterColumnDecimalWidenCase(preparation), + alterColumnReorderFirstCase(preparation)) + } + +} diff --git a/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioTableProperty.scala b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioTableProperty.scala new file mode 100644 index 000000000..48af60b85 --- /dev/null +++ b/integrations/spark/delta-harness/src/main/scala/harness/openhouse/scenarios/ScenarioTableProperty.scala @@ -0,0 +1,146 @@ +package harness + +import org.apache.iceberg.exceptions.BadRequestException + +/** + * Table properties: which properties a table keeps as written, which ones the catalog owns and overrides, and which + * ones it refuses to change. + * + * Operations: SET and UNSET TBLPROPERTIES for a user property; SET TBLPROPERTIES on the reserved openhouse.tableUUID + * property; reading format-version back from a table created with format-version=1; reading + * write.metadata.previous-versions-max back from a table that requested 7; reading write.target-file-size-bytes back + * from a table that requested 1048576; and SET TBLPROPERTIES on openhouse.tableType. + * + * Preparation axes: in each columnar format, the standard seeded core table for the two families that + * change properties after creation, plus one purpose-built table per family that asserts a property requested at + * creation. + * + * Case families: six families contributing 12 cases. + */ +trait ScenarioTableProperty extends ScenarioKit { + + /** Every table-property case, one file format at a time. */ + lazy val tablePropertyCases: List[TestCase] = + fileFormats.flatMap { format => + List( + userRoundTripCase(preparedStandardTable(format)), + reservedPropertyRejectedCase(preparedStandardTable(format)), + tableTypeImmutableCase(preparedStandardTable(format)), + formatVersionForcedCase(format), + previousVersionsHonoredCase(format), + targetFileSizeCase(format)) + } + + // --- the preparations, shared helpers and case bodies the surface above composes --- + + /** SET TBLPROPERTIES adds a user property that reads back, and UNSET TBLPROPERTIES removes it. */ + private def userRoundTripCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("tableProperty.userRoundTrip") { table => + table.spark.sql( + s"ALTER TABLE ${table.name} SET TBLPROPERTIES ('my_key'='my_val')") + assert( + tableProps(table.spark, table.name).get("my_key").contains("my_val"), + "user property was not set") + + table.spark.sql(s"ALTER TABLE ${table.name} UNSET TBLPROPERTIES ('my_key')") + assert( + !tableProps(table.spark, table.name).contains("my_key"), + "user property was not removed") + } + + /** + * SET TBLPROPERTIES on the reserved openhouse.tableUUID property is rejected with a BadRequestException about the + * restriction. + */ + private def reservedPropertyRejectedCase( + preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("tableProperty.reservedOpenhouse.rejected") { table => + val exception = Check.intercept[BadRequestException]( + table.spark.sql( + s"ALTER TABLE ${table.name} SET TBLPROPERTIES (" + + "'openhouse.tableUUID'='deadbeef')")) + + assert( + exception.getMessage.toLowerCase.contains("restriction"), + s"unexpected message: ${exception.getMessage.take(200)}") + } + + /** + * ALTER TABLE SET TBLPROPERTIES ('openhouse.tableType'='REPLICA_TABLE') is rejected with a BadRequestException, + * since the table type is fixed at creation. + */ + private def tableTypeImmutableCase(preparation: TablePreparation[CoreTable.type]): TestCase = + preparation.test("tableProperty.tableTypeImmutable") { table => + val exception = Check.intercept[BadRequestException]( + table.spark.sql( + s"ALTER TABLE ${table.name} SET TBLPROPERTIES (" + + "'openhouse.tableType'='REPLICA_TABLE')")) + + assert( + exception.getMessage.contains("restriction"), + s"unexpected message: ${exception.getMessage.take(160)}") + } + + /** + * Even though format-version=1 was requested at creation, the catalog stores the table at format-version=2 and the + * table remains writable there. + */ + private def formatVersionForcedCase(format: String): TestCase = + TablePreparation( + format, + TableTest(Core) + .sql("create")(table => + s"CREATE TABLE $table ($columnDefinitions) USING $dataSource TBLPROPERTIES (" + + s"'write.format.default'='$format', 'format-version'='1')")() + .insert(standardSeedRowCount)()) + .test("tableProperty.formatVersionForced") { table => + val formatVersion = tableProps(table.spark, table.name).get("format-version") + + assert( + formatVersion.contains("2"), + s"expected the catalog to store format-version=2, got $formatVersion") + assert( + table.rows.size == standardSeedRowCount, + "table not writable at the stored format-version") + } + + /** The write.metadata.previous-versions-max property requested at creation is honored and reads back as 7. */ + private def previousVersionsHonoredCase(format: String): TestCase = + TablePreparation( + format, + TableTest(Core).sql("create")(table => + s"CREATE TABLE $table ($columnDefinitions) USING $dataSource TBLPROPERTIES (" + + s"'write.format.default'='$format', 'write.metadata.previous-versions-max'='7')")()) + .test("tableProperty.previousVersionsHonored") { table => + val previousVersions = + tableProps(table.spark, table.name).get("write.metadata.previous-versions-max") + + assert( + previousVersions.contains("7"), + s"expected previous-versions-max=7, got $previousVersions") + } + + /** + * The write.target-file-size-bytes=1048576 property requested at creation is retained and the table holds its 3 seed + * rows. + */ + private def targetFileSizeCase(format: String): TestCase = + TablePreparation( + format, + TableTest(Core) + .sql("create")(table => + s"CREATE TABLE $table ($columnDefinitions) USING $dataSource TBLPROPERTIES (" + + s"'write.format.default'='$format', 'write.target-file-size-bytes'='1048576')")() + .insert(standardSeedRowCount)()) + .test("tableProperty.targetFileSize") { table => + assert( + tableProps(table.spark, table.name) + .get("write.target-file-size-bytes") + .contains("1048576"), + "target file size should be retained") + assert( + table.rows.size == standardSeedRowCount, + "the custom target-size table should hold its seed rows") + } + +} diff --git a/integrations/spark/delta-harness/src/test/scala/harness/framework/TableLifecycleTest.scala b/integrations/spark/delta-harness/src/test/scala/harness/framework/TableLifecycleTest.scala new file mode 100644 index 000000000..5da2938c8 --- /dev/null +++ b/integrations/spark/delta-harness/src/test/scala/harness/framework/TableLifecycleTest.scala @@ -0,0 +1,43 @@ +package harness + +import org.junit.jupiter.api.Assertions.{assertEquals, assertSame, assertThrows} +import org.junit.jupiter.api.Test + +final class TableLifecycleTest { + @Test + def cleanupRunsOnlyForAnOwnedTable(): Unit = { + var cleanupCount = 0 + val createFailure = new Exception("table already exists") + + val thrown = assertThrows( + classOf[Exception], + () => + OwnedTableLifecycle.withOwnership(cleanupCount += 1)(_ => + throw createFailure)) + + assertSame(createFailure, thrown) + assertEquals(0, cleanupCount) + + OwnedTableLifecycle.withOwnership(cleanupCount += 1)(markTableCreated => + markTableCreated()) + + assertEquals(1, cleanupCount) + } + + @Test + def cleanupFailureIsSuppressedBehindTheBodyFailure(): Unit = { + val bodyFailure = new Exception("test failed") + val cleanupFailure = new Exception("cleanup failed") + + val thrown = assertThrows( + classOf[Exception], + () => + OwnedTableLifecycle.withOwnership(throw cleanupFailure) { markTableCreated => + markTableCreated() + throw bodyFailure + }) + + assertSame(bodyFailure, thrown) + assertEquals(List(cleanupFailure), thrown.getSuppressed.toList) + } +} diff --git a/integrations/spark/delta-harness/src/test/scala/harness/scenarios/CaseCatalogTest.scala b/integrations/spark/delta-harness/src/test/scala/harness/scenarios/CaseCatalogTest.scala new file mode 100644 index 000000000..751fa1530 --- /dev/null +++ b/integrations/spark/delta-harness/src/test/scala/harness/scenarios/CaseCatalogTest.scala @@ -0,0 +1,21 @@ +package harness + +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +final class CaseCatalogTest { + @Test + def everyCaseIdIsUnique(): Unit = { + val duplicateCaseIds = ScenarioCatalog.caseIds + .groupBy(identity) + .collect { + case (caseId, occurrences) if occurrences.size > 1 => caseId + } + .toList + .sorted + + assertTrue( + duplicateCaseIds.isEmpty, + s"case IDs must be unique; duplicates=${duplicateCaseIds.mkString(", ")}") + } +} diff --git a/settings.gradle b/settings.gradle index 9acc07d67..4f9b29e12 100644 --- a/settings.gradle +++ b/settings.gradle @@ -40,6 +40,7 @@ include ':integrations:spark:spark-3.1:openhouse-spark-runtime' include ':integrations:spark:spark-3.1:openhouse-spark-itest' include ':integrations:spark:spark-3.5:openhouse-spark-runtime' include ':integrations:spark:spark-3.5:openhouse-spark-itest' +include ':integrations:spark:delta-harness' include ':iceberg:openhouse:htscatalog' include ':iceberg:openhouse:internalcatalog' @@ -77,4 +78,5 @@ project(':integrations:java:iceberg-1.5:openhouse-java-runtime').name = 'openhou project(':integrations:java:iceberg-1.5:openhouse-java-itest').name = 'openhouse-java-iceberg-1.5-itest' project(':integrations:spark:spark-3.5:openhouse-spark-runtime').name = 'openhouse-spark-3.5-runtime_2.12' project(':integrations:spark:spark-3.5:openhouse-spark-itest').name = 'openhouse-spark-3.5-itest' +project(':integrations:spark:delta-harness').name = 'openhouse-spark-delta-harness_2.12' project(':tables-test-fixtures:tables-test-fixtures-iceberg-1.5').name = 'tables-test-fixtures-iceberg-1.5_2.12'