test(delta-harness): add harness-core framework and scenarios

build.gradle

@@ -153,19 +153,23 @@ allprojects {
   }
 
   afterEvaluate {
-    for (def task in it.tasks) {
-      if (task != rootProject.tasks.CopyGitHooksTask) {
+    it.tasks
+      .findAll { task -> task != rootProject.tasks.CopyGitHooksTask }
+      .each { task ->
         task.dependsOn rootProject.tasks.CopyGitHooksTask
       }
-    }
   }
 }
 
-// Local Git Hooks cannot be shared, as .git directory is gitignore'd.
+// Resolve the repository hooks directory through Git so linked worktrees and standard checkouts use the same task.
+def gitHooksDirectory = providers.exec {
+  commandLine 'git', 'rev-parse', '--git-path', 'hooks'
+}.standardOutput.asText.map { it.trim() }
+
 tasks.register('CopyGitHooksTask', Copy) {
-  println 'Make the git hook available in .git/hooks directory.'
+  println 'Make the Git hooks available in the repository hooks directory.'
   from file('scripts/git-hooks')
-  into file('.git/hooks/')
+  into gitHooksDirectory
 }
 
 // =============================================================================

integrations/spark/delta-harness/build.gradle

@@ -0,0 +1,155 @@
+plugins {
+  id 'openhouse.java-minimal-conventions'
+  id 'openhouse.maven-publish'
+  id 'scala'
+}
+
+// The delta-harness behavioral matrix is published as a portable Scala library. This repository runs
+// it against the embedded catalog, and the LinkedIn acceptance tests supply a remote environment.
+//
+// Only the portable scenario/framework sources are published. Env.scala boots the embedded OpenHouse
+// server and pulls in its test fixtures, and LocalRunner.scala carries the harness.Main launch class
+// that drives it, so both are excluded from the published library and compiled in the local source
+// set. Runner.scala remains portable so its retry contract is exercised by the module tests.
+
+ext {
+  icebergVersion = rootProject.ext.iceberg_1_5_version
+  sparkVersion = '3.5.2'
+  scalaLibVersion = '2.12.18'
+}
+
+// The embedded boot wiring and the local run loop. They are excluded from the portable library and
+// are the whole of the local source set.
+def embeddedOnlySources = ['harness/openhouse/Env.scala', 'harness/openhouse/LocalRunner.scala']
+
+sourceSets {
+  main {
+    scala {
+      srcDirs = ['src/main/scala']
+      exclude embeddedOnlySources
+    }
+  }
+  local {
+    scala {
+      srcDirs = ['src/main/scala']
+      include embeddedOnlySources
+    }
+    compileClasspath += sourceSets.main.output
+    runtimeClasspath += sourceSets.main.output
+  }
+}
+
+dependencies {
+  implementation "org.scala-lang:scala-library:${scalaLibVersion}"
+
+  // The consumer provides the Spark, Iceberg, and OpenHouse runtime. The library jar carries only the
+  // harness classes.
+  compileOnly("org.apache.spark:spark-sql_2.12:${sparkVersion}") {
+    exclude group: 'io.netty'
+  }
+  compileOnly("com.linkedin.iceberg:iceberg-spark-runtime-3.5_2.12:${icebergVersion}") {
+    exclude group: 'io.netty'
+  }
+  // Provides com.linkedin.openhouse.javaclient.* (WebClientResponseWithMessageException, etc.).
+  compileOnly(project(path: ':integrations:spark:spark-3.5:openhouse-spark-3.5-runtime_2.12', configuration: 'shadow'))
+
+  // The catalog tests read Catalog contributions and cases without starting Spark. Their JVM still
+  // loads the harness signatures, so the compile-only harness dependencies must be present on the test classpath.
+  testImplementation("org.apache.spark:spark-sql_2.12:${sparkVersion}") {
+    exclude group: 'io.netty'
+  }
+  testImplementation("com.linkedin.iceberg:iceberg-spark-runtime-3.5_2.12:${icebergVersion}") {
+    exclude group: 'io.netty'
+  }
+  testImplementation(
+    project(path: ':integrations:spark:spark-3.5:openhouse-spark-3.5-runtime_2.12',
+            configuration: 'shadow'))
+
+  localImplementation sourceSets.main.output
+  localImplementation "org.scala-lang:scala-library:${scalaLibVersion}"
+  localImplementation("org.apache.spark:spark-sql_2.12:${sparkVersion}") {
+    exclude group: 'io.netty'
+  }
+  localImplementation(
+    project(path: ':integrations:spark:spark-3.5:openhouse-spark-3.5-runtime_2.12',
+            configuration: 'shadow')) {
+    exclude group: 'org.apache.commons', module: 'commons-lang3'
+  }
+  localImplementation project(':tables-test-fixtures:tables-test-fixtures-iceberg-1.5_2.12')
+}
+
+configurations.localRuntimeClasspath {
+  exclude group: 'com.linkedin.iceberg', module: 'iceberg-core'
+  exclude group: 'com.linkedin.iceberg', module: 'iceberg-api'
+  exclude group: 'com.linkedin.iceberg', module: 'iceberg-common'
+  exclude group: 'com.linkedin.iceberg', module: 'iceberg-data'
+  resolutionStrategy.force(
+    'com.fasterxml.jackson.core:jackson-annotations:2.15.2',
+    'com.fasterxml.jackson.core:jackson-core:2.15.2',
+    'com.fasterxml.jackson.core:jackson-databind:2.15.2',
+    'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.15.2',
+    'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.15.2',
+    'com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.15.2',
+    'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.2',
+    'com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.15.2',
+    'com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.15.2',
+    'com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.15.2',
+    'com.fasterxml.jackson.module:jackson-module-parameter-names:2.15.2',
+    'com.fasterxml.jackson.module:jackson-module-scala_2.12:2.15.2')
+}
+
+def configureEmbeddedHarnessRun = { JavaExec task ->
+  task.dependsOn localClasses
+  task.classpath = sourceSets.local.runtimeClasspath
+  task.mainClass = 'harness.Main'
+  if (JavaVersion.current() >= JavaVersion.VERSION_1_9) {
+    task.jvmArgs(
+      '--add-opens=java.base/java.nio=ALL-UNNAMED',
+      '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED',
+      '--add-opens=java.base/sun.util.calendar=ALL-UNNAMED',
+      '--add-exports=java.base/sun.util.calendar=ALL-UNNAMED')
+  }
+}
+
+tasks.register('runOpenHouse', JavaExec) {
+  group = 'verification'
+  description = 'Runs the delta harness against an embedded OpenHouse catalog.'
+  configureEmbeddedHarnessRun(delegate)
+}
+
+def verifyOpenHouseFoundation = tasks.register('verifyOpenHouseFoundation', JavaExec) {
+  group = 'verification'
+  description = 'Runs the foundation scenarios against an embedded OpenHouse catalog.'
+  configureEmbeddedHarnessRun(delegate)
+  args '--catalog=foundation'
+}
+
+def verifyPortableJar = tasks.register('verifyPortableJar') {
+  group = 'verification'
+  description = 'Checks that the published harness jar contains only portable entry points.'
+  dependsOn tasks.named('jar')
+  doLast {
+    def jarFile = tasks.named('jar').get().archiveFile.get().asFile
+    def entries = new java.util.zip.ZipFile(jarFile).withCloseable { archive ->
+      archive.entries().collect { entry -> entry.name }.toSet()
+    }
+    ['harness/Plan.class', 'harness/Runner.class'].each { requiredEntry ->
+      if (!entries.contains(requiredEntry)) {
+        throw new GradleException("Portable harness jar is missing ${requiredEntry}")
+      }
+    }
+    ['harness/Main.class', 'harness/Main$.class', 'harness/OpenHouseEnv.class', 'harness/OpenHouseEnv$.class']
+      .each { embeddedEntry ->
+        if (entries.contains(embeddedEntry)) {
+          throw new GradleException("Portable harness jar contains embedded-only ${embeddedEntry}")
+        }
+      }
+  }
+}
+
+tasks.named('check') {
+  dependsOn verifyOpenHouseFoundation
+  dependsOn verifyPortableJar
+}
+
+jar.enabled = true

integrations/spark/delta-harness/run-openhouse.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+JAVA_HOME="${JAVA17_HOME:-${JAVA_HOME:-}}"
+if [[ -z "$JAVA_HOME" ]]; then
+  echo "Set JAVA17_HOME or JAVA_HOME to a JDK 17 installation." >&2
+  exit 2
+fi
+export JAVA_HOME
+
+JAVA_VERSION="$("$JAVA_HOME/bin/java" -version 2>&1 | sed -n '1s/.*version "\([0-9][0-9]*\).*/\1/p')"
+if [[ "$JAVA_VERSION" != "17" ]]; then
+  echo "The delta harness requires JDK 17; $JAVA_HOME reports Java $JAVA_VERSION." >&2
+  exit 2
+fi
+
+cd "$REPO_ROOT"
+if (( $# == 0 )); then
+  exec ./gradlew --no-daemon \
+    :integrations:spark:openhouse-spark-delta-harness_2.12:runOpenHouse
+fi
+
+printf -v FILTERS ' %q' "$@"
+exec ./gradlew --no-daemon \
+  :integrations:spark:openhouse-spark-delta-harness_2.12:runOpenHouse \
+  --args="${FILTERS# }"

integrations/spark/delta-harness/src/main/scala/harness/openhouse/Env.scala

@@ -0,0 +1,72 @@
+package harness
+
+import org.apache.spark.sql.SparkSession
+import scala.io.Source
+
+/**
+ * Embedded environment wiring: boots the in-process OpenHouse server and hands back a SparkSession pointed at its
+ * catalog. This file is compiled into the `local` source set only, because it pulls in the server test fixtures that
+ * the published portable library leaves out.
+ */
+object OpenHouseEnv {
+  import com.linkedin.openhouse.tablestest.OpenHouseLocalServer
+
+  private def authToken(): String =
+    Option(getClass.getClassLoader.getResourceAsStream("dummy.token"))
+      .map(tokenStream => Source.fromInputStream(tokenStream, "UTF-8").mkString.trim)
+      .getOrElse("default-token")
+
+  private def wireCatalog(
+      builder: SparkSession.Builder,
+      name: String,
+      uri: String,
+      token: String): SparkSession.Builder =
+    builder
+      .config(s"spark.sql.catalog.$name", "org.apache.iceberg.spark.SparkCatalog")
+      .config(
+        s"spark.sql.catalog.$name.catalog-impl",
+        "com.linkedin.openhouse.spark.OpenHouseCatalog")
+      .config(s"spark.sql.catalog.$name.uri", uri)
+      .config(s"spark.sql.catalog.$name.cluster", "local-cluster")
+      .config(s"spark.sql.catalog.$name.auth-token", token)
+
+  def start(): (OpenHouseLocalServer, SparkSession) = {
+    // The embedded server uses Hibernate to create its H2 schema. Hibernate owns initialization for this process, so
+    // classpath SQL initialization stays disabled.
+    System.setProperty("spring.sql.init.mode", "never")
+    System.setProperty("spring.jpa.hibernate.ddl-auto", "create-drop")
+
+    val server = new OpenHouseLocalServer()
+    server.start()
+    try {
+      val uri = s"http://localhost:${server.getPort}"
+      val token = authToken()
+
+      val base = SparkSession.builder()
+        .appName("delta-harness-openhouse")
+        .master("local[2]")
+        .config("spark.sql.extensions",
+          "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions," +
+            "com.linkedin.openhouse.spark.extensions.OpenhouseSparkSessionExtensions")
+        .config("spark.hadoop.fs.defaultFS", "file:///")
+        .config("spark.sql.session.timeZone", "UTC")
+        .config("spark.sql.autoBroadcastJoinThreshold", "-1")
+        .config("spark.driver.bindAddress", "127.0.0.1")
+        .config("spark.ui.enabled", "false")
+
+      val wired =
+        Seq("openhouse", "default_iceberg")
+          .foldLeft(base)(wireCatalog(_, _, uri, token))
+      (server, wired.getOrCreate())
+    } catch {
+      case startupFailure: Throwable =>
+        try {
+          server.stop()
+        } catch {
+          case cleanupFailure: Throwable =>
+            startupFailure.addSuppressed(cleanupFailure)
+        }
+        throw startupFailure
+    }
+  }
+}