diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json
index 3714c27d2a98b..26a3ff7df7d8a 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -239,6 +239,30 @@
     ],
     "sqlState" : "22023"
   },
+  "AUTOCDC_IGNORE_NULL_CANNOT_SELECT_RESERVED_COLUMN" : {
+    "message" : [
+      "In AutoCDC flow <flowName>, the ignore-null selection names column `<columnName>` which collides with the reserved column name prefix `<reservedColumnNamePrefix>` (<caseSensitivity>). Reserved columns may not appear in an ignore-null selection."
+    ],
+    "sqlState" : "42710"
+  },
+  "AUTOCDC_IGNORE_NULL_COLUMN_NOT_IN_OUTPUT_COLUMNS" : {
+    "message" : [
+      "In AutoCDC flow <flowName>, the ignore-null selection names column `<columnName>` which is not present in the flow's selected output columns (<caseSensitivity>). Only user-data columns surviving the column selection may appear in an ignore-null selection."
+    ],
+    "sqlState" : "42703"
+  },
+  "AUTOCDC_IGNORE_NULL_EMPTY_COLUMN_LIST" : {
+    "message" : [
+      "AutoCDC flow specifies an empty `ignore_null_updates_column_list`. Provide at least one column, or omit the option entirely to disable ignore-null updates. To ignore nulls for every column, specify an empty `ignore_null_updates_except_column_list` instead."
+    ],
+    "sqlState" : "22023"
+  },
+  "AUTOCDC_IGNORE_NULL_SELECTION_CONTAINS_KEY_COLUMN" : {
+    "message" : [
+      "In AutoCDC flow <flowName>, the ignore-null selection names key column `<columnName>` (<caseSensitivity>). Key columns identify rows and may not appear in an ignore-null selection. Key columns: <keyColumnNames>."
+    ],
+    "sqlState" : "22023"
+  },
   "AUTOCDC_INVALID_STATE" : {
     "message" : [
       "Detected an invalid AutoCDC state for target table <tableName>:"
@@ -6742,6 +6766,14 @@
     ],
     "sqlState" : "0A000"
   },
+  "PIPELINE_NESTED_COMPLEX_TYPE_SCHEMA_EVOLUTION_UNSUPPORTED" : {
+    "message" : [
+      "Schema evolution within a complex type (array or map) whose element is itself a complex type is not supported.",
+      "Column path <columnPath> has element type <currentType> but the target element type is <targetType>.",
+      "Flatten the nesting or apply the change manually."
+    ],
+    "sqlState" : "0A000"
+  },
   "PIPELINE_RUN_FAILED" : {
     "message" : [
       "<message>"
@@ -6762,6 +6794,12 @@
     ],
     "sqlState" : "42K03"
   },
+  "PIPELINE_TIGHTEN_NULLABILITY_UNSUPPORTED" : {
+    "message" : [
+      "Cannot tighten nullability of <columnPath> from nullable to non-nullable. Existing data may already contain nulls."
+    ],
+    "sqlState" : "0A000"
+  },
   "PIPE_OPERATOR_AGGREGATE_EXPRESSION_CONTAINS_NO_AGGREGATE_FUNCTION" : {
     "message" : [
       "Non-grouping expression <expr> is provided as an argument to the |> AGGREGATE pipe operator but does not contain any aggregate function; please update it to include an aggregate function and then retry the query again."
diff --git a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/ChangeArgs.scala b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/ChangeArgs.scala
index fe28decab357c..3c4ab81404541 100644
--- a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/ChangeArgs.scala
+++ b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/ChangeArgs.scala
@@ -64,6 +64,15 @@ object ColumnSelection {
   case class ExcludeColumns(columns: Seq[UnqualifiedColumnName])
       extends ColumnSelection
 
+  /** Extracts the explicitly named columns, or `Nil` when the selection is `None`. */
+  def namedColumns(
+      selection: Option[ColumnSelection]): Seq[UnqualifiedColumnName] =
+    selection match {
+      case None => Nil
+      case Some(IncludeColumns(cols)) => cols
+      case Some(ExcludeColumns(cols)) => cols
+    }
+
   /**
    * Applies [[ColumnSelection]] to a [[StructType]] and returns the filtered schema. Field order
    * follows the original schema; only matching fields are retained in the returned schema.
@@ -186,6 +195,13 @@ object ScdType {
  *                               which has no run concept and therefore no history-tracking columns.
  *                               See the "run of upsert events" concept in the `Scd2BatchProcessor`
  *                               scaladoc for the precise definition of a run.
+ * @param ignoreNullSelection    Selects the columns whose nulls are treated as declined
+ *                               authorship rather than authored nulls. None means ignore-null
+ *                               is off completely. An empty include list is rejected; an empty
+ *                               exclude list selects every eligible column. Eligible columns
+ *                               are those surviving the column selection that are neither keys
+ *                               nor columns whose names start with the reserved AutoCDC prefix.
+ *                               Naming a struct selects every leaf beneath it.
  */
 case class ChangeArgs(
     keys: Seq[UnqualifiedColumnName],
@@ -193,10 +209,12 @@ case class ChangeArgs(
     storedAsScdType: ScdType,
     deleteCondition: Option[Column] = None,
     columnSelection: Option[ColumnSelection] = None,
-    trackHistorySelection: Option[ColumnSelection] = None
+    trackHistorySelection: Option[ColumnSelection] = None,
+    ignoreNullSelection: Option[ColumnSelection] = None
 ) {
   ChangeArgs.validateNonEmptyKeys(keys)
   ChangeArgs.validateTrackHistoryOnlyForScd2(storedAsScdType, trackHistorySelection)
+  ChangeArgs.validateNonEmptyIgnoreNullIncludeList(ignoreNullSelection)
 }
 
 object ChangeArgs {
@@ -231,4 +249,20 @@ object ChangeArgs {
       )
     }
   }
+
+  /**
+   * Rejects an empty ignore-null include list; "ignore-null off" is spelled
+   * as None.
+   */
+  private def validateNonEmptyIgnoreNullIncludeList(
+      ignoreNullSelection: Option[ColumnSelection]): Unit = {
+    ignoreNullSelection match {
+      case Some(ColumnSelection.IncludeColumns(columns)) if columns.isEmpty =>
+        throw new AnalysisException(
+          errorClass = "AUTOCDC_IGNORE_NULL_EMPTY_COLUMN_LIST",
+          messageParameters = Map.empty
+        )
+      case _ => ()
+    }
+  }
 }
diff --git a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala
index 3f2a2e7415323..2910d0664bc22 100644
--- a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala
+++ b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala
@@ -194,6 +194,9 @@ case class Scd2BatchProcessor(
       colName = AutoCdcReservedNames.cdcMetadataColName,
       col = Scd2BatchProcessor.constructCdcMetadataCol(
         recordStartAt = changeArgs.sequencing,
+        // TODO (SPARK-59183): actually populate version map according to ignore-null selection and
+        // actual authorship in microbatch.
+        versionMap = F.lit(null),
         sequencingType = resolvedSequencingType
       )
     )
@@ -514,6 +517,9 @@ case class Scd2BatchProcessor(
             colName,
             Scd2BatchProcessor.constructCdcMetadataCol(
               recordStartAt = F.lit(null).cast(resolvedSequencingType),
+              // Decomposition tails are synthetic rows that carry no column-level authorship; they
+              // are instead row wide delete markers. They should always hold a null version map.
+              versionMap = F.lit(null),
               sequencingType = resolvedSequencingType
             )
           )
@@ -922,6 +928,9 @@ case class Scd2BatchProcessor(
           isDecompositionTail,
           Scd2BatchProcessor.constructCdcMetadataCol(
             recordStartAt = endAt,
+            // Tombstone rows carry no column-level authorship; they are instead row wide delete
+            // markers. They should always hold a null version map.
+            versionMap = F.lit(null),
             sequencingType = resolvedSequencingType
           )
         ).otherwise(F.col(c)).as(c, metadata)
@@ -1357,6 +1366,9 @@ object Scd2BatchProcessor {
    */
   private[pipelines] val recordStartAtFieldName: String = "__RECORD_START_AT"
 
+  /** CDC metadata field for the ignore-null version map. */
+  private[pipelines] val versionMapFieldName: String = "__VERSION_MAP"
+
   /**
    * Aux-table only column that holds the microbatch id by which a row was logically
    * deleted (null if the row is still live). Future microbatches must treat any row with a
@@ -1567,6 +1579,10 @@ object Scd2BatchProcessor {
   private def recordStartAtOf(cdcMetadataCol: Column): Column =
     cdcMetadataCol.getField(recordStartAtFieldName)
 
+  /** Project the [[versionMapFieldName]] out of an SCD2 CDC metadata column. */
+  private[autocdc] def versionMapOf(cdcMetadataCol: Column): Column =
+    cdcMetadataCol.getField(versionMapFieldName)
+
   /**
    * The [[Scd2IntervalColumns]] of a row read from either the auxiliary or the target table, in
    * the canonical SCD2 row schema. The columns are unresolved name references, so they read from
@@ -1587,7 +1603,16 @@ object Scd2BatchProcessor {
         // The sequence value of the originating CDC event for this row. Nullable because
         // decomposition tails, which are temporarily and synthetically constructed during
         // reconciliation, have a null record start at.
-        StructField(recordStartAtFieldName, sequencingType, nullable = true)
+        StructField(recordStartAtFieldName, sequencingType, nullable = true),
+        // The version map representing null-authorship for the row. If the version map is null for
+        // a row, that row was ingested with ignore-null off, and all columns are considered
+        // explicitly authored (null or not). If the version map is non-null, the row was ingested
+        // with ignore-null on, and contents of the map comply with the contract defined in
+        // [[Scd2VersionMap]].
+        //
+        // Tombstones and decomposition tails also always hold null version maps because column
+        // authorship is not applicable - they are delete markers.
+        StructField(versionMapFieldName, Scd2VersionMap.mapType, nullable = true)
       )
     )
 
@@ -1597,11 +1622,13 @@ object Scd2BatchProcessor {
    */
   private[pipelines] def constructCdcMetadataCol(
       recordStartAt: Column,
+      versionMap: Column,
       sequencingType: DataType
   ): Column = {
     val cdcMetadataFieldsInOrder = cdcMetadataColSchema(sequencingType).fields.map { field =>
       val value = field.name match {
         case `recordStartAtFieldName` => recordStartAt
+        case `versionMapFieldName` => versionMap
         case other =>
           throw SparkException.internalError(
             s"Unable to construct SCD2 CDC metadata column due to unknown `${other}` field."
diff --git a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2VersionMap.scala b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2VersionMap.scala
new file mode 100644
index 0000000000000..5926d9e108ad0
--- /dev/null
+++ b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2VersionMap.scala
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.pipelines.autocdc
+
+import org.apache.spark.sql.types.{BooleanType, MapType, StringType}
+
+/**
+ * Per-row column authorship tracker for SCD2 ignore-null semantics.
+ *
+ * Recall in SCD2, every materialized row traces back to an upsert event that created it (and
+ * if the row is closed then also a delete/succession event that closed it, but that's not
+ * relevant here). Every data column in the row is at least partially derived by the
+ * corresponding data column in the upsert event that spawned the row.
+ *
+ * For columns where ignore-null was not applied, the data column in the row is fully derived
+ * (authored) by the corresponding data column in the upsert event. For columns where
+ * ignore-null was applied however, if the data column in the upsert event was null
+ * (unauthored), then we need to look backwards to deduce the corresponding inherited data
+ * column for the row.
+ *
+ * Non-null values in an event are always considered authored, regardless of whether the column
+ * in the event was included in the ignore-null configuration or not. Null values however, as
+ * mentioned above, may or may not be considered authored -- it depends on whether they are
+ * specified for a column that was included in the ignore-null configuration.
+ *
+ * In SCD2 the version map helps us answer per row: for all the columns that received a null
+ * value in the upsert event that created this row, which nulls are considered authored vs
+ * unauthored?
+ *
+ * Concretely, the contract of the version map is as follows.
+ * 1. Every column that received a null in the event but is considered authored (i.e. not part
+ *    of ignore-null selection at ingestion), receives an entry of (column name, true) in the
+ *    version map.
+ * 2. Every column that received a null in the event but is considered unauthored (i.e. part
+ *    of the ignore-null selection at ingestion), receives an entry of (column name, false) in
+ *    the version map.
+ * 3. Every column that was not present in the event, but schema evolved in later with a null
+ *    value, will be treated as unauthored BUT does not yet have any entry in the version map.
+ *    An entry will be added as per (2).
+ *
+ * In a single sentence: if a null column in the SCD2 row is either absent from the version
+ * map or has a false value in the version map, the null is considered unauthored by the
+ * upsert event that spawned this row. Otherwise the null value was explicitly authored by
+ * the row.
+ *
+ * As mentioned above, authorship is dependent on the configured ignore-null selection, which
+ * is free to change between pipeline runs for the same AutoCDC flow. As such, we choose that
+ * the version map strictly reflects authorship as of the ignore-null selection that was active
+ * when the upsert event that produced this row was ingested. This means the authorship
+ * information the version map encoded at creation time is invariant/frozen -- even if the
+ * ignore-null selection changes on a future run, the version map is not rewritten (unless the
+ * table is full refreshed).
+ *
+ * It's worth noting that while contract case (3) materializes new entries in the version map
+ * after creation, it does not change the set of columns whose null values are considered
+ * authored/unauthored. Therefore authorship information encoded by the mutated version map is
+ * still invariant, and independent of a changing ignore-null configuration. New rows materialized
+ * in the map are still compliant with whatever the ignore-null selection was at ingestion time.
+ */
+private[pipelines] object Scd2VersionMap {
+
+  /**
+   * Schema of the version map: `Map(String, Boolean)`.
+   *
+   * Keys are dot-delimited paths to *leaf* columns that received a null value in their
+   * corresponding upsert event (e.g. `"address.city"`, `` "`has space`.city" ``). Paths
+   * must be formatted by [[org.apache.spark.sql.catalyst.util.QuotingUtils.quoted]] to
+   * ensure segments that need quoting are back-tick escaped.
+   *
+   * Values indicate authorship. I.e, `true` => authored-null, `false` => unauthored-null.
+   *
+   * Lack of entry in the map for a null-valued leaf column implies the column was
+   * schema-evolved with an unauthored-null.
+   */
+  def mapType: MapType = MapType(StringType, BooleanType, valueContainsNull = false)
+}
diff --git a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DatasetManager.scala b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DatasetManager.scala
index 61d188f302e13..23d58c1598ab0 100644
--- a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DatasetManager.scala
+++ b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DatasetManager.scala
@@ -690,7 +690,11 @@ object DatasetManager extends Logging {
     // above has already folded a case-only-differing incoming field onto the persisted one. On the
     // non-merging paths (materialized views, full refresh), `targetSchema` is the declared schema
     // as-is, where exact-name matching keeps a case-only rename visible as a schema change.
-    val columnChanges = diffSchemas(currentSchema, targetSchema)
+    val columnChanges = diffSchemas(
+      currentSchema,
+      targetSchema,
+      rejectNullabilityTightening = mergeWithExistingSchema
+    )
 
     val existingProperties = existingTable.properties()
 
diff --git a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala
index a8be0a4c2ec33..627a70698e5f4 100644
--- a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala
+++ b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala
@@ -263,6 +263,8 @@ class AutoCdcMergeFlow(
 
   requireReservedPrefixAbsentInSourceColumns()
   requireReservedFrameworkColumnsAbsentInSourceColumns()
+  requireKeysAbsentInIgnoreNullSelection()
+  requireReservedPrefixAbsentInIgnoreNullSelection()
 
   def changeArgs: ChangeArgs = flow.changeArgs
 
@@ -277,6 +279,7 @@ class AutoCdcMergeFlow(
     // AutoCDC flows require all key columns to be present in the user-selected source schema,
     // so that they survive into the target table where SCD reconciliation needs them.
     requireKeysPresentInSelectedSchema(selectedSchema)
+    requireIgnoreNullColumnsInSelectedSchema(selectedSchema)
     selectedSchema
   }
 
@@ -402,6 +405,7 @@ class AutoCdcMergeFlow(
           F.lit(null).cast(sequencingType).as(Scd2BatchProcessor.endAtColName)
         val emptyCdcMetadataCol: Column = Scd2BatchProcessor.constructCdcMetadataCol(
           recordStartAt = F.lit(null),
+          versionMap = F.lit(null),
           sequencingType = sequencingType
         ).as(AutoCdcReservedNames.cdcMetadataColName)
 
@@ -409,29 +413,23 @@ class AutoCdcMergeFlow(
     }
   }
 
-  /**
-   * Validate that the resolved source dataframe for the AutoCDC flow does not contain any column
-   * names that use the reserved Spark AutoCDC prefix.
-   */
-  private def requireReservedPrefixAbsentInSourceColumns(): Unit = {
-    val resolver = effectiveResolver
+  /** Whether `name` starts with [[AutoCdcReservedNames.prefix]], honoring case sensitivity. */
+  private def nameHasReservedPrefix(name: String): Boolean = {
     val reservedPrefix = AutoCdcReservedNames.prefix
+    name.length >= reservedPrefix.length &&
+      effectiveResolver(name.substring(0, reservedPrefix.length), reservedPrefix)
+  }
 
-    def nameContainsReservedPrefix(name: String): Boolean = {
-      name.length >= reservedPrefix.length && resolver(
-        name.substring(0, reservedPrefix.length),
-        reservedPrefix
-      )
-    }
-
-    df.schema.fieldNames.find(nameContainsReservedPrefix).foreach { conflictingColumnName =>
+  /** Rejects any source column whose name uses the reserved AutoCDC prefix. */
+  private def requireReservedPrefixAbsentInSourceColumns(): Unit = {
+    df.schema.fieldNames.find(nameHasReservedPrefix).foreach { conflictingColumnName =>
       throw new AnalysisException(
         errorClass = "AUTOCDC_RESERVED_COLUMN_NAME_PREFIX_CONFLICT",
         messageParameters = Map(
-          "caseSensitivity" -> CaseSensitivityLabels.of(resolver),
+          "caseSensitivity" -> CaseSensitivityLabels.of(effectiveResolver),
           "columnName" -> conflictingColumnName,
           "schemaName" -> "changeDataFeed",
-          "reservedColumnNamePrefix" -> reservedPrefix
+          "reservedColumnNamePrefix" -> AutoCdcReservedNames.prefix
         )
       )
     }
@@ -493,4 +491,60 @@ class AutoCdcMergeFlow(
       }
   }
 
+  /** Rejects any ignore-null column that is also a key column. */
+  private def requireKeysAbsentInIgnoreNullSelection(): Unit = {
+    val resolver = effectiveResolver
+    ColumnSelection.namedColumns(changeArgs.ignoreNullSelection).foreach { column =>
+      if (changeArgs.keys.exists(key => resolver(key.name, column.name))) {
+        throw new AnalysisException(
+          errorClass = "AUTOCDC_IGNORE_NULL_SELECTION_CONTAINS_KEY_COLUMN",
+          messageParameters = Map(
+            "flowName" -> identifier.unquotedString,
+            "caseSensitivity" -> CaseSensitivityLabels.of(resolver),
+            "columnName" -> column.name,
+            "keyColumnNames" -> changeArgs.keys.map(_.name).mkString(", ")
+          )
+        )
+      }
+    }
+  }
+
+  /** Rejects any ignore-null column whose name uses the reserved prefix. */
+  private def requireReservedPrefixAbsentInIgnoreNullSelection(): Unit = {
+    ColumnSelection.namedColumns(changeArgs.ignoreNullSelection)
+      .find(col => nameHasReservedPrefix(col.name))
+      .foreach { column =>
+        throw new AnalysisException(
+          errorClass = "AUTOCDC_IGNORE_NULL_CANNOT_SELECT_RESERVED_COLUMN",
+          messageParameters = Map(
+            "flowName" -> identifier.unquotedString,
+            "caseSensitivity" -> CaseSensitivityLabels.of(effectiveResolver),
+            "columnName" -> column.name,
+            "reservedColumnNamePrefix" -> AutoCdcReservedNames.prefix
+          )
+        )
+      }
+  }
+
+  /**
+   * Validate every column named by [[ChangeArgs.ignoreNullSelection]] is present in the
+   * user-selected schema.
+   */
+  private def requireIgnoreNullColumnsInSelectedSchema(
+      selectedSchema: StructType): Unit = {
+    val resolver = effectiveResolver
+    ColumnSelection.namedColumns(changeArgs.ignoreNullSelection)
+      .find(col => !selectedSchema.fieldNames.exists(resolver(_, col.name)))
+      .foreach { missing =>
+        throw new AnalysisException(
+          errorClass = "AUTOCDC_IGNORE_NULL_COLUMN_NOT_IN_OUTPUT_COLUMNS",
+          messageParameters = Map(
+            "flowName" -> identifier.unquotedString,
+            "caseSensitivity" -> CaseSensitivityLabels.of(resolver),
+            "columnName" -> missing.name
+          )
+        )
+      }
+  }
+
 }
diff --git a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/util/SchemaInferenceUtils.scala b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/util/SchemaInferenceUtils.scala
index fd60684150742..ce989cba057eb 100644
--- a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/util/SchemaInferenceUtils.scala
+++ b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/util/SchemaInferenceUtils.scala
@@ -19,6 +19,8 @@ package org.apache.spark.sql.pipelines.util
 
 import scala.util.control.NonFatal
 
+import org.apache.spark.SparkUnsupportedOperationException
+import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.analysis.{
   caseInsensitiveResolution,
@@ -34,10 +36,13 @@ import org.apache.spark.sql.pipelines.graph.{
   GraphErrors,
   ResolvedFlow
 }
-import org.apache.spark.sql.types.{StructField, StructType}
+import org.apache.spark.sql.types.{
+  ArrayType, DataType, MapType, Metadata, MetadataBuilder,
+  StructField, StructType
+}
 
 
-object SchemaInferenceUtils {
+object SchemaInferenceUtils extends Logging {
 
   def resolverFor(caseSensitive: Boolean): Resolver = {
     if (caseSensitive) {
@@ -207,12 +212,12 @@ object SchemaInferenceUtils {
   }
 
   /**
-   * Determines the column changes needed to transform the current schema into the target schema.
-   *
-   * This function compares the current schema with the target schema and produces a sequence of
-   * TableChange objects representing:
-   * 1. New columns that need to be added
-   * 2. Existing columns that need type updates
+   * Produces the [[TableChange]] sequence needed to transform `currentSchema` into
+   * `targetSchema`: additions, type updates, deletions, nullability and comment changes.
+   * Recurses into structs, arrays, and maps so changes are emitted at the leaf level.
+   * Similar to [[org.apache.spark.sql.catalyst.analysis.ResolveSchemaEvolution]], but
+   * produces a full bidirectional sync (deletes, nullability, and comment changes) rather
+   * than additive-only evolution.
    *
    * Column identity is keyed on the exact field name, not on a case-normalized one. On the
    * incremental streaming-table path, `targetSchema` is the merge of the current and desired
@@ -225,61 +230,252 @@ object SchemaInferenceUtils {
    *
    * @param currentSchema The current schema of the table
    * @param targetSchema The target schema that we want the table to have
+   * @param rejectNullabilityTightening When true, throws if any field changes
+   *        from nullable to non-nullable. Callers should set this when existing
+   *        rows are retained (incremental streaming tables) because those rows
+   *        may already contain nulls. Full-refresh and materialized-view paths
+   *        truncate the table first, so tightening is safe.
    * @return A sequence of TableChange objects representing the necessary changes
    */
-  def diffSchemas(currentSchema: StructType, targetSchema: StructType): Seq[TableChange] = {
-    val changes = scala.collection.mutable.ArrayBuffer.empty[TableChange]
-
-    // Helper function to get a map of field name to field
-    def getFieldMap(schema: StructType): Map[String, StructField] = {
-      schema.fields.map(field => field.name -> field).toMap
+  def diffSchemas(
+      currentSchema: StructType,
+      targetSchema: StructType,
+      rejectNullabilityTightening: Boolean = false
+  ): Seq[TableChange] = {
+    val changes = diffStructs(
+      currentStruct = currentSchema,
+      targetStruct = targetSchema,
+      // Root call: path is empty because current and target are the top-level schemas.
+      pathToStruct = Seq.empty
+    )
+    if (rejectNullabilityTightening) {
+      changes.foreach {
+        case nc: TableChange.UpdateColumnNullability if !nc.nullable() =>
+          throw new SparkUnsupportedOperationException(
+            errorClass = "PIPELINE_TIGHTEN_NULLABILITY_UNSUPPORTED",
+            messageParameters =
+              Map("columnPath" -> nc.fieldNames().mkString(".")))
+        case _ =>
+      }
     }
+    changes
+  }
 
-    val currentFields = getFieldMap(currentSchema)
-    val targetFields = getFieldMap(targetSchema)
+  /**
+   * Diffs two structs field-by-field, matching fields by exact name.
+   *
+   * @param currentStruct The struct as it exists in the current schema.
+   * @param targetStruct The struct as it should look in the target schema.
+   * @param pathToStruct Path segments from the top-level schema to this
+   *                     struct, if this is a nested struct. Empty for the
+   *                     root call.
+   */
+  private def diffStructs(
+      currentStruct: StructType,
+      targetStruct: StructType,
+      pathToStruct: Seq[String]): Seq[TableChange] = {
+    val topLevelFieldsInCurrent = currentStruct.fields.map(field => field.name -> field).toMap
+    val topLevelFieldsInTarget = targetStruct.fields.map(field => field.name -> field).toMap
+
+    // Fields present in target but not in current are columns that need to be added.
+    val columnsAdded = topLevelFieldsInTarget.values.toSeq
+      .filterNot(fieldInTarget =>
+        topLevelFieldsInCurrent.contains(fieldInTarget.name)
+      )
+      .map { fieldInTarget =>
+        TableChange.addColumn(
+          (pathToStruct :+ fieldInTarget.name).toArray,
+          fieldInTarget.dataType,
+          fieldInTarget.nullable,
+          fieldInTarget.getComment().orNull
+        )
+      }
 
-    // Find columns to add (in target but not in current)
-    val columnsToAdd = targetFields.keySet.diff(currentFields.keySet)
-    columnsToAdd.foreach { columnName =>
-      val field = targetFields(columnName)
-      changes += TableChange.addColumn(
-        Array(columnName),
-        field.dataType,
-        field.nullable,
-        field.getComment().orNull
+    // Fields present in current but not in target are columns that need to be removed.
+    val columnsDeleted = topLevelFieldsInCurrent.values.toSeq
+      .filterNot(fieldInCurrent =>
+        topLevelFieldsInTarget.contains(fieldInCurrent.name)
+      )
+      .map(fieldInCurrent =>
+        TableChange
+          .deleteColumn(
+            (pathToStruct :+ fieldInCurrent.name).toArray,
+            false
+          )
       )
+
+    // Fields in both current and target but vary in metadata or nested sub-fields represent
+    // columns that need to be updated.
+    val columnsUpdated = topLevelFieldsInCurrent.values.toSeq.flatMap {
+      fieldInCurrent =>
+        topLevelFieldsInTarget.get(fieldInCurrent.name).toSeq.flatMap {
+          fieldInTarget =>
+            diffField(
+              currentField = fieldInCurrent,
+              targetField = fieldInTarget,
+              pathToField = pathToStruct :+ fieldInCurrent.name
+            )
+        }
     }
 
-    // Find columns to delete (in current but not in target)
-    val columnsToDelete = currentFields.keySet.diff(targetFields.keySet)
-    columnsToDelete.foreach { columnName =>
-      changes += TableChange.deleteColumn(Array(columnName), false)
+    columnsAdded ++ columnsDeleted ++ columnsUpdated
+  }
+
+  /**
+   * Diffs the type, nullability, and comment of one field present in both schemas. Other
+   * StructField.metadata entries (defaults, generated-column expressions, connector-specific
+   * metadata) are not diffed: pipeline schema synchronization does not support propagating
+   * them, and Spark's own ResolveSchemaEvolution likewise ignores them.
+   */
+  private def diffField(
+      currentField: StructField,
+      targetField: StructField,
+      pathToField: Seq[String]): Seq[TableChange] = {
+    warnOnFieldMetadataDrift(currentField, targetField, pathToField)
+    diffDataTypes(currentField.dataType, targetField.dataType, pathToField) ++
+      diffNullability(currentField.nullable, targetField.nullable, pathToField) ++
+      diffComment(currentField.getComment(), targetField.getComment(), pathToField)
+  }
+
+  /**
+   * Logs a warning when two fields' metadata bags differ beyond the "comment" key
+   * (which is already handled by [[diffComment]]). Pipeline schema synchronization does not
+   * support propagating other metadata entries (defaults, generated-column expressions,
+   * connector-specific metadata), so these differences are left for the user to reconcile.
+   */
+  private def warnOnFieldMetadataDrift(
+      currentField: StructField,
+      targetField: StructField,
+      pathToField: Seq[String]): Unit = {
+    val current = stripMetadataComment(currentField.metadata)
+    val target = stripMetadataComment(targetField.metadata)
+    if (current != target) {
+      logWarning(
+        s"Field ${pathToField.mkString(".")} has metadata changes that pipeline schema " +
+          s"synchronization does not propagate and will be ignored. " +
+          s"Current: ${current.json}, Target: ${target.json}")
     }
+  }
 
-    // Find columns with type changes (in both but with different types)
-    val commonColumns = currentFields.keySet.intersect(targetFields.keySet)
-    commonColumns.foreach { columnName =>
-      val currentField = currentFields(columnName)
-      val targetField = targetFields(columnName)
+  private def stripMetadataComment(m: Metadata): Metadata =
+    new MetadataBuilder().withMetadata(m).remove("comment").build()
 
-      // If data types are different, add a type update change
-      if (currentField.dataType != targetField.dataType) {
-        changes += TableChange.updateColumnType(Array(columnName), targetField.dataType)
-      }
+  private def diffNullability(
+      currentNullable: Boolean,
+      targetNullable: Boolean,
+      pathToField: Seq[String]
+  ): Option[TableChange] = {
+    Option.when(currentNullable != targetNullable)(
+      TableChange.updateColumnNullability(pathToField.toArray, targetNullable)
+    )
+  }
 
-      // If nullability is different, add a nullability update change
-      if (currentField.nullable != targetField.nullable) {
-        changes += TableChange.updateColumnNullability(Array(columnName), targetField.nullable)
-      }
+  private def diffComment(
+      currentComment: Option[String],
+      targetComment: Option[String],
+      pathToField: Seq[String]
+  ): Option[TableChange] = {
+    Option.when(currentComment != targetComment)(
+      TableChange.updateColumnComment(pathToField.toArray, targetComment.orNull)
+    )
+  }
+
+  /**
+   * Diffs two data types at `path`, descending through matching complex types.
+   *
+   * Recurses freely through structs, arrays, and maps. After recursing into an array element
+   * or map key/value, any actual changes are rejected when the child type is itself an array
+   * or map, because [[org.apache.spark.sql.connector.catalog.CatalogV2Util]] cannot resolve
+   * paths with consecutive `element`/`key`/`value` segments (see SPARK-59188).
+   */
+  private def diffDataTypes(
+      currentType: DataType,
+      targetType: DataType,
+      pathToField: Seq[String]): Seq[TableChange] = (currentType, targetType) match {
+    case (currentStruct: StructType, targetStruct: StructType) =>
+      diffStructs(currentStruct, targetStruct, pathToField)
+
+    case (currentArray: ArrayType, targetArray: ArrayType) =>
+      val elementPath = pathToField :+ "element"
+      val dataTypeChanges = diffDataTypes(
+        currentType = currentArray.elementType,
+        targetType = targetArray.elementType,
+        pathToField = elementPath
+      )
+      val nullabilityChanges = diffNullability(
+        currentNullable = currentArray.containsNull,
+        targetNullable = targetArray.containsNull,
+        pathToField = elementPath
+      )
+      rejectUnsupportedNestedTypeChanges(
+        currentType = currentArray.elementType,
+        targetType = targetArray.elementType,
+        typeChanges = dataTypeChanges,
+        pathToElement = elementPath
+      )
+      dataTypeChanges ++ nullabilityChanges
+
+    case (currentMap: MapType, targetMap: MapType) =>
+      val keyPath = pathToField :+ "key"
+      val valuePath = pathToField :+ "value"
+      val keyTypeChanges = diffDataTypes(
+        currentType = currentMap.keyType,
+        targetType = targetMap.keyType,
+        pathToField = keyPath
+      )
+      val valueTypeChanges = diffDataTypes(
+        currentType = currentMap.valueType,
+        targetType = targetMap.valueType,
+        pathToField = valuePath
+      )
+      val valueNullabilityChanges = diffNullability(
+        currentNullable = currentMap.valueContainsNull,
+        targetNullable = targetMap.valueContainsNull,
+        pathToField = valuePath
+      )
+      rejectUnsupportedNestedTypeChanges(
+        currentType = currentMap.keyType,
+        targetType = targetMap.keyType,
+        typeChanges = keyTypeChanges,
+        pathToElement = keyPath
+      )
+      rejectUnsupportedNestedTypeChanges(
+        currentType = currentMap.valueType,
+        targetType = targetMap.valueType,
+        typeChanges = valueTypeChanges,
+        pathToElement = valuePath
+      )
+      keyTypeChanges ++ valueTypeChanges ++ valueNullabilityChanges
 
-      // If comments are different, add a comment update change
-      val currentComment = currentField.getComment().orNull
-      val targetComment = targetField.getComment().orNull
-      if (currentComment != targetComment) {
-        changes += TableChange.updateColumnComment(Array(columnName), targetComment)
+    case _ if currentType == targetType =>
+      Seq.empty
+
+    case _ =>
+      Seq(TableChange.updateColumnType(pathToField.toArray, targetType))
+  }
+
+  /**
+   * Throws when an array element or map key/value is itself an array or map and the
+   * recursive diff found changes. The resulting paths would contain consecutive
+   * `element`/`key`/`value` segments that
+   * [[org.apache.spark.sql.connector.catalog.CatalogV2Util]] cannot resolve (SPARK-59188).
+   */
+  private def rejectUnsupportedNestedTypeChanges(
+      currentType: DataType,
+      targetType: DataType,
+      typeChanges: Seq[TableChange],
+      pathToElement: Seq[String]): Unit = {
+    if (typeChanges.nonEmpty) {
+      (currentType, targetType) match {
+        case (_: ArrayType | _: MapType, _: ArrayType | _: MapType) =>
+          throw new SparkUnsupportedOperationException(
+            errorClass = "PIPELINE_NESTED_COMPLEX_TYPE_SCHEMA_EVOLUTION_UNSUPPORTED",
+            messageParameters = Map(
+              "columnPath" -> pathToElement.mkString("."),
+              "currentType" -> currentType.simpleString,
+              "targetType" -> targetType.simpleString))
+        case _ =>
       }
     }
-
-    changes.toSeq
   }
 }
diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala
index afe1915a22ab5..a38e0935d7f71 100644
--- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala
+++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala
@@ -168,14 +168,16 @@ class AutoCdcFlowSuite extends QueryTest with SharedSparkSession {
       sequencing: Column = F.col("seq"),
       storedAsScdType: ScdType = ScdType.Type1,
       columnSelection: Option[ColumnSelection] = None,
-      trackHistorySelection: Option[ColumnSelection] = None): AutoCdcMergeFlow = {
+      trackHistorySelection: Option[ColumnSelection] = None,
+      ignoreNullSelection: Option[ColumnSelection] = None): AutoCdcMergeFlow = {
     val flow = newAutoCdcFlow(
       changeArgs = ChangeArgs(
         keys = keys,
         sequencing = sequencing,
         storedAsScdType = storedAsScdType,
         columnSelection = columnSelection,
-        trackHistorySelection = trackHistorySelection
+        trackHistorySelection = trackHistorySelection,
+        ignoreNullSelection = ignoreNullSelection
       )
     )
     new AutoCdcMergeFlow(
@@ -995,4 +997,246 @@ class AutoCdcFlowSuite extends QueryTest with SharedSparkSession {
       )
     }
   }
+
+  // ===========================================================================================
+  // AutoCdcMergeFlow ignore-null validation tests
+  // ===========================================================================================
+
+
+  // ---------- key and reserved-prefix checks ----------
+
+  test("AutoCdcMergeFlow rejects key in ignore-null include list") {
+    checkError(
+      exception = intercept[AnalysisException] {
+        newAutoCdcMergeFlow(
+          sourceDf = threeColumnSourceDf(),
+          ignoreNullSelection = Some(
+            ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("id")))
+          )
+        )
+      },
+      condition = "AUTOCDC_IGNORE_NULL_SELECTION_CONTAINS_KEY_COLUMN",
+      sqlState = "22023",
+      parameters = Map(
+        "flowName" -> testIdentifier.unquotedString,
+        "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive,
+        "columnName" -> "id",
+        "keyColumnNames" -> "id"
+      )
+    )
+  }
+
+  test("AutoCdcMergeFlow rejects key in ignore-null exclude list") {
+    checkError(
+      exception = intercept[AnalysisException] {
+        newAutoCdcMergeFlow(
+          sourceDf = threeColumnSourceDf(),
+          ignoreNullSelection = Some(
+            ColumnSelection.ExcludeColumns(Seq(UnqualifiedColumnName("id")))
+          )
+        )
+      },
+      condition = "AUTOCDC_IGNORE_NULL_SELECTION_CONTAINS_KEY_COLUMN",
+      sqlState = "22023",
+      parameters = Map(
+        "flowName" -> testIdentifier.unquotedString,
+        "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive,
+        "columnName" -> "id",
+        "keyColumnNames" -> "id"
+      )
+    )
+  }
+
+  test("AutoCdcMergeFlow rejects ignore-null column with reserved prefix") {
+    checkError(
+      exception = intercept[AnalysisException] {
+        newAutoCdcMergeFlow(
+          sourceDf = threeColumnSourceDf(),
+          ignoreNullSelection = Some(
+            ColumnSelection.IncludeColumns(
+              Seq(UnqualifiedColumnName(s"${AutoCdcReservedNames.prefix}foo"))
+            )
+          )
+        )
+      },
+      condition = "AUTOCDC_IGNORE_NULL_CANNOT_SELECT_RESERVED_COLUMN",
+      sqlState = "42710",
+      parameters = Map(
+        "flowName" -> testIdentifier.unquotedString,
+        "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive,
+        "columnName" -> s"${AutoCdcReservedNames.prefix}foo",
+        "reservedColumnNamePrefix" -> AutoCdcReservedNames.prefix
+      )
+    )
+  }
+
+  test("AutoCdcMergeFlow accepts ignore-null None selection") {
+    val flow = newAutoCdcMergeFlow(sourceDf = threeColumnSourceDf())
+    assert(flow.changeArgs.ignoreNullSelection.isEmpty)
+  }
+
+  test("AutoCdcMergeFlow accepts ignore-null valid include list") {
+    val flow = newAutoCdcMergeFlow(
+      sourceDf = threeColumnSourceDf(),
+      ignoreNullSelection = Some(
+        ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("name")))
+      )
+    )
+    assert(flow.changeArgs.ignoreNullSelection.isDefined)
+  }
+
+  test("AutoCdcMergeFlow accepts ignore-null valid exclude list") {
+    val flow = newAutoCdcMergeFlow(
+      sourceDf = threeColumnSourceDf(),
+      ignoreNullSelection = Some(
+        ColumnSelection.ExcludeColumns(Seq(UnqualifiedColumnName("name")))
+      )
+    )
+    assert(flow.changeArgs.ignoreNullSelection.isDefined)
+  }
+
+  // ---------- schema-level checks ----------
+
+  test("AutoCdcMergeFlow rejects ignore-null column excluded by columnSelection") {
+    checkError(
+      exception = intercept[AnalysisException] {
+        newAutoCdcMergeFlow(
+          sourceDf = threeColumnSourceDf(),
+          ignoreNullSelection = Some(
+            ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("name")))
+          ),
+          columnSelection = Some(
+            ColumnSelection.ExcludeColumns(Seq(UnqualifiedColumnName("name")))
+          )
+        )
+      },
+      condition = "AUTOCDC_IGNORE_NULL_COLUMN_NOT_IN_OUTPUT_COLUMNS",
+      sqlState = "42703",
+      parameters = Map(
+        "flowName" -> testIdentifier.unquotedString,
+        "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive,
+        "columnName" -> "name"
+      )
+    )
+  }
+
+  test("AutoCdcMergeFlow rejects ignore-null column not in include list") {
+    checkError(
+      exception = intercept[AnalysisException] {
+        newAutoCdcMergeFlow(
+          sourceDf = threeColumnSourceDf(),
+          ignoreNullSelection = Some(
+            ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("name")))
+          ),
+          columnSelection = Some(
+            ColumnSelection.IncludeColumns(
+              Seq(UnqualifiedColumnName("id"), UnqualifiedColumnName("seq"))
+            )
+          )
+        )
+      },
+      condition = "AUTOCDC_IGNORE_NULL_COLUMN_NOT_IN_OUTPUT_COLUMNS",
+      sqlState = "42703",
+      parameters = Map(
+        "flowName" -> testIdentifier.unquotedString,
+        "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive,
+        "columnName" -> "name"
+      )
+    )
+  }
+
+  test("AutoCdcMergeFlow rejects ignore-null column absent from source schema") {
+    checkError(
+      exception = intercept[AnalysisException] {
+        newAutoCdcMergeFlow(
+          sourceDf = threeColumnSourceDf(),
+          ignoreNullSelection = Some(
+            ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("nonexistent")))
+          )
+        )
+      },
+      condition = "AUTOCDC_IGNORE_NULL_COLUMN_NOT_IN_OUTPUT_COLUMNS",
+      sqlState = "42703",
+      parameters = Map(
+        "flowName" -> testIdentifier.unquotedString,
+        "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive,
+        "columnName" -> "nonexistent"
+      )
+    )
+  }
+
+  test("AutoCdcMergeFlow accepts ignore-null column present in selected schema") {
+    val flow = newAutoCdcMergeFlow(
+      sourceDf = threeColumnSourceDf(),
+      ignoreNullSelection = Some(
+        ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("name")))
+      )
+    )
+    assert(flow.changeArgs.ignoreNullSelection.isDefined)
+  }
+
+  gridTest("AutoCdcMergeFlow rejects SCD2 framework column in ignore-null selection")(
+    Seq(Scd2BatchProcessor.startAtColName, Scd2BatchProcessor.endAtColName)
+  ) { colName =>
+    checkError(
+      exception = intercept[AnalysisException] {
+        newAutoCdcMergeFlow(
+          sourceDf = threeColumnSourceDf(),
+          storedAsScdType = ScdType.Type2,
+          ignoreNullSelection = Some(
+            ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName(colName)))
+          )
+        )
+      },
+      condition = "AUTOCDC_IGNORE_NULL_COLUMN_NOT_IN_OUTPUT_COLUMNS",
+      sqlState = "42703",
+      parameters = Map(
+        "flowName" -> testIdentifier.unquotedString,
+        "caseSensitivity" -> CaseSensitivityLabels.CaseInsensitive,
+        "columnName" -> colName
+      )
+    )
+  }
+
+  test("AutoCdcMergeFlow rejects ignore-null key case-sensitively") {
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+      checkError(
+        exception = intercept[AnalysisException] {
+          newAutoCdcMergeFlow(
+            sourceDf = threeColumnSourceDf(),
+            ignoreNullSelection = Some(
+              ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("id")))
+            )
+          )
+        },
+        condition = "AUTOCDC_IGNORE_NULL_SELECTION_CONTAINS_KEY_COLUMN",
+        sqlState = "22023",
+        parameters = Map(
+          "flowName" -> testIdentifier.unquotedString,
+          "caseSensitivity" -> CaseSensitivityLabels.CaseSensitive,
+          "columnName" -> "id",
+          "keyColumnNames" -> "id"
+        )
+      )
+    }
+  }
+
+  test("AutoCdcMergeFlow accepts valid SCD2 ignore-null selection") {
+    val flow = newAutoCdcMergeFlow(
+      sourceDf = threeColumnSourceDf(),
+      storedAsScdType = ScdType.Type2,
+      ignoreNullSelection = Some(
+        ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("name")))
+      )
+    )
+    assert(flow.changeArgs.ignoreNullSelection.isDefined)
+  }
+
+  test("AutoCdcMergeFlow accepts empty ignore-null exclude list") {
+    val flow = newAutoCdcMergeFlow(
+      sourceDf = threeColumnSourceDf(),
+      ignoreNullSelection = Some(ColumnSelection.ExcludeColumns(Seq.empty))
+    )
+    assert(flow.changeArgs.ignoreNullSelection.isDefined)
+  }
 }
diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/ChangeArgsSuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/ChangeArgsSuite.scala
index 7deda6fb0d26d..2751554a995d2 100644
--- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/ChangeArgsSuite.scala
+++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/ChangeArgsSuite.scala
@@ -418,6 +418,60 @@ class ChangeArgsSuite extends SparkFunSuite with SharedSparkSession {
     assert(args.trackHistorySelection.isEmpty)
   }
 
+  test("ChangeArgs rejects an empty ignoreNullSelection include list") {
+    checkError(
+      exception = intercept[AnalysisException] {
+        ChangeArgs(
+          keys = Seq(UnqualifiedColumnName("id")),
+          sequencing = F.col("seq"),
+          storedAsScdType = ScdType.Type2,
+          ignoreNullSelection = Some(
+            ColumnSelection.IncludeColumns(Seq.empty)
+          )
+        )
+      },
+      condition = "AUTOCDC_IGNORE_NULL_EMPTY_COLUMN_LIST",
+      sqlState = "22023",
+      parameters = Map.empty
+    )
+  }
+
+  test("ChangeArgs allows non-empty ignoreNullSelection include list") {
+    val args = ChangeArgs(
+      keys = Seq(UnqualifiedColumnName("id")),
+      sequencing = F.col("seq"),
+      storedAsScdType = ScdType.Type2,
+      ignoreNullSelection = Some(
+        ColumnSelection.IncludeColumns(
+          Seq(UnqualifiedColumnName("Name"))
+        )
+      )
+    )
+    assert(args.ignoreNullSelection.isDefined)
+  }
+
+  test("ChangeArgs allows empty ignoreNullSelection exclude list") {
+    // An empty exclude list means "every eligible column", which is valid.
+    val args = ChangeArgs(
+      keys = Seq(UnqualifiedColumnName("id")),
+      sequencing = F.col("seq"),
+      storedAsScdType = ScdType.Type2,
+      ignoreNullSelection = Some(
+        ColumnSelection.ExcludeColumns(Seq.empty)
+      )
+    )
+    assert(args.ignoreNullSelection.isDefined)
+  }
+
+  test("ChangeArgs allows None ignoreNullSelection") {
+    val args = ChangeArgs(
+      keys = Seq(UnqualifiedColumnName("id")),
+      sequencing = F.col("seq"),
+      storedAsScdType = ScdType.Type2
+    )
+    assert(args.ignoreNullSelection.isEmpty)
+  }
+
   test("UnqualifiedColumnName lets a ParseException from the SQL parser propagate") {
     checkError(
       exception = intercept[ParseException] {
diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessorMergeSuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessorMergeSuite.scala
index f77a35fecd15a..cd0fb805528b8 100644
--- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessorMergeSuite.scala
+++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessorMergeSuite.scala
@@ -143,56 +143,56 @@ class Scd2BatchProcessorMergeSuite
 
     // A freshly-routed tombstone with no matching aux row is inserted live (no logical-delete
     // marker).
-    val tagged = taggedOf(Row(1, "x", 5L, 5L, Row(5L), true))
+    val tagged = taggedOf(Row(1, "x", 5L, 5L, Row(5L, null), true))
     val affected = canonicalOf()
 
     processor.mergeRowsIntoAuxiliaryTable(tagged, affected, defaultAuxTableIdentifier, batchId = 1L)
 
-    checkAnswer(auxTable, Row(1, "x", 5L, 5L, Row(5L), null))
+    checkAnswer(auxTable, Row(1, "x", 5L, 5L, Row(5L, null), null))
   }
 
   test("mergeRowsIntoAuxiliaryTable updates a surviving routed row in place") {
     // An existing hidden no-op upsert at recordStartAt=5 that survives reconciliation.
-    createAuxTable(Row(1, "old", 5L, null, Row(5L), null))
+    createAuxTable(Row(1, "old", 5L, null, Row(5L, null), null))
 
-    val tagged = taggedOf(Row(1, "new", 5L, null, Row(5L), true))
+    val tagged = taggedOf(Row(1, "new", 5L, null, Row(5L, null), true))
     // The affected row survives (still present in the routed set), so it is not logically
     // deleted; only its non-key columns are refreshed.
-    val affected = canonicalOf(Row(1, "old", 5L, null, Row(5L)))
+    val affected = canonicalOf(Row(1, "old", 5L, null, Row(5L, null)))
 
     processor.mergeRowsIntoAuxiliaryTable(tagged, affected, defaultAuxTableIdentifier, batchId = 2L)
 
-    checkAnswer(auxTable, Row(1, "new", 5L, null, Row(5L), null))
+    checkAnswer(auxTable, Row(1, "new", 5L, null, Row(5L, null), null))
   }
 
   test("mergeRowsIntoAuxiliaryTable logically deletes an affected row that did not survive") {
     // A tombstone pulled in as affected but absent from this batch's routed set.
-    createAuxTable(Row(1, "gone", 7L, 7L, Row(7L), null))
+    createAuxTable(Row(1, "gone", 7L, 7L, Row(7L, null), null))
 
     // The single tagged row is not routed to the aux table, so the affected aux row has no
     // surviving counterpart and must be logically deleted.
-    val tagged = taggedOf(Row(1, "vis", 10L, null, Row(10L), false))
-    val affected = canonicalOf(Row(1, "gone", 7L, 7L, Row(7L)))
+    val tagged = taggedOf(Row(1, "vis", 10L, null, Row(10L, null), false))
+    val affected = canonicalOf(Row(1, "gone", 7L, 7L, Row(7L, null)))
 
     processor.mergeRowsIntoAuxiliaryTable(tagged, affected, defaultAuxTableIdentifier, batchId = 3L)
 
     // Logical, not physical: the row stays but is stamped with this batch's id.
-    checkAnswer(auxTable, Row(1, "gone", 7L, 7L, Row(7L), 3L))
+    checkAnswer(auxTable, Row(1, "gone", 7L, 7L, Row(7L, null), 3L))
   }
 
   test("mergeRowsIntoAuxiliaryTable garbage-collects rows logically deleted by an older batch") {
     createAuxTable(
       // Logically deleted by an older batch (2 != 5) -> physically garbage-collected.
-      Row(1, "gc-old", 7L, 7L, Row(7L), 2L),
+      Row(1, "gc-old", 7L, 7L, Row(7L, null), 2L),
       // Still live (no marker) -> retained.
-      Row(2, "live", 3L, null, Row(3L), null),
+      Row(2, "live", 3L, null, Row(3L, null), null),
       // Logically deleted by the current batch (5 == 5) -> retained for replay-stability.
-      Row(3, "this-batch", 4L, 4L, Row(4L), 5L)
+      Row(3, "this-batch", 4L, 4L, Row(4L, null), 5L)
     )
 
     // One brand-new routed insert keeps the merge source non-empty; none of the seeded rows are
     // in the affected set, so they are all evaluated by the not-matched-by-source GC clause.
-    val tagged = taggedOf(Row(10, "newtomb", 8L, 8L, Row(8L), true))
+    val tagged = taggedOf(Row(10, "newtomb", 8L, 8L, Row(8L, null), true))
     val affected = canonicalOf()
 
     processor.mergeRowsIntoAuxiliaryTable(tagged, affected, defaultAuxTableIdentifier, batchId = 5L)
@@ -200,9 +200,9 @@ class Scd2BatchProcessorMergeSuite
     checkAnswer(
       auxTable,
       Seq(
-        Row(2, "live", 3L, null, Row(3L), null),
-        Row(3, "this-batch", 4L, 4L, Row(4L), 5L),
-        Row(10, "newtomb", 8L, 8L, Row(8L), null)
+        Row(2, "live", 3L, null, Row(3L, null), null),
+        Row(3, "this-batch", 4L, 4L, Row(4L, null), 5L),
+        Row(10, "newtomb", 8L, 8L, Row(8L, null), null)
       )
     )
   }
@@ -210,15 +210,15 @@ class Scd2BatchProcessorMergeSuite
   test("mergeRowsIntoAuxiliaryTable is replay-stable: re-running the same batchId is a no-op") {
     createAuxTable(
       // Survives reconciliation -> re-upserted in place.
-      Row(1, "old", 5L, null, Row(5L), null),
+      Row(1, "old", 5L, null, Row(5L, null), null),
       // Affected but does not survive -> logically deleted by this batch.
-      Row(2, "gone", 7L, 7L, Row(7L), null)
+      Row(2, "gone", 7L, 7L, Row(7L, null), null)
     )
 
-    val tagged = taggedOf(Row(1, "new", 5L, null, Row(5L), true))
+    val tagged = taggedOf(Row(1, "new", 5L, null, Row(5L, null), true))
     val affected = canonicalOf(
-      Row(1, "old", 5L, null, Row(5L)),
-      Row(2, "gone", 7L, 7L, Row(7L))
+      Row(1, "old", 5L, null, Row(5L, null)),
+      Row(2, "gone", 7L, 7L, Row(7L, null))
     )
 
     // First application of the batch.
@@ -238,8 +238,8 @@ class Scd2BatchProcessorMergeSuite
     checkAnswer(
       auxTable,
       Seq(
-        Row(1, "new", 5L, null, Row(5L), null),
-        Row(2, "gone", 7L, 7L, Row(7L), 7L)
+        Row(1, "new", 5L, null, Row(5L, null), null),
+        Row(2, "gone", 7L, 7L, Row(7L, null), 7L)
       )
     )
   }
@@ -247,23 +247,23 @@ class Scd2BatchProcessorMergeSuite
   test("mergeRowsIntoAuxiliaryTable applies insert, update, logical-delete, and GC in one merge") {
     createAuxTable(
       // Affected and survives reconciliation -> non-key columns updated in place.
-      Row(1, "old", 5L, null, Row(5L), null),
+      Row(1, "old", 5L, null, Row(5L, null), null),
       // Affected but does not survive -> logically deleted by this batch (stamped with batchId).
-      Row(2, "gone", 7L, 7L, Row(7L), null),
+      Row(2, "gone", 7L, 7L, Row(7L, null), null),
       // Not affected, logically deleted by an older batch (4 != 9) -> physically GC'd.
-      Row(3, "gc", 3L, 3L, Row(3L), 4L)
+      Row(3, "gc", 3L, 3L, Row(3L, null), 4L)
     )
 
     val tagged = taggedOf(
       // Matches key 1 -> update.
-      Row(1, "new", 5L, null, Row(5L), true),
+      Row(1, "new", 5L, null, Row(5L, null), true),
       // New key 4 routed to aux -> insert.
-      Row(4, "ins", 9L, 9L, Row(9L), true)
+      Row(4, "ins", 9L, 9L, Row(9L, null), true)
     )
     // Keys 1 and 2 were pulled in as affected; key 1 survives in the routed set, key 2 does not.
     val affected = canonicalOf(
-      Row(1, "old", 5L, null, Row(5L)),
-      Row(2, "gone", 7L, 7L, Row(7L))
+      Row(1, "old", 5L, null, Row(5L, null)),
+      Row(2, "gone", 7L, 7L, Row(7L, null))
     )
 
     processor.mergeRowsIntoAuxiliaryTable(tagged, affected, defaultAuxTableIdentifier, batchId = 9L)
@@ -271,9 +271,9 @@ class Scd2BatchProcessorMergeSuite
     checkAnswer(
       auxTable,
       Seq(
-        Row(1, "new", 5L, null, Row(5L), null),
-        Row(2, "gone", 7L, 7L, Row(7L), 9L),
-        Row(4, "ins", 9L, 9L, Row(9L), null)
+        Row(1, "new", 5L, null, Row(5L, null), null),
+        Row(2, "gone", 7L, 7L, Row(7L, null), 9L),
+        Row(4, "ins", 9L, 9L, Row(9L, null), null)
         // key 3 physically garbage-collected.
       )
     )
@@ -284,21 +284,21 @@ class Scd2BatchProcessorMergeSuite
       defaultAuxIdent,
       defaultAuxTableIdentifier,
       wideAuxSchema,
-      Row(1, "old-name", "active", 10, 5L, null, Row(5L), null)
+      Row(1, "old-name", "active", 10, 5L, null, Row(5L, null), null)
     )
 
     // Every non-key column (name, status, score) differs from the seeded row; the in-place update
     // must refresh all of them, not just the first.
     val tagged = microbatchOf(wideTaggedSchema)(
-      Row(1, "new-name", "inactive", 42, 5L, null, Row(5L), true)
+      Row(1, "new-name", "inactive", 42, 5L, null, Row(5L, null), true)
     )
     val affected = microbatchOf(wideCanonicalSchema)(
-      Row(1, "old-name", "active", 10, 5L, null, Row(5L))
+      Row(1, "old-name", "active", 10, 5L, null, Row(5L, null))
     )
 
     processor.mergeRowsIntoAuxiliaryTable(tagged, affected, defaultAuxTableIdentifier, batchId = 1L)
 
-    checkAnswer(auxTable, Row(1, "new-name", "inactive", 42, 5L, null, Row(5L), null))
+    checkAnswer(auxTable, Row(1, "new-name", "inactive", 42, 5L, null, Row(5L, null), null))
   }
 
   test("mergeRowsIntoAuxiliaryTable handles user columns whose names contain a dot") {
@@ -307,12 +307,12 @@ class Scd2BatchProcessorMergeSuite
     // nested-field access and the MERGE would fail to resolve the column.
     createTable(defaultAuxIdent, defaultAuxTableIdentifier, dottedAuxSchema)
 
-    val tagged = microbatchOf(dottedTaggedSchema)(Row(1, "alice", 5L, 5L, Row(5L), true))
+    val tagged = microbatchOf(dottedTaggedSchema)(Row(1, "alice", 5L, 5L, Row(5L, null), true))
     val affected = microbatchOf(dottedCanonicalSchema)()
 
     processor.mergeRowsIntoAuxiliaryTable(tagged, affected, defaultAuxTableIdentifier, batchId = 1L)
 
-    checkAnswer(auxTable, Row(1, "alice", 5L, 5L, Row(5L), null))
+    checkAnswer(auxTable, Row(1, "alice", 5L, 5L, Row(5L, null), null))
   }
 
   // ===========================================================================================
@@ -322,34 +322,34 @@ class Scd2BatchProcessorMergeSuite
   test("mergeRowsIntoTargetTable inserts a visible upsert absent from the target table") {
     createTargetTable()
 
-    val tagged = taggedOf(Row(1, "v", 5L, null, Row(5L), false))
+    val tagged = taggedOf(Row(1, "v", 5L, null, Row(5L, null), false))
     val affected = canonicalOf()
 
     processor.mergeRowsIntoTargetTable(tagged, affected, defaultTargetTableIdentifier)
 
-    checkAnswer(targetTable, Row(1, "v", 5L, null, Row(5L)))
+    checkAnswer(targetTable, Row(1, "v", 5L, null, Row(5L, null)))
   }
 
   test("mergeRowsIntoTargetTable updates a visible row matched at the same recordStartAt") {
-    createTargetTable(Row(1, "old", 5L, null, Row(5L)))
+    createTargetTable(Row(1, "old", 5L, null, Row(5L, null)))
 
     // The run head at recordStartAt=5 is now closed at 20 with refreshed data; it matches and
     // updates the existing target row in place.
-    val tagged = taggedOf(Row(1, "new", 5L, 20L, Row(5L), false))
-    val affected = canonicalOf(Row(1, "old", 5L, null, Row(5L)))
+    val tagged = taggedOf(Row(1, "new", 5L, 20L, Row(5L, null), false))
+    val affected = canonicalOf(Row(1, "old", 5L, null, Row(5L, null)))
 
     processor.mergeRowsIntoTargetTable(tagged, affected, defaultTargetTableIdentifier)
 
-    checkAnswer(targetTable, Row(1, "new", 5L, 20L, Row(5L)))
+    checkAnswer(targetTable, Row(1, "new", 5L, 20L, Row(5L, null)))
   }
 
   test("mergeRowsIntoTargetTable deletes an affected row reconciled away") {
-    createTargetTable(Row(1, "old", 5L, null, Row(5L)))
+    createTargetTable(Row(1, "old", 5L, null, Row(5L, null)))
 
     // The only tagged row routes to the aux table (e.g. closed into a tombstone), so no visible
     // row survives for key 1: the previously-affected target row must be deleted.
-    val tagged = taggedOf(Row(1, "x", 5L, 5L, Row(5L), true))
-    val affected = canonicalOf(Row(1, "old", 5L, null, Row(5L)))
+    val tagged = taggedOf(Row(1, "x", 5L, 5L, Row(5L, null), true))
+    val affected = canonicalOf(Row(1, "old", 5L, null, Row(5L, null)))
 
     processor.mergeRowsIntoTargetTable(tagged, affected, defaultTargetTableIdentifier)
 
@@ -363,34 +363,34 @@ class Scd2BatchProcessorMergeSuite
     // are filtered by the routing flag; the decomposition tail is filtered because it is not an
     // upsert-representing row.
     val tagged = taggedOf(
-      Row(1, "vis", 5L, null, Row(5L), false),
-      Row(2, "tomb", 7L, 7L, Row(7L), true),
-      Row(3, "tail", null, 9L, Row(null), false),
-      Row(4, "hidden", 5L, null, Row(5L), true)
+      Row(1, "vis", 5L, null, Row(5L, null), false),
+      Row(2, "tomb", 7L, 7L, Row(7L, null), true),
+      Row(3, "tail", null, 9L, Row(null, null), false),
+      Row(4, "hidden", 5L, null, Row(5L, null), true)
     )
     val affected = canonicalOf()
 
     processor.mergeRowsIntoTargetTable(tagged, affected, defaultTargetTableIdentifier)
 
-    checkAnswer(targetTable, Row(1, "vis", 5L, null, Row(5L)))
+    checkAnswer(targetTable, Row(1, "vis", 5L, null, Row(5L, null)))
   }
 
   test("mergeRowsIntoTargetTable applies insert, update, and delete branches in one merge") {
     createTargetTable(
-      Row(1, "old1", 5L, null, Row(5L)),
-      Row(2, "old2", 8L, null, Row(8L))
+      Row(1, "old1", 5L, null, Row(5L, null)),
+      Row(2, "old2", 8L, null, Row(8L, null))
     )
 
     val tagged = taggedOf(
       // Matches key 1 at recordStartAt=5 -> update.
-      Row(1, "new1", 5L, 30L, Row(5L), false),
+      Row(1, "new1", 5L, 30L, Row(5L, null), false),
       // New key 3 -> insert.
-      Row(3, "ins3", 12L, null, Row(12L), false)
+      Row(3, "ins3", 12L, null, Row(12L, null), false)
       // Key 2 has no surviving visible row -> delete.
     )
     val affected = canonicalOf(
-      Row(1, "old1", 5L, null, Row(5L)),
-      Row(2, "old2", 8L, null, Row(8L))
+      Row(1, "old1", 5L, null, Row(5L, null)),
+      Row(2, "old2", 8L, null, Row(8L, null))
     )
 
     processor.mergeRowsIntoTargetTable(tagged, affected, defaultTargetTableIdentifier)
@@ -398,8 +398,8 @@ class Scd2BatchProcessorMergeSuite
     checkAnswer(
       targetTable,
       Seq(
-        Row(1, "new1", 5L, 30L, Row(5L)),
-        Row(3, "ins3", 12L, null, Row(12L))
+        Row(1, "new1", 5L, 30L, Row(5L, null)),
+        Row(3, "ins3", 12L, null, Row(12L, null))
       )
     )
   }
@@ -410,31 +410,31 @@ class Scd2BatchProcessorMergeSuite
     // parsed as a nested-field access and the MERGE would fail to resolve the column.
     createTable(defaultTargetIdent, defaultTargetTableIdentifier, dottedCanonicalSchema)
 
-    val tagged = microbatchOf(dottedTaggedSchema)(Row(1, "alice", 5L, null, Row(5L), false))
+    val tagged = microbatchOf(dottedTaggedSchema)(Row(1, "alice", 5L, null, Row(5L, null), false))
     val affected = microbatchOf(dottedCanonicalSchema)()
 
     processor.mergeRowsIntoTargetTable(tagged, affected, defaultTargetTableIdentifier)
 
-    checkAnswer(targetTable, Row(1, "alice", 5L, null, Row(5L)))
+    checkAnswer(targetTable, Row(1, "alice", 5L, null, Row(5L, null)))
   }
 
   test("mergeRowsIntoTargetTable updates every non-key column of a matched row") {
     createTable(defaultTargetIdent, defaultTargetTableIdentifier, wideCanonicalSchema)
     microbatchOf(wideCanonicalSchema)(
-      Row(1, "old-name", "active", 10, 5L, null, Row(5L))
+      Row(1, "old-name", "active", 10, 5L, null, Row(5L, null))
     ).writeTo(defaultTargetTableIdentifier.quotedString).append()
 
     // Every non-key column (name, status, score) differs from the existing row; the in-place
     // update must refresh all of them, exercising the full non-key update assignment map.
     val tagged = microbatchOf(wideTaggedSchema)(
-      Row(1, "new-name", "inactive", 42, 5L, 30L, Row(5L), false)
+      Row(1, "new-name", "inactive", 42, 5L, 30L, Row(5L, null), false)
     )
     val affected = microbatchOf(wideCanonicalSchema)(
-      Row(1, "old-name", "active", 10, 5L, null, Row(5L))
+      Row(1, "old-name", "active", 10, 5L, null, Row(5L, null))
     )
 
     processor.mergeRowsIntoTargetTable(tagged, affected, defaultTargetTableIdentifier)
 
-    checkAnswer(targetTable, Row(1, "new-name", "inactive", 42, 5L, 30L, Row(5L)))
+    checkAnswer(targetTable, Row(1, "new-name", "inactive", 42, 5L, 30L, Row(5L, null)))
   }
 }
diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessorSuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessorSuite.scala
index de82f05047cd5..71d837740321d 100644
--- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessorSuite.scala
+++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessorSuite.scala
@@ -162,9 +162,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // (recordStartAt = null, endAt = 10) takes its endAt as its effective ordering sequence,
     // so the expected per-key window order is 5, tail(10), 15.
     val df = targetTableOf(userSchema)(
-      Row(1, "v15", 15L, null, Row(15L)),
-      Row(1, "tail", null, 10L, Row(null)),
-      Row(1, "v5", 5L, null, Row(5L))
+      Row(1, "v15", 15L, null, Row(15L, null)),
+      Row(1, "tail", null, 10L, Row(null, null)),
+      Row(1, "v5", 5L, null, Row(5L, null))
     )
 
     val withRn = df.withColumn(
@@ -174,9 +174,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = withRn,
       expectedAnswer = Seq(
-        Row(1, "v5", 5L, null, Row(5L), 1),
-        Row(1, "tail", null, 10L, Row(null), 2),
-        Row(1, "v15", 15L, null, Row(15L), 3)
+        Row(1, "v5", 5L, null, Row(5L, null), 1),
+        Row(1, "tail", null, 10L, Row(null, null), 2),
+        Row(1, "v15", 15L, null, Row(15L, null), 3)
       )
     )
   }
@@ -192,12 +192,12 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     //   key=2: upsert-representing-first (open variant)   - open upsert vs tombstone.
     //   key=3: upsert-representing-first (closed variant) - closed run head vs tombstone.
     val df = targetTableOf(userSchema)(
-      Row(1, "tomb", 10L, 10L, Row(10L)),
-      Row(1, "tail", null, 10L, Row(null)),
-      Row(2, "tomb", 10L, 10L, Row(10L)),
-      Row(2, "open", 10L, null, Row(10L)),
-      Row(3, "tomb", 10L, 10L, Row(10L)),
-      Row(3, "closed", 10L, 20L, Row(10L))
+      Row(1, "tomb", 10L, 10L, Row(10L, null)),
+      Row(1, "tail", null, 10L, Row(null, null)),
+      Row(2, "tomb", 10L, 10L, Row(10L, null)),
+      Row(2, "open", 10L, null, Row(10L, null)),
+      Row(3, "tomb", 10L, 10L, Row(10L, null)),
+      Row(3, "closed", 10L, 20L, Row(10L, null))
     )
 
     val withRn = df.withColumn(
@@ -207,12 +207,12 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = withRn,
       expectedAnswer = Seq(
-        Row(1, "tail", null, 10L, Row(null), 1),
-        Row(1, "tomb", 10L, 10L, Row(10L), 2),
-        Row(2, "open", 10L, null, Row(10L), 1),
-        Row(2, "tomb", 10L, 10L, Row(10L), 2),
-        Row(3, "closed", 10L, 20L, Row(10L), 1),
-        Row(3, "tomb", 10L, 10L, Row(10L), 2)
+        Row(1, "tail", null, 10L, Row(null, null), 1),
+        Row(1, "tomb", 10L, 10L, Row(10L, null), 2),
+        Row(2, "open", 10L, null, Row(10L, null), 1),
+        Row(2, "tomb", 10L, 10L, Row(10L, null), 2),
+        Row(3, "closed", 10L, 20L, Row(10L, null), 1),
+        Row(3, "tomb", 10L, 10L, Row(10L, null), 2)
       )
     )
   }
@@ -225,12 +225,12 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // by effective recordStartAt - rows from key 2 must not influence row_number positions
     // of rows for key 1, and vice versa.
     val df = targetTableOf(userSchema)(
-      Row(1, "k1-15", 15L, null, Row(15L)),
-      Row(2, "k2-7", 7L, null, Row(7L)),
-      Row(1, "k1-5", 5L, null, Row(5L)),
-      Row(2, "k2-3", 3L, null, Row(3L)),
-      Row(1, "k1-10", 10L, null, Row(10L)),
-      Row(2, "k2-20", 20L, null, Row(20L))
+      Row(1, "k1-15", 15L, null, Row(15L, null)),
+      Row(2, "k2-7", 7L, null, Row(7L, null)),
+      Row(1, "k1-5", 5L, null, Row(5L, null)),
+      Row(2, "k2-3", 3L, null, Row(3L, null)),
+      Row(1, "k1-10", 10L, null, Row(10L, null)),
+      Row(2, "k2-20", 20L, null, Row(20L, null))
     )
 
     val withRn = df.withColumn(
@@ -240,12 +240,12 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = withRn,
       expectedAnswer = Seq(
-        Row(1, "k1-5", 5L, null, Row(5L), 1),
-        Row(1, "k1-10", 10L, null, Row(10L), 2),
-        Row(1, "k1-15", 15L, null, Row(15L), 3),
-        Row(2, "k2-3", 3L, null, Row(3L), 1),
-        Row(2, "k2-7", 7L, null, Row(7L), 2),
-        Row(2, "k2-20", 20L, null, Row(20L), 3)
+        Row(1, "k1-5", 5L, null, Row(5L, null), 1),
+        Row(1, "k1-10", 10L, null, Row(10L, null), 2),
+        Row(1, "k1-15", 15L, null, Row(15L, null), 3),
+        Row(2, "k2-3", 3L, null, Row(3L, null), 1),
+        Row(2, "k2-7", 7L, null, Row(7L, null), 2),
+        Row(2, "k2-20", 20L, null, Row(20L, null), 3)
       )
     )
   }
@@ -278,6 +278,18 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
       Scd2BatchProcessor.endAtColName,
       AutoCdcReservedNames.cdcMetadataColName
     ))
+
+    val cdcMetadataSchema =
+      result.schema(AutoCdcReservedNames.cdcMetadataColName).dataType.asInstanceOf[StructType]
+    assert(
+      cdcMetadataSchema.fieldNames.sameElements(
+        Array(Scd2BatchProcessor.recordStartAtFieldName, Scd2BatchProcessor.versionMapFieldName)
+      )
+    )
+
+    val versionMapField = cdcMetadataSchema(Scd2BatchProcessor.versionMapFieldName)
+    assert(versionMapField.dataType == MapType(StringType, BooleanType, valueContainsNull = false))
+    assert(versionMapField.nullable)
   }
 
   test("preprocessMicrobatch returns an empty DataFrame with the full preprocessed schema") {
@@ -343,9 +355,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = processor.preprocessMicrobatch(batch),
       expectedAnswer = Seq(
-        Row(1, 10L, "first-upsert", false, 10L, null, Row(10L)),
-        Row(1, 20L, "second-upsert", false, 20L, null, Row(20L)),
-        Row(1, 30L, null, true, 30L, 30L, Row(30L))
+        Row(1, 10L, "first-upsert", false, 10L, null, Row(10L, null)),
+        Row(1, 20L, "second-upsert", false, 20L, null, Row(20L, null)),
+        Row(1, 30L, null, true, 30L, 30L, Row(30L, null))
       )
     )
   }
@@ -381,8 +393,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = processor.preprocessMicrobatch(batch),
       expectedAnswer = Seq(
-        Row(1, 10L, "alice", false, 10L, null, Row(10L)),
-        Row(1, 10L, "alice", false, 10L, null, Row(10L))
+        Row(1, 10L, "alice", false, 10L, null, Row(10L, null)),
+        Row(1, 10L, "alice", false, 10L, null, Row(10L, null))
       )
     )
   }
@@ -547,7 +559,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     ))
     checkAnswer(
       df = result,
-      expectedAnswer = Row(1, 30, 10L, null, Row(10L))
+      expectedAnswer = Row(1, 30, 10L, null, Row(10L, null))
     )
   }
 
@@ -579,7 +591,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     ))
     checkAnswer(
       df = result,
-      expectedAnswer = Row(1, 30, 10L, 10L, null, Row(10L))
+      expectedAnswer = Row(1, 30, 10L, 10L, null, Row(10L, null))
     )
   }
 
@@ -683,7 +695,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     ))
     checkAnswer(
       df = result,
-      expectedAnswer = Row(1, "u-100", 10L, null, Row(10L))
+      expectedAnswer = Row(1, "u-100", 10L, null, Row(10L, null))
     )
   }
 
@@ -728,8 +740,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 10L, null, Row(10L)),
-        Row(1, null, 20L, 20L, Row(20L))
+        Row(1, "alice", 10L, null, Row(10L, null)),
+        Row(1, null, 20L, 20L, Row(20L, null))
       )
     )
   }
@@ -876,10 +888,10 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // back to minSeq itself.
     //   - 7  -> sits at the cutoff; included.
     val aux = auxTableOf(userSchema)(
-      Row(1, "v1.3", 3L, null, Row(3L), null),
-      Row(1, "v1.5", 5L, null, Row(5L), null),
-      Row(1, "v1.10", 10L, null, Row(10L), null),
-      Row(2, "v2.7", 7L, null, Row(7L), null)
+      Row(1, "v1.3", 3L, null, Row(3L, null), null),
+      Row(1, "v1.5", 5L, null, Row(5L, null), null),
+      Row(1, "v1.10", 10L, null, Row(10L, null), null),
+      Row(2, "v2.7", 7L, null, Row(7L, null), null)
     )
     val minSeq = minSeqOf(keySchema)(
       Row(1, 10L),
@@ -892,9 +904,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "v1.5", 5L, null, Row(5L)), // sits at key=1's cutoff
-        Row(1, "v1.10", 10L, null, Row(10L)), // after key=1's cutoff
-        Row(2, "v2.7", 7L, null, Row(7L))     // sits at key=2's cutoff, which fell back to minSeq
+        Row(1, "v1.5", 5L, null, Row(5L, null)), // sits at key=1's cutoff
+        Row(1, "v1.10", 10L, null, Row(10L, null)), // after key=1's cutoff
+        Row(2, "v2.7", 7L, null, Row(7L, null)) // key=2's cutoff (fell back to minSeq)
       )
     )
   }
@@ -910,14 +922,14 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     val aux = auxTableOf(userSchema)(
       // Tombstone at recordStartAt = 3 (deleted at sequence 3): startAt = endAt = 3.
       // Below the cutoff; dropped.
-      Row(1, null, 3L, 3L, Row(3L), null),
+      Row(1, null, 3L, 3L, Row(3L, null), null),
       // No-op upsert continuation at recordStartAt = 7: startAt inherits its run head's
       // recordStartAt, endAt is null. Sets the cutoff for minSeq=10 (nearest below it).
-      Row(1, "alice", 5L, null, Row(7L), null),
+      Row(1, "alice", 5L, null, Row(7L, null), null),
       // Tombstone at recordStartAt = 12: after the cutoff; included.
-      Row(1, null, 12L, 12L, Row(12L), null),
+      Row(1, null, 12L, 12L, Row(12L, null), null),
       // No-op upsert continuation at recordStartAt = 15: after the cutoff; included.
-      Row(1, "bob", 13L, null, Row(15L), null)
+      Row(1, "bob", 13L, null, Row(15L, null), null)
     )
     val minSeq = minSeqOf(keySchema)(Row(1, 10L))
     val target = targetTableOf(userSchema)()
@@ -927,9 +939,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, null, Row(7L)),
-        Row(1, null, 12L, 12L, Row(12L)),
-        Row(1, "bob", 13L, null, Row(15L))
+        Row(1, "alice", 5L, null, Row(7L, null)),
+        Row(1, null, 12L, 12L, Row(12L, null)),
+        Row(1, "bob", 13L, null, Row(15L, null))
       )
     )
   }
@@ -941,8 +953,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     val userSchema = keySchema.add("value", StringType)
 
     val aux = auxTableOf(userSchema)(
-      Row(1, "alice", 2L, null, Row(8L), null),
-      Row(1, "alice", 2L, null, Row(12L), null)
+      Row(1, "alice", 2L, null, Row(8L, null), null),
+      Row(1, "alice", 2L, null, Row(12L, null), null)
     )
     val minSeq = minSeqOf(keySchema)(Row(1, 10L))
     val target = targetTableOf(userSchema)()
@@ -953,9 +965,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
       df = result,
       expectedAnswer = Seq(
         // Row with record start at of 8 sets the cutoff,
-        Row(1, "alice", 2L, null, Row(8L)),
+        Row(1, "alice", 2L, null, Row(8L, null)),
         // Row with record start at of 12 sits after the cutoff.
-        Row(1, "alice", 2L, null, Row(12L))
+        Row(1, "alice", 2L, null, Row(12L, null))
       )
     )
   }
@@ -973,8 +985,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // pull it in as a harmless side effect of the range filter, and this behavior is
     // documented via test.
     val aux = auxTableOf(userSchema)(
-      Row(1, null, 7L, 7L, Row(7L), null),
-      Row(1, null, 12L, 12L, Row(12L), null)
+      Row(1, null, 7L, 7L, Row(7L, null), null),
+      Row(1, null, 12L, 12L, Row(12L, null), null)
     )
     val minSeq = minSeqOf(keySchema)(Row(1, 10L))
     val target = targetTableOf(userSchema)()
@@ -985,9 +997,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
       df = result,
       expectedAnswer = Seq(
         // Sets the cutoff.
-        Row(1, null, 7L, 7L, Row(7L)),
+        Row(1, null, 7L, 7L, Row(7L, null)),
         // Sits after the cutoff.
-        Row(1, null, 12L, 12L, Row(12L))
+        Row(1, null, 12L, 12L, Row(12L, null))
       )
     )
   }
@@ -1005,11 +1017,11 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // applies uniformly to the row at the cutoff and to the rows after it.
     val aux = auxTableOf(userSchema)(
       // Cutoff candidate (recordStartAt < minSeq):
-      Row(1, "anchor", 5L, null, Row(5L), currentBatchId), // deleted by current -> kept
+      Row(1, "anchor", 5L, null, Row(5L, null), currentBatchId), // deleted by current -> kept
       // At-or-after minSeq:
-      Row(1, "live", 10L, null, Row(10L), null), // not deleted -> kept
-      Row(1, "retried", 11L, null, Row(11L), currentBatchId), // deleted by current -> kept
-      Row(1, "ignored", 12L, null, Row(12L), differentBatchId)   // deleted by another -> dropped
+      Row(1, "live", 10L, null, Row(10L, null), null), // not deleted -> kept
+      Row(1, "retried", 11L, null, Row(11L, null), currentBatchId), // deleted by current -> kept
+      Row(1, "ignored", 12L, null, Row(12L, null), differentBatchId) // other batch -> dropped
     )
     val minSeq = minSeqOf(keySchema)(Row(1, 10L))
     val target = targetTableOf(userSchema)()
@@ -1025,9 +1037,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "anchor", 5L, null, Row(5L)),
-        Row(1, "live", 10L, null, Row(10L)),
-        Row(1, "retried", 11L, null, Row(11L))
+        Row(1, "anchor", 5L, null, Row(5L, null)),
+        Row(1, "live", 10L, null, Row(10L, null)),
+        Row(1, "retried", 11L, null, Row(11L, null))
       )
     )
   }
@@ -1049,8 +1061,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // because row 7 would set the cutoff and then be dropped, leaving nothing at the cutoff
     // at all.
     val aux = auxTableOf(userSchema)(
-      Row(1, "live3", 3L, null, Row(3L), null),
-      Row(1, "stale7", 7L, null, Row(7L), differentBatchId)
+      Row(1, "live3", 3L, null, Row(3L, null), null),
+      Row(1, "stale7", 7L, null, Row(7L, null), differentBatchId)
     )
     val minSeq = minSeqOf(keySchema)(Row(1, 10L))
     val target = targetTableOf(userSchema)()
@@ -1065,7 +1077,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
 
     checkAnswer(
       df = result,
-      expectedAnswer = Seq(Row(1, "live3", 3L, null, Row(3L)))
+      expectedAnswer = Seq(Row(1, "live3", 3L, null, Row(3L, null)))
     )
   }
 
@@ -1079,7 +1091,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // drop that aux-only column so the result is union-compatible with target-table rows
     // and preprocessed-microbatch rows downstream, while leaving the (now-shared)
     // `_cdc_metadata` struct schema untouched.
-    val aux = auxTableOf(userSchema)(Row(1, "v", 5L, null, Row(5L), null))
+    val aux = auxTableOf(userSchema)(Row(1, "v", 5L, null, Row(5L, null), null))
     val minSeq = minSeqOf(keySchema)(Row(1, 10L))
     val target = targetTableOf(userSchema)()
 
@@ -1095,7 +1107,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     val keySchema = new StructType().add("a.b", IntegerType)
     val userSchema = keySchema.add("value", StringType)
 
-    val aux = auxTableOf(userSchema)(Row(1, "v", 5L, null, Row(5L), null))
+    val aux = auxTableOf(userSchema)(Row(1, "v", 5L, null, Row(5L, null), null))
     val minSeq = minSeqOf(keySchema)(Row(1, 10L))
     val target = targetTableOf(userSchema)()
 
@@ -1104,7 +1116,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // The lone aux row sets the cutoff (recordStartAt=5 < minSeq=10, no other candidates).
     checkAnswer(
       df = result,
-      expectedAnswer = Seq(Row(1, "v", 5L, null, Row(5L)))
+      expectedAnswer = Seq(Row(1, "v", 5L, null, Row(5L, null)))
     )
   }
 
@@ -1121,9 +1133,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     //   (EU, 1): cutoff at 4; nothing follows it, so only the cutoff row is selected.
     //   (US, 2): no aux rows -> contributes nothing.
     val aux = auxTableOf(userSchema)(
-      Row("US", 1, "us1.3", 3L, null, Row(3L), null),
-      Row("US", 1, "us1.10", 10L, null, Row(10L), null),
-      Row("EU", 1, "eu1.4", 4L, null, Row(4L), null)
+      Row("US", 1, "us1.3", 3L, null, Row(3L, null), null),
+      Row("US", 1, "us1.10", 10L, null, Row(10L, null), null),
+      Row("EU", 1, "eu1.4", 4L, null, Row(4L, null), null)
     )
     val minSeq = minSeqOf(keySchema)(
       Row("US", 1, 10L),
@@ -1137,9 +1149,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row("US", 1, "us1.3", 3L, null, Row(3L)),
-        Row("US", 1, "us1.10", 10L, null, Row(10L)),
-        Row("EU", 1, "eu1.4", 4L, null, Row(4L))
+        Row("US", 1, "us1.3", 3L, null, Row(3L, null)),
+        Row("US", 1, "us1.10", 10L, null, Row(10L, null)),
+        Row("EU", 1, "eu1.4", 4L, null, Row(4L, null))
       )
     )
   }
@@ -1165,7 +1177,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     val userSchema = keySchema.add("value", StringType)
 
     // Aux only has rows for key=1. Microbatch only sees key=2.
-    val aux = auxTableOf(userSchema)(Row(1, "v", 5L, null, Row(5L), null))
+    val aux = auxTableOf(userSchema)(Row(1, "v", 5L, null, Row(5L, null), null))
     val minSeq = minSeqOf(keySchema)(Row(2, 10L))
     val target = targetTableOf(userSchema)()
 
@@ -1182,8 +1194,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // Aux has rows for keys 1 and 2. Microbatch only mentions key=1, so key=2's aux rows
     // must be dropped (the inner join with minSeq strips them).
     val aux = auxTableOf(userSchema)(
-      Row(1, "v1", 5L, null, Row(5L), null),
-      Row(2, "v2", 7L, null, Row(7L), null)
+      Row(1, "v1", 5L, null, Row(5L, null), null),
+      Row(2, "v2", 7L, null, Row(7L, null), null)
     )
     val minSeq = minSeqOf(keySchema)(Row(1, 10L))
     val target = targetTableOf(userSchema)()
@@ -1192,7 +1204,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
 
     checkAnswer(
       df = result,
-      expectedAnswer = Seq(Row(1, "v1", 5L, null, Row(5L)))
+      expectedAnswer = Seq(Row(1, "v1", 5L, null, Row(5L, null)))
     )
   }
 
@@ -1211,10 +1223,10 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     //   - recordStartAt=10 -> after the cutoff  -> included
     //   - recordStartAt=15 -> after the cutoff  -> included (and is the active row)
     val target = targetTableOf(userSchema)(
-      Row(1, "old", 1L, 5L, Row(1L)),
-      Row(1, "edge", 5L, 10L, Row(5L)),
-      Row(1, "recent", 10L, 15L, Row(10L)),
-      Row(1, "active", 15L, null, Row(15L))
+      Row(1, "old", 1L, 5L, Row(1L, null)),
+      Row(1, "edge", 5L, 10L, Row(5L, null)),
+      Row(1, "recent", 10L, 15L, Row(10L, null)),
+      Row(1, "active", 15L, null, Row(15L, null))
     )
     val minSeq = minSeqOf(keySchema)(Row(1, 10L))
     val aux = auxTableOf(userSchema)()
@@ -1224,9 +1236,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "edge", 5L, 10L, Row(5L)),
-        Row(1, "recent", 10L, 15L, Row(10L)),
-        Row(1, "active", 15L, null, Row(15L))
+        Row(1, "edge", 5L, 10L, Row(5L, null)),
+        Row(1, "recent", 10L, 15L, Row(10L, null)),
+        Row(1, "active", 15L, null, Row(15L, null))
       )
     )
   }
@@ -1238,14 +1250,14 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
 
     // A standalone delete at 40 found nothing live to close, so it survives in the auxiliary
     // table as a tombstone. The upsert at 42 then opened a run in the gap after it.
-    val aux = auxTableOf(userSchema)(Row(1, null, 40L, 40L, Row(40L), null))
-    val target = targetTableOf(userSchema)(Row(1, "target", 42L, null, Row(42L)))
+    val aux = auxTableOf(userSchema)(Row(1, null, 40L, 40L, Row(40L, null), null))
+    val target = targetTableOf(userSchema)(Row(1, "target", 42L, null, Row(42L, null)))
     val minSeq = minSeqOf(keySchema)(Row(1, 50L))
 
     // The target's row at 42 is the cutoff, so the auxiliary tombstone at 40 falls below it.
     checkAnswer(
       df = findAffectedTargetRows(processor, target = target, aux = aux, minSeq = minSeq),
-      expectedAnswer = Seq(Row(1, "target", 42L, null, Row(42L)))
+      expectedAnswer = Seq(Row(1, "target", 42L, null, Row(42L, null)))
     )
     checkAnswer(
       df = findAffectedAuxRows(processor, aux = aux, target = target, minSeq = minSeq),
@@ -1261,15 +1273,15 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // A delete at 41 closed the target's run, leaving no tombstone of its own since the closed
     // row already carries that boundary. A later standalone delete at 42 landed in the gap after
     // it with nothing to close, and so survives in the auxiliary table.
-    val aux = auxTableOf(userSchema)(Row(1, null, 42L, 42L, Row(42L), null))
-    val target = targetTableOf(userSchema)(Row(1, "target", 40L, 41L, Row(40L)))
+    val aux = auxTableOf(userSchema)(Row(1, null, 42L, 42L, Row(42L, null), null))
+    val target = targetTableOf(userSchema)(Row(1, "target", 40L, 41L, Row(40L, null)))
     val minSeq = minSeqOf(keySchema)(Row(1, 50L))
 
     // The tombstone at 42 is the cutoff, so the target's row at 40 falls below it - correctly,
     // since that interval already closed at 41, before anything in the microbatch.
     checkAnswer(
       df = findAffectedAuxRows(processor, aux = aux, target = target, minSeq = minSeq),
-      expectedAnswer = Seq(Row(1, null, 42L, 42L, Row(42L)))
+      expectedAnswer = Seq(Row(1, null, 42L, 42L, Row(42L, null)))
     )
     checkAnswer(
       df = findAffectedTargetRows(processor, target = target, aux = aux, minSeq = minSeq),
@@ -1287,13 +1299,13 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     val target = targetTableOf(userSchema)(
       // Key 1: minSeq=10, so the cutoff is recordStartAt=5. "k1.recent" sits at it and
       // "k1.active" follows it; "k1.old" at recordStartAt=1 falls below it.
-      Row(1, "k1.old", 1L, 5L, Row(1L)),
-      Row(1, "k1.recent", 5L, 15L, Row(5L)),
-      Row(1, "k1.active", 15L, null, Row(15L)),
+      Row(1, "k1.old", 1L, 5L, Row(1L, null)),
+      Row(1, "k1.recent", 5L, 15L, Row(5L, null)),
+      Row(1, "k1.active", 15L, null, Row(15L, null)),
       // Key 2: minSeq=20, so the cutoff is recordStartAt=18. Only "k2.active" survives.
-      Row(2, "k2.old", 1L, 10L, Row(1L)),
-      Row(2, "k2.recent", 10L, 18L, Row(10L)),
-      Row(2, "k2.active", 18L, null, Row(18L))
+      Row(2, "k2.old", 1L, 10L, Row(1L, null)),
+      Row(2, "k2.recent", 10L, 18L, Row(10L, null)),
+      Row(2, "k2.active", 18L, null, Row(18L, null))
     )
     val minSeq = minSeqOf(keySchema)(
       Row(1, 10L),
@@ -1306,9 +1318,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "k1.recent", 5L, 15L, Row(5L)),
-        Row(1, "k1.active", 15L, null, Row(15L)),
-        Row(2, "k2.active", 18L, null, Row(18L))
+        Row(1, "k1.recent", 5L, 15L, Row(5L, null)),
+        Row(1, "k1.active", 15L, null, Row(15L, null)),
+        Row(2, "k2.active", 18L, null, Row(18L, null))
       )
     )
   }
@@ -1326,9 +1338,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // (EU, 1)'s cutoff for minSeq=12 is recordStartAt=5, so its older row at
     // recordStartAt=1 falls below it. (US, 2) has no target rows.
     val target = targetTableOf(userSchema)(
-      Row("US", 1, "us1", 1L, null, Row(1L)),
-      Row("EU", 1, "eu1.old", 1L, 5L, Row(1L)),
-      Row("EU", 1, "eu1", 5L, null, Row(5L))
+      Row("US", 1, "us1", 1L, null, Row(1L, null)),
+      Row("EU", 1, "eu1.old", 1L, 5L, Row(1L, null)),
+      Row("EU", 1, "eu1", 5L, null, Row(5L, null))
     )
     val minSeq = minSeqOf(keySchema)(
       Row("US", 1, 10L),
@@ -1342,8 +1354,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row("US", 1, "us1", 1L, null, Row(1L)),
-        Row("EU", 1, "eu1", 5L, null, Row(5L))
+        Row("US", 1, "us1", 1L, null, Row(1L, null)),
+        Row("EU", 1, "eu1", 5L, null, Row(5L, null))
       )
     )
   }
@@ -1369,7 +1381,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     val userSchema = keySchema.add("value", StringType)
 
     // Target only has rows for key=1. Microbatch only sees key=2.
-    val target = targetTableOf(userSchema)(Row(1, "v", 1L, null, Row(1L)))
+    val target = targetTableOf(userSchema)(Row(1, "v", 1L, null, Row(1L, null)))
     val minSeq = minSeqOf(keySchema)(Row(2, 10L))
     val aux = auxTableOf(userSchema)()
 
@@ -1393,9 +1405,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     //   - "tomb":  startAt == endAt, so it is excluded by the strict `<` closed check
     //   - "last":  closed, but is the last row in its window partition (no successor)
     val df = targetTableOf(userSchema)(
-      Row(1, "open", 100, 5L, null, Row(5L)),
-      Row(1, "tomb", 200, 10L, 10L, Row(10L)),
-      Row(1, "last", 300, 15L, 25L, Row(15L))
+      Row(1, "open", 100, 5L, null, Row(5L, null)),
+      Row(1, "tomb", 200, 10L, 10L, Row(10L, null)),
+      Row(1, "last", 300, 15L, 25L, Row(15L, null))
     )
 
     val result = processor.decomposeOutOfOrderRows(df)
@@ -1403,9 +1415,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "open", 100, 5L, null, Row(5L)),
-        Row(1, "tomb", 200, 10L, 10L, Row(10L)),
-        Row(1, "last", 300, 15L, 25L, Row(15L))
+        Row(1, "open", 100, 5L, null, Row(5L, null)),
+        Row(1, "tomb", 200, 10L, 10L, Row(10L, null)),
+        Row(1, "last", 300, 15L, 25L, Row(15L, null))
       )
     )
   }
@@ -1422,8 +1434,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // than 10. Here the successor lands at exactly 10, which means it doesn't actually
     // bisect the closed row and therefore shouldn't decompose it.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 42, 5L, 10L, Row(5L)),
-      Row(1, "bob", 99, 10L, null, Row(10L))
+      Row(1, "alice", 42, 5L, 10L, Row(5L, null)),
+      Row(1, "bob", 99, 10L, null, Row(10L, null))
     )
 
     val result = processor.decomposeOutOfOrderRows(df)
@@ -1431,8 +1443,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 42, 5L, 10L, Row(5L)),
-        Row(1, "bob", 99, 10L, null, Row(10L))
+        Row(1, "alice", 42, 5L, 10L, Row(5L, null)),
+        Row(1, "bob", 99, 10L, null, Row(10L, null))
       )
     )
   }
@@ -1452,8 +1464,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // Both head and tail must carry the parent's data columns (value="alice", amount=42)
     // identically.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 42, 5L, 30L, Row(5L)),
-      Row(1, "bob", 99, 15L, null, Row(15L))
+      Row(1, "alice", 42, 5L, 30L, Row(5L, null)),
+      Row(1, "bob", 99, 15L, null, Row(15L, null))
     )
 
     val result = processor.decomposeOutOfOrderRows(df)
@@ -1461,9 +1473,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 42, 5L, null, Row(5L)), // head
-        Row(1, "alice", 42, null, 30L, Row(null)), // tail
-        Row(1, "bob", 99, 15L, null, Row(15L))    // bisecting successor
+        Row(1, "alice", 42, 5L, null, Row(5L, null)), // head
+        Row(1, "alice", 42, null, 30L, Row(null, null)), // tail
+        Row(1, "bob", 99, 15L, null, Row(15L, null))    // bisecting successor
       )
     )
   }
@@ -1480,16 +1492,16 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // own row kind (the bisection check looks only at recordStartAt < parent.endAt).
     val df = targetTableOf(userSchema)(
       // Key 1: bisected by an open upsert.
-      Row(1, "alice", 1, 5L, 50L, Row(5L)),
-      Row(1, "bob", 2, 10L, null, Row(10L)),
+      Row(1, "alice", 1, 5L, 50L, Row(5L, null)),
+      Row(1, "bob", 2, 10L, null, Row(10L, null)),
 
       // Key 2: bisected by a tombstone.
-      Row(2, "carol", 3, 5L, 50L, Row(5L)),
-      Row(2, "dave", 4, 20L, 20L, Row(20L)),
+      Row(2, "carol", 3, 5L, 50L, Row(5L, null)),
+      Row(2, "dave", 4, 20L, 20L, Row(20L, null)),
 
       // Key 3: bisected by another closed non-tombstone.
-      Row(3, "eve", 5, 5L, 50L, Row(5L)),
-      Row(3, "frank", 6, 30L, 40L, Row(30L))
+      Row(3, "eve", 5, 5L, 50L, Row(5L, null)),
+      Row(3, "frank", 6, 30L, 40L, Row(30L, null))
     )
 
     val result = processor.decomposeOutOfOrderRows(df)
@@ -1498,17 +1510,17 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
       df = result,
       expectedAnswer = Seq(
         // Key 1.
-        Row(1, "alice", 1, 5L, null, Row(5L)),
-        Row(1, "alice", 1, null, 50L, Row(null)),
-        Row(1, "bob", 2, 10L, null, Row(10L)),
+        Row(1, "alice", 1, 5L, null, Row(5L, null)),
+        Row(1, "alice", 1, null, 50L, Row(null, null)),
+        Row(1, "bob", 2, 10L, null, Row(10L, null)),
         // Key 2.
-        Row(2, "carol", 3, 5L, null, Row(5L)),
-        Row(2, "carol", 3, null, 50L, Row(null)),
-        Row(2, "dave", 4, 20L, 20L, Row(20L)),
+        Row(2, "carol", 3, 5L, null, Row(5L, null)),
+        Row(2, "carol", 3, null, 50L, Row(null, null)),
+        Row(2, "dave", 4, 20L, 20L, Row(20L, null)),
         // Key 3.
-        Row(3, "eve", 5, 5L, null, Row(5L)),
-        Row(3, "eve", 5, null, 50L, Row(null)),
-        Row(3, "frank", 6, 30L, 40L, Row(30L))
+        Row(3, "eve", 5, 5L, null, Row(5L, null)),
+        Row(3, "eve", 5, null, 50L, Row(null, null)),
+        Row(3, "frank", 6, 30L, 40L, Row(30L, null))
       )
     )
   }
@@ -1524,8 +1536,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // non-chronological order. The window orders rows by effective recordStartAt, so the
     // result must still recognize that [5, 30] is bisected by the row at recordStartAt = 15.
     val df = targetTableOf(userSchema)(
-      Row(1, "bob", 99, 15L, null, Row(15L)), // appears first in input
-      Row(1, "alice", 42, 5L, 30L, Row(5L))    // appears last in input but lower in window
+      Row(1, "bob", 99, 15L, null, Row(15L, null)), // appears first in input
+      Row(1, "alice", 42, 5L, 30L, Row(5L, null))    // appears last in input but lower in window
     )
 
     val result = processor.decomposeOutOfOrderRows(df)
@@ -1533,9 +1545,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 42, 5L, null, Row(5L)),
-        Row(1, "alice", 42, null, 30L, Row(null)),
-        Row(1, "bob", 99, 15L, null, Row(15L))
+        Row(1, "alice", 42, 5L, null, Row(5L, null)),
+        Row(1, "alice", 42, null, 30L, Row(null, null)),
+        Row(1, "bob", 99, 15L, null, Row(15L, null))
       )
     )
   }
@@ -1552,11 +1564,11 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // bisecting successor must NOT bleed into key 2's partition.
     val df = targetTableOf(userSchema)(
       // Key 1: closed [5, 30] bisected by recordStartAt = 15.
-      Row(1, "alice", 42, 5L, 30L, Row(5L)),
-      Row(1, "bob", 99, 15L, null, Row(15L)),
+      Row(1, "alice", 42, 5L, 30L, Row(5L, null)),
+      Row(1, "bob", 99, 15L, null, Row(15L, null)),
 
       // Key 2: a single closed [5, 30] with no successor in its own partition.
-      Row(2, "carol", 7, 5L, 30L, Row(5L))
+      Row(2, "carol", 7, 5L, 30L, Row(5L, null))
     )
 
     val result = processor.decomposeOutOfOrderRows(df)
@@ -1565,11 +1577,11 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
       df = result,
       expectedAnswer = Seq(
         // Key 1 decomposes.
-        Row(1, "alice", 42, 5L, null, Row(5L)),
-        Row(1, "alice", 42, null, 30L, Row(null)),
-        Row(1, "bob", 99, 15L, null, Row(15L)),
+        Row(1, "alice", 42, 5L, null, Row(5L, null)),
+        Row(1, "alice", 42, null, 30L, Row(null, null)),
+        Row(1, "bob", 99, 15L, null, Row(15L, null)),
         // Key 2 passes through.
-        Row(2, "carol", 7, 5L, 30L, Row(5L))
+        Row(2, "carol", 7, 5L, 30L, Row(5L, null))
       )
     )
   }
@@ -1587,9 +1599,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     //   [10, 25] bisected by [15, 20]   -> decomposes
     //   [15, 20] is the last row        -> passes through
     val df = targetTableOf(userSchema)(
-      Row(1, "outer", 1, 5L, 30L, Row(5L)),
-      Row(1, "middle", 2, 10L, 25L, Row(10L)),
-      Row(1, "inner", 3, 15L, 20L, Row(15L))
+      Row(1, "outer", 1, 5L, 30L, Row(5L, null)),
+      Row(1, "middle", 2, 10L, 25L, Row(10L, null)),
+      Row(1, "inner", 3, 15L, 20L, Row(15L, null))
     )
 
     val result = processor.decomposeOutOfOrderRows(df)
@@ -1598,13 +1610,13 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
       df = result,
       expectedAnswer = Seq(
         // outer decomposes.
-        Row(1, "outer", 1, 5L, null, Row(5L)),
-        Row(1, "outer", 1, null, 30L, Row(null)),
+        Row(1, "outer", 1, 5L, null, Row(5L, null)),
+        Row(1, "outer", 1, null, 30L, Row(null, null)),
         // middle decomposes.
-        Row(1, "middle", 2, 10L, null, Row(10L)),
-        Row(1, "middle", 2, null, 25L, Row(null)),
+        Row(1, "middle", 2, 10L, null, Row(10L, null)),
+        Row(1, "middle", 2, null, 25L, Row(null, null)),
         // inner passes through.
-        Row(1, "inner", 3, 15L, 20L, Row(15L))
+        Row(1, "inner", 3, 15L, 20L, Row(15L, null))
       )
     )
   }
@@ -1631,12 +1643,19 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     def commentMetadata(comment: String): Metadata =
       new MetadataBuilder().putString("comment", comment).build()
 
-    val cdcMetadataInnerSchema = new StructType().add(
-      Scd2BatchProcessor.recordStartAtFieldName,
-      LongType,
-      nullable = true,
-      metadata = commentMetadata("inner __RECORD_START_AT")
-    )
+    val cdcMetadataInnerSchema = new StructType()
+      .add(
+        Scd2BatchProcessor.recordStartAtFieldName,
+        LongType,
+        nullable = true,
+        metadata = commentMetadata("inner __RECORD_START_AT")
+      )
+      .add(
+        Scd2BatchProcessor.versionMapFieldName,
+        Scd2VersionMap.mapType,
+        nullable = true,
+        metadata = commentMetadata("inner __VERSION_MAP")
+      )
 
     val schema = new StructType()
       .add("id", IntegerType, nullable = false, metadata = commentMetadata("user key"))
@@ -1653,8 +1672,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
 
     // Closed [5, 30] bisected by recordStartAt = 15.
     val df = microbatchOf(schema)(
-      Row(1, "alice", 5L, 30L, Row(5L)),
-      Row(1, "bob", 15L, null, Row(15L))
+      Row(1, "alice", 5L, 30L, Row(5L, null)),
+      Row(1, "bob", 15L, null, Row(15L, null))
     )
 
     val result = processor.decomposeOutOfOrderRows(df)
@@ -1682,19 +1701,19 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     //   closed upsert:       startAt < endAt, recordStartAt non-null
     //   decomposition tail:  startAt and recordStartAt null, endAt non-null
     val df = targetTableOf(userSchema)(
-      Row(1, "tomb", 10L, 10L, Row(10L)),
-      Row(2, "open", 20L, null, Row(20L)),
-      Row(3, "closed", 30L, 40L, Row(30L)),
-      Row(4, "tail", null, 50L, Row(null))
+      Row(1, "tomb", 10L, 10L, Row(10L, null)),
+      Row(2, "open", 20L, null, Row(20L, null)),
+      Row(3, "closed", 30L, 40L, Row(30L, null)),
+      Row(4, "tail", null, 50L, Row(null, null))
     )
 
     checkAnswer(
       df = processor.assertWellFormedRowsPostDecomposition(df, batchId = 0),
       expectedAnswer = Seq(
-        Row(1, "tomb", 10L, 10L, Row(10L)),
-        Row(2, "open", 20L, null, Row(20L)),
-        Row(3, "closed", 30L, 40L, Row(30L)),
-        Row(4, "tail", null, 50L, Row(null))
+        Row(1, "tomb", 10L, 10L, Row(10L, null)),
+        Row(2, "open", 20L, null, Row(20L, null)),
+        Row(3, "closed", 30L, 40L, Row(30L, null)),
+        Row(4, "tail", null, 50L, Row(null, null))
       )
     )
   }
@@ -1707,7 +1726,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // the decomposition-tail kind rejects on startAt non-null, while the tombstone, open
     // upsert, and closed upsert kinds all reject on recordStartAt null.
     val df = targetTableOf(userSchema)(
-      Row(1, "malformed", 5L, 10L, Row(null))
+      Row(1, "malformed", 5L, 10L, Row(null, null))
     )
 
     val wrapper = intercept[SparkException] {
@@ -1727,7 +1746,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     val userSchema = new StructType().add("id", IntegerType).add("value", StringType)
 
     val df = targetTableOf(userSchema)(
-      Row(1, "open", 10L, null, Row(10L))
+      Row(1, "open", 10L, null, Row(10L, null))
     )
 
     val result = processor.assertWellFormedRowsPostDecomposition(df, batchId = 0)
@@ -1744,9 +1763,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // Distinct effective recordStartAts within the dataframe, so no redundancies - identity
     // transformation expected.
     val df = targetTableOf(userSchema)(
-      Row(1, "v5", 5L, null, Row(5L)),
-      Row(1, "v10", 10L, null, Row(10L)),
-      Row(1, "v15", 15L, 20L, Row(15L))
+      Row(1, "v5", 5L, null, Row(5L, null)),
+      Row(1, "v10", 10L, null, Row(10L, null)),
+      Row(1, "v15", 15L, 20L, Row(15L, null))
     )
 
     val result = processor.dropRedundantRowsPostDecomposition(df)
@@ -1754,9 +1773,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "v5", 5L, null, Row(5L)),
-        Row(1, "v10", 10L, null, Row(10L)),
-        Row(1, "v15", 15L, 20L, Row(15L))
+        Row(1, "v5", 5L, null, Row(5L, null)),
+        Row(1, "v10", 10L, null, Row(10L, null)),
+        Row(1, "v15", 15L, 20L, Row(15L, null))
       )
     )
   }
@@ -1770,8 +1789,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // survives because the window's tiebreaker among truly-identical rows is intentionally
     // undefined.
     val df = targetTableOf(userSchema)(
-      Row(1, "first", 10L, null, Row(10L)),
-      Row(1, "second", 10L, null, Row(10L))
+      Row(1, "first", 10L, null, Row(10L, null)),
+      Row(1, "second", 10L, null, Row(10L, null))
     )
 
     val result = processor.dropRedundantRowsPostDecomposition(df)
@@ -1790,8 +1809,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // overtakes, leaving it 0-width: it ties with its successor on effective sequence
     // and is dropped. The tombstone (last in partition) survives.
     val df = targetTableOf(userSchema)(
-      Row(1, "open", 10L, null, Row(10L)),
-      Row(1, "tomb", 10L, 10L, Row(10L))
+      Row(1, "open", 10L, null, Row(10L, null)),
+      Row(1, "tomb", 10L, 10L, Row(10L, null))
     )
 
     val result = processor.dropRedundantRowsPostDecomposition(df)
@@ -1799,7 +1818,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "tomb", 10L, 10L, Row(10L))
+        Row(1, "tomb", 10L, 10L, Row(10L, null))
       )
     )
   }
@@ -1814,8 +1833,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // tail encodes is already represented by the coincident event, so the leading tail is
     // dropped. The event survives.
     val df = targetTableOf(userSchema)(
-      Row(1, "tail", null, 30L, Row(null)),
-      Row(1, "event", 30L, null, Row(30L))
+      Row(1, "tail", null, 30L, Row(null, null)),
+      Row(1, "event", 30L, null, Row(30L, null))
     )
 
     val result = processor.dropRedundantRowsPostDecomposition(df)
@@ -1823,7 +1842,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "event", 30L, null, Row(30L))
+        Row(1, "event", 30L, null, Row(30L, null))
       )
     )
   }
@@ -1838,10 +1857,10 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // successor on effective recordStartAt, so the redundancy filter doesn't fire. Every
     // row survives.
     val df = targetTableOf(userSchema)(
-      Row(1, "open", 10L, null, Row(10L)),
-      Row(1, "next1", 15L, null, Row(15L)),
-      Row(1, "tail", null, 30L, Row(null)),
-      Row(1, "next2", 35L, null, Row(35L))
+      Row(1, "open", 10L, null, Row(10L, null)),
+      Row(1, "next1", 15L, null, Row(15L, null)),
+      Row(1, "tail", null, 30L, Row(null, null)),
+      Row(1, "next2", 35L, null, Row(35L, null))
     )
 
     val result = processor.dropRedundantRowsPostDecomposition(df)
@@ -1849,10 +1868,10 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "open", 10L, null, Row(10L)),
-        Row(1, "next1", 15L, null, Row(15L)),
-        Row(1, "tail", null, 30L, Row(null)),
-        Row(1, "next2", 35L, null, Row(35L))
+        Row(1, "open", 10L, null, Row(10L, null)),
+        Row(1, "next1", 15L, null, Row(15L, null)),
+        Row(1, "tail", null, 30L, Row(null, null)),
+        Row(1, "next2", 35L, null, Row(35L, null))
       )
     )
   }
@@ -1866,8 +1885,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // see key 2's row as its successor and tie on effective sequence; proper partitioning
     // preserves both.
     val df = targetTableOf(userSchema)(
-      Row(1, "v10", 10L, null, Row(10L)),
-      Row(2, "v10", 10L, null, Row(10L))
+      Row(1, "v10", 10L, null, Row(10L, null)),
+      Row(2, "v10", 10L, null, Row(10L, null))
     )
 
     val result = processor.dropRedundantRowsPostDecomposition(df)
@@ -1875,8 +1894,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "v10", 10L, null, Row(10L)),
-        Row(2, "v10", 10L, null, Row(10L))
+        Row(1, "v10", 10L, null, Row(10L, null)),
+        Row(2, "v10", 10L, null, Row(10L, null))
       )
     )
   }
@@ -1890,10 +1909,10 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // duplicate plus the distinct event. We don't assert which user-data variant of the
     // duplicate survives.
     val df = targetTableOf(userSchema)(
-      Row(1, "dup1", 5L, null, Row(5L)),
-      Row(1, "dup2", 5L, null, Row(5L)),
-      Row(1, "dup3", 5L, null, Row(5L)),
-      Row(1, "different", 10L, null, Row(10L))
+      Row(1, "dup1", 5L, null, Row(5L, null)),
+      Row(1, "dup2", 5L, null, Row(5L, null)),
+      Row(1, "dup3", 5L, null, Row(5L, null)),
+      Row(1, "different", 10L, null, Row(10L, null))
     )
 
     val result = processor.dropRedundantRowsPostDecomposition(df)
@@ -1920,14 +1939,14 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     //          covered).
     //   key=4: two tombstones.
     val df = targetTableOf(userSchema)(
-      Row(1, "openA", 10L, null, Row(10L)),
-      Row(1, "openB", 10L, null, Row(10L)),
-      Row(2, "open", 10L, null, Row(10L)),
-      Row(2, "closed", 10L, 20L, Row(10L)),
-      Row(3, "closedA", 10L, 20L, Row(10L)),
-      Row(3, "closedB", 10L, 20L, Row(10L)),
-      Row(4, "tombA", 10L, 10L, Row(10L)),
-      Row(4, "tombB", 10L, 10L, Row(10L))
+      Row(1, "openA", 10L, null, Row(10L, null)),
+      Row(1, "openB", 10L, null, Row(10L, null)),
+      Row(2, "open", 10L, null, Row(10L, null)),
+      Row(2, "closed", 10L, 20L, Row(10L, null)),
+      Row(3, "closedA", 10L, 20L, Row(10L, null)),
+      Row(3, "closedB", 10L, 20L, Row(10L, null)),
+      Row(4, "tombA", 10L, 10L, Row(10L, null)),
+      Row(4, "tombB", 10L, 10L, Row(10L, null))
     )
 
     val expectedSurvivorsPerKey: Map[Int, Set[String]] = Map(
@@ -1957,9 +1976,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // redundancy filter then drops every row whose successor shares its effective sequence,
     // leaving only the trailing tombstone.
     val df = targetTableOf(userSchema)(
-      Row(1, "tail", null, 10L, Row(null)),
-      Row(1, "open", 10L, null, Row(10L)),
-      Row(1, "tomb", 10L, 10L, Row(10L))
+      Row(1, "tail", null, 10L, Row(null, null)),
+      Row(1, "open", 10L, null, Row(10L, null)),
+      Row(1, "tomb", 10L, 10L, Row(10L, null))
     )
 
     val result = processor.dropRedundantRowsPostDecomposition(df)
@@ -1967,7 +1986,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "tomb", 10L, 10L, Row(10L))
+        Row(1, "tomb", 10L, 10L, Row(10L, null))
       )
     )
   }
@@ -1983,8 +2002,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // values. The first row begins a fresh run with startAt=5. The second row, sharing the
     // tracked value, is a continuation of that run and inherits the run head's startAt.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, null, Row(5L)),
-      Row(1, "alice", 10L, null, Row(10L))
+      Row(1, "alice", 5L, null, Row(5L, null)),
+      Row(1, "alice", 10L, null, Row(10L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -1992,8 +2011,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, null, Row(5L)),
-        Row(1, "alice", 5L, null, Row(10L))
+        Row(1, "alice", 5L, null, Row(5L, null)),
+        Row(1, "alice", 5L, null, Row(10L, null))
       )
     )
   }
@@ -2009,8 +2028,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // its existing startAt encodes the true global run start and must be preserved -
     // and propagated to the in-window continuation.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 2L, null, Row(5L)),
-      Row(1, "alice", 10L, null, Row(10L))
+      Row(1, "alice", 2L, null, Row(5L, null)),
+      Row(1, "alice", 10L, null, Row(10L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2018,8 +2037,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 2L, null, Row(5L)),
-        Row(1, "alice", 2L, null, Row(10L))
+        Row(1, "alice", 2L, null, Row(5L, null)),
+        Row(1, "alice", 2L, null, Row(10L, null))
       )
     )
   }
@@ -2033,8 +2052,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // first run at the second event's effective recordStartAt and starts a new run whose
     // startAt is the new event's recordStartAt.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, null, Row(5L)),
-      Row(1, "bob", 10L, null, Row(10L))
+      Row(1, "alice", 5L, null, Row(5L, null)),
+      Row(1, "bob", 10L, null, Row(10L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2042,8 +2061,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, 10L, Row(5L)),
-        Row(1, "bob", 10L, null, Row(10L))
+        Row(1, "alice", 5L, 10L, Row(5L, null)),
+        Row(1, "bob", 10L, null, Row(10L, null))
       )
     )
   }
@@ -2056,9 +2075,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // Three consecutive open upserts all agreeing on the tracked column form one no-op
     // run. Every row in the run must end up with the run head's startAt.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, null, Row(5L)),
-      Row(1, "alice", 10L, null, Row(10L)),
-      Row(1, "alice", 15L, null, Row(15L))
+      Row(1, "alice", 5L, null, Row(5L, null)),
+      Row(1, "alice", 10L, null, Row(10L, null)),
+      Row(1, "alice", 15L, null, Row(15L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2066,9 +2085,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, null, Row(5L)),
-        Row(1, "alice", 5L, null, Row(10L)),
-        Row(1, "alice", 5L, null, Row(15L))
+        Row(1, "alice", 5L, null, Row(5L, null)),
+        Row(1, "alice", 5L, null, Row(10L, null)),
+        Row(1, "alice", 5L, null, Row(15L, null))
       )
     )
   }
@@ -2087,10 +2106,10 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     //  - the last `alice` row (the run tail) must close at bob's recordStartAt (20), not stay
     //    open and not close at any interior sequence.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, null, Row(5L)),
-      Row(1, "alice", 10L, null, Row(10L)),
-      Row(1, "alice", 15L, null, Row(15L)),
-      Row(1, "bob", 20L, null, Row(20L))
+      Row(1, "alice", 5L, null, Row(5L, null)),
+      Row(1, "alice", 10L, null, Row(10L, null)),
+      Row(1, "alice", 15L, null, Row(15L, null)),
+      Row(1, "bob", 20L, null, Row(20L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2098,10 +2117,10 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, null, Row(5L)),
-        Row(1, "alice", 5L, null, Row(10L)),
-        Row(1, "alice", 5L, 20L, Row(15L)),
-        Row(1, "bob", 20L, null, Row(20L))
+        Row(1, "alice", 5L, null, Row(5L, null)),
+        Row(1, "alice", 5L, null, Row(10L, null)),
+        Row(1, "alice", 5L, 20L, Row(15L, null)),
+        Row(1, "bob", 20L, null, Row(20L, null))
       )
     )
   }
@@ -2118,8 +2137,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // as tracked. With no columnSelection, both `name` and `status` are selected. The
     // two rows agree on `name` but disagree on `status`, so they start distinct runs.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", "active", 5L, null, Row(5L)),
-      Row(1, "alice", "inactive", 10L, null, Row(10L))
+      Row(1, "alice", "active", 5L, null, Row(5L, null)),
+      Row(1, "alice", "inactive", 10L, null, Row(10L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2127,8 +2146,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", "active", 5L, 10L, Row(5L)),
-        Row(1, "alice", "inactive", 10L, null, Row(10L))
+        Row(1, "alice", "active", 5L, 10L, Row(5L, null)),
+        Row(1, "alice", "inactive", 10L, null, Row(10L, null))
       )
     )
   }
@@ -2161,8 +2180,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, null, Row(5L)),
-        Row(1, "alice", 5L, null, Row(10L))
+        Row(1, "alice", 5L, null, Row(5L, null)),
+        Row(1, "alice", 5L, null, Row(10L, null))
       )
     )
   }
@@ -2211,8 +2230,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // `status` is excluded from tracking, so the two rows are tracked-equal on the
     // remaining columns (`name`). They should collapse into a single no-op run.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", "active", 5L, null, Row(5L)),
-      Row(1, "alice", "inactive", 10L, null, Row(10L))
+      Row(1, "alice", "active", 5L, null, Row(5L, null)),
+      Row(1, "alice", "inactive", 10L, null, Row(10L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2220,8 +2239,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", "active", 5L, null, Row(5L)),
-        Row(1, "alice", "inactive", 5L, null, Row(10L))
+        Row(1, "alice", "active", 5L, null, Row(5L, null)),
+        Row(1, "alice", "inactive", 5L, null, Row(10L, null))
       )
     )
   }
@@ -2245,8 +2264,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // nothing to compare, every consecutive upsert pair collapses into a single run -
     // even when the user-visible data differs on every column.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", "active", 5L, null, Row(5L)),
-      Row(1, "bob", "inactive", 10L, null, Row(10L))
+      Row(1, "alice", "active", 5L, null, Row(5L, null)),
+      Row(1, "bob", "inactive", 10L, null, Row(10L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2254,8 +2273,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", "active", 5L, null, Row(5L)),
-        Row(1, "bob", "inactive", 5L, null, Row(10L))
+        Row(1, "alice", "active", 5L, null, Row(5L, null)),
+        Row(1, "bob", "inactive", 5L, null, Row(10L, null))
       )
     )
   }
@@ -2269,9 +2288,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // pass through identically. The bracketing upserts close (10) and reopen (15) around
     // the tombstone.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, null, Row(5L)),
-      Row(1, "alice", 10L, 10L, Row(10L)),
-      Row(1, "alice", 15L, null, Row(15L))
+      Row(1, "alice", 5L, null, Row(5L, null)),
+      Row(1, "alice", 10L, 10L, Row(10L, null)),
+      Row(1, "alice", 15L, null, Row(15L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2279,9 +2298,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, 10L, Row(5L)),
-        Row(1, "alice", 10L, 10L, Row(10L)),
-        Row(1, "alice", 15L, null, Row(15L))
+        Row(1, "alice", 5L, 10L, Row(5L, null)),
+        Row(1, "alice", 10L, 10L, Row(10L, null)),
+        Row(1, "alice", 15L, null, Row(15L, null))
       )
     )
   }
@@ -2295,8 +2314,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // null) is excluded from upsert reconciliation. The tail's startAt must stay null
     // and its endAt must pass through unchanged.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, null, Row(5L)),
-      Row(1, "alice", null, 30L, Row(null))
+      Row(1, "alice", 5L, null, Row(5L, null)),
+      Row(1, "alice", null, 30L, Row(null, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2304,8 +2323,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, 30L, Row(5L)),
-        Row(1, "alice", null, 30L, Row(null))
+        Row(1, "alice", 5L, 30L, Row(5L, null)),
+        Row(1, "alice", null, 30L, Row(null, null))
       )
     )
   }
@@ -2323,9 +2342,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // opens a new run in between. The bisecting event in turn closes at the tail boundary
     // (30), and the tail passes through unchanged.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, null, Row(5L)),
-      Row(1, "bob", 15L, null, Row(15L)),
-      Row(1, "alice", null, 30L, Row(null))
+      Row(1, "alice", 5L, null, Row(5L, null)),
+      Row(1, "bob", 15L, null, Row(15L, null)),
+      Row(1, "alice", null, 30L, Row(null, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2333,9 +2352,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, 15L, Row(5L)),
-        Row(1, "bob", 15L, 30L, Row(15L)),
-        Row(1, "alice", null, 30L, Row(null))
+        Row(1, "alice", 5L, 15L, Row(5L, null)),
+        Row(1, "bob", 15L, 30L, Row(15L, null)),
+        Row(1, "alice", null, 30L, Row(null, null))
       )
     )
   }
@@ -2349,8 +2368,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // The closed upsert already ended at 15 - strictly before the next event - so its
     // endAt is left intact.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, 15L, Row(5L)),
-      Row(1, "bob", 20L, null, Row(20L))
+      Row(1, "alice", 5L, 15L, Row(5L, null)),
+      Row(1, "bob", 20L, null, Row(20L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2358,8 +2377,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, 15L, Row(5L)),
-        Row(1, "bob", 20L, null, Row(20L))
+        Row(1, "alice", 5L, 15L, Row(5L, null)),
+        Row(1, "bob", 20L, null, Row(20L, null))
       )
     )
   }
@@ -2372,8 +2391,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // at recordStartAt=20. The first run head must be closed at 20 because the run ends
     // there.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, null, Row(5L)),
-      Row(1, "bob", 20L, null, Row(20L))
+      Row(1, "alice", 5L, null, Row(5L, null)),
+      Row(1, "bob", 20L, null, Row(20L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2381,8 +2400,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, 20L, Row(5L)),
-        Row(1, "bob", 20L, null, Row(20L))
+        Row(1, "alice", 5L, 20L, Row(5L, null)),
+        Row(1, "bob", 20L, null, Row(20L, null))
       )
     )
   }
@@ -2397,8 +2416,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // the now-redundant tombstone for the next transform to drop based on the
     // shape locked in here.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, 20L, Row(5L)),
-      Row(1, "alice", 20L, 20L, Row(20L))
+      Row(1, "alice", 5L, 20L, Row(5L, null)),
+      Row(1, "alice", 20L, 20L, Row(20L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2406,8 +2425,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, 20L, Row(5L)),
-        Row(1, "alice", 20L, 20L, Row(20L))
+        Row(1, "alice", 5L, 20L, Row(5L, null)),
+        Row(1, "alice", 20L, 20L, Row(20L, null))
       )
     )
   }
@@ -2420,8 +2439,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // Downstream transforms identify decomposition tails by recordStartAt = null, so
     // reconciliation must not synthesize a value into the tail's _cdc_metadata.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, null, Row(5L)),
-      Row(1, "alice", null, 30L, Row(null))
+      Row(1, "alice", 5L, null, Row(5L, null)),
+      Row(1, "alice", null, 30L, Row(null, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2438,12 +2457,19 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     def commentMetadata(comment: String): Metadata =
       new MetadataBuilder().putString("comment", comment).build()
 
-    val cdcMetadataInnerSchema = new StructType().add(
-      Scd2BatchProcessor.recordStartAtFieldName,
-      LongType,
-      nullable = true,
-      metadata = commentMetadata("inner __RECORD_START_AT")
-    )
+    val cdcMetadataInnerSchema = new StructType()
+      .add(
+        Scd2BatchProcessor.recordStartAtFieldName,
+        LongType,
+        nullable = true,
+        metadata = commentMetadata("inner __RECORD_START_AT")
+      )
+      .add(
+        Scd2BatchProcessor.versionMapFieldName,
+        Scd2VersionMap.mapType,
+        nullable = true,
+        metadata = commentMetadata("inner __VERSION_MAP")
+      )
 
     val schema = new StructType()
       .add("id", IntegerType, nullable = false, metadata = commentMetadata("user key"))
@@ -2461,9 +2487,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // Mix of canonical post-decomposition row shapes so we exercise multiple reconciliation
     // branches under the schema-preservation contract.
     val df = microbatchOf(schema)(
-      Row(1, "alice", 5L, null, Row(5L)),
-      Row(1, "alice", null, 30L, Row(null)),
-      Row(1, "alice", 30L, 30L, Row(30L))
+      Row(1, "alice", 5L, null, Row(5L, null)),
+      Row(1, "alice", null, 30L, Row(null, null)),
+      Row(1, "alice", 30L, 30L, Row(30L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2486,8 +2512,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // or successor. Reconciliation must handle the missing neighbors cleanly and pass the
     // single rows through.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, null, Row(5L)),
-      Row(2, "bob", 10L, 20L, Row(10L))
+      Row(1, "alice", 5L, null, Row(5L, null)),
+      Row(2, "bob", 10L, 20L, Row(10L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2495,8 +2521,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, null, Row(5L)),
-        Row(2, "bob", 10L, 20L, Row(10L))
+        Row(1, "alice", 5L, null, Row(5L, null)),
+        Row(2, "bob", 10L, 20L, Row(10L, null))
       )
     )
   }
@@ -2516,8 +2542,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // Only `name` is tracked. Two rows agreeing on name but differing on status are
     // tracked-equal and should collapse into one run.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", "active", 5L, null, Row(5L)),
-      Row(1, "alice", "inactive", 10L, null, Row(10L))
+      Row(1, "alice", "active", 5L, null, Row(5L, null)),
+      Row(1, "alice", "inactive", 10L, null, Row(10L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2525,8 +2551,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", "active", 5L, null, Row(5L)),
-        Row(1, "alice", "inactive", 5L, null, Row(10L))
+        Row(1, "alice", "active", 5L, null, Row(5L, null)),
+        Row(1, "alice", "inactive", 5L, null, Row(10L, null))
       )
     )
   }
@@ -2550,8 +2576,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // `F.col("user.name")` would be parsed as a nested-field access (struct `user`, field
     // `name`) and fail to resolve.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", "active", 5L, null, Row(5L)),
-      Row(1, "alice", "inactive", 10L, null, Row(10L))
+      Row(1, "alice", "active", 5L, null, Row(5L, null)),
+      Row(1, "alice", "inactive", 10L, null, Row(10L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2559,8 +2585,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", "active", 5L, null, Row(5L)),
-        Row(1, "alice", "inactive", 5L, null, Row(10L))
+        Row(1, "alice", "active", 5L, null, Row(5L, null)),
+        Row(1, "alice", "inactive", 5L, null, Row(10L, null))
       )
     )
   }
@@ -2575,7 +2601,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     )
     val userSchema = new StructType().add("id", IntegerType).add("value", StringType)
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, null, Row(5L))
+      Row(1, "alice", 5L, null, Row(5L, null))
     )
 
     val ex = intercept[AnalysisException] {
@@ -2589,8 +2615,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     val userSchema = new StructType().add("id", IntegerType).add("name", StringType)
 
     val df = targetTableOf(userSchema)(
-      Row(1, null, 5L, null, Row(5L)),
-      Row(1, null, 10L, null, Row(10L))
+      Row(1, null, 5L, null, Row(5L, null)),
+      Row(1, null, 10L, null, Row(10L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2598,8 +2624,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, null, 5L, null, Row(5L)),
-        Row(1, null, 5L, null, Row(10L))
+        Row(1, null, 5L, null, Row(5L, null)),
+        Row(1, null, 5L, null, Row(10L, null))
       )
     )
   }
@@ -2610,9 +2636,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     val userSchema = new StructType().add("id", IntegerType).add("value", StringType)
 
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, null, Row(5L)),
-      Row(1, "alice", 10L, null, Row(10L)),
-      Row(1, "alice", 15L, 15L, Row(15L))
+      Row(1, "alice", 5L, null, Row(5L, null)),
+      Row(1, "alice", 10L, null, Row(10L, null)),
+      Row(1, "alice", 15L, 15L, Row(15L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2620,9 +2646,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, null, Row(5L)),
-        Row(1, "alice", 5L, 15L, Row(10L)),
-        Row(1, "alice", 15L, 15L, Row(15L))
+        Row(1, "alice", 5L, null, Row(5L, null)),
+        Row(1, "alice", 5L, 15L, Row(10L, null)),
+        Row(1, "alice", 15L, 15L, Row(15L, null))
       )
     )
   }
@@ -2634,10 +2660,10 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // Two keys, each with a fresh-key run head + tracked-equal continuation. The two
     // partitions must reconcile independently.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, null, Row(5L)),
-      Row(1, "alice", 10L, null, Row(10L)),
-      Row(2, "bob", 20L, null, Row(20L)),
-      Row(2, "bob", 25L, null, Row(25L))
+      Row(1, "alice", 5L, null, Row(5L, null)),
+      Row(1, "alice", 10L, null, Row(10L, null)),
+      Row(2, "bob", 20L, null, Row(20L, null)),
+      Row(2, "bob", 25L, null, Row(25L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2645,10 +2671,10 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, null, Row(5L)),
-        Row(1, "alice", 5L, null, Row(10L)),
-        Row(2, "bob", 20L, null, Row(20L)),
-        Row(2, "bob", 20L, null, Row(25L))
+        Row(1, "alice", 5L, null, Row(5L, null)),
+        Row(1, "alice", 5L, null, Row(10L, null)),
+        Row(2, "bob", 20L, null, Row(20L, null)),
+        Row(2, "bob", 20L, null, Row(25L, null))
       )
     )
   }
@@ -2666,8 +2692,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // open" transition, which every other no-op-continuation test leaves as a no-op by
     // starting from an already-open row.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, 20L, Row(5L)),
-      Row(1, "alice", 20L, null, Row(20L))
+      Row(1, "alice", 5L, 20L, Row(5L, null)),
+      Row(1, "alice", 20L, null, Row(20L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2675,8 +2701,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, null, Row(5L)),
-        Row(1, "alice", 5L, null, Row(20L))
+        Row(1, "alice", 5L, null, Row(5L, null)),
+        Row(1, "alice", 5L, null, Row(20L, null))
       )
     )
   }
@@ -2693,8 +2719,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // at its own recordStartAt. This exercises run-head startAt propagation for a closed
     // upsert, which the other run-head tests only cover with open rows.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, 10L, Row(5L)),
-      Row(1, "alice", 15L, null, Row(15L))
+      Row(1, "alice", 5L, 10L, Row(5L, null)),
+      Row(1, "alice", 15L, null, Row(15L, null))
     )
 
     val result = processor.reconcileStartAndEndAt(df)
@@ -2702,8 +2728,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, 10L, Row(5L)),
-        Row(1, "alice", 15L, null, Row(15L))
+        Row(1, "alice", 5L, 10L, Row(5L, null)),
+        Row(1, "alice", 15L, null, Row(15L, null))
       )
     )
   }
@@ -2730,8 +2756,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // A closed upsert [10, 15) immediately followed by a tombstone at 15. The upsert's reconciled
     // endAt already encodes the delete boundary, so the standalone tombstone is redundant.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 10L, 15L, Row(10L)),
-      Row(1, "alice", 15L, 15L, Row(15L))
+      Row(1, "alice", 10L, 15L, Row(10L, null)),
+      Row(1, "alice", 15L, 15L, Row(15L, null))
     )
 
     val result = processor.dropLeftoverDeletesPostReconciliation(df)
@@ -2739,7 +2765,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 10L, 15L, Row(10L))
+        Row(1, "alice", 10L, 15L, Row(10L, null))
       )
     )
   }
@@ -2753,9 +2779,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // at 15, the event closes at the tail's boundary 20, leaving the [null, 20) tail redundant
     // because the [15, 20) upsert already encodes the boundary.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 10L, 15L, Row(10L)),
-      Row(1, "bob", 15L, 20L, Row(15L)),
-      Row(1, "alice", null, 20L, Row(null))
+      Row(1, "alice", 10L, 15L, Row(10L, null)),
+      Row(1, "bob", 15L, 20L, Row(15L, null)),
+      Row(1, "alice", null, 20L, Row(null, null))
     )
 
     val result = processor.dropLeftoverDeletesPostReconciliation(df)
@@ -2763,8 +2789,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 10L, 15L, Row(10L)),
-        Row(1, "bob", 15L, 20L, Row(15L))
+        Row(1, "alice", 10L, 15L, Row(10L, null)),
+        Row(1, "bob", 15L, 20L, Row(15L, null))
       )
     )
   }
@@ -2777,8 +2803,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // The closed upsert ends at 15 but the tombstone is at 20, so the delete boundary is not
     // encoded by the preceding row and the tombstone must survive.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, 15L, Row(5L)),
-      Row(1, "alice", 20L, 20L, Row(20L))
+      Row(1, "alice", 5L, 15L, Row(5L, null)),
+      Row(1, "alice", 20L, 20L, Row(20L, null))
     )
 
     val result = processor.dropLeftoverDeletesPostReconciliation(df)
@@ -2786,8 +2812,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, 15L, Row(5L)),
-        Row(1, "alice", 20L, 20L, Row(20L))
+        Row(1, "alice", 5L, 15L, Row(5L, null)),
+        Row(1, "alice", 20L, 20L, Row(20L, null))
       )
     )
   }
@@ -2800,8 +2826,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // The preceding closed upsert ends at 12, strictly before the tail's boundary 20, so the
     // tail is an unmatched delete boundary that must survive for promotion to a tombstone.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, 12L, Row(5L)),
-      Row(1, "alice", null, 20L, Row(null))
+      Row(1, "alice", 5L, 12L, Row(5L, null)),
+      Row(1, "alice", null, 20L, Row(null, null))
     )
 
     val result = processor.dropLeftoverDeletesPostReconciliation(df)
@@ -2809,8 +2835,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, 12L, Row(5L)),
-        Row(1, "alice", null, 20L, Row(null))
+        Row(1, "alice", 5L, 12L, Row(5L, null)),
+        Row(1, "alice", null, 20L, Row(null, null))
       )
     )
   }
@@ -2823,8 +2849,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // With no predecessor, previousEndAt is null and `null <=> endAt` is false, so a leading
     // tombstone (key 1) and a leading decomposition tail (key 2) both survive.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 15L, 15L, Row(15L)),
-      Row(2, "bob", null, 20L, Row(null))
+      Row(1, "alice", 15L, 15L, Row(15L, null)),
+      Row(2, "bob", null, 20L, Row(null, null))
     )
 
     val result = processor.dropLeftoverDeletesPostReconciliation(df)
@@ -2832,8 +2858,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 15L, 15L, Row(15L)),
-        Row(2, "bob", null, 20L, Row(null))
+        Row(1, "alice", 15L, 15L, Row(15L, null)),
+        Row(2, "bob", null, 20L, Row(null, null))
       )
     )
   }
@@ -2847,8 +2873,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // row is delete-encoded, so the `isDeleteEncodedRow` guard must keep both. The overlapping
     // intervals are synthetic here purely to isolate the guard.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, 20L, Row(5L)),
-      Row(1, "bob", 10L, 20L, Row(10L))
+      Row(1, "alice", 5L, 20L, Row(5L, null)),
+      Row(1, "bob", 10L, 20L, Row(10L, null))
     )
 
     val result = processor.dropLeftoverDeletesPostReconciliation(df)
@@ -2856,8 +2882,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, 20L, Row(5L)),
-        Row(1, "bob", 10L, 20L, Row(10L))
+        Row(1, "alice", 5L, 20L, Row(5L, null)),
+        Row(1, "bob", 10L, 20L, Row(10L, null))
       )
     )
   }
@@ -2870,9 +2896,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // key 2: a tombstone at 15 that is the first row in its window (kept) - the key-1 upsert
     // must not be treated as its predecessor.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 10L, 15L, Row(10L)),
-      Row(1, "alice", 15L, 15L, Row(15L)),
-      Row(2, "bob", 15L, 15L, Row(15L))
+      Row(1, "alice", 10L, 15L, Row(10L, null)),
+      Row(1, "alice", 15L, 15L, Row(15L, null)),
+      Row(2, "bob", 15L, 15L, Row(15L, null))
     )
 
     val result = processor.dropLeftoverDeletesPostReconciliation(df)
@@ -2880,8 +2906,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 10L, 15L, Row(10L)),
-        Row(2, "bob", 15L, 15L, Row(15L))
+        Row(1, "alice", 10L, 15L, Row(10L, null)),
+        Row(2, "bob", 15L, 15L, Row(15L, null))
       )
     )
   }
@@ -2895,10 +2921,10 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // redundant tail at 20. Both delete-encoded rows are encoded by their immediate predecessors
     // and drop together in one window pass.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, 10L, Row(5L)),
-      Row(1, "alice", 10L, 10L, Row(10L)),
-      Row(1, "bob", 15L, 20L, Row(15L)),
-      Row(1, "alice", null, 20L, Row(null))
+      Row(1, "alice", 5L, 10L, Row(5L, null)),
+      Row(1, "alice", 10L, 10L, Row(10L, null)),
+      Row(1, "bob", 15L, 20L, Row(15L, null)),
+      Row(1, "alice", null, 20L, Row(null, null))
     )
 
     val result = processor.dropLeftoverDeletesPostReconciliation(df)
@@ -2906,8 +2932,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, 10L, Row(5L)),
-        Row(1, "bob", 15L, 20L, Row(15L))
+        Row(1, "alice", 5L, 10L, Row(5L, null)),
+        Row(1, "bob", 15L, 20L, Row(15L, null))
       )
     )
   }
@@ -2923,9 +2949,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // survive: only a preceding upsert makes a delete boundary redundant. This guards the
     // documented "immediately preceding upsert" invariant against adjacent delete-encoded rows.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, 15L, Row(5L)),
-      Row(1, "alice", 15L, 15L, Row(15L)),
-      Row(1, "alice", 15L, 15L, Row(15L))
+      Row(1, "alice", 5L, 15L, Row(5L, null)),
+      Row(1, "alice", 15L, 15L, Row(15L, null)),
+      Row(1, "alice", 15L, 15L, Row(15L, null))
     )
 
     val result = processor.dropLeftoverDeletesPostReconciliation(df)
@@ -2933,8 +2959,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, 15L, Row(5L)),
-        Row(1, "alice", 15L, 15L, Row(15L))
+        Row(1, "alice", 5L, 15L, Row(5L, null)),
+        Row(1, "alice", 15L, 15L, Row(15L, null))
       )
     )
   }
@@ -2948,8 +2974,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // The [null, 20) tail becomes a tombstone [20, 20] with recordStartAt = 20; the closed
     // upsert is untouched.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 10L, 20L, Row(10L)),
-      Row(1, "alice", null, 20L, Row(null))
+      Row(1, "alice", 10L, 20L, Row(10L, null)),
+      Row(1, "alice", null, 20L, Row(null, null))
     )
 
     val result = processor.promoteDecompositionTailsToTombstones(df)
@@ -2957,8 +2983,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 10L, 20L, Row(10L)),
-        Row(1, "alice", 20L, 20L, Row(20L))
+        Row(1, "alice", 10L, 20L, Row(10L, null)),
+        Row(1, "alice", 20L, 20L, Row(20L, null))
       )
     )
   }
@@ -2970,9 +2996,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // No decomposition tails present: a tombstone, an open upsert, and a closed upsert must all
     // pass through identically.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 15L, 15L, Row(15L)),
-      Row(2, "bob", 5L, null, Row(5L)),
-      Row(3, "carol", 5L, 15L, Row(5L))
+      Row(1, "alice", 15L, 15L, Row(15L, null)),
+      Row(2, "bob", 5L, null, Row(5L, null)),
+      Row(3, "carol", 5L, 15L, Row(5L, null))
     )
 
     val result = processor.promoteDecompositionTailsToTombstones(df)
@@ -2980,9 +3006,9 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 15L, 15L, Row(15L)),
-        Row(2, "bob", 5L, null, Row(5L)),
-        Row(3, "carol", 5L, 15L, Row(5L))
+        Row(1, "alice", 15L, 15L, Row(15L, null)),
+        Row(2, "bob", 5L, null, Row(5L, null)),
+        Row(3, "carol", 5L, 15L, Row(5L, null))
       )
     )
   }
@@ -2997,7 +3023,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // Only the framework columns (startAt and the cdc-metadata recordStartAt) are rewritten to
     // the tail's boundary; the inherited user columns must survive verbatim.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", "active", null, 20L, Row(null))
+      Row(1, "alice", "active", null, 20L, Row(null, null))
     )
 
     val result = processor.promoteDecompositionTailsToTombstones(df)
@@ -3005,7 +3031,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", "active", 20L, 20L, Row(20L))
+        Row(1, "alice", "active", 20L, 20L, Row(20L, null))
       )
     )
   }
@@ -3033,8 +3059,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
 
     // A tail (rewritten) plus a non-tail (passed through) so both projection paths are exercised.
     val df = microbatchOf(schema)(
-      Row(1, "alice", null, 20L, Row(null)),
-      Row(1, "alice", 5L, 20L, Row(5L))
+      Row(1, "alice", null, 20L, Row(null, null)),
+      Row(1, "alice", 5L, 20L, Row(5L, null))
     )
 
     val result = processor.promoteDecompositionTailsToTombstones(df)
@@ -3054,10 +3080,10 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
 
     // Each key carries a closed upsert plus a decomposition tail; only the tails are rewritten.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, 20L, Row(5L)),
-      Row(1, "alice", null, 20L, Row(null)),
-      Row(2, "bob", 8L, 30L, Row(8L)),
-      Row(2, "bob", null, 30L, Row(null))
+      Row(1, "alice", 5L, 20L, Row(5L, null)),
+      Row(1, "alice", null, 20L, Row(null, null)),
+      Row(2, "bob", 8L, 30L, Row(8L, null)),
+      Row(2, "bob", null, 30L, Row(null, null))
     )
 
     val result = processor.promoteDecompositionTailsToTombstones(df)
@@ -3065,10 +3091,10 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, 20L, Row(5L)),
-        Row(1, "alice", 20L, 20L, Row(20L)),
-        Row(2, "bob", 8L, 30L, Row(8L)),
-        Row(2, "bob", 30L, 30L, Row(30L))
+        Row(1, "alice", 5L, 20L, Row(5L, null)),
+        Row(1, "alice", 20L, 20L, Row(20L, null)),
+        Row(2, "bob", 8L, 30L, Row(8L, null)),
+        Row(2, "bob", 30L, 30L, Row(30L, null))
       )
     )
   }
@@ -3086,7 +3112,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // and startAt null, endAt=20) is promoted to a tombstone at 20 while the dotted user column
     // survives verbatim.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", null, 20L, Row(null))
+      Row(1, "alice", null, 20L, Row(null, null))
     )
 
     val result = processor.promoteDecompositionTailsToTombstones(df)
@@ -3094,7 +3120,7 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 20L, 20L, Row(20L))
+        Row(1, "alice", 20L, 20L, Row(20L, null))
       )
     )
   }
@@ -3119,13 +3145,13 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // A tombstone (startAt == endAt == recordStartAt) is delete-encoded and must live in the
     // aux table, never the target table.
     val df = targetTableOf(userSchema)(
-      Row(1, "t", 5L, 5L, Row(5L))
+      Row(1, "t", 5L, 5L, Row(5L, null))
     )
 
     checkAnswer(
       df = withRouteFlag(processor.identifyAndTagAuxRows(df)),
       expectedAnswer = Seq(
-        Row(1, "t", 5L, 5L, Row(5L), true)
+        Row(1, "t", 5L, 5L, Row(5L, null), true)
       )
     )
   }
@@ -3138,13 +3164,13 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // upsert-representing row, so it is tagged false. (Tails are promoted/dropped upstream of
     // the merges; this pins that the router itself never claims them for the aux table.)
     val df = targetTableOf(userSchema)(
-      Row(1, "tail", null, 10L, Row(null))
+      Row(1, "tail", null, 10L, Row(null, null))
     )
 
     checkAnswer(
       df = withRouteFlag(processor.identifyAndTagAuxRows(df)),
       expectedAnswer = Seq(
-        Row(1, "tail", null, 10L, Row(null), false)
+        Row(1, "tail", null, 10L, Row(null, null), false)
       )
     )
   }
@@ -3156,13 +3182,13 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // A single open upsert with no following row in its key window cannot be a hidden no-op
     // continuation - it is the visible tail of its (size-1) run.
     val df = targetTableOf(userSchema)(
-      Row(1, "v", 5L, null, Row(5L))
+      Row(1, "v", 5L, null, Row(5L, null))
     )
 
     checkAnswer(
       df = withRouteFlag(processor.identifyAndTagAuxRows(df)),
       expectedAnswer = Seq(
-        Row(1, "v", 5L, null, Row(5L), false)
+        Row(1, "v", 5L, null, Row(5L, null), false)
       )
     )
   }
@@ -3175,17 +3201,17 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // run head's startAt=5 and open endAt. The first two coalesce into hidden aux rows; only
     // the last (the run tail) stays visible in the target table.
     val df = targetTableOf(userSchema)(
-      Row(1, "a", 5L, null, Row(5L)),
-      Row(1, "a", 5L, null, Row(10L)),
-      Row(1, "a", 5L, null, Row(15L))
+      Row(1, "a", 5L, null, Row(5L, null)),
+      Row(1, "a", 5L, null, Row(10L, null)),
+      Row(1, "a", 5L, null, Row(15L, null))
     )
 
     checkAnswer(
       df = withRouteFlag(processor.identifyAndTagAuxRows(df)),
       expectedAnswer = Seq(
-        Row(1, "a", 5L, null, Row(5L), true),
-        Row(1, "a", 5L, null, Row(10L), true),
-        Row(1, "a", 5L, null, Row(15L), false)
+        Row(1, "a", 5L, null, Row(5L, null), true),
+        Row(1, "a", 5L, null, Row(10L, null), true),
+        Row(1, "a", 5L, null, Row(15L, null), false)
       )
     )
   }
@@ -3197,15 +3223,15 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // A real state change between the two rows (value a -> b) breaks the run, so the earlier
     // closed upsert is a visible run tail rather than a hidden no-op continuation.
     val df = targetTableOf(userSchema)(
-      Row(1, "a", 5L, 10L, Row(5L)),
-      Row(1, "b", 10L, null, Row(10L))
+      Row(1, "a", 5L, 10L, Row(5L, null)),
+      Row(1, "b", 10L, null, Row(10L, null))
     )
 
     checkAnswer(
       df = withRouteFlag(processor.identifyAndTagAuxRows(df)),
       expectedAnswer = Seq(
-        Row(1, "a", 5L, 10L, Row(5L), false),
-        Row(1, "b", 10L, null, Row(10L), false)
+        Row(1, "a", 5L, 10L, Row(5L, null), false),
+        Row(1, "b", 10L, null, Row(10L, null), false)
       )
     )
   }
@@ -3218,15 +3244,15 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // though both rows agree on the tracked `value`, the earlier row cannot be hidden - dropping
     // it would erase the gap from the visible timeline.
     val df = targetTableOf(userSchema)(
-      Row(1, "a", 5L, 8L, Row(5L)),
-      Row(1, "a", 10L, null, Row(10L))
+      Row(1, "a", 5L, 8L, Row(5L, null)),
+      Row(1, "a", 10L, null, Row(10L, null))
     )
 
     checkAnswer(
       df = withRouteFlag(processor.identifyAndTagAuxRows(df)),
       expectedAnswer = Seq(
-        Row(1, "a", 5L, 8L, Row(5L), false),
-        Row(1, "a", 10L, null, Row(10L), false)
+        Row(1, "a", 5L, 8L, Row(5L, null), false),
+        Row(1, "a", 10L, null, Row(10L, null), false)
       )
     )
   }
@@ -3243,15 +3269,15 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // With `value` excluded the effective tracked set is empty, so consecutive gapless upserts
     // are always tracked-equal: the earlier one is hidden even though the user data differs.
     val df = targetTableOf(userSchema)(
-      Row(1, "a", 5L, null, Row(5L)),
-      Row(1, "b", 5L, null, Row(10L))
+      Row(1, "a", 5L, null, Row(5L, null)),
+      Row(1, "b", 5L, null, Row(10L, null))
     )
 
     checkAnswer(
       df = withRouteFlag(processor.identifyAndTagAuxRows(df)),
       expectedAnswer = Seq(
-        Row(1, "a", 5L, null, Row(5L), true),
-        Row(1, "b", 5L, null, Row(10L), false)
+        Row(1, "a", 5L, null, Row(5L, null), true),
+        Row(1, "b", 5L, null, Row(10L, null), false)
       )
     )
   }
@@ -3266,19 +3292,19 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // 2's successor only. A window that leaked across keys would mistag one of these boundary
     // rows.
     val df = targetTableOf(userSchema)(
-      Row(1, "a", 5L, null, Row(5L)),
-      Row(1, "a", 5L, null, Row(10L)),
-      Row(2, "b", 7L, null, Row(7L)),
-      Row(2, "b", 7L, null, Row(20L))
+      Row(1, "a", 5L, null, Row(5L, null)),
+      Row(1, "a", 5L, null, Row(10L, null)),
+      Row(2, "b", 7L, null, Row(7L, null)),
+      Row(2, "b", 7L, null, Row(20L, null))
     )
 
     checkAnswer(
       df = withRouteFlag(processor.identifyAndTagAuxRows(df)),
       expectedAnswer = Seq(
-        Row(1, "a", 5L, null, Row(5L), true),
-        Row(1, "a", 5L, null, Row(10L), false),
-        Row(2, "b", 7L, null, Row(7L), true),
-        Row(2, "b", 7L, null, Row(20L), false)
+        Row(1, "a", 5L, null, Row(5L, null), true),
+        Row(1, "a", 5L, null, Row(10L, null), false),
+        Row(2, "b", 7L, null, Row(7L, null), true),
+        Row(2, "b", 7L, null, Row(20L, null), false)
       )
     )
   }
@@ -3300,8 +3326,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // parse `user.name` as a nested-field access (struct `user`, field `name`) and fail to
     // resolve.
     val df = targetTableOf(userSchema)(
-      Row(1, "alice", 5L, null, Row(5L)),
-      Row(1, "alice", 5L, null, Row(10L))
+      Row(1, "alice", 5L, null, Row(5L, null)),
+      Row(1, "alice", 5L, null, Row(10L, null))
     )
 
     val result = processor.identifyAndTagAuxRows(df)
@@ -3316,8 +3342,8 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
         F.col(Scd2BatchProcessor.shouldRouteToAuxTableColName)
       ),
       expectedAnswer = Seq(
-        Row(1, "alice", 5L, null, Row(5L), true),
-        Row(1, "alice", 5L, null, Row(10L), false)
+        Row(1, "alice", 5L, null, Row(5L, null), true),
+        Row(1, "alice", 5L, null, Row(10L, null), false)
       )
     )
   }
@@ -3330,18 +3356,18 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     val userSchema = new StructType().add("id", IntegerType).add("value", StringType)
 
     val left = targetTableOf(userSchema)(
-      Row(1, "L5", 5L, null, Row(5L)),
-      Row(1, "L9", 9L, null, Row(9L))
+      Row(1, "L5", 5L, null, Row(5L, null)),
+      Row(1, "L9", 9L, null, Row(9L, null))
     )
     // Right matches the left row at recordStartAt=5 only; the left row at 9 has no counterpart.
     val right = targetTableOf(userSchema)(
-      Row(1, "R5", 5L, null, Row(5L))
+      Row(1, "R5", 5L, null, Row(5L, null))
     )
 
     checkAnswer(
       df = processor.antiJoinRowsByRecordStartAtPerKey(left, right),
       expectedAnswer = Seq(
-        Row(1, "L9", 9L, null, Row(9L))
+        Row(1, "L9", 9L, null, Row(9L, null))
       )
     )
   }
@@ -3352,18 +3378,18 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     val userSchema = new StructType().add("id", IntegerType).add("value", StringType)
 
     val left = targetTableOf(userSchema)(
-      Row(1, "a", 5L, null, Row(5L)),
-      Row(2, "b", 5L, null, Row(5L))
+      Row(1, "a", 5L, null, Row(5L, null)),
+      Row(2, "b", 5L, null, Row(5L, null))
     )
     // Only key 1 at recordStartAt=5 matches; key 2 at the same recordStartAt must survive.
     val right = targetTableOf(userSchema)(
-      Row(1, "r", 5L, null, Row(5L))
+      Row(1, "r", 5L, null, Row(5L, null))
     )
 
     checkAnswer(
       df = processor.antiJoinRowsByRecordStartAtPerKey(left, right),
       expectedAnswer = Seq(
-        Row(2, "b", 5L, null, Row(5L))
+        Row(2, "b", 5L, null, Row(5L, null))
       )
     )
   }
@@ -3375,10 +3401,10 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // Both sides carry a null recordStartAt for the same key. A null-safe equality treats the
     // two as matching, so the left row is anti-joined away (an ordinary `=` would keep it).
     val left = targetTableOf(userSchema)(
-      Row(1, "tailL", null, 10L, Row(null))
+      Row(1, "tailL", null, 10L, Row(null, null))
     )
     val right = targetTableOf(userSchema)(
-      Row(1, "tailR", null, 20L, Row(null))
+      Row(1, "tailR", null, 20L, Row(null, null))
     )
 
     assert(processor.antiJoinRowsByRecordStartAtPerKey(left, right).collect().isEmpty)
@@ -3394,17 +3420,17 @@ class Scd2BatchProcessorSuite extends QueryTest with SharedSparkSession {
     // Rows share id and recordStartAt but differ on the second key column `grp`. Only the exact
     // composite-key match (1, "g") is removed; (1, "h") survives.
     val left = targetTableOf(userSchema)(
-      Row(1, "g", "a", 5L, null, Row(5L)),
-      Row(1, "h", "b", 5L, null, Row(5L))
+      Row(1, "g", "a", 5L, null, Row(5L, null)),
+      Row(1, "h", "b", 5L, null, Row(5L, null))
     )
     val right = targetTableOf(userSchema)(
-      Row(1, "g", "r", 5L, null, Row(5L))
+      Row(1, "g", "r", 5L, null, Row(5L, null))
     )
 
     checkAnswer(
       df = processor.antiJoinRowsByRecordStartAtPerKey(left, right),
       expectedAnswer = Seq(
-        Row(1, "h", "b", 5L, null, Row(5L))
+        Row(1, "h", "b", 5L, null, Row(5L, null))
       )
     )
   }
diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd2ForeachBatchHandlerSuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd2ForeachBatchHandlerSuite.scala
index 1384bcd078aaf..008313b1ee175 100644
--- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd2ForeachBatchHandlerSuite.scala
+++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd2ForeachBatchHandlerSuite.scala
@@ -52,7 +52,7 @@ class Scd2ForeachBatchHandlerSuite
     .add("seq", LongType)
     .add("is_delete", BooleanType)
 
-  /** The SCD2 cdc-metadata struct carries a single `recordStartAt` field (unlike SCD1's two). */
+  /** SCD2 cdc-metadata struct: `recordStartAt` + `versionMap` (unlike SCD1's two sequences). */
   private val scd2MetadataSchema: StructType = Scd2BatchProcessor.cdcMetadataColSchema(LongType)
 
   /** Canonical SCD2 row schema: persisted user columns + framework start/end + cdc metadata. */
@@ -111,7 +111,8 @@ class Scd2ForeachBatchHandlerSuite
   private def del(id: Int, seq: Long): Row = Row(id, null, seq, true)
 
   /** The cdc-metadata struct value for a given `recordStartAt`. */
-  private def meta(recordStartAt: Long): Row = Row(recordStartAt)
+  private def meta(recordStartAt: Long, versionMap: Any = null): Row =
+    Row(recordStartAt, versionMap)
 
   /** A canonical target row `(id, value, startAt, endAt, meta(recordStartAt))`. */
   private def targetRow(
diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcGraphExecutionTestMixin.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcGraphExecutionTestMixin.scala
index c789627005b6b..9eeba36b73526 100644
--- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcGraphExecutionTestMixin.scala
+++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcGraphExecutionTestMixin.scala
@@ -168,7 +168,9 @@ trait AutoCdcGraphExecutionTestMixin extends BeforeAndAfterEach {
     val startAt = Scd2BatchProcessor.startAtColName
     val endAt = Scd2BatchProcessor.endAtColName
     val recordStartAt = Scd2BatchProcessor.recordStartAtFieldName
-    s"$startAt BIGINT, $endAt BIGINT, $col STRUCT<$recordStartAt:BIGINT> NOT NULL"
+    val versionMap = Scd2BatchProcessor.versionMapFieldName
+    s"$startAt BIGINT, $endAt BIGINT, " +
+      s"$col STRUCT<$recordStartAt:BIGINT,$versionMap:MAP<STRING,BOOLEAN>> NOT NULL"
   }
 
   /**
diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2AuxiliaryTableDurabilitySuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2AuxiliaryTableDurabilitySuite.scala
index e6b1119383ced..863a6d5b3f75e 100644
--- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2AuxiliaryTableDurabilitySuite.scala
+++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2AuxiliaryTableDurabilitySuite.scala
@@ -45,7 +45,7 @@ class AutoCdcScd2AuxiliaryTableDurabilitySuite
   import testImplicits._
 
   /** The SCD2 target's `_cdc_metadata` struct value for a given recordStartAt. */
-  private def scd2Meta(recordStartAt: Long): Row = Row(recordStartAt)
+  private def scd2Meta(recordStartAt: Long): Row = Row(recordStartAt, null)
 
   test("a higher-sequence event in a later pipeline run correctly closes and opens records") {
     spark.sql(
diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2ColumnEvolutionSuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2ColumnEvolutionSuite.scala
index 25cb185a7ff49..8f573e8919667 100644
--- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2ColumnEvolutionSuite.scala
+++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2ColumnEvolutionSuite.scala
@@ -60,7 +60,7 @@ class AutoCdcScd2ColumnEvolutionSuite
   import testImplicits._
 
   /** The SCD2 target's `_cdc_metadata` struct value for a given recordStartAt. */
-  private def scd2Meta(recordStartAt: Long): Row = Row(recordStartAt)
+  private def scd2Meta(recordStartAt: Long): Row = Row(recordStartAt, null)
 
   /** An explicit SCD2 `TRACK HISTORY ON (name)` selection, shared across the scenarios below. */
   private val trackName: Option[ColumnSelection] =
diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2FullRefreshSuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2FullRefreshSuite.scala
index 3b6151d6c929f..903537ea3547f 100644
--- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2FullRefreshSuite.scala
+++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2FullRefreshSuite.scala
@@ -42,7 +42,7 @@ class AutoCdcScd2FullRefreshSuite
   import testImplicits._
 
   /** The SCD2 target's `_cdc_metadata` struct value for a given recordStartAt. */
-  private def scd2Meta(recordStartAt: Long): Row = Row(recordStartAt)
+  private def scd2Meta(recordStartAt: Long): Row = Row(recordStartAt, null)
 
   /** Create an SCD2 target with user columns `(id, name, version)` plus the framework columns. */
   private def createScd2Target(table: String): Unit = {
diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2MultiPipelineSuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2MultiPipelineSuite.scala
index cd161282e7294..7e3f34b3d9f59 100644
--- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2MultiPipelineSuite.scala
+++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2MultiPipelineSuite.scala
@@ -38,7 +38,7 @@ class AutoCdcScd2MultiPipelineSuite
   import testImplicits._
 
   /** The SCD2 target's `_cdc_metadata` struct value for a given recordStartAt. */
-  private def scd2Meta(recordStartAt: Long): Row = Row(recordStartAt)
+  private def scd2Meta(recordStartAt: Long): Row = Row(recordStartAt, null)
 
   test("two AutoCDC pipelines targeting separate tables maintain independent target and " +
     "auxiliary tables") {
diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2SchemaEvolutionSuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2SchemaEvolutionSuite.scala
index 838a5f8ee2f5b..4115121eebe9d 100644
--- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2SchemaEvolutionSuite.scala
+++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2SchemaEvolutionSuite.scala
@@ -23,9 +23,16 @@ import org.apache.spark.sql.Row
 import org.apache.spark.sql.execution.streaming.runtime.MemoryStream
 import org.apache.spark.sql.functions
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.pipelines.autocdc.{ColumnSelection, ScdType, UnqualifiedColumnName}
+import org.apache.spark.sql.pipelines.autocdc.{
+  AutoCdcReservedNames,
+  ColumnSelection,
+  Scd2BatchProcessor,
+  ScdType,
+  UnqualifiedColumnName
+}
 import org.apache.spark.sql.pipelines.utils.{ExecutionTest, TestGraphRegistrationContext}
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.LongType
 
 /**
  * Tests covering SCD Type 2 AutoCDC's interaction with non-key schema evolution across pipeline
@@ -64,7 +71,78 @@ class AutoCdcScd2SchemaEvolutionSuite
   import testImplicits._
 
   /** The SCD2 target's `_cdc_metadata` struct value for a given recordStartAt. */
-  private def scd2Meta(recordStartAt: Long): Row = Row(recordStartAt)
+  private def scd2Meta(recordStartAt: Long): Row = Row(recordStartAt, null)
+
+  test("legacy SCD2 CDC metadata schema evolves to include the version map") {
+    val targetName = s"$catalog.$namespace.target"
+    val auxiliaryName = auxTableNameFor("target")
+    val cdcMetadataCol = AutoCdcReservedNames.cdcMetadataColName
+    val recordStartAt = Scd2BatchProcessor.recordStartAtFieldName
+    val legacyMetadataDdl =
+      s"${Scd2BatchProcessor.startAtColName} BIGINT, " +
+      s"${Scd2BatchProcessor.endAtColName} BIGINT, " +
+      s"$cdcMetadataCol STRUCT<$recordStartAt:BIGINT> NOT NULL"
+    spark.sql(
+      s"CREATE TABLE $targetName " +
+      s"(id INT NOT NULL, name STRING, version BIGINT NOT NULL, " +
+      s"$legacyMetadataDdl)"
+    )
+    // Pre-create the auxiliary table too, so both persisted SCD2 schemas take the upgrade path.
+    spark.sql(
+      s"""CREATE TABLE $auxiliaryName """ +
+      s"""(id INT, name STRING, version BIGINT, $legacyMetadataDdl, """ +
+      s"""${Scd2BatchProcessor.deletedByBatchIdColName} BIGINT) """ +
+      s"""TBLPROPERTIES (""" +
+      s"""'${AutoCdcAuxiliaryTable.scdTypePropertyKey}' = '${ScdType.Type2.label}', """ +
+      s"""'${AutoCdcAuxiliaryTable.keyColumnNamesProperty}' = '["id"]', """ +
+      s"""'${AutoCdcAuxiliaryTable.trackHistoryColumnNamesProperty}' = '["name"]')"""
+    )
+    spark.sql(
+      s"INSERT INTO $targetName SELECT 1, 'alice', CAST(5 AS BIGINT), " +
+      s"CAST(1 AS BIGINT), CAST(NULL AS BIGINT), " +
+      s"named_struct('$recordStartAt', CAST(5 AS BIGINT))"
+    )
+    spark.sql(
+      s"INSERT INTO $auxiliaryName SELECT 1, 'alice', CAST(1 AS BIGINT), " +
+      s"CAST(1 AS BIGINT), CAST(NULL AS BIGINT), " +
+      s"named_struct('$recordStartAt', CAST(1 AS BIGINT)), CAST(NULL AS BIGINT)"
+    )
+
+    val stream = MemoryStream[(Int, String, Long)]
+    // Reconcile both legacy rows: one event extends their run, and the next closes it.
+    stream.addData((1, "alice", 3L), (1, "alicia", 6L))
+    runPipeline(singleAutoCdcFlowPipeline(
+      flowName = "auto_cdc_flow",
+      target = "target",
+      sourceDf = stream.toDF().toDF("id", "name", "version"),
+      keys = Seq("id"),
+      sequencing = functions.col("version"),
+      scdType = ScdType.Type2,
+      trackHistorySelection =
+        Option(ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("name"))))))
+
+    val target = spark.table(targetName)
+    val expectedMetadataSchema = Scd2BatchProcessor.cdcMetadataColSchema(LongType)
+    assert(
+      target.schema(cdcMetadataCol).dataType === expectedMetadataSchema.asNullable)
+    assert(
+      spark.table(auxiliaryName).schema(cdcMetadataCol).dataType ===
+        expectedMetadataSchema)
+    checkAnswer(
+      target,
+      Seq(
+        Row(1, "alice", 5L, 1L, 6L, scd2Meta(5L)),
+        Row(1, "alicia", 6L, 6L, null, scd2Meta(6L))
+      )
+    )
+    checkAnswer(
+      spark.table(auxiliaryName),
+      Seq(
+        Row(1, "alice", 1L, 1L, null, scd2Meta(1L), null),
+        Row(1, "alice", 3L, 1L, null, scd2Meta(3L), null)
+      )
+    )
+  }
 
   test("a nullable non-key column merges correctly with mixed NULL and non-NULL values") {
     spark.sql(
diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2SinglePipelineSuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2SinglePipelineSuite.scala
index 595a3df45e28b..3754156dfa8e6 100644
--- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2SinglePipelineSuite.scala
+++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2SinglePipelineSuite.scala
@@ -48,7 +48,7 @@ class AutoCdcScd2SinglePipelineSuite
   import testImplicits._
 
   /** The SCD2 target's `_cdc_metadata` struct value for a given recordStartAt. */
-  private def scd2Meta(recordStartAt: Long): Row = Row(recordStartAt)
+  private def scd2Meta(recordStartAt: Long): Row = Row(recordStartAt, null)
 
   /**
    * DDL for an SCD2 target table with user columns `(id, name, version)` plus the framework
diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2TargetTableDurabilitySuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2TargetTableDurabilitySuite.scala
index ff4e903117406..d3c4b00c3f974 100644
--- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2TargetTableDurabilitySuite.scala
+++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd2TargetTableDurabilitySuite.scala
@@ -38,7 +38,7 @@ class AutoCdcScd2TargetTableDurabilitySuite
   import testImplicits._
 
   /** The SCD2 target's `_cdc_metadata` struct value for a given recordStartAt. */
-  private def scd2Meta(recordStartAt: Long): Row = Row(recordStartAt)
+  private def scd2Meta(recordStartAt: Long): Row = Row(recordStartAt, null)
 
   /** Create an SCD2 target with user columns `(id, name, version)` plus the framework columns. */
   private def createScd2Target(table: String): Unit = {
@@ -61,10 +61,12 @@ class AutoCdcScd2TargetTableDurabilitySuite
   private def insertPreloadedCurrentRecord(
       table: String, colValues: String, sequence: Long): Unit = {
     val recordStartAt = Scd2BatchProcessor.recordStartAtFieldName
+    val versionMap = Scd2BatchProcessor.versionMapFieldName
     spark.sql(
       s"INSERT INTO $table SELECT $colValues, " +
       s"CAST($sequence AS BIGINT), CAST(NULL AS BIGINT), " +
-      s"named_struct('$recordStartAt', CAST($sequence AS BIGINT))"
+      s"named_struct('$recordStartAt', CAST($sequence AS BIGINT), " +
+      s"'$versionMap', CAST(NULL AS MAP<STRING, BOOLEAN>))"
     )
   }
 
@@ -83,10 +85,12 @@ class AutoCdcScd2TargetTableDurabilitySuite
   private def insertPreloadedClosedRecord(
       table: String, colValues: String, startAt: Long, endAt: Long): Unit = {
     val recordStartAt = Scd2BatchProcessor.recordStartAtFieldName
+    val versionMap = Scd2BatchProcessor.versionMapFieldName
     spark.sql(
       s"INSERT INTO $table SELECT $colValues, " +
       s"CAST($startAt AS BIGINT), CAST($endAt AS BIGINT), " +
-      s"named_struct('$recordStartAt', CAST($startAt AS BIGINT))"
+      s"named_struct('$recordStartAt', CAST($startAt AS BIGINT), " +
+      s"'$versionMap', CAST(NULL AS MAP<STRING, BOOLEAN>))"
     )
   }
 
diff --git a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/util/SchemaInferenceUtilsSuite.scala b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/util/SchemaInferenceUtilsSuite.scala
index 92e1c700b99ba..1996463f07f66 100644
--- a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/util/SchemaInferenceUtilsSuite.scala
+++ b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/util/SchemaInferenceUtilsSuite.scala
@@ -19,10 +19,16 @@ package org.apache.spark.sql.pipelines.util
 
 import scala.util.Success
 
-import org.apache.spark.SparkException
+import org.apache.spark.{SparkException, SparkUnsupportedOperationException}
 import org.apache.spark.sql.{QueryTest, Row}
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.connector.catalog.TableChange
+import org.apache.spark.sql.connector.catalog.{
+  CatalogV2Util,
+  Identifier,
+  InMemoryTableCatalog,
+  TableChange,
+  TableInfo
+}
 import org.apache.spark.sql.pipelines.graph.{
   FlowFunction,
   FlowFunctionResult,
@@ -35,8 +41,10 @@ import org.apache.spark.sql.pipelines.graph.{
 }
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
 class SchemaInferenceUtilsSuite extends QueryTest with SharedSparkSession {
+  import TableChangeExtractors._
 
   /** A [[FlowFunction]] that throws if invoked; the inferSchemaFromFlows test builds resolved
    * flows directly. */
@@ -395,19 +403,19 @@ class SchemaInferenceUtilsSuite extends QueryTest with SharedSparkSession {
     assert(merged === new StructType().add("s", expectedStruct))
 
     // Unlike the case-insensitive test above (where the merge is a no-op and no changes are
-    // derived), evolution here must rewrite the top-level `s` column. `diffSchemas` compares nested
-    // types wholesale, so the growth of a nested field surfaces as a single UpdateColumnType on `s`
-    // carrying the full new struct -- not as an add of `s.Value`.
+    // derived), evolution here must grow the `s` struct. The growth surfaces as an add of the
+    // nested leaf `s.Value`, which is the portable shape; retyping `s` with the whole new struct
+    // would be rejected by `CheckAnalysis`, which fails ALTER COLUMN ... TYPE on a struct.
     val changes = SchemaInferenceUtils.diffSchemas(currentSchema, merged)
     assert(changes.length === 1)
-    val typeChange = changes.collect { case tc: TableChange.UpdateColumnType => tc }
-    assert(typeChange.length === 1)
-    assert(typeChange.head.fieldNames() === Array("s"))
-    assert(typeChange.head.newDataType() === expectedStruct)
+    val addChange = changes.collect { case ac: TableChange.AddColumn => ac }
+    assert(addChange.length === 1)
+    assert(addChange.head.fieldNames() === Array("s", "Value"))
+    assert(addChange.head.dataType() === StringType)
   }
 
   test("mergeSchemas - a nested case-only field whose type also changes fails to merge, and " +
-    "diffSchemas reports it as a type change") {
+    "diffSchemas reports it as a drop-then-add") {
     // A nested field that differs only in case AND changes type is rejected rather than silently
     // resolved. Note this is a *type* incompatibility, not a case one: `StructType.merge` never
     // widens numeric types, so `int` -> `long` fails identically for a same-cased field and at the
@@ -438,22 +446,402 @@ class SchemaInferenceUtilsSuite extends QueryTest with SharedSparkSession {
     }
 
     // Diffing the two schemas directly (rather than diffing against their merge, which fails
-    // above) reports a TYPE change on the enclosing `s` column -- not a field-name mismatch, i.e.
-    // not an add of `s.Value` plus a delete of `s.value`. `diffSchemas` keys column identity only
-    // at the top level and compares nested types wholesale, so the case difference inside the
-    // struct never surfaces as an add/delete pair.
+    // above) reports a drop-then-add of the nested leaf, since `diffSchemas` keys column identity
+    // on the exact name at every level. This is the nested analog of the top-level rule pinned by
+    // "a case-only difference is a drop-then-add, not a match", and it is only reachable on the
+    // non-merging paths, where the declared schema is used as-is.
     {
       val changes = SchemaInferenceUtils.diffSchemas(currentSchema, dataSchema)
-      assert(changes.length === 1, s"changes=$changes")
-      val typeChange = changes.collect { case tc: TableChange.UpdateColumnType => tc }
-      assert(typeChange.length === 1, s"changes=$changes")
-      assert(typeChange.head.fieldNames() === Array("s"))
-      assert(typeChange.head.newDataType() === new StructType().add("Value", LongType))
-      assert(!changes.exists(_.isInstanceOf[TableChange.AddColumn]))
-      assert(!changes.exists(_.isInstanceOf[TableChange.DeleteColumn]))
+      assert(changes.length === 2, s"changes=$changes")
+      val added = changes.collect { case ac: TableChange.AddColumn => ac }
+      assert(added.length === 1, s"changes=$changes")
+      assert(added.head.fieldNames() === Array("s", "Value"))
+      assert(added.head.dataType() === LongType)
+      val deleted = changes.collect { case dc: TableChange.DeleteColumn => dc }
+      assert(deleted.length === 1, s"changes=$changes")
+      assert(deleted.head.fieldNames() === Array("s", "value"))
+      assert(!changes.exists(_.isInstanceOf[TableChange.UpdateColumnType]))
     }
   }
 
+  test("diffSchemas - a leaf added to a struct is a nested add, not a retype of the parent") {
+    val currentSchema = new StructType()
+      .add("id", IntegerType)
+      .add("point", new StructType().add("x", DoubleType).add("y", DoubleType))
+    val targetSchema = new StructType()
+      .add("id", IntegerType)
+      .add(
+        "point",
+        new StructType()
+          .add("x", DoubleType)
+          .add("y", DoubleType)
+          .add("z", DoubleType))
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(changes.length === 1, s"changes=$changes")
+    assert(addsOf(changes) === Map(Seq("point", "z") -> ((DoubleType, true, null))))
+  }
+
+  test("diffSchemas - a leaf added to a deeply nested struct carries the full path") {
+    val inner = new StructType().add("c", IntegerType)
+    val currentSchema = new StructType().add("a", new StructType().add("b", inner))
+    val targetSchema = new StructType()
+      .add("a", new StructType().add("b", inner.add("d", StringType)))
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(addsOf(changes).keySet === Set(Seq("a", "b", "d")))
+  }
+
+  test("diffSchemas - an added nested leaf keeps its declared nullability and comment") {
+    val currentSchema = new StructType()
+      .add("s", new StructType().add("a", IntegerType))
+    val targetSchema = new StructType().add(
+      "s",
+      new StructType()
+        .add("a", IntegerType)
+        .add("b", StringType, nullable = false, "a comment"))
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(addsOf(changes) === Map(Seq("s", "b") -> ((StringType, false, "a comment"))))
+  }
+
+  test("diffSchemas - a leaf removed from a struct is a nested delete") {
+    val currentSchema = new StructType()
+      .add("s", new StructType().add("a", IntegerType).add("b", StringType))
+    val targetSchema = new StructType()
+      .add("s", new StructType().add("a", IntegerType))
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(changes.length === 1, s"changes=$changes")
+    assert(deletesOf(changes) === Set(Seq("s", "b")))
+  }
+
+  test("diffSchemas - a leaf type change inside a struct is a nested type update") {
+    val currentSchema = new StructType()
+      .add("s", new StructType().add("a", IntegerType))
+    val targetSchema = new StructType()
+      .add("s", new StructType().add("a", LongType))
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(changes.length === 1, s"changes=$changes")
+    assert(typeUpdatesOf(changes) === Map(Seq("s", "a") -> LongType))
+  }
+
+  test("diffSchemas - nested leaf nullability and comment changes are emitted at the leaf") {
+    val currentSchema = new StructType()
+      .add("s", new StructType().add("a", IntegerType, nullable = true, "old"))
+    val targetSchema = new StructType()
+      .add("s", new StructType().add("a", IntegerType, nullable = false, "new"))
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(changes.length === 2, s"changes=$changes")
+    assert(nullabilityUpdatesOf(changes) === Map(Seq("s", "a") -> false))
+    assert(commentUpdatesOf(changes) === Map(Seq("s", "a") -> "new"))
+  }
+
+  test("diffSchemas - a field added to a struct inside an array uses the element path") {
+    val currentSchema = new StructType()
+      .add("points", ArrayType(new StructType().add("x", DoubleType)))
+    val targetSchema = new StructType()
+      .add("points", ArrayType(new StructType().add("x", DoubleType).add("y", DoubleType)))
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(addsOf(changes).keySet === Set(Seq("points", "element", "y")))
+  }
+
+  test("diffSchemas - a field added to a struct inside a map value uses the value path") {
+    val currentSchema = new StructType().add(
+      "points",
+      MapType(StringType, new StructType().add("x", DoubleType)))
+    val targetSchema = new StructType()
+      .add(
+        "points",
+        MapType(StringType, new StructType().add("x", DoubleType).add("y", DoubleType)))
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(addsOf(changes).keySet === Set(Seq("points", "value", "y")))
+  }
+
+  test("diffSchemas - a field added to a struct inside a map key uses the key path") {
+    val currentSchema = new StructType().add(
+      "points",
+      MapType(new StructType().add("x", DoubleType), LongType))
+    val targetSchema = new StructType()
+      .add(
+        "points",
+        MapType(new StructType().add("x", DoubleType).add("y", DoubleType), LongType))
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(addsOf(changes).keySet === Set(Seq("points", "key", "y")))
+  }
+
+  test("diffSchemas - an element type change inside an array uses the element path") {
+    val currentSchema = new StructType().add("vals", ArrayType(IntegerType))
+    val targetSchema = new StructType().add("vals", ArrayType(LongType))
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(typeUpdatesOf(changes) === Map(Seq("vals", "element") -> LongType))
+  }
+
+  test("diffSchemas - a map value type change uses the value path") {
+    val currentSchema = new StructType()
+      .add("m", MapType(StringType, IntegerType))
+    val targetSchema = new StructType()
+      .add("m", MapType(StringType, LongType))
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(typeUpdatesOf(changes) === Map(Seq("m", "value") -> LongType))
+  }
+
+  test("diffSchemas - an array containsNull change is a nullability update on the element") {
+    val currentSchema = new StructType()
+      .add("vals", ArrayType(IntegerType, containsNull = false))
+    val targetSchema = new StructType()
+      .add("vals", ArrayType(IntegerType, containsNull = true))
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(changes.length === 1, s"changes=$changes")
+    assert(nullabilityUpdatesOf(changes) === Map(Seq("vals", "element") -> true))
+  }
+
+  test("diffSchemas - a map valueContainsNull change is a nullability update on the value") {
+    val currentSchema = new StructType().add(
+      "m", MapType(StringType, IntegerType, valueContainsNull = false))
+    val targetSchema = new StructType().add(
+      "m", MapType(StringType, IntegerType, valueContainsNull = true))
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(changes.length === 1, s"changes=$changes")
+    assert(nullabilityUpdatesOf(changes) === Map(Seq("m", "value") -> true))
+  }
+
+  test("diffSchemas - a wholly new struct column stays a single top-level add") {
+    val newStruct = new StructType().add("x", DoubleType).add("y", DoubleType)
+    val currentSchema = new StructType().add("id", IntegerType)
+    val targetSchema = new StructType()
+      .add("id", IntegerType)
+      .add("point", newStruct)
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(changes.length === 1, s"changes=$changes")
+    assert(addsOf(changes) === Map(Seq("point") -> ((newStruct, true, null))))
+  }
+
+  test("diffSchemas - a whole struct column removed stays a single top-level delete") {
+    val currentSchema = new StructType()
+      .add("id", IntegerType)
+      .add("point", new StructType().add("x", DoubleType))
+    val targetSchema = new StructType().add("id", IntegerType)
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(changes.length === 1, s"changes=$changes")
+    assert(deletesOf(changes) === Set(Seq("point")))
+  }
+
+  test("diffSchemas - a struct replaced by an atomic type is a type update on the column") {
+    val currentSchema = new StructType()
+      .add("s", new StructType().add("a", IntegerType))
+    val targetSchema = new StructType().add("s", StringType)
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(changes.length === 1, s"changes=$changes")
+    assert(typeUpdatesOf(changes) === Map(Seq("s") -> StringType))
+  }
+
+  test("diffSchemas - identical nested schemas produce no changes") {
+    val schema = new StructType()
+      .add("s", new StructType().add("a", IntegerType).add("b", StringType))
+      .add("arr", ArrayType(new StructType().add("x", DoubleType)))
+      .add("m", MapType(StringType, new StructType().add("y", DoubleType)))
+
+    assert(SchemaInferenceUtils.diffSchemas(schema, schema).isEmpty)
+  }
+
+  test("diffSchemas - independent nested changes are all emitted at their own leaves") {
+    val currentSchema = new StructType()
+      .add(
+        "s",
+        new StructType()
+          .add("a", IntegerType)
+          .add("gone", StringType))
+      .add("arr", ArrayType(new StructType().add("x", DoubleType)))
+      .add("top", IntegerType)
+    val targetSchema = new StructType()
+      .add(
+        "s",
+        new StructType()
+          .add("a", IntegerType)
+          .add("added", StringType))
+      .add("arr", ArrayType(new StructType().add("x", DoubleType).add("z", DoubleType)))
+      .add("top", IntegerType)
+      .add("brand_new", BooleanType)
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(
+      addsOf(changes).keySet === Set(
+        Seq("s", "added"),
+        Seq("arr", "element", "z"),
+        Seq("brand_new")),
+      s"changes=$changes")
+    assert(
+      deletesOf(changes) === Set(Seq("s", "gone")),
+      s"changes=$changes")
+    assert(typeUpdatesOf(changes).isEmpty, s"changes=$changes")
+  }
+
+  test("diffSchemas - DSv2 catalog respects emitted nested changes") {
+    val currentNestedStruct =
+      new StructType().add("x", DoubleType, nullable = false, "old comment")
+    val currentSchema = new StructType()
+      .add("struct", currentNestedStruct)
+      .add("array", ArrayType(currentNestedStruct))
+      .add("map", MapType(currentNestedStruct, currentNestedStruct))
+
+    val targetNestedStruct =
+      new StructType()
+        .add("x", DoubleType, nullable = true, "new comment")
+        .add("y", DoubleType)
+    // Add `y`, make `x` nullable, and update its comment in every nested struct.
+    val targetSchema = new StructType()
+      .add("struct", targetNestedStruct)
+      .add("array", ArrayType(targetNestedStruct))
+      .add("map", MapType(targetNestedStruct, targetNestedStruct))
+
+    val catalog = new InMemoryTableCatalog
+    catalog.initialize("test", CaseInsensitiveStringMap.empty())
+    val ident = Identifier.of(Array.empty, "t")
+    catalog.createTable(ident, new TableInfo.Builder().withSchema(currentSchema).build())
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    val updated = catalog.alterTable(ident, changes: _*)
+
+    assert(
+      CatalogV2Util.clearIds(updated.columns()) ===
+        CatalogV2Util.structTypeToV2Columns(targetSchema, keepIds = false))
+  }
+
+  test("diffSchemas - array<array<struct>> throws unsupported error") {
+    val innerCurrent =
+      ArrayType(new StructType().add("x", IntegerType).add("y", StringType))
+    val innerTarget = ArrayType(new StructType().add("x", IntegerType))
+
+    val currentSchema = new StructType().add("a", ArrayType(innerCurrent))
+    val targetSchema = new StructType().add("a", ArrayType(innerTarget))
+
+    // CatalogV2Util cannot resolve paths with consecutive element/key/value
+    // segments (SPARK-59188), so diffSchemas rejects nested complex type
+    // evolution rather than emitting changes the catalog cannot apply.
+    checkError(
+      intercept[SparkUnsupportedOperationException](
+        SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)),
+      condition =
+        "PIPELINE_NESTED_COMPLEX_TYPE_SCHEMA_EVOLUTION_UNSUPPORTED",
+      parameters = Map(
+        "columnPath" -> "a.element",
+        "currentType" -> innerCurrent.simpleString,
+        "targetType" -> innerTarget.simpleString))
+  }
+
+  test("diffSchemas - map<string, array<struct>> throws unsupported error") {
+    val valCurrent = ArrayType(new StructType().add("x", IntegerType))
+    val valTarget = ArrayType(
+      new StructType().add("x", IntegerType).add("y", StringType))
+
+    val currentSchema = new StructType()
+      .add("m", MapType(StringType, valCurrent))
+    val targetSchema = new StructType()
+      .add("m", MapType(StringType, valTarget))
+
+    checkError(
+      intercept[SparkUnsupportedOperationException](
+        SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)),
+      condition =
+        "PIPELINE_NESTED_COMPLEX_TYPE_SCHEMA_EVOLUTION_UNSUPPORTED",
+      parameters = Map(
+        "columnPath" -> "m.value",
+        "currentType" -> valCurrent.simpleString,
+        "targetType" -> valTarget.simpleString))
+  }
+
+  test("diffSchemas - map<struct, map<string, int>> recurses into struct key") {
+    val keyCurrent = new StructType().add("a", IntegerType)
+    val keyTarget =
+      new StructType().add("a", IntegerType).add("b", StringType)
+
+    val currentSchema = new StructType()
+      .add("m", MapType(keyCurrent, MapType(StringType, IntegerType)))
+    val targetSchema = new StructType()
+      .add("m", MapType(keyTarget, MapType(StringType, IntegerType)))
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    // The key is a struct, so leaf-level diff is emitted.
+    assert(changes.length === 1, s"changes=$changes")
+    assert(
+      addsOf(changes) ===
+        Map(Seq("m", "key", "b") -> ((StringType, true, null))))
+  }
+
+  test("diffSchemas - identical nested array<array<struct>> produces no changes") {
+    val inner = ArrayType(new StructType().add("x", IntegerType))
+    val schema = new StructType().add("a", ArrayType(inner))
+    assert(SchemaInferenceUtils.diffSchemas(schema, schema).isEmpty)
+  }
+
+  test("diffSchemas - non-type changes on nested elements are supported") {
+    // Due to SPARK-59188, type changes on nested array/map elements are unsupported. But other
+    // schema changes like nullability are still supported.
+    val currentSchema = new StructType()
+      .add("a", ArrayType(ArrayType(IntegerType), containsNull = false))
+    val targetSchema = new StructType()
+      .add("a", ArrayType(ArrayType(IntegerType), containsNull = true))
+
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(changes.length === 1, s"changes=$changes")
+    assert(nullabilityUpdatesOf(changes) === Map(Seq("a", "element") -> true))
+  }
+
+  test("diffSchemas - tightening nullability throws when flag is set") {
+    val currentSchema = new StructType()
+      .add("a", IntegerType, nullable = true)
+    val targetSchema = new StructType()
+      .add("a", IntegerType, nullable = false)
+
+    // Without the flag, tightening is allowed (full-refresh / MV path).
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(nullabilityUpdatesOf(changes) === Map(Seq("a") -> false))
+
+    // With the flag, tightening is rejected (incremental path).
+    checkError(
+      intercept[SparkUnsupportedOperationException](
+        SchemaInferenceUtils.diffSchemas(
+          currentSchema,
+          targetSchema,
+          rejectNullabilityTightening = true)),
+      condition = "PIPELINE_TIGHTEN_NULLABILITY_UNSUPPORTED",
+      parameters = Map("columnPath" -> "a"))
+  }
+
+  test("diffSchemas - tightening nested nullability throws when flag is set") {
+    val currentSchema = new StructType()
+      .add("s", new StructType().add("x", IntegerType, nullable = true))
+    val targetSchema = new StructType()
+      .add("s", new StructType().add("x", IntegerType, nullable = false))
+
+    // Without the flag, allowed.
+    val changes = SchemaInferenceUtils.diffSchemas(currentSchema, targetSchema)
+    assert(nullabilityUpdatesOf(changes) === Map(Seq("s", "x") -> false))
+
+    // With the flag, rejected.
+    checkError(
+      intercept[SparkUnsupportedOperationException](
+        SchemaInferenceUtils.diffSchemas(
+          currentSchema,
+          targetSchema,
+          rejectNullabilityTightening = true)),
+      condition = "PIPELINE_TIGHTEN_NULLABILITY_UNSUPPORTED",
+      parameters = Map("columnPath" -> "s.x"))
+  }
+
   test("inferSchemaFromFlows folds a case-only column to the same spelling regardless of flow " +
     "order, even when identifier names contain dots") {
     // The merge order decides which spelling of a case-only-differing column survives, so it must
@@ -483,3 +871,35 @@ class SchemaInferenceUtilsSuite extends QueryTest with SharedSparkSession {
     }
   }
 }
+
+private[util] object TableChangeExtractors {
+  /** Added columns as path -> (type, nullable, comment). */
+  def addsOf(changes: Seq[TableChange]): Map[Seq[String], (DataType, Boolean, String)] =
+    changes.collect {
+      case ac: TableChange.AddColumn =>
+        ac.fieldNames().toSeq -> ((ac.dataType(), ac.isNullable(), ac.comment()))
+    }.toMap
+
+  def deletesOf(changes: Seq[TableChange]): Set[Seq[String]] =
+    changes.collect {
+      case dc: TableChange.DeleteColumn => dc.fieldNames().toSeq
+    }.toSet
+
+  def typeUpdatesOf(changes: Seq[TableChange]): Map[Seq[String], DataType] =
+    changes.collect {
+      case tc: TableChange.UpdateColumnType =>
+        tc.fieldNames().toSeq -> tc.newDataType()
+    }.toMap
+
+  def nullabilityUpdatesOf(changes: Seq[TableChange]): Map[Seq[String], Boolean] =
+    changes.collect {
+      case nc: TableChange.UpdateColumnNullability =>
+        nc.fieldNames().toSeq -> nc.nullable()
+    }.toMap
+
+  def commentUpdatesOf(changes: Seq[TableChange]): Map[Seq[String], String] =
+    changes.collect {
+      case cc: TableChange.UpdateColumnComment =>
+        cc.fieldNames().toSeq -> cc.newComment()
+    }.toMap
+}