Spark: Add native CDC for copy-on-write tables

docs/docs/spark-queries.md

@@ -103,6 +103,47 @@ For `string` and `binary` inputs, it keeps the first `width` characters or bytes
 These functions are especially useful when you want to inspect how Iceberg transforms values or
 when writing filters for queries and row-level operations that align with partition transforms.
 
+### Native change data capture in Spark 4.2
+
+Native CDC reads use Iceberg snapshot sequence numbers as commit versions. The bounds below
+include commits 20 through 25. Required snapshots and their data files must still be retained.
+The current implementation supports copy-on-write changes in format v2 and later; delete-file
+changelog scans are not supported.
+
+Without row lineage, raw mode returns all added and removed rows, including unchanged rows
+copied during a file rewrite:
+
+```sql
+SELECT * FROM prod.db.table CHANGES FROM VERSION 20 TO VERSION 25
+WITH (deduplicationMode = 'none', computeUpdates = 'false');
+```
+
+To remove carry-over and compute update images using business keys, enable
+[Iceberg SQL extensions](spark-configuration.md#sql-extensions) and explicitly supply
+`identifier-columns`:
+
+```sql
+SELECT * FROM prod.db.table CHANGES FROM VERSION 20 TO VERSION 25
+WITH (`identifier-columns` = 'id',
+      deduplicationMode = 'dropCarryovers',
+      computeUpdates = 'true');
+```
+
+`identifier-columns` is a comma-separated list of top-level primitive column names, such as
+`tenant_id,id`. Names follow Spark's case-sensitivity setting. This mode uses the same iterators
+as `create_changelog_view`: remove equal DELETE/INSERT carry-over rows, then pair changes by
+the business key within each commit. Keys must identify rows unambiguously; ambiguous duplicate
+keys can fail update reconstruction. Changing the key produces a delete and a separate insert.
+Setting `computeUpdates=false` still removes carry-over but leaves the `delete` and `insert` labels.
+This mode supports batch reads with `dropCarryovers`; it rejects streaming, `none`, and `netChanges`.
+
+Native output contains user columns plus `_change_type`, `_commit_version`, `_commit_timestamp`,
+`_row_id`, and `_last_updated_sequence_number`. Update images use `update_preimage` and
+`update_postimage`. Business-key mode returns null for both lineage columns, even when the source
+has lineage, because it processes rows by values and business keys. It preserves commit versions
+and timestamps. Without `identifier-columns`, native post-processing uses Iceberg row lineage
+and requires valid lineage in the selected snapshots and files.
+
 ### Time travel Queries with SQL
 Spark supports time travel in SQL queries using `TIMESTAMP AS OF` or `VERSION AS OF` clauses.
 The `VERSION AS OF` clause can contain a long snapshot ID or a string branch or tag name.

spark/v4.2/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala

@@ -22,6 +22,7 @@ import org.apache.spark.sql.SparkSessionExtensions
 import org.apache.spark.sql.catalyst.analysis.CheckViews
 import org.apache.spark.sql.catalyst.analysis.ResolveBranch
 import org.apache.spark.sql.catalyst.analysis.ResolveViews
+import org.apache.spark.sql.catalyst.analysis.RewriteBusinessKeyChangelog
 import org.apache.spark.sql.catalyst.optimizer.ReplaceStaticInvoke
 import org.apache.spark.sql.catalyst.parser.extensions.IcebergSparkSqlExtensionsParser
 import org.apache.spark.sql.execution.datasources.v2.ExtendedDataSourceV2Strategy
@@ -35,6 +36,7 @@ class IcebergSparkSessionExtensions extends (SparkSessionExtensions => Unit) {
     // analyzer extensions
     extensions.injectResolutionRule { spark => ResolveViews(spark) }
     extensions.injectPostHocResolutionRule { spark => ResolveBranch(spark) }
+    extensions.injectPostHocResolutionRule { spark => RewriteBusinessKeyChangelog(spark) }
     extensions.injectCheckRule(_ => CheckViews)
 
     // optimizer extensions

spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestBusinessKeyChangelog.java

@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark.extensions;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.sql.Timestamp;
+import java.util.List;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.Table;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.RowFactory;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.TestTemplate;
+
+class TestBusinessKeyChangelog extends ExtensionsTestBase {
+
+  @AfterEach
+  void removeTableAndView() {
+    spark.catalog().dropTempView("business_key_expected");
+    sql("DROP TABLE IF EXISTS %s", tableName);
+  }
+
+  @TestTemplate
+  void matchesChangelogViewWithoutLineage() {
+    createTable();
+    Table table = validationCatalog.loadTable(tableIdent);
+    long startSnapshot = table.currentSnapshot().snapshotId();
+    sql("UPDATE %s SET data = 'updated' WHERE id = 1", tableName);
+    table.refresh();
+    Snapshot end = table.currentSnapshot();
+    Dataset<Row> changes = changes(end.sequenceNumber(), end.sequenceNumber(), true);
+    Timestamp timestamp = new Timestamp(end.timestampMillis());
+    assertThat(changes.collectAsList())
+        .containsExactlyInAnyOrder(
+            RowFactory.create(
+                1L, "a", null, null, "update_preimage", end.sequenceNumber(), timestamp),
+            RowFactory.create(
+                1L, "updated", null, null, "update_postimage", end.sequenceNumber(), timestamp));
+    assertThat(changes.schema().apply("_row_id").nullable()).isTrue();
+    assertThat(changes.schema().apply("_last_updated_sequence_number").nullable()).isTrue();
+
+    sql(
+        "CALL %s.system.create_changelog_view(table => '%s', "
+            + "changelog_view => 'business_key_expected', compute_updates => true, "
+            + "identifier_columns => array('id'), "
+            + "options => map('start-snapshot-id', '%d', 'end-snapshot-id', '%d'))",
+        catalogName, tableName, startSnapshot, end.snapshotId());
+    List<Row> expected =
+        spark
+            .sql(
+                "SELECT id, data, "
+                    + "CASE _change_type WHEN 'UPDATE_BEFORE' THEN 'update_preimage' "
+                    + "WHEN 'UPDATE_AFTER' THEN 'update_postimage' ELSE lower(_change_type) END AS _change_type "
+                    + "FROM business_key_expected")
+            .collectAsList();
+    assertThat(changes.select("id", "data", "_change_type").collectAsList())
+        .containsExactlyInAnyOrderElementsOf(expected);
+    assertThat(changes.filter("data = 'updated'").select("_change_type").collectAsList())
+        .containsExactly(RowFactory.create("update_postimage"));
+    assertThat(changes.select("_change_type").collectAsList())
+        .containsExactlyInAnyOrder(
+            RowFactory.create("update_preimage"), RowFactory.create("update_postimage"));
+  }
+
+  @TestTemplate
+  void keepsDeleteInsertWhenUpdateImagesAreDisabled() {
+    createTable();
+    sql("UPDATE %s SET data = 'updated' WHERE id = 1", tableName);
+    long version = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber();
+    assertThat(
+            changes(version, version, false).select("id", "data", "_change_type").collectAsList())
+        .containsExactlyInAnyOrder(
+            RowFactory.create(1L, "a", "delete"), RowFactory.create(1L, "updated", "insert"));
+  }
+
+  @TestTemplate
+  void treatsBusinessKeyChangeAsDeleteAndInsert() {
+    createTable();
+    sql("UPDATE %s SET id = 3 WHERE id = 1", tableName);
+    long version = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber();
+    assertThat(changes(version, version, true).select("id", "data", "_change_type").collectAsList())
+        .containsExactlyInAnyOrder(
+            RowFactory.create(1L, "a", "delete"), RowFactory.create(3L, "a", "insert"));
+  }
+
+  @TestTemplate
+  void pairsWithinEachCommitAndRemovesSameValueRewrites() {
+    createTable();
+    sql("UPDATE %s SET data = 'updated' WHERE id = 1", tableName);
+    long first = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber();
+    sql("UPDATE %s SET data = 'a' WHERE id = 1", tableName);
+    long second = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber();
+    assertThat(
+            changes(first, second, true)
+                .select("data", "_change_type", "_commit_version")
+                .collectAsList())
+        .containsExactlyInAnyOrder(
+            RowFactory.create("a", "update_preimage", first),
+            RowFactory.create("updated", "update_postimage", first),
+            RowFactory.create("updated", "update_preimage", second),
+            RowFactory.create("a", "update_postimage", second));
+    sql("UPDATE %s SET data = 'a' WHERE id = 1", tableName);
+    long sameValue = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber();
+    assertThat(changes(sameValue, sameValue, true).collectAsList()).isEmpty();
+  }
+
+  @TestTemplate
+  void rejectsAmbiguousBusinessKeys() {
+    createTable();
+    sql("INSERT INTO %s VALUES (1, 'other')", tableName);
+    sql("UPDATE %s SET data = 'updated' WHERE id = 1", tableName);
+    long version = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber();
+    assertThatThrownBy(() -> changes(version, version, true).collectAsList())
+        .hasStackTraceContaining("multiple rows with the same identifier");
+  }
+
+  @TestTemplate
+  void validatesBusinessKeyOptionsAndRejectsStreaming() {
+    createTable();
+    for (String identifiers : List.of("", "missing", "id,id", "data.nested", "_row_id")) {
+      assertThatThrownBy(
+              () ->
+                  spark
+                      .read()
+                      .option("identifier-columns", identifiers)
+                      .option("computeUpdates", "true")
+                      .changes(tableName)
+                      .collectAsList())
+          .hasStackTraceContaining("identifier column");
+    }
+    for (String mode : List.of("none", "netChanges")) {
+      assertThatThrownBy(
+              () ->
+                  spark
+                      .read()
+                      .option("identifier-columns", "id")
+                      .option("deduplicationMode", mode)
+                      .changes(tableName)
+                      .collectAsList())
+          .hasStackTraceContaining("requires deduplicationMode=dropCarryovers");
+    }
+    assertThatThrownBy(
+            () ->
+                spark
+                    .readStream()
+                    .option("identifier-columns", "id")
+                    .option("computeUpdates", "true")
+                    .changes(tableName)
+                    .explain())
+        .hasStackTraceContaining("Business-key CDC currently supports batch reads only");
+  }
+
+  @TestTemplate
+  void resolvesQuotedCompositeKeysAndNullableValues() {
+    sql(
+        "CREATE TABLE %s (`Tenant.ID` string, id bigint, data string) USING iceberg "
+            + "TBLPROPERTIES ('format-version'='2', 'write.update.mode'='copy-on-write')",
+        tableName);
+    sql(
+        "INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM VALUES "
+            + "('x', CAST(NULL AS BIGINT), CAST(NULL AS STRING)), ('y', 1, 'b')",
+        tableName);
+    sql("UPDATE %s SET data = 'updated' WHERE `Tenant.ID` = 'x'", tableName);
+    long version = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber();
+    Dataset<Row> result =
+        spark
+            .read()
+            .option("identifier-columns", "tenant.id, ID")
+            .option("computeUpdates", "true")
+            .option("startingVersion", String.valueOf(version))
+            .option("endingVersion", String.valueOf(version))
+            .changes(tableName);
+    assertThat(result.selectExpr("`Tenant.ID`", "id", "data", "_change_type").collectAsList())
+        .containsExactlyInAnyOrder(
+            RowFactory.create("x", null, null, "update_preimage"),
+            RowFactory.create("x", null, "updated", "update_postimage"));
+  }
+
+  @TestTemplate
+  void usesValueSemanticsWhenLineageIsAvailable() {
+    sql(
+        "CREATE TABLE %s (id bigint, data string) USING iceberg "
+            + "TBLPROPERTIES ('format-version'='3', 'write.update.mode'='copy-on-write')",
+        tableName);
+    sql("INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM VALUES (1, 'a'), (2, 'b')", tableName);
+    sql("UPDATE %s SET data = 'a' WHERE id = 1", tableName);
+    long version = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber();
+    assertThat(changes(version, version, true).collectAsList()).isEmpty();
+    sql("UPDATE %s SET data = 'updated' WHERE id = 1", tableName);
+    long updated = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber();
+    assertThat(
+            changes(updated, updated, true)
+                .select("_row_id", "_last_updated_sequence_number")
+                .collectAsList())
+        .containsExactly(RowFactory.create(null, null), RowFactory.create(null, null));
+  }
+
+  @TestTemplate
+  void supportsCompositeKeys() {
+    sql(
+        "CREATE TABLE %s (tenant string, id bigint, data string) USING iceberg "
+            + "TBLPROPERTIES ('format-version'='2', 'write.update.mode'='copy-on-write')",
+        tableName);
+    sql(
+        "INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM VALUES " + "('x', 1, 'a'), ('y', 1, 'b')",
+        tableName);
+    sql("UPDATE %s SET data = 'updated' WHERE tenant = 'x'", tableName);
+    long version = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber();
+    Dataset<Row> result =
+        spark
+            .read()
+            .option("identifier-columns", "tenant, id")
+            .option("computeUpdates", "true")
+            .option("startingVersion", String.valueOf(version))
+            .option("endingVersion", String.valueOf(version))
+            .changes(tableName);
+    assertThat(result.select("tenant", "id", "data", "_change_type").collectAsList())
+        .containsExactlyInAnyOrder(
+            RowFactory.create("x", 1L, "a", "update_preimage"),
+            RowFactory.create("x", 1L, "updated", "update_postimage"));
+  }
+
+  private void createTable() {
+    sql(
+        "CREATE TABLE %s (id bigint, data string) USING iceberg "
+            + "TBLPROPERTIES ('format-version'='2', 'write.update.mode'='copy-on-write')",
+        tableName);
+    sql("INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM VALUES (1, 'a'), (2, 'b')", tableName);
+  }
+
+  private Dataset<Row> changes(long start, long end, boolean computeUpdates) {
+    return spark.sql(
+        String.format(
+            "SELECT * FROM %s CHANGES FROM VERSION %d TO VERSION %d "
+                + "WITH (`identifier-columns` = 'id', computeUpdates = '%s')",
+            tableName, start, end, computeUpdates));
+  }
+}

spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java

@@ -73,6 +73,8 @@
 import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException;
 import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException;
 import org.apache.spark.sql.catalyst.analysis.ViewUtil;
+import org.apache.spark.sql.connector.catalog.Changelog;
+import org.apache.spark.sql.connector.catalog.ChangelogContext;
 import org.apache.spark.sql.connector.catalog.Identifier;
 import org.apache.spark.sql.connector.catalog.NamespaceChange;
 import org.apache.spark.sql.connector.catalog.StagedTable;
@@ -192,6 +194,18 @@ public Table loadTable(Identifier ident, long timestampMicros) throws NoSuchTabl
     return load(ident, TimeTravel.timestampMicros(timestampMicros));
   }
 
+  @Override
+  public Changelog loadChangelog(
+      Identifier ident, ChangelogContext context, CaseInsensitiveStringMap options)
+      throws NoSuchTableException {
+    try {
+      return new SparkChangelogTable(
+          icebergCatalog.loadTable(buildIdentifier(ident)), context, options);
+    } catch (org.apache.iceberg.exceptions.NoSuchTableException e) {
+      throw new NoSuchTableException(ident);
+    }
+  }
+
   @Override
   public boolean tableExists(Identifier ident) {
     if (isPathIdentifier(ident)) {

spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkReadOptions.java

@@ -23,6 +23,9 @@ public class SparkReadOptions {
 
   private SparkReadOptions() {}
 
+  // Comma-separated top-level fields for business-key CDC processing in Spark 4.2
+  public static final String CDC_IDENTIFIER_COLUMNS = "identifier-columns";
+
   // legacy time travel option that is no longer supported
   public static final String LEGACY_SNAPSHOT_ID = "snapshot-id";
 

spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkSessionCatalog.java

@@ -43,6 +43,8 @@
 import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException;
 import org.apache.spark.sql.connector.catalog.CatalogExtension;
 import org.apache.spark.sql.connector.catalog.CatalogPlugin;
+import org.apache.spark.sql.connector.catalog.Changelog;
+import org.apache.spark.sql.connector.catalog.ChangelogContext;
 import org.apache.spark.sql.connector.catalog.FunctionCatalog;
 import org.apache.spark.sql.connector.catalog.Identifier;
 import org.apache.spark.sql.connector.catalog.NamespaceChange;
@@ -224,6 +226,17 @@ public Table loadTable(Identifier ident, long timestamp) throws NoSuchTableExcep
     }
   }
 
+  @Override
+  public Changelog loadChangelog(
+      Identifier ident, ChangelogContext context, CaseInsensitiveStringMap options)
+      throws NoSuchTableException {
+    try {
+      return icebergCatalog.loadChangelog(ident, context, options);
+    } catch (NoSuchTableException e) {
+      return getSessionCatalog().loadChangelog(ident, context, options);
+    }
+  }
+
   @Override
   public void invalidateTable(Identifier ident) {
     // We do not need to check whether the table exists and whether