diff --git a/docs/docs/spark-queries.md b/docs/docs/spark-queries.md index 91f1759f568c..0de94e138f3c 100644 --- a/docs/docs/spark-queries.md +++ b/docs/docs/spark-queries.md @@ -103,6 +103,47 @@ For `string` and `binary` inputs, it keeps the first `width` characters or bytes These functions are especially useful when you want to inspect how Iceberg transforms values or when writing filters for queries and row-level operations that align with partition transforms. +### Native change data capture in Spark 4.2 + +Native CDC reads use Iceberg snapshot sequence numbers as commit versions. The bounds below +include commits 20 through 25. Required snapshots and their data files must still be retained. +The current implementation supports copy-on-write changes in format v2 and later; delete-file +changelog scans are not supported. + +Without row lineage, raw mode returns all added and removed rows, including unchanged rows +copied during a file rewrite: + +```sql +SELECT * FROM prod.db.table CHANGES FROM VERSION 20 TO VERSION 25 +WITH (deduplicationMode = 'none', computeUpdates = 'false'); +``` + +To remove carry-over and compute update images using business keys, enable +[Iceberg SQL extensions](spark-configuration.md#sql-extensions) and explicitly supply +`identifier-columns`: + +```sql +SELECT * FROM prod.db.table CHANGES FROM VERSION 20 TO VERSION 25 +WITH (`identifier-columns` = 'id', + deduplicationMode = 'dropCarryovers', + computeUpdates = 'true'); +``` + +`identifier-columns` is a comma-separated list of top-level primitive column names, such as +`tenant_id,id`. Names follow Spark's case-sensitivity setting. This mode uses the same iterators +as `create_changelog_view`: remove equal DELETE/INSERT carry-over rows, then pair changes by +the business key within each commit. Keys must identify rows unambiguously; ambiguous duplicate +keys can fail update reconstruction. Changing the key produces a delete and a separate insert. +Setting `computeUpdates=false` still removes carry-over but leaves the `delete` and `insert` labels. +This mode supports batch reads with `dropCarryovers`; it rejects streaming, `none`, and `netChanges`. + +Native output contains user columns plus `_change_type`, `_commit_version`, `_commit_timestamp`, +`_row_id`, and `_last_updated_sequence_number`. Update images use `update_preimage` and +`update_postimage`. Business-key mode returns null for both lineage columns, even when the source +has lineage, because it processes rows by values and business keys. It preserves commit versions +and timestamps. Without `identifier-columns`, native post-processing uses Iceberg row lineage +and requires valid lineage in the selected snapshots and files. + ### Time travel Queries with SQL Spark supports time travel in SQL queries using `TIMESTAMP AS OF` or `VERSION AS OF` clauses. The `VERSION AS OF` clause can contain a long snapshot ID or a string branch or tag name. diff --git a/spark/v4.2/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala b/spark/v4.2/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala index 81824e05e92d..7eed382b702e 100644 --- a/spark/v4.2/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala +++ b/spark/v4.2/spark-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala @@ -22,6 +22,7 @@ import org.apache.spark.sql.SparkSessionExtensions import org.apache.spark.sql.catalyst.analysis.CheckViews import org.apache.spark.sql.catalyst.analysis.ResolveBranch import org.apache.spark.sql.catalyst.analysis.ResolveViews +import org.apache.spark.sql.catalyst.analysis.RewriteBusinessKeyChangelog import org.apache.spark.sql.catalyst.optimizer.ReplaceStaticInvoke import org.apache.spark.sql.catalyst.parser.extensions.IcebergSparkSqlExtensionsParser import org.apache.spark.sql.execution.datasources.v2.ExtendedDataSourceV2Strategy @@ -35,6 +36,7 @@ class IcebergSparkSessionExtensions extends (SparkSessionExtensions => Unit) { // analyzer extensions extensions.injectResolutionRule { spark => ResolveViews(spark) } extensions.injectPostHocResolutionRule { spark => ResolveBranch(spark) } + extensions.injectPostHocResolutionRule { spark => RewriteBusinessKeyChangelog(spark) } extensions.injectCheckRule(_ => CheckViews) // optimizer extensions diff --git a/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestBusinessKeyChangelog.java b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestBusinessKeyChangelog.java new file mode 100644 index 000000000000..725ea93753fe --- /dev/null +++ b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestBusinessKeyChangelog.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.extensions; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.sql.Timestamp; +import java.util.List; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.Table; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.TestTemplate; + +class TestBusinessKeyChangelog extends ExtensionsTestBase { + + @AfterEach + void removeTableAndView() { + spark.catalog().dropTempView("business_key_expected"); + sql("DROP TABLE IF EXISTS %s", tableName); + } + + @TestTemplate + void matchesChangelogViewWithoutLineage() { + createTable(); + Table table = validationCatalog.loadTable(tableIdent); + long startSnapshot = table.currentSnapshot().snapshotId(); + sql("UPDATE %s SET data = 'updated' WHERE id = 1", tableName); + table.refresh(); + Snapshot end = table.currentSnapshot(); + Dataset changes = changes(end.sequenceNumber(), end.sequenceNumber(), true); + Timestamp timestamp = new Timestamp(end.timestampMillis()); + assertThat(changes.collectAsList()) + .containsExactlyInAnyOrder( + RowFactory.create( + 1L, "a", null, null, "update_preimage", end.sequenceNumber(), timestamp), + RowFactory.create( + 1L, "updated", null, null, "update_postimage", end.sequenceNumber(), timestamp)); + assertThat(changes.schema().apply("_row_id").nullable()).isTrue(); + assertThat(changes.schema().apply("_last_updated_sequence_number").nullable()).isTrue(); + + sql( + "CALL %s.system.create_changelog_view(table => '%s', " + + "changelog_view => 'business_key_expected', compute_updates => true, " + + "identifier_columns => array('id'), " + + "options => map('start-snapshot-id', '%d', 'end-snapshot-id', '%d'))", + catalogName, tableName, startSnapshot, end.snapshotId()); + List expected = + spark + .sql( + "SELECT id, data, " + + "CASE _change_type WHEN 'UPDATE_BEFORE' THEN 'update_preimage' " + + "WHEN 'UPDATE_AFTER' THEN 'update_postimage' ELSE lower(_change_type) END AS _change_type " + + "FROM business_key_expected") + .collectAsList(); + assertThat(changes.select("id", "data", "_change_type").collectAsList()) + .containsExactlyInAnyOrderElementsOf(expected); + assertThat(changes.filter("data = 'updated'").select("_change_type").collectAsList()) + .containsExactly(RowFactory.create("update_postimage")); + assertThat(changes.select("_change_type").collectAsList()) + .containsExactlyInAnyOrder( + RowFactory.create("update_preimage"), RowFactory.create("update_postimage")); + } + + @TestTemplate + void keepsDeleteInsertWhenUpdateImagesAreDisabled() { + createTable(); + sql("UPDATE %s SET data = 'updated' WHERE id = 1", tableName); + long version = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber(); + assertThat( + changes(version, version, false).select("id", "data", "_change_type").collectAsList()) + .containsExactlyInAnyOrder( + RowFactory.create(1L, "a", "delete"), RowFactory.create(1L, "updated", "insert")); + } + + @TestTemplate + void treatsBusinessKeyChangeAsDeleteAndInsert() { + createTable(); + sql("UPDATE %s SET id = 3 WHERE id = 1", tableName); + long version = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber(); + assertThat(changes(version, version, true).select("id", "data", "_change_type").collectAsList()) + .containsExactlyInAnyOrder( + RowFactory.create(1L, "a", "delete"), RowFactory.create(3L, "a", "insert")); + } + + @TestTemplate + void pairsWithinEachCommitAndRemovesSameValueRewrites() { + createTable(); + sql("UPDATE %s SET data = 'updated' WHERE id = 1", tableName); + long first = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber(); + sql("UPDATE %s SET data = 'a' WHERE id = 1", tableName); + long second = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber(); + assertThat( + changes(first, second, true) + .select("data", "_change_type", "_commit_version") + .collectAsList()) + .containsExactlyInAnyOrder( + RowFactory.create("a", "update_preimage", first), + RowFactory.create("updated", "update_postimage", first), + RowFactory.create("updated", "update_preimage", second), + RowFactory.create("a", "update_postimage", second)); + sql("UPDATE %s SET data = 'a' WHERE id = 1", tableName); + long sameValue = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber(); + assertThat(changes(sameValue, sameValue, true).collectAsList()).isEmpty(); + } + + @TestTemplate + void rejectsAmbiguousBusinessKeys() { + createTable(); + sql("INSERT INTO %s VALUES (1, 'other')", tableName); + sql("UPDATE %s SET data = 'updated' WHERE id = 1", tableName); + long version = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber(); + assertThatThrownBy(() -> changes(version, version, true).collectAsList()) + .hasStackTraceContaining("multiple rows with the same identifier"); + } + + @TestTemplate + void validatesBusinessKeyOptionsAndRejectsStreaming() { + createTable(); + for (String identifiers : List.of("", "missing", "id,id", "data.nested", "_row_id")) { + assertThatThrownBy( + () -> + spark + .read() + .option("identifier-columns", identifiers) + .option("computeUpdates", "true") + .changes(tableName) + .collectAsList()) + .hasStackTraceContaining("identifier column"); + } + for (String mode : List.of("none", "netChanges")) { + assertThatThrownBy( + () -> + spark + .read() + .option("identifier-columns", "id") + .option("deduplicationMode", mode) + .changes(tableName) + .collectAsList()) + .hasStackTraceContaining("requires deduplicationMode=dropCarryovers"); + } + assertThatThrownBy( + () -> + spark + .readStream() + .option("identifier-columns", "id") + .option("computeUpdates", "true") + .changes(tableName) + .explain()) + .hasStackTraceContaining("Business-key CDC currently supports batch reads only"); + } + + @TestTemplate + void resolvesQuotedCompositeKeysAndNullableValues() { + sql( + "CREATE TABLE %s (`Tenant.ID` string, id bigint, data string) USING iceberg " + + "TBLPROPERTIES ('format-version'='2', 'write.update.mode'='copy-on-write')", + tableName); + sql( + "INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM VALUES " + + "('x', CAST(NULL AS BIGINT), CAST(NULL AS STRING)), ('y', 1, 'b')", + tableName); + sql("UPDATE %s SET data = 'updated' WHERE `Tenant.ID` = 'x'", tableName); + long version = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber(); + Dataset result = + spark + .read() + .option("identifier-columns", "tenant.id, ID") + .option("computeUpdates", "true") + .option("startingVersion", String.valueOf(version)) + .option("endingVersion", String.valueOf(version)) + .changes(tableName); + assertThat(result.selectExpr("`Tenant.ID`", "id", "data", "_change_type").collectAsList()) + .containsExactlyInAnyOrder( + RowFactory.create("x", null, null, "update_preimage"), + RowFactory.create("x", null, "updated", "update_postimage")); + } + + @TestTemplate + void usesValueSemanticsWhenLineageIsAvailable() { + sql( + "CREATE TABLE %s (id bigint, data string) USING iceberg " + + "TBLPROPERTIES ('format-version'='3', 'write.update.mode'='copy-on-write')", + tableName); + sql("INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM VALUES (1, 'a'), (2, 'b')", tableName); + sql("UPDATE %s SET data = 'a' WHERE id = 1", tableName); + long version = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber(); + assertThat(changes(version, version, true).collectAsList()).isEmpty(); + sql("UPDATE %s SET data = 'updated' WHERE id = 1", tableName); + long updated = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber(); + assertThat( + changes(updated, updated, true) + .select("_row_id", "_last_updated_sequence_number") + .collectAsList()) + .containsExactly(RowFactory.create(null, null), RowFactory.create(null, null)); + } + + @TestTemplate + void supportsCompositeKeys() { + sql( + "CREATE TABLE %s (tenant string, id bigint, data string) USING iceberg " + + "TBLPROPERTIES ('format-version'='2', 'write.update.mode'='copy-on-write')", + tableName); + sql( + "INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM VALUES " + "('x', 1, 'a'), ('y', 1, 'b')", + tableName); + sql("UPDATE %s SET data = 'updated' WHERE tenant = 'x'", tableName); + long version = validationCatalog.loadTable(tableIdent).currentSnapshot().sequenceNumber(); + Dataset result = + spark + .read() + .option("identifier-columns", "tenant, id") + .option("computeUpdates", "true") + .option("startingVersion", String.valueOf(version)) + .option("endingVersion", String.valueOf(version)) + .changes(tableName); + assertThat(result.select("tenant", "id", "data", "_change_type").collectAsList()) + .containsExactlyInAnyOrder( + RowFactory.create("x", 1L, "a", "update_preimage"), + RowFactory.create("x", 1L, "updated", "update_postimage")); + } + + private void createTable() { + sql( + "CREATE TABLE %s (id bigint, data string) USING iceberg " + + "TBLPROPERTIES ('format-version'='2', 'write.update.mode'='copy-on-write')", + tableName); + sql("INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM VALUES (1, 'a'), (2, 'b')", tableName); + } + + private Dataset changes(long start, long end, boolean computeUpdates) { + return spark.sql( + String.format( + "SELECT * FROM %s CHANGES FROM VERSION %d TO VERSION %d " + + "WITH (`identifier-columns` = 'id', computeUpdates = '%s')", + tableName, start, end, computeUpdates)); + } +} diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index 67c75c3a63d1..eef0e0842bdc 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -73,6 +73,8 @@ import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException; import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException; import org.apache.spark.sql.catalyst.analysis.ViewUtil; +import org.apache.spark.sql.connector.catalog.Changelog; +import org.apache.spark.sql.connector.catalog.ChangelogContext; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.NamespaceChange; import org.apache.spark.sql.connector.catalog.StagedTable; @@ -192,6 +194,18 @@ public Table loadTable(Identifier ident, long timestampMicros) throws NoSuchTabl return load(ident, TimeTravel.timestampMicros(timestampMicros)); } + @Override + public Changelog loadChangelog( + Identifier ident, ChangelogContext context, CaseInsensitiveStringMap options) + throws NoSuchTableException { + try { + return new SparkChangelogTable( + icebergCatalog.loadTable(buildIdentifier(ident)), context, options); + } catch (org.apache.iceberg.exceptions.NoSuchTableException e) { + throw new NoSuchTableException(ident); + } + } + @Override public boolean tableExists(Identifier ident) { if (isPathIdentifier(ident)) { diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkReadOptions.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkReadOptions.java index e6d02d104766..524edb93fe2b 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkReadOptions.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkReadOptions.java @@ -23,6 +23,9 @@ public class SparkReadOptions { private SparkReadOptions() {} + // Comma-separated top-level fields for business-key CDC processing in Spark 4.2 + public static final String CDC_IDENTIFIER_COLUMNS = "identifier-columns"; + // legacy time travel option that is no longer supported public static final String LEGACY_SNAPSHOT_ID = "snapshot-id"; diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkSessionCatalog.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkSessionCatalog.java index d754f84b276d..375df452d57c 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkSessionCatalog.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkSessionCatalog.java @@ -43,6 +43,8 @@ import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException; import org.apache.spark.sql.connector.catalog.CatalogExtension; import org.apache.spark.sql.connector.catalog.CatalogPlugin; +import org.apache.spark.sql.connector.catalog.Changelog; +import org.apache.spark.sql.connector.catalog.ChangelogContext; import org.apache.spark.sql.connector.catalog.FunctionCatalog; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.NamespaceChange; @@ -224,6 +226,17 @@ public Table loadTable(Identifier ident, long timestamp) throws NoSuchTableExcep } } + @Override + public Changelog loadChangelog( + Identifier ident, ChangelogContext context, CaseInsensitiveStringMap options) + throws NoSuchTableException { + try { + return icebergCatalog.loadChangelog(ident, context, options); + } catch (NoSuchTableException e) { + return getSessionCatalog().loadChangelog(ident, context, options); + } + } + @Override public void invalidateTable(Identifier ident) { // We do not need to check whether the table exists and whether diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java index eb8e5e63f430..190f16a29893 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/ChangelogRowReader.java @@ -30,24 +30,37 @@ import org.apache.iceberg.DeleteFile; import org.apache.iceberg.DeletedDataFileScanTask; import org.apache.iceberg.DeletedRowsScanTask; +import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.ScanTaskGroup; import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.spark.SparkSchemaUtil; +import org.apache.iceberg.types.Types; import org.apache.spark.rdd.InputFileBlockHolder; import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.catalyst.expressions.BoundReference; +import org.apache.spark.sql.catalyst.expressions.Expression; import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; import org.apache.spark.sql.catalyst.expressions.JoinedRow; +import org.apache.spark.sql.catalyst.expressions.UnsafeProjection; +import org.apache.spark.sql.connector.catalog.Changelog; import org.apache.spark.sql.connector.read.PartitionReader; import org.apache.spark.unsafe.types.UTF8String; +import scala.jdk.javaapi.CollectionConverters; class ChangelogRowReader extends BaseRowReader implements PartitionReader { + private final SparkChangelogReadMode readMode; + private final UnsafeProjection projection; + ChangelogRowReader(SparkInputPartition partition) { this( partition.table(), @@ -69,9 +82,35 @@ class ChangelogRowReader extends BaseRowReader table, fileIO, taskGroup, - ChangelogUtil.dropChangelogMetadata(expectedSchema), + dataSchema(expectedSchema), caseSensitive, cacheDeleteFilesOnExecutors); + this.readMode = + expectedSchema.findField(SparkChangelogTable.COMMIT_VERSION_ID) != null + ? SparkChangelogReadMode.SPARK_CDC + : SparkChangelogReadMode.ICEBERG_CHANGELOG; + List metadataIds = + readMode.isSparkCdc() + ? List.of( + MetadataColumns.CHANGE_TYPE.fieldId(), + SparkChangelogTable.COMMIT_VERSION_ID, + SparkChangelogTable.COMMIT_TIMESTAMP_ID) + : List.of( + MetadataColumns.CHANGE_TYPE.fieldId(), + MetadataColumns.CHANGE_ORDINAL.fieldId(), + MetadataColumns.COMMIT_SNAPSHOT_ID.fieldId()); + List fieldIds = Lists.newArrayList(); + dataSchema(expectedSchema).columns().forEach(field -> fieldIds.add(field.fieldId())); + fieldIds.addAll(metadataIds); + List expressions = Lists.newArrayList(); + for (Types.NestedField field : expectedSchema.columns()) { + expressions.add( + new BoundReference( + fieldIds.indexOf(field.fieldId()), + SparkSchemaUtil.convert(field.type()), + field.isOptional())); + } + this.projection = UnsafeProjection.create(CollectionConverters.asScala(expressions).toSeq()); } @Override @@ -81,19 +120,49 @@ protected CloseableIterator open(ChangelogScanTask task) { cdcRow.withRight(changelogMetadata(task)); CloseableIterable rows = openChangelogScanTask(task); - CloseableIterable cdcRows = CloseableIterable.transform(rows, cdcRow::withLeft); + CloseableIterable cdcRows = + CloseableIterable.transform(rows, row -> projection.apply(cdcRow.withLeft(row))); return cdcRows.iterator(); } - private static InternalRow changelogMetadata(ChangelogScanTask task) { - InternalRow metadataRow = new GenericInternalRow(3); + private InternalRow changelogMetadata(ChangelogScanTask task) { + Object[] values; + if (readMode.isSparkCdc()) { + Snapshot snapshot = table().snapshot(task.commitSnapshotId()); + Preconditions.checkNotNull( + snapshot, "Cannot find snapshot for changelog task: %s", task.commitSnapshotId()); + values = + new Object[] { + UTF8String.fromString(changeType(task)), + snapshot.sequenceNumber(), + snapshot.timestampMillis() * 1000 + }; + } else { + values = + new Object[] { + UTF8String.fromString(task.operation().name()), + task.changeOrdinal(), + task.commitSnapshotId() + }; + } - metadataRow.update(0, UTF8String.fromString(task.operation().name())); - metadataRow.update(1, task.changeOrdinal()); - metadataRow.update(2, task.commitSnapshotId()); + return new GenericInternalRow(values); + } + + private static Schema dataSchema(Schema expectedSchema) { + return expectedSchema.findField(SparkChangelogTable.COMMIT_VERSION_ID) != null + ? SparkChangelogTable.dropCdcMetadata(expectedSchema) + : ChangelogUtil.dropChangelogMetadata(expectedSchema); + } - return metadataRow; + private static String changeType(ChangelogScanTask task) { + return switch (task.operation()) { + case INSERT -> Changelog.CHANGE_TYPE_INSERT; + case DELETE -> Changelog.CHANGE_TYPE_DELETE; + case UPDATE_BEFORE -> Changelog.CHANGE_TYPE_UPDATE_PREIMAGE; + case UPDATE_AFTER -> Changelog.CHANGE_TYPE_UPDATE_POSTIMAGE; + }; } private CloseableIterable openChangelogScanTask(ChangelogScanTask task) { diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogMicroBatchStream.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogMicroBatchStream.java new file mode 100644 index 000000000000..ee0247d71a36 --- /dev/null +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogMicroBatchStream.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.source; + +import java.util.Collections; +import java.util.List; +import org.apache.iceberg.ChangelogScanTask; +import org.apache.iceberg.ChangelogUtil; +import org.apache.iceberg.IncrementalChangelogScan; +import org.apache.iceberg.MetadataColumns; +import org.apache.iceberg.ScanTaskGroup; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotSummary; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.spark.SparkReadConf; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.PropertyUtil; +import org.apache.iceberg.util.SnapshotUtil; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.sql.connector.read.streaming.Offset; +import org.apache.spark.sql.connector.read.streaming.ReadAllAvailable; +import org.apache.spark.sql.connector.read.streaming.ReadLimit; + +/** + * A minimal changelog stream that advances at Iceberg snapshot boundaries. + * + *

Each planned range contains complete snapshots, ensuring that all rows from a commit remain in + * the same Spark micro-batch. + */ +class SparkChangelogMicroBatchStream extends SparkMicroBatchStreamBase { + + private Broadcast plannedTableBroadcast = null; + private final Schema dataSchema; + private final SparkChangelogRange range; + + SparkChangelogMicroBatchStream( + JavaSparkContext sparkContext, + Table table, + SparkReadConf readConf, + Schema projection, + String checkpointLocation, + SparkChangelogRange range) { + super( + sparkContext, + table, + table::io, + readConf, + projection, + checkpointLocation, + () -> StreamingOffset.START_OFFSET); + this.dataSchema = SparkChangelogTable.dropCdcMetadata(projection); + this.range = range; + } + + @Override + protected StreamingOffset latestStreamingOffset() { + table().refresh(); + Snapshot latest = table().currentSnapshot(); + return latest != null + ? new StreamingOffset(latest.snapshotId(), 0, false) + : StreamingOffset.START_OFFSET; + } + + @Override + public Offset latestOffset(Offset startOffset, ReadLimit limit) { + Preconditions.checkArgument( + startOffset instanceof StreamingOffset, "Invalid start offset: %s", startOffset); + + StreamingOffset latestOffset = (StreamingOffset) latestOffset(); + if (latestOffset.equals(StreamingOffset.START_OFFSET) || latestOffset.equals(startOffset)) { + return null; + } else if (limit instanceof ReadAllAvailable) { + return latestOffset; + } + + BaseSparkMicroBatchPlanner.UnpackedLimits limits = + new BaseSparkMicroBatchPlanner.UnpackedLimits(limit); + List snapshots = snapshotsBetween((StreamingOffset) startOffset, latestOffset); + long rows = 0; + long files = 0; + Snapshot last = null; + for (Snapshot snapshot : snapshots) { + // Admission limits are soft: a single snapshot is never split across batches. + if (last != null + && (rows >= limits.getMaxRows() || files >= limits.getMaxFiles()) + && snapshot.timestampMillis() != last.timestampMillis()) { + break; + } + + last = snapshot; + if (range.includes(snapshot)) { + rows += + PropertyUtil.propertyAsLong(snapshot.summary(), SnapshotSummary.ADDED_RECORDS_PROP, 0) + + PropertyUtil.propertyAsLong( + snapshot.summary(), SnapshotSummary.DELETED_RECORDS_PROP, 0); + files += + PropertyUtil.propertyAsLong(snapshot.summary(), SnapshotSummary.ADDED_FILES_PROP, 0) + + PropertyUtil.propertyAsLong( + snapshot.summary(), SnapshotSummary.DELETED_FILES_PROP, 0); + } + } + + return last != null ? new StreamingOffset(last.snapshotId(), 0, false) : null; + } + + @Override + public ReadLimit getDefaultReadLimit() { + return ReadLimit.compositeLimit( + new ReadLimit[] { + ReadLimit.maxFiles(readConf().maxFilesPerMicroBatch()), + ReadLimit.maxRows(readConf().maxRecordsPerMicroBatch()) + }); + } + + @Override + protected List> planTaskGroups( + StreamingOffset startOffset, StreamingOffset endOffset) { + if (endOffset.equals(StreamingOffset.START_OFFSET) || startOffset.equals(endOffset)) { + return Lists.newArrayList(); + } + + table().refresh(); + if (!startOffset.equals(StreamingOffset.START_OFFSET)) { + Preconditions.checkState( + table().snapshot(startOffset.snapshotId()) != null, + "Cannot load changelog start offset at expired or removed snapshot: %s", + startOffset.snapshotId()); + } + + Preconditions.checkState( + table().snapshot(endOffset.snapshotId()) != null, + "Cannot load changelog end offset at expired or removed snapshot: %s", + endOffset.snapshotId()); + + validateReadSchema(); + if (startOffset.equals(StreamingOffset.START_OFFSET)) { + range.validateVersions(table()); + } + + if (range.requiresPostProcessing()) { + validateCommitTimestamps(startOffset, endOffset); + } + + IncrementalChangelogScan scan = + table() + .newIncrementalChangelogScan() + .caseSensitive(readConf().caseSensitive()) + .project(ChangelogUtil.changelogSchema(dataSchema)) + .option(TableProperties.SPLIT_SIZE, String.valueOf(readConf().splitSize())) + .option(TableProperties.SPLIT_LOOKBACK, String.valueOf(readConf().splitLookback())) + .option( + TableProperties.SPLIT_OPEN_FILE_COST, + String.valueOf(readConf().splitOpenFileCost())); + Long startSnapshotId = + startOffset.equals(StreamingOffset.START_OFFSET) + ? null + : Long.valueOf(startOffset.snapshotId()); + return range.planTasks(table(), scan, startSnapshotId, endOffset.snapshotId()); + } + + private List snapshotsBetween(StreamingOffset start, StreamingOffset end) { + Long startSnapshotId = + start.equals(StreamingOffset.START_OFFSET) ? null : Long.valueOf(start.snapshotId()); + if (startSnapshotId != null) { + Preconditions.checkState( + SnapshotUtil.isParentAncestorOf(table(), end.snapshotId(), startSnapshotId), + "Cannot read CDC: snapshot %s is not an ancestor of %s", + startSnapshotId, + end.snapshotId()); + } + + List snapshots = Lists.newArrayList(); + SnapshotUtil.ancestorsBetween(table(), end.snapshotId(), startSnapshotId) + .forEach(snapshots::add); + Collections.reverse(snapshots); + return snapshots; + } + + private void validateCommitTimestamps(StreamingOffset start, StreamingOffset end) { + Long previousTimestamp = + start.equals(StreamingOffset.START_OFFSET) + ? null + : Long.valueOf(table().snapshot(start.snapshotId()).timestampMillis()); + boolean first = true; + for (Snapshot snapshot : snapshotsBetween(start, end)) { + long timestamp = snapshot.timestampMillis(); + // Spark's zero-delay CDC watermark drops timestamps <= the preceding batch's maximum. + Preconditions.checkState( + previousTimestamp == null + || (first ? timestamp > previousTimestamp : timestamp >= previousTimestamp), + "Cannot stream CDC post-processing at snapshot %s: commit timestamp %s does not advance " + + "past %s. Use batch CDC or deduplicationMode=none with computeUpdates=false", + snapshot.snapshotId(), + timestamp, + previousTimestamp); + previousTimestamp = timestamp; + first = false; + } + } + + private void validateReadSchema() { + for (Types.NestedField field : dataSchema.columns()) { + if (field.fieldId() != MetadataColumns.ROW_ID.fieldId() + && field.fieldId() != MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId()) { + Types.NestedField current = table().schema().findField(field.fieldId()); + Preconditions.checkState( + current != null && current.type().equals(field.type()), + "Cannot continue CDC after an incompatible schema change to field %s", + field.name()); + } + } + } + + @Override + protected Broadcast
tableBroadcast() { + if (plannedTableBroadcast != null) { + plannedTableBroadcast.unpersist(false); + } + + this.plannedTableBroadcast = + sparkContext().broadcast(SerializableTableWithSize.copyOf(table())); + return plannedTableBroadcast; + } + + @Override + protected void stopStream() { + if (plannedTableBroadcast != null) { + plannedTableBroadcast.unpersist(false); + plannedTableBroadcast = null; + } + } +} diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogRange.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogRange.java new file mode 100644 index 000000000000..1a4aa77da900 --- /dev/null +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogRange.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.source; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import org.apache.iceberg.BaseScanTaskGroup; +import org.apache.iceberg.ChangelogScanTask; +import org.apache.iceberg.ContentScanTask; +import org.apache.iceberg.IncrementalChangelogScan; +import org.apache.iceberg.ScanTaskGroup; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.Table; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.util.SnapshotUtil; +import org.apache.spark.sql.connector.catalog.ChangelogContext; +import org.apache.spark.sql.connector.catalog.ChangelogContext.DeduplicationMode; +import org.apache.spark.sql.connector.catalog.ChangelogRange; + +/** Keeps CDC bounds independent of the snapshots available when a stream is created. */ +class SparkChangelogRange { + private final ChangelogContext context; + private final Long startVersion; + private final Long endVersion; + + SparkChangelogRange(ChangelogContext context) { + this.context = context; + if (context.range() instanceof ChangelogRange.VersionRange versions) { + this.startVersion = parseVersion(versions.startingVersion()); + this.endVersion = + versions.endingVersion().map(SparkChangelogRange::parseVersion).orElse(null); + } else { + this.startVersion = null; + this.endVersion = null; + } + } + + boolean requiresPostProcessing() { + return context.deduplicationMode() != DeduplicationMode.NONE || context.computeUpdates(); + } + + boolean includes(Snapshot snapshot) { + ChangelogRange range = context.range(); + if (range instanceof ChangelogRange.VersionRange) { + long version = snapshot.sequenceNumber(); + return (range.startingBoundInclusive() ? version >= startVersion : version > startVersion) + && (endVersion == null + || (range.endingBoundInclusive() ? version <= endVersion : version < endVersion)); + } else if (range instanceof ChangelogRange.TimestampRange timestamps) { + long timestamp = snapshot.timestampMillis() * 1000; + return (timestamps.startingBoundInclusive() + ? timestamp >= timestamps.startingTimestamp() + : timestamp > timestamps.startingTimestamp()) + && (timestamps.endingTimestamp().isEmpty() + || (timestamps.endingBoundInclusive() + ? timestamp <= timestamps.endingTimestamp().get() + : timestamp < timestamps.endingTimestamp().get())); + } + + return true; + } + + void validateVersions(Table table) { + Set versions = Sets.newHashSet(); + for (Snapshot snapshot : SnapshotUtil.currentAncestors(table)) { + versions.add(snapshot.sequenceNumber()); + } + + Preconditions.checkArgument( + startVersion == null || versions.contains(startVersion), + "Cannot find Iceberg snapshot with sequence number: %s", + startVersion); + Preconditions.checkArgument( + endVersion == null || versions.contains(endVersion), + "Cannot find Iceberg snapshot with sequence number: %s", + endVersion); + } + + List> planTasks( + Table table, IncrementalChangelogScan scan, Long startExclusive, long endInclusive) { + Preconditions.checkState( + startExclusive == null + || SnapshotUtil.isParentAncestorOf(table, endInclusive, startExclusive), + "Cannot read CDC: snapshot %s is not an ancestor of %s", + startExclusive, + endInclusive); + List snapshots = Lists.newArrayList(); + for (Snapshot snapshot : SnapshotUtil.ancestorsBetween(table, endInclusive, startExclusive)) { + if (includes(snapshot)) { + snapshots.add(snapshot); + } + } + + if (snapshots.isEmpty()) { + return Collections.emptyList(); + } + + Set snapshotIds = Sets.newHashSet(); + for (Snapshot snapshot : snapshots) { + validateSnapshot(snapshot); + snapshotIds.add(snapshot.snapshotId()); + } + + IncrementalChangelogScan boundedScan = + scan.fromSnapshotInclusive(snapshots.get(snapshots.size() - 1).snapshotId()) + .toSnapshot(snapshots.get(0).snapshotId()); + List> result = Lists.newArrayList(); + try (CloseableIterable> groups = boundedScan.planTasks()) { + for (ScanTaskGroup group : groups) { + List tasks = Lists.newArrayList(); + for (ChangelogScanTask task : group.tasks()) { + // Commit timestamps need not follow snapshot order. Filter whole commits, not rows. + if (snapshotIds.contains(task.commitSnapshotId())) { + validateTaskLineage(task); + tasks.add(task); + } + } + + if (!tasks.isEmpty()) { + result.add(new BaseScanTaskGroup<>(group.groupingKey(), tasks)); + } + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to close Spark CDC scan tasks", e); + } + + return result; + } + + private void validateSnapshot(Snapshot snapshot) { + Preconditions.checkArgument( + snapshot.sequenceNumber() > 0, + "Cannot read Spark CDC from snapshot %s without a commit sequence number", + snapshot.snapshotId()); + Preconditions.checkArgument( + !requiresPostProcessing() || snapshot.firstRowId() != null, + "Cannot read Spark CDC from snapshot %s without row lineage", + snapshot.snapshotId()); + } + + private void validateTaskLineage(ChangelogScanTask task) { + Preconditions.checkArgument( + !requiresPostProcessing() + || (task instanceof ContentScanTask contentTask + && contentTask.file().firstRowId() != null), + "Cannot read Spark CDC from a file without row lineage in snapshot %s", + task.commitSnapshotId()); + } + + private static long parseVersion(String version) { + try { + return Long.parseLong(version); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid Iceberg snapshot sequence number: " + version, e); + } + } + + @Override + public boolean equals(Object other) { + return other instanceof SparkChangelogRange that && context.equals(that.context); + } + + @Override + public int hashCode() { + return context.hashCode(); + } + + @Override + public String toString() { + return context.toString(); + } +} diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogReadMode.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogReadMode.java new file mode 100644 index 000000000000..6347b8a5b438 --- /dev/null +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogReadMode.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.source; + +enum SparkChangelogReadMode { + ICEBERG_CHANGELOG, + SPARK_CDC; + + boolean isSparkCdc() { + return this != ICEBERG_CHANGELOG; + } +} diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogScan.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogScan.java index 57ccf92b9651..83f813524c44 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogScan.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogScan.java @@ -28,6 +28,7 @@ import org.apache.iceberg.IncrementalChangelogScan; import org.apache.iceberg.ScanTaskGroup; import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.io.CloseableIterable; @@ -42,6 +43,7 @@ import org.apache.spark.sql.connector.read.Scan; import org.apache.spark.sql.connector.read.Statistics; import org.apache.spark.sql.connector.read.SupportsReportStatistics; +import org.apache.spark.sql.connector.read.streaming.MicroBatchStream; import org.apache.spark.sql.types.StructType; class SparkChangelogScan implements Scan, SupportsReportStatistics { @@ -56,6 +58,8 @@ class SparkChangelogScan implements Scan, SupportsReportStatistics { private final List filters; private final Long startSnapshotId; private final Long endSnapshotId; + private final SparkChangelogRange cdcRange; + private final Long batchEndSnapshotId; // lazy variables private List> taskGroups = null; @@ -68,7 +72,17 @@ class SparkChangelogScan implements Scan, SupportsReportStatistics { SparkReadConf readConf, Schema projection, List filters) { + this(spark, table, scan, readConf, projection, filters, null); + } + SparkChangelogScan( + SparkSession spark, + Table table, + IncrementalChangelogScan scan, + SparkReadConf readConf, + Schema projection, + List filters, + SparkChangelogRange cdcRange) { SparkSchemaUtil.validateMetadataColumnReferences(table.schema(), projection); this.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); @@ -79,6 +93,12 @@ class SparkChangelogScan implements Scan, SupportsReportStatistics { this.filters = filters != null ? filters : Collections.emptyList(); this.startSnapshotId = readConf.startSnapshotId(); this.endSnapshotId = readConf.endSnapshotId(); + this.cdcRange = cdcRange; + Snapshot currentSnapshot = table.currentSnapshot(); + this.batchEndSnapshotId = + cdcRange != null && currentSnapshot != null + ? Long.valueOf(currentSnapshot.snapshotId()) + : null; if (scan == null) { this.taskGroups = Collections.emptyList(); } @@ -113,7 +133,25 @@ public Batch toBatch() { hashCode()); } + @Override + public MicroBatchStream toMicroBatchStream(String checkpointLocation) { + if (cdcRange == null) { + throw new UnsupportedOperationException("Changelog streaming is only supported through CDC"); + } + + return new SparkChangelogMicroBatchStream( + sparkContext, table, readConf, projection, checkpointLocation, cdcRange); + } + private List> taskGroups() { + if (taskGroups == null && cdcRange != null) { + cdcRange.validateVersions(table); + this.taskGroups = + batchEndSnapshotId == null + ? Collections.emptyList() + : cdcRange.planTasks(table, scan, null, batchEndSnapshotId); + } + if (taskGroups == null) { try (CloseableIterable> groups = scan.planTasks()) { this.taskGroups = Lists.newArrayList(groups); @@ -129,11 +167,12 @@ private List> taskGroups() { public String description() { return String.format( Locale.ROOT, - "IcebergChangelogScan(table=%s, fromSnapshotId=%d, toSnapshotId=%d, filters=%s)", + "IcebergChangelogScan(table=%s, fromSnapshotId=%d, toSnapshotId=%d, filters=%s, cdcRange=%s)", table, startSnapshotId, endSnapshotId, - filtersDesc()); + filtersDesc(), + cdcRange); } @Override @@ -152,13 +191,22 @@ public boolean equals(Object o) { && readSchema().equals(that.readSchema()) // compare Spark schemas to ignore field IDs && filtersDesc().equals(that.filtersDesc()) && Objects.equals(startSnapshotId, that.startSnapshotId) - && Objects.equals(endSnapshotId, that.endSnapshotId); + && Objects.equals(endSnapshotId, that.endSnapshotId) + && Objects.equals(cdcRange, that.cdcRange) + && Objects.equals(batchEndSnapshotId, that.batchEndSnapshotId); } @Override public int hashCode() { return Objects.hash( - table.name(), table.uuid(), readSchema(), filtersDesc(), startSnapshotId, endSnapshotId); + table.name(), + table.uuid(), + readSchema(), + filtersDesc(), + startSnapshotId, + endSnapshotId, + cdcRange, + batchEndSnapshotId); } private String filtersDesc() { diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogScanBuilder.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogScanBuilder.java index 43b8a36507db..5c11be8e6ea1 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogScanBuilder.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogScanBuilder.java @@ -18,6 +18,7 @@ */ package org.apache.iceberg.spark.source; +import org.apache.iceberg.ChangelogUtil; import org.apache.iceberg.IncrementalChangelogScan; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; @@ -26,22 +27,52 @@ import org.apache.iceberg.spark.SparkReadOptions; import org.apache.iceberg.util.SnapshotUtil; import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.connector.expressions.filter.Predicate; import org.apache.spark.sql.connector.read.Scan; import org.apache.spark.sql.connector.read.SupportsPushDownLimit; import org.apache.spark.sql.connector.read.SupportsPushDownRequiredColumns; import org.apache.spark.sql.connector.read.SupportsPushDownV2Filters; +import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; public class SparkChangelogScanBuilder extends BaseSparkScanBuilder implements SupportsPushDownV2Filters, SupportsPushDownRequiredColumns, SupportsPushDownLimit { + private final SparkChangelogRange cdcRange; + SparkChangelogScanBuilder( SparkSession spark, Table table, Schema schema, CaseInsensitiveStringMap options) { + this(spark, table, schema, options, null); + } + + SparkChangelogScanBuilder( + SparkSession spark, + Table table, + Schema schema, + CaseInsensitiveStringMap options, + SparkChangelogRange cdcRange) { super(spark, table, schema, options); + this.cdcRange = cdcRange; + } + + @Override + public void pruneColumns(StructType requestedType) { + if (cdcRange == null) { + super.pruneColumns(requestedType); + } + } + + @Override + public Predicate[] pushPredicates(Predicate[] predicates) { + return cdcRange != null ? predicates : super.pushPredicates(predicates); } @Override public Scan build() { + if (cdcRange != null) { + return buildCdcScan(); + } + Long startSnapshotId = readConf().startSnapshotId(); Long endSnapshotId = readConf().endSnapshotId(); Long startTimestamp = readConf().startTimestamp(); @@ -84,6 +115,23 @@ public Scan build() { return new SparkChangelogScan(spark(), table(), scan, readConf(), projection, filters()); } + private SparkChangelogScan buildCdcScan() { + Preconditions.checkArgument( + readConf().startSnapshotId() == null + && readConf().endSnapshotId() == null + && readConf().startTimestamp() == null + && readConf().endTimestamp() == null, + "Use Spark CDC startingVersion/endingVersion or startingTimestamp/endingTimestamp options"); + Schema readProjection = projectionWithMetadataColumns(); + IncrementalChangelogScan scan = + buildIcebergScan( + ChangelogUtil.changelogSchema(SparkChangelogTable.dropCdcMetadata(readProjection)), + null, + null); + return new SparkChangelogScan( + spark(), table(), scan, readConf(), readProjection, filters(), cdcRange); + } + private IncrementalChangelogScan buildIcebergScan( Schema projection, Long startSnapshotId, Long endSnapshotId) { IncrementalChangelogScan scan = diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogTable.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogTable.java index bdafca27fbb8..2959905b4b3c 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogTable.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkChangelogTable.java @@ -18,38 +18,197 @@ */ package org.apache.iceberg.spark.source; +import java.util.Arrays; +import java.util.List; import java.util.Set; import org.apache.iceberg.ChangelogUtil; +import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; +import org.apache.iceberg.TableUtil; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.base.Splitter; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.spark.Spark3Util; +import org.apache.iceberg.spark.SparkReadOptions; import org.apache.iceberg.spark.SparkSchemaUtil; +import org.apache.iceberg.spark.SparkUtil; +import org.apache.iceberg.types.TypeUtil; +import org.apache.iceberg.types.Types; import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.connector.catalog.Changelog; +import org.apache.spark.sql.connector.catalog.ChangelogContext; +import org.apache.spark.sql.connector.catalog.ChangelogContext.DeduplicationMode; +import org.apache.spark.sql.connector.catalog.Column; import org.apache.spark.sql.connector.catalog.MetadataColumn; import org.apache.spark.sql.connector.catalog.SupportsMetadataColumns; import org.apache.spark.sql.connector.catalog.SupportsRead; import org.apache.spark.sql.connector.catalog.TableCapability; +import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.connector.read.ScanBuilder; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; +/** + * Iceberg changelog relation used both as {@code table.changes} and Spark 4.2 {@link Changelog}. + * + *

The Table API keeps Iceberg's changelog columns ({@code _change_type}, {@code + * _change_ordinal}, {@code _commit_snapshot_id}). The Changelog API maps those onto Spark CDC + * columns ({@code _change_type}, {@code _commit_version}, {@code _commit_timestamp}). + */ public class SparkChangelogTable - implements org.apache.spark.sql.connector.catalog.Table, SupportsRead, SupportsMetadataColumns { + implements org.apache.spark.sql.connector.catalog.Table, + SupportsRead, + SupportsMetadataColumns, + Changelog { public static final String TABLE_NAME = "changes"; + static final int COMMIT_VERSION_ID = Integer.MAX_VALUE - 109; + static final int COMMIT_TIMESTAMP_ID = Integer.MAX_VALUE - 110; + + static final String COMMIT_VERSION = "_commit_version"; + static final String COMMIT_TIMESTAMP = "_commit_timestamp"; + private static final Set CAPABILITIES = - ImmutableSet.of(TableCapability.BATCH_READ); + ImmutableSet.of(TableCapability.BATCH_READ, TableCapability.MICRO_BATCH_READ); + + private static final Types.NestedField ROW_ID_FIELD = + Types.NestedField.required( + MetadataColumns.ROW_ID.fieldId(), + MetadataColumns.ROW_ID.name(), + MetadataColumns.ROW_ID.type(), + MetadataColumns.ROW_ID.doc()); + private static final Types.NestedField ROW_VERSION_FIELD = + Types.NestedField.required( + MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId(), + MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.name(), + MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.type(), + MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.doc()); + + private static final Types.NestedField COMMIT_VERSION_FIELD = + Types.NestedField.required( + COMMIT_VERSION_ID, + COMMIT_VERSION, + Types.LongType.get(), + "Iceberg snapshot sequence number"); + private static final Types.NestedField COMMIT_TIMESTAMP_FIELD = + Types.NestedField.required( + COMMIT_TIMESTAMP_ID, + COMMIT_TIMESTAMP, + Types.TimestampType.withZone(), + "Iceberg snapshot commit timestamp"); private final Table table; - private final Schema schema; + private final ChangelogContext context; + private final String[] identifierColumns; + private final Schema icebergChangelogSchema; + private final SparkChangelogRange cdcRange; + private final Schema sparkCdcSchema; + private final Column[] sparkCdcColumns; private SparkSession lazySpark = null; private StructType lazySparkSchema = null; public SparkChangelogTable(Table table) { + this(table, null); + } + + public SparkChangelogTable(Table table, ChangelogContext context) { + this(table, context, CaseInsensitiveStringMap.empty()); + } + + public SparkChangelogTable( + Table table, ChangelogContext context, CaseInsensitiveStringMap options) { this.table = table; - this.schema = ChangelogUtil.changelogSchema(table.schema()); + this.context = context; + this.identifierColumns = identifierColumns(table.schema(), options); + Preconditions.checkArgument( + identifierColumns.length == 0 + || (context != null + && context.deduplicationMode() == DeduplicationMode.DROP_CARRYOVERS), + "Business-key CDC requires deduplicationMode=dropCarryovers"); + this.icebergChangelogSchema = ChangelogUtil.changelogSchema(table.schema()); + this.cdcRange = context != null ? new SparkChangelogRange(context) : null; + Preconditions.checkArgument( + cdcRange == null || TableUtil.formatVersion(table) >= 2, + "Spark CDC requires format version 2 or later for commit sequence numbers"); + Preconditions.checkArgument( + cdcRange == null + || !cdcRange.requiresPostProcessing() + || identifierColumns.length > 0 + || TableUtil.supportsRowLineage(table), + "Spark CDC post-processing requires row lineage. " + + "Use deduplicationMode=none with computeUpdates=false for raw changes"); + this.sparkCdcSchema = + cdcRange != null + ? TypeUtil.join( + cdcDataSchema( + table, identifierColumns.length == 0 && cdcRange.requiresPostProcessing()), + new Schema( + MetadataColumns.CHANGE_TYPE, COMMIT_VERSION_FIELD, COMMIT_TIMESTAMP_FIELD)) + : null; + this.sparkCdcColumns = cdcRange != null ? toColumns(sparkCdcSchema) : null; + } + + private static String[] identifierColumns(Schema schema, CaseInsensitiveStringMap options) { + String value = options.get(SparkReadOptions.CDC_IDENTIFIER_COLUMNS); + if (value == null) { + return new String[0]; + } + + List names = Splitter.on(',').trimResults().splitToList(value); + String[] resolved = new String[names.size()]; + boolean caseSensitive = SparkUtil.caseSensitive(SparkSession.active()); + Set seen = Sets.newHashSet(); + for (int index = 0; index < names.size(); index++) { + String name = names.get(index); + Types.NestedField field = + caseSensitive + ? schema.asStruct().field(name) + : schema.asStruct().caseInsensitiveField(name); + Preconditions.checkArgument( + field != null && field.type().isPrimitiveType(), + "CDC identifier column must be an existing top-level primitive field: %s", + name); + Preconditions.checkArgument( + seen.add(field.name()), "Duplicate CDC identifier column: %s", name); + resolved[index] = field.name(); + } + + return resolved; + } + + /** Identifier fields consumed by the Iceberg CDC resolution rule. */ + public String[] identifierColumns() { + return identifierColumns.clone(); + } + + /** Raw input for the Iceberg business-key CDC resolution rule. */ + public SparkChangelogTable rawChangelog() { + Preconditions.checkState(identifierColumns.length > 0, "Not a business-key changelog"); + return new SparkChangelogTable( + table, new ChangelogContext(context.range(), DeduplicationMode.NONE, false)); + } + + static Schema cdcDataSchema(Table table) { + return cdcDataSchema(table, true); + } + + private static Schema cdcDataSchema(Table table, boolean requiresRowLineage) { + // Raw CDC can expose older files whose lineage is unknown without inventing an identity. + Schema lineageSchema = + requiresRowLineage + ? new Schema(ROW_ID_FIELD, ROW_VERSION_FIELD) + : new Schema(MetadataColumns.ROW_ID, MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER); + return TypeUtil.join(table.schema(), lineageSchema); + } + + static Schema dropCdcMetadata(Schema schema) { + return TypeUtil.selectNot( + schema, + Set.of(MetadataColumns.CHANGE_TYPE.fieldId(), COMMIT_VERSION_ID, COMMIT_TIMESTAMP_ID)); } @Override @@ -60,20 +219,63 @@ public String name() { @Override public StructType schema() { if (lazySparkSchema == null) { + Schema schema = cdcRange != null ? sparkCdcSchema : icebergChangelogSchema; this.lazySparkSchema = SparkSchemaUtil.convert(schema); } return lazySparkSchema; } + @Override + public Column[] columns() { + return cdcRange != null ? sparkCdcColumns : toColumns(icebergChangelogSchema); + } + @Override public Set capabilities() { - return CAPABILITIES; + return cdcRange != null ? CAPABILITIES : ImmutableSet.of(TableCapability.BATCH_READ); + } + + @Override + public boolean containsCarryoverRows() { + return identifierColumns.length == 0; + } + + @Override + public boolean containsIntermediateChanges() { + return true; + } + + @Override + public boolean representsUpdateAsDeleteAndInsert() { + return identifierColumns.length == 0 || !context.computeUpdates(); + } + + @Override + public NamedReference[] rowId() { + if (identifierColumns.length > 0) { + return Arrays.stream(identifierColumns) + .map(name -> Spark3Util.toNamedReference("`" + name.replace("`", "``") + "`")) + .toArray(NamedReference[]::new); + } + + return new NamedReference[] {Spark3Util.toNamedReference(MetadataColumns.ROW_ID.name())}; + } + + @Override + public NamedReference rowVersion() { + return Spark3Util.toNamedReference(MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.name()); } @Override public ScanBuilder newScanBuilder(CaseInsensitiveStringMap options) { - return new SparkChangelogScanBuilder(spark(), table, schema, options); + Preconditions.checkState( + identifierColumns.length == 0, "Business-key CDC requires IcebergSparkSessionExtensions"); + if (cdcRange == null) { + return new SparkChangelogScanBuilder(spark(), table, icebergChangelogSchema, options); + } + + return new SparkChangelogScanBuilder(spark(), table, sparkCdcSchema, options, cdcRange); } private SparkSession spark() { @@ -94,4 +296,11 @@ public MetadataColumn[] metadataColumns() { SparkMetadataColumns.IS_DELETED, }; } + + private static Column[] toColumns(Schema schema) { + StructType sparkSchema = SparkSchemaUtil.convert(schema); + return Arrays.stream(sparkSchema.fields()) + .map(field -> Column.create(field.name(), field.dataType(), field.nullable())) + .toArray(Column[]::new); + } } diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java index 7adf3c633cd0..18bcd1ef07db 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java @@ -18,67 +18,36 @@ */ package org.apache.iceberg.spark.source; -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; import java.util.List; import java.util.function.Supplier; -import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CombinedScanTask; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.Schema; -import org.apache.iceberg.SchemaParser; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; -import org.apache.iceberg.hadoop.HadoopFileIO; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.FileIO; -import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.io.OutputFile; -import org.apache.iceberg.relocated.com.google.common.base.Joiner; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.spark.SparkReadConf; -import org.apache.iceberg.types.Types; import org.apache.iceberg.util.TableScanUtil; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.broadcast.Broadcast; -import org.apache.spark.sql.connector.read.InputPartition; -import org.apache.spark.sql.connector.read.PartitionReaderFactory; -import org.apache.spark.sql.connector.read.streaming.MicroBatchStream; import org.apache.spark.sql.connector.read.streaming.Offset; import org.apache.spark.sql.connector.read.streaming.ReadLimit; -import org.apache.spark.sql.connector.read.streaming.SupportsTriggerAvailableNow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class SparkMicroBatchStream implements MicroBatchStream, SupportsTriggerAvailableNow { - private static final Joiner SLASH = Joiner.on("/"); +public class SparkMicroBatchStream extends SparkMicroBatchStreamBase { private static final Logger LOG = LoggerFactory.getLogger(SparkMicroBatchStream.class); - private static final Types.StructType EMPTY_GROUPING_KEY_TYPE = Types.StructType.of(); - private final Table table; - private final Supplier fileIO; - private final SparkReadConf readConf; - private final boolean caseSensitive; - private final String projection; - private final Broadcast

tableBroadcast; - private final Broadcast fileIOBroadcast; private final long splitSize; private final int splitLookback; private final long splitOpenFileCost; private final boolean localityPreferred; - private final StreamingOffset initialOffset; private final long fromTimestamp; private final int maxFilesPerMicroBatch; private final int maxRecordsPerMicroBatch; - private final boolean cacheDeleteFilesOnExecutors; private SparkMicroBatchPlanner planner; - private StreamingOffset lastOffsetForTriggerAvailableNow; SparkMicroBatchStream( JavaSparkContext sparkContext, @@ -87,61 +56,52 @@ public class SparkMicroBatchStream implements MicroBatchStream, SupportsTriggerA SparkReadConf readConf, Schema projection, String checkpointLocation) { - this.table = table; - this.fileIO = fileIO; - this.readConf = readConf; - this.caseSensitive = readConf.caseSensitive(); - this.projection = SchemaParser.toJson(projection); + super( + sparkContext, + table, + fileIO, + readConf, + projection, + checkpointLocation, + () -> { + table.refresh(); + return MicroBatchUtils.determineStartingOffset(table, readConf.streamFromTimestamp()); + }); this.localityPreferred = readConf.localityEnabled(); - this.tableBroadcast = sparkContext.broadcast(SerializableTableWithSize.copyOf(table)); - this.fileIOBroadcast = sparkContext.broadcast(SerializableFileIOWithSize.wrap(fileIO.get())); this.splitSize = readConf.splitSize(); this.splitLookback = readConf.splitLookback(); this.splitOpenFileCost = readConf.splitOpenFileCost(); this.fromTimestamp = readConf.streamFromTimestamp(); this.maxFilesPerMicroBatch = readConf.maxFilesPerMicroBatch(); this.maxRecordsPerMicroBatch = readConf.maxRecordsPerMicroBatch(); - this.cacheDeleteFilesOnExecutors = readConf.cacheDeleteFilesOnExecutors(); - - InitialOffsetStore initialOffsetStore = - new InitialOffsetStore( - table, checkpointLocation, fromTimestamp, sparkContext.hadoopConfiguration()); - this.initialOffset = initialOffsetStore.initialOffset(); } @Override - public Offset latestOffset() { - table.refresh(); - if (table.currentSnapshot() == null) { + protected StreamingOffset latestStreamingOffset() { + table().refresh(); + if (table().currentSnapshot() == null) { return StreamingOffset.START_OFFSET; } - if (table.currentSnapshot().timestampMillis() < fromTimestamp) { + if (table().currentSnapshot().timestampMillis() < fromTimestamp) { return StreamingOffset.START_OFFSET; } - Snapshot latestSnapshot = table.currentSnapshot(); + Snapshot latestSnapshot = table().currentSnapshot(); return new StreamingOffset( - latestSnapshot.snapshotId(), MicroBatchUtils.addedFilesCount(table, latestSnapshot), false); + latestSnapshot.snapshotId(), + MicroBatchUtils.addedFilesCount(table(), latestSnapshot), + false); } @Override - public InputPartition[] planInputPartitions(Offset start, Offset end) { - Preconditions.checkArgument( - end instanceof StreamingOffset, "Invalid end offset: %s is not a StreamingOffset", end); - Preconditions.checkArgument( - start instanceof StreamingOffset, - "Invalid start offset: %s is not a StreamingOffset", - start); - - if (end.equals(StreamingOffset.START_OFFSET)) { - return new InputPartition[0]; + protected List planTaskGroups( + StreamingOffset startOffset, StreamingOffset endOffset) { + if (endOffset.equals(StreamingOffset.START_OFFSET)) { + return Lists.newArrayList(); } - StreamingOffset endOffset = (StreamingOffset) end; - StreamingOffset startOffset = (StreamingOffset) start; - // Initialize planner if not already done (for resume scenarios) if (planner == null) { initializePlanner(startOffset, endOffset); @@ -151,68 +111,30 @@ public InputPartition[] planInputPartitions(Offset start, Offset end) { CloseableIterable splitTasks = TableScanUtil.splitFiles(CloseableIterable.withNoopClose(fileScanTasks), splitSize); - List combinedScanTasks = - Lists.newArrayList( - TableScanUtil.planTasks(splitTasks, splitSize, splitLookback, splitOpenFileCost)); - String[][] locations = computePreferredLocations(combinedScanTasks); - - InputPartition[] partitions = new InputPartition[combinedScanTasks.size()]; - - for (int index = 0; index < combinedScanTasks.size(); index++) { - partitions[index] = - new SparkInputPartition( - EMPTY_GROUPING_KEY_TYPE, - combinedScanTasks.get(index), - tableBroadcast, - fileIOBroadcast, - projection, - caseSensitive, - locations != null ? locations[index] : SparkPlanningUtil.NO_LOCATION_PREFERENCE, - cacheDeleteFilesOnExecutors); - } - - return partitions; - } - - private String[][] computePreferredLocations(List taskGroups) { - return localityPreferred - ? SparkPlanningUtil.fetchBlockLocations(fileIO.get(), taskGroups) - : null; - } - - @Override - public PartitionReaderFactory createReaderFactory() { - return new SparkRowReaderFactory(); - } - - @Override - public Offset initialOffset() { - return initialOffset; + return Lists.newArrayList( + TableScanUtil.planTasks(splitTasks, splitSize, splitLookback, splitOpenFileCost)); } @Override - public Offset deserializeOffset(String json) { - return StreamingOffset.fromJson(json); + protected boolean localityPreferred() { + return localityPreferred; } @Override - public void commit(Offset end) {} - - @Override - public void stop() { + protected void stopStream() { if (planner != null) { planner.stop(); } } private void initializePlanner(StreamingOffset startOffset, StreamingOffset endOffset) { - if (readConf.asyncMicroBatchPlanningEnabled()) { + if (readConf().asyncMicroBatchPlanningEnabled()) { this.planner = new AsyncSparkMicroBatchPlanner( - table, readConf, startOffset, endOffset, lastOffsetForTriggerAvailableNow); + table(), readConf(), startOffset, endOffset, lastOffsetForTriggerAvailableNow()); } else { this.planner = - new SyncSparkMicroBatchPlanner(table, readConf, lastOffsetForTriggerAvailableNow); + new SyncSparkMicroBatchPlanner(table(), readConf(), lastOffsetForTriggerAvailableNow()); } } @@ -249,69 +171,22 @@ public ReadLimit getDefaultReadLimit() { } @Override - public void prepareForTriggerAvailableNow() { + protected StreamingOffset availableNowEndOffset() { LOG.info("The streaming query reports to use Trigger.AvailableNow"); - lastOffsetForTriggerAvailableNow = - (StreamingOffset) latestOffset(initialOffset, ReadLimit.allAvailable()); + StreamingOffset endOffset = + (StreamingOffset) latestOffset(initialStreamingOffset(), ReadLimit.allAvailable()); - LOG.info("lastOffset for Trigger.AvailableNow is {}", lastOffsetForTriggerAvailableNow.json()); + LOG.info("lastOffset for Trigger.AvailableNow is {}", endOffset.json()); + return endOffset; + } + @Override + protected void availableNowPrepared() { // Reset planner so it gets recreated with the cap on next call if (planner != null) { planner.stop(); planner = null; } } - - private static class InitialOffsetStore { - private final Table table; - private final FileIO io; - private final String initialOffsetLocation; - private final long fromTimestamp; - - InitialOffsetStore( - Table table, String checkpointLocation, long fromTimestamp, Configuration conf) { - this.table = table; - this.io = new HadoopFileIO(conf); - this.initialOffsetLocation = SLASH.join(checkpointLocation, "offsets/0"); - this.fromTimestamp = fromTimestamp; - } - - public StreamingOffset initialOffset() { - InputFile inputFile = io.newInputFile(initialOffsetLocation); - if (inputFile.exists()) { - return readOffset(inputFile); - } - - table.refresh(); - StreamingOffset offset = MicroBatchUtils.determineStartingOffset(table, fromTimestamp); - - OutputFile outputFile = io.newOutputFile(initialOffsetLocation); - writeOffset(offset, outputFile); - - return offset; - } - - private void writeOffset(StreamingOffset offset, OutputFile file) { - try (OutputStream outputStream = file.create()) { - BufferedWriter writer = - new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8)); - writer.write(offset.json()); - writer.flush(); - } catch (IOException ioException) { - throw new UncheckedIOException( - String.format("Failed writing offset to: %s", initialOffsetLocation), ioException); - } - } - - private StreamingOffset readOffset(InputFile file) { - try (InputStream in = file.newStream()) { - return StreamingOffset.fromJson(in); - } catch (IOException ioException) { - throw new UncheckedIOException( - String.format("Failed reading offset from: %s", initialOffsetLocation), ioException); - } - } - } } diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStreamBase.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStreamBase.java new file mode 100644 index 000000000000..a063116d52ea --- /dev/null +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStreamBase.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.source; + +import java.util.List; +import java.util.function.Supplier; +import org.apache.iceberg.ScanTaskGroup; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SchemaParser; +import org.apache.iceberg.Table; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.spark.SparkReadConf; +import org.apache.iceberg.types.Types; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.sql.connector.read.InputPartition; +import org.apache.spark.sql.connector.read.PartitionReaderFactory; +import org.apache.spark.sql.connector.read.streaming.MicroBatchStream; +import org.apache.spark.sql.connector.read.streaming.Offset; +import org.apache.spark.sql.connector.read.streaming.SupportsTriggerAvailableNow; + +abstract class SparkMicroBatchStreamBase implements MicroBatchStream, SupportsTriggerAvailableNow { + + private static final Types.StructType EMPTY_GROUPING_KEY_TYPE = Types.StructType.of(); + + private final JavaSparkContext sparkContext; + private final Table table; + private final Supplier fileIO; + private final SparkReadConf readConf; + private final String projection; + private Broadcast
tableBroadcast = null; + private final Broadcast fileIOBroadcast; + private final StreamingOffset initialOffset; + private StreamingOffset lastOffsetForTriggerAvailableNow = null; + + SparkMicroBatchStreamBase( + JavaSparkContext sparkContext, + Table table, + Supplier fileIO, + SparkReadConf readConf, + Schema projection, + String checkpointLocation, + Supplier initialOffsetSupplier) { + this.sparkContext = sparkContext; + this.table = table; + this.fileIO = fileIO; + this.readConf = readConf; + this.projection = SchemaParser.toJson(projection); + this.fileIOBroadcast = sparkContext.broadcast(SerializableFileIOWithSize.wrap(fileIO.get())); + this.initialOffset = + new StreamingInitialOffsetStore( + checkpointLocation, sparkContext.hadoopConfiguration(), initialOffsetSupplier) + .initialOffset(); + } + + protected final Table table() { + return table; + } + + protected final JavaSparkContext sparkContext() { + return sparkContext; + } + + protected final FileIO fileIO() { + return fileIO.get(); + } + + protected final SparkReadConf readConf() { + return readConf; + } + + protected final StreamingOffset initialStreamingOffset() { + return initialOffset; + } + + protected final StreamingOffset lastOffsetForTriggerAvailableNow() { + return lastOffsetForTriggerAvailableNow; + } + + @Override + public final Offset latestOffset() { + return lastOffsetForTriggerAvailableNow != null + ? lastOffsetForTriggerAvailableNow + : latestStreamingOffset(); + } + + protected abstract StreamingOffset latestStreamingOffset(); + + @Override + public final InputPartition[] planInputPartitions(Offset start, Offset end) { + Preconditions.checkArgument( + start instanceof StreamingOffset, "Invalid start offset: %s", start); + Preconditions.checkArgument(end instanceof StreamingOffset, "Invalid end offset: %s", end); + + List> taskGroups = + planTaskGroups((StreamingOffset) start, (StreamingOffset) end); + if (taskGroups.isEmpty()) { + return new InputPartition[0]; + } + + String[][] locations = + localityPreferred() ? SparkPlanningUtil.fetchBlockLocations(fileIO(), taskGroups) : null; + Broadcast
currentTableBroadcast = tableBroadcast(); + InputPartition[] partitions = new InputPartition[taskGroups.size()]; + for (int index = 0; index < taskGroups.size(); index++) { + partitions[index] = + new SparkInputPartition( + EMPTY_GROUPING_KEY_TYPE, + taskGroups.get(index), + currentTableBroadcast, + fileIOBroadcast, + projection, + readConf.caseSensitive(), + locations != null ? locations[index] : SparkPlanningUtil.NO_LOCATION_PREFERENCE, + readConf.cacheDeleteFilesOnExecutors()); + } + + return partitions; + } + + protected abstract List> planTaskGroups( + StreamingOffset startOffset, StreamingOffset endOffset); + + protected boolean localityPreferred() { + return false; + } + + protected Broadcast
tableBroadcast() { + if (tableBroadcast == null) { + this.tableBroadcast = sparkContext.broadcast(SerializableTableWithSize.copyOf(table)); + } + + return tableBroadcast; + } + + @Override + public final PartitionReaderFactory createReaderFactory() { + return new SparkRowReaderFactory(); + } + + @Override + public final Offset initialOffset() { + return initialOffset; + } + + @Override + public final Offset deserializeOffset(String json) { + return StreamingOffset.fromJson(json); + } + + @Override + public final void commit(Offset end) {} + + @Override + public final void stop() { + stopStream(); + } + + protected void stopStream() {} + + @Override + public final void prepareForTriggerAvailableNow() { + this.lastOffsetForTriggerAvailableNow = availableNowEndOffset(); + availableNowPrepared(); + } + + protected StreamingOffset availableNowEndOffset() { + return latestStreamingOffset(); + } + + protected void availableNowPrepared() {} +} diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/StreamingInitialOffsetStore.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/StreamingInitialOffsetStore.java new file mode 100644 index 000000000000..9d34033c5ea5 --- /dev/null +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/StreamingInitialOffsetStore.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.source; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.function.Supplier; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.hadoop.HadoopFileIO; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.relocated.com.google.common.base.Joiner; + +class StreamingInitialOffsetStore { + private static final Joiner SLASH = Joiner.on("/"); + + private final FileIO io; + private final String initialOffsetLocation; + private final Supplier offsetSupplier; + + StreamingInitialOffsetStore( + String checkpointLocation, Configuration conf, Supplier offsetSupplier) { + this.io = new HadoopFileIO(conf); + this.initialOffsetLocation = SLASH.join(checkpointLocation, "offsets/0"); + this.offsetSupplier = offsetSupplier; + } + + StreamingOffset initialOffset() { + InputFile inputFile = io.newInputFile(initialOffsetLocation); + if (inputFile.exists()) { + return readOffset(inputFile); + } + + StreamingOffset offset = offsetSupplier.get(); + writeOffset(offset, io.newOutputFile(initialOffsetLocation)); + return offset; + } + + private void writeOffset(StreamingOffset offset, OutputFile file) { + try (OutputStream outputStream = file.create(); + BufferedWriter writer = + new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8))) { + writer.write(offset.json()); + writer.flush(); + } catch (IOException e) { + throw new UncheckedIOException("Failed writing offset to: " + initialOffsetLocation, e); + } + } + + private StreamingOffset readOffset(InputFile file) { + try (InputStream in = file.newStream()) { + return StreamingOffset.fromJson(in); + } catch (IOException e) { + throw new UncheckedIOException("Failed reading offset from: " + initialOffsetLocation, e); + } + } +} diff --git a/spark/v4.2/spark/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteBusinessKeyChangelog.scala b/spark/v4.2/spark/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteBusinessKeyChangelog.scala new file mode 100644 index 000000000000..ab85ae39784a --- /dev/null +++ b/spark/v4.2/spark/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteBusinessKeyChangelog.scala @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.catalyst.analysis + +import org.apache.iceberg.spark.ChangelogIterator +import org.apache.iceberg.spark.source.SparkChangelogTable +import org.apache.spark.sql.Encoders +import org.apache.spark.sql.Row +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.expressions.Alias +import org.apache.spark.sql.catalyst.expressions.OrderUtils +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.plans.logical.Project +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.catalyst.streaming.StreamingRelationV2 +import org.apache.spark.sql.classic.{SparkSession => ClassicSparkSession} +import org.apache.spark.sql.classic.Dataset +import org.apache.spark.sql.connector.catalog.ChangelogContext +import org.apache.spark.sql.connector.catalog.ChangelogContext.DeduplicationMode +import org.apache.spark.sql.connector.catalog.Table +import org.apache.spark.sql.execution.datasources.v2.ChangelogTable +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation +import org.apache.spark.sql.functions.col +import org.apache.spark.sql.functions.lit +import org.apache.spark.sql.functions.lower +import org.apache.spark.sql.functions.upper +import org.apache.spark.sql.functions.when +import scala.jdk.CollectionConverters._ + +/** Applies the changelog view's value/key semantics to an explicitly requested CDC batch read. */ +case class RewriteBusinessKeyChangelog(spark: SparkSession) extends Rule[LogicalPlan] { + + override def apply(plan: LogicalPlan): LogicalPlan = plan.transformUp { + case rel: DataSourceV2Relation if businessKeyTable(rel.table).isDefined => + rewrite(rel, rel.table.asInstanceOf[ChangelogTable]) + + case rel: StreamingRelationV2 if businessKeyTable(rel.table).isDefined => + throw new UnsupportedOperationException( + "Business-key CDC currently supports batch reads only") + } + + private def businessKeyTable(table: Table): Option[SparkChangelogTable] = table match { + case wrapper: ChangelogTable => + wrapper.changelog match { + case iceberg: SparkChangelogTable if iceberg.identifierColumns().nonEmpty => Some(iceberg) + case _ => None + } + case _ => None + } + + private def rewrite(rel: DataSourceV2Relation, wrapper: ChangelogTable): LogicalPlan = { + val iceberg = businessKeyTable(wrapper).get + val rawContext = + new ChangelogContext(wrapper.changelogContext.range(), DeduplicationMode.NONE, false) + val raw = rel.copy(table = wrapper.copy( + changelog = iceberg.rawChangelog(), + changelogContext = rawContext, + resolved = true)) + val input = Dataset + .ofRows(spark.asInstanceOf[ClassicSparkSession], raw) + .drop("_row_id", "_last_updated_sequence_number") + .withColumn("_change_type", upper(col("_change_type"))) + val schema = input.schema + val computeUpdates = wrapper.changelogContext.computeUpdates() + val identifiers = iceberg.identifierColumns() :+ "_commit_version" + val groupNames = + if (computeUpdates) identifiers else schema.fieldNames.filterNot(_ == "_change_type") + require( + groupNames.forall(name => OrderUtils.isOrderable(schema(name).dataType)), + "Business-key CDC grouping columns must be orderable") + val groupColumns = groupNames.map(name => col("`" + name.replace("`", "``") + "`")) + val sorted = input + .repartition(groupColumns.toIndexedSeq: _*) + .sortWithinPartitions((groupColumns :+ col("_change_type")).toIndexedSeq: _*) + val processed = sorted + .mapPartitions { rows: Iterator[Row] => + val result = if (computeUpdates) { + ChangelogIterator.computeUpdates(rows.asJava, schema, identifiers) + } else { + ChangelogIterator.removeCarryovers(rows.asJava, schema) + } + result.asScala + }(Encoders.row(schema)) + .withColumn( + "_change_type", + when(col("_change_type") === "UPDATE_BEFORE", "update_preimage") + .when(col("_change_type") === "UPDATE_AFTER", "update_postimage") + .otherwise(lower(col("_change_type")))) + .withColumn("_row_id", lit(null).cast("long")) + .withColumn("_last_updated_sequence_number", lit(null).cast("long")) + .queryExecution + .analyzed + val output = rel.output.map { expected => + val actual = processed.output.find(_.name == expected.name).get + Alias(actual, expected.name)( + exprId = expected.exprId, + explicitMetadata = Some(expected.metadata)) + } + Project(output, processed) + } +} diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestChangelogReader.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestChangelogReader.java index b12fdd443ff4..4392fb915487 100644 --- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestChangelogReader.java +++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestChangelogReader.java @@ -29,6 +29,7 @@ import java.util.stream.Collectors; import org.apache.iceberg.ChangelogOperation; import org.apache.iceberg.ChangelogScanTask; +import org.apache.iceberg.ChangelogUtil; import org.apache.iceberg.DataFile; import org.apache.iceberg.Files; import org.apache.iceberg.IncrementalChangelogScan; @@ -105,7 +106,13 @@ public void testInsert() throws IOException { for (ScanTaskGroup taskGroup : taskGroups) { ChangelogRowReader reader = - new ChangelogRowReader(table, table.io(), taskGroup, table.schema(), false, true); + new ChangelogRowReader( + table, + table.io(), + taskGroup, + ChangelogUtil.changelogSchema(table.schema()), + false, + true); while (reader.next()) { rows.add(reader.get().copy()); } @@ -136,7 +143,13 @@ public void testDelete() throws IOException { for (ScanTaskGroup taskGroup : taskGroups) { ChangelogRowReader reader = - new ChangelogRowReader(table, table.io(), taskGroup, table.schema(), false, true); + new ChangelogRowReader( + table, + table.io(), + taskGroup, + ChangelogUtil.changelogSchema(table.schema()), + false, + true); while (reader.next()) { rows.add(reader.get().copy()); } @@ -170,7 +183,13 @@ public void testDataFileRewrite() throws IOException { for (ScanTaskGroup taskGroup : taskGroups) { ChangelogRowReader reader = - new ChangelogRowReader(table, table.io(), taskGroup, table.schema(), false, true); + new ChangelogRowReader( + table, + table.io(), + taskGroup, + ChangelogUtil.changelogSchema(table.schema()), + false, + true); while (reader.next()) { rows.add(reader.get().copy()); } @@ -197,7 +216,13 @@ public void testMixDeleteAndInsert() throws IOException { for (ScanTaskGroup taskGroup : taskGroups) { ChangelogRowReader reader = - new ChangelogRowReader(table, table.io(), taskGroup, table.schema(), false, true); + new ChangelogRowReader( + table, + table.io(), + taskGroup, + ChangelogUtil.changelogSchema(table.schema()), + false, + true); while (reader.next()) { rows.add(reader.get().copy()); } diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkChangelog.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkChangelog.java new file mode 100644 index 000000000000..559873b4ca0b --- /dev/null +++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkChangelog.java @@ -0,0 +1,762 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.source; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.AdditionalAnswers.delegatesTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.sql.Timestamp; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.iceberg.ChangelogScanTask; +import org.apache.iceberg.ChangelogUtil; +import org.apache.iceberg.ScanTaskGroup; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.Table; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.spark.SparkReadConf; +import org.apache.iceberg.spark.TestBaseWithCatalog; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.catalog.ChangelogContext; +import org.apache.spark.sql.connector.catalog.ChangelogContext.DeduplicationMode; +import org.apache.spark.sql.connector.catalog.ChangelogRange; +import org.apache.spark.sql.connector.read.InputPartition; +import org.apache.spark.sql.connector.read.PartitionReader; +import org.apache.spark.sql.connector.read.streaming.MicroBatchStream; +import org.apache.spark.sql.connector.read.streaming.Offset; +import org.apache.spark.sql.connector.read.streaming.ReadLimit; +import org.apache.spark.sql.streaming.StreamingQuery; +import org.apache.spark.sql.streaming.Trigger; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.TestTemplate; + +class TestSparkChangelog extends TestBaseWithCatalog { + + @AfterEach + void removeTable() { + sql("DROP TABLE IF EXISTS %s", tableName); + } + + @TestTemplate + void readsRawCopyOnWriteChangesWithoutRowLineage() { + sql( + "CREATE TABLE %s (id bigint, data string) USING iceberg " + + "TBLPROPERTIES ('format-version'='2', 'write.update.mode'='copy-on-write')", + tableName); + sql("INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM VALUES (1, 'a'), (2, 'b')", tableName); + sql("UPDATE %s SET data = 'updated' WHERE id = 1", tableName); + Table table = validationCatalog.loadTable(tableIdent); + Snapshot snapshot = table.currentSnapshot(); + long version = snapshot.sequenceNumber(); + Timestamp timestamp = new Timestamp(snapshot.timestampMillis()); + Dataset changes = + spark.sql( + String.format( + "SELECT * FROM %s CHANGES FROM VERSION %d TO VERSION %d " + + "WITH (deduplicationMode = 'none', computeUpdates = 'false')", + tableName, version, version)); + + assertThat(changes.schema().fieldNames()) + .containsExactly( + "id", + "data", + "_row_id", + "_last_updated_sequence_number", + "_change_type", + "_commit_version", + "_commit_timestamp"); + assertThat(changes.schema().apply("_row_id").nullable()).isTrue(); + assertThat(changes.schema().apply("_last_updated_sequence_number").nullable()).isTrue(); + assertThat(changes.collectAsList()) + .containsExactlyInAnyOrder( + RowFactory.create(1L, "a", null, null, "delete", version, timestamp), + RowFactory.create(1L, "updated", null, null, "insert", version, timestamp), + RowFactory.create(2L, "b", null, null, "delete", version, timestamp), + RowFactory.create(2L, "b", null, null, "insert", version, timestamp)); + } + + @TestTemplate + void resumesRawStreamWithoutRowLineage() throws Exception { + sql( + "CREATE TABLE %s (id bigint, data string) USING iceberg " + + "TBLPROPERTIES ('format-version'='2', 'write.delete.mode'='copy-on-write')", + tableName); + sql("INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM VALUES (1, 'a'), (2, 'b')", tableName); + Table table = validationCatalog.loadTable(tableIdent); + long firstVersion = table.currentSnapshot().sequenceNumber(); + Dataset changes = + spark + .readStream() + .option("deduplicationMode", "none") + .option("computeUpdates", "false") + .option("startingVersion", String.valueOf(firstVersion)) + .changes(tableName); + String checkpoint = temp.resolve("raw-cdc-checkpoint").toString(); + String output = temp.resolve("raw-cdc-output").toString(); + writeAvailableChanges(changes, checkpoint, output); + assertThat(spark.read().parquet(output).count()).isEqualTo(2); + + sql("DELETE FROM %s WHERE id = 1", tableName); + table.refresh(); + long deleteVersion = table.currentSnapshot().sequenceNumber(); + writeAvailableChanges(changes, checkpoint, output); + Dataset result = spark.read().parquet(output); + assertThat(result.select("id", "data", "_change_type", "_commit_version").collectAsList()) + .containsExactlyInAnyOrder( + RowFactory.create(1L, "a", "insert", firstVersion), + RowFactory.create(2L, "b", "insert", firstVersion), + RowFactory.create(1L, "a", "delete", deleteVersion), + RowFactory.create(2L, "b", "delete", deleteVersion), + RowFactory.create(2L, "b", "insert", deleteVersion)); + assertThat( + result + .filter("_row_id IS NOT NULL OR _last_updated_sequence_number IS NOT NULL") + .count()) + .isZero(); + } + + private void writeAvailableChanges(Dataset changes, String checkpoint, String output) + throws Exception { + StreamingQuery query = + changes + .writeStream() + .format("parquet") + .option("checkpointLocation", checkpoint) + .trigger(Trigger.AvailableNow()) + .start(output); + try { + assertThat(query.awaitTermination(60_000)).isTrue(); + } finally { + query.stop(); + } + } + + @TestTemplate + void rejectsBusinessKeyReadWithoutExtensions() { + sql( + "CREATE TABLE %s (id bigint, data string) USING iceberg " + + "TBLPROPERTIES ('format-version'='2')", + tableName); + sql("INSERT INTO %s VALUES (1, 'a')", tableName); + assertThatThrownBy( + () -> + spark + .read() + .option("identifier-columns", "id") + .option("computeUpdates", "true") + .changes(tableName) + .collectAsList()) + .hasStackTraceContaining("Business-key CDC requires IcebergSparkSessionExtensions"); + } + + @TestTemplate + void rejectsPostProcessingWithoutRowLineage() { + sql( + "CREATE TABLE %s (id bigint, data string) USING iceberg " + + "TBLPROPERTIES ('format-version'='2')", + tableName); + sql("INSERT INTO %s VALUES (1, 'a')", tableName); + List> options = + List.of( + Map.of(), + Map.of("deduplicationMode", "dropCarryovers"), + Map.of("deduplicationMode", "netChanges"), + Map.of("computeUpdates", "true"), + Map.of("deduplicationMode", "none", "computeUpdates", "true")); + for (Map option : options) { + assertThatThrownBy(() -> spark.read().options(option).changes(tableName).collectAsList()) + .hasStackTraceContaining("Spark CDC post-processing requires row lineage"); + } + } + + @TestTemplate + void rejectsRawCdcWithoutCommitSequenceNumbers() { + sql( + "CREATE TABLE %s (id bigint, data string) USING iceberg " + + "TBLPROPERTIES ('format-version'='1')", + tableName); + sql("INSERT INTO %s VALUES (1, 'a')", tableName); + assertThatThrownBy( + () -> + spark.read().option("deduplicationMode", "none").changes(tableName).collectAsList()) + .hasStackTraceContaining("format version 2 or later"); + + sql("ALTER TABLE %s SET TBLPROPERTIES ('format-version'='2')", tableName); + assertThatThrownBy( + () -> + spark.read().option("deduplicationMode", "none").changes(tableName).collectAsList()) + .hasStackTraceContaining("without a commit sequence number"); + } + + @TestTemplate + void rawCdcStillRejectsDeleteFiles() { + sql( + "CREATE TABLE %s (id bigint, data string) USING iceberg " + + "TBLPROPERTIES ('format-version'='2', 'write.delete.mode'='merge-on-read')", + tableName); + sql("INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM VALUES (1, 'a'), (2, 'b')", tableName); + sql("DELETE FROM %s WHERE id = 1", tableName); + Table table = validationCatalog.loadTable(tableIdent); + assertThat(table.currentSnapshot().deleteManifests(table.io())).isNotEmpty(); + assertThatThrownBy( + () -> + spark.read().option("deduplicationMode", "none").changes(tableName).collectAsList()) + .hasStackTraceContaining("Delete files are currently not supported in changelog scans"); + } + + @TestTemplate + void readsChangesUsingSparkCdcSyntax() { + sql( + "CREATE TABLE %s (id bigint, data string) USING iceberg " + + "TBLPROPERTIES ('format-version'='3')", + tableName); + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + Table table = validationCatalog.loadTable(tableIdent); + long firstVersion = table.currentSnapshot().sequenceNumber(); + + sql("INSERT INTO %s VALUES (3, 'c')", tableName); + table.refresh(); + long secondVersion = table.currentSnapshot().sequenceNumber(); + + assertThat( + sql( + "SELECT id, data, _change_type, _commit_version " + + "FROM %s CHANGES FROM VERSION %d TO VERSION %d ORDER BY id", + tableName, firstVersion, secondVersion)) + .containsExactly( + row(1L, "a", "insert", firstVersion), + row(2L, "b", "insert", firstVersion), + row(3L, "c", "insert", secondVersion)); + } + + @TestTemplate + void readsCopyOnWriteChangesUsingSparkCdcSyntax() { + sql( + "CREATE TABLE %s (id bigint, data string) USING iceberg " + + "TBLPROPERTIES ('format-version'='3')", + tableName); + sql("INSERT INTO %s SELECT /*+ COALESCE(1) */ * FROM VALUES (1, 'a'), (2, 'b')", tableName); + + sql("DELETE FROM %s WHERE id = 1", tableName); + Table table = validationCatalog.loadTable(tableIdent); + long deleteVersion = table.currentSnapshot().sequenceNumber(); + assertThat(table.currentSnapshot().deleteManifests(table.io())).isEmpty(); + + assertThat( + sql( + "SELECT id, data, _change_type, _commit_version " + + "FROM %s CHANGES FROM VERSION %d TO VERSION %d " + + "ORDER BY _change_type, id", + tableName, deleteVersion, deleteVersion)) + .containsExactly(row(1L, "a", "delete", deleteVersion)); + + assertThat( + sql( + "SELECT id, data, _change_type, _commit_version " + + "FROM %s CHANGES FROM VERSION %d TO VERSION %d " + + "WITH (deduplicationMode = 'none') " + + "ORDER BY _change_type, id", + tableName, deleteVersion, deleteVersion)) + .containsExactly( + row(1L, "a", "delete", deleteVersion), + row(2L, "b", "delete", deleteVersion), + row(2L, "b", "insert", deleteVersion)); + } + + @TestTemplate + void computesUpdatesForCopyOnWriteChanges() { + sql( + "CREATE TABLE %s (id bigint, data string) USING iceberg " + + "TBLPROPERTIES ('format-version'='3')", + tableName); + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + + sql("UPDATE %s SET data = 'updated' WHERE id = 1", tableName); + Table table = validationCatalog.loadTable(tableIdent); + long updateVersion = table.currentSnapshot().sequenceNumber(); + + assertThat( + sql( + "SELECT id, data, _change_type, _commit_version " + + "FROM %s CHANGES FROM VERSION %d TO VERSION %d " + + "WITH (computeUpdates = 'true') ORDER BY data", + tableName, updateVersion, updateVersion)) + .containsExactly( + row(1L, "a", "update_preimage", updateVersion), + row(1L, "updated", "update_postimage", updateVersion)); + } + + @TestTemplate + void availableNowPinsLatestSnapshot() { + sql( + "CREATE TABLE %s (id bigint, data string) USING iceberg " + + "TBLPROPERTIES ('format-version'='3')", + tableName); + sql("INSERT INTO %s VALUES (1, 'a')", tableName); + Table table = validationCatalog.loadTable(tableIdent); + long availableSnapshotId = table.currentSnapshot().snapshotId(); + SparkReadConf readConf = new SparkReadConf(spark, table, CaseInsensitiveStringMap.empty()); + SparkChangelogMicroBatchStream stream = + new SparkChangelogMicroBatchStream( + JavaSparkContext.fromSparkContext(spark.sparkContext()), + table, + readConf, + SparkChangelogTable.cdcDataSchema(table), + temp.resolve("cdc-available-now").toString(), + new SparkChangelogRange(context(new ChangelogRange.UnboundedRange()))); + + stream.prepareForTriggerAvailableNow(); + sql("INSERT INTO %s VALUES (2, 'b')", tableName); + + StreamingOffset latestOffset = (StreamingOffset) stream.latestOffset(); + assertThat(latestOffset.snapshotId()).isEqualTo(availableSnapshotId); + assertThat(latestOffset.position()).isZero(); + assertThat(stream.latestOffset(stream.initialOffset(), ReadLimit.allAvailable())) + .isEqualTo(latestOffset); + stream.stop(); + } + + @TestTemplate + void streamsChangesUsingSparkCdcApi() throws Exception { + String queryName = "iceberg_cdc_changes"; + sql( + "CREATE TABLE %s (id bigint, data string) USING iceberg " + + "TBLPROPERTIES ('format-version'='3')", + tableName); + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + + Dataset changes = spark.readStream().changes(tableName); + StreamingQuery query = + changes + .writeStream() + .format("memory") + .queryName(queryName) + .trigger(Trigger.AvailableNow()) + .start(); + query.awaitTermination(); + + assertThat(sql("SELECT id, data, _change_type FROM %s ORDER BY id", queryName)) + .containsExactly(row(1L, "a", "insert"), row(2L, "b", "insert")); + spark.catalog().dropTempView(queryName); + } + + @TestTemplate + void readsExclusiveVersionBounds() { + Table table = createCdcTable(); + long first = table.currentSnapshot().sequenceNumber(); + sql("INSERT INTO %s VALUES (2, 'b')", tableName); + table.refresh(); + long second = table.currentSnapshot().sequenceNumber(); + ChangelogContext empty = + context( + new ChangelogRange.VersionRange( + String.valueOf(first), Optional.of(String.valueOf(first)), false, false)); + assertThat( + new SparkChangelogTable(table, empty) + .newScanBuilder(CaseInsensitiveStringMap.empty()) + .build() + .toBatch() + .planInputPartitions()) + .isEmpty(); + assertThat( + spark + .read() + .option("startingVersion", first) + .option("endingVersion", second) + .option("startingBoundInclusive", false) + .changes(tableName) + .select("id") + .collectAsList()) + .containsExactly(RowFactory.create(2L)); + } + + @TestTemplate + void streamsNewCommitsWithOnlyStartingVersion() throws Exception { + Table table = createCdcTable(); + ChangelogContext context = + context( + new ChangelogRange.VersionRange( + String.valueOf(table.currentSnapshot().sequenceNumber()), + Optional.empty(), + true, + true)); + MicroBatchStream stream = stream(table, context, "open-version"); + try { + Offset firstEnd = stream.latestOffset(); + assertThat(readChanges(stream, stream.initialOffset(), firstEnd)) + .containsExactly(row(1L, "a", "insert", table.currentSnapshot().sequenceNumber())); + sql("INSERT INTO %s VALUES (2, 'b')", tableName); + Offset secondEnd = stream.latestOffset(); + assertThat(secondEnd).isNotEqualTo(firstEnd); + assertThat(readChanges(stream, firstEnd, secondEnd)) + .containsExactly(row(2L, "b", "insert", table.currentSnapshot().sequenceNumber())); + } finally { + stream.stop(); + } + } + + @TestTemplate + void waitsForFutureTimestampWithoutReadingHistory() throws Exception { + Table table = createCdcTable(); + long future = Long.MAX_VALUE; + MicroBatchStream stream = + stream( + table, + context(new ChangelogRange.TimestampRange(future, Optional.empty(), true, true)), + "future"); + try { + assertThat(readChanges(stream, stream.initialOffset(), stream.latestOffset())).isEmpty(); + } finally { + stream.stop(); + } + } + + @TestTemplate + void preservesEmptyRangeInStreams() throws Exception { + Table table = createCdcTable(); + String version = String.valueOf(table.currentSnapshot().sequenceNumber()); + MicroBatchStream stream = + stream( + table, + context(new ChangelogRange.VersionRange(version, Optional.of(version), false, false)), + "empty"); + try { + assertThat(readChanges(stream, stream.initialOffset(), stream.latestOffset())).isEmpty(); + } finally { + stream.stop(); + } + } + + @TestTemplate + void keepsReadSchemaAfterAddingColumn() throws Exception { + Table table = createCdcTable(); + MicroBatchStream stream = stream(table, context(new ChangelogRange.UnboundedRange()), "schema"); + try { + Offset firstEnd = stream.latestOffset(); + readChanges(stream, stream.initialOffset(), firstEnd); + sql("ALTER TABLE %s ADD COLUMN extra string", tableName); + sql("INSERT INTO %s VALUES (2, 'b', 'extra')", tableName); + Offset secondEnd = stream.latestOffset(); + assertThat(readChanges(stream, firstEnd, secondEnd)) + .containsExactly(row(2L, "b", "insert", table.currentSnapshot().sequenceNumber())); + } finally { + stream.stop(); + } + } + + @TestTemplate + void rejectsIncompatibleStreamingSchemaChange() { + Table table = createCdcTable(); + MicroBatchStream stream = stream(table, context(new ChangelogRange.UnboundedRange()), "drop"); + try { + Offset firstEnd = stream.latestOffset(); + table.updateSchema().deleteColumn("data").commit(); + sql("INSERT INTO %s VALUES (2)", tableName); + Offset secondEnd = stream.latestOffset(); + assertThatThrownBy(() -> stream.planInputPartitions(firstEnd, secondEnd)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("incompatible schema change"); + } finally { + stream.stop(); + } + } + + @TestTemplate + void readsRawHistoryButRejectsPostProcessingAfterUpgrade() { + sql( + "CREATE TABLE %s (id bigint, data string) USING iceberg " + + "TBLPROPERTIES ('format-version'='2')", + tableName); + sql("INSERT INTO %s VALUES (1, 'a')", tableName); + Table table = validationCatalog.loadTable(tableIdent); + long version = table.currentSnapshot().sequenceNumber(); + sql("ALTER TABLE %s SET TBLPROPERTIES ('format-version'='3')", tableName); + table.refresh(); + assertThat( + spark + .read() + .option("deduplicationMode", "none") + .option("startingVersion", String.valueOf(version)) + .option("endingVersion", String.valueOf(version)) + .changes(tableName) + .select("id", "data", "_row_id", "_last_updated_sequence_number", "_change_type") + .collectAsList()) + .containsExactly(RowFactory.create(1L, "a", null, null, "insert")); + ChangelogContext context = + new ChangelogContext( + new ChangelogRange.VersionRange( + String.valueOf(version), Optional.of(String.valueOf(version)), true, true), + DeduplicationMode.DROP_CARRYOVERS, + false); + assertThatThrownBy( + () -> + new SparkChangelogTable(table, context) + .newScanBuilder(CaseInsensitiveStringMap.empty()) + .build() + .toBatch() + .planInputPartitions()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("without row lineage"); + } + + @TestTemplate + void filtersTimestampRangesWhenCommitTimeMovesBackwards() { + Table table = createCdcTable(); + Snapshot first = table.currentSnapshot(); + sql("INSERT INTO %s VALUES (2, 'b')", tableName); + table.refresh(); + Snapshot second = table.currentSnapshot(); + sql("INSERT INTO %s VALUES (3, 'c')", tableName); + table.refresh(); + Snapshot third = table.currentSnapshot(); + Table skewed = + withTimestamps( + table, + Map.of(first.snapshotId(), 100L, second.snapshotId(), 90L, third.snapshotId(), 110L)); + SparkChangelogRange range = + new SparkChangelogRange( + context(new ChangelogRange.TimestampRange(95000, Optional.of(115000L), true, true))); + List> groups = + range.planTasks( + skewed, + table + .newIncrementalChangelogScan() + .project(ChangelogUtil.changelogSchema(SparkChangelogTable.cdcDataSchema(table))), + null, + third.snapshotId()); + assertThat(groups).isNotEmpty(); + assertThat(groups.stream().flatMap(group -> group.tasks().stream())) + .extracting(ChangelogScanTask::commitSnapshotId) + .containsOnly(first.snapshotId(), third.snapshotId()); + } + + @TestTemplate + void rejectsLateCommitBeforeStreamingPostProcessing() { + Table table = createCdcTable(); + Snapshot first = table.currentSnapshot(); + sql("INSERT INTO %s VALUES (2, 'b')", tableName); + table.refresh(); + Snapshot second = table.currentSnapshot(); + Table skewed = + withTimestamps(table, Map.of(first.snapshotId(), 100L, second.snapshotId(), 100L)); + ChangelogContext context = + new ChangelogContext( + new ChangelogRange.UnboundedRange(), DeduplicationMode.DROP_CARRYOVERS, false); + SparkChangelogMicroBatchStream stream = + new SparkChangelogMicroBatchStream( + JavaSparkContext.fromSparkContext(spark.sparkContext()), + skewed, + new SparkReadConf(spark, table, CaseInsensitiveStringMap.empty()), + SparkChangelogTable.cdcDataSchema(table), + temp.resolve("late").toString(), + new SparkChangelogRange(context)); + try { + assertThatThrownBy( + () -> + stream.planTaskGroups( + new StreamingOffset(first.snapshotId(), 0, false), + new StreamingOffset(second.snapshotId(), 0, false))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Cannot stream CDC post-processing"); + } finally { + stream.stop(); + } + } + + @TestTemplate + void resumesFromCheckpointOffsetWithOpenRange() throws Exception { + Table table = createCdcTable(); + ChangelogContext context = context(new ChangelogRange.UnboundedRange()); + MicroBatchStream first = stream(table, context, "restart"); + String endJson; + try { + Offset end = first.latestOffset(); + readChanges(first, first.initialOffset(), end); + endJson = end.json(); + } finally { + first.stop(); + } + sql("INSERT INTO %s VALUES (2, 'b')", tableName); + MicroBatchStream resumed = stream(table, context, "restart"); + try { + assertThat(readChanges(resumed, resumed.deserializeOffset(endJson), resumed.latestOffset())) + .containsExactly(row(2L, "b", "insert", table.currentSnapshot().sequenceNumber())); + } finally { + resumed.stop(); + } + } + + @TestTemplate + void legacyChangelogAllowsCommitVersionDataColumn() { + sql("CREATE TABLE %s (id bigint, _commit_version string) USING iceberg", tableName); + sql("INSERT INTO %s VALUES (1, 'business-version')", tableName); + assertThat(sql("SELECT id, _commit_version, _change_type FROM %s.changes", tableName)) + .containsExactly(row(1L, "business-version", "INSERT")); + } + + @TestTemplate + void readsInclusiveStartAfterItsParentExpires() throws Exception { + Table table = createCdcTable(); + long parentId = table.currentSnapshot().snapshotId(); + sql("INSERT INTO %s VALUES (2, 'b')", tableName); + table.refresh(); + String version = String.valueOf(table.currentSnapshot().sequenceNumber()); + table.expireSnapshots().expireSnapshotId(parentId).cleanExpiredFiles(false).commit(); + MicroBatchStream stream = + stream( + table, + context(new ChangelogRange.VersionRange(version, Optional.empty(), true, true)), + "expired-parent"); + try { + assertThat(readChanges(stream, stream.initialOffset(), stream.latestOffset())) + .containsExactly(row(2L, "b", "insert", table.currentSnapshot().sequenceNumber())); + } finally { + stream.stop(); + } + } + + @TestTemplate + void admissionKeepsEqualTimestampSnapshotsTogether() { + Table table = createCdcTable(); + Snapshot first = table.currentSnapshot(); + sql("INSERT INTO %s VALUES (2, 'b')", tableName); + table.refresh(); + Snapshot second = table.currentSnapshot(); + sql("INSERT INTO %s VALUES (3, 'c'), (4, 'd')", tableName); + table.refresh(); + Snapshot third = table.currentSnapshot(); + Table controlled = + withTimestamps( + table, + Map.of(first.snapshotId(), 100L, second.snapshotId(), 100L, third.snapshotId(), 110L)); + SparkChangelogMicroBatchStream stream = + new SparkChangelogMicroBatchStream( + JavaSparkContext.fromSparkContext(spark.sparkContext()), + controlled, + new SparkReadConf(spark, table, CaseInsensitiveStringMap.empty()), + SparkChangelogTable.cdcDataSchema(table), + temp.resolve("admission").toString(), + new SparkChangelogRange(context(new ChangelogRange.UnboundedRange()))); + try { + StreamingOffset end = + (StreamingOffset) stream.latestOffset(stream.initialOffset(), ReadLimit.maxRows(1)); + assertThat(end.snapshotId()).isEqualTo(second.snapshotId()); + StreamingOffset next = (StreamingOffset) stream.latestOffset(end, ReadLimit.maxRows(1)); + assertThat(next.snapshotId()).isEqualTo(third.snapshotId()); + assertThat(next.position()).isZero(); + // Raw CDC does not use Spark's event-time watermark and can read equal timestamps. + assertThat(stream.planTaskGroups(new StreamingOffset(first.snapshotId(), 0, false), end)) + .isNotEmpty(); + } finally { + stream.stop(); + } + } + + @TestTemplate + void legacyChangelogProjectsFileAndChangeMetadataTogether() { + Table table = createCdcTable(); + assertThat( + sql("SELECT id, _change_type, _commit_snapshot_id, _file FROM %s.changes", tableName)) + .singleElement() + .satisfies( + row -> { + assertThat(row[0]).isEqualTo(1L); + assertThat(row[1]).isEqualTo("INSERT"); + assertThat(row[2]).isEqualTo(table.currentSnapshot().snapshotId()); + assertThat(row[3]).isInstanceOf(String.class); + assertThat((String) row[3]).contains(".parquet"); + }); + } + + private Table createCdcTable() { + sql( + "CREATE TABLE %s (id bigint, data string) USING iceberg " + + "TBLPROPERTIES ('format-version'='3')", + tableName); + sql("INSERT INTO %s VALUES (1, 'a')", tableName); + return validationCatalog.loadTable(tableIdent); + } + + private static ChangelogContext context(ChangelogRange range) { + return new ChangelogContext(range, DeduplicationMode.NONE, false); + } + + private MicroBatchStream stream(Table table, ChangelogContext context, String checkpoint) { + return new SparkChangelogTable(table, context) + .newScanBuilder(CaseInsensitiveStringMap.empty()) + .build() + .toMicroBatchStream(temp.resolve(checkpoint).toString()); + } + + private List readChanges(MicroBatchStream stream, Offset start, Offset end) + throws Exception { + List rows = Lists.newArrayList(); + for (InputPartition partition : stream.planInputPartitions(start, end)) { + try (PartitionReader reader = + stream.createReaderFactory().createReader(partition)) { + while (reader.next()) { + InternalRow row = reader.get(); + rows.add( + row( + row.getLong(0), + row.getUTF8String(1).toString(), + row.getUTF8String(4).toString(), + row.getLong(5))); + } + } + } + return rows; + } + + private static Table withTimestamps(Table table, Map timestamps) { + Table controlled = mock(Table.class, delegatesTo(table)); + for (Snapshot snapshot : table.snapshots()) { + Snapshot timed = mock(Snapshot.class, delegatesTo(snapshot)); + when(timed.timestampMillis()).thenReturn(timestamps.get(snapshot.snapshotId())); + when(controlled.snapshot(snapshot.snapshotId())).thenReturn(timed); + if (snapshot.snapshotId() == table.currentSnapshot().snapshotId()) { + when(controlled.currentSnapshot()).thenReturn(timed); + } + } + return controlled; + } + + @TestTemplate + void tableChangesKeepsIcebergChangelogColumns() { + sql("CREATE TABLE %s (id bigint, data string) USING iceberg", tableName); + sql("INSERT INTO %s VALUES (1, 'a')", tableName); + + assertThat(sql("SELECT id, data, _change_type, _commit_snapshot_id FROM %s.changes", tableName)) + .hasSize(1) + .allSatisfy( + row -> { + assertThat(row[2]).isEqualTo("INSERT"); + assertThat(row[3]).isInstanceOf(Long.class); + }); + } +} diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestStreamingInitialOffsetStore.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestStreamingInitialOffsetStore.java new file mode 100644 index 000000000000..dd8f2dca0ca3 --- /dev/null +++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestStreamingInitialOffsetStore.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.source; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TestStreamingInitialOffsetStore { + + @TempDir private Path checkpointDir; + + @Test + void restoresStoredOffset() { + AtomicInteger initializations = new AtomicInteger(); + StreamingOffset expected = new StreamingOffset(34L, 0L, false); + StreamingInitialOffsetStore firstStore = + new StreamingInitialOffsetStore( + checkpointDir.toString(), + new Configuration(), + () -> { + initializations.incrementAndGet(); + return expected; + }); + + assertThat(firstStore.initialOffset()).isEqualTo(expected); + + StreamingInitialOffsetStore restoredStore = + new StreamingInitialOffsetStore( + checkpointDir.toString(), + new Configuration(), + () -> { + initializations.incrementAndGet(); + return StreamingOffset.START_OFFSET; + }); + + assertThat(restoredStore.initialOffset()).isEqualTo(expected); + assertThat(initializations).hasValue(1); + } +}