diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 20333882b5..9cc3557e07 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -3141,11 +3141,15 @@ def _check_pyarrow_schema_compatible( provided_schema: pa.Schema, downcast_ns_timestamp_to_us: bool = False, format_version: TableVersion = TableProperties.DEFAULT_FORMAT_VERSION, + *, + reject_unknown_field_ids: bool = False, ) -> None: """ Check if the `requested_schema` is compatible with `provided_schema`. Two schemas are considered compatible when they are equal in terms of the Iceberg Schema type. + `reject_unknown_field_ids` also rejects the fields of `provided_schema` whose field ID is not in `requested_schema`. + Writes set it, because the writer only writes the fields of `requested_schema`. Raises: ValueError: If the schemas are not compatible. @@ -3168,6 +3172,13 @@ def _check_pyarrow_schema_compatible( f"PyArrow table contains more columns: {', '.join(sorted(additional_names))}. " "Update the schema first (hint, use union_by_name)." ) from e + if reject_unknown_field_ids and (unknown_field_ids := provided_schema.field_ids - requested_schema.field_ids): + unknown_columns = sorted( + f"{provided_schema.find_column_name(field_id)} (field ID {field_id})" for field_id in unknown_field_ids + ) + raise ValueError( + f"PyArrow table contains columns whose field IDs are not in the table schema: {', '.join(unknown_columns)}." + ) _check_schema_compatible(requested_schema, provided_schema) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 3ea41e3f22..b6ce0b8f89 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -569,6 +569,9 @@ def append( df: An Arrow Table or a RecordBatchReader of records to append. snapshot_properties: Custom properties to be added to the snapshot summary branch: Branch Reference to run the append operation + + Raises: + ValueError: If the schema of ``df`` is not compatible with the table schema. """ try: import pyarrow as pa @@ -586,6 +589,7 @@ def append( provided_schema=df.schema, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=self.table_metadata.format_version, + reject_unknown_field_ids=True, ) with self._append_snapshot_producer(snapshot_properties, branch=branch) as append_files: @@ -615,6 +619,9 @@ def dynamic_partition_overwrite( df: The Arrow dataframe that will be used to overwrite the table snapshot_properties: Custom properties to be added to the snapshot summary branch: Branch Reference to run the dynamic partition overwrite operation + + Raises: + ValueError: If the schema of ``df`` is not compatible with the table schema. """ try: import pyarrow as pa @@ -642,6 +649,7 @@ def dynamic_partition_overwrite( provided_schema=df.schema, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=self.table_metadata.format_version, + reject_unknown_field_ids=True, ) # If dataframe does not have data, there is no need to overwrite @@ -734,6 +742,9 @@ def overwrite( snapshot_properties: Custom properties to be added to the snapshot summary case_sensitive: A bool determine if the provided `overwrite_filter` is case-sensitive branch: Branch Reference to run the overwrite operation + + Raises: + ValueError: If the schema of ``df`` is not compatible with the table schema. """ try: import pyarrow as pa @@ -751,6 +762,7 @@ def overwrite( provided_schema=df.schema, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=self.table_metadata.format_version, + reject_unknown_field_ids=True, ) if overwrite_filter != AlwaysFalse(): @@ -928,6 +940,9 @@ def upsert( Returns: An UpsertResult class (contains details of rows updated and inserted) + + Raises: + ValueError: If the schema of ``df`` is not compatible with the table schema. """ try: import pyarrow as pa # noqa: F401 @@ -963,6 +978,7 @@ def upsert( provided_schema=df.schema, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=self.table_metadata.format_version, + reject_unknown_field_ids=True, ) # get list of rows that exist so we don't have to load the entire target table @@ -1805,6 +1821,9 @@ def upsert( Returns: An UpsertResult class (contains details of rows updated and inserted) + + Raises: + ValueError: If the schema of ``df`` is not compatible with the table schema. """ with self.transaction() as tx: return tx.upsert( @@ -1834,6 +1853,9 @@ def append( df: An Arrow Table or a RecordBatchReader of records to append. snapshot_properties: Custom properties to be added to the snapshot summary branch: Branch Reference to run the append operation + + Raises: + ValueError: If the schema of ``df`` is not compatible with the table schema. """ with self.transaction() as tx: tx.append(df=df, snapshot_properties=snapshot_properties, branch=branch) @@ -1848,6 +1870,9 @@ def dynamic_partition_overwrite( df: The Arrow dataframe that will be used to overwrite the table snapshot_properties: Custom properties to be added to the snapshot summary branch: Branch Reference to run the dynamic partition overwrite operation + + Raises: + ValueError: If the schema of ``df`` is not compatible with the table schema. """ with self.transaction() as tx: tx.dynamic_partition_overwrite(df=df, snapshot_properties=snapshot_properties, branch=branch) @@ -1880,6 +1905,9 @@ def overwrite( snapshot_properties: Custom properties to be added to the snapshot summary case_sensitive: A bool determine if the provided `overwrite_filter` is case-sensitive branch: Branch Reference to run the overwrite operation + + Raises: + ValueError: If the schema of ``df`` is not compatible with the table schema. """ with self.transaction() as tx: tx.overwrite( diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index a5099e4e6b..a34ae7efcd 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -1005,6 +1005,54 @@ def test_write_and_evolve(catalog: Catalog, format_version: int) -> None: txn.delete("foo = 'a'") +def test_write_rejects_field_ids_not_in_table_schema(catalog: Catalog) -> None: + catalog.create_namespace("default") + identifier = f"default.write_unknown_field_ids_{catalog.name}" + old_schema = Schema(NestedField(1, "id", LongType()), NestedField(2, "x", LongType()), NestedField(3, "y", LongType())) + spec = PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="id")) + tbl = catalog.create_table(identifier, schema=old_schema, partition_spec=spec) + # Drop x and add it back, so the new x has field ID 4, and rename y (field ID 3) to w + with tbl.update_schema() as update: + update.delete_column("x") + update.rename_column("y", "w") + with tbl.update_schema() as update: + update.add_column("x", LongType()) + + writes: list[Callable[[pa.Table], Any]] = [ + tbl.append, + tbl.overwrite, + tbl.dynamic_partition_overwrite, + # The table is empty and inserts are off, so only the check in upsert itself can reject the input + lambda df: tbl.upsert(df, join_cols=["id"], when_not_matched_insert_all=False), + ] + schema_with_z = Schema(*tbl.schema().fields, NestedField(99, "z", LongType())) + for input_schema, unknown_columns in [(schema_with_z, "z (field ID 99)"), (old_schema, "x (field ID 2)")]: + df = pa.Table.from_pylist([{"id": 1, "x": 2, "y": 3, "w": 3, "z": 4}], schema=input_schema.as_arrow()) + for write in writes: + with pytest.raises(ValueError) as exc_info: + write(df) + assert ( + str(exc_info.value) + == f"PyArrow table contains columns whose field IDs are not in the table schema: {unknown_columns}." + ) + assert catalog.load_table(identifier).metadata.snapshots == [] + + # Still accepted: y is the old name of w (field ID 3), the columns are reordered, y is an int and x is missing + tbl.append( + pa.Table.from_pylist( + [{"y": 5, "id": 1}], schema=Schema(NestedField(3, "y", IntegerType()), NestedField(1, "id", LongType())).as_arrow() + ) + ) + # Without field IDs, the columns are matched by name + tbl.append( + pa.Table.from_pylist([{"id": 2, "x": 6}], schema=pa.schema([pa.field("id", pa.int64()), pa.field("x", pa.int64())])) + ) + assert sorted(catalog.load_table(identifier).scan().to_arrow().to_pylist(), key=lambda row: row["id"]) == [ + {"id": 1, "w": 5, "x": None}, + {"id": 2, "w": None, "x": 6}, + ] + + # Merge manifests @pytest.mark.parametrize("format_version", [1, 2]) def test_merge_manifests_local_file_system(catalog: Catalog, arrow_table_with_null: pa.Table, format_version: int) -> None: diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 439405d121..7c8954b593 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3320,6 +3320,41 @@ def test_schema_compatible_nested(table_schema_nested: Schema) -> None: pytest.fail("Unexpected Exception raised when calling `_check_pyarrow_schema_compatible`") +def test_schema_mismatch_unknown_field_ids() -> None: + table_schema = Schema( + NestedField(1, "id", LongType()), + NestedField(2, "payload", StructType(NestedField(3, "qty", LongType()))), + NestedField(4, "tags", ListType(5, LongType(), element_required=False)), + NestedField(6, "props", MapType(7, StringType(), 8, LongType(), value_required=False)), + ) + for provided_schema, unknown_columns in [ + (Schema(NestedField(1, "id", LongType()), NestedField(99, "z", LongType())), "z (field ID 99)"), + # A struct field, list element and map value whose parent is in the table + ( + Schema( + NestedField(2, "payload", StructType(NestedField(3, "qty", LongType()), NestedField(99, "extra", LongType()))) + ), + "payload.extra (field ID 99)", + ), + (Schema(NestedField(4, "tags", ListType(99, LongType(), element_required=False))), "tags.element (field ID 99)"), + ( + Schema(NestedField(6, "props", MapType(7, StringType(), 99, LongType(), value_required=False))), + "props.value (field ID 99)", + ), + # A struct that is not in the table, and its field + ( + Schema(NestedField(98, "extra", StructType(NestedField(99, "qty", LongType())))), + "extra (field ID 98), extra.qty (field ID 99)", + ), + ]: + with pytest.raises(ValueError) as exc_info: + _check_pyarrow_schema_compatible(table_schema, schema_to_pyarrow(provided_schema), reject_unknown_field_ids=True) + assert ( + str(exc_info.value) + == f"PyArrow table contains columns whose field IDs are not in the table schema: {unknown_columns}." + ) + + def test_schema_mismatch_additional_field(table_schema_simple: Schema) -> None: other_schema = pa.schema( (