Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -3141,11 +3141,15 @@ def _check_pyarrow_schema_compatible(
provided_schema: pa.Schema,
downcast_ns_timestamp_to_us: bool = False,
format_version: TableVersion = TableProperties.DEFAULT_FORMAT_VERSION,
*,
reject_unknown_field_ids: bool = False,
) -> None:
"""
Check if the `requested_schema` is compatible with `provided_schema`.

Two schemas are considered compatible when they are equal in terms of the Iceberg Schema type.
`reject_unknown_field_ids` also rejects the fields of `provided_schema` whose field ID is not in `requested_schema`.
Writes set it, because the writer only writes the fields of `requested_schema`.

Raises:
ValueError: If the schemas are not compatible.
Expand All @@ -3168,6 +3172,13 @@ def _check_pyarrow_schema_compatible(
f"PyArrow table contains more columns: {', '.join(sorted(additional_names))}. "
"Update the schema first (hint, use union_by_name)."
) from e
if reject_unknown_field_ids and (unknown_field_ids := provided_schema.field_ids - requested_schema.field_ids):
unknown_columns = sorted(
f"{provided_schema.find_column_name(field_id)} (field ID {field_id})" for field_id in unknown_field_ids
)
raise ValueError(
f"PyArrow table contains columns whose field IDs are not in the table schema: {', '.join(unknown_columns)}."
)
_check_schema_compatible(requested_schema, provided_schema)


Expand Down
28 changes: 28 additions & 0 deletions pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -569,6 +569,9 @@ def append(
df: An Arrow Table or a RecordBatchReader of records to append.
snapshot_properties: Custom properties to be added to the snapshot summary
branch: Branch Reference to run the append operation

Raises:
ValueError: If the schema of ``df`` is not compatible with the table schema.
"""
try:
import pyarrow as pa
Expand All @@ -586,6 +589,7 @@ def append(
provided_schema=df.schema,
downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us,
format_version=self.table_metadata.format_version,
reject_unknown_field_ids=True,
)

with self._append_snapshot_producer(snapshot_properties, branch=branch) as append_files:
Expand Down Expand Up @@ -615,6 +619,9 @@ def dynamic_partition_overwrite(
df: The Arrow dataframe that will be used to overwrite the table
snapshot_properties: Custom properties to be added to the snapshot summary
branch: Branch Reference to run the dynamic partition overwrite operation

Raises:
ValueError: If the schema of ``df`` is not compatible with the table schema.
"""
try:
import pyarrow as pa
Expand Down Expand Up @@ -642,6 +649,7 @@ def dynamic_partition_overwrite(
provided_schema=df.schema,
downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us,
format_version=self.table_metadata.format_version,
reject_unknown_field_ids=True,
)

# If dataframe does not have data, there is no need to overwrite
Expand Down Expand Up @@ -734,6 +742,9 @@ def overwrite(
snapshot_properties: Custom properties to be added to the snapshot summary
case_sensitive: A bool determine if the provided `overwrite_filter` is case-sensitive
branch: Branch Reference to run the overwrite operation

Raises:
ValueError: If the schema of ``df`` is not compatible with the table schema.
"""
try:
import pyarrow as pa
Expand All @@ -751,6 +762,7 @@ def overwrite(
provided_schema=df.schema,
downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us,
format_version=self.table_metadata.format_version,
reject_unknown_field_ids=True,
)

if overwrite_filter != AlwaysFalse():
Expand Down Expand Up @@ -928,6 +940,9 @@ def upsert(

Returns:
An UpsertResult class (contains details of rows updated and inserted)

Raises:
ValueError: If the schema of ``df`` is not compatible with the table schema.
"""
try:
import pyarrow as pa # noqa: F401
Expand Down Expand Up @@ -963,6 +978,7 @@ def upsert(
provided_schema=df.schema,
downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us,
format_version=self.table_metadata.format_version,
reject_unknown_field_ids=True,
)

# get list of rows that exist so we don't have to load the entire target table
Expand Down Expand Up @@ -1805,6 +1821,9 @@ def upsert(

Returns:
An UpsertResult class (contains details of rows updated and inserted)

Raises:
ValueError: If the schema of ``df`` is not compatible with the table schema.
"""
with self.transaction() as tx:
return tx.upsert(
Expand Down Expand Up @@ -1834,6 +1853,9 @@ def append(
df: An Arrow Table or a RecordBatchReader of records to append.
snapshot_properties: Custom properties to be added to the snapshot summary
branch: Branch Reference to run the append operation

Raises:
ValueError: If the schema of ``df`` is not compatible with the table schema.
"""
with self.transaction() as tx:
tx.append(df=df, snapshot_properties=snapshot_properties, branch=branch)
Expand All @@ -1848,6 +1870,9 @@ def dynamic_partition_overwrite(
df: The Arrow dataframe that will be used to overwrite the table
snapshot_properties: Custom properties to be added to the snapshot summary
branch: Branch Reference to run the dynamic partition overwrite operation

Raises:
ValueError: If the schema of ``df`` is not compatible with the table schema.
"""
with self.transaction() as tx:
tx.dynamic_partition_overwrite(df=df, snapshot_properties=snapshot_properties, branch=branch)
Expand Down Expand Up @@ -1880,6 +1905,9 @@ def overwrite(
snapshot_properties: Custom properties to be added to the snapshot summary
case_sensitive: A bool determine if the provided `overwrite_filter` is case-sensitive
branch: Branch Reference to run the overwrite operation

Raises:
ValueError: If the schema of ``df`` is not compatible with the table schema.
"""
with self.transaction() as tx:
tx.overwrite(
Expand Down
48 changes: 48 additions & 0 deletions tests/catalog/test_catalog_behaviors.py
Original file line number Diff line number Diff line change
Expand Up @@ -1005,6 +1005,54 @@ def test_write_and_evolve(catalog: Catalog, format_version: int) -> None:
txn.delete("foo = 'a'")


def test_write_rejects_field_ids_not_in_table_schema(catalog: Catalog) -> None:
catalog.create_namespace("default")
identifier = f"default.write_unknown_field_ids_{catalog.name}"
old_schema = Schema(NestedField(1, "id", LongType()), NestedField(2, "x", LongType()), NestedField(3, "y", LongType()))
spec = PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="id"))
tbl = catalog.create_table(identifier, schema=old_schema, partition_spec=spec)
# Drop x and add it back, so the new x has field ID 4, and rename y (field ID 3) to w
with tbl.update_schema() as update:
update.delete_column("x")
update.rename_column("y", "w")
with tbl.update_schema() as update:
update.add_column("x", LongType())

writes: list[Callable[[pa.Table], Any]] = [
tbl.append,
tbl.overwrite,
tbl.dynamic_partition_overwrite,
# The table is empty and inserts are off, so only the check in upsert itself can reject the input
lambda df: tbl.upsert(df, join_cols=["id"], when_not_matched_insert_all=False),
]
schema_with_z = Schema(*tbl.schema().fields, NestedField(99, "z", LongType()))
for input_schema, unknown_columns in [(schema_with_z, "z (field ID 99)"), (old_schema, "x (field ID 2)")]:
df = pa.Table.from_pylist([{"id": 1, "x": 2, "y": 3, "w": 3, "z": 4}], schema=input_schema.as_arrow())
for write in writes:
with pytest.raises(ValueError) as exc_info:
write(df)
assert (
str(exc_info.value)
== f"PyArrow table contains columns whose field IDs are not in the table schema: {unknown_columns}."
)
assert catalog.load_table(identifier).metadata.snapshots == []

# Still accepted: y is the old name of w (field ID 3), the columns are reordered, y is an int and x is missing
tbl.append(
pa.Table.from_pylist(
[{"y": 5, "id": 1}], schema=Schema(NestedField(3, "y", IntegerType()), NestedField(1, "id", LongType())).as_arrow()
)
)
# Without field IDs, the columns are matched by name
tbl.append(
pa.Table.from_pylist([{"id": 2, "x": 6}], schema=pa.schema([pa.field("id", pa.int64()), pa.field("x", pa.int64())]))
)
assert sorted(catalog.load_table(identifier).scan().to_arrow().to_pylist(), key=lambda row: row["id"]) == [
{"id": 1, "w": 5, "x": None},
{"id": 2, "w": None, "x": 6},
]


# Merge manifests
@pytest.mark.parametrize("format_version", [1, 2])
def test_merge_manifests_local_file_system(catalog: Catalog, arrow_table_with_null: pa.Table, format_version: int) -> None:
Expand Down
35 changes: 35 additions & 0 deletions tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -3320,6 +3320,41 @@ def test_schema_compatible_nested(table_schema_nested: Schema) -> None:
pytest.fail("Unexpected Exception raised when calling `_check_pyarrow_schema_compatible`")


def test_schema_mismatch_unknown_field_ids() -> None:
table_schema = Schema(
NestedField(1, "id", LongType()),
NestedField(2, "payload", StructType(NestedField(3, "qty", LongType()))),
NestedField(4, "tags", ListType(5, LongType(), element_required=False)),
NestedField(6, "props", MapType(7, StringType(), 8, LongType(), value_required=False)),
)
for provided_schema, unknown_columns in [
(Schema(NestedField(1, "id", LongType()), NestedField(99, "z", LongType())), "z (field ID 99)"),
# A struct field, list element and map value whose parent is in the table
(
Schema(
NestedField(2, "payload", StructType(NestedField(3, "qty", LongType()), NestedField(99, "extra", LongType())))
),
"payload.extra (field ID 99)",
),
(Schema(NestedField(4, "tags", ListType(99, LongType(), element_required=False))), "tags.element (field ID 99)"),
(
Schema(NestedField(6, "props", MapType(7, StringType(), 99, LongType(), value_required=False))),
"props.value (field ID 99)",
),
# A struct that is not in the table, and its field
(
Schema(NestedField(98, "extra", StructType(NestedField(99, "qty", LongType())))),
"extra (field ID 98), extra.qty (field ID 99)",
),
]:
with pytest.raises(ValueError) as exc_info:
_check_pyarrow_schema_compatible(table_schema, schema_to_pyarrow(provided_schema), reject_unknown_field_ids=True)
assert (
str(exc_info.value)
== f"PyArrow table contains columns whose field IDs are not in the table schema: {unknown_columns}."
)


def test_schema_mismatch_additional_field(table_schema_simple: Schema) -> None:
other_schema = pa.schema(
(
Expand Down
Loading