From ea61c07991646cf78350a4539dca372ee17e9343 Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Fri, 20 Nov 2026 19:03:50 +0000 Subject: [PATCH] Find partition source columns the same way as the writer _determine_partitions read the partition source from the input by its full name in the table schema, split on dots, while the writer finds each column by following the table's struct path with the input's own names. A dotted struct name raised KeyError, and an input with field IDs from before a rename got its partition values from another column, so filtered scans missed rows. Follow the writer's path and names instead. Co-authored-by: Isaac --- pyiceberg/io/pyarrow.py | 54 +++++++++++--------- tests/catalog/test_catalog_behaviors.py | 66 +++++++++++++++++++++++++ tests/io/test_pyarrow.py | 26 ++++++++++ 3 files changed, 124 insertions(+), 22 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 20333882b5..1eae25fff9 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -3324,7 +3324,9 @@ def _dataframe_to_data_files( ), ) else: - partitions = _determine_partitions(spec=table_metadata.spec(), schema=table_metadata.schema(), arrow_table=df) + partitions = _determine_partitions( + spec=table_metadata.spec(), schema=table_metadata.schema(), arrow_table=df, input_schema=task_schema + ) yield from write_file( io=io, table_metadata=table_metadata, @@ -3348,9 +3350,14 @@ class _TablePartition: arrow_table_partition: pa.Table -def _determine_partitions(spec: PartitionSpec, schema: Schema, arrow_table: pa.Table) -> Iterable[_TablePartition]: +def _determine_partitions( + spec: PartitionSpec, schema: Schema, arrow_table: pa.Table, input_schema: Schema | None = None +) -> Iterable[_TablePartition]: """Based on the iceberg table partition spec, filter the arrow table into partitions with their keys. + The source columns are found by field ID in input_schema, the Iceberg schema of the arrow table (schema if not + given), whose names can differ from schema, e.g. when the arrow table has field IDs from before a rename. + Example: Input: An arrow table with partition key of ['n_legs', 'year'] and with data of @@ -3378,10 +3385,7 @@ def _determine_partitions(spec: PartitionSpec, schema: Schema, arrow_table: pa.T for partition, name in zip(spec.fields, partition_fields, strict=True): source_field = schema.find_field(partition.source_id) - full_field_name = schema.find_column_name(partition.source_id) - if full_field_name is None: - raise ValueError(f"Could not find column name for field ID: {partition.source_id}") - field_array = _get_field_from_arrow_table(arrow_table, full_field_name) + field_array = _get_field_from_arrow_table(arrow_table, schema, partition.source_id, input_schema) partition_array = partition.transform.pyarrow_transform(source_field.field_type)(field_array) # PyArrow has no group by or equality kernels for extension types such as arrow.uuid, so use their storage type if isinstance(partition_array.type, pa.BaseExtensionType): @@ -3427,30 +3431,36 @@ def _partition_value_filter(partition_field_name: str, value: Any) -> pc.Express return pc.field(partition_field_name) == value -def _get_field_from_arrow_table(arrow_table: pa.Table, field_path: str) -> pa.Array: - """Get a field from an Arrow table, supporting both literal field names and nested field paths. +def _get_field_from_arrow_table( + arrow_table: pa.Table, schema: Schema, field_id: int, input_schema: Schema | None = None +) -> pa.Array: + """Get a field from an Arrow table by its field ID, where the field can be nested in structs. - This function handles two cases: - 1. Literal field names that may contain dots (e.g., "some.id") - 2. Nested field paths using dot notation (e.g., "bar.baz" for nested access) + Follow the target schema's parent IDs and resolve each name in input_schema, matching the writer. + A name can contain dots (e.g., "some.id"), so a nested field's full name cannot be split into these names. Args: arrow_table: The Arrow table containing the field - field_path: Field name or dot-separated path + schema: The target Iceberg table schema + field_id: The ID of the field + input_schema: The Iceberg schema of the Arrow table, if different from schema Returns: The field as a PyArrow Array Raises: + ValueError: If a field ID on the path is missing from either schema KeyError: If the field path cannot be resolved """ - # Try exact column name match (handles field names containing literal dots) - if field_path in arrow_table.column_names: - return arrow_table[field_path] - - # If not found as exact name, treat as nested field path - path_parts = field_path.split(".") - # Get the struct column from the table (e.g., "bar" from "bar.baz") - field_array = arrow_table[path_parts[0]] - # Navigate into the struct using the remaining path parts - return pc.struct_field(field_array, path_parts[1:]) + source_schema = input_schema if input_schema is not None else schema + # Match ArrowAccessor: follow the target's parent IDs using each field's input name. + names = [source_schema.find_field(field_id).name] + parent_id = schema._lazy_id_to_parent.get(field_id) + while parent_id is not None: + names.append(source_schema.find_field(parent_id).name) + parent_id = schema._lazy_id_to_parent.get(parent_id) + + # Get the column from the table (e.g., "bar" for "bar.baz") and navigate into its structs + column_name, *nested_names = reversed(names) + field_array = arrow_table[column_name] + return pc.struct_field(field_array, nested_names) if nested_names else field_array diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index a5099e4e6b..6eaf7f1bfb 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -744,6 +744,72 @@ def test_table_properties_raise_for_none_value( # Append table +def test_append_to_identity_partitioned_table_with_dotted_names(catalog: Catalog) -> None: + catalog.create_namespace("default") + # The names from the top-level column down to the partition source, where a name can contain dots + paths = [("a.b",), ("bar", "baz"), ("foo", "bar", "baz"), ("a.b", "c"), ("a", "b.c"), ("root", "a.b", "c")] + for i, path in enumerate(paths): + column_type: pa.DataType = pa.string() + values: list[Any] = ["x", "y", None] + for name in reversed(path[1:]): + column_type = pa.struct([(name, column_type)]) + values = [{name: value} for value in values] + arrow_schema = pa.schema([("id", pa.int64()), (path[0], column_type)]) + df = pa.Table.from_pydict({"id": [1, 2, 3], path[0]: values}, schema=arrow_schema) + tbl = catalog.create_table(f"default.dotted_partition_source_{i}", schema=df.schema) + with tbl.update_spec() as update: + update.add_field(".".join(path), IdentityTransform(), "p") + + tbl.append(df) + + assert sorted(tbl.scan().to_arrow().to_pylist(), key=lambda row: row["id"]) == df.to_pylist(), path + assert {task.file.partition[0] for task in tbl.scan().plan_files()} == {"x", "y", None}, path + + +def test_append_with_field_ids_after_swapping_column_names(catalog: Catalog) -> None: + catalog.create_namespace("default") + struct_type = pa.struct([("c", pa.string())]) + # Each case swaps the names of the two columns after the Arrow table took its field IDs from the table schema + cases: list[tuple[pa.Schema, dict[str, Any], str, str]] = [ + (pa.schema([("a", pa.string()), ("b", pa.string())]), {"a": "x", "b": "y"}, "a", "b"), + (pa.schema([("left", struct_type), ("a.b", struct_type)]), {"left": {"c": "x"}, "a.b": {"c": "y"}}, "left.c", "a.b.c"), + ] + for i, (arrow_schema, row, source_name, renamed_source_name) in enumerate(cases): + tbl = catalog.create_table(f"default.swapped_names_{i}", schema=arrow_schema) + with tbl.update_spec() as spec_update: + spec_update.add_field(source_name, IdentityTransform(), "p") + df = pa.Table.from_pylist([row], schema=tbl.schema().as_arrow()) + first, second = arrow_schema.names + for old_name, new_name in [(first, "tmp"), (second, first), ("tmp", second)]: + with tbl.update_schema() as schema_update: + schema_update.rename_column(old_name, new_name) + + tbl.append(df) + + # The partition value comes from the column that the source field is written from + assert [task.file.partition[0] for task in tbl.scan().plan_files()] == ["x"], source_name + swapped_row = {first: row[second], second: row[first]} + assert tbl.scan(row_filter=f"{renamed_source_name} = 'x'").to_arrow().to_pylist() == [swapped_row], source_name + + +def test_partition_values_follow_writer_struct_path(catalog: Catalog) -> None: + catalog.create_namespace("default") + struct_type = pa.struct([("x", pa.string())]) + tbl = catalog.create_table("default.partition_input_parents", schema=pa.schema([("s", struct_type), ("t", struct_type)])) + with tbl.update_spec() as spec_update: + spec_update.add_field("s.x", IdentityTransform(), "p") + # Swap the struct types, so that each leaf field ID of the input is under the other parent + s_field, t_field = tbl.schema().as_arrow() + input_schema = pa.schema([s_field.with_type(t_field.type), t_field.with_type(s_field.type)]) + tbl.append(pa.Table.from_pylist([{"s": {"x": "X"}, "t": {"x": "Y"}}], schema=input_schema)) + + # The partition value and the filtered scan agree with the row that was written + written_rows = tbl.scan().to_arrow().to_pylist() + value = written_rows[0]["s"]["x"] + assert [task.file.partition[0] for task in tbl.scan().plan_files()] == [value] + assert tbl.scan(row_filter=f"s.x = '{value}'").to_arrow().to_pylist() == written_rows + + def test_append_table(catalog: Catalog, table_schema_simple: Schema, test_table_identifier: Identifier) -> None: namespace = Catalog.namespace_from(test_table_identifier) catalog.create_namespace(namespace) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 439405d121..18dad07f17 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3436,6 +3436,32 @@ def test_partition_for_nested_field() -> None: assert partition_values == {486729, 486730} +def test_partition_for_nested_field_with_dotted_names() -> None: + # The full names a.b.c, d.e.f and g.h.i.j cannot be split on dots into the names of the nested fields + schema = Schema( + NestedField(1, "a.b", StructType(NestedField(2, "c", StringType()))), + NestedField(3, "d", StructType(NestedField(4, "e.f", StringType()))), + NestedField(5, "g", StructType(NestedField(6, "h.i", StructType(NestedField(7, "j", StringType()))))), + ) + spec = PartitionSpec( + PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="c"), + PartitionField(source_id=4, field_id=1001, transform=IdentityTransform(), name="e_f"), + PartitionField(source_id=7, field_id=1002, transform=IdentityTransform(), name="j"), + ) + rows = [ + {"a.b": {"c": "x"}, "d": {"e.f": "y"}, "g": {"h.i": {"j": "z"}}}, + {"a.b": {"c": "x"}, "d": {"e.f": None}, "g": None}, + ] + arrow_table = pa.Table.from_pylist(rows, schema=schema.as_arrow()) + + partitions = list(_determine_partitions(spec, schema, arrow_table)) + + assert {p.partition_key.partition: p.arrow_table_partition.to_pylist() for p in partitions} == { + Record("x", "y", "z"): [rows[0]], + Record("x", None, None): [rows[1]], + } + + def test_partition_for_deep_nested_field() -> None: schema = Schema( NestedField(