From a25126386eb0a3c6977df1c7bad80fa2890560e2 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Sat, 21 Nov 2026 02:26:49 +0530 Subject: [PATCH 1/2] Core: Accept Arrow PyCapsule exporters in append and overwrite Writes rejected anything that was not a pyarrow Table or RecordBatchReader, which forced users of Polars, arro3, and other Arrow implementations to import pyarrow themselves and convert first. Accept objects that implement the Arrow PyCapsule stream interface and import them as a pyarrow Table. RecordBatchReader keeps the streaming path, and the error message names the accepted types. --- mkdocs/docs/api.md | 2 ++ pyiceberg/table/__init__.py | 35 ++++++++++++++++--------- pyiceberg/typedef.py | 8 ++++++ tests/table/test_init.py | 52 +++++++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 12 deletions(-) diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index aa7c111329..a659e51eb5 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -324,6 +324,8 @@ tbl.append(df) tbl.overwrite(df) ``` +`append` and `overwrite` also accept any object that implements the Arrow PyCapsule stream interface, such as a Polars DataFrame or an arro3 table, and import it as a PyArrow table. + Now, the data is written to the table, and the table can be read using `tbl.scan().to_arrow()`: ```python diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 3ea41e3f22..d231aa5963 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -106,6 +106,7 @@ from pyiceberg.transforms import IdentityTransform from pyiceberg.typedef import ( EMPTY_DICT, + ArrowStreamExportable, IcebergBaseModel, IcebergRootModel, Identifier, @@ -522,15 +523,16 @@ def update_partition_statistics(self) -> UpdatePartitionStatistics: def append( self, - df: pa.Table | pa.RecordBatchReader, + df: pa.Table | pa.RecordBatchReader | ArrowStreamExportable, snapshot_properties: dict[str, str] = EMPTY_DICT, branch: str | None = MAIN_BRANCH, ) -> None: """ Shorthand API for appending PyArrow data to a table transaction. - Accepts either a fully materialised ``pa.Table`` or a streaming - ``pa.RecordBatchReader``. Streaming is microbatched by + Accepts either a fully materialised ``pa.Table``, a streaming + ``pa.RecordBatchReader``, or an object implementing the Arrow PyCapsule + stream interface, which is imported as a ``pa.Table``. Streaming is microbatched by ``write.target-file-size-bytes`` so memory stays bounded; the reader is consumed once and cannot be reused. @@ -577,8 +579,11 @@ def append( from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible, _dataframe_to_data_files + if isinstance(df, ArrowStreamExportable) and not isinstance(df, (pa.Table, pa.RecordBatchReader)): + df = pa.table(df) + if not isinstance(df, (pa.Table, pa.RecordBatchReader)): - raise ValueError(f"Expected pa.Table or pa.RecordBatchReader, got: {df}") + raise ValueError(f"Expected pa.Table, pa.RecordBatchReader, or an Arrow PyCapsule exporter, got: {df}") downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) or False _check_pyarrow_schema_compatible( @@ -676,7 +681,7 @@ def dynamic_partition_overwrite( def overwrite( self, - df: pa.Table | pa.RecordBatchReader, + df: pa.Table | pa.RecordBatchReader | ArrowStreamExportable, overwrite_filter: BooleanExpression | str = ALWAYS_TRUE, snapshot_properties: dict[str, str] = EMPTY_DICT, case_sensitive: bool = True, @@ -685,8 +690,9 @@ def overwrite( """ Shorthand for adding a table overwrite with a PyArrow table or RecordBatchReader to the transaction. - Accepts either a fully materialised ``pa.Table`` or a streaming - ``pa.RecordBatchReader``. Streaming is microbatched by + Accepts either a fully materialised ``pa.Table``, a streaming + ``pa.RecordBatchReader``, or an object implementing the Arrow PyCapsule + stream interface, which is imported as a ``pa.Table``. Streaming is microbatched by ``write.target-file-size-bytes`` so memory stays bounded; the reader is consumed once and cannot be reused. @@ -742,8 +748,11 @@ def overwrite( from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible, _dataframe_to_data_files + if isinstance(df, ArrowStreamExportable) and not isinstance(df, (pa.Table, pa.RecordBatchReader)): + df = pa.table(df) + if not isinstance(df, (pa.Table, pa.RecordBatchReader)): - raise ValueError(f"Expected pa.Table or pa.RecordBatchReader, got: {df}") + raise ValueError(f"Expected pa.Table, pa.RecordBatchReader, or an Arrow PyCapsule exporter, got: {df}") downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) or False _check_pyarrow_schema_compatible( @@ -1819,14 +1828,15 @@ def upsert( def append( self, - df: pa.Table | pa.RecordBatchReader, + df: pa.Table | pa.RecordBatchReader | ArrowStreamExportable, snapshot_properties: dict[str, str] = EMPTY_DICT, branch: str | None = MAIN_BRANCH, ) -> None: """ Shorthand API for appending PyArrow data to the table. - Accepts either a ``pa.Table`` or a streaming ``pa.RecordBatchReader``. + Accepts a ``pa.Table``, a streaming ``pa.RecordBatchReader``, or any object + implementing the Arrow PyCapsule stream interface. See :meth:`Transaction.append` for streaming semantics and partition limitations. @@ -1854,7 +1864,7 @@ def dynamic_partition_overwrite( def overwrite( self, - df: pa.Table | pa.RecordBatchReader, + df: pa.Table | pa.RecordBatchReader | ArrowStreamExportable, overwrite_filter: BooleanExpression | str = ALWAYS_TRUE, snapshot_properties: dict[str, str] = EMPTY_DICT, case_sensitive: bool = True, @@ -1863,7 +1873,8 @@ def overwrite( """ Shorthand for overwriting the table with a PyArrow Table or RecordBatchReader. - Accepts either a ``pa.Table`` or a streaming ``pa.RecordBatchReader``. + Accepts a ``pa.Table``, a streaming ``pa.RecordBatchReader``, or any object + implementing the Arrow PyCapsule stream interface. See :meth:`Transaction.overwrite` for streaming semantics and partition limitations. diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index de4d036fe7..3acd14ecdf 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -100,6 +100,14 @@ def __missing__(self, key: K) -> V: L = TypeVar("L", str, bool, int, float, bytes, UUID, Decimal, datetime, date, time, covariant=True) +@runtime_checkable +class ArrowStreamExportable(Protocol): # pragma: no cover + """An object that exports an Arrow stream through the Arrow PyCapsule interface.""" + + def __arrow_c_stream__(self, requested_schema: object | None = None) -> object: + """Export this object as an Arrow C stream capsule.""" + + @runtime_checkable class StructProtocol(Protocol): # pragma: no cover """A generic protocol used by accessors to get and set at positions of an object.""" diff --git a/tests/table/test_init.py b/tests/table/test_init.py index 547ca94fc9..9d7c3b96b8 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -2595,6 +2595,58 @@ def test_overwrite_delete_data_file_on_bucket_partition(catalog: Catalog) -> Non assert table.scan().to_arrow().num_rows == 0 +def test_append_accepts_arrow_capsule_exporter(catalog: Catalog) -> None: + import pyarrow as pa + + from pyiceberg.io.pyarrow import schema_to_pyarrow + + class ArrowCapsuleTable: + def __init__(self, table: pa.Table) -> None: + self._table = table + + def __arrow_c_stream__(self, requested_schema: Any = None) -> Any: + return self._table.__arrow_c_stream__(requested_schema) + + catalog.create_namespace("default") + schema = Schema(NestedField(1, "x", IntegerType(), required=False)) + table = catalog.create_table("default.arrow_capsule_append", schema=schema) + + table.append(ArrowCapsuleTable(pa.table({"x": [1, 2]}, schema=schema_to_pyarrow(schema)))) + + assert table.scan().to_arrow().to_pylist() == [{"x": 1}, {"x": 2}] + + +def test_overwrite_accepts_arrow_capsule_exporter(catalog: Catalog) -> None: + import pyarrow as pa + + from pyiceberg.io.pyarrow import schema_to_pyarrow + + class ArrowCapsuleTable: + def __init__(self, table: pa.Table) -> None: + self._table = table + + def __arrow_c_stream__(self, requested_schema: Any = None) -> Any: + return self._table.__arrow_c_stream__(requested_schema) + + catalog.create_namespace("default") + schema = Schema(NestedField(1, "x", IntegerType(), required=False)) + table = catalog.create_table("default.arrow_capsule_overwrite", schema=schema) + table.append(pa.table({"x": [1, 2]}, schema=schema_to_pyarrow(schema))) + + table.overwrite(ArrowCapsuleTable(pa.table({"x": [3]}, schema=schema_to_pyarrow(schema)))) + + assert table.scan().to_arrow().to_pylist() == [{"x": 3}] + + +def test_append_rejects_object_without_arrow_capsule(catalog: Catalog) -> None: + catalog.create_namespace("default") + schema = Schema(NestedField(1, "x", IntegerType(), required=False)) + table = catalog.create_table("default.arrow_capsule_rejected", schema=schema) + + with pytest.raises(ValueError, match="Arrow PyCapsule"): + table.append(object()) # type: ignore[arg-type] + + def test_partial_delete_keeps_earlier_deletes_outside_the_filter(catalog: Catalog) -> None: """Rewriting a file applies all of its delete files, not only those whose bounds can match the delete filter.""" import pyarrow as pa From 781f3f701fe4016b2b24e84ecf5c8a537aabc24b Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Sat, 21 Nov 2026 03:18:54 +0530 Subject: [PATCH 2/2] Update the expected error for unsupported write inputs --- tests/catalog/test_catalog_behaviors.py | 2 +- tests/integration/test_writes/test_partitioned_writes.py | 2 +- tests/integration/test_writes/test_writes.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index a5099e4e6b..25a26bf230 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -1475,7 +1475,7 @@ def test_append_invalid_input_type_raises(catalog: Catalog) -> None: identifier = f"default.append_invalid_input_{catalog.name}" pa_table = _simple_arrow_table() tbl = catalog.create_table(identifier=identifier, schema=pa_table.schema) - with pytest.raises(ValueError, match="Expected pa.Table or pa.RecordBatchReader"): + with pytest.raises(ValueError, match="Expected pa.Table, pa.RecordBatchReader, or an Arrow PyCapsule exporter"): tbl.append("not an arrow object") diff --git a/tests/integration/test_writes/test_partitioned_writes.py b/tests/integration/test_writes/test_partitioned_writes.py index 1d1488255f..798f61fbe4 100644 --- a/tests/integration/test_writes/test_partitioned_writes.py +++ b/tests/integration/test_writes/test_partitioned_writes.py @@ -768,7 +768,7 @@ def test_invalid_arguments(spark: SparkSession, session_catalog: Catalog) -> Non properties={"format-version": "1"}, ) - with pytest.raises(ValueError, match="Expected pa.Table or pa.RecordBatchReader, got: not a df"): + with pytest.raises(ValueError, match="Expected pa.Table, pa.RecordBatchReader, or an Arrow PyCapsule exporter"): tbl.append("not a df") diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index e0d647c2e9..824a8b985d 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -792,10 +792,10 @@ def test_invalid_arguments(spark: SparkSession, session_catalog: Catalog, arrow_ identifier = "default.arrow_data_files" tbl = _create_table(session_catalog, identifier, {"format-version": "1"}, []) - with pytest.raises(ValueError, match="Expected pa.Table or pa.RecordBatchReader, got: not a df"): + with pytest.raises(ValueError, match="Expected pa.Table, pa.RecordBatchReader, or an Arrow PyCapsule exporter"): tbl.overwrite("not a df") - with pytest.raises(ValueError, match="Expected pa.Table or pa.RecordBatchReader, got: not a df"): + with pytest.raises(ValueError, match="Expected pa.Table, pa.RecordBatchReader, or an Arrow PyCapsule exporter"): tbl.append("not a df")