From c5ae115c6ac05ad11bbdaa2e4aa31b5d72565379 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Sat, 21 Nov 2026 01:59:40 +0530 Subject: [PATCH 1/2] Core: Control inferred column metrics and collect nested bounds Nested fields were downgraded to counts-only statistics, which prevents bounds pruning for struct fields such as GeoParquet bounding boxes. Add write.metadata.metrics.max-inferred-column-defaults, defaulting to 100, and apply the default mode to at most that many primitive fields, matching the Java MetricsConfig. An explicitly configured default mode still applies to every column, and per-column overrides still work beyond the limit. With the limit in place, nested fields collect the same bounds as top-level fields. --- mkdocs/docs/configuration.md | 3 +- pyiceberg/io/pyarrow.py | 71 +++++++++++++++++++++++++++---- pyiceberg/table/__init__.py | 2 + tests/io/test_pyarrow_stats.py | 77 ++++++++++++++++++++++++++++++---- 4 files changed, 137 insertions(+), 16 deletions(-) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index c2f3ef5870..d6c1dbad79 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -88,8 +88,9 @@ Iceberg tables support table properties to configure table behavior. | `write.target-file-size-bytes` | Size in bytes | 536870912 (512MB) | Target size of generated data files. Measured in uncompressed in-memory Arrow bytes, so files on disk are typically smaller after compression | | `write.format.default` | `{parquet}` | parquet | File format for data files written by PyIceberg. Only Parquet is supported | | `write.avro.compression-codec` | `{gzip,zstd,snappy}` | gzip | Compression codec for manifest files and manifest lists. For `snappy`, install with `pip install pyiceberg[snappy]` | -| `write.metadata.metrics.default` | `{none,counts,truncate(N),full}` | truncate(16) | Default column metrics for data files: `none` records only column sizes, `counts` adds value and null counts, `full` adds lower/upper bounds, and `truncate(N)` truncates string and binary bounds to N characters or bytes. Nested fields collect at most counts | +| `write.metadata.metrics.default` | `{none,counts,truncate(N),full}` | truncate(16) | Default column metrics for data files: `none` records only column sizes, `counts` adds value and null counts, `full` adds lower/upper bounds, and `truncate(N)` truncates string and binary bounds to N characters or bytes. Applies to nested fields as well | | `write.metadata.metrics.column.{column-name}` | `{none,counts,truncate(N),full}` | (value of `write.metadata.metrics.default`) | Overrides `write.metadata.metrics.default` for a single column. Use the dot-separated name for nested fields | +| `write.metadata.metrics.max-inferred-column-defaults` | Integer | 100 | Maximum number of columns that get `write.metadata.metrics.default` when it is not configured explicitly. Further columns collect no metrics unless they have a `write.metadata.metrics.column` override | | `write.summary.partition-limit` | Integer | 0 | Maximum number of changed partitions for which per-partition summaries are included in the snapshot summary. The default of 0 disables them | | `write.metadata.previous-versions-max` | Integer | 100 | The max number of previous version metadata files to keep before deleting after commit. | | `write.metadata.delete-after-commit.enabled` | Boolean | False | Whether to automatically delete old *tracked* metadata files after each table commit. It will retain a number of the most recent metadata files, which can be set using property `write.metadata.previous-versions-max`. | diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 20333882b5..89ebbea69d 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -2565,6 +2565,48 @@ def match_metrics_mode(mode: str) -> MetricsMode: raise ValueError(f"Unsupported metrics mode: {mode}") +def _limit_metrics_field_ids(schema: Schema, limit: int) -> set[int]: + """Return the primitive field ids that get the default metrics mode, in schema order. + + Mirrors Java's MetricsConfig.limitFieldIds: struct fields add their primitive children + before descending into nested fields, and list and map add their primitive element, key, + or value ids before descending. + """ + field_ids: set[int] = set() + + def add(field_id: int) -> None: + if len(field_ids) < limit: + field_ids.add(field_id) + + def visit(iceberg_type: IcebergType) -> None: + if isinstance(iceberg_type, StructType): + for field in iceberg_type.fields: + if isinstance(field.field_type, PrimitiveType): + add(field.field_id) + for field in iceberg_type.fields: + if len(field_ids) >= limit: + return + if not isinstance(field.field_type, PrimitiveType): + visit(field.field_type) + elif isinstance(iceberg_type, ListType): + if isinstance(iceberg_type.element_type, PrimitiveType): + add(iceberg_type.element_id) + else: + visit(iceberg_type.element_type) + elif isinstance(iceberg_type, MapType): + if isinstance(iceberg_type.key_type, PrimitiveType): + add(iceberg_type.key_id) + if isinstance(iceberg_type.value_type, PrimitiveType): + add(iceberg_type.value_id) + if not isinstance(iceberg_type.key_type, PrimitiveType): + visit(iceberg_type.key_type) + if not isinstance(iceberg_type.value_type, PrimitiveType): + visit(iceberg_type.value_type) + + visit(schema.as_struct()) + return field_ids + + @dataclass(frozen=True) class StatisticsCollector: field_id: int @@ -2578,15 +2620,32 @@ class PyArrowStatisticsCollector(PreOrderSchemaVisitor[list[StatisticsCollector] _schema: Schema _properties: dict[str, str] _default_mode: str + _configured_default: bool + _inferred_field_ids: set[int] def __init__(self, schema: Schema, properties: dict[str, str]): from pyiceberg.table import TableProperties self._schema = schema self._properties = properties + self._configured_default = TableProperties.DEFAULT_WRITE_METRICS_MODE in self._properties self._default_mode = self._properties.get( TableProperties.DEFAULT_WRITE_METRICS_MODE, TableProperties.DEFAULT_WRITE_METRICS_MODE_DEFAULT ) + max_inferred = property_as_int( + properties, + TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, + TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT, + ) + if max_inferred < 0: + logger.warning( + "Invalid value for %s (negative): %s, falling back to %s", + TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, + max_inferred, + TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT, + ) + max_inferred = TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT + self._inferred_field_ids = _limit_metrics_field_ids(schema, max_inferred) def schema( self, schema: Schema, struct_result: Callable[[], builtins.list[StatisticsCollector]] @@ -2629,11 +2688,14 @@ def primitive(self, primitive: PrimitiveType) -> builtins.list[StatisticsCollect if column_name is None: return [] - metrics_mode = match_metrics_mode(self._default_mode) - col_mode = self._properties.get(f"{TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX}.{column_name}") if col_mode: metrics_mode = match_metrics_mode(col_mode) + elif not self._configured_default and self._field_id not in self._inferred_field_ids: + # Only the first max-inferred-column-defaults primitive fields get the default mode. + return [] + else: + metrics_mode = match_metrics_mode(self._default_mode) if ( not (isinstance(primitive, StringType) or isinstance(primitive, BinaryType)) @@ -2641,11 +2703,6 @@ def primitive(self, primitive: PrimitiveType) -> builtins.list[StatisticsCollect ): metrics_mode = MetricsMode(MetricModeTypes.FULL) - is_nested = column_name.find(".") >= 0 - - if is_nested and metrics_mode.type in [MetricModeTypes.TRUNCATE, MetricModeTypes.FULL]: - metrics_mode = MetricsMode(MetricModeTypes.COUNTS) - return [StatisticsCollector(field_id=self._field_id, iceberg_type=primitive, mode=metrics_mode, column_name=column_name)] diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 3ea41e3f22..b86d6c1603 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -188,6 +188,8 @@ class TableProperties: DEFAULT_WRITE_METRICS_MODE_DEFAULT = "truncate(16)" METRICS_MODE_COLUMN_CONF_PREFIX = "write.metadata.metrics.column" + METRICS_MAX_INFERRED_COLUMN_DEFAULTS = "write.metadata.metrics.max-inferred-column-defaults" + METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT = 100 WRITE_PARTITION_SUMMARY_LIMIT = "write.summary.partition-limit" WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT = 0 diff --git a/tests/io/test_pyarrow_stats.py b/tests/io/test_pyarrow_stats.py index 1c5e802a26..62d61db636 100644 --- a/tests/io/test_pyarrow_stats.py +++ b/tests/io/test_pyarrow_stats.py @@ -58,6 +58,7 @@ ) from pyiceberg.manifest import DataFile, FileFormat from pyiceberg.schema import Schema, pre_order_visit +from pyiceberg.table import TableProperties from pyiceberg.table.metadata import ( TableMetadata, TableMetadataUtil, @@ -336,11 +337,11 @@ def test_bounds() -> None: ) datafile = DataFile.from_args(**statistics.to_serialized_dict()) - assert len(datafile.lower_bounds) == 2 + assert len(datafile.lower_bounds) == 7 assert datafile.lower_bounds[1].decode() == "aaaaaaaaaaaaaaaa" assert datafile.lower_bounds[2] == STRUCT_FLOAT.pack(1.69) - assert len(datafile.upper_bounds) == 2 + assert len(datafile.upper_bounds) == 7 assert datafile.upper_bounds[1].decode() == "zzzzzzzzzzzzzzz{" assert datafile.upper_bounds[2] == STRUCT_FLOAT.pack(100) @@ -422,11 +423,11 @@ def test_metrics_mode_full() -> None: assert len(datafile.null_value_counts) == 7 assert len(datafile.nan_value_counts) == 0 - assert len(datafile.lower_bounds) == 2 + assert len(datafile.lower_bounds) == 7 assert datafile.lower_bounds[1].decode() == "aaaaaaaaaaaaaaaaaaaa" assert datafile.lower_bounds[2] == STRUCT_FLOAT.pack(1.69) - assert len(datafile.upper_bounds) == 2 + assert len(datafile.upper_bounds) == 7 assert datafile.upper_bounds[1].decode() == "zzzzzzzzzzzzzzzzzzzz" assert datafile.upper_bounds[2] == STRUCT_FLOAT.pack(100) @@ -447,11 +448,11 @@ def test_metrics_mode_non_default_trunc() -> None: assert len(datafile.null_value_counts) == 7 assert len(datafile.nan_value_counts) == 0 - assert len(datafile.lower_bounds) == 2 + assert len(datafile.lower_bounds) == 7 assert datafile.lower_bounds[1].decode() == "aa" assert datafile.lower_bounds[2] == STRUCT_FLOAT.pack(1.69) - assert len(datafile.upper_bounds) == 2 + assert len(datafile.upper_bounds) == 7 assert datafile.upper_bounds[1].decode() == "z{" assert datafile.upper_bounds[2] == STRUCT_FLOAT.pack(100) @@ -473,15 +474,75 @@ def test_column_metrics_mode() -> None: assert len(datafile.null_value_counts) == 6 assert len(datafile.nan_value_counts) == 0 - assert len(datafile.lower_bounds) == 1 + assert len(datafile.lower_bounds) == 6 assert datafile.lower_bounds[2] == STRUCT_FLOAT.pack(1.69) assert 1 not in datafile.lower_bounds - assert len(datafile.upper_bounds) == 1 + assert len(datafile.upper_bounds) == 6 assert datafile.upper_bounds[2] == STRUCT_FLOAT.pack(100) assert 1 not in datafile.upper_bounds +def test_metrics_max_inferred_column_defaults_limits_stats() -> None: + schema = Schema( + NestedField(1, "a", IntegerType(), required=False), + NestedField(2, "b", IntegerType(), required=False), + NestedField(3, "c", IntegerType(), required=False), + ) + + plan = compute_statistics_plan(schema, {TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS: "2"}) + + assert set(plan) == {1, 2} + + +def test_metrics_max_inferred_column_defaults_keeps_column_overrides() -> None: + schema = Schema( + NestedField(1, "a", IntegerType(), required=False), + NestedField(2, "b", IntegerType(), required=False), + NestedField(3, "c", IntegerType(), required=False), + ) + + plan = compute_statistics_plan( + schema, + { + TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS: "1", + f"{TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX}.c": "full", + }, + ) + + assert set(plan) == {1, 3} + + +def test_configured_metrics_default_is_not_limited() -> None: + schema = Schema( + NestedField(1, "a", IntegerType(), required=False), + NestedField(2, "b", IntegerType(), required=False), + NestedField(3, "c", IntegerType(), required=False), + ) + + plan = compute_statistics_plan( + schema, + { + TableProperties.DEFAULT_WRITE_METRICS_MODE: "counts", + TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS: "1", + }, + ) + + assert set(plan) == {1, 2, 3} + + +def test_negative_metrics_max_inferred_column_defaults_falls_back() -> None: + schema = Schema( + NestedField(1, "a", IntegerType(), required=False), + NestedField(2, "b", IntegerType(), required=False), + NestedField(3, "c", IntegerType(), required=False), + ) + + plan = compute_statistics_plan(schema, {TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS: "-1"}) + + assert set(plan) == {1, 2, 3} + + def construct_test_table_primitive_types() -> tuple[pq.FileMetaData, TableMetadataV1 | TableMetadataV2]: table_metadata = { "format-version": 2, From e762dc3308c9537438de8387fe65ba09a9f1b19b Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Sat, 21 Nov 2026 03:05:46 +0530 Subject: [PATCH 2/2] Narrow the inferred metrics limit before using it property_as_int can return None, which mypy flagged on Python 3.11. --- pyiceberg/io/pyarrow.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 89ebbea69d..b62c19a6e2 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -2637,7 +2637,9 @@ def __init__(self, schema: Schema, properties: dict[str, str]): TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT, ) - if max_inferred < 0: + if max_inferred is None: + max_inferred = TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT + elif max_inferred < 0: logger.warning( "Invalid value for %s (negative): %s, falling back to %s", TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS,