From fe8c8e27aa35d29c8c827376203e844b1ba7d939 Mon Sep 17 00:00:00 2001 From: Shivadarshan Date: Wed, 18 Nov 2026 12:16:39 +0530 Subject: [PATCH 1/3] GH-3970: Match the file's list and map structure when converting an Avro requested projection --- .../apache/parquet/avro/AvroReadSupport.java | 109 ++++++++- .../parquet/avro/TestArrayCompatibility.java | 226 ++++++++++++++++++ .../org/apache/parquet/cli/ReadCliTest.java | 39 +++ 3 files changed, 372 insertions(+), 2 deletions(-) diff --git a/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java b/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java index ee8f9ab272..d2d65d8cbf 100644 --- a/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java +++ b/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java @@ -18,7 +18,9 @@ */ package org.apache.parquet.avro; +import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -140,8 +142,13 @@ public ReadContext init( String requestedProjectionString = configuration.get(AVRO_REQUESTED_PROJECTION); if (requestedProjectionString != null) { Schema avroRequestedProjection = new Schema.Parser().parse(requestedProjectionString); - projection = new AvroSchemaConverter(getDerivedListEncodingConf(configuration, fileSchema)) - .convert(avroRequestedProjection); + ParquetConfiguration listEncodingConf = getDerivedListEncodingConf(configuration, fileSchema); + projection = new AvroSchemaConverter(listEncodingConf).convert(avroRequestedProjection); + boolean oldListStructure = listEncodingConf.getBoolean( + AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE, AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE_DEFAULT); + projection = new MessageType( + projection.getName(), + alignListStructures(projection, fileSchema, avroRequestedProjection, oldListStructure)); } String avroReadSchema = configuration.get(AVRO_READ_SCHEMA); @@ -269,6 +276,104 @@ private static ParquetConfiguration getDerivedListEncodingConf( return copiedConfiguration; } + // The projection converted from the Avro schema writes every list with the same structure, so it + // only selects a list's columns if the file wrote the list that way too. Use the repeated and + // element names of each list in the file instead; the converters read all the + // backward-compatible list structures. + private static List alignListStructures( + GroupType projected, GroupType file, Schema record, boolean oldListStructure) { + List fields = new ArrayList<>(projected.getFieldCount()); + for (Type field : projected.getFields()) { + Schema.Field avroField = record.getField(field.getName()); + if (avroField != null && file.containsField(field.getName())) { + fields.add(alignListStructures( + field, file.getType(field.getName()), avroField.schema(), oldListStructure)); + } else { + fields.add(field); + } + } + return fields; + } + + private static Type alignListStructures(Type projected, Type file, Schema schema, boolean oldListStructure) { + if (projected.isPrimitive() || file.isPrimitive()) { + return projected; + } + GroupType projectedGroup = projected.asGroupType(); + GroupType fileGroup = file.asGroupType(); + Schema nonNullSchema = AvroSchemaConverter.getNonNull(schema); + switch (nonNullSchema.getType()) { + case RECORD: + return projectedGroup.withNewFields( + alignListStructures(projectedGroup, fileGroup, nonNullSchema, oldListStructure)); + case ARRAY: + return alignList(projectedGroup, fileGroup, nonNullSchema.getElementType(), oldListStructure); + case MAP: + return alignMap(projectedGroup, fileGroup, nonNullSchema.getValueType(), oldListStructure); + default: + return projected; + } + } + + private static Type alignList(GroupType projected, GroupType file, Schema elementSchema, boolean oldListStructure) { + if (!(projected.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) + || !(file.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) + || file.isRepetition(Type.Repetition.REPEATED) + || file.getFieldCount() != 1 + || !file.getType(0).isRepetition(Type.Repetition.REPEATED)) { + return projected; + } + Type projectedRepeated = projected.getType(0); + Type projectedElement = oldListStructure + ? projectedRepeated + : projectedRepeated.asGroupType().getType(0); + Type fileRepeated = file.getType(0); + Type alignedRepeated; + // Decide which field is the element the same way AvroCollectionConverter does + if (AvroRecordConverter.isElementType(fileRepeated, AvroSchemaConverter.getNonNull(elementSchema))) { + alignedRepeated = alignElement(projectedElement, fileRepeated, elementSchema, oldListStructure); + } else { + GroupType fileRepeatedGroup = fileRepeated.asGroupType(); + Type alignedElement = + alignElement(projectedElement, fileRepeatedGroup.getType(0), elementSchema, oldListStructure); + alignedRepeated = alignedElement == null ? null : fileRepeatedGroup.withNewFields(alignedElement); + } + return alignedRepeated == null ? projected : projected.withNewFields(alignedRepeated); + } + + private static Type alignMap(GroupType projected, GroupType file, Schema valueSchema, boolean oldListStructure) { + if (projected.getFieldCount() != 1 + || file.getFieldCount() != 1 + || file.getType(0).isPrimitive() + || !file.getType(0).isRepetition(Type.Repetition.REPEATED)) { + return projected; + } + GroupType projectedKeyValue = projected.getType(0).asGroupType(); + GroupType fileKeyValue = file.getType(0).asGroupType(); + if (projectedKeyValue.getFieldCount() != 2 || fileKeyValue.getFieldCount() != 2) { + return projected; + } + Type alignedValue = + alignElement(projectedKeyValue.getType(1), fileKeyValue.getType(1), valueSchema, oldListStructure); + return alignedValue == null + ? projected + : projected.withNewFields(fileKeyValue.withNewFields(fileKeyValue.getType(0), alignedValue)); + } + + // Returns the projected element with the name and repetition of the file's element, or null if + // one of them is a group and the other is not. + private static Type alignElement(Type projected, Type file, Schema schema, boolean oldListStructure) { + if (projected.isPrimitive() != file.isPrimitive()) { + return null; + } + if (file.isPrimitive()) { + return file; + } + GroupType aligned = + alignListStructures(projected, file, schema, oldListStructure).asGroupType(); + return file.asGroupType().withNewFields(aligned.getFields()); + } + private static boolean writesNewListStructure(MessageType schema) { return Boolean.TRUE.equals(allListStructuresAreThreeLevel(schema)); } diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java b/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java index 242fd2fcf0..e4247fef1c 100644 --- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java +++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java @@ -36,11 +36,13 @@ import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.parquet.DirectWriterTest; import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.io.InvalidRecordException; +import org.apache.parquet.io.api.Binary; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.MessageTypeParser; import org.junit.jupiter.api.BeforeAll; @@ -1610,6 +1612,230 @@ public void testAutoDetectTwoLevelRepeatedPrimitive() throws Exception { assertReaderContains(new AvroParquetReader<>(newAutoDetectConf(), test), expectedSchema, expectedRecord); } + @Test + public void testProjectionReadsTwoLevelListOfPrimitives() throws Exception { + // The projection is converted with the old list structure ("repeated binary array"), which + // must follow the file's repeated field name to find the column. + Path test = writeDirect( + "message ProjectTwoLevelList {" + + " required int32 id;" + + " optional group tags (LIST) {" + + " repeated binary element (UTF8);" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("id", 0); + rc.addInteger(1); + rc.endField("id", 0); + rc.startField("tags", 1); + rc.startGroup(); + rc.startField("element", 0); + rc.addBinary(Binary.fromString("a")); + rc.addBinary(Binary.fromString("b")); + rc.endField("element", 0); + rc.endGroup(); + rc.endField("tags", 1); + rc.endMessage(); + }); + + Schema projection = record("ProjectTwoLevelList", optionalField("tags", array(primitive(Schema.Type.STRING)))); + + assertReaderContains( + projectionReader(test, projection), + projection, + instance(projection, "tags", Arrays.asList(new Utf8("a"), new Utf8("b")))); + } + + @Test + public void testProjectionReadsSingleFieldGroupsWithOtherNames() throws Exception { + // A synthetic group that is not named "list" or whose element is not named "element", such as + // "bag" or the "list" / "item" names older PyArrow versions write + for (String repeatedName : Arrays.asList("bag", "list")) { + String elementName = repeatedName.equals("bag") ? "x" : "item"; + Path test = writeDirect( + "message ProjectSingleFieldGroup {" + + " required int32 id;" + + " optional group tags (LIST) {" + + " repeated group " + repeatedName + " {" + + " optional binary " + elementName + " (UTF8);" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("id", 0); + rc.addInteger(1); + rc.endField("id", 0); + rc.startField("tags", 1); + rc.startGroup(); + rc.startField(repeatedName, 0); + rc.startGroup(); + rc.startField(elementName, 0); + rc.addBinary(Binary.fromString("a")); + rc.endField(elementName, 0); + rc.endGroup(); + rc.startGroup(); // null element + rc.endGroup(); + rc.endField(repeatedName, 0); + rc.endGroup(); + rc.endField("tags", 1); + rc.endMessage(); + }); + + Schema projection = record( + "ProjectSingleFieldGroup", optionalField("tags", array(optional(primitive(Schema.Type.STRING))))); + + assertReaderContains( + projectionReader(test, projection), + projection, + instance(projection, "tags", Arrays.asList(new Utf8("a"), null))); + } + } + + @Test + public void testProjectionReadsMixedListStructures() throws Exception { + // A standard 3-level list next to a non-standard one disables auto-detection for the file + Path test = writeDirect( + "message ProjectMixedLists {" + + " optional group standard (LIST) {" + + " repeated group list {" + + " optional int32 element;" + + " }" + + " }" + + " optional group legacy (LIST) {" + + " repeated group list {" + + " optional int32 item;" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("standard", 0); + rc.startGroup(); + rc.startField("list", 0); + rc.startGroup(); + rc.startField("element", 0); + rc.addInteger(1); + rc.endField("element", 0); + rc.endGroup(); + rc.endField("list", 0); + rc.endGroup(); + rc.endField("standard", 0); + rc.startField("legacy", 1); + rc.startGroup(); + rc.startField("list", 0); + rc.startGroup(); + rc.startField("item", 0); + rc.addInteger(2); + rc.endField("item", 0); + rc.endGroup(); + rc.endField("list", 0); + rc.endGroup(); + rc.endField("legacy", 1); + rc.endMessage(); + }); + + Schema ints = array(optional(primitive(Schema.Type.INT))); + Schema projection = record("ProjectMixedLists", optionalField("standard", ints), optionalField("legacy", ints)); + + assertReaderContains( + projectionReader(test, projection), + projection, + instance(projection, "standard", Arrays.asList(1), "legacy", Arrays.asList(2))); + } + + @Test + public void testProjectionPrunesRecordsInNonStandardList() throws Exception { + Path test = writeDirect( + "message ProjectRecordsInList {" + + " optional group points (LIST) {" + + " repeated group list {" + + " optional group item {" + + " required int32 x;" + + " required int32 y;" + + " }" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("points", 0); + rc.startGroup(); + rc.startField("list", 0); + rc.startGroup(); + rc.startField("item", 0); + rc.startGroup(); + rc.startField("x", 0); + rc.addInteger(3); + rc.endField("x", 0); + rc.startField("y", 1); + rc.addInteger(4); + rc.endField("y", 1); + rc.endGroup(); + rc.endField("item", 0); + rc.endGroup(); + rc.endField("list", 0); + rc.endGroup(); + rc.endField("points", 0); + rc.endMessage(); + }); + + Schema point = record("item", field("x", primitive(Schema.Type.INT))); + Schema projection = record("ProjectRecordsInList", optionalField("points", array(optional(point)))); + + assertReaderContains( + projectionReader(test, projection), + projection, + instance(projection, "points", Arrays.asList(instance(point, "x", 3)))); + } + + @Test + public void testProjectionReadsMapWithOtherNames() throws Exception { + Path test = writeDirect( + "message ProjectMap {" + + " optional group counts (MAP_KEY_VALUE) {" + + " repeated group map {" + + " required binary key (UTF8);" + + " optional int32 value;" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("counts", 0); + rc.startGroup(); + rc.startField("map", 0); + rc.startGroup(); + rc.startField("key", 0); + rc.addBinary(Binary.fromString("a")); + rc.endField("key", 0); + rc.startField("value", 1); + rc.addInteger(5); + rc.endField("value", 1); + rc.endGroup(); + rc.endField("map", 0); + rc.endGroup(); + rc.endField("counts", 0); + rc.endMessage(); + }); + + Schema projection = + record("ProjectMap", optionalField("counts", Schema.createMap(optional(primitive(Schema.Type.INT))))); + Map counts = new HashMap<>(); + counts.put(new Utf8("a"), 5); + + assertReaderContains(projectionReader(test, projection), projection, instance(projection, "counts", counts)); + } + + private AvroParquetReader projectionReader(Path path, Schema projection) + throws IOException { + Configuration conf = new Configuration(); + AvroReadSupport.setRequestedProjection(conf, projection); + AvroReadSupport.setAvroReadSchema(conf, projection); + return new AvroParquetReader<>(conf, path); + } + private static Configuration newAutoDetectConf() { Configuration conf = new Configuration(); conf.setBoolean(AvroReadSupport.AUTO_DETECT_LIST_STRUCTURE, true); diff --git a/parquet-cli/src/test/java/org/apache/parquet/cli/ReadCliTest.java b/parquet-cli/src/test/java/org/apache/parquet/cli/ReadCliTest.java index 42a7feb680..8cc1b43be6 100644 --- a/parquet-cli/src/test/java/org/apache/parquet/cli/ReadCliTest.java +++ b/parquet-cli/src/test/java/org/apache/parquet/cli/ReadCliTest.java @@ -36,6 +36,7 @@ import org.apache.parquet.hadoop.example.ExampleParquetWriter; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; import org.apache.parquet.schema.Types; import org.junit.jupiter.api.BeforeEach; @@ -152,6 +153,44 @@ void catReadsStandardListsWithoutAvroSchema() throws Exception { assertThat(readRecords(result.output())).containsExactlyElementsOf(expected); } + @Test + void catReadsNonStandardListsWithoutAvroSchema() throws Exception { + // A 2-level list and the "list" / "item" names older PyArrow versions write + Path path = getTempFolder().toPath().resolve("legacy-lists.parquet"); + MessageType schema = MessageTypeParser.parseMessageType("message record {" + + " required int32 id;" + + " optional group two_level (LIST) { repeated binary element (STRING); }" + + " optional group items (LIST) { repeated group list { optional binary item (STRING); } }" + + "}"); + SimpleGroupFactory groups = new SimpleGroupFactory(schema); + try (ParquetWriter writer = ExampleParquetWriter.builder(new org.apache.hadoop.fs.Path(path.toUri())) + .withType(schema) + .build()) { + Group record = groups.newGroup().append("id", 0); + record.addGroup("two_level").append("element", "a").append("element", "b"); + Group items = record.addGroup("items"); + items.addGroup("list").append("item", "c"); + items.addGroup("list"); + writer.write(record); + } + + Map expected = new HashMap<>(); + expected.put("id", 0); + expected.put("two_level", List.of("a", "b")); + expected.put("items", Arrays.asList("c", null)); + + CliResult all = cli("cat", path.toString()); + assertThat(all.exitCode()).as(all.output()).isZero(); + assertThat(readRecords(all.output())).containsExactly(expected); + + CliResult projected = cli("cat", "--columns", "id,items", path.toString()); + assertThat(projected.exitCode()).as(projected.output()).isZero(); + Map expectedProjected = new HashMap<>(); + expectedProjected.put("id", 0); + expectedProjected.put("items", Arrays.asList("c", null)); + assertThat(readRecords(projected.output())).containsExactly(expectedProjected); + } + private Path writeRecords(String name, int firstId, int count) throws IOException { Path path = getTempFolder().toPath().resolve(name); MessageType schema = Types.buildMessage() From 6a4dd2f5cd880aa7f64f5f6443ff7b658b86a6e9 Mon Sep 17 00:00:00 2001 From: Shivadarshan Date: Wed, 18 Nov 2026 22:38:45 +0530 Subject: [PATCH 2/3] GH-3970: Keep projected element types and repetition when aligning with the file --- .../apache/parquet/avro/AvroReadSupport.java | 36 +++- .../parquet/avro/TestArrayCompatibility.java | 176 ++++++++++++++++++ 2 files changed, 208 insertions(+), 4 deletions(-) diff --git a/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java b/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java index d2d65d8cbf..7db41a5e8a 100644 --- a/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java +++ b/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java @@ -37,7 +37,9 @@ import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Type; +import org.apache.parquet.schema.Types; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -360,18 +362,44 @@ private static Type alignMap(GroupType projected, GroupType file, Schema valueSc : projected.withNewFields(fileKeyValue.withNewFields(fileKeyValue.getType(0), alignedValue)); } - // Returns the projected element with the name and repetition of the file's element, or null if - // one of them is a group and the other is not. + // Returns the projected element under the name of the file's element, or null if one of them is a + // group and the other is not. The type stays the projected one, so the requested schema is still + // checked against the file. The repetition is the one the converted projection gives the element: + // repeated if the file's repeated field is the element, otherwise optional only if the Avro schema + // allows null. private static Type alignElement(Type projected, Type file, Schema schema, boolean oldListStructure) { if (projected.isPrimitive() != file.isPrimitive()) { return null; } + Type.Repetition repetition; + if (file.isRepetition(Type.Repetition.REPEATED)) { + repetition = Type.Repetition.REPEATED; + } else if (isNullable(schema)) { + repetition = Type.Repetition.OPTIONAL; + } else { + repetition = Type.Repetition.REQUIRED; + } if (file.isPrimitive()) { - return file; + PrimitiveType primitive = projected.asPrimitiveType(); + Types.PrimitiveBuilder builder = Types.primitive( + primitive.getPrimitiveTypeName(), repetition) + .as(primitive.getLogicalTypeAnnotation()); + if (primitive.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) { + builder = builder.length(primitive.getTypeLength()); + } + return builder.named(file.getName()); } GroupType aligned = alignListStructures(projected, file, schema, oldListStructure).asGroupType(); - return file.asGroupType().withNewFields(aligned.getFields()); + return Types.buildGroup(repetition) + .as(aligned.getLogicalTypeAnnotation()) + .addFields(aligned.getFields().toArray(new Type[0])) + .named(file.getName()); + } + + private static boolean isNullable(Schema schema) { + return schema.getType() == Schema.Type.UNION + && schema.getTypes().stream().anyMatch(s -> s.getType() == Schema.Type.NULL); } private static boolean writesNewListStructure(MessageType schema) { diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java b/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java index e4247fef1c..5ff658eae3 100644 --- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java +++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java @@ -42,6 +42,7 @@ import org.apache.parquet.DirectWriterTest; import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.io.InvalidRecordException; +import org.apache.parquet.io.ParquetDecodingException; import org.apache.parquet.io.api.Binary; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.MessageTypeParser; @@ -1828,6 +1829,181 @@ public void testProjectionReadsMapWithOtherNames() throws Exception { assertReaderContains(projectionReader(test, projection), projection, instance(projection, "counts", counts)); } + @Test + public void testProjectionStillChecksElementTypes() throws Exception { + // The projected element keeps its own type, so a type that doesn't match the file still fails + // the requested-schema check instead of reaching a converter. + Path test = writeDirect( + "message ProjectWrongElementType {" + + " optional group items (LIST) {" + + " repeated group list {" + + " optional int32 element;" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("items", 0); + rc.startGroup(); + rc.startField("list", 0); + rc.startGroup(); + rc.startField("element", 0); + rc.addInteger(5); + rc.endField("element", 0); + rc.endGroup(); + rc.endField("list", 0); + rc.endGroup(); + rc.endField("items", 0); + rc.endMessage(); + }); + + Schema projection = record( + "ProjectWrongElementType", optionalField("items", array(optional(primitive(Schema.Type.STRING))))); + + AvroParquetReader reader = projectionReader(test, projection); + assertThatThrownBy(reader::read) + .isInstanceOf(ParquetDecodingException.class) + .cause() + .isInstanceOf(ParquetDecodingException.class) + .hasMessageContaining("incompatible types"); + } + + @Test + public void testProjectionKeepsNonNullElementsRequired() throws Exception { + // A non-null Avro element type is still requested as required, so optional elements in the file + // fail the requested-schema check as they did before. + Path test = writeDirect( + "message ProjectRequiredElements {" + + " optional group items (LIST) {" + + " repeated group list {" + + " optional int32 item;" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("items", 0); + rc.startGroup(); + rc.startField("list", 0); + rc.startGroup(); + rc.startField("item", 0); + rc.addInteger(5); + rc.endField("item", 0); + rc.endGroup(); + rc.endField("list", 0); + rc.endGroup(); + rc.endField("items", 0); + rc.endMessage(); + }); + + Schema projection = + record("ProjectRequiredElements", optionalField("items", array(primitive(Schema.Type.INT)))); + + AvroParquetReader reader = projectionReader(test, projection); + assertThatThrownBy(reader::read) + .isInstanceOf(ParquetDecodingException.class) + .cause() + .isInstanceOf(ParquetDecodingException.class) + .hasMessageContaining("incompatible types"); + } + + @Test + public void testProjectionReadsNestedNonStandardLists() throws Exception { + // A "list" / "item" list whose elements are 2-level lists + Path test = writeDirect( + "message ProjectNestedLists {" + + " optional group matrix (LIST) {" + + " repeated group list {" + + " optional group item (LIST) {" + + " repeated int32 element;" + + " }" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("matrix", 0); + rc.startGroup(); + rc.startField("list", 0); + rc.startGroup(); + rc.startField("item", 0); + rc.startGroup(); + rc.startField("element", 0); + rc.addInteger(1); + rc.addInteger(2); + rc.endField("element", 0); + rc.endGroup(); + rc.endField("item", 0); + rc.endGroup(); + rc.endField("list", 0); + rc.endGroup(); + rc.endField("matrix", 0); + rc.endMessage(); + }); + + Schema projection = record( + "ProjectNestedLists", optionalField("matrix", array(optional(array(primitive(Schema.Type.INT)))))); + + assertReaderContains( + projectionReader(test, projection), + projection, + instance(projection, "matrix", Arrays.asList(Arrays.asList(1, 2)))); + } + + @Test + public void testProjectionReadsMapOfNonStandardLists() throws Exception { + // A map whose values are "bag" lists + Path test = writeDirect( + "message ProjectMapOfLists {" + + " optional group lists (MAP) {" + + " repeated group key_value {" + + " required binary key (UTF8);" + + " optional group value (LIST) {" + + " repeated group bag {" + + " optional int32 x;" + + " }" + + " }" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("lists", 0); + rc.startGroup(); + rc.startField("key_value", 0); + rc.startGroup(); + rc.startField("key", 0); + rc.addBinary(Binary.fromString("a")); + rc.endField("key", 0); + rc.startField("value", 1); + rc.startGroup(); + rc.startField("bag", 0); + rc.startGroup(); + rc.startField("x", 0); + rc.addInteger(3); + rc.endField("x", 0); + rc.endGroup(); + rc.startGroup(); // null element + rc.endGroup(); + rc.endField("bag", 0); + rc.endGroup(); + rc.endField("value", 1); + rc.endGroup(); + rc.endField("key_value", 0); + rc.endGroup(); + rc.endField("lists", 0); + rc.endMessage(); + }); + + Schema projection = record( + "ProjectMapOfLists", + optionalField("lists", Schema.createMap(optional(array(optional(primitive(Schema.Type.INT))))))); + Map lists = new HashMap<>(); + lists.put(new Utf8("a"), Arrays.asList(3, null)); + + assertReaderContains(projectionReader(test, projection), projection, instance(projection, "lists", lists)); + } + private AvroParquetReader projectionReader(Path path, Schema projection) throws IOException { Configuration conf = new Configuration(); From eacd64092628dad076a9d3f6b2dfa38999670baf Mon Sep 17 00:00:00 2001 From: Shivadarshan Date: Thu, 19 Nov 2026 19:04:51 +0530 Subject: [PATCH 3/3] GH-3970: Keep the projection of a list the read schema would read differently --- .../apache/parquet/avro/AvroReadSupport.java | 111 +++++++++++++++--- .../parquet/avro/TestArrayCompatibility.java | 70 +++++++++++ 2 files changed, 162 insertions(+), 19 deletions(-) diff --git a/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java b/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java index 7db41a5e8a..44f697e048 100644 --- a/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java +++ b/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java @@ -150,7 +150,12 @@ public ReadContext init( AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE, AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE_DEFAULT); projection = new MessageType( projection.getName(), - alignListStructures(projection, fileSchema, avroRequestedProjection, oldListStructure)); + alignListStructures( + projection, + fileSchema, + avroRequestedProjection, + materializedSchema(configuration, keyValueMetaData), + oldListStructure)); } String avroReadSchema = configuration.get(AVRO_READ_SCHEMA); @@ -283,13 +288,17 @@ private static ParquetConfiguration getDerivedListEncodingConf( // element names of each list in the file instead; the converters read all the // backward-compatible list structures. private static List alignListStructures( - GroupType projected, GroupType file, Schema record, boolean oldListStructure) { + GroupType projected, GroupType file, Schema record, Schema readRecord, boolean oldListStructure) { List fields = new ArrayList<>(projected.getFieldCount()); for (Type field : projected.getFields()) { Schema.Field avroField = record.getField(field.getName()); if (avroField != null && file.containsField(field.getName())) { fields.add(alignListStructures( - field, file.getType(field.getName()), avroField.schema(), oldListStructure)); + field, + file.getType(field.getName()), + avroField.schema(), + readField(readRecord, field.getName()), + oldListStructure)); } else { fields.add(field); } @@ -297,7 +306,8 @@ private static List alignListStructures( return fields; } - private static Type alignListStructures(Type projected, Type file, Schema schema, boolean oldListStructure) { + private static Type alignListStructures( + Type projected, Type file, Schema schema, Schema readSchema, boolean oldListStructure) { if (projected.isPrimitive() || file.isPrimitive()) { return projected; } @@ -307,17 +317,32 @@ private static Type alignListStructures(Type projected, Type file, Schema schema switch (nonNullSchema.getType()) { case RECORD: return projectedGroup.withNewFields( - alignListStructures(projectedGroup, fileGroup, nonNullSchema, oldListStructure)); + alignListStructures(projectedGroup, fileGroup, nonNullSchema, readSchema, oldListStructure)); case ARRAY: - return alignList(projectedGroup, fileGroup, nonNullSchema.getElementType(), oldListStructure); + return alignList( + projectedGroup, + fileGroup, + nonNullSchema.getElementType(), + readChild(readSchema, Schema.Type.ARRAY), + oldListStructure); case MAP: - return alignMap(projectedGroup, fileGroup, nonNullSchema.getValueType(), oldListStructure); + return alignMap( + projectedGroup, + fileGroup, + nonNullSchema.getValueType(), + readChild(readSchema, Schema.Type.MAP), + oldListStructure); default: return projected; } } - private static Type alignList(GroupType projected, GroupType file, Schema elementSchema, boolean oldListStructure) { + private static Type alignList( + GroupType projected, + GroupType file, + Schema elementSchema, + Schema readElementSchema, + boolean oldListStructure) { if (!(projected.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) || !(file.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) || file.isRepetition(Type.Repetition.REPEATED) @@ -332,18 +357,32 @@ private static Type alignList(GroupType projected, GroupType file, Schema elemen Type fileRepeated = file.getType(0); Type alignedRepeated; // Decide which field is the element the same way AvroCollectionConverter does - if (AvroRecordConverter.isElementType(fileRepeated, AvroSchemaConverter.getNonNull(elementSchema))) { - alignedRepeated = alignElement(projectedElement, fileRepeated, elementSchema, oldListStructure); + boolean repeatedIsElement = + AvroRecordConverter.isElementType(fileRepeated, AvroSchemaConverter.getNonNull(elementSchema)); + if (repeatedIsElement) { + alignedRepeated = + alignElement(projectedElement, fileRepeated, elementSchema, readElementSchema, oldListStructure); } else { GroupType fileRepeatedGroup = fileRepeated.asGroupType(); - Type alignedElement = - alignElement(projectedElement, fileRepeatedGroup.getType(0), elementSchema, oldListStructure); + Type alignedElement = alignElement( + projectedElement, fileRepeatedGroup.getType(0), elementSchema, readElementSchema, oldListStructure); alignedRepeated = alignedElement == null ? null : fileRepeatedGroup.withNewFields(alignedElement); } - return alignedRepeated == null ? projected : projected.withNewFields(alignedRepeated); + // The converter decides again on the aligned repeated field, with the schema it reads with. Pruning + // a group element can change that decision, for example when the read schema requires a field the + // projection leaves out, so keep the converted projection rather than misread the list. + if (alignedRepeated == null + || (readElementSchema != null + && AvroRecordConverter.isElementType( + alignedRepeated, AvroSchemaConverter.getNonNull(readElementSchema)) + != repeatedIsElement)) { + return projected; + } + return projected.withNewFields(alignedRepeated); } - private static Type alignMap(GroupType projected, GroupType file, Schema valueSchema, boolean oldListStructure) { + private static Type alignMap( + GroupType projected, GroupType file, Schema valueSchema, Schema readValueSchema, boolean oldListStructure) { if (projected.getFieldCount() != 1 || file.getFieldCount() != 1 || file.getType(0).isPrimitive() @@ -355,8 +394,8 @@ private static Type alignMap(GroupType projected, GroupType file, Schema valueSc if (projectedKeyValue.getFieldCount() != 2 || fileKeyValue.getFieldCount() != 2) { return projected; } - Type alignedValue = - alignElement(projectedKeyValue.getType(1), fileKeyValue.getType(1), valueSchema, oldListStructure); + Type alignedValue = alignElement( + projectedKeyValue.getType(1), fileKeyValue.getType(1), valueSchema, readValueSchema, oldListStructure); return alignedValue == null ? projected : projected.withNewFields(fileKeyValue.withNewFields(fileKeyValue.getType(0), alignedValue)); @@ -367,7 +406,8 @@ private static Type alignMap(GroupType projected, GroupType file, Schema valueSc // checked against the file. The repetition is the one the converted projection gives the element: // repeated if the file's repeated field is the element, otherwise optional only if the Avro schema // allows null. - private static Type alignElement(Type projected, Type file, Schema schema, boolean oldListStructure) { + private static Type alignElement( + Type projected, Type file, Schema schema, Schema readSchema, boolean oldListStructure) { if (projected.isPrimitive() != file.isPrimitive()) { return null; } @@ -389,14 +429,47 @@ private static Type alignElement(Type projected, Type file, Schema schema, boole } return builder.named(file.getName()); } - GroupType aligned = - alignListStructures(projected, file, schema, oldListStructure).asGroupType(); + GroupType aligned = alignListStructures(projected, file, schema, readSchema, oldListStructure) + .asGroupType(); return Types.buildGroup(repetition) .as(aligned.getLogicalTypeAnnotation()) .addFields(aligned.getFields().toArray(new Type[0])) .named(file.getName()); } + // The Avro schema the converters read with, if it is not converted from the projection itself, as + // prepareForRead chooses it. + private static Schema materializedSchema(ParquetConfiguration configuration, Map keyValueMetaData) { + String schema = configuration.get(AVRO_READ_SCHEMA); + if (schema == null) { + schema = keyValueMetaData.get(AVRO_SCHEMA_METADATA_KEY); + } + if (schema == null) { + schema = keyValueMetaData.get(OLD_AVRO_SCHEMA_METADATA_KEY); + } + return schema == null ? null : new Schema.Parser().parse(schema); + } + + private static Schema readField(Schema readRecord, String name) { + if (readRecord == null) { + return null; + } + Schema record = AvroSchemaConverter.getNonNull(readRecord); + Schema.Field field = record.getType() == Schema.Type.RECORD ? record.getField(name) : null; + return field == null ? null : field.schema(); + } + + private static Schema readChild(Schema readSchema, Schema.Type type) { + if (readSchema == null) { + return null; + } + Schema schema = AvroSchemaConverter.getNonNull(readSchema); + if (schema.getType() != type) { + return null; + } + return type == Schema.Type.ARRAY ? schema.getElementType() : schema.getValueType(); + } + private static boolean isNullable(Schema schema) { return schema.getType() == Schema.Type.UNION && schema.getTypes().stream().anyMatch(s -> s.getType() == Schema.Type.NULL); diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java b/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java index 5ff658eae3..8367cd3a3e 100644 --- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java +++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java @@ -2004,6 +2004,68 @@ public void testProjectionReadsMapOfNonStandardLists() throws Exception { assertReaderContains(projectionReader(test, projection), projection, instance(projection, "lists", lists)); } + @Test + public void testProjectionOfTwoLevelRecordsFollowsTheReadSchema() throws Exception { + // A 2-level list of records with one of their fields projected. The converters decide again + // whether the pruned repeated group is the element, with the read schema. + Path test = writeDirect( + "message ProjectTwoLevelRecords {" + + " optional group points (LIST) {" + + " repeated group point {" + + " required int32 x;" + + " required int32 y;" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("points", 0); + rc.startGroup(); + rc.startField("point", 0); + rc.startGroup(); + rc.startField("x", 0); + rc.addInteger(1); + rc.endField("x", 0); + rc.startField("y", 1); + rc.addInteger(2); + rc.endField("y", 1); + rc.endGroup(); + rc.endField("point", 0); + rc.endGroup(); + rc.endField("points", 0); + rc.endMessage(); + }); + + Schema x = record("point", field("x", primitive(Schema.Type.INT))); + Schema projection = record("ProjectTwoLevelRecords", optionalField("points", array(x))); + + // the read schema is the projection + assertReaderContains( + projectionReader(test, projection), + projection, + instance(projection, "points", Arrays.asList(instance(x, "x", 1)))); + + // the read schema adds a field with a default + Schema.Field yWithDefault = new Schema.Field("y", primitive(Schema.Type.INT), null, 7); + Schema xy = Schema.createRecord( + "point", null, null, false, Arrays.asList(field("x", primitive(Schema.Type.INT)), yWithDefault)); + Schema readSchema = record("ProjectTwoLevelRecords", optionalField("points", array(xy))); + assertReaderContains( + projectionReader(test, projection, readSchema), + readSchema, + instance(readSchema, "points", Arrays.asList(instance(xy, "x", 1, "y", 7)))); + + // the read schema requires a field the projection leaves out, so the pruned group would not be + // read as the element: the list keeps the converted projection, as before the alignment + Schema xyRequired = + record("point", field("x", primitive(Schema.Type.INT)), field("y", primitive(Schema.Type.INT))); + Schema requiredReadSchema = record("ProjectTwoLevelRecords", optionalField("points", array(xyRequired))); + assertReaderContains( + projectionReader(test, projection, requiredReadSchema), + requiredReadSchema, + instance(requiredReadSchema, "points", null)); + } + private AvroParquetReader projectionReader(Path path, Schema projection) throws IOException { Configuration conf = new Configuration(); @@ -2012,6 +2074,14 @@ private AvroParquetReader projectionReader(Path pat return new AvroParquetReader<>(conf, path); } + private AvroParquetReader projectionReader( + Path path, Schema projection, Schema readSchema) throws IOException { + Configuration conf = new Configuration(); + AvroReadSupport.setRequestedProjection(conf, projection); + AvroReadSupport.setAvroReadSchema(conf, readSchema); + return new AvroParquetReader<>(conf, path); + } + private static Configuration newAutoDetectConf() { Configuration conf = new Configuration(); conf.setBoolean(AvroReadSupport.AUTO_DETECT_LIST_STRUCTURE, true);