diff --git a/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java b/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java index ee8f9ab272..44f697e048 100644 --- a/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java +++ b/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java @@ -18,7 +18,9 @@ */ package org.apache.parquet.avro; +import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -35,7 +37,9 @@ import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Type; +import org.apache.parquet.schema.Types; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -140,8 +144,18 @@ public ReadContext init( String requestedProjectionString = configuration.get(AVRO_REQUESTED_PROJECTION); if (requestedProjectionString != null) { Schema avroRequestedProjection = new Schema.Parser().parse(requestedProjectionString); - projection = new AvroSchemaConverter(getDerivedListEncodingConf(configuration, fileSchema)) - .convert(avroRequestedProjection); + ParquetConfiguration listEncodingConf = getDerivedListEncodingConf(configuration, fileSchema); + projection = new AvroSchemaConverter(listEncodingConf).convert(avroRequestedProjection); + boolean oldListStructure = listEncodingConf.getBoolean( + AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE, AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE_DEFAULT); + projection = new MessageType( + projection.getName(), + alignListStructures( + projection, + fileSchema, + avroRequestedProjection, + materializedSchema(configuration, keyValueMetaData), + oldListStructure)); } String avroReadSchema = configuration.get(AVRO_READ_SCHEMA); @@ -269,6 +283,198 @@ private static ParquetConfiguration getDerivedListEncodingConf( return copiedConfiguration; } + // The projection converted from the Avro schema writes every list with the same structure, so it + // only selects a list's columns if the file wrote the list that way too. Use the repeated and + // element names of each list in the file instead; the converters read all the + // backward-compatible list structures. + private static List alignListStructures( + GroupType projected, GroupType file, Schema record, Schema readRecord, boolean oldListStructure) { + List fields = new ArrayList<>(projected.getFieldCount()); + for (Type field : projected.getFields()) { + Schema.Field avroField = record.getField(field.getName()); + if (avroField != null && file.containsField(field.getName())) { + fields.add(alignListStructures( + field, + file.getType(field.getName()), + avroField.schema(), + readField(readRecord, field.getName()), + oldListStructure)); + } else { + fields.add(field); + } + } + return fields; + } + + private static Type alignListStructures( + Type projected, Type file, Schema schema, Schema readSchema, boolean oldListStructure) { + if (projected.isPrimitive() || file.isPrimitive()) { + return projected; + } + GroupType projectedGroup = projected.asGroupType(); + GroupType fileGroup = file.asGroupType(); + Schema nonNullSchema = AvroSchemaConverter.getNonNull(schema); + switch (nonNullSchema.getType()) { + case RECORD: + return projectedGroup.withNewFields( + alignListStructures(projectedGroup, fileGroup, nonNullSchema, readSchema, oldListStructure)); + case ARRAY: + return alignList( + projectedGroup, + fileGroup, + nonNullSchema.getElementType(), + readChild(readSchema, Schema.Type.ARRAY), + oldListStructure); + case MAP: + return alignMap( + projectedGroup, + fileGroup, + nonNullSchema.getValueType(), + readChild(readSchema, Schema.Type.MAP), + oldListStructure); + default: + return projected; + } + } + + private static Type alignList( + GroupType projected, + GroupType file, + Schema elementSchema, + Schema readElementSchema, + boolean oldListStructure) { + if (!(projected.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) + || !(file.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) + || file.isRepetition(Type.Repetition.REPEATED) + || file.getFieldCount() != 1 + || !file.getType(0).isRepetition(Type.Repetition.REPEATED)) { + return projected; + } + Type projectedRepeated = projected.getType(0); + Type projectedElement = oldListStructure + ? projectedRepeated + : projectedRepeated.asGroupType().getType(0); + Type fileRepeated = file.getType(0); + Type alignedRepeated; + // Decide which field is the element the same way AvroCollectionConverter does + boolean repeatedIsElement = + AvroRecordConverter.isElementType(fileRepeated, AvroSchemaConverter.getNonNull(elementSchema)); + if (repeatedIsElement) { + alignedRepeated = + alignElement(projectedElement, fileRepeated, elementSchema, readElementSchema, oldListStructure); + } else { + GroupType fileRepeatedGroup = fileRepeated.asGroupType(); + Type alignedElement = alignElement( + projectedElement, fileRepeatedGroup.getType(0), elementSchema, readElementSchema, oldListStructure); + alignedRepeated = alignedElement == null ? null : fileRepeatedGroup.withNewFields(alignedElement); + } + // The converter decides again on the aligned repeated field, with the schema it reads with. Pruning + // a group element can change that decision, for example when the read schema requires a field the + // projection leaves out, so keep the converted projection rather than misread the list. + if (alignedRepeated == null + || (readElementSchema != null + && AvroRecordConverter.isElementType( + alignedRepeated, AvroSchemaConverter.getNonNull(readElementSchema)) + != repeatedIsElement)) { + return projected; + } + return projected.withNewFields(alignedRepeated); + } + + private static Type alignMap( + GroupType projected, GroupType file, Schema valueSchema, Schema readValueSchema, boolean oldListStructure) { + if (projected.getFieldCount() != 1 + || file.getFieldCount() != 1 + || file.getType(0).isPrimitive() + || !file.getType(0).isRepetition(Type.Repetition.REPEATED)) { + return projected; + } + GroupType projectedKeyValue = projected.getType(0).asGroupType(); + GroupType fileKeyValue = file.getType(0).asGroupType(); + if (projectedKeyValue.getFieldCount() != 2 || fileKeyValue.getFieldCount() != 2) { + return projected; + } + Type alignedValue = alignElement( + projectedKeyValue.getType(1), fileKeyValue.getType(1), valueSchema, readValueSchema, oldListStructure); + return alignedValue == null + ? projected + : projected.withNewFields(fileKeyValue.withNewFields(fileKeyValue.getType(0), alignedValue)); + } + + // Returns the projected element under the name of the file's element, or null if one of them is a + // group and the other is not. The type stays the projected one, so the requested schema is still + // checked against the file. The repetition is the one the converted projection gives the element: + // repeated if the file's repeated field is the element, otherwise optional only if the Avro schema + // allows null. + private static Type alignElement( + Type projected, Type file, Schema schema, Schema readSchema, boolean oldListStructure) { + if (projected.isPrimitive() != file.isPrimitive()) { + return null; + } + Type.Repetition repetition; + if (file.isRepetition(Type.Repetition.REPEATED)) { + repetition = Type.Repetition.REPEATED; + } else if (isNullable(schema)) { + repetition = Type.Repetition.OPTIONAL; + } else { + repetition = Type.Repetition.REQUIRED; + } + if (file.isPrimitive()) { + PrimitiveType primitive = projected.asPrimitiveType(); + Types.PrimitiveBuilder builder = Types.primitive( + primitive.getPrimitiveTypeName(), repetition) + .as(primitive.getLogicalTypeAnnotation()); + if (primitive.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) { + builder = builder.length(primitive.getTypeLength()); + } + return builder.named(file.getName()); + } + GroupType aligned = alignListStructures(projected, file, schema, readSchema, oldListStructure) + .asGroupType(); + return Types.buildGroup(repetition) + .as(aligned.getLogicalTypeAnnotation()) + .addFields(aligned.getFields().toArray(new Type[0])) + .named(file.getName()); + } + + // The Avro schema the converters read with, if it is not converted from the projection itself, as + // prepareForRead chooses it. + private static Schema materializedSchema(ParquetConfiguration configuration, Map keyValueMetaData) { + String schema = configuration.get(AVRO_READ_SCHEMA); + if (schema == null) { + schema = keyValueMetaData.get(AVRO_SCHEMA_METADATA_KEY); + } + if (schema == null) { + schema = keyValueMetaData.get(OLD_AVRO_SCHEMA_METADATA_KEY); + } + return schema == null ? null : new Schema.Parser().parse(schema); + } + + private static Schema readField(Schema readRecord, String name) { + if (readRecord == null) { + return null; + } + Schema record = AvroSchemaConverter.getNonNull(readRecord); + Schema.Field field = record.getType() == Schema.Type.RECORD ? record.getField(name) : null; + return field == null ? null : field.schema(); + } + + private static Schema readChild(Schema readSchema, Schema.Type type) { + if (readSchema == null) { + return null; + } + Schema schema = AvroSchemaConverter.getNonNull(readSchema); + if (schema.getType() != type) { + return null; + } + return type == Schema.Type.ARRAY ? schema.getElementType() : schema.getValueType(); + } + + private static boolean isNullable(Schema schema) { + return schema.getType() == Schema.Type.UNION + && schema.getTypes().stream().anyMatch(s -> s.getType() == Schema.Type.NULL); + } + private static boolean writesNewListStructure(MessageType schema) { return Boolean.TRUE.equals(allListStructuresAreThreeLevel(schema)); } diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java b/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java index 242fd2fcf0..8367cd3a3e 100644 --- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java +++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java @@ -36,11 +36,14 @@ import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.parquet.DirectWriterTest; import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.io.InvalidRecordException; +import org.apache.parquet.io.ParquetDecodingException; +import org.apache.parquet.io.api.Binary; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.MessageTypeParser; import org.junit.jupiter.api.BeforeAll; @@ -1610,6 +1613,475 @@ public void testAutoDetectTwoLevelRepeatedPrimitive() throws Exception { assertReaderContains(new AvroParquetReader<>(newAutoDetectConf(), test), expectedSchema, expectedRecord); } + @Test + public void testProjectionReadsTwoLevelListOfPrimitives() throws Exception { + // The projection is converted with the old list structure ("repeated binary array"), which + // must follow the file's repeated field name to find the column. + Path test = writeDirect( + "message ProjectTwoLevelList {" + + " required int32 id;" + + " optional group tags (LIST) {" + + " repeated binary element (UTF8);" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("id", 0); + rc.addInteger(1); + rc.endField("id", 0); + rc.startField("tags", 1); + rc.startGroup(); + rc.startField("element", 0); + rc.addBinary(Binary.fromString("a")); + rc.addBinary(Binary.fromString("b")); + rc.endField("element", 0); + rc.endGroup(); + rc.endField("tags", 1); + rc.endMessage(); + }); + + Schema projection = record("ProjectTwoLevelList", optionalField("tags", array(primitive(Schema.Type.STRING)))); + + assertReaderContains( + projectionReader(test, projection), + projection, + instance(projection, "tags", Arrays.asList(new Utf8("a"), new Utf8("b")))); + } + + @Test + public void testProjectionReadsSingleFieldGroupsWithOtherNames() throws Exception { + // A synthetic group that is not named "list" or whose element is not named "element", such as + // "bag" or the "list" / "item" names older PyArrow versions write + for (String repeatedName : Arrays.asList("bag", "list")) { + String elementName = repeatedName.equals("bag") ? "x" : "item"; + Path test = writeDirect( + "message ProjectSingleFieldGroup {" + + " required int32 id;" + + " optional group tags (LIST) {" + + " repeated group " + repeatedName + " {" + + " optional binary " + elementName + " (UTF8);" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("id", 0); + rc.addInteger(1); + rc.endField("id", 0); + rc.startField("tags", 1); + rc.startGroup(); + rc.startField(repeatedName, 0); + rc.startGroup(); + rc.startField(elementName, 0); + rc.addBinary(Binary.fromString("a")); + rc.endField(elementName, 0); + rc.endGroup(); + rc.startGroup(); // null element + rc.endGroup(); + rc.endField(repeatedName, 0); + rc.endGroup(); + rc.endField("tags", 1); + rc.endMessage(); + }); + + Schema projection = record( + "ProjectSingleFieldGroup", optionalField("tags", array(optional(primitive(Schema.Type.STRING))))); + + assertReaderContains( + projectionReader(test, projection), + projection, + instance(projection, "tags", Arrays.asList(new Utf8("a"), null))); + } + } + + @Test + public void testProjectionReadsMixedListStructures() throws Exception { + // A standard 3-level list next to a non-standard one disables auto-detection for the file + Path test = writeDirect( + "message ProjectMixedLists {" + + " optional group standard (LIST) {" + + " repeated group list {" + + " optional int32 element;" + + " }" + + " }" + + " optional group legacy (LIST) {" + + " repeated group list {" + + " optional int32 item;" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("standard", 0); + rc.startGroup(); + rc.startField("list", 0); + rc.startGroup(); + rc.startField("element", 0); + rc.addInteger(1); + rc.endField("element", 0); + rc.endGroup(); + rc.endField("list", 0); + rc.endGroup(); + rc.endField("standard", 0); + rc.startField("legacy", 1); + rc.startGroup(); + rc.startField("list", 0); + rc.startGroup(); + rc.startField("item", 0); + rc.addInteger(2); + rc.endField("item", 0); + rc.endGroup(); + rc.endField("list", 0); + rc.endGroup(); + rc.endField("legacy", 1); + rc.endMessage(); + }); + + Schema ints = array(optional(primitive(Schema.Type.INT))); + Schema projection = record("ProjectMixedLists", optionalField("standard", ints), optionalField("legacy", ints)); + + assertReaderContains( + projectionReader(test, projection), + projection, + instance(projection, "standard", Arrays.asList(1), "legacy", Arrays.asList(2))); + } + + @Test + public void testProjectionPrunesRecordsInNonStandardList() throws Exception { + Path test = writeDirect( + "message ProjectRecordsInList {" + + " optional group points (LIST) {" + + " repeated group list {" + + " optional group item {" + + " required int32 x;" + + " required int32 y;" + + " }" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("points", 0); + rc.startGroup(); + rc.startField("list", 0); + rc.startGroup(); + rc.startField("item", 0); + rc.startGroup(); + rc.startField("x", 0); + rc.addInteger(3); + rc.endField("x", 0); + rc.startField("y", 1); + rc.addInteger(4); + rc.endField("y", 1); + rc.endGroup(); + rc.endField("item", 0); + rc.endGroup(); + rc.endField("list", 0); + rc.endGroup(); + rc.endField("points", 0); + rc.endMessage(); + }); + + Schema point = record("item", field("x", primitive(Schema.Type.INT))); + Schema projection = record("ProjectRecordsInList", optionalField("points", array(optional(point)))); + + assertReaderContains( + projectionReader(test, projection), + projection, + instance(projection, "points", Arrays.asList(instance(point, "x", 3)))); + } + + @Test + public void testProjectionReadsMapWithOtherNames() throws Exception { + Path test = writeDirect( + "message ProjectMap {" + + " optional group counts (MAP_KEY_VALUE) {" + + " repeated group map {" + + " required binary key (UTF8);" + + " optional int32 value;" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("counts", 0); + rc.startGroup(); + rc.startField("map", 0); + rc.startGroup(); + rc.startField("key", 0); + rc.addBinary(Binary.fromString("a")); + rc.endField("key", 0); + rc.startField("value", 1); + rc.addInteger(5); + rc.endField("value", 1); + rc.endGroup(); + rc.endField("map", 0); + rc.endGroup(); + rc.endField("counts", 0); + rc.endMessage(); + }); + + Schema projection = + record("ProjectMap", optionalField("counts", Schema.createMap(optional(primitive(Schema.Type.INT))))); + Map counts = new HashMap<>(); + counts.put(new Utf8("a"), 5); + + assertReaderContains(projectionReader(test, projection), projection, instance(projection, "counts", counts)); + } + + @Test + public void testProjectionStillChecksElementTypes() throws Exception { + // The projected element keeps its own type, so a type that doesn't match the file still fails + // the requested-schema check instead of reaching a converter. + Path test = writeDirect( + "message ProjectWrongElementType {" + + " optional group items (LIST) {" + + " repeated group list {" + + " optional int32 element;" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("items", 0); + rc.startGroup(); + rc.startField("list", 0); + rc.startGroup(); + rc.startField("element", 0); + rc.addInteger(5); + rc.endField("element", 0); + rc.endGroup(); + rc.endField("list", 0); + rc.endGroup(); + rc.endField("items", 0); + rc.endMessage(); + }); + + Schema projection = record( + "ProjectWrongElementType", optionalField("items", array(optional(primitive(Schema.Type.STRING))))); + + AvroParquetReader reader = projectionReader(test, projection); + assertThatThrownBy(reader::read) + .isInstanceOf(ParquetDecodingException.class) + .cause() + .isInstanceOf(ParquetDecodingException.class) + .hasMessageContaining("incompatible types"); + } + + @Test + public void testProjectionKeepsNonNullElementsRequired() throws Exception { + // A non-null Avro element type is still requested as required, so optional elements in the file + // fail the requested-schema check as they did before. + Path test = writeDirect( + "message ProjectRequiredElements {" + + " optional group items (LIST) {" + + " repeated group list {" + + " optional int32 item;" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("items", 0); + rc.startGroup(); + rc.startField("list", 0); + rc.startGroup(); + rc.startField("item", 0); + rc.addInteger(5); + rc.endField("item", 0); + rc.endGroup(); + rc.endField("list", 0); + rc.endGroup(); + rc.endField("items", 0); + rc.endMessage(); + }); + + Schema projection = + record("ProjectRequiredElements", optionalField("items", array(primitive(Schema.Type.INT)))); + + AvroParquetReader reader = projectionReader(test, projection); + assertThatThrownBy(reader::read) + .isInstanceOf(ParquetDecodingException.class) + .cause() + .isInstanceOf(ParquetDecodingException.class) + .hasMessageContaining("incompatible types"); + } + + @Test + public void testProjectionReadsNestedNonStandardLists() throws Exception { + // A "list" / "item" list whose elements are 2-level lists + Path test = writeDirect( + "message ProjectNestedLists {" + + " optional group matrix (LIST) {" + + " repeated group list {" + + " optional group item (LIST) {" + + " repeated int32 element;" + + " }" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("matrix", 0); + rc.startGroup(); + rc.startField("list", 0); + rc.startGroup(); + rc.startField("item", 0); + rc.startGroup(); + rc.startField("element", 0); + rc.addInteger(1); + rc.addInteger(2); + rc.endField("element", 0); + rc.endGroup(); + rc.endField("item", 0); + rc.endGroup(); + rc.endField("list", 0); + rc.endGroup(); + rc.endField("matrix", 0); + rc.endMessage(); + }); + + Schema projection = record( + "ProjectNestedLists", optionalField("matrix", array(optional(array(primitive(Schema.Type.INT)))))); + + assertReaderContains( + projectionReader(test, projection), + projection, + instance(projection, "matrix", Arrays.asList(Arrays.asList(1, 2)))); + } + + @Test + public void testProjectionReadsMapOfNonStandardLists() throws Exception { + // A map whose values are "bag" lists + Path test = writeDirect( + "message ProjectMapOfLists {" + + " optional group lists (MAP) {" + + " repeated group key_value {" + + " required binary key (UTF8);" + + " optional group value (LIST) {" + + " repeated group bag {" + + " optional int32 x;" + + " }" + + " }" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("lists", 0); + rc.startGroup(); + rc.startField("key_value", 0); + rc.startGroup(); + rc.startField("key", 0); + rc.addBinary(Binary.fromString("a")); + rc.endField("key", 0); + rc.startField("value", 1); + rc.startGroup(); + rc.startField("bag", 0); + rc.startGroup(); + rc.startField("x", 0); + rc.addInteger(3); + rc.endField("x", 0); + rc.endGroup(); + rc.startGroup(); // null element + rc.endGroup(); + rc.endField("bag", 0); + rc.endGroup(); + rc.endField("value", 1); + rc.endGroup(); + rc.endField("key_value", 0); + rc.endGroup(); + rc.endField("lists", 0); + rc.endMessage(); + }); + + Schema projection = record( + "ProjectMapOfLists", + optionalField("lists", Schema.createMap(optional(array(optional(primitive(Schema.Type.INT))))))); + Map lists = new HashMap<>(); + lists.put(new Utf8("a"), Arrays.asList(3, null)); + + assertReaderContains(projectionReader(test, projection), projection, instance(projection, "lists", lists)); + } + + @Test + public void testProjectionOfTwoLevelRecordsFollowsTheReadSchema() throws Exception { + // A 2-level list of records with one of their fields projected. The converters decide again + // whether the pruned repeated group is the element, with the read schema. + Path test = writeDirect( + "message ProjectTwoLevelRecords {" + + " optional group points (LIST) {" + + " repeated group point {" + + " required int32 x;" + + " required int32 y;" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("points", 0); + rc.startGroup(); + rc.startField("point", 0); + rc.startGroup(); + rc.startField("x", 0); + rc.addInteger(1); + rc.endField("x", 0); + rc.startField("y", 1); + rc.addInteger(2); + rc.endField("y", 1); + rc.endGroup(); + rc.endField("point", 0); + rc.endGroup(); + rc.endField("points", 0); + rc.endMessage(); + }); + + Schema x = record("point", field("x", primitive(Schema.Type.INT))); + Schema projection = record("ProjectTwoLevelRecords", optionalField("points", array(x))); + + // the read schema is the projection + assertReaderContains( + projectionReader(test, projection), + projection, + instance(projection, "points", Arrays.asList(instance(x, "x", 1)))); + + // the read schema adds a field with a default + Schema.Field yWithDefault = new Schema.Field("y", primitive(Schema.Type.INT), null, 7); + Schema xy = Schema.createRecord( + "point", null, null, false, Arrays.asList(field("x", primitive(Schema.Type.INT)), yWithDefault)); + Schema readSchema = record("ProjectTwoLevelRecords", optionalField("points", array(xy))); + assertReaderContains( + projectionReader(test, projection, readSchema), + readSchema, + instance(readSchema, "points", Arrays.asList(instance(xy, "x", 1, "y", 7)))); + + // the read schema requires a field the projection leaves out, so the pruned group would not be + // read as the element: the list keeps the converted projection, as before the alignment + Schema xyRequired = + record("point", field("x", primitive(Schema.Type.INT)), field("y", primitive(Schema.Type.INT))); + Schema requiredReadSchema = record("ProjectTwoLevelRecords", optionalField("points", array(xyRequired))); + assertReaderContains( + projectionReader(test, projection, requiredReadSchema), + requiredReadSchema, + instance(requiredReadSchema, "points", null)); + } + + private AvroParquetReader projectionReader(Path path, Schema projection) + throws IOException { + Configuration conf = new Configuration(); + AvroReadSupport.setRequestedProjection(conf, projection); + AvroReadSupport.setAvroReadSchema(conf, projection); + return new AvroParquetReader<>(conf, path); + } + + private AvroParquetReader projectionReader( + Path path, Schema projection, Schema readSchema) throws IOException { + Configuration conf = new Configuration(); + AvroReadSupport.setRequestedProjection(conf, projection); + AvroReadSupport.setAvroReadSchema(conf, readSchema); + return new AvroParquetReader<>(conf, path); + } + private static Configuration newAutoDetectConf() { Configuration conf = new Configuration(); conf.setBoolean(AvroReadSupport.AUTO_DETECT_LIST_STRUCTURE, true); diff --git a/parquet-cli/src/test/java/org/apache/parquet/cli/ReadCliTest.java b/parquet-cli/src/test/java/org/apache/parquet/cli/ReadCliTest.java index 42a7feb680..8cc1b43be6 100644 --- a/parquet-cli/src/test/java/org/apache/parquet/cli/ReadCliTest.java +++ b/parquet-cli/src/test/java/org/apache/parquet/cli/ReadCliTest.java @@ -36,6 +36,7 @@ import org.apache.parquet.hadoop.example.ExampleParquetWriter; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; import org.apache.parquet.schema.Types; import org.junit.jupiter.api.BeforeEach; @@ -152,6 +153,44 @@ void catReadsStandardListsWithoutAvroSchema() throws Exception { assertThat(readRecords(result.output())).containsExactlyElementsOf(expected); } + @Test + void catReadsNonStandardListsWithoutAvroSchema() throws Exception { + // A 2-level list and the "list" / "item" names older PyArrow versions write + Path path = getTempFolder().toPath().resolve("legacy-lists.parquet"); + MessageType schema = MessageTypeParser.parseMessageType("message record {" + + " required int32 id;" + + " optional group two_level (LIST) { repeated binary element (STRING); }" + + " optional group items (LIST) { repeated group list { optional binary item (STRING); } }" + + "}"); + SimpleGroupFactory groups = new SimpleGroupFactory(schema); + try (ParquetWriter writer = ExampleParquetWriter.builder(new org.apache.hadoop.fs.Path(path.toUri())) + .withType(schema) + .build()) { + Group record = groups.newGroup().append("id", 0); + record.addGroup("two_level").append("element", "a").append("element", "b"); + Group items = record.addGroup("items"); + items.addGroup("list").append("item", "c"); + items.addGroup("list"); + writer.write(record); + } + + Map expected = new HashMap<>(); + expected.put("id", 0); + expected.put("two_level", List.of("a", "b")); + expected.put("items", Arrays.asList("c", null)); + + CliResult all = cli("cat", path.toString()); + assertThat(all.exitCode()).as(all.output()).isZero(); + assertThat(readRecords(all.output())).containsExactly(expected); + + CliResult projected = cli("cat", "--columns", "id,items", path.toString()); + assertThat(projected.exitCode()).as(projected.output()).isZero(); + Map expectedProjected = new HashMap<>(); + expectedProjected.put("id", 0); + expectedProjected.put("items", Arrays.asList("c", null)); + assertThat(readRecords(projected.output())).containsExactly(expectedProjected); + } + private Path writeRecords(String name, int firstId, int count) throws IOException { Path path = getTempFolder().toPath().resolve(name); MessageType schema = Types.buildMessage()