Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
9ee42ed
Define the VECTOR schema contract
divjotarora Oct 22, 2026
1c8d398
Preserve metadata fallback for VECTOR elements
divjotarora Oct 22, 2026
d7d8da6
Enforce VECTOR validity during record writes
divjotarora Oct 22, 2026
8a44a99
Validate stored VECTOR boundaries during record reads
divjotarora Oct 22, 2026
4f95e9c
Reject contradictory VECTOR counts in selected chunks
divjotarora Oct 22, 2026
5e5daec
Read VECTOR files as Avro arrays
divjotarora Oct 22, 2026
1b31654
Map VECTOR files through existing model LIST paths
divjotarora Oct 22, 2026
36e534d
Cover VECTOR file contracts and malformed boundaries
divjotarora Oct 22, 2026
99ef065
Reject invalid VECTOR adapter interpretations
divjotarora Oct 22, 2026
4b1e872
Consolidate VECTOR schema and count validation
divjotarora Oct 22, 2026
5bdd8f1
Keep distinct VECTOR validation regressions
divjotarora Oct 22, 2026
0b4bba9
Use fixed VECTOR resources for focused file coverage
divjotarora Oct 22, 2026
5565d64
GH-3877: Align vector support with FINITE_VECTOR proposal
divjotarora Nov 2, 2026
974f3fb
GH-3877: Address FINITE_VECTOR schema and adapter review
divjotarora Nov 2, 2026
869eb6c
GH-3877: Validate FINITE_VECTOR schemas in the group builder
divjotarora Nov 4, 2026
362317e
GH-3877: Address FINITE_VECTOR validation and compatibility review
divjotarora Nov 11, 2026
c42087c
GH-3877: Remove per-value FINITE_VECTOR validation
divjotarora Nov 11, 2026
a7d8129
GH-3877: Simplify FINITE_VECTOR adapters and tests
divjotarora Nov 11, 2026
6493ae2
GH-3877: Validate FINITE_VECTOR lengths while reading records
divjotarora Nov 16, 2026
ed6bcde
GH-3877: Address FINITE_VECTOR review feedback
divjotarora Nov 20, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ class List3Levels {
* @param list the Parquet List
*/
public List3Levels(GroupType list) {
if (list.getLogicalTypeAnnotation() != LogicalTypeAnnotation.listType()
LogicalTypeAnnotation logicalType = list.getLogicalTypeAnnotation();
if (!(logicalType instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation)
|| list.getFields().size() != 1) {
throw new IllegalArgumentException("invalid list type: " + list);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,21 @@ public Optional<TypeMapping> visit(
return of(new ListTypeMapping(arrowField, list3Levels, child));
}

@Override
public Optional<TypeMapping> visit(
LogicalTypeAnnotation.FiniteVectorLogicalTypeAnnotation vectorLogicalType) {
List3Levels list3Levels = new List3Levels(type);
Type element = list3Levels.getElement();
TypeMapping child = fromParquet(element, null, element.getRepetition());
ArrowType.FixedSizeList fixedSizeList =
new ArrowType.FixedSizeList(vectorLogicalType.getNumElements());

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice, this maps to FixedSizeList now 👍 The other direction still loses the length: fromArrow maps ArrowType.FixedSizeList to a plain LIST (createListTypeMapping()), so FINITE_VECTOR → Arrow → Parquet gives a LIST. Should we map it back to FINITE_VECTOR(listSize) when the child is a non-nullable supported primitive, or do we keep that for a follow-up?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the main issue here is we wouldn't guarantee the values in the Arrow FixedSizeList are all finite, so we could end up with an invalid FINITE_VECTOR on the Parquet side. I chose to go with LIST to stay conservative and provide something that's guaranteed to be correct, but I agree the round-trip inconsistency is confusing. Let me know what you'd prefer here, also happy to take this as a follow-up.

Field arrowField = new Field(
name,
new FieldType(type.isRepetition(OPTIONAL), fixedSizeList, null),
Collections.singletonList(child.getArrowField()));
return of(new ListTypeMapping(arrowField, list3Levels, child));
}

@Override
public Optional<TypeMapping> visit(
LogicalTypeAnnotation.MapLogicalTypeAnnotation mapLogicalType) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,34 @@
import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;
import org.apache.parquet.schema.Types;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;

/**
* @see SchemaConverter
*/
public class TestSchemaConverter {
@ParameterizedTest
@ValueSource(strings = {"required", "optional"})
public void testVectorAsFixedSizeList(String repetition) {
String children = "repeated group list { required float element; }";
MessageType vector = MessageTypeParser.parseMessageType(
"message m { " + repetition + " group v (FINITE_VECTOR(5)) { " + children + " } }");
SchemaMapping mapping = converter.fromParquet(vector);
Field field = mapping.getArrowSchema().findField("v");
assertThat(field.getType()).isEqualTo(new ArrowType.FixedSizeList(5));
assertThat(field.isNullable()).isEqualTo(repetition.equals("optional"));
assertThat(field.getChildren()).hasSize(1);
assertThat(field.getChildren().get(0).getType())
.isEqualTo(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE));
assertThat(field.getChildren().get(0).isNullable()).isFalse();
assertThat(mapping.getParquetSchema()).isEqualTo(vector);
assertThat(converter.map(mapping.getArrowSchema(), vector).getParquetSchema())
.isEqualTo(vector);
}

private static Field field(String name, boolean nullable, ArrowType type, Field... children) {
if (nullable) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,8 @@ private static Boolean allListStructuresAreThreeLevel(Type type) {
return null;
}
GroupType group = type.asGroupType();
if (group.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) {
LogicalTypeAnnotation logicalType = group.getLogicalTypeAnnotation();
if (logicalType instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) {
if (group.isRepetition(Type.Repetition.REPEATED) || group.getFieldCount() != 1) {
return false;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -470,7 +470,7 @@ public Optional<Schema> visit(
if (!repeatedType.isRepetition(REPEATED)) {
throw new UnsupportedOperationException("Invalid list type " + parquetGroupType);
}
if (isElementType(repeatedType, parquetGroupType.getName())) {
if (isElementType(repeatedType, parquetGroupType)) {
// repeated element types are always required
return of(Schema.createArray(convertField(repeatedType, names)));
} else {
Expand Down Expand Up @@ -654,7 +654,10 @@ public Optional<LogicalType> visit(UUIDLogicalTypeAnnotation uuidLogicalType) {
* the same. This ensures that the LIST rules are followed when there is no
* schema and that a schema can be provided to override the default behavior.
*/
private boolean isElementType(Type repeatedType, String parentName) {
private boolean isElementType(Type repeatedType, GroupType parent) {
if (parent.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.FiniteVectorLogicalTypeAnnotation) {
return false;
}
return (
// can't be a synthetic layer because it would be invalid
repeatedType.isPrimitive()
Expand All @@ -663,7 +666,7 @@ private boolean isElementType(Type repeatedType, String parentName) {
||
// known patterns without the synthetic layer
repeatedType.getName().equals("array")
|| repeatedType.getName().equals(parentName + "_tuple")
|| repeatedType.getName().equals(parent.getName() + "_tuple")
||
// default assumption
assumeRepeatedIsListElement);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,20 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.DirectWriterTest;
import org.apache.parquet.example.data.Group;
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.example.ExampleParquetWriter;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.io.InvalidRecordException;
import org.apache.parquet.io.LocalInputFile;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

public class TestArrayCompatibility extends DirectWriterTest {

Expand Down Expand Up @@ -1610,6 +1617,53 @@ public void testAutoDetectTwoLevelRepeatedPrimitive() throws Exception {
assertReaderContains(new AvroParquetReader<>(newAutoDetectConf(), test), expectedSchema, expectedRecord);
}

@Test
public void testVectorWithoutAvroSchema(@TempDir java.nio.file.Path dir) throws IOException {
MessageType schema = MessageTypeParser.parseMessageType("message VectorWithoutAvroSchema {"
+ " required group v (FINITE_VECTOR(2)) { repeated group list { required float element; } }"
+ " optional group o (FINITE_VECTOR(1)) { repeated group list { required float element; } } }");
Path path = new Path(dir.resolve("vector.parquet").toUri());
SimpleGroupFactory factory = new SimpleGroupFactory(schema);
Group first = factory.newGroup();
first.addGroup("v").addGroup("list").append("element", 1.0f);
first.getGroup("v", 0).addGroup("list").append("element", 2.0f);
first.addGroup("o").addGroup("list").append("element", 9.0f);
Group second = factory.newGroup();
second.addGroup("v").addGroup("list").append("element", 3.0f);
second.getGroup("v", 0).addGroup("list").append("element", 4.0f);
try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(path)
.withType(schema)
.withWriterVersion(org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_1_0)
.withDictionaryEncoding(false)
.withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
.withValidation(false)
.build()) {
writer.write(first);
writer.write(second);
}
try (ParquetFileReader reader = ParquetFileReader.open(new LocalInputFile(dir.resolve("vector.parquet")))) {
assertThat(reader.getFileMetaData().getKeyValueMetaData())
.doesNotContainKeys(
AvroReadSupport.AVRO_SCHEMA_METADATA_KEY, AvroReadSupport.OLD_AVRO_SCHEMA_METADATA_KEY);
}
Schema expectedSchema = record(
"VectorWithoutAvroSchema",
field("v", array(primitive(Schema.Type.FLOAT))),
optionalField("o", array(primitive(Schema.Type.FLOAT))));
assertThat(new AvroSchemaConverter().convert(schema)).isEqualTo(expectedSchema);
for (boolean assumeRepeatedIsListElement : new boolean[] {true, false}) {
assertThat(new AvroSchemaConverter(assumeRepeatedIsListElement, false).convert(schema))
.isEqualTo(expectedSchema);
}
try (AvroParquetReader<GenericRecord> reader = autoDetectReader(path)) {
assertReaderContains(
reader,
expectedSchema,
instance(expectedSchema, "v", Arrays.asList(1.0f, 2.0f), "o", Arrays.asList(9.0f)),
instance(expectedSchema, "v", Arrays.asList(3.0f, 4.0f), "o", null));
}
}

private static Configuration newAutoDetectConf() {
Configuration conf = new Configuration();
conf.setBoolean(AvroReadSupport.AUTO_DETECT_LIST_STRUCTURE, true);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,16 +84,18 @@ private void skipToMatch() {
State currentState = getState(0);
do {
ColumnReader columnReader = currentState.column;
int d = columnReader.getCurrentDefinitionLevel();

// currentLevel = depth + 1 at this point
// set the current value
if (columnReader.getCurrentDefinitionLevel() >= currentState.maxDefinitionLevel) {
if (d >= currentState.maxDefinitionLevel) {
columnReader.skip();
}
columnReader.consume();

// Based on repetition level work out next state to go to
int nextR = currentState.maxRepetitionLevel == 0 ? 0 : columnReader.getCurrentRepetitionLevel();
currentState.validateVector(d, nextR);
currentState = currentState.getNextState(nextR);
} while (currentState != null);
++recordsRead;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
import org.apache.parquet.io.api.PrimitiveConverter;
import org.apache.parquet.io.api.RecordConsumer;
import org.apache.parquet.io.api.RecordMaterializer;
import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.LogicalTypeAnnotation.FiniteVectorLogicalTypeAnnotation;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.slf4j.Logger;
Expand Down Expand Up @@ -192,6 +194,8 @@ public static class State {
private Case[][][] caseLookup;
private List<Case> definedCases;
private List<Case> undefinedCases;
private final int vectorLength;
private long vectorSize;

private State(
int id,
Expand All @@ -213,6 +217,25 @@ private State(
this.primitiveField = fieldPath[fieldPath.length - 1];
this.indexFieldPath = primitiveColumnIO.getIndexFieldPath();
this.primitiveFieldIndex = indexFieldPath[indexFieldPath.length - 1];
LogicalTypeAnnotation logical = fieldPath.length < 3
? null
: primitiveColumnIO.getParent().getParent().getType().getLogicalTypeAnnotation();
this.vectorLength = logical instanceof FiniteVectorLogicalTypeAnnotation
? ((FiniteVectorLogicalTypeAnnotation) logical).getNumElements()
: 0;
}

void validateVector(int defLevel, int repLevel) {
if (vectorLength == 0) return;
if (defLevel == maxDefinitionLevel) ++vectorSize;
if (vectorSize > vectorLength
|| (repLevel < maxRepetitionLevel
&& (vectorSize > 0 || defLevel == maxDefinitionLevel - 1)
&& vectorSize != vectorLength)) {
throw new ParquetDecodingException("Invalid FINITE_VECTOR length for " + Arrays.toString(fieldPath)
+ ": expected " + vectorLength + " elements, got " + vectorSize);
}
if (repLevel < maxRepetitionLevel) vectorSize = 0;
}

public int getDepth(int definitionLevel) {
Expand Down Expand Up @@ -427,6 +450,7 @@ public T read() {
columnReader.consume();

int nextR = currentState.maxRepetitionLevel == 0 ? 0 : columnReader.getCurrentRepetitionLevel();
currentState.validateVector(d, nextR);
// level to go to close current groups
int next = currentState.nextLevel[nextR];
for (; currentLevel > next; currentLevel--) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,13 @@ protected LogicalTypeAnnotation fromString(List<String> params) {
return listType();
}
},
FINITE_VECTOR {
@Override
protected LogicalTypeAnnotation fromString(List<String> params) {
Preconditions.checkArgument(params.size() == 1, "Expecting 1 parameter for FINITE_VECTOR: %s", params);
return finiteVectorType(Integer.parseInt(params.get(0)));
}
},
VARIANT {
@Override
protected LogicalTypeAnnotation fromString(List<String> params) {
Expand Down Expand Up @@ -314,6 +321,10 @@ public static ListLogicalTypeAnnotation listType() {
return ListLogicalTypeAnnotation.INSTANCE;
}

public static FiniteVectorLogicalTypeAnnotation finiteVectorType(int numElements) {
return new FiniteVectorLogicalTypeAnnotation(numElements);
}

public static VariantLogicalTypeAnnotation variantType(byte specVersion) {
return new VariantLogicalTypeAnnotation(specVersion);
}
Expand Down Expand Up @@ -503,7 +514,7 @@ LogicalTypeToken getType() {

@Override
public boolean equals(Object obj) {
return obj instanceof ListLogicalTypeAnnotation;
return obj != null && obj.getClass() == ListLogicalTypeAnnotation.class;
}

@Override
Expand All @@ -513,6 +524,50 @@ public int hashCode() {
}
}

/**
* Each non-null vector must contain exactly numElements non-null, finite elements. Writers are
* responsible for meeting these requirements. Record readers validate vector lengths but do not
* check finiteness.
*/
public static class FiniteVectorLogicalTypeAnnotation extends ListLogicalTypeAnnotation {
Comment thread
divjotarora marked this conversation as resolved.
Resolved
private final int numElements;

private FiniteVectorLogicalTypeAnnotation(int numElements) {
Preconditions.checkArgument(numElements > 0, "Invalid FINITE_VECTOR length: %s", numElements);
this.numElements = numElements;
}

public int getNumElements() {
return numElements;
}

@Override
public <T> Optional<T> accept(LogicalTypeAnnotationVisitor<T> visitor) {
return visitor.visit(this);
}

@Override
LogicalTypeToken getType() {
return LogicalTypeToken.FINITE_VECTOR;
}

@Override
protected String typeParametersAsString() {
return "(" + numElements + ")";
}

@Override
public boolean equals(Object obj) {
return obj instanceof FiniteVectorLogicalTypeAnnotation
&& numElements == ((FiniteVectorLogicalTypeAnnotation) obj).numElements;
}

@Override
public int hashCode() {
return 31 * getClass().hashCode() + numElements;
}
}

public static class EnumLogicalTypeAnnotation extends LogicalTypeAnnotation {
private static final EnumLogicalTypeAnnotation INSTANCE = new EnumLogicalTypeAnnotation();

Expand Down Expand Up @@ -1536,6 +1591,10 @@ default Optional<T> visit(ListLogicalTypeAnnotation listLogicalType) {
return empty();
}

default Optional<T> visit(FiniteVectorLogicalTypeAnnotation vectorLogicalType) {
return visit((ListLogicalTypeAnnotation) vectorLogicalType);
}
Comment thread
divjotarora marked this conversation as resolved.
Resolved

default Optional<T> visit(VariantLogicalTypeAnnotation variantLogicalType) {
return empty();
}
Expand Down
Loading
Loading