Skip to content

Commit 17c93ce

Browse files
committed
GH-390: Refresh vector lookup when synchronizing the schema
Promoted child types change Field equality, leaving the lookup cache keyed by the old schema. Rebuild the cache when the schema changes so Field-based lookups and JSON serialization resolve the current vectors. Generated-by: Codex and DeepSeek
1 parent 617edd5 commit 17c93ce

2 files changed

Lines changed: 87 additions & 0 deletions

File tree

‎vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,12 @@ public boolean syncSchema() {
312312
this.fieldVectors.stream().map(ValueVector::getField).collect(Collectors.toList());
313313
if (!oldFields.equals(newFields)) {
314314
this.schema = new Schema(newFields);
315+
// Field equality is structural, and promotion (e.g. a promoted Struct/List child)
316+
// produces new Field instances, so the existing keys would no longer match.
317+
fieldVectorsMap.clear();
318+
for (int i = 0; i < newFields.size(); i++) {
319+
fieldVectorsMap.put(newFields.get(i), fieldVectors.get(i));
320+
}
315321
return true;
316322
}
317323
return false;

‎vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,13 @@
1919
import static org.junit.jupiter.api.Assertions.assertEquals;
2020
import static org.junit.jupiter.api.Assertions.assertFalse;
2121
import static org.junit.jupiter.api.Assertions.assertNotEquals;
22+
import static org.junit.jupiter.api.Assertions.assertNotNull;
23+
import static org.junit.jupiter.api.Assertions.assertNull;
24+
import static org.junit.jupiter.api.Assertions.assertSame;
2225
import static org.junit.jupiter.api.Assertions.assertThrows;
2326
import static org.junit.jupiter.api.Assertions.assertTrue;
2427

28+
import java.nio.file.Path;
2529
import java.util.ArrayList;
2630
import java.util.Arrays;
2731
import java.util.List;
@@ -30,16 +34,21 @@
3034
import org.apache.arrow.memory.RootAllocator;
3135
import org.apache.arrow.vector.complex.ListVector;
3236
import org.apache.arrow.vector.complex.impl.UnionListWriter;
37+
import org.apache.arrow.vector.ipc.JsonFileReader;
38+
import org.apache.arrow.vector.ipc.JsonFileWriter;
3339
import org.apache.arrow.vector.types.pojo.ArrowType;
3440
import org.apache.arrow.vector.types.pojo.Field;
3541
import org.apache.arrow.vector.types.pojo.FieldType;
3642
import org.apache.arrow.vector.types.pojo.Schema;
3743
import org.junit.jupiter.api.AfterEach;
3844
import org.junit.jupiter.api.BeforeEach;
3945
import org.junit.jupiter.api.Test;
46+
import org.junit.jupiter.api.io.TempDir;
4047

4148
public class TestVectorSchemaRoot {
4249

50+
@TempDir public Path tempDir;
51+
4352
private BufferAllocator allocator;
4453

4554
@BeforeEach
@@ -344,4 +353,76 @@ public void testSchemaSync() {
344353
assertFalse(schemaRoot.syncSchema());
345354
}
346355
}
356+
357+
@Test
358+
public void testSchemaSyncRefreshesFieldVectorLookup() {
359+
try (VectorSchemaRoot root = createBatch()) {
360+
final FieldVector listVector = root.getVector("listCol");
361+
// createBatch() writes two list rows but leaves the root row count unset.
362+
root.setRowCount(listVector.getValueCount());
363+
final int rowCount = root.getRowCount();
364+
assertEquals(2, rowCount);
365+
final Field staleField = root.getSchema().getFields().get(0);
366+
final Object oldValue = listVector.getObject(0);
367+
368+
assertTrue(root.syncSchema());
369+
370+
// Promotion produced a new Field instance for the same, unchanged vector.
371+
final Field syncedField = root.getSchema().getFields().get(0);
372+
assertNotEquals(staleField, syncedField);
373+
374+
// Looking up by the synced Field must resolve to the very same vector instance.
375+
assertNotNull(root.getVector(syncedField));
376+
assertSame(listVector, root.getVector(syncedField));
377+
assertSame(listVector, root.getVector("listCol"));
378+
379+
// The stale structural key is gone because the Field itself changed.
380+
assertNull(root.getVector(staleField));
381+
382+
// Syncing the schema leaves the row count and the vector data untouched.
383+
assertEquals(rowCount, root.getRowCount());
384+
assertEquals(oldValue, listVector.getObject(0));
385+
386+
// A second sync is a no-op and the refreshed lookup stays correct.
387+
assertFalse(root.syncSchema());
388+
assertSame(listVector, root.getVector(syncedField));
389+
}
390+
}
391+
392+
@Test
393+
public void testJsonWriteAndReadAfterSchemaSync() throws Exception {
394+
final Path jsonFile = tempDir.resolve("promoted-schema.json");
395+
try (VectorSchemaRoot root = createBatch()) {
396+
final FieldVector expectedVector = root.getVector("listCol");
397+
// createBatch() writes two list rows but leaves the root row count unset.
398+
root.setRowCount(expectedVector.getValueCount());
399+
final int rowCount = root.getRowCount();
400+
assertEquals(2, rowCount);
401+
assertTrue(root.syncSchema());
402+
final Schema syncedSchema = root.getSchema();
403+
404+
// Writing resolves every vector through VectorSchemaRoot#getVector(Field), so a
405+
// stale Field-keyed lookup makes this write fail.
406+
try (JsonFileWriter writer =
407+
new JsonFileWriter(jsonFile.toFile(), JsonFileWriter.config().pretty(true))) {
408+
writer.start(syncedSchema, null);
409+
writer.write(root);
410+
}
411+
412+
try (BufferAllocator readerAllocator =
413+
allocator.newChildAllocator("json-reader", 0, Long.MAX_VALUE);
414+
JsonFileReader reader = new JsonFileReader(jsonFile.toFile(), readerAllocator)) {
415+
assertEquals(syncedSchema, reader.start());
416+
try (VectorSchemaRoot readRoot = reader.read()) {
417+
assertNotNull(readRoot);
418+
assertEquals(rowCount, readRoot.getRowCount());
419+
final FieldVector actualVector = readRoot.getVector("listCol");
420+
// The JSON schema round-trips exactly, so the nested rows compare structurally.
421+
assertEquals(expectedVector.getObject(0), actualVector.getObject(0));
422+
assertEquals(expectedVector.getObject(1), actualVector.getObject(1));
423+
}
424+
assertNull(reader.read());
425+
}
426+
}
427+
}
347428
}

0 commit comments

Comments
 (0)