From 9ee9281751bcfa3a7e46208854665aa14286fa19 Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Sun, 18 Oct 2026 16:46:39 -0700 Subject: [PATCH 1/4] Put dictionary value extension metadata on the C Data dictionary child For a dictionary-encoded extension type, Arrow C++ puts the value type's ARROW:extension:name on the dictionary child ArrowSchema, but the C Data exporter put it on the field, where IPC keeps it. pyarrow rejected such fields, and the importer never resolved them in either direction. Export: write a dictionary field's extension keys on the dictionary child, and have ExportType emit extension metadata for extension types. Import: resolve the dictionary child's extension metadata against the value type and copy it to the field metadata, matching the IPC reader. For data exported by older versions, a field-level extension name on a dictionary field is also tried against the value type. Fixes #473 Co-Authored-By: Claude Opus 5.5 --- src/Apache.Arrow/C/CArrowSchemaExporter.cs | 95 ++++++--- src/Apache.Arrow/C/CArrowSchemaImporter.cs | 75 ++++++- .../CDataInterfacePythonTests.cs | 94 +++++++++ test/Apache.Arrow.Tests/ExtensionTypeTests.cs | 183 ++++++++++++++++++ 4 files changed, 410 insertions(+), 37 deletions(-) diff --git a/src/Apache.Arrow/C/CArrowSchemaExporter.cs b/src/Apache.Arrow/C/CArrowSchemaExporter.cs index acc44e50..e6c77758 100644 --- a/src/Apache.Arrow/C/CArrowSchemaExporter.cs +++ b/src/Apache.Arrow/C/CArrowSchemaExporter.cs @@ -27,6 +27,9 @@ namespace Apache.Arrow.C { public static class CArrowSchemaExporter { + private const string ExtensionNameKey = "ARROW:extension:name"; + private const string ExtensionMetadataKey = "ARROW:extension:metadata"; + #if NET5_0_OR_GREATER private static unsafe delegate* unmanaged ReleaseSchemaPtr => &ReleaseCArrowSchema; #else @@ -49,6 +52,11 @@ public static class CArrowSchemaExporter /// /// public static unsafe void ExportType(IArrowType datatype, CArrowSchema* schema) + { + ExportType(datatype, schema, null); + } + + private static unsafe void ExportType(IArrowType datatype, CArrowSchema* schema, IReadOnlyDictionary metadata) { if (datatype == null) { @@ -59,9 +67,17 @@ public static unsafe void ExportType(IArrowType datatype, CArrowSchema* schema) throw new ArgumentNullException(nameof(schema)); } + // A dictionary-encoded field's extension metadata describes its value type. IPC + // keeps it on the field, but the C Data Interface puts it on the dictionary child. + IReadOnlyDictionary dictionaryMetadata = null; + if (datatype is DictionaryType) + { + metadata = SplitExtensionMetadata(metadata, out dictionaryMetadata); + } + schema->format = StringUtil.ToCStringUtf8(GetFormat(datatype)); schema->name = null; - schema->metadata = null; + schema->metadata = ConstructMetadata(WithExtensionMetadata(datatype, metadata)); IArrowType physicalType = datatype is ExtensionType ext ? ext.StorageType : datatype; schema->flags = GetFlags(physicalType); @@ -69,7 +85,7 @@ public static unsafe void ExportType(IArrowType datatype, CArrowSchema* schema) schema->children = ConstructChildren(physicalType, out var numChildren); schema->n_children = numChildren; - schema->dictionary = ConstructDictionary(physicalType); + schema->dictionary = ConstructDictionary(physicalType, dictionaryMetadata); schema->release = ReleaseSchemaPtr; @@ -91,29 +107,9 @@ public static unsafe void ExportType(IArrowType datatype, CArrowSchema* schema) /// public static unsafe void ExportField(Field field, CArrowSchema* schema) { - ExportType(field.DataType, schema); + ExportType(field.DataType, schema, field.Metadata); schema->name = StringUtil.ToCStringUtf8(field.Name); - // Inject extension metadata if the field type is an ExtensionType - IReadOnlyDictionary metadata = field.Metadata; - ExtensionType extType = field.DataType as ExtensionType; - if (extType == null && field.DataType is DictionaryType dt && dt.ValueType is ExtensionType dext) - extType = dext; - - if (extType != null) - { - var merged = metadata != null - ? new Dictionary((IDictionary)metadata) - : new Dictionary(); - if (!merged.ContainsKey("ARROW:extension:name")) - merged["ARROW:extension:name"] = extType.Name; - if (!merged.ContainsKey("ARROW:extension:metadata")) - merged["ARROW:extension:metadata"] = extType.ExtensionMetadata ?? ""; - metadata = merged; - } - - schema->metadata = ConstructMetadata(metadata); - IArrowType physicalType = field.DataType is ExtensionType ext ? ext.StorageType : field.DataType; schema->flags = GetFlags(physicalType, field.IsNullable); } @@ -134,8 +130,7 @@ public static unsafe void ExportField(Field field, CArrowSchema* schema) public static unsafe void ExportSchema(Schema schema, CArrowSchema* out_schema) { var structType = new StructType(schema.FieldsList); - ExportType(structType, out_schema); - out_schema->metadata = ConstructMetadata(schema.Metadata); + ExportType(structType, out_schema, schema.Metadata); } private static char FormatTimeUnit(TimeUnit unit) => unit switch @@ -301,12 +296,12 @@ private static long GetFlags(IArrowType datatype, bool nullable = true) } } - private static unsafe CArrowSchema* ConstructDictionary(IArrowType datatype) + private static unsafe CArrowSchema* ConstructDictionary(IArrowType datatype, IReadOnlyDictionary metadata) { if (datatype is DictionaryType dictType) { CArrowSchema* cSchema = CArrowSchema.Create(); - ExportType(dictType.ValueType, cSchema); + ExportType(dictType.ValueType, cSchema, metadata); return cSchema; } else @@ -315,6 +310,52 @@ private static long GetFlags(IArrowType datatype, bool nullable = true) } } + /// + /// Adds the extension name and metadata of , if it is an + /// , to . Existing keys are kept. + /// + private static IReadOnlyDictionary WithExtensionMetadata(IArrowType datatype, IReadOnlyDictionary metadata) + { + if (!(datatype is ExtensionType extType)) + { + return metadata; + } + + var merged = metadata != null + ? new Dictionary((IDictionary)metadata) + : new Dictionary(); + if (!merged.ContainsKey(ExtensionNameKey)) + merged[ExtensionNameKey] = extType.Name; + if (!merged.ContainsKey(ExtensionMetadataKey)) + merged[ExtensionMetadataKey] = extType.ExtensionMetadata ?? ""; + return merged; + } + + /// + /// Splits the extension name and metadata keys out of . + /// + /// The remaining metadata, or null if none remains. + private static IReadOnlyDictionary SplitExtensionMetadata( + IReadOnlyDictionary metadata, out IReadOnlyDictionary extensionMetadata) + { + extensionMetadata = null; + if (metadata == null || + !(metadata.ContainsKey(ExtensionNameKey) || metadata.ContainsKey(ExtensionMetadataKey))) + { + return metadata; + } + + var remaining = new Dictionary(); + var extension = new Dictionary(); + foreach (KeyValuePair pair in metadata) + { + bool isExtensionKey = pair.Key == ExtensionNameKey || pair.Key == ExtensionMetadataKey; + (isExtensionKey ? extension : remaining)[pair.Key] = pair.Value; + } + extensionMetadata = extension; + return remaining.Count > 0 ? remaining : null; + } + private unsafe static byte* ConstructMetadata(IReadOnlyDictionary metadata) { if (metadata == null || metadata.Count == 0) diff --git a/src/Apache.Arrow/C/CArrowSchemaImporter.cs b/src/Apache.Arrow/C/CArrowSchemaImporter.cs index 4bf7f44b..e90c2a48 100644 --- a/src/Apache.Arrow/C/CArrowSchemaImporter.cs +++ b/src/Apache.Arrow/C/CArrowSchemaImporter.cs @@ -26,6 +26,9 @@ namespace Apache.Arrow.C { public static class CArrowSchemaImporter { + private const string ExtensionNameKey = "ARROW:extension:name"; + private const string ExtensionMetadataKey = "ARROW:extension:metadata"; + /// /// Import C pointer as an . /// @@ -186,7 +189,10 @@ public ArrowType GetAsType(ExtensionTypeRegistry extensionRegistry) }; var dictionarySchema = new ImportedArrowSchema(_cSchema->dictionary, isRoot: false); - ArrowType dictionaryType = dictionarySchema.GetAsType(extensionRegistry); + IArrowType dictionaryType = ResolveExtensionType( + dictionarySchema.GetAsType(extensionRegistry), + GetMetadata(_cSchema->dictionary->metadata), + extensionRegistry); bool ordered = _cSchema->GetFlag(CArrowSchema.ArrowFlagDictionaryOrdered); @@ -384,20 +390,26 @@ public Field GetAsField(ExtensionTypeRegistry extensionRegistry) IArrowType type = GetAsType(extensionRegistry); IReadOnlyDictionary metadata = GetMetadata(_cSchema->metadata); - // Resolve extension type from metadata - if (metadata != null && - extensionRegistry != null && - metadata.TryGetValue("ARROW:extension:name", out string extName) && - extensionRegistry.TryGetDefinition(extName, out ExtensionDefinition extDef)) + IArrowType resolvedType = ResolveExtensionType(type, metadata, extensionRegistry); + if (resolvedType == type && type is DictionaryType dictionaryType && !(dictionaryType.ValueType is ExtensionType)) { - metadata.TryGetValue("ARROW:extension:metadata", out string extMeta); - if (extDef.TryCreateType(type, extMeta, out ExtensionType extType)) + // Older versions of this library exported a dictionary value type's extension + // metadata on the field instead of on the dictionary child. + IArrowType valueType = ResolveExtensionType(dictionaryType.ValueType, metadata, extensionRegistry); + if (valueType != dictionaryType.ValueType) { - type = extType; + resolvedType = new DictionaryType(dictionaryType.IndexType, valueType, dictionaryType.Ordered); } } - return new Field(fieldName, type, nullable, metadata); + if (_cSchema->dictionary != null) + { + // The C Data Interface puts a dictionary value type's extension metadata on + // the dictionary child. Keep it on the field, which is where IPC puts it. + metadata = WithDictionaryExtensionMetadata(metadata, GetMetadata(_cSchema->dictionary->metadata)); + } + + return new Field(fieldName, resolvedType, nullable, metadata); } public Schema GetAsSchema() @@ -418,6 +430,49 @@ public Schema GetAsSchema(ExtensionTypeRegistry extensionRegistry) } } + /// + /// Returns the extension type named in if + /// defines it for ; + /// otherwise returns . + /// + private static IArrowType ResolveExtensionType( + IArrowType storageType, IReadOnlyDictionary metadata, ExtensionTypeRegistry extensionRegistry) + { + if (metadata != null && + extensionRegistry != null && + metadata.TryGetValue(ExtensionNameKey, out string extName) && + extensionRegistry.TryGetDefinition(extName, out ExtensionDefinition extDef)) + { + metadata.TryGetValue(ExtensionMetadataKey, out string extMeta); + if (extDef.TryCreateType(storageType, extMeta, out ExtensionType extType)) + { + return extType; + } + } + return storageType; + } + + private static IReadOnlyDictionary WithDictionaryExtensionMetadata( + IReadOnlyDictionary metadata, IReadOnlyDictionary dictionaryMetadata) + { + if (dictionaryMetadata == null || + !dictionaryMetadata.TryGetValue(ExtensionNameKey, out string extName) || + (metadata != null && metadata.ContainsKey(ExtensionNameKey))) + { + return metadata; + } + + var merged = metadata != null + ? new Dictionary((IDictionary)metadata) + : new Dictionary(); + merged[ExtensionNameKey] = extName; + if (dictionaryMetadata.TryGetValue(ExtensionMetadataKey, out string extMeta)) + { + merged[ExtensionMetadataKey] = extMeta; + } + return merged; + } + private List ParseChildren(string typeName, ExtensionTypeRegistry extensionRegistry) { var child_schemas = new ImportedArrowSchema[_cSchema->n_children]; diff --git a/test/Apache.Arrow.Tests/CDataInterfacePythonTests.cs b/test/Apache.Arrow.Tests/CDataInterfacePythonTests.cs index ee5134ac..a6e8b802 100644 --- a/test/Apache.Arrow.Tests/CDataInterfacePythonTests.cs +++ b/test/Apache.Arrow.Tests/CDataInterfacePythonTests.cs @@ -1225,6 +1225,100 @@ public unsafe void ImportArrayStreamWithExtensionTypes() Assert.Equal(new Guid?[] { guid2 }, Assert.IsType(uuidLists.GetSlicedValues(1))); } + [SkippableFact] + public unsafe void ExportDictionaryOfGuidArray() + { + // Export a dictionary-encoded GuidArray and verify that pyarrow sees a + // dictionary whose value type is arrow.uuid. + + var guid1 = Guid.Parse("01234567-89ab-cdef-0123-456789abcdef"); + var guid2 = Guid.Parse("fedcba98-7654-3210-fedc-ba9876543210"); + GuidArray dictionary = new GuidArray.Builder().Append(guid1).Append(guid2).Build(); + Int32Array indices = new Int32Array.Builder().Append(1).AppendNull().Append(0).Build(); + var dictionaryType = new DictionaryType(Int32Type.Default, GuidType.Default, false); + var array = new DictionaryArray(dictionaryType, indices, dictionary); + var schema = new Schema(new[] { new Field("uuids", dictionaryType, true) }, null); + var batch = new RecordBatch(schema, new[] { array }, array.Length); + + CArrowArray* cArray = CArrowArray.Create(); + CArrowArrayExporter.ExportRecordBatch(batch, cArray); + CArrowSchema* cSchema = CArrowSchema.Create(); + CArrowSchemaExporter.ExportSchema(batch.Schema, cSchema); + + long arrayPtr = ((IntPtr)cArray).ToInt64(); + long schemaPtr = ((IntPtr)cSchema).ToInt64(); + + using (Py.GIL()) + { + dynamic pa = Py.Import("pyarrow"); + dynamic uuid_mod = Py.Import("uuid"); + + dynamic pyBatch = pa.RecordBatch._import_from_c(arrayPtr, schemaPtr); + dynamic pyArray = pyBatch.column(0); + + Assert.True((bool)pyArray.type.equals(pa.dictionary(pa.int32(), pa.uuid()))); + + dynamic expectedArray = pa.DictionaryArray.from_arrays( + pa.array(new PyList(new PyObject[] { new PyInt(1), PyObject.None, new PyInt(0) }), type: pa.int32()), + pa.array( + new PyList(new PyObject[] + { + (PyObject)uuid_mod.UUID(guid1.ToString()).bytes, + (PyObject)uuid_mod.UUID(guid2.ToString()).bytes, + }), + type: pa.uuid())); + + Assert.True((bool)pyArray.equals(expectedArray)); + } + + CArrowArray.Free(cArray); + CArrowSchema.Free(cSchema); + } + + [SkippableFact] + public unsafe void ImportDictionaryOfGuidArray() + { + // Import a pyarrow dictionary array and verify that the + // dictionary values resolve to a GuidArray. + + var guid1 = Guid.Parse("01234567-89ab-cdef-0123-456789abcdef"); + var guid2 = Guid.Parse("fedcba98-7654-3210-fedc-ba9876543210"); + + CArrowArray* cArray = CArrowArray.Create(); + CArrowSchema* cSchema = CArrowSchema.Create(); + + using (Py.GIL()) + { + dynamic pa = Py.Import("pyarrow"); + dynamic uuid_mod = Py.Import("uuid"); + + dynamic pyArray = pa.DictionaryArray.from_arrays( + pa.array(new[] { 1, 0, 1 }, type: pa.int32()), + pa.array( + new PyList(new PyObject[] + { + (PyObject)uuid_mod.UUID(guid1.ToString()).bytes, + (PyObject)uuid_mod.UUID(guid2.ToString()).bytes, + }), + type: pa.uuid())); + + pyArray._export_to_c(((IntPtr)cArray).ToInt64(), ((IntPtr)cSchema).ToInt64()); + } + + var registry = new ExtensionTypeRegistry(); + registry.Register(GuidExtensionDefinition.Instance); + + Field field = CArrowSchemaImporter.ImportField(cSchema, registry); + DictionaryType dictionaryType = Assert.IsType(field.DataType); + Assert.IsType(dictionaryType.ValueType); + + var importedArray = (DictionaryArray)CArrowArrayImporter.ImportArray(cArray, field.DataType); + Assert.Equal(new Guid?[] { guid1, guid2 }, Assert.IsType(importedArray.Dictionary)); + Assert.Equal(new int?[] { 1, 0, 1 }, (Int32Array)importedArray.Indices); + + CArrowArray.Free(cArray); + } + [SkippableFact] public unsafe void ExportBool8Array() { diff --git a/test/Apache.Arrow.Tests/ExtensionTypeTests.cs b/test/Apache.Arrow.Tests/ExtensionTypeTests.cs index 308ca265..d4cc39f9 100644 --- a/test/Apache.Arrow.Tests/ExtensionTypeTests.cs +++ b/test/Apache.Arrow.Tests/ExtensionTypeTests.cs @@ -492,6 +492,189 @@ public async Task CDataArrayStreamResolvesListOfExtensionType() } } + private static readonly DictionaryType s_guidDictionaryType = new DictionaryType(Int32Type.Default, GuidType.Default, false); + + private static ExtensionTypeRegistry GuidRegistry() + { + var registry = new ExtensionTypeRegistry(); + registry.Register(GuidExtensionDefinition.Instance); + return registry; + } + + private static unsafe Dictionary ReadMetadata(byte* metadata) + { + var result = new Dictionary(); + if (metadata == null) + { + return result; + } + + int count = *(int*)metadata; + byte* ptr = metadata + 4; + for (int i = 0; i < count; i++) + { + int keyLength = *(int*)ptr; + string key = System.Text.Encoding.UTF8.GetString(ptr + 4, keyLength); + ptr += 4 + keyLength; + int valueLength = *(int*)ptr; + string value = System.Text.Encoding.UTF8.GetString(ptr + 4, valueLength); + ptr += 4 + valueLength; + result[key] = value; + } + return result; + } + + [Fact] + public unsafe void CDataExportTypeIncludesExtensionMetadata() + { + CArrowSchema* cSchema = CArrowSchema.Create(); + try + { + CArrowSchemaExporter.ExportType(GuidType.Default, cSchema); + + Dictionary metadata = ReadMetadata(cSchema->metadata); + Assert.Equal("arrow.uuid", metadata["ARROW:extension:name"]); + Assert.Equal("", metadata["ARROW:extension:metadata"]); + } + finally + { + CArrowSchema.Free(cSchema); + } + } + + [Fact] + public unsafe void CDataExportDictionaryOfExtensionPutsMetadataOnDictionary() + { + var field = new Field("f", s_guidDictionaryType, true, + new Dictionary { ["other"] = "value" }); + + CArrowSchema* cSchema = CArrowSchema.Create(); + try + { + CArrowSchemaExporter.ExportField(field, cSchema); + + Assert.Equal(new Dictionary { ["other"] = "value" }, ReadMetadata(cSchema->metadata)); + Dictionary dictionaryMetadata = ReadMetadata(cSchema->dictionary->metadata); + Assert.Equal("arrow.uuid", dictionaryMetadata["ARROW:extension:name"]); + } + finally + { + CArrowSchema.Free(cSchema); + } + } + + [Fact] + public unsafe void CDataExportMovesDictionaryFieldExtensionMetadataToDictionary() + { + // A dictionary field read from IPC without a registry keeps the value type's + // extension name in the field metadata. + var field = new Field("f", new DictionaryType(Int32Type.Default, new FixedSizeBinaryType(16), false), true, + new Dictionary + { + ["ARROW:extension:name"] = "arrow.uuid", + ["ARROW:extension:metadata"] = "", + }); + + CArrowSchema* cSchema = CArrowSchema.Create(); + try + { + CArrowSchemaExporter.ExportField(field, cSchema); + + Assert.Empty(ReadMetadata(cSchema->metadata)); + Assert.Equal(field.Metadata, ReadMetadata(cSchema->dictionary->metadata)); + } + finally + { + CArrowSchema.Free(cSchema); + } + } + + [Fact] + public unsafe void CDataDictionaryOfExtensionRoundTrip() + { + var field = new Field("f", s_guidDictionaryType, true); + + CArrowSchema* cSchema = CArrowSchema.Create(); + CArrowSchemaExporter.ExportField(field, cSchema); + Field imported = CArrowSchemaImporter.ImportField(cSchema, GuidRegistry()); + CArrowSchema.Free(cSchema); + + DictionaryType dictionaryType = Assert.IsType(imported.DataType); + Assert.IsType(dictionaryType.IndexType); + Assert.IsType(dictionaryType.ValueType); + // Like IPC, the field metadata names the value type's extension. + Assert.Equal("arrow.uuid", imported.Metadata["ARROW:extension:name"]); + } + + [Fact] + public unsafe void CDataDictionaryOfExtensionRoundTripWithoutRegistry() + { + var field = new Field("f", s_guidDictionaryType, true); + + CArrowSchema* cSchema = CArrowSchema.Create(); + CArrowSchemaExporter.ExportField(field, cSchema); + Field imported = CArrowSchemaImporter.ImportField(cSchema); + CArrowSchema.Free(cSchema); + + DictionaryType dictionaryType = Assert.IsType(imported.DataType); + Assert.IsType(dictionaryType.ValueType); + Assert.Equal("arrow.uuid", imported.Metadata["ARROW:extension:name"]); + } + + [Fact] + public unsafe void CDataImportDictionaryWithLegacyFieldExtensionMetadata() + { + // Older versions of this library put the value type's extension metadata on the + // dictionary field. Build such a schema by swapping in the metadata of another export. + var storageField = new Field("f", new DictionaryType(Int32Type.Default, new FixedSizeBinaryType(16), false), true); + var metadataField = new Field("m", Int32Type.Default, true, + new Dictionary + { + ["ARROW:extension:name"] = "arrow.uuid", + ["ARROW:extension:metadata"] = "", + }); + + CArrowSchema* cSchema = CArrowSchema.Create(); + CArrowSchema* cMetadataSchema = CArrowSchema.Create(); + CArrowSchemaExporter.ExportField(storageField, cSchema); + CArrowSchemaExporter.ExportField(metadataField, cMetadataSchema); + byte* metadata = cSchema->metadata; + cSchema->metadata = cMetadataSchema->metadata; + cMetadataSchema->metadata = metadata; + CArrowSchema.Free(cMetadataSchema); + + Field imported = CArrowSchemaImporter.ImportField(cSchema, GuidRegistry()); + CArrowSchema.Free(cSchema); + + DictionaryType dictionaryType = Assert.IsType(imported.DataType); + Assert.IsType(dictionaryType.ValueType); + } + + [Fact] + public unsafe void CDataDictionaryOfExtensionArrayRoundTrip() + { + var values = new Guid?[] { Guid.NewGuid(), Guid.NewGuid() }; + GuidArray dictionary = (GuidArray)BuildGuidRecordBatch(values).Column(0); + Int32Array indices = new Int32Array.Builder().Append(1).AppendNull().Append(0).Append(1).Build(); + var array = new DictionaryArray(s_guidDictionaryType, indices, dictionary); + var schema = new Schema(new[] { new Field("f", s_guidDictionaryType, true) }, null); + var batch = new RecordBatch(schema, new[] { array }, array.Length); + + CArrowArray* cArray = CArrowArray.Create(); + CArrowSchema* cSchema = CArrowSchema.Create(); + CArrowArrayExporter.ExportRecordBatch(batch, cArray); + CArrowSchemaExporter.ExportSchema(schema, cSchema); + + Schema importedSchema = CArrowSchemaImporter.ImportSchema(cSchema, GuidRegistry()); + using RecordBatch importedBatch = CArrowArrayImporter.ImportRecordBatch(cArray, importedSchema); + CArrowSchema.Free(cSchema); + CArrowArray.Free(cArray); + + DictionaryArray importedArray = Assert.IsType(importedBatch.Column(0)); + Assert.Equal(values, Assert.IsType(importedArray.Dictionary)); + Assert.Equal(new int?[] { 1, null, 0, 1 }, (Int32Array)importedArray.Indices); + } + private sealed class SingleBatchStream : IArrowArrayStream { private RecordBatch _batch; From 4ad0d8bed8da4caadc56550d4fce28d738700cb2 Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Sun, 18 Oct 2026 16:59:40 -0700 Subject: [PATCH 2/4] Keep unresolved dictionary field extension metadata on the field On a dictionary field, C Data extension metadata means the extension's storage is the whole dictionary. The exporter moved any such metadata to the dictionary child, so an unresolved extension over a dictionary was re-exported as an extension over its values. Move it only when it names the resolved value type, as the IPC reader leaves it. The importer no longer copies dictionary child extension metadata to the field, where it would take on that other meaning. As in Arrow C++, an unresolved value type's extension metadata is dropped. Co-Authored-By: Claude Opus 5.5 --- src/Apache.Arrow/C/CArrowSchemaExporter.cs | 12 ++- src/Apache.Arrow/C/CArrowSchemaImporter.cs | 31 +------ test/Apache.Arrow.Tests/ExtensionTypeTests.cs | 85 +++++++++++++++++-- 3 files changed, 88 insertions(+), 40 deletions(-) diff --git a/src/Apache.Arrow/C/CArrowSchemaExporter.cs b/src/Apache.Arrow/C/CArrowSchemaExporter.cs index e6c77758..da875c01 100644 --- a/src/Apache.Arrow/C/CArrowSchemaExporter.cs +++ b/src/Apache.Arrow/C/CArrowSchemaExporter.cs @@ -67,10 +67,16 @@ private static unsafe void ExportType(IArrowType datatype, CArrowSchema* schema, throw new ArgumentNullException(nameof(schema)); } - // A dictionary-encoded field's extension metadata describes its value type. IPC - // keeps it on the field, but the C Data Interface puts it on the dictionary child. + // The C Data Interface puts the extension metadata of a dictionary's value type on + // the dictionary child; on the field itself, it means the extension's storage is the + // whole dictionary. The IPC reader leaves a value type's extension metadata on the + // field, so move it only when it names the value type. Unresolved metadata stays put. IReadOnlyDictionary dictionaryMetadata = null; - if (datatype is DictionaryType) + if (datatype is DictionaryType dictionaryType && + dictionaryType.ValueType is ExtensionType valueType && + metadata != null && + metadata.TryGetValue(ExtensionNameKey, out string extName) && + extName == valueType.Name) { metadata = SplitExtensionMetadata(metadata, out dictionaryMetadata); } diff --git a/src/Apache.Arrow/C/CArrowSchemaImporter.cs b/src/Apache.Arrow/C/CArrowSchemaImporter.cs index e90c2a48..e8b5b101 100644 --- a/src/Apache.Arrow/C/CArrowSchemaImporter.cs +++ b/src/Apache.Arrow/C/CArrowSchemaImporter.cs @@ -402,13 +402,9 @@ public Field GetAsField(ExtensionTypeRegistry extensionRegistry) } } - if (_cSchema->dictionary != null) - { - // The C Data Interface puts a dictionary value type's extension metadata on - // the dictionary child. Keep it on the field, which is where IPC puts it. - metadata = WithDictionaryExtensionMetadata(metadata, GetMetadata(_cSchema->dictionary->metadata)); - } - + // Extension metadata on the dictionary child isn't copied to the field, where it + // would mean the extension's storage is the whole dictionary. Like Arrow C++, an + // unresolved value type's extension metadata is dropped. return new Field(fieldName, resolvedType, nullable, metadata); } @@ -452,27 +448,6 @@ private static IArrowType ResolveExtensionType( return storageType; } - private static IReadOnlyDictionary WithDictionaryExtensionMetadata( - IReadOnlyDictionary metadata, IReadOnlyDictionary dictionaryMetadata) - { - if (dictionaryMetadata == null || - !dictionaryMetadata.TryGetValue(ExtensionNameKey, out string extName) || - (metadata != null && metadata.ContainsKey(ExtensionNameKey))) - { - return metadata; - } - - var merged = metadata != null - ? new Dictionary((IDictionary)metadata) - : new Dictionary(); - merged[ExtensionNameKey] = extName; - if (dictionaryMetadata.TryGetValue(ExtensionMetadataKey, out string extMeta)) - { - merged[ExtensionMetadataKey] = extMeta; - } - return merged; - } - private List ParseChildren(string typeName, ExtensionTypeRegistry extensionRegistry) { var child_schemas = new ImportedArrowSchema[_cSchema->n_children]; diff --git a/test/Apache.Arrow.Tests/ExtensionTypeTests.cs b/test/Apache.Arrow.Tests/ExtensionTypeTests.cs index d4cc39f9..3aae0a00 100644 --- a/test/Apache.Arrow.Tests/ExtensionTypeTests.cs +++ b/test/Apache.Arrow.Tests/ExtensionTypeTests.cs @@ -564,15 +564,15 @@ public unsafe void CDataExportDictionaryOfExtensionPutsMetadataOnDictionary() } [Fact] - public unsafe void CDataExportMovesDictionaryFieldExtensionMetadataToDictionary() + public unsafe void CDataExportMovesValueExtensionMetadataFromDictionaryField() { - // A dictionary field read from IPC without a registry keeps the value type's - // extension name in the field metadata. - var field = new Field("f", new DictionaryType(Int32Type.Default, new FixedSizeBinaryType(16), false), true, + // The IPC reader leaves a dictionary value type's extension metadata on the field. + var field = new Field("f", s_guidDictionaryType, true, new Dictionary { ["ARROW:extension:name"] = "arrow.uuid", ["ARROW:extension:metadata"] = "", + ["other"] = "value", }); CArrowSchema* cSchema = CArrowSchema.Create(); @@ -580,8 +580,36 @@ public unsafe void CDataExportMovesDictionaryFieldExtensionMetadataToDictionary( { CArrowSchemaExporter.ExportField(field, cSchema); - Assert.Empty(ReadMetadata(cSchema->metadata)); - Assert.Equal(field.Metadata, ReadMetadata(cSchema->dictionary->metadata)); + Assert.Equal(new Dictionary { ["other"] = "value" }, ReadMetadata(cSchema->metadata)); + Assert.Equal( + new Dictionary { ["ARROW:extension:name"] = "arrow.uuid", ["ARROW:extension:metadata"] = "" }, + ReadMetadata(cSchema->dictionary->metadata)); + } + finally + { + CArrowSchema.Free(cSchema); + } + } + + [Fact] + public unsafe void CDataExportKeepsUnresolvedDictionaryFieldExtensionMetadata() + { + // Without a resolved value type, extension metadata on a dictionary field means + // the extension's storage is the whole dictionary, so it must stay on the field. + var metadata = new Dictionary + { + ["ARROW:extension:name"] = "test.dict_storage", + ["ARROW:extension:metadata"] = "", + }; + var field = new Field("f", new DictionaryType(Int32Type.Default, StringType.Default, false), true, metadata); + + CArrowSchema* cSchema = CArrowSchema.Create(); + try + { + CArrowSchemaExporter.ExportField(field, cSchema); + + Assert.Equal(metadata, ReadMetadata(cSchema->metadata)); + Assert.Empty(ReadMetadata(cSchema->dictionary->metadata)); } finally { @@ -589,6 +617,44 @@ public unsafe void CDataExportMovesDictionaryFieldExtensionMetadataToDictionary( } } + [Fact] + public unsafe void CDataDictionaryStorageExtensionPassThroughWithoutRegistry() + { + // Arrow C++ exports an extension type whose storage is a dictionary with its + // metadata on the field. Importing it without the definition and exporting it + // again must not move that metadata to the dictionary values. + var metadata = new Dictionary + { + ["ARROW:extension:name"] = "test.dict_storage", + ["ARROW:extension:metadata"] = "", + }; + CArrowSchema* cSchema = CArrowSchema.Create(); + CArrowSchema* cMetadataSchema = CArrowSchema.Create(); + CArrowSchemaExporter.ExportField(new Field("f", new DictionaryType(Int32Type.Default, StringType.Default, false), true), cSchema); + CArrowSchemaExporter.ExportField(new Field("m", Int32Type.Default, true, metadata), cMetadataSchema); + byte* swapped = cSchema->metadata; + cSchema->metadata = cMetadataSchema->metadata; + cMetadataSchema->metadata = swapped; + CArrowSchema.Free(cMetadataSchema); + + Field imported = CArrowSchemaImporter.ImportField(cSchema, GuidRegistry()); + CArrowSchema.Free(cSchema); + Assert.Equal(metadata, imported.Metadata); + + CArrowSchema* cReexported = CArrowSchema.Create(); + try + { + CArrowSchemaExporter.ExportField(imported, cReexported); + + Assert.Equal(metadata, ReadMetadata(cReexported->metadata)); + Assert.Empty(ReadMetadata(cReexported->dictionary->metadata)); + } + finally + { + CArrowSchema.Free(cReexported); + } + } + [Fact] public unsafe void CDataDictionaryOfExtensionRoundTrip() { @@ -602,8 +668,7 @@ public unsafe void CDataDictionaryOfExtensionRoundTrip() DictionaryType dictionaryType = Assert.IsType(imported.DataType); Assert.IsType(dictionaryType.IndexType); Assert.IsType(dictionaryType.ValueType); - // Like IPC, the field metadata names the value type's extension. - Assert.Equal("arrow.uuid", imported.Metadata["ARROW:extension:name"]); + Assert.Null(imported.Metadata); } [Fact] @@ -616,9 +681,11 @@ public unsafe void CDataDictionaryOfExtensionRoundTripWithoutRegistry() Field imported = CArrowSchemaImporter.ImportField(cSchema); CArrowSchema.Free(cSchema); + // Like Arrow C++, the unresolved value type's extension metadata is dropped rather + // than moved to the field, where it would describe the whole dictionary. DictionaryType dictionaryType = Assert.IsType(imported.DataType); Assert.IsType(dictionaryType.ValueType); - Assert.Equal("arrow.uuid", imported.Metadata["ARROW:extension:name"]); + Assert.Null(imported.Metadata); } [Fact] From afb482e0a6ad831e0b84b76cbf56267da08bf175 Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Sun, 18 Oct 2026 17:08:53 -0700 Subject: [PATCH 3/4] Drop the legacy fallback for dictionary field extension metadata Extension metadata on a dictionary field describes the whole dictionary. Stop also trying it against the value type to resolve schemas exported by earlier versions of this library; C Data schemas aren't persisted, and Arrow C++ has no such fallback. Co-Authored-By: Claude Opus 5.5 --- src/Apache.Arrow/C/CArrowSchemaImporter.cs | 20 +++++-------------- test/Apache.Arrow.Tests/ExtensionTypeTests.cs | 11 ++++++---- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/src/Apache.Arrow/C/CArrowSchemaImporter.cs b/src/Apache.Arrow/C/CArrowSchemaImporter.cs index e8b5b101..c92c0fa2 100644 --- a/src/Apache.Arrow/C/CArrowSchemaImporter.cs +++ b/src/Apache.Arrow/C/CArrowSchemaImporter.cs @@ -390,22 +390,12 @@ public Field GetAsField(ExtensionTypeRegistry extensionRegistry) IArrowType type = GetAsType(extensionRegistry); IReadOnlyDictionary metadata = GetMetadata(_cSchema->metadata); - IArrowType resolvedType = ResolveExtensionType(type, metadata, extensionRegistry); - if (resolvedType == type && type is DictionaryType dictionaryType && !(dictionaryType.ValueType is ExtensionType)) - { - // Older versions of this library exported a dictionary value type's extension - // metadata on the field instead of on the dictionary child. - IArrowType valueType = ResolveExtensionType(dictionaryType.ValueType, metadata, extensionRegistry); - if (valueType != dictionaryType.ValueType) - { - resolvedType = new DictionaryType(dictionaryType.IndexType, valueType, dictionaryType.Ordered); - } - } + // On a dictionary field, extension metadata means the extension's storage is the + // whole dictionary. Extension metadata on the dictionary child isn't copied to the + // field for that reason; like Arrow C++, an unresolved value type's is dropped. + type = ResolveExtensionType(type, metadata, extensionRegistry); - // Extension metadata on the dictionary child isn't copied to the field, where it - // would mean the extension's storage is the whole dictionary. Like Arrow C++, an - // unresolved value type's extension metadata is dropped. - return new Field(fieldName, resolvedType, nullable, metadata); + return new Field(fieldName, type, nullable, metadata); } public Schema GetAsSchema() diff --git a/test/Apache.Arrow.Tests/ExtensionTypeTests.cs b/test/Apache.Arrow.Tests/ExtensionTypeTests.cs index 3aae0a00..d87823f6 100644 --- a/test/Apache.Arrow.Tests/ExtensionTypeTests.cs +++ b/test/Apache.Arrow.Tests/ExtensionTypeTests.cs @@ -689,10 +689,12 @@ public unsafe void CDataDictionaryOfExtensionRoundTripWithoutRegistry() } [Fact] - public unsafe void CDataImportDictionaryWithLegacyFieldExtensionMetadata() + public unsafe void CDataImportDoesNotApplyDictionaryFieldExtensionMetadataToValues() { - // Older versions of this library put the value type's extension metadata on the - // dictionary field. Build such a schema by swapping in the metadata of another export. + // On a dictionary field, extension metadata describes the whole dictionary, so as in + // Arrow C++ a field-level arrow.uuid doesn't resolve. (Older versions of this library + // exported a value type's extension metadata this way.) Build such a schema by + // swapping in the metadata of another export. var storageField = new Field("f", new DictionaryType(Int32Type.Default, new FixedSizeBinaryType(16), false), true); var metadataField = new Field("m", Int32Type.Default, true, new Dictionary @@ -714,7 +716,8 @@ public unsafe void CDataImportDictionaryWithLegacyFieldExtensionMetadata() CArrowSchema.Free(cSchema); DictionaryType dictionaryType = Assert.IsType(imported.DataType); - Assert.IsType(dictionaryType.ValueType); + Assert.IsType(dictionaryType.ValueType); + Assert.Equal("arrow.uuid", imported.Metadata["ARROW:extension:name"]); } [Fact] From 221f30bda1b04b1ad6fea9b64ff9ea79949b8ac4 Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Sun, 18 Oct 2026 17:10:24 -0700 Subject: [PATCH 4/4] Free the imported schema in ImportDictionaryOfGuidArray Co-Authored-By: Claude Opus 5.5 --- test/Apache.Arrow.Tests/CDataInterfacePythonTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/Apache.Arrow.Tests/CDataInterfacePythonTests.cs b/test/Apache.Arrow.Tests/CDataInterfacePythonTests.cs index a6e8b802..48db2b00 100644 --- a/test/Apache.Arrow.Tests/CDataInterfacePythonTests.cs +++ b/test/Apache.Arrow.Tests/CDataInterfacePythonTests.cs @@ -1309,6 +1309,7 @@ public unsafe void ImportDictionaryOfGuidArray() registry.Register(GuidExtensionDefinition.Instance); Field field = CArrowSchemaImporter.ImportField(cSchema, registry); + CArrowSchema.Free(cSchema); DictionaryType dictionaryType = Assert.IsType(field.DataType); Assert.IsType(dictionaryType.ValueType);