fix: preserve kernel decimal precision and scale metadata

Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com>

Author: cathleeny

lib/kernel/KernelArrowIpc.ts

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 import { RecordBatchReader, MessageReader, MessageHeader, Schema, Field, DataType, TypeMap } from 'apache-arrow';
-import { TTableSchema, TTypeId, TPrimitiveTypeEntry } from '../../thrift/TCLIService_types';
+import { TTableSchema, TTypeId, TPrimitiveTypeEntry, TTypeQualifiers } from '../../thrift/TCLIService_types';
 import { rewriteDurationToInt64, DURATION_UNIT_METADATA_KEY } from './KernelArrowIpcDurationFix';
 import HiveDriverError from '../errors/HiveDriverError';
 
@@ -264,6 +264,20 @@ export function arrowSchemaToThriftSchema(arrowSchema: Schema<TypeMap>): TTableS
     const primitiveEntry: TPrimitiveTypeEntry = {
       type: arrowTypeToTTypeId(field),
     };
+    if (primitiveEntry.type === TTypeId.DECIMAL_TYPE) {
+      const qualifiers: TTypeQualifiers['qualifiers'] = {};
+      const decimal = DataType.isDecimal(field.type) ? field.type : undefined;
+      for (const key of ['precision', 'scale'] as const) {
+        // Prefer manifest qualifiers, including for text-encoded decimals.
+        // Native Arrow decimals also carry these when metadata is absent.
+        const text = field.metadata.get(`databricks.type_${key}`);
+        const parsed = text !== undefined && /^-?\d+$/.test(text) ? Number(text) : NaN;
+        const minimum = key === 'precision' ? 1 : -0x80000000;
+        const value = Number.isInteger(parsed) && parsed >= minimum && parsed <= 0x7fffffff ? parsed : decimal?.[key];
+        if (value !== undefined) qualifiers[key] = { i32Value: value };
+      }
+      if (Object.keys(qualifiers).length > 0) primitiveEntry.typeQualifiers = { qualifiers };
+    }
     return {
       columnName: field.name,
       typeDesc: {

tests/unit/kernel/KernelArrowIpc.test.ts

@@ -13,8 +13,14 @@
 // limitations under the License.
 
 import { expect } from 'chai';
-import { tableFromArrays, RecordBatchStreamWriter } from 'apache-arrow';
-import { countRowsInIpc, decodeIpcSchema, patchIpcBytes } from '../../../lib/kernel/KernelArrowIpc';
+import { tableFromArrays, RecordBatchStreamWriter, Schema, Field, Decimal, Utf8, Int32 } from 'apache-arrow';
+import {
+  arrowSchemaToThriftSchema,
+  countRowsInIpc,
+  decodeIpcSchema,
+  patchIpcBytes,
+} from '../../../lib/kernel/KernelArrowIpc';
+import { TTypeId } from '../../../thrift/TCLIService_types';
 import { rewriteDurationToInt64 } from '../../../lib/kernel/KernelArrowIpcDurationFix';
 import HiveDriverError from '../../../lib/errors/HiveDriverError';
 
@@ -100,3 +106,57 @@ describe('KernelArrowIpc.decodeIpcSchema', () => {
     expect(() => decodeIpcSchema(Buffer.alloc(0))).to.throw(HiveDriverError);
   });
 });
+
+describe('KernelArrowIpc.arrowSchemaToThriftSchema decimal qualifiers', () => {
+  for (const [precision, scale] of [
+    [10, 2],
+    [38, 0],
+    [38, 38],
+    [10, -2],
+  ]) {
+    it(`preserves Arrow DECIMAL(${precision},${scale}) precision and scale`, () => {
+      const schema = new Schema([new Field('amount', new Decimal(scale, precision, 128), true)]);
+      expect(arrowSchemaToThriftSchema(schema).columns[0].typeDesc.types[0].primitiveEntry).to.deep.equal({
+        type: TTypeId.DECIMAL_TYPE,
+        typeQualifiers: { qualifiers: { precision: { i32Value: precision }, scale: { i32Value: scale } } },
+      });
+    });
+  }
+
+  for (const type of [new Utf8(), new Decimal(2, 10, 128)]) {
+    it(`prefers logical decimal qualifiers over the ${type} physical type`, () => {
+      const metadata = new Map([
+        ['databricks.type_name', 'DECIMAL'],
+        ['databricks.type_precision', '20'],
+        ['databricks.type_scale', '0'],
+      ]);
+      const schema = new Schema([new Field('amount', type, true, metadata)]);
+      expect(arrowSchemaToThriftSchema(schema).columns[0].typeDesc.types[0].primitiveEntry).to.deep.equal({
+        type: TTypeId.DECIMAL_TYPE,
+        typeQualifiers: { qualifiers: { precision: { i32Value: 20 }, scale: { i32Value: 0 } } },
+      });
+    });
+  }
+
+  for (const invalid of ['', 'invalid', '10.5', '2147483648']) {
+    it(`falls back to Arrow for invalid decimal qualifiers ${JSON.stringify(invalid)}`, () => {
+      const metadata = new Map([
+        ['databricks.type_precision', invalid],
+        ['databricks.type_scale', invalid],
+      ]);
+      const schema = new Schema([new Field('amount', new Decimal(2, 10, 128), true, metadata)]);
+      expect(
+        arrowSchemaToThriftSchema(schema).columns[0].typeDesc.types[0].primitiveEntry?.typeQualifiers,
+      ).to.deep.equal({ qualifiers: { precision: { i32Value: 10 }, scale: { i32Value: 2 } } });
+    });
+  }
+
+  it('does not invent missing qualifiers or attach decimal qualifiers to other types', () => {
+    const schema = new Schema([
+      new Field('unknown_decimal', new Utf8(), true, new Map([['databricks.type_name', 'DECIMAL']])),
+      new Field('integer', new Int32(), true, new Map([['databricks.type_precision', '10']])),
+    ]);
+    const entries = arrowSchemaToThriftSchema(schema).columns.map((column) => column.typeDesc.types[0].primitiveEntry);
+    expect(entries).to.deep.equal([{ type: TTypeId.DECIMAL_TYPE }, { type: TTypeId.INT_TYPE }]);
+  });
+});

tests/unit/kernel/KernelOperationBackend.test.ts

@@ -211,6 +211,11 @@ describe('KernelOperationBackend — M0 datatype round-trip via napi → ArrowRe
     expect((row.bin as Buffer).equals(Buffer.from([0xde, 0xad, 0xbe, 0xef]))).to.equal(true);
     // DECIMAL: Phase-1 scale-aware coercion via Arrow's Decimal type → 1.5
     expect(row.dec).to.equal(1.5);
+    const metadata = await backend.getResultMetadata();
+    const decimalColumn = metadata.schema?.columns.find((column) => column.columnName === 'dec');
+    expect(decimalColumn?.typeDesc.types[0].primitiveEntry?.typeQualifiers).to.deep.equal({
+      qualifiers: { precision: { i32Value: 10 }, scale: { i32Value: 2 } },
+    });
     // TIMESTAMP: Phase-1 produces JS Date for arrow timestamps
     expect(row.ts).to.be.instanceOf(Date);
     expect((row.ts as Date).toISOString()).to.equal('2026-05-15T12:00:00.000Z');
@@ -221,6 +226,19 @@ describe('KernelOperationBackend — M0 datatype round-trip via napi → ArrowRe
     expect(await backend.hasMore()).to.equal(false);
   });
 
+  it('preserves decimal qualifiers for a schema-only result with zero scale', async () => {
+    const schema = new Schema([new Field('amount', new Decimal(0, 38, 128), true)]);
+    const backend = new KernelOperationBackend({
+      statement: new StatementStub(ipcSchemaOnly(schema), []),
+      context: new ClientContextStub(),
+    });
+    const metadata = await backend.getResultMetadata();
+    expect(metadata.schema?.columns[0].typeDesc.types[0].primitiveEntry?.typeQualifiers).to.deep.equal({
+      qualifiers: { precision: { i32Value: 38 }, scale: { i32Value: 0 } },
+    });
+    expect(await backend.hasMore()).to.equal(false);
+  });
+
   it('round-trips ARRAY / MAP / STRUCT via the converter Phase-2 JSON fallback', async () => {
     // ARRAY / MAP / STRUCT have two possible wire encodings in M0:
     //   (a) native Arrow `List` / `Map` / `Struct` — Phase 1 produces plain