diff --git a/.github/workflows/vector-plugins.yml b/.github/workflows/vector-plugins.yml index cc57e97ffd..e2ce935890 100644 --- a/.github/workflows/vector-plugins.yml +++ b/.github/workflows/vector-plugins.yml @@ -26,7 +26,7 @@ jobs: strategy: fail-fast: false matrix: - java: [ '17' ] + java: [ '17', '21', '25' ] codes: [ 'uncompressed' ] name: Build Parquet with JDK ${{ matrix.java }} and ${{ matrix.codes }} diff --git a/.gitignore b/.gitignore index 2fd06049ea..568aa2a323 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.class .project .classpath +.factorypath .settings target # Package Files # @@ -20,4 +21,3 @@ target/ mvn_install.log .vscode/* .DS_Store - diff --git a/parquet-plugins/parquet-encoding-vector/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBitPacking512VectorLE.java b/parquet-plugins/parquet-encoding-vector/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBitPacking512VectorLE.java index eb1690a4e1..4a084cf9e5 100644 --- a/parquet-plugins/parquet-encoding-vector/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBitPacking512VectorLE.java +++ b/parquet-plugins/parquet-encoding-vector/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBitPacking512VectorLE.java @@ -177,7 +177,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(BYTE_SPECIES_64, in, inPos, in.order()); + ByteVector byteVector = fromByteBuffer(BYTE_SPECIES_64, in, inPos); ShortVector tempRes = byteVector .castShape(SHORT_SPECIES_512, 0) .reinterpretAsBytes() @@ -260,7 +260,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(BYTE_SPECIES, in, inPos, in.order()); + ByteVector byteVector = fromByteBuffer(BYTE_SPECIES, in, inPos); ShortVector tempRes = byteVector .castShape(LONG_SPECIES, 0) .reinterpretAsBytes() @@ -377,7 +377,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B128, in, inPos, in.order()) + ByteVector byteVector = fromByteBuffer(B128, in, inPos, inp_mask) .castShape(S512, 0) .reinterpretAsBytes(); ShortVector tempRes1 = byteVector @@ -466,7 +466,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(BSPECIES, in, inPos, in.order()); + ByteVector byteVector = fromByteBuffer(BSPECIES, in, inPos); ShortVector tempRes = byteVector .castShape(ISPECIES, 0) .reinterpretAsBytes() @@ -582,7 +582,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B256, in, inPos, in.order(), inp_mask) + ByteVector byteVector = fromByteBuffer(B256, in, inPos, inp_mask) .castShape(S512, 0) .reinterpretAsBytes(); @@ -705,7 +705,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B256, in, inPos, in.order(), inp_mask) + ByteVector byteVector = fromByteBuffer(B256, in, inPos, inp_mask) .castShape(S512, 0) .reinterpretAsBytes(); @@ -827,7 +827,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B256, in, inPos, in.order(), inp_mask) + ByteVector byteVector = fromByteBuffer(B256, in, inPos, inp_mask) .castShape(S512, 0) .reinterpretAsBytes(); ShortVector tempRes1 = byteVector @@ -914,7 +914,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order()); + ByteVector byteVector = fromByteBuffer(B512, in, inPos); byteVector .castShape(ISPECIES, 0) .lanewise(VectorOperators.AND, 255) @@ -1004,7 +1004,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); ShortVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsShorts() @@ -1084,7 +1084,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order()); + ByteVector byteVector = fromByteBuffer(B512, in, inPos); ShortVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsShorts() @@ -1194,7 +1194,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); ShortVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsShorts() @@ -1280,7 +1280,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order()); + ByteVector byteVector = fromByteBuffer(B512, in, inPos); ShortVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsShorts() @@ -1388,7 +1388,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); ShortVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsShorts() @@ -1512,7 +1512,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); ShortVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsShorts() @@ -1630,7 +1630,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); ShortVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsShorts() @@ -1703,7 +1703,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); ShortVector shortVector = byteVector.reinterpretAsShorts(); shortVector .castShape(I512, 0) @@ -1783,7 +1783,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); IntVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsInts() @@ -1866,7 +1866,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); IntVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsInts() @@ -1944,7 +1944,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); IntVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsInts() @@ -2022,7 +2022,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); IntVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsInts() @@ -2102,7 +2102,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); IntVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsInts() @@ -2182,7 +2182,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); IntVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsInts() @@ -2261,7 +2261,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); IntVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsInts() @@ -2332,7 +2332,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); IntVector tempRes1 = byteVector.rearrange(perm_mask0).reinterpretAsInts().lanewise(VectorOperators.AND, 16777215); tempRes1.intoArray(out, outPos, out_mask); @@ -2407,7 +2407,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); IntVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsInts() @@ -2486,7 +2486,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); IntVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsInts() @@ -2603,7 +2603,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); IntVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsInts() @@ -2718,7 +2718,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); IntVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsInts() @@ -2832,7 +2832,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); IntVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsInts() @@ -2960,7 +2960,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); IntVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsInts() @@ -3089,7 +3089,7 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); IntVector tempRes1 = byteVector .rearrange(perm_mask0) .reinterpretAsInts() @@ -3175,13 +3175,30 @@ public final void unpackValuesUsingVector(final byte[] in, final int inPos, fina public final void unpackValuesUsingVector( final ByteBuffer in, final int inPos, final int[] out, final int outPos) { - ByteVector byteVector = ByteVector.fromByteBuffer(B512, in, inPos, in.order(), inp_mask); + ByteVector byteVector = fromByteBuffer(B512, in, inPos, inp_mask); IntVector tempRes1 = byteVector.rearrange(perm_mask0).reinterpretAsInts(); tempRes1.intoArray(out, outPos, out_mask); } } + private static ByteVector fromByteBuffer(VectorSpecies species, ByteBuffer input, int inPos) { + return ByteVector.fromArray(species, readInputBytes(input, inPos, species.length()), 0); + } + + private static ByteVector fromByteBuffer( + VectorSpecies species, ByteBuffer input, int inPos, VectorMask mask) { + return ByteVector.fromArray(species, readInputBytes(input, inPos, mask.trueCount()), 0, mask); + } + + private static byte[] readInputBytes(ByteBuffer input, int inPos, int byteCount) { + byte[] bytes = new byte[byteCount]; + ByteBuffer source = input.duplicate(); + source.position(inPos); + source.get(bytes); + return bytes; + } + private static void notSupport() { throw new RuntimeException( "ByteBitPacking512VectorLE doesn't support the function, please use ByteBitPackingLE!");