diff --git a/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties b/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties index 2836b8c1a..54f9728be 100644 --- a/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties +++ b/java/common/src/main/resources/org/apache/tsfile/i18n/messages.properties @@ -1117,6 +1117,44 @@ error.encrypt.encrypt_no_constructor = Get constructor for encryptor failed: %1$ # EncryptUtils.getEncrypt — instantiation failed (encryptType arg) error.encrypt.encrypt_instantiation_failed = New encryptor instance failed: %1$s +# EncryptionProviderRegistry — provider registration and lookup +error.encrypt.invalid_provider_id = Encryption provider id must not be null or empty +error.encrypt.duplicate_provider = Encryption provider id is already registered: %1$s +error.encrypt.provider_not_found = Encryption provider is not registered: %1$s +error.encrypt.provider_returned_null = Encryption provider returned null: %1$s + +# EncryptParameter — use after sensitive state was destroyed +error.encrypt.parameter_destroyed = Encryption parameters have been destroyed +error.encrypt.invalid_page_body_overhead = Invalid page body overhead %1$s for encryption profile %2$s +error.encrypt.page_body_too_short = Encrypted page body size %1$s is smaller than AEAD overhead %2$s +error.encrypt.page_context_invalid_parameter = Page crypto context requires page AEAD parameters +error.encrypt.page_aead_encrypt_unsupported = This encryptor does not implement authenticated page encryption +error.encrypt.page_aead_decrypt_unsupported = This decryptor does not implement authenticated page decryption +error.encrypt.page_context_invalid_file_id = Page crypto context requires a 16-byte file crypto id +error.encrypt.page_context_invalid_chunk_ordinal = Page crypto context requires a nonnegative chunk ordinal +error.encrypt.page_read_requires_chunk_ordinal = Page-AEAD files require readPage(header, type, pageIndex, chunkOrdinal) +error.encrypt.chunk_ordinal_exhausted = File chunk ordinal space is exhausted +error.write.invalid_chunk_ordinal = Encrypted chunks require a file-unique chunk ordinal +error.write.unencrypted_chunk_has_ordinal = Unencrypted chunks must not have a chunk ordinal +error.write.aead_append_requires_file_header = Cannot append encrypted data to a file without an encryption header +error.write.encrypted_chunk_copy_unsupported = Encrypted chunks must be decrypted and re-encrypted before copying +error.encrypt.page_context_invalid_sizes = Invalid page crypto context values, uncompressed size: %1$s, compressed plaintext size: %2$s, page index: %3$s +error.encrypt.page_output_size_mismatch = AEAD page output size mismatch, expected %1$s bytes but got %2$s +error.encrypt.page_plaintext_size_mismatch = AEAD page plaintext size mismatch, expected %1$s bytes but got %2$s +error.chunk.merge_page_aead_unsupported = Page-AEAD encrypted chunks must be decrypted and rewritten before merging + +# FileEncryptionHeader — malformed or unsupported file encryption metadata +error.file.encryption_header_invalid_size = Invalid file encryption header size: %1$s +error.file.encryption_header_unsupported_version = Unsupported file encryption header version: %1$s +error.file.encryption_header_malformed = Malformed file encryption header +error.file.encryption_header_trailing_bytes = File encryption header contains trailing bytes +error.file.encryption_header_invalid_component_size = Invalid file encryption header component size: %1$s +error.file.encryption_header_truncated = File encryption header is truncated, expected %1$s bytes but read %2$s +error.file.encryption_header_invalid_field = Invalid file encryption header field: %1$s +error.write.encryption_header_must_precede_data = File encryption header must be written before TsFile data +error.write.encrypted_chunk_context_mismatch = Encrypted chunks must use the destination file encryption context +error.write.encryption_context_immutable = The file encryption context cannot change after its header is written + # === compress === # ICompressor.getCompressor / IUnCompressor.getUnCompressor — null type diff --git a/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties b/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties index 502df532e..b86387e0a 100644 --- a/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties +++ b/java/common/src/main/resources/org/apache/tsfile/i18n/messages_zh.properties @@ -1117,6 +1117,44 @@ error.encrypt.encrypt_no_constructor = 获取 encryptor 构造函数失败: %1$s # EncryptUtils.getEncrypt — instantiation failed (encryptType arg) error.encrypt.encrypt_instantiation_failed = 实例化 encryptor 失败: %1$s +# EncryptionProviderRegistry — provider registration and lookup +error.encrypt.invalid_provider_id = encryption provider id 不能为空 +error.encrypt.duplicate_provider = encryption provider id 已注册: %1$s +error.encrypt.provider_not_found = encryption provider 未注册: %1$s +error.encrypt.provider_returned_null = encryption provider 返回 null: %1$s + +# EncryptParameter — use after sensitive state was destroyed +error.encrypt.parameter_destroyed = encryption parameters 已销毁 +error.encrypt.invalid_page_body_overhead = encryption profile %2$s 的 page body 额外开销 %1$s 无效 +error.encrypt.page_body_too_short = encrypted page body 大小 %1$s 小于 AEAD 额外开销 %2$s +error.encrypt.page_context_invalid_parameter = page crypto context 需要 page AEAD 参数 +error.encrypt.page_aead_encrypt_unsupported = 此加密器未实现经过认证的 Page 加密 +error.encrypt.page_aead_decrypt_unsupported = 此解密器未实现经过认证的 Page 解密 +error.encrypt.page_context_invalid_file_id = page crypto context 需要 16 字节的 file crypto id +error.encrypt.page_context_invalid_chunk_ordinal = page crypto context 需要非负的 chunk ordinal +error.encrypt.page_read_requires_chunk_ordinal = Page-AEAD 文件需要调用 readPage(header, type, pageIndex, chunkOrdinal) +error.encrypt.chunk_ordinal_exhausted = 文件 Chunk 序号已耗尽 +error.write.invalid_chunk_ordinal = 加密 Chunk 必须具有文件内唯一的 Chunk 序号 +error.write.unencrypted_chunk_has_ordinal = 未加密 Chunk 不得携带 Chunk 序号 +error.write.aead_append_requires_file_header = 无加密头的文件不能追加加密数据 +error.write.encrypted_chunk_copy_unsupported = 复制加密 Chunk 前必须先解密并重新加密 +error.encrypt.page_context_invalid_sizes = page crypto context 值无效，uncompressed size: %1$s，compressed plaintext size: %2$s，page index: %3$s +error.encrypt.page_output_size_mismatch = AEAD page 输出大小不匹配，预期 %1$s 字节，实际 %2$s 字节 +error.encrypt.page_plaintext_size_mismatch = AEAD page plaintext 大小不匹配，预期 %1$s 字节，实际 %2$s 字节 +error.chunk.merge_page_aead_unsupported = Page-AEAD encrypted chunk 合并前必须先解密并重写 + +# FileEncryptionHeader — malformed or unsupported file encryption metadata +error.file.encryption_header_invalid_size = file encryption header 大小无效: %1$s +error.file.encryption_header_unsupported_version = 不支持的 file encryption header 版本: %1$s +error.file.encryption_header_malformed = file encryption header 格式错误 +error.file.encryption_header_trailing_bytes = file encryption header 包含多余字节 +error.file.encryption_header_invalid_component_size = file encryption header 组件大小无效: %1$s +error.file.encryption_header_truncated = file encryption header 已截断，预期 %1$s 字节，实际读取 %2$s 字节 +error.file.encryption_header_invalid_field = file encryption header 字段无效: %1$s +error.write.encryption_header_must_precede_data = file encryption header 必须写在 TsFile data 之前 +error.write.encrypted_chunk_context_mismatch = encrypted Chunk 必须使用目标文件的加密上下文 +error.write.encryption_context_immutable = 文件加密头写入后不能更改文件加密上下文 + # === compress === # ICompressor.getCompressor / IUnCompressor.getUnCompressor — null type diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptParameter.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptParameter.java index f7adb5221..70cf0951c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptParameter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptParameter.java @@ -18,14 +18,116 @@ */ package org.apache.tsfile.encrypt; -public class EncryptParameter { +import org.apache.tsfile.exception.encrypt.EncryptException; +import org.apache.tsfile.i18n.Messages; + +import javax.security.auth.Destroyable; + +import java.util.Arrays; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Parameters required to encrypt or decrypt one TsFile. + * + *

The two-argument constructor preserves the legacy encryption API. New file-level encryption + * schemes should use {@link #pageAeadBuilder()} and identify an {@link IEncryptProvider} and one of + * its immutable encryption profiles by stable ids. Runtime implementation class names and algorithm + * implementation details are deliberately not part of the persisted parameters. + */ +public class EncryptParameter implements AutoCloseable, Destroyable { + + public static final int FILE_CRYPTO_ID_LENGTH = 16; private final String type; private final byte[] key; + private final boolean pageAead; + private final String providerId; + private final String profileId; + private final String keyId; + private final String keyVersion; + private final byte[] wrappedDataKey; + private final byte[] fileCryptoId; + private final AtomicLong nextChunkOrdinal; + + private transient volatile IEncrypt fileEncrypt; + private transient volatile boolean destroyed; public EncryptParameter(String type, byte[] key) { this.type = type; this.key = key; + this.pageAead = false; + this.providerId = null; + this.profileId = null; + this.keyId = null; + this.keyVersion = null; + this.wrappedDataKey = null; + this.fileCryptoId = null; + this.nextChunkOrdinal = new AtomicLong(); + } + + private EncryptParameter(Builder builder) { + this.type = null; + this.key = copy(builder.key); + this.pageAead = true; + this.providerId = builder.providerId; + this.profileId = builder.profileId; + this.keyId = builder.keyId; + this.keyVersion = builder.keyVersion; + this.wrappedDataKey = copy(builder.wrappedDataKey); + this.fileCryptoId = copy(builder.fileCryptoId); + this.nextChunkOrdinal = + builder.nextChunkOrdinal == null ? new AtomicLong() : builder.nextChunkOrdinal; + } + + public static Builder pageAeadBuilder() { + return new Builder(); + } + + /** Returns an independent parameter object with no initialized runtime provider state. */ + public EncryptParameter copy() { + if (destroyed) { + throw new EncryptException(Messages.get("error.encrypt.parameter_destroyed")); + } + if (!isTdePageAead()) { + return new EncryptParameter(type, key); + } + return pageAeadBuilder() + .key(key) + .providerId(providerId) + .profileId(profileId) + .keyId(keyId) + .keyVersion(keyVersion) + .wrappedDataKey(wrappedDataKey) + .fileCryptoId(fileCryptoId) + .nextChunkOrdinal(nextChunkOrdinal) + .build(); + } + + /** Allocates a file-unique identity before the first page of a chunk is encrypted. */ + public long nextChunkOrdinal() { + if (!pageAead || destroyed) { + throw new EncryptException(Messages.get("error.encrypt.page_context_invalid_parameter")); + } + long ordinal = + nextChunkOrdinal.getAndUpdate(value -> value == Long.MAX_VALUE ? value : value + 1); + if (ordinal == Long.MAX_VALUE) { + throw new EncryptException(Messages.get("error.encrypt.chunk_ordinal_exhausted")); + } + return ordinal; + } + + /** Continues allocation after the highest retained chunk during append or crash recovery. */ + public void resumeAfterChunkOrdinal(long ordinal) { + if (!pageAead || destroyed) { + throw new EncryptException(Messages.get("error.encrypt.page_context_invalid_parameter")); + } + if (ordinal == Long.MAX_VALUE) { + throw new EncryptException(Messages.get("error.encrypt.chunk_ordinal_exhausted")); + } + if (ordinal >= 0) { + nextChunkOrdinal.accumulateAndGet(ordinal + 1, Math::max); + } } public byte[] getKey() { @@ -35,4 +137,169 @@ public byte[] getKey() { public String getType() { return type; } + + public String getProviderId() { + return providerId; + } + + public String getProfileId() { + return profileId; + } + + public String getKeyId() { + return keyId; + } + + public String getKeyVersion() { + return keyVersion; + } + + public byte[] getWrappedDataKey() { + return copy(wrappedDataKey); + } + + public byte[] getFileCryptoId() { + return copy(fileCryptoId); + } + + public int getPageBodyOverhead() { + int overhead = getOrCreateFileEncrypt().getPageBodyOverhead(); + if (overhead <= 0) { + throw new EncryptException( + Messages.format("error.encrypt.invalid_page_body_overhead", overhead, profileId)); + } + return overhead; + } + + public boolean isTdePageAead() { + return pageAead; + } + + /** Copies of one file share the ordinal allocator; independently created files never do. */ + public boolean sharesPageAeadFileContext(EncryptParameter other) { + return other != null + && pageAead + && other.pageAead + && !destroyed + && !other.destroyed + && nextChunkOrdinal == other.nextChunkOrdinal + && Objects.equals(providerId, other.providerId) + && Objects.equals(profileId, other.profileId) + && Objects.equals(keyId, other.keyId) + && Objects.equals(keyVersion, other.keyVersion) + && Arrays.equals(fileCryptoId, other.fileCryptoId) + && Arrays.equals(wrappedDataKey, other.wrappedDataKey) + && Arrays.equals(key, other.key); + } + + IEncrypt getOrCreateFileEncrypt() { + if (destroyed) { + throw new EncryptException(Messages.get("error.encrypt.parameter_destroyed")); + } + IEncrypt current = fileEncrypt; + if (current == null) { + synchronized (this) { + if (destroyed) { + throw new EncryptException(Messages.get("error.encrypt.parameter_destroyed")); + } + current = fileEncrypt; + if (current == null) { + current = EncryptionProviderRegistry.create(this); + fileEncrypt = current; + } + } + } + return current; + } + + @Override + public boolean isDestroyed() { + return destroyed; + } + + @Override + public void destroy() { + IEncrypt current; + synchronized (this) { + if (destroyed) { + return; + } + destroyed = true; + current = fileEncrypt; + fileEncrypt = null; + if (key != null && isTdePageAead()) { + Arrays.fill(key, (byte) 0); + } + } + if (current != null) { + current.close(); + } + } + + @Override + public void close() { + destroy(); + } + + private static byte[] copy(byte[] value) { + return value == null ? null : Arrays.copyOf(value, value.length); + } + + public static final class Builder { + + private byte[] key; + private String providerId; + private String profileId; + private String keyId; + private String keyVersion; + private byte[] wrappedDataKey; + private byte[] fileCryptoId; + private AtomicLong nextChunkOrdinal; + + private Builder() {} + + public Builder key(byte[] key) { + this.key = copy(key); + return this; + } + + public Builder providerId(String providerId) { + this.providerId = providerId; + return this; + } + + public Builder profileId(String profileId) { + this.profileId = profileId; + return this; + } + + public Builder keyId(String keyId) { + this.keyId = keyId; + return this; + } + + public Builder keyVersion(String keyVersion) { + this.keyVersion = keyVersion; + return this; + } + + public Builder wrappedDataKey(byte[] wrappedDataKey) { + this.wrappedDataKey = copy(wrappedDataKey); + return this; + } + + public Builder fileCryptoId(byte[] fileCryptoId) { + this.fileCryptoId = copy(fileCryptoId); + return this; + } + + private Builder nextChunkOrdinal(AtomicLong nextChunkOrdinal) { + this.nextChunkOrdinal = nextChunkOrdinal; + return this; + } + + public EncryptParameter build() { + return new EncryptParameter(this); + } + } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java index 7c8dbcd25..daa79d1dd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java @@ -260,6 +260,12 @@ public static EncryptParameter getEncryptParameter() { /** Get the second EncryptParameter object according to the given type and first key. */ public static EncryptParameter getEncryptParameter(EncryptParameter param) { + if (param == null) { + return new EncryptParameter("org.apache.tsfile.encrypt.UNENCRYPTED", null); + } + if (param.isTdePageAead()) { + return param; + } return encryptParamCache.computeIfAbsent(param, EncryptUtils::generateEncryptParameter); } @@ -297,6 +303,16 @@ public static IEncrypt getEncrypt() { return getEncrypt(TSFileDescriptor.getInstance().getConfig()); } + public static IEncrypt getEncrypt(EncryptParameter encryptParameter) { + if (encryptParameter == null) { + return getEncrypt("org.apache.tsfile.encrypt.UNENCRYPTED", null); + } + if (encryptParameter.isTdePageAead()) { + return encryptParameter.getOrCreateFileEncrypt(); + } + return getEncrypt(encryptParameter.getType(), encryptParameter.getKey()); + } + public static IEncrypt getEncrypt(String encryptType, byte[] dataEncryptKey) { try { String className = getEncryptClass(encryptType); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptionProviderRegistry.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptionProviderRegistry.java new file mode 100644 index 000000000..49a097cf7 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptionProviderRegistry.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.encrypt; + +import org.apache.tsfile.exception.encrypt.EncryptException; +import org.apache.tsfile.i18n.Messages; + +import java.util.ServiceLoader; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; + +/** Registry for file-level encryption providers identified by stable provider ids. */ +public final class EncryptionProviderRegistry { + + private static final ConcurrentHashMap PROVIDERS = + new ConcurrentHashMap<>(); + private static final AtomicBoolean SERVICES_LOADED = new AtomicBoolean(); + + private EncryptionProviderRegistry() {} + + public static void registerProvider(IEncryptProvider provider) { + String providerId = provider == null ? null : provider.getProviderId(); + if (providerId == null || providerId.trim().isEmpty()) { + throw new EncryptException(Messages.get("error.encrypt.invalid_provider_id")); + } + providerId = providerId.trim(); + IEncryptProvider existing = PROVIDERS.putIfAbsent(providerId, provider); + if (existing != null && !existing.getClass().equals(provider.getClass())) { + throw new EncryptException(Messages.format("error.encrypt.duplicate_provider", providerId)); + } + } + + public static void unregisterProvider(String providerId) { + if (providerId != null) { + PROVIDERS.remove(providerId.trim()); + } + } + + static IEncrypt create(EncryptParameter encryptParameter) { + loadServices(); + String providerId = encryptParameter.getProviderId(); + IEncryptProvider provider = providerId == null ? null : PROVIDERS.get(providerId); + if (provider == null) { + throw new EncryptException(Messages.format("error.encrypt.provider_not_found", providerId)); + } + IEncrypt encrypt = provider.create(encryptParameter); + if (encrypt == null) { + throw new EncryptException( + Messages.format("error.encrypt.provider_returned_null", providerId)); + } + return encrypt; + } + + private static void loadServices() { + if (SERVICES_LOADED.compareAndSet(false, true)) { + for (IEncryptProvider provider : ServiceLoader.load(IEncryptProvider.class)) { + registerProvider(provider); + } + } + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IDecryptor.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IDecryptor.java index de7735dae..464dbb507 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IDecryptor.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IDecryptor.java @@ -57,6 +57,12 @@ static IDecryptor getDecryptor(String type, byte[] key) { } static IDecryptor getDecryptor(EncryptParameter encryptParam) { + if (encryptParam == null) { + return getDecryptor("org.apache.tsfile.encrypt.UNENCRYPTED", null); + } + if (encryptParam.isTdePageAead()) { + return EncryptUtils.getEncrypt(encryptParam).getDecryptor(); + } String type = encryptParam.getType(); byte[] key = encryptParam.getKey(); return getDecryptor(type, key); @@ -66,5 +72,10 @@ static IDecryptor getDecryptor(EncryptParameter encryptParam) { byte[] decrypt(byte[] data, int offset, int size); + default byte[] decryptPage( + byte[] data, int offset, int size, PageCryptoContext pageCryptoContext) { + throw new EncryptException(Messages.get("error.encrypt.page_aead_decrypt_unsupported")); + } + EncryptionType getEncryptionType(); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncrypt.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncrypt.java index 0ea6bbef7..7f8b5e0a3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncrypt.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncrypt.java @@ -20,7 +20,7 @@ import java.util.concurrent.ConcurrentHashMap; -public interface IEncrypt { +public interface IEncrypt extends AutoCloseable { static ConcurrentHashMap> encryptMap = new ConcurrentHashMap<>(); @@ -30,4 +30,17 @@ public interface IEncrypt { IDecryptor getDecryptor(); IEncryptor getEncryptor(); + + /** + * Returns the fixed number of bytes added to every encrypted page body by this profile. + * + *

Page AEAD providers must override this method. The value is part of the profile contract and + * must not change while files using that profile exist. + */ + default int getPageBodyOverhead() { + return -1; + } + + @Override + default void close() {} } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncryptProvider.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncryptProvider.java new file mode 100644 index 000000000..d10262f51 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncryptProvider.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.encrypt; + +/** + * Service provider interface for file-level encryption implementations. + * + *

Provider ids are persisted in a TsFile and therefore must remain stable across releases. A + * provider implementation may be registered explicitly through {@link EncryptionProviderRegistry} + * or discovered through {@link java.util.ServiceLoader}. Providers own the encryption profiles + * referenced by {@link EncryptParameter#getProfileId()}; a profile's algorithms, page layout, key + * envelope format, and page body overhead must remain immutable while files using it exist. + */ +public interface IEncryptProvider { + + String getProviderId(); + + IEncrypt create(EncryptParameter encryptParameter); +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncryptor.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncryptor.java index 7d60ecfc4..c5c16ae64 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncryptor.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncryptor.java @@ -57,6 +57,12 @@ static IEncryptor getEncryptor(String type, byte[] key) { } static IEncryptor getEncryptor(EncryptParameter encryptParam) { + if (encryptParam == null) { + return getEncryptor("org.apache.tsfile.encrypt.UNENCRYPTED", null); + } + if (encryptParam.isTdePageAead()) { + return EncryptUtils.getEncrypt(encryptParam).getEncryptor(); + } String type = encryptParam.getType(); byte[] key = encryptParam.getKey(); return getEncryptor(type, key); @@ -66,5 +72,10 @@ static IEncryptor getEncryptor(EncryptParameter encryptParam) { byte[] encrypt(byte[] data, int offset, int size); + default byte[] encryptPage( + byte[] data, int offset, int size, PageCryptoContext pageCryptoContext) { + throw new EncryptException(Messages.get("error.encrypt.page_aead_encrypt_unsupported")); + } + EncryptionType getEncryptionType(); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/PageCryptoContext.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/PageCryptoContext.java new file mode 100644 index 000000000..d4c503b62 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/PageCryptoContext.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.encrypt; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.exception.encrypt.EncryptException; +import org.apache.tsfile.i18n.Messages; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +/** + * Canonical context for page-level authenticated encryption. + * + *

Encryption and decryption construct the same associated data from persisted file encryption + * metadata and stable page fields. Providers should pass {@link #getAssociatedData()} to their AEAD + * implementation unchanged. + */ +public final class PageCryptoContext { + + private static final int ASSOCIATED_DATA_VERSION = 2; + + private final int uncompressedSize; + private final int compressedPlaintextSize; + private final int encryptedPageBodySize; + private final int pageIndex; + private final byte[] associatedData; + + private PageCryptoContext( + EncryptParameter parameter, + int uncompressedSize, + int compressedPlaintextSize, + int encryptedPageBodySize, + int pageIndex, + long chunkOrdinal) { + validate(parameter, uncompressedSize, compressedPlaintextSize, pageIndex, chunkOrdinal); + this.uncompressedSize = uncompressedSize; + this.compressedPlaintextSize = compressedPlaintextSize; + this.encryptedPageBodySize = encryptedPageBodySize; + this.pageIndex = pageIndex; + this.associatedData = + buildAssociatedData( + parameter, uncompressedSize, compressedPlaintextSize, pageIndex, chunkOrdinal); + } + + public static PageCryptoContext forEncryption( + EncryptParameter parameter, + int uncompressedSize, + int compressedPlaintextSize, + int pageIndex, + long chunkOrdinal) { + validate(parameter, uncompressedSize, compressedPlaintextSize, pageIndex, chunkOrdinal); + int encryptedPageBodySize = + Math.addExact(compressedPlaintextSize, parameter.getPageBodyOverhead()); + return new PageCryptoContext( + parameter, + uncompressedSize, + compressedPlaintextSize, + encryptedPageBodySize, + pageIndex, + chunkOrdinal); + } + + public static PageCryptoContext forDecryption( + EncryptParameter parameter, + int uncompressedSize, + int encryptedPageBodySize, + int pageIndex, + long chunkOrdinal) { + validate(parameter, uncompressedSize, 0, pageIndex, chunkOrdinal); + int compressedPlaintextSize = encryptedPageBodySize - parameter.getPageBodyOverhead(); + if (compressedPlaintextSize < 0) { + throw new EncryptException( + Messages.format( + "error.encrypt.page_body_too_short", + encryptedPageBodySize, + parameter.getPageBodyOverhead())); + } + return new PageCryptoContext( + parameter, + uncompressedSize, + compressedPlaintextSize, + encryptedPageBodySize, + pageIndex, + chunkOrdinal); + } + + public int getUncompressedSize() { + return uncompressedSize; + } + + /** Returns the compressed plaintext size on both encryption and decryption paths. */ + public int getInputSize() { + return compressedPlaintextSize; + } + + public int getCompressedPlaintextSize() { + return compressedPlaintextSize; + } + + public int getEncryptedPageBodySize() { + return encryptedPageBodySize; + } + + public int getPageIndex() { + return pageIndex; + } + + public boolean isFirstPage() { + return pageIndex == 0; + } + + /** Returns a defensive copy so providers cannot mutate the authenticated page identity. */ + public byte[] getAssociatedData() { + return Arrays.copyOf(associatedData, associatedData.length); + } + + private static void validate( + EncryptParameter parameter, + int uncompressedSize, + int compressedPlaintextSize, + int pageIndex, + long chunkOrdinal) { + if (parameter == null || !parameter.isTdePageAead()) { + throw new EncryptException(Messages.get("error.encrypt.page_context_invalid_parameter")); + } + if (parameter.getFileCryptoId() == null + || parameter.getFileCryptoId().length != EncryptParameter.FILE_CRYPTO_ID_LENGTH) { + throw new EncryptException(Messages.get("error.encrypt.page_context_invalid_file_id")); + } + if (chunkOrdinal < 0) { + throw new EncryptException(Messages.get("error.encrypt.page_context_invalid_chunk_ordinal")); + } + if (uncompressedSize < 0 || compressedPlaintextSize < 0 || pageIndex < 0) { + throw new EncryptException( + Messages.format( + "error.encrypt.page_context_invalid_sizes", + uncompressedSize, + compressedPlaintextSize, + pageIndex)); + } + } + + private static byte[] buildAssociatedData( + EncryptParameter parameter, + int uncompressedSize, + int compressedPlaintextSize, + int pageIndex, + long chunkOrdinal) { + byte[][] components = + new byte[][] { + bytes(parameter.getProviderId()), + bytes(parameter.getProfileId()), + bytes(parameter.getKeyId()), + bytes(parameter.getKeyVersion()), + parameter.getFileCryptoId() + }; + int size = Integer.BYTES * (4 + components.length) + Long.BYTES; + for (byte[] component : components) { + size = Math.addExact(size, component.length); + } + + ByteBuffer buffer = ByteBuffer.allocate(size); + buffer.putInt(ASSOCIATED_DATA_VERSION); + for (byte[] component : components) { + buffer.putInt(component.length); + buffer.put(component); + } + buffer.putLong(chunkOrdinal); + buffer.putInt(pageIndex); + buffer.putInt(uncompressedSize); + buffer.putInt(compressedPlaintextSize); + return buffer.array(); + } + + private static byte[] bytes(String value) { + return value == null ? new byte[0] : value.getBytes(TSFileConfig.STRING_CHARSET); + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/MetaMarker.java b/java/tsfile/src/main/java/org/apache/tsfile/file/MetaMarker.java index 5cca70594..ce8bf8b37 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/MetaMarker.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/MetaMarker.java @@ -51,6 +51,9 @@ public class MetaMarker { /** Chunk header marker and this chunk has only one page. */ public static final byte ONLY_ONE_PAGE_CHUNK_HEADER = 5; + /** Optional file encryption header immediately following the TsFile version byte. */ + public static final byte ENCRYPTION_HEADER = 6; + /** Time Chunk header marker and this chunk has more than one page. */ public static final byte TIME_CHUNK_HEADER = (byte) (CHUNK_HEADER | TsFileConstant.TIME_COLUMN_MASK); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkHeader.java b/java/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkHeader.java index a03209fdc..12e19460e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkHeader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkHeader.java @@ -34,6 +34,7 @@ import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -63,6 +64,7 @@ public class ChunkHeader { private final TSDataType dataType; private final CompressionType compressionType; private final TSEncoding encodingType; + private final long chunkOrdinal; // the following fields do not need to be serialized. private int numOfPages; @@ -87,16 +89,29 @@ public ChunkHeader( TSEncoding encoding, int numOfPages, int mask) { + this(measurementID, dataSize, dataType, compressionType, encoding, numOfPages, mask, -1); + } + + public ChunkHeader( + String measurementID, + int dataSize, + TSDataType dataType, + CompressionType compressionType, + TSEncoding encoding, + int numOfPages, + int mask, + long chunkOrdinal) { this( (byte) ((numOfPages <= 1 ? MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER : MetaMarker.CHUNK_HEADER) | (byte) mask), measurementID, dataSize, - getSerializedSize(measurementID, dataSize), + getSerializedSize(measurementID, dataSize, chunkOrdinal >= 0), dataType, compressionType, - encoding); + encoding, + chunkOrdinal); this.numOfPages = numOfPages; } @@ -117,6 +132,25 @@ public ChunkHeader( encoding); } + public ChunkHeader( + byte chunkType, + String measurementID, + int dataSize, + TSDataType dataType, + CompressionType compressionType, + TSEncoding encoding, + long chunkOrdinal) { + this( + chunkType, + measurementID, + dataSize, + getSerializedSize(measurementID, dataSize, chunkOrdinal >= 0), + dataType, + compressionType, + encoding, + chunkOrdinal); + } + public ChunkHeader( byte chunkType, String measurementID, @@ -125,12 +159,25 @@ public ChunkHeader( TSDataType dataType, CompressionType compressionType, TSEncoding encoding) { + this(chunkType, measurementID, dataSize, headerSize, dataType, compressionType, encoding, -1); + } + + private ChunkHeader( + byte chunkType, + String measurementID, + int dataSize, + int headerSize, + TSDataType dataType, + CompressionType compressionType, + TSEncoding encoding, + long chunkOrdinal) { this.chunkType = chunkType; this.measurementID = measurementID; this.dataSize = dataSize; this.dataType = dataType; this.compressionType = compressionType; this.encodingType = encoding; + this.chunkOrdinal = chunkOrdinal; this.serializedSize = headerSize; } @@ -147,6 +194,10 @@ public static int getSerializedSize(String measurementID, int dataSize) { + TSEncoding.getSerializedSize(); // encodingType } + public static int getSerializedSize(String measurementID, int dataSize, boolean pageAead) { + return getSerializedSize(measurementID, dataSize) + (pageAead ? Long.BYTES : 0); + } + /** * The estimated serialized size of chunk header. Only used when we don't know the actual dataSize * attribute @@ -176,13 +227,34 @@ public int getSerializedSize() { */ public static ChunkHeader deserializeFrom(InputStream inputStream, byte chunkType) throws IOException { + return deserializeFrom(inputStream, chunkType, false); + } + + public static ChunkHeader deserializeFrom( + InputStream inputStream, byte chunkType, boolean pageAead) throws IOException { // read measurementID String measurementID = ReadWriteIOUtils.readVarIntString(inputStream); int dataSize = ReadWriteForEncodingUtils.readUnsignedVarInt(inputStream); TSDataType dataType = ReadWriteIOUtils.readDataType(inputStream); CompressionType type = ReadWriteIOUtils.readCompressionType(inputStream); TSEncoding encoding = ReadWriteIOUtils.readEncoding(inputStream); - return new ChunkHeader(chunkType, measurementID, dataSize, dataType, type, encoding); + long chunkOrdinal = -1; + if (pageAead) { + byte[] ordinalBytes = inputStream.readNBytes(Long.BYTES); + if (ordinalBytes.length != Long.BYTES) { + throw new EOFException(); + } + chunkOrdinal = ByteBuffer.wrap(ordinalBytes).getLong(); + } + return new ChunkHeader( + chunkType, + measurementID, + dataSize, + getSerializedSize(measurementID, dataSize, pageAead), + dataType, + type, + encoding, + chunkOrdinal); } /** @@ -194,7 +266,7 @@ public static ChunkHeader deserializeFrom(InputStream inputStream, byte chunkTyp * @throws IOException IOException */ public static ChunkHeader deserializeFrom(TsFileInput input, long offset) throws IOException { - return deserializeFrom(input, offset, null); + return deserializeFrom(input, offset, null, false); } /** @@ -208,6 +280,12 @@ public static ChunkHeader deserializeFrom(TsFileInput input, long offset) throws */ public static ChunkHeader deserializeFrom( TsFileInput input, long offset, LongConsumer ioSizeRecorder) throws IOException { + return deserializeFrom(input, offset, ioSizeRecorder, false); + } + + public static ChunkHeader deserializeFrom( + TsFileInput input, long offset, LongConsumer ioSizeRecorder, boolean pageAead) + throws IOException { // only 6 bytes, no need to call ioSizeRecorder.accept alone, combine into the remaining read // operation @@ -226,7 +304,8 @@ public static ChunkHeader deserializeFrom( + 1 // uVarInt dataSize + TSDataType.getSerializedSize() // dataType + CompressionType.getSerializedSize() // compressionType - + TSEncoding.getSerializedSize(); + + TSEncoding.getSerializedSize() + + (pageAead ? Long.BYTES : 0); buffer = ByteBuffer.allocate(remainingBytes); if (ioSizeRecorder != null) { @@ -242,9 +321,23 @@ public static ChunkHeader deserializeFrom( TSDataType dataType = ReadWriteIOUtils.readDataType(buffer); CompressionType type = ReadWriteIOUtils.readCompressionType(buffer); TSEncoding encoding = ReadWriteIOUtils.readEncoding(buffer); + long chunkOrdinal = -1; + if (pageAead) { + if (buffer.remaining() < Long.BYTES) { + throw new EOFException(); + } + chunkOrdinal = buffer.getLong(); + } int chunkHeaderSize = alreadyReadLength + buffer.position(); return new ChunkHeader( - chunkType, measurementID, dataSize, chunkHeaderSize, dataType, type, encoding); + chunkType, + measurementID, + dataSize, + chunkHeaderSize, + dataType, + type, + encoding, + chunkOrdinal); } /** @@ -269,6 +362,10 @@ public String getMeasurementID() { return measurementID; } + public long getChunkOrdinal() { + return chunkOrdinal; + } + public void setMeasurementID(String measurementID) { this.measurementID = measurementID; } @@ -312,6 +409,9 @@ public int serializeTo(OutputStream outputStream) throws IOException { length += ReadWriteIOUtils.write(dataType, outputStream); length += ReadWriteIOUtils.write(compressionType, outputStream); length += ReadWriteIOUtils.write(encodingType, outputStream); + if (chunkOrdinal >= 0) { + length += ReadWriteIOUtils.write(chunkOrdinal, outputStream); + } return length; } @@ -329,6 +429,9 @@ public int serializeTo(ByteBuffer buffer) { length += ReadWriteIOUtils.write(dataType, buffer); length += ReadWriteIOUtils.write(compressionType, buffer); length += ReadWriteIOUtils.write(encodingType, buffer); + if (chunkOrdinal >= 0) { + length += ReadWriteIOUtils.write(chunkOrdinal, buffer); + } return length; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/header/FileEncryptionHeader.java b/java/tsfile/src/main/java/org/apache/tsfile/file/header/FileEncryptionHeader.java new file mode 100644 index 000000000..97e5016ff --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/header/FileEncryptionHeader.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.file.header; + +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.exception.encrypt.EncryptException; +import org.apache.tsfile.file.MetaMarker; +import org.apache.tsfile.i18n.Messages; +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteForEncodingUtils; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +/** + * Versioned file encryption metadata stored before the first TsFile data marker. + * + *

The header contains only public identifiers and a wrapped data key. Plaintext key material and + * runtime implementation class names are never serialized. + */ +public final class FileEncryptionHeader { + + public static final byte VERSION = 2; + public static final int MAX_HEADER_SIZE = 1024 * 1024; + + private FileEncryptionHeader() {} + + public static int serialize(EncryptParameter parameter, OutputStream outputStream) + throws IOException { + validate(parameter); + + PublicBAOS payload = new PublicBAOS(); + int payloadSize = 0; + payloadSize += ReadWriteIOUtils.write(VERSION, payload); + payloadSize += ReadWriteIOUtils.writeVar(parameter.getProviderId(), payload); + payloadSize += ReadWriteIOUtils.writeVar(parameter.getProfileId(), payload); + payloadSize += ReadWriteIOUtils.writeVar(parameter.getKeyId(), payload); + payloadSize += ReadWriteIOUtils.writeVar(parameter.getKeyVersion(), payload); + payloadSize += writeBytes(parameter.getFileCryptoId(), payload); + payloadSize += writeBytes(parameter.getWrappedDataKey(), payload); + + if (payloadSize != payload.size() || payloadSize > MAX_HEADER_SIZE) { + throw new EncryptException( + Messages.format("error.file.encryption_header_invalid_size", payloadSize)); + } + + int size = ReadWriteIOUtils.write(MetaMarker.ENCRYPTION_HEADER, outputStream); + size += ReadWriteForEncodingUtils.writeUnsignedVarInt(payloadSize, outputStream); + payload.writeTo(outputStream); + return size + payloadSize; + } + + /** Deserializes a header after its {@link MetaMarker#ENCRYPTION_HEADER} marker was consumed. */ + public static EncryptParameter deserialize(InputStream inputStream) throws IOException { + int payloadSize = ReadWriteForEncodingUtils.readUnsignedVarInt(inputStream); + if (payloadSize <= 0 || payloadSize > MAX_HEADER_SIZE) { + throw new EncryptException( + Messages.format("error.file.encryption_header_invalid_size", payloadSize)); + } + + byte[] payload = readExactly(inputStream, payloadSize); + ByteBuffer buffer = ByteBuffer.wrap(payload); + byte version = buffer.get(); + if (version != VERSION) { + throw new EncryptException( + Messages.format("error.file.encryption_header_unsupported_version", version)); + } + + EncryptParameter parameter; + try { + parameter = + EncryptParameter.pageAeadBuilder() + .providerId(ReadWriteIOUtils.readVarIntString(buffer)) + .profileId(ReadWriteIOUtils.readVarIntString(buffer)) + .keyId(ReadWriteIOUtils.readVarIntString(buffer)) + .keyVersion(ReadWriteIOUtils.readVarIntString(buffer)) + .fileCryptoId(readBytes(buffer)) + .wrappedDataKey(readBytes(buffer)) + .build(); + } catch (RuntimeException e) { + throw new EncryptException(Messages.get("error.file.encryption_header_malformed"), e); + } + if (buffer.hasRemaining()) { + throw new EncryptException(Messages.get("error.file.encryption_header_trailing_bytes")); + } + validate(parameter); + return parameter; + } + + private static int writeBytes(byte[] bytes, OutputStream outputStream) throws IOException { + int size = ReadWriteForEncodingUtils.writeUnsignedVarInt(bytes.length, outputStream); + outputStream.write(bytes); + return size + bytes.length; + } + + private static byte[] readBytes(ByteBuffer buffer) { + int length = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer); + if (length < 0 || length > MAX_HEADER_SIZE || length > buffer.remaining()) { + throw new EncryptException( + Messages.format("error.file.encryption_header_invalid_component_size", length)); + } + byte[] bytes = new byte[length]; + buffer.get(bytes); + return bytes; + } + + private static byte[] readExactly(InputStream inputStream, int size) throws IOException { + byte[] bytes = new byte[size]; + int offset = 0; + while (offset < size) { + int read = inputStream.read(bytes, offset, size - offset); + if (read < 0) { + throw new EncryptException( + Messages.format("error.file.encryption_header_truncated", size, offset)); + } + if (read == 0) { + int value = inputStream.read(); + if (value < 0) { + throw new EncryptException( + Messages.format("error.file.encryption_header_truncated", size, offset)); + } + bytes[offset++] = (byte) value; + } else { + offset += read; + } + } + return bytes; + } + + private static void validate(EncryptParameter parameter) { + if (parameter == null || !parameter.isTdePageAead()) { + invalidField("pageAead"); + } + requireText(parameter.getProviderId(), "providerId"); + requireText(parameter.getProfileId(), "profileId"); + requireText(parameter.getKeyId(), "keyId"); + requireText(parameter.getKeyVersion(), "keyVersion"); + + byte[] fileCryptoId = parameter.getFileCryptoId(); + if (fileCryptoId == null || fileCryptoId.length != EncryptParameter.FILE_CRYPTO_ID_LENGTH) { + invalidField("fileCryptoId"); + } + byte[] wrappedDataKey = parameter.getWrappedDataKey(); + if (wrappedDataKey == null || wrappedDataKey.length == 0) { + invalidField("wrappedDataKey"); + } + } + + private static void requireText(String value, String field) { + if (value == null || value.trim().isEmpty()) { + invalidField(field); + } + } + + private static void invalidField(String field) { + throw new EncryptException( + Messages.format("error.file.encryption_header_invalid_field", field)); + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 2eea4089f..ea82ce82b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -30,6 +30,7 @@ import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.encrypt.IDecryptor; +import org.apache.tsfile.encrypt.PageCryptoContext; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.NotCompatibleTsFileException; import org.apache.tsfile.exception.StopReadTsFileByInterruptException; @@ -40,6 +41,7 @@ import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkGroupHeader; import org.apache.tsfile.file.header.ChunkHeader; +import org.apache.tsfile.file.header.FileEncryptionHeader; import org.apache.tsfile.file.header.PageHeader; import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; @@ -114,6 +116,9 @@ public class TsFileSequenceReader implements AutoCloseable { + private static final int BASE_FILE_HEADER_SIZE = + TSFileConfig.MAGIC_STRING.getBytes(TSFileConfig.STRING_CHARSET).length + Byte.BYTES; + private static final Logger logger = LoggerFactory.getLogger(TsFileSequenceReader.class); private static final Logger resourceLogger = LoggerFactory.getLogger("FileMonitor"); protected static final TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); @@ -147,6 +152,11 @@ public class TsFileSequenceReader implements AutoCloseable { private EncryptParameter dataEncryptParam = null; + private EncryptParameter fileEncryptionParam; + private long maxChunkOrdinal = -1; + + private long dataStartOffset = BASE_FILE_HEADER_SIZE; + /** * Create a file reader of the given file. The reader will read the tail of the file to get the * file metadata size.Then the reader will skip the first @@ -168,7 +178,7 @@ public Map countChunksPerChunkGroup() throws IOException { return result; } - int headerLength = TSFileConfig.MAGIC_STRING.getBytes().length + Byte.BYTES; + long headerLength = getDataStartOffset(); if (checkFile.length() < headerLength) { return result; } @@ -315,6 +325,7 @@ public TsFileSequenceReader(String file, boolean loadMetadataSize, LongConsumer try { loadFileVersion(ioSizeRecorder); + loadFileEncryptionHeader(ioSizeRecorder); if (loadMetadataSize) { loadMetadataSize(ioSizeRecorder); } @@ -379,6 +390,7 @@ public TsFileSequenceReader(TsFileInput input, boolean loadMetadataSize) throws this.tsFileInput = input; this.file = input.getFilePath(); try { + loadFileEncryptionHeader(null); if (loadMetadataSize) { // NOTE no autoRepair here loadMetadataSize(); } @@ -403,6 +415,7 @@ public TsFileSequenceReader( try { loadFileVersion(ioSizeRecorder); + loadFileEncryptionHeader(ioSizeRecorder); if (loadMetadataSize) { loadMetadataSize(ioSizeRecorder); } @@ -428,17 +441,26 @@ public TsFileSequenceReader( * of the input to the current position * @param fileMetadataSize the byte size of the file metadata in the input */ - public TsFileSequenceReader(TsFileInput input, long fileMetadataPos, int fileMetadataSize) { + public TsFileSequenceReader(TsFileInput input, long fileMetadataPos, int fileMetadataSize) + throws IOException { this.tsFileInput = input; + this.file = input.getFilePath(); this.fileMetadataPos = fileMetadataPos; this.fileMetadataSize = fileMetadataSize; + try { + loadFileEncryptionHeader(null); + } catch (IOException | RuntimeException e) { + tsFileInput.close(); + throw e; + } } public TsFileSequenceReader( TsFileInput input, long fileMetadataPos, int fileMetadataSize, - EncryptParameter firstEncryptParam) { + EncryptParameter firstEncryptParam) + throws IOException { this(input, fileMetadataPos, fileMetadataSize); this.firstEncryptParam = firstEncryptParam; } @@ -465,6 +487,42 @@ private void loadFileVersion(LongConsumer ioSizeRecorder) throws IOException { } } + private void loadFileEncryptionHeader(LongConsumer ioSizeRecorder) throws IOException { + long savedPosition = tsFileInput.position(); + dataStartOffset = BASE_FILE_HEADER_SIZE; + fileEncryptionParam = null; + try { + if (tsFileInput.size() <= BASE_FILE_HEADER_SIZE) { + return; + } + ByteBuffer marker = ByteBuffer.allocate(Byte.BYTES); + if (tsFileInput.read(marker, BASE_FILE_HEADER_SIZE) != Byte.BYTES) { + return; + } + marker.flip(); + if (marker.get() != MetaMarker.ENCRYPTION_HEADER) { + return; + } + + tsFileInput.position(BASE_FILE_HEADER_SIZE + Byte.BYTES); + fileEncryptionParam = FileEncryptionHeader.deserialize(tsFileInput.wrapAsInputStream()); + dataStartOffset = tsFileInput.position(); + if (ioSizeRecorder != null) { + ioSizeRecorder.accept(dataStartOffset - BASE_FILE_HEADER_SIZE); + } + } finally { + tsFileInput.position(savedPosition); + } + } + + public long getDataStartOffset() { + return dataStartOffset; + } + + public boolean hasFileEncryptionHeader() { + return fileEncryptionParam != null; + } + private void configDeserializer() { if (fileVersion == TSFileConfig.VERSION_NUMBER_V3) { deserializeConfig = CompatibilityUtils.v3DeserializeConfig; @@ -672,6 +730,9 @@ public EncryptParameter getEncryptParam() throws IOException { public EncryptParameter getEncryptParam(LongConsumer ioSizeRecorder) throws IOException { if (dataEncryptParam != null) { return dataEncryptParam; + } else if (fileEncryptionParam != null) { + dataEncryptParam = fileEncryptionParam; + return dataEncryptParam; } else { if (fileMetadataSize != 0) { readFileMetadata(ioSizeRecorder); @@ -2035,7 +2096,11 @@ public void readPlanIndex() throws IOException { */ public ChunkHeader readChunkHeader(byte chunkType) throws IOException { try { - return ChunkHeader.deserializeFrom(tsFileInput.wrapAsInputStream(), chunkType); + ChunkHeader header = + ChunkHeader.deserializeFrom( + tsFileInput.wrapAsInputStream(), chunkType, fileEncryptionParam != null); + recordChunkOrdinal(header); + return header; } catch (StopReadTsFileByInterruptException e) { throw e; } catch (Throwable t) { @@ -2054,7 +2119,11 @@ public ChunkHeader readChunkHeader(byte chunkType) throws IOException { private ChunkHeader readChunkHeader(long position, LongConsumer ioSizeRecorder) throws IOException { try { - return ChunkHeader.deserializeFrom(tsFileInput, position, ioSizeRecorder); + ChunkHeader header = + ChunkHeader.deserializeFrom( + tsFileInput, position, ioSizeRecorder, fileEncryptionParam != null); + recordChunkOrdinal(header); + return header; } catch (StopReadTsFileByInterruptException e) { throw e; } catch (Throwable t) { @@ -2064,6 +2133,17 @@ private ChunkHeader readChunkHeader(long position, LongConsumer ioSizeRecorder) } } + private void recordChunkOrdinal(ChunkHeader header) throws IOException { + if (fileEncryptionParam != null && header.getChunkOrdinal() < 0) { + throw new IOException(Messages.get("error.write.invalid_chunk_ordinal")); + } + maxChunkOrdinal = Math.max(maxChunkOrdinal, header.getChunkOrdinal()); + } + + public long getMaxChunkOrdinal() { + return maxChunkOrdinal; + } + /** * notice, this function will modify channel's position. * @@ -2098,7 +2178,8 @@ public ByteBuffer readChunk(long position, int dataSize, LongConsumer ioSizeReco /** * read memory chunk. * - * @return -chunk + * @return a chunk whose page-AEAD context remains valid after this reader closes; close the chunk + * after use */ public Chunk readMemChunk(long offset) throws IOException { return readMemChunk(offset, null); @@ -2125,7 +2206,8 @@ public Chunk readMemChunk(long offset, LongConsumer ioSizeRecorder) throws IOExc * read memory chunk. * * @param metaData -given chunk meta data - * @return -chunk + * @return a chunk whose page-AEAD context remains valid after this reader closes; close the chunk + * after use */ public Chunk readMemChunk(ChunkMetadata metaData) throws IOException { try { @@ -2151,7 +2233,7 @@ public Chunk readMemChunk(ChunkMetadata metaData) throws IOException { * read memory chunk. * * @param chunkCacheKey given key of chunk LRUCache - * @return chunk + * @return a chunk with an independent page-AEAD context; close it after use */ public Chunk readMemChunk(CachedChunkLoaderImpl.ChunkCacheKey chunkCacheKey) throws IOException { ChunkHeader header = readChunkHeader(chunkCacheKey.getOffsetOfChunkHeader(), null); @@ -2235,17 +2317,63 @@ public ByteBuffer readCompressedPage(PageHeader header) throws IOException { } public ByteBuffer readPage(PageHeader header, CompressionType type) throws IOException { + return readPage(header, type, 0); + } + + public ByteBuffer readPage(PageHeader header, CompressionType type, int pageIndex) + throws IOException { + if (fileEncryptionParam != null && fileEncryptionParam.isTdePageAead()) { + throw new IOException(Messages.get("error.encrypt.page_read_requires_chunk_ordinal")); + } + return readPage(header, type, pageIndex, -1); + } + + public ByteBuffer readPage( + PageHeader header, CompressionType type, int pageIndex, long chunkOrdinal) + throws IOException { ByteBuffer buffer = readData(-1, header.getCompressedSize()); - IDecryptor decryptor = IDecryptor.getDecryptor(getEncryptParam()); + EncryptParameter encryptParameter = getEncryptParam(); + IDecryptor decryptor = IDecryptor.getDecryptor(encryptParameter); if (header.getUncompressedSize() == 0) { return buffer; } - ByteBuffer finalBuffer = decrypt(decryptor, buffer); + ByteBuffer finalBuffer = + decrypt(decryptor, encryptParameter, buffer, header, pageIndex, chunkOrdinal); finalBuffer = uncompress(type, finalBuffer, header.getUncompressedSize()); return finalBuffer; } - private static ByteBuffer decrypt(IDecryptor decryptor, ByteBuffer buffer) { + private static ByteBuffer decrypt( + IDecryptor decryptor, + EncryptParameter encryptParameter, + ByteBuffer buffer, + PageHeader header, + int pageIndex, + long chunkOrdinal) + throws IOException { + if (encryptParameter != null && encryptParameter.isTdePageAead()) { + PageCryptoContext pageCryptoContext = + PageCryptoContext.forDecryption( + encryptParameter, + header.getUncompressedSize(), + buffer.remaining(), + pageIndex, + chunkOrdinal); + byte[] plaintext = + decryptor.decryptPage( + buffer.array(), + buffer.arrayOffset() + buffer.position(), + buffer.remaining(), + pageCryptoContext); + if (plaintext.length != pageCryptoContext.getCompressedPlaintextSize()) { + throw new IOException( + Messages.format( + "error.encrypt.page_plaintext_size_mismatch", + pageCryptoContext.getCompressedPlaintextSize(), + plaintext.length)); + } + return ByteBuffer.wrap(plaintext); + } if (decryptor == null || decryptor.getEncryptionType() == EncryptionType.UNENCRYPTED) { return buffer; } @@ -2296,7 +2424,13 @@ public void close() throws IOException { if (resourceLogger.isDebugEnabled()) { resourceLogger.debug("{} reader is closed.", file); } - this.tsFileInput.close(); + try { + this.tsFileInput.close(); + } finally { + if (dataEncryptParam != null && dataEncryptParam.isTdePageAead()) { + dataEncryptParam.close(); + } + } } public String getFileName() { @@ -2437,7 +2571,7 @@ public long selfCheck( // ChunkMetadata of current ChunkGroup List chunkMetadataList = new ArrayList<>(); - int headerLength = TSFileConfig.MAGIC_STRING.getBytes().length + Byte.BYTES; + long headerLength = getDataStartOffset(); if (fileSize < headerLength) { return TsFileCheckStatus.INCOMPATIBLE_FILE; } @@ -2524,7 +2658,12 @@ public long selfCheck( Decoder valueDecoder = Decoder.getDecoderByType( chunkHeader.getEncodingType(), chunkHeader.getDataType()); - ByteBuffer pageData = readPage(pageHeader, chunkHeader.getCompressionType()); + ByteBuffer pageData = + readPage( + pageHeader, + chunkHeader.getCompressionType(), + 0, + chunkHeader.getChunkOrdinal()); TSEncoding configuredTimeEncoding = TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()); boolean isTimeColumn = @@ -2699,7 +2838,7 @@ public long selfCheckWithInfo( long fileSize = checkFile.length(); logger.info(Messages.get("log.read.sequence_reader_file_length"), fileSize); - int headerLength = TSFileConfig.MAGIC_STRING.getBytes().length + Byte.BYTES; + long headerLength = getDataStartOffset(); if (fileSize < headerLength) { return TsFileCheckStatus.INCOMPATIBLE_FILE; } @@ -2776,7 +2915,8 @@ public long checkChunkAndPagesStatistics(IChunkMetadata chunkMetadata) throws IO PageHeader pageHeader = this.readPageHeader(chunkHeader.getDataType(), false); Decoder valueDecoder = Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()); - ByteBuffer pageData = readPage(pageHeader, chunkHeader.getCompressionType()); + ByteBuffer pageData = + readPage(pageHeader, chunkHeader.getCompressionType(), 0, chunkHeader.getChunkOrdinal()); Decoder timeDecoder = Decoder.getDecoderByType( TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/UnClosedTsFileReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/UnClosedTsFileReader.java index 230a2e164..fcf11c9d6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/UnClosedTsFileReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/UnClosedTsFileReader.java @@ -21,7 +21,6 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.encrypt.EncryptParameter; -import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.exception.NotImplementedException; import org.apache.tsfile.file.metadata.TsFileMetadata; @@ -46,7 +45,7 @@ public UnClosedTsFileReader(String file, LongConsumer ioSizeRecorder) throws IOE public UnClosedTsFileReader( String file, EncryptParameter encryptParam, LongConsumer ioSizeRecorder) throws IOException { super(file, false, ioSizeRecorder, encryptParam); - this.dataEncryptParam = EncryptUtils.getEncryptParameter(encryptParam); + this.dataEncryptParam = super.getEncryptParam(); } /** unclosed file has no tail magic data. */ diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java index d41ad14a3..da2bdccd1 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java @@ -49,8 +49,8 @@ import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfByteArray; -/** used in query. */ -public class Chunk { +/** A materialized chunk. Close it after use to destroy its independent page-AEAD context. */ +public class Chunk implements AutoCloseable { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(Chunk.class) @@ -88,7 +88,7 @@ public Chunk( this.chunkData = buffer; this.deleteIntervalList = deleteIntervalList; this.chunkStatistic = chunkStatistic; - this.encryptParam = encryptParam; + this.encryptParam = copyPageAeadParameter(encryptParam); } public Chunk(ChunkHeader header, ByteBuffer buffer) { @@ -100,7 +100,18 @@ public Chunk(ChunkHeader header, ByteBuffer buffer) { public Chunk(ChunkHeader header, ByteBuffer buffer, EncryptParameter encryptParam) { this.chunkHeader = header; this.chunkData = buffer; - this.encryptParam = encryptParam; + this.encryptParam = copyPageAeadParameter(encryptParam); + } + + private static EncryptParameter copyPageAeadParameter(EncryptParameter parameter) { + return parameter != null && parameter.isTdePageAead() ? parameter.copy() : parameter; + } + + @Override + public void close() { + if (encryptParam != null && encryptParam.isTdePageAead()) { + encryptParam.close(); + } } public EncryptParameter getEncryptParam() { @@ -124,6 +135,10 @@ public void setDeleteIntervalList(List list) { } public void mergeChunkByAppendPage(Chunk chunk) throws IOException { + if ((encryptParam != null && encryptParam.isTdePageAead()) + || (chunk.encryptParam != null && chunk.encryptParam.isTdePageAead())) { + throw new IOException(Messages.get("error.chunk.merge_page_aead_unsupported")); + } int dataSize = 0; // from where the page data of the merged chunk starts, if -1, it means the merged chunk has // more than one page @@ -220,6 +235,10 @@ public Chunk rewrite(TSDataType newType, Chunk timeChunk) throws IOException { if (newType == null || newType == chunkHeader.getDataType()) { return this; } + if (encryptParam != null && encryptParam.isTdePageAead()) { + encryptParam.resumeAfterChunkOrdinal(chunkHeader.getChunkOrdinal()); + encryptParam.resumeAfterChunkOrdinal(timeChunk.getHeader().getChunkOrdinal()); + } TSEncoding encoding = TSFileDescriptor.getInstance().getConfig().getValueEncoder(newType); IMeasurementSchema schema = new MeasurementSchema( @@ -270,7 +289,8 @@ public Chunk rewrite(TSDataType newType, Chunk timeChunk) throws IOException { newChunkData.capacity(), newType, chunkHeader.getCompressionType(), - encoding); + encoding, + chunkWriter.getChunkOrdinal()); chunkData.flip(); timeChunk.chunkData.flip(); return new Chunk( @@ -285,6 +305,9 @@ public Chunk rewrite(TSDataType newType) throws IOException { if (newType == null || newType == chunkHeader.getDataType()) { return this; } + if (encryptParam != null && encryptParam.isTdePageAead()) { + encryptParam.resumeAfterChunkOrdinal(chunkHeader.getChunkOrdinal()); + } TSEncoding encoding = TSFileDescriptor.getInstance().getConfig().getValueEncoder(newType); IMeasurementSchema schema = new MeasurementSchema( @@ -317,7 +340,8 @@ public Chunk rewrite(TSDataType newType) throws IOException { newChunkData.capacity(), newType, chunkHeader.getCompressionType(), - encoding); + encoding, + chunkWriter.getChunkOrdinal()); chunkData.flip(); return new Chunk( newChunkHeader, diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/CachedChunkLoaderImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/CachedChunkLoaderImpl.java index 92925ea82..43ef003c8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/CachedChunkLoaderImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/CachedChunkLoaderImpl.java @@ -20,6 +20,7 @@ package org.apache.tsfile.read.controller; import org.apache.tsfile.common.cache.LRUCache; +import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.statistics.Statistics; @@ -61,7 +62,9 @@ public CachedChunkLoaderImpl(TsFileSequenceReader fileSequenceReader, int cacheS @Override protected Chunk loadObjectByKey(ChunkCacheKey chunkCacheKey) throws IOException { - return reader.readMemChunk(chunkCacheKey); + try (Chunk chunk = reader.readMemChunk(chunkCacheKey)) { + return new Chunk(chunk.getHeader(), chunk.getData(), (EncryptParameter) null); + } } }; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/TsFileLastReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/TsFileLastReader.java index 8700a850e..0b30ea634 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/TsFileLastReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/TsFileLastReader.java @@ -19,14 +19,13 @@ package org.apache.tsfile.read.reader; -import org.apache.tsfile.compress.IUnCompressor; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.header.PageHeader; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TimeseriesMetadata; -import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.i18n.Messages; import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.TsFileSequenceReader; @@ -181,6 +180,8 @@ private TimeValuePair readAlignedLastPoint(Chunk chunk, ChunkMetadata chunkMetad ByteBuffer chunkData = chunk.getData(); PageHeader lastPageHeader = null; ByteBuffer lastPageData = null; + int pageIndex = 0; + int lastPageIndex = -1; while (chunkData.hasRemaining()) { PageHeader pageHeader; if (chunk.isSinglePageChunk()) { @@ -196,17 +197,20 @@ private TimeValuePair readAlignedLastPoint(Chunk chunk, ChunkMetadata chunkMetad || (pageHeader.getStatistics() != null && pageHeader.getStatistics().getCount() > 0)) { lastPageHeader = pageHeader; lastPageData = pageData; + lastPageIndex = pageIndex; } + pageIndex++; } if (lastPageHeader != null) { - CompressionType compressionType = chunk.getHeader().getCompressionType(); - if (compressionType != CompressionType.UNCOMPRESSED) { - ByteBuffer uncompressedPage = ByteBuffer.allocate(lastPageHeader.getUncompressedSize()); - IUnCompressor.getUnCompressor(compressionType).uncompress(lastPageData, uncompressedPage); - lastPageData = uncompressedPage; - lastPageData.flip(); - } + lastPageData = + ChunkReader.deserializePageData( + lastPageHeader, + lastPageData, + chunk.getHeader(), + IDecryptor.getDecryptor(chunk.getEncryptParam()), + chunk.getEncryptParam(), + lastPageIndex); ValuePageReader valuePageReader = new ValuePageReader( @@ -265,15 +269,15 @@ private Pair readLastPoint( return new Pair<>(seriesMeta.getMeasurementId(), null); } - Chunk chunk = sequenceReader.readMemChunk(lastNonEmptyChunkMetadata); - - if (!isAligned) { - return new Pair<>(seriesMeta.getMeasurementId(), readNonAlignedLastPoint(chunk)); - } else { - return new Pair<>( - seriesMeta.getMeasurementId(), - readAlignedLastPoint( - chunk, lastNonEmptyChunkMetadata, seriesMeta.getStatistics().getEndTime())); + try (Chunk chunk = sequenceReader.readMemChunk(lastNonEmptyChunkMetadata)) { + if (!isAligned) { + return new Pair<>(seriesMeta.getMeasurementId(), readNonAlignedLastPoint(chunk)); + } else { + return new Pair<>( + seriesMeta.getMeasurementId(), + readAlignedLastPoint( + chunk, lastNonEmptyChunkMetadata, seriesMeta.getStatistics().getEndTime())); + } } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AbstractAlignedChunkReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AbstractAlignedChunkReader.java index acc9789e4..ab7a77ff4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AbstractAlignedChunkReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/AbstractAlignedChunkReader.java @@ -90,12 +90,15 @@ private void initAllPageReaders( List> valueChunkStatisticsList) throws IOException { // construct next satisfied page header + int pageIndex = 0; while (timeChunkDataBuffer.remaining() > 0) { // deserialize PageHeader from chunkDataBuffer AbstractAlignedPageReader alignedPageReader = isSinglePageChunk() - ? deserializeFromSinglePageChunk(timeChunkStatistics, valueChunkStatisticsList) - : deserializeFromMultiPageChunk(); + ? deserializeFromSinglePageChunk( + timeChunkStatistics, valueChunkStatisticsList, pageIndex) + : deserializeFromMultiPageChunk(pageIndex); + pageIndex++; if (alignedPageReader != null) { pageReaderList.add(alignedPageReader); } @@ -108,7 +111,8 @@ private boolean isSinglePageChunk() { private AbstractAlignedPageReader deserializeFromSinglePageChunk( Statistics timeChunkStatistics, - List> valueChunkStatisticsList) + List> valueChunkStatisticsList, + int pageIndex) throws IOException { PageHeader timePageHeader = PageHeader.deserializeFrom(timeChunkDataBuffer, timeChunkStatistics); @@ -132,12 +136,13 @@ private AbstractAlignedPageReader deserializeFromSinglePageChunk( skipCurrentPage(timePageHeader, valuePageHeaderList); return null; } - return constructAlignedPageReader(timePageHeader, valuePageHeaderList); + return constructAlignedPageReader(timePageHeader, valuePageHeaderList, pageIndex); } abstract boolean needSkipForSinglePageChunk(boolean isAllNull, PageHeader timePageHeader); - private AbstractAlignedPageReader deserializeFromMultiPageChunk() throws IOException { + private AbstractAlignedPageReader deserializeFromMultiPageChunk(int pageIndex) + throws IOException { PageHeader timePageHeader = PageHeader.deserializeFrom(timeChunkDataBuffer, timeChunkHeader.getDataType()); List valuePageHeaderList = new ArrayList<>(); @@ -158,7 +163,7 @@ private AbstractAlignedPageReader deserializeFromMultiPageChunk() throws IOExcep skipCurrentPage(timePageHeader, valuePageHeaderList); return null; } - return constructAlignedPageReader(timePageHeader, valuePageHeaderList); + return constructAlignedPageReader(timePageHeader, valuePageHeaderList, pageIndex); } abstract boolean needSkipForMultiPageChunk(boolean isAllNull, PageHeader timePageHeader); @@ -193,11 +198,17 @@ private void skipCurrentPage(PageHeader timePageHeader, List valuePa } private AbstractAlignedPageReader constructAlignedPageReader( - PageHeader timePageHeader, List rawValuePageHeaderList) throws IOException { + PageHeader timePageHeader, List rawValuePageHeaderList, int pageIndex) + throws IOException { IDecryptor decrytor = IDecryptor.getDecryptor(encryptParam); ByteBuffer timePageData = ChunkReader.deserializePageData( - timePageHeader, timeChunkDataBuffer, timeChunkHeader, decrytor); + timePageHeader, + timeChunkDataBuffer, + timeChunkHeader, + decrytor, + encryptParam, + pageIndex); List valuePageHeaderList = new ArrayList<>(); LazyLoadPageData[] lazyLoadPageDataArray = new LazyLoadPageData[rawValuePageHeaderList.size()]; @@ -238,7 +249,9 @@ private AbstractAlignedPageReader constructAlignedPageReader( valueChunkDataBufferList.get(i).array(), currentPagePosition, IUnCompressor.getUnCompressor(valueChunkHeader.getCompressionType()), - encryptParam); + encryptParam, + pageIndex, + valueChunkHeader.getChunkOrdinal()); valueDataTypeList.add(valueChunkHeader.getDataType()); valueDecoderList.add(valueChunkHeader.calculateDecoderForNonTimeChunk()); isAllNull = false; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java index b555a25e1..77e8b18d6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/chunk/ChunkReader.java @@ -22,6 +22,7 @@ import org.apache.tsfile.compress.IUnCompressor; import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.encrypt.IDecryptor; +import org.apache.tsfile.encrypt.PageCryptoContext; import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; @@ -81,7 +82,9 @@ public ChunkReader(Chunk chunk, long readStopTime) { private void initAllPageReaders(Statistics chunkStatistic) { // construct next satisfied page header + int pageIndex = 0; while (chunkDataBuffer.remaining() > 0) { + int currentPageIndex = pageIndex++; // deserialize a PageHeader from chunkDataBuffer PageHeader pageHeader; if (((byte) (chunkHeader.getChunkType() & 0x3F)) == MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER) { @@ -100,7 +103,7 @@ private void initAllPageReaders(Statistics chunkStatisti if (pageDeleted(pageHeader)) { skipCurrentPage(pageHeader); } else { - pageReaderList.add(constructPageReader(pageHeader)); + pageReaderList.add(constructPageReader(pageHeader, currentPageIndex)); } } } @@ -141,7 +144,7 @@ private void skipCurrentPage(PageHeader pageHeader) { chunkDataBuffer.position(chunkDataBuffer.position() + pageHeader.getCompressedSize()); } - private PageReader constructPageReader(PageHeader pageHeader) { + private PageReader constructPageReader(PageHeader pageHeader, int pageIndex) { IUnCompressor unCompressor = IUnCompressor.getUnCompressor(chunkHeader.getCompressionType()); // record the current position of chunkDataBuffer, use this to get the page data in PageReader // through directly accessing the buffer array @@ -151,7 +154,12 @@ private PageReader constructPageReader(PageHeader pageHeader) { new PageReader( pageHeader, new LazyLoadPageData( - chunkDataBuffer.array(), currentPagePosition, unCompressor, encryptParam), + chunkDataBuffer.array(), + currentPagePosition, + unCompressor, + encryptParam, + pageIndex, + chunkHeader.getChunkOrdinal()), chunkHeader.getDataType(), chunkHeader.calculateDecoderForNonTimeChunk(), getConfiguredTimeDecoder(), @@ -221,7 +229,55 @@ public static ByteBuffer decryptAndUncompressPageData( compressedPageData.arrayOffset() + compressedPageData.position(), compressedPageBodyLength); unCompressor.uncompress( - decryptedPageData, 0, compressedPageBodyLength, uncompressedPageData, 0); + decryptedPageData, 0, decryptedPageData.length, uncompressedPageData, 0); + } catch (Exception e) { + throw new IOException( + Messages.format( + "error.read.uncompress_error_with_header", + pageHeader.getUncompressedSize(), + pageHeader.getCompressedSize(), + pageHeader, + e.getMessage()), + e); + } + compressedPageData.position(compressedPageData.position() + compressedPageBodyLength); + return ByteBuffer.wrap(uncompressedPageData); + } + + public static ByteBuffer decryptAndUncompressPageData( + PageHeader pageHeader, + IUnCompressor unCompressor, + ByteBuffer compressedPageData, + IDecryptor decryptor, + EncryptParameter encryptParameter, + int pageIndex, + long chunkOrdinal) + throws IOException { + int compressedPageBodyLength = pageHeader.getCompressedSize(); + byte[] uncompressedPageData = new byte[pageHeader.getUncompressedSize()]; + try { + PageCryptoContext pageCryptoContext = + PageCryptoContext.forDecryption( + encryptParameter, + pageHeader.getUncompressedSize(), + compressedPageBodyLength, + pageIndex, + chunkOrdinal); + byte[] decryptedPageData = + decryptor.decryptPage( + compressedPageData.array(), + compressedPageData.arrayOffset() + compressedPageData.position(), + compressedPageBodyLength, + pageCryptoContext); + if (decryptedPageData.length != pageCryptoContext.getCompressedPlaintextSize()) { + throw new IOException( + Messages.format( + "error.encrypt.page_plaintext_size_mismatch", + pageCryptoContext.getCompressedPlaintextSize(), + decryptedPageData.length)); + } + unCompressor.uncompress( + decryptedPageData, 0, decryptedPageData.length, uncompressedPageData, 0); } catch (Exception e) { throw new IOException( Messages.format( @@ -243,6 +299,31 @@ public static ByteBuffer deserializePageData( ByteBuffer compressedPageBody = readCompressedPageData(pageHeader, chunkBuffer); if (decryptor == null || decryptor.getEncryptionType() == EncryptionType.UNENCRYPTED) { return uncompressPageData(pageHeader, unCompressor, compressedPageBody); + } + return decryptAndUncompressPageData(pageHeader, unCompressor, compressedPageBody, decryptor); + } + + public static ByteBuffer deserializePageData( + PageHeader pageHeader, + ByteBuffer chunkBuffer, + ChunkHeader chunkHeader, + IDecryptor decryptor, + EncryptParameter encryptParameter, + int pageIndex) + throws IOException { + IUnCompressor unCompressor = IUnCompressor.getUnCompressor(chunkHeader.getCompressionType()); + ByteBuffer compressedPageBody = readCompressedPageData(pageHeader, chunkBuffer); + if (encryptParameter != null && encryptParameter.isTdePageAead()) { + return decryptAndUncompressPageData( + pageHeader, + unCompressor, + compressedPageBody, + decryptor, + encryptParameter, + pageIndex, + chunkHeader.getChunkOrdinal()); + } else if (decryptor == null || decryptor.getEncryptionType() == EncryptionType.UNENCRYPTED) { + return uncompressPageData(pageHeader, unCompressor, compressedPageBody); } else { return decryptAndUncompressPageData(pageHeader, unCompressor, compressedPageBody, decryptor); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/LazyLoadPageData.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/LazyLoadPageData.java index 7b01a67f5..cfc185504 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/LazyLoadPageData.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/LazyLoadPageData.java @@ -23,7 +23,9 @@ import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.encrypt.IDecryptor; +import org.apache.tsfile.encrypt.PageCryptoContext; import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.file.metadata.enums.EncryptionType; import org.apache.tsfile.i18n.Messages; import java.io.IOException; @@ -39,11 +41,16 @@ public class LazyLoadPageData { private final EncryptParameter encryptParam; + private final int pageIndex; + private final long chunkOrdinal; + public LazyLoadPageData(byte[] data, int offset, IUnCompressor unCompressor) { this.chunkData = data; this.pageDataOffset = offset; this.unCompressor = unCompressor; this.encryptParam = EncryptUtils.getEncryptParameter(); + this.pageIndex = -1; + this.chunkOrdinal = -1; } public LazyLoadPageData( @@ -52,17 +59,67 @@ public LazyLoadPageData( this.pageDataOffset = offset; this.unCompressor = unCompressor; this.encryptParam = encryptParam; + this.pageIndex = -1; + this.chunkOrdinal = -1; + } + + public LazyLoadPageData( + byte[] data, + int offset, + IUnCompressor unCompressor, + EncryptParameter encryptParam, + int pageIndex) { + this(data, offset, unCompressor, encryptParam, pageIndex, -1); + } + + public LazyLoadPageData( + byte[] data, + int offset, + IUnCompressor unCompressor, + EncryptParameter encryptParam, + int pageIndex, + long chunkOrdinal) { + this.chunkData = data; + this.pageDataOffset = offset; + this.unCompressor = unCompressor; + this.encryptParam = encryptParam; + this.pageIndex = pageIndex; + this.chunkOrdinal = chunkOrdinal; } public ByteBuffer uncompressPageData(PageHeader pageHeader) throws IOException { int compressedPageBodyLength = pageHeader.getCompressedSize(); byte[] uncompressedPageData = new byte[pageHeader.getUncompressedSize()]; IDecryptor decryptor = IDecryptor.getDecryptor(encryptParam); - byte[] decryptedPageData = - decryptor.decrypt(chunkData, pageDataOffset, compressedPageBodyLength); + byte[] decryptedPageData; + if (encryptParam != null && encryptParam.isTdePageAead()) { + PageCryptoContext pageCryptoContext = + PageCryptoContext.forDecryption( + encryptParam, + pageHeader.getUncompressedSize(), + compressedPageBodyLength, + pageIndex, + chunkOrdinal); + decryptedPageData = + decryptor.decryptPage( + chunkData, pageDataOffset, compressedPageBodyLength, pageCryptoContext); + if (decryptedPageData.length != pageCryptoContext.getCompressedPlaintextSize()) { + throw new IOException( + Messages.format( + "error.encrypt.page_plaintext_size_mismatch", + pageCryptoContext.getCompressedPlaintextSize(), + decryptedPageData.length)); + } + } else if (decryptor.getEncryptionType() == EncryptionType.UNENCRYPTED) { + decryptedPageData = + java.util.Arrays.copyOfRange( + chunkData, pageDataOffset, pageDataOffset + compressedPageBodyLength); + } else { + decryptedPageData = decryptor.decrypt(chunkData, pageDataOffset, compressedPageBodyLength); + } try { unCompressor.uncompress( - decryptedPageData, 0, compressedPageBodyLength, uncompressedPageData, 0); + decryptedPageData, 0, decryptedPageData.length, uncompressedPageData, 0); } catch (Exception e) { throw new IOException( Messages.format( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileSketchTool.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileSketchTool.java index a20c0e888..a04c13aa5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileSketchTool.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TsFileSketchTool.java @@ -311,8 +311,7 @@ private void printFileInfo() { private void printChunk(List allChunkGroupMetadata) { try { - long nextChunkGroupHeaderPos = - (long) TSFileConfig.MAGIC_STRING.getBytes().length + Byte.BYTES; + long nextChunkGroupHeaderPos = reader.getDataStartOffset(); // ChunkGroup begins for (ChunkGroupMetadata chunkGroupMetadata : allChunkGroupMetadata) { printlnBoth( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java index 5521a4f72..b931c3b73 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java @@ -118,7 +118,7 @@ public TsFileWriter(File file) throws IOException { public TsFileWriter(File file, EncryptParameter firstEncryptParam) throws IOException { this( - new TsFileIOWriter(file), + new TsFileIOWriter(file, copyEncryptParameter(firstEncryptParam)), new Schema(), TSFileDescriptor.getInstance().getConfig(), firstEncryptParam); @@ -151,7 +151,7 @@ public TsFileWriter(File file, Schema schema) throws IOException { public TsFileWriter(File file, Schema schema, EncryptParameter firstEncryptParam) throws IOException { this( - new TsFileIOWriter(file), + new TsFileIOWriter(file, copyEncryptParameter(firstEncryptParam)), schema, TSFileDescriptor.getInstance().getConfig(), firstEncryptParam); @@ -170,7 +170,7 @@ public TsFileWriter(TsFileOutput output, Schema schema) throws IOException { public TsFileWriter(TsFileOutput output, Schema schema, EncryptParameter firstEncryptParam) throws IOException { this( - new TsFileIOWriter(output), + new TsFileIOWriter(output, copyEncryptParameter(firstEncryptParam)), schema, TSFileDescriptor.getInstance().getConfig(), firstEncryptParam); @@ -190,7 +190,11 @@ public TsFileWriter(File file, Schema schema, TSFileConfig conf) throws IOExcept public TsFileWriter( File file, Schema schema, TSFileConfig conf, EncryptParameter firstEncryptParam) throws IOException { - this(new TsFileIOWriter(file), schema, conf, firstEncryptParam); + this( + new TsFileIOWriter(file, conf, copyEncryptParameter(firstEncryptParam)), + schema, + conf, + firstEncryptParam); } /** @@ -231,23 +235,39 @@ protected TsFileWriter( if (this.pageSize >= chunkGroupSizeThreshold) { LOG.warn(Messages.get("log.write.page_size_warn"), pageSize, chunkGroupSizeThreshold); } - this.secondEncryptParam = EncryptUtils.getEncryptParameter(firstEncryptParam); - String encryptLevel; - if (firstEncryptParam != null - && !Objects.equals(firstEncryptParam.getType(), "UNENCRYPTED") - && !Objects.equals(firstEncryptParam.getType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) { - encryptLevel = "2"; - String str = - EncryptUtils.getKeyStr( - IEncryptor.getEncryptor(firstEncryptParam.getType(), firstEncryptParam.getKey()) - .encrypt(secondEncryptParam.getKey())); - fileWriter.setEncryptParam(encryptLevel, secondEncryptParam.getType(), str); + EncryptParameter fileEncryptParameter = fileWriter.getEncryptParameter(); + EncryptParameter effectiveEncryptParameter = + fileEncryptParameter != null && fileEncryptParameter.isTdePageAead() + ? fileEncryptParameter + : firstEncryptParam; + if (effectiveEncryptParameter != null && effectiveEncryptParameter.isTdePageAead()) { + this.secondEncryptParam = effectiveEncryptParameter; + fileWriter.setEncryptParam(effectiveEncryptParameter); + fileWriter.writeEncryptionHeaderIfNecessary(); } else { - encryptLevel = "0"; - fileWriter.setEncryptParam(encryptLevel, "org.apache.tsfile.encrypt.UNENCRYPTED", ""); + this.secondEncryptParam = EncryptUtils.getEncryptParameter(firstEncryptParam); + String encryptLevel; + if (firstEncryptParam != null + && !Objects.equals(firstEncryptParam.getType(), "UNENCRYPTED") + && !Objects.equals( + firstEncryptParam.getType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) { + encryptLevel = "2"; + String str = + EncryptUtils.getKeyStr( + IEncryptor.getEncryptor(firstEncryptParam.getType(), firstEncryptParam.getKey()) + .encrypt(secondEncryptParam.getKey())); + fileWriter.setEncryptParam(encryptLevel, secondEncryptParam.getType(), str); + } else { + encryptLevel = "0"; + fileWriter.setEncryptParam(encryptLevel, "org.apache.tsfile.encrypt.UNENCRYPTED", ""); + } } } + private static EncryptParameter copyEncryptParameter(EncryptParameter parameter) { + return parameter == null ? null : parameter.copy(); + } + public void setChunkGroupSizeThreshold(long chunkGroupSizeThreshold) { this.chunkGroupSizeThreshold = chunkGroupSizeThreshold; } @@ -729,8 +749,14 @@ private void reset() { @TsFileApi public void close() throws IOException { LOG.info(Messages.get("log.write.close_file")); - flush(); - fileWriter.endFile(); + try { + flush(); + fileWriter.endFile(); + } finally { + if (secondEncryptParam != null && secondEncryptParam.isTdePageAead()) { + secondEncryptParam.close(); + } + } } /** diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/ChunkWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/ChunkWriterImpl.java index 7d7912b14..2eba37f3a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/ChunkWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/ChunkWriterImpl.java @@ -55,6 +55,7 @@ public class ChunkWriterImpl implements IChunkWriter { private final ICompressor compressor; private final EncryptParameter encryptParam; + private long chunkOrdinal = -1; /** all pages of this chunk. */ private final PublicBAOS pageBuffer; @@ -106,6 +107,7 @@ public ChunkWriterImpl(IMeasurementSchema schema) { this.measurementSchema = schema; this.compressor = ICompressor.getCompressor(schema.getCompressor()); this.encryptParam = EncryptUtils.getEncryptParameter(); + this.chunkOrdinal = nextChunkOrdinal(); this.pageBuffer = new PublicBAOS(); this.pageSizeThreshold = TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); @@ -118,6 +120,7 @@ public ChunkWriterImpl(IMeasurementSchema schema) { this.statistics = Statistics.getStatsByType(measurementSchema.getType()); this.pageWriter = new PageWriter(measurementSchema, encryptParam); + this.pageWriter.setChunkOrdinal(chunkOrdinal); this.pageWriter.setTimeEncoder(measurementSchema.getTimeEncoder()); this.pageWriter.setValueEncoder(measurementSchema.getValueEncoder()); @@ -130,6 +133,7 @@ public ChunkWriterImpl(IMeasurementSchema schema, EncryptParameter encryptParam) this.measurementSchema = schema; this.compressor = ICompressor.getCompressor(schema.getCompressor()); this.encryptParam = encryptParam; + this.chunkOrdinal = nextChunkOrdinal(); this.pageBuffer = new PublicBAOS(); this.pageSizeThreshold = TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); @@ -142,6 +146,7 @@ public ChunkWriterImpl(IMeasurementSchema schema, EncryptParameter encryptParam) this.statistics = Statistics.getStatsByType(measurementSchema.getType()); this.pageWriter = new PageWriter(measurementSchema, this.encryptParam); + this.pageWriter.setChunkOrdinal(chunkOrdinal); this.pageWriter.setTimeEncoder(measurementSchema.getTimeEncoder()); this.pageWriter.setValueEncoder(measurementSchema.getValueEncoder()); @@ -161,6 +166,12 @@ public ChunkWriterImpl( this.isMerging = isMerging; } + private long nextChunkOrdinal() { + return encryptParam != null && encryptParam.isTdePageAead() + ? encryptParam.nextChunkOrdinal() + : -1; + } + private void checkSdtEncoding() { if (measurementSchema.getProps() != null && !isMerging) { if (measurementSchema.getProps().getOrDefault(LOSS, "").equals(SDT)) { @@ -322,17 +333,18 @@ private void writePageToPageBuffer() { try { if (numOfPages == 0) { // record the firstPageStatistics this.firstPageStatistics = pageWriter.getStatistics(); - this.sizeWithoutStatistic = pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, true); + this.sizeWithoutStatistic = + pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, true, numOfPages); } else if (numOfPages == 1) { // put the firstPageStatistics into pageBuffer byte[] b = pageBuffer.toByteArray(); pageBuffer.reset(); pageBuffer.write(b, 0, this.sizeWithoutStatistic); firstPageStatistics.serialize(pageBuffer); pageBuffer.write(b, this.sizeWithoutStatistic, b.length - this.sizeWithoutStatistic); - pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, false); + pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, false, numOfPages); firstPageStatistics = null; } else { - pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, false); + pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, false, numOfPages); } // update statistics of this chunk @@ -364,6 +376,8 @@ public void writeToFileWriter( sizeWithoutStatistic = 0; firstPageStatistics = null; this.statistics = Statistics.getStatsByType(measurementSchema.getType()); + this.chunkOrdinal = nextChunkOrdinal(); + pageWriter.setChunkOrdinal(chunkOrdinal); } @Override @@ -380,7 +394,8 @@ public long getSerializedChunkSize() { return 0; } // return the serialized size of the chunk header + all pages - return ChunkHeader.getSerializedSize(measurementSchema.getMeasurementName(), pageBuffer.size()) + return ChunkHeader.getSerializedSize( + measurementSchema.getMeasurementName(), pageBuffer.size(), chunkOrdinal >= 0) + (long) pageBuffer.size(); } @@ -505,7 +520,9 @@ private void writeAllPagesOfChunkToTsFile( statistics, pageBuffer.size(), numOfPages, - 0); + 0, + chunkOrdinal, + encryptParam); long dataOffset = writer.getPos(); @@ -542,6 +559,10 @@ public int getNumOfPages() { return numOfPages; } + public long getChunkOrdinal() { + return chunkOrdinal; + } + public ByteBuffer getByteBuffer() { return ByteBuffer.wrap(pageBuffer.toByteArray()); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/TimeChunkWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/TimeChunkWriter.java index 392a0cf45..0796162b3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/TimeChunkWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/TimeChunkWriter.java @@ -57,6 +57,7 @@ public class TimeChunkWriter { private CompressionType compressionType; private EncryptParameter encryptParam; + private long chunkOrdinal = -1; /** all pages of this chunk. */ private PublicBAOS pageBuffer; @@ -96,6 +97,7 @@ public TimeChunkWriter( this.encodingType = encodingType; this.compressionType = compressionType; this.encryptParam = EncryptUtils.getEncryptParameter(); + this.chunkOrdinal = nextChunkOrdinal(); this.pageBuffer = new PublicBAOS(); this.pageSizeThreshold = TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); @@ -110,6 +112,7 @@ public TimeChunkWriter( this.pageWriter = new TimePageWriter( timeEncoder, ICompressor.getCompressor(compressionType), this.encryptParam); + this.pageWriter.setChunkOrdinal(chunkOrdinal); } public TimeChunkWriter( @@ -122,6 +125,7 @@ public TimeChunkWriter( this.encodingType = encodingType; this.compressionType = compressionType; this.encryptParam = encryptParam; + this.chunkOrdinal = nextChunkOrdinal(); this.pageBuffer = new PublicBAOS(); this.pageSizeThreshold = TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); @@ -135,12 +139,19 @@ public TimeChunkWriter( this.pageWriter = new TimePageWriter(timeEncoder, ICompressor.getCompressor(compressionType), encryptParam); + this.pageWriter.setChunkOrdinal(chunkOrdinal); } public void write(long time) { pageWriter.write(time); } + private long nextChunkOrdinal() { + return encryptParam != null && encryptParam.isTdePageAead() + ? encryptParam.nextChunkOrdinal() + : -1; + } + public void write(long[] timestamps, int batchSize, int arrayOffset) { pageWriter.write(timestamps, batchSize, arrayOffset); } @@ -184,17 +195,18 @@ public void writePageToPageBuffer() { try { if (numOfPages == 0) { // record the firstPageStatistics this.firstPageStatistics = pageWriter.getStatistics(); - this.sizeWithoutStatistic = pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, true); + this.sizeWithoutStatistic = + pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, true, numOfPages); } else if (numOfPages == 1) { // put the firstPageStatistics into pageBuffer byte[] b = pageBuffer.toByteArray(); pageBuffer.reset(); pageBuffer.write(b, 0, this.sizeWithoutStatistic); firstPageStatistics.serialize(pageBuffer); pageBuffer.write(b, this.sizeWithoutStatistic, b.length - this.sizeWithoutStatistic); - pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, false); + pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, false, numOfPages); firstPageStatistics = null; } else { - pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, false); + pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, false, numOfPages); } // update statistics of this chunk @@ -262,6 +274,8 @@ public void writeToFileWriter(TsFileIOWriter tsfileWriter) throws IOException { sizeWithoutStatistic = 0; firstPageStatistics = null; this.statistics = new TimeStatistics(); + this.chunkOrdinal = nextChunkOrdinal(); + pageWriter.setChunkOrdinal(chunkOrdinal); } public long estimateMaxSeriesMemSize() { @@ -276,7 +290,7 @@ public long getCurrentChunkSize() { return 0; } // return the serialized size of the chunk header + all pages - return ChunkHeader.getSerializedSize(measurementId, pageBuffer.size()) + return ChunkHeader.getSerializedSize(measurementId, pageBuffer.size(), chunkOrdinal >= 0) + (long) pageBuffer.size(); } @@ -322,7 +336,9 @@ public void writeAllPagesOfChunkToTsFile(TsFileIOWriter writer) throws IOExcepti statistics, pageBuffer.size(), numOfPages, - TsFileConstant.TIME_COLUMN_MASK); + TsFileConstant.TIME_COLUMN_MASK, + chunkOrdinal, + encryptParam); long dataOffset = writer.getPos(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/ValueChunkWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/ValueChunkWriter.java index 382df6e80..c7709daf4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/ValueChunkWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/ValueChunkWriter.java @@ -61,6 +61,7 @@ public class ValueChunkWriter { private final CompressionType compressionType; private final EncryptParameter encryptParam; + private long chunkOrdinal = -1; /** all pages of this chunk. */ private final PublicBAOS pageBuffer; @@ -100,6 +101,7 @@ public ValueChunkWriter( this.dataType = dataType; this.compressionType = compressionType; this.encryptParam = EncryptUtils.getEncryptParameter(); + this.chunkOrdinal = nextChunkOrdinal(); this.pageBuffer = new PublicBAOS(); this.pageSizeThreshold = TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); this.maxNumberOfPointsInPage = @@ -112,6 +114,7 @@ public ValueChunkWriter( this.pageWriter = new ValuePageWriter( valueEncoder, ICompressor.getCompressor(compressionType), dataType, this.encryptParam); + this.pageWriter.setChunkOrdinal(chunkOrdinal); } public ValueChunkWriter( @@ -126,6 +129,7 @@ public ValueChunkWriter( this.dataType = dataType; this.compressionType = compressionType; this.encryptParam = encryptParam; + this.chunkOrdinal = nextChunkOrdinal(); this.pageBuffer = new PublicBAOS(); this.pageSizeThreshold = TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); this.maxNumberOfPointsInPage = @@ -138,6 +142,13 @@ public ValueChunkWriter( this.pageWriter = new ValuePageWriter( valueEncoder, ICompressor.getCompressor(compressionType), dataType, this.encryptParam); + this.pageWriter.setChunkOrdinal(chunkOrdinal); + } + + private long nextChunkOrdinal() { + return encryptParam != null && encryptParam.isTdePageAead() + ? encryptParam.nextChunkOrdinal() + : -1; } public void write(long time, long value, boolean isNull) { @@ -214,7 +225,8 @@ public void writePageToPageBuffer() { // record the firstPageStatistics if it is not empty page this.firstPageStatistics = pageWriter.getStatistics(); } - this.sizeWithoutStatistic = pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, true); + this.sizeWithoutStatistic = + pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, true, numOfPages); } else if (numOfPages == 1) { // put the firstPageStatistics into pageBuffer if (firstPageStatistics != null) { // Consider previous page is an empty page byte[] b = pageBuffer.toByteArray(); @@ -223,10 +235,10 @@ public void writePageToPageBuffer() { firstPageStatistics.serialize(pageBuffer); pageBuffer.write(b, this.sizeWithoutStatistic, b.length - this.sizeWithoutStatistic); } - pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, false); + pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, false, numOfPages); firstPageStatistics = null; } else { - pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, false); + pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, false, numOfPages); } // update statistics of this chunk @@ -314,6 +326,8 @@ public void writeToFileWriter( sizeWithoutStatistic = 0; firstPageStatistics = null; this.statistics = Statistics.getStatsByType(dataType); + this.chunkOrdinal = nextChunkOrdinal(); + pageWriter.setChunkOrdinal(chunkOrdinal); } public long estimateMaxSeriesMemSize() { @@ -335,11 +349,11 @@ public long getCurrentChunkSize() { // Empty chunk, it may happen if pageBuffer stores empty bits and only chunk header will be // flushed. if (statistics.getCount() == 0) { - return ChunkHeader.getSerializedSize(measurementId, 0); + return ChunkHeader.getSerializedSize(measurementId, 0, chunkOrdinal >= 0); } // return the serialized size of the chunk header + all pages - return ChunkHeader.getSerializedSize(measurementId, pageBuffer.size()) + return ChunkHeader.getSerializedSize(measurementId, pageBuffer.size(), chunkOrdinal >= 0) + (long) pageBuffer.size(); } @@ -385,6 +399,10 @@ public int getNumOfPages() { return numOfPages; } + public long getChunkOrdinal() { + return chunkOrdinal; + } + public TSDataType getDataType() { return dataType; } @@ -421,7 +439,9 @@ public void writeAllPagesOfChunkToTsFile( statistics, 0, 0, - TsFileConstant.VALUE_COLUMN_MASK); + TsFileConstant.VALUE_COLUMN_MASK, + chunkOrdinal, + encryptParam); writer.endCurrentChunk(); return; } @@ -435,7 +455,9 @@ public void writeAllPagesOfChunkToTsFile( statistics, pageBuffer.size(), numOfPages, - TsFileConstant.VALUE_COLUMN_MASK); + TsFileConstant.VALUE_COLUMN_MASK, + chunkOrdinal, + encryptParam); long dataOffset = writer.getPos(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/page/EncodedPageBody.java b/java/tsfile/src/main/java/org/apache/tsfile/write/page/EncodedPageBody.java new file mode 100644 index 000000000..6089de625 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/page/EncodedPageBody.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.write.page; + +class EncodedPageBody { + + private final byte[] data; + private final int offset; + private final int size; + + EncodedPageBody(byte[] data) { + this(data, 0, data.length); + } + + EncodedPageBody(byte[] data, int offset, int size) { + this.data = data; + this.offset = offset; + this.size = size; + } + + byte[] getData() { + return data; + } + + int getOffset() { + return offset; + } + + int size() { + return size; + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/page/PageBodyEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/write/page/PageBodyEncoder.java new file mode 100644 index 000000000..0122936d3 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/page/PageBodyEncoder.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.write.page; + +import org.apache.tsfile.compress.ICompressor; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.IEncryptor; +import org.apache.tsfile.encrypt.PageCryptoContext; +import org.apache.tsfile.exception.encrypt.EncryptException; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.EncryptionType; +import org.apache.tsfile.i18n.Messages; + +import java.io.IOException; +import java.nio.ByteBuffer; + +class PageBodyEncoder { + + private PageBodyEncoder() {} + + static EncodedPageBody encode( + ByteBuffer pageData, + int uncompressedSize, + ICompressor compressor, + EncryptParameter encryptParameter, + int pageIndex, + long chunkOrdinal) + throws IOException { + byte[] plaintext; + int plaintextOffset = 0; + int compressedSize; + + if (compressor.getType().equals(CompressionType.UNCOMPRESSED)) { + plaintext = pageData.array(); + plaintextOffset = pageData.position(); + compressedSize = uncompressedSize; + } else if (compressor.getType().equals(CompressionType.GZIP)) { + plaintext = compressor.compress(pageData.array(), pageData.position(), uncompressedSize); + compressedSize = plaintext.length; + } else { + plaintext = new byte[compressor.getMaxBytesForCompression(uncompressedSize)]; + compressedSize = + compressor.compress(pageData.array(), pageData.position(), uncompressedSize, plaintext); + } + + IEncryptor encryptor = IEncryptor.getEncryptor(encryptParameter); + byte[] encryptedPageBody; + if (encryptParameter != null && encryptParameter.isTdePageAead()) { + PageCryptoContext pageCryptoContext = + PageCryptoContext.forEncryption( + encryptParameter, uncompressedSize, compressedSize, pageIndex, chunkOrdinal); + encryptedPageBody = + encryptor.encryptPage(plaintext, plaintextOffset, compressedSize, pageCryptoContext); + if (encryptedPageBody.length != pageCryptoContext.getEncryptedPageBodySize()) { + throw new EncryptException( + Messages.format( + "error.encrypt.page_output_size_mismatch", + pageCryptoContext.getEncryptedPageBodySize(), + encryptedPageBody.length)); + } + } else if (encryptor.getEncryptionType() == EncryptionType.UNENCRYPTED) { + return new EncodedPageBody(plaintext, plaintextOffset, compressedSize); + } else { + encryptedPageBody = encryptor.encrypt(plaintext, plaintextOffset, compressedSize); + } + return new EncodedPageBody(encryptedPageBody); + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/page/PageWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/page/PageWriter.java index d479ad8d4..2c6f35680 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/page/PageWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/page/PageWriter.java @@ -22,10 +22,7 @@ import org.apache.tsfile.encoding.encoder.Encoder; import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.encrypt.EncryptUtils; -import org.apache.tsfile.encrypt.IEncryptor; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.enums.CompressionType; -import org.apache.tsfile.file.metadata.enums.EncryptionType; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.PublicBAOS; @@ -38,8 +35,6 @@ import java.io.IOException; import java.io.Serializable; import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.WritableByteChannel; /** * This writer is used to write time-value into a page. It consists of a time encoder, a value @@ -52,6 +47,7 @@ public class PageWriter { private ICompressor compressor; private EncryptParameter encryptParam; + private long chunkOrdinal = -1; // time private Encoder timeEncoder; @@ -231,28 +227,22 @@ public ByteBuffer getUncompressedBytes() throws IOException { /** write the page header and data into the PageWriter's output stream. */ public int writePageHeaderAndDataIntoBuff(PublicBAOS pageBuffer, boolean first) throws IOException { + return writePageHeaderAndDataIntoBuff(pageBuffer, first, first ? 0 : -1); + } + + /** write the page header and data into the PageWriter's output stream. */ + public int writePageHeaderAndDataIntoBuff(PublicBAOS pageBuffer, boolean first, int pageIndex) + throws IOException { if (statistics.getCount() == 0) { return 0; } ByteBuffer pageData = getUncompressedBytes(); int uncompressedSize = pageData.remaining(); - int compressedSize; - byte[] compressedBytes = null; - - if (compressor.getType().equals(CompressionType.UNCOMPRESSED)) { - compressedSize = uncompressedSize; - } else if (compressor.getType().equals(CompressionType.GZIP)) { - compressedBytes = - compressor.compress(pageData.array(), pageData.position(), uncompressedSize); - compressedSize = compressedBytes.length; - } else { - compressedBytes = new byte[compressor.getMaxBytesForCompression(uncompressedSize)]; - // data is never a directByteBuffer now, so we can use data.array() - compressedSize = - compressor.compress( - pageData.array(), pageData.position(), uncompressedSize, compressedBytes); - } + EncodedPageBody pageBody = + PageBodyEncoder.encode( + pageData, uncompressedSize, compressor, encryptParam, pageIndex, chunkOrdinal); + int pageBodySize = pageBody.size(); // write the page header to IOWriter int sizeWithoutStatistic = 0; @@ -260,41 +250,16 @@ public int writePageHeaderAndDataIntoBuff(PublicBAOS pageBuffer, boolean first) sizeWithoutStatistic += ReadWriteForEncodingUtils.writeUnsignedVarInt(uncompressedSize, pageBuffer); sizeWithoutStatistic += - ReadWriteForEncodingUtils.writeUnsignedVarInt(compressedSize, pageBuffer); + ReadWriteForEncodingUtils.writeUnsignedVarInt(pageBodySize, pageBuffer); } else { ReadWriteForEncodingUtils.writeUnsignedVarInt(uncompressedSize, pageBuffer); - ReadWriteForEncodingUtils.writeUnsignedVarInt(compressedSize, pageBuffer); + ReadWriteForEncodingUtils.writeUnsignedVarInt(pageBodySize, pageBuffer); statistics.serialize(pageBuffer); } - IEncryptor encryptor = IEncryptor.getEncryptor(encryptParam); - // write page content to temp PBAOS logger.trace("start to flush a page data into buffer, buffer position {} ", pageBuffer.size()); - if (compressor.getType().equals(CompressionType.UNCOMPRESSED)) { - if (encryptor.getEncryptionType().equals(EncryptionType.UNENCRYPTED)) { - try (WritableByteChannel channel = Channels.newChannel(pageBuffer)) { - channel.write(pageData); - } - } else { - byte[] encryptedBytes = null; - encryptedBytes = encryptor.encrypt(pageData.array(), pageData.position(), uncompressedSize); - // data is never a directByteBuffer now, so we can use data.array() - int encryptedSize = encryptedBytes.length; - pageBuffer.write(encryptedBytes, 0, encryptedSize); - } - - } else { - if (encryptor.getEncryptionType().equals(EncryptionType.UNENCRYPTED)) { - pageBuffer.write(compressedBytes, 0, compressedSize); - } else { - byte[] encryptedBytes = null; - encryptedBytes = encryptor.encrypt(compressedBytes, 0, compressedSize); - // data is never a directByteBuffer now, so we can use data.array() - int encryptedSize = encryptedBytes.length; - pageBuffer.write(encryptedBytes, 0, encryptedSize); - } - } + pageBuffer.write(pageBody.getData(), pageBody.getOffset(), pageBodySize); logger.trace("start to flush a page data into buffer, buffer position {} ", pageBuffer.size()); return sizeWithoutStatistic; } @@ -319,6 +284,10 @@ public void reset(IMeasurementSchema measurementSchema) { statistics = Statistics.getStatsByType(measurementSchema.getType()); } + public void setChunkOrdinal(long chunkOrdinal) { + this.chunkOrdinal = chunkOrdinal; + } + public void setTimeEncoder(Encoder encoder) { this.timeEncoder = encoder; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/page/TimePageWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/page/TimePageWriter.java index e09e66cdf..d75bda019 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/page/TimePageWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/page/TimePageWriter.java @@ -22,9 +22,6 @@ import org.apache.tsfile.encoding.encoder.Encoder; import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.encrypt.EncryptUtils; -import org.apache.tsfile.encrypt.IEncryptor; -import org.apache.tsfile.file.metadata.enums.CompressionType; -import org.apache.tsfile.file.metadata.enums.EncryptionType; import org.apache.tsfile.file.metadata.statistics.TimeStatistics; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; @@ -34,8 +31,6 @@ import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.WritableByteChannel; /** * This writer is used to write time into a page. It consists of a time encoder and respective @@ -48,6 +43,7 @@ public class TimePageWriter { private final ICompressor compressor; private final EncryptParameter encryptParam; + private long chunkOrdinal = -1; // time private Encoder timeEncoder; @@ -114,28 +110,22 @@ public ByteBuffer getUncompressedBytes() throws IOException { /** write the page header and data into the PageWriter's output stream. */ public int writePageHeaderAndDataIntoBuff(PublicBAOS pageBuffer, boolean first) throws IOException { + return writePageHeaderAndDataIntoBuff(pageBuffer, first, first ? 0 : -1); + } + + /** write the page header and data into the PageWriter's output stream. */ + public int writePageHeaderAndDataIntoBuff(PublicBAOS pageBuffer, boolean first, int pageIndex) + throws IOException { if (statistics.getCount() == 0) { return 0; } ByteBuffer pageData = getUncompressedBytes(); int uncompressedSize = pageData.remaining(); - int compressedSize; - byte[] compressedBytes = null; - - if (compressor.getType().equals(CompressionType.UNCOMPRESSED)) { - compressedSize = uncompressedSize; - } else if (compressor.getType().equals(CompressionType.GZIP)) { - compressedBytes = - compressor.compress(pageData.array(), pageData.position(), uncompressedSize); - compressedSize = compressedBytes.length; - } else { - compressedBytes = new byte[compressor.getMaxBytesForCompression(uncompressedSize)]; - // data is never a directByteBuffer now, so we can use data.array() - compressedSize = - compressor.compress( - pageData.array(), pageData.position(), uncompressedSize, compressedBytes); - } + EncodedPageBody pageBody = + PageBodyEncoder.encode( + pageData, uncompressedSize, compressor, encryptParam, pageIndex, chunkOrdinal); + int pageBodySize = pageBody.size(); // write the page header to IOWriter int sizeWithoutStatistic = 0; @@ -143,41 +133,17 @@ public int writePageHeaderAndDataIntoBuff(PublicBAOS pageBuffer, boolean first) sizeWithoutStatistic += ReadWriteForEncodingUtils.writeUnsignedVarInt(uncompressedSize, pageBuffer); sizeWithoutStatistic += - ReadWriteForEncodingUtils.writeUnsignedVarInt(compressedSize, pageBuffer); + ReadWriteForEncodingUtils.writeUnsignedVarInt(pageBodySize, pageBuffer); } else { ReadWriteForEncodingUtils.writeUnsignedVarInt(uncompressedSize, pageBuffer); - ReadWriteForEncodingUtils.writeUnsignedVarInt(compressedSize, pageBuffer); + ReadWriteForEncodingUtils.writeUnsignedVarInt(pageBodySize, pageBuffer); statistics.serialize(pageBuffer); } // write page content to temp PBAOS logger.trace( "start to flush a time page data into buffer, buffer position {} ", pageBuffer.size()); - IEncryptor encryptor = IEncryptor.getEncryptor(encryptParam); - if (compressor.getType().equals(CompressionType.UNCOMPRESSED)) { - if (encryptor.getEncryptionType().equals(EncryptionType.UNENCRYPTED)) { - try (WritableByteChannel channel = Channels.newChannel(pageBuffer)) { - channel.write(pageData); - } - } else { - byte[] encryptedBytes = null; - encryptedBytes = encryptor.encrypt(pageData.array(), pageData.position(), uncompressedSize); - // data is never a directByteBuffer now, so we can use data.array() - int encryptedSize = encryptedBytes.length; - pageBuffer.write(encryptedBytes, 0, encryptedSize); - } - - } else { - if (encryptor.getEncryptionType().equals(EncryptionType.UNENCRYPTED)) { - pageBuffer.write(compressedBytes, 0, compressedSize); - } else { - byte[] encryptedBytes = null; - encryptedBytes = encryptor.encrypt(compressedBytes, 0, compressedSize); - // data is never a directByteBuffer now, so we can use data.array() - int encryptedSize = encryptedBytes.length; - pageBuffer.write(encryptedBytes, 0, encryptedSize); - } - } + pageBuffer.write(pageBody.getData(), pageBody.getOffset(), pageBodySize); logger.trace( "finish flushing a time page data into buffer, buffer position {} ", pageBuffer.size()); return sizeWithoutStatistic; @@ -199,6 +165,10 @@ public void reset() { statistics = new TimeStatistics(); } + public void setChunkOrdinal(long chunkOrdinal) { + this.chunkOrdinal = chunkOrdinal; + } + public void setTimeEncoder(Encoder encoder) { this.timeEncoder = encoder; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/page/ValuePageWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/page/ValuePageWriter.java index f7f375759..265186f9f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/page/ValuePageWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/page/ValuePageWriter.java @@ -22,10 +22,7 @@ import org.apache.tsfile.encoding.encoder.Encoder; import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.encrypt.EncryptUtils; -import org.apache.tsfile.encrypt.IEncryptor; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.enums.CompressionType; -import org.apache.tsfile.file.metadata.enums.EncryptionType; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; @@ -38,8 +35,6 @@ import java.io.IOException; import java.io.Serializable; import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.WritableByteChannel; /** * This writer is used to write value into a page. It consists of a value encoder and respective @@ -51,6 +46,7 @@ public class ValuePageWriter { private final ICompressor compressor; private final EncryptParameter encryptParam; + private long chunkOrdinal = -1; // value private Encoder valueEncoder; @@ -405,6 +401,12 @@ public int writeEmptyPageIntoBuff(PublicBAOS pageBuffer) { /** write the page header and data into the PageWriter's output stream. */ public int writePageHeaderAndDataIntoBuff(PublicBAOS pageBuffer, boolean first) throws IOException { + return writePageHeaderAndDataIntoBuff(pageBuffer, first, first ? 0 : -1); + } + + /** write the page header and data into the PageWriter's output stream. */ + public int writePageHeaderAndDataIntoBuff(PublicBAOS pageBuffer, boolean first, int pageIndex) + throws IOException { if (size == 0) { return 0; } else if (statistics.getCount() == 0) { @@ -414,22 +416,10 @@ public int writePageHeaderAndDataIntoBuff(PublicBAOS pageBuffer, boolean first) ByteBuffer pageData = getUncompressedBytes(); int uncompressedSize = pageData.remaining(); - int compressedSize; - byte[] compressedBytes = null; - - if (compressor.getType().equals(CompressionType.UNCOMPRESSED)) { - compressedSize = uncompressedSize; - } else if (compressor.getType().equals(CompressionType.GZIP)) { - compressedBytes = - compressor.compress(pageData.array(), pageData.position(), uncompressedSize); - compressedSize = compressedBytes.length; - } else { - compressedBytes = new byte[compressor.getMaxBytesForCompression(uncompressedSize)]; - // data is never a directByteBuffer now, so we can use data.array() - compressedSize = - compressor.compress( - pageData.array(), pageData.position(), uncompressedSize, compressedBytes); - } + EncodedPageBody pageBody = + PageBodyEncoder.encode( + pageData, uncompressedSize, compressor, encryptParam, pageIndex, chunkOrdinal); + int pageBodySize = pageBody.size(); // write the page header to IOWriter int sizeWithoutStatistic = 0; @@ -437,39 +427,15 @@ public int writePageHeaderAndDataIntoBuff(PublicBAOS pageBuffer, boolean first) sizeWithoutStatistic += ReadWriteForEncodingUtils.writeUnsignedVarInt(uncompressedSize, pageBuffer); sizeWithoutStatistic += - ReadWriteForEncodingUtils.writeUnsignedVarInt(compressedSize, pageBuffer); + ReadWriteForEncodingUtils.writeUnsignedVarInt(pageBodySize, pageBuffer); } else { ReadWriteForEncodingUtils.writeUnsignedVarInt(uncompressedSize, pageBuffer); - ReadWriteForEncodingUtils.writeUnsignedVarInt(compressedSize, pageBuffer); + ReadWriteForEncodingUtils.writeUnsignedVarInt(pageBodySize, pageBuffer); statistics.serialize(pageBuffer); } - IEncryptor encryptor = IEncryptor.getEncryptor(encryptParam); // write page content to temp PBAOS logger.trace("start to flush a page data into buffer, buffer position {} ", pageBuffer.size()); - if (compressor.getType().equals(CompressionType.UNCOMPRESSED)) { - if (encryptor.getEncryptionType().equals(EncryptionType.UNENCRYPTED)) { - try (WritableByteChannel channel = Channels.newChannel(pageBuffer)) { - channel.write(pageData); - } - } else { - byte[] encryptedBytes = null; - encryptedBytes = encryptor.encrypt(pageData.array(), pageData.position(), uncompressedSize); - // data is never a directByteBuffer now, so we can use data.array() - int encryptedSize = encryptedBytes.length; - pageBuffer.write(encryptedBytes, 0, encryptedSize); - } - - } else { - if (encryptor.getEncryptionType().equals(EncryptionType.UNENCRYPTED)) { - pageBuffer.write(compressedBytes, 0, compressedSize); - } else { - byte[] encryptedBytes = null; - encryptedBytes = encryptor.encrypt(compressedBytes, 0, compressedSize); - // data is never a directByteBuffer now, so we can use data.array() - int encryptedSize = encryptedBytes.length; - pageBuffer.write(encryptedBytes, 0, encryptedSize); - } - } + pageBuffer.write(pageBody.getData(), pageBody.getOffset(), pageBodySize); logger.trace("start to flush a page data into buffer, buffer position {} ", pageBuffer.size()); return sizeWithoutStatistic; } @@ -493,6 +459,10 @@ public void reset(TSDataType dataType) { statistics = Statistics.getStatsByType(dataType); } + public void setChunkOrdinal(long chunkOrdinal) { + this.chunkOrdinal = chunkOrdinal; + } + public void setValueEncoder(Encoder encoder) { this.valueEncoder = encoder; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java index cf7e3b651..eb13b63ed 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java @@ -94,7 +94,10 @@ protected AbstractTableModelTsFileWriter(File file, long chunkGroupSizeThreshold protected AbstractTableModelTsFileWriter( File file, long chunkGroupSizeThreshold, EncryptParameter firstEncryptParam) throws IOException { - this(new TsFileIOWriter(file), chunkGroupSizeThreshold, firstEncryptParam); + this( + new TsFileIOWriter(file, copyEncryptParameter(firstEncryptParam)), + chunkGroupSizeThreshold, + firstEncryptParam); } @TsFileApi @@ -123,23 +126,39 @@ protected AbstractTableModelTsFileWriter( LOG.warn(Messages.get("log.write.page_size_warn"), pageSize, chunkGroupSizeThreshold); } - this.secondEncryptParam = EncryptUtils.getEncryptParameter(firstEncryptParam); - String encryptLevel; - if (firstEncryptParam != null - && !Objects.equals(firstEncryptParam.getType(), "UNENCRYPTED") - && !Objects.equals(firstEncryptParam.getType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) { - encryptLevel = "2"; - String str = - EncryptUtils.getKeyStr( - IEncryptor.getEncryptor(firstEncryptParam.getType(), firstEncryptParam.getKey()) - .encrypt(secondEncryptParam.getKey())); - fileWriter.setEncryptParam(encryptLevel, secondEncryptParam.getType(), str); + EncryptParameter fileEncryptParameter = fileWriter.getEncryptParameter(); + EncryptParameter effectiveEncryptParameter = + fileEncryptParameter != null && fileEncryptParameter.isTdePageAead() + ? fileEncryptParameter + : firstEncryptParam; + if (effectiveEncryptParameter != null && effectiveEncryptParameter.isTdePageAead()) { + this.secondEncryptParam = effectiveEncryptParameter; + fileWriter.setEncryptParam(effectiveEncryptParameter); + fileWriter.writeEncryptionHeaderIfNecessary(); } else { - encryptLevel = "0"; - fileWriter.setEncryptParam(encryptLevel, "org.apache.tsfile.encrypt.UNENCRYPTED", ""); + this.secondEncryptParam = EncryptUtils.getEncryptParameter(firstEncryptParam); + String encryptLevel; + if (firstEncryptParam != null + && !Objects.equals(firstEncryptParam.getType(), "UNENCRYPTED") + && !Objects.equals( + firstEncryptParam.getType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) { + encryptLevel = "2"; + String str = + EncryptUtils.getKeyStr( + IEncryptor.getEncryptor(firstEncryptParam.getType(), firstEncryptParam.getKey()) + .encrypt(secondEncryptParam.getKey())); + fileWriter.setEncryptParam(encryptLevel, secondEncryptParam.getType(), str); + } else { + encryptLevel = "0"; + fileWriter.setEncryptParam(encryptLevel, "org.apache.tsfile.encrypt.UNENCRYPTED", ""); + } } } + private static EncryptParameter copyEncryptParameter(EncryptParameter parameter) { + return parameter == null ? null : parameter.copy(); + } + protected IChunkGroupWriter tryToInitialGroupWriter( IDeviceID deviceId, boolean isAligned, boolean isTableModel) throws IOException { IChunkGroupWriter groupWriter = groupWriters.get(deviceId); @@ -273,7 +292,13 @@ public void addTsFileProperty(String key, byte[] value) { @TsFileApi public void close() throws IOException { LOG.info(Messages.get("log.write.close_file")); - flush(); - fileWriter.endFile(); + try { + flush(); + fileWriter.endFile(); + } finally { + if (secondEncryptParam != null && secondEncryptParam.isTdePageAead()) { + secondEncryptParam.close(); + } + } } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/ForceAppendTsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/ForceAppendTsFileWriter.java index 1f627d693..cc94ea857 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/ForceAppendTsFileWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/ForceAppendTsFileWriter.java @@ -45,6 +45,7 @@ public class ForceAppendTsFileWriter extends TsFileIOWriter { private long truncatePosition; private static Logger logger = LoggerFactory.getLogger(ForceAppendTsFileWriter.class); + private EncryptParameter ownedEncryptParameter; public ForceAppendTsFileWriter(File file) throws IOException { this( @@ -58,18 +59,27 @@ public ForceAppendTsFileWriter(File file, EncryptParameter param) throws IOExcep if (logger.isDebugEnabled()) { logger.debug(Messages.get("log.write.writer_opened"), file.getName()); } - this.out = FSFactoryProducer.getFileOutputFactory().getTsFileOutput(file.getPath(), true); this.file = file; - setEncryptParam(param); // file doesn't exist if (file.length() == 0 || !file.exists()) { throw new TsFileNotCompleteException( Messages.format("error.write.force_append_not_complete", file.getPath())); } + this.out = FSFactoryProducer.getFileOutputFactory().getTsFileOutput(file.getPath(), true); + setEncryptParam(param); try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath(), param, true)) { + if (!reader.hasFileEncryptionHeader() && param != null && param.isTdePageAead()) { + throw new IOException(Messages.get("error.write.aead_append_requires_file_header")); + } + EncryptParameter recoveredParameter = reader.getEncryptParam(); + if (recoveredParameter != null && recoveredParameter.isTdePageAead()) { + ownedEncryptParameter = recoveredParameter.copy(); + setEncryptParam(ownedEncryptParameter); + } + markExistingFileStarted(reader.hasFileEncryptionHeader()); // this tsfile is not complete if (!reader.isComplete()) { @@ -85,9 +95,21 @@ public ForceAppendTsFileWriter(File file, EncryptParameter param) throws IOExcep for (IDeviceID device : devices) { List chunkMetadataList = new ArrayList<>(); reader.readChunkMetadataInDevice(device).values().forEach(chunkMetadataList::addAll); + if (ownedEncryptParameter != null) { + for (ChunkMetadata chunkMetadata : chunkMetadataList) { + reader.position(chunkMetadata.getOffsetOfChunkHeader()); + reader.readChunkHeader(reader.readMarker()); + } + } ChunkGroupMetadata chunkGroupMetadata = new ChunkGroupMetadata(device, chunkMetadataList); chunkGroupMetadataList.add(chunkGroupMetadata); } + if (ownedEncryptParameter != null) { + ownedEncryptParameter.resumeAfterChunkOrdinal(reader.getMaxChunkOrdinal()); + } + } catch (IOException | RuntimeException e) { + closeAfterFailedInitialization(e); + throw e; } } @@ -98,4 +120,38 @@ public void doTruncate() throws IOException { public long getTruncatePosition() { return truncatePosition; } + + @Override + public void endFile() throws IOException { + try { + super.endFile(); + } finally { + closeOwnedEncryptParameter(); + } + } + + @Override + public void close() throws IOException { + try { + super.close(); + } finally { + closeOwnedEncryptParameter(); + } + } + + private void closeOwnedEncryptParameter() { + if (ownedEncryptParameter != null) { + ownedEncryptParameter.close(); + } + } + + private void closeAfterFailedInitialization(Exception exception) { + try { + out.close(); + } catch (IOException e) { + exception.addSuppressed(e); + } finally { + closeOwnedEncryptParameter(); + } + } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriter.java index 09866b3eb..be8429915 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriter.java @@ -126,23 +126,45 @@ public RestorableTsFileIOWriter(File file, boolean truncate, EncryptParameter pa } this.file = file; this.out = FSFactoryProducer.getFileOutputFactory().getTsFileOutput(file.getPath(), true); - this.param = param; - setEncryptParam(param); + this.param = param != null && param.isTdePageAead() ? param.copy() : param; + setEncryptParam(this.param); // file doesn't exist if (file.length() == 0) { - startFile(); - crashed = true; - canWrite = true; - return; + try { + startFile(); + crashed = true; + canWrite = true; + return; + } catch (IOException | RuntimeException e) { + closeAfterFailedInitialization(e); + throw e; + } } try { if (file.exists()) { try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath(), param, false)) { + if (!reader.hasFileEncryptionHeader() + && this.param != null + && this.param.isTdePageAead()) { + throw new IOException(Messages.get("error.write.aead_append_requires_file_header")); + } + EncryptParameter recoveredParameter = reader.getEncryptParam(); + if (recoveredParameter != null && recoveredParameter.isTdePageAead()) { + if (this.param != null && this.param.isTdePageAead()) { + this.param.close(); + } + this.param = recoveredParameter.copy(); + setEncryptParam(this.param); + } + markExistingFileStarted(reader.hasFileEncryptionHeader()); schema.setEnabledUpdateSchema(false); truncatedSize = reader.selfCheck(schema, chunkGroupMetadataList, true); + if (this.param != null && this.param.isTdePageAead()) { + this.param.resumeAfterChunkOrdinal(reader.getMaxChunkOrdinal()); + } minPlanIndex = reader.getMinPlanIndex(); maxPlanIndex = reader.getMaxPlanIndex(); if (truncatedSize == TsFileCheckStatus.COMPLETE_FILE) { @@ -165,7 +187,7 @@ public RestorableTsFileIOWriter(File file, boolean truncate, EncryptParameter pa } } } catch (Exception e) { - out.close(); + closeAfterFailedInitialization(e); throw e; } } @@ -209,6 +231,45 @@ public Schema getKnownSchema() { return schema; } + @Override + public EncryptParameter getEncryptParameter() { + return param; + } + + @Override + public void endFile() throws IOException { + try { + super.endFile(); + } finally { + closeOwnedEncryptParameter(); + } + } + + @Override + public void close() throws IOException { + try { + super.close(); + } finally { + closeOwnedEncryptParameter(); + } + } + + private void closeOwnedEncryptParameter() { + if (param != null && param.isTdePageAead()) { + param.close(); + } + } + + private void closeAfterFailedInitialization(Exception exception) { + try { + out.close(); + } catch (IOException e) { + exception.addSuppressed(e); + } finally { + closeOwnedEncryptParameter(); + } + } + /** * For query. * diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java index 93ddec569..ab94f3642 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java @@ -25,10 +25,12 @@ import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.encrypt.IEncryptor; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.encrypt.EncryptException; import org.apache.tsfile.external.commons.io.FileUtils; import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkGroupHeader; import org.apache.tsfile.file.header.ChunkHeader; +import org.apache.tsfile.file.header.FileEncryptionHeader; import org.apache.tsfile.file.metadata.ChunkGroupMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; @@ -136,6 +138,12 @@ public class TsFileIOWriter implements AutoCloseable { protected String encryptKey; + protected EncryptParameter encryptParameter; + + private boolean fileStarted; + + private boolean encryptionHeaderWritten; + private final List flushListeners = new ArrayList<>(); protected String currentTable; @@ -236,14 +244,28 @@ public TsFileIOWriter(File file, long maxMetadataSize, EncryptParameter param) } public void setEncryptParam(String encryptLevel, String encryptType, String encryptKey) { + if (encryptionHeaderWritten) { + throw new EncryptException(Messages.get("error.write.encryption_context_immutable")); + } this.encryptLevel = encryptLevel; this.encryptType = encryptType; this.encryptKey = encryptKey; + this.encryptParameter = new EncryptParameter(encryptType, null); } public void setEncryptParam(EncryptParameter param) { + if (encryptionHeaderWritten && encryptParameter != param) { + throw new EncryptException(Messages.get("error.write.encryption_context_immutable")); + } + this.encryptParameter = param; if (param == null) { setEncryptParam("0", "org.apache.tsfile.encrypt.UNENCRYPTED", null); + } else if (param.isTdePageAead()) { + // File-level encryption metadata is persisted in FileEncryptionHeader. Legacy footer fields + // stay explicitly unencrypted so wrapped key material is not duplicated. + encryptLevel = "0"; + encryptType = "org.apache.tsfile.encrypt.UNENCRYPTED"; + encryptKey = ""; } else { if (!Objects.equals(param.getType(), "UNENCRYPTED") && !Objects.equals(param.getType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) { @@ -260,6 +282,29 @@ public void setEncryptParam(EncryptParameter param) { } } + public EncryptParameter getEncryptParameter() { + return encryptParameter; + } + + protected void markExistingFileStarted(boolean hasEncryptionHeader) { + fileStarted = true; + encryptionHeaderWritten = hasEncryptionHeader; + } + + public void writeEncryptionHeaderIfNecessary() throws IOException { + if (encryptParameter == null || !encryptParameter.isTdePageAead() || encryptionHeaderWritten) { + return; + } + long baseHeaderSize = MAGIC_STRING_BYTES.length + Byte.BYTES; + if (!fileStarted || out.getPosition() != baseHeaderSize) { + throw new IOException(Messages.get("error.write.encryption_header_must_precede_data")); + } + FileEncryptionHeader.serialize(encryptParameter, out.wrapAsStream()); + // A recoverable encrypted file must persist its wrapped data key before any encrypted page. + out.force(); + encryptionHeaderWritten = true; + } + /** Add a custom property to the TsFile metadata. */ public void addTsFileProperty(String key, byte[] value) { tsFileProperties.put(key, value); @@ -277,15 +322,19 @@ public void addFlushListener(FlushChunkMetadataListener listener) { * @throws IOException if an I/O error occurs. */ public void writeBytesToStream(PublicBAOS bytes) throws IOException { + writeEncryptionHeaderIfNecessary(); bytes.writeTo(out.wrapAsStream()); } protected void startFile() throws IOException { out.write(MAGIC_STRING_BYTES); out.write(VERSION_NUMBER_BYTE); + fileStarted = true; + writeEncryptionHeaderIfNecessary(); } public int startChunkGroup(IDeviceID deviceId) throws IOException { + writeEncryptionHeaderIfNecessary(); updateTableSize(deviceId); this.currentChunkGroupDeviceId = deviceId; if (logger.isDebugEnabled()) { @@ -346,6 +395,64 @@ public void startFlushChunk( int numOfPages, int mask) throws IOException { + startFlushChunk( + measurementId, + compressionCodecName, + tsDataType, + encodingType, + statistics, + dataSize, + numOfPages, + mask, + -1); + } + + public void startFlushChunk( + String measurementId, + CompressionType compressionCodecName, + TSDataType tsDataType, + TSEncoding encodingType, + Statistics statistics, + int dataSize, + int numOfPages, + int mask, + long chunkOrdinal) + throws IOException { + startFlushChunk( + measurementId, + compressionCodecName, + tsDataType, + encodingType, + statistics, + dataSize, + numOfPages, + mask, + chunkOrdinal, + null); + } + + public void startFlushChunk( + String measurementId, + CompressionType compressionCodecName, + TSDataType tsDataType, + TSEncoding encodingType, + Statistics statistics, + int dataSize, + int numOfPages, + int mask, + long chunkOrdinal, + EncryptParameter chunkEncryptParameter) + throws IOException { + + boolean pageAead = encryptParameter != null && encryptParameter.isTdePageAead(); + if (pageAead != (chunkOrdinal >= 0)) { + throw new IOException(Messages.get("error.write.invalid_chunk_ordinal")); + } + if (pageAead && !encryptParameter.sharesPageAeadFileContext(chunkEncryptParameter)) { + throw new IOException(Messages.get("error.write.encrypted_chunk_context_mismatch")); + } + + writeEncryptionHeaderIfNecessary(); currentChunkMetadata = new ChunkMetadata( @@ -365,12 +472,15 @@ public void startFlushChunk( compressionCodecName, encodingType, numOfPages, - mask); + mask, + chunkOrdinal); header.serializeTo(out.wrapAsStream()); } /** Write a whole chunk in another file into this file. Providing fast merge for IoTDB. */ public void writeChunk(Chunk chunk, ChunkMetadata chunkMetadata) throws IOException { + writeEncryptionHeaderIfNecessary(); + validateChunkEncryptionContext(chunk); ChunkHeader chunkHeader = chunk.getHeader(); currentChunkMetadata = new ChunkMetadata( @@ -400,6 +510,7 @@ public void writeEmptyValueChunk( TSEncoding encodingType, Statistics statistics) throws IOException { + writeEncryptionHeaderIfNecessary(); currentChunkMetadata = new ChunkMetadata( measurementId, @@ -417,12 +528,17 @@ public void writeEmptyValueChunk( compressionType, encodingType, 0, - TsFileConstant.VALUE_COLUMN_MASK); + TsFileConstant.VALUE_COLUMN_MASK, + encryptParameter != null && encryptParameter.isTdePageAead() + ? encryptParameter.nextChunkOrdinal() + : -1); emptyChunkHeader.serializeTo(out.wrapAsStream()); endCurrentChunk(); } public void writeChunk(Chunk chunk) throws IOException { + writeEncryptionHeaderIfNecessary(); + validateChunkEncryptionContext(chunk); ChunkHeader chunkHeader = chunk.getHeader(); currentChunkMetadata = new ChunkMetadata( @@ -468,6 +584,7 @@ public void endFile() throws IOException { if (!canWrite) { return; } + writeEncryptionHeaderIfNecessary(); updateTableSize(null); checkInMemoryPathCount(); @@ -712,12 +829,26 @@ public void setFile(File file) { } public void writePlanIndices() throws IOException { + writeEncryptionHeaderIfNecessary(); ReadWriteIOUtils.write(MetaMarker.OPERATION_INDEX_RANGE, out.wrapAsStream()); ReadWriteIOUtils.write(minPlanIndex, out.wrapAsStream()); ReadWriteIOUtils.write(maxPlanIndex, out.wrapAsStream()); out.flush(); } + private void validateChunkEncryptionContext(Chunk chunk) throws IOException { + EncryptParameter sourceParameter = chunk.getEncryptParam(); + boolean targetUsesPageAead = encryptParameter != null && encryptParameter.isTdePageAead(); + boolean sourceUsesPageAead = sourceParameter != null && sourceParameter.isTdePageAead(); + if (!targetUsesPageAead && !sourceUsesPageAead) { + if (chunk.getHeader().getChunkOrdinal() >= 0) { + throw new IOException(Messages.get("error.write.unencrypted_chunk_has_ordinal")); + } + return; + } + throw new IOException(Messages.get("error.write.encrypted_chunk_copy_unsupported")); + } + public void truncate(long offset) throws IOException { out.truncate(offset); } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/PageCryptoContextTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/PageCryptoContextTest.java new file mode 100644 index 000000000..f37952e51 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/PageCryptoContextTest.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.encrypt; + +import org.apache.tsfile.exception.encrypt.EncryptException; +import org.apache.tsfile.file.metadata.enums.EncryptionType; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class PageCryptoContextTest { + + @BeforeClass + public static void setUpProvider() { + EncryptionProviderRegistry.registerProvider(TestAeadEncryptionProvider.INSTANCE); + } + + @AfterClass + public static void tearDownProvider() { + EncryptionProviderRegistry.unregisterProvider(TestAeadEncryptionProvider.PROVIDER_ID); + } + + @Before + public void resetProviderCount() { + TestAeadEncryptionProvider.resetCreateCount(); + } + + @Test + public void testAeadRejectsTamperingAndPageSwap() { + byte[] dataKey = new byte[16]; + Arrays.fill(dataKey, (byte) 0x5A); + byte[] fileCryptoId = new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]; + Arrays.fill(fileCryptoId, (byte) 0x2C); + EncryptParameter parameter = TestAeadEncryptionProvider.createParameter(dataKey, fileCryptoId); + byte[] plaintext = new byte[] {1, 3, 5, 7, 9, 11, 13, 15}; + + try { + IEncryptor encryptor = IEncryptor.getEncryptor(parameter); + IDecryptor decryptor = IDecryptor.getDecryptor(parameter); + PageCryptoContext page0 = + PageCryptoContext.forEncryption(parameter, plaintext.length, plaintext.length, 0, 0); + PageCryptoContext page1 = + PageCryptoContext.forEncryption(parameter, plaintext.length, plaintext.length, 1, 0); + PageCryptoContext anotherChunkPage0 = + PageCryptoContext.forEncryption(parameter, plaintext.length, plaintext.length, 0, 1); + + byte[] encryptedPage0 = encryptor.encryptPage(plaintext, 0, plaintext.length, page0); + byte[] encryptedPage1 = encryptor.encryptPage(plaintext, 0, plaintext.length, page1); + assertArrayEquals( + plaintext, decryptor.decryptPage(encryptedPage0, 0, encryptedPage0.length, page0)); + assertArrayEquals( + plaintext, decryptor.decryptPage(encryptedPage1, 0, encryptedPage1.length, page1)); + + assertThrows( + EncryptException.class, + () -> decryptor.decryptPage(encryptedPage0, 0, encryptedPage0.length, page1)); + assertThrows( + EncryptException.class, + () -> decryptor.decryptPage(encryptedPage0, 0, encryptedPage0.length, anotherChunkPage0)); + + encryptedPage0[encryptedPage0.length - 1] ^= 1; + assertThrows( + EncryptException.class, + () -> decryptor.decryptPage(encryptedPage0, 0, encryptedPage0.length, page0)); + + assertEquals(1, TestAeadEncryptionProvider.getCreateCount()); + } finally { + parameter.close(); + } + } + + @Test + public void testChunkOrdinalAllocationSurvivesCopyAndRecovery() { + EncryptParameter parameter = + TestAeadEncryptionProvider.createParameter( + new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]); + try { + assertEquals(0, parameter.nextChunkOrdinal()); + try (EncryptParameter copy = parameter.copy()) { + assertEquals(1, copy.nextChunkOrdinal()); + parameter.resumeAfterChunkOrdinal(10); + assertEquals(11, copy.nextChunkOrdinal()); + } + assertEquals(12, parameter.nextChunkOrdinal()); + } finally { + parameter.close(); + } + } + + @Test + public void testPageAeadFileContextRequiresSharedAllocatorAndKey() { + byte[] fileId = new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]; + EncryptParameter parameter = TestAeadEncryptionProvider.createParameter(new byte[16], fileId); + EncryptParameter independent = TestAeadEncryptionProvider.createParameter(new byte[16], fileId); + try (EncryptParameter copy = parameter.copy()) { + assertTrue(parameter.sharesPageAeadFileContext(copy)); + assertFalse(parameter.sharesPageAeadFileContext(independent)); + copy.getKey()[0] ^= 1; + assertFalse(parameter.sharesPageAeadFileContext(copy)); + } finally { + parameter.close(); + independent.close(); + } + } + + @Test + public void testAssociatedDataCannotBeModifiedByProvider() { + EncryptParameter parameter = + TestAeadEncryptionProvider.createParameter( + new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]); + try { + PageCryptoContext context = PageCryptoContext.forEncryption(parameter, 8, 8, 0, 0); + byte[] expected = context.getAssociatedData(); + byte[] providerCopy = context.getAssociatedData(); + providerCopy[0] ^= 1; + assertArrayEquals(expected, context.getAssociatedData()); + } finally { + parameter.close(); + } + } + + @Test + public void testLegacyCipherCannotSilentlyHandleAuthenticatedPages() { + EncryptParameter parameter = + TestAeadEncryptionProvider.createParameter( + new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]); + try { + PageCryptoContext context = PageCryptoContext.forEncryption(parameter, 1, 1, 0, 0); + IEncryptor legacyEncryptor = + new IEncryptor() { + @Override + public byte[] encrypt(byte[] data) { + return data; + } + + @Override + public byte[] encrypt(byte[] data, int offset, int size) { + throw new AssertionError("Legacy encryption must not be called for AEAD pages"); + } + + @Override + public EncryptionType getEncryptionType() { + return EncryptionType.NewWay; + } + }; + IDecryptor legacyDecryptor = + new IDecryptor() { + @Override + public byte[] decrypt(byte[] data) { + return data; + } + + @Override + public byte[] decrypt(byte[] data, int offset, int size) { + throw new AssertionError("Legacy decryption must not be called for AEAD pages"); + } + + @Override + public EncryptionType getEncryptionType() { + return EncryptionType.NewWay; + } + }; + byte[] page = {1}; + assertThrows(EncryptException.class, () -> legacyEncryptor.encryptPage(page, 0, 1, context)); + assertThrows(EncryptException.class, () -> legacyDecryptor.decryptPage(page, 0, 1, context)); + } finally { + parameter.close(); + } + } + + @Test + public void testUnregisterProviderTrimsId() { + EncryptParameter parameter = + TestAeadEncryptionProvider.createParameter( + new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]); + try { + EncryptionProviderRegistry.create(parameter); + EncryptionProviderRegistry.unregisterProvider( + " " + TestAeadEncryptionProvider.PROVIDER_ID + " "); + assertThrows(EncryptException.class, () -> EncryptionProviderRegistry.create(parameter)); + } finally { + EncryptionProviderRegistry.registerProvider(TestAeadEncryptionProvider.INSTANCE); + parameter.close(); + } + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/TestAeadEncryptionProvider.java b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/TestAeadEncryptionProvider.java new file mode 100644 index 000000000..1e56df2d8 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/TestAeadEncryptionProvider.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.encrypt; + +import org.apache.tsfile.exception.encrypt.EncryptException; +import org.apache.tsfile.file.metadata.enums.EncryptionType; + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +import java.security.GeneralSecurityException; +import java.security.SecureRandom; +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicInteger; + +/** JDK-only AEAD provider used to exercise the generic TsFile encryption SPI. */ +public final class TestAeadEncryptionProvider implements IEncryptProvider { + + public static final TestAeadEncryptionProvider INSTANCE = new TestAeadEncryptionProvider(); + public static final String PROVIDER_ID = "test-aead-provider"; + public static final String PROFILE_ID = "AES_GCM_128_AES_WRAP_128_V1"; + + private static final int IV_LENGTH = 12; + private static final int TAG_LENGTH = 16; + + private static final byte[] TEST_KEY_ENCRYPTION_KEY = + new byte[] { + 0x10, 0x32, 0x54, 0x76, 0x01, 0x23, 0x45, 0x67, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, + 0x00 + }; + private static final SecureRandom SECURE_RANDOM = new SecureRandom(); + private static final AtomicInteger CREATE_COUNT = new AtomicInteger(); + + private TestAeadEncryptionProvider() {} + + @Override + public String getProviderId() { + return PROVIDER_ID; + } + + @Override + public IEncrypt create(EncryptParameter encryptParameter) { + if (!PROFILE_ID.equals(encryptParameter.getProfileId())) { + throw new EncryptException("unsupported test encryption profile"); + } + CREATE_COUNT.incrementAndGet(); + return new TestAeadEncrypt(resolveDataKey(encryptParameter)); + } + + public static EncryptParameter createParameter(byte[] dataKey, byte[] fileCryptoId) { + return EncryptParameter.pageAeadBuilder() + .key(dataKey) + .providerId(PROVIDER_ID) + .profileId(PROFILE_ID) + .keyId("test-key") + .keyVersion("1") + .wrappedDataKey(wrapDataKey(dataKey)) + .fileCryptoId(fileCryptoId) + .build(); + } + + public static int getCreateCount() { + return CREATE_COUNT.get(); + } + + public static void resetCreateCount() { + CREATE_COUNT.set(0); + } + + private static byte[] resolveDataKey(EncryptParameter parameter) { + byte[] dataKey = parameter.getKey(); + if (dataKey != null) { + return Arrays.copyOf(dataKey, dataKey.length); + } + try { + Cipher cipher = Cipher.getInstance("AESWrap"); + cipher.init(Cipher.UNWRAP_MODE, new SecretKeySpec(TEST_KEY_ENCRYPTION_KEY, "AES")); + SecretKey unwrapped = + (SecretKey) cipher.unwrap(parameter.getWrappedDataKey(), "AES", Cipher.SECRET_KEY); + return unwrapped.getEncoded(); + } catch (GeneralSecurityException e) { + throw new EncryptException("test data key unwrap failed", e); + } + } + + private static byte[] wrapDataKey(byte[] dataKey) { + try { + Cipher cipher = Cipher.getInstance("AESWrap"); + cipher.init(Cipher.WRAP_MODE, new SecretKeySpec(TEST_KEY_ENCRYPTION_KEY, "AES")); + return cipher.wrap(new SecretKeySpec(dataKey, "AES")); + } catch (GeneralSecurityException e) { + throw new EncryptException("test data key wrap failed", e); + } + } + + private static final class TestAeadEncrypt implements IEncrypt { + + private final byte[] dataKey; + + private TestAeadEncrypt(byte[] dataKey) { + this.dataKey = dataKey; + } + + @Override + public IDecryptor getDecryptor() { + return new AeadDecryptor(dataKey); + } + + @Override + public IEncryptor getEncryptor() { + return new AeadEncryptor(dataKey); + } + + @Override + public int getPageBodyOverhead() { + return IV_LENGTH + TAG_LENGTH; + } + + @Override + public void close() { + Arrays.fill(dataKey, (byte) 0); + } + } + + private static final class AeadEncryptor implements IEncryptor { + + private final byte[] dataKey; + + private AeadEncryptor(byte[] dataKey) { + this.dataKey = dataKey; + } + + @Override + public byte[] encrypt(byte[] data) { + return encrypt(data, 0, data.length); + } + + @Override + public byte[] encrypt(byte[] data, int offset, int size) { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] encryptPage( + byte[] data, int offset, int size, PageCryptoContext pageCryptoContext) { + byte[] iv = new byte[IV_LENGTH]; + SECURE_RANDOM.nextBytes(iv); + try { + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); + cipher.init( + Cipher.ENCRYPT_MODE, + new SecretKeySpec(dataKey, "AES"), + new GCMParameterSpec(TAG_LENGTH * Byte.SIZE, iv)); + cipher.updateAAD(pageCryptoContext.getAssociatedData()); + byte[] ciphertext = cipher.doFinal(data, offset, size); + byte[] pageBody = new byte[iv.length + ciphertext.length]; + System.arraycopy(iv, 0, pageBody, 0, iv.length); + System.arraycopy(ciphertext, 0, pageBody, iv.length, ciphertext.length); + return pageBody; + } catch (GeneralSecurityException e) { + throw new EncryptException("test page encryption failed", e); + } + } + + @Override + public EncryptionType getEncryptionType() { + return EncryptionType.NewWay; + } + } + + private static final class AeadDecryptor implements IDecryptor { + + private final byte[] dataKey; + + private AeadDecryptor(byte[] dataKey) { + this.dataKey = dataKey; + } + + @Override + public byte[] decrypt(byte[] data) { + return decrypt(data, 0, data.length); + } + + @Override + public byte[] decrypt(byte[] data, int offset, int size) { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] decryptPage( + byte[] data, int offset, int size, PageCryptoContext pageCryptoContext) { + try { + GCMParameterSpec spec = + new GCMParameterSpec(TAG_LENGTH * Byte.SIZE, data, offset, IV_LENGTH); + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); + cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(dataKey, "AES"), spec); + cipher.updateAAD(pageCryptoContext.getAssociatedData()); + return cipher.doFinal(data, offset + IV_LENGTH, size - IV_LENGTH); + } catch (GeneralSecurityException e) { + throw new EncryptException("test page decryption failed", e); + } + } + + @Override + public EncryptionType getEncryptionType() { + return EncryptionType.NewWay; + } + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/file/header/FileEncryptionHeaderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/file/header/FileEncryptionHeaderTest.java new file mode 100644 index 000000000..82bd6a33c --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/file/header/FileEncryptionHeaderTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.file.header; + +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.TestAeadEncryptionProvider; +import org.apache.tsfile.exception.encrypt.EncryptException; +import org.apache.tsfile.file.MetaMarker; + +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; + +public class FileEncryptionHeaderTest { + + @Test + public void testRoundTrip() throws IOException { + byte[] dataKey = new byte[16]; + byte[] fileCryptoId = new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]; + for (int i = 0; i < dataKey.length; i++) { + dataKey[i] = (byte) (i + 1); + fileCryptoId[i] = (byte) (0x40 + i); + } + EncryptParameter parameter = TestAeadEncryptionProvider.createParameter(dataKey, fileCryptoId); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + FileEncryptionHeader.serialize(parameter, output); + byte[] serialized = output.toByteArray(); + assertEquals(MetaMarker.ENCRYPTION_HEADER, serialized[0]); + assertFalse(contains(serialized, dataKey)); + + ByteArrayInputStream input = new ByteArrayInputStream(serialized); + assertEquals(MetaMarker.ENCRYPTION_HEADER, input.read()); + EncryptParameter restored = FileEncryptionHeader.deserialize(input); + + assertNull(restored.getType()); + assertNull(restored.getKey()); + assertEquals(parameter.getProviderId(), restored.getProviderId()); + assertEquals(parameter.getProfileId(), restored.getProfileId()); + assertEquals(parameter.getKeyId(), restored.getKeyId()); + assertEquals(parameter.getKeyVersion(), restored.getKeyVersion()); + assertArrayEquals(parameter.getFileCryptoId(), restored.getFileCryptoId()); + assertArrayEquals(parameter.getWrappedDataKey(), restored.getWrappedDataKey()); + } + + @Test + public void testRejectInvalidFileCryptoId() { + EncryptParameter parameter = + TestAeadEncryptionProvider.createParameter(new byte[16], new byte[8]); + assertThrows( + EncryptException.class, + () -> FileEncryptionHeader.serialize(parameter, new ByteArrayOutputStream())); + } + + private static boolean contains(byte[] data, byte[] target) { + for (int i = 0; i <= data.length - target.length; i++) { + boolean matches = true; + for (int j = 0; j < target.length; j++) { + if (data[i + j] != target[j]) { + matches = false; + break; + } + } + if (matches) { + return true; + } + } + return false; + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/UnClosedTsFileReaderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/UnClosedTsFileReaderTest.java index 8e865413a..5fadd06dc 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/UnClosedTsFileReaderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/UnClosedTsFileReaderTest.java @@ -20,6 +20,9 @@ package org.apache.tsfile.read; import org.apache.tsfile.constant.TestConstant; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.EncryptionProviderRegistry; +import org.apache.tsfile.encrypt.TestAeadEncryptionProvider; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID.Factory; @@ -30,6 +33,8 @@ import org.apache.tsfile.write.schema.MeasurementSchema; import org.apache.tsfile.write.writer.TsFileIOWriter; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import java.io.File; @@ -41,6 +46,16 @@ public class UnClosedTsFileReaderTest { + @BeforeClass + public static void setUpEncryptionProvider() { + EncryptionProviderRegistry.registerProvider(TestAeadEncryptionProvider.INSTANCE); + } + + @AfterClass + public static void tearDownEncryptionProvider() { + EncryptionProviderRegistry.unregisterProvider(TestAeadEncryptionProvider.PROVIDER_ID); + } + @Test public void testRead() throws IOException { File file = new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "test.tsfile"); @@ -71,4 +86,47 @@ public void testRead() throws IOException { file.delete(); } + + @Test + public void testReadEncryptedPagesUsingFileHeader() throws IOException { + File file = new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "test-encrypted.tsfile"); + byte[] fileCryptoId = new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]; + fileCryptoId[0] = 1; + EncryptParameter encryptParameter = + TestAeadEncryptionProvider.createParameter(new byte[16], fileCryptoId); + + try { + TsFileIOWriter writer = new TsFileIOWriter(file, encryptParameter); + writer.startChunkGroup(Factory.DEFAULT_FACTORY.create("root.sg1.d1")); + ChunkWriterImpl chunkWriter = + new ChunkWriterImpl(new MeasurementSchema("s1", TSDataType.INT64), encryptParameter); + chunkWriter.write(1, 1L); + chunkWriter.sealCurrentPage(); + chunkWriter.write(2, 2L); + chunkWriter.sealCurrentPage(); + chunkWriter.writeToFileWriter(writer); + writer.endChunkGroup(); + writer.close(); + ChunkMetadata chunkMetadata = + writer.getChunkGroupMetadataList().get(0).getChunkMetadataList().get(0); + + try (UnClosedTsFileReader reader = new UnClosedTsFileReader(file.getAbsolutePath(), null)) { + assertEquals("test-key", reader.getEncryptParam().getKeyId()); + ChunkReader chunkReader = new ChunkReader(reader.readMemChunk(chunkMetadata)); + + BatchData firstPage = chunkReader.nextPageData(); + assertTrue(firstPage.hasCurrent()); + assertEquals(1, firstPage.currentTime()); + assertEquals(1L, firstPage.currentValue()); + + BatchData secondPage = chunkReader.nextPageData(); + assertTrue(secondPage.hasCurrent()); + assertEquals(2, secondPage.currentTime()); + assertEquals(2L, secondPage.currentValue()); + } + } finally { + encryptParameter.close(); + file.delete(); + } + } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TDEPageAeadTsFileTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TDEPageAeadTsFileTest.java new file mode 100644 index 000000000..abe940c43 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TDEPageAeadTsFileTest.java @@ -0,0 +1,572 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tsfile.write; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.constant.TestConstant; +import org.apache.tsfile.encoding.encoder.PlainEncoder; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.EncryptionProviderRegistry; +import org.apache.tsfile.encrypt.TestAeadEncryptionProvider; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.encrypt.EncryptException; +import org.apache.tsfile.file.MetaMarker; +import org.apache.tsfile.file.header.ChunkHeader; +import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.file.metadata.ChunkMetadata; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Chunk; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.read.reader.BufferedTsFileInput; +import org.apache.tsfile.read.reader.IPageReader; +import org.apache.tsfile.read.reader.IPointReader; +import org.apache.tsfile.read.reader.TsFileLastReader; +import org.apache.tsfile.read.reader.chunk.ChunkReader; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.chunk.TimeChunkWriter; +import org.apache.tsfile.write.chunk.ValueChunkWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.record.datapoint.LongDataPoint; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.writer.TsFileIOWriter; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class TDEPageAeadTsFileTest { + + private final File file = + new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "tde-page-aead.tsfile"); + + @BeforeClass + public static void setUpEncryptionProvider() { + EncryptionProviderRegistry.registerProvider(TestAeadEncryptionProvider.INSTANCE); + } + + @AfterClass + public static void tearDownEncryptionProvider() { + EncryptionProviderRegistry.unregisterProvider(TestAeadEncryptionProvider.PROVIDER_ID); + } + + @Before + public void setUp() { + if (!file.getParentFile().exists()) { + assertTrue(file.getParentFile().mkdirs()); + } + } + + @After + public void tearDown() { + if (file.exists()) { + assertTrue(file.delete()); + } + } + + @Test + public void testAlignedMultiPageReadWrite() throws Exception { + TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); + int previousMaxPointsInPage = config.getMaxNumberOfPointsInPage(); + config.setMaxNumberOfPointsInPage(1); + byte[] fileCryptoId = new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]; + fileCryptoId[0] = 1; + EncryptParameter encryptParameter = + TestAeadEncryptionProvider.createParameter(new byte[16], fileCryptoId); + + try { + List schemas = + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE), + new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE)); + try (TsFileWriter writer = new TsFileWriter(file, encryptParameter)) { + writer.registerAlignedTimeseries(new Path("d1"), schemas); + for (int i = 1; i <= 3; i++) { + writer.writeRecord( + new TSRecord("d1", i) + .addTuple(new LongDataPoint("s1", i * 10L)) + .addTuple(new LongDataPoint("s2", i * 100L))); + } + } + + try (TsFileReader reader = new TsFileReader(new TsFileSequenceReader(file.getPath()))) { + QueryDataSet dataSet = + reader.query( + QueryExpression.create( + Arrays.asList(new Path("d1", "s1", true), new Path("d1", "s2", true)), null)); + for (int i = 1; i <= 3; i++) { + RowRecord record = dataSet.next(); + assertEquals(i, record.getTimestamp()); + assertEquals(i * 10L, record.getFields().get(0).getLongV()); + assertEquals(i * 100L, record.getFields().get(1).getLongV()); + } + assertFalse(dataSet.hasNext()); + } + } finally { + encryptParameter.close(); + config.setMaxNumberOfPointsInPage(previousMaxPointsInPage); + } + } + + @Test + public void testBufferedInputConstructorLoadsEncryptionHeader() throws Exception { + byte[] fileCryptoId = new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]; + fileCryptoId[0] = 1; + EncryptParameter encryptParameter = + TestAeadEncryptionProvider.createParameter(new byte[16], fileCryptoId); + + try { + try (TsFileWriter writer = new TsFileWriter(file, encryptParameter)) { + writer.registerTimeseries( + new Path("d1"), new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE)); + writer.writeRecord(new TSRecord("d1", 1).addTuple(new LongDataPoint("s1", 1L))); + } + + try (TsFileSequenceReader reader = + new TsFileSequenceReader(new BufferedTsFileInput(file.toPath()), false, false, null)) { + reader.position(reader.getDataStartOffset()); + assertEquals(MetaMarker.CHUNK_GROUP_HEADER, reader.readMarker()); + } + } finally { + encryptParameter.close(); + } + } + + @Test + public void testMetadataOffsetConstructorLoadsEncryptionHeader() throws Exception { + try (EncryptParameter parameter = + TestAeadEncryptionProvider.createParameter( + new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH])) { + try (TsFileWriter writer = new TsFileWriter(file, parameter)) { + writer.registerTimeseries( + new Path("d1"), new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE)); + writer.writeRecord(new TSRecord("d1", 1).addTuple(new LongDataPoint("s1", 11L))); + } + try (TsFileSequenceReader original = new TsFileSequenceReader(file.getPath())) { + ChunkMetadata metadata = original.getChunkMetadataList(new Path("d1", "s1", true)).get(0); + try (TsFileSequenceReader reader = + new TsFileSequenceReader( + new BufferedTsFileInput(file.toPath()), + original.getFileMetadataPos(), + original.getTsFileMetadataSize())) { + assertTrue(reader.hasFileEncryptionHeader()); + original.position(metadata.getOffsetOfChunkHeader()); + reader.position(metadata.getOffsetOfChunkHeader()); + assertEquals( + original.readChunkHeader(original.readMarker()).getChunkOrdinal(), + reader.readChunkHeader(reader.readMarker()).getChunkOrdinal()); + } + } + } + } + + @Test + public void testMaterializedChunkSurvivesReaderClose() throws Exception { + try (EncryptParameter parameter = + TestAeadEncryptionProvider.createParameter( + new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH])) { + try (TsFileWriter writer = new TsFileWriter(file, parameter)) { + writer.registerTimeseries( + new Path("d1"), new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN)); + writer.writeRecord(new TSRecord("d1", 1).addTuple(new LongDataPoint("s1", 11L))); + } + Chunk chunk; + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getPath())) { + chunk = reader.readMemChunk(reader.getChunkMetadataList(new Path("d1", "s1", true)).get(0)); + } + try (Chunk ownedChunk = chunk) { + BatchData data = new ChunkReader(ownedChunk).nextPageData(); + IPointReader points = data.getBatchDataIterator(); + assertTrue(points.hasNextTimeValuePair()); + assertEquals(11L, points.nextTimeValuePair().getValue().getLong()); + } + assertTrue(chunk.getEncryptParam().isDestroyed()); + chunk.getData().rewind(); + assertThrows(EncryptException.class, () -> new ChunkReader(chunk).nextPageData()); + } + } + + @Test + public void testAlignedBlobLastPointDecryptsPage() throws Exception { + TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); + int previousMaxPointsInPage = config.getMaxNumberOfPointsInPage(); + config.setMaxNumberOfPointsInPage(1); + try (EncryptParameter parameter = + TestAeadEncryptionProvider.createParameter( + new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH])) { + List schemas = + Arrays.asList(new MeasurementSchema("s1", TSDataType.BLOB, TSEncoding.PLAIN)); + try (TsFileWriter writer = new TsFileWriter(file, parameter)) { + writer.registerAlignedTimeseries(new Path("d1"), schemas); + Tablet tablet = new Tablet("d1", schemas, 2); + for (int i = 0; i < 2; i++) { + tablet.addTimestamp(i, i + 1); + tablet.addValue(i, 0, ("value-" + i).getBytes(StandardCharsets.UTF_8)); + } + writer.writeTree(tablet); + } + try (TsFileLastReader reader = new TsFileLastReader(file.getPath(), true, false)) { + Pair>> last = reader.next(); + assertEquals("s1", last.right.get(1).left); + assertEquals(2L, last.right.get(1).right.getTimestamp()); + assertEquals( + new Binary("value-1", StandardCharsets.UTF_8), + last.right.get(1).right.getValue().getBinary()); + } + } finally { + config.setMaxNumberOfPointsInPage(previousMaxPointsInPage); + } + } + + @Test + public void testUnencryptedWriterRejectsChunkWithOrdinal() throws Exception { + try (TsFileIOWriter writer = new TsFileIOWriter(file)) { + ChunkHeader header = + new ChunkHeader( + "s1", 0, TSDataType.INT64, CompressionType.UNCOMPRESSED, TSEncoding.PLAIN, 1, 0, 0); + Chunk chunk = new Chunk(header, ByteBuffer.allocate(0)); + assertThrows(IOException.class, () -> writer.writeChunk(chunk)); + assertThrows(IOException.class, () -> writer.writeChunk(chunk, null)); + } + } + + @Test + public void testLegacyReadPageOverloadsRejectPageAeadBeforeReading() throws Exception { + EncryptParameter encryptParameter = + TestAeadEncryptionProvider.createParameter( + new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]); + try { + try (TsFileWriter writer = new TsFileWriter(file, encryptParameter)) { + writer.registerTimeseries( + new Path("d1"), + new MeasurementSchema( + "s1", TSDataType.INT64, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED)); + writer.writeRecord(new TSRecord("d1", 1).addTuple(new LongDataPoint("s1", 11L))); + } + + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getPath())) { + ChunkMetadata metadata = reader.getChunkMetadataList(new Path("d1", "s1", true)).get(0); + reader.position(metadata.getOffsetOfChunkHeader()); + ChunkHeader chunkHeader = reader.readChunkHeader(reader.readMarker()); + PageHeader pageHeader = reader.readPageHeader(chunkHeader.getDataType(), false); + long pageBodyOffset = reader.position(); + + IOException noPageIndex = + assertThrows( + IOException.class, + () -> reader.readPage(pageHeader, chunkHeader.getCompressionType())); + assertTrue(noPageIndex.getMessage().contains("chunkOrdinal")); + assertEquals(pageBodyOffset, reader.position()); + + IOException noChunkOrdinal = + assertThrows( + IOException.class, + () -> reader.readPage(pageHeader, chunkHeader.getCompressionType(), 0)); + assertTrue(noChunkOrdinal.getMessage().contains("chunkOrdinal")); + assertEquals(pageBodyOffset, reader.position()); + + assertTrue( + reader + .readPage( + pageHeader, + chunkHeader.getCompressionType(), + 0, + chunkHeader.getChunkOrdinal()) + .remaining() + > 0); + } + } finally { + encryptParameter.close(); + } + } + + @Test + public void testChunkWriterReuseAllocatesDistinctOrdinals() throws Exception { + EncryptParameter encryptParameter = + TestAeadEncryptionProvider.createParameter( + new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]); + try { + try (TsFileWriter writer = new TsFileWriter(file, encryptParameter)) { + writer.registerTimeseries( + new Path("d1"), new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE)); + writer.writeRecord(new TSRecord("d1", 1).addTuple(new LongDataPoint("s1", 11L))); + writer.flush(); + writer.writeRecord(new TSRecord("d1", 2).addTuple(new LongDataPoint("s1", 22L))); + } + + try (TsFileSequenceReader sequenceReader = new TsFileSequenceReader(file.getPath()); + TsFileReader reader = new TsFileReader(sequenceReader)) { + List chunks = + sequenceReader.getChunkMetadataList(new Path("d1", "s1", true)); + assertEquals(2, chunks.size()); + long firstOrdinal = + sequenceReader.readMemChunk(chunks.get(0)).getHeader().getChunkOrdinal(); + long secondOrdinal = + sequenceReader.readMemChunk(chunks.get(1)).getHeader().getChunkOrdinal(); + assertTrue(firstOrdinal >= 0); + assertNotEquals(firstOrdinal, secondOrdinal); + + QueryDataSet dataSet = + reader.query(QueryExpression.create(Arrays.asList(new Path("d1", "s1", true)), null)); + assertEquals(11L, dataSet.next().getFields().get(0).getLongV()); + assertEquals(22L, dataSet.next().getFields().get(0).getLongV()); + assertFalse(dataSet.hasNext()); + } + } finally { + encryptParameter.close(); + } + } + + @Test + public void testEncryptedChunkRewriteUsesNewOrdinal() throws Exception { + EncryptParameter encryptParameter = + TestAeadEncryptionProvider.createParameter( + new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]); + try { + try (TsFileWriter writer = new TsFileWriter(file, encryptParameter)) { + writer.registerTimeseries( + new Path("d1"), new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN)); + writer.writeRecord(new TSRecord("d1", 1).addTuple(new LongDataPoint("s1", 11L))); + } + + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getPath())) { + Chunk source = + reader.readMemChunk(reader.getChunkMetadataList(new Path("d1", "s1", true)).get(0)); + Chunk rewritten = source.rewrite(TSDataType.DOUBLE); + assertNotEquals( + source.getHeader().getChunkOrdinal(), rewritten.getHeader().getChunkOrdinal()); + for (IPageReader page : new ChunkReader(rewritten).loadPageReaderList()) { + BatchData data = page.getAllSatisfiedPageData(true); + IPointReader points = data.getBatchDataIterator(); + assertTrue(points.hasNextTimeValuePair()); + TimeValuePair point = points.nextTimeValuePair(); + assertEquals(1L, point.getTimestamp()); + assertEquals(11.0, point.getValue().getDouble(), 0.0); + assertFalse(points.hasNextTimeValuePair()); + } + } + } finally { + encryptParameter.close(); + } + } + + @Test + public void testSwappingEqualSizedPagesBetweenChunksFailsAuthentication() throws Exception { + EncryptParameter encryptParameter = + TestAeadEncryptionProvider.createParameter( + new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]); + try { + try (TsFileWriter writer = new TsFileWriter(file, encryptParameter)) { + writer.registerTimeseries( + new Path("d1"), + new MeasurementSchema( + "s1", TSDataType.INT64, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED)); + writer.writeRecord(new TSRecord("d1", 1).addTuple(new LongDataPoint("s1", 11L))); + writer.flush(); + writer.writeRecord(new TSRecord("d1", 2).addTuple(new LongDataPoint("s1", 22L))); + } + + long[] pageBodyOffsets = new long[2]; + int[] pageBodySizes = new int[2]; + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getPath())) { + List chunks = reader.getChunkMetadataList(new Path("d1", "s1", true)); + assertEquals(2, chunks.size()); + for (int i = 0; i < chunks.size(); i++) { + Chunk chunk = reader.readMemChunk(chunks.get(i)); + ByteBuffer chunkData = chunk.getData(); + PageHeader pageHeader = PageHeader.deserializeFrom(chunkData, chunk.getChunkStatistic()); + pageBodyOffsets[i] = + chunks.get(i).getOffsetOfChunkHeader() + + chunk.getHeader().getSerializedSize() + + chunkData.position(); + pageBodySizes[i] = pageHeader.getCompressedSize(); + } + } + assertEquals(pageBodySizes[0], pageBodySizes[1]); + try (RandomAccessFile data = new RandomAccessFile(file, "rw")) { + byte[] firstPage = new byte[pageBodySizes[0]]; + byte[] secondPage = new byte[pageBodySizes[1]]; + data.seek(pageBodyOffsets[0]); + data.readFully(firstPage); + data.seek(pageBodyOffsets[1]); + data.readFully(secondPage); + data.seek(pageBodyOffsets[0]); + data.write(secondPage); + data.seek(pageBodyOffsets[1]); + data.write(firstPage); + } + + try (TsFileReader reader = new TsFileReader(new TsFileSequenceReader(file.getPath()))) { + assertThrows( + Exception.class, + () -> + reader + .query(QueryExpression.create(Arrays.asList(new Path("d1", "s1", true)), null)) + .next()); + } + } finally { + encryptParameter.close(); + } + } + + @Test + public void testRejectUnsafeEncryptedChunkReuse() throws Exception { + File targetFile = new File(file.getPath() + ".target"); + byte[] sourceFileCryptoId = new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]; + byte[] targetFileCryptoId = new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]; + sourceFileCryptoId[0] = 1; + targetFileCryptoId[0] = 2; + EncryptParameter sourceParameter = + TestAeadEncryptionProvider.createParameter(new byte[16], sourceFileCryptoId); + EncryptParameter targetParameter = + TestAeadEncryptionProvider.createParameter(new byte[16], targetFileCryptoId); + + try (TsFileIOWriter writer = new TsFileIOWriter(targetFile, targetParameter)) { + ChunkHeader header = + new ChunkHeader( + "s1", 0, TSDataType.INT64, CompressionType.UNCOMPRESSED, TSEncoding.PLAIN, 1); + Chunk sourceChunk = new Chunk(header, ByteBuffer.allocate(0), sourceParameter); + + assertThrows(IOException.class, () -> writer.writeChunk(sourceChunk)); + Chunk sameContextChunk = new Chunk(header, ByteBuffer.allocate(0), targetParameter); + assertThrows(IOException.class, () -> writer.writeChunk(sameContextChunk)); + assertThrows(IOException.class, () -> sourceChunk.mergeChunkByAppendPage(sourceChunk)); + } finally { + sourceParameter.close(); + targetParameter.close(); + if (targetFile.exists()) { + assertTrue(targetFile.delete()); + } + } + } + + @Test + public void testFileEncryptionContextCannotChangeAfterHeader() throws Exception { + EncryptParameter first = + TestAeadEncryptionProvider.createParameter( + new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]); + byte[] secondFileId = new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]; + secondFileId[0] = 1; + EncryptParameter second = + TestAeadEncryptionProvider.createParameter(new byte[16], secondFileId); + try (TsFileIOWriter writer = new TsFileIOWriter(file, first); + EncryptParameter firstCopy = first.copy()) { + writer.setEncryptParam(first); + assertThrows(EncryptException.class, () -> writer.setEncryptParam(second)); + assertThrows(EncryptException.class, () -> writer.setEncryptParam(firstCopy)); + assertThrows( + EncryptException.class, + () -> writer.setEncryptParam("0", "org.apache.tsfile.encrypt.UNENCRYPTED", null)); + assertTrue(writer.getEncryptParameter() == first); + } finally { + first.close(); + second.close(); + } + } + + @Test + public void testChunkFromDifferentFileContextIsRejectedBeforeWriting() throws Exception { + EncryptParameter target = + TestAeadEncryptionProvider.createParameter( + new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]); + byte[] sourceFileId = new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]; + sourceFileId[0] = 1; + EncryptParameter source = + TestAeadEncryptionProvider.createParameter(new byte[16], sourceFileId); + try (TsFileIOWriter writer = new TsFileIOWriter(file, target)) { + ChunkWriterImpl chunk = + new ChunkWriterImpl(new MeasurementSchema("s1", TSDataType.INT64), source); + chunk.write(1, 11L); + long position = writer.getPos(); + assertThrows(IOException.class, () -> chunk.writeToFileWriter(writer)); + assertEquals(position, writer.getPos()); + + TimeChunkWriter timeChunk = + new TimeChunkWriter( + "", + CompressionType.UNCOMPRESSED, + TSEncoding.PLAIN, + new PlainEncoder(TSDataType.INT64, 0), + source); + timeChunk.write(1); + timeChunk.sealCurrentPage(); + assertThrows(IOException.class, () -> timeChunk.writeAllPagesOfChunkToTsFile(writer)); + assertEquals(position, writer.getPos()); + + ValueChunkWriter valueChunk = + new ValueChunkWriter( + "s1", + CompressionType.UNCOMPRESSED, + TSDataType.INT64, + TSEncoding.PLAIN, + new PlainEncoder(TSDataType.INT64, 0), + source); + valueChunk.write(1, 11L, false); + valueChunk.sealCurrentPage(); + assertThrows(IOException.class, () -> valueChunk.writeAllPagesOfChunkToTsFile(writer, null)); + assertEquals(position, writer.getPos()); + + assertThrows( + IOException.class, + () -> + writer.startFlushChunk( + "s1", + CompressionType.UNCOMPRESSED, + TSDataType.INT64, + TSEncoding.PLAIN, + null, + 0, + 0, + 0, + 0)); + assertEquals(position, writer.getPos()); + } finally { + source.close(); + target.close(); + } + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/writer/ForceAppendTsFileWriterTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/writer/ForceAppendTsFileWriterTest.java index 68d05cefd..223d68a39 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/writer/ForceAppendTsFileWriterTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/writer/ForceAppendTsFileWriterTest.java @@ -18,7 +18,11 @@ */ package org.apache.tsfile.write.writer; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.EncryptionProviderRegistry; +import org.apache.tsfile.encrypt.TestAeadEncryptionProvider; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.fileSystem.FSFactoryProducer; import org.apache.tsfile.fileSystem.fsFactory.FSFactory; @@ -34,15 +38,21 @@ import org.apache.tsfile.write.record.datapoint.FloatDataPoint; import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.AfterClass; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; import java.io.File; +import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -51,6 +61,16 @@ public class ForceAppendTsFileWriterTest { TsFileGeneratorForTest.getTestTsFilePath("root.sg1", 0, 0, 1); private static FSFactory fsFactory = FSFactoryProducer.getFSFactory(); + @BeforeClass + public static void setUpEncryptionProvider() { + EncryptionProviderRegistry.registerProvider(TestAeadEncryptionProvider.INSTANCE); + } + + @AfterClass + public static void tearDownEncryptionProvider() { + EncryptionProviderRegistry.unregisterProvider(TestAeadEncryptionProvider.PROVIDER_ID); + } + @Test public void test() throws Exception { File file = fsFactory.getFile(FILE_NAME); @@ -120,4 +140,90 @@ record = dataSet.next(); assertTrue(file.delete()); } + + @Test + public void testHeaderlessForceAppendRejectsPageAeadParameter() throws Exception { + File file = fsFactory.getFile(FILE_NAME + ".headerless"); + if (!file.getParentFile().exists()) { + assertTrue(file.getParentFile().mkdirs()); + } + try { + try (TsFileWriter writer = new TsFileWriter(file)) { + writer.registerTimeseries( + new Path("d1"), new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.RLE)); + writer.writeRecord(new TSRecord("d1", 1).addTuple(new FloatDataPoint("s1", 5))); + } + long originalLength = file.length(); + try (EncryptParameter parameter = + TestAeadEncryptionProvider.createParameter( + new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH])) { + assertThrows(IOException.class, () -> new ForceAppendTsFileWriter(file, parameter)); + } + assertEquals(originalLength, file.length()); + } finally { + if (file.exists()) { + assertTrue(file.delete()); + } + } + } + + @Test + public void testEncryptedForceAppend() throws Exception { + File file = fsFactory.getFile(FILE_NAME + ".encrypted"); + if (!file.getParentFile().exists()) { + Assert.assertTrue(file.getParentFile().mkdirs()); + } + byte[] fileCryptoId = new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]; + fileCryptoId[0] = 1; + EncryptParameter encryptParameter = + TestAeadEncryptionProvider.createParameter(new byte[16], fileCryptoId); + EncryptParameter appendParameter = + TestAeadEncryptionProvider.createParameter(new byte[16], fileCryptoId); + + try { + try (TsFileWriter writer = new TsFileWriter(file, encryptParameter)) { + writer.registerTimeseries( + new Path("d1"), new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.RLE)); + writer.writeRecord(new TSRecord("d1", 1).addTuple(new FloatDataPoint("s1", 5))); + } + + ForceAppendTsFileWriter appendWriter = new ForceAppendTsFileWriter(file, appendParameter); + EncryptParameter ownedParameter = appendWriter.getEncryptParameter(); + Assert.assertNotSame(encryptParameter, ownedParameter); + appendWriter.doTruncate(); + try (TsFileWriter writer = new TsFileWriter(appendWriter, appendParameter)) { + writer.registerTimeseries( + new Path("d1"), new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.RLE)); + writer.writeRecord(new TSRecord("d1", 2).addTuple(new FloatDataPoint("s1", 6))); + } + assertTrue(ownedParameter.isDestroyed()); + assertFalse(encryptParameter.isDestroyed()); + assertFalse(appendParameter.isDestroyed()); + + try (TsFileSequenceReader sequenceReader = new TsFileSequenceReader(file.getPath()); + TsFileReader reader = new TsFileReader(sequenceReader)) { + List chunks = + sequenceReader.getChunkMetadataList(new Path("d1", "s1", true)); + assertEquals(2, chunks.size()); + assertNotEquals( + sequenceReader.readMemChunk(chunks.get(0)).getHeader().getChunkOrdinal(), + sequenceReader.readMemChunk(chunks.get(1)).getHeader().getChunkOrdinal()); + QueryDataSet dataSet = + reader.query( + QueryExpression.create( + Collections.singletonList(new Path("d1", "s1", true)), null)); + RowRecord first = dataSet.next(); + assertEquals(1, first.getTimestamp()); + assertEquals(5.0f, first.getFields().get(0).getFloatV(), 0.001); + RowRecord second = dataSet.next(); + assertEquals(2, second.getTimestamp()); + assertEquals(6.0f, second.getFields().get(0).getFloatV(), 0.001); + assertFalse(dataSet.hasNext()); + } + } finally { + encryptParameter.close(); + appendParameter.close(); + file.delete(); + } + } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriterTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriterTest.java index 0e1279fae..b2a370eac 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriterTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriterTest.java @@ -20,6 +20,9 @@ package org.apache.tsfile.write.writer; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.EncryptionProviderRegistry; +import org.apache.tsfile.encrypt.TestAeadEncryptionProvider; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.NotCompatibleTsFileException; import org.apache.tsfile.file.MetaMarker; @@ -44,8 +47,10 @@ import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import java.io.File; @@ -58,6 +63,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; @SuppressWarnings("squid:S4042") // Suppress use java.nio.Files#delete warning public class RestorableTsFileIOWriterTest { @@ -67,6 +73,16 @@ public class RestorableTsFileIOWriterTest { private static final FSFactory fsFactory = FSFactoryProducer.getFSFactory(); File file = fsFactory.getFile(FILE_NAME); + @BeforeClass + public static void setUpEncryptionProvider() { + EncryptionProviderRegistry.registerProvider(TestAeadEncryptionProvider.INSTANCE); + } + + @AfterClass + public static void tearDownEncryptionProvider() { + EncryptionProviderRegistry.unregisterProvider(TestAeadEncryptionProvider.PROVIDER_ID); + } + @Before public void setUp() throws IOException { if (!file.getParentFile().exists()) { @@ -105,6 +121,116 @@ public void testOnlyHeadMagic() throws Exception { rWriter.close(); } + @Test + public void testHeaderlessRecoveryRejectsPageAeadParameter() throws Exception { + TsFileWriter writer = new TsFileWriter(file); + writer.getIOWriter().close(); + long originalLength = file.length(); + try (EncryptParameter parameter = + TestAeadEncryptionProvider.createParameter( + new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH])) { + assertThrows(IOException.class, () -> new RestorableTsFileIOWriter(file, false, parameter)); + } + assertEquals(originalLength, file.length()); + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getPath(), false)) { + assertFalse(reader.hasFileEncryptionHeader()); + } + } + + @Test + public void testEncryptedHeaderSurvivesUnclosedFile() throws Exception { + byte[] fileCryptoId = new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]; + for (int i = 0; i < fileCryptoId.length; i++) { + fileCryptoId[i] = (byte) (i + 1); + } + EncryptParameter parameter = + TestAeadEncryptionProvider.createParameter(new byte[16], fileCryptoId); + TsFileWriter writer = new TsFileWriter(file, parameter); + long expectedDataStartOffset; + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getPath(), false)) { + expectedDataStartOffset = reader.getDataStartOffset(); + assertNotEquals( + TSFileConfig.MAGIC_STRING.getBytes().length + Byte.BYTES, expectedDataStartOffset); + assertEquals("test-key", reader.getEncryptParam().getKeyId()); + Assert.assertArrayEquals(fileCryptoId, reader.getEncryptParam().getFileCryptoId()); + } + writer.getIOWriter().close(); + writer.getIOWriter().getEncryptParameter().close(); + + RestorableTsFileIOWriter restorableWriter = new RestorableTsFileIOWriter(file, parameter); + assertEquals(expectedDataStartOffset, restorableWriter.getTruncatedSize()); + assertEquals("test-key", restorableWriter.getEncryptParameter().getKeyId()); + EncryptParameter ownedParameter = restorableWriter.getEncryptParameter(); + Assert.assertNotSame(parameter, ownedParameter); + try (TsFileWriter recoveredWriter = new TsFileWriter(restorableWriter, parameter)) { + // Closing writes a valid footer without replacing the file encryption header. + } + Assert.assertTrue(ownedParameter.isDestroyed()); + Assert.assertFalse(parameter.isDestroyed()); + parameter.close(); + } + + @Test + public void testEncryptedChunkOrdinalContinuesAfterRecovery() throws Exception { + EncryptParameter parameter = + TestAeadEncryptionProvider.createParameter( + new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]); + try { + TsFileWriter writer = new TsFileWriter(file, parameter); + writer.registerTimeseries( + new Path("d1"), new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.RLE)); + writer.writeRecord(new TSRecord("d1", 1).addTuple(new FloatDataPoint("s1", 1.0f))); + writer.flush(); + writer.getIOWriter().writePlanIndices(); + ChunkMetadata firstMetadata = + writer.getIOWriter().getChunkGroupMetadataList().get(0).getChunkMetadataList().get(0); + long firstOrdinal; + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getPath(), false)) { + firstOrdinal = reader.readMemChunk(firstMetadata).getHeader().getChunkOrdinal(); + } + writer.getIOWriter().close(); + writer.getIOWriter().getEncryptParameter().close(); + + try (EncryptParameter recoveryParameter = + TestAeadEncryptionProvider.createParameter( + new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH])) { + RestorableTsFileIOWriter restorableWriter = + new RestorableTsFileIOWriter(file, recoveryParameter); + try (TsFileWriter recoveredWriter = new TsFileWriter(restorableWriter, recoveryParameter)) { + recoveredWriter.writeRecord( + new TSRecord("d1", 2).addTuple(new FloatDataPoint("s1", 2.0f))); + } + } + + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getPath())) { + List chunks = reader.getChunkMetadataList(new Path("d1", "s1", true)); + assertEquals(2, chunks.size()); + assertEquals( + firstOrdinal, reader.readMemChunk(chunks.get(0)).getHeader().getChunkOrdinal()); + Assert.assertTrue( + reader.readMemChunk(chunks.get(1)).getHeader().getChunkOrdinal() > firstOrdinal); + } + } finally { + parameter.close(); + } + } + + @Test + public void testCloseDestroysOwnedTdeParameter() throws Exception { + EncryptParameter parameter = + TestAeadEncryptionProvider.createParameter( + new byte[16], new byte[EncryptParameter.FILE_CRYPTO_ID_LENGTH]); + RestorableTsFileIOWriter writer = new RestorableTsFileIOWriter(file, parameter); + EncryptParameter ownedParameter = writer.getEncryptParameter(); + + writer.close(); + + Assert.assertNotSame(parameter, ownedParameter); + Assert.assertTrue(ownedParameter.isDestroyed()); + Assert.assertFalse(parameter.isDestroyed()); + parameter.close(); + } + @Test public void testOnlyFirstMask() throws Exception { TsFileWriter writer = new TsFileWriter(file); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/writer/TimePageWriterTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/writer/TimePageWriterTest.java index 878b9d7a7..c09ccf28f 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/writer/TimePageWriterTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/writer/TimePageWriterTest.java @@ -23,14 +23,24 @@ import org.apache.tsfile.encoding.decoder.PlainDecoder; import org.apache.tsfile.encoding.encoder.Encoder; import org.apache.tsfile.encoding.encoder.PlainEncoder; +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.EncryptionProviderRegistry; +import org.apache.tsfile.encrypt.IDecryptor; +import org.apache.tsfile.encrypt.TestAeadEncryptionProvider; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.header.ChunkHeader; +import org.apache.tsfile.file.header.PageHeader; import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.TimeStatistics; +import org.apache.tsfile.read.reader.chunk.ChunkReader; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.page.TimePageWriter; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import java.io.IOException; @@ -41,6 +51,16 @@ public class TimePageWriterTest { + @BeforeClass + public static void setUpEncryptionProvider() { + EncryptionProviderRegistry.registerProvider(TestAeadEncryptionProvider.INSTANCE); + } + + @AfterClass + public static void tearDownEncryptionProvider() { + EncryptionProviderRegistry.unregisterProvider(TestAeadEncryptionProvider.PROVIDER_ID); + } + @Test public void testWrite() { Encoder timeEncoder = new PlainEncoder(TSDataType.INT64, 0); @@ -168,4 +188,50 @@ public void testWritePageHeaderAndDataIntoBuffWithSnappy() { fail(); } } + + @Test + public void testWriteAndReadPageWithAeadEncryption() throws IOException { + Encoder timeEncoder = new PlainEncoder(TSDataType.INT64, 0); + ICompressor compressor = ICompressor.getCompressor(CompressionType.UNCOMPRESSED); + EncryptParameter encryptParameter = + TestAeadEncryptionProvider.createParameter(new byte[16], new byte[16]); + TimePageWriter pageWriter = new TimePageWriter(timeEncoder, compressor, encryptParameter); + long chunkOrdinal = encryptParameter.nextChunkOrdinal(); + pageWriter.setChunkOrdinal(chunkOrdinal); + PublicBAOS publicBAOS = new PublicBAOS(); + + try { + pageWriter.write(1L); + pageWriter.write(2L); + pageWriter.write(3L); + pageWriter.writePageHeaderAndDataIntoBuff(publicBAOS, false, 0); + + ByteBuffer pageBuffer = ByteBuffer.wrap(publicBAOS.getBuf(), 0, publicBAOS.size()); + PageHeader pageHeader = PageHeader.deserializeFrom(pageBuffer, TSDataType.VECTOR); + assertEquals(24, pageHeader.getUncompressedSize()); + assertEquals(52, pageHeader.getCompressedSize()); + + ChunkHeader chunkHeader = + new ChunkHeader( + "time", + pageHeader.getCompressedSize(), + TSDataType.INT64, + CompressionType.UNCOMPRESSED, + TSEncoding.PLAIN, + 1, + 0, + chunkOrdinal); + IDecryptor decryptor = IDecryptor.getDecryptor(encryptParameter); + ByteBuffer uncompressedPageData = + ChunkReader.deserializePageData( + pageHeader, pageBuffer, chunkHeader, decryptor, encryptParameter, 0); + + assertEquals(1L, ReadWriteIOUtils.readLong(uncompressedPageData)); + assertEquals(2L, ReadWriteIOUtils.readLong(uncompressedPageData)); + assertEquals(3L, ReadWriteIOUtils.readLong(uncompressedPageData)); + assertEquals(0, uncompressedPageData.remaining()); + } finally { + encryptParameter.close(); + } + } }