Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cpp/src/dataset/dataset_index.h
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,8 @@ struct ColumnSchemaRecord {
uint32_t name_sid;
uint32_t column_ordinal;
uint16_t logical_type;
// 0xFFFF denotes mixed physical types on independent tree devices.
// Decode each series using its located TimeseriesMetadata.
uint16_t physical_type;
uint16_t encoding;
uint16_t compression;
Expand Down
15 changes: 8 additions & 7 deletions cpp/src/reader/aligned_chunk_reader.cc
Original file line number Diff line number Diff line change
Expand Up @@ -37,16 +37,15 @@ int AlignedChunkReader::init(RandomAccessReadFile* read_file, String m_name,
TSDataType data_type, Filter* time_filter) {
read_file_ = read_file;
measurement_name_.shallow_copy_from(m_name);
time_decoder_ = DecoderFactory::alloc_time_decoder();
// Aligned time chunks carry their own encoding. Allocate after reading the
// chunk header instead of fixing the decoder to the process default.
time_decoder_ = nullptr;
value_decoder_ = nullptr;
time_compressor_ = nullptr;
value_compressor_ = nullptr;
time_filter_ = time_filter;
time_uncompressed_buf_ = nullptr;
value_uncompressed_buf_ = nullptr;
if (IS_NULL(time_decoder_)) {
return E_OOM;
}
return E_OK;
}

Expand Down Expand Up @@ -317,7 +316,7 @@ int AlignedChunkReader::load_by_aligned_meta(ChunkMeta* time_chunk_meta,
} else if (RET_FAIL(alloc_compressor_and_decoder(
time_decoder_, time_compressor_,
time_chunk_header_.encoding_type_,
time_chunk_header_.data_type_,
common::INT64,
time_chunk_header_.compression_type_))) {
} else if (RET_FAIL(alloc_compressor_and_decoder(
value_decoder_, value_compressor_,
Expand Down Expand Up @@ -1264,7 +1263,7 @@ int AlignedChunkReader::load_by_aligned_meta_multi(
if (RET_FAIL(alloc_compressor_and_decoder(
time_decoder_, time_compressor_,
time_chunk_header_.encoding_type_,
time_chunk_header_.data_type_,
common::INT64,
time_chunk_header_.compression_type_))) {
return ret;
}
Expand Down Expand Up @@ -1833,7 +1832,9 @@ int AlignedChunkReader::decode_all_planned_pages() {
for (size_t w = 0; w < worker_count; w++) {
if (time_decoder_pool_[w] == nullptr) {
time_decoder_pool_[w] =
DecoderFactory::alloc_time_decoder();
DecoderFactory::alloc_value_decoder(
time_chunk_header_.encoding_type_,
common::INT64);
if (time_decoder_pool_[w] == nullptr) return E_OOM;
}
if (time_compressor_pool_[w] == nullptr) {
Expand Down
20 changes: 20 additions & 0 deletions python/README-zh.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,26 @@ mvn -P with-cpp,with-python clean verify
python setup.py build_ext --inplace
```

## 树模型数值序列

V4 树模型 TsFile 可以使用与表模型相同的 `TsFileDataFrame` 序列读取和窗口切片接口。
通过完整逻辑路径选择测点。不同设备的同名测点可以具有不同的物理类型;
数值统一返回为 `float64` 数组,空值保留为 `NaN`。

```python
from tsfile import TsFileDataFrame

with TsFileDataFrame("tree_data/", use_index=True) as dataset:
with dataset["root.training.d1.temperature"] as series:
window = series[0:128]
timestamps = series.timestamps
statistics = series.stats # start_time、end_time、count
```

统计信息和按位置切片以序列时间线为准,包括对齐数据中的空值位置。
索引仍会校验同一完整路径在不同文件分片中的物理类型是否一致。
表模型与树模型文件应分别加载到不同的 DataFrame;数值视图不包含非数值测点。

## Dataset 索引

通过 `use_index=True` 启用持久化 Dataset 索引。第一次打开时构建索引,之后
Expand Down
22 changes: 22 additions & 0 deletions python/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,28 @@ with TsFileReader("example.tsfile") as reader:
Values do not carry a data type; use an explicit portable encoding when storing
numbers or structures.

## Numeric tree-model sequences

V4 tree-model TsFiles can use the same `TsFileDataFrame` sequence and window APIs
as table-model files. Select each measurement by its full logical path. Different
devices may have different physical types for the same measurement name; numeric
values are returned as `float64` arrays, with null values preserved as `NaN`.

```python
from tsfile import TsFileDataFrame

with TsFileDataFrame("tree_data/", use_index=True) as dataset:
with dataset["root.training.d1.temperature"] as series:
window = series[0:128]
timestamps = series.timestamps
statistics = series.stats # start_time, end_time, count
```

Statistics and positional slices follow the sequence's timeline, including null
positions in aligned data. The index validates physical types across shards of
the same full path. Load table-model and tree-model files in separate DataFrames;
non-numeric measurements are omitted from the numeric dataset view.

## Dataset indexes

Enable the persistent dataset index with `use_index=True`. The first open
Expand Down
15 changes: 14 additions & 1 deletion python/tests/resources/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,20 @@

-->

## Aligned numeric tree fixtures

`tree_aligned_mixed.tsfile` and `tree_aligned_mixed_ts2diff.tsfile` are V4 files
generated by `TreeDatasetFixtureGenerator.java` using the Java TsFile writer.
The generator accepts the output path and an optional time encoding (`PLAIN`
by default, or `TS_2DIFF`). Both use uncompressed two-row pages.

The fixtures contain six timestamps (0 through 5) on two aligned devices:
`root.training.d1` has DOUBLE temperature and INT32 status, while
`root.training.d2.deep` has INT32 temperature and BOOLEAN status. Temperature
is null at timestamp 2 on the first device and timestamp 3 on the second.
They verify that the Python numeric dataset reads each sequence with its own
physical type, preserves nulls, and counts the shared timeline consistently.

In `simple_tree.tsfile`:

```sql
Expand Down Expand Up @@ -332,4 +346,3 @@ Total line number = 4
It costs 0.065s
IoTDB:mydb>
```

69 changes: 69 additions & 0 deletions python/tests/resources/TreeDatasetFixtureGenerator.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.read.common.Path;
import org.apache.tsfile.write.TsFileWriter;
import org.apache.tsfile.write.record.TSRecord;
import org.apache.tsfile.write.schema.MeasurementSchema;

import java.io.File;
import java.util.Arrays;

/** Generates aligned tree fixtures for the Python sequence training tests. */
public class TreeDatasetFixtureGenerator {
public static void main(String[] args) throws Exception {
TSFileDescriptor.getInstance()
.getConfig()
.setTimeEncoder(args.length > 1 ? args[1] : "PLAIN");
TSFileDescriptor.getInstance().getConfig().setCompressor("UNCOMPRESSED");
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(2);
TSFileDescriptor.getInstance().getConfig().setMaxDegreeOfIndexNode(2);
try (TsFileWriter writer = new TsFileWriter(new File(args[0]))) {
writer.registerAlignedTimeseries(
new Path("root.training.d1"),
Arrays.asList(
schema("temperature", TSDataType.DOUBLE), schema("status", TSDataType.INT32)));
writer.registerAlignedTimeseries(
new Path("root.training.d2.deep"),
Arrays.asList(
schema("temperature", TSDataType.INT32), schema("status", TSDataType.BOOLEAN)));
for (int t = 0; t < 6; t++) {
TSRecord first = new TSRecord("root.training.d1", t).addPoint("status", t);
if (t != 2) {
first.addPoint("temperature", t + 0.25);
}
writer.writeRecord(first);
TSRecord second =
new TSRecord("root.training.d2.deep", t).addPoint("status", t % 2 == 0);
if (t != 3) {
second.addPoint("temperature", t * 10 + 100);
}
writer.writeRecord(second);
}
}
}

private static MeasurementSchema schema(String name, TSDataType type) {
return new MeasurementSchema(name, type, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED);
}
}
Binary file added python/tests/resources/tree_aligned_mixed.tsfile
Binary file not shown.
Binary file not shown.
Loading