Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -26,3 +26,4 @@ junit.xml
/quickstart/data
/.devcontainer
.perf-loop/
.codegraph/
8 changes: 2 additions & 6 deletions frac/sealed/token/table_loader.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,13 +147,9 @@ func (l *TableLoader) loadBlocksLegacy() ([]TableBlock, error) {
}

func (l *TableLoader) loadBlocks() ([]TableBlock, error) {
blocksCount, err := l.reader.BlocksCount()
if err != nil {
return nil, err
}

var blocks []TableBlock
for blockIndex := l.tableIndex; blockIndex < uint32(blocksCount); blockIndex++ {
blocksCount := uint32(l.reader.BlocksCount())
for blockIndex := l.tableIndex; blockIndex < blocksCount; blockIndex++ {
data, err := l.readBlock(blockIndex)
if err != nil {
return nil, err
Expand Down
18 changes: 2 additions & 16 deletions frac/sealed_loader.go
Original file line number Diff line number Diff line change
Expand Up @@ -241,14 +241,7 @@ func (l *Loader) loadIDsTable(r storage.IndexReader, info *common.Info) seqids.T
IDsTotal: info.DocsTotal + 1, // Increment by one for [seq.SystemID]
}

blocksCount, err := r.BlocksCount()
if err != nil {
logger.Fatal(
"cannot get block count",
zap.Error(err),
)
}

blocksCount := r.BlocksCount()
for blockIdx := 0; blockIdx < blocksCount; blockIdx += 3 {
header, err := r.GetBlockHeader(uint32(blockIdx))
if err != nil {
Expand Down Expand Up @@ -281,14 +274,7 @@ func (l *Loader) loadLIDsTable(r storage.IndexReader, fracVer config.BinaryDataV
isContinued []bool
)

blocksCount, err := r.BlocksCount()
if err != nil {
logger.Fatal(
"cannot get block count",
zap.Error(err),
)
}

blocksCount := r.BlocksCount()
for blockIdx := 0; blockIdx < blocksCount; blockIdx++ {
header, err := r.GetBlockHeader(uint32(blockIdx))
if err != nil {
Expand Down
39 changes: 17 additions & 22 deletions storage/index_reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@ import (
"fmt"
"io"

"go.uber.org/zap"

"github.com/ozontech/seq-db/bytespool"
"github.com/ozontech/seq-db/cache"
"github.com/ozontech/seq-db/logger"
"github.com/ozontech/seq-db/util"
)

Expand All @@ -18,19 +21,25 @@ type IndexReader struct {
reader io.ReaderAt
readerName string

cache cache.Cache[[]byte]
registry []byte
}

func NewIndexReader(
limiter *ReadLimiter, readerName string,
reader io.ReaderAt, registryCache cache.Cache[[]byte],
) IndexReader {
return IndexReader{
r := IndexReader{
limiter: limiter,
reader: reader,
readerName: readerName,
cache: registryCache,
}

var err error
if r.registry, err = registryCache.Get(registryCacheKey, (*registryLoader)(&r)); err != nil {
logger.Fatal("can't init IndexReader", zap.Error(err))
}

return r
}

type registryLoader IndexReader
Expand Down Expand Up @@ -82,25 +91,16 @@ func (rl *registryLoader) Load(uint32) ([]byte, int, error) {
return buf, cap(buf), nil
}

func (r *IndexReader) registry() ([]byte, error) {
return r.cache.Get(registryCacheKey, (*registryLoader)(r))
}

func (r *IndexReader) GetBlockHeader(index uint32) (IndexBlockHeader, error) {
reg, err := r.registry()
if err != nil {
return nil, err
}

if (uint64(index)+1)*IndexBlockHeaderSize > uint64(len(reg)) {
if (uint64(index)+1)*IndexBlockHeaderSize > uint64(len(r.registry)) {
return nil, fmt.Errorf(
"too large index block in file %s, with index %d, registry size %d",
r.readerName, index, len(reg),
r.readerName, index, len(r.registry),
)
}

pos := index * IndexBlockHeaderSize
return reg[pos : pos+IndexBlockHeaderSize], nil
return r.registry[pos : pos+IndexBlockHeaderSize], nil
}

func (r *IndexReader) ReadIndexBlock(blockIndex uint32, dst []byte) ([]byte, uint64, error) {
Expand Down Expand Up @@ -129,11 +129,6 @@ func (r *IndexReader) ReadIndexBlock(blockIndex uint32, dst []byte) ([]byte, uin
return dst, uint64(n), err
}

func (r *IndexReader) BlocksCount() (int, error) {
reg, err := r.registry()
if err != nil {
return 0, err
}

return len(reg) / IndexBlockHeaderSize, nil
func (r *IndexReader) BlocksCount() int {
return len(r.registry) / IndexBlockHeaderSize
}
Loading