Skip to content

Commit e5379d5

Browse files
committed
feat: add SSTable format and S3 persistence (Phase 1c)
Add persistence layer so MemTables flush to immutable SSTable files on S3-compatible storage. When s3Bucket is not configured the store stays in pure in-memory mode and all existing tests remain unaffected. Key additions: - ObjectStore interface with S3ObjectStore (MinIO) and FileSystemObjectStore (test double) - SSTableWriter/SSTable: binary format with block index for range scans - MergeIterator: K-way merge across MemTable + SSTables with deduplication and tombstone suppression - Manifest: versioned JSON manifest tracking SSTables on S3 - S3ValueStore and S3NamespaceStore serialization/deserialization - S3StoreConfig: S3 connectivity properties (bucket, endpoint, etc.) - S3SailStore: flush path, merged read path, startup loading - 29 new tests (unit + persistence); 541 total tests pass
1 parent 2b2f929 commit e5379d5

27 files changed

Lines changed: 2876 additions & 30 deletions

core/sail/s3/pom.xml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,10 @@
4848
<groupId>com.google.guava</groupId>
4949
<artifactId>guava</artifactId>
5050
</dependency>
51+
<dependency>
52+
<groupId>com.fasterxml.jackson.core</groupId>
53+
<artifactId>jackson-databind</artifactId>
54+
</dependency>
5155
<dependency>
5256
<groupId>${project.groupId}</groupId>
5357
<artifactId>rdf4j-sail-testsuite</artifactId>
@@ -71,5 +75,17 @@
7175
<artifactId>junit-jupiter-params</artifactId>
7276
<scope>test</scope>
7377
</dependency>
78+
<dependency>
79+
<groupId>org.testcontainers</groupId>
80+
<artifactId>testcontainers</artifactId>
81+
<version>${testcontainers.version}</version>
82+
<scope>test</scope>
83+
</dependency>
84+
<dependency>
85+
<groupId>org.testcontainers</groupId>
86+
<artifactId>testcontainers-junit-jupiter</artifactId>
87+
<version>${testcontainers.version}</version>
88+
<scope>test</scope>
89+
</dependency>
7490
</dependencies>
7591
</project>

core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3NamespaceStore.java

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,17 @@
1010
*******************************************************************************/
1111
package org.eclipse.rdf4j.sail.s3;
1212

13+
import java.io.IOException;
14+
import java.io.UncheckedIOException;
1315
import java.util.Iterator;
1416
import java.util.LinkedHashMap;
17+
import java.util.List;
1518
import java.util.Map;
1619

1720
import org.eclipse.rdf4j.model.impl.SimpleNamespace;
21+
import org.eclipse.rdf4j.sail.s3.storage.ObjectStore;
22+
23+
import com.fasterxml.jackson.databind.ObjectMapper;
1824

1925
/**
2026
* In-memory store for namespace prefix information. All operations are synchronized for thread safety.
@@ -52,4 +58,37 @@ public synchronized Iterator<SimpleNamespace> iterator() {
5258
public synchronized void clear() {
5359
namespacesMap.clear();
5460
}
61+
62+
@SuppressWarnings("unchecked")
63+
synchronized void deserialize(ObjectStore objectStore, ObjectMapper mapper) {
64+
byte[] data = objectStore.get("namespaces/current");
65+
if (data == null) {
66+
return;
67+
}
68+
try {
69+
List<Map<String, String>> entries = mapper.readValue(data, List.class);
70+
for (Map<String, String> entry : entries) {
71+
String prefix = entry.get("prefix");
72+
String name = entry.get("name");
73+
namespacesMap.put(prefix, new SimpleNamespace(prefix, name));
74+
}
75+
} catch (IOException e) {
76+
throw new UncheckedIOException("Failed to deserialize namespaces", e);
77+
}
78+
}
79+
80+
synchronized void serialize(ObjectStore objectStore, ObjectMapper mapper) {
81+
try {
82+
List<Map<String, String>> entries = new java.util.ArrayList<>();
83+
for (SimpleNamespace ns : namespacesMap.values()) {
84+
Map<String, String> entry = new LinkedHashMap<>();
85+
entry.put("prefix", ns.getPrefix());
86+
entry.put("name", ns.getName());
87+
entries.add(entry);
88+
}
89+
objectStore.put("namespaces/current", mapper.writeValueAsBytes(entries));
90+
} catch (IOException e) {
91+
throw new UncheckedIOException("Failed to serialize namespaces", e);
92+
}
93+
}
5594
}

0 commit comments

Comments
 (0)