Skip to content

Commit 81a7e40

Browse files
hmottestadhmottestad
committed
working on new ID based join iterator
1 parent 8a00d62 commit 81a7e40

11 files changed

Lines changed: 189 additions & 34 deletions

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbDelegatingSailDataset.java‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -95,24 +95,36 @@ public RecordIterator getRecordIterator(org.eclipse.rdf4j.query.algebra.Statemen
9595
@Override
9696
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
9797
long[] patternIds) throws QueryEvaluationException {
98+
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, null);
99+
}
100+
101+
@Override
102+
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
103+
long[] patternIds, long[] reuse) throws QueryEvaluationException {
98104
if (delegate instanceof LmdbEvaluationDataset) {
99105
return ((LmdbEvaluationDataset) delegate).getRecordIterator(binding, subjIndex, predIndex, objIndex,
100-
ctxIndex, patternIds);
106+
ctxIndex, patternIds, reuse);
101107
}
102108
// Fallback via TripleSource with Value conversion
103109
return new LmdbSailDatasetTripleSource(valueStore, delegate)
104-
.getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds);
110+
.getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, reuse);
105111
}
106112

107113
@Override
108114
public RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex,
109115
int ctxIndex, long[] patternIds, StatementOrder order) throws QueryEvaluationException {
116+
return getOrderedRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, order, null);
117+
}
118+
119+
@Override
120+
public RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex,
121+
int ctxIndex, long[] patternIds, StatementOrder order, long[] reuse) throws QueryEvaluationException {
110122
if (delegate instanceof LmdbEvaluationDataset) {
111123
return ((LmdbEvaluationDataset) delegate).getOrderedRecordIterator(binding, subjIndex, predIndex, objIndex,
112-
ctxIndex, patternIds, order);
124+
ctxIndex, patternIds, order, reuse);
113125
}
114126
return LmdbEvaluationDataset.super.getOrderedRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex,
115-
patternIds, order);
127+
patternIds, order, reuse);
116128
}
117129

118130
@Override

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbEvaluationDataset.java‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,10 @@ public interface LmdbEvaluationDataset {
4242
* read-only. The {@code patternIds} array contains four entries (subject, predicate, object, context) where a value
4343
* of {@link org.eclipse.rdf4j.sail.lmdb.model.LmdbValue#UNKNOWN_ID} indicates that the corresponding position is
4444
* unbound in the pattern. The index arguments point to the slots in {@code binding} where the resolved IDs should
45-
* be written (or {@code -1} if that pattern position does not correspond to a variable).
45+
* be written (or {@code -1} if that pattern position does not correspond to a variable). Implementations may reuse
46+
* internal buffers by copying {@code binding} into a scratch array before mutating; callers can supply such an
47+
* array via {@link #getRecordIterator(long[], int, int, int, int, long[], long[])} to avoid per-iterator
48+
* allocation.
4649
* </p>
4750
*
4851
* @param binding the current binding snapshot; implementations must copy before mutating
@@ -58,6 +61,20 @@ public interface LmdbEvaluationDataset {
5861
RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
5962
long[] patternIds) throws QueryEvaluationException;
6063

64+
/**
65+
* Variant of {@link #getRecordIterator(long[], int, int, int, int, long[])} that allows callers to supply a
66+
* reusable scratch buffer. Implementations should treat {@code binding} as read-only and (when {@code reuse} is
67+
* non-null and large enough) seed the scratch buffer with the binding state before producing rows from this
68+
* iterator.
69+
*
70+
* @param reuse optional scratch buffer that may be mutated and returned from {@link RecordIterator#next()}
71+
*/
72+
@InternalUseOnly
73+
default RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
74+
long[] patternIds, long[] reuse) throws QueryEvaluationException {
75+
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds);
76+
}
77+
6178
/**
6279
* Create an ordered {@link RecordIterator} for the supplied pattern expressed via internal IDs and binding indexes.
6380
* Implementations may fall back to the unordered iterator when the requested order is unsupported.
@@ -71,6 +88,19 @@ default RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, i
7188
return null;
7289
}
7390

91+
/**
92+
* Variant of {@link #getOrderedRecordIterator(long[], int, int, int, int, long[], StatementOrder)} that accepts a
93+
* reusable scratch buffer.
94+
*/
95+
@InternalUseOnly
96+
default RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex,
97+
int ctxIndex, long[] patternIds, StatementOrder order, long[] reuse) throws QueryEvaluationException {
98+
if (order == null) {
99+
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, reuse);
100+
}
101+
return null;
102+
}
103+
74104
/**
75105
* Create an ordered {@link RecordIterator} for the supplied pattern. Implementations may fall back to the unordered
76106
* iterator when the requested order is unsupported.

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbIdTripleSource.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,14 @@ public interface LmdbIdTripleSource {
3333
RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
3434
long[] patternIds) throws QueryEvaluationException;
3535

36+
/**
37+
* Variant that accepts a reusable scratch buffer.
38+
*/
39+
default RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
40+
long[] patternIds, long[] reuse) throws QueryEvaluationException {
41+
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds);
42+
}
43+
3644
/**
3745
* Create an ordered iterator over ID-level bindings; may fall back to the unordered iterator if unsupported.
3846
*/
@@ -43,4 +51,15 @@ default RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, i
4351
}
4452
return null;
4553
}
54+
55+
/**
56+
* Variant that accepts a reusable scratch buffer.
57+
*/
58+
default RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex,
59+
int ctxIndex, long[] patternIds, StatementOrder order, long[] reuse) throws QueryEvaluationException {
60+
if (order == null) {
61+
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, reuse);
62+
}
63+
return null;
64+
}
4665
}

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbIdTripleSourceAdapter.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,17 +82,25 @@ public ValueFactory getValueFactory() {
8282
@Override
8383
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
8484
long[] patternIds) throws QueryEvaluationException {
85+
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, null);
86+
}
87+
88+
@Override
89+
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
90+
long[] patternIds, long[] reuse) throws QueryEvaluationException {
8591
// Prefer direct ID-level access if the delegate already supports it
8692
if (delegate instanceof LmdbIdTripleSource) {
8793
return ((LmdbIdTripleSource) delegate).getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex,
88-
patternIds);
94+
patternIds, reuse);
8995
}
9096

9197
// If no active connection changes, delegate to the current LMDB dataset to avoid materialization
9298
if (!LmdbEvaluationStrategy.hasActiveConnectionChanges()) {
9399
var dsOpt = LmdbEvaluationStrategy.getCurrentDataset();
94100
if (dsOpt.isPresent()) {
95-
return dsOpt.get().getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds);
101+
return dsOpt.get()
102+
.getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds,
103+
reuse);
96104
}
97105
}
98106

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbOverlayEvaluationDataset.java‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,12 +131,20 @@ public void close() {
131131
@Override
132132
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
133133
long[] patternIds) throws QueryEvaluationException {
134+
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, null);
135+
}
136+
137+
@Override
138+
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
139+
long[] patternIds, long[] reuse) throws QueryEvaluationException {
134140
// Fast path: no active connection changes → use the current LMDB dataset's ID-level iterator
135141
try {
136142
if (!LmdbEvaluationStrategy.hasActiveConnectionChanges()) {
137143
var dsOpt = LmdbEvaluationStrategy.getCurrentDataset();
138144
if (dsOpt.isPresent()) {
139-
return dsOpt.get().getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds);
145+
return dsOpt.get()
146+
.getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds,
147+
reuse);
140148
}
141149
}
142150
} catch (Exception ignore) {
@@ -148,7 +156,7 @@ public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predI
148156
// current branch dataset state (including transaction overlays). Therefore, using ID-level access here is
149157
// correct when available.
150158
RecordIterator viaIds = ((LmdbIdTripleSource) tripleSource)
151-
.getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds);
159+
.getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, reuse);
152160
if (viaIds != null) {
153161
return viaIds;
154162
}
@@ -244,6 +252,13 @@ public RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, in
244252
patternIds, order);
245253
}
246254

255+
@Override
256+
public RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex,
257+
int ctxIndex, long[] patternIds, StatementOrder order, long[] reuse) throws QueryEvaluationException {
258+
return LmdbEvaluationDataset.super.getOrderedRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex,
259+
patternIds, order, reuse);
260+
}
261+
247262
@Override
248263
public ValueStore getValueStore() {
249264
return valueStore;

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailDatasetTripleSource.java‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,17 @@ public LmdbSailDatasetTripleSource(ValueFactory vf, SailDataset dataset) {
4343
@Override
4444
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
4545
long[] patternIds) throws QueryEvaluationException {
46+
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, null);
47+
}
48+
49+
@Override
50+
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
51+
long[] patternIds, long[] reuse) throws QueryEvaluationException {
4652

4753
// Fast path: backing dataset supports ID-level access
4854
if (dataset instanceof LmdbEvaluationDataset) {
4955
return ((LmdbEvaluationDataset) dataset).getRecordIterator(binding, subjIndex, predIndex, objIndex,
50-
ctxIndex, patternIds);
56+
ctxIndex, patternIds, reuse);
5157
}
5258

5359
// Fallback path: value-level iteration converted to IDs using ValueStore
@@ -146,12 +152,18 @@ public void close() {
146152
@Override
147153
public RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex,
148154
int ctxIndex, long[] patternIds, StatementOrder order) throws QueryEvaluationException {
155+
return getOrderedRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, order, null);
156+
}
157+
158+
@Override
159+
public RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex,
160+
int ctxIndex, long[] patternIds, StatementOrder order, long[] reuse) throws QueryEvaluationException {
149161
if (dataset instanceof LmdbEvaluationDataset) {
150162
return ((LmdbEvaluationDataset) dataset).getOrderedRecordIterator(binding, subjIndex, predIndex, objIndex,
151-
ctxIndex, patternIds, order);
163+
ctxIndex, patternIds, order, reuse);
152164
}
153165
return LmdbIdTripleSource.super.getOrderedRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex,
154-
patternIds, order);
166+
patternIds, order, reuse);
155167
}
156168

157169
private long selectQueryId(long patternId, long[] binding, int index) {

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailStore.java‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1370,6 +1370,13 @@ public RecordIterator getRecordIterator(StatementPattern pattern, BindingSet bin
13701370
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex,
13711371
int ctxIndex,
13721372
long[] patternIds) throws QueryEvaluationException {
1373+
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, null);
1374+
}
1375+
1376+
@Override
1377+
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex,
1378+
int ctxIndex,
1379+
long[] patternIds, long[] reuse) throws QueryEvaluationException {
13731380
try {
13741381
long subjQuery = selectQueryId(patternIds[TripleStore.SUBJ_IDX], binding, subjIndex);
13751382
long predQuery = selectQueryId(patternIds[TripleStore.PRED_IDX], binding, predIndex);
@@ -1379,9 +1386,15 @@ public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predI
13791386
RecordIterator base = tripleStore.getTriples(txn, subjQuery, predQuery, objQuery, ctxQuery, explicit);
13801387

13811388
if (!"false".equalsIgnoreCase(System.getProperty("rdf4j.lmdb.experimentalScratchReuse", "true"))) {
1382-
return new RecordIterator() {
1383-
private final long[] scratch = Arrays.copyOf(binding, binding.length);
1389+
final long[] scratch;
1390+
if (reuse != null && reuse.length >= binding.length) {
1391+
System.arraycopy(binding, 0, reuse, 0, binding.length);
1392+
scratch = reuse;
1393+
} else {
1394+
scratch = Arrays.copyOf(binding, binding.length);
1395+
}
13841396

1397+
return new RecordIterator() {
13851398
@Override
13861399
public long[] next() throws QueryEvaluationException {
13871400
try {
@@ -1482,8 +1495,14 @@ public void close() {
14821495
@Override
14831496
public RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex,
14841497
int ctxIndex, long[] patternIds, StatementOrder order) throws QueryEvaluationException {
1498+
return getOrderedRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, order, null);
1499+
}
1500+
1501+
@Override
1502+
public RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex,
1503+
int ctxIndex, long[] patternIds, StatementOrder order, long[] reuse) throws QueryEvaluationException {
14851504
if (order == null) {
1486-
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds);
1505+
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, reuse);
14871506
}
14881507
try {
14891508
long subjQuery = selectQueryId(patternIds[TripleStore.SUBJ_IDX], binding, subjIndex);
@@ -1500,7 +1519,7 @@ public RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, in
15001519
int sortIndex = indexForOrder(order, subjIndex, predIndex, objIndex, ctxIndex);
15011520
if (sortIndex >= 0) {
15021521
RecordIterator fallback = getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex,
1503-
patternIds);
1522+
patternIds, reuse);
15041523
return sortedRecordIterator(fallback, sortIndex);
15051524
}
15061525
}

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbUnionSailDataset.java‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -131,36 +131,49 @@ public RecordIterator getRecordIterator(org.eclipse.rdf4j.query.algebra.Statemen
131131
@Override
132132
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
133133
long[] patternIds) throws org.eclipse.rdf4j.query.QueryEvaluationException {
134+
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, null);
135+
}
136+
137+
@Override
138+
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
139+
long[] patternIds, long[] reuse) throws org.eclipse.rdf4j.query.QueryEvaluationException {
134140
boolean d1 = dataset1 instanceof LmdbEvaluationDataset;
135141
boolean d2 = dataset2 instanceof LmdbEvaluationDataset;
136142
if (d1 && d2) {
137143
RecordIterator r1 = ((LmdbEvaluationDataset) dataset1).getRecordIterator(binding, subjIndex, predIndex,
138-
objIndex, ctxIndex, patternIds);
144+
objIndex, ctxIndex, patternIds, reuse);
139145
RecordIterator r2 = ((LmdbEvaluationDataset) dataset2).getRecordIterator(binding, subjIndex, predIndex,
140-
objIndex, ctxIndex, patternIds);
146+
objIndex, ctxIndex, patternIds, reuse);
141147
return chain(r1, r2);
142148
}
143149
LmdbDelegatingSailDataset a = new LmdbDelegatingSailDataset(dataset1, valueStore);
144150
LmdbDelegatingSailDataset b = new LmdbDelegatingSailDataset(dataset2, valueStore);
145-
return chain(a.getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds),
146-
b.getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds));
151+
return chain(a.getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, reuse),
152+
b.getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, reuse));
147153
}
148154

149155
@Override
150156
public RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex,
151157
int ctxIndex, long[] patternIds, StatementOrder order)
152158
throws org.eclipse.rdf4j.query.QueryEvaluationException {
159+
return getOrderedRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, order, null);
160+
}
161+
162+
@Override
163+
public RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex,
164+
int ctxIndex, long[] patternIds, StatementOrder order, long[] reuse)
165+
throws org.eclipse.rdf4j.query.QueryEvaluationException {
153166
boolean d1 = dataset1 instanceof LmdbEvaluationDataset;
154167
boolean d2 = dataset2 instanceof LmdbEvaluationDataset;
155168
if (d1 && d2) {
156169
RecordIterator r1 = ((LmdbEvaluationDataset) dataset1).getOrderedRecordIterator(binding, subjIndex,
157-
predIndex, objIndex, ctxIndex, patternIds, order);
170+
predIndex, objIndex, ctxIndex, patternIds, order, reuse);
158171
RecordIterator r2 = ((LmdbEvaluationDataset) dataset2).getOrderedRecordIterator(binding, subjIndex,
159-
predIndex, objIndex, ctxIndex, patternIds, order);
172+
predIndex, objIndex, ctxIndex, patternIds, order, reuse);
160173
return chain(r1, r2);
161174
}
162175
return LmdbEvaluationDataset.super.getOrderedRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex,
163-
patternIds, order);
176+
patternIds, order, reuse);
164177
}
165178

166179
@Override

0 commit comments

Comments
 (0)