Skip to content

Commit 945ecc6

Browse files
hmottestadhmottestad
committed
allow users to use the old eval factory in lmdb and disable the sketches
1 parent 79a1ac7 commit 945ecc6

7 files changed

Lines changed: 516 additions & 105 deletions

File tree

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbEvaluationStatistics.java‎

Lines changed: 45 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,9 @@ public LmdbEvaluationStatistics(ValueStore valueStore, TripleStore tripleStore,
8888

8989
@Override
9090
public boolean supportsJoinEstimation() {
91+
if (sketchBasedJoinEstimator == null) {
92+
return false;
93+
}
9194
long now = System.currentTimeMillis();
9295
long revisionId = valueStore.getRevision().getRevisionId();
9396
if (now < joinSupportCacheExpiryMs && revisionId == joinSupportCacheRevisionId) {
@@ -104,11 +107,14 @@ public boolean supportsJoinEstimation() {
104107

105108
@Override
106109
public boolean supportsFilterSelectivityCosting() {
107-
return true;
110+
return sketchBasedJoinEstimator != null;
108111
}
109112

110113
@Override
111114
public QueryOptimizationScope beginQueryOptimizationScope() {
115+
if (sketchBasedJoinEstimator == null) {
116+
return QueryOptimizationScopeProvider.NO_OP_SCOPE;
117+
}
112118
return sketchBasedJoinEstimator.beginQueryOptimizationScope();
113119
}
114120

@@ -193,11 +199,17 @@ private boolean hasAccessMetrics(JoinOrderPlanner.PlanStep step) {
193199

194200
@Override
195201
public Optional<FactorCostEstimate> estimateFactorCost(TupleExpr factor, Set<String> currentlyBoundVars) {
202+
if (sketchBasedJoinEstimator == null) {
203+
return estimatePageWalkingFactorCost(factor);
204+
}
196205
return estimateLmdbFactorCost(factor, currentlyBoundVars);
197206
}
198207

199208
@Override
200209
public Optional<FactorCostEstimate> estimateFactorCost(TupleExpr factor, JoinFactorCostModel.CostContext context) {
210+
if (sketchBasedJoinEstimator == null) {
211+
return estimatePageWalkingFactorCost(factor);
212+
}
201213
if (context == null) {
202214
return estimateLmdbFactorCost(factor, Set.of());
203215
}
@@ -208,6 +220,15 @@ public Optional<FactorCostEstimate> estimateFactorCost(TupleExpr factor, JoinFac
208220
return estimate.map(factorCostEstimate -> accountNestedInvocationWork(factor, factorCostEstimate, context));
209221
}
210222

223+
private Optional<FactorCostEstimate> estimatePageWalkingFactorCost(TupleExpr factor) {
224+
if (factor == null) {
225+
return Optional.empty();
226+
}
227+
double outputRows = estimateFactorOutputRows(factor, Set.of());
228+
return isFiniteNonNegative(outputRows) ? Optional.of(new FactorCostEstimate(outputRows, outputRows))
229+
: Optional.empty();
230+
}
231+
211232
private Optional<FactorCostEstimate> estimateLmdbFactorCost(TupleExpr factor,
212233
JoinFactorCostModel.CostContext context) {
213234
if (context.hasCurrentlyBoundVarMask()) {
@@ -292,6 +313,9 @@ private double predicateBoundedRepeatedPrefixWorkRows(TupleExpr factor, FactorCo
292313

293314
private SketchBasedJoinEstimator.AccessShape accessShapeForContext(TupleExpr factor,
294315
JoinFactorCostModel.CostContext context) {
316+
if (sketchBasedJoinEstimator == null) {
317+
return null;
318+
}
295319
if (context.hasCurrentlyBoundVarMask()) {
296320
return sketchBasedJoinEstimator.accessShapeForJoinOrdering(factor, context.getVariableNames(),
297321
context.getCurrentlyBoundVarMask());
@@ -567,9 +591,11 @@ private String accessMode(AccessPathEstimate accessPathEstimate) {
567591
}
568592

569593
private double estimateFactorOutputRows(TupleExpr factor, Set<String> boundVars) {
570-
double sketchRows = sketchBasedJoinEstimator.factorOutputRowsForJoinOrdering(factor, boundVars);
571-
if (isFiniteNonNegative(sketchRows)) {
572-
return sketchRows;
594+
if (sketchBasedJoinEstimator != null) {
595+
double sketchRows = sketchBasedJoinEstimator.factorOutputRowsForJoinOrdering(factor, boundVars);
596+
if (isFiniteNonNegative(sketchRows)) {
597+
return sketchRows;
598+
}
573599
}
574600
if (factor instanceof BindingSetAssignment assignment) {
575601
double rows = 0.0d;
@@ -582,10 +608,12 @@ private double estimateFactorOutputRows(TupleExpr factor, Set<String> boundVars)
582608
}
583609

584610
private double estimateFactorOutputRows(TupleExpr factor, String[] variableNames, long boundVarMask) {
585-
double sketchRows = sketchBasedJoinEstimator.factorOutputRowsForJoinOrdering(factor, variableNames,
586-
boundVarMask);
587-
if (isFiniteNonNegative(sketchRows)) {
588-
return sketchRows;
611+
if (sketchBasedJoinEstimator != null) {
612+
double sketchRows = sketchBasedJoinEstimator.factorOutputRowsForJoinOrdering(factor, variableNames,
613+
boundVarMask);
614+
if (isFiniteNonNegative(sketchRows)) {
615+
return sketchRows;
616+
}
589617
}
590618
if (factor instanceof BindingSetAssignment assignment) {
591619
double rows = 0.0d;
@@ -869,6 +897,9 @@ public boolean isJoinEstimationReady() {
869897
}
870898

871899
public Object joinEstimationDiagnostics() {
900+
if (sketchBasedJoinEstimator == null) {
901+
return Map.of("estimator", "disabled");
902+
}
872903
SketchBasedJoinEstimator.Staleness staleness = sketchBasedJoinEstimator.staleness();
873904
Map<String, Object> diagnostics = new LinkedHashMap<>();
874905
diagnostics.put("estimator", "SketchBasedJoinEstimator");
@@ -889,12 +920,18 @@ protected CardinalityCalculator createCardinalityCalculator() {
889920

890921
@Override
891922
public double estimateFilterPassRatio(Filter filter) {
923+
if (sketchBasedJoinEstimator == null) {
924+
return -1.0d;
925+
}
892926
double ratio = sketchBasedJoinEstimator.estimateFilterPassRatio(filter);
893927
return Double.isFinite(ratio) && ratio >= 0.0d && ratio <= 1.0d ? ratio : -1.0d;
894928
}
895929

896930
@Override
897931
public FilterPassEstimate estimateFilterPass(Filter filter) {
932+
if (sketchBasedJoinEstimator == null) {
933+
return super.estimateFilterPass(filter);
934+
}
898935
return sketchBasedJoinEstimator.estimateFilterPass(filter);
899936
}
900937

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailStore.java‎

Lines changed: 45 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -290,9 +290,17 @@ abstract static class StatefulOperation implements Operation {
290290
*/
291291
public LmdbSailStore(File dataDir, StoreProperties properties, LmdbStoreConfig config)
292292
throws IOException, SailException {
293+
this(dataDir, properties, config, true);
294+
}
295+
296+
public LmdbSailStore(File dataDir, StoreProperties properties, LmdbStoreConfig config,
297+
boolean sketchBasedJoinEstimatorEnabled)
298+
throws IOException, SailException {
293299
this.setFactory = new PersistentSetFactory<>(dataDir);
294300
this.bulkOperationSize = config.getBulkOperationSize();
295-
this.sketchBasedJoinEstimator = new SketchBasedJoinEstimator(this, sketchEstimatorConfig(config));
301+
this.sketchBasedJoinEstimator = sketchBasedJoinEstimatorEnabled
302+
? new SketchBasedJoinEstimator(this, sketchEstimatorConfig(config))
303+
: null;
296304
Function<Long, byte[]> encode = element -> {
297305
ByteBuffer bb = ByteBuffer.allocate(Long.BYTES).order(ByteOrder.BIG_ENDIAN);
298306
bb.putLong(element);
@@ -310,18 +318,20 @@ public LmdbSailStore(File dataDir, StoreProperties properties, LmdbStoreConfig c
310318
statementPatternCardinalitySource = new LmdbStatementPatternCardinalitySource(valueStore, tripleStore);
311319
mayHaveInferred = tripleStore.hasTriples(false);
312320
initialized = true;
313-
Path estimatorPath = new File(dataDir, JOIN_ESTIMATOR_FILE_NAME).toPath();
314-
boolean snapshotExists = Files.isRegularFile(estimatorPath.resolve("metadata.bin"));
315-
filterSelectivityStats = new LmdbFilterSelectivityStats(estimatorPath, tripleStore, valueStore);
316-
sketchBasedJoinEstimator.setRebuildAllowedSupplier(() -> !storeTxnStarted.get());
317-
sketchBasedJoinEstimator.setLearnedStatsProvider(filterSelectivityStats);
318-
sketchBasedJoinEstimator.setPatternFilterSamplingEstimator(filterSelectivityStats);
319-
sketchBasedJoinEstimator.setPatternCardinalityProvider(statementPatternCardinalitySource::estimate);
320-
sketchBasedJoinEstimator.configurePersistence(estimatorPath, snapshotExists);
321-
if (!snapshotExists) {
322-
sketchBasedJoinEstimator.rebuild();
321+
if (sketchBasedJoinEstimator != null) {
322+
Path estimatorPath = new File(dataDir, JOIN_ESTIMATOR_FILE_NAME).toPath();
323+
boolean snapshotExists = Files.isRegularFile(estimatorPath.resolve("metadata.bin"));
324+
filterSelectivityStats = new LmdbFilterSelectivityStats(estimatorPath, tripleStore, valueStore);
325+
sketchBasedJoinEstimator.setRebuildAllowedSupplier(() -> !storeTxnStarted.get());
326+
sketchBasedJoinEstimator.setLearnedStatsProvider(filterSelectivityStats);
327+
sketchBasedJoinEstimator.setPatternFilterSamplingEstimator(filterSelectivityStats);
328+
sketchBasedJoinEstimator.setPatternCardinalityProvider(statementPatternCardinalitySource::estimate);
329+
sketchBasedJoinEstimator.configurePersistence(estimatorPath, snapshotExists);
330+
if (!snapshotExists) {
331+
sketchBasedJoinEstimator.rebuild();
332+
}
333+
sketchBasedJoinEstimator.startBackgroundRefresh(3);
323334
}
324-
sketchBasedJoinEstimator.startBackgroundRefresh(3);
325335
} finally {
326336
if (!initialized) {
327337
close();
@@ -392,7 +402,9 @@ public void close() throws SailException {
392402
try {
393403
cancelAndDrainScheduledEstimatorPersist();
394404
persistEstimatorState();
395-
sketchBasedJoinEstimator.close();
405+
if (sketchBasedJoinEstimator != null) {
406+
sketchBasedJoinEstimator.close();
407+
}
396408
} finally {
397409
try {
398410
if (namespaceStore != null) {
@@ -466,7 +478,9 @@ private void cancelAndDrainScheduledEstimatorPersist() {
466478
}
467479

468480
private void persistEstimatorState() {
469-
sketchBasedJoinEstimator.persistIfDirty();
481+
if (sketchBasedJoinEstimator != null) {
482+
sketchBasedJoinEstimator.persistIfDirty();
483+
}
470484
if (filterSelectivityStats != null) {
471485
filterSelectivityStats.persistIfDirty();
472486
}
@@ -624,8 +638,8 @@ public SailSink sink(IsolationLevel level) throws SailException {
624638

625639
@Override
626640
public LmdbSailDataset dataset(IsolationLevel level) throws SailException {
627-
boolean isEstimatorRefresh = SketchBasedJoinEstimator.REFRESH_THREAD_NAME
628-
.equals(Thread.currentThread().getName());
641+
boolean isEstimatorRefresh = sketchBasedJoinEstimator != null
642+
&& SketchBasedJoinEstimator.REFRESH_THREAD_NAME.equals(Thread.currentThread().getName());
629643
// Refresh reader transactions can remain open across write commits and must not
630644
// participate in the active txn reset/renew cycle.
631645
boolean trackActive = !isEstimatorRefresh;
@@ -647,7 +661,7 @@ public LmdbSailSink(boolean explicit, IsolationLevel level) throws SailException
647661
}
648662

649663
private void queueEstimatorAdd(Statement st) {
650-
if (!explicit) {
664+
if (!explicit || sketchBasedJoinEstimator == null) {
651665
return;
652666
}
653667
if (nonIsolated) {
@@ -665,7 +679,7 @@ private void queueEstimatorAdd(Statement st) {
665679
}
666680

667681
private void queueEstimatorRemove(Statement st) {
668-
if (!explicit) {
682+
if (!explicit || sketchBasedJoinEstimator == null) {
669683
return;
670684
}
671685
if (nonIsolated) {
@@ -705,7 +719,7 @@ private void clearEstimatorUpdates() {
705719
if (nonIsolated) {
706720
// In NONE isolation updates are applied eagerly; rollback cannot replay inverse deltas safely.
707721
// Discard the derived state and rebuild it from authoritative store data.
708-
if (nonIsolatedUpdatesApplied) {
722+
if (nonIsolatedUpdatesApplied && sketchBasedJoinEstimator != null) {
709723
sketchBasedJoinEstimator.discardAndMarkForRebuild();
710724
nonIsolatedUpdatesApplied = false;
711725
}
@@ -720,7 +734,9 @@ private void recoverEstimatorAfterFailure(String action, RuntimeException e) {
720734
synchronized (pendingEstimatorUpdates) {
721735
pendingEstimatorUpdates.clear();
722736
}
723-
sketchBasedJoinEstimator.discardAndMarkForRebuild();
737+
if (sketchBasedJoinEstimator != null) {
738+
sketchBasedJoinEstimator.discardAndMarkForRebuild();
739+
}
724740
nonIsolatedUpdatesApplied = false;
725741
logger.warn("Discarded join estimator state after failure while {}; store data remains authoritative",
726742
action,
@@ -808,12 +824,16 @@ public void flush() throws SailException {
808824
// The triple/value stores are authoritative once both commits succeed.
809825
storeTxnStarted.set(false);
810826
applyEstimatorUpdates();
811-
filterSelectivityStats.recordStoreMutation();
827+
if (filterSelectivityStats != null) {
828+
filterSelectivityStats.recordStoreMutation();
829+
}
812830
nonIsolatedUpdatesApplied = false;
813-
try {
814-
scheduleEstimatorPersist();
815-
} catch (RuntimeException e) {
816-
logger.warn("Failed to schedule join estimator persistence after commit", e);
831+
if (sketchBasedJoinEstimator != null || filterSelectivityStats != null) {
832+
try {
833+
scheduleEstimatorPersist();
834+
} catch (RuntimeException e) {
835+
logger.warn("Failed to schedule join estimator persistence after commit", e);
836+
}
817837
}
818838
}
819839
}

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbStore.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,8 @@ protected void initializeInternal() throws SailException {
265265
properties.setVersion(String.valueOf(VERSION));
266266
}
267267

268-
backingStore = new LmdbSailStore(dataDir, properties, config);
268+
boolean useSketchBasedJoinEstimator = usesLmdbEvaluationStrategyFactory();
269+
backingStore = new LmdbSailStore(dataDir, properties, config, useSketchBasedJoinEstimator);
269270

270271
// update version afer loading and potential internal migration within value and triple store
271272
if (updateVersion) {
@@ -277,7 +278,8 @@ protected void initializeInternal() throws SailException {
277278
@Override
278279
protected LmdbSailStore createSailStore(File dataDir) throws IOException, SailException {
279280
// Model can't fit into memory, use another LmdbSailStore to store delta
280-
LmdbSailStore lmdbSailStore = new LmdbSailStore(dataDir, new StoreProperties(), config);
281+
LmdbSailStore lmdbSailStore = new LmdbSailStore(dataDir, new StoreProperties(), config,
282+
useSketchBasedJoinEstimator);
281283
lmdbSailStore.enableMultiThreading = false;
282284
return lmdbSailStore;
283285
}
@@ -421,6 +423,10 @@ LmdbSailStore getBackingStore() {
421423
return backingStore;
422424
}
423425

426+
private boolean usesLmdbEvaluationStrategyFactory() {
427+
return getEvaluationStrategyFactory() instanceof LmdbEvaluationStrategyFactory;
428+
}
429+
424430
private boolean upgradeStore(File dataDir, String version) throws SailException {
425431
// nothing to do, just update version number
426432
return true;

0 commit comments

Comments
 (0)