Skip to content

Commit 8618d7a

Browse files
hmottestadhmottestad
committed
keep fine tuning
1 parent d968ff5 commit 8618d7a

7 files changed

Lines changed: 9597 additions & 11 deletions

File tree

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbDeferredFilterPlacer.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ TupleExpr buildSegmentRoot(Deque<TupleExpr> orderedArgs, List<DeferredFilter> fi
5151
Set<String> prefixBindingNames = new HashSet<>(boundBeforeSegment);
5252
for (int i = 0; i < orderedJoinArgs.size(); i++) {
5353
TupleExpr optimized = factorOptimizer.optimize(orderedJoinArgs.get(i), prefixBindingNames);
54+
optimized = applyCompatibleLocalDeferredFilters(optimized, pendingFilters);
5455
Set<StatementPattern> currentPatterns = LmdbJoinPlanSupport.collectPatternIdentities(optimized);
5556
optimized = applyPrefixBindingDeferredFilters(optimized, pendingFilters, prefixBindingNames,
5657
currentPatterns, remainingPatternsAfterFactor.get(i));
@@ -111,6 +112,29 @@ private TupleExpr applyPrefixBindingDeferredFilters(TupleExpr tupleExpr,
111112
: filterWrapper.wrap(tupleExpr, prefixFilters, "bindingPrefix");
112113
}
113114

115+
private TupleExpr applyCompatibleLocalDeferredFilters(TupleExpr tupleExpr, List<DeferredFilter> deferredFilters) {
116+
if (deferredFilters.isEmpty()) {
117+
return tupleExpr;
118+
}
119+
List<DeferredFilter> localFilters = new ArrayList<>();
120+
for (int i = 0; i < deferredFilters.size();) {
121+
DeferredFilter deferredFilter = deferredFilters.get(i);
122+
if (!canApplyDeferredFilterToTupleExpr(deferredFilter, tupleExpr)) {
123+
i++;
124+
continue;
125+
}
126+
localFilters.add(deferredFilter);
127+
deferredFilters.remove(i);
128+
}
129+
return localFilters.isEmpty() ? tupleExpr : filterWrapper.wrap(tupleExpr, localFilters, "localPattern");
130+
}
131+
132+
private boolean canApplyDeferredFilterToTupleExpr(DeferredFilter deferredFilter, TupleExpr tupleExpr) {
133+
return tupleExpr instanceof StatementPattern
134+
&& deferredFilter.conditionCost == JoinOrderPlanner.FILTER_COST_CHEAP
135+
&& deferredFilter.patternLocalBase == tupleExpr;
136+
}
137+
114138
private boolean hasPendingSplitExistsFilter(List<DeferredFilter> deferredFilters, DeferredFilter candidate,
115139
Set<String> availableNames, Set<String> assignmentBindingNames) {
116140
if (LmdbJoinPlanSupport.containsExists(candidate.condition)) {

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSketchJoinOptimizer.java‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,28 @@
2222
import java.util.Optional;
2323
import java.util.Set;
2424

25+
import org.eclipse.rdf4j.model.Literal;
26+
import org.eclipse.rdf4j.model.Value;
27+
import org.eclipse.rdf4j.model.vocabulary.FN;
28+
import org.eclipse.rdf4j.model.vocabulary.XMLSchema;
2529
import org.eclipse.rdf4j.query.BindingSet;
2630
import org.eclipse.rdf4j.query.Dataset;
2731
import org.eclipse.rdf4j.query.algebra.AbstractQueryModelNode;
32+
import org.eclipse.rdf4j.query.algebra.And;
33+
import org.eclipse.rdf4j.query.algebra.Difference;
2834
import org.eclipse.rdf4j.query.algebra.Filter;
35+
import org.eclipse.rdf4j.query.algebra.FunctionCall;
2936
import org.eclipse.rdf4j.query.algebra.Join;
3037
import org.eclipse.rdf4j.query.algebra.LeftJoin;
38+
import org.eclipse.rdf4j.query.algebra.Not;
3139
import org.eclipse.rdf4j.query.algebra.QueryRoot;
40+
import org.eclipse.rdf4j.query.algebra.StatementPattern;
41+
import org.eclipse.rdf4j.query.algebra.Str;
3242
import org.eclipse.rdf4j.query.algebra.TupleExpr;
43+
import org.eclipse.rdf4j.query.algebra.UnaryTupleOperator;
44+
import org.eclipse.rdf4j.query.algebra.ValueConstant;
3345
import org.eclipse.rdf4j.query.algebra.ValueExpr;
46+
import org.eclipse.rdf4j.query.algebra.Var;
3447
import org.eclipse.rdf4j.query.algebra.evaluation.QueryOptimizer;
3548
import org.eclipse.rdf4j.query.algebra.evaluation.impl.EvaluationStatistics;
3649
import org.eclipse.rdf4j.query.algebra.evaluation.optimizer.JoinOrderPlanner;
@@ -107,11 +120,114 @@ public void meet(LeftJoin leftJoin) {
107120
}
108121
}
109122

123+
@Override
124+
public void meet(Difference difference) {
125+
TupleExpr replacement = rewriteRedundantPatternMinusFilter(difference);
126+
if (replacement != null) {
127+
replacement.visit(this);
128+
return;
129+
}
130+
131+
difference.getLeftArg().visit(this);
132+
difference.getRightArg().visit(this);
133+
}
134+
110135
@Override
111136
public void meet(Join join) {
112137
optimizeJoinReplacement(join, join, List.of());
113138
}
114139

140+
private TupleExpr rewriteRedundantPatternMinusFilter(Difference difference) {
141+
if (!(difference.getRightArg() instanceof Filter)) {
142+
return null;
143+
}
144+
145+
Filter rightFilter = (Filter) difference.getRightArg();
146+
if (!(rightFilter.getArg() instanceof StatementPattern)) {
147+
return null;
148+
}
149+
150+
StatementPattern rightPattern = (StatementPattern) rightFilter.getArg();
151+
Set<String> conditionVars = VarNameCollector.process(rightFilter.getCondition());
152+
if (!rightPattern.getBindingNames().containsAll(conditionVars)
153+
|| !difference.getLeftArg().getAssuredBindingNames().containsAll(conditionVars)
154+
|| !isSafeTotalMinusLocalCondition(rightFilter.getCondition(), conditionVars)
155+
|| !containsEquivalentRequiredPattern(difference.getLeftArg(), rightPattern)) {
156+
return null;
157+
}
158+
159+
Filter replacement = new Filter(difference.getLeftArg().clone(),
160+
new Not(rightFilter.getCondition().clone()));
161+
difference.replaceWith(replacement);
162+
return replacement;
163+
}
164+
165+
private boolean containsEquivalentRequiredPattern(TupleExpr tupleExpr, StatementPattern expectedPattern) {
166+
if (tupleExpr instanceof StatementPattern) {
167+
return sameStatementPattern((StatementPattern) tupleExpr, expectedPattern);
168+
}
169+
if (tupleExpr instanceof Join) {
170+
Join join = (Join) tupleExpr;
171+
return containsEquivalentRequiredPattern(join.getLeftArg(), expectedPattern)
172+
|| containsEquivalentRequiredPattern(join.getRightArg(), expectedPattern);
173+
}
174+
if (tupleExpr instanceof LeftJoin) {
175+
return containsEquivalentRequiredPattern(((LeftJoin) tupleExpr).getLeftArg(), expectedPattern);
176+
}
177+
if (tupleExpr instanceof Difference) {
178+
return containsEquivalentRequiredPattern(((Difference) tupleExpr).getLeftArg(), expectedPattern);
179+
}
180+
if (tupleExpr instanceof UnaryTupleOperator) {
181+
return containsEquivalentRequiredPattern(((UnaryTupleOperator) tupleExpr).getArg(), expectedPattern);
182+
}
183+
return false;
184+
}
185+
186+
private boolean sameStatementPattern(StatementPattern left, StatementPattern right) {
187+
return samePatternVar(left.getSubjectVar(), right.getSubjectVar())
188+
&& samePatternVar(left.getPredicateVar(), right.getPredicateVar())
189+
&& samePatternVar(left.getObjectVar(), right.getObjectVar())
190+
&& samePatternVar(left.getContextVar(), right.getContextVar());
191+
}
192+
193+
private boolean samePatternVar(Var left, Var right) {
194+
if (left == null || right == null) {
195+
return left == right;
196+
}
197+
if (left.hasValue() || right.hasValue()) {
198+
return left.hasValue() && right.hasValue() && left.getValue().equals(right.getValue());
199+
}
200+
return left.getName() != null && left.getName().equals(right.getName());
201+
}
202+
203+
private boolean isSafeTotalMinusLocalCondition(ValueExpr condition, Set<String> assuredConditionVars) {
204+
if (condition instanceof And) {
205+
And and = (And) condition;
206+
return isSafeTotalMinusLocalCondition(and.getLeftArg(), assuredConditionVars)
207+
&& isSafeTotalMinusLocalCondition(and.getRightArg(), assuredConditionVars);
208+
}
209+
if (condition instanceof FunctionCall) {
210+
FunctionCall functionCall = (FunctionCall) condition;
211+
return FN.CONTAINS.stringValue().equals(functionCall.getURI()) && functionCall.getArgs().size() == 2
212+
&& isSafeStringExpression(functionCall.getArgs().get(0), assuredConditionVars)
213+
&& isSafeStringExpression(functionCall.getArgs().get(1), assuredConditionVars);
214+
}
215+
return false;
216+
}
217+
218+
private boolean isSafeStringExpression(ValueExpr expression, Set<String> assuredConditionVars) {
219+
if (expression instanceof Str) {
220+
ValueExpr arg = ((Str) expression).getArg();
221+
return arg instanceof Var && assuredConditionVars.contains(((Var) arg).getName());
222+
}
223+
if (expression instanceof ValueConstant) {
224+
Value value = ((ValueConstant) expression).getValue();
225+
return value instanceof Literal && !((Literal) value).getLanguage().isPresent()
226+
&& XMLSchema.STRING.equals(((Literal) value).getDatatype());
227+
}
228+
return false;
229+
}
230+
115231
private void optimizeJoinReplacement(TupleExpr replaceTarget, Join join, List<Filter> additionalFilters) {
116232
Set<String> originalBoundVars = boundVars;
117233
try {
@@ -148,6 +264,9 @@ private void collectJoinArgs(TupleExpr tupleExpr, CollectedJoinArgs collected) {
148264
if (filterArg instanceof Join) {
149265
collectJoinArgs(filterArg, collected);
150266
collected.deferredFilters.addAll(filters);
267+
} else if (canDeferSinglePatternFilter(filterArg)) {
268+
collected.joinArgs.add(filterArg);
269+
collected.deferredFilters.addAll(filters);
151270
} else {
152271
collected.joinArgs.add(tupleExpr);
153272
}
@@ -156,6 +275,11 @@ private void collectJoinArgs(TupleExpr tupleExpr, CollectedJoinArgs collected) {
156275
}
157276
}
158277

278+
private boolean canDeferSinglePatternFilter(TupleExpr filterArg) {
279+
return !LmdbJoinPlanSupport.isJoinOrderSeparator(filterArg)
280+
&& LmdbJoinPlanSupport.collectPatternIdentities(filterArg).size() == 1;
281+
}
282+
159283
private TupleExpr unwrapJoinFilterChain(Filter filter, List<Filter> filters) {
160284
TupleExpr current = filter;
161285
while (current instanceof Filter && !TupleExprs.isVariableScopeChange(current)) {
@@ -353,6 +477,7 @@ private TupleExpr applyFilter(TupleExpr root, DeferredFilter deferredFilter, Str
353477
if (LmdbJoinPlanSupport.isValidPassRatio(passRatio)) {
354478
filter.setDoubleMetricPlanned(TelemetryMetricNames.PLANNED_FILTER_PASS_RATIO, passRatio);
355479
}
480+
filter.setStringMetricPlanned(TelemetryMetricNames.DEFERRED_FILTER_SCOPE, placement);
356481
filter.setStringMetricPlanned(TelemetryMetricNames.OPTIMIZER_STRATEGY, "lmdb-sketch-filter-" + placement);
357482
filter.setStringMetricPlanned(TelemetryMetricNames.FILTER_SELECTIVITY_SOURCE,
358483
deferredFilter.passEstimate.getSource().name().toLowerCase());

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSmallLiteralFilterAnchors.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,6 @@ static void add(List<TupleExpr> joinArgs, List<DeferredFilter> filters) {
2929
return;
3030
}
3131
for (DeferredFilter filter : filters) {
32-
if (filter.patternLocalBase != null) {
33-
continue;
34-
}
3532
BindingSetAssignment anchor = LmdbJoinPlanSupport.smallLiteralFilterAnchor(filter.condition);
3633
if (anchor == null) {
3734
continue;

‎core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/benchmark/LmdbEngineeringThemeQueryRegressionTest.java‎

Lines changed: 79 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,36 @@ void assemblyOptionalMinusUsesDevelopPlanShape(@TempDir Path dataDir) throws Exc
110110
}
111111
}
112112

113+
@Test
114+
void componentNameFilterUsesFiniteValuesAnchor(@TempDir Path dataDir) throws Exception {
115+
Theme theme = Theme.ENGINEERING;
116+
Path themeDir = dataDir.resolve(theme.name());
117+
try {
118+
LmdbStore store = new LmdbStore(themeDir.toFile(), ConfigUtil.createConfig());
119+
SailRepository repository = new SailRepository(store);
120+
try {
121+
BenchmarkJoinEstimatorSupport.prepareEstimatorForBulkLoad(repository, store);
122+
loadData(repository, theme);
123+
BenchmarkJoinEstimatorSupport.persistEstimatorAfterBulkLoad(repository, store);
124+
primeLearnedFilterStats(repository, theme, 4);
125+
BenchmarkJoinEstimatorSupport.persistStoreStatistics(store);
126+
} finally {
127+
shutdownAndRelease(repository, store);
128+
}
129+
130+
store = new LmdbStore(themeDir.toFile(), ConfigUtil.createConfig());
131+
repository = new SailRepository(store);
132+
try {
133+
OptimizerSnapshot snapshot = explainOptimized(repository, theme, 4);
134+
assertEngineeringQ4FastPlanShape(snapshot.renderedQuery().trim(), snapshot.plan());
135+
} finally {
136+
shutdownAndRelease(repository, store);
137+
}
138+
} finally {
139+
BenchmarkJoinEstimatorSupport.deleteStoreDirectory(themeDir);
140+
}
141+
}
142+
113143
private static void loadData(SailRepository repository, Theme theme) throws IOException {
114144
try (SailRepositoryConnection connection = repository.getConnection()) {
115145
connection.begin(IsolationLevels.NONE);
@@ -160,18 +190,26 @@ private static void shutdownAndRelease(SailRepository repository, LmdbStore stor
160190
private static void assertEngineeringQ7DevelopPlanShape(String renderedQuery, String plan) {
161191
assertContains(renderedQuery, "VALUES ?name { \"REQ-1000\" \"REQ-1001\" }");
162192
assertBefore(renderedQuery,
163-
"?requirement a <http://example.com/theme/engineering/Requirement> .",
193+
"VALUES ?name { \"REQ-1000\" \"REQ-1001\" }",
164194
"?requirement <http://example.com/theme/engineering/name> ?name .",
165-
"Engineering q7 should keep the Requirement rdf:type anchor before the bound name filter\n" + plan);
195+
"Engineering q7 should bind the finite requirement-name set before the name lookup\n" + plan);
196+
assertBefore(renderedQuery,
197+
"?requirement <http://example.com/theme/engineering/name> ?name .",
198+
"?requirement a <http://example.com/theme/engineering/Requirement> .",
199+
"Engineering q7 should use the bound name lookup before the rdf:type direct lookup\n" + plan);
166200
assertBefore(renderedQuery,
167201
"?requirement <http://example.com/theme/engineering/name> ?name .",
168202
"FILTER EXISTS",
169203
"Engineering q7 should apply the name filter before evaluating EXISTS\n" + plan);
170204
assertBefore(plan,
171-
"value=http://www.w3.org/1999/02/22-rdf-syntax-ns#type",
205+
"BindingSetAssignment ([[name=\"REQ-1000\"], [name=\"REQ-1001\"]])",
206+
"value=http://example.com/theme/engineering/name",
207+
"Engineering q7 should anchor the finite name set before the name access");
208+
assertBefore(plan,
172209
"value=http://example.com/theme/engineering/name",
173-
"Engineering q7 plan should keep the Requirement rdf:type anchor before the name filter");
174-
assertNamePatternUsesBoundSubject(plan, "Engineering q7", "requirement");
210+
"value=http://www.w3.org/1999/02/22-rdf-syntax-ns#type",
211+
"Engineering q7 plan should use the bound name lookup before the rdf:type direct lookup");
212+
assertNamePatternUsesBoundValue(plan, "Engineering q7", false);
175213
assertContains(plan, "ValueConstant (value=\"REQ-1000\")");
176214
}
177215

@@ -189,6 +227,23 @@ private static void assertEngineeringQ10DevelopPlanShape(String plan) {
189227
assertMinusSatisfiesStaysNewScope(plan);
190228
}
191229

230+
private static void assertEngineeringQ4FastPlanShape(String renderedQuery, String plan) {
231+
assertContains(renderedQuery, "VALUES ?name { \"Component 1\" \"Component 2\" }");
232+
assertBefore(renderedQuery,
233+
"VALUES ?name { \"Component 1\" \"Component 2\" }",
234+
"?component <http://example.com/theme/engineering/name> ?name .",
235+
"Engineering q4 should bind the finite component-name set before the name lookup\n" + plan);
236+
assertBefore(renderedQuery,
237+
"?component <http://example.com/theme/engineering/name> ?name .",
238+
"?component a <http://example.com/theme/engineering/Component> .",
239+
"Engineering q4 should use the bound name lookup before broad rdf:type scan\n" + plan);
240+
assertBefore(plan,
241+
"BindingSetAssignment ([[name=\"Component 1\"], [name=\"Component 2\"]])",
242+
"value=http://example.com/theme/engineering/name",
243+
"Engineering q4 should anchor the finite name set before the name access");
244+
assertNamePatternUsesBoundValue(plan, "Engineering q4", false);
245+
}
246+
192247
private static void assertDevelopOperatorSkeleton(String plan) {
193248
assertOrdered(plan,
194249
"Projection",
@@ -203,7 +258,7 @@ private static void assertDevelopOperatorSkeleton(String plan) {
203258
"LeftJoin",
204259
"Join (JoinIterator)",
205260
"BindingSetAssignment ([[name=\"Assembly 1\"], [name=\"Assembly 2\"]])",
206-
"deferredFilterScope=localPattern) [right]",
261+
"deferredFilterScope=localPattern)",
207262
"Or",
208263
"ValueConstant (value=\"Assembly 1\")",
209264
"ValueConstant (value=\"Assembly 2\")",
@@ -221,15 +276,21 @@ private static void assertDevelopOperatorSkeleton(String plan) {
221276
}
222277

223278
private static void assertNamePatternUsesBoundValue(String plan, String label) {
279+
assertNamePatternUsesBoundValue(plan, label, true);
280+
}
281+
282+
private static void assertNamePatternUsesBoundValue(String plan, String label, boolean requirePlannerMetrics) {
224283
int predicateIndex = plan.indexOf("value=http://example.com/theme/engineering/name");
225284
if (predicateIndex < 0) {
226285
throw new AssertionError(label + " plan should include the name pattern:\n" + plan);
227286
}
228287

229288
String pattern = statementPatternWindow(plan, predicateIndex, "StatementPattern");
230289
assertContains(pattern, "o: Var (name=name) (bindingState=bound)");
231-
assertContains(pattern, "plannedBoundVars=[name]");
232-
assertContains(pattern, "plannedLookupComponents=[P, O]");
290+
if (requirePlannerMetrics) {
291+
assertContainsAny(pattern, "plannedBoundVars=[name]", "plannedBoundVars=name");
292+
assertContains(pattern, "plannedLookupComponents=[P, O]");
293+
}
233294
}
234295

235296
private static void assertNamePatternUsesBoundSubject(String plan, String label, String subjectName) {
@@ -284,6 +345,16 @@ private static void assertContains(String value, String expected) {
284345
}
285346
}
286347

348+
private static void assertContainsAny(String value, String... expectedTokens) {
349+
for (String expected : expectedTokens) {
350+
if (value.contains(expected)) {
351+
return;
352+
}
353+
}
354+
throw new AssertionError("Expected to find one of `" + String.join("`, `", expectedTokens) + "` in:\n"
355+
+ value);
356+
}
357+
287358
private static void assertBefore(String value, String first, String second, String message) {
288359
int firstIndex = value.indexOf(first);
289360
int secondIndex = value.indexOf(second);

0 commit comments

Comments
 (0)