Skip to content

Commit 2918c14

Browse files
ikholopov-omniikholopov-omni
authored
JiT contexts (#2088)
Implementation of context objects accessible at JiT compilation stage for supported action types. They expose access to adapter/data and context methods, equivalent to AoT context, but with a more restrictive selection of resolvable targets.
1 parent cc92f4a commit 2918c14

5 files changed

Lines changed: 300 additions & 78 deletions

File tree

‎core/BUILD‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ ts_library(
3333
"contextables.ts",
3434
"extension.ts",
3535
"index.ts",
36+
"jit_context.ts",
3637
"main.ts",
3738
"path.ts",
3839
"session.ts",
@@ -76,6 +77,7 @@ ts_test_suite(
7677
"actions/table_test.ts",
7778
"actions/test_test.ts",
7879
"actions/view_test.ts",
80+
"jit_context_test.ts",
7981
],
8082
data = [
8183
":node_modules",

‎core/jit_context.ts‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
import { IActionContext, ITableContext, JitContext, Resolvable } from "df/core/contextables";
2+
import { ambiguousActionNameMsg, resolvableAsTarget, ResolvableMap, stringifyResolvable, toResolvable } from "df/core/utils";
3+
import { dataform } from "df/protos/ts";
4+
5+
function canonicalTargetValue(target: dataform.ITarget): string {
6+
return `${target.database}.${target.schema}.${target.name}`;
7+
}
8+
9+
function schemaTargetValue(target: dataform.ITarget): string {
10+
return `${target.schema}.${target.name}`;
11+
}
12+
13+
/** Generate SQL action JiT context. */
14+
export class SqlActionJitContext implements JitContext<IActionContext> {
15+
private readonly resolvableMap: ResolvableMap<string>;
16+
17+
constructor(
18+
public readonly adapter: dataform.DbAdapter,
19+
public readonly data: { [k: string]: any } | undefined,
20+
private readonly target: dataform.ITarget,
21+
dependencies: dataform.ITarget[],
22+
) {
23+
const resolvables = [target, ...dependencies];
24+
this.resolvableMap = new ResolvableMap(resolvables.map(dep => ({
25+
actionTarget: dep,
26+
value: canonicalTargetValue(dep)
27+
})));
28+
}
29+
30+
public self(): string {
31+
return this.resolve(this.name());
32+
}
33+
34+
public name(): string {
35+
return this.target.name;
36+
}
37+
38+
public ref(ref: Resolvable | string[], ...rest: string[]): string {
39+
return this.resolve(ref, ...rest);
40+
}
41+
42+
public resolve(ref: Resolvable | string[], ...rest: string[]): string {
43+
ref = toResolvable(ref, rest);
44+
const refTarget = resolvableAsTarget(ref);
45+
const candidates = this.resolvableMap.find(refTarget);
46+
47+
if (candidates.length > 1) {
48+
throw new Error(ambiguousActionNameMsg(ref, candidates));
49+
}
50+
51+
return this.resolveReference(
52+
stringifyResolvable(ref),
53+
candidates.length > 0 ? candidates[0] : undefined);
54+
}
55+
56+
public schema(): string {
57+
return this.target.schema;
58+
}
59+
60+
public database(): string {
61+
return this.target.database;
62+
}
63+
64+
65+
private resolveReference(name: string, resolvedName?: string): string {
66+
if (!!resolvedName) {
67+
return `\`${resolvedName}\``;
68+
}
69+
throw new Error(`Undeclared dependency referenced: ${name}.\n` +
70+
"JiT action must have its dependencies declared explicitly.");
71+
}
72+
}
73+
74+
/** JiT context for table and view actions. */
75+
export class TableJitContext extends SqlActionJitContext implements JitContext<ITableContext> {
76+
constructor(
77+
adapter: dataform.DbAdapter,
78+
data: { [k: string]: any } | undefined,
79+
target: dataform.ITarget,
80+
dependencies: dataform.ITarget[],
81+
) {
82+
super(adapter, data, target, dependencies);
83+
}
84+
85+
public when(cond: boolean, trueCase: string, falseCase?: string) {
86+
return cond ? trueCase : falseCase || "";
87+
}
88+
89+
public incremental(): boolean {
90+
return false;
91+
}
92+
}
93+
94+
/** JiT context for incremental table actions. */
95+
export class IncrementalTableJitContext extends TableJitContext {
96+
constructor(adapter: dataform.DbAdapter,
97+
data: { [k: string]: any } | undefined,
98+
target: dataform.ITarget,
99+
dependencies: dataform.ITarget[],
100+
private readonly isIncrementalContext: boolean,
101+
) {
102+
super(adapter, data, target, dependencies);
103+
}
104+
105+
public incremental(): boolean {
106+
return this.isIncrementalContext;
107+
}
108+
}

‎core/jit_context_test.ts‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
import { expect } from "chai";
2+
3+
import { IncrementalTableJitContext, SqlActionJitContext, TableJitContext } from "df/core/jit_context";
4+
import { dataform } from "df/protos/ts";
5+
import { suite, test } from "df/testing";
6+
7+
suite("jit_context", () => {
8+
suite("SqlActionJitContext", () => {
9+
const adapter = {} as dataform.DbAdapter;
10+
const target = dataform.Target.create({
11+
database: "db",
12+
schema: "schema",
13+
name: "name"
14+
});
15+
const dependency = dataform.Target.create({
16+
database: "db",
17+
schema: "schema",
18+
name: "dep"
19+
});
20+
21+
test("resolve by name only", () => {
22+
const context = new SqlActionJitContext(adapter, {}, target, [dependency]);
23+
expect(context.resolve("dep")).to.equal("`db.schema.dep`");
24+
});
25+
26+
test("resolve by schema and name", () => {
27+
const context = new SqlActionJitContext(adapter, {}, target, [dependency]);
28+
expect(context.resolve(["schema", "dep"])).to.equal("`db.schema.dep`");
29+
});
30+
31+
test("resolve by database, schema and name", () => {
32+
const context = new SqlActionJitContext(adapter, {}, target, [dependency]);
33+
expect(context.resolve(["db", "schema", "dep"])).to.equal("`db.schema.dep`");
34+
});
35+
36+
test("resolve throws for undeclared dependency", () => {
37+
const context = new SqlActionJitContext(adapter, {}, target, []);
38+
expect(() => context.resolve("dep")).to.throw("Undeclared dependency referenced: dep");
39+
});
40+
41+
test("ref", () => {
42+
const context = new SqlActionJitContext(adapter, {}, target, [dependency]);
43+
expect(context.ref("dep")).to.equal("`db.schema.dep`");
44+
});
45+
46+
test("self", () => {
47+
const context = new SqlActionJitContext(adapter, {}, target, []);
48+
expect(context.self()).to.equal("`db.schema.name`");
49+
});
50+
51+
test("name", () => {
52+
const context = new SqlActionJitContext(adapter, {}, target, []);
53+
expect(context.name()).to.equal("name");
54+
});
55+
56+
test("schema", () => {
57+
const context = new SqlActionJitContext(adapter, {}, target, []);
58+
expect(context.schema()).to.equal("schema");
59+
});
60+
61+
test("database", () => {
62+
const context = new SqlActionJitContext(adapter, {}, target, []);
63+
expect(context.database()).to.equal("db");
64+
});
65+
});
66+
67+
suite("TableJitContext", () => {
68+
const adapter = {} as dataform.DbAdapter;
69+
const target = dataform.Target.create({
70+
database: "db",
71+
schema: "schema",
72+
name: "name"
73+
});
74+
75+
test("when", () => {
76+
const context = new TableJitContext(adapter, {}, target, []);
77+
expect(context.when(true, "true", "false")).to.equal("true");
78+
expect(context.when(false, "true", "false")).to.equal("false");
79+
expect(context.when(false, "true")).to.equal("");
80+
});
81+
82+
test("incremental", () => {
83+
const context = new TableJitContext(adapter, {}, target, []);
84+
expect(context.incremental()).to.equal(false);
85+
});
86+
});
87+
88+
suite("IncrementalTableJitContext", () => {
89+
const adapter = {} as dataform.DbAdapter;
90+
const target = dataform.Target.create({
91+
database: "db",
92+
schema: "schema",
93+
name: "name"
94+
});
95+
96+
test("incremental", () => {
97+
let context = new IncrementalTableJitContext(adapter, {}, target, [], true);
98+
expect(context.incremental()).to.equal(true);
99+
100+
context = new IncrementalTableJitContext(adapter, {}, target, [], false);
101+
expect(context.incremental()).to.equal(false);
102+
});
103+
});
104+
});

‎core/session.ts‎

Lines changed: 6 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import { CompilationSql } from "df/core/compilation_sql";
1818
import { Contextable, IActionContext, ITableContext, Resolvable } from "df/core/contextables";
1919
import { targetAsReadableString, targetStringifier } from "df/core/targets";
2020
import * as utils from "df/core/utils";
21-
import { toResolvable } from "df/core/utils";
21+
import { ResolvableMap, toResolvable } from "df/core/utils";
2222
import { version as dataformCoreVersion } from "df/core/version";
2323
import { dataform, google } from "df/protos/ts";
2424

@@ -49,12 +49,12 @@ export class Session {
4949
public canonicalProjectConfig: dataform.ProjectConfig;
5050

5151
public actions: Action[];
52-
public indexedActions: ActionMap;
52+
public indexedActions: ResolvableMap<Action>;
5353
public tests: { [name: string]: Test };
5454

5555
// This map holds information about what assertions are dependent
5656
// upon a certain action in our actions list. We use this later to resolve dependencies.
57-
public actionAssertionMap = new ActionMap([]);
57+
public actionAssertionMap = new ResolvableMap<Action>();
5858

5959
public graphErrors: dataform.IGraphErrors;
6060

@@ -473,7 +473,9 @@ export class Session {
473473
}
474474

475475
public compile(): dataform.CompiledGraph {
476-
this.indexedActions = new ActionMap(this.actions);
476+
this.indexedActions = new ResolvableMap(
477+
this.actions.map(action => ({ actionTarget: action.getTarget(), value: action }))
478+
);
477479

478480
// defaultLocation is no longer a required parameter to support location auto-selection.
479481

@@ -821,77 +823,3 @@ function getCanonicalProjectConfig(originalProjectConfig: dataform.ProjectConfig
821823
assertionSchema: originalProjectConfig.assertionSchema
822824
});
823825
}
824-
825-
class ActionMap {
826-
private byName: Map<string, Action[]> = new Map();
827-
private bySchemaAndName: Map<string, Map<string, Action[]>> = new Map();
828-
private byDatabaseAndName: Map<string, Map<string, Action[]>> = new Map();
829-
private byDatabaseSchemaAndName: Map<string, Map<string, Map<string, Action[]>>> = new Map();
830-
831-
public constructor(actions: Action[]) {
832-
for (const action of actions) {
833-
this.set(action.getTarget(), action);
834-
}
835-
}
836-
837-
public set(actionTarget: dataform.ITarget, assertionTarget: Action) {
838-
this.setByNameLevel(this.byName, actionTarget.name, assertionTarget);
839-
840-
if (!!actionTarget.schema) {
841-
this.setBySchemaLevel(this.bySchemaAndName, actionTarget, assertionTarget);
842-
}
843-
844-
if (!!actionTarget.database) {
845-
if (!this.byDatabaseAndName.has(actionTarget.database)) {
846-
this.byDatabaseAndName.set(actionTarget.database, new Map());
847-
}
848-
const forDatabaseNoSchema = this.byDatabaseAndName.get(actionTarget.database);
849-
this.setByNameLevel(forDatabaseNoSchema, actionTarget.name, assertionTarget);
850-
851-
if (!!actionTarget.schema) {
852-
if (!this.byDatabaseSchemaAndName.has(actionTarget.database)) {
853-
this.byDatabaseSchemaAndName.set(actionTarget.database, new Map());
854-
}
855-
const forDatabase = this.byDatabaseSchemaAndName.get(actionTarget.database);
856-
this.setBySchemaLevel(forDatabase, actionTarget, assertionTarget);
857-
}
858-
}
859-
}
860-
861-
public find(target: dataform.ITarget) {
862-
if (!!target.database) {
863-
if (!!target.schema) {
864-
return (
865-
this.byDatabaseSchemaAndName
866-
.get(target.database)
867-
?.get(target.schema)
868-
?.get(target.name) || []
869-
);
870-
}
871-
return this.byDatabaseAndName.get(target.database)?.get(target.name) || [];
872-
}
873-
if (!!target.schema) {
874-
return this.bySchemaAndName.get(target.schema)?.get(target.name) || [];
875-
}
876-
return this.byName.get(target.name) || [];
877-
}
878-
879-
private setByNameLevel(targetMap: Map<string, Action[]>, name: string, assertionTarget: Action) {
880-
if (!targetMap.has(name)) {
881-
targetMap.set(name, []);
882-
}
883-
targetMap.get(name).push(assertionTarget);
884-
}
885-
886-
private setBySchemaLevel(
887-
targetMap: Map<string, Map<string, Action[]>>,
888-
actionTarget: dataform.ITarget,
889-
assertionTarget: Action
890-
) {
891-
if (!targetMap.has(actionTarget.schema)) {
892-
targetMap.set(actionTarget.schema, new Map());
893-
}
894-
const forSchema = targetMap.get(actionTarget.schema);
895-
this.setByNameLevel(forSchema, actionTarget.name, assertionTarget);
896-
}
897-
}

0 commit comments

Comments
 (0)