Skip to content

Commit bb35744

Browse files
ikholopov-omniikholopov-omni
authored
JiT context RPC service and protos (#2079)
Protos for JiT DB adapter and JiT compilation messages.
1 parent 91d80dd commit bb35744

5 files changed

Lines changed: 215 additions & 86 deletions

File tree

‎packages/@dataform/core/BUILD‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,18 @@ add_license_header_to_file(
6767
to_file = "configs.proto",
6868
)
6969

70+
add_license_header_to_file(
71+
name = "db_adapter_proto_with_license",
72+
from_file = "//protos:db_adapter.proto",
73+
to_file = "db_adapter.proto",
74+
)
75+
76+
add_license_header_to_file(
77+
name = "jit_proto_with_license",
78+
from_file = "//protos:jit.proto",
79+
to_file = "jit.proto",
80+
)
81+
7082
add_license_header_to_file(
7183
name = "bundle_with_license",
7284
from_file = "bundle",
@@ -95,6 +107,8 @@ pkg_npm_tar(
95107
":bundle_with_license",
96108
":configs_proto_with_license",
97109
":core_proto_with_license",
110+
":db_adapter_proto_with_license",
111+
":jit_proto_with_license",
98112
":package.json",
99113
"//:version.bzl",
100114
],

‎protos/BUILD‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
load("@rules_proto//proto:defs.bzl", "proto_library")
2-
load("//testing:build_test.bzl", "build_test")
32
load("//tools:ts_proto_library.bzl", "ts_proto_library")
43

54
package(default_visibility = ["//visibility:public"])
@@ -9,11 +8,16 @@ proto_library(
98
srcs = [
109
"configs.proto",
1110
"core.proto",
11+
"db_adapter.proto",
1212
"evaluation.proto",
1313
"execution.proto",
14+
"jit.proto",
1415
"profiles.proto",
1516
],
16-
deps = ["@com_google_protobuf//:struct_proto"],
17+
deps = [
18+
"@com_google_protobuf//:empty_proto",
19+
"@com_google_protobuf//:struct_proto",
20+
],
1721
)
1822

1923
ts_proto_library(

‎protos/db_adapter.proto‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
syntax = "proto3";
2+
3+
package dataform;
4+
5+
import "protos/core.proto";
6+
7+
// Common structures for Evaluation, execution and compilation.
8+
9+
message Field {
10+
enum Primitive {
11+
UNKNOWN = 0;
12+
INTEGER = 1;
13+
FLOAT = 2;
14+
NUMERIC = 5;
15+
BOOLEAN = 3;
16+
STRING = 4;
17+
DATE = 6;
18+
DATETIME = 7;
19+
TIMESTAMP = 8;
20+
TIME = 9;
21+
BYTES = 10;
22+
ANY = 11;
23+
GEOGRAPHY = 12;
24+
}
25+
26+
enum Flag {
27+
UNKNOWN_FLAG = 0;
28+
REPEATED = 1;
29+
}
30+
31+
string name = 1;
32+
33+
repeated Flag flags = 6;
34+
35+
oneof type {
36+
Primitive primitive = 7;
37+
Fields struct = 3;
38+
}
39+
40+
string description = 5;
41+
42+
reserved 2, 4;
43+
}
44+
45+
message Fields {
46+
repeated Field fields = 1;
47+
}
48+
49+
message TableMetadata {
50+
Target target = 1;
51+
enum Type {
52+
UNKNOWN = 0;
53+
TABLE = 1;
54+
VIEW = 2;
55+
}
56+
Type type = 6;
57+
58+
repeated Field fields = 3;
59+
string description = 5;
60+
map<string, string> labels = 7;
61+
int64 last_updated_millis = 4;
62+
63+
message BigQuery {
64+
bool has_streaming_buffer = 1;
65+
}
66+
BigQuery bigquery = 8;
67+
68+
reserved 2;
69+
}
70+
71+
message ExecutionMetadata {
72+
message BigqueryMetadata {
73+
string job_id = 1;
74+
int64 total_bytes_processed = 2;
75+
int64 total_bytes_billed = 3;
76+
}
77+
BigqueryMetadata bigquery = 1;
78+
}

‎protos/execution.proto‎

Lines changed: 9 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
1-
syntax="proto3";
1+
syntax = "proto3";
22

33
package dataform;
44

55
import "protos/core.proto";
6+
import "protos/db_adapter.proto";
67

78
option go_package = "github.com/dataform-co/dataform/protos/dataform";
89

9-
1010
message RunConfig {
1111
repeated string actions = 1;
1212
repeated string tags = 5;
1313
bool include_dependencies = 3;
1414
bool include_dependents = 11;
1515
bool full_refresh = 2;
1616
int32 timeout_millis = 7;
17-
17+
1818
// For internal use only, will be removed at a later date.
1919
bool disable_set_metadata = 9;
2020

@@ -36,7 +36,6 @@ message ExecutionAction {
3636
string type = 4;
3737
string table_type = 6;
3838

39-
4039
repeated Target dependency_targets = 11;
4140
ActionHermeticity hermeticity = 10;
4241

@@ -82,7 +81,6 @@ message RunResult {
8281
}
8382

8483
message ActionResult {
85-
8684
Target target = 5;
8785

8886
enum ExecutionStatus {
@@ -103,24 +101,14 @@ message ActionResult {
103101
reserved 1, 6, 7;
104102
}
105103

106-
107-
message ExecutionMetadata {
108-
message BigqueryMetadata {
109-
string job_id = 1;
110-
int64 total_bytes_processed = 2;
111-
int64 total_bytes_billed = 3;
112-
}
113-
BigqueryMetadata bigquery = 1;
114-
}
115-
116104
message TaskResult {
117105
enum ExecutionStatus {
118-
UNKNOWN = 0;
119-
RUNNING = 1;
120-
SUCCESSFUL = 2;
121-
FAILED = 3;
122-
SKIPPED = 4;
123-
CANCELLED = 5;
106+
UNKNOWN = 0;
107+
RUNNING = 1;
108+
SUCCESSFUL = 2;
109+
FAILED = 3;
110+
SKIPPED = 4;
111+
CANCELLED = 5;
124112
}
125113
ExecutionStatus status = 1;
126114
string error_message = 2;
@@ -133,66 +121,3 @@ message TestResult {
133121
bool successful = 2;
134122
repeated string messages = 3;
135123
}
136-
137-
message Field {
138-
139-
enum Primitive {
140-
UNKNOWN = 0;
141-
INTEGER = 1;
142-
FLOAT = 2;
143-
NUMERIC = 5;
144-
BOOLEAN = 3;
145-
STRING = 4;
146-
DATE = 6;
147-
DATETIME = 7;
148-
TIMESTAMP = 8;
149-
TIME = 9;
150-
BYTES = 10;
151-
ANY = 11;
152-
GEOGRAPHY = 12;
153-
}
154-
155-
enum Flag {
156-
UNKNOWN_FLAG = 0;
157-
REPEATED = 1;
158-
}
159-
160-
string name = 1;
161-
162-
repeated Flag flags = 6;
163-
164-
oneof type {
165-
Primitive primitive = 7;
166-
Fields struct = 3;
167-
}
168-
169-
string description = 5;
170-
171-
reserved 2, 4;
172-
}
173-
174-
message Fields {
175-
repeated Field fields = 1;
176-
}
177-
178-
message TableMetadata {
179-
Target target = 1;
180-
enum Type {
181-
UNKNOWN = 0;
182-
TABLE = 1;
183-
VIEW = 2;
184-
}
185-
Type type = 6;
186-
187-
repeated Field fields = 3;
188-
string description = 5;
189-
map<string, string> labels = 7;
190-
int64 last_updated_millis = 4;
191-
192-
message BigQuery {
193-
bool has_streaming_buffer = 1;
194-
}
195-
BigQuery bigquery = 8;
196-
197-
reserved 2;
198-
}

‎protos/jit.proto‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
syntax = "proto3";
2+
3+
package dataform;
4+
5+
import "google/protobuf/empty.proto";
6+
import "google/protobuf/struct.proto";
7+
import "protos/core.proto";
8+
import "protos/db_adapter.proto";
9+
10+
// Database adapter, available in JiT compilation context.
11+
service DbAdapter {
12+
// Executes a query.
13+
rpc Execute(ExecuteRequest) returns (ExecuteResponse) {}
14+
15+
// Lists table in schema.
16+
rpc ListTables(ListTablesRequest) returns (ListTablesResponse) {}
17+
18+
// Gets table for target.
19+
rpc GetTable(GetTableRequest) returns (TableMetadata) {}
20+
21+
// Drops table of specified target.
22+
rpc DeleteTable(DeleteTableRequest) returns (google.protobuf.Empty) {}
23+
}
24+
25+
// Request to execute synchronous SQL query.
26+
message ExecuteRequest {
27+
// SQL statement query.
28+
string statement = 1;
29+
// Max rows to return in the job.
30+
int64 row_limit = 2;
31+
// Max bytes to process.
32+
int64 byte_limit = 3;
33+
// Whether or not fetch result rows.
34+
bool fetch_results = 4;
35+
36+
BigQueryExecuteOptions big_query_options = 5;
37+
}
38+
39+
message BigQueryExecuteOptions {
40+
// Priority - interactive if true or batch if false (default).
41+
bool interactive = 1;
42+
// See https://docs.cloud.google.com/bigquery/docs/reference/legacy-sql.
43+
bool use_legacy_sql = 2;
44+
// Labels to add to the job.
45+
map<string, string> labels = 3;
46+
// Location to execute the job at.
47+
string location = 4;
48+
// Job prefix to add.
49+
string job_prefix = 5;
50+
// Is dry run job.
51+
bool dry_run = 6;
52+
}
53+
54+
// Synchronous execution response result.
55+
message ExecuteResponse {
56+
// Job metadata.
57+
ExecutionMetadata execution_metadata = 1;
58+
// Rows. For BigQuery, see
59+
// https://docs.cloud.google.com/bigquery/docs/reference/rest/v2/jobs/getQueryResults.
60+
repeated google.protobuf.Struct rows = 2;
61+
repeated string errors = 3;
62+
}
63+
64+
// Request to list tables in schema.
65+
message ListTablesRequest {
66+
string database = 1;
67+
string schema = 2;
68+
}
69+
70+
// Tables metadata in schema.
71+
message ListTablesResponse {
72+
repeated TableMetadata tables = 1;
73+
}
74+
75+
// Request to get table metadata for target.
76+
message GetTableRequest {
77+
Target target = 1;
78+
}
79+
80+
// Request to drop the table for target.
81+
message DeleteTableRequest {
82+
Target target = 1;
83+
}
84+
85+
// JiT compilation result for table actions (including views).
86+
// Fields match the Table message.
87+
message JitTableResult {
88+
// SQL Select query of the table.
89+
string query = 1;
90+
// Optional pre-operation statements.
91+
repeated string pre_ops = 3;
92+
// Optional post-operation statements.
93+
repeated string post_ops = 4;
94+
}
95+
96+
// JiT compilation result for incremental table actions.
97+
message JitIncrementalTableResult {
98+
// Fields match the Table message.
99+
JitTableResult regular = 1;
100+
// Fields match incremental_ fields in the Table message.
101+
JitTableResult incremental = 2;
102+
}
103+
104+
// JiT compilation result for operation actions.
105+
message JitOperationResult {
106+
// Sequence of SQL operations.
107+
repeated string queries = 1;
108+
}

0 commit comments

Comments
 (0)