Skip to content
This repository was archived by the owner on Jun 13, 2025. It is now read-only.

Commit 29bc03b

Browse files
author
DanLee
authored
Merge pull request #13 from dataform-co/dataform_move_to_dataform_core
Dataform move to dataform core
2 parents 547d21f + 7084b6a commit 29bc03b

7 files changed

Lines changed: 109 additions & 144 deletions

File tree

‎includes/crossdb.js‎

Lines changed: 0 additions & 59 deletions
This file was deleted.

‎includes/sessionized_events.js‎

Lines changed: 24 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
const crossdb = require("./crossdb");
1+
const sql = require("@dataform/sql")();
22
const segmentCommon = require("./common");
33

44
module.exports = (params) => {
@@ -42,16 +42,18 @@ select
4242
*,
4343
coalesce(
4444
(
45-
${crossdb.timestampDiff(`millisecond`,
46-
crossdb.windowFunction({
47-
func: "lag",
48-
value: "timestamp",
49-
ignore_nulls: false,
50-
partition_fields: "user_id",
51-
order_fields: "timestamp asc",
52-
frame_clause: " " // supplying empty frame clause as frame clause is not valid for a lag
53-
}),
54-
`segment_events_mapped.timestamp`
45+
${sql.timestamps.diff(`millisecond`,
46+
sql.windowFunction(
47+
"lag",
48+
"timestamp",
49+
false,
50+
{
51+
partitionFields: ["user_id"],
52+
orderFields: ["timestamp asc"]
53+
}
54+
),
55+
`
56+
segment_events_mapped.timestamp `
5557
)}
5658
) >= ${params.sessionTimeoutMillis},
5759
true
@@ -64,22 +66,24 @@ with_session_index as (
6466
-- add a session_index (users first session = 1, users second session = 2 etc)
6567
select
6668
*,
67-
${crossdb.windowFunction({
68-
func: "sum",
69-
value: "case when session_start_event then 1 else 0 end",
70-
ignore_nulls: false,
71-
partition_fields: "user_id",
72-
order_fields: 'session_starts.timestamp asc',
73-
frame_clause: "rows between unbounded preceding and current row"
74-
})} as session_index
69+
${sql.windowFunction(
70+
"sum",
71+
"case when session_start_event then 1 else 0 end",
72+
false,
73+
{
74+
partitionFields: ["user_id"],
75+
orderFields: ['session_starts.timestamp asc'],
76+
frameClause: "rows between unbounded preceding and current row"
77+
}
78+
)} as session_index
7579
from
7680
session_starts
7781
)
7882
7983
-- add a unique session_id to each session
8084
select
8185
*,
82-
${crossdb.generateSurrogateKey(["session_index", "user_id"])} as session_id
86+
${sql.surrogateKey(["session_index", "user_id"])} as session_id
8387
from
8488
with_session_index
8589

‎includes/sessions.js‎

Lines changed: 43 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
const segmentCommon = require("./common");
2-
const crossdb = require("./crossdb");
2+
const sql = require("@dataform/sql")();
33

44
module.exports = (params) => {
55

@@ -24,23 +24,27 @@ ${params.includePages ?
2424
select distinct
2525
session_id,
2626
${Object.entries(segmentCommon.allPageFields(params)).map(
27-
([key, value]) => `${crossdb.windowFunction({
28-
func: "first_value",
29-
value: value,
30-
ignore_nulls: false,
31-
partition_fields: "session_id",
32-
order_fields: 'sessionized_pages.timestamp asc',
33-
frame_clause: "rows between unbounded preceding and unbounded following",
34-
})} as first_${value}`).join(",\n ")},
27+
([key, value]) => `${sql.windowFunction(
28+
"first_value",
29+
value,
30+
false,
31+
{
32+
partitionFields: ["session_id"],
33+
orderFields: ['sessionized_pages.timestamp asc'],
34+
frameClause: "rows between unbounded preceding and unbounded following"
35+
}
36+
)} as first_${value}`).join(",\n ")},
3537
${Object.entries(segmentCommon.allPageFields(params)).map(
36-
([key, value]) => `${crossdb.windowFunction({
37-
func: "last_value",
38-
value: value,
39-
ignore_nulls: false,
40-
partition_fields: "session_id",
41-
order_fields: 'sessionized_pages.timestamp asc',
42-
frame_clause: "rows between unbounded preceding and unbounded following",
43-
})} as last_${value}`).join(",\n ")}
38+
([key, value]) => `${sql.windowFunction(
39+
"last_value",
40+
value,
41+
false,
42+
{
43+
partitionFields: ["session_id"],
44+
orderFields: ['sessionized_pages.timestamp asc'],
45+
frameClause: "rows between unbounded preceding and unbounded following"
46+
}
47+
)} as last_${value}`).join(",\n ")}
4448
from
4549
${ctx.ref(params.defaultConfig.schema, "segment_sessionized_pages")} as sessionized_pages
4650
),` : ``}
@@ -50,23 +54,27 @@ ${params.includeScreens ?
5054
select distinct
5155
session_id,
5256
${Object.entries(segmentCommon.allScreenFields(params)).map(
53-
([key, value]) => `${crossdb.windowFunction({
54-
func: "first_value",
55-
value: value,
56-
ignore_nulls: false,
57-
partition_fields: "session_id",
58-
order_fields: 'sessionized_screens.timestamp asc',
59-
frame_clause: "rows between unbounded preceding and unbounded following",
60-
})} as first_${value}`).join(",\n ")},
57+
([key, value]) => `${sql.windowFunction(
58+
"first_value",
59+
value,
60+
false,
61+
{
62+
partitionFields: ["session_id"],
63+
orderFields: ['sessionized_screens.timestamp asc'],
64+
frameClause: "rows between unbounded preceding and unbounded following"
65+
}
66+
)} as first_${value}`).join(",\n ")},
6167
${Object.entries(segmentCommon.allScreenFields(params)).map(
62-
([key, value]) => `${crossdb.windowFunction({
63-
func: "last_value",
64-
value: value,
65-
ignore_nulls: false,
66-
partition_fields: "session_id",
67-
order_fields: 'sessionized_screens.timestamp asc',
68-
frame_clause: "rows between unbounded preceding and unbounded following",
69-
})} as last_${value}`).join(",\n ")}
68+
([key, value]) => `${sql.windowFunction(
69+
"last_value",
70+
value,
71+
false,
72+
{
73+
partitionFields: ["session_id"],
74+
orderFields: ['sessionized_screens.timestamp asc'],
75+
frameClause: "rows between unbounded preceding and unbounded following"
76+
}
77+
)} as last_${value}`).join(",\n ")}
7078
from
7179
${ctx.ref(params.defaultConfig.schema, "segment_sessionized_screens")} as sessionized_screens
7280
),` : ``}
@@ -84,7 +92,7 @@ select
8492
${ctx.when(global.dataform.projectConfig.warehouse == "bigquery", `struct(\n `)}
8593
${segmentCommon.enabledEvents(params).map((event) =>
8694
`count(segment_sessionized_events.${event}_id) as total_${event}s`).join(`,\n `)},
87-
${crossdb.timestampDiff("millisecond", "min(segment_sessionized_events.timestamp)", "max(segment_sessionized_events.timestamp)")} as duration_millis
95+
${sql.timestamps.diff("millisecond", "min(segment_sessionized_events.timestamp)", "max(segment_sessionized_events.timestamp)")} as duration_millis
8896
${ctx.when(global.dataform.projectConfig.warehouse == "bigquery", `) as stats`)}
8997
9098
-- first values in the session for page fields
@@ -168,4 +176,4 @@ group by
168176
)
169177
170178
select * from output`)
171-
}
179+
}

‎includes/user_map.js‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
const crossdb = require("./crossdb");
1+
const sql = require("@dataform/sql")();
22
const segmentCommon = require("./common");
33

44
module.exports = (params) => {
@@ -33,14 +33,16 @@ ${segmentCommon.enabledEvents(params).map((event) =>
3333
3434
select distinct
3535
anonymous_id,
36-
${crossdb.windowFunction({
37-
func: "last_value",
38-
value: "user_id",
39-
ignore_nulls: false,
40-
partition_fields: "anonymous_id",
41-
order_fields: "anonymous_id_user_id_pairs.timestamp asc",
42-
frame_clause: "rows between unbounded preceding and unbounded following"
43-
})} as user_id
36+
${sql.windowFunction(
37+
"last_value",
38+
"user_id",
39+
false,
40+
{
41+
partitionFields: ["anonymous_id"],
42+
orderFields: ["anonymous_id_user_id_pairs.timestamp asc"],
43+
frameClause: "rows between unbounded preceding and unbounded following"
44+
}
45+
)} as user_id
4446
from
4547
anonymous_id_user_id_pairs
4648
where

‎includes/users.js‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
const crossdb = require("./crossdb");
1+
const sql = require("@dataform/sql")();
22

33
let USER = `coalesce(
44
identifies.user_id,
@@ -21,23 +21,27 @@ module.exports = (params) => {
2121
2222
select distinct
2323
${USER} as user_id,
24-
${crossdb.windowFunction({
25-
func: "first_value",
26-
value: "identifies.timestamp",
27-
ignore_nulls: true,
28-
partition_fields: USER,
29-
order_fields: "identifies.timestamp asc",
30-
frame_clause: "rows between unbounded preceding and unbounded following",
31-
})} as first_seen_at
24+
${sql.windowFunction(
25+
"first_value",
26+
"identifies.timestamp",
27+
true,
28+
{
29+
partitionFields: [USER],
30+
orderFields: ["identifies.timestamp asc"],
31+
frameClause: "rows between unbounded preceding and unbounded following"
32+
}
33+
)} as first_seen_at
3234
${params.customUserFields.length ? `,` : ``}
33-
${params.customUserFields.map(f => `${crossdb.windowFunction({
34-
func: "first_value",
35-
value: f,
36-
ignore_nulls: true,
37-
partition_fields: USER,
38-
order_fields: "identifies.timestamp desc",
39-
frame_clause: "rows between unbounded preceding and unbounded following",
40-
})} as ${f}`).join(",\n ")}
35+
${params.customUserFields.map(f => `${sql.windowFunction(
36+
"first_value",
37+
f,
38+
true,
39+
{
40+
partitionFields: [USER],
41+
orderFields: ["identifies.timestamp desc"],
42+
frameClause: "rows between unbounded preceding and unbounded following",
43+
}
44+
)} as ${f}`).join(",\n ")}
4145
from
4246
${ctx.ref(params.defaultConfig.schema, "segment_user_map")} as segment_user_anonymous_map
4347
left join ${ctx.ref(params.segmentSchema, "identifies")} as identifies

‎package-lock.json‎

Lines changed: 8 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
22
"dependencies": {
3-
"@dataform/core": "1.15.3"
3+
"@dataform/core": "1.15.5",
4+
"@dataform/sql": "0.2.0"
45
}
56
}

0 commit comments

Comments
 (0)