dagster/copy.bara.sky at master · GIScience/dagster · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
"""
Copybara configuration for syncing commits from dagster-io/dagster master to dagster-io/internal.
"""

def _skip_synced_from_internal(ctx):
    """Skip commits that originated from dagster-io/internal so we don't double-commit them."""
    if ctx.find_label("Internal-RevId"):
        core.fail_with_noop("Skipping commit that originated from dagster-io/internal.")

core.workflow(
    name="sync-internal",
    origin = git.origin(
        url =  "file:///workspace/build/buildkite/dagster/copybara-dagster-to-internal",
        ref = "master",
    ),
    destination = git.destination(
        url = "https://github.com/dagster-io/internal.git",
        fetch = "master",
        push = "master",
    ),
    origin_files = glob(["**"]),
    destination_files = glob(["dagster-oss/**"]),

    # Preserve the author of the origin commit as the author of the destination
    # commit. Note that the "Dagster Devtools" setting here is setting a
    # default param that is required by Copybara for all authoring.* calls. For
    # authoring.pass_thru, this default is never actually used.
    authoring=authoring.pass_thru("Dagster Devtools <devtools@dagsterlabs.com>"),

    # Transformation pipeline for commits.
    transformations = [

        # Skip any commit that originated from dagster-io/internal. This
        # prevents echoing the same commit back to dagster-io/internal.
        core.dynamic_transform(impl = _skip_synced_from_internal),

        # Yarn workspace files for OSS in dagster-io/internal use "oss.yml" so
        # that they are not detected by yarn, which instead uses a workspace
        # defined at the repo root. Add this prefix when exporting the commit
        # to dagster-io/internal.
        core.move("js_modules/.yarnrc.yml", "js_modules/.yarnrc.oss.yml"),
        core.move("js_modules/package.json", "js_modules/package.oss.json"),
        core.move("js_modules/yarn.lock", "js_modules/yarn.oss.lock"),

        # Mount all contents of this repo on the dagster-oss subdirectory for
        # the dagster-io/internal commit.
        core.move("", "dagster-oss"),
    ],

    # Follows this algorithm:
    # - Find the most recent commit in destination.fetch having
    #   GitOrigin-RevId: <HASH> in its commit message. <HASH> refers to a
    #   commit in origin.
    # - Find <HASH> in origin.ref. Iterate up from this commit to ref. For each
    #   commit, apply the transformation pipeline. Commits that are transformed
    #   into an empty diff (usually because no changes are made to the subset
    #   of files targeted by `origin_files`) are skipped.
    # - Push all the transformed commits to destination.push.
    mode = "ITERATIVE",

    # This will cause the sync to fail if any transformations are not
    # reversible (i.e. can be run in reverse and arrive at the original commit
    # diff).
    reversible_check = True,

    # Commits written to dagster-io/internal record the corresponding
    # dagster-io/dagster commit hash under "Dagster-RevId" instead of the
    # default "GitOrigin-RevId". This affects ITERATIVE mode syncing, which by
    # default identifies the most recent commit in the destination ref
    # containing this string. We override the default so that multiple repos
    # can sync with dagster-io/dagster. Without overriding, ITERATIVE mode
    # would mistakenly identify commits synced from another repo, which would
    # also use the default GitOrigin-RevId, as having originated from this
    # repo.
    custom_rev_id = "Dagster-RevId",
)