Promotion 2026-10-20 prod (#8420)

UPGRADING.rst

@@ -23,6 +23,17 @@ Note that as of September 2026, the issue number(s) must be occur in parentheses
 and at the end of section titles, instead of the beginning.
 
 
+Four inaccessible LungMAP snapshots in dev (#8421)
+==================================================
+
+Everyone
+~~~~~~~~
+
+Drop the ``lungmap`` atlas from personal deployments colocated with ``dev``. Use
+the ``sandbox`` deployment's ``environment.py`` as a model. Remove the ``lm2``
+and ``lm2-it`` catalogs from your personal deployments.
+
+
 Claude Code monitors run without the environment the hook provides (#8323)
 ==========================================================================
 

deployments/anvilprod/environment.py

@@ -1408,6 +1408,12 @@ def union(previous_catalog: dict[DatasetName, SourceItem | None],
     source('ee5a23e4', 'DepMap_HMB_20240827_ANV6_202607080056', no_ma_mirror),
     source('ce3811eb', 'ENCORE_293T_20260708_ANV6_202607081759'),
     source('42923ab3', 'ENCORE_RS293_20260708_ANV6_202607081737'),
+    source('c112acbb', 'GTEx_V7_hg19_20260821_ANV6_202608241432', no_ma_mirror),
+    source('c81bf322', 'GTEx_V8_hg38_20240116_ANV6_202608211533', no_ma_mirror),
+    source('532caa61', 'GTEx_V9_hg38_20260821_ANV6_202608251459', no_ma_mirror),
+    source('cb6c0630', 'GTEx_public_data_20260821_ANV6_202608211401'),
+    source('5aa27a7c', 'GTEx_v10_hg38_20260821_ANV6_202608241349', no_ma_mirror),
+    source('63c68539', 'GTEx_v11_hg38_20260126_ANV6_202608211606', no_ma_mirror),
     source('6ae39c14', 'GUARDD_US_GRU_R2_20260709_ANV6_202607091806', no_ma_mirror),
     source('2de712ea', 'GUARDD_US_HMB_R2_20260709_ANV6_202607091814', no_ma_mirror),
     source('6e7af669', 'HPRC_20260121_ANV6_202607080129'),

deployments/dev/environment.py

@@ -308,7 +308,9 @@ def env() -> Mapping[str, str | None]:
                                        sources=condense(sources))
             for atlas, catalog, sources, mirror_limit, it_mirror_limit, in [
                 ('hca', 'dcp3', dcp3_min_sources, int(1.5 * 1024 ** 3), int(1.5 * 1024 ** 3)),
-                ('lungmap', 'lm2', lm2_sources, -1, -1)
+                # FIXME: Decommission LungMAP
+                #        https://github.com/DataBiosphere/azul/issues/8412
+                # ('lungmap', 'lm2', lm2_sources, -1, -1)
             ]
             for suffix, is_it in [
                 ('', False),

deployments/hammerbox/environment.py

@@ -1410,6 +1410,12 @@ def union(previous_catalog: dict[DatasetName, SourceItem | None],
     source('ee5a23e4', 'DepMap_HMB_20240827_ANV6_202607080056', no_ma_mirror),
     source('ce3811eb', 'ENCORE_293T_20260708_ANV6_202607081759'),
     source('42923ab3', 'ENCORE_RS293_20260708_ANV6_202607081737'),
+    source('c112acbb', 'GTEx_V7_hg19_20260821_ANV6_202608241432', no_ma_mirror),
+    source('c81bf322', 'GTEx_V8_hg38_20240116_ANV6_202608211533', no_ma_mirror),
+    source('532caa61', 'GTEx_V9_hg38_20260821_ANV6_202608251459', no_ma_mirror),
+    source('cb6c0630', 'GTEx_public_data_20260821_ANV6_202608211401'),
+    source('5aa27a7c', 'GTEx_v10_hg38_20260821_ANV6_202608241349', no_ma_mirror),
+    source('63c68539', 'GTEx_v11_hg38_20260126_ANV6_202608211606', no_ma_mirror),
     source('6ae39c14', 'GUARDD_US_GRU_R2_20260709_ANV6_202607091806', no_ma_mirror),
     source('2de712ea', 'GUARDD_US_HMB_R2_20260709_ANV6_202607091814', no_ma_mirror),
     source('6e7af669', 'HPRC_20260121_ANV6_202607080129'),

deployments/sandbox/environment.py

@@ -321,7 +321,9 @@ def env() -> Mapping[str, str | None]:
                                        sources=condense(sources))
             for atlas, catalog, sources, mirror_limit, it_mirror_limit, in [
                 ('hca', 'dcp3', dcp3_min_sources, int(1.5 * 1024 ** 3), int(1.5 * 1024 ** 3)),
-                ('lungmap', 'lm2', lm2_sources, -1, -1)
+                # FIXME: Decommission LungMAP
+                #        https://github.com/DataBiosphere/azul/issues/8412
+                # ('lungmap', 'lm2', lm2_sources, -1, -1)
             ]
             for suffix, is_it in [
                 ('', False),

scripts/can_bundle.py

@@ -39,6 +39,7 @@
     AnyJSON,
     AnyMutableJSON,
     json_dict,
+    json_mapping,
 )
 from azul.logging import (
     configure_script_logging,
@@ -150,6 +151,15 @@ def save_bundle(bundle: Bundle, output_dir: str) -> None:
     bundle_json = bundle.to_json()
     # We can bundles without the FQID so that we can mock it during tests
     bundle_json.pop('fqid')
+    # Plugins group these entries by entity type, in the iteration order of
+    # a set, which varies based on Python's hash seed.
+    for key in ('manifest', 'metadata'):
+        try:
+            entries = json_mapping(bundle_json[key])
+        except KeyError:
+            pass
+        else:
+            bundle_json[key] = dict(sorted(entries.items()))
     with write_file_atomically(path) as f:
         json.dump(bundle_json, f, indent=4)
     log.info('Successfully wrote %s', path)

src/azul/plugins/repository/tdr_hca/__init__.py

@@ -467,11 +467,18 @@ def _find_upstream_bundles(self,
     def _merge_links(self, links_jsons: MutableJSONs) -> MutableJSON:
         """
         Merge the links.json documents from multiple stitched bundles into a
-        single document.
+        single document, ensuring that the result does not depend on the order
+        of the stitched bundles.
+
+        :param links_jsons: The links entities of the root bundle, followed by
+                            those of the bundles stitched to it.
         """
         root, *stitched = links_jsons
         if stitched:
-            source_contents = [row['content'] for row in links_jsons]
+            # The stitched bundles are in the iteration order of a set, which
+            # varies between processes.
+            stitched.sort(key=itemgetter('links_id'))
+            source_contents = [row['content'] for row in [root, *stitched]]
             # FIXME: Explicitly verify compatible schema versions for stitched subgraphs
             #        https://github.com/DataBiosphere/azul/issues/3215
             schema_type = 'links'