From 0ee7d89c70ea2393588bdf269c59211dfee9088a Mon Sep 17 00:00:00 2001 From: Vineet Bansal Date: Thu, 18 Jun 2026 19:18:08 -0400 Subject: [PATCH 1/5] Support for science file versioning from imap-data-access. Permissive behavior to allow gradual adoption. --- imap_processing/cdf/utils.py | 17 +- imap_processing/cli.py | 252 +++++++++++------- .../tests/codice/test_codice_l1b.py | 11 +- imap_processing/tests/hi/test_hi_l1a.py | 6 +- imap_processing/tests/idex/test_idex_l1a.py | 8 +- imap_processing/tests/idex/test_idex_l1b.py | 2 +- imap_processing/tests/mag/test_mag_l1d.py | 15 +- imap_processing/tests/swapi/test_swapi_l1.py | 8 +- imap_processing/tests/swapi/test_swapi_l2.py | 6 +- imap_processing/tests/swe/test_swe_l1a.py | 6 +- imap_processing/tests/swe/test_swe_l1b.py | 2 +- .../tests/ultra/unit/test_ultra_l1a.py | 4 +- poetry.lock | 8 +- pyproject.toml | 2 +- 14 files changed, 214 insertions(+), 133 deletions(-) diff --git a/imap_processing/cdf/utils.py b/imap_processing/cdf/utils.py index 99900e08d3..0c111ecf65 100644 --- a/imap_processing/cdf/utils.py +++ b/imap_processing/cdf/utils.py @@ -14,6 +14,7 @@ from cdflib.logging import logger as cdflib_logger from cdflib.xarray import cdf_to_xarray, xarray_to_cdf from cdflib.xarray.cdf_to_xarray import ISTP_TO_XARRAY_ATTRS +from imap_data_access.file_validation import Version import imap_processing from imap_processing._version import __version__, __version_tuple__ # noqa: F401 @@ -121,14 +122,18 @@ def write_cdf( version = dataset.attrs.get("Data_version", None) if version is None: warnings.warn( - "No Data_version attribute found in dataset. Using default v999.", + "No Data_version attribute found in dataset. Using default v999.9999", stacklevel=2, ) - version = "999" + version = "v999.9999" dataset.attrs["Data_version"] = version - elif not re.match(r"\d{3}", version): + + # Data_version may be stored without the leading 'v'; add it before validating. + version_string = version if str(version).startswith("v") else f"v{version}" + if not Version.is_valid_version(version_string): raise ValueError( - f"The Data_version attribute {version} does not match expected format XXX." + rf"The Data_version attribute {version} does not match the expected " + "regex {Version.version_regex()}" ) repointing = dataset.attrs.get("Repointing", None) @@ -139,7 +144,7 @@ def write_cdf( data_level=data_level, descriptor=descriptor, start_time=start_date, - version=f"v{version}", # Ensure version is prefixed with 'v' + version=version_string, repointing=repointing_int, ) file_path = Path(science_file.construct_path()) @@ -218,7 +223,7 @@ def parse_filename_like(filename_like: str) -> re.Match: r"(?P[^_]+)" # Required descriptor r"(_(?P\d{8}))?" # Optional start date r"(-repoint(?P\d{5}))?" # Optional repointing field - r"(?:_(?Pv\d{3}))?" # Optional version + rf"(?:_(?P{Version.version_regex()}))?" # Optional version r"(?:\.(?Pcdf|pkts))?$" # Optional extension ) match = re.match(regex_str, filename_like) diff --git a/imap_processing/cli.py b/imap_processing/cli.py index 8e1bafc491..44ecd8d90b 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -15,7 +15,6 @@ import argparse import json import logging -import re import sys from abc import ABC, abstractmethod from pathlib import Path @@ -27,6 +26,7 @@ import xarray as xr from cdflib.xarray import xarray_to_cdf from cdflib.xarray.xarray_to_cdf import ISTPError +from imap_data_access.file_validation import Version from imap_data_access.io import IMAPDataAccessError, download from imap_data_access.processing_input import ( ProcessingInputCollection, @@ -105,23 +105,27 @@ def _parse_args() -> argparse.Namespace: --data-level "l1a" --descriptor "all" --start-date "20231212" - --version "v001" - --dependency '[ - { - "type": "ancillary", - "files": [ - "imap_mag_l1b-cal_20250101_v001.cdf", - "imap_mag_l1b-cal_20250103_20250104_v002.cdf" - ] - }, - { - "type": "science", - "files": [ - "imap_idex_l2_sci_20240312_v000.cdf", - "imap_idex_l2_sci_20240312_v001.cdf" - ] + --dependency '{ + "dependency": [ + { + "type": "ancillary", + "files": [ + "imap_mag_l1b-cal_20250101_v001.cdf", + "imap_mag_l1b-cal_20250103_20250104_v002.cdf" + ] + }, + { + "type": "science", + "files": [ + "imap_idex_l2_sci_20240312_v000.cdf", + "imap_idex_l2_sci_20240312_v001.cdf" + ] + } + ], + "version": { + "": {"major_version": 2, "minor_version": 1} } - ]' + }' --upload-to-sdc Returns @@ -137,23 +141,27 @@ def _parse_args() -> argparse.Namespace: '--descriptor "all" ' ' --start-date "20231212" ' '--repointing "repoint12345" ' - '--version "v001" ' - '--dependency "[' - " {" - ' "type": "ancillary",' - ' "files": [' - ' "imap_mag_l1b-cal_20250101_v001.cdf",' - ' "imap_mag_l1b-cal_20250103_20250104_v002.cdf"' - " ]" - " }," - " {" - ' "type": "science",' - ' "files": [' - ' "imap_idex_l2_sci_20240312_v000.cdf",' - ' "imap_idex_l2_sci_20240312_v001.cdf"' - " ]" + '--dependency "{' + ' "dependency": [' + " {" + ' "type": "ancillary",' + ' "files": [' + ' "imap_mag_l1b-cal_20250101_v001.cdf",' + ' "imap_mag_l1b-cal_20250103_20250104_v002.cdf"' + " ]" + " }," + " {" + ' "type": "science",' + ' "files": [' + ' "imap_idex_l2_sci_20240312_v000.cdf",' + ' "imap_idex_l2_sci_20240312_v001.cdf"' + " ]" + " }" + " ]," + ' "version": {' + ' "": {"major_version": 2, "minor_version": 1}' " }" - "]" + "}" ' --upload-to-sdc"' ) instrument_help = ( @@ -169,24 +177,29 @@ def _parse_args() -> argparse.Namespace: "descriptor like 'sci-1min'. Default is 'all'." ) dependency_help = ( - "Dependency information in str format." + "Dependency information in str format. This is an object that wraps the" + " dependency list together with the per-descriptor output versions." "Example:" - "'[" - " {" - ' "type": "ancillary",' - ' "files": [' - ' "imap_mag_l1b-cal_20250101_v001.cdf",' - ' "imap_mag_l1b-cal_20250103_20250104_v002.cdf"' - " ]" - " }," - " {" - ' "type": "science",' - ' "files": [' - ' "imap_idex_l2_sci_20240312_v000.cdf",' - ' "imap_idex_l2_sci_20240312_v001.cdf"' - " ]" - " }" - "]'" + "'{" + ' "dependency": [' + " {" + ' "type": "ancillary",' + ' "files": [' + ' "imap_mag_l1b-cal_20250101_v001.cdf",' + ' "imap_mag_l1b-cal_20250103_20250104_v002.cdf"' + " ]" + " }," + " {" + ' "type": "science",' + ' "files": [' + ' "imap_idex_l2_sci_20240312_v000.cdf",' + ' "imap_idex_l2_sci_20240312_v001.cdf"' + " ]" + " }" + " ]," + ' "version": {"": ' + '{"major_version": 2, "minor_version": 1}}' + "}'" " A path to a JSON file containing this same information may also be" "passed in. If dependency is a string ending in '.json', it will be interpreted" " as such a file path." @@ -239,6 +252,7 @@ def _parse_args() -> argparse.Namespace: "provided. Format: repoint#####", ) + # TODO - Remove support for this eventually parser.add_argument( "--version", type=str, @@ -337,44 +351,52 @@ class ProcessInstrument(ABC): data_descriptor : str The descriptor of the data to process (e.g. ``sci``). dependency_str : str - A string representation of the dependencies for the instrument in the - format: - '[ - { - "type": "ancillary", - "files": [ - "imap_mag_l1b-cal_20250101_v001.cdf", - "imap_mag_l1b-cal_20250103_20250104_v002.cdf" - ] - }, - { - "type": "ancillary", - "files": [ - "imap_mag_l1b-lut_20250101_v001.cdf", - ] - }, - { - "type": "science", - "files": [ - "imap_mag_l1a_norm-magi_20240312_v000.cdf", - "imap_mag_l1a_norm-magi_20240312_v001.cdf" - ] - }, - { - "type": "science", - "files": [ - "imap_idex_l2_sci_20240312_v000.cdf", - "imap_idex_l2_sci_20240312_v001.cdf" - ] + A string representation of an object that wraps the dependency list + produced by ProcessingInputCollection.serialize() together with the + per-descriptor output versions produced by the job: + '{ + "dependency": [ + { + "type": "ancillary", + "files": [ + "imap_mag_l1b-cal_20250101_v001.cdf", + "imap_mag_l1b-cal_20250103_20250104_v002.cdf" + ] + }, + { + "type": "ancillary", + "files": [ + "imap_mag_l1b-lut_20250101_v001.cdf", + ] + }, + { + "type": "science", + "files": [ + "imap_mag_l1a_norm-magi_20240312_v000.cdf", + "imap_mag_l1a_norm-magi_20240312_v001.cdf" + ] + }, + { + "type": "science", + "files": [ + "imap_idex_l2_sci_20240312_v000.cdf", + "imap_idex_l2_sci_20240312_v001.cdf" + ] + } + ], + "version": { + "": {"major_version": 2, "minor_version": 1}, + ... } - ]' - This is what ProcessingInputCollection.serialize() outputs. + }' + The version block determines the version of each produced product + (matched by descriptor). start_date : str The start date for the output data in YYYYMMDD format. repointing : str The repointing for the output data in the format 'repoint#####'. version : str - The version of the data in vXXX format. + Fallback version string. upload_to_sdc : bool A flag indicating whether to upload the output file to the SDC. """ @@ -391,20 +413,56 @@ def __init__( dependency_str: str, start_date: str, repointing: str | None, - version: str, + version: str, # TODO - remove this argument eventually upload_to_sdc: bool, ) -> None: self.data_level = data_level self.descriptor = data_descriptor - self.dependency_str = dependency_str - self.start_date = start_date self.repointing = repointing - - self.version = version self.upload_to_sdc = upload_to_sdc + # The dependency argument is an object of the form + # {"dependency": [...], "version": {: {...}}}. The + # per-descriptor product versions are extracted here and the dependency + # list is preserved as `self.dependency_str` so that + # ProcessingInputCollection.deserialize continues to work unchanged. + parsed_dependency = json.loads(dependency_str) if dependency_str else {} + # Tolerate the old format where the dependency argument is a bare list + # of dependency objects (no wrapping object and no version block). + if isinstance(parsed_dependency, list): + logger.warning("Old dependency format detected, converting to new format.") + parsed_dependency = {"dependency": parsed_dependency} + version_block = parsed_dependency.get("version", {}) + self.dependency_str = json.dumps(parsed_dependency.get("dependency", [])) + self._fallback_version = version + + # Map each produced descriptor to a Version built from major/minor + self.version_map: dict[str, Version] = { + descriptor: Version(info["major_version"], info["minor_version"]) + for descriptor, info in version_block.items() + } + + def _resolve_version(self, descriptor: str) -> Version: + """ + Return the Version to use for a product with the given descriptor. + + Parameters + ---------- + descriptor : str + The product descriptor (the final field of ``Logical_source``). + + Returns + ------- + Version + The Version to use for the product. + """ + if descriptor not in self.version_map: + msg = f"No version provided for descriptor: {descriptor}" + logger.warning(msg) + return self.version_map.get(descriptor, self._fallback_version) + def upload_products(self, products: list[Path]) -> None: """ Upload data products to the IMAP SDC. @@ -560,16 +618,10 @@ def post_processing( logger.info("Writing products to local storage") - logger.info("Dataset version: %s", self.version) # Parent files used to create these datasets # https://spdf.gsfc.nasa.gov/istp_guide/gattributes.html. parent_files = [p.name for p in dependencies.get_file_paths()] logger.info("Parent files: %s", parent_files) - # Format version to vXXX if not already in that format. Eg. - # If version is passed in as 1 or 001, it will be converted to v001. - r = re.compile(r"v\d{3}") - if not isinstance(self.version, str) or r.match(self.version) is None: - self.version = f"v{int(self.version):03d}" # vXXX # Start date is either the start date or the repointing. # if it is the repointing, default to using the first epoch in the file as @@ -578,7 +630,14 @@ def post_processing( for ds in processed_data: if isinstance(ds, xr.Dataset): - ds.attrs["Data_version"] = self.version[1:] # Strip 'v' from version + # Look up the version for this product by its descriptor (the + # final field of Logical_source). + descriptor = ds.attrs.get("Logical_source", "").split("_")[-1] + if descriptor == "": + logger.warning("No descriptor found in dataset.") + version = self._resolve_version(descriptor) + logger.info(f"Product {descriptor} version: {version}") + ds.attrs["Data_version"] = str(version) if self.repointing is not None: ds.attrs["Repointing"] = self.repointing ds.attrs["Start_date"] = self.start_date @@ -1503,7 +1562,10 @@ def post_processing( "Logical_source" ].split("_")[1:] start_date = self.start_date - version = self.version + # Ancillary filenames only support the minor-only (vXXX) + # version format, so render the resolved version that way. + resolved_version = self._resolve_version(descriptor) + version = str(Version(None, resolved_version.minor)) output_filepath = ( imap_data_access.AncillaryFilePath.generate_from_inputs( diff --git a/imap_processing/tests/codice/test_codice_l1b.py b/imap_processing/tests/codice/test_codice_l1b.py index 3c4532d8a4..9c259d698b 100644 --- a/imap_processing/tests/codice/test_codice_l1b.py +++ b/imap_processing/tests/codice/test_codice_l1b.py @@ -112,7 +112,9 @@ def test_l1b_hi_omni(mock_get_file_paths, codice_lut_path): ) cdf_file = write_cdf(processed_data) - assert cdf_file.name == f"imap_codice_l1b_hi-omni_{VALIDATION_FILE_DATE}_v999.cdf" + assert ( + cdf_file.name == f"imap_codice_l1b_hi-omni_{VALIDATION_FILE_DATE}_v999.9999.cdf" + ) @pytest.mark.xfail(reason="Need to revisit in future PR") @@ -171,7 +173,8 @@ def test_l1b_hi_priorities(mock_get_file_paths, codice_lut_path): cdf_file = write_cdf(processed_data) assert ( - cdf_file.name == f"imap_codice_l1b_hi-priority_{VALIDATION_FILE_DATE}_v999.cdf" + cdf_file.name + == f"imap_codice_l1b_hi-priority_{VALIDATION_FILE_DATE}_v999.9999.cdf" ) @@ -202,7 +205,7 @@ def test_l1b_nsw_lo_priorities(mock_get_file_paths, codice_lut_path): cdf_file = write_cdf(processed_data) assert ( cdf_file.name - == f"imap_codice_l1b_lo-nsw-priority_{VALIDATION_FILE_DATE}_v999.cdf" + == f"imap_codice_l1b_lo-nsw-priority_{VALIDATION_FILE_DATE}_v999.9999.cdf" ) @@ -233,7 +236,7 @@ def test_l1b_sw_lo_priorities(mock_get_file_paths, codice_lut_path): cdf_file = write_cdf(processed_data) assert ( cdf_file.name - == f"imap_codice_l1b_lo-sw-priority_{VALIDATION_FILE_DATE}_v999.cdf" + == f"imap_codice_l1b_lo-sw-priority_{VALIDATION_FILE_DATE}_v999.9999.cdf" ) diff --git a/imap_processing/tests/hi/test_hi_l1a.py b/imap_processing/tests/hi/test_hi_l1a.py index d86f649a01..893716c378 100644 --- a/imap_processing/tests/hi/test_hi_l1a.py +++ b/imap_processing/tests/hi/test_hi_l1a.py @@ -33,7 +33,7 @@ def test_sci_de_decom(hi_l0_test_data_path): # TODO: Verify correct unpacking of sample data. Issue: #1186 # Write to CDF - cdf_filename = "imap_hi_l1a_90sensor-de_20241105_v999.cdf" + cdf_filename = "imap_hi_l1a_90sensor-de_20241105_v999.9999.cdf" cdf_filepath = write_cdf(processed_data[0]) assert cdf_filepath.name == cdf_filename @@ -72,7 +72,7 @@ def test_diag_fee_decom(hi_l0_test_data_path): processed_data = hi_l1a(packet_file_path=bin_data_path) dataset = processed_data[0] cdf_filepath = write_cdf(processed_data[0], istp=False) - assert cdf_filepath.name == "imap_hi_l1a_45sensor-diagfee_20250208_v999.cdf" + assert cdf_filepath.name == "imap_hi_l1a_45sensor-diagfee_20250208_v999.9999.cdf" assert np.unique(processed_data[0]["pkt_apid"].values) == HIAPID.H45_DIAG_FEE.value @@ -99,7 +99,7 @@ def test_app_nhk_decom(hi_l0_test_data_path): # Write CDF cem_raw_cdf_filepath = write_cdf(processed_data[0], istp=False) - assert cem_raw_cdf_filepath.name == "imap_hi_l1a_90sensor-hk_20241105_v999.cdf" + assert cem_raw_cdf_filepath.name == "imap_hi_l1a_90sensor-hk_20241105_v999.9999.cdf" validation_df = pd.read_csv(hi_l0_test_data_path / "H90_NHK_20241104_verify.csv") for col_name, series in validation_df.items(): diff --git a/imap_processing/tests/idex/test_idex_l1a.py b/imap_processing/tests/idex/test_idex_l1a.py index 97e6074e80..f82488d9f7 100644 --- a/imap_processing/tests/idex/test_idex_l1a.py +++ b/imap_processing/tests/idex/test_idex_l1a.py @@ -55,7 +55,7 @@ def test_idex_cdf_file(decom_test_data_sci: xr.Dataset): file_name = write_cdf(decom_test_data_sci) assert file_name.exists() - assert file_name.name == "imap_idex_l1a_sci-10days_20231218_v999.cdf" + assert file_name.name == "imap_idex_l1a_sci-10days_20231218_v999.9999.cdf" written_dataset = load_cdf(file_name) assert written_dataset["time_low_sample_rate"].attrs["VAR_NOTES"] == ( "The low sample rate is 4.0625 MHz, so adjacent samples are separated " @@ -532,10 +532,10 @@ def test_catlst_dataset(decom_test_data_catlst: list[xr.Dataset]): np.testing.assert_array_equal(ds.epoch, expected_epoch) # Assert that the dataset can be written to a CDF file filename_l1a = write_cdf(decom_test_data_catlst[0]) - assert filename_l1a.name == "imap_idex_l1a_catlst-10days_20241206_v999.cdf" + assert filename_l1a.name == "imap_idex_l1a_catlst-10days_20241206_v999.9999.cdf" filename_l1b = write_cdf(decom_test_data_catlst[1]) - assert filename_l1b.name == "imap_idex_l1b_catlst-10days_20241206_v999.cdf" + assert filename_l1b.name == "imap_idex_l1b_catlst-10days_20241206_v999.9999.cdf" def test_msg_dataset(decom_test_data_msg: xr.Dataset): @@ -556,7 +556,7 @@ def test_msg_dataset(decom_test_data_msg: xr.Dataset): np.testing.assert_array_equal(decom_test_data_msg.epoch, expected_epoch) # Assert that the dataset can be written to a CDF file filename_l1a = write_cdf(decom_test_data_msg) - assert filename_l1a.name == "imap_idex_l1a_msg-10days_20100101_v999.cdf" + assert filename_l1a.name == "imap_idex_l1a_msg-10days_20100101_v999.9999.cdf" # Validate the messages with the IDEX team example data example_data = pd.read_csv( diff --git a/imap_processing/tests/idex/test_idex_l1b.py b/imap_processing/tests/idex/test_idex_l1b.py index e1c2db9f03..e2b590d1e7 100644 --- a/imap_processing/tests/idex/test_idex_l1b.py +++ b/imap_processing/tests/idex/test_idex_l1b.py @@ -70,7 +70,7 @@ def test_idex_cdf_file(l1b_dataset: xr.Dataset): file_name = write_cdf(l1b_dataset) assert file_name.exists() - assert file_name.name == "imap_idex_l1b_sci-10days_20231218_v999.cdf" + assert file_name.name == "imap_idex_l1b_sci-10days_20231218_v999.9999.cdf" def test_idex_waveform_units(l1b_dataset: xr.Dataset): diff --git a/imap_processing/tests/mag/test_mag_l1d.py b/imap_processing/tests/mag/test_mag_l1d.py index 2ebe5efe76..fbb94d48e8 100644 --- a/imap_processing/tests/mag/test_mag_l1d.py +++ b/imap_processing/tests/mag/test_mag_l1d.py @@ -1,3 +1,4 @@ +import json import logging from unittest.mock import patch @@ -205,11 +206,21 @@ def test_mag_l1d_attributes( # Verify xarray_to_cdf was called for each dataset assert mock_xarray_to_cdf.call_count == len(l1d_datasets) - # Test that Mag.post_processing can be called on the datasets + # Test that Mag.post_processing can be called on the datasets. + dependency_str = json.dumps( + { + "dependency": [], + "version": { + "spin-offsets": {"major_version": 1, "minor_version": 1}, + "gradiometry-offsets-norm": {"major_version": 1, "minor_version": 1}, + "gradiometry-offsets-burst": {"major_version": 1, "minor_version": 1}, + }, + } + ) mag_processor = Mag( data_level="l1d", data_descriptor="all", - dependency_str="[]", + dependency_str=dependency_str, start_date="20000101", repointing=None, version="v001", diff --git a/imap_processing/tests/swapi/test_swapi_l1.py b/imap_processing/tests/swapi/test_swapi_l1.py index 49ddd2fa50..a18de5ea63 100644 --- a/imap_processing/tests/swapi/test_swapi_l1.py +++ b/imap_processing/tests/swapi/test_swapi_l1.py @@ -174,7 +174,7 @@ def test_process_swapi_science(decom_test_data): } # Test CDF File - cdf_filename = "imap_swapi_l1_sci_20240924_v999.cdf" + cdf_filename = "imap_swapi_l1_sci_20240924_v999.9999.cdf" cdf_path = write_cdf(processed_data) assert cdf_path.name == cdf_filename @@ -205,10 +205,10 @@ def first_get_file_paths_side_effect(descriptor): ) processed_data = swapi_l1(collection_obj, descriptor="hk") # hk cdf file - l1a_hk_cdf_filename = "imap_swapi_l1a_hk_20240924_v999.cdf" + l1a_hk_cdf_filename = "imap_swapi_l1a_hk_20240924_v999.9999.cdf" hk_cdf_path = write_cdf(processed_data[0]) assert hk_cdf_path.name == l1a_hk_cdf_filename - l1b_hk_cdf_filename = "imap_swapi_l1b_hk_20240924_v999.cdf" + l1b_hk_cdf_filename = "imap_swapi_l1b_hk_20240924_v999.9999.cdf" l1b_hk_cdf_path = write_cdf(processed_data[1]) assert l1b_hk_cdf_path.name == l1b_hk_cdf_filename @@ -236,6 +236,6 @@ def second_get_file_paths_side_effect(descriptor): assert processed_data[0].attrs["Apid"] == f"{SWAPIAPID.SWP_SCI}" # Test CDF File - cdf_filename = "imap_swapi_l1_sci_20240924_v999.cdf" + cdf_filename = "imap_swapi_l1_sci_20240924_v999.9999.cdf" cdf_path = write_cdf(processed_data[0]) assert cdf_path.name == cdf_filename diff --git a/imap_processing/tests/swapi/test_swapi_l2.py b/imap_processing/tests/swapi/test_swapi_l2.py index af953a7b4f..fe214f925b 100644 --- a/imap_processing/tests/swapi/test_swapi_l2.py +++ b/imap_processing/tests/swapi/test_swapi_l2.py @@ -86,7 +86,7 @@ def first_get_file_paths_side_effect(descriptor): ) # Create HK CDF File processed_hk_data = swapi_l1(collection_obj, descriptor="hk") - hk_cdf_filename = "imap_swapi_l1a_hk_20240924_v999.cdf" + hk_cdf_filename = "imap_swapi_l1a_hk_20240924_v999.9999.cdf" hk_cdf_path = write_cdf(processed_hk_data[0]) assert hk_cdf_path.name == hk_cdf_filename @@ -110,7 +110,7 @@ def second_get_file_paths_side_effect(descriptor): ) # Create L1 CDF File processed_sci_data = swapi_l1(collection_obj, descriptor="sci") - cdf_filename = "imap_swapi_l1_sci_20240924_v999.cdf" + cdf_filename = "imap_swapi_l1_sci_20240924_v999.9999.cdf" cdf_path = write_cdf(processed_sci_data[0]) assert cdf_path.name == cdf_filename @@ -127,7 +127,7 @@ def second_get_file_paths_side_effect(descriptor): lut_notes_df=lut_notes_table, ) l2_cdf = write_cdf(l2_dataset) - assert l2_cdf.name == "imap_swapi_l2_sci_20240924_v999.cdf" + assert l2_cdf.name == "imap_swapi_l2_sci_20240924_v999.9999.cdf" cdf_file = cdflib.CDF(l2_cdf) esa_energy_info = cdf_file.varinq("esa_energy") esa_energy_attrs = cdf_file.varattsget("esa_energy") diff --git a/imap_processing/tests/swe/test_swe_l1a.py b/imap_processing/tests/swe/test_swe_l1a.py index 359cbc92d7..8c5249f303 100644 --- a/imap_processing/tests/swe/test_swe_l1a.py +++ b/imap_processing/tests/swe/test_swe_l1a.py @@ -9,7 +9,7 @@ def test_cdf_creation(): cem_raw_cdf_filepath = write_cdf(processed_data[0]) - assert cem_raw_cdf_filepath.name == "imap_swe_l1a_sci_20240510_v999.cdf" + assert cem_raw_cdf_filepath.name == "imap_swe_l1a_sci_20240510_v999.9999.cdf" def test_cdf_creation_hk(): @@ -18,7 +18,7 @@ def test_cdf_creation_hk(): hk_cdf_filepath = write_cdf(processed_data[0]) - assert hk_cdf_filepath.name == "imap_swe_l1a_hk_20240510_v999.cdf" + assert hk_cdf_filepath.name == "imap_swe_l1a_hk_20240510_v999.9999.cdf" def test_cdf_creation_cem_raw(): @@ -27,4 +27,4 @@ def test_cdf_creation_cem_raw(): cem_raw_cdf_filepath = write_cdf(processed_data[0]) - assert cem_raw_cdf_filepath.name == "imap_swe_l1a_cem-raw_20240510_v999.cdf" + assert cem_raw_cdf_filepath.name == "imap_swe_l1a_cem-raw_20240510_v999.9999.cdf" diff --git a/imap_processing/tests/swe/test_swe_l1b.py b/imap_processing/tests/swe/test_swe_l1b.py index 11e010db24..b80a490582 100644 --- a/imap_processing/tests/swe/test_swe_l1b.py +++ b/imap_processing/tests/swe/test_swe_l1b.py @@ -289,7 +289,7 @@ def get_file_paths_side_effect(descriptor): hk_datasets = swe_l1b(input_collection) # Looks like HK data has some science data. That's why we use index 1. l1b_hk_filepath = write_cdf(hk_datasets[1]) - assert l1b_hk_filepath.name == "imap_swe_l1b_hk_20240510_v999.cdf" + assert l1b_hk_filepath.name == "imap_swe_l1b_hk_20240510_v999.9999.cdf" def test_count_rate(): diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1a.py b/imap_processing/tests/ultra/unit/test_ultra_l1a.py index 42c71b7802..5a1bb489c2 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1a.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1a.py @@ -190,7 +190,7 @@ def test_cdf_energy_rates(ccsds_path_functional): assert test_data_path.exists() assert ( test_data_path.name - == "imap_ultra_l1a_45sensor-energy-rates_20240122-repoint99999_v999.cdf" + == "imap_ultra_l1a_45sensor-energy-rates_20240122-repoint99999_v999.9999.cdf" ) # L1b dataset @@ -201,7 +201,7 @@ def test_cdf_energy_rates(ccsds_path_functional): assert test_data_path.exists() assert ( test_data_path.name - == "imap_ultra_l1b_45sensor-energy-rates_20240122-repoint99999_v999.cdf" + == "imap_ultra_l1b_45sensor-energy-rates_20240122-repoint99999_v999.9999.cdf" ) diff --git a/poetry.lock b/poetry.lock index c9bf31cf63..e6091591b1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -689,14 +689,14 @@ files = [ [[package]] name = "imap-data-access" -version = "0.37.0" +version = "0.42.0.dev0" description = "IMAP SDC Data Access" optional = false python-versions = "*" groups = ["main"] files = [ - {file = "imap_data_access-0.37.0-py3-none-any.whl", hash = "sha256:9801aba311311815866336636fdad9e37b530498f4c3f21abce283d860a7c643"}, - {file = "imap_data_access-0.37.0.tar.gz", hash = "sha256:7887ad43c4404c6465035af73621237aef7a8d88d506439dd4618aef1db8cf87"}, + {file = "imap_data_access-0.42.0.dev0-py3-none-any.whl", hash = "sha256:ba6618e809033cf8effa1ed0a63add082ff32ce5cde239a59530ec7065237da2"}, + {file = "imap_data_access-0.42.0.dev0.tar.gz", hash = "sha256:d75b5f2df51fb220d58c5fb009fd5d2eec24c22abdeac5035bc32df55ee2c338"}, ] [package.dependencies] @@ -2428,4 +2428,4 @@ tools = ["openpyxl", "pandas"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "ade584e6eb5f56e1447b73329767a3dffadf1e842eada0049215534e8ddd5e01" +content-hash = "170e9de255b8ac27e0e2d38f35415b9758a126aad8ee93efcea04d7bbf9ddb26" diff --git a/pyproject.toml b/pyproject.toml index 297cc7417f..079ccc5e23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ classifiers = [ dependencies = [ "astropy-healpix>=1.0", "cdflib>=1.3.11", - "imap-data-access>=0.37.0", + "imap-data-access>=0.42.0.dev0", "space_packet_parser>=6.0.0", "spiceypy>=6.0.0", "xarray>=2024.10.0,<2026", From 3a21dd8a641d6bb5ca1317da1f94e9d1ec3a1b9b Mon Sep 17 00:00:00 2001 From: Vineet Bansal Date: Thu, 18 Jun 2026 22:40:48 -0400 Subject: [PATCH 2/5] --version optional; tests modified to pass version in dep string --- imap_processing/cli.py | 3 +- imap_processing/tests/test_cli.py | 76 +++++++++++++++++-------------- 2 files changed, 43 insertions(+), 36 deletions(-) diff --git a/imap_processing/cli.py b/imap_processing/cli.py index 44ecd8d90b..cb7e3544da 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -256,7 +256,8 @@ def _parse_args() -> argparse.Namespace: parser.add_argument( "--version", type=str, - required=True, + required=False, + default=None, help="Version of the data. Format: vXXX", ) parser.add_argument( diff --git a/imap_processing/tests/test_cli.py b/imap_processing/tests/test_cli.py index f2027522b1..d73a996819 100644 --- a/imap_processing/tests/test_cli.py +++ b/imap_processing/tests/test_cli.py @@ -76,22 +76,27 @@ def test_main(mock_instrument): "--instrument", "mag", "--dependency", - ( - "[" - "{" - '"type": "ancillary",' - '"files": [' - '"imap_mag_l1b-cal_20250101_v001.cdf",' - '"imap_mag_l1b-cal_20250103-20250104_v002.cdf"' - "]" - "}," - "{" - '"type": "science",' - '"files": [' - '"imap_mag_l0_raw_20240430_v001.cdf",' - "]" - "}" - "]" + json.dumps( + { + "dependency": [ + { + "type": "ancillary", + "files": [ + "imap_mag_l1b-cal_20250101_v001.cdf", + "imap_mag_l1b-cal_20250103-20250104_v002.cdf", + ], + }, + { + "type": "science", + "files": [ + "imap_mag_l0_raw_20240430_v001.cdf", + ], + }, + ], + "version": { + "all": {"major_version": 1, "minor_version": 1}, + }, + } ), "--data-level", "l1a", @@ -99,8 +104,6 @@ def test_main(mock_instrument): "20240430", "--repointing", "repoint12345", - "--version", - "v001", "--upload-to-sdc", ] with mock.patch.object(sys, "argv", test_args): @@ -113,22 +116,27 @@ def test_parse_args_dependency_json_file(caplog, tmp_path): # Set caplog to capture all log levels caplog.set_level(logging.DEBUG) """Test imap_processing.cli.main() with --dependency as a JSON file path.""" - test_json_content = [ - { - "type": "ancillary", - "files": [ - "imap_mag_l1b-cal_20250101_v001.cdf", - "imap_mag_l1b-cal_20250103_20250104_v002.cdf", - ], + test_json_content = { + "dependency": [ + { + "type": "ancillary", + "files": [ + "imap_mag_l1b-cal_20250101_v001.cdf", + "imap_mag_l1b-cal_20250103_20250104_v002.cdf", + ], + }, + { + "type": "science", + "files": [ + "imap_idex_l2_sci_20240312_v000.cdf", + "imap_idex_l2_sci_20240312_v001.cdf", + ], + }, + ], + "version": { + "all": {"major_version": 1, "minor_version": 1}, }, - { - "type": "science", - "files": [ - "imap_idex_l2_sci_20240312_v000.cdf", - "imap_idex_l2_sci_20240312_v001.cdf", - ], - }, - ] + } test_json_filename = "imap_ultra_l2_test-dependency-json_20250520_v999.json" test_json_dir = tmp_path / "imap/dependency/ultra/l2/2025/05/" test_json_dir.mkdir(parents=True, exist_ok=True) @@ -149,8 +157,6 @@ def test_parse_args_dependency_json_file(caplog, tmp_path): "20240430", "--repointing", "repoint12345", - "--version", - "v001", "--upload-to-sdc", ] with mock.patch.object(sys, "argv", test_args): From 4337e79f90c68eb70de8af402edfd61f95f94481 Mon Sep 17 00:00:00 2001 From: Vineet Bansal Date: Thu, 18 Jun 2026 23:25:38 -0400 Subject: [PATCH 3/5] default version changed to v001.0001 --- imap_processing/cdf/utils.py | 4 ++-- imap_processing/tests/codice/test_codice_l1b.py | 8 ++++---- imap_processing/tests/hi/test_hi_l1a.py | 6 +++--- imap_processing/tests/idex/test_idex_l1a.py | 8 ++++---- imap_processing/tests/idex/test_idex_l1b.py | 2 +- imap_processing/tests/swapi/test_swapi_l1.py | 8 ++++---- imap_processing/tests/swapi/test_swapi_l2.py | 6 +++--- imap_processing/tests/swe/test_swe_l1a.py | 6 +++--- imap_processing/tests/swe/test_swe_l1b.py | 2 +- imap_processing/tests/ultra/unit/test_ultra_l1a.py | 4 ++-- 10 files changed, 27 insertions(+), 27 deletions(-) diff --git a/imap_processing/cdf/utils.py b/imap_processing/cdf/utils.py index 0c111ecf65..50f5264252 100644 --- a/imap_processing/cdf/utils.py +++ b/imap_processing/cdf/utils.py @@ -122,10 +122,10 @@ def write_cdf( version = dataset.attrs.get("Data_version", None) if version is None: warnings.warn( - "No Data_version attribute found in dataset. Using default v999.9999", + "No Data_version attribute found in dataset. Using default v001.0001", stacklevel=2, ) - version = "v999.9999" + version = "v001.0001" dataset.attrs["Data_version"] = version # Data_version may be stored without the leading 'v'; add it before validating. diff --git a/imap_processing/tests/codice/test_codice_l1b.py b/imap_processing/tests/codice/test_codice_l1b.py index 9c259d698b..761e231411 100644 --- a/imap_processing/tests/codice/test_codice_l1b.py +++ b/imap_processing/tests/codice/test_codice_l1b.py @@ -113,7 +113,7 @@ def test_l1b_hi_omni(mock_get_file_paths, codice_lut_path): cdf_file = write_cdf(processed_data) assert ( - cdf_file.name == f"imap_codice_l1b_hi-omni_{VALIDATION_FILE_DATE}_v999.9999.cdf" + cdf_file.name == f"imap_codice_l1b_hi-omni_{VALIDATION_FILE_DATE}_v001.0001.cdf" ) @@ -174,7 +174,7 @@ def test_l1b_hi_priorities(mock_get_file_paths, codice_lut_path): cdf_file = write_cdf(processed_data) assert ( cdf_file.name - == f"imap_codice_l1b_hi-priority_{VALIDATION_FILE_DATE}_v999.9999.cdf" + == f"imap_codice_l1b_hi-priority_{VALIDATION_FILE_DATE}_v001.0001.cdf" ) @@ -205,7 +205,7 @@ def test_l1b_nsw_lo_priorities(mock_get_file_paths, codice_lut_path): cdf_file = write_cdf(processed_data) assert ( cdf_file.name - == f"imap_codice_l1b_lo-nsw-priority_{VALIDATION_FILE_DATE}_v999.9999.cdf" + == f"imap_codice_l1b_lo-nsw-priority_{VALIDATION_FILE_DATE}_v001.0001.cdf" ) @@ -236,7 +236,7 @@ def test_l1b_sw_lo_priorities(mock_get_file_paths, codice_lut_path): cdf_file = write_cdf(processed_data) assert ( cdf_file.name - == f"imap_codice_l1b_lo-sw-priority_{VALIDATION_FILE_DATE}_v999.9999.cdf" + == f"imap_codice_l1b_lo-sw-priority_{VALIDATION_FILE_DATE}_v001.0001.cdf" ) diff --git a/imap_processing/tests/hi/test_hi_l1a.py b/imap_processing/tests/hi/test_hi_l1a.py index 893716c378..51553095b0 100644 --- a/imap_processing/tests/hi/test_hi_l1a.py +++ b/imap_processing/tests/hi/test_hi_l1a.py @@ -33,7 +33,7 @@ def test_sci_de_decom(hi_l0_test_data_path): # TODO: Verify correct unpacking of sample data. Issue: #1186 # Write to CDF - cdf_filename = "imap_hi_l1a_90sensor-de_20241105_v999.9999.cdf" + cdf_filename = "imap_hi_l1a_90sensor-de_20241105_v001.0001.cdf" cdf_filepath = write_cdf(processed_data[0]) assert cdf_filepath.name == cdf_filename @@ -72,7 +72,7 @@ def test_diag_fee_decom(hi_l0_test_data_path): processed_data = hi_l1a(packet_file_path=bin_data_path) dataset = processed_data[0] cdf_filepath = write_cdf(processed_data[0], istp=False) - assert cdf_filepath.name == "imap_hi_l1a_45sensor-diagfee_20250208_v999.9999.cdf" + assert cdf_filepath.name == "imap_hi_l1a_45sensor-diagfee_20250208_v001.0001.cdf" assert np.unique(processed_data[0]["pkt_apid"].values) == HIAPID.H45_DIAG_FEE.value @@ -99,7 +99,7 @@ def test_app_nhk_decom(hi_l0_test_data_path): # Write CDF cem_raw_cdf_filepath = write_cdf(processed_data[0], istp=False) - assert cem_raw_cdf_filepath.name == "imap_hi_l1a_90sensor-hk_20241105_v999.9999.cdf" + assert cem_raw_cdf_filepath.name == "imap_hi_l1a_90sensor-hk_20241105_v001.0001.cdf" validation_df = pd.read_csv(hi_l0_test_data_path / "H90_NHK_20241104_verify.csv") for col_name, series in validation_df.items(): diff --git a/imap_processing/tests/idex/test_idex_l1a.py b/imap_processing/tests/idex/test_idex_l1a.py index f82488d9f7..828423e1cc 100644 --- a/imap_processing/tests/idex/test_idex_l1a.py +++ b/imap_processing/tests/idex/test_idex_l1a.py @@ -55,7 +55,7 @@ def test_idex_cdf_file(decom_test_data_sci: xr.Dataset): file_name = write_cdf(decom_test_data_sci) assert file_name.exists() - assert file_name.name == "imap_idex_l1a_sci-10days_20231218_v999.9999.cdf" + assert file_name.name == "imap_idex_l1a_sci-10days_20231218_v001.0001.cdf" written_dataset = load_cdf(file_name) assert written_dataset["time_low_sample_rate"].attrs["VAR_NOTES"] == ( "The low sample rate is 4.0625 MHz, so adjacent samples are separated " @@ -532,10 +532,10 @@ def test_catlst_dataset(decom_test_data_catlst: list[xr.Dataset]): np.testing.assert_array_equal(ds.epoch, expected_epoch) # Assert that the dataset can be written to a CDF file filename_l1a = write_cdf(decom_test_data_catlst[0]) - assert filename_l1a.name == "imap_idex_l1a_catlst-10days_20241206_v999.9999.cdf" + assert filename_l1a.name == "imap_idex_l1a_catlst-10days_20241206_v001.0001.cdf" filename_l1b = write_cdf(decom_test_data_catlst[1]) - assert filename_l1b.name == "imap_idex_l1b_catlst-10days_20241206_v999.9999.cdf" + assert filename_l1b.name == "imap_idex_l1b_catlst-10days_20241206_v001.0001.cdf" def test_msg_dataset(decom_test_data_msg: xr.Dataset): @@ -556,7 +556,7 @@ def test_msg_dataset(decom_test_data_msg: xr.Dataset): np.testing.assert_array_equal(decom_test_data_msg.epoch, expected_epoch) # Assert that the dataset can be written to a CDF file filename_l1a = write_cdf(decom_test_data_msg) - assert filename_l1a.name == "imap_idex_l1a_msg-10days_20100101_v999.9999.cdf" + assert filename_l1a.name == "imap_idex_l1a_msg-10days_20100101_v001.0001.cdf" # Validate the messages with the IDEX team example data example_data = pd.read_csv( diff --git a/imap_processing/tests/idex/test_idex_l1b.py b/imap_processing/tests/idex/test_idex_l1b.py index e2b590d1e7..2210d923e6 100644 --- a/imap_processing/tests/idex/test_idex_l1b.py +++ b/imap_processing/tests/idex/test_idex_l1b.py @@ -70,7 +70,7 @@ def test_idex_cdf_file(l1b_dataset: xr.Dataset): file_name = write_cdf(l1b_dataset) assert file_name.exists() - assert file_name.name == "imap_idex_l1b_sci-10days_20231218_v999.9999.cdf" + assert file_name.name == "imap_idex_l1b_sci-10days_20231218_v001.0001.cdf" def test_idex_waveform_units(l1b_dataset: xr.Dataset): diff --git a/imap_processing/tests/swapi/test_swapi_l1.py b/imap_processing/tests/swapi/test_swapi_l1.py index a18de5ea63..36437afbf3 100644 --- a/imap_processing/tests/swapi/test_swapi_l1.py +++ b/imap_processing/tests/swapi/test_swapi_l1.py @@ -174,7 +174,7 @@ def test_process_swapi_science(decom_test_data): } # Test CDF File - cdf_filename = "imap_swapi_l1_sci_20240924_v999.9999.cdf" + cdf_filename = "imap_swapi_l1_sci_20240924_v001.0001.cdf" cdf_path = write_cdf(processed_data) assert cdf_path.name == cdf_filename @@ -205,10 +205,10 @@ def first_get_file_paths_side_effect(descriptor): ) processed_data = swapi_l1(collection_obj, descriptor="hk") # hk cdf file - l1a_hk_cdf_filename = "imap_swapi_l1a_hk_20240924_v999.9999.cdf" + l1a_hk_cdf_filename = "imap_swapi_l1a_hk_20240924_v001.0001.cdf" hk_cdf_path = write_cdf(processed_data[0]) assert hk_cdf_path.name == l1a_hk_cdf_filename - l1b_hk_cdf_filename = "imap_swapi_l1b_hk_20240924_v999.9999.cdf" + l1b_hk_cdf_filename = "imap_swapi_l1b_hk_20240924_v001.0001.cdf" l1b_hk_cdf_path = write_cdf(processed_data[1]) assert l1b_hk_cdf_path.name == l1b_hk_cdf_filename @@ -236,6 +236,6 @@ def second_get_file_paths_side_effect(descriptor): assert processed_data[0].attrs["Apid"] == f"{SWAPIAPID.SWP_SCI}" # Test CDF File - cdf_filename = "imap_swapi_l1_sci_20240924_v999.9999.cdf" + cdf_filename = "imap_swapi_l1_sci_20240924_v001.0001.cdf" cdf_path = write_cdf(processed_data[0]) assert cdf_path.name == cdf_filename diff --git a/imap_processing/tests/swapi/test_swapi_l2.py b/imap_processing/tests/swapi/test_swapi_l2.py index fe214f925b..75be5a473e 100644 --- a/imap_processing/tests/swapi/test_swapi_l2.py +++ b/imap_processing/tests/swapi/test_swapi_l2.py @@ -86,7 +86,7 @@ def first_get_file_paths_side_effect(descriptor): ) # Create HK CDF File processed_hk_data = swapi_l1(collection_obj, descriptor="hk") - hk_cdf_filename = "imap_swapi_l1a_hk_20240924_v999.9999.cdf" + hk_cdf_filename = "imap_swapi_l1a_hk_20240924_v001.0001.cdf" hk_cdf_path = write_cdf(processed_hk_data[0]) assert hk_cdf_path.name == hk_cdf_filename @@ -110,7 +110,7 @@ def second_get_file_paths_side_effect(descriptor): ) # Create L1 CDF File processed_sci_data = swapi_l1(collection_obj, descriptor="sci") - cdf_filename = "imap_swapi_l1_sci_20240924_v999.9999.cdf" + cdf_filename = "imap_swapi_l1_sci_20240924_v001.0001.cdf" cdf_path = write_cdf(processed_sci_data[0]) assert cdf_path.name == cdf_filename @@ -127,7 +127,7 @@ def second_get_file_paths_side_effect(descriptor): lut_notes_df=lut_notes_table, ) l2_cdf = write_cdf(l2_dataset) - assert l2_cdf.name == "imap_swapi_l2_sci_20240924_v999.9999.cdf" + assert l2_cdf.name == "imap_swapi_l2_sci_20240924_v001.0001.cdf" cdf_file = cdflib.CDF(l2_cdf) esa_energy_info = cdf_file.varinq("esa_energy") esa_energy_attrs = cdf_file.varattsget("esa_energy") diff --git a/imap_processing/tests/swe/test_swe_l1a.py b/imap_processing/tests/swe/test_swe_l1a.py index 8c5249f303..e95b90be96 100644 --- a/imap_processing/tests/swe/test_swe_l1a.py +++ b/imap_processing/tests/swe/test_swe_l1a.py @@ -9,7 +9,7 @@ def test_cdf_creation(): cem_raw_cdf_filepath = write_cdf(processed_data[0]) - assert cem_raw_cdf_filepath.name == "imap_swe_l1a_sci_20240510_v999.9999.cdf" + assert cem_raw_cdf_filepath.name == "imap_swe_l1a_sci_20240510_v001.0001.cdf" def test_cdf_creation_hk(): @@ -18,7 +18,7 @@ def test_cdf_creation_hk(): hk_cdf_filepath = write_cdf(processed_data[0]) - assert hk_cdf_filepath.name == "imap_swe_l1a_hk_20240510_v999.9999.cdf" + assert hk_cdf_filepath.name == "imap_swe_l1a_hk_20240510_v001.0001.cdf" def test_cdf_creation_cem_raw(): @@ -27,4 +27,4 @@ def test_cdf_creation_cem_raw(): cem_raw_cdf_filepath = write_cdf(processed_data[0]) - assert cem_raw_cdf_filepath.name == "imap_swe_l1a_cem-raw_20240510_v999.9999.cdf" + assert cem_raw_cdf_filepath.name == "imap_swe_l1a_cem-raw_20240510_v001.0001.cdf" diff --git a/imap_processing/tests/swe/test_swe_l1b.py b/imap_processing/tests/swe/test_swe_l1b.py index b80a490582..e88ce7f0ed 100644 --- a/imap_processing/tests/swe/test_swe_l1b.py +++ b/imap_processing/tests/swe/test_swe_l1b.py @@ -289,7 +289,7 @@ def get_file_paths_side_effect(descriptor): hk_datasets = swe_l1b(input_collection) # Looks like HK data has some science data. That's why we use index 1. l1b_hk_filepath = write_cdf(hk_datasets[1]) - assert l1b_hk_filepath.name == "imap_swe_l1b_hk_20240510_v999.9999.cdf" + assert l1b_hk_filepath.name == "imap_swe_l1b_hk_20240510_v001.0001.cdf" def test_count_rate(): diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1a.py b/imap_processing/tests/ultra/unit/test_ultra_l1a.py index 5a1bb489c2..c88f3738a9 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1a.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1a.py @@ -190,7 +190,7 @@ def test_cdf_energy_rates(ccsds_path_functional): assert test_data_path.exists() assert ( test_data_path.name - == "imap_ultra_l1a_45sensor-energy-rates_20240122-repoint99999_v999.9999.cdf" + == "imap_ultra_l1a_45sensor-energy-rates_20240122-repoint99999_v001.0001.cdf" ) # L1b dataset @@ -201,7 +201,7 @@ def test_cdf_energy_rates(ccsds_path_functional): assert test_data_path.exists() assert ( test_data_path.name - == "imap_ultra_l1b_45sensor-energy-rates_20240122-repoint99999_v999.9999.cdf" + == "imap_ultra_l1b_45sensor-energy-rates_20240122-repoint99999_v001.0001.cdf" ) From b8e37c38fde19b31944781d2de74d8c10dba759c Mon Sep 17 00:00:00 2001 From: Vineet Bansal Date: Thu, 18 Jun 2026 23:26:54 -0400 Subject: [PATCH 4/5] changed File_naming_convention in global attrs yaml --- imap_processing/cdf/config/imap_default_global_cdf_attrs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/cdf/config/imap_default_global_cdf_attrs.yaml b/imap_processing/cdf/config/imap_default_global_cdf_attrs.yaml index da0825024a..3457beef02 100644 --- a/imap_processing/cdf/config/imap_default_global_cdf_attrs.yaml +++ b/imap_processing/cdf/config/imap_default_global_cdf_attrs.yaml @@ -8,7 +8,7 @@ Acknowledgement: > Prof. David J. McComas of Princeton University (Contract #80GSFC19C0027). IMAP Rules-of-the-Road for data usage can be found in the IMAP CMAD. Discipline: Solar Physics>Heliospheric Physics -File_naming_convention: source_descriptor_datatype_yyyyMMdd_vNNN +File_naming_convention: source_descriptor_datatype_yyyyMMdd_vMMM.mmmm HTTP_LINK: https://imap.princeton.edu/ LINK_TITLE: IMAP The Interstellar Mapping and Acceleration Probe Mission_group: IMAP From 57a11be4c81c589eec664d1300a32b44bf54ec70 Mon Sep 17 00:00:00 2001 From: Vineet Bansal Date: Thu, 18 Jun 2026 23:34:23 -0400 Subject: [PATCH 5/5] fix failing test --- imap_processing/tests/hit/test_hit_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/tests/hit/test_hit_utils.py b/imap_processing/tests/hit/test_hit_utils.py index 0239472fbf..1ec1fda6cb 100644 --- a/imap_processing/tests/hit/test_hit_utils.py +++ b/imap_processing/tests/hit/test_hit_utils.py @@ -202,7 +202,7 @@ def test_process_housekeeping(housekeeping_dataset, attribute_manager): "Data_version": None, "Descriptor": "HIT>IMAP High-energy Ion Telescope", "Discipline": "Solar Physics>Heliospheric Physics", - "File_naming_convention": "source_descriptor_datatype_yyyyMMdd_vNNN", + "File_naming_convention": "source_descriptor_datatype_yyyyMMdd_vMMM.mmmm", "HTTP_LINK": "https://imap.princeton.edu/", "Instrument_type": "Particles (space)", "LINK_TITLE": "IMAP The Interstellar Mapping and Acceleration Probe",