From 0b2d6dffa7ce66b3af2cca56afb46e2a5bb92b7b Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 11 Jun 2026 14:43:22 -0400 Subject: [PATCH 1/4] fix(codice): address SPDF hi-sectored spin metadata comments (A1.1) For "spin_sector" variable, could you change FORMAT to "I3" to cover FILLVAL=255 (A1.2) This is related to comment (3.2) below. Is it correct that "spin_angle" variable is defined as VAR_TYPE="support_data" and not VAR_TYPE="data". This seems to be the variable that shows what Spin Angle, and is not visible to users if VAR_TYPE="support_data". Implementation notes: - widened spin_sector FORMAT to I3 so FILLVAL=255 is representable. - promoted hi-sectored spin_angle to VAR_TYPE=data with DISPLAY_TYPE=spectrogram so it is visible to users. - broadcast the static spin-angle grid across epoch so the written CDF satisfies the ISTP DEPEND_0/1/2 requirements for a visible data variable. --- ...map_codice_l2-hi-sectored_variable_attrs.yaml | 6 ++++-- imap_processing/codice/codice_l2.py | 16 ++++++++++++++-- .../tests/codice/test_codice_hi_l2.py | 11 ++++++++--- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/imap_processing/cdf/config/imap_codice_l2-hi-sectored_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l2-hi-sectored_variable_attrs.yaml index 305450d02..79e5840c9 100644 --- a/imap_processing/cdf/config/imap_codice_l2-hi-sectored_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l2-hi-sectored_variable_attrs.yaml @@ -68,7 +68,7 @@ spin_sector: CATDESC: Spin Sector Index FIELDNAM: Spin Sector Index FILLVAL: *uint8_fillval - FORMAT: I2 + FORMAT: I3 LABLAXIS: " " SCALETYP: linear UNITS: " " @@ -424,9 +424,11 @@ energy_he3he4_plus: # spin angles: spin_angle: CATDESC: Spin Angle + DEPEND_0: epoch DEPEND_1: spin_sector DEPEND_2: elevation_angle DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:ArrivalDirection,Qualifier:DirectionAngle.AzimuthAngle + DISPLAY_TYPE: spectrogram FIELDNAM: Spin Angle FILLVAL: *real_fillval FORMAT: F32.1 @@ -436,4 +438,4 @@ spin_angle: UNITS: degrees VALIDMAX: 360.0 VALIDMIN: 0.0 - VAR_TYPE: support_data + VAR_TYPE: data diff --git a/imap_processing/codice/codice_l2.py b/imap_processing/codice/codice_l2.py index f6e067ff0..c41a55d94 100644 --- a/imap_processing/codice/codice_l2.py +++ b/imap_processing/codice/codice_l2.py @@ -1113,8 +1113,20 @@ def process_hi_sectored(dependencies: ProcessingInputCollection) -> xr.Dataset: # column remained same but spin angle incremented by 30 degrees for each # elevation angle. spin_angle = spin_angle.T - # Add spin angle variable using the new elevation_angle dimension - l2_dataset["spin_angle"] = (("spin_sector", "elevation_angle"), spin_angle) + # Broadcast the static spin-angle grid across epoch so the variable can be + # written as a visible ISTP data variable with DEPEND_0/1/2. + spin_angle = np.broadcast_to( + spin_angle[np.newaxis, :, :], + ( + l2_dataset.sizes["epoch"], + l2_dataset.sizes["spin_sector"], + l2_dataset.sizes["elevation_angle"], + ), + ) + l2_dataset["spin_angle"] = ( + ("epoch", "spin_sector", "elevation_angle"), + spin_angle, + ) l2_dataset["spin_angle"].attrs = cdf_attrs.get_variable_attributes( "spin_angle", check_schema=False ) diff --git a/imap_processing/tests/codice/test_codice_hi_l2.py b/imap_processing/tests/codice/test_codice_hi_l2.py index 8bc712475..ebfe9a5df 100644 --- a/imap_processing/tests/codice/test_codice_hi_l2.py +++ b/imap_processing/tests/codice/test_codice_hi_l2.py @@ -219,8 +219,10 @@ def test_l2_hi_sectored(mock_get_file_paths): # The external validation file has outdated spin_angle values, but we # still verify structure and basic numeric sanity to guard against # regressions in the spin angle computation. - assert processed_l2[variable].dims == val_data[variable].dims, ( - f"Dimension mismatch in variable '{variable}'" + assert processed_l2[variable].dims == ( + "epoch", + "spin_sector", + "elevation_angle", ) spin_vals = processed_l2[variable].values # All values should be finite and lie within a reasonable angular range. @@ -285,7 +287,10 @@ def test_l2_hi_sectored(mock_get_file_paths): assert data_quality_attrs["VAR_TYPE"] == "data" assert data_quality_attrs["FORMAT"] == "I3" spin_sector_attrs = cdf_file.varattsget("spin_sector") - assert spin_sector_attrs["FORMAT"] == "I2" + assert spin_sector_attrs["FORMAT"] == "I3" + spin_angle_attrs = cdf_file.varattsget("spin_angle") + assert spin_angle_attrs["VAR_TYPE"] == "data" + assert spin_angle_attrs["DISPLAY_TYPE"] == "spectrogram" assert cdf_file.varattsget("energy_h")["FORMAT"] == "F12.6" assert cdf_file.varattsget("energy_h_minus")["FORMAT"] == "F12.6" assert cdf_file.varattsget("energy_h_plus")["FORMAT"] == "F12.6" From 729a8fba0658a498aa2c9c2679fd53dae5aaf402 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 11 Jun 2026 14:43:42 -0400 Subject: [PATCH 2/4] fix(mag): address SPDF RTN direction_label and quality_bitmask comments (A2.1) "direction_label" variable values should be ["B_R", "B_T", "B_N"], or something like that, not ["Bx", "By", "Bz"], since RTN is a special case with named components. And same in imap_mag_l2_burst-rtn_20260422_v002.cdf (A3) For all imap_mag datasets, for "quality_bitmask" variable, FILLVAL attribute value has to be outside of [VALIDMIN, VALIDMAX] range. Now FILLVAL is same as VALIDMAX. Implementation notes: - RTN now writes ["B_R", "B_T", "B_N"], while SRF/DSRF/GSE/GSM keep ["Bx", "By", "Bz"] because those remain Cartesian component labels. - quality_bitmask now writes as CDF_UINT2 with VALIDMIN=0, VALIDMAX=255, and a non-science fill value outside that range. - the final fill value is 65535 rather than 256 because SPDF validation on the written CDF preferred the standard unsigned-16 fill value. - this change affects both MAG L2 and L1D because they share the same dataset-generation path. --- .../config/imap_mag_l2_variable_attrs.yaml | 3 ++- imap_processing/mag/l2/mag_l2_data.py | 8 ++++-- imap_processing/tests/mag/test_mag_l1d.py | 25 +++++++++++++++++++ imap_processing/tests/mag/test_mag_l2.py | 19 ++++++++++++-- 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/imap_processing/cdf/config/imap_mag_l2_variable_attrs.yaml b/imap_processing/cdf/config/imap_mag_l2_variable_attrs.yaml index 0d869c751..bbd8658f8 100644 --- a/imap_processing/cdf/config/imap_mag_l2_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_mag_l2_variable_attrs.yaml @@ -141,9 +141,10 @@ pri_sens: qf_bitmask: <<: *support_default CATDESC: Bitmask indicating when spacecraft related activities influenced the measurement. + CDF_DATA_TYPE: CDF_UINT2 DICT_KEY: SPASE>Support>SupportQuantity:DataQuality FIELDNAM: Quality Bitmask - FILLVAL: 255 + FILLVAL: 65535 FORMAT: I3 LABLAXIS: QB VALIDMAX: 255 diff --git a/imap_processing/mag/l2/mag_l2_data.py b/imap_processing/mag/l2/mag_l2_data.py index ea5bf6f80..beee1f2b4 100644 --- a/imap_processing/mag/l2/mag_l2_data.py +++ b/imap_processing/mag/l2/mag_l2_data.py @@ -212,7 +212,11 @@ def generate_dataset( ) direction_label = xr.DataArray( - np.array(["Bx", "By", "Bz"]), + np.array( + ["B_R", "B_T", "B_N"] + if self.frame == ValidFrames.RTN + else ["Bx", "By", "Bz"] + ), name="direction_label", dims=["direction_label"], attrs=attribute_manager.get_variable_attributes( @@ -246,7 +250,7 @@ def generate_dataset( ) quality_bitmask = xr.DataArray( - self.quality_bitmask, + self.quality_bitmask.astype(np.uint16), name="quality_bitmask", dims=["epoch"], attrs=attribute_manager.get_variable_attributes("qf_bitmask"), diff --git a/imap_processing/tests/mag/test_mag_l1d.py b/imap_processing/tests/mag/test_mag_l1d.py index 2ebe5efe7..6550a0c86 100644 --- a/imap_processing/tests/mag/test_mag_l1d.py +++ b/imap_processing/tests/mag/test_mag_l1d.py @@ -1,6 +1,7 @@ import logging from unittest.mock import patch +import cdflib import numpy as np import pytest import xarray as xr @@ -548,6 +549,30 @@ def test_mago_magi_no_swap_functionality(mag_l1d_test_class): assert np.array_equal(result["epoch"].data, mago_epoch) +def test_mag_l1d_rtn_direction_label_written_cdf(mag_l1d_test_class): + """Test that shared MAG L1D metadata writes RTN component labels.""" + mag_l1d_test_class.frame = ValidFrames.RTN + + with patch( + "imap_processing.mag.l1d.mag_l1d_data.MagL2L1dBase.truncate_to_24h", + return_value=None, + ): + attributes = ImapCdfAttributes() + attributes.add_instrument_global_attrs("mag") + attributes.add_instrument_variable_attrs("mag", "l2") + + result = mag_l1d_test_class.generate_dataset( + attributes, np.datetime64("2000-01-01") + ) + + result.attrs["Data_version"] = "001" + cdf_filepath = write_cdf(result) + with cdflib.CDF(cdf_filepath) as cdf_file: + direction_label = cdf_file.varget("direction_label") + + np.testing.assert_array_equal(direction_label, np.array(["B_R", "B_T", "B_N"])) + + def test_enhanced_gradiometry_with_quality_flags_detailed(): """Test enhanced gradiometry calculation with quality flags and magnitude.""" # Test data with known differences diff --git a/imap_processing/tests/mag/test_mag_l2.py b/imap_processing/tests/mag/test_mag_l2.py index 03a0688e3..0520b62cf 100644 --- a/imap_processing/tests/mag/test_mag_l2.py +++ b/imap_processing/tests/mag/test_mag_l2.py @@ -130,9 +130,14 @@ def test_mag_l2_attributes( dataset["magnitude"].attrs["CATDESC"] == "Magnitude of the magnetic field" ) assert dataset["range"].attrs["CATDESC"] == "Range of the magnetometer sensor" + expected_direction_label = ( + np.array(["B_R", "B_T", "B_N"]) + if frame == "RTN" + else np.array(["Bx", "By", "Bz"]) + ) np.testing.assert_array_equal( dataset["direction_label"].data, - np.array(["Bx", "By", "Bz"]), + expected_direction_label, ) assert dataset["range"].attrs["DICT_KEY"] == ( "SPASE>Support>SupportQuantity:InstrumentMode" @@ -183,9 +188,14 @@ def test_mag_l2(norm_dataset, mag_test_l2_data): assert np.isclose(vector_attrs["VALIDMIN"], np.float32(-1.0e5)) assert np.isclose(vector_attrs["VALIDMAX"], np.float32(1.0e5)) assert vector_attrs["UNITS"] == "nT" + expected_direction_label = ( + np.array(["B_R", "B_T", "B_N"]) + if expected_frames[i] == ValidFrames.RTN + else np.array(["Bx", "By", "Bz"]) + ) np.testing.assert_array_equal( direction_label, - np.array(["Bx", "By", "Bz"]), + expected_direction_label, ) @@ -642,6 +652,7 @@ def test_qf(norm_dataset): ) assert output["quality_bitmask"].attrs["FIELDNAM"] == "Quality Bitmask" assert output["quality_bitmask"].attrs["LABLAXIS"] == "QB" + assert output["quality_bitmask"].dtype == np.uint16 assert output["quality_bitmask"].attrs["CATDESC"] == ( "Bitmask indicating when spacecraft related activities influenced " "the measurement." @@ -660,10 +671,14 @@ def test_qf(norm_dataset): with cdflib.CDF(cdf_filepath) as cdf_file: qf_attrs = cdf_file.varattsget("quality_flags") qf_bitmask_attrs = cdf_file.varattsget("quality_bitmask") + qf_bitmask_info = cdf_file.varinq("quality_bitmask") assert qf_attrs["FORMAT"] == "I1" assert int(qf_attrs["VALIDMAX"]) == 1 + assert qf_bitmask_info.Data_Type_Description == "CDF_UINT2" assert qf_bitmask_attrs["FORMAT"] == "I3" + assert int(qf_bitmask_attrs["FILLVAL"]) == 65535 + assert int(qf_bitmask_attrs["VALIDMIN"]) == 0 assert int(qf_bitmask_attrs["VALIDMAX"]) == 255 From 195d0e88220e045c74ad0c241113e7f1d1a07a38 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 11 Jun 2026 14:44:00 -0400 Subject: [PATCH 3/4] fix(swapi): address SPDF esa_energy dependency comment (A4.1) Related to (12.2) below. I see that you now have "esa_energy" variable defined as VAR_TYPE="data", so visible to users. But that "esa_energy" variable (time varying 1-D, size 72) could also be used as DEPEND_1="esa_energy", instead of DEPEND_1="esa_step" (time non-varying 1-D, size 72). Depend variables in ISTP can be time-varying (1-D + time). Implementation notes: - swp_l1a_flags and the L2 rate/rate-uncertainty variables now use the time-varying esa_energy axis as DEPEND_1. - esa_energy itself remains indexed by esa_step because it is still the per-sweep mapping from step id to energy. - this was implemented in swapi_l2.py rather than by changing the shared SWAPI YAML defaults, because the shared defaults are also used by the L1 science writer and changing them there broke the L1 CDF path. --- imap_processing/swapi/l2/swapi_l2.py | 15 +++++++++++++++ imap_processing/tests/swapi/test_swapi_l2.py | 6 ++++++ 2 files changed, 21 insertions(+) diff --git a/imap_processing/swapi/l2/swapi_l2.py b/imap_processing/swapi/l2/swapi_l2.py index 5440e8293..e6f8d9a60 100644 --- a/imap_processing/swapi/l2/swapi_l2.py +++ b/imap_processing/swapi/l2/swapi_l2.py @@ -290,6 +290,21 @@ def swapi_l2( "swp_coin_rate_stat_uncert_minus" ].attrs = cdf_manager.get_variable_attributes("swp_coin_rate_stat_uncert_minus") + depend_on_esa_energy = [ + "swp_l1a_flags", + "swp_pcem_rate", + "swp_scem_rate", + "swp_coin_rate", + "swp_pcem_rate_stat_uncert_plus", + "swp_pcem_rate_stat_uncert_minus", + "swp_scem_rate_stat_uncert_plus", + "swp_scem_rate_stat_uncert_minus", + "swp_coin_rate_stat_uncert_plus", + "swp_coin_rate_stat_uncert_minus", + ] + for variable in depend_on_esa_energy: + l2_dataset[variable].attrs["DEPEND_1"] = "esa_energy" + # TODO: add thruster firing flag # TODO: add other flags logger.info("SWAPI L2 processing complete") diff --git a/imap_processing/tests/swapi/test_swapi_l2.py b/imap_processing/tests/swapi/test_swapi_l2.py index af953a7b4..065f24764 100644 --- a/imap_processing/tests/swapi/test_swapi_l2.py +++ b/imap_processing/tests/swapi/test_swapi_l2.py @@ -141,6 +141,12 @@ def second_get_file_paths_side_effect(descriptor): assert esa_energy_attrs["VALIDMIN"] == np.float64(0.0) assert esa_energy_attrs["VAR_TYPE"] == "data" assert esa_energy_attrs["DEPEND_1"] == "esa_step" + assert cdf_file.varattsget("swp_l1a_flags")["DEPEND_1"] == "esa_energy" + assert cdf_file.varattsget("swp_pcem_rate")["DEPEND_1"] == "esa_energy" + assert ( + cdf_file.varattsget("swp_pcem_rate_stat_uncert_plus")["DEPEND_1"] + == "esa_energy" + ) assert esa_step_attrs["SCALETYP"] == "linear" assert "SCALE_TYP" not in esa_step_attrs assert esa_energy_attrs["CATDESC"] == ( From fc6f9f7c6b83a337a3b77188a7b920cf2713da4e Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 12 Jun 2026 00:25:17 -0400 Subject: [PATCH 4/4] fix(codice): keep hi-sectored spin_angle as support_data SPDF follow-up: Thanks, Dave. For (A1.2) comment, I missed that "spin_angle" variable is time non-varying, and cannot be easily changed to VAR_TYPE="data", so no change is needed in this case. Implementation notes: - restore hi-sectored spin_angle as a static 2-D support_data variable - remove the epoch broadcast and visible-data metadata that were added for the earlier interpretation of (A1.2) - keep the separate spin_sector FORMAT=I3 fix from (A1.1) - retain the focused test guard for spin_angle because the checked-in validation CDF still carries outdated spin_angle values --- ..._codice_l2-hi-sectored_variable_attrs.yaml | 4 +--- imap_processing/codice/codice_l2.py | 15 +------------ .../tests/codice/test_codice_hi_l2.py | 22 +++++++------------ 3 files changed, 10 insertions(+), 31 deletions(-) diff --git a/imap_processing/cdf/config/imap_codice_l2-hi-sectored_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l2-hi-sectored_variable_attrs.yaml index 79e5840c9..6eb25485b 100644 --- a/imap_processing/cdf/config/imap_codice_l2-hi-sectored_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l2-hi-sectored_variable_attrs.yaml @@ -424,11 +424,9 @@ energy_he3he4_plus: # spin angles: spin_angle: CATDESC: Spin Angle - DEPEND_0: epoch DEPEND_1: spin_sector DEPEND_2: elevation_angle DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:ArrivalDirection,Qualifier:DirectionAngle.AzimuthAngle - DISPLAY_TYPE: spectrogram FIELDNAM: Spin Angle FILLVAL: *real_fillval FORMAT: F32.1 @@ -438,4 +436,4 @@ spin_angle: UNITS: degrees VALIDMAX: 360.0 VALIDMIN: 0.0 - VAR_TYPE: data + VAR_TYPE: support_data diff --git a/imap_processing/codice/codice_l2.py b/imap_processing/codice/codice_l2.py index c41a55d94..a4ad81a41 100644 --- a/imap_processing/codice/codice_l2.py +++ b/imap_processing/codice/codice_l2.py @@ -1113,20 +1113,7 @@ def process_hi_sectored(dependencies: ProcessingInputCollection) -> xr.Dataset: # column remained same but spin angle incremented by 30 degrees for each # elevation angle. spin_angle = spin_angle.T - # Broadcast the static spin-angle grid across epoch so the variable can be - # written as a visible ISTP data variable with DEPEND_0/1/2. - spin_angle = np.broadcast_to( - spin_angle[np.newaxis, :, :], - ( - l2_dataset.sizes["epoch"], - l2_dataset.sizes["spin_sector"], - l2_dataset.sizes["elevation_angle"], - ), - ) - l2_dataset["spin_angle"] = ( - ("epoch", "spin_sector", "elevation_angle"), - spin_angle, - ) + l2_dataset["spin_angle"] = (("spin_sector", "elevation_angle"), spin_angle) l2_dataset["spin_angle"].attrs = cdf_attrs.get_variable_attributes( "spin_angle", check_schema=False ) diff --git a/imap_processing/tests/codice/test_codice_hi_l2.py b/imap_processing/tests/codice/test_codice_hi_l2.py index ebfe9a5df..7713dda85 100644 --- a/imap_processing/tests/codice/test_codice_hi_l2.py +++ b/imap_processing/tests/codice/test_codice_hi_l2.py @@ -216,28 +216,23 @@ def test_l2_hi_sectored(mock_get_file_paths): if variable.startswith("unc_"): continue if variable == "spin_angle": - # The external validation file has outdated spin_angle values, but we - # still verify structure and basic numeric sanity to guard against - # regressions in the spin angle computation. assert processed_l2[variable].dims == ( - "epoch", "spin_sector", "elevation_angle", ) spin_vals = processed_l2[variable].values - # All values should be finite and lie within a reasonable angular range. assert np.all(np.isfinite(spin_vals)), ( "spin_angle contains non-finite values" ) assert np.min(spin_vals) >= 0.0, "spin_angle has values below 0 degrees" assert np.max(spin_vals) <= 360.0, "spin_angle has values above 360 degrees" - continue - np.testing.assert_allclose( - processed_l2[variable].values, - val_data[variable].values, - rtol=1e-5, - err_msg=f"Mismatch in variable '{variable}'", - ) + else: + np.testing.assert_allclose( + processed_l2[variable].values, + val_data[variable].values, + rtol=1e-5, + err_msg=f"Mismatch in variable '{variable}'", + ) # Tests that dimensions match if variable in ["epoch_delta_plus", "epoch_delta_minus"]: continue @@ -289,8 +284,7 @@ def test_l2_hi_sectored(mock_get_file_paths): spin_sector_attrs = cdf_file.varattsget("spin_sector") assert spin_sector_attrs["FORMAT"] == "I3" spin_angle_attrs = cdf_file.varattsget("spin_angle") - assert spin_angle_attrs["VAR_TYPE"] == "data" - assert spin_angle_attrs["DISPLAY_TYPE"] == "spectrogram" + assert spin_angle_attrs["VAR_TYPE"] == "support_data" assert cdf_file.varattsget("energy_h")["FORMAT"] == "F12.6" assert cdf_file.varattsget("energy_h_minus")["FORMAT"] == "F12.6" assert cdf_file.varattsget("energy_h_plus")["FORMAT"] == "F12.6"