From 6ecfa77894368af83c4d5fb068813fb254b0b155 Mon Sep 17 00:00:00 2001 From: Tenzin Choedon Date: Thu, 4 Jun 2026 15:37:16 -0600 Subject: [PATCH 1/4] CoDICE: L1A bug that caused l1b to fail for CoDICE Lo --- imap_processing/codice/utils.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/imap_processing/codice/utils.py b/imap_processing/codice/utils.py index c4854f428..c888de678 100644 --- a/imap_processing/codice/utils.py +++ b/imap_processing/codice/utils.py @@ -246,7 +246,17 @@ def process_by_table_id( ] if len(processed) == 1: return processed[0] - return xr.concat(processed, dim="epoch").sortby("epoch") + # Keep non-epoch support variables as it is, 1-D arrays, + # instead of expanding them along the epoch dimension. + # Eg. voltage_table and k_factor are 1-D arrays that apply to all + # epochs in the same way. + return xr.concat( + processed, + dim="epoch", + data_vars="minimal", + coords="minimal", + compat="equals", + ).sortby("epoch") def get_collapse_pattern_shape( From 94ae3b92e1c316bd523b56fc7092b3f84ba1d07f Mon Sep 17 00:00:00 2001 From: Tenzin Choedon Date: Thu, 4 Jun 2026 16:34:44 -0600 Subject: [PATCH 2/4] test for new code --- .../tests/codice/test_codice_l1a.py | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index 82d33dd8e..b328bd86a 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -9,18 +9,20 @@ """ import logging +from pathlib import Path from unittest.mock import patch import cdflib import numpy as np import pytest +import xarray as xr from imap_data_access import ProcessingInputCollection from imap_processing import imap_module_directory from imap_processing.cdf.utils import load_cdf, write_cdf from imap_processing.codice.codice_l1a import process_l1a from imap_processing.codice.codice_l1a_de import l1a_direct_event -from imap_processing.codice.utils import CODICEAPID +from imap_processing.codice.utils import CODICEAPID, process_by_table_id from imap_processing.tests.codice.conftest import ( VALIDATION_FILE_DATE, VALIDATION_FILE_VERSION, @@ -36,6 +38,40 @@ EXPECTED_EPOCH_DELTA_VALIDMAX = 128000000000 +def test_process_by_table_id_keeps_voltage_table_1d(): + """Shared support data should not be expanded across epoch groups.""" + + unpacked_dataset = xr.Dataset( + data_vars={ + "view_id": ("epoch", np.array([0, 0], dtype=np.int16)), + "pkt_apid": ("epoch", np.array([1156, 1156], dtype=np.int16)), + "plan_id": ("epoch", np.array([7, 7], dtype=np.int16)), + "plan_step": ("epoch", np.array([3, 3], dtype=np.int16)), + "table_id": ("epoch", np.array([1, 2], dtype=np.int16)), + "sample": ("epoch", np.array([11, 22], dtype=np.int16)), + }, + coords={"epoch": np.array([20, 10], dtype=np.int64)}, + ) + + voltage_table = np.array([1.0, 2.0, 3.0], dtype=np.float64) + + def _process_fn(group_ds, lut_file, table_id, view_id, apid, plan_id, plan_step): + # Mock processed data with non-epoch data variables + return xr.Dataset( + data_vars={ + "sample": ("epoch", group_ds["sample"].values), + "voltage_table": ("esa_step", voltage_table), + }, + coords={"epoch": group_ds["epoch"].values, "esa_step": np.arange(3)}, + ) + + processed = process_by_table_id(unpacked_dataset, Path("unused.json"), _process_fn) + + assert processed["epoch"].values.tolist() == [10, 20] + assert processed["voltage_table"].dims == ("esa_step",) + np.testing.assert_array_equal(processed["voltage_table"].values, voltage_table) + + def assert_epoch_delta_cdf_metadata(cdf_file): """Assert the written epoch delta variables use integer duration metadata.""" with cdflib.CDF(cdf_file) as cdf: From de5280feece3a22eb5106e3868f5899302f32f34 Mon Sep 17 00:00:00 2001 From: Tenzin Choedon Date: Thu, 4 Jun 2026 16:50:30 -0600 Subject: [PATCH 3/4] one more try to get codecove/patch to be higher --- imap_processing/codice/constants.py | 41 ++++++++++++++++++- imap_processing/codice/utils.py | 40 +----------------- .../tests/codice/test_codice_l1a.py | 38 +---------------- .../tests/codice/test_process_by_table_id.py | 41 +++++++++++++++++++ 4 files changed, 83 insertions(+), 77 deletions(-) create mode 100644 imap_processing/tests/codice/test_process_by_table_id.py diff --git a/imap_processing/codice/constants.py b/imap_processing/codice/constants.py index 49eed824e..e0604bc56 100644 --- a/imap_processing/codice/constants.py +++ b/imap_processing/codice/constants.py @@ -13,11 +13,50 @@ ESA = ElectroStatic Analyzer """ +from enum import IntEnum from typing import Any import numpy as np -from imap_processing.codice.utils import CODICEAPID + +class CODICEAPID(IntEnum): + """Create ENUM for CoDICE APIDs.""" + + COD_AUT = 1120 + COD_BOOT_HK = 1121 + COD_BOOT_MEMDMP = 1122 + COD_COUNTS_COMMON = 1135 + COD_NHK = 1136 + COD_EVTMSG = 1137 + COD_MEMDMP = 1138 + COD_SHK = 1139 + COD_RTS = 1141 + COD_DIAG_CDHFPGA = 1144 + COD_DIAG_SNSR_HV = 1145 + COD_DIAG_OPTC_HV = 1146 + COD_DIAG_APDFPGA = 1147 + COD_DIAG_SSDFPGA = 1148 + COD_DIAG_FSW = 1149 + COD_DIAG_SYSVARS = 1150 + COD_LO_IAL = 1152 + COD_LO_PHA = 1153 + COD_LO_SW_PRIORITY_COUNTS = 1155 + COD_LO_SW_SPECIES_COUNTS = 1156 + COD_LO_NSW_SPECIES_COUNTS = 1157 + COD_LO_SW_ANGULAR_COUNTS = 1158 + COD_LO_NSW_ANGULAR_COUNTS = 1159 + COD_LO_NSW_PRIORITY_COUNTS = 1160 + COD_LO_INST_COUNTS_AGGREGATED = 1161 + COD_LO_INST_COUNTS_SINGLES = 1162 + COD_HI_IAL = 1168 + COD_HI_PHA = 1169 + COD_HI_INST_COUNTS_AGGREGATED = 1170 + COD_HI_INST_COUNTS_SINGLES = 1171 + COD_HI_OMNI_SPECIES_COUNTS = 1172 + COD_HI_SECT_SPECIES_COUNTS = 1173 + COD_HI_INST_COUNTS_PRIORITIES = 1174 + COD_CSTOL_CONFIG = 2457 + # -------L1A Constants------- # Numerical constants diff --git a/imap_processing/codice/utils.py b/imap_processing/codice/utils.py index c888de678..52527c018 100644 --- a/imap_processing/codice/utils.py +++ b/imap_processing/codice/utils.py @@ -15,6 +15,7 @@ import xarray as xr from imap_processing.codice import constants +from imap_processing.codice.constants import CODICEAPID @dataclass @@ -46,45 +47,6 @@ class ViewTabInfo: view_id: int -class CODICEAPID(IntEnum): - """Create ENUM for CoDICE APIDs.""" - - COD_AUT = 1120 - COD_BOOT_HK = 1121 - COD_BOOT_MEMDMP = 1122 - COD_COUNTS_COMMON = 1135 - COD_NHK = 1136 - COD_EVTMSG = 1137 - COD_MEMDMP = 1138 - COD_SHK = 1139 - COD_RTS = 1141 - COD_DIAG_CDHFPGA = 1144 - COD_DIAG_SNSR_HV = 1145 - COD_DIAG_OPTC_HV = 1146 - COD_DIAG_APDFPGA = 1147 - COD_DIAG_SSDFPGA = 1148 - COD_DIAG_FSW = 1149 - COD_DIAG_SYSVARS = 1150 - COD_LO_IAL = 1152 - COD_LO_PHA = 1153 - COD_LO_SW_PRIORITY_COUNTS = 1155 - COD_LO_SW_SPECIES_COUNTS = 1156 - COD_LO_NSW_SPECIES_COUNTS = 1157 - COD_LO_SW_ANGULAR_COUNTS = 1158 - COD_LO_NSW_ANGULAR_COUNTS = 1159 - COD_LO_NSW_PRIORITY_COUNTS = 1160 - COD_LO_INST_COUNTS_AGGREGATED = 1161 - COD_LO_INST_COUNTS_SINGLES = 1162 - COD_HI_IAL = 1168 - COD_HI_PHA = 1169 - COD_HI_INST_COUNTS_AGGREGATED = 1170 - COD_HI_INST_COUNTS_SINGLES = 1171 - COD_HI_OMNI_SPECIES_COUNTS = 1172 - COD_HI_SECT_SPECIES_COUNTS = 1173 - COD_HI_INST_COUNTS_PRIORITIES = 1174 - COD_CSTOL_CONFIG = 2457 - - class CoDICECompression(IntEnum): """Create ENUM for CoDICE compression algorithms.""" diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index b328bd86a..82d33dd8e 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -9,20 +9,18 @@ """ import logging -from pathlib import Path from unittest.mock import patch import cdflib import numpy as np import pytest -import xarray as xr from imap_data_access import ProcessingInputCollection from imap_processing import imap_module_directory from imap_processing.cdf.utils import load_cdf, write_cdf from imap_processing.codice.codice_l1a import process_l1a from imap_processing.codice.codice_l1a_de import l1a_direct_event -from imap_processing.codice.utils import CODICEAPID, process_by_table_id +from imap_processing.codice.utils import CODICEAPID from imap_processing.tests.codice.conftest import ( VALIDATION_FILE_DATE, VALIDATION_FILE_VERSION, @@ -38,40 +36,6 @@ EXPECTED_EPOCH_DELTA_VALIDMAX = 128000000000 -def test_process_by_table_id_keeps_voltage_table_1d(): - """Shared support data should not be expanded across epoch groups.""" - - unpacked_dataset = xr.Dataset( - data_vars={ - "view_id": ("epoch", np.array([0, 0], dtype=np.int16)), - "pkt_apid": ("epoch", np.array([1156, 1156], dtype=np.int16)), - "plan_id": ("epoch", np.array([7, 7], dtype=np.int16)), - "plan_step": ("epoch", np.array([3, 3], dtype=np.int16)), - "table_id": ("epoch", np.array([1, 2], dtype=np.int16)), - "sample": ("epoch", np.array([11, 22], dtype=np.int16)), - }, - coords={"epoch": np.array([20, 10], dtype=np.int64)}, - ) - - voltage_table = np.array([1.0, 2.0, 3.0], dtype=np.float64) - - def _process_fn(group_ds, lut_file, table_id, view_id, apid, plan_id, plan_step): - # Mock processed data with non-epoch data variables - return xr.Dataset( - data_vars={ - "sample": ("epoch", group_ds["sample"].values), - "voltage_table": ("esa_step", voltage_table), - }, - coords={"epoch": group_ds["epoch"].values, "esa_step": np.arange(3)}, - ) - - processed = process_by_table_id(unpacked_dataset, Path("unused.json"), _process_fn) - - assert processed["epoch"].values.tolist() == [10, 20] - assert processed["voltage_table"].dims == ("esa_step",) - np.testing.assert_array_equal(processed["voltage_table"].values, voltage_table) - - def assert_epoch_delta_cdf_metadata(cdf_file): """Assert the written epoch delta variables use integer duration metadata.""" with cdflib.CDF(cdf_file) as cdf: diff --git a/imap_processing/tests/codice/test_process_by_table_id.py b/imap_processing/tests/codice/test_process_by_table_id.py new file mode 100644 index 000000000..3fe28d21d --- /dev/null +++ b/imap_processing/tests/codice/test_process_by_table_id.py @@ -0,0 +1,41 @@ +"""Unit tests for CoDICE table-id grouping helpers.""" + +from pathlib import Path + +import numpy as np +import xarray as xr + +from imap_processing.codice.utils import process_by_table_id + + +def test_process_by_table_id_keeps_voltage_table_1d(): + """Shared support data should not be expanded across epoch groups.""" + + unpacked_dataset = xr.Dataset( + data_vars={ + "view_id": ("epoch", np.array([0, 0], dtype=np.int16)), + "pkt_apid": ("epoch", np.array([1156, 1156], dtype=np.int16)), + "plan_id": ("epoch", np.array([7, 7], dtype=np.int16)), + "plan_step": ("epoch", np.array([3, 3], dtype=np.int16)), + "table_id": ("epoch", np.array([1, 2], dtype=np.int16)), + "sample": ("epoch", np.array([11, 22], dtype=np.int16)), + }, + coords={"epoch": np.array([20, 10], dtype=np.int64)}, + ) + + voltage_table = np.array([1.0, 2.0, 3.0], dtype=np.float64) + + def _process_fn(group_ds, lut_file, table_id, view_id, apid, plan_id, plan_step): + return xr.Dataset( + data_vars={ + "sample": ("epoch", group_ds["sample"].values), + "voltage_table": ("esa_step", voltage_table), + }, + coords={"epoch": group_ds["epoch"].values, "esa_step": np.arange(3)}, + ) + + processed = process_by_table_id(unpacked_dataset, Path("unused.json"), _process_fn) + + assert processed["epoch"].values.tolist() == [10, 20] + assert processed["voltage_table"].dims == ("esa_step",) + np.testing.assert_array_equal(processed["voltage_table"].values, voltage_table) From 96bb1442647e1de4e0c5336082ca254fad5e0134 Mon Sep 17 00:00:00 2001 From: Tenzin Choedon Date: Thu, 4 Jun 2026 16:58:43 -0600 Subject: [PATCH 4/4] pre-commit fixes --- imap_processing/codice/codice_l1a.py | 6 ++---- imap_processing/codice/codice_l1a_de.py | 2 +- imap_processing/codice/codice_l1a_lo_priority.py | 3 +-- imap_processing/codice/codice_l1a_lo_species.py | 2 +- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index 794c09730..fba72d288 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -27,10 +27,8 @@ ) from imap_processing.codice.codice_l1a_lo_priority import l1a_lo_priority from imap_processing.codice.codice_l1a_lo_species import l1a_lo_species -from imap_processing.codice.utils import ( - CODICEAPID, - process_by_table_id, -) +from imap_processing.codice.constants import CODICEAPID +from imap_processing.codice.utils import process_by_table_id from imap_processing.utils import packet_file_to_datasets logger = logging.getLogger(__name__) diff --git a/imap_processing/codice/codice_l1a_de.py b/imap_processing/codice/codice_l1a_de.py index 25e437594..5fbba71f8 100644 --- a/imap_processing/codice/codice_l1a_de.py +++ b/imap_processing/codice/codice_l1a_de.py @@ -7,9 +7,9 @@ from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes from imap_processing.codice import constants +from imap_processing.codice.constants import CODICEAPID from imap_processing.codice.decompress import decompress from imap_processing.codice.utils import ( - CODICEAPID, CoDICECompression, ViewTabInfo, apply_replacements_to_attrs, diff --git a/imap_processing/codice/codice_l1a_lo_priority.py b/imap_processing/codice/codice_l1a_lo_priority.py index 34a32349a..1603d8b18 100644 --- a/imap_processing/codice/codice_l1a_lo_priority.py +++ b/imap_processing/codice/codice_l1a_lo_priority.py @@ -8,10 +8,9 @@ from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes from imap_processing.codice import constants -from imap_processing.codice.constants import HALF_SPIN_FILLVAL +from imap_processing.codice.constants import CODICEAPID, HALF_SPIN_FILLVAL from imap_processing.codice.decompress import decompress from imap_processing.codice.utils import ( - CODICEAPID, CoDICECompression, calculate_acq_time_per_step, get_codice_epoch_time, diff --git a/imap_processing/codice/codice_l1a_lo_species.py b/imap_processing/codice/codice_l1a_lo_species.py index d5f9fbeef..187d0aa2c 100644 --- a/imap_processing/codice/codice_l1a_lo_species.py +++ b/imap_processing/codice/codice_l1a_lo_species.py @@ -9,13 +9,13 @@ from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes from imap_processing.codice import constants from imap_processing.codice.constants import ( + CODICEAPID, HALF_SPIN_FILLVAL, LO_IALIRT_VARIABLE_NAMES, LO_SW_SPECIES_VARIABLE_NAMES, ) from imap_processing.codice.decompress import decompress from imap_processing.codice.utils import ( - CODICEAPID, CoDICECompression, calculate_acq_time_per_step, get_codice_epoch_time,