Lo L1A - Refactor combin_segemented_packets to handle MET (IMAP-Science-Operations-Center#2415)

sdhoyt · web-flow · commit 2b5e21773e7b · 2025-11-11T16:18:40.000-07:00
* refactored combine segmented packets

* added log per packet

* removed check for unsegemented packets
diff --git a/imap_processing/lo/l0/lo_science.py b/imap_processing/lo/l0/lo_science.py
@@ -241,6 +241,10 @@ def parse_events(dataset: xr.Dataset, attr_mgr: ImapCdfAttributes) -> xr.Dataset
     pointing_de = 0
 
     for pkt_idx, de_count in enumerate(de_count_values):
+        logger.info(
+            f"Parsing packet {pkt_idx} of {len(de_count_values)} "
+            f"with {de_count} direct events"
+        )
         raw_data = data_values[pkt_idx]
 
         # Parse all direct events in this packet using bytewise operations
@@ -358,13 +362,16 @@ def extract_bits_from_bytes(
 
 def combine_segmented_packets(dataset: xr.Dataset) -> xr.Dataset:
     """
-    Combine segmented packets.
+    Combine segmented packets and set MET field.
 
     If the number of bits needed to pack the direct events exceeds the
     maximum number of bits allowed in a packet, the direct events
     will be spread across multiple packets. This function will combine
     the segmented binary into a single binary string for each epoch.
 
+    This function also sets the MET field based on segment start times,
+    even when no segmentation is present.
+
     Parameters
     ----------
     dataset : xr.Dataset
@@ -373,7 +380,7 @@ def combine_segmented_packets(dataset: xr.Dataset) -> xr.Dataset:
     Returns
     -------
     dataset : xr.Dataset
-        Updated dataset with any segmented direct events combined.
+        Updated dataset with any segmented direct events combined and MET field set.
     """
     seq_flgs = dataset.seq_flgs.values
     seq_ctrs = dataset.src_seq_ctr.values
@@ -384,21 +391,19 @@ def combine_segmented_packets(dataset: xr.Dataset) -> xr.Dataset:
     # 3 = unsegmented packet
     seg_starts = np.nonzero((seq_flgs == 1) | (seq_flgs == 3))[0]
     seg_ends = np.nonzero((seq_flgs == 2) | (seq_flgs == 3))[0]
+
     # Swap the epoch dimension for the shcoarse
     # the epoch dimension will be reduced to the
     # first epoch in each segment
     dataset.coords["shcoarse"] = dataset["shcoarse"]
     dataset = dataset.swap_dims({"epoch": "shcoarse"})
 
     # Find the valid groups of segmented packets
-    # returns a list of booleans for each group of segmented packets
-    # where true means the group is valid
     valid_groups = find_valid_groups(seq_ctrs, seg_starts, seg_ends)
 
     # Combine the segmented packets into raw bytes directly
     combined_data_list = []
     for start, end in zip(seg_starts, seg_ends, strict=False):
-        # Concatenate raw bytes data directly
         combined_bytes = b"".join(dataset["data"].values[start : end + 1])
         combined_data_list.append(combined_bytes)
 
@@ -414,11 +419,11 @@ def combine_segmented_packets(dataset: xr.Dataset) -> xr.Dataset:
 
     # Create the data DataArray with combined raw bytes
     dataset["data"] = xr.DataArray(
-        valid_combined_data, dims=["epoch"], coords={"epoch": dataset.coords["epoch"]}
+        valid_combined_data,
+        dims=["epoch"],
+        coords={"epoch": dataset.coords["epoch"]},
     )
-    # Set met to the first segment start times for the valid groups.
-    # shcoarse will be retained as a per packet coordinate and met
-    # is used as the mission elapsed time for each segment
+    # Set met to the first segment start times for the valid groups
     dataset["met"] = xr.DataArray(
         dataset["shcoarse"].values[seg_starts][valid_groups], dims="epoch"
     )
diff --git a/imap_processing/lo/l1a/lo_l1a.py b/imap_processing/lo/l1a/lo_l1a.py
@@ -86,12 +86,9 @@ def lo_l1a(dependency: Path) -> list[xr.Dataset]:
         logical_source = "imap_lo_l1a_de"
         ds = datasets_by_apid[LoAPID.ILO_SCI_DE]
 
-        # Check if we need to combine segmented packets first
-        needs_combine = not all(ds.seq_flgs.values == 3)  # 3 = unsegmented
-
-        if needs_combine:
-            # For segmented packets, combine raw bytes directly
-            ds = combine_segmented_packets(ds)
+        # For segmented packets, combine raw bytes directly
+        # Always call combine_segmented_packets to set MET field
+        ds = combine_segmented_packets(ds)
 
         ds = parse_events(ds, attr_mgr)
         ds = add_dataset_attrs(ds, attr_mgr, logical_source)
diff --git a/imap_processing/tests/lo/test_lo_science.py b/imap_processing/tests/lo/test_lo_science.py
@@ -227,6 +227,18 @@ def test_combine_segmented_packets(segmented_pkts_fake_data):
     np.testing.assert_array_equal(dataset["met"].values, np.array([0, 10, 30]))
 
 
+def test_combin_segmented_packets_only_met(segmented_pkts_fake_data):
+    segmented_pkts_fake_data["seq_flgs"].values = np.full(10, 3)
+    dataset = combine_segmented_packets(segmented_pkts_fake_data)
+
+    np.testing.assert_array_equal(
+        dataset["epoch"].values, segmented_pkts_fake_data["epoch"].values
+    )
+    np.testing.assert_array_equal(
+        dataset["met"].values, segmented_pkts_fake_data["shcoarse"].values
+    )
+
+
 def test_validate_parse_events(sample_data, attr_mgr):
     de_data = sample_data[LoAPID.ILO_SCI_DE]
     validation_path = (
