Skip to content

Commit 4a5e6bc

Browse files
bourquebourque
authored
Codice L1A updates for SIT-3 (IMAP-Science-Operations-Center#567)
* Updated explanation of variable length data in packet definition * Updated data_type values for l1a products * Updated APID mapping * Updated telemetry definitions files * Now using custom launch_time * Added esa_sweep and acq_times values to CDF file as data variables * Changed a bunch of CDF attributes and formatting * Some changes to counters CDF attrs in anticipation of updates to cdflib * Renamed filenames of simulated data to match expected L0 filename conventions; also added a few new files * Added support for lo-nsw-angular-counts and lo-nsw-priority-counts * Rearranged some variable names to alphabetical order * Using new simulated data * Updated CoDICE valid descriptors in docs
1 parent 0a0a02d commit 4a5e6bc

33 files changed

Lines changed: 1340 additions & 1179 deletions

‎docs/source/development-guide/style-guide/naming-conventions.rst‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,9 @@ The general filename convention is as follows::
3434
can have any text or numbers as long as it doesn't include underscores. The following is a list of expected
3535
descriptors for each instrument:
3636

37-
* CoDICE: ``hskp``, ``eng``, ``lo-priority``, ``lo-sw-angular``, ``lo-nsw-angular``, ``lo-pha``, ``hi-pha``,
38-
``lo-sw-species``, ``lo-nsw-species``, ``hi-omni``, ``hi-sectored``
37+
* CoDICE: ``hskp``, ``lo-counters-aggregated``, ``lo-counters-singles``, ``hi-counters-aggregated``,
38+
``hi-counters-singles``, ``lo-sw-priority``, ``lo-nsw-priority``, ``lo-sw-angular``, ``lo-nsw-angular``, ``lo-pha``,
39+
``hi-pha``, ``lo-sw-species``, ``lo-nsw-species``, ``hi-omni``, ``hi-sectored``, ``lo-ialirt``, ``hi-ialirt``
3940
* GLOWS: ``histogram``, ``de``
4041
* HIT: TBD
4142
* IDEX: ``sci``

‎imap_processing/codice/cdf_attrs.py‎

Lines changed: 30 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -47,20 +47,20 @@
4747
instrument_base=codice_base,
4848
)
4949

50-
l1a_lo_sw_species_counts_attrs = GlobalDataLevelAttrs(
51-
data_type="L1A_lo-sw-species-counts->Level-1A Lo Sunward Species Counts Data",
52-
logical_source="imap_codice_l1a_lo-sw-species-counts",
50+
l1a_lo_sw_angular_counts_attrs = GlobalDataLevelAttrs(
51+
data_type="L1A_lo-sw-angular-counts->Level-1A Lo Sunward Angular Counts Data",
52+
logical_source="imap_codice_l1a_lo-sw-angular-counts",
5353
logical_source_desc=(
54-
"IMAP Mission CoDICE Instrument Level-1A Lo Sunward Species Counts Data"
54+
"IMAP Mission CoDICE Instrument Level-1A Lo Sunward Angular Counts Data"
5555
),
5656
instrument_base=codice_base,
5757
)
5858

59-
l1a_lo_nsw_species_counts_attrs = GlobalDataLevelAttrs(
60-
data_type="L1A_lo-nsw-species-counts->Level-1A Lo Non-sunward Species Counts Data",
61-
logical_source="imap_codice_l1a_lo-nsw-species-counts",
59+
l1a_lo_nsw_angular_counts_attrs = GlobalDataLevelAttrs(
60+
data_type="L1A_lo-nsw-angular-counts->Level-1A Lo Non-Sunward Angular Counts Data",
61+
logical_source="imap_codice_l1a_lo-nsw-angular-counts",
6262
logical_source_desc=(
63-
"IMAP Mission CoDICE Instrument Level-1A Lo Non-sunward Species Counts Data"
63+
"IMAP Mission CoDICE Instrument Level-1A Lo Non-sunward Angular Counts Data"
6464
),
6565
instrument_base=codice_base,
6666
)
@@ -74,11 +74,29 @@
7474
instrument_base=codice_base,
7575
)
7676

77-
l1a_lo_sw_angular_counts_attrs = GlobalDataLevelAttrs(
78-
data_type="L1A_lo-sw-angular-counts->Level-1A Lo Sunward Angular Counts Data",
79-
logical_source="imap_codice_l1a_lo-sw-angular-counts",
77+
l1a_lo_nsw_priority_counts_attrs = GlobalDataLevelAttrs(
78+
data_type="L1A_lo-sw-priority-counts->Level-1A Lo Non-Sunward Priority Counts Data",
79+
logical_source="imap_codice_l1a_lo-nsw-priority-counts",
8080
logical_source_desc=(
81-
"IMAP Mission CoDICE Instrument Level-1A Lo Sunward Angular Counts Data"
81+
"IMAP Mission CoDICE Instrument Level-1A Lo Non-Sunward Priority Counts Data"
82+
),
83+
instrument_base=codice_base,
84+
)
85+
86+
l1a_lo_sw_species_counts_attrs = GlobalDataLevelAttrs(
87+
data_type="L1A_lo-sw-species-counts->Level-1A Lo Sunward Species Counts Data",
88+
logical_source="imap_codice_l1a_lo-sw-species-counts",
89+
logical_source_desc=(
90+
"IMAP Mission CoDICE Instrument Level-1A Lo Sunward Species Counts Data"
91+
),
92+
instrument_base=codice_base,
93+
)
94+
95+
l1a_lo_nsw_species_counts_attrs = GlobalDataLevelAttrs(
96+
data_type="L1A_lo-nsw-species-counts->Level-1A Lo Non-Sunward Species Counts Data",
97+
logical_source="imap_codice_l1a_lo-nsw-species-counts",
98+
logical_source_desc=(
99+
"IMAP Mission CoDICE Instrument Level-1A Lo Non-Sunward Species Counts Data"
82100
),
83101
instrument_base=codice_base,
84102
)

‎imap_processing/codice/codice_l0.py‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,6 @@
1818

1919
from imap_processing import decom, imap_module_directory
2020

21-
PACKET_TO_XTCE_MAPPING = {
22-
"raw_ccsds_20230822_122700Z_idle.bin": "P_COD_NHK.xml",
23-
"lo_fsw_view_3_ccsds.bin": "P_COD_LO_PRIORITY_COUNTS.xml",
24-
"lo_fsw_view_4_ccsds.bin": "P_COD_LO_PRIORITY_COUNTS.xml",
25-
"lo_fsw_view_5_ccsds.bin": "P_COD_LO_SW_SPECIES_COUNTS.xml",
26-
"lo_fsw_view_6_ccsds.bin": "P_COD_LO_NSW_SPECIES_COUNTS.xml",
27-
"lo_fsw_view_7_ccsds.bin": "P_COD_LO_SW_ANGULAR_COUNTS.xml",
28-
"lo_fsw_view_8_ccsds.bin": "P_COD_LO_NSW_ANGULAR_COUNTS.xml",
29-
}
30-
3121

3222
def decom_packets(packet_file: Path) -> list:
3323
"""Decom CoDICE data packets using CoDICE packet definition.
@@ -42,7 +32,19 @@ def decom_packets(packet_file: Path) -> list:
4232
list : list
4333
all the unpacked data.
4434
"""
35+
# Determine the mapping between L0 filename and corresponding XML filename
36+
# via the descriptor. For example:
37+
# packet_file = imap_codice_l0_lo-sw-species_YYYYMMDD_v001.pkts
38+
descriptor = (
39+
packet_file.stem.split("imap_codice_l0_")[
40+
-1
41+
] # lo-sw-species_YYYYMMDD_v001.pkts
42+
.split("_")[0] # lo-sw-species
43+
.upper() # LO-SW-SPECIES
44+
.replace("-", "_") # LO_SW_SPECIES
45+
)
46+
filename = f"P_COD_{descriptor}.xml" # P_COD_LO_SW_SPECIES.xml
4547
xtce_document = Path(
46-
f"{imap_module_directory}/codice/packet_definitions/{PACKET_TO_XTCE_MAPPING[packet_file.name]}"
48+
f"{imap_module_directory}/codice/packet_definitions/{filename}"
4749
)
4850
return decom.decom_packets(packet_file, xtce_document)

‎imap_processing/codice/codice_l1a.py‎

Lines changed: 53 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929
ESA_SWEEP_TABLE_ID_LOOKUP,
3030
LO_COLLAPSE_TABLE_ID_LOOKUP,
3131
LO_COMPRESSION_ID_LOOKUP,
32+
LO_NSW_ANGULAR_NAMES,
33+
LO_NSW_PRIORITY_NAMES,
3234
LO_NSW_SPECIES_NAMES,
3335
LO_STEPPING_TABLE_ID_LOOKUP,
3436
LO_SW_ANGULAR_NAMES,
@@ -41,10 +43,10 @@
4143
logger = logging.getLogger(__name__)
4244
logger.setLevel(logging.INFO)
4345

44-
# TODO: Data array lengths should all be 128 * num_counters
45-
# (see notes in unpack_science_data)
46-
# TODO: Try new simulated data
46+
# TODO: Decom data arrays need to be decompressed
4747
# TODO: Add metadata attrs to science dataset?
48+
# TODO: In decommutation, how to have a variable length data and then a checksum
49+
# after it?
4850

4951

5052
class CoDICEL1aPipeline:
@@ -87,29 +89,23 @@ def __init__(self, table_id: int, plan_id: int, plan_step: int, view_id: int):
8789
self.plan_step = plan_step
8890
self.view_id = view_id
8991

90-
def create_science_dataset(self, packets: list) -> xr.Dataset:
92+
def create_science_dataset(self, start_time: np.datetime64) -> xr.Dataset:
9193
"""Create an ``xarray`` dataset for the unpacked science data.
9294
9395
The dataset can then be written to a CDF file.
9496
9597
Parameters
9698
----------
97-
packet : list[space_packet_parser.parser.Packet]
98-
List of packets for the APID of interest
99+
start_time : np.datetime64
100+
The start time of the packet, used to determine epoch data variable
99101
100102
Returns
101103
-------
102104
xr.Dataset
103105
``xarray`` dataset containing the science data and supporting metadata
104106
"""
105107
epoch = xr.DataArray(
106-
[
107-
calc_start_time(
108-
packet.data["ACQ_START_SECONDS"].raw_value,
109-
launch_time=np.datetime64("2010-01-01T00:01:06.184", "ns"),
110-
)
111-
for packet in packets
112-
],
108+
[start_time],
113109
name="epoch",
114110
dims=["epoch"],
115111
attrs=ConstantCoordinates.EPOCH,
@@ -241,55 +237,38 @@ def get_lo_data_products(self, apid: int):
241237
apid : int
242238
The APID of interest.
243239
"""
244-
# TODO: There are some discrepancies here to doublecheck with Joey:
245-
246-
# LO_SW_SPECIES_COUNTS all checks out
247-
# PKT_LEN = 283 (really 284 because its zero-indexed)
248-
# BYTE_COUNT = 256
249-
# 256 * 8 = 2048 bits
250-
# 2048 bits / 16 species = 128 bits per species
251-
252-
# LO_NSW_SPECIES_COUNTS
253-
# PKT_LEN = 140
254-
# BYTE_COUNT = 112
255-
# 112 * 8 = 896 bits
256-
# 896 bits / 8 species = 112 bits per species (should be 128)
257-
# Math works out if there are 7 species
258-
259-
# LO_SW_PRIORITY_COUNTS
260-
# PKT_LEN = 160
261-
# BYTE_COUNT = 132
262-
# 132 * 8 = 1056 bits
263-
# 1056 bits / 5 counters = 211.2 bits per counter ???
264-
265-
# LO_SW_ANGULAR_COUNTS
266-
# PKT_LEN = 2535
267-
# BYTE_COUNT = 2508
268-
# 2508 * 8 = 20064 bits
269-
# 20064 bits / 4 counters = 5016 bits per counter ???
270-
271240
if apid == CODICEAPID.COD_LO_SW_SPECIES_COUNTS:
272241
self.num_counters = 16
273242
self.num_energy_steps = 128
274243
self.variable_names = LO_SW_SPECIES_NAMES
275244
self.cdf_attrs = cdf_attrs.l1a_lo_sw_species_counts_attrs
276245
elif apid == CODICEAPID.COD_LO_NSW_SPECIES_COUNTS:
277246
self.num_counters = 8
278-
self.num_energy_steps = 112
247+
self.num_energy_steps = 128
279248
self.variable_names = LO_NSW_SPECIES_NAMES
280249
self.cdf_attrs = cdf_attrs.l1a_lo_nsw_species_counts_attrs
281250
elif apid == CODICEAPID.COD_LO_SW_PRIORITY_COUNTS:
282251
self.num_counters = 5
283-
self.num_energy_steps = 211
252+
self.num_energy_steps = 128
284253
self.variable_names = LO_SW_PRIORITY_NAMES
285254
self.cdf_attrs = cdf_attrs.l1a_lo_sw_priority_counts_attrs
255+
elif apid == CODICEAPID.COD_LO_NSW_PRIORITY_COUNTS:
256+
self.num_counters = 2
257+
self.num_energy_steps = 128
258+
self.variable_names = LO_NSW_PRIORITY_NAMES
259+
self.cdf_attrs = cdf_attrs.l1a_lo_nsw_priority_counts_attrs
286260
elif apid == CODICEAPID.COD_LO_SW_ANGULAR_COUNTS:
287261
self.num_counters = 4
288-
self.num_energy_steps = 5016
262+
self.num_energy_steps = 128
289263
self.variable_names = LO_SW_ANGULAR_NAMES
290264
self.cdf_attrs = cdf_attrs.l1a_lo_sw_angular_counts_attrs
265+
elif apid == CODICEAPID.COD_LO_NSW_ANGULAR_COUNTS:
266+
self.num_counters = 1
267+
self.num_energy_steps = 128
268+
self.variable_names = LO_NSW_ANGULAR_NAMES
269+
self.cdf_attrs = cdf_attrs.l1a_lo_nsw_angular_counts_attrs
291270

292-
def unpack_science_data(self, packets: list):
271+
def unpack_science_data(self, science_values: str):
293272
"""Unpack the science data from the packet.
294273
295274
For LO SW Species Counts data, the science data within the packet is a
@@ -299,21 +278,22 @@ def unpack_science_data(self, packets: list):
299278
300279
Parameters
301280
----------
302-
packet : list[space_packet_parser.parser.Packet]
303-
List of packets for the APID of interest
281+
science_values : str
282+
A string of binary data representing the science values of the data
304283
"""
305284
self.compression_algorithm = LO_COMPRESSION_ID_LOOKUP[self.view_id]
306285
self.collapse_table_id = LO_COLLAPSE_TABLE_ID_LOOKUP[self.view_id]
307286

308-
science_values = packets[0].data["DATA"].raw_value
309-
287+
# TODO: Turn this back on after SIT-3
288+
# For SIT-3, just create appropriate length data arrays of all ones
310289
# Divide up the data by the number of priorities or species
311-
num_bits = len(science_values)
312-
chunk_size = len(science_values) // self.num_counters
313-
314-
self.data = [
315-
science_values[i : i + chunk_size] for i in range(0, num_bits, chunk_size)
316-
]
290+
# science_values = packets[0].data["DATA"].raw_value
291+
# num_bits = len(science_values)
292+
# chunk_size = len(science_values) // self.num_counters
293+
# self.data = [
294+
# science_values[i : i + chunk_size] for i in range(0, num_bits, chunk_size)
295+
# ]
296+
self.data = [["1"] * 128] * self.num_counters
317297

318298

319299
def get_params(packet) -> tuple[int, int, int, int]:
@@ -366,11 +346,12 @@ def process_codice_l1a(file_path: Path | str) -> xr.Dataset:
366346
``xarray`` dataset containing the science data and supporting metadata
367347
"""
368348
apids_for_lo_science_processing = [
369-
CODICEAPID.COD_LO_SW_SPECIES_COUNTS,
370-
CODICEAPID.COD_LO_NSW_SPECIES_COUNTS,
349+
CODICEAPID.COD_LO_SW_ANGULAR_COUNTS,
350+
CODICEAPID.COD_LO_NSW_ANGULAR_COUNTS,
371351
CODICEAPID.COD_LO_SW_PRIORITY_COUNTS,
372352
CODICEAPID.COD_LO_NSW_PRIORITY_COUNTS,
373-
CODICEAPID.COD_LO_SW_ANGULAR_COUNTS,
353+
CODICEAPID.COD_LO_SW_SPECIES_COUNTS,
354+
CODICEAPID.COD_LO_NSW_SPECIES_COUNTS,
374355
]
375356

376357
# Decom the packets, group data by APID, and sort by time
@@ -386,8 +367,18 @@ def process_codice_l1a(file_path: Path | str) -> xr.Dataset:
386367
dataset = create_hskp_dataset(packets=sorted_packets)
387368

388369
elif apid in apids_for_lo_science_processing:
370+
# Sort the packets by time
389371
packets = sort_by_time(grouped_data[apid], "SHCOARSE")
390372

373+
# Determine the start time of the packet
374+
start_time = calc_start_time(
375+
packets[0].data["ACQ_START_SECONDS"].raw_value,
376+
launch_time=np.datetime64("2010-01-01T00:01:06.184", "ns"),
377+
)
378+
379+
# Extract the data
380+
science_values = packets[0].data["DATA"].raw_value
381+
391382
# Get the four "main" parameters for processing
392383
table_id, plan_id, plan_step, view_id = get_params(packets[0])
393384

@@ -396,22 +387,18 @@ def process_codice_l1a(file_path: Path | str) -> xr.Dataset:
396387
pipeline.get_esa_sweep_values()
397388
pipeline.get_acquisition_times()
398389
pipeline.get_lo_data_products(apid)
399-
pipeline.unpack_science_data(packets)
400-
dataset = pipeline.create_science_dataset(packets)
401-
402-
elif apid == CODICEAPID.COD_LO_PHA:
403-
logger.info(f"{apid} is currently not supported")
404-
continue
390+
pipeline.unpack_science_data(science_values)
391+
dataset = pipeline.create_science_dataset(start_time)
405392

406-
elif apid == CODICEAPID.COD_LO_NSW_PRIORITY_COUNTS:
393+
elif apid == CODICEAPID.COD_LO_INSTRUMENT_COUNTERS:
407394
logger.info(f"{apid} is currently not supported")
408395
continue
409396

410-
elif apid == CODICEAPID.COD_LO_SW_ANGULAR_COUNTS:
397+
elif apid == CODICEAPID.COD_LO_PHA:
411398
logger.info(f"{apid} is currently not supported")
412399
continue
413400

414-
elif apid == CODICEAPID.COD_LO_NSW_ANGULAR_COUNTS:
401+
elif apid == CODICEAPID.COD_HI_INSTRUMENT_COUNTERS:
415402
logger.info(f"{apid} is currently not supported")
416403
continue
417404

0 commit comments

Comments
 (0)