Skip to content

Commit 28a8a91

Browse files
greglucasgreglucas
committed
MNT: Update file naming convention via imap-data-access upgrade
Bumps imap-data-access to v0.5.0 where there is a new filename requirement on the mission.
1 parent 12a0d69 commit 28a8a91

21 files changed

Lines changed: 139 additions & 177 deletions

‎docs/source/code-documentation/tools/xarray-to-cdf.rst‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -126,15 +126,15 @@ The following python code is the **minimum** code you'd need to store this data
126126
# The following attributes are specific to JIM.
127127
"Instrument_type": "Particles (space)",
128128
"Data_type": "L1_Mode_Description>Level-1 Mode Description",
129-
"Data_version": "01",
129+
"Data_version": "001",
130130
"Descriptor": "JIM>Just an Ion Monitor",
131131
"TEXT": (
132132
"JIM is a fictitious instrument that counts ions at 3 different energies on "
133133
"the IMAP mission. This is where a detailed description of the instrument "
134134
"goes, as well as the type of data in the file. For example, if a (mode) "
135135
"or (descriptor) exist they can be described here."
136136
),
137-
"Logical_file_id": "imap_jim_l1_mode_description_20250101_v01",
137+
"Logical_file_id": "imap_jim_l1_mode_description_20250101_v001",
138138
"Logical_source": "imap_jim_l1_mode_description",
139139
"Logical_source_description": "IMAP Mission JIM Instrument Level-1 (mode) (description) Data.",
140140
}

‎imap_processing/cdf/global_attrs.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ class GlobalDataLevelAttrs:
149149
data_type : str
150150
The level of data, ex "L1>Level-1"
151151
logical_source : str
152-
The source of the data, ex "imap_idex_l1"
152+
The source of the data, ex "imap_idex_l1_sci"
153153
logical_source_desc : str
154154
The description of the data, ex "IMAP Mission IDEX Instrument Level-1 Data."
155155
instrument_base : GlobalInstrumentAttrs

‎imap_processing/cdf/utils.py‎

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22

33
import logging
44
from pathlib import Path
5+
from typing import Optional
56

7+
import imap_data_access
68
import numpy as np
79
import xarray as xr
810
from cdflib.xarray import xarray_to_cdf
@@ -39,7 +41,7 @@ def calc_start_time(shcoarse_time: int):
3941
return launch_time + time_delta
4042

4143

42-
def write_cdf(data: xr.Dataset, filepath: Path):
44+
def write_cdf(dataset: xr.Dataset, directory: Optional[Path] = None):
4345
"""Write the contents of "data" to a CDF file using cdflib.xarray_to_cdf.
4446
4547
This function determines the file name to use from the global attributes,
@@ -51,7 +53,7 @@ def write_cdf(data: xr.Dataset, filepath: Path):
5153
5254
Parameters
5355
----------
54-
data : xarray.Dataset
56+
dataset : xarray.Dataset
5557
The dataset object to convert to a CDF
5658
filepath: Path
5759
The output path, including filename, to write the CDF to.
@@ -61,20 +63,39 @@ def write_cdf(data: xr.Dataset, filepath: Path):
6163
pathlib.Path
6264
Path to the file created
6365
"""
64-
if not filepath.parent.exists():
65-
logger.info("The directory does not exist, creating directory %s", filepath)
66-
filepath.parent.mkdir(parents=True)
67-
66+
# Use the directory if provided, otherwise use the default
67+
directory = directory or imap_data_access.config["DATA_DIR"]
68+
69+
# Create the filename from the global attributes
70+
# Logical_source looks like "imap_swe_l2_counts-1min"
71+
instrument, data_level, descriptor = dataset.attrs["Logical_source"].split("_")[1:]
72+
start_time = np.datetime_as_string(dataset["Epoch"].values[0], unit="D").replace(
73+
"-", ""
74+
)
75+
version = f"v{int(dataset.attrs['Data_version']):03d}" # vXXX
76+
repointing = dataset.attrs.get("Repointing", None)
77+
science_file = imap_data_access.ScienceFilePath.generate_from_inputs(
78+
instrument=instrument,
79+
data_level=data_level,
80+
descriptor=descriptor,
81+
start_time=start_time,
82+
version=version,
83+
repointing=repointing,
84+
)
85+
file_path = directory / science_file.construct_path()
86+
if not file_path.parent.exists():
87+
logger.info("The directory does not exist, creating directory %s", directory)
88+
file_path.parent.mkdir(parents=True)
6889
# Insert the final attribute:
6990
# The Logical_file_id is always the name of the file without the extension
70-
data.attrs["Logical_file_id"] = filepath.stem
91+
dataset.attrs["Logical_file_id"] = file_path.stem
7192

7293
# Convert the xarray object to a CDF
7394
xarray_to_cdf(
74-
data,
75-
str(filepath),
95+
dataset,
96+
str(file_path),
7697
datetime64_to_cdftt2000=True,
7798
terminate_on_warning=True,
7899
) # Terminate if not ISTP compliant
79100

80-
return filepath
101+
return file_path

‎imap_processing/cli.py‎

Lines changed: 14 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,8 @@ def _parse_args():
5353
'instrument': 'mag',
5454
'data_level': 'l0',
5555
'descriptor': 'sci',
56-
'version': 'v00-01',
57-
'start_date': '20231212',
58-
'end_date': '20231212'
56+
'version': 'v001',
57+
'start_date': '20231212'
5958
}]"
6059
--upload-to-sdc
6160
@@ -70,15 +69,13 @@ def _parse_args():
7069
'"imap_cli --instrument "mag" '
7170
'--data-level "l1a"'
7271
' --start-date "20231212"'
73-
'--end-date "20231212"'
74-
'--version "v00-01"'
72+
'--version "v001"'
7573
'--dependency "['
7674
' {"instrument": "mag",'
7775
' "data_level": "l0"',
7876
' "descriptor": "sci"',
79-
' "version": "v00-01"',
77+
' "version": "v001"',
8078
' "start_date": "20231212"',
81-
' "end_date": "20231212"',
8279
'}]" --upload-to-sdc"',
8380
)
8481
instrument_help = (
@@ -94,9 +91,8 @@ def _parse_args():
9491
"Example: '[{'instrument': 'mag',"
9592
"'data_level': 'l0',"
9693
"'descriptor': 'sci',"
97-
"'version': 'v00-01',"
98-
"'start_date': '20231212',"
99-
"'end_date': '20231212'}]"
94+
"'version': 'v001',"
95+
"'start_date': '20231212'}]"
10096
)
10197

10298
parser = argparse.ArgumentParser(prog="imap_cli", description=description)
@@ -123,7 +119,7 @@ def _parse_args():
123119
"--version",
124120
type=str,
125121
required=True,
126-
help="Version of the data. Format: vxx-xx",
122+
help="Version of the data. Format: vXXX",
127123
)
128124
parser.add_argument(
129125
"--dependency",
@@ -154,7 +150,7 @@ def _validate_args(args):
154150
if args.instrument not in imap_data_access.VALID_INSTRUMENTS:
155151
raise ValueError(
156152
f"{args.instrument} is not in the supported instrument list: "
157-
f"{imap_processing.INSTRUMENTS}"
153+
f"{imap_data_access.VALID_INSTRUMENTS}"
158154
)
159155
if args.data_level not in imap_processing.PROCESSING_LEVELS[args.instrument]:
160156
raise ValueError(
@@ -178,15 +174,14 @@ class ProcessInstrument(ABC):
178174
'data_level': 'l0',
179175
'descriptor': 'sci',
180176
'version': 'v00-01',
181-
'start_date': '20231212',
182-
'end_date': '20231212'
177+
'start_date': '20231212'
183178
}]"
184179
start_date : str
185180
The start date for the output data. Format: YYYYMMDD
186181
end_date : str
187182
The end date for the output data. Format: YYYYMMDD
188183
version : str
189-
The version of the data. Format: vxx-xx
184+
The version of the data. Format: vXXX
190185
upload_to_sdc : bool
191186
A flag indicating whether to upload the output file to the SDC.
192187
"""
@@ -317,10 +312,10 @@ def process(self):
317312
f"{file_paths}. Expected only one dependency."
318313
)
319314
filename_norm = imap_data_access.ScienceFilePath.generate_from_inputs(
320-
"mag", "l1a", "raw-norm", self.start_date, self.end_date, self.version
315+
"mag", "l1a", "raw-norm", self.start_date, self.version
321316
).construct_path()
322317
filename_burst = imap_data_access.ScienceFilePath.generate_from_inputs(
323-
"mag", "l1a", "raw-burst", self.start_date, self.end_date, self.version
318+
"mag", "l1a", "raw-burst", self.start_date, self.version
324319
).construct_path()
325320
mag_l1a(file_paths[0], filename_norm, filename_burst)
326321

@@ -349,7 +344,7 @@ class Swe(ProcessInstrument):
349344
def process(self):
350345
"""Perform SWE specific processing."""
351346
# self.file_path example:
352-
# imap/swe/l1a/2023/09/imap_swe_l1a_sci_20230927_20230927_v01-00.cdf
347+
# imap/swe/l1a/2023/09/imap_swe_l1a_sci_20230927_v001.cdf
353348
dependencies = self.download_dependencies()
354349
print(f"Processing SWE {self.data_level}")
355350

@@ -364,7 +359,6 @@ def process(self):
364359
"l1a",
365360
data["descriptor"],
366361
self.start_date,
367-
self.end_date,
368362
self.version,
369363
)
370364

@@ -385,7 +379,7 @@ def process(self):
385379
# TODO: Update this descriptor
386380
descriptor = "test"
387381
file = imap_data_access.ScienceFilePath.generate_from_inputs(
388-
"swe", "l1b", descriptor, self.start_date, self.end_date, self.version
382+
"swe", "l1b", descriptor, self.start_date, self.version
389383
)
390384

391385
cdf_file_path = write_cdf(

‎imap_processing/codice/cdf_attrs.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,14 @@
4444

4545
codice_l1a_global_attrs = GlobalDataLevelAttrs(
4646
data_type="L1A->Level-1A",
47-
logical_source="imap_codice_l1a",
47+
logical_source="imap_codice_l1a_sci",
4848
logical_source_desc="IMAP Mission CoDICE Instrument Level-1A Data",
4949
instrument_base=codice_base,
5050
)
5151

5252
codice_l1b_global_attrs = GlobalDataLevelAttrs(
5353
data_type="L1B->Level-1B",
54-
logical_source="imap_cpdice_l1b",
54+
logical_source="imap_cpdice_l1b_sci",
5555
logical_source_desc="IMAP Mission CoDICE Instrument Level-1B Data",
5656
instrument_base=codice_base,
5757
)

‎imap_processing/codice/codice_l1a.py‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414

1515
import logging
1616

17-
import imap_data_access
1817
import space_packet_parser
1918

2019
from imap_processing.cdf.utils import write_cdf
@@ -47,10 +46,7 @@ def codice_l1a(packets: list[space_packet_parser.parser.Packet]) -> str:
4746
else:
4847
logger.debug(f"{apid} is currently not supported")
4948

50-
file = imap_data_access.ScienceFilePath.generate_from_inputs(
51-
"codice", "l1a", "hk", "20210101", "20210102", "v01-01"
52-
)
5349
# Write data to CDF
54-
cdf_filename = write_cdf(data, file.construct_path())
50+
cdf_filename = write_cdf(data)
5551

5652
return cdf_filename

‎imap_processing/idex/idex_cdf_attrs.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,16 @@
2828

2929
idex_l1_global_attrs = GlobalDataLevelAttrs(
3030
"L1>Level-1",
31-
"imap_idex_l1",
31+
"imap_idex_l1_sci",
3232
"IMAP Mission IDEX Instrument Level-1 Data.",
3333
idex_base,
3434
)
3535

3636
idex_l2_global_attrs = GlobalDataLevelAttrs(
37-
"L2>Level-2", "imap_idex_l2", "IMAP Mission IDEX Instrument Level-2 Data", idex_base
37+
"L2>Level-2",
38+
"imap_idex_l2_sci",
39+
"IMAP Mission IDEX Instrument Level-2 Data",
40+
idex_base,
3841
)
3942

4043
l1_data_base = ScienceAttrs(

‎imap_processing/idex/idex_packet_parser.py‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -473,11 +473,12 @@ class PacketParser:
473473
474474
Examples
475475
--------
476-
>>> # Print out the data in a L0 file
477-
>>> from imap_processing.idex.idex_packet_parser import PacketParser
478-
>>> l0_file = "imap_processing/tests/idex/imap_idex_l0_20230725_v01-00.pkts"
479-
>>> l1_data = PacketParser(l0_file)
480-
>>> l1_data.write_l1_cdf()
476+
.. code-block:: python
477+
# Print out the data in a L0 file
478+
from imap_processing.idex.idex_packet_parser import PacketParser
479+
l0_file = "imap_processing/tests/idex/imap_idex_l0_20230725_v001.pkts"
480+
l1_data = PacketParser(l0_file)
481+
l1_data.write_l1_cdf()
481482
482483
"""
483484

‎imap_processing/mag/mag_cdf_attrs.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,21 +29,21 @@
2929
# TODO: data type should include "norm" and "burst" L1A-norm>Level-1A-normal-rate
3030
"L1A>Level-1A",
3131
# Should also include data type
32-
logical_source="imap_mag_l1a",
32+
logical_source="imap_mag_l1a_sci",
3333
logical_source_desc="IMAP Mission MAG Instrument Level-1A Data.",
3434
instrument_base=mag_base,
3535
)
3636

3737
mag_l1b_attrs = GlobalDataLevelAttrs(
3838
"L1A>Level-1B",
39-
logical_source="imap_mag_l1b",
39+
logical_source="imap_mag_l1b_sci",
4040
logical_source_desc="IMAP Mission MAG Instrument Level-1B Data.",
4141
instrument_base=mag_base,
4242
)
4343

4444
mag_l1c_attrs = GlobalDataLevelAttrs(
4545
"L1A>Level-1C",
46-
logical_source="imap_mag_l1c",
46+
logical_source="imap_mag_l1c_sci",
4747
logical_source_desc="IMAP Mission MAG Instrument Level-1C Data.",
4848
instrument_base=mag_base,
4949
)

‎imap_processing/swe/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
__version__ = "01-00"
1+
__version__ = "001"

0 commit comments

Comments
 (0)