Skip to content

Commit 217a24f

Browse files
lacoak21lacoak21
authored
ULTRA L1c and L2 testing and validation work (IMAP-Science-Operations-Center#2172)
* add new map variables
1 parent d0d7c03 commit 217a24f

13 files changed

Lines changed: 223 additions & 79 deletions

‎imap_processing/cdf/config/imap_ultra_l1c_variable_attrs.yaml‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ geometric_function:
7676
CATDESC: The effective sensitive area as a function of theta and phi in instrument frame (energy independent).
7777
FIELDNAM: geometric_function
7878
LABLAXIS: Geometric Factor
79-
UNITS: " "
79+
UNITS: cm^2
8080
VALIDMIN: 0.0
8181

8282
efficiency:
@@ -177,3 +177,23 @@ energy_bin_delta:
177177
FIELDNAM: energy_bin_delta
178178
LABLAXIS: energy bin delta
179179
UNITS: keV
180+
181+
energy_delta_minus:
182+
<<: *default_float32
183+
VAR_TYPE: support_data
184+
CATDESC: Difference between the energy bin center and lower edge.
185+
LABLAXIS: energy
186+
UNITS: KeV
187+
FIELDNAM: energy_bin_delta_minus
188+
DISPLAY_TYPE: no_plot
189+
DEPEND_1: energy_bin_geometric_mean
190+
191+
energy_delta_plus:
192+
<<: *default_float32
193+
VAR_TYPE: support_data
194+
CATDESC: Difference between the energy bin center and upper edge.
195+
LABLAXIS: energy
196+
UNITS: KeV
197+
FIELDNAM: energy_bin_delta_plus
198+
DISPLAY_TYPE: no_plot
199+
DEPEND_1: energy_bin_geometric_mean

‎imap_processing/cli.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1463,7 +1463,10 @@ def do_processing(
14631463
}
14641464
science_files = dependencies.get_file_paths(source="ultra", data_type="l1b")
14651465
l1b_dict = {
1466-
dataset.attrs["Logical_source"]: dataset
1466+
# TODO remove
1467+
dataset.attrs["Logical_source"].replace(
1468+
"cullingmask", "goodtimes"
1469+
): dataset
14671470
for dataset in [load_cdf(sci_file) for sci_file in science_files]
14681471
}
14691472
combined = {**l1a_dict, **l1b_dict}
@@ -1472,11 +1475,12 @@ def do_processing(
14721475
for path in anc_paths:
14731476
ancillary_files[path.stem.split("_")[2]] = path
14741477
spice_paths = dependencies.get_file_paths(data_type="spice")
1475-
if spice_paths:
1476-
has_spice = True
1478+
# Only the helio pset needs IMAP frames
1479+
if any("imap_frames" in path.as_posix() for path in spice_paths):
1480+
imap_frames = True
14771481
else:
1478-
has_spice = False
1479-
datasets = ultra_l1c.ultra_l1c(combined, ancillary_files, has_spice)
1482+
imap_frames = False
1483+
datasets = ultra_l1c.ultra_l1c(combined, ancillary_files, imap_frames)
14801484
elif self.data_level == "l2":
14811485
all_pset_filepaths = dependencies.get_file_paths(
14821486
source="ultra", descriptor="pset"

‎imap_processing/tests/ultra/unit/conftest.py‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -594,10 +594,16 @@ def deadtime_datasets():
594594
@pytest.fixture
595595
def random_spin_data():
596596
"""Fixture for random spin data."""
597-
with mock.patch(
598-
"imap_processing.ultra.l1c.ultra_l1c_pset_bins.get_spacecraft_spin_phase"
599-
) as mock_spin_phases:
597+
with (
598+
mock.patch(
599+
"imap_processing.ultra.l1c.ultra_l1c_pset_bins.get_spacecraft_spin_phase"
600+
) as mock_spin_phases,
601+
mock.patch(
602+
"imap_processing.ultra.l1c.ultra_l1c_pset_bins.ttj2000ns_to_met"
603+
) as mock_met,
604+
):
600605
mock_spin_phases.side_effect = lambda time: np.random.random(time.shape)
606+
mock_met.side_effect = lambda time: time
601607
yield
602608

603609

‎imap_processing/tests/ultra/unit/test_spacecraft_pset.py‎

Lines changed: 40 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
"""Tests Spacecraft PSET for ULTRA L1c."""
22

3+
from unittest import mock
4+
35
import numpy as np
46
import pandas as pd
57
import pytest
@@ -85,16 +87,25 @@ def test_calculate_spacecraft_pset(
8587
"component": ("component", ["vx", "vy", "vz"]),
8688
},
8789
)
88-
89-
spacecraft_pset = calculate_spacecraft_pset(
90-
test_l1b_de_dataset,
91-
test_l1b_de_dataset, # placeholder for goodtimes_dataset
92-
deadtime_datasets["rates"],
93-
deadtime_datasets["params"],
94-
"imap_ultra_l1c_45sensor-spacecraftpset",
95-
ancillary_files,
96-
45,
97-
)
90+
with (
91+
mock.patch(
92+
"imap_processing.ultra.l1c.spacecraft_pset.get_pointing_times",
93+
return_value=(482374890.0, 482374000.0),
94+
),
95+
mock.patch(
96+
"imap_processing.ultra.l1c.ultra_l1c_pset_bins.ttj2000ns_to_met",
97+
side_effect=lambda x: x,
98+
),
99+
):
100+
spacecraft_pset = calculate_spacecraft_pset(
101+
test_l1b_de_dataset,
102+
test_l1b_de_dataset, # placeholder for goodtimes_dataset
103+
deadtime_datasets["rates"],
104+
deadtime_datasets["params"],
105+
"imap_ultra_l1c_45sensor-spacecraftpset",
106+
ancillary_files,
107+
45,
108+
)
98109
assert "pixel_index" in spacecraft_pset.coords
99110
assert "epoch" in spacecraft_pset.coords
100111
assert "energy_bin_geometric_mean" in spacecraft_pset.coords
@@ -163,16 +174,25 @@ def test_calculate_spacecraft_pset_with_cdf(
163174

164175
name = "imap_ultra_l1b_45sensor-de"
165176
dataset = create_dataset(de_dict, name, "l1b")
166-
167-
spacecraft_pset = calculate_spacecraft_pset(
168-
dataset,
169-
dataset, # placeholder for goodtimes_dataset
170-
deadtime_datasets["rates"],
171-
deadtime_datasets["params"],
172-
"imap_ultra_l1c_45sensor-spacecraftpset",
173-
ancillary_files,
174-
45,
175-
)
177+
with (
178+
mock.patch(
179+
"imap_processing.ultra.l1c.spacecraft_pset.get_pointing_times",
180+
return_value=(482374890.0, 482374000.0),
181+
),
182+
mock.patch(
183+
"imap_processing.ultra.l1c.ultra_l1c_pset_bins.ttj2000ns_to_met",
184+
side_effect=lambda x: x,
185+
),
186+
):
187+
spacecraft_pset = calculate_spacecraft_pset(
188+
dataset,
189+
dataset, # placeholder for goodtimes_dataset
190+
deadtime_datasets["rates"],
191+
deadtime_datasets["params"],
192+
"imap_ultra_l1c_45sensor-spacecraftpset",
193+
ancillary_files,
194+
45,
195+
)
176196
# TODO: validate with output histogram data once we have it in healpix.
177197
assert (
178198
spacecraft_pset.attrs["Logical_source"]

‎imap_processing/tests/ultra/unit/test_ultra_l1c.py‎

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@
88
from imap_processing import imap_module_directory
99
from imap_processing.cdf.utils import write_cdf
1010
from imap_processing.spice.geometry import SpiceFrame
11-
from imap_processing.spice.time import et_to_met
11+
from imap_processing.spice.time import (
12+
et_to_met,
13+
)
1214
from imap_processing.ultra.l1b.ultra_l1b_annotated import (
1315
get_annotated_particle_velocity,
1416
)
@@ -112,7 +114,7 @@ def test_ultra_l1c_error(mock_data_l1b_dict):
112114
with pytest.raises(
113115
ValueError, match="Data dictionary does not contain the expected keys."
114116
):
115-
ultra_l1c(mock_data_l1b_dict, ancillary_files, has_spice=False)
117+
ultra_l1c(mock_data_l1b_dict, ancillary_files, imap_frames=False)
116118

117119

118120
@pytest.mark.external_test_data
@@ -183,8 +185,17 @@ def test_calculate_spacecraft_pset_with_cdf(
183185
"imap_ultra_l1a_45sensor-rates": deadtime_datasets["rates"],
184186
"imap_ultra_l1a_45sensor-params": deadtime_datasets["params"],
185187
}
186-
187-
output_datasets = ultra_l1c(data_dict, ancillary_files, has_spice=False)
188+
with (
189+
mock.patch(
190+
"imap_processing.ultra.l1c.spacecraft_pset.get_pointing_times",
191+
return_value=(482374890.0, 482374000.0),
192+
),
193+
mock.patch(
194+
"imap_processing.ultra.l1c.ultra_l1c_pset_bins.ttj2000ns_to_met",
195+
side_effect=lambda x: x,
196+
),
197+
):
198+
output_datasets = ultra_l1c(data_dict, ancillary_files, imap_frames=False)
188199
output_datasets[0].attrs["Data_version"] = "999"
189200
output_datasets[0].attrs["Repointing"] = f"repoint{pointing + 1:05d}"
190201
output_datasets[0].attrs["Start_date"] = "20250415"
@@ -267,11 +278,18 @@ def test_calculate_helio_pset_with_cdf(
267278
"imap_ultra_l1a_45sensor-rates": deadtime_datasets["rates"],
268279
"imap_ultra_l1a_45sensor-params": deadtime_datasets["params"],
269280
}
270-
with mock.patch(
271-
"imap_processing.ultra.l1c.helio_pset.get_pointing_times",
272-
return_value=(482374890.0, 482374000.0),
281+
282+
with (
283+
mock.patch(
284+
"imap_processing.ultra.l1c.helio_pset.get_pointing_times",
285+
return_value=(482374890.0, 482374000.0),
286+
),
287+
mock.patch(
288+
"imap_processing.ultra.l1c.ultra_l1c_pset_bins.ttj2000ns_to_met",
289+
side_effect=lambda x: x,
290+
),
273291
):
274-
output_datasets = ultra_l1c(data_dict, ancillary_files, has_spice=True)
292+
output_datasets = ultra_l1c(data_dict, ancillary_files, imap_frames=True)
275293
output_datasets[0].attrs["Data_version"] = "999"
276294
output_datasets[0].attrs["Repointing"] = f"repoint{pointing + 1:05d}"
277295
test_data_path = write_cdf(output_datasets[0], istp=True)

‎imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,9 @@ def test_get_sectored_rates():
151151
sectored_rates = get_sectored_rates(test_l1a_rates_dataset, test_l1a_params_dataset)
152152
np.testing.assert_array_equal(
153153
sectored_rates["test_data"].data,
154-
np.hstack([np.arange(10, 20), np.arange(30, 40), np.arange(50, 60)]),
154+
np.arange(
155+
10, 20
156+
), # Make sure duplicate epochs with the same mode are filtered out
155157
)
156158
# Test with one mode shift in the middle of the dataset.
157159
modes = np.array([1, 3, 1])
@@ -226,7 +228,7 @@ def test_get_deadtime_interpolator(random_spin_data):
226228
# Assert value error is raised for NaN values
227229
with pytest.raises(
228230
ValueError,
229-
match="Dead time ratios contain NaN values, cannot create interpolator.",
231+
match="All dead time ratios are NaN, cannot interpolate",
230232
):
231233
get_deadtime_ratios_by_spin_phase(sectored_rates_ds)
232234

‎imap_processing/tests/ultra/unit/test_ultra_l2.py‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ def _mock_multiple_psets(self, _setup_spice_kernels_list, furnish_kernels):
5353
peak_exposure=1000,
5454
timestr=manual_timestrs[i],
5555
head=("90"),
56+
energy_dependent_exposure=True,
5657
)
5758
for i, mid_latitude in enumerate(
5859
np.arange(
@@ -90,13 +91,18 @@ def test_generate_ultra_healpix_skymap_single_pset(
9091
):
9192
# Avoid modifying the original pset
9293
pset = self.ultra_pset.copy(deep=True)
93-
9494
# Set the values in the single input PSET for easy calculation
9595
# of the expected ena_intensity and ena_intensity statistical uncertainty
9696
pset["counts"].values = np.full_like(pset["counts"].values, 10)
97-
pset["exposure_factor"].values = np.ones_like(pset["exposure_factor"].values)
97+
pset["exposure_factor"].values = np.ones_like(pset["exposure_factor"])
9898
pset["background_rates"].values = np.ones_like(pset["background_rates"].values)
9999
pset["sensitivity"].values = np.ones_like(pset["sensitivity"].values)
100+
pset["energy_bin_delta"].values = np.ones_like(pset["energy_bin_delta"].values)
101+
pset["efficiency"] = xr.ones_like(pset["exposure_factor"])
102+
pset["geometric_function"] = xr.ones_like(pset["exposure_factor"])
103+
pset["scatter_theta"] = xr.ones_like(pset["exposure_factor"])
104+
pset["scatter_phi"] = xr.ones_like(pset["exposure_factor"])
105+
100106
pset["energy_bin_delta"].values = np.ones_like(pset["energy_bin_delta"].values)
101107
if epoch_dim_for_energy_delta:
102108
# add an extra dim to the start
@@ -120,6 +126,10 @@ def test_generate_ultra_healpix_skymap_single_pset(
120126
"values_to_pull_project": [
121127
"exposure_factor",
122128
"sensitivity",
129+
"geometric_function",
130+
"efficiency",
131+
"scatter_theta",
132+
"scatter_phi",
123133
"background_rates",
124134
],
125135
"nside": 32,
@@ -139,6 +149,11 @@ def test_generate_ultra_healpix_skymap_single_pset(
139149
"obs_date_range",
140150
"ena_intensity_stat_unc",
141151
"exposure_factor",
152+
"sensitivity",
153+
"geometric_function",
154+
"efficiency",
155+
"scatter_theta",
156+
"scatter_phi",
142157
"obs_date",
143158
]
144159
for var in expected_vars:
@@ -319,10 +334,7 @@ def test_generate_ultra_healpix_skymap_multiple_psets(self, furnish_kernels):
319334
assert hp_skymap.data_1d["counts"].dims == counts_dims
320335
assert hp_skymap.data_1d["ena_intensity"].dims == counts_dims
321336
assert hp_skymap.data_1d["ena_intensity_stat_unc"].dims == counts_dims
322-
assert hp_skymap.data_1d["exposure_factor"].dims == (
323-
CoordNames.TIME.value,
324-
CoordNames.GENERIC_PIXEL.value,
325-
)
337+
assert hp_skymap.data_1d["exposure_factor"].dims == counts_dims
326338

327339
@pytest.mark.usefixtures("_setup_spice_kernels_list")
328340
def test_ultra_l2_output_unbinned_healpix(self, mock_data_dict, furnish_kernels):
@@ -448,11 +460,7 @@ def test_ultra_l2_rectangular(self, mock_data_dict, furnish_kernels):
448460
rect_map_dataset["ena_intensity_stat_unc"].dims
449461
== expected_ena_intensity_dims
450462
)
451-
assert rect_map_dataset["exposure_factor"].dims == (
452-
CoordNames.TIME.value,
453-
CoordNames.AZIMUTH_L2.value,
454-
CoordNames.ELEVATION_L2.value,
455-
)
463+
assert rect_map_dataset["exposure_factor"].dims == expected_ena_intensity_dims
456464

457465
# Check that '_label' coordinates were added for all coordinates except 'epoch'
458466
for coord_var in expected_ena_intensity_dims[1:]:

‎imap_processing/ultra/l1c/helio_pset.py‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
from imap_processing.spice.time import (
1212
et_to_met,
1313
met_to_ttj2000ns,
14-
sct_to_et,
1514
ttj2000ns_to_et,
1615
)
1716
from imap_processing.ultra.l1b.ultra_l1b_culling import get_de_rejection_mask
@@ -23,6 +22,7 @@
2322
from imap_processing.ultra.l1c.ultra_l1c_pset_bins import (
2423
build_energy_bins,
2524
get_efficiencies_and_geometric_function,
25+
get_energy_delta_minus_plus,
2626
get_helio_adjusted_data,
2727
get_spacecraft_exposure_times,
2828
get_spacecraft_histogram,
@@ -139,11 +139,11 @@ def calculate_helio_pset(
139139
ancillary_files,
140140
)
141141
# Get midpoint timestamp for pointing.
142-
# TODO remove sct_to_et conversion
143142
pointing_start, pointing_stop = get_pointing_times(
144-
et_to_met(sct_to_et(species_dataset["event_times"].data[0]))
143+
et_to_met(species_dataset["event_times"].data[0])
145144
)
146145
mid_time = ttj2000ns_to_et(met_to_ttj2000ns((pointing_start + pointing_stop) / 2))
146+
147147
logger.info("Adjusting data for helio frame.")
148148
exposure_time, efficiency, geometric_function = get_helio_adjusted_data(
149149
mid_time,
@@ -169,9 +169,9 @@ def calculate_helio_pset(
169169
helio_pset_quality_flags,
170170
nside=nside,
171171
)
172-
173-
# For ISTP, epoch should be the center of the time bin.
174-
pset_dict["epoch"] = de_dataset.epoch.data[:1].astype(np.int64)
172+
pointing_start = met_to_ttj2000ns(pointing_start)
173+
# Epoch should be the start of the pointing
174+
pset_dict["epoch"] = np.atleast_1d(pointing_start).astype(np.int64)
175175
pset_dict["counts"] = counts[np.newaxis, ...]
176176
pset_dict["latitude"] = latitude[np.newaxis, ...]
177177
pset_dict["longitude"] = longitude[np.newaxis, ...]
@@ -192,6 +192,11 @@ def calculate_helio_pset(
192192
pset_dict["scatter_phi"] = scattering_phi
193193
pset_dict["scatter_threshold"] = scattering_thresholds
194194

195+
# Add the energy delta plus/minus to the dataset
196+
energy_delta_minus, energy_delta_plus = get_energy_delta_minus_plus()
197+
pset_dict["energy_delta_minus"] = energy_delta_minus
198+
pset_dict["energy_delta_plus"] = energy_delta_plus
199+
195200
dataset = create_dataset(pset_dict, name, "l1c")
196201

197202
return dataset

0 commit comments

Comments
 (0)