Backport PR #9602 on branch release/1.19.x (JP-4047: Update regtests for niriss and nircam wfss)

changes/9602.datamodels.rst

@@ -0,0 +1 @@
+Fix a bug where DQ table column was not padded properly where no valid data for multi-spec type tables

jwst/datamodels/utils/flat_multispec.py

@@ -1,7 +1,6 @@
 """Utilities for re-organizing spectral products into a flat structure."""
 
 import logging
-import warnings
 from copy import deepcopy
 
 import numpy as np
@@ -103,7 +102,7 @@ def make_empty_recarray(n_rows, n_spec, columns, is_vector, defaults=0):
     return arr
 
 
-def populate_recarray(output_table, input_spec, n_rows, columns, is_vector, ignore_columns=None):
+def populate_recarray(output_table, input_spec, columns, is_vector, ignore_columns=None):
     """
     Populate the output table in-place with data from the input spectrum.
 
@@ -118,9 +117,6 @@ def populate_recarray(output_table, input_spec, n_rows, columns, is_vector, igno
         The output table to be populated with the spectral data.
     input_spec : `~jwst.datamodels.SpecModel` or `~jwst.datamodels.CombinedSpecModel`
         The input data model containing the spectral data.
-    n_rows : int
-        The number of rows in the output table; this is the maximum number of
-        data points for any spectrum in the exposure.
     columns : np.ndarray[tuple]
         Array of tuples containing the column names and their dtypes.
     is_vector : np.ndarray[bool]
@@ -138,17 +134,12 @@ def populate_recarray(output_table, input_spec, n_rows, columns, is_vector, igno
     vector_columns = columns[is_vector]
     meta_columns = columns[~is_vector]
 
-    # Copy the data into the new table with NaN padding
+    # Copy the data into the new table
     for col, _ in vector_columns:
         if col in ignore_columns:
             continue
-        padded_data = np.full(n_rows, np.nan)
-        padded_data[: input_table.shape[0]] = input_table[col]
-        with warnings.catch_warnings():
-            warnings.filterwarnings(
-                "ignore", category=RuntimeWarning, message="invalid value encountered in cast"
-            )
-            output_table[col] = padded_data
+
+        output_table[col][: input_table.shape[0]] = input_table[col]
 
     # Copy the metadata into the new table
     # Metadata columns must have identical names to spec_meta columns

jwst/datamodels/utils/tests/test_flat_multispec.py

@@ -27,7 +27,12 @@ def empty_recarray(request):
     """
     n_rows = 10
     n_sources = 5
-    vector_columns = [("FLUX", np.float32), ("WAVELENGTH", np.float64)]
+    vector_columns = [
+        ("FLUX", np.float32),
+        ("WAVELENGTH", np.float64),
+        ("DQ", np.uint32),
+        ("UNKNOWN", np.int32),
+    ]
     meta_columns = [("NAME", "S20"), ("SOURCE_ID", np.int32)]
     if request.param:
         # Add a column that is not in the input model
@@ -173,30 +178,35 @@ def test_populate_recarray(empty_recarray, ignore_columns, monkeypatch):
     all_datatypes = [output_table[name].dtype for name in all_names]
     all_columns = [(name, dtype) for name, dtype in zip(all_names, all_datatypes)]
     all_columns = np.array(all_columns)
-    vector_columns = [(name, dtype) for name, dtype in zip(all_names[:2], all_datatypes[:2])]
-    meta_columns = [(name, dtype) for name, dtype in zip(all_names[2:], all_datatypes[2:])]
+    vector_columns = [(name, dtype) for name, dtype in zip(all_names[:4], all_datatypes[:4])]
+    meta_columns = [(name, dtype) for name, dtype in zip(all_names[4:], all_datatypes[4:])]
     is_vector = [True] * len(vector_columns) + [False] * len(meta_columns)
     is_vector = np.array(is_vector, dtype=bool)
     (n_sources, n_rows) = output_table["FLUX"].shape
 
+    schema_dtype = [
+        {"name": "WAVELENGTH", "datatype": "float64"},
+        {"name": "FLUX", "datatype": "float64"},
+        {"name": "DQ", "datatype": "uint32"},
+        {"name": "UNKNOWN", "datatype": "int32"},
+    ]
+
     for i in range(n_sources):
         input_spec = dm.SpecModel()
         input_spec.name = f"Source {i}"
         input_spec.source_id = i
 
         this_output = output_table[i]
 
-        # hack input model schema to have only the flux and wavelength columns
-        input_spec.schema["properties"]["spec_table"]["datatype"] = input_spec.schema["properties"][
-            "spec_table"
-        ]["datatype"][:2]
+        # hack input model schema to have only the columns we want
+        input_spec.schema["properties"]["spec_table"]["datatype"] = schema_dtype
 
         # give all the input spectra different numbers of rows
         input_spec.spec_table = np.ones(
             (n_rows - i,), dtype=[(name, dtype) for name, dtype in vector_columns]
         )
         populate_recarray(
-            this_output, input_spec, n_rows, all_columns, is_vector, ignore_columns=ignore_columns
+            this_output, input_spec, all_columns, is_vector, ignore_columns=ignore_columns
         )
 
     if ignore_columns is None:
@@ -207,7 +217,7 @@ def test_populate_recarray(empty_recarray, ignore_columns, monkeypatch):
         for name, _ in vector_columns:
             if name not in ignore_columns:
                 # ensure proper nan-padding of output
-                expected = np.ones((n_rows,)) * np.nan
+                expected = this_output[name].copy()
                 expected[: n_rows - i] = 1.0
                 assert np.array_equal(output_table[name][i], expected, equal_nan=True)
             else:

jwst/datamodels/utils/tso_multispec.py

@@ -56,7 +56,6 @@ def make_tso_specmodel(spec_list, segment=None):
         flat_multispec.populate_recarray(
             this_output,
             input_spec,
-            n_rows,
             all_cols,
             is_vector,
             ignore_columns=ignore_columns,

jwst/datamodels/utils/wfss_multispec.py

@@ -124,7 +124,6 @@ def make_wfss_multiexposure(input_list):
             populate_recarray(
                 fltdata[spec_idx],
                 spec,
-                n_rows,
                 all_columns,
                 is_vector,
                 ignore_columns=["SOURCE_ID", "N_ALONGDISP"],
@@ -261,7 +260,6 @@ def make_wfss_multicombined(results_list):
             populate_recarray(
                 fltdata[j],
                 spec,
-                n_rows,
                 all_columns,
                 is_vector,
                 ignore_columns=["N_ALONGDISP"],

jwst/regtest/test_nircam_wfss_spec3.py

@@ -0,0 +1,39 @@
+import pytest
+from jwst.regtest.st_fitsdiff import STFITSDiff as FITSDiff
+
+from jwst.stpipe import Step
+
+
+# Mark all tests in this module
+pytestmark = [pytest.mark.bigdata]
+
+
+@pytest.fixture(scope="module")
+def run_nircam_wfss_spec3(rtdata_module, resource_tracker):
+    """Run the calwebb_spec3 pipeline"""
+    rtdata = rtdata_module
+
+    # Get the level3 association file and run the spec3 pipeline on it.
+    rtdata.get_asn("nircam/wfss/jw02279-o001_spec3_00001_asn.json")
+    args = ["calwebb_spec3", rtdata.input]
+    with resource_tracker.track():
+        Step.from_cmdline(args)
+
+
+def test_log_tracked_resources(log_tracked_resources, run_nircam_wfss_spec3):
+    log_tracked_resources()
+
+
+@pytest.mark.parametrize("suffix", ["x1d", "c1d"])
+def test_nircam_wfss_spec3(run_nircam_wfss_spec3, rtdata_module, suffix, fitsdiff_default_kwargs):
+    """Regression test of the calwebb_spec3 pipeline applied to NIRISS WFSS data"""
+    rtdata = rtdata_module
+    rtdata.input = "jw02279-o001_spec3_00001_asn.json"
+    output = "jw02279-o001_t001_nircam_grismr_" + suffix + ".fits"
+    rtdata.output = output
+    rtdata.get_truth(f"truth/test_nircam_wfss_spec3/{output}")
+
+    # Compare the results
+    fitsdiff_default_kwargs["atol"] = 1e-5
+    diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
+    assert diff.identical, diff.report()

jwst/regtest/test_niriss_wfss.py

@@ -14,10 +14,9 @@ def run_nis_wfss_spec2(rtdata_module, resource_tracker):
     """Run the calwebb_spec2 pipeline on NIRISS WFSS exposures"""
     rtdata = rtdata_module
 
-    # These are the 4 WFSS exposures we'll be processing
+    # These are the WFSS exposures we'll be processing
     spec2_asns = [
         "niriss/wfss/jw01324-o001_20220629t171902_spec2_001_asn.json",
-        "niriss/wfss/jw01324-o001_20220629t171902_spec2_002_asn.json",
         "niriss/wfss/jw01324-o001_20220629t171902_spec2_005_asn.json",
         "niriss/wfss/jw01324-o001_20220629t171902_spec2_007_asn.json",
     ]
@@ -57,7 +56,7 @@ def run_nis_wfss_spec3(run_nis_wfss_spec2, rtdata_module, resource_tracker):
     # Get the level3 association file and run the spec3 pipeline on it.
     # We don't need to retrieve any of the cal members of the association,
     # because they were all just created by the preceding spec2 test.
-    rtdata.get_data("niriss/wfss/jw01324-o001_20220629t171902_spec3_003_asn.json")
+    rtdata.get_data("niriss/wfss/jw01324-o001_spec3_00005_asn.json")
     args = ["calwebb_spec3", rtdata.input]
     with resource_tracker.track():
         Step.from_cmdline(args)
@@ -91,8 +90,8 @@ def test_nis_wfss_spec2(run_nis_wfss_spec2, rtdata_module, fitsdiff_default_kwar
 def test_nis_wfss_spec3(run_nis_wfss_spec3, rtdata_module, suffix, fitsdiff_default_kwargs):
     """Regression test of the calwebb_spec3 pipeline applied to NIRISS WFSS data"""
     rtdata = rtdata_module
-    rtdata.input = "jw01324-o001_20220629t171902_spec3_003_asn.json"
-    output = "jw01324-o001_t0000_niriss_f115w-gr150c-gr150r_" + suffix + ".fits"
+    rtdata.input = "jw01324-o001_spec3_00005_asn.json"
+    output = "jw01324-o001_t001_niriss_f115w-gr150c_" + suffix + ".fits"
     rtdata.output = output
     rtdata.get_truth(f"truth/test_niriss_wfss/{output}")