expand coverage for NIR instruments

Author: tapastro

jwst/regtest/test_miri_mrs_tso.py

@@ -8,9 +8,12 @@
 INPUT_PATH = 'miri/mrs'
 TRUTH_PATH = 'truth/test_miri_mrs_tso'
 
+# Mark all tests in this module
+pytestmark = [pytest.mark.bigdata]
+
 
 @pytest.fixture(scope='module')
-def run_spec2(rtdata_module):
+def run_spec2(rtdata_module, resource_tracker):
     """Run the Spec2Pipeline on a single exposure"""
     rtdata = rtdata_module
 
@@ -29,7 +32,12 @@ def run_spec2(rtdata_module):
             ]
     # FIXME: Handle warnings properly.
     # Example: RuntimeWarning: invalid value encountered in add
-    Step.from_cmdline(args)
+    with resource_tracker.track():
+        Step.from_cmdline(args)
+
+
+def test_log_tracked_resources_spec2(log_tracked_resources, run_spec2):
+    log_tracked_resources()
 
 
 @pytest.mark.bigdata

jwst/regtest/test_nircam_dhs.py

@@ -3,9 +3,12 @@
 
 from jwst.stpipe import Step
 
+# Mark all tests in this module
+pytestmark = [pytest.mark.bigdata]
+
 
 @pytest.fixture(scope="module")
-def run_detector1pipeline(rtdata_module):
+def run_detector1pipeline(rtdata_module, resource_tracker):
     """Run calwebb_detector1 on NIRCam imaging long data"""
     rtdata = rtdata_module
     rtdata.get_data("nircam/dhs/sub164stripe4_dhs_mock_dark.fits")
@@ -23,10 +26,14 @@ def run_detector1pipeline(rtdata_module):
             "--steps.jump.save_results=True",
             "--steps.jump.rejection_threshold=50.0",
             ]
-    Step.from_cmdline(args)
+    with resource_tracker.track():
+        Step.from_cmdline(args)
+
+
+def test_log_tracked_resources_det1(log_tracked_resources, run_detector1pipeline):
+    log_tracked_resources()
 
 
-@pytest.mark.bigdata
 @pytest.mark.parametrize("suffix", ["dq_init", "saturation", "superbias",
                                     "refpix", "linearity",
                                     "dark_current", "jump", "rate", "rateints"])

jwst/regtest/test_nircam_tsgrism.py

@@ -5,26 +5,30 @@
 from jwst.lib.set_telescope_pointing import add_wcs
 from jwst.stpipe import Step
 
+# Mark all tests in this module
+pytestmark = [pytest.mark.bigdata]
+
 
 @pytest.fixture(scope="module")
-def run_pipelines(rtdata_module):
+def run_pipelines(rtdata_module, resource_tracker):
     """Run stage 2-3 tso pipelines on NIRCAM TSO grism data."""
     rtdata = rtdata_module
 
-    # Run tso-spec2 pipeline on the _rateints file, saving intermediate products
-    rtdata.get_data("nircam/tsgrism/jw01366002001_04103_00001-seg001_nrcalong_rateints.fits")
-    args = ["calwebb_spec2", rtdata.input,
-            "--steps.flat_field.save_results=True",
-            "--steps.extract_2d.save_results=True",
-            "--steps.srctype.save_results=True"
-            ]
-    Step.from_cmdline(args)
-
-    # Get the level3 association json file (though not its members) and run
-    # the tso3 pipeline on all _calints files listed in association
-    rtdata.get_data("nircam/tsgrism/jw01366-o002_20230107t004627_tso3_00001_asn.json")
-    args = ["calwebb_tso3", rtdata.input]
-    Step.from_cmdline(args)
+    with resource_tracker.track():
+        # Run tso-spec2 pipeline on the _rateints file, saving intermediate products
+        rtdata.get_data("nircam/tsgrism/jw01366002001_04103_00001-seg001_nrcalong_rateints.fits")
+        args = ["calwebb_spec2", rtdata.input,
+                "--steps.flat_field.save_results=True",
+                "--steps.extract_2d.save_results=True",
+                "--steps.srctype.save_results=True"
+                ]
+        Step.from_cmdline(args)
+
+        # Get the level3 association json file (though not its members) and run
+        # the tso3 pipeline on all _calints files listed in association
+        rtdata.get_data("nircam/tsgrism/jw01366-o002_20230107t004627_tso3_00001_asn.json")
+        args = ["calwebb_tso3", rtdata.input]
+        Step.from_cmdline(args)
 
     return rtdata
 
@@ -41,7 +45,10 @@ def run_pipeline_offsetSR(request, rtdata_module):
     return rtdata
 
 
-@pytest.mark.bigdata
+def test_log_tracked_resources_pipelines(log_tracked_resources, run_pipelines):
+    log_tracked_resources()
+
+
 def test_nircam_tsgrism_stage2_offsetSR(run_pipeline_offsetSR, fitsdiff_default_kwargs):
     """
     Test coverage for offset special requirement specifying nonzero offset in X.
@@ -58,7 +65,6 @@ def test_nircam_tsgrism_stage2_offsetSR(run_pipeline_offsetSR, fitsdiff_default_
     assert diff.identical, diff.report()
 
 
-@pytest.mark.bigdata
 @pytest.mark.parametrize("suffix", ["calints", "extract_2d", "flat_field",
                                     "o002_crfints", "srctype", "x1dints"])
 def test_nircam_tsgrism_stage2(run_pipelines, fitsdiff_default_kwargs, suffix):
@@ -74,7 +80,6 @@ def test_nircam_tsgrism_stage2(run_pipelines, fitsdiff_default_kwargs, suffix):
     assert diff.identical, diff.report()
 
 
-@pytest.mark.bigdata
 def test_nircam_tsgrism_stage3_x1dints(run_pipelines, fitsdiff_default_kwargs):
     rtdata = run_pipelines
     rtdata.input = "jw01366-o002_20230107t004627_tso3_00001_asn.json"
@@ -85,7 +90,6 @@ def test_nircam_tsgrism_stage3_x1dints(run_pipelines, fitsdiff_default_kwargs):
     assert diff.identical, diff.report()
 
 
-@pytest.mark.bigdata
 def test_nircam_tsgrism_stage3_whtlt(run_pipelines):
     rtdata = run_pipelines
     rtdata.input = "jw01366-o002_20230107t004627_tso3_00001_asn.json"
@@ -99,7 +103,6 @@ def test_nircam_tsgrism_stage3_whtlt(run_pipelines):
     assert len(setdiff(table, table_truth)) == 0
 
 
-@pytest.mark.bigdata
 def test_nircam_setpointing_tsgrism(rtdata, fitsdiff_default_kwargs):
     """
     Regression test of the set_telescope_pointing script on a level-1b NIRCam file.

jwst/regtest/test_nircam_tsimg.py

@@ -4,9 +4,12 @@
 from jwst.lib.set_telescope_pointing import add_wcs
 from jwst.stpipe import Step
 
+# Mark all tests in this module
+pytestmark = [pytest.mark.bigdata]
+
 
 @pytest.fixture(scope="module")
-def run_pipelines(rtdata_module):
+def run_pipelines(rtdata_module, resource_tracker):
     """Run stage 2 and 3 pipelines on NIRCam TSO image data."""
 
     rtdata = rtdata_module
@@ -24,11 +27,16 @@ def run_pipelines(rtdata_module):
     # the tso3 pipeline on all _calints files listed in association
     rtdata.get_data("nircam/tsimg/jw01068-o006_20240401t151322_tso3_00002_asn.json")
     args = ["calwebb_tso3", rtdata.input]
-    Step.from_cmdline(args)
+    with resource_tracker.track():
+        Step.from_cmdline(args)
 
     return rtdata
 
 
+def test_log_tracked_resources_tsimg(log_tracked_resources, run_pipelines):
+    log_tracked_resources()
+
+
 @pytest.mark.bigdata
 @pytest.mark.parametrize("suffix", ["calints", "o006_crfints"])
 def test_nircam_tsimg_stage2(run_pipelines, fitsdiff_default_kwargs, suffix):

jwst/regtest/test_nircam_wfss_contam.py

@@ -3,9 +3,12 @@
 
 from jwst.regtest import regtestdata as rt
 
+# Mark all tests in this module
+pytestmark = [pytest.mark.bigdata]
+
 
 @pytest.fixture(scope='module')
-def run_wfss_contam(rtdata_module):
+def run_wfss_contam(rtdata_module, resource_tracker):
     """Run the wfss_contam step"""
     rtdata = rtdata_module
 
@@ -24,11 +27,17 @@ def run_wfss_contam(rtdata_module):
             '--skip=False',
         ]
     }
-    rtdata = rt.run_step_from_dict(rtdata, **step_params)
+    with resource_tracker.track():
+        rtdata = rt.run_step_from_dict(rtdata, **step_params)
     return rtdata
 
-@pytest.mark.skip(reason='Test too slow until stdatamodels PR#165 merged')
-@pytest.mark.bigdata
+
+@pytest.mark.skip(reason='Test too slow until contam is updated')
+def test_log_tracked_resources_tsimg(log_tracked_resources, run_wfss_contam):
+    log_tracked_resources()
+
+
+@pytest.mark.skip(reason='Test too slow until contam is updated')
 @pytest.mark.parametrize(
     'suffix',
     ['simul', 'contam', 'wfsscontamstep']

jwst/regtest/test_nircam_wfss_spec2.py

@@ -4,9 +4,12 @@
 
 from jwst.stpipe import Step
 
+# Mark all tests in this module
+pytestmark = [pytest.mark.bigdata]
+
 
 @pytest.fixture(scope="module")
-def run_pipeline(rtdata_module):
+def run_pipeline(rtdata_module, resource_tracker):
     """Run the calwebb_spec2 pipeline on a single NIRSpec WFSS exposure."""
 
     rtdata = rtdata_module
@@ -27,12 +30,16 @@ def run_pipeline(rtdata_module):
             "--steps.extract_2d.wfss_nbright=20",
             "--steps.srctype.save_results=true",
             "--steps.flat_field.save_results=true"]
-    Step.from_cmdline(args)
+    with resource_tracker.track():
+        Step.from_cmdline(args)
 
     return rtdata
 
 
-@pytest.mark.bigdata
+def test_log_tracked_resources_spec2(log_tracked_resources, run_pipeline):
+    log_tracked_resources()
+
+
 @pytest.mark.parametrize("suffix", [
     "assign_wcs", "bsub", "extract_2d", "flat_field", "srctype",
     "cal", "x1d"])

jwst/regtest/test_niriss_ami3.py

@@ -3,16 +3,20 @@
 
 from jwst.stpipe import Step
 
+# Mark all tests in this module
+pytestmark = [pytest.mark.bigdata]
+
 
 @pytest.fixture(scope="module")
-def run_pipeline(rtdata_module):
+def run_pipeline(rtdata_module, resource_tracker):
     """Run calwebb_ami3 on NIRISS AMI data."""
     rtdata = rtdata_module
     rtdata.get_asn("niriss/ami/ami3_test_asn.json")
 
     # Run the calwebb_ami3 pipeline on the association
     args = ["calwebb_ami3", rtdata.input]
-    Step.from_cmdline(args)
+    with resource_tracker.track():
+        Step.from_cmdline(args)
 
     return rtdata
 
@@ -32,7 +36,10 @@ def run_step_with_cal(rtdata_module):
     return rtdata
 
 
-@pytest.mark.bigdata
+def test_log_tracked_resources_ami3(log_tracked_resources, run_pipeline):
+    log_tracked_resources()
+
+
 @pytest.mark.parametrize("obs, suffix", [("012", "ami-oi"), ("015", "psf-ami-oi")])
 def test_niriss_ami3_exp(run_pipeline, obs, suffix, fitsdiff_default_kwargs):
     """Check exposure-level results of calwebb_ami3"""
@@ -47,7 +54,6 @@ def test_niriss_ami3_exp(run_pipeline, obs, suffix, fitsdiff_default_kwargs):
     assert diff.identical, diff.report()
 
 
-@pytest.mark.bigdata
 def test_niriss_ami3_product(run_pipeline, fitsdiff_default_kwargs):
     """Check final products of calwebb_ami3"""
     rtdata = run_pipeline
@@ -61,7 +67,6 @@ def test_niriss_ami3_product(run_pipeline, fitsdiff_default_kwargs):
     assert diff.identical, diff.report()
 
 
-@pytest.mark.bigdata
 @pytest.mark.parametrize("suffix", ("ami-oi", "amimulti-oi", "amilg"))
 def test_niriss_ami3_cal(run_step_with_cal, suffix, fitsdiff_default_kwargs):
     rtdata = run_step_with_cal