JP-3775: Expand usage of resource_tracker for benchmarking

jwst/regtest/test_miri_coron3.py

@@ -3,9 +3,12 @@
 
 from jwst.stpipe import Step
 
+# Mark all tests in this module
+pytestmark = [pytest.mark.bigdata]
+
 
 @pytest.fixture(scope="module")
-def run_pipeline(rtdata_module):
+def run_pipeline(rtdata_module, resource_tracker):
     """Run calwebb_coron3 on MIRI 4QPM coronographic data."""
     rtdata = rtdata_module
     rtdata.get_asn("miri/coron/jw01386-c1002_20230109t015044_coron3_00001_asn.json")
@@ -16,12 +19,16 @@ def run_pipeline(rtdata_module):
     ]
     # FIXME: Handle warnings properly.
     # Example: RuntimeWarning: Mean of empty slice
-    Step.from_cmdline(args)
+    with resource_tracker.track():
+        Step.from_cmdline(args)
 
     return rtdata
 
 
-@pytest.mark.bigdata
+def test_log_tracked_resources_coron3(log_tracked_resources, run_pipeline):
+    log_tracked_resources()
+
+
 @pytest.mark.parametrize("suffix", ["crfints", "psfalign", "psfsub"])
 @pytest.mark.parametrize("exposure", ["4", "5"])
 def test_miri_coron3_sci_exp(run_pipeline, suffix, exposure, fitsdiff_default_kwargs):
@@ -37,7 +44,6 @@ def test_miri_coron3_sci_exp(run_pipeline, suffix, exposure, fitsdiff_default_kw
     assert diff.identical, diff.report()
 
 
-@pytest.mark.bigdata
 @pytest.mark.parametrize("suffix", ["psfstack", "i2d"])
 def test_miri_coron3_product(run_pipeline, suffix, fitsdiff_default_kwargs):
     """Check final products of calwebb_coron3"""

jwst/regtest/test_miri_coron_image2.py

@@ -6,8 +6,11 @@
 
 from jwst.stpipe import Step
 
+# Mark all tests in this module
+pytestmark = [pytest.mark.bigdata]
+
 @pytest.fixture(scope='module')
-def run_image2(rtdata_module):
+def run_image2(rtdata_module, resource_tracker):
     """Run the calwebb_image2 pipeline"""
 
     rtdata = rtdata_module
@@ -18,12 +21,16 @@ def run_image2(rtdata_module):
             "--steps.bkg_subtract.save_combined_background=true",
             "--steps.assign_wcs.save_results=true",
             "--steps.flat_field.save_results=true"]
-    Step.from_cmdline(args)
+    with resource_tracker.track():
+        Step.from_cmdline(args)
 
     return rtdata
 
 
-@pytest.mark.bigdata
+def test_log_tracked_resources_image2(log_tracked_resources, run_image2):
+    log_tracked_resources()
+
+
 @pytest.mark.parametrize("suffix", ["bsubints", "combinedbackground", "assign_wcs", "flat_field", "calints"])
 def test_miri_coron_image2(run_image2, fitsdiff_default_kwargs, suffix):
     """Regression test for image2 processing of MIRI coronagraphic data with background exposures"""

jwst/regtest/test_miri_image.py

@@ -131,7 +131,9 @@ def run_image2(run_detector1, rtdata_module, resource_tracker):
             "--steps.flat_field.save_results=True"
             ]
     with warnings.catch_warnings():
-        warnings.filterwarnings("ignore", message="Failed to achieve requested SIP approximation accuracy")
+        warnings.filterwarnings(
+            "ignore", message="Failed to achieve requested SIP approximation accuracy"
+        )
         Step.from_cmdline(args)
 
         # Grab rest of _rate files for the asn and run image2 pipeline on each to
@@ -151,15 +153,17 @@ def run_image2(run_detector1, rtdata_module, resource_tracker):
 
 
 @pytest.fixture(scope="module")
-def run_image3(run_image2, rtdata_module):
+def run_image3(run_image2, rtdata_module, resource_tracker):
     """Get the level3 association json file (though not its members) and run
     image3 pipeline on all _cal files listed in association"""
     rtdata = rtdata_module
     rtdata.get_data("miri/image/jw01024-o001_20220501t155404_image3_001_asn.json")
     args = ["jwst.pipeline.Image3Pipeline", rtdata.input]
     with warnings.catch_warnings():
-        warnings.filterwarnings("ignore", message="Failed to achieve requested SIP approximation accuracy")
-        Step.from_cmdline(args)
+        warnings.filterwarnings("ignore",
+                                message="Failed to achieve requested SIP approximation accuracy")
+        with resource_tracker.track():
+            Step.from_cmdline(args)
 
 
 def test_log_tracked_resources_det1(log_tracked_resources, run_detector1_with_average_dark_current):
@@ -170,6 +174,10 @@ def test_log_tracked_resources_image2(log_tracked_resources, run_image2):
     log_tracked_resources()
 
 
+def test_log_tracked_resources_image3(log_tracked_resources, run_image3):
+    log_tracked_resources()
+
+
 @pytest.mark.parametrize("suffix", ["dq_init", "saturation", "firstframe", "lastframe", "reset",
                                     "linearity", "rscd", "dark_current", "ramp", "rate",
                                     "rateints"])
@@ -188,38 +196,6 @@ def test_miri_image_detector1_multiprocess_jump(run_detector1_multiprocess_jump,
     _assert_is_same(rtdata_module, fitsdiff_default_kwargs, "rate")
 
 
-def test_detector1_mem_usage(rtdata_module):
-    """Determine the memory usage for Detector 1"""
-    rtdata = rtdata_module
-    rtdata.get_data("miri/image/jw01024001001_04101_00001_mirimage_uncal.fits")
-    args = ["jwst.pipeline.Detector1Pipeline", rtdata.input]
-
-    # starting the monitoring
-    tracemalloc.start()
-
-    # run Detector1
-    Step.from_cmdline(args)
-
-    # displaying the memory
-    current_mem, peak_mem = tracemalloc.get_traced_memory()
-    # convert bytes to GB
-    peak_mem *= 1e-9
-    peak_mem = np.round(peak_mem, decimals=1)
-
-    # stopping the monitoring
-    tracemalloc.stop()
-
-    # set comparison values in GB
-    mem_threshold = 16.0  # average user's available memory
-    mem_benchmark = 11.0   # benchmark run with build 1.15.1 + 1 additional GB
-
-    # test that max memory is less than threshold
-    assert peak_mem < mem_threshold, "Max memory used is greater than 16 GB!"
-
-    # test that max memory is less or equal to benchmark
-    assert peak_mem <= mem_benchmark, "Max memory used is greater than 11 GB"
-
-
 @pytest.mark.parametrize("suffix", ["dark_current", "ramp", "rate",
                                     "rateints"])
 def test_miri_image_detector1_with_avg_dark_current(run_detector1_with_average_dark_current,

jwst/regtest/test_miri_lrs_slit_spec2.py

@@ -9,8 +9,12 @@
 from jwst.extract_1d import Extract1dStep
 from stcal.alignment import util
 
+# Mark all tests in this module
+pytestmark = [pytest.mark.bigdata]
+
+
 @pytest.fixture(scope="module")
-def run_pipeline(rtdata_module):
+def run_pipeline(rtdata_module, resource_tracker):
     """Run the calwebb_spec2 pipeline on an ASN of nodded MIRI LRS
        fixedslit exposures."""
     rtdata = rtdata_module
@@ -29,10 +33,14 @@ def run_pipeline(rtdata_module):
             "--steps.pixel_replace.save_results=true",
             "--steps.bkg_subtract.save_combined_background=true"
             ]
-    Step.from_cmdline(args)
+    with resource_tracker.track():
+        Step.from_cmdline(args)
+
+
+def test_log_tracked_resources_spec2(log_tracked_resources, run_pipeline):
+    log_tracked_resources()
 
 
-@pytest.mark.bigdata
 @pytest.mark.parametrize("suffix", [
     "assign_wcs", "combinedbackground", "bsub", "srctype", "flat_field", "pathloss",
     "cal", "pixel_replace", "s2d", "x1d"])
@@ -52,7 +60,6 @@ def test_miri_lrs_slit_spec2(run_pipeline, fitsdiff_default_kwargs, suffix, rtda
     assert diff.identical, diff.report()
 
 
-@pytest.mark.bigdata
 def test_miri_lrs_extract1d_from_cal(run_pipeline, rtdata_module, fitsdiff_default_kwargs):
     rtdata = rtdata_module
     rtdata.input = "jw01530005001_03103_00001_mirimage_cal.fits"
@@ -64,7 +71,6 @@ def test_miri_lrs_extract1d_from_cal(run_pipeline, rtdata_module, fitsdiff_defau
     assert diff.identical, diff.report()
 
 
-@pytest.mark.bigdata
 def test_miri_lrs_slit_wcs(run_pipeline, rtdata_module, fitsdiff_default_kwargs):
     rtdata = rtdata_module
     # get input assign_wcs and truth file

jwst/regtest/test_miri_lrs_slit_spec3.py

@@ -2,20 +2,24 @@
     This takes an association and generates the level 3 products."""
 import pytest
 import numpy as np
-from gwcs import wcstools
 
+
+from gwcs import wcstools
 import asdf
 from astropy.io.fits.diff import FITSDiff
 
 from jwst.stpipe import Step
 from jwst import datamodels
 
+# Mark all tests in this module
+pytestmark = [pytest.mark.bigdata]
+
 
 @pytest.fixture(
     scope="module",
     params=["default_wcs", "user_wcs", "user_wcs+shape", "user_wcs+shape1"]
 )
-def run_pipeline(rtdata_module, request):
+def run_pipeline(rtdata_module, request, resource_tracker):
     """
     Run the calwebb_spec3 pipeline on an ASN of nodded MIRI LRS
     fixed-slit exposures using different options for the WCS and output
@@ -74,10 +78,14 @@ def run_pipeline(rtdata_module, request):
         args.append(f"--steps.resample_spec.output_file={output_file}")
 
     # Run the calwebb_spec3 pipeline; save results from intermediate steps
-    Step.from_cmdline(args)
+    with resource_tracker.track():
+        Step.from_cmdline(args)
+
+
+def test_log_tracked_resources_spec3(log_tracked_resources, run_pipeline):
+    log_tracked_resources()
 
 
-@pytest.mark.bigdata
 @pytest.mark.parametrize("suffix", ["s2d", "x1d"])
 def test_miri_lrs_slit_spec3(run_pipeline, rtdata_module, fitsdiff_default_kwargs, suffix):
     """Regression test of the calwebb_spec3 pipeline on MIRI

jwst/regtest/test_miri_lrs_slitless.py

@@ -15,6 +15,9 @@
 PRODUCT_NAME = "jw01536-o028_t008_miri_p750l-slitlessprism"
 ASN_ID = "o028"
 
+# Mark all tests in this module
+pytestmark = [pytest.mark.bigdata]
+
 
 @pytest.fixture(scope="module")
 def run_tso1_pipeline(rtdata_module):
@@ -72,7 +75,7 @@ def run_detector1_pipeline_emicorr_joint(rtdata_module):
 
 
 @pytest.fixture(scope="module")
-def run_tso_spec2_pipeline(run_tso1_pipeline, rtdata_module):
+def run_tso_spec2_pipeline(run_tso1_pipeline, rtdata_module, resource_tracker):
     """Run the calwebb_tso-spec2 pipeline on a MIRI LRS slitless exposure."""
     rtdata = rtdata_module
 
@@ -89,11 +92,12 @@ def run_tso_spec2_pipeline(run_tso1_pipeline, rtdata_module):
     ]
     # FIXME: Handle warnings properly.
     # Example: RuntimeWarning: invalid value encountered in multiply
-    Step.from_cmdline(args)
+    with resource_tracker.track():
+        Step.from_cmdline(args)
 
 
 @pytest.fixture(scope="module")
-def run_tso3_pipeline(run_tso_spec2_pipeline, rtdata_module):
+def run_tso3_pipeline(run_tso_spec2_pipeline, rtdata_module, resource_tracker):
     """Run the calwebb_tso3 pipeline on the output of run_spec2_pipeline."""
     rtdata = rtdata_module
     rtdata.get_data(f"miri/lrs/{DATASET2_ID}_calints.fits")
@@ -105,10 +109,18 @@ def run_tso3_pipeline(run_tso_spec2_pipeline, rtdata_module):
         "--steps.outlier_detection.save_results=true",
         "--steps.outlier_detection.save_intermediate_results=true"
     ]
-    Step.from_cmdline(args)
+    with resource_tracker.track():
+        Step.from_cmdline(args)
+
+
+def test_log_tracked_resources_spec2(log_tracked_resources, run_tso_spec2_pipeline):
+    log_tracked_resources()
+
+
+def test_log_tracked_resources_spec3(log_tracked_resources, run_tso3_pipeline):
+    log_tracked_resources()
 
 
-@pytest.mark.bigdata
 @pytest.mark.parametrize("step_suffix", ['dq_init', 'saturation', 'lastframe', 'reset', 'linearity',
                                          'dark_current', 'ramp', 'rate', 'rateints'])
 def test_miri_lrs_slitless_tso1(run_tso1_pipeline, rtdata_module, fitsdiff_default_kwargs, step_suffix):
@@ -123,7 +135,6 @@ def test_miri_lrs_slitless_tso1(run_tso1_pipeline, rtdata_module, fitsdiff_defau
     assert diff.identical, diff.report()
 
 
-@pytest.mark.bigdata
 @pytest.mark.parametrize("step_suffix", ['rscd', 'emicorr',
                                          'dark_current', 'ramp', 'rate', 'rateints'])
 def test_miri_lrs_slitless_detector1(run_detector1_pipeline, rtdata_module,
@@ -144,7 +155,6 @@ def test_miri_lrs_slitless_detector1(run_detector1_pipeline, rtdata_module,
     assert diff.identical, diff.report()
 
 
-@pytest.mark.bigdata
 @pytest.mark.parametrize("step_suffix", ['emicorr', 'rate', 'rateints'])
 def test_miri_lrs_slitless_detector1_emicorr_joint(
         run_detector1_pipeline_emicorr_joint, rtdata_module,
@@ -160,7 +170,6 @@ def test_miri_lrs_slitless_detector1_emicorr_joint(
     assert diff.identical, diff.report()
 
 
-@pytest.mark.bigdata
 @pytest.mark.parametrize("step_suffix", ["assign_wcs", "srctype", "flat_field",
                                          "pixel_replace", "calints", "x1dints"])
 def test_miri_lrs_slitless_tso_spec2(run_tso_spec2_pipeline, rtdata_module, fitsdiff_default_kwargs,
@@ -176,7 +185,6 @@ def test_miri_lrs_slitless_tso_spec2(run_tso_spec2_pipeline, rtdata_module, fits
     assert diff.identical, diff.report()
 
 
-@pytest.mark.bigdata
 @pytest.mark.parametrize("step_suffix", ["outlier_detection", "crfints"])
 def test_miri_lrs_slitless_tso3(run_tso3_pipeline, rtdata_module,
                                 fitsdiff_default_kwargs, step_suffix):
@@ -194,7 +202,6 @@ def test_miri_lrs_slitless_tso3(run_tso3_pipeline, rtdata_module,
     assert diff.identical, diff.report()
 
 
-@pytest.mark.bigdata
 def test_miri_lrs_slitless_tso3_x1dints(run_tso3_pipeline, rtdata_module,
                                         fitsdiff_default_kwargs):
     """Compare the output of a MIRI LRS slitless calwebb_tso3 pipeline."""
@@ -208,7 +215,6 @@ def test_miri_lrs_slitless_tso3_x1dints(run_tso3_pipeline, rtdata_module,
     assert diff.identical, diff.report()
 
 
-@pytest.mark.bigdata
 def test_miri_lrs_slitless_tso3_whtlt(run_tso3_pipeline,
                                       rtdata_module, diff_astropy_tables):
     """Compare the whitelight output of a MIRI LRS slitless calwebb_tso3 pipeline."""
@@ -221,7 +227,6 @@ def test_miri_lrs_slitless_tso3_whtlt(run_tso3_pipeline,
     assert diff_astropy_tables(rtdata.output, rtdata.truth)
 
 
-@pytest.mark.bigdata
 def test_miri_lrs_slitless_wcs(run_tso_spec2_pipeline, fitsdiff_default_kwargs,
                                rtdata_module):
     """Compare the assign_wcs output of a MIRI LRS slitless calwebb_tso3 pipeline."""