From 4f9582bbf39fc5babbbc2f66faac08fb57e59a8a Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Wed, 20 Sep 2023 18:26:39 -0400 Subject: [PATCH 01/12] Update the memory used by the atmospheric screen. Use the production instead of test value in the default config. --- config/imsim-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/imsim-config.yaml b/config/imsim-config.yaml index ee3f7f63..8120aed9 100644 --- a/config/imsim-config.yaml +++ b/config/imsim-config.yaml @@ -80,7 +80,7 @@ input: t0: 0 # seconds exptime: $exptime kcrit: 0.2 # in units of 1/r0 - screen_size: 409.6 # Default=812.2, which takes a lot of memory, so use this for testing. + screen_size: 812.2 # If you want less memory for testing, you can try 409.6 screen_scale: 0.1 # meters doOpt: False nproc: 1 # Default (None) means one proc per screen. From 1f6bbb0dee65ccd3ed194c17859a950b4a992567 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Wed, 20 Sep 2023 19:23:06 -0400 Subject: [PATCH 02/12] Update ci file New download URL (forgot to this in the other PR) change to pip without a -e to try to fix a strange CI error. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59603f24..fc2d41f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,13 +61,13 @@ jobs: git clone https://github.com/lsst/rubin_sim.git cd rubin_sim git checkout 1.1.1 - pip install -e . + pip install . cd .. mkdir rubin_sim_data mkdir rubin_sim_data/sims_sed_library # Just get the skybrightness, throughputs, and SED data for now. curl https://s3df.slac.stanford.edu/groups/rubin/static/sim-data/rubin_sim_data/skybrightness_may_2021.tgz | tar -C rubin_sim_data -xz - curl https://s3df.slac.stanford.edu/groups/rubin/static/sim-data/rubin_sim_data/throughputs_aug_2021.tgz | tar -C rubin_sim_data -xz + curl https://s3df.slac.stanford.edu/groups/rubin/static/sim-data/rubin_sim_data/throughputs_2023_09_07.tgz | tar -C rubin_sim_data -xz curl https://s3df.slac.stanford.edu/groups/rubin/static/sim-data/sed_library/seds_170124.tar.gz | tar -C rubin_sim_data/sims_sed_library -xz - name: Install imSim From 79085006158031879aa2efd8b1a0f7b10d84a6a0 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Thu, 21 Sep 2023 11:25:03 -0400 Subject: [PATCH 03/12] Minor text change in config description. --- config/imsim-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/imsim-config.yaml b/config/imsim-config.yaml index 8120aed9..cb5ec8a0 100644 --- a/config/imsim-config.yaml +++ b/config/imsim-config.yaml @@ -80,7 +80,7 @@ input: t0: 0 # seconds exptime: $exptime kcrit: 0.2 # in units of 1/r0 - screen_size: 812.2 # If you want less memory for testing, you can try 409.6 + screen_size: 812.2 # To use less memory for testing/development, you can try 409.6 screen_scale: 0.1 # meters doOpt: False nproc: 1 # Default (None) means one proc per screen. From e69146220c4e2985a63a6813fbc6933429353485 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Thu, 21 Sep 2023 12:10:50 -0400 Subject: [PATCH 04/12] Change CI to install rubin-sim with conda. There are breaking changes in the new version of pip that are incompatible with the current version of rubin-sim. --- .github/workflows/ci.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc2d41f1..2f228a8a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,11 +58,18 @@ jobs: run: | # Do the minimal installation from source to avoid installing # a bunch of unneeded packages. - git clone https://github.com/lsst/rubin_sim.git - cd rubin_sim - git checkout 1.1.1 - pip install . - cd .. + ## + ## Pip has changed how it operates in a breaking way for us. + ## Try using conda instead. + ## + ## git clone https://github.com/lsst/rubin_sim.git + ## cd rubin_sim + ## git checkout 1.1.1 + ## pip -e install . + ## cd .. + ## + conda install rubin-sim=1.1.1 + ## mkdir rubin_sim_data mkdir rubin_sim_data/sims_sed_library # Just get the skybrightness, throughputs, and SED data for now. From c30b098ee6d30554b122e34000f6a7f52b770703 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Thu, 21 Sep 2023 13:53:30 -0400 Subject: [PATCH 05/12] Switch to using mamba for the installs for speed. --- .github/workflows/ci.yml | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f228a8a..45481a10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,7 @@ jobs: conda install -y mamba mamba install -y --file etc/standalone_conda_requirements.txt conda info - conda list + mamba list - name: Install skyCatalogs from source run: | @@ -56,20 +56,9 @@ jobs: - name: Install rubin_sim and rubin_sim_data run: | - # Do the minimal installation from source to avoid installing - # a bunch of unneeded packages. - ## - ## Pip has changed how it operates in a breaking way for us. - ## Try using conda instead. - ## - ## git clone https://github.com/lsst/rubin_sim.git - ## cd rubin_sim - ## git checkout 1.1.1 - ## pip -e install . - ## cd .. - ## - conda install rubin-sim=1.1.1 - ## + # Install a fixed version so we control the code for tests etc. + mamba install rubin-sim=1.1.1 + # mkdir rubin_sim_data mkdir rubin_sim_data/sims_sed_library # Just get the skybrightness, throughputs, and SED data for now. From f74037e264e51d76dfd6259994408c2aca9329aa Mon Sep 17 00:00:00 2001 From: Mike Jarvis Date: Thu, 21 Sep 2023 14:33:39 -0400 Subject: [PATCH 06/12] Truncate the limits of the new filters more aggressively --- imsim/bandpass.py | 1 + 1 file changed, 1 insertion(+) diff --git a/imsim/bandpass.py b/imsim/bandpass.py index 1bc582f5..00e0ae02 100644 --- a/imsim/bandpass.py +++ b/imsim/bandpass.py @@ -28,6 +28,7 @@ def RubinBandpass(band, logger=None): logger.warning("Warning: Using the old bandpass files from GalSim, not lsst.sims") file_name = f"LSST_{band}.dat" bp = galsim.Bandpass(file_name, wave_type='nm') + bp = bp.truncate(relative_throughput=1.e-3) bp = bp.thin() bp = bp.withZeropoint('AB') return bp From 2d0d190f0441cd20ccb4f135b6cf820f5d8fa89e Mon Sep 17 00:00:00 2001 From: Mike Jarvis Date: Thu, 21 Sep 2023 14:34:28 -0400 Subject: [PATCH 07/12] Loosen the test tolerances for the agreement of new filters vs GalSim lsst filters --- tests/test_bandpass.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/test_bandpass.py b/tests/test_bandpass.py index 0564861a..5bd40828 100644 --- a/tests/test_bandpass.py +++ b/tests/test_bandpass.py @@ -11,14 +11,19 @@ def test_rubin_bandpass(): print(f'Band = {band}') print('Rubin_bp: ',rubin_bp.blue_limit, rubin_bp.effective_wavelength, rubin_bp.red_limit) print('galsim_bp: ',galsim_bp.blue_limit, galsim_bp.effective_wavelength, galsim_bp.red_limit) - # These are quite close. The only exception is the blue_limit of the u band, where the - # old GalSim bandpass is too red. - if band == 'u': - assert np.isclose(rubin_bp.blue_limit, galsim_bp.blue_limit, atol=20) + #print('Rubin_bp: ',repr(rubin_bp)) + + # These are reasonably close. Especially in the effective wavelength. + # The limits of the actual filters are a bit different in some cases though from what was + # estimated many years ago. Especially for u and y bands. + if band in 'uy': + assert np.isclose(rubin_bp.blue_limit, galsim_bp.blue_limit, atol=30) + assert np.isclose(rubin_bp.red_limit, galsim_bp.red_limit, atol=10) + assert np.isclose(rubin_bp.effective_wavelength, galsim_bp.effective_wavelength, atol=5) else: - assert np.isclose(rubin_bp.blue_limit, galsim_bp.blue_limit, atol=1) - assert np.isclose(rubin_bp.red_limit, galsim_bp.red_limit, atol=1) - assert np.isclose(rubin_bp.effective_wavelength, galsim_bp.effective_wavelength, atol=3) + assert np.isclose(rubin_bp.blue_limit, galsim_bp.blue_limit, atol=15) + assert np.isclose(rubin_bp.red_limit, galsim_bp.red_limit, atol=8) + assert np.isclose(rubin_bp.effective_wavelength, galsim_bp.effective_wavelength, atol=3) if __name__ == "__main__": From a02db6df1749d784a20b7ab118586536dfb97068 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Fri, 22 Sep 2023 10:03:14 -0700 Subject: [PATCH 08/12] use current rubin_sim.skybrightness interface, and ensure that GalSim versions of LSST bandpasses are used --- tests/sky_level_reference_values.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/sky_level_reference_values.py b/tests/sky_level_reference_values.py index da022efb..a9663dc9 100644 --- a/tests/sky_level_reference_values.py +++ b/tests/sky_level_reference_values.py @@ -2,8 +2,11 @@ import warnings import numpy as np import json +from unittest import mock import galsim from rubin_sim import skybrightness +import imsim +from imsim_test_helpers import CaptureLog RUBIN_AREA = 0.25 * np.pi * 649**2 # cm^2 @@ -19,12 +22,14 @@ sky_model = skybrightness.SkyModel() with warnings.catch_warnings(): warnings.simplefilter('ignore') - sky_model.setRaDecMjd(ra, dec, mjd, degrees=True) + sky_model.set_ra_dec_mjd(ra, dec, mjd, degrees=True) sky_levels = {} for band in 'ugrizy': - bandpass = galsim.RubinBandpass(band) - wave, spec = sky_model.returnWaveSpec() + with mock.patch('os.getenv', return_value=''): + with CaptureLog() as cl: + bandpass = imsim.RubinBandpass(band, logger=cl.logger) + wave, spec = sky_model.return_wave_spec() lut = galsim.LookupTable(wave, spec[0]) sed = galsim.SED(lut, wave_type='nm', flux_type='flambda') sky_levels[band] = sed.calculateFlux(bandpass)*RUBIN_AREA*exptime From bd1c169aef6c2262e32a95b35a278926a98793fe Mon Sep 17 00:00:00 2001 From: James Chiang Date: Fri, 22 Sep 2023 10:04:05 -0700 Subject: [PATCH 09/12] update sky model reference sky levels --- tests/data/reference_sky_levels.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/reference_sky_levels.json b/tests/data/reference_sky_levels.json index d1b09677..cb2c990e 100644 --- a/tests/data/reference_sky_levels.json +++ b/tests/data/reference_sky_levels.json @@ -1 +1 @@ -{"u": 940.8063879564563, "g": 8041.5851235691625, "r": 16237.67541651461, "i": 28067.137304469303, "z": 47780.00064197293, "y": 46624.55973988414} \ No newline at end of file +{"u": 938.8341725029875, "g": 8028.292239852839, "r": 16211.09380977137, "i": 27918.842213779477, "z": 47721.355949474804, "y": 46615.285459051345} \ No newline at end of file From 952920dcae9945215cacea2eb73bf92227be946b Mon Sep 17 00:00:00 2001 From: Mike Jarvis Date: Fri, 22 Sep 2023 16:03:51 -0400 Subject: [PATCH 10/12] Add a test of the fallback to galsim.Bandpass when RUBIN_SIM_DATA is not defined. --- tests/test_bandpass.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_bandpass.py b/tests/test_bandpass.py index 5bd40828..7a7bce24 100644 --- a/tests/test_bandpass.py +++ b/tests/test_bandpass.py @@ -1,6 +1,8 @@ import numpy as np import galsim from imsim import RubinBandpass +from unittest import mock +from imsim_test_helpers import CaptureLog def test_rubin_bandpass(): """Check the RubinBandpass functions to check that they are similar to GalSim lsst_*.dat files. @@ -26,6 +28,22 @@ def test_rubin_bandpass(): assert np.isclose(rubin_bp.effective_wavelength, galsim_bp.effective_wavelength, atol=3) + # If RUBIN_SIM_DATA isn't availble, then RubinBandpass reverts to the galsim files, + # and everything should match exactly. + with mock.patch('os.getenv', return_value=''): + with CaptureLog() as cl: + alt_rubin_bp = RubinBandpass(band, logger=cl.logger) + assert "Using the old bandpass files" in cl.output + print('Alt_Rubin_bp: ',alt_rubin_bp.blue_limit, alt_rubin_bp.effective_wavelength, + alt_rubin_bp.red_limit) + galsim_bp = galsim.Bandpass(f"LSST_{band}.dat", wave_type='nm') + galsim_bp = galsim_bp.truncate(relative_throughput=1.e-3) + galsim_bp = galsim_bp.thin() + galsim_bp = galsim_bp.withZeropoint('AB') + assert alt_rubin_bp == galsim_bp + + + if __name__ == "__main__": testfns = [v for k, v in vars().items() if k[:5] == 'test_' and callable(v)] for testfn in testfns: From 4aa0a0df9283baba7bf549e1a95790ddbc049c7b Mon Sep 17 00:00:00 2001 From: James Chiang Date: Fri, 22 Sep 2023 17:36:03 -0700 Subject: [PATCH 11/12] use throughputs in RUBIN_SIM_DATA_DIR --- tests/data/reference_sky_levels.json | 2 +- tests/sky_level_reference_values.py | 6 +----- tests/test_sky_model.py | 12 ++---------- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/tests/data/reference_sky_levels.json b/tests/data/reference_sky_levels.json index cb2c990e..04ada9b5 100644 --- a/tests/data/reference_sky_levels.json +++ b/tests/data/reference_sky_levels.json @@ -1 +1 @@ -{"u": 938.8341725029875, "g": 8028.292239852839, "r": 16211.09380977137, "i": 27918.842213779477, "z": 47721.355949474804, "y": 46615.285459051345} \ No newline at end of file +{"u": 966.8740383242452, "g": 9565.16844049534, "r": 20335.64330578381, "i": 39770.05348346839, "z": 66820.68250072759, "y": 62387.30799288845} \ No newline at end of file diff --git a/tests/sky_level_reference_values.py b/tests/sky_level_reference_values.py index a9663dc9..1b777ade 100644 --- a/tests/sky_level_reference_values.py +++ b/tests/sky_level_reference_values.py @@ -2,11 +2,9 @@ import warnings import numpy as np import json -from unittest import mock import galsim from rubin_sim import skybrightness import imsim -from imsim_test_helpers import CaptureLog RUBIN_AREA = 0.25 * np.pi * 649**2 # cm^2 @@ -26,9 +24,7 @@ sky_levels = {} for band in 'ugrizy': - with mock.patch('os.getenv', return_value=''): - with CaptureLog() as cl: - bandpass = imsim.RubinBandpass(band, logger=cl.logger) + bandpass = imsim.RubinBandpass(band) wave, spec = sky_model.return_wave_spec() lut = galsim.LookupTable(wave, spec[0]) sed = galsim.SED(lut, wave_type='nm', flux_type='flambda') diff --git a/tests/test_sky_model.py b/tests/test_sky_model.py index 8a261a26..d542c431 100644 --- a/tests/test_sky_model.py +++ b/tests/test_sky_model.py @@ -5,8 +5,6 @@ import logging import galsim from imsim import SkyModel, SkyGradient, make_batoid_wcs, RubinBandpass -from unittest import mock -from imsim_test_helpers import CaptureLog DATA_DIR = Path(__file__).parent / 'data' @@ -32,12 +30,7 @@ def test_sky_model(): expected_sky_levels = json.load(fobj) for band in 'ugrizy': - # This regression test used the GalSim bandpasses. - # So use them here by mocking that RUBIN_DATA_DIR is not defined. - with mock.patch('os.getenv', return_value=''): - with CaptureLog() as cl: - bandpass = RubinBandpass(band, logger=cl.logger) - assert "Using the old bandpass files" in cl.output + bandpass = RubinBandpass(band) sky_model = SkyModel(exptime, mjd, bandpass) sky_level = sky_model.get_sky_level(skyCoord) np.testing.assert_approx_equal(sky_level, expected_sky_levels[band], @@ -50,7 +43,6 @@ def test_sky_gradient(): # http://astro-lsst-01.astro.washington.edu:8080/ ra = 54.9348753510528 dec = -35.8385705255579 - skyCoord = galsim.CelestialCoord(ra*galsim.degrees, dec*galsim.degrees) mjd = 60232.3635999295 rottelpos = 350.946271812373 exptime = 30. @@ -91,7 +83,6 @@ def test_sky_gradient(): sky_gradient(*r[:2]), significant=4) - # Check that it gets applied by LSST_Image config = { 'input': { @@ -126,6 +117,7 @@ def test_sky_gradient(): image2 = galsim.config.BuildImage(config, logger=logger) assert image2 == image + if __name__ == "__main__": testfns = [v for k, v in vars().items() if k[:5] == 'test_' and callable(v)] for testfn in testfns: From 15b73d9c45ffa24582e20cb518f662e5ef1fddd5 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Sun, 24 Sep 2023 16:10:54 -0400 Subject: [PATCH 12/12] Fix a screen value that was copied from a comment. Update the screen size from 812.2 to 819.2. This was copied from a comment but was a typo there. From Josh: "Should be 2^13 * 0.1 = 819.2. 812.2 must be a typo." --- config/imsim-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/imsim-config.yaml b/config/imsim-config.yaml index cb5ec8a0..44d9b920 100644 --- a/config/imsim-config.yaml +++ b/config/imsim-config.yaml @@ -80,7 +80,7 @@ input: t0: 0 # seconds exptime: $exptime kcrit: 0.2 # in units of 1/r0 - screen_size: 812.2 # To use less memory for testing/development, you can try 409.6 + screen_size: 819.2 # To use less memory for testing/development, you can try 409.6 screen_scale: 0.1 # meters doOpt: False nproc: 1 # Default (None) means one proc per screen.