Merge remote-tracking branch 'upstream/master' into lsst_sims

.github/workflows/python-package.yml

@@ -7,9 +7,9 @@ name: build
 
 on:
   push:
-    branches: [ master, actions ]
   pull_request:
-    branches: [ master ]
+  schedule:
+  - cron: '0 0 1 * *'
 
 jobs:
   build:
@@ -32,25 +32,21 @@ jobs:
         conda info
     - name: Create conda environment
       run: |
-        conda create -q -n env python=${{ matrix.python-version }} numpy scipy matplotlib astropy healpy pyyaml emcee fitsio corner -c conda-forge -c kadrlica
+        conda create -y -q -n env python=${{ matrix.python-version }} numpy scipy matplotlib astropy healpy pyyaml emcee fitsio corner -c conda-forge -c kadrlica
+        # Add UGALIDIR to environment
+        conda env config vars set UGALIDIR=$HOME/.ugali -n env
     - name: Install package
       run: |
         source activate env
-        export UGALIDIR="$HOME/.ugali"
-        python setup.py -q install --isochrones --catalogs
-    - name: Install test data
-      run: |
-        wget https://github.com/DarkEnergySurvey/ugali/releases/download/v1.7.0/ugali-test-data.tar.gz -O ugali-test-data.tar.gz
-        tar -xzf ugali-test-data.tar.gz
+        conda env config vars list
+        python setup.py -q install --isochrones --catalogs --tests
     - name: Lint with flake8
-      if: ${{ false }}
+      if: ${{ true }}
       run: |
         source activate env
         conda install -q flake8 -c conda-forge
         # stop the build if there are Python syntax errors or undefined names
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Test with nose
       run: |
         source activate env

README.md

@@ -50,7 +50,7 @@ The `ugali` source code is distributed with several auxiliary libraries for isoc
 
 ```
 cd $UGALIDIR
-wget https://github.com/kadrlica/ugali/releases/download/v1.7.0rc0/ugali-des-bressan2012.tar.gz
+wget https://github.com/DarkEnergySurvey/ugali/releases/download/v1.8.0/ugali-des-bressan2012.tar.gz
 tar -xzf ugali-des-bressan2012.tar.gz
 ```
 

setup.py

@@ -24,25 +24,27 @@ def find_packages():
 HERE = os.path.abspath(os.path.dirname(__file__))
 URL = 'https://github.com/DarkEnergySurvey/ugali'
 DESC = "Ultra-faint galaxy likelihood toolkit."
-LONG_DESC = "See %s"%URL
+LONG_DESC = "%s\n%s"%(DESC,URL)
 CLASSIFIERS = """\
-Development Status :: 3 - Alpha
+Development Status :: 4 - Beta
 Intended Audience :: Science/Research
 Intended Audience :: Developers
 License :: OSI Approved :: MIT License
 Natural Language :: English
 Operating System :: MacOS :: MacOS X
 Operating System :: POSIX :: Linux
-Programming Language :: Python :: 2.7
+Programming Language :: Python :: 2
+Programming Language :: Python :: 3
 Topic :: Scientific/Engineering
 Topic :: Scientific/Engineering :: Astronomy
 Topic :: Scientific/Engineering :: Physics
 """
 
-RELEASE = URL+'/releases/download/v1.7.0'
+RELEASE_URL = URL+'/releases/download/v1.8.0'
 UGALIDIR = os.getenv("UGALIDIR","$HOME/.ugali")
 ISOSIZE = "~1MB" 
 CATSIZE = "~20MB"
+TSTSIZE = "~1MB"
 # Could find file size dynamically, but it's a bit slow...
 # int(urllib.urlopen(ISOCHRONES).info().getheaders("Content-Length")[0])/1024**2
 SURVEYS = ['des','ps1','sdss','lsst']
@@ -62,7 +64,7 @@ def read(self, size):
     def progress_bar(count, block_size, total_size):
         block = 100*block_size/float(total_size)
         progress = count*block
-        if progress % 1 < 1.01*block:
+        if progress % 5 < 1.01*block:
             msg = '\r[{:51}] ({:d}%)'.format(int(progress//2)*'='+'>',int(progress))
             sys.stdout.write(msg)
             sys.stdout.flush()
@@ -77,7 +79,7 @@ class TarballCommand(distutils.cmd.Command,object):
          'force installation (overwrite any existing files)')
         ]
     boolean_options = ['force']
-    release = RELEASE
+    release_url = RELEASE_URL
     _tarball = None
     _dirname = None
 
@@ -114,7 +116,7 @@ def install_tarball(self, tarball):
             os.makedirs(self.ugali_dir)
         os.chdir(self.ugali_dir)
 
-        url = os.path.join(self.release,tarball)
+        url = os.path.join(self.release_url,tarball)
 
         print("downloading %s..."%url)
         if urlopen(url).getcode() >= 400:
@@ -158,6 +160,12 @@ class CatalogCommand(TarballCommand):
     _tarball = 'ugali-catalogs.tar.gz'
     _dirname = 'catalogs'
 
+class TestsCommand(TarballCommand):
+    """ Command for downloading catalog files """
+    description = "install test data"
+    _tarball = 'ugali-test-data.tar.gz'
+    _dirname = 'testdata'
+
 class IsochroneCommand(TarballCommand):
     """ Command for downloading isochrone files """
     description = "install isochrone files"
@@ -223,6 +231,7 @@ class install(_install):
     user_options = _install.user_options + [
         ('isochrones',None,"install isochrone files (%s)"%ISOSIZE),
         ('catalogs',None,"install catalog files (%s)"%CATSIZE),
+        ('tests',None,"install test data (%s)"%TSTSIZE),
         ('ugali-dir=',None,"install file directory [default: %s]"%UGALIDIR),
     ]
     boolean_options = _install.boolean_options + ['isochrones','catalogs']
@@ -232,6 +241,7 @@ def initialize_options(self):
         self.ugali_dir = os.path.expandvars(UGALIDIR)
         self.isochrones = False
         self.catalogs = False
+        self.tests = False
 
     def run(self):
         # run superclass install
@@ -246,7 +256,10 @@ def run(self):
 
         if self.catalogs: 
             self.install_catalogs()
-
+
+        if self.tests:
+            self.install_tests()
+
     def install_isochrones(self):
         """
         Call to isochrone install command:
@@ -266,10 +279,21 @@ def install_catalogs(self):
         cmd_obj.force = self.force
         if self.ugali_dir: cmd_obj.ugali_dir = self.ugali_dir
         self.run_command('catalogs')
-
+
+    def install_tests(self):
+        """
+        Call to catalog install command:
+        http://stackoverflow.com/a/24353921/4075339
+        """
+        cmd_obj = self.distribution.get_command_obj('tests')
+        cmd_obj.force = self.force
+        if self.ugali_dir: cmd_obj.ugali_dir = self.ugali_dir
+        self.run_command('tests')
+
 CMDCLASS = versioneer.get_cmdclass()
 CMDCLASS['isochrones'] = IsochroneCommand
 CMDCLASS['catalogs'] = CatalogCommand
+CMDCLASS['tests'] = TestsCommand
 CMDCLASS['install'] = install
 
 setup(
@@ -278,17 +302,18 @@ def install_catalogs(self):
     cmdclass=CMDCLASS,
     url=URL,
     author='Keith Bechtol & Alex Drlica-Wagner',
-    author_email='bechtol@kicp.uchicago.edu, [email protected]',
+    author_email='bechtol@wisc.edu, [email protected]',
     scripts = [],
     install_requires=[
+        'astropy',
+        'matplotlib',
         'numpy >= 1.9.0',
         'scipy >= 0.14.0',
         'healpy >= 1.6.0',
         'fitsio >= 0.9.10',
         'emcee >= 2.1.0',
         'corner >= 1.0.0',
         'pyyaml >= 3.10',
-        # Add astropy, fitsio, matplotlib, ...
     ],
     packages=find_packages(),
     description=DESC,

tests/config.yaml

@@ -27,7 +27,7 @@ coords:
 
 catalog:
   # Color always defined as mag_1 - mag_2
-  dirname: ./healpix                    # directory of catalog files
+  dirname: ${UGALIDIR}/testdata/healpix          # directory of catalog files
   basename: "catalog_hpx%04i.fits"      # catalog file basename format
   objid_field       : COADD_OBJECT_ID   # unique object identifier
   lon_field         : RA                # name of longitude field
@@ -45,24 +45,19 @@ catalog:
   selection         : "(np.abs(self.data['WAVG_SPREAD_MODEL_I']) < 0.003)"
 
 data: 
-  dirname: ./raw
+  dirname: ${UGALIDIR}/raw
   script : ./ugali/preprocess/database.py
   survey : des
   release: y3a2
   density: ./density/density_hpx%04i.fits
   footprint: ./maps/footprint_nside4096_equ.fits.gz
 
 mask:
-  dirname    : ./mask
+  dirname    : ${UGALIDIR}/testdata/mask
   basename_1 : "maglim_g_hpx%04i.fits"
   basename_2 : "maglim_r_hpx%04i.fits"
   minimum_solid_angle: 0.1 # deg^2
 
-mangle:
-  dirname    : ./maps
-  filename_1 : 'y3a2_g_o.4096_t.32768_maglim_EQU.fits.gz'
-  filename_2 : 'y3a2_r_o.4096_t.32768_maglim_EQU.fits.gz'
-
 #ADW: Depricated in favor of 'binning'
 color:
   min   : &cmin -0.5

tests/test_config.py

@@ -15,12 +15,12 @@ def test_config():
     np.testing.assert_equal(len(config.filenames),768)
 
     np.testing.assert_equal(config.filenames['pix'].compressed()[0],687)
-    np.testing.assert_equal(config.filenames['catalog'].compressed()[0],
-                            './healpix/catalog_hpx0687.fits')
-    np.testing.assert_equal(config.filenames['mask_1'].compressed()[0],
-                            './mask/maglim_g_hpx0687.fits')
-    np.testing.assert_equal(config.filenames['mask_2'].compressed()[0],
-                            './mask/maglim_r_hpx0687.fits')
+    catfile = os.path.basename(config.filenames['catalog'].compressed()[0])
+    np.testing.assert_equal(catfile,'catalog_hpx0687.fits')
+    maskfile = os.path.basename(config.filenames['mask_1'].compressed()[0])
+    np.testing.assert_equal(maskfile,'maglim_g_hpx0687.fits')
+    maskfile = os.path.basename(config.filenames['mask_2'].compressed()[0])
+    np.testing.assert_equal(maskfile,'maglim_r_hpx0687.fits')
 
     np.testing.assert_equal(config.likefile,'./scan/scan_%08i_%s.fits')
     np.testing.assert_equal(config.mergefile,'./scan/merged_scan.fits')

tests/test_observation.py

@@ -44,8 +44,13 @@ def test_catalog():
     """ Test ugali.observation.catalog """
     import ugali.observation.roi
     import ugali.observation.catalog
+    import ugali.utils.config
 
-    filename='healpix/catalog_hpx0687.fits'
+    # Get catalog filename
+    config = ugali.utils.config.Config(CONFIG)
+    filename = config.filenames['catalog'].compressed()[0]
+
+    # Create catalog from filename
     catalog = ugali.observation.catalog.Catalog(CONFIG,filenames=filename)
 
     roi = ugali.observation.roi.ROI(CONFIG, LON, LAT)

ugali/analysis/prior.py

@@ -1,4 +1,9 @@
 #!/usr/bin/env python
+"""
+Incomplete module for dealing with priors...
+"""
+
+import scipy.stats
 
 class Prior(object):
     def __call__(self, value):

ugali/analysis/results.py

@@ -18,7 +18,8 @@
 
 import ugali.utils.stats
 from ugali.utils.stats import Samples
-from ugali.utils.projector import dist2mod,mod2dist,gal2cel,gal2cel_angle
+from ugali.utils.projector import dist2mod,mod2dist
+from ugali.utils.projector import cel2gal,gal2cel,gal2cel_angle
 from ugali.utils.projector import ang2const, ang2iau
 from ugali.utils.config import Config
 from ugali.utils.logger import logger

ugali/analysis/scan.py

@@ -130,10 +130,12 @@ def search(self, coords=None, distance_modulus=None, extension=None, tolerance=1
         self.stellar_mass_array         = np.zeros([nmoduli,npixels],dtype='f4')
         self.fraction_observable_array  = np.zeros([nmoduli,npixels],dtype='f4')
         self.extension_fit_array        = np.zeros([nmoduli,npixels],dtype='f4')
-        # DEPRECATED: ADW 2019-04-27
-        self.richness_lower_array       = np.zeros([nmoduli,npixels],dtype='f4')
-        self.richness_upper_array       = np.zeros([nmoduli,npixels],dtype='f4')
-        self.richness_ulimit_array      = np.zeros([nmoduli,npixels],dtype='f4')
+        if self.config['scan']['full_pdf']:
+            # DEPRECATED: ADW 2019-04-27
+            DeprecationWarning("'full_pdf' is deprecated.")
+            self.richness_lower_array       = np.zeros([nmoduli,npixels],dtype='f4')
+            self.richness_upper_array       = np.zeros([nmoduli,npixels],dtype='f4')
+            self.richness_ulimit_array      = np.zeros([nmoduli,npixels],dtype='f4')
 
         # Specific pixel/distance_modulus
         coord_idx, distance_modulus_idx, extension_idx = None, None, None
@@ -315,6 +317,7 @@ def write(self, outfile):
         data['PIXEL']=self.roi.pixels_target
         # Full data output (too large for survey)
         if self.config['scan']['full_pdf']:
+            DeprecationWarning("'full_pdf' is deprecated.")
             data['LOG_LIKELIHOOD']=self.loglike_array.T
             data['RICHNESS']=self.richness_array.T
             data['RICHNESS_LOWER']=self.richness_lower_array.T

ugali/candidate/associate.py

@@ -54,7 +54,7 @@
 #            glon, glat = ugali.utils.projector.celToGal(lon,lat)
 #        else:
 #            glon,glat = lon, lat
-#        return ugali.utils.projector.match(glon,glat,self.data['glon'],self.data['glat'],tol)
+#        return ugali.utils.projector.match(glon,glat,self['glon'],self['glat'],tol)
 
 
 
@@ -452,7 +452,7 @@ def catalogFactory(name, **kwargs):
     catalogs = odict(inspect.getmembers(sys.modules[__name__], fn))
 
     if name not in list(catalogs.keys()):
-        msg = "%s not found in catalogs:\n %s"%(name,list(kernels.keys()))
+        msg = "%s not found in catalogs:\n %s"%(name,list(catalogs.keys()))
         logger.error(msg)
         msg = "Unrecognized catalog: %s"%name
         raise Exception(msg)

ugali/isochrone/dartmouth.py

@@ -165,7 +165,7 @@ def _parse(self,filename):
         try:
             columns = self.columns[self.survey.lower()]
         except KeyError as e:
-            logger.warning('Unrecognized survey: %s'%(survey))
+            logger.warning('Unrecognized survey: %s'%(self.survey))
             raise(e)
 
         kwargs = dict(comments='#',usecols=list(columns.keys()),dtype=list(columns.values()))

ugali/isochrone/empirical.py

@@ -3,7 +3,10 @@
 Generic python script.
 """
 import os
+from collections import OrderedDict as odict
 
+from ugali.analysis.model import Model, Parameter
+from ugali.utils.shell import mkdir, get_ugali_dir, get_iso_dir
 from ugali.isochrone.parsec import PadovaIsochrone
 
 class EmpiricalPadova(PadovaIsochrone):

ugali/isochrone/mesa.py

@@ -79,7 +79,7 @@ class Dotter2016(Isochrone):
             des = odict([
                 (2, ('mass_init',float)),
                 (3, ('mass_act',float)),
-                (8, ('log_lum',float)),
+                (6, ('log_lum',float)),
                 (9, ('u',float)),
                 (10,('g',float)),
                 (11,('r',float)),
@@ -134,7 +134,7 @@ def _parse(self,filename):
         try:
             columns = self.columns[self.survey.lower()]
         except KeyError as e:
-            logger.warning('Unrecognized survey: %s'%(survey))
+            logger.warning('Unrecognized survey: %s'%(self.survey))
             raise(e)
 
         kwargs = dict(comments='#',usecols=list(columns.keys()),dtype=list(columns.values()))