add delve as possible phot cat

SAGA/database/external.py

@@ -406,42 +406,17 @@ def run_casjobs_with_sciserver(
                 shutil.copyfileobj(r, f_out)
 
 
-class DesQuery(DownloadableBase):
+class DatalabQuery(DownloadableBase):
 
-    _query_template = """select
-        d.COADD_OBJECT_ID as OBJID,
-        d.ALPHAWIN_J2000 as RA,
-        d.DELTAWIN_J2000 as DEC,
-        d.FLUX_RADIUS_G * 0.263 as radius_g,
-        d.FLUX_RADIUS_R * 0.263 as radius_r,
-        d.FLUX_RADIUS_I * 0.263 as radius_i,
-        d.MAG_AUTO_G_DERED as g_mag,
-        d.MAG_AUTO_R_DERED as r_mag,
-        d.MAG_AUTO_I_DERED as i_mag,
-        d.MAG_AUTO_Z_DERED as z_mag,
-        d.MAG_AUTO_Y_DERED as y_mag,
-        d.MAGERR_AUTO_G as g_err,
-        d.MAGERR_AUTO_R as r_err,
-        d.MAGERR_AUTO_I as i_err,
-        d.MAGERR_AUTO_Z as z_err,
-        d.MAGERR_AUTO_Y as y_err,
-        d.A_IMAGE,
-        d.B_IMAGE,
-        d.THETA_J2000,
-        d.flags_r,
-        d.imaflags_iso_r,
-        (CASE WHEN d.wavg_spread_model_r=-99 THEN d.spread_model_r ELSE d.wavg_spread_model_r END) as spread_model_r,
-        (CASE WHEN d.wavg_spreaderr_model_r=-99 THEN d.spreaderr_model_r ELSE d.wavg_spreaderr_model_r END) as spreaderr_model_r
-        from {table_name} d where
-        q3c_radial_query(d.ra, d.dec, {ra:.7g}, {dec:.7g}, {r_deg:.7g})"""
-
-    _default_table_name = "des_dr1.main"
+    _query_template = None  # to be implemented by subclass
 
-    def __init__(self, ra, dec, radius=1.0, table_name=None):
-        self.query = self.construct_query(ra, dec, radius, table_name)
+    def __init__(self, *args, **kwargs):
+        self.query = None
+        if self._query_template:
+            self.set_query(self.construct_query(*args, **kwargs))
 
     @classmethod
-    def construct_query(cls, ra, dec, radius=1.0, table_name=None):
+    def construct_query(cls, ra, dec, radius=1.0, **kwargs):
         """
         Generates the query to send to the DES to get the full DES catalog around
         a target.
@@ -458,22 +433,26 @@ def construct_query(cls, ra, dec, radius=1.0, table_name=None):
         Returns
         -------
         query : str
-            The SQL query to send to the SDSS skyserver
+            The SQL query to send to the Datalab skyserver
         """
         # pylint: disable=possibly-unused-variable
         ra = ensure_deg(ra)
         dec = ensure_deg(dec)
         r_deg = ensure_deg(radius)
-        table_name = str(table_name or cls._default_table_name)
 
         # ``**locals()`` means "use the local variable names to fill the template"
         q = cls._query_template.format(**locals())
         q = re.sub(r"\s+", " ", q).strip()
         q = re.sub(", ", ",", q)
         return q
 
-    @staticmethod
-    def get_colnames_from_query(query):
+    def set_query(self, query):
+        self.query = str(query)
+
+    def get_colnames_from_query(self, query=None):
+        if query is None:
+            query = self.query
+
         cols = []
         for item in query.lower().partition(" from ")[0].split(","):
             _, has_as, name = item.rpartition(" as ")
@@ -484,17 +463,20 @@ def get_colnames_from_query(query):
             cols.append(name.strip())
         return cols
 
-    def run_query(self):
+    def run_query(self, query=None):
+        if query is None:
+            query = self.query
+
         if _HAS_DATALAB_:
-            raw = dl.queryClient.query(sql=self.query)
+            raw = dl.queryClient.query(sql=query)
             return Table.read(raw, format="ascii.fast_csv")
 
-        cols = self.get_colnames_from_query(self.query)
+        cols = self.get_colnames_from_query(query)
 
         # taken from https://github.com/noaodatalab/datalab/blob/master/dl/queryClient.py#L1791
         r = requests.get(
             "https://datalab.noao.edu/query/query",
-            {"sql": self.query, "ofmt": "ascii", "async": False},
+            {"sql": query, "ofmt": "ascii", "async": False},
             headers={
                 "Content-Type": "application/octet-stream",
                 "X-DL-AuthToken": "anonymous.0.0.anon_access",
@@ -519,6 +501,59 @@ def download_as_file(self, file_path, overwrite=False, compress=True):
             t.write(f, format="fits")
 
 
+class DesQuery(DatalabQuery):
+    _query_template = """select
+        d.COADD_OBJECT_ID as OBJID,
+        d.ALPHAWIN_J2000 as RA,
+        d.DELTAWIN_J2000 as DEC,
+        d.FLUX_RADIUS_G * 0.263 as radius_g,
+        d.FLUX_RADIUS_R * 0.263 as radius_r,
+        d.FLUX_RADIUS_I * 0.263 as radius_i,
+        d.MAG_AUTO_G_DERED as g_mag,
+        d.MAG_AUTO_R_DERED as r_mag,
+        d.MAG_AUTO_I_DERED as i_mag,
+        d.MAG_AUTO_Z_DERED as z_mag,
+        d.MAG_AUTO_Y_DERED as y_mag,
+        d.MAGERR_AUTO_G as g_err,
+        d.MAGERR_AUTO_R as r_err,
+        d.MAGERR_AUTO_I as i_err,
+        d.MAGERR_AUTO_Z as z_err,
+        d.MAGERR_AUTO_Y as y_err,
+        d.A_IMAGE,
+        d.B_IMAGE,
+        d.THETA_J2000,
+        d.flags_r,
+        d.imaflags_iso_r,
+        (CASE WHEN d.wavg_spread_model_r=-99 THEN d.spread_model_r ELSE d.wavg_spread_model_r END) as spread_model_r,
+        (CASE WHEN d.wavg_spreaderr_model_r=-99 THEN d.spreaderr_model_r ELSE d.wavg_spreaderr_model_r END) as spreaderr_model_r
+        from des_dr1.main d where
+        q3c_radial_query(d.ra, d.dec, {ra:.7g}, {dec:.7g}, {r_deg:.7g})"""
+
+
+class DelveQuery(DatalabQuery):
+    _query_template = """select
+        d.quick_object_id as OBJID,
+        d.ra as RA,
+        d.dec as DEC,
+        (d.mag_auto_g - d.extinction_g) as g_mag,
+        (d.mag_auto_r - d.extinction_r) as r_mag,
+        (d.mag_auto_i - d.extinction_i) as i_mag,
+        (d.mag_auto_z - d.extinction_z) as z_mag,
+        d.magerr_auto_g as g_err,
+        d.magerr_auto_r as r_err,
+        d.magerr_auto_i as i_err,
+        d.magerr_auto_z as z_err,
+        d.a_image_r as radius,
+        (d.b_image_r / d.a_image_r) as ba,
+        d.theta_image_r as phi,
+        d.extended_class_r as morphology_info,
+        (CASE WHEN d.extended_class_r=3 THEN 1 ELSE 0 END) as is_galaxy,
+        (d.flags_g * 100 + d.flags_r) as REMOVE
+        from delve_dr1.objects d where
+        q3c_radial_query(d.ra, d.dec, {ra:.7g}, {dec:.7g}, {r_deg:.7g})
+        and d.mag_auto_r < 23.5"""
+
+
 class DecalsPrebuilt(DownloadableBase):
 
     requires_host_id = True

SAGA/database/saga_database.py

@@ -424,6 +424,7 @@ def __init__(self, shared_dir=None, local_dir=None):
             "sdss_dr12": os.path.join(self._local_dir, "external_catalogs", "sdss_dr12", "{}.fits.gz"),
             "wise": os.path.join(self._local_dir, "external_catalogs", "wise", "{}.fits.gz"),
             "des_dr1": os.path.join(self._local_dir, "external_catalogs", "des_dr1", "{}_des_dr1.fits.gz"),
+            "delve_dr1": os.path.join(self._local_dir, "external_catalogs", "delve_dr1", "{}_delve_dr1.fits.gz"),
             "decals_dr67": os.path.join(self._local_dir, "external_catalogs", "decals", "{}_decals.fits.gz"),
             "decals_dr8": os.path.join(
                 self._local_dir,
@@ -591,5 +592,6 @@ def set_default_base_version(self, version=None):
             self._file_path_pattern["sdss"] = self._file_path_pattern["sdss_dr16"]
             self._file_path_pattern["des"] = self._file_path_pattern["des_dr1"]
             self._file_path_pattern["decals"] = self._file_path_pattern["decals_dr9"]
+            self._file_path_pattern["delve"] = self._file_path_pattern["delve_dr1"]
 
         self.base_file_path_pattern = self._file_path_pattern["base_" + version_postfix]

SAGA/objects/build3.py

@@ -186,6 +186,36 @@ def prepare_decals_catalog_for_merging(catalog, to_remove=None, to_recover=None,
     return catalog
 
 
+def prepare_delve_catalog_for_merging(catalog, to_remove=None, to_recover=None, trim=True):
+    cols = ["objid", "ra", "dec", "remove"]
+    catalog.rename_columns(cols, [c.upper() for c in cols])
+    for b in "griz":
+        catalog[f"{b}_mag"] = catalog[f"{b}_mag"].astype(np.float32)
+        catalog[f"{b}_err"] = catalog[f"{b}_err"].astype(np.float32)
+    catalog["u_mag"] = np.float32(99.0)
+    catalog["u_err"] = np.float32(99.0)
+    catalog["morphology_info"] = catalog["morphology_info"].astype(np.int32)
+    catalog["is_galaxy"] = catalog["is_galaxy"].astype(bool)
+    catalog["radius_err"] = catalog["radius"] * 1.0e-4
+    catalog["REF_CAT"] = '  '
+    catalog["SGA_ID"] = -1
+
+    if trim:
+        catalog = catalog[MERGED_CATALOG_COLUMNS]
+    catalog["survey"] = "delve"
+    catalog["OBJID_delve"] = catalog["OBJID"]
+    catalog["REMOVE_delve"] = catalog["REMOVE"]
+
+    if to_remove is not None:
+        catalog["REMOVE"] |= np.isin(catalog["OBJID"], to_remove).astype(np.int)
+
+    if to_recover is not None:
+        idx = np.flatnonzero(np.isin(catalog["OBJID"], to_recover))
+        catalog["REMOVE"][idx] = 0
+
+    return catalog
+
+
 SPEC_MATCHING_ORDER = (
     (Query("SPEC_Z < 0.002", "REMOVE == 0", "is_galaxy == 0", "sep < 5"), "sep"),
     (Query("SPEC_Z < 0.002", "REMOVE % 2 == 0", "is_galaxy == 0", "sep < 5"), "sep"),
@@ -374,6 +404,9 @@ def build_full_stack(  # pylint: disable=unused-argument
     halpha=None,
     shreds_recover=None,
     debug=None,
+    delve=None,
+    delve_remove=None,
+    delve_recover=None,
     **kwargs,
 ):
     """
@@ -384,10 +417,14 @@ def build_full_stack(  # pylint: disable=unused-argument
     -------
     base : astropy.table.Table
     """
-    if decals is None:
+    if decals is not None:
+        base = prepare_decals_catalog_for_merging(decals, decals_remove, decals_recover)
+        del decals, decals_remove, decals_recover
+    elif delve is not None:
+        base = prepare_delve_catalog_for_merging(delve, delve_remove, delve_recover)
+        del delve, delve_remove, delve_recover
+    else:
         raise ValueError("No photometry catalog to build!")
-    base = prepare_decals_catalog_for_merging(decals, decals_remove, decals_recover)
-    del decals, decals_remove, decals_recover
 
     base = build.add_host_info(base, host)
     base = build2.add_columns_for_spectra(base)

SAGA/objects/object_catalog.py

@@ -151,7 +151,10 @@ def _determine_raw_catalogs_lowz_v2(host_id, **kwargs):
     return ("des",)
 
 
-def _determine_raw_catalogs_lowz_v3(**kwargs):
+def _determine_raw_catalogs_lowz_v3(host_id, **kwargs):
+    field_name = str(host_id)
+    if any(field_name.startswith(s) for s in ["Pal15"]):
+        return ("delve_dr1",)
     return ("decals_dr9",)
 
 

SAGA/version.py

@@ -1,4 +1,4 @@
 """
 SAGA package version
 """
-__version__ = "0.40.4"
+__version__ = "0.40.5"