support zonal statistics and fs-based raster db

bin/compile-requirements.bash

@@ -40,7 +40,7 @@ main () {
 
     cd "${THIS_DIR}/.."
 
-    local compile="pip-compile --strip-extras --generate-hashes --allow-unsafe"
+    local compile="pip-compile --strip-extras --allow-unsafe"
 
     $compile pyproject.toml "$@"
 

bin/snodas_batch_import.bash → bin/snodas-batch-import.bash

@@ -29,7 +29,7 @@ dates=$(ls *.grz 2>/dev/null | grep -oP '(?<!\d)\d{8}' | sort -u)
 for date in ${dates[@]}; do
     (
         tmp=$(mktemp -d)
-	trap "rm -r $tmp" EXIT
+        trap "rm -r $tmp" EXIT
 
         year=${date:0:4}
         month=${date:4:2}
@@ -51,8 +51,8 @@ for date in ${dates[@]}; do
         file=SNODAS_${year}${month}${day}.tar
         tar -cf $file -C $tmp .
 
-	# and we can remove the original grz files
-	rm *$date*.grz
+        # and we can remove the original grz files
+        rm *$date*.grz
     )
 done
 

bin/snodas-download-import.bash → bin/snodas-download.bash

@@ -7,7 +7,6 @@
 # Simple.
 #
 # Date format must be YYYY-MM-DD.
-# Make sure snodas is on your path (`source activate` the env).
 
 
 # Need to build a full url like
@@ -39,9 +38,4 @@ esac
 filename="SNODAS_${year}${month}${day}.tar"
 url="${url}/${year}/${mo_name}/${filename}"
 
-dir=$(mktemp -d)
-trap "rm -r ${dir}" EXIT
-
-filepath="${dir}/${filename}"
-curl "${url}" -o "${filepath}"
-snodas loadraster "${filepath}"
+curl "${url}"

conda-requirements.txt

@@ -1 +1 @@
-libgdal=3.0.2
+conda-forge::gdal=3.8.5

pyproject.toml

@@ -12,14 +12,18 @@ authors = [
 description = 'Interface to SNODAS daily and aggregate rasters as a tile service and for analytical functions.'
 requires-python = '>=3.12'
 dependencies = [
+    'area',
     'Django',
     'django-ninja',
     'django-split-settings',
     'django-cors-headers',
+    'gdal',
+    'numpy',
     'psycopg2-binary',
     'pydantic',
     'pyyaml',
     'sqlparse',
+    'waitress',
 ]
 dynamic = [
   'readme',
@@ -35,6 +39,12 @@ Repository = 'https://github.com/PSU-CSAR/django-snodas'
 [tool.setuptools]
 packages = ['snodas']
 py-modules = ['manage']
+script-files = [
+    'bin/snodas-dump-pourpoints.bash',
+    'bin/snodas-batch-import.bash',
+    'bin/snodas-download.bash',
+    'bin/snodas-streamflow-import.bash',
+]
 
 [tool.setuptools.dynamic]
 readme = {file = 'README.md'}

requirements-dev.in

@@ -1,3 +1,4 @@
+-c requirements.txt
 mypy
 pip-tools
 pre-commit

snodas/apps.py

@@ -1,3 +1,5 @@
+import os
+
 from django.apps import AppConfig
 from django.utils.translation import gettext_lazy as _
 
@@ -7,4 +9,8 @@ class SnodasConfig(AppConfig):
     verbose_name = _('SNODAS')
 
     def ready(self):
-        pass
+        # This ensures we can fully leverage supported parallelism in gdal
+        # when running gdal functions by default. It also allows the user
+        # to override the amount of parallelism by setting the var themselves.
+        if 'GDAL_NUM_THREADS' not in os.environ:
+            os.environ['GDAL_NUM_THREADS'] = 'ALL_CPUS'

snodas/management/commands/createrasterdb.py

@@ -0,0 +1,42 @@
+from argparse import ArgumentParser
+from pathlib import Path
+from typing import Self
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from snodas.management import utils
+from snodas.snodas.db import RasterDatabase
+
+
+class Command(BaseCommand):
+    help = (
+        'Create the raster database for SNODAS COGs, AOI rasters, '
+        'and other required datasets.'
+    )
+
+    requires_system_checks = []  # type: ignore  # noqa: RUF012
+    can_import_settings = True
+
+    def add_arguments(self: Self, parser: ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument(
+            '--dem',
+            required=True,
+            type=utils.path_exists,
+            help='Path to reference DEM layer',
+        )
+        parser.add_argument(
+            '-F',
+            '--force',
+            action='store_true',
+            default=False,
+            help='Attempt database creation even if already exists',
+        )
+
+    def handle(self: Self, dem: Path, *_, force: bool = False, **__) -> None:
+        RasterDatabase.create(
+            path=settings.SNODAS_RASTERDB,
+            input_dem_path=dem,
+            force=force,
+        )

snodas/management/commands/install.py

@@ -59,6 +59,7 @@ class Settings:
     SITE_DOMAIN_NAME: str
     ADDITIONAL_SETTINGS_FILES: list[str]
     CONDA_ENV_NAME: str
+    SNODAS_RASTERDB: str
     DATABASE_HOST: str | None = None
     DATABASE_PORT: int | None = None
 
@@ -78,6 +79,7 @@ def from_options(
         *,
         deployment_type: str,
         project_name: str,
+        raster_db: Path,
         db_name: str | None = None,
         db_user: str | None = None,
         db_password: str | None = None,
@@ -108,6 +110,7 @@ def from_options(
                 additional_settings_file if additional_settings_file else []
             ),
             CONDA_ENV_NAME=(env_name if env_name else project_name),
+            SNODAS_RASTERDB=str(raster_db),
         )
 
 
@@ -214,6 +217,16 @@ def add_arguments(cls, parser: argparse.ArgumentParser) -> None:
                 'Default is None, which will use postgres default.'
             ),
         )
+        parser.add_argument(
+            '--raster-db',
+            required=True,
+            type=Path,
+            help=(
+                'Path of the directory containing (or to contain) '
+                'the raster database. New raster databases can be '
+                'initialized with the `createrasterdb` command.'
+            ),
+        )
         parser.add_argument(
             '-t',
             '--deployment-type',
@@ -518,6 +531,7 @@ def execute(self, *_, **options) -> int:
             '\nNext, activate the new conda env for the project:\n\n'
             f'`conda activate {self.settings.PROJECT_NAME}`\n\n'
             'Then, setup any required services for this instance:\n\n'
+            '`snodas createrasterdb INPUT_DEM  # creates raster db`\n'
             '`snodas createdb [options]  # creates a postgres DB`\n'
             '`snodas setupiis [options]  # creates a site in IIS`\n\n'
             'Once the services are configured, '