Switch to using pytest-regressions for test_fluxes (#36)

* Remove restriction on tables version by switching runner to macos-13
* Add gh action to add a test report to PR
* fix boolean trap in corsika_weighter
* remove asfarray, switch to asarray(dtype=np.float64)
* switch to pytest.parallelization from using a bunch of unittest functions in dataset test
* sum all of the species for global spline fit test
* move tox ini to pyproject

.github/workflows/tests.yml

@@ -20,10 +20,8 @@ jobs:
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         os: [ubuntu-22.04]
         include:
-          - python-version: "3.11"
-            os: macos-12
           - python-version: "3.12"
-            os: macos-12
+            os: macos-13
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
@@ -41,7 +39,14 @@ jobs:
       - name: Run Tests
         env:
           SIMWEIGHTS_TESTDATA: .
-        run: python3 -m pytest  --cov-report=xml
+        run: python3 -m pytest  --cov-report=xml --junit-xml=test-results-${{matrix.os}}-${{matrix.python-version}}.junit.xml
+      - name: Upload Test Results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          if-no-files-found: error
+          name: test-results-${{matrix.os}}-${{matrix.python-version}}
+          path: test-results-${{matrix.os}}-${{matrix.python-version}}.junit.xml
       - name: Upload Coverage to Codecov
         if: ${{ !github.event.act }}
         uses: codecov/codecov-action@v4
@@ -50,3 +55,24 @@ jobs:
         with:
           fail_ci_if_error: false
           verbose: true
+  publish-test-results:
+    name: "Publish Tests Results"
+    needs: Tests
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: read
+    if: always()
+    steps:
+      - name: Download Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: .
+          pattern: test-results-*
+          merge-multiple: true
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          files: "*.xml"
+          deduplicate_classes_by_file_name: true

.pre-commit-config.yaml

@@ -44,7 +44,7 @@ repos:
         exclude: ^contrib/
         additional_dependencies: [numpy, pandas]
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.14
+    rev: v0.2.1
     hooks:
       - id: ruff
         args: [--fix, --show-fixes]

.reuse/dep5

@@ -3,10 +3,10 @@ Upstream-Name: SimWeights
 Upstream-Contact: the SimWeights contributors
 Source: https://github.com/icecube/simweights
 
-Files: docs/*.svg
+Files: docs/*.svg tests/flux_values.json
 Copyright: 2022 the SimWeights contributors
 License: BSD-2-Clause
 
-Files: tests/flux_values.json
-Copyright: 2022 the SimWeights contributors
+Files: tests/test_fluxes/*.npz
+Copyright: 2024 the SimWeights contributors
 License: BSD-2-Clause

pyproject.toml

@@ -34,7 +34,15 @@ requires-python = "~=3.7"
 [project.optional-dependencies]
 dev = ["pytest", "pre-commit", "reuse", "black", "ruff", "pylint", "mypy"]
 docs = ["sphinx", "sphinx-rtd-theme", "pandas"]
-test = ["h5py", "tables < 3.8; python_version < '3.9'", "tables <= 3.9.2", "pandas", "uproot", "pytest-cov"]
+test = [
+  "h5py",
+  "tables < 3.8; python_version < '3.9'",
+  "tables",
+  "pandas",
+  "uproot",
+  "pytest-cov",
+  'pytest-regressions'
+]
 
 [project.scripts]
 simweights = "simweights.cmdline:main"
@@ -89,7 +97,6 @@ target-version = "py38"
 fixable = ["I"]
 ignore = [
   "ANN401",  # any-type
-  "FBT",  # flake8-boolean-trap
   "S101",  # assert-used
   "COM812",  # confilcts with ruff formatter
   "ISC001",  # confilcts with ruff formatter
@@ -118,3 +125,15 @@ select = ["ALL"]
 
 [tool.ruff.lint.pydocstyle]
 convention = "google"
+
+[tool.tox]
+legacy_tox_ini = """
+[tox]
+envlist = py3{8,9,10,11,12}
+isolated_build = True
+
+[testenv]
+passenv = SIMWEIGHTS_TESTDATA, HDF5_DIR
+deps = .[test]
+commands = pytest
+"""

src/simweights/__init__.py

@@ -28,6 +28,7 @@
     "GaisserHillas",
     "GlobalFitGST",
     "GlobalFitGST_IT",
+    "GlobalSplineFit",
     "GlobalSplineFit5Comp",
     "GlobalSplineFit_IT",
     "Hoerandel",
@@ -54,6 +55,7 @@
     GaisserHillas,
     GlobalFitGST,
     GlobalFitGST_IT,
+    GlobalSplineFit,
     GlobalSplineFit5Comp,
     GlobalSplineFit_IT,
     Hoerandel,

src/simweights/_corsika_weighter.py

@@ -13,11 +13,11 @@
 from ._generation_surface import GenerationSurface, generation_surface
 from ._powerlaw import PowerLaw
 from ._spatial import NaturalRateCylinder
-from ._utils import Column, constcol, get_column, get_table, has_table
+from ._utils import Column, constcol, get_column, get_table, has_column, has_table
 from ._weighter import Weighter
 
 
-def sframe_corsika_surface(table: Any, oversampling: bool) -> GenerationSurface:
+def sframe_corsika_surface(table: Any) -> GenerationSurface:
     """Inspect the rows of a CORSIKA S-Frame table object to generate a surface object.
 
     This function works on files generated with either triggered CORSIKA or corsika-reader because
@@ -40,7 +40,7 @@ def sframe_corsika_surface(table: Any, oversampling: bool) -> GenerationSurface:
             get_column(table, "max_energy")[i],
             "energy",
         )
-        oversampling_val = get_column(table, "oversampling")[i] if oversampling else 1
+        oversampling_val = get_column(table, "oversampling")[i] if has_column(table, "oversampling") else 1
         pdgid = int(get_column(table, "primary_type")[i])
         surfaces.append(
             get_column(table, "n_events")[i]
@@ -108,7 +108,7 @@ def CorsikaWeighter(file_obj: Any, nfiles: float | None = None) -> Weighter:  #
             )
             raise RuntimeError(mesg)
 
-        surface = sframe_corsika_surface(get_table(file_obj, info_obj), oversampling=False)
+        surface = sframe_corsika_surface(get_table(file_obj, info_obj))
         triggered = True
 
     elif nfiles is None:
@@ -119,7 +119,7 @@ def CorsikaWeighter(file_obj: Any, nfiles: float | None = None) -> Weighter:  #
                 "nfiles and no I3CorsikaInfo table was found."
             )
             raise RuntimeError(msg)
-        surface = sframe_corsika_surface(get_table(file_obj, info_obj), oversampling=True)
+        surface = sframe_corsika_surface(get_table(file_obj, info_obj))
         triggered = False
 
     else:

src/simweights/_fluxes.py

@@ -21,7 +21,7 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Callable, Mapping, Sequence
 
-from numpy import asfarray, bool_, broadcast_arrays, exp, float64, genfromtxt, int32, piecewise, sqrt
+from numpy import asarray, bool_, broadcast_arrays, exp, float64, genfromtxt, int32, piecewise, sqrt
 from numpy import sum as nsum
 from scipy.interpolate import CubicSpline  # pylint: disable=import-error
 
@@ -398,7 +398,7 @@ def __init__(
         self: FixedFractionFlux,
         fractions: Mapping[PDGCode, float],
         basis: CosmicRayFlux | None = None,
-        normalized: bool = True,
+        normalized: bool = True,  # noqa: FBT001, FBT002
     ) -> None:
         """Flux that is a fixed fraction of another flux.
 
@@ -422,7 +422,7 @@ def __call__(self: FixedFractionFlux, energy: ArrayLike, pdgid: ArrayLike) -> ND
         energy_arr, pdgid_arr = broadcast_arrays(energy, pdgid)
         fluxsum = sum(self.flux(energy_arr, p) for p in self.pdgids)
         cond = self._condition(energy_arr, pdgid_arr)
-        return asfarray(fluxsum * piecewise(energy, cond, self.fracs))
+        return asarray(fluxsum * piecewise(energy, cond, self.fracs), dtype=float64)
 
 
 class _references:

src/simweights/_generation_surface.py

@@ -101,13 +101,13 @@ def get_epdf(
         cols = {}
         shape = None
         for key, value in kwargs.items():
-            cols[key] = np.asfarray(value)
+            cols[key] = np.asarray(value, dtype=np.float64)
             if shape is None:
                 shape = cols[key].shape
             else:
                 assert shape == cols[key].shape  # type: ignore[unreachable]
         assert shape is not None
-        count = np.zeros(shape, dtype=np.float_)
+        count = np.zeros(shape, dtype=np.float64)
         # loop over particle type
         for ptype in np.unique(pdgid):
             mask = ptype == pdgid

src/simweights/_powerlaw.py

@@ -52,17 +52,17 @@ def __init__(self: PowerLaw, g: float, a: float, b: float, colname: str | None =
         self.columns = (colname,)
 
     def _pdf(self: PowerLaw, x: NDArray[np.float64]) -> NDArray[np.float64]:
-        return np.asfarray(x**self.g / self.integral)
+        return np.asarray(x**self.g / self.integral, dtype=np.float64)
 
     def _cdf(self: PowerLaw, x: NDArray[np.float64]) -> NDArray[np.float64]:
         if self.G == 0:
-            return np.asfarray(np.log(x / self.a) / self.integral)
-        return np.asfarray((x**self.G - self.a**self.G) / self.G / self.integral)
+            return np.asarray(np.log(x / self.a) / self.integral, dtype=np.float64)
+        return np.asarray((x**self.G - self.a**self.G) / self.G / self.integral, dtype=np.float64)
 
     def _ppf(self: PowerLaw, q: NDArray[np.float64]) -> NDArray[np.float64]:
         if self.G == 0:
-            return np.asfarray(self.a * np.exp(q * self.integral))
-        return np.asfarray((q * self.G * self.integral + self.a**self.G) ** (1 / self.G))
+            return np.asarray(self.a * np.exp(q * self.integral), dtype=np.float64)
+        return np.asarray((q * self.G * self.integral + self.a**self.G) ** (1 / self.G), np.float64)
 
     def pdf(self: PowerLaw, x: ArrayLike) -> NDArray[np.float64]:
         r"""Probability density function.
@@ -74,7 +74,7 @@ def pdf(self: PowerLaw, x: ArrayLike) -> NDArray[np.float64]:
         Returns:
             array_like: Probability density function evaluated at `x`
         """
-        xa = np.asfarray(x)
+        xa = np.asarray(x, dtype=np.float64)
         return np.piecewise(xa, [(xa >= self.a) & (xa <= self.b)], [self._pdf])
 
     def cdf(self: PowerLaw, x: ArrayLike) -> NDArray[np.float64]:
@@ -86,7 +86,7 @@ def cdf(self: PowerLaw, x: ArrayLike) -> NDArray[np.float64]:
         Returns:
             array_like: Cumulative distribution function evaluated at `x`
         """
-        qa = np.asfarray(x)
+        qa = np.asarray(x, dtype=np.float64)
         return np.piecewise(qa, [qa < self.a, qa > self.b], [0, 1, self._cdf])
 
     def ppf(self: PowerLaw, q: ArrayLike) -> NDArray[np.float64]:
@@ -98,7 +98,7 @@ def ppf(self: PowerLaw, q: ArrayLike) -> NDArray[np.float64]:
         Returns:
             array_like: quantile corresponding to the lower tail probability `q`.
         """
-        qa = np.asfarray(q)
+        qa = np.asarray(q, dtype=np.float64)
         return np.piecewise(qa, [(qa >= 0) & (qa <= 1)], [self._ppf, np.nan])
 
     def rvs(self: PowerLaw, size: Any = None, random_state: SeedType = None) -> NDArray[np.float64]:
@@ -116,7 +116,7 @@ def rvs(self: PowerLaw, size: Any = None, random_state: SeedType = None) -> NDAr
                 Default is None.
         """
         rand_state = check_random_state(random_state)
-        return self._ppf(np.asfarray(rand_state.uniform(0, 1, size)))
+        return self._ppf(np.asarray(rand_state.uniform(0, 1, size), dtype=np.float64))
 
     def __repr__(self: PowerLaw) -> str:
         return f"{self.__class__.__name__}({self.g} ,{self.a}, {self.b})"

src/simweights/_spatial.py

@@ -42,17 +42,18 @@ def projected_area(self: CylinderBase, cos_zen: ArrayLike) -> NDArray[np.float64
 
         As seen from the angle described by cos_zen.
         """
-        cosz = np.asfarray(cos_zen)
+        cosz = np.asarray(cos_zen, dtype=np.float64)
         assert np.all(cosz >= -1)
         assert np.all(cosz <= +1)
-        return np.asfarray(self._cap * np.abs(cosz) + self._side * np.sqrt(1 - cosz**2))
+        return np.asarray(self._cap * np.abs(cosz) + self._side * np.sqrt(1 - cosz**2), dtype=np.float64)
 
     def _diff_etendue(self: CylinderBase, cos_zen: ArrayLike) -> NDArray[np.float64]:
-        cosz = np.asfarray(cos_zen)
+        cosz = np.asarray(cos_zen, dtype=np.float64)
         assert np.all(cosz >= -1)
         assert np.all(cosz <= +1)
-        return np.asfarray(
+        return np.asarray(
             np.pi * (self._cap * cosz * np.abs(cosz) + self._side * (cosz * np.sqrt(1 - cosz**2) - np.arccos(cosz))),
+            dtype=np.float64,
         )
 
     def pdf(self: CylinderBase, cos_zen: ArrayLike) -> NDArray[np.float64]:
@@ -91,7 +92,7 @@ def _pdf(self: UniformSolidAngleCylinder, cos_zen: NDArray[np.float64]) -> NDArr
         return 1 / (2 * np.pi * (self.cos_zen_max - self.cos_zen_min) * self.projected_area(cos_zen))
 
     def pdf(self: UniformSolidAngleCylinder, cos_zen: ArrayLike) -> NDArray[np.float64]:
-        cosz = np.asfarray(cos_zen)
+        cosz = np.asarray(cos_zen, dtype=np.float64)
         return np.piecewise(cosz, [(cosz >= self.cos_zen_min) & (cosz <= self.cos_zen_max)], [self._pdf])
 
 
@@ -123,7 +124,7 @@ def __init__(
         self._normalization = 1 / self.etendue
 
     def pdf(self: NaturalRateCylinder, cos_zen: ArrayLike) -> NDArray[np.float64]:
-        cosz = np.asfarray(cos_zen)
+        cosz = np.asarray(cos_zen, dtype=np.float64)
         return np.piecewise(
             cosz,
             [(cosz >= self.cos_zen_min) & (cosz <= self.cos_zen_max)],
@@ -157,7 +158,7 @@ def projected_area(self: CircleInjector, cos_zen: float) -> float:  # noqa: ARG0
 
     def pdf(self: CircleInjector, cos_zen: ArrayLike) -> NDArray[np.float64]:
         """The probability density function for the given zenith angle."""
-        cosz = np.asfarray(cos_zen)
+        cosz = np.asarray(cos_zen, dtype=np.float64)
         return np.piecewise(
             cosz,
             [(cosz >= self.cos_zen_min) & (cosz <= self.cos_zen_max)],

src/simweights/_utils.py

@@ -29,7 +29,7 @@ def __init__(self: Column, colname: str | None = None) -> None:
 
     def pdf(self: Column, value: ArrayLike) -> NDArray[np.float64]:
         r"""Probability density function."""
-        return 1 / np.asfarray(value)
+        return 1 / np.asarray(value, dtype=np.float64)
 
     def __eq__(self: Column, other: object) -> bool:
         return isinstance(other, Column) and self.columns == other.columns
@@ -47,7 +47,7 @@ def __init__(self: Const, v: float) -> None:
 
     def pdf(self: Const) -> NDArray[np.float64]:
         r"""Probability density function."""
-        return np.asfarray(self.v)
+        return np.asarray(self.v, dtype=np.float64)
 
     def __eq__(self: Const, other: object) -> bool:
         return isinstance(other, Const) and self.v == other.v
@@ -84,11 +84,11 @@ def has_column(table: Any, name: str) -> bool:
 def get_column(table: Any, name: str) -> NDArray[np.float64]:
     """Helper function getting a column from a table, works with h5py, pytables, and pandas."""
     if hasattr(table, "cols"):
-        return np.asfarray(getattr(table.cols, name)[:])
+        return np.asarray(getattr(table.cols, name)[:], dtype=np.float64)
     column = table[name]
     if hasattr(column, "array") and callable(column.array):
-        return np.asfarray(column.array(library="np"))
-    return np.asfarray(column)
+        return np.asarray(column.array(library="np"), dtype=np.float64)
+    return np.asarray(column, dtype=np.float64)
 
 
 def constcol(table: Any, colname: str, mask: NDArray[np.bool_] | None = None) -> float:

src/simweights/_weighter.py

@@ -199,7 +199,7 @@ def effective_area(
         assert np.array_equal(enbin, energy_bins)
         assert np.array_equal(czbin, cos_zenith_bins)
         e_width, z_width = np.meshgrid(np.ediff1d(enbin), np.ediff1d(czbin))
-        return np.asfarray(hist_val / (e_width * 2 * np.pi * z_width * nspecies))
+        return np.asarray(hist_val / (e_width * 2 * np.pi * z_width * nspecies), dtype=np.float64)
 
     def __add__(self: Weighter, other: Weighter | int) -> Weighter:
         if other == 0: