0.10.3 (#160)

* Update poetry.lock

* ⬆️ Pump simplug to 0.2

* 🔖 0.10.1

* 🚑 Fix false warning when importing from all

* 🔖 0.10.2

* ✨ Add `array_ufunc` hook

* 💥 Change hook `data_api` to `load_dataset`

* ✨ Allow backend for `c[]`

* ✨ Add `c.with_backend()` for `c[]`

* ✨ Add `DatarOperator.with_backend()` to select backend for operators

* ⬆️ Bump simplug to 0.2.2

* ✅ Add tests

* 📝 Update docs for backends

* 📝 Fix docs

* 🔖 0.10.3rc0

* 💚 Fix CI

* 💚 Try add six in CI

* 💚 Add dev-deps

* 💚 Add python-slugify as dev-deps

* 💚 Fix coverage file in CI

* 🔖 0.10.3

* 📝 [0.10.3] Fix docs building

* 💚 Try to ignore optional deps in CI

* 💚 Install dev-deps in CI

.coveragerc

@@ -0,0 +1,6 @@
+[report]
+exclude_lines =
+    pragma: no cover
+    if TYPE_CHECKING:
+omit =
+    datar/datasets.py

.github/workflows/ci.yml

@@ -7,17 +7,55 @@ on:
     types: [published]
 
 jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8, 3.9, "3.10"]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Python # Set Python version
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install poetry
+          poetry config virtualenvs.create false
+          # poetry install -v
+          python -m pip install .
+          python -m pip install flake8 pytest pytest-cov six numpy python-slugify
+      - name: Run flake8
+        run: flake8 datar
+      - name: Test with pytest
+        run: poetry run pytest tests/ --junitxml=junit/test-results-${{ matrix.python-version }}.xml
+      - name: Upload pytest test results
+        uses: actions/upload-artifact@v3
+        with:
+          name: pytest-results-${{ matrix.python-version }}
+          path: junit/test-results-${{ matrix.python-version }}.xml
+        # Use always() to always run this step to publish test results when there are test failures
+        if: ${{ always() }}
+      - name: Run codacy-coverage-reporter
+        uses: codacy/codacy-coverage-reporter-action@master
+        if: matrix.python-version == 3.9
+        with:
+          project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
+          coverage-reports: cov.xml
 
   deploy:
+    needs: build
     runs-on: ubuntu-20.04
     if: github.event_name == 'release'
     strategy:
       matrix:
         python-version: [3.9]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Setup Python # Set Python version
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/docs.yml

@@ -10,17 +10,19 @@ jobs:
       matrix:
         python-version: [3.9]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Setup Python # Set Python version
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
           python -m pip install poetry
           poetry config virtualenvs.create false
-          poetry install -v
+          python -m pip install --upgrade pip
+          # Can't skip optional deps with poetry install -v
+          # poetry install -v
+          python -m pip install .
       - name: Build docs
         run: |
           python -m pip install -r docs/requirements.txt
@@ -47,7 +49,7 @@ jobs:
       matrix:
         python-version: [3.9]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           ref: gh-pages
       - name: Fix index.html

datar/__init__.py

@@ -4,7 +4,7 @@
 from .core.defaults import f
 from .core.options import options, get_option, options_context
 
-__version__ = "0.10.2"
+__version__ = "0.10.3"
 
 
 def get_versions(prnt: bool = True) -> _Mapping[str, str]:

datar/all.py

@@ -12,6 +12,7 @@
 from .forcats import *
 from .tibble import *
 from .tidyr import *
+from .other import *
 
 _locs.update(
     {

datar/apis/forcats.py

@@ -239,7 +239,14 @@ def fct_collapse(_f, other_level=None, **kwargs):
 
 
 @_register_func(pipeable=True, dispatchable=True)
-def fct_lump(_f, n, prop, w, other_level="Other", ties_method: str = "min"):
+def fct_lump(
+    _f,
+    n=None,
+    prop=None,
+    w=None,
+    other_level="Other",
+    ties_method: str = "min",
+):
     """Lump together factor levels into "other"
 
     Args:

datar/apis/other.py

@@ -0,0 +1,26 @@
+from contextlib import contextmanager
+
+from pipda import register_func
+
+
+@contextmanager
+def _array_ufunc_with_backend(backend: str):
+    """Use a backend for the operator"""
+    old_backend = array_ufunc.backend
+    array_ufunc.backend = backend
+    yield
+    array_ufunc.backend = old_backend
+
+
+@register_func(cls=object, dispatchable="first")
+def array_ufunc(x, ufunc, *args, **kwargs):
+    """Implement the array ufunc
+
+    Allow other backends to override the behavior of the ufunc on
+    different types of data.
+    """
+    return ufunc(x, *args, **kwargs)
+
+
+array_ufunc.backend = None
+array_ufunc.with_backend = _array_ufunc_with_backend

datar/apis/tidyr.py

@@ -374,6 +374,7 @@ def drop_na(
     Returns:
         Dataframe with rows with NAs dropped and indexes dropped
     """
+    raise _NotImplementedByCurrentBackendError("drop_na", _data)
 
 
 @_register_verb()

datar/core/names.py

@@ -20,7 +20,7 @@ def _isnan(x: Any) -> bool:
 
 def _is_scalar(x: Any) -> bool:
     """Check if x is scalar"""
-    if isinstance(x, str):
+    if isinstance(x, str):  # pragma: no cover
         return True
     try:
         iter(x)

datar/core/operator.py

@@ -1,5 +1,6 @@
 """Operators for datar"""
 from typing import Callable
+from contextlib import contextmanager
 
 from pipda import register_operator, Operator
 
@@ -8,6 +9,22 @@
 class DatarOperator(Operator):
     """Operator class for datar"""
 
+    backend = None
+
+    @classmethod
+    @contextmanager
+    def with_backend(cls, backend: str):
+        """Use a backend for the operator"""
+        old_backend = cls.backend
+        cls.backend = backend
+        yield
+        cls.backend = old_backend
+
     def __getattr__(self, name: str) -> Callable:
         from .plugin import plugin
-        return lambda x, y=None: plugin.hooks.operate(name, x, y)
+        return lambda x, y=None: plugin.hooks.operate(
+            name,
+            x,
+            y,
+            __plugin=self.__class__.backend,
+        )

datar/core/plugin.py

@@ -2,6 +2,7 @@
 from typing import Any, List, Mapping, Tuple, Callable
 
 from simplug import Simplug, SimplugResult, makecall
+from pipda import register_array_ufunc
 
 from .options import get_option
 
@@ -18,6 +19,19 @@ def _collect(calls: List[Tuple[Callable, Tuple, Mapping]]) -> Mapping[str, Any]:
     return collected
 
 
+def _array_ufunc_to_register(ufunc, x, *args, **kwargs):
+    """Register the array ufunc to pipda"""
+    from ..apis.other import array_ufunc
+
+    return array_ufunc(
+        x,
+        ufunc,
+        *args,
+        **kwargs,
+        __backend=array_ufunc.backend,
+    )
+
+
 @plugin.spec
 def setup():
     """Initialize the backend"""
@@ -28,8 +42,8 @@ def get_versions():
     """Return the versions of the dependencies of the plugin."""
 
 
-@plugin.spec
-def data_api():
+@plugin.spec(result=SimplugResult.TRY_SINGLE)
+def load_dataset(name: str, metadata: Mapping):
     """Implementations for load_dataset()"""
 
 
@@ -63,16 +77,17 @@ def other_api():
     """What is implemented the other APIs."""
 
 
-@plugin.spec(result=SimplugResult.LAST)
+@plugin.spec(result=SimplugResult.SINGLE)
 def c_getitem(item):
     """Get item for c"""
 
 
-@plugin.spec(result=SimplugResult.LAST)
-def operate(op, x, y=None):
+@plugin.spec(result=SimplugResult.SINGLE)
+def operate(op: str, x: Any, y: Any = None):
     """Operate on x and y"""
 
 
 plugin.load_entrypoints(only=get_option("backends"))
 
 plugin.hooks.setup()
+register_array_ufunc(_array_ufunc_to_register)

datar/core/utils.py

@@ -2,6 +2,7 @@
 import sys
 import logging
 from typing import Any, Callable
+from contextlib import contextmanager
 
 from .plugin import plugin
 
@@ -38,14 +39,23 @@ class CollectionFunction:
 
     def __init__(self, c_func: Callable) -> None:
         self.c = c_func
+        self.backend = None
 
     def __call__(self, *args, **kwargs):
         kwargs["__ast_fallback"] = "normal"
         return self.c(*args, **kwargs)
 
+    @contextmanager
+    def with_backend(self, backend: str):
+        """Set the backend for c[]"""
+        _backend = self.backend
+        self.backend = backend
+        yield
+        self.backend = _backend
+
     def __getitem__(self, item):
         """Allow c[1:3] to be interpreted as 1:3"""
-        return plugin.hooks.c_getitem(item)
+        return plugin.hooks.c_getitem(item, __plugin=self.backend)
 
 
 def arg_match(arg, argname, values, errmsg=None):

datar/data/__init__.py

@@ -2,17 +2,13 @@
 import functools
 from typing import List
 
-from ..apis.data import load_dataset as _load_dataset
 from ..core.plugin import plugin
 from .metadata import Metadata, metadata
 
 
 # Should never do `from datar.data import *`
 __all__ = []  # type: List[str]
 
-# Load implementations for _load_dataset
-plugin.hooks.data_api()
-
 
 def descr_datasets(*names: str):
     """Get the information of the given datasets
@@ -40,23 +36,17 @@ def add_dataset(name: str, meta: Metadata):
 @functools.lru_cache()
 def load_dataset(name: str, __backend: str = None):
     """Load the specific dataset"""
-    if name not in metadata:
-        raise ImportError(
-            f"No such dataset: {name}, "
-            f"available: {list(metadata)}"
-        ) from None
-
-    loaded = _load_dataset(name, metadata[name], __backend=__backend)
+    loaded = plugin.hooks.load_dataset(name, metadata, __plugin=__backend)
     if loaded is None:
         from ..core.utils import NotImplementedByCurrentBackendError
         raise NotImplementedByCurrentBackendError(f"loading dataset '{name}'")
 
     return loaded
 
 
-def __getattr__(name):
+def __getattr__(name: str):
     # mkapi accesses quite a lot of attributes starting with _
-    try:
-        return load_dataset(name.lower())
-    except ImportError as imerr:
-        raise AttributeError from imerr
+    if not name.isidentifier() or name.startswith("__"):
+        raise AttributeError(name)
+
+    return load_dataset(name.lower())

datar/datasets.py

@@ -1,3 +1,4 @@
+# pragma: no cover
 import warnings
 
 

docs/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Change Log
 
+## 0.10.3
+
+- ⬆️ Bump simplug to 0.2.2
+- ✨ Add `apis.other.array_ufunc` to support numpy ufuncs
+- 💥 Change hook `data_api` to `load_dataset`
+- ✨ Allow backend for `c[]`
+- ✨ Add `DatarOperator.with_backend()` to select backend for operators
+- ✅ Add tests
+- 📝 Update docs for backend supports
+
 ## 0.10.2
 
 - 🚑 Fix false warning when importing from all