From 715160950f5c762dccc88b47375cd49116dbe04b Mon Sep 17 00:00:00 2001 From: Russ Allbery Date: Thu, 12 Jan 2023 14:21:42 -0800 Subject: [PATCH] Update typing to avoid deprecations Fix the deprecations in PEP 585, and use the | syntax instead of Union and some Optional cases. --- src/vocutouts/actors.py | 18 +++++++------- src/vocutouts/cli.py | 4 +-- src/vocutouts/config.py | 5 ++-- src/vocutouts/exceptions.py | 4 +-- src/vocutouts/handlers/external.py | 34 +++++++++++++------------- src/vocutouts/models/parameters.py | 9 +++---- src/vocutouts/models/stencils.py | 12 ++++----- src/vocutouts/policy.py | 3 +-- src/vocutouts/uws/config.py | 5 ++-- src/vocutouts/uws/dependencies.py | 4 +-- src/vocutouts/uws/exceptions.py | 4 +-- src/vocutouts/uws/handlers.py | 12 ++++----- src/vocutouts/uws/jobs.py | 10 ++++---- src/vocutouts/uws/models.py | 22 ++++++++--------- src/vocutouts/uws/policy.py | 5 ++-- src/vocutouts/uws/responses.py | 3 +-- src/vocutouts/uws/schema/job.py | 23 +++++++++-------- src/vocutouts/uws/schema/job_result.py | 6 ++--- src/vocutouts/uws/service.py | 18 +++++++------- src/vocutouts/uws/storage.py | 17 +++++++------ src/vocutouts/uws/utils.py | 3 +-- src/vocutouts/workers.py | 26 ++++++++++---------- tests/conftest.py | 9 ++++--- tests/handlers/async_test.py | 3 +-- tests/handlers/sync_test.py | 4 +-- tests/support/uws.py | 10 ++++---- tests/uws/conftest.py | 2 +- tests/uws/errors_test.py | 3 +-- tests/uws/job_error_test.py | 8 +++--- tests/uws/long_polling_test.py | 4 +-- tests/uws/policy_test.py | 3 +-- 31 files changed, 138 insertions(+), 155 deletions(-) diff --git a/src/vocutouts/actors.py b/src/vocutouts/actors.py index ab768a5..350be0c 100644 --- a/src/vocutouts/actors.py +++ b/src/vocutouts/actors.py @@ -34,7 +34,7 @@ from __future__ import annotations -from typing import Any, Dict, List +from typing import Any import dramatiq import structlog @@ -55,9 +55,9 @@ @dramatiq.actor(queue_name="cutout", max_retries=1, store_results=True) def cutout( job_id: str, - dataset_ids: List[str], - stencils: List[Dict[str, Any]], -) -> List[Dict[str, str]]: + dataset_ids: list[str], + stencils: list[dict[str, Any]], +) -> list[dict[str, str]]: """Stub for a circle cutout. This is only a stub, existing to define the actor in the Dramatiq broker @@ -68,11 +68,11 @@ def cutout( ---------- job_id : `str` The UWS job ID, used as the key for storing results. - dataset_ids : List[`str`] + dataset_ids : list[`str`] The data objects on which to perform cutouts. These are opaque identifiers passed as-is to the backend. The user will normally discover them via some service such as ObsTAP. - stencils : List[Dict[`str`, Any]] + stencils : list[dict[`str`, Any]] Serialized stencils for the cutouts to perform. These are JSON-serializable (a requirement for Dramatiq) representations of the `~vocutouts.models.stencils.Stencil` objects corresponding to the @@ -80,7 +80,7 @@ def cutout( Returns ------- - result : List[Dict[`str`, `str`]] + result : list[dict[`str`, `str`]] The results of the job. This must be a list of dict representations of `~vocutouts.uws.models.JobResult` objects. @@ -117,7 +117,7 @@ def job_started(job_id: str, message_id: str, start_time: str) -> None: @dramatiq.actor(queue_name="uws", priority=10) def job_completed( - message: Dict[str, Any], result: List[Dict[str, str]] + message: dict[str, Any], result: list[dict[str, str]] ) -> None: """Wrapper around the UWS function to mark a job as completed.""" logger = structlog.get_logger(config.logger_name) @@ -127,7 +127,7 @@ def job_completed( @dramatiq.actor(queue_name="uws", priority=20) -def job_failed(message: Dict[str, Any], exception: Dict[str, str]) -> None: +def job_failed(message: dict[str, Any], exception: dict[str, str]) -> None: """Wrapper around the UWS function to mark a job as errored.""" logger = structlog.get_logger(config.logger_name) job_id = message["args"][0] diff --git a/src/vocutouts/cli.py b/src/vocutouts/cli.py index f3b0223..6cac483 100644 --- a/src/vocutouts/cli.py +++ b/src/vocutouts/cli.py @@ -2,8 +2,6 @@ from __future__ import annotations -from typing import Optional - import click import structlog import uvicorn @@ -27,7 +25,7 @@ def main() -> None: @main.command() @click.argument("topic", default=None, required=False, nargs=1) @click.pass_context -def help(ctx: click.Context, topic: Optional[str]) -> None: +def help(ctx: click.Context, topic: str | None) -> None: """Show help for any command.""" # The help command implementation is taken from # https://www.burgundywall.com/post/having-click-help-subcommand diff --git a/src/vocutouts/config.py b/src/vocutouts/config.py index ce3e4ee..54fde79 100644 --- a/src/vocutouts/config.py +++ b/src/vocutouts/config.py @@ -4,7 +4,6 @@ import os from dataclasses import dataclass -from typing import Optional from .uws.config import UWSConfig @@ -29,7 +28,7 @@ class Configuration: is mandatory. """ - database_password: Optional[str] = os.getenv("CUTOUT_DATABASE_PASSWORD") + database_password: str | None = os.getenv("CUTOUT_DATABASE_PASSWORD") """The password for the UWS job database. Set with the ``CUTOUT_DATABASE_PASSWORD`` environment variable. @@ -64,7 +63,7 @@ class Configuration: mandatory. """ - redis_password: Optional[str] = os.getenv("CUTOUT_REDIS_PASSWORD") + redis_password: str | None = os.getenv("CUTOUT_REDIS_PASSWORD") """Password for the Redis server used by Dramatiq. Set with the ``CUTOUT_REDIS_PASSWORD`` environment variable. diff --git a/src/vocutouts/exceptions.py b/src/vocutouts/exceptions.py index 308d261..0890244 100644 --- a/src/vocutouts/exceptions.py +++ b/src/vocutouts/exceptions.py @@ -2,8 +2,6 @@ from __future__ import annotations -from typing import List - from .uws.exceptions import ParameterError from .uws.models import JobParameter @@ -13,7 +11,7 @@ class InvalidCutoutParameterError(ParameterError): """The parameters for the cutout were invalid.""" - def __init__(self, message: str, params: List[JobParameter]) -> None: + def __init__(self, message: str, params: list[JobParameter]) -> None: detail = "\n".join(f"{p.parameter_id}={p.value}" for p in params) super().__init__(message, detail) self.params = params diff --git a/src/vocutouts/handlers/external.py b/src/vocutouts/handlers/external.py index c22989a..27ba399 100644 --- a/src/vocutouts/handlers/external.py +++ b/src/vocutouts/handlers/external.py @@ -5,7 +5,7 @@ application knows the job parameters. """ -from typing import List, Literal, Optional +from typing import Literal, Optional from fastapi import APIRouter, Depends, Form, Query, Request, Response from fastapi.responses import PlainTextResponse, RedirectResponse @@ -118,9 +118,9 @@ async def get_capabilities(request: Request) -> Response: async def _sync_request( - params: List[JobParameter], + params: list[JobParameter], user: str, - runid: Optional[str], + runid: str | None, uws_factory: UWSFactory, logger: BoundLogger, ) -> Response: @@ -193,7 +193,7 @@ async def _sync_request( ) async def get_sync( request: Request, - id: List[str] = Query( + id: list[str] = Query( ..., title="Source ID", description=( @@ -201,7 +201,7 @@ async def get_sync( " parameter is mandatory." ), ), - pos: Optional[List[str]] = Query( + pos: Optional[list[str]] = Query( None, title="Cutout positions", description=( @@ -213,7 +213,7 @@ async def get_sync( " numbers expressed as strings." ), ), - circle: Optional[List[str]] = Query( + circle: Optional[list[str]] = Query( None, title="Cutout circle positions", description=( @@ -223,7 +223,7 @@ async def get_sync( " strings and separated by spaces." ), ), - polygon: Optional[List[str]] = Query( + polygon: Optional[list[str]] = Query( None, title="Cutout polygon positions", description=( @@ -274,7 +274,7 @@ async def get_sync( ) async def post_sync( request: Request, - id: Optional[List[str]] = Form( + id: Optional[list[str]] = Form( None, title="Source ID", description=( @@ -282,7 +282,7 @@ async def post_sync( " parameter is mandatory." ), ), - pos: Optional[List[str]] = Form( + pos: Optional[list[str]] = Form( None, title="Cutout positions", description=( @@ -294,7 +294,7 @@ async def post_sync( " numbers expressed as strings." ), ), - circle: Optional[List[str]] = Form( + circle: Optional[list[str]] = Form( None, title="Cutout circle positions", description=( @@ -304,7 +304,7 @@ async def post_sync( " strings and separated by spaces." ), ), - polygon: Optional[List[str]] = Form( + polygon: Optional[list[str]] = Form( None, title="Cutout polygon positions", description=( @@ -324,7 +324,7 @@ async def post_sync( " with specific larger operations." ), ), - params: List[JobParameter] = Depends(uws_post_params_dependency), + params: list[JobParameter] = Depends(uws_post_params_dependency), user: str = Depends(auth_dependency), uws_factory: UWSFactory = Depends(uws_dependency), logger: BoundLogger = Depends(auth_logger_dependency), @@ -346,7 +346,7 @@ async def post_sync( ) async def create_job( request: Request, - id: Optional[List[str]] = Form( + id: Optional[list[str]] = Form( None, title="Source ID", description=( @@ -354,7 +354,7 @@ async def create_job( " parameter is mandatory." ), ), - pos: Optional[List[str]] = Form( + pos: Optional[list[str]] = Form( None, title="Cutout positions", description=( @@ -366,7 +366,7 @@ async def create_job( " numbers expressed as strings." ), ), - circle: Optional[List[str]] = Form( + circle: Optional[list[str]] = Form( None, title="Cutout circle positions", description=( @@ -376,7 +376,7 @@ async def create_job( " strings and separated by spaces." ), ), - polygon: Optional[List[str]] = Form( + polygon: Optional[list[str]] = Form( None, title="Cutout polygon positions", description=( @@ -399,7 +399,7 @@ async def create_job( " with specific larger operations." ), ), - params: List[JobParameter] = Depends(uws_post_params_dependency), + params: list[JobParameter] = Depends(uws_post_params_dependency), user: str = Depends(auth_dependency), uws_factory: UWSFactory = Depends(uws_dependency), logger: BoundLogger = Depends(auth_logger_dependency), diff --git a/src/vocutouts/models/parameters.py b/src/vocutouts/models/parameters.py index 10934bf..4f93476 100644 --- a/src/vocutouts/models/parameters.py +++ b/src/vocutouts/models/parameters.py @@ -3,7 +3,6 @@ from __future__ import annotations from dataclasses import dataclass -from typing import List from ..exceptions import InvalidCutoutParameterError from ..uws.models import JobParameter @@ -14,21 +13,21 @@ class CutoutParameters: """The parameters to a cutout request.""" - ids: List[str] + ids: list[str] """The dataset IDs on which to operate.""" - stencils: List[Stencil] + stencils: list[Stencil] """The cutout stencils to apply.""" @classmethod def from_job_parameters( - cls, params: List[JobParameter] + cls, params: list[JobParameter] ) -> CutoutParameters: """Convert generic UWS parameters to the iamge cutout parameters. Parameters ---------- - params : List[`vocutouts.uws.models.JobParameter`] + params : list[`vocutouts.uws.models.JobParameter`] Generic input job parameters. Returns diff --git a/src/vocutouts/models/stencils.py b/src/vocutouts/models/stencils.py index cb94108..0eb2955 100644 --- a/src/vocutouts/models/stencils.py +++ b/src/vocutouts/models/stencils.py @@ -4,12 +4,12 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Any, Dict, Tuple +from typing import Any from astropy import units as u from astropy.coordinates import Angle, SkyCoord -Range = Tuple[float, float] +Range = tuple[float, float] class Stencil(ABC): @@ -21,7 +21,7 @@ def from_string(cls, params: str) -> Stencil: """Parse a string representation of stencil parameters to an object.""" @abstractmethod - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: """Convert the stencil to a JSON-serializable form for queuing.""" @@ -40,7 +40,7 @@ def from_string(cls, params: str) -> CircleStencil: radius=Angle(radius * u.degree), ) - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: return { "type": "circle", "center": { @@ -78,7 +78,7 @@ def from_string(cls, params: str) -> PolygonStencil: vertices = SkyCoord(ras * u.degree, decs * u.degree, frame="icrs") return cls(vertices=vertices) - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: return { "type": "polygon", "vertices": [(v.ra.degree, v.dec.degree) for v in self.vertices], @@ -100,7 +100,7 @@ def from_string(cls, params: str) -> RangeStencil: dec=(dec_min, dec_max), ) - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: return { "type": "range", "ra": self.ra, diff --git a/src/vocutouts/policy.py b/src/vocutouts/policy.py index deff798..ea0a5bb 100644 --- a/src/vocutouts/policy.py +++ b/src/vocutouts/policy.py @@ -3,7 +3,6 @@ from __future__ import annotations from datetime import datetime -from typing import List from dramatiq import Actor, Message from structlog.stdlib import BoundLogger @@ -80,7 +79,7 @@ def validate_execution_duration( ) -> int: return job.execution_duration - def validate_params(self, params: List[JobParameter]) -> None: + def validate_params(self, params: list[JobParameter]) -> None: try: cutout_params = CutoutParameters.from_job_parameters(params) except InvalidCutoutParameterError as e: diff --git a/src/vocutouts/uws/config.py b/src/vocutouts/uws/config.py index aee05dc..5ce9cee 100644 --- a/src/vocutouts/uws/config.py +++ b/src/vocutouts/uws/config.py @@ -3,7 +3,6 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Optional __all__ = ["UWSConfig"] @@ -49,10 +48,10 @@ class encapsulates the configuration of the UWS component that may vary by with this email. """ - database_password: Optional[str] = None + database_password: str | None = None """Password for the database.""" - redis_password: Optional[str] = None + redis_password: str | None = None """Password for the Redis server used by Dramatiq.""" url_lifetime: int = 15 * 60 diff --git a/src/vocutouts/uws/dependencies.py b/src/vocutouts/uws/dependencies.py index 0c66d58..b0d0ed8 100644 --- a/src/vocutouts/uws/dependencies.py +++ b/src/vocutouts/uws/dependencies.py @@ -6,7 +6,7 @@ objects. """ -from typing import List, Optional +from typing import Optional from fastapi import Depends, Request from safir.dependencies.db_session import db_session_dependency @@ -140,7 +140,7 @@ def override_policy(self, policy: UWSPolicy) -> None: uws_dependency = UWSDependency() -async def uws_post_params_dependency(request: Request) -> List[JobParameter]: +async def uws_post_params_dependency(request: Request) -> list[JobParameter]: """Parse POST parameters. UWS requires that all POST parameters be case-insensitive, which is not diff --git a/src/vocutouts/uws/exceptions.py b/src/vocutouts/uws/exceptions.py index f43a556..c31a18a 100644 --- a/src/vocutouts/uws/exceptions.py +++ b/src/vocutouts/uws/exceptions.py @@ -6,7 +6,7 @@ from __future__ import annotations -from typing import Dict, Optional +from typing import Optional from .models import ErrorCode, ErrorType, JobError @@ -61,7 +61,7 @@ class TaskError(UWSError): """An error occurred during background task processing.""" @classmethod - def from_callback(cls, exception: Dict[str, str]) -> TaskError: + def from_callback(cls, exception: dict[str, str]) -> TaskError: """Reconstitute the exception passed to an on_failure callback. Notes diff --git a/src/vocutouts/uws/handlers.py b/src/vocutouts/uws/handlers.py index 9a37e87..c00fd53 100644 --- a/src/vocutouts/uws/handlers.py +++ b/src/vocutouts/uws/handlers.py @@ -17,7 +17,7 @@ """ from datetime import datetime -from typing import List, Literal, Optional +from typing import Literal, Optional from fastapi import APIRouter, Depends, Form, Query, Request, Response from fastapi.responses import PlainTextResponse, RedirectResponse @@ -53,7 +53,7 @@ ) async def get_job_list( request: Request, - phase: Optional[List[ExecutionPhase]] = Query( + phase: Optional[list[ExecutionPhase]] = Query( None, title="Execution phase", description="Limit results to the provided execution phases", @@ -151,7 +151,7 @@ async def delete_job_via_post( title="Action to perform", description="Mandatory, must be set to DELETE", ), - params: List[JobParameter] = Depends(uws_post_params_dependency), + params: list[JobParameter] = Depends(uws_post_params_dependency), user: str = Depends(auth_dependency), uws_factory: UWSFactory = Depends(uws_dependency), logger: BoundLogger = Depends(auth_logger_dependency), @@ -206,7 +206,7 @@ async def post_job_destruction( description="Must be in ISO 8601 format.", example="2021-09-10T10:01:02Z", ), - params: List[JobParameter] = Depends(uws_post_params_dependency), + params: list[JobParameter] = Depends(uws_post_params_dependency), user: str = Depends(auth_dependency), uws_factory: UWSFactory = Depends(uws_dependency), logger: BoundLogger = Depends(auth_logger_dependency), @@ -291,7 +291,7 @@ async def post_job_execution_duration( description="Integer seconds of wall clock time.", example=14400, ), - params: List[JobParameter] = Depends(uws_post_params_dependency), + params: list[JobParameter] = Depends(uws_post_params_dependency), user: str = Depends(auth_dependency), uws_factory: UWSFactory = Depends(uws_dependency), logger: BoundLogger = Depends(auth_logger_dependency), @@ -388,7 +388,7 @@ async def post_job_phase( title="Job state change", summary="RUN to start the job, ABORT to abort the job.", ), - params: List[JobParameter] = Depends(uws_post_params_dependency), + params: list[JobParameter] = Depends(uws_post_params_dependency), user: str = Depends(auth_dependency), uws_factory: UWSFactory = Depends(uws_dependency), logger: BoundLogger = Depends(auth_logger_dependency), diff --git a/src/vocutouts/uws/jobs.py b/src/vocutouts/uws/jobs.py index cfd6b73..ef71c2b 100644 --- a/src/vocutouts/uws/jobs.py +++ b/src/vocutouts/uws/jobs.py @@ -29,7 +29,7 @@ from __future__ import annotations from datetime import datetime -from typing import Any, Dict, List +from typing import Any from sqlalchemy.orm import scoped_session from structlog.stdlib import BoundLogger @@ -77,7 +77,7 @@ def uws_job_started( def uws_job_completed( job_id: str, - result: List[Dict[str, Any]], + result: list[dict[str, Any]], session: scoped_session, logger: BoundLogger, ) -> None: @@ -87,7 +87,7 @@ def uws_job_completed( ---------- job_id : `str` The identifier of the job that was started. - result : List[Dict[`str`, Any]] + result : list[dict[`str`, Any]] The results of the job. This must be a list of dict representations of `~vocutouts.uws.models.JobResult` objects. session : `sqlalchemy.orm.scoped_session` @@ -106,7 +106,7 @@ def uws_job_completed( def uws_job_failed( job_id: str, - exception: Dict[str, str], + exception: dict[str, str], session: scoped_session, logger: BoundLogger, ) -> None: @@ -116,7 +116,7 @@ def uws_job_failed( ---------- job_id : `str` The identifier of the job that was started. - exception : Dict[`str`, `str`] + exception : dict[`str`, `str`] Exception information as passed to a Dramatiq ``on_failure`` callback. session : `sqlalchemy.orm.scoped_session` A synchronous session to the UWS database. diff --git a/src/vocutouts/uws/models.py b/src/vocutouts/uws/models.py index 61ec42f..e8b59ac 100644 --- a/src/vocutouts/uws/models.py +++ b/src/vocutouts/uws/models.py @@ -9,7 +9,7 @@ from dataclasses import asdict, dataclass from datetime import datetime from enum import Enum -from typing import Dict, List, Optional, Union +from typing import Optional @dataclass @@ -155,7 +155,7 @@ class JobParameter: is_post: bool = False """Whether the parameter was provided via POST.""" - def to_dict(self) -> Dict[str, Union[str, bool]]: + def to_dict(self) -> dict[str, str | bool]: """Convert to a dictionary, primarily for logging.""" return asdict(self) @@ -177,7 +177,7 @@ class JobDescription: phase: ExecutionPhase """Execution phase of the job.""" - run_id: Optional[str] + run_id: str | None """Optional opaque string provided by the client. The RunId is intended for the client to add a unique identifier to all @@ -197,7 +197,7 @@ class Job: job_id: str """Unique identifier of the job.""" - message_id: Optional[str] + message_id: str | None """Internal message identifier for the work queuing system.""" owner: str @@ -206,7 +206,7 @@ class Job: phase: ExecutionPhase """Execution phase of the job.""" - run_id: Optional[str] + run_id: str | None """Optional opaque string provided by the client. The RunId is intended for the client to add a unique identifier to all @@ -218,10 +218,10 @@ class Job: creation_time: datetime """When the job was created.""" - start_time: Optional[datetime] + start_time: datetime | None """When the job started executing (if it has started).""" - end_time: Optional[datetime] + end_time: datetime | None """When the job stopped executing (if it has stopped).""" destruction_time: datetime @@ -242,7 +242,7 @@ class Job: aborted. """ - quote: Optional[datetime] + quote: datetime | None """Expected completion time of the job if it were started now. May be `None` to indicate that the expected duration of the job is not @@ -250,11 +250,11 @@ class Job: not possible due to resource constraints. """ - error: Optional[JobError] + error: JobError | None """Error information if the job failed.""" - parameters: List[JobParameter] + parameters: list[JobParameter] """The parameters of the job.""" - results: List[JobResult] + results: list[JobResult] """The results of the job.""" diff --git a/src/vocutouts/uws/policy.py b/src/vocutouts/uws/policy.py index 91185aa..ed12bb4 100644 --- a/src/vocutouts/uws/policy.py +++ b/src/vocutouts/uws/policy.py @@ -4,7 +4,6 @@ from abc import ABC, abstractmethod from datetime import datetime -from typing import List from dramatiq import Message @@ -91,12 +90,12 @@ def validate_execution_duration( """ @abstractmethod - def validate_params(self, params: List[JobParameter]) -> None: + def validate_params(self, params: list[JobParameter]) -> None: """Validate parameters for a job. Parameters ---------- - params : List[`vocutouts.uws.models.JobParameter`] + params : list[`vocutouts.uws.models.JobParameter`] The new parameters. Raises diff --git a/src/vocutouts/uws/responses.py b/src/vocutouts/uws/responses.py index 5ecc396..6967012 100644 --- a/src/vocutouts/uws/responses.py +++ b/src/vocutouts/uws/responses.py @@ -3,7 +3,6 @@ from __future__ import annotations from pathlib import Path -from typing import List from fastapi import Request, Response from fastapi.templating import Jinja2Templates @@ -60,7 +59,7 @@ async def job(self, request: Request, job: Job) -> Response: ) def job_list( - self, request: Request, jobs: List[JobDescription], base_url: str + self, request: Request, jobs: list[JobDescription], base_url: str ) -> Response: """Return a list of jobs as an XML response.""" return _templates.TemplateResponse( diff --git a/src/vocutouts/uws/schema/job.py b/src/vocutouts/uws/schema/job.py index 486e136..7cc6d9c 100644 --- a/src/vocutouts/uws/schema/job.py +++ b/src/vocutouts/uws/schema/job.py @@ -9,7 +9,6 @@ from __future__ import annotations from datetime import datetime -from typing import List, Optional from sqlalchemy import Column, DateTime, Enum, Index, Integer, String, Text from sqlalchemy.orm import relationship @@ -26,25 +25,25 @@ class Job(Base): __tablename__ = "job" id: int = Column(Integer, primary_key=True, autoincrement=True) - message_id: Optional[str] = Column(String(64)) + message_id: str | None = Column(String(64)) owner: str = Column(String(64), nullable=False) phase: ExecutionPhase = Column(Enum(ExecutionPhase), nullable=False) - run_id: Optional[str] = Column(String(64)) + run_id: str | None = Column(String(64)) creation_time: datetime = Column(DateTime, nullable=False) - start_time: Optional[datetime] = Column(DateTime) - end_time: Optional[datetime] = Column(DateTime) + start_time: datetime | None = Column(DateTime) + end_time: datetime | None = Column(DateTime) destruction_time: datetime = Column(DateTime, nullable=False) execution_duration: int = Column(Integer, nullable=False) - quote: Optional[datetime] = Column(DateTime) - error_type: Optional[ErrorType] = Column(Enum(ErrorType)) - error_code: Optional[ErrorCode] = Column(Enum(ErrorCode)) - error_message: Optional[str] = Column(Text) - error_detail: Optional[str] = Column(Text) + quote: datetime | None = Column(DateTime) + error_type: ErrorType | None = Column(Enum(ErrorType)) + error_code: ErrorCode | None = Column(Enum(ErrorCode)) + error_message: str | None = Column(Text) + error_detail: str | None = Column(Text) - parameters: List[JobParameter] = relationship( + parameters: list[JobParameter] = relationship( "JobParameter", cascade="delete", lazy="selectin", uselist=True ) - results: List[JobResult] = relationship( + results: list[JobResult] = relationship( "JobResult", cascade="delete", lazy="selectin", uselist=True ) diff --git a/src/vocutouts/uws/schema/job_result.py b/src/vocutouts/uws/schema/job_result.py index 1d1c7d3..82483df 100644 --- a/src/vocutouts/uws/schema/job_result.py +++ b/src/vocutouts/uws/schema/job_result.py @@ -2,8 +2,6 @@ from __future__ import annotations -from typing import Optional - from sqlalchemy import Column, ForeignKey, Index, Integer, String from .base import Base @@ -21,8 +19,8 @@ class JobResult(Base): result_id: str = Column(String(64), nullable=False) sequence: int = Column(Integer, nullable=False) url: str = Column(String(256), nullable=False) - size: Optional[int] = Column(Integer) - mime_type: Optional[str] = Column(String(64)) + size: int | None = Column(Integer) + mime_type: str | None = Column(String(64)) __table_args__ = ( Index("by_sequence", "job_id", "sequence", unique=True), diff --git a/src/vocutouts/uws/service.py b/src/vocutouts/uws/service.py index a359a20..1cad981 100644 --- a/src/vocutouts/uws/service.py +++ b/src/vocutouts/uws/service.py @@ -4,7 +4,7 @@ import asyncio from datetime import datetime, timedelta, timezone -from typing import List, Optional +from typing import Optional from dramatiq import Message @@ -73,7 +73,7 @@ async def create( user: str, *, run_id: Optional[str] = None, - params: List[JobParameter], + params: list[JobParameter], ) -> Job: """Create a pending job. @@ -86,7 +86,7 @@ async def create( User on behalf this operation is performed. run_id : `str`, optional A client-supplied opaque identifier to record with the job. - params : List[`vocutouts.uws.models.JobParameter`] + params : list[`vocutouts.uws.models.JobParameter`] The input parameters to the job. Returns @@ -216,17 +216,17 @@ async def list_jobs( self, user: str, *, - phases: Optional[List[ExecutionPhase]] = None, + phases: Optional[list[ExecutionPhase]] = None, after: Optional[datetime] = None, count: Optional[int] = None, - ) -> List[JobDescription]: + ) -> list[JobDescription]: """List the jobs for a particular user. Parameters ---------- user : `str` Name of the user whose jobs to load. - phases : List[`vocutouts.uws.models.ExecutionPhase`], optional + phases : list[`vocutouts.uws.models.ExecutionPhase`], optional Limit the result to jobs in this list of possible execution phases. after : `datetime.datetime`, optional @@ -236,7 +236,7 @@ async def list_jobs( Returns ------- - descriptions : List[`vocutouts.uws.models.JobDescription`] + descriptions : list[`vocutouts.uws.models.JobDescription`] List of job descriptions matching the search criteria. """ return await self._storage.list_jobs( @@ -275,7 +275,7 @@ async def start(self, user: str, job_id: str) -> Message: async def update_destruction( self, user: str, job_id: str, destruction: datetime - ) -> Optional[datetime]: + ) -> datetime | None: """Update the destruction time of a job. Parameters @@ -313,7 +313,7 @@ async def update_destruction( async def update_execution_duration( self, user: str, job_id: str, duration: int - ) -> Optional[int]: + ) -> int | None: """Update the execution duration time of a job. Parameters diff --git a/src/vocutouts/uws/storage.py b/src/vocutouts/uws/storage.py index 1a65c52..6aa57e7 100644 --- a/src/vocutouts/uws/storage.py +++ b/src/vocutouts/uws/storage.py @@ -2,9 +2,10 @@ from __future__ import annotations +from collections.abc import Awaitable, Callable from datetime import datetime, timedelta, timezone from functools import wraps -from typing import Any, Awaitable, Callable, List, Optional, TypeVar, cast +from typing import Any, Optional, TypeVar, cast from safir.database import datetime_from_db, datetime_to_db from sqlalchemy import delete @@ -159,7 +160,7 @@ async def add( *, owner: str, run_id: Optional[str] = None, - params: List[JobParameter], + params: list[JobParameter], execution_duration: int, lifetime: int, ) -> Job: @@ -173,7 +174,7 @@ async def add( The username of the owner of the job. run_id : `str`, optional A client-supplied opaque identifier to record with the job. - params : List[`vocutouts.uws.models.JobParameter`] + params : list[`vocutouts.uws.models.JobParameter`] The input parameters to the job. execution_duration : `int` The maximum length of time for which a job is allowed to run in @@ -241,17 +242,17 @@ async def list_jobs( self, user: str, *, - phases: Optional[List[ExecutionPhase]] = None, + phases: Optional[list[ExecutionPhase]] = None, after: Optional[datetime] = None, count: Optional[int] = None, - ) -> List[JobDescription]: + ) -> list[JobDescription]: """List the jobs for a particular user. Parameters ---------- user : `str` Name of the user whose jobs to load. - phases : List[`vocutouts.uws.models.ExecutionPhase`], optional + phases : list[`vocutouts.uws.models.ExecutionPhase`], optional Limit the result to jobs in this list of possible execution phases. after : `datetime.datetime`, optional @@ -261,7 +262,7 @@ async def list_jobs( Returns ------- - descriptions : List[`vocutouts.uws.models.JobDescription`] + descriptions : list[`vocutouts.uws.models.JobDescription`] List of job descriptions matching the search criteria. """ stmt = select( @@ -398,7 +399,7 @@ def __init__(self, session: scoped_session) -> None: self._session = session @retry_transaction - def mark_completed(self, job_id: str, results: List[JobResult]) -> None: + def mark_completed(self, job_id: str, results: list[JobResult]) -> None: """Mark a job as completed.""" with self._session.begin(): job = self._get_job(job_id) diff --git a/src/vocutouts/uws/utils.py b/src/vocutouts/uws/utils.py index 9184132..9a3c6f1 100644 --- a/src/vocutouts/uws/utils.py +++ b/src/vocutouts/uws/utils.py @@ -3,7 +3,6 @@ from __future__ import annotations from datetime import datetime, timezone -from typing import Optional def isodatetime(timestamp: datetime) -> str: @@ -12,7 +11,7 @@ def isodatetime(timestamp: datetime) -> str: return timestamp.strftime("%Y-%m-%dT%H:%M:%SZ") -def parse_isodatetime(time_string: str) -> Optional[datetime]: +def parse_isodatetime(time_string: str) -> datetime | None: """Parse a string in the UWS ISO date format. Returns diff --git a/src/vocutouts/workers.py b/src/vocutouts/workers.py index 6f82393..0d24103 100644 --- a/src/vocutouts/workers.py +++ b/src/vocutouts/workers.py @@ -16,7 +16,7 @@ import os from datetime import datetime, timezone -from typing import Any, Dict, List, Tuple +from typing import Any from urllib.parse import urlparse from uuid import UUID @@ -37,7 +37,7 @@ ) from safir.logging import configure_logging -BACKENDS: Dict[str, ImageCutoutBackend] = {} +BACKENDS: dict[str, ImageCutoutBackend] = {} """Cache of image cutout backends by Butler repository label.""" configure_logging( @@ -65,13 +65,13 @@ def job_started(job_id: str, message_id: str, start_time: str) -> None: @dramatiq.actor(queue_name="uws") def job_completed( - message: Dict[str, Any], result: List[Dict[str, str]] + message: dict[str, Any], result: list[dict[str, str]] ) -> None: pass @dramatiq.actor(queue_name="uws") -def job_failed(message: Dict[str, Any], exception: Dict[str, str]) -> None: +def job_failed(message: dict[str, Any], exception: dict[str, str]) -> None: pass @@ -114,7 +114,7 @@ def get_backend(butler_label: str) -> ImageCutoutBackend: return backend -def parse_uri(uri: str) -> Tuple[str, UUID]: +def parse_uri(uri: str) -> tuple[str, UUID]: """Parse a Butler URI. Parameters @@ -124,7 +124,7 @@ def parse_uri(uri: str) -> Tuple[str, UUID]: Returns ------- - data : Tuple[`str`, `uuid.UUID`] + data : tuple[`str`, `uuid.UUID`] The Butler label and the object UUID. """ parsed_uri = urlparse(uri) @@ -134,9 +134,9 @@ def parse_uri(uri: str) -> Tuple[str, UUID]: @dramatiq.actor(queue_name="cutout", max_retries=1, store_results=True) def cutout( job_id: str, - dataset_ids: List[str], - stencils: List[Dict[str, Any]], -) -> List[Dict[str, str]]: + dataset_ids: list[str], + stencils: list[dict[str, Any]], +) -> list[dict[str, str]]: """Perform a cutout. This is a queue worker for the vo-cutouts service. It takes a serialized @@ -149,11 +149,11 @@ def cutout( ---------- job_id : `str` The UWS job ID, used as the key for storing results. - dataset_ids : List[`str`] + dataset_ids : list[`str`] The data objects on which to perform cutouts. These are opaque identifiers passed as-is to the backend. The user will normally discover them via some service such as ObsTAP. - stencils : List[Dict[`str`, Any]] + stencils : list[dict[`str`, Any]] Serialized stencils for the cutouts to perform. These are JSON-serializable (a requirement for Dramatiq) representations of the `~vocutouts.models.stencils.Stencil` objects corresponding to the @@ -161,7 +161,7 @@ def cutout( Returns ------- - result : List[Dict[`str`, `str`]] + result : list[dict[`str`, `str`]] The results of the job. This must be a list of dict representations of `~vocutouts.uws.models.JobResult` objects. """ @@ -190,7 +190,7 @@ def cutout( backend = get_backend(butler_label) # Convert the stencils to SkyStencils. - sky_stencils: List[SkyStencil] = [] + sky_stencils: list[SkyStencil] = [] for stencil_dict in stencils: if stencil_dict["type"] == "circle": center = SkyCoord( diff --git a/tests/conftest.py b/tests/conftest.py index f05ecae..d7edd9c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,8 +2,9 @@ from __future__ import annotations +from collections.abc import AsyncIterator, Iterator from datetime import datetime, timedelta, timezone -from typing import Any, AsyncIterator, Dict, Iterator, List +from typing import Any import dramatiq import pytest @@ -29,9 +30,9 @@ @dramatiq.actor(queue_name="cutout", store_results=True) def cutout_test( job_id: str, - dataset_ids: List[str], - stencils: List[Dict[str, Any]], -) -> List[Dict[str, Any]]: + dataset_ids: list[str], + stencils: list[dict[str, Any]], +) -> list[dict[str, Any]]: message = CurrentMessage.get_current_message() now = isodatetime(datetime.now(tz=timezone.utc)) job_started.send(job_id, message.message_id, now) diff --git a/tests/handlers/async_test.py b/tests/handlers/async_test.py index 69849ea..205a9ef 100644 --- a/tests/handlers/async_test.py +++ b/tests/handlers/async_test.py @@ -4,7 +4,6 @@ import asyncio import re -from typing import Dict, List import pytest from dramatiq import Worker @@ -156,7 +155,7 @@ async def test_redirect(app: FastAPI) -> None: @pytest.mark.asyncio async def test_bad_parameters(client: AsyncClient) -> None: - bad_params: List[Dict[str, str]] = [ + bad_params: list[dict[str, str]] = [ {}, {"pos": "RANGE 0 360 -2 2"}, {"id": "foo", "foo": "bar"}, diff --git a/tests/handlers/sync_test.py b/tests/handlers/sync_test.py index 6c8789d..b2b5421 100644 --- a/tests/handlers/sync_test.py +++ b/tests/handlers/sync_test.py @@ -2,8 +2,6 @@ from __future__ import annotations -from typing import Dict, List - import pytest from dramatiq import Worker from httpx import AsyncClient @@ -40,7 +38,7 @@ async def test_sync(client: AsyncClient) -> None: @pytest.mark.asyncio async def test_bad_parameters(client: AsyncClient) -> None: - bad_params: List[Dict[str, str]] = [ + bad_params: list[dict[str, str]] = [ {}, {"pos": "RANGE 0 360 -2 2"}, {"id": "5:6:a:b", "foo": "bar"}, diff --git a/tests/support/uws.py b/tests/support/uws.py index 7aa2e25..fff17b3 100644 --- a/tests/support/uws.py +++ b/tests/support/uws.py @@ -5,7 +5,7 @@ import asyncio import os from datetime import datetime, timezone -from typing import Any, Dict, List, Optional +from typing import Any, Optional import dramatiq import structlog @@ -69,7 +69,7 @@ def before_worker_boot(self, broker: Broker, worker: Worker) -> None: @dramatiq.actor(broker=uws_broker, queue_name="job", store_results=True) -def trivial_job(job_id: str) -> List[Dict[str, Any]]: +def trivial_job(job_id: str) -> list[dict[str, Any]]: message = CurrentMessage.get_current_message() now = datetime.now(tz=timezone.utc) job_started.send(job_id, message.message_id, isodatetime(now)) @@ -93,7 +93,7 @@ def job_started(job_id: str, message_id: str, start_time: str) -> None: @dramatiq.actor(broker=uws_broker, queue_name="uws", priority=10) def job_completed( - message: Dict[str, Any], result: List[Dict[str, str]] + message: dict[str, Any], result: list[dict[str, str]] ) -> None: logger = structlog.get_logger("uws") job_id = message["args"][0] @@ -102,7 +102,7 @@ def job_completed( @dramatiq.actor(broker=uws_broker, queue_name="uws", priority=20) -def job_failed(message: Dict[str, Any], exception: Dict[str, str]) -> None: +def job_failed(message: dict[str, Any], exception: dict[str, str]) -> None: logger = structlog.get_logger("uws") job_id = message["args"][0] assert worker_session @@ -131,7 +131,7 @@ def validate_execution_duration( ) -> int: return execution_duration - def validate_params(self, params: List[JobParameter]) -> None: + def validate_params(self, params: list[JobParameter]) -> None: pass diff --git a/tests/uws/conftest.py b/tests/uws/conftest.py index 5df5e41..516f275 100644 --- a/tests/uws/conftest.py +++ b/tests/uws/conftest.py @@ -11,8 +11,8 @@ from __future__ import annotations +from collections.abc import AsyncIterator, Iterator from datetime import timedelta -from typing import AsyncIterator, Iterator import pytest import pytest_asyncio diff --git a/tests/uws/errors_test.py b/tests/uws/errors_test.py index 55a6e2b..696c810 100644 --- a/tests/uws/errors_test.py +++ b/tests/uws/errors_test.py @@ -3,7 +3,6 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Dict import pytest from httpx import AsyncClient @@ -17,7 +16,7 @@ class PostTest: """Encapsulates the data a test POST.""" url: str - data: Dict[str, str] + data: dict[str, str] @pytest.mark.asyncio diff --git a/tests/uws/job_error_test.py b/tests/uws/job_error_test.py index 779c9b0..e42501c 100644 --- a/tests/uws/job_error_test.py +++ b/tests/uws/job_error_test.py @@ -4,7 +4,7 @@ import time from datetime import datetime, timezone -from typing import Any, Dict, List +from typing import Any import dramatiq import pytest @@ -80,7 +80,7 @@ async def test_temporary_error( # Create a backend worker that raises a transient error. @dramatiq.actor(broker=uws_broker, queue_name="job") - def error_transient_job(job_id: str) -> List[Dict[str, Any]]: + def error_transient_job(job_id: str) -> list[dict[str, Any]]: message = CurrentMessage.get_current_message() now = datetime.now(tz=timezone.utc) job_started.send(job_id, message.message_id, isodatetime(now)) @@ -145,7 +145,7 @@ async def test_fatal_error( # Create a backend worker that raises a fatal error with detail. @dramatiq.actor(broker=uws_broker, queue_name="job") - def error_fatal_job(job_id: str) -> List[Dict[str, Any]]: + def error_fatal_job(job_id: str) -> list[dict[str, Any]]: message = CurrentMessage.get_current_message() now = datetime.now(tz=timezone.utc) job_started.send(job_id, message.message_id, isodatetime(now)) @@ -208,7 +208,7 @@ async def test_unknown_error( # Create a backend worker that raises a fatal error with detail. @dramatiq.actor(broker=uws_broker, queue_name="job") - def error_unknown_job(job_id: str) -> List[Dict[str, Any]]: + def error_unknown_job(job_id: str) -> list[dict[str, Any]]: message = CurrentMessage.get_current_message() now = datetime.now(tz=timezone.utc) time.sleep(0.5) diff --git a/tests/uws/long_polling_test.py b/tests/uws/long_polling_test.py index 45a683c..a3e3436 100644 --- a/tests/uws/long_polling_test.py +++ b/tests/uws/long_polling_test.py @@ -4,7 +4,7 @@ import time from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List +from typing import Any import dramatiq import pytest @@ -119,7 +119,7 @@ async def test_poll( ) @dramatiq.actor(broker=uws_broker, queue_name="job", store_results=True) - def wait_job(job_id: str) -> List[Dict[str, Any]]: + def wait_job(job_id: str) -> list[dict[str, Any]]: message = CurrentMessage.get_current_message() now = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") job_started.send(job_id, message.message_id, now) diff --git a/tests/uws/policy_test.py b/tests/uws/policy_test.py index 01e53e4..cdb5878 100644 --- a/tests/uws/policy_test.py +++ b/tests/uws/policy_test.py @@ -3,7 +3,6 @@ from __future__ import annotations from datetime import datetime, timedelta, timezone -from typing import List import pytest from httpx import AsyncClient @@ -34,7 +33,7 @@ def validate_execution_duration( else: return execution_duration - def validate_params(self, params: List[JobParameter]) -> None: + def validate_params(self, params: list[JobParameter]) -> None: for param in params: if param.parameter_id != "id": msg = f"Invalid parameter f{param.parameter_id}"