add ruff formatting and minor improvements

api/README.md

@@ -4,7 +4,8 @@ This folder contains all HTTP-api code. The Flask App `edge_http.py` will import
 
 Relevant code in `edge_http.py`:
 
-```
+```python
 from api import deckhand
+
 app.register_blueprint(deckhand, url_prefix='/deckhand')
 ```

api/deckhand.py

@@ -1,21 +1,15 @@
 import json
-from http.client import BAD_REQUEST
 
-from flask import Blueprint, g, make_response, request
-from sqlalchemy.orm import scoped_session, sessionmaker
+from flask import Blueprint, Response, make_response, request
 
 from db import db
 from model import DeckhandEventRaw
 
 blueprint = Blueprint("DeckhandApi", __name__)
 
 
-# ORM Session
-# orm_session = scoped_session(sessionmaker())
-
-
 @blueprint.route("/", methods=["PUT", "POST"])
-def event():
+def event() -> Response:
     d = request.get_json()
 
     event = DeckhandEventRaw()

config/README.md

@@ -1,43 +1,44 @@
 # About config
 
-This folder contains environment files. One environment per boat. 
+This folder contains environment files. One environment per boat.
 
-_Boat specific config files are not checked into git._ This is a precautionary procedure to prevent overwriting configs in the production environment. Boat specific config filenames are listed in `.gitignore`. 
+_Boat specific config files are not checked into git._ This is a precautionary procedure to prevent overwriting configs in the production environment. Boat specific config filenames are listed in `.gitignore`.
 
 # Using Configs
 
 Create a new config file. It is a good idea to copy the contents from `defaults.py`.
 
 Export the filename (with the `config/` prefix) into an environmnent variable named `ENVIRONMENT`
 
-```
-$ export ENVIRONMENT='config/queen_mary.py'
+```bash
+export ENVIRONMENT='config/queen_mary.py'
 ```
 
 All code in this repository should pick up both the defaults and the the referenced `ENVIRONMENT` file for config values. Example python code:
 
-```
+```python
+import click
+
 from flask.config import Config as FlaskConfig
+
 flaskconfig = FlaskConfig(root_path='')
 
 flaskconfig.from_object('config.defaults')
 if 'ENVIRONMENT' in os.environ:
     flaskconfig.from_envvar('ENVIRONMENT')
 
-import click
-
 @click.command()
 @click.option('--dbname', default=flaskconfig.get('DBNAME'))
 @click.option('--dbuser', default=flaskconfig.get('DBUSER'))
 def main(dbname, dbuser):
     pass
 ```
 
-# Boat specific examples:
+# Boat specific examples
 
 ### brancol.py
 
-```
+```python
 DEBUG=False
 SECRET_KEY=''
 THALOS_VIDEO_DIR="/thalos/brancol/videos"
@@ -50,7 +51,7 @@ BOAT_NAME='brancol'
 
 ### stpatrick.py
 
-```
+```python
 DEBUG=False
 THALOS_VIDEO_DIR="/thalos/saintpatrick/videos"
 THALOS_CAM_NAME='cam1'

edge_http.py

@@ -27,7 +27,6 @@
 db.init_app(app)
 
 from model import *
-from vector import GpsVector
 
 with app.app_context():
     # Base.metadata.create_all(engine)

gps_fetch.py

@@ -1,7 +1,7 @@
 import os
 import re
 import time
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from pathlib import Path
 
 import click
@@ -19,7 +19,7 @@
 
 
 def thalos_gps_filename_date(filename: str) -> datetime:
-    m = re.match(".*(\d{8}).?(\d{6})\.txt", filename)
+    m = re.match(r".*(\d{8}).?(\d{6})\.txt", filename)
     if not m:
         return None
     return isoparse(m[1] + " " + m[2] + "+00:00")
@@ -30,7 +30,7 @@ def gps_fetch(cpool: SimpleConnectionPool, thalos_dir: Path):
     gps_files = [x for x in thalos_dir.iterdir()]
     dt_index = {}
     for gps_file in gps_files:
-        m = re.match(".*(\d{8}).?(\d{6})\.txt", gps_file.name)
+        m = re.match(r".*(\d{8}).?(\d{6})\.txt", gps_file.name)
         if not m:
             continue
         dt = datetime.strptime(m[1] + " " + m[2] + "Z", "%Y%m%d %H%M%S%z")
@@ -42,7 +42,7 @@ def gps_fetch(cpool: SimpleConnectionPool, thalos_dir: Path):
     if len(dt_index.keys()) > 0:
         try:
             with conn.cursor() as cur:
-                args = ",".join(cur.mogrify("(%s)", [dt]).decode("utf-8") for dt in dt_index.keys())
+                args = ",".join(cur.mogrify("(%s)", [dt]).decode("utf-8") for dt in dt_index)
                 cur.execute(
                     """WITH t (file_dt) AS ( VALUES """
                     + args
@@ -59,10 +59,10 @@ def gps_fetch(cpool: SimpleConnectionPool, thalos_dir: Path):
             insert_tuples = []
 
             for new_dt in new_dts:
-                new_file: Path = dt_index[new_dt.astimezone(timezone.utc)]
+                new_file: Path = dt_index[new_dt.astimezone(UTC)]
                 with new_file.open() as data:
                     line = data.readline()
-                    m = re.match("([+-]?(\d+(\.\d*)?|\.\d+)).*,.*?([+-]?(\d+(\.\d*)?|\.\d+))", line)
+                    m = re.match(r"([+-]?(\d+(\.\d*)?|\.\d+)).*,.*?([+-]?(\d+(\.\d*)?|\.\d+))", line)
                     if m:
                         lat = m[1]
                         lon = m[4]
@@ -75,7 +75,7 @@ def gps_fetch(cpool: SimpleConnectionPool, thalos_dir: Path):
                         )
 
             if len(insert_tuples) > 0:
-                click.echo("inserting {} new gps coords".format(len(insert_tuples)))
+                click.echo(f"inserting {len(insert_tuples)} new gps coords")
                 with conn.cursor() as cur:
                     cur.executemany(
                         "INSERT INTO gpsdata (gps_datetime, lat, lon) VALUES (%s, %s, %s);",
@@ -111,7 +111,7 @@ def runonce(cpool, thalos_gps_dir):
             break
         elif n > 0:
             # sleep exactly the right amount of time
-            click.echo("sleeping for: {}".format(n))
+            click.echo(f"sleeping for: {n}")
             time.sleep(n)
         schedule.run_pending()
 

migrations/versions/04eaff9bcc55_.py

@@ -5,8 +5,6 @@
 Create Date: 2023-04-10 12:56:26.377798
 
 """
-from alembic import op
-import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.

migrations/versions/17911f3ffb3b_new_vector_rows_1.py

@@ -5,9 +5,8 @@
 Create Date: 2023-06-02 14:22:38.910122
 
 """
-from alembic import op
 import sqlalchemy as sa
-
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = '17911f3ffb3b'
@@ -22,16 +21,16 @@ def upgrade() -> None:
     for row in op.get_bind().execute("select id, name from vectors where name = 'InternetVector';"):
         if row:
             found_id = row[0]
-    
+
     if found_id is None:
         op.get_bind().execute('insert into vectors (name, configblob) values (\'InternetVector\', \'{"target_ips":["8.8.8.8","1.1.1.1","208.67.222.222","9.9.9.9"],"run_traceroute":false}\');')
-    
-    
+
+
 
 
 def downgrade() -> None:
-    
-    op.get_bind().execute("delete from tests where vector_id = (select id from vectors where name = 'InternetVector');");
+
+    op.get_bind().execute("delete from tests where vector_id = (select id from vectors where name = 'InternetVector');")
 
     t = sa.table('vectors')
     op.get_bind().execute("delete from vectors where name = 'InternetVector';")

migrations/versions/1d1b10e054af_add_timezones.py

@@ -6,7 +6,6 @@
 
 """
 
-import sqlalchemy as sa
 from alembic import op
 
 # revision identifiers, used by Alembic.

migrations/versions/47ff3fca73a4_vector_schedulestring.py

@@ -5,9 +5,8 @@
 Create Date: 2023-06-06 13:07:09.704169
 
 """
-from alembic import op
 import sqlalchemy as sa
-
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = '47ff3fca73a4'

migrations/versions/495235ece5f0_ondeckdata_unique.py

@@ -6,8 +6,6 @@
 
 """
 from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision = '495235ece5f0'
@@ -29,7 +27,7 @@ def upgrade() -> None:
                and id not in (select id from duped)
                ;""")
     op.create_unique_constraint(None, 'ondeckdata', ['video_uri'])
-    
+
 
 
 def downgrade() -> None:

migrations/versions/58dd42108a22_new_vid_file_table.py

@@ -5,9 +5,8 @@
 Create Date: 2023-06-16 18:15:23.314916
 
 """
-from alembic import op
 import sqlalchemy as sa
-
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = '58dd42108a22'

migrations/versions/5e4898954923_ondeckdata_table.py

@@ -5,9 +5,8 @@
 Create Date: 2023-06-05 14:09:26.594081
 
 """
-from alembic import op
 import sqlalchemy as sa
-
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = '5e4898954923'

migrations/versions/5fdb864a1bbb_refactor_aifish.py

@@ -5,8 +5,8 @@
 Create Date: 2023-12-12 12:43:34.309532
 
 """
-from alembic import op
 import sqlalchemy as sa
+from alembic import op
 from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
@@ -17,7 +17,7 @@
 
 
 def upgrade() -> None:
-    
+
     op.create_table('aifishdata',
     sa.Column('id', sa.Integer(), nullable=False),
     sa.Column('video_uri', sa.String(), nullable=True),
@@ -33,7 +33,7 @@ def upgrade() -> None:
     sa.UniqueConstraint('video_uri')
     )
     op.drop_table('fishaidata')
-    
+
 
 
 def downgrade() -> None:

migrations/versions/643148911953_deckhand_json_views.py

@@ -6,8 +6,6 @@
 
 """
 from alembic import op
-import sqlalchemy as sa
-
 
 # revision identifiers, used by Alembic.
 revision = '643148911953'
@@ -50,7 +48,7 @@ def upgrade() -> None:
         WHERE a.jsonblob->>'eventType' = 'longlineEvent'
         GROUP BY a.id,  a.jsonblob, a.datetime, bycatchcount;
     """)
-    
+
 
 
 def downgrade() -> None:

migrations/versions/677a2f2884e1_s3uploadstable.py

@@ -5,9 +5,8 @@
 Create Date: 2023-08-02 17:08:34.590190
 
 """
-from alembic import op
 import sqlalchemy as sa
-
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = '677a2f2884e1'
@@ -18,7 +17,7 @@
 
 def upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
-    
+
     op.create_table('s3uploads',
     sa.Column('datetime', sa.DateTime(timezone=True), nullable=False),
     sa.Column('tablename', sa.String(), nullable=False),

migrations/versions/81b92a299311_gps_data_types.py

@@ -5,9 +5,8 @@
 Create Date: 2023-07-26 16:51:17.527649
 
 """
-from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = '81b92a299311'

migrations/versions/8304966281aa_reencode_files.py

@@ -5,9 +5,8 @@
 Create Date: 2023-09-20 15:15:56.043600
 
 """
-from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = '8304966281aa'

migrations/versions/97b633de0899_video_cam_name.py

@@ -5,9 +5,8 @@
 Create Date: 2023-07-27 15:50:13.450935
 
 """
-from alembic import op
 import sqlalchemy as sa
-
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = '97b633de0899'

migrations/versions/b2f76c38a4a0_deckhand_gaps_score.py

@@ -6,8 +6,6 @@
 
 """
 from alembic import op
-import sqlalchemy as sa
-
 
 # revision identifiers, used by Alembic.
 revision = 'b2f76c38a4a0'
@@ -68,5 +66,5 @@ def upgrade() -> None:
 
 
 def downgrade() -> None:
-    op.get_bind().execute('drop view elog_time_gap_score');
-    op.get_bind().execute('drop function elog_time_gap_sigmoid');
+    op.get_bind().execute('drop view elog_time_gap_score')
+    op.get_bind().execute('drop function elog_time_gap_sigmoid')