reformar env vars picking and other improvements

config/README.md

@@ -4,7 +4,7 @@ This folder contains environment files. One environment per boat.
 
 _Boat specific config files are not checked into git._ This is a precautionary procedure to prevent overwriting configs in the production environment. Boat specific config filenames are listed in `.gitignore`.
 
-# Using Configs
+## Using Configs
 
 Create a new config file. It is a good idea to copy the contents from `defaults.py`.
 
@@ -34,7 +34,7 @@ def main(dbname, dbuser):
     pass
 ```
 
-# Boat specific examples
+## Boat specific examples
 
 ### brancol.py
 

config/defaults.py

@@ -1,11 +1,13 @@
-SECRET_KEY = "not_so_secret"
-DEBUG = True
-DBUSER = "edge"
-DBNAME = "edge"
-THALOS_VIDEO_DIR = "/thalos/videos"
-THALOS_CAM_NAME = "cam1"
-VIDEO_OUTPUT_DIR = "/videos"
-VIDEO_PASSPHRASE_FILE = "/dev/null"
-THALOS_VIDEO_SUFFIX = ".avi.done"
-BOAT_NAME = ""
-DB_TABLES_VERSION = "v1"
+import os
+
+SECRET_KEY = os.getenv("SECRET_KEY", "")
+DEBUG = os.getenv("DEBUG", "true")
+DBUSER = os.getenv("DBUSER", "edge")
+DBNAME = os.getenv("DBNAME", "edge")
+THALOS_VIDEO_DIR = os.getenv("THALOS_VIDEO_DIR", "/thalos/videos")
+THALOS_CAM_NAME = os.getenv("THALOS_CAM_NAME", "cam1")
+VIDEO_OUTPUT_DIR = os.getenv("VIDEO_OUTPUT_DIR", "/videos")
+VIDEO_PASSPHRASE_FILE = os.getenv("VIDEO_PASSPHRASE_FILE", "/dev/null")
+THALOS_VIDEO_SUFFIX = os.getenv("THALOS_VIDEO_SUFFIX", ".avi.done")
+BOAT_NAME = os.getenv("BOAT_NAME", "")
+DB_TABLES_VERSION = os.getenv("DB_TABLES_VERSION", "v1")

dbdumps/reset_schema.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
 
-SCRIPTNAME="$0"
 SCRIPTDIR="$(dirname -- "$(readlink -f -- "$0")")"
 
 DBNAME=edge
@@ -21,7 +20,10 @@ while (("$#")); do
    shift
 done
 
-cd "$SCRIPTDIR/.."
+cd "$SCRIPTDIR/.." || {
+   echo "Failed to cd to project root"
+   return
+}
 
 if [ "$VIRTUAL_ENV" != "$(pwd)/venv" ]; then
    if [ "x$VIRTUAL_ENV" != "x" ]; then

dbdumps/seed_db.py

@@ -44,9 +44,11 @@ def cli(cleardb, dbname, dbuser, force):
         sys.exit(1)
 
     # engine = create_engine("sqlite:///db.db", echo=True)
-    engine = create_engine("postgresql+psycopg2://%s@/%s" % (dbuser, dbname), echo=True)
+    engine = create_engine(f"postgresql+psycopg2://{dbuser}@/{dbname}", echo=True)
+
     SessionMaker = sessionmaker(engine)
     session = SessionMaker()
+
     if cleardb:
         clear_db(session)
         Base.metadata.drop_all(engine)

edge_http.py

@@ -6,7 +6,19 @@
 from flask_admin.contrib.sqla import ModelView
 from sqlalchemy import text
 
+from api import deckhand
 from db import db
+from model import (
+    AifishData,
+    BoatSchedule,
+    DeckhandEventRaw,
+    DeckhandEventView,
+    GpsData,
+    InternetDataView,
+    OndeckData,
+    RiskVectorModelView,
+    TestModelView,
+)
 
 app = Flask(__name__)
 
@@ -17,7 +29,7 @@
 # Set optional bootswatch theme
 app.config["FLASK_ADMIN_SWATCH"] = "cerulean"
 
-app.config["SQLALCHEMY_DATABASE_URI"] = "postgresql+psycopg2://%s@/%s" % (
+app.config["SQLALCHEMY_DATABASE_URI"] = "postgresql+psycopg2://{}@/{}".format(
     app.config["DBUSER"],
     app.config["DBNAME"],
 )
@@ -26,7 +38,6 @@
 # SessionMaker = scoped_session(sessionmaker(bind=engine))
 db.init_app(app)
 
-from model import *
 
 with app.app_context():
     # Base.metadata.create_all(engine)
@@ -57,14 +68,11 @@
         connection.execute(text("SELECT setval('vectors_id_seq', (SELECT MAX(id) FROM vectors));"))
 
 
-from api import deckhand
-
 app.register_blueprint(deckhand, url_prefix="/deckhand")
 
 
 admin = Admin(app, name="Risk Assesment", template_mode="bootstrap3")
 
-
 # work with session
 admin.add_view(RiskVectorModelView(db.session))
 admin.add_view(TestModelView(db.session))
@@ -74,13 +82,12 @@
 admin.add_view(InternetDataView(db.session))
 admin.add_view(ModelView(DeckhandEventRaw, db.session))
 admin.add_view(ModelView(DeckhandEventView, db.session))
-
 admin.add_view(ModelView(BoatSchedule, db.session))
 
 
 @click.command()
 @click.option("--port", default=50000)
-def serve(port):
+def serve(port: int | str) -> None:
     app.run(host="0.0.0.0", port=port)
 
 

migrations/README.md

@@ -2,15 +2,15 @@
 
 This folder is reserved for Alembic database migration scripts and configs.
 
-# Using Alembic
+## Using Alembic
 
 Alembic is installed as one of the pip requirements.txt packages. Using a venv is suggested. Exporting the `ENVIRONMENT` config is required.
 
-The `alembic` cli command provides a suite of tools to manage database migrations. 
+The `alembic` cli command provides a suite of tools to manage database migrations.
 
 Alembic migration scripts can be run forwards or backwards with `upgrade` and `downgrade`
 
-```
+```bash
 (venv) $ export ENVIRONMENT="config/queen_mary.py"
 (venv) $ alembic downgrade ba08d4e11cc7
 INFO  [alembic.runtime.migration] Context impl PostgresqlImpl.
@@ -22,13 +22,12 @@ INFO  [alembic.runtime.migration] Context impl PostgresqlImpl.
 INFO  [alembic.runtime.migration] Will assume transactional DDL.
 INFO  [alembic.runtime.migration] Running upgrade ba08d4e11cc7 -> fdfd9e708602, add_elog_timegap_vector_row
 INFO  [alembic.runtime.migration] Running upgrade fdfd9e708602 -> e718ddd7c0bd, add_track_table
-(venv) $ 
+(venv) $
 ```
 
 Alembic provides a tool that auto-generates new migration scripts from detected differences between the db schema and the python db model classes.
 
-
-```
+```bash
 (venv) $ export ENVIRONMENT="config/queen_mary.py"
 (venv) $ alembic revision --autogenerate -m new_migration_filename
 ```

migrations/env.py

@@ -36,7 +36,7 @@
 if "ENVIRONMENT" in os.environ:
     flaskconfig.from_envvar("ENVIRONMENT")
 
-url = "postgresql+psycopg2://%s@/%s" % (flaskconfig["DBUSER"], flaskconfig["DBNAME"])
+url = "postgresql+psycopg2://{}@/{}".format(flaskconfig["DBUSER"], flaskconfig["DBNAME"])
 
 
 def run_migrations_offline() -> None:

pyproject.toml

@@ -198,6 +198,7 @@ ignore = [
     "PLR0913",   # too many arguments in function definition
     "S311",   # Standard pseudo-random generators are not suitable for cryptographic purposes
     "E501",   # Line too long
+    "ERA001", # Found commented-out code
 ]
 
 # Exclude a variety of commonly ignored directories.

reencode.py

@@ -135,7 +135,7 @@ def run_reencode(output_dir: Path, sessionmaker: SessionMaker):
 def main(dbname, dbuser, output_dir, print_queue):
     output_dir = Path(output_dir)
 
-    sa_engine = sa.create_engine("postgresql+psycopg2://%s@/%s" % (dbuser, dbname), echo=True)
+    sa_engine = sa.create_engine(f"postgresql+psycopg2://{dbuser}@/{dbname}", echo=True)
     sessionmaker = SessionMaker(sa_engine)
 
     ModelBase.metadata.create_all(sa_engine)

run_aifish.py

@@ -469,7 +469,7 @@ def main(
     if engine:
         engine = Path(engine)
 
-    sa_engine = sa.create_engine("postgresql+psycopg2://%s@/%s" % (dbuser, dbname), echo=True)
+    sa_engine = sa.create_engine(f"postgresql+psycopg2://{dbuser}@/{dbname}", echo=True)
     sessionmaker = SessionMaker(sa_engine)
 
     ModelBase.metadata.create_all(sa_engine)

run_ondeck.py

@@ -501,7 +501,7 @@ def main(
     if engine:
         engine = Path(engine)
 
-    sa_engine = sa.create_engine("postgresql+psycopg2://%s@/%s" % (dbuser, dbname), echo=True)
+    sa_engine = sa.create_engine(f"postgresql+psycopg2://{dbuser}@/{dbname}", echo=True)
     sessionmaker = SessionMaker(sa_engine)
 
     ModelBase.metadata.create_all(sa_engine)

tests/ondeck_json_to_tracks.py

@@ -20,7 +20,7 @@
 @click.option("--dbname", default=flaskconfig.get("DBNAME"))
 @click.option("--dbuser", default=flaskconfig.get("DBUSER"))
 def main(ctx, dbname, dbuser):
-    sa_engine = sa.create_engine("postgresql+psycopg2://%s@/%s" % (dbuser, dbname), echo=True)
+    sa_engine = sa.create_engine(f"postgresql+psycopg2://{dbuser}@/{dbname}", echo=True)
     sessionmaker = SessionMaker(sa_engine)
 
     Base.metadata.create_all(sa_engine)

vector/catchcountA.py

@@ -256,7 +256,7 @@ def main(dbname, dbuser):
         import sqlalchemy as sa
         from sqlalchemy.orm import sessionmaker as SessionMaker
 
-        sa_engine = sa.create_engine("postgresql+psycopg2://%s@/%s" % (dbuser, dbname), echo=True)
+        sa_engine = sa.create_engine(f"postgresql+psycopg2://{dbuser}@/{dbname}", echo=True)
         sessionmaker = SessionMaker(sa_engine)
 
         Base.metadata.create_all(sa_engine)

vector/thalos_mount.py

@@ -70,7 +70,7 @@ def main(dbname, dbuser):
         import sqlalchemy as sa
         from sqlalchemy.orm import sessionmaker as SessionMaker
 
-        sa_engine = sa.create_engine("postgresql+psycopg2://%s@/%s" % (dbuser, dbname), echo=True)
+        sa_engine = sa.create_engine(f"postgresql+psycopg2://{dbuser}@/{dbname}", echo=True)
         sessionmaker = SessionMaker(sa_engine)
 
         Base.metadata.create_all(sa_engine)

vector/thalos_vids_exist.py

@@ -119,7 +119,7 @@ def main(dbname, dbuser):
         import sqlalchemy as sa
         from sqlalchemy.orm import sessionmaker as SessionMaker
 
-        sa_engine = sa.create_engine("postgresql+psycopg2://%s@/%s" % (dbuser, dbname), echo=True)
+        sa_engine = sa.create_engine(f"postgresql+psycopg2://{dbuser}@/{dbname}", echo=True)
         sessionmaker = SessionMaker(sa_engine)
 
         Base.metadata.create_all(sa_engine)

vector_schedule.py

@@ -55,7 +55,7 @@ def main(dbname, dbuser):
     # engine = create_engine("sqlite:///db.db", echo=True)
     # print(os.environ, dbuser, dbname)
 
-    engine = create_engine("postgresql+psycopg2://%s@/%s" % (dbuser, dbname), echo=True)
+    engine = create_engine(f"postgresql+psycopg2://{dbuser}@/{dbname}", echo=True)
     SessionMaker = sessionmaker(engine)
 
     ModelBase.metadata.create_all(engine)