Demo notebook

echodataflow/deployment/demo.ipynb

@@ -0,0 +1,81 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from echodataflow.deployment.deployment_engine import deploy_echodataflow\n",
+    "\n",
+    "config_file = 'C:/Users/soham/Desktop/Soham/Projects/OSO/temp/echodataflow/echodataflow/deployment/deployment_demo.yaml'\n",
+    "logging_yaml = 'C:/Users/soham/Desktop/Soham/Projects/OSO/temp/echodataflow/echodataflow/deployment/logging.yaml'\n",
+    "d = await deploy_echodataflow(deployment_yaml=config_file, logging_yaml=logging_yaml)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from echodataflow.deployment.service import edf_service\n",
+    "from echodataflow.models.deployment.stage import Stage\n",
+    "\n",
+    "data = [ { \"name\": \"Sv_flow\", \"module\": \"echodataflow.deployment.flow\", \"stage_params\": { \"prefect_config\": { \"retries\": 3, 'task_runner': 'DaskTaskRunner()'}, \"key\": \"value\", \"ky2\": \"value2\", \"options\": { \"save_offline\": True, \"use_offline\": True, \"group\": False } }, \"source\": { \"path\": \"s3://ncei-wcsd-archive/data/raw/Bell_M._Shimada/SH1707/EK60/Summer2017-D20170623-T153732.raw\", \"storage_options\": { \"anon\": True }, \"raw_regex\": \"(.*)-?D(?P<date>\\\\w{1,8})-T(?P<time>\\\\w{1,6})\" }, \"destination\": { \"path\": \"./processed\" }, \"data_quality\": { \"checks\": [ { \"name\": \"NULL_CHECK\" }, { \"name\": \"MIN_MAX_CHECK\" }, { \"name\": \"echodataflow.data_quality.checks.custom_check\" } ], \"block_downstream\": False, \"out_path\": \"s3://\", \"storage_options\": { \"block_type\": \"AWS\", \"block_name\": \"ABCD\", \"anon\": False } }, \"options\": {}, \"prefect_config\": {}, \"tasks\": [ { \"name\": \"edf_open_raw\", \"module\": \"echodataflow.tasklib.echopype\", \"task_params\": { \"sonar_model\": \"EK60\", \"xml_path\": \"s3//\" } }, { \"name\": \"edf_Sv\", \"module\": \"echodataflow.tasklib.echopype\", \"task_params\": {} } ] } ]\n",
+    "\n",
+    "stages = [Stage(**s) for s in data]\n",
+    "\n",
+    "edf_logger = '{ \"disable_existing_loggers\": false, \"formatters\": { \"json\": { \"format\": \"[%(asctime)s] %(process)d %(levelname)s%(mod_name)s:%(func_name)s:%(lineno)s - %(message)s\" }, \"plaintext\": { \"format\": \"[%(asctime)s] %(process)d %(levelname)s%(mod_name)s:%(func_name)s:%(lineno)s - %(message)s\" }, \"handlers\": { \"logfile\": { \"class\": \"logging.handlers.RotatingFileHandler\", \"formatter\": \"plaintext\", \"level\": \"DEBUG\", \"filename\": \"raw_to_Sv.log\", \"maxBytes\": 1000000, \"backupCount\": 3 } } }, \"loggers\": { \"echodataflow\": { \"level\": \"DEBUG\", \"propagate\": false, \"handlers\": [ \"logfile\" ] } } }'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stages"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "edf_service(stages=stages)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from echodataflow.tasklib.echopype import edf_open_raw"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "echodataflow_redesign",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}