From 68a4a1a8218342b5e9a6bd3efc861a0e41a1cd74 Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 26 Nov 2024 18:31:28 +0100 Subject: [PATCH] add data loading flag for multiio data. add script to compute MOC based on errie NG5 data --- tools/eerie/template_eerie_transp_zmoc.ipynb | 2956 ++++++++++++++++++ tripyview/sub_data.py | 23 +- 2 files changed, 2978 insertions(+), 1 deletion(-) create mode 100644 tools/eerie/template_eerie_transp_zmoc.ipynb diff --git a/tools/eerie/template_eerie_transp_zmoc.ipynb b/tools/eerie/template_eerie_transp_zmoc.ipynb new file mode 100644 index 0000000..e694f37 --- /dev/null +++ b/tools/eerie/template_eerie_transp_zmoc.ipynb @@ -0,0 +1,2956 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/albedo/home/pscholz/tripyview/tripyview/sub_data.py:8: UserWarning: The seawater library is deprecated! Please use gsw instead.\n", + " import seawater as sw\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/albedo/home/pscholz/tripyview\n" + ] + } + ], + "source": [ + "%matplotlib inline\n", + "%load_ext autoreload\n", + "%autoreload 2\n", + "\n", + "#___________________________________________________________________________________________________________________\n", + "import os\n", + "import tripyview as tpv\n", + "import numpy as np\n", + "import xarray as xr\n", + "import time as clock\n", + "import shapefile as shp\n", + "import datetime \n", + "import warnings\n", + "xr.set_options(keep_attrs=True)\n", + "client_runs = False" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "# parameters\n", + "#___Dask Client Parameters____________________________________________________________\n", + "do_papermill = False\n", + "do_parallel = True\n", + "parallel_tnprc = 128 # total number of available CPUs\n", + "parallel_nprc = 48 # number of dask workers\n", + "parallel_nprc_bin = parallel_tnprc-parallel_nprc # number of processor used to parallize the binning loop\n", + "parallel_tmem = 256 # max. available RAM\n", + "\n", + "#___Mesh Path & Save Path_____________________________________________________________\n", + "# mesh_path ='/work/ollie/projects/clidyn/FESOM2/meshes/core2/'\n", + "mesh_path = '/albedo/work/user/pscholz/data_errie_rohit/mesh/NG5/'\n", + "save_path = '/albedo/work/user/pscholz/data_errie_rohit/' # None #'~/figures/test_papermill/'\n", + "save_fname = None # filename from papermill come in through save_fname\n", + "tripyrun_name = None # papermill workflow name of notebook \n", + "tripyrun_analysis = None # papermill diagnostic driver\n", + "tripyrun_spath_nb = None # papermill path to processed notebooks\n", + "tripyrun_spath_fig= None # papermill path to processed figures\n", + "\n", + "#_____________________________________________________________________________________\n", + "# set datapath\n", + "input_paths= list()\n", + "input_paths.append('/albedo/work/user/pscholz/data_errie_rohit/')\n", + "\n", + "input_names = list()\n", + "input_names.append('Phase1-CTL-FESOM')\n", + "\n", + "#_____________________________________________________________________________________\n", + "# set run infos\n", + "vname = 'amoc' # gmoc, amoc, ipmoc, imoc, pmoc\n", + "year = [1950, 1987]\n", + "\n", + "mon = None\n", + "day = None\n", + "record = None \n", + "box = None\n", + "depth = None\n", + "\n", + "#___Define Colormap Parameters________________________________________________________\n", + "# papermill doesnt like multi variable alignment in a single line\n", + "cstr = 'blue2red'\n", + "cnum = 15\n", + "cref = 0\n", + "crange = None\n", + "cmin = None\n", + "cmax = None\n", + "cfac = None\n", + "climit = None\n", + "chist = True\n", + "ctresh = 0.995\n", + "\n", + "ref_cstr = 'wbgyr'\n", + "ref_cnum = 15\n", + "ref_cref = 0\n", + "ref_crange= None\n", + "ref_cmin = None\n", + "ref_cmax = None\n", + "ref_cfac = None\n", + "ref_climit= None\n", + "ref_chist = True\n", + "ref_ctresh= 0.995\n", + "\n", + "#_____________________________________________________________________________________\n", + "ncol = 2 # number of pannel columns in figure\n", + "nrow = None\n", + "do_plt = 'tcf' # plot pcolor (tpc) or contourf (tcf)\n", + "plt_contb = True # background contour line (thin)\n", + "plt_contf = True # contour line of main colorbar steps \n", + "plt_contr = False # contour line of reference value \n", + "plt_contl = False # label contourline of main colorbar steps \n", + "do_rescale = None # rescale data: None, 'log10', 'slog10', np.array(...)\n", + "do_lsm ='fesom' # 'fesom', 'bluemarble', 'etopo', 'stock'\n", + "do_mesh, mesh_opt = False, dict({'color':'k', 'linewidth':0.10})\n", + "do_enum = False # do enumeration of panels\n", + "do_reffig = True # plot reference fig when doing anomalies \n", + "# do_clim = False # plot climatolgy values when doing absoluts\n", + "ax_title = None\n", + "cb_label = None\n", + "save_dpi = 300\n", + "save_fmt = ['png']\n", + "do_bolus = True\n", + "do_netcdf = True\n", + "do_info = True" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " --> memory_limit: 3.733 GB\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "
\n", + "
\n", + "

Client

\n", + "

Client-4bacb0c8-ac1b-11ef-9488-3cecefaf0fac

\n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "
Connection method: Cluster objectCluster type: distributed.LocalCluster
\n", + " Dashboard: http://127.0.0.1:8787/status\n", + "
\n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "

Cluster Info

\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

LocalCluster

\n", + "

c29c11e1

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "\n", + " \n", + "
\n", + " Dashboard: http://127.0.0.1:8787/status\n", + " \n", + " Workers: 48\n", + "
\n", + " Total threads: 48\n", + " \n", + " Total memory: 166.88 GiB\n", + "
Status: runningUsing processes: True
\n", + "\n", + "
\n", + " \n", + "

Scheduler Info

\n", + " \n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

Scheduler

\n", + "

Scheduler-ffd5c6eb-c90f-4ff5-802c-61ae0b474e82

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " Comm: tcp://127.0.0.1:46439\n", + " \n", + " Workers: 48\n", + "
\n", + " Dashboard: http://127.0.0.1:8787/status\n", + " \n", + " Total threads: 48\n", + "
\n", + " Started: Just now\n", + " \n", + " Total memory: 166.88 GiB\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "

Workers

\n", + " \n", + "\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 0

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:34647\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:45661/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:33707\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-fxjot1_3\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 1

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:35825\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:37077/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:34707\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-p6xj6lwg\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 2

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:33533\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:44729/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:43255\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-fcien9j3\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 3

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:40061\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:36171/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:36689\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-58uup_1e\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 4

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:40105\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:41805/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:46461\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-d5i619bt\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 5

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:45487\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:37375/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:45557\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-0n02wt29\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 6

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:43869\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:32821/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:42125\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-w7hsp__i\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 7

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:43701\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:39137/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:46571\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-fmtk7xxh\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 8

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:43795\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:46705/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:41487\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-w0dacbno\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 9

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:32965\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:37329/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:45919\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-9z0bfesv\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 10

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:37253\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:37459/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:37565\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-92ukzfwd\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 11

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:40369\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:38491/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:34095\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-q_pf16v2\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 12

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:33757\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:34445/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:39283\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-ke5e9exw\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 13

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:32981\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:39647/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:46709\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-uafx6qjn\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 14

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:43917\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:34343/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:45761\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-yozud2g2\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 15

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:34263\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:45395/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:45651\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-0sm6_28f\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 16

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:34885\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:37221/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:35783\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-ho1qzszt\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 17

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:41917\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:45749/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:34771\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-bz67a3jq\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 18

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:41457\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:42479/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:35023\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-k7sbc4cs\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 19

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:37043\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:38549/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:33097\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-bom8mcyz\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 20

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:34559\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:35591/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:45267\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-36yq6b0b\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 21

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:45507\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:43877/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:35899\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-df8pnsiu\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 22

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:33875\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:35463/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:35901\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-iy_hevau\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 23

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:45763\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:37983/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:42419\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-pa6o8px9\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 24

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:45431\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:33521/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:39929\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-24uzc65n\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 25

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:44751\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:38129/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:33249\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-8fxlnlrt\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 26

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:45333\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:32841/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:36063\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-gzsbe7lq\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 27

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:38511\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:41893/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:43109\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-3x0812p9\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 28

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:46421\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:45847/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:33563\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-e2yb3m3l\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 29

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:40945\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:38615/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:43507\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-ysbdrsw9\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 30

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:45227\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:35617/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:43131\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-nbekd3zl\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 31

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:45235\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:37207/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:36881\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-x28cz8_i\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 32

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:37501\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:40659/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:37765\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-3q2gosiv\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 33

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:40079\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:39811/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:35845\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-znskwfeg\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 34

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:44249\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:45119/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:42109\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-psdbrt9p\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 35

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:45367\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:36137/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:42891\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-qmhwr8js\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 36

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:34801\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:37465/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:34819\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-7lv2inm9\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 37

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:36209\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:37397/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:33259\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-kewtv10z\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 38

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:36989\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:45841/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:44667\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-jgp0eizb\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 39

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:36107\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:35453/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:42745\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-0g90zcgs\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 40

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:35903\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:39793/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:41599\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-b8b59hd_\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 41

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:44915\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:37771/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:38833\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-821anch_\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 42

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:36799\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:43867/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:33959\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-u7ju4lvp\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 43

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:38293\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:35055/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:41589\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-ns5ekvi_\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 44

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:34509\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:39425/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:45379\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-nb_ftc22\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 45

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:33721\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:44829/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:45729\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-3io5jwxw\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 46

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:43575\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:44967/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:33437\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-lbbbhn7d\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 47

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:42463\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:36167/status\n", + " \n", + " Memory: 3.48 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:37757\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-pt_kvee3\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "\n", + "
\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# start parallel dask client\n", + "safety = 0.3\n", + "if do_parallel and not client_runs:\n", + " from dask.distributed import Client\n", + " \n", + " #import dask\n", + " #dask.config.config.get('distributed').get('dashboard').update({'link':'{JUPYTERHUB_SERVICE_PREFIX}/proxy/{port}/status'})\n", + " print(' --> memory_limit: {:3.3f} GB'.format(parallel_tmem*(1.0-safety)/(parallel_nprc)))\n", + " client = Client(n_workers=parallel_nprc, threads_per_worker=1, memory_limit='{:3.3f} GB'.format(parallel_tmem*(1.0-safety)/parallel_nprc))\n", + " client_runs = True\n", + " display(client)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## > Load mesh:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " > warning: pickle5 module could not be found, no do_pickle \n", + " is possible! Therefor switch to joblib saving/loading\n", + " > found *.jlib file: /albedo/work/user/pscholz/data_errie_rohit/mesh/NG5\n", + " > load *.jlib file: tripyview_fesom2_NG5_focus0.jlib\n", + "___FESOM2 MESH INFO________________________\n", + " > path = /albedo/work/user/pscholz/rohit_errie/mesh/NG5\n", + " > id = NG5\n", + " > do rot = None\n", + " > [al,be,ga] = 50, 15, -90\n", + " > do augmpbnd = True\n", + " > do cavity = False\n", + " > do lsmask = True\n", + " > do earea,eresol = True, False\n", + " > do narea,nresol = True, False\n", + "___________________________________________\n", + " > #node = 7402886\n", + " > #elem = 14741520\n", + " > #lvls = 70\n", + "___________________________________________\n" + ] + } + ], + "source": [ + "#___LOAD FESOM2 MESH___________________________________________________________________________________\n", + "mesh=tpv.load_mesh_fesom2(mesh_path, do_rot='None', focus=0, do_info=True, do_pickle=True)\n", + "\n", + "#______________________________________________________________________________________________________ \n", + "cinfo=tpv.set_cinfo(cstr, cnum, crange, cmin, cmax, cref, cfac, climit, chist, ctresh)\n", + "\n", + "#______________________________________________________________________________________________________ \n", + "if save_path is not None:\n", + " if not os.path.isdir(save_path): os.makedirs(save_path)\n", + "\n", + "#______________________________________________________________________________________________________\n", + "chunks ={'time' :'auto', \n", + " 'nod2' :'auto', \n", + " 'elem' :'auto', \n", + " 'ncells' :'auto', \n", + " 'ntriags':'auto', } " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## > Compute: MOC vs. time\n", + "- compute annual mean MOC from monthly mean FESOM1.4 AWI-CM-1-1-HR CMIP6 vertical velocities (var:wo)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/albedo/work/user/pscholz/data_errie_rohit/\n", + " --> open file:\n", + " > fesom_avg_wo_195001_monthly-NG5.nc\n", + " > fesom_avg_wo_195002_monthly-NG5.nc\n", + " > fesom_avg_wo_195003_monthly-NG5.nc\n", + " > fesom_avg_wo_195004_monthly-NG5.nc\n", + " > fesom_avg_wo_195005_monthly-NG5.nc\n", + " > fesom_avg_wo_195006_monthly-NG5.nc\n", + " > fesom_avg_wo_195007_monthly-NG5.nc\n", + " > fesom_avg_wo_195008_monthly-NG5.nc\n", + " > fesom_avg_wo_195009_monthly-NG5.nc\n", + " > fesom_avg_wo_195010_monthly-NG5.nc\n", + " > fesom_avg_wo_195011_monthly-NG5.nc\n", + " > fesom_avg_wo_195012_monthly-NG5.nc\n", + " --> compute: amoc\n", + "_____calc. AMOC from vertical velocities via meridional bins_____\n", + "\n", + " ___parallel loop over longitudinal bins____\n", + " --> do cumsum over latitudes\n", + " --> total time:17.283 s\n", + " --> elapsed time/file: 57.01978826522827 sec\n", + "\n", + "\n", + " --> open file:\n", + " > fesom_avg_wo_195101_monthly-NG5.nc\n", + " > fesom_avg_wo_195102_monthly-NG5.nc\n", + " > fesom_avg_wo_195103_monthly-NG5.nc\n", + " > fesom_avg_wo_195104_monthly-NG5.nc\n", + " > fesom_avg_wo_195105_monthly-NG5.nc\n", + " > fesom_avg_wo_195106_monthly-NG5.nc\n", + " > fesom_avg_wo_195107_monthly-NG5.nc\n", + " > fesom_avg_wo_195108_monthly-NG5.nc\n", + " > fesom_avg_wo_195109_monthly-NG5.nc\n", + " > fesom_avg_wo_195110_monthly-NG5.nc\n", + " > fesom_avg_wo_195111_monthly-NG5.nc\n", + " > fesom_avg_wo_195112_monthly-NG5.nc\n", + " --> compute: amoc\n", + "_____calc. AMOC from vertical velocities via meridional bins_____\n", + "\n", + " ___parallel loop over longitudinal bins____\n", + " --> do cumsum over latitudes\n", + " --> total time:13.641 s\n", + " --> elapsed time/file: 52.19856405258179 sec\n", + "\n", + "\n", + " --> open file:\n", + " > fesom_avg_wo_195201_monthly-NG5.nc\n", + " > fesom_avg_wo_195202_monthly-NG5.nc\n", + " > fesom_avg_wo_195203_monthly-NG5.nc\n", + " > fesom_avg_wo_195204_monthly-NG5.nc\n", + " > fesom_avg_wo_195205_monthly-NG5.nc\n", + " > fesom_avg_wo_195206_monthly-NG5.nc\n", + " > fesom_avg_wo_195207_monthly-NG5.nc\n", + " > fesom_avg_wo_195208_monthly-NG5.nc\n", + " > fesom_avg_wo_195209_monthly-NG5.nc\n", + " > fesom_avg_wo_195210_monthly-NG5.nc\n", + " > fesom_avg_wo_195211_monthly-NG5.nc\n", + " > fesom_avg_wo_195212_monthly-NG5.nc\n", + " --> compute: amoc\n", + "_____calc. AMOC from vertical velocities via meridional bins_____\n", + "\n", + " ___parallel loop over longitudinal bins____\n", + " --> do cumsum over latitudes\n", + " --> total time:14.224 s\n", + " --> elapsed time/file: 52.965959787368774 sec\n", + "\n", + "\n", + " --> open file:\n", + " > fesom_avg_wo_195301_monthly-NG5.nc\n", + " > fesom_avg_wo_195302_monthly-NG5.nc\n", + " > fesom_avg_wo_195303_monthly-NG5.nc\n", + " > fesom_avg_wo_195304_monthly-NG5.nc\n", + " > fesom_avg_wo_195305_monthly-NG5.nc\n", + " > fesom_avg_wo_195306_monthly-NG5.nc\n", + " > fesom_avg_wo_195307_monthly-NG5.nc\n", + " > fesom_avg_wo_195308_monthly-NG5.nc\n", + " > fesom_avg_wo_195309_monthly-NG5.nc\n", + " > fesom_avg_wo_195310_monthly-NG5.nc\n", + " > fesom_avg_wo_195311_monthly-NG5.nc\n", + " > fesom_avg_wo_195312_monthly-NG5.nc\n", + " --> compute: amoc\n", + "_____calc. AMOC from vertical velocities via meridional bins_____\n", + "\n", + " ___parallel loop over longitudinal bins____\n", + " --> do cumsum over latitudes\n", + " --> total time:13.891 s\n", + " --> elapsed time/file: 52.406697511672974 sec\n", + "\n", + "\n", + " --> open file:\n", + " > fesom_avg_wo_195401_monthly-NG5.nc\n", + " > fesom_avg_wo_195402_monthly-NG5.nc\n", + " > fesom_avg_wo_195403_monthly-NG5.nc\n", + " > fesom_avg_wo_195404_monthly-NG5.nc\n", + " > fesom_avg_wo_195405_monthly-NG5.nc\n", + " > fesom_avg_wo_195406_monthly-NG5.nc\n", + " > fesom_avg_wo_195407_monthly-NG5.nc\n", + " > fesom_avg_wo_195408_monthly-NG5.nc\n", + " > fesom_avg_wo_195409_monthly-NG5.nc\n", + " > fesom_avg_wo_195410_monthly-NG5.nc\n", + " > fesom_avg_wo_195411_monthly-NG5.nc\n", + " > fesom_avg_wo_195412_monthly-NG5.nc\n", + " --> compute: amoc\n", + "_____calc. AMOC from vertical velocities via meridional bins_____\n", + "\n", + " ___parallel loop over longitudinal bins____\n", + " --> do cumsum over latitudes\n", + " --> total time:14.377 s\n", + " --> elapsed time/file: 52.8844530582428 sec\n", + "\n", + "\n", + " --> open file:\n", + " > fesom_avg_wo_195501_monthly-NG5.nc\n", + " > fesom_avg_wo_195502_monthly-NG5.nc\n", + " > fesom_avg_wo_195503_monthly-NG5.nc\n", + " > fesom_avg_wo_195504_monthly-NG5.nc\n", + " > fesom_avg_wo_195505_monthly-NG5.nc\n", + " > fesom_avg_wo_195506_monthly-NG5.nc\n", + " > fesom_avg_wo_195507_monthly-NG5.nc\n", + " > fesom_avg_wo_195508_monthly-NG5.nc\n", + " > fesom_avg_wo_195509_monthly-NG5.nc\n", + " > fesom_avg_wo_195510_monthly-NG5.nc\n", + " > fesom_avg_wo_195511_monthly-NG5.nc\n", + " > fesom_avg_wo_195512_monthly-NG5.nc\n" + ] + } + ], + "source": [ + "#______________________________________________________________________________________________________\n", + "ts = clock.time()\n", + "zmoc_t = xr.Dataset()\n", + "for jj, (datapath, descript) in enumerate(zip(input_paths, input_names)): \n", + " print(datapath)\n", + "\n", + " #__________________________________________________________________________________________________\n", + " # loop over year to open 10-year bundled files \n", + " t1s = clock.time()\n", + " for yr in range(year[0], year[1]+1):\n", + " #______________________________________________________________________________________________\n", + " # create rohit filename path\n", + " print(' --> open file:')\n", + " filepath = list()\n", + " for auxmon in range(1,12+1):\n", + " fname = 'fesom_avg_wo_{:4d}{:02d}_monthly-NG5.nc'.format(yr, auxmon)\n", + " print(' > ', fname)\n", + " filepath.append(os.path.join(datapath+'/{:4d}/native/monthly/'.format(yr), fname))\n", + " del auxmon \n", + " \n", + " #______________________________________________________________________________________________\n", + " # load cmip6 filename path\n", + " data = tpv.load_data_fesom2(mesh, filepath, do_filename=True, do_multiio=True, do_tarithm=None, do_zarithm=None,\n", + " vname='wo', year=year, mon=None, day=None, record=None, chunks=chunks,\n", + " depth=depth, descript=descript, do_info=False, do_parallel=do_parallel)\n", + " data = data.rename({list(data.data_vars)[0]:'w'})\n", + " \n", + " #______________________________________________________________________________________________\n", + " # compute time mean --> data are monthly mean, AMOC usually computed on annual mean\n", + " # --> annual means can be computed with do_arithm='mean'\n", + " \n", + " #______________________________________________________________________________________________\n", + " # compute zmoc \n", + " if do_info: print(' --> compute: ', vname)\n", + " zmoc = tpv.calc_zmoc(mesh, data, dlat=0.5, which_moc=vname, do_info=do_info, do_checkbasin=False, do_onelem=False, diagpath=False, \n", + " do_load=True, do_parallel=do_parallel, n_workers=parallel_nprc_bin)\n", + " \n", + " del(data)\n", + " \n", + " #______________________________________________________________________________________________\n", + " # accumulate to zmoc over time\n", + " zmoc_t = xr.merge([zmoc_t, zmoc], combine_attrs='drop_conflicts') #no_conflicts\n", + " del(zmoc)\n", + " \n", + " #______________________________________________________________________________________________\n", + " t1e = clock.time()\n", + " print(' --> elapsed time/file:', t1e-t1s, ' sec')\n", + " t1s = t1e\n", + " print('\\n')\n", + " \n", + " #__________________________________________________________________________________________________\n", + " # save to netcdf\n", + " if do_netcdf and save_path is not None:\n", + " auxtime = zmoc_t['time.year'].values\n", + " auxrun = descript.replace(' ','_').lower()\n", + " sfname = f'{vname}_vs_t_{auxrun}_y{auxtime[0]}-{auxtime[-1]}.nc'\n", + " # sfilepath = os.path.join(datapath, sfname)\n", + " sfilepath = os.path.join(save_path, sfname)\n", + " print(' --> save zmoc_t to file:', sfname)\n", + " zmoc_t.to_netcdf(path=sfilepath, mode='w')\n", + " \n", + "print(' --> tot. elapsed time:', clock.time()-ts, ' sec')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## > Plot: mean MOC " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#__________________________________________________________________________________________________\n", + "# do save filename path\n", + "spath = save_path\n", + "svname = vname\n", + "sfpath = None\n", + "which_run = descript.replace(' ','_').lower()\n", + "auxtime = zmoc_t['time.year'].values\n", + "slabel = f'y{auxtime[0]}-{auxtime[-1]}'\n", + "nrow0, ncol0= 1,1\n", + "\n", + "if spath!=None: \n", + " sfpath=list()\n", + " for sfmt in save_fmt: sfpath.append( os.path.join(spath,'{:s}_{:s}_{:s}.{:s}'.format(svname, which_run ,slabel, sfmt)) )\n", + " \n", + "#__________________________________________________________________________________________________\n", + "# do colorbar either single cbar or ref_cbar + anom_cbar\n", + "cb_plt, cb_plt_single, cinfo0 = True, True, cinfo.copy() \n", + " \n", + "#__________________________________________________________________________________________________ \n", + "hfig, hax, hcb = tpv.plot_vslice(mesh, [zmoc_t.mean(dim='time', keep_attrs=True)], nrow=nrow0, ncol=ncol0, cinfo=cinfo0, \n", + " do_plt=do_plt, plt_contb=plt_contb, plt_contf=plt_contf, plt_contr=plt_contr, plt_contl=plt_contl, do_enum=do_enum, \n", + " ax_opt=dict({'fig_sizefac':2.0, 'cb_plt':cb_plt, 'cb_plt_single':cb_plt_single, 'cb_pos':'vertical', 'cb_h':'auto',}), # 'fs_label':14, 'fs_ticks':14, 'ax_dt':1.0}),\n", + " cbl_opt=dict(), cb_label=cb_label, cbtl_opt=dict(),\n", + " do_save=sfpath, save_dpi=save_dpi ) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## > Compute: MOC time series @ latitude" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "zdefuc,zdeflc = [500,2000], [1500, 6000] # depth range definition for upper/lower cell\n", + "\n", + "#____________________________________________________________________________________________________\n", + "# compute time-series\n", + "if vname == 'ipmoc' or vname == 'pmoc':\n", + " zmoc_maxatlat = zmoc_t.where((zmoc_t['depth']>=zdeflc[0]) & (zmoc_t['depth']<=zdeflc[1])).min(dim='nz',skipna=True).rename({'zmoc':'zmoc_lcell'})\n", + "elif vname == 'amoc' or vname == 'aamoc':\n", + " zmoc_ucell = zmoc_t.where((zmoc_t['depth']>=zdefuc[0]) & (zmoc_t['depth']<=zdefuc[1])).max(dim='nz',skipna=True).rename({'zmoc':'zmoc_ucell'})\n", + " zmoc_lcell = zmoc_t.where((zmoc_t['depth']>=zdeflc[0]) & (zmoc_t['depth']<=zdeflc[1])).min(dim='nz',skipna=True).rename({'zmoc':'zmoc_lcell'})\n", + " zmoc_maxatlat = xr.merge([zmoc_ucell, zmoc_lcell], compat='minimal' )\n", + " del(zmoc_ucell, zmoc_lcell)\n", + "\n", + "#_____________________________________________________________________________________________________\n", + "# save to netcdf\n", + "if (do_netcdf) and (save_path is not None) and (vname in ['amoc', 'aamoc', 'ipmoc', 'pmoc']):\n", + " auxtime = zmoc_t['time.year'].values\n", + " sfname = f'{vname}_at_lat_vs_t__{which_run}_y{auxtime[0]}-{auxtime[-1]}.nc'\n", + " sfilepath = os.path.join(save_path, sfname)\n", + " print(' --> save max. zmoc_t @ lat. to file:', sfname)\n", + " zmoc_maxatlat.to_netcdf(path=sfilepath, mode='w') " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## > Plot: MOC time-series @ latitude" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#___PLOT FESOM2 DATA___________________________________________________________________________________\n", + "which_lat = ['max'] # 'max' # 'max', 26.0\n", + "\n", + "#___PLOT FESOM2 DATA________________________________________________________\n", + "# plot strength of AMOC at lat\n", + "for lat in which_lat: \n", + " lat_list = list()\n", + " for data in [zmoc_maxatlat]:\n", + " for var in data.data_vars:\n", + " #_______________________________________________________________\n", + " if var in ['zmoc_nadw','zmoc_ucell']:\n", + " data_lat = zmoc_maxatlat[var].to_dataset(promote_attrs=True)\n", + " if lat=='max':\n", + " data_lat = data_lat.isel(lat=(data.lat>40) & (data.lat<60)).max(dim='lat') \n", + " str_label= f'@{40}°Nlat[0]) & (data.lat=0: str_label= f'@{lat}°N'\n", + " else : str_label= f'@{lat}°S' \n", + " data_lat[var].attrs['add2ylabel'] = str_label\n", + " lat_list.append([data_lat]) \n", + " \n", + " #___PLOT TRANSECT__________________________________________________________________________________\n", + " svname = list(lat_list[0][0].data_vars)[0]\n", + " auxtime = zmoc_t['time.year'].values\n", + " slabel = f'y{auxtime[0]}-{auxtime[-1]}'\n", + "\n", + " #__________________________________________________________________________________________________\n", + " # do save filename path\n", + " spath = save_path\n", + " sfpath = None\n", + " if spath!=None: \n", + " sfpath=list()\n", + " for sfmt in save_fmt: sfpath.append( os.path.join(spath,'{:s}_{:s}_{:s}_{:s}.{:s}'.format(svname, 'which_model', which_run ,slabel, sfmt)) )\n", + "\n", + " #__________________________________________________________________________________________________\n", + " fig,ax=tpv.plot_tline(lat_list, None, nrow=1, ncol=1 , \n", + " do_allcycl = False , \n", + " do_concat = False , \n", + " do_shdw = False ,\n", + " do_mean = True ,\n", + " do_std = False ,\n", + " ax_opt = dict({'fig_sizefac':3}) ,\n", + " do_enum = do_enum ,\n", + " do_save = sfpath, save_dpi=save_dpi, \n", + " )\n", + "\n", + " \n", + " \n", + "#___PLOT FESOM2 DATA________________________________________________________\n", + "# plot strength of AABW at lat\n", + "for lat in which_lat: \n", + " lat_list = list()\n", + " for data in [zmoc_maxatlat]:\n", + " for var in data.data_vars:\n", + " #_______________________________________________________________\n", + " if var in ['zmoc_aabw','zmoc_lcell']:\n", + " data_lat = data[var].to_dataset(promote_attrs=True)\n", + " if lat=='max':\n", + " data_lat = data_lat.isel(lat=(data.lat>40) & (data.lat<60)).min(dim='lat') \n", + " str_label= f'@{40}°Nlat[0]) & (data.lat=0: str_label= f'@{lat}°N'\n", + " else : str_label= f'@{lat}°S' \n", + " data_lat[var].attrs['add2ylabel'] = str_label\n", + " data_lat[var].attrs['short_name'] = 'AABW'\n", + " data_lat[var].attrs['long_name'] = 'AABW'\n", + " lat_list.append([data_lat]) \n", + " \n", + " #___PLOT TRANSECT__________________________________________________________________________________\n", + " svname = list(lat_list[0][0].data_vars)[0]\n", + " auxtime = zmoc_t['time.year'].values\n", + " slabel = f'y{auxtime[0]}-{auxtime[-1]}'\n", + "\n", + " #__________________________________________________________________________________________________\n", + " # do save filename path\n", + " spath = save_path\n", + " sfpath = None\n", + " if spath!=None: \n", + " sfpath=list()\n", + " for sfmt in save_fmt: sfpath.append( os.path.join(spath,'{:s}_{:s}_{:s}_{:s}.{:s}'.format(svname, 'which_model', which_run ,slabel, sfmt)) )\n", + "\n", + " #__________________________________________________________________________________________________\n", + " fig,ax=tpv.plot_tline(lat_list, None, nrow=1, ncol=1 , \n", + " do_allcycl = False , \n", + " do_concat = False , \n", + " do_shdw = False ,\n", + " do_mean = True ,\n", + " do_std = False ,\n", + " ax_opt = dict({'fig_sizefac':3}) ,\n", + " grid_opt = dict({'do_yinv':True}) ,\n", + " do_enum = do_enum ,\n", + " do_save = sfpath, save_dpi=save_dpi, \n", + " )" + ] + } + ], + "metadata": { + "celltoolbar": "Tags", + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.20" + }, + "latex_envs": { + "LaTeX_envs_menu_present": true, + "autoclose": false, + "autocomplete": true, + "bibliofile": "biblio.bib", + "cite_by": "apalike", + "current_citInitial": 1, + "eqLabelWithNumbers": true, + "eqNumInitial": 1, + "hotkeys": { + "equation": "Ctrl-E", + "itemize": "Ctrl-I" + }, + "labels_anchors": false, + "latex_user_defs": false, + "report_style_numbering": false, + "user_envs_cfg": false + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/tripyview/sub_data.py b/tripyview/sub_data.py index 11c2f4d..1a1ffce 100755 --- a/tripyview/sub_data.py +++ b/tripyview/sub_data.py @@ -39,6 +39,7 @@ def load_data_fesom2(mesh, runid = 'fesom' , do_prec = 'float32' , do_f14cmip6 = False , + do_multiio = False , do_compute = False , do_load = True , do_persist = False , @@ -388,6 +389,25 @@ def _preprocess(x, do_prec): ('nod2' in data.dims) and \ ('nz' in data.dims): data = data.transpose('time', 'nod2', 'nz') data = data.unify_chunks() + + # convert dimensions name from multiio --> fesom2 + if do_multiio: + data = data.drop_vars(['time_bnds']) + if ('ncells' in data.dims ): data = data.rename_dims({'ncells':'nod2'}) + + # rename coordinate: depth --> nz, do this first than rename dimension otherwise + # it triggers a warning message + if ('lev' in data.coords): + data = data.rename({'lev' :'nz' }) + if 'nz' not in data.indexes: + data = data.set_index(nz='nz') + + if ('lev' in data.dims ): data = data.swap_dims({'lev': 'nz'}) + + if ('time' in data.dims) and \ + ('nod2' in data.dims) and \ + ('nz' in data.dims): data = data.transpose('time', 'nod2', 'nz') + data = data.unify_chunks() #___________________________________________________________________________ # check if mesh and data fit together @@ -410,7 +430,7 @@ def _preprocess(x, do_prec): # years are selected by the files that are open, need to select mon or day # or record data, mon, day, str_ltim = do_select_time(data, mon, day, record, str_ltim) - + # do time arithmetic on data if 'time' in data.dims: data, str_atim = do_time_arithmetic(data, do_tarithm) @@ -899,6 +919,7 @@ def do_select_time(data, mon, day, record, str_mtim): ____________________________________________________________________________ """ + #___________________________________________________________________________ # select no time use entire yearly file if (mon is None) and (day is None) and (record is None):