Merge branch 'develop' into develop_satellite

examples/jupyter_notebooks/Monet-analysis-example-plots-wrfChemexample_develop_aircraft_Latest.py

@@ -0,0 +1,18 @@
+from melodies_monet import driver
+
+an = driver.analysis()
+
+an.control = '../examples/yaml/control_wrfchem_aircraft_Latestfor_develop_aircraft.yaml'
+
+an.read_control() 
+
+an.control_dict   
+
+an.open_models()
+
+an.open_obs()
+
+#This just pairs the data
+an.pair_data()
+
+an.plotting()

examples/submit_jobs/run_aircraft_pairing_loop.py

@@ -0,0 +1,25 @@
+# This code uses MELODIES-MONET to read in a .yaml file 
+# and produces sets of paired aircraft observations. 
+# For an interactive script see jupyter notebooks in main directory.
+
+from melodies_monet import driver
+from melodies_monet.util.tools import loop_pairing
+import os
+import dask
+an = driver.analysis()
+# -- Update the yaml file below
+control_fn = '../yaml/control_aircraft_looping.yaml'
+file_pairs_yaml='../yaml/supplementary_yaml/aircraft_looping_file_pairs.yaml'
+an.control = control_fn
+an.read_control()
+
+# -- Lines below make a copy of the namelist in the plot directory for reference later
+cmd = 'cp ' + an.control + ' ' + an.control_dict['analysis']['output_dir']
+os.system(cmd)
+
+cmd = 'cp ' + file_pairs_yaml + ' ' + an.control_dict['analysis']['output_dir']
+os.system(cmd)
+
+dask.config.set(**{'array.slicing.split_large_chunks': True})
+loop_pairing(control=control_fn,file_pairs_yaml=file_pairs_yaml)
+

examples/yaml/control_aircraft_looping.yaml

@@ -0,0 +1,88 @@
+# General Description:  
+# Any key that is specific for a plot type will begin with ts for timeseries, ty for taylor
+# Opt: Specifying the variable or variable group is optional
+# For now all plots except time series average over the analysis window. 
+# Seting axis values - If set_axis = True in data_proc section of each plot_grp the yaxis for the plot will be set based on the values specified in the obs section for each variable. If set_axis is set to False, then defaults will be used. 'vmin_plot' and 'vmax_plot' are needed for 'timeseries', 'spatial_overlay', and 'boxplot'. 'vdiff_plot' is needed for 'spatial_bias' plots and'ty_scale' is needed for 'taylor' plots. 'nlevels' or the number of levels used in the contour plot can also optionally be provided for spatial_overlay plot. If set_axis = True and the proper limits are not provided in the obs section, a warning will print, and the plot will be created using the default limits.
+analysis:
+  #start_time: '2019-09-05-12:00:00' #UTC
+  #end_time: '2019-09-06-00:00:00' #UTC
+  output_dir:  /wrk/charkins/melodies_monet/aircraft/develop_aircraft_looping/output #/wrk/charkins/melodies_monet/aircraft/analysis #Opt if not specified plots stored in code directory.
+  debug: True
+
+model:
+  wrfchem_v4.2: # model label
+    mod_type: 'wrfchem'
+    mod_kwargs: 
+      mech: 'racm_esrl_vcp'
+    radius_of_influence: 12000 #meters
+    mapping: #model species name : obs species name
+      firexaq:
+        no2: NO2_CL_RYERSON
+        'no': 'NO_CL_RYERSON'
+        #PM2_5_DRY: "PM2.5"
+        o3: "O3_CL_RYERSON"
+    variables:
+        'pres_pa_mid':
+            rename: pressure_model # name to convert this variable to, use 'pressure_model' for aircraft obs
+            unit_scale: 1 #Opt Scaling factor 
+            unit_scale_method: '*' #Opt Multiply = '*' , Add = '+', subtract = '-', divide = '/'
+        'temperature_k':
+            rename: temp_model # name to convert this variable to
+            unit_scale: 1 #Opt Scaling factor 
+            unit_scale_method: '*' #Opt Multiply = '*' , Add = '+', subtract = '-', divide = '/'
+    projection: null
+    #plot_kwargs: #Opt
+    #  color: 'dodgerblue'
+    #  marker: '^'
+    #  linestyle: ':' 
+obs:
+  firexaq: # obs label
+    obs_type: aircraft
+    resample: '600S' #'600S' #10 min so works on Hera as a test. Can comment this if submitting a job. 
+    variables: #Opt 
+      'O3_CL_RYERSON':
+        unit_scale: 1 #Opt Scaling factor 
+        unit_scale_method: '*' #Opt Multiply = '*' , Add = '+', subtract = '-', divide = '/'
+        nan_value: -777777 # Opt Set this value to NaN
+        LLOD_value: -888888 # Opt Set this value to LLOD_setvalue
+        LLOD_setvalue: 0.0 # Opt Set LLOD_value=LLOD_setvalue, applied AFTER unit_scale and obs_unit
+        #ylabel_plot: 'Ozone (ppbv)'
+        #vmin_plot: 15.0 #Opt Min for y-axis during plotting. To apply to a plot, change restrict_yaxis = True.
+        #vmax_plot: 55.0 #Opt Max for y-axis during plotting. To apply to a plot, change restrict_yaxis = True.
+        #vdiff_plot: 20.0 #Opt +/- range to use in bias plots. To apply to a plot, change restrict_yaxis = True.
+        # nlevels_plot: 21 #Opt number of levels used in colorbar for contourf plot.
+      'NO_CL_RYERSON':
+        nan_value: -777777 # Set this value to NaN
+        LLOD_value: -888888 # Opt Set this value to LLOD_setvalue
+        LLOD_setvalue: 0.0 # Opt Set LLOD_value=LLOD_setvalue, applied AFTER unit_scale and obs_unit
+        #ylabel_plot: 'NO (ppbv)' #Optional to set ylabel so can include units and/or instr etc.
+        #vmin_plot: 0.0 #Opt Min for y-axis during plotting. To apply to a plot, change restrict_yaxis = True.
+        #vmax_plot: 20.0 #Opt Max for y-axis during plotting. To apply to a plot, change restrict_yaxis = True.
+        #vdiff_plot: 15.0 #Opt +/- range to use in bias plots. To apply to a plot, change restrict_yaxis = True.
+        #nlevels_plot: 21 #Opt number of levels used in colorbar for contourf plot.
+      'NO2_CL_RYERSON':
+        nan_value: -777777 # Set this value to NaN
+        LLOD_value: -888888 # Opt Set this value to LLOD_setvalue
+        LLOD_setvalue: 0.0 # Opt Set LLOD_value=LLOD_setvalue, applied AFTER unit_scale and obs_unit
+        #ylabel_plot: 'NO2 (ppbv)' #Optional to set ylabel so can include units and/or instr etc.
+        #vmin_plot: 0.0 #Opt Min for y-axis during plotting. To apply to a plot, change restrict_yaxis = True.
+        #vmax_plot: 20.0 #Opt Max for y-axis during plotting. To apply to a plot, change restrict_yaxis = True.
+        #vdiff_plot: 15.0 #Opt +/- range to use in bias plots. To apply to a plot, change restrict_yaxis = True.
+        #nlevels_plot: 21 #Opt number of levels used in colorbar for contourf plot.
+      'Latitude_YANG':
+            rename: latitude # name to convert this variable to
+            unit_scale: 1 #Opt Scaling factor 
+            unit_scale_method: '*' #Opt Multiply = '*' , Add = '+', subtract = '-', divide = '/'
+      'Longitude_YANG':
+            rename: longitude # name to convert this variable to
+            unit_scale: 1 #Opt Scaling factor 
+            unit_scale_method: '*' #Opt Multiply = '*' , Add = '+', subtract = '-', divide = '/'
+      'P_BUI':
+            rename: pressure_obs # name to convert this variable to
+            unit_scale: 100 #Opt Scaling factor 
+            unit_scale_method: '*' #Opt Multiply = '*' , Add = '+', subtract = '-', divide = '/'
+      'MSL_GPS_Altitude_YANG':
+          rename: altitude # name to convert this variable to
+          unit_scale: 1 #Opt Scaling factor 
+          unit_scale_method: '*' #Opt Multiply = '*' , Add = '+', subtract = '-', divide = '/'
+

examples/yaml/control_hrrr-smoke_mobile.yaml

@@ -0,0 +1,109 @@
+# General Description:  
+# Any key that is specific for a plot type will begin with ts for timeseries, ty for taylor
+# Opt: Specifying the variable or variable group is optional
+# For now all plots except time series average over the analysis window. 
+# Seting axis values - If set_axis = True in data_proc section of each plot_grp the yaxis for the plot will be set based on the values specified in the obs section for each variable. If set_axis is set to False, then defaults will be used. 'vmin_plot' and 'vmax_plot' are needed for 'timeseries', 'spatial_overlay', and 'boxplot'. 'vdiff_plot' is needed for 'spatial_bias' plots and'ty_scale' is needed for 'taylor' plots. 'nlevels' or the number of levels used in the contour plot can also optionally be provided for spatial_overlay plot. If set_axis = True and the proper limits are not provided in the obs section, a warning will print, and the plot will be created using the default limits.
+analysis:
+  start_time: '2022-09-11-06:00:00' #UTC
+  end_time: '2022-09-12-06:00:00' #UTC
+  output_dir: /wrk/charkins/melodies_monet/aircraft/develop_aircraft_mobile/Output_hrrr-smoke #Opt if not specified plots stored in code directory.
+  debug: True
+model:
+  hrrr-smoke: # model label
+    files: /wrk/users/charkins/megan_mobile_data/HRRR/aqm_HRRR-Smoke_2022091106.nc.subset.nc
+    mod_type: random
+    mapping: #model species name : obs species name
+      mobile_fire:
+        PM2_5_DRY: 'PM2.5_5min'
+        'TMP_2maboveground': 'Temperature_5min'
+    plot_kwargs: #Opt
+      color: 'dodgerblue'
+      marker: '^'
+      linestyle: ':' 
+obs:
+  mobile_fire: # obs label
+    filename: '/wrk/users/charkins/megan_mobile_data/Fire_obs_all/treated_final_modtime.csv'
+    obs_type: mobile
+    resample: '600S' 
+    time_var: 'time_5min'
+    variables: #Opt 
+      'PM2.5_5min':
+        unit_scale: 1 #Opt Scaling factor 
+        unit_scale_method: '*' #Opt Multiply = '*' , Add = '+', subtract = '-', divide = '/'
+
+        ylabel_plot: 'PM2.5 (units?)'
+      'Temperature_5min':
+        unit_scale: 273.15 #Opt Scaling factor 
+        unit_scale_method: '+' #Opt Multiply = '*' , Add = '+', subtract = '-', divide = '/'
+        ylabel_plot: 'Temperature (K)' #Optional to set ylabel so can include units and/or instr etc.
+      'Latitude_5min':
+            rename: latitude # name to convert this variable to
+            unit_scale: 1 #Opt Scaling factor 
+            unit_scale_method: '*' #Opt Multiply = '*' , Add = '+', subtract = '-', divide = '/'
+      'Longitude_5min':
+            rename: longitude # name to convert this variable to
+            unit_scale: 1 #Opt Scaling factor 
+            unit_scale_method: '*' #Opt Multiply = '*' , Add = '+', subtract = '-', divide = '/'
+plots:
+  plot_grp1:
+    type: 'timeseries' # plot type
+    fig_kwargs: #Opt to define figure options
+      figsize: [12,6] # figure size if multiple plots
+    default_plot_kwargs: # Opt to define defaults for all plots. Model kwargs overwrite these.
+      linewidth: 2.0
+      markersize: 10.
+    text_kwargs: #Opt
+      fontsize: 18.
+    domain_type: ['all'] #List of domain types: 'all' or any domain in obs file. (e.g., airnow: epa_region, state_name, siteid, etc.) 
+    domain_name: ['CONUS'] #List of domain names. If domain_type = all domain_name is used in plot title.
+    data: ['mobile_fire_hrrr-smoke'] # make this a list of pairs in obs_model where the obs is the obs label and model is the model_label
+    data_proc:
+      rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
+      ts_select_time: 'time' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local'
+      ts_avg_window: null # Options: None for no averaging or list pandas resample rule (e.g., 'H', 'D')
+      set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs.
+  plot_grp2:
+    type: 'taylor' # plot type
+    fig_kwargs: #Opt to define figure options
+      figsize: [8,8] # figure size if multiple plots
+    default_plot_kwargs: # Opt to define defaults for all plots. Model kwargs overwrite these.
+      linewidth: 2.0
+      markersize: 10.
+    text_kwargs: #Opt
+      fontsize: 16.
+    domain_type: ['all'] #List of domain types: 'all' or any domain in obs file. (e.g., airnow: epa_region, state_name, siteid, etc.) 
+    domain_name: ['CONUS'] #List of domain names. If domain_type = all domain_name is used in plot title.
+    data: ['mobile_fire_hrrr-smoke'] # make this a list of pairs in obs_model where the obs is the obs label and model is the model_label
+    data_proc:
+      rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
+      set_axis: True #If select True, add ty_scale for each variable in obs.
+  plot_grp3:
+    type: 'boxplot' # plot type
+    fig_kwargs: #Opt to define figure options
+      figsize: [8, 6] # figure size 
+    text_kwargs: #Opt
+      fontsize: 20.
+    domain_type: ['all'] #List of domain types: 'all' or any domain in obs file. (e.g., airnow: epa_region, state_name, siteid, etc.) 
+    domain_name: ['CONUS'] #List of domain names. If domain_type = all domain_name is used in plot title.
+    data: ['mobile_fire_hrrr-smoke'] # make this a list of pairs in obs_model where the obs is the obs label and model is the model_label
+    data_proc:
+      rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
+      set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs.
+
+stats:
+  #Stats require positive numbers, so if you want to calculate temperature use Kelvin!
+  #Wind direction has special calculations for AirNow if obs name is 'WD'
+  stat_list: ['MB', 'MdnB','R2', 'RMSE'] #List stats to calculate. Dictionary of definitions included in plots/proc_stats.py Only stats listed below are currently working.
+  #Full calc list ['STDO', 'STDP', 'MdnNB','MdnNE','NMdnGE', 'NO','NOP','NP','MO','MP', 'MdnO', 'MdnP', 'RM', 'RMdn', 'MB', 'MdnB', 'NMB', 'NMdnB', 'FB', 'ME','MdnE','NME', 'NMdnE', 'FE', 'R2', 'RMSE','d1','E1', 'IOA', 'AC']
+  round_output: 2 #Opt, defaults to rounding to 3rd decimal place.
+  output_table: False #Always outputs a .txt file. Optional to also output as a table.
+  output_table_kwargs: #Opt 
+    figsize: [7, 3] # figure size 
+    fontsize: 12.
+    xscale: 1.4
+    yscale: 1.4
+    edges: 'horizontal'
+  domain_type: ['all'] #List of domain types: 'all' or any domain in obs file. (e.g., airnow: epa_region, state_name, siteid, etc.) 
+  domain_name: ['CONUS'] #List of domain names. If domain_type = all domain_name is used in plot title.
+  data: ['mobile_fire_hrrr-smoke'] # make this a list of pairs in obs_model where the obs is the obs label and model is the model_label
+