Merge pull request #259 from blychs/develop

Add numeric_only=True to relevant pandas operations

docs/develop/developers_guide.rst

@@ -74,7 +74,7 @@ these instructions:
 
        $ conda create --name melodies-monet python=3.9
        $ conda activate melodies-monet
-       $ conda install -y -c conda-forge pyyaml pandas=1 monet monetio netcdf4 wrf-python typer rich pooch jupyterlab
+       $ conda install -y -c conda-forge pyyaml pandas=1 'matplotlib-base<3.9' monet monetio netcdf4 wrf-python typer rich pooch jupyterlab
 
 (b) Clone [#clone]_ and link the latest development versions of MONET and MONETIO from GitHub to
     your conda environment::

docs/environment-docs.yml

@@ -7,6 +7,7 @@ dependencies:
   - python=3.9
   #
   # melodies_monet deps
+  - matplotlib-base<3.9  # for pandas v1 compat
   - monet
   - monetio
   - netcdf4

docs/examples/control_camchem.yaml

@@ -86,7 +86,7 @@ plots:
     data_proc:
       rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
       ts_select_time: 'time' # Time used for avg and plotting: Options: 'time' for UTC or 'time_local'
-      ts_avg_window: 'H' # Options: None for no averaging or list pandas resample rule (e.g., 'H', 'D')
+      ts_avg_window: 'h' # Options: None for no averaging or list pandas resample rule (e.g., 'h', 'D')
       set_axis: False # If select True, add vmin_plot and vmax_plot for each variable in obs.
   plot_grp2:
     type: 'taylor' # plot type

docs/examples/control_camchem_se.yaml

@@ -86,7 +86,7 @@ plots:
     data_proc:
       rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
       ts_select_time: 'time_local' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local'
-      ts_avg_window: 'H' # Options: None for no averaging or list pandas resample rule (e.g., 'H', 'D')
+      ts_avg_window: 'h' # Options: None for no averaging or list pandas resample rule (e.g., 'h', 'D')
       set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs.
   plot_grp2:
     type: 'taylor' # plot type

docs/examples/control_idealized.yaml

@@ -61,7 +61,7 @@ plots:
     data_proc:  # These four seem to be required for time series
       rem_obs_nan: True  # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
       ts_select_time: 'time'  # 'time' for UTC or 'time_local'
-      ts_avg_window: '3H'  # Options: None for no averaging or list pandas resample rule (e.g., 'H', 'D')
+      ts_avg_window: '3h'  # Options: None for no averaging or list pandas resample rule (e.g., 'h', 'D')
       # ^ TODO: null setting seems not working
       set_axis: False  # If True, add vmin_plot and vmax_plot for each variable in obs.
 

docs/examples/control_wrfchem_mech-0905_2.yaml

@@ -126,8 +126,8 @@ plots:
       ts_select_time: "time_local"  # `ts_` indicates this is time series plot-specific
       # ^ Time used for avg and plotting
       #   Options: 'time' for UTC or 'time_local'
-      ts_avg_window: "H"
-      # ^ Options: None for no averaging, pandas resample rule (e.g., 'H', 'D')
+      ts_avg_window: "h"
+      # ^ Options: None for no averaging, pandas resample rule (e.g., 'h', 'D')
       set_axis: True
       # ^ If true, add `vmin_plot` and `vmax_plot` for each variable in obs.
 

docs/examples/control_wrfchem_saveandread.yaml

@@ -147,8 +147,8 @@ plots:
       ts_select_time: "time_local"  # `ts_` indicates this is time series plot-specific
       # ^ Time used for avg and plotting
       #   Options: 'time' for UTC or 'time_local'
-      ts_avg_window: "H"
-      # ^ Options: None for no averaging, pandas resample rule (e.g., 'H', 'D')
+      ts_avg_window: "h"
+      # ^ Options: None for no averaging, pandas resample rule (e.g., 'h', 'D')
       set_axis: True
       # ^ If true, add `vmin_plot` and `vmax_plot` for each variable in obs.
 

docs/tutorial/installation.rst

@@ -34,7 +34,7 @@ First create and activate a conda environment::
 
 Add dependencies from conda-forge::
 
-    $ conda install -y -c conda-forge pyyaml pandas=1 monet monetio netcdf4 wrf-python typer rich pooch
+    $ conda install -y -c conda-forge pyyaml pandas=1 'matplotlib-base<3.9' monet monetio netcdf4 wrf-python typer rich pooch
 
 Now, install the stable branch of MELODIES MONET to the environment::
 

examples/forecast_evaluation/reformat_aeronet_rapchemtest.py

@@ -15,16 +15,16 @@
 end_time_reformat=sys.argv[2]
 print(sys.argv[1])
 print(sys.argv[2])
-dates = pd.date_range(start=start_time_reformat,end=end_time_reformat,freq='H')
+dates = pd.date_range(start=start_time_reformat,end=end_time_reformat,freq='h')
 
-#dates = pd.date_range(start='2021-08-01',end='2021-09-01',freq='H')
+#dates = pd.date_range(start='2021-08-01',end='2021-09-01',freq='h')
 # set the output filename
 #outname = 'AERONET_L15_20190901_20190930.nc'
 outname = 'test5.nc'
 # set standard wavelengths
 standard_wavelengths = np.array([0.34, 0.44, 0.55, 0.66, 0.86, 1.63, 11.1])* 1000. # convert from micron to nm
 # get the data
-df = mio.aeronet.add_data(dates, interp_to_aod_values=standard_wavelengths, freq='H') # ,n_procs=12)
+df = mio.aeronet.add_data(dates, interp_to_aod_values=standard_wavelengths, freq='h') # ,n_procs=12)
 
 
 # dfp = df.rename({'siteid':'x'},axis=1).set_index(['time','x']).drop_duplicates()

examples/forecast_evaluation/reformat_airnow_rapchemtest.py

@@ -16,8 +16,8 @@
 end_time_reformat=sys.argv[2] 
 print(sys.argv[1])
 print(sys.argv[2])
-dates = pd.date_range(start=start_time_reformat,end=end_time_reformat,freq='H')
-#dates = pd.date_range(start='2021-05-28',end='2021-05-29',freq='H')
+dates = pd.date_range(start=start_time_reformat,end=end_time_reformat,freq='h')
+#dates = pd.date_range(start='2021-05-28',end='2021-05-29',freq='h')
 
 
 # helper function for local time.  Could be important for EPA statistics\n"
@@ -28,7 +28,7 @@ def get_local_time(ds):
         o = tim.expand_dims({'x':t.x.values}).transpose('time','x')
         on = xr.Dataset({'time_local':o,'utcoffset':t.utcoffset})
         y = on.to_dataframe()
-        y['time_local'] = y.time_local + pd.to_timedelta(y.utcoffset, unit='H')
+        y['time_local'] = y.time_local + pd.to_timedelta(y.utcoffset, unit='h')
         time_local = y[['time_local']].to_xarray()
         ds = xr.merge([ds,time_local])
     return ds

examples/subset/airnow_wrfchem.yaml

@@ -82,7 +82,7 @@ plots:
     data_proc:
       rem_obs_nan: True
       ts_select_time: 'time_local'
-      ts_avg_window: 'H'
+      ts_avg_window: 'h'
       set_axis: True
 
   plot_overlay_group:

examples/yaml/control_cmaq-rrfs_surface-all-short_test_jupyter.yaml

@@ -163,7 +163,7 @@ plots:
       #rem_obs_by_nan_pct: {'group_var': 'siteid','pct_cutoff': 25,'times':'hourly'} # Groups by group_var, then removes all instances of groupvar where obs variable is > pct_cutoff % nan values
       rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
       ts_select_time: 'time_local' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local'
-      ts_avg_window: 'H' # pandas resample rule (e.g., 'H', 'D'). No averaging is done if ts_avg_window is null or not specified.
+      ts_avg_window: 'h' # pandas resample rule (e.g., 'h', 'D'). No averaging is done if ts_avg_window is null or not specified.
       set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs.
   plot_grp2:
     type: 'taylor' # plot type

examples/yaml/control_cmaq-rrfs_surface-all.yaml

@@ -163,7 +163,7 @@ plots:
       #rem_obs_by_nan_pct: {'group_var': 'siteid','pct_cutoff': 25,'times':'hourly'} # Groups by group_var, then removes all instances of groupvar where obs variable is > pct_cutoff % nan values
       rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
       ts_select_time: 'time_local' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local'
-      ts_avg_window: 'H' # pandas resample rule (e.g., 'H', 'D'). No averaging is done if ts_avg_window is null or not specified.
+      ts_avg_window: 'h' # pandas resample rule (e.g., 'h', 'D'). No averaging is done if ts_avg_window is null or not specified.
       set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs.
   plot_grp2:
     type: 'taylor' # plot type

examples/yaml/control_cmaq-rrfs_surface.yaml

@@ -201,7 +201,7 @@ plots:
       #rem_obs_by_nan_pct: {'group_var': 'siteid','pct_cutoff': 25,'times':'hourly'} # Groups by group_var, then removes all instances of groupvar where obs variable is > pct_cutoff % nan values
       rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
       ts_select_time: 'time_local' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local'
-      ts_avg_window: 'H' # pandas resample rule (e.g., 'H', 'D'). No averaging is done if ts_avg_window is null or not specified.
+      ts_avg_window: 'h' # pandas resample rule (e.g., 'h', 'D'). No averaging is done if ts_avg_window is null or not specified.
       set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs.
   plot_grp2:
     type: 'taylor' # plot type

examples/yaml/control_cmaq.yaml

@@ -155,7 +155,7 @@ plots:
       #rem_obs_by_nan_pct: {'group_var': 'siteid','pct_cutoff': 25,'times':'hourly'} # Groups by group_var, then removes all instances of groupvar where obs variable is > pct_cutoff % nan values
       rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
       ts_select_time: 'time_local' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local'
-      ts_avg_window: 'H' # pandas resample rule (e.g., 'H', 'D'). No averaging is done if ts_avg_window is null or not specified.
+      ts_avg_window: 'h' # pandas resample rule (e.g., 'h', 'D'). No averaging is done if ts_avg_window is null or not specified.
       set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs.
   plot_grp2:
     type: 'taylor' # plot type

examples/yaml/control_hrrr-smoke_mobile.yaml

@@ -60,7 +60,7 @@ plots:
     data_proc:
       rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
       ts_select_time: 'time' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local'
-      ts_avg_window: null # Options: None for no averaging or list pandas resample rule (e.g., 'H', 'D')
+      ts_avg_window: null # Options: None for no averaging or list pandas resample rule (e.g., 'h', 'D')
       set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs.
   plot_grp2:
     type: 'taylor' # plot type

examples/yaml/control_omps_limb.yaml

@@ -48,7 +48,7 @@ plots:
     data_proc:
       rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
       ts_select_time: 'time_local' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local'
-      ts_avg_window: 'H' # Options: None for no averaging or list pandas resample rule (e.g., 'H', 'D')
+      ts_avg_window: 'h' # Options: None for no averaging or list pandas resample rule (e.g., 'h', 'D')
       set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs.
   plot_grp2:
     type: 'taylor' # plot type
@@ -106,4 +106,4 @@ plots:
 stats:
   rmse: True
   mse: True
-  ioa: True 
+  ioa: True 

examples/yaml/control_omps_nm-raqms.yaml

@@ -54,7 +54,7 @@ plots:
 #    data_proc:
 #      rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
 #      ts_select_time: 'time' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local'
-#      ts_avg_window: 'min' # Options: None for no averaging or list pandas resample rule (e.g., 'H', 'D')
+#      ts_avg_window: 'min' # Options: None for no averaging or list pandas resample rule (e.g., 'h', 'D')
 #      set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs.
   plot_grp2:
     type: 'taylor' # plot type

examples/yaml/control_read_looped_aircraft.yaml

@@ -117,7 +117,7 @@ plots:
       #altitude_ticks: 1000  # Altitude tick interval in meters (for secondary y-axis for altitude (m))
       rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
       ts_select_time: 'time' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local'
-      ts_avg_window: #'H' # pandas resample rule (e.g., 'H', 'D'). No averaging is done if ts_avg_window is null or not specified.
+      ts_avg_window: #'h' # pandas resample rule (e.g., 'h', 'D'). No averaging is done if ts_avg_window is null or not specified.
       set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs.
       #vmin2, vmax2 filter not needed as filter_dict option added in 'altitude_yax2' to subset the paireddf as per altitude secondary-axis limits
       #vmin2: #0  #Optional
@@ -133,4 +133,4 @@ plots:
         #filter_dict: #Default is min and max if filter_dict doesn't define the options below (or if they are commented out)
         #  altitude:
         #    oper: "between"
-        #    value: [2000,8000] # values are [vmin_y2, vmax_y2] 
+        #    value: [2000,8000] # values are [vmin_y2, vmax_y2] 

examples/yaml/control_rrfs_cmaq_airnow_norm.yaml

@@ -167,7 +167,7 @@ plots:
       #rem_obs_by_nan_pct: {'group_var': 'siteid','pct_cutoff': 25,'times':'hourly'} # Groups by group_var, then removes all instances of groupvar where obs variable is > pct_cutoff % nan values
       rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
       ts_select_time: 'time_local' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local'
-      ts_avg_window: 'H' # pandas resample rule (e.g., 'H', 'D'). No averaging is done if ts_avg_window is null or not specified.
+      ts_avg_window: 'h' # pandas resample rule (e.g., 'h', 'D'). No averaging is done if ts_avg_window is null or not specified.
       set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs.
   plot_grp2:
     type: 'taylor' # plot type

examples/yaml/control_rrfs_cmaq_airnow_reg.yaml

@@ -167,7 +167,7 @@ plots:
       #rem_obs_by_nan_pct: {'group_var': 'siteid','pct_cutoff': 25,'times':'hourly'} # Groups by group_var, then removes all instances of groupvar where obs variable is > pct_cutoff % nan values
       rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
       ts_select_time: 'time_local' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local'
-      ts_avg_window: 'D' # pandas resample rule (e.g., 'H', 'D'). No averaging is done if ts_avg_window is null or not specified.
+      ts_avg_window: 'D' # pandas resample rule (e.g., 'h', 'D'). No averaging is done if ts_avg_window is null or not specified.
       set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs.
   plot_grp2:
     type: 'taylor' # plot type

examples/yaml/control_wrfchem.yaml

@@ -117,7 +117,7 @@ plots:
       #rem_obs_by_nan_pct: {'group_var': 'siteid','pct_cutoff': 25,'times':'hourly'} # Groups by group_var, then removes all instances of groupvar where obs variable is > pct_cutoff % nan values
       rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
       ts_select_time: 'time_local' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local'
-      ts_avg_window: 'H' # pandas resample rule (e.g., 'H', 'D'). No averaging is done if ts_avg_window is null or not specified.
+      ts_avg_window: 'h' # pandas resample rule (e.g., 'h', 'D'). No averaging is done if ts_avg_window is null or not specified.
       set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs.
   plot_grp2:
     type: 'taylor' # plot type

examples/yaml/control_wrfchem_aircraft.yaml

@@ -99,7 +99,7 @@ plots:
     data_proc:
       rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
       ts_select_time: 'time' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local'
-      ts_avg_window: 'H' # Options: None for no averaging or list pandas resample rule (e.g., 'H', 'D')
+      ts_avg_window: 'h' # Options: None for no averaging or list pandas resample rule (e.g., 'h', 'D')
       set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs.
   plot_grp2:
     type: 'taylor' # plot type

examples/yaml/control_wrfchem_aircraft_Latestfor_develop_aircraft.yaml

@@ -107,7 +107,7 @@ plots:
       #altitude_ticks: 1000  # Altitude tick interval in meters (for secondary y-axis for altitude (m))
       rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
       ts_select_time: 'time' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local'
-      ts_avg_window: #'H' # pandas resample rule (e.g., 'H', 'D'). No averaging is done if ts_avg_window is null or not specified.
+      ts_avg_window: #'h' # pandas resample rule (e.g., 'h', 'D'). No averaging is done if ts_avg_window is null or not specified.
       set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs.
       #vmin2, vmax2 filter not needed as filter_dict option added in 'altitude_yax2' to subset the paireddf as per altitude secondary-axis limits
       #vmin2: #0  #Optional

examples/yaml/control_wrfchem_aircraft_Latestfor_develop_aircraft_testmultiplemodelgroups.yaml

@@ -144,7 +144,7 @@ plots:
       #altitude_ticks: 1000  # Altitude tick interval in meters (for secondary y-axis for altitude (m))
       rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
       ts_select_time: 'time' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local'
-      ts_avg_window: #'H' # pandas resample rule (e.g., 'H', 'D'). No averaging is done if ts_avg_window is null or not specified.
+      ts_avg_window: #'h' # pandas resample rule (e.g., 'h', 'D'). No averaging is done if ts_avg_window is null or not specified.
       set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs.
       #vmin2, vmax2 filter not needed as filter_dict option added in 'altitude_yax2' to subset the paireddf as per altitude secondary-axis limits
       #vmin2: #0  #Optional

examples/yaml/control_wrfchem_ground.yaml

@@ -49,7 +49,7 @@ plots:
     data_proc:
       rem_obs_nan: True # True: Remove all points where model or obs variable is NaN. False: Remove only points where model variable is NaN.
       ts_select_time: 'time' #Time used for avg and plotting: Options: 'time' for UTC or 'time_local'
-      ts_avg_window: 'H' # Options: None for no averaging or list pandas resample rule (e.g., 'H', 'D')
+      ts_avg_window: 'h' # Options: None for no averaging or list pandas resample rule (e.g., 'h', 'D')
       set_axis: False #If select True, add vmin_plot and vmax_plot for each variable in obs.
   plot_grp2:
     type: 'taylor' # plot type