From: Alexander Vasarab Date: Tue, 16 Nov 2021 04:16:29 +0000 (-0800) Subject: Merge branch 'hn24-and-wind-avg' X-Git-Tag: v3.3.0~1 X-Git-Url: https://wylark.com/src/infoex-autowx.git/commitdiff_plain/555278b3e0ff7279a8963485b870f2970f4fd689?hp=88809dafd11be98b51e399a8be3a46cf9d1e5ea3 Merge branch 'hn24-and-wind-avg' --- diff --git a/README.md b/README.md index 54d92b6..ed40451 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,8 @@ options. `units = # either english or metric -- only applies when type is mesowest #` `tz = # any entry from the Olson tz database e.g. America/Denver #` `path = # the filesystem path to the Python program -- only applies when type is python #` +`wind_mode = # normal or average #` +`hn24 = # yes or no #` `[infoex]` `host = # InfoEx FTP host address #` @@ -193,6 +195,40 @@ indicates that I'd like to import "Temperature" and "Precipitation accumulated" from the MesoWest station at Santiam Pass, OR, into InfoEx, and that I want that data in imperial units. +Three- versus 24-hour ranges +---------------------------- + +By default, this program will fetch three hours of data from the +provider. This way, if the most recent record has any missing data, it +can examine the two hours prior, using whatever data it can find. + +There are two features which will cause the program to expand the time +range of fetched data from three to 24 hours. Please be aware of this +expansion as it may cause a rise in data/API usage. + +**NOTE: Only MesoWest stations have the benefit of wind averaging and + HN24 calculation at this time, because generally NRCS SNOTEL + stations do not provide wind data. HN24 support for NRCS SNOTEL + is planned. + +### Wind mode +If you go to submit a Wx observation in InfoEx at e.g. 05:05, and have +so configured InfoEx, it will take the wind speed, wind gust speed, and +wind direction, from that hour and auto-fill it for the observation. + +Some operations may find it more important to know the averages for +those values over the prior 24 hour period. Setting `wind_mode` to +`average` will enable that. + +### HN24 +As most stations do not provide HN24 on their own, this program provides +a configuration option for calculating this. Simply add `hn24 = true` to +the configuration file. + +*NOTE: This is its own configuration option, rather than a new value for + desired_data, because it's not technically provided by MesoWest + or NRCS SNOTEL.* + Custom weather station support ------------------------------ diff --git a/infoex-autowx.py b/infoex-autowx.py index d9adcff..e584933 100755 --- a/infoex-autowx.py +++ b/infoex-autowx.py @@ -128,9 +128,45 @@ def setup_config(config): LOG.critical("%s is not a valid timezone", tz) sys.exit(1) + # By default, fetch three hours of data + # + # If user wants hn24 or wind averaging, then + # we need more. + station['num_hrs_to_fetch'] = 3 + + # HN24 + if 'hn24' in config['station']: + if config['station']['hn24'] not in ['true', 'false']: + raise ValueError("hn24 must be either 'true' or 'false'") + + if config['station']['hn24'] == "true": + station['hn24'] = True + station['num_hrs_to_fetch'] = 24 + else: + station['hn24'] = False + else: + # default to False + station['hn24'] = False + + # Wind mode + if 'wind_mode' in config['station']: + if config['station']['wind_mode'] not in ['normal', 'average']: + raise ValueError("wind_mode must be either 'normal' or 'average'") + + station['wind_mode'] = config['station']['wind_mode'] + + if station['wind_mode'] == "average": + station['num_hrs_to_fetch'] = 24 + else: + # default to False + station['wind_mode'] = "normal" + except KeyError as err: LOG.critical("%s not defined in configuration file", err) sys.exit(1) + except ValueError as err: + LOG.critical("%s", err) + sys.exit(1) # all sections/values present in config file, final sanity check try: @@ -292,7 +328,8 @@ def main(): # Avoid transforming None values if element_cd in ['wind_speed', 'WSPD', 'wind_direction', 'RHUM', 'relative_humidity', 'WDIR', - 'wind_gust', 'SNWD', 'snow_depth']: + 'wind_gust', 'SNWD', 'snow_depth', + 'hn24']: infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd]) elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']: infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1) @@ -404,6 +441,10 @@ def setup_infoex_counterparts_mapping(provider): iemap['wind_speed'] = 'windSpeedNum' iemap['wind_direction'] = 'windDirectionNum' iemap['wind_gust'] = 'windGustSpeedNum' + + # NOTE: this doesn't exist in MesoWest, we create it in this + # program, so add it to the map here + iemap['hn24'] = 'hn24Auto' elif provider == 'python': # we expect Python programs to use the InfoEx data type names iemap['precipitationGauge'] = 'precipitationGauge' @@ -500,8 +541,21 @@ def get_mesowest_data(begin, end, station): LOG.error("Bad JSON in MesoWest response: '%s'", exc) sys.exit(1) + # pos represents the last item in the array, aka the most recent pos = len(observations['date_time']) - 1 + # while these values only apply in certain cases, init them here + wind_speed_values = [] + wind_gust_speed_values = [] + wind_direction_values = [] + hn24_values = [] + + # results + wind_speed_avg = None + wind_gust_speed_avg = None + wind_direction_avg = None + hn24 = None + for element_cd in station['desired_data'].split(','): # sort and isolate the most recent, see note above in NRCS for how and # why this is done @@ -520,23 +574,84 @@ def get_mesowest_data(begin, end, station): key_name = element_cd + '_set_1' if key_name in observations: - if observations[key_name][pos]: - remote_data[element_cd] = observations[key_name][pos] + # val is what will make it into the dataset, after + # conversions... it gets defined here because in certain + # cases we need to look at all of the data to calculate HN24 + # or wind averages, but for the rest of the data, we only + # take the most recent + val = None + + # loop through all observations for this key_name + # record relevant values for wind averaging or hn24, but + # otherwise only persist the data if it's the last datum in + # the set + for idx, _ in enumerate(observations[key_name]): + val = observations[key_name][idx] + + # skip bunk vals + if val is None: + continue # mesowest by default provides wind_speed in m/s, but # we specify 'english' units in the request; either way, # we want mph if element_cd in ('wind_speed', 'wind_gust'): - remote_data[element_cd] = kn_to_mph(remote_data[element_cd]) + val = kn_to_mph(val) # mesowest provides HS in mm, not cm; we want cm if element_cd == 'snow_depth' and station['units'] == 'metric': - remote_data[element_cd] = mm_to_cm(remote_data[element_cd]) - else: + val = mm_to_cm(val) + + # HN24 / wind_mode transformations, once the data has + # completed unit conversions + if station['wind_mode'] == "average": + if element_cd == 'wind_speed' and val is not None: + wind_speed_values.append(val) + elif element_cd == 'wind_gust' and val is not None: + wind_gust_speed_values.append(val) + elif element_cd == 'wind_direction' and val is not None: + wind_direction_values.append(val) + + if element_cd == 'snow_depth': + hn24_values.append(val) + + # again, only persist this datum to the final data if + # it's from the most recent date + if idx == pos: + remote_data[element_cd] = val + + # ensure that the data is filled out + if not observations[key_name][pos]: remote_data[element_cd] = None else: remote_data[element_cd] = None + if len(hn24_values) > 0: + hn24 = max(hn24_values) - min(hn24_values) + + if len(wind_speed_values) > 0: + wind_speed_avg = sum(wind_speed_values) / len(wind_speed_values) + + if len(wind_gust_speed_values) > 0: + wind_gust_speed_avg = sum(wind_gust_speed_values) / len(wind_gust_speed_values) + + if len(wind_direction_values) > 0: + wind_direction_avg = sum(wind_direction_values) / len(wind_direction_values) + + if hn24 is not None: + remote_data['hn24'] = hn24 + + # overwrite the following with the respective averages, if + # applicable + if wind_speed_avg is not None: + remote_data['wind_speed'] = wind_speed_avg + + if wind_gust_speed_avg is not None: + remote_data['wind_gust'] = wind_gust_speed_avg + + if wind_direction_avg is not None: + remote_data['wind_direction'] = wind_direction_avg + return remote_data def switch_units_to_metric(data_map, mapping): @@ -606,7 +721,7 @@ def setup_time_values(station): end_date = date_time - datetime.timedelta(minutes=date_time.minute % 60, seconds=date_time.second, microseconds=date_time.microsecond) - begin_date = end_date - datetime.timedelta(hours=3) + begin_date = end_date - datetime.timedelta(hours=station['num_hrs_to_fetch']) return (begin_date, end_date) def f_to_c(f):