X-Git-Url: https://wylark.com/src/infoex-autowx.git/blobdiff_plain/d8cdec693274f345101d6a9a5efe337593cf17a7..d0f0a9f2b3f26fce76dedc4e6a510e17f65df24c:/infoex-autowx.py?ds=sidebyside diff --git a/infoex-autowx.py b/infoex-autowx.py index 1cdc513..3d3da27 100755 --- a/infoex-autowx.py +++ b/infoex-autowx.py @@ -43,7 +43,7 @@ import zeep import zeep.cache import zeep.transports -__version__ = '3.1.1' +__version__ = '3.4.1' LOG = logging.getLogger(__name__) LOG.setLevel(logging.NOTSET) @@ -89,28 +89,47 @@ def setup_config(config): } station = dict() + station['provider'] = config['station']['type'] if station['provider'] not in ['nrcs', 'mesowest', 'python']: print("Please specify either nrcs or mesowest as the station type.") sys.exit(1) + # massage units config items first + # + # NOTE: custom providers don't require units to be specified + # because they can do whatever they please with the units + # within their own program + if station['provider'] != "custom": + station['units'] = config['station']['units'] + + if station['units'] not in ['metric', 'english', 'american']: + print("Please specify metric, english, or american for the units.") + sys.exit(1) + else: + # if units are specified as "American" then we simply + # default to metric for the requests + if station['units'] == 'american': + station['units_requested'] = 'metric' + else: + station['units_requested'] = station['units'] + + # massage provider config items if station['provider'] == 'nrcs': station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL' station['station_id'] = config['station']['station_id'] station['desired_data'] = config['station']['desired_data'].split(',') - station['units'] = config['station']['units'] if station['provider'] == 'mesowest': station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries' station['station_id'] = config['station']['station_id'] - station['units'] = config['station']['units'] station['desired_data'] = config['station']['desired_data'] # construct full API URL (sans start/end time, added later) station['source'] = station['source'] + '?token=' + \ config['station']['token'] + \ - '&within=60&units=' + station['units'] + \ + '&within=60&units=' + station['units_requested'] + \ '&stid=' + station['station_id'] + \ '&vars=' + station['desired_data'] @@ -128,9 +147,45 @@ def setup_config(config): LOG.critical("%s is not a valid timezone", tz) sys.exit(1) + # By default, fetch three hours of data + # + # If user wants hn24 or wind averaging, then + # we need more. + station['num_hrs_to_fetch'] = 3 + + # HN24 + if 'hn24' in config['station']: + if config['station']['hn24'] not in ['true', 'false']: + raise ValueError("hn24 must be either 'true' or 'false'") + + if config['station']['hn24'] == "true": + station['hn24'] = True + station['num_hrs_to_fetch'] = 24 + else: + station['hn24'] = False + else: + # default to False + station['hn24'] = False + + # Wind mode + if 'wind_mode' in config['station']: + if config['station']['wind_mode'] not in ['normal', 'average']: + raise ValueError("wind_mode must be either 'normal' or 'average'") + + station['wind_mode'] = config['station']['wind_mode'] + + if station['wind_mode'] == "average": + station['num_hrs_to_fetch'] = 24 + else: + # default to False + station['wind_mode'] = "normal" + except KeyError as err: LOG.critical("%s not defined in configuration file", err) sys.exit(1) + except ValueError as err: + LOG.critical("%s", err) + sys.exit(1) # all sections/values present in config file, final sanity check try: @@ -155,6 +210,11 @@ def setup_logging(log_level): #LOG.addHandler(logging.handlers.SysLogHandler()) # fallback to stdout handler = logging.StreamHandler(sys.stdout) + formatter = logging.Formatter('%(asctime)s.%(msecs)03d ' + '%(levelname)s %(module)s - ' + '%(funcName)s: %(message)s', + '%Y-%m-%d %H:%M:%S') + handler.setFormatter(formatter) LOG.addHandler(handler) # ugly, but passable @@ -200,15 +260,13 @@ def main(): iemap = setup_infoex_counterparts_mapping(station['provider']) # override units if user selected metric - if 'units' not in station: - if station['provider'] != 'python': - LOG.error("Please specify the units in the configuration " - "file") - sys.exit(1) - - if station['units'] == 'metric': + if station['provider'] != 'python' and station['units'] == 'metric': final_data = switch_units_to_metric(final_data, fmap) + # likewise for "American" units + if station['provider'] != 'python' and station['units'] == 'american': + final_data = switch_units_to_american(final_data, fmap) + (begin_date, end_date) = setup_time_values(station) if station['provider'] == 'python': @@ -277,10 +335,16 @@ def main(): LOG.warning("BAD KEY wx_data['%s']", element_cd) continue + if infoex['wx_data'][element_cd] is None: + continue + # do the conversion before the rounding if station['provider'] == 'nrcs' and station['units'] == 'metric': infoex['wx_data'][element_cd] = convert_nrcs_units_to_metric(element_cd, infoex['wx_data'][element_cd]) + if station['provider'] != 'custom' and station['units'] == 'american': + infoex['wx_data'][element_cd] = convert_units_to_american(element_cd, infoex['wx_data'][element_cd]) + # Massage precision of certain values to fit InfoEx's # expectations # @@ -288,14 +352,15 @@ def main(): # direction, wind gust, snow depth # 1 decimal place: air temp, baro # Avoid transforming None values - if infoex['wx_data'][element_cd] is None: - continue - elif element_cd in ['wind_speed', 'WSPD', 'wind_direction', - 'RHUM', 'relative_humidity', 'WDIR', - 'wind_gust', 'SNWD', 'snow_depth']: + if element_cd in ['wind_speed', 'WSPD', 'wind_direction', + 'RHUM', 'relative_humidity', 'WDIR', + 'wind_gust', 'SNWD', 'snow_depth', + 'hn24']: infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd]) elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']: infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1) + elif element_cd in ['PREC', 'precip_accum']: + infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 2) # CONSIDER: Casting every value to Float() -- need to investigate if # any possible elementCds we may want are any other data @@ -391,6 +456,10 @@ def setup_infoex_counterparts_mapping(provider): iemap['WDIR'] = 'windDirectionNum' # unsupported by NRCS: # windGustSpeedNum + + # NOTE: this doesn't exist in NRCS SNOTEL, we create it in this + # program, so add it to the map here + iemap['hn24'] = 'hn24Auto' elif provider == 'mesowest': iemap['precip_accum'] = 'precipitationGauge' iemap['air_temp'] = 'tempPres' @@ -402,6 +471,10 @@ def setup_infoex_counterparts_mapping(provider): iemap['wind_speed'] = 'windSpeedNum' iemap['wind_direction'] = 'windDirectionNum' iemap['wind_gust'] = 'windGustSpeedNum' + + # NOTE: this doesn't exist in MesoWest, we create it in this + # program, so add it to the map here + iemap['hn24'] = 'hn24Auto' elif provider == 'python': # we expect Python programs to use the InfoEx data type names iemap['precipitationGauge'] = 'precipitationGauge' @@ -456,6 +529,44 @@ def get_nrcs_data(begin, end, station): else: remote_data[element_cd] = None + + # calc hn24, if applicable + hn24 = None + + if station['hn24']: + hn24_values = [] + + if element_cd == "SNWD": + for idx, _ in enumerate(values): + val = values[idx] + if val is None: + continue + hn24_values.append(val['value']) + + if len(hn24_values) > 0: + # instead of taking MAX - MIN, we want the first + # value (most distant) - the last value (most + # recent) + # + # if the result is positive, then we have + # settlement; if it's not, then we have HN24 + hn24 = hn24_values[0] - hn24_values[len(hn24_values)-1] + + if hn24 < 0.0: + hn24 = abs(hn24) + else: + # this case represents HS settlement + hn24 = 0.0 + + # finally, if user wants hn24 and it's set to None at this + # point, then force it to 0.0 + if hn24 is None: + hn24 = 0.0 + + if hn24 is not None: + if station['hn24']: + remote_data['hn24'] = hn24 + return remote_data def get_mesowest_data(begin, end, station): @@ -468,7 +579,12 @@ def get_mesowest_data(begin, end, station): # construct final, completed API URL api_req_url = station['source'] + '&start=' + begin_date_str + '&end=' + end_date_str - req = requests.get(api_req_url) + + try: + req = requests.get(api_req_url) + except requests.exceptions.ConnectionError: + LOG.error("Could not connect to '%s'", api_req_url) + sys.exit(1) try: json = req.json() @@ -493,8 +609,21 @@ def get_mesowest_data(begin, end, station): LOG.error("Bad JSON in MesoWest response: '%s'", exc) sys.exit(1) + # pos represents the last item in the array, aka the most recent pos = len(observations['date_time']) - 1 + # while these values only apply in certain cases, init them here + wind_speed_values = [] + wind_gust_speed_values = [] + wind_direction_values = [] + hn24_values = [] + + # results + wind_speed_avg = None + wind_gust_speed_avg = None + wind_direction_avg = None + hn24 = None + for element_cd in station['desired_data'].split(','): # sort and isolate the most recent, see note above in NRCS for how and # why this is done @@ -513,19 +642,102 @@ def get_mesowest_data(begin, end, station): key_name = element_cd + '_set_1' if key_name in observations: - if observations[key_name][pos]: - remote_data[element_cd] = observations[key_name][pos] + # val is what will make it into the dataset, after + # conversions... it gets defined here because in certain + # cases we need to look at all of the data to calculate HN24 + # or wind averages, but for the rest of the data, we only + # take the most recent + val = None + + # loop through all observations for this key_name + # record relevant values for wind averaging or hn24, but + # otherwise only persist the data if it's the last datum in + # the set + for idx, _ in enumerate(observations[key_name]): + val = observations[key_name][idx] + + # skip bunk vals + if val is None: + continue # mesowest by default provides wind_speed in m/s, but # we specify 'english' units in the request; either way, # we want mph if element_cd in ('wind_speed', 'wind_gust'): - remote_data[element_cd] = kn_to_mph(remote_data[element_cd]) - else: + val = kn_to_mph(val) + + # mesowest provides HS in mm, not cm; we want cm + if element_cd == 'snow_depth' and station['units'] == 'metric': + val = mm_to_cm(val) + + # HN24 / wind_mode transformations, once the data has + # completed unit conversions + if station['wind_mode'] == "average": + if element_cd == 'wind_speed' and val is not None: + wind_speed_values.append(val) + elif element_cd == 'wind_gust' and val is not None: + wind_gust_speed_values.append(val) + elif element_cd == 'wind_direction' and val is not None: + wind_direction_values.append(val) + + if element_cd == 'snow_depth': + hn24_values.append(val) + + # again, only persist this datum to the final data if + # it's from the most recent date + if idx == pos: + remote_data[element_cd] = val + + # ensure that the data is filled out + if not observations[key_name][pos]: remote_data[element_cd] = None else: remote_data[element_cd] = None + if len(hn24_values) > 0: + # instead of taking MAX - MIN, we want the first value (most + # distant) - the last value (most recent) + # + # if the result is positive, then we have settlement; if it's not, + # then we have HN24 + hn24 = hn24_values[0] - hn24_values[len(hn24_values)-1] + + if hn24 < 0.0: + hn24 = abs(hn24) + else: + # this case represents HS settlement + hn24 = 0.0 + + + # finally, if user wants hn24 and it's set to None at this + # point, then force it to 0.0 + if station['hn24'] and hn24 is None: + hn24 = 0.0 + + if len(wind_speed_values) > 0: + wind_speed_avg = sum(wind_speed_values) / len(wind_speed_values) + + if len(wind_gust_speed_values) > 0: + wind_gust_speed_avg = sum(wind_gust_speed_values) / len(wind_gust_speed_values) + + if len(wind_direction_values) > 0: + wind_direction_avg = sum(wind_direction_values) / len(wind_direction_values) + + if hn24 is not None: + if station['hn24']: + remote_data['hn24'] = hn24 + + # overwrite the following with the respective averages, if + # applicable + if wind_speed_avg is not None: + remote_data['wind_speed'] = wind_speed_avg + + if wind_gust_speed_avg is not None: + remote_data['wind_gust'] = wind_gust_speed_avg + + if wind_direction_avg is not None: + remote_data['wind_direction'] = wind_direction_avg + return remote_data def switch_units_to_metric(data_map, mapping): @@ -546,6 +758,33 @@ def switch_units_to_metric(data_map, mapping): return data_map +def switch_units_to_american(data_map, mapping): + """ + replace units with the American mixture of metric and imperial + + Precip values = metric + Wind values = imperial + """ + + # precip values + data_map[mapping['tempMaxHourUnit']] = 'C' + data_map[mapping['tempMinHourUnit']] = 'C' + data_map[mapping['tempPresUnit']] = 'C' + data_map[mapping['dewPointUnit']] = 'C' + + data_map[mapping['precipitationGaugeUnit']] = 'cm' + data_map[mapping['hsUnit']] = 'cm' + data_map[mapping['hn24AutoUnit']] = 'cm' + data_map[mapping['hstAutoUnit']] = 'cm' + + data_map[mapping['baroUnit']] = 'inHg' + + # wind values + data_map[mapping['windSpeedUnit']] = 'mph' + data_map[mapping['windGustSpeedNumUnit']] = 'mph' + + return data_map + def convert_nrcs_units_to_metric(element_cd, value): """convert NRCS values from English to metric""" if element_cd == 'TOBS': @@ -556,6 +795,35 @@ def convert_nrcs_units_to_metric(element_cd, value): value = in_to_mm(value) return value +def convert_units_to_american(element_cd, value): + """ + convert value to 'American' units + + The original unit is always metric. + + Precip values = metric + Wind values = imperial + """ + + # no need to convert precip values, as they will arrive in metric + # units in "American" units mode + # # temp values + # if element_cd in ['TMAX', 'TMIN', 'TOBS', 'air_temp', 'air_temp_high_24_hour', 'air_temp_low_24_hour']: + # value = c_to_f(value) + + # # snow values + # if element_cd in ['SNWD', 'snow_depth']: + # value = cm_to_in(mm_to_cm(value)) + + # baro values also arrive in metric, so convert to imperial + if element_cd in ['PRES', 'pressure']: + value = inhg_to_pascal(value) + + if element_cd in ['WSPD', 'wind_speed', 'wind_gust']: + value = ms_to_mph(value) + + return value + # CSV operations def write_local_csv(path_to_file, data): """Write the specified CSV file to disk""" @@ -595,17 +863,33 @@ def setup_time_values(station): end_date = date_time - datetime.timedelta(minutes=date_time.minute % 60, seconds=date_time.second, microseconds=date_time.microsecond) - begin_date = end_date - datetime.timedelta(hours=3) + begin_date = end_date - datetime.timedelta(hours=station['num_hrs_to_fetch']) return (begin_date, end_date) def f_to_c(f): """convert Fahrenheit to Celsius""" return (float(f) - 32) * 5.0/9.0 +def c_to_f(c): + """convert Celsius to Fahrenheit""" + return (float(c) * 1.8) + 32 + def in_to_cm(inches): - """convert inches to centimetrs""" + """convert inches to centimeters""" return float(inches) * 2.54 +def cm_to_in(cms): + """convert centimeters to inches""" + return float(cms) / 2.54 + +def pascal_to_inhg(pa): + """convert pascals to inches of mercury""" + return float(pa) * 0.00029530 + +def inhg_to_pascal(inhg): + """convert inches of mercury to pascals""" + return float(inhg) / 0.00029530 + def in_to_mm(inches): """convert inches to millimeters""" return (float(inches) * 2.54) * 10.0 @@ -618,5 +902,9 @@ def kn_to_mph(kn): """convert knots to miles per hour""" return kn * 1.150779 +def mm_to_cm(mm): + """convert millimeters to centimeters""" + return mm / 10.0 + if __name__ == "__main__": sys.exit(main())