X-Git-Url: https://wylark.com/src/infoex-autowx.git/blobdiff_plain/4847e91a9b3981136e2f0cebd544efb67b8e872b..ab9d94ed5106de49bd048896e8f4c7e00c0398d5:/infoex-autowx.py?ds=sidebyside diff --git a/infoex-autowx.py b/infoex-autowx.py index c5ec48d..f2be003 100755 --- a/infoex-autowx.py +++ b/infoex-autowx.py @@ -33,13 +33,15 @@ import time from ftplib import FTP from argparse import ArgumentParser +import pytz + import requests import zeep import zeep.cache import zeep.transports -__version__ = '2.0.2' +__version__ = '3.1.0' LOG = logging.getLogger(__name__) LOG.setLevel(logging.NOTSET) @@ -85,7 +87,7 @@ def setup_config(config): station = dict() station['provider'] = config['station']['type'] - if station['provider'] not in ['nrcs', 'mesowest']: + if station['provider'] not in ['nrcs', 'mesowest', 'python']: print("Please specify either nrcs or mesowest as the station type.") sys.exit(1) @@ -112,9 +114,23 @@ def setup_config(config): '&stid=' + station['station_id'] + \ '&vars=' + station['desired_data'] + if station['provider'] == 'python': + station['path'] = config['station']['path'] + + tz = 'America/Los_Angeles' + + if 'tz' in config['station']: + tz = config['station']['tz'] + + try: + station['tz'] = pytz.timezone(tz) + except pytz.exceptions.UnknownTimeZoneError: + LOG.critical("%s is not a valid timezone", tz) + sys.exit(1) + except KeyError as err: LOG.critical("%s not defined in configuration file", err) - exit(1) + sys.exit(1) # all sections/values present in config file, final sanity check try: @@ -124,7 +140,7 @@ def setup_config(config): raise ValueError except ValueError: LOG.critical("Config value '%s.%s' is empty", key, subkey) - exit(1) + sys.exit(1) return (infoex, station) @@ -184,14 +200,23 @@ def main(): iemap = setup_infoex_counterparts_mapping(station['provider']) # override units if user selected metric - if station['units'] == 'metric': - final_data = switch_units_to_metric(final_data, fmap) + try: + if station['units'] == 'metric': + final_data = switch_units_to_metric(final_data, fmap) + except KeyError: + if station['provider'] != 'python': + LOG.error("Please specify the units in the configuration " + "file") + sys.exit(1) - (begin_date, end_date) = setup_time_values() + (begin_date, end_date) = setup_time_values(station) - # get the data - LOG.debug("Getting %s data from %s to %s", str(station['desired_data']), - str(begin_date), str(end_date)) + if station['provider'] == 'python': + LOG.debug("Getting custom data from external Python program") + else: + LOG.debug("Getting %s data from %s to %s (%s)", + str(station['desired_data']), + str(begin_date), str(end_date), end_date.tzinfo.zone) time_all_elements = time.time() @@ -201,23 +226,73 @@ def main(): elif station['provider'] == 'mesowest': infoex['wx_data'] = get_mesowest_data(begin_date, end_date, station) + elif station['provider'] == 'python': + try: + import importlib.util + + spec = importlib.util.spec_from_file_location('custom_wx', + station['path']) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + mod.LOG = LOG + + try: + infoex['wx_data'] = mod.get_custom_data() + + if infoex['wx_data'] is None: + infoex['wx_data'] = [] + except Exception as exc: + LOG.error("Python program for custom Wx data failed in " + "execution: %s", str(exc)) + sys.exit(1) + + LOG.info("Successfully executed external Python program") + except ImportError: + LOG.error("Please upgrade to Python 3.3 or later") + sys.exit(1) + except FileNotFoundError: + LOG.error("Specified Python program for custom Wx data " + "was not found") + sys.exit(1) + except Exception as exc: + LOG.error("A problem was encountered when attempting to " + "load your custom Wx program: %s", str(exc)) + sys.exit(1) LOG.info("Time taken to get all data : %.3f sec", time.time() - time_all_elements) LOG.debug("infoex[wx_data]: %s", str(infoex['wx_data'])) + # timezone massaging + final_end_date = end_date.astimezone(station['tz']) + # Now we only need to add in what we want to change thanks to that # abomination of a variable declaration earlier final_data[fmap['Location UUID']] = infoex['location_uuid'] - final_data[fmap['obDate']] = end_date.strftime('%m/%d/%Y') - final_data[fmap['obTime']] = end_date.strftime('%H:%M') + final_data[fmap['obDate']] = final_end_date.strftime('%m/%d/%Y') + final_data[fmap['obTime']] = final_end_date.strftime('%H:%M') for element_cd in infoex['wx_data']: if element_cd not in iemap: LOG.warning("BAD KEY wx_data['%s']", element_cd) continue + # Massage precision of certain values to fit InfoEx's + # expectations + # + # 0 decimal places: wind speed, wind direction, wind gust, snow depth + # 1 decimal place: air temp, relative humidity, baro + # Avoid transforming None values + if infoex['wx_data'][element_cd] is None: + continue + elif element_cd in ['wind_speed', 'WSPD', 'wind_direction', + 'WDIR', 'wind_gust', 'SNWD', 'snow_depth']: + infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd]) + elif element_cd in ['TOBS', 'air_temp', 'RHUM', + 'relative_humidity', 'PRES', 'pressure']: + infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1) + # CONSIDER: Casting every value to Float() -- need to investigate if # any possible elementCds we may want are any other data # type than float. @@ -231,13 +306,14 @@ def main(): LOG.debug("final_data: %s", str(final_data)) - if not write_local_csv(infoex['csv_filename'], final_data): - LOG.warning('Could not write local CSV file: %s', - infoex['csv_filename']) - return 1 + if infoex['wx_data']: + if not write_local_csv(infoex['csv_filename'], final_data): + LOG.warning('Could not write local CSV file: %s', + infoex['csv_filename']) + return 1 - if not options.dry_run: - upload_csv(infoex['csv_filename'], infoex) + if not options.dry_run: + upload_csv(infoex['csv_filename'], infoex) LOG.debug('DONE') return 0 @@ -302,6 +378,8 @@ def setup_infoex_counterparts_mapping(provider): if provider == 'nrcs': iemap['PREC'] = 'precipitationGauge' iemap['TOBS'] = 'tempPres' + iemap['TMAX'] = 'tempMaxHour' + iemap['TMIN'] = 'tempMinHour' iemap['SNWD'] = 'hS' iemap['PRES'] = 'baro' iemap['RHUM'] = 'rH' @@ -312,12 +390,26 @@ def setup_infoex_counterparts_mapping(provider): elif provider == 'mesowest': iemap['precip_accum'] = 'precipitationGauge' iemap['air_temp'] = 'tempPres' + iemap['air_temp_high_24_hour'] = 'tempMaxHour' + iemap['air_temp_low_24_hour'] = 'tempMinHour' iemap['snow_depth'] = 'hS' iemap['pressure'] = 'baro' iemap['relative_humidity'] = 'rH' iemap['wind_speed'] = 'windSpeedNum' iemap['wind_direction'] = 'windDirectionNum' iemap['wind_gust'] = 'windGustSpeedNum' + elif provider == 'python': + # we expect Python programs to use the InfoEx data type names + iemap['precipitationGauge'] = 'precipitationGauge' + iemap['tempPres'] = 'tempPres' + iemap['tempMaxHour'] = 'tempMaxHour' + iemap['tempMinHour'] = 'tempMinHour' + iemap['hS'] = 'hS' + iemap['baro'] = 'baro' + iemap['rH'] = 'rH' + iemap['windSpeedNum'] = 'windSpeedNum' + iemap['windDirectionNum'] = 'windDirectionNum' + iemap['windGustSpeedNum'] = 'windGustSpeedNum' return iemap @@ -328,6 +420,10 @@ def get_nrcs_data(begin, end, station): client = zeep.Client(wsdl=station['source'], transport=transport) remote_data = {} + # massage begin/end date format + begin_date_str = begin.strftime('%Y-%m-%d %H:%M:00') + end_date_str = end.strftime('%Y-%m-%d %H:%M:00') + for element_cd in station['desired_data']: time_element = time.time() @@ -336,8 +432,8 @@ def get_nrcs_data(begin, end, station): stationTriplets=[station['station_id']], elementCd=element_cd, ordinal=1, - beginDate=begin, - endDate=end) + beginDate=begin_date_str, + endDate=end_date_str) LOG.info("Time to get NRCS elementCd '%s': %.3f sec", element_cd, time.time() - time_element) @@ -399,9 +495,16 @@ def get_mesowest_data(begin, end, station): # we may not have the data at all key_name = element_cd + '_set_1' + if key_name in observations: if observations[key_name][pos]: remote_data[element_cd] = observations[key_name][pos] + + # mesowest by default provides wind_speed in m/s, but + # we specify 'english' units in the request; either way, + # we want mph + if element_cd in ('wind_speed', 'wind_gust'): + remote_data[element_cd] = kn_to_mph(remote_data[element_cd]) else: remote_data[element_cd] = None else: @@ -450,15 +553,31 @@ def upload_csv(path_to_file, infoex_data): os.remove(path_to_file) # other miscellaneous routines -def setup_time_values(): +def setup_time_values(station): """establish time bounds of data request(s)""" + + # default timezone to UTC (for MesoWest) + tz = pytz.utc + + # but for NRCS, use the config-specified timezone + if station['provider'] == 'nrcs': + tz = station['tz'] + # floor time to nearest hour - date_time = datetime.datetime.now() + date_time = datetime.datetime.now(tz=tz) end_date = date_time - datetime.timedelta(minutes=date_time.minute % 60, seconds=date_time.second, microseconds=date_time.microsecond) begin_date = end_date - datetime.timedelta(hours=3) return (begin_date, end_date) +def ms_to_mph(ms): + """convert meters per second to miles per hour""" + return ms * 2.236936 + +def kn_to_mph(kn): + """convert knots to miles per hour""" + return kn * 1.150779 + if __name__ == "__main__": sys.exit(main())