X-Git-Url: https://wylark.com/src/infoex-autowx.git/blobdiff_plain/dd8c6cc56ecf1248b52b67c388d0ba18954f6854..cff4d6287459aa92308e558b8e7bf614b03fcb36:/infoex-autowx.py diff --git a/infoex-autowx.py b/infoex-autowx.py index dfb22a7..fd01e04 100755 --- a/infoex-autowx.py +++ b/infoex-autowx.py @@ -29,21 +29,27 @@ import logging import os import sys import time +import urllib3 +import importlib.util from ftplib import FTP from argparse import ArgumentParser +import pytz + import requests import zeep import zeep.cache import zeep.transports -__version__ = '3.0.2' +__version__ = '3.2.3' LOG = logging.getLogger(__name__) LOG.setLevel(logging.NOTSET) +urllib3.disable_warnings() + def get_parser(): """Return OptionParser for this program""" parser = ArgumentParser() @@ -93,11 +99,7 @@ def setup_config(config): station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL' station['station_id'] = config['station']['station_id'] station['desired_data'] = config['station']['desired_data'].split(',') - - # XXX: For NRCS, we're manually overriding units for now! Once - # unit conversion is supported for NRCS, REMOVE THIS! - if 'units' not in station: - station['units'] = 'imperial' + station['units'] = config['station']['units'] if station['provider'] == 'mesowest': station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries' @@ -115,6 +117,17 @@ def setup_config(config): if station['provider'] == 'python': station['path'] = config['station']['path'] + tz = 'America/Los_Angeles' + + if 'tz' in config['station']: + tz = config['station']['tz'] + + try: + station['tz'] = pytz.timezone(tz) + except pytz.exceptions.UnknownTimeZoneError: + LOG.critical("%s is not a valid timezone", tz) + sys.exit(1) + except KeyError as err: LOG.critical("%s not defined in configuration file", err) sys.exit(1) @@ -142,6 +155,11 @@ def setup_logging(log_level): #LOG.addHandler(logging.handlers.SysLogHandler()) # fallback to stdout handler = logging.StreamHandler(sys.stdout) + formatter = logging.Formatter('%(asctime)s.%(msecs)03d ' + '%(levelname)s %(module)s - ' + '%(funcName)s: %(message)s', + '%Y-%m-%d %H:%M:%S') + handler.setFormatter(formatter) LOG.addHandler(handler) # ugly, but passable @@ -187,23 +205,17 @@ def main(): iemap = setup_infoex_counterparts_mapping(station['provider']) # override units if user selected metric - try: - if station['units'] == 'metric': - final_data = switch_units_to_metric(final_data, fmap) - except KeyError: - if station['provider'] != 'python': - LOG.error("Please specify the units in the configuration " - "file") - sys.exit(1) + if station['provider'] != 'python' and station['units'] == 'metric': + final_data = switch_units_to_metric(final_data, fmap) - (begin_date, end_date) = setup_time_values(station['provider']) + (begin_date, end_date) = setup_time_values(station) if station['provider'] == 'python': LOG.debug("Getting custom data from external Python program") else: - LOG.debug("Getting %s data from %s to %s", + LOG.debug("Getting %s data from %s to %s (%s)", str(station['desired_data']), - str(begin_date), str(end_date)) + str(begin_date), str(end_date), end_date.tzinfo.zone) time_all_elements = time.time() @@ -215,8 +227,6 @@ def main(): station) elif station['provider'] == 'python': try: - import importlib.util - spec = importlib.util.spec_from_file_location('custom_wx', station['path']) mod = importlib.util.module_from_spec(spec) @@ -251,31 +261,43 @@ def main(): LOG.debug("infoex[wx_data]: %s", str(infoex['wx_data'])) + # timezone massaging + final_end_date = end_date.astimezone(station['tz']) + # Now we only need to add in what we want to change thanks to that # abomination of a variable declaration earlier final_data[fmap['Location UUID']] = infoex['location_uuid'] - final_data[fmap['obDate']] = end_date.strftime('%m/%d/%Y') - final_data[fmap['obTime']] = end_date.strftime('%H:%M') + final_data[fmap['obDate']] = final_end_date.strftime('%m/%d/%Y') + final_data[fmap['obTime']] = final_end_date.strftime('%H:%M') + final_data[fmap['timeZone']] = station['tz'].zone for element_cd in infoex['wx_data']: if element_cd not in iemap: LOG.warning("BAD KEY wx_data['%s']", element_cd) continue + if infoex['wx_data'][element_cd] is None: + continue + + # do the conversion before the rounding + if station['provider'] == 'nrcs' and station['units'] == 'metric': + infoex['wx_data'][element_cd] = convert_nrcs_units_to_metric(element_cd, infoex['wx_data'][element_cd]) + # Massage precision of certain values to fit InfoEx's # expectations # - # 0 decimal places: wind speed, wind direction, wind gust, snow depth - # 1 decimal place: air temp, relative humidity, baro + # 0 decimal places: relative humidity, wind speed, wind + # direction, wind gust, snow depth + # 1 decimal place: air temp, baro # Avoid transforming None values - if infoex['wx_data'][element_cd] is None: - continue - elif element_cd in ['wind_speed', 'WSPD', 'wind_direction', - 'WDIR', 'wind_gust', 'SNWD', 'snow_depth']: + if element_cd in ['wind_speed', 'WSPD', 'wind_direction', + 'RHUM', 'relative_humidity', 'WDIR', + 'wind_gust', 'SNWD', 'snow_depth']: infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd]) - elif element_cd in ['TOBS', 'air_temp', 'RHUM', - 'relative_humidity', 'PRES', 'pressure']: + elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']: infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1) + elif element_cd in ['PREC', 'precip_accum']: + infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 2) # CONSIDER: Casting every value to Float() -- need to investigate if # any possible elementCds we may want are any other data @@ -401,6 +423,7 @@ def setup_infoex_counterparts_mapping(provider): def get_nrcs_data(begin, end, station): """get the data we're after from the NRCS WSDL""" transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache()) + transport.session.verify = False client = zeep.Client(wsdl=station['source'], transport=transport) remote_data = {} @@ -447,7 +470,12 @@ def get_mesowest_data(begin, end, station): # construct final, completed API URL api_req_url = station['source'] + '&start=' + begin_date_str + '&end=' + end_date_str - req = requests.get(api_req_url) + + try: + req = requests.get(api_req_url) + except requests.exceptions.ConnectionError: + LOG.error("Could not connect to '%s'", api_req_url) + sys.exit(1) try: json = req.json() @@ -457,8 +485,19 @@ def get_mesowest_data(begin, end, station): try: observations = json['STATION'][0]['OBSERVATIONS'] - except ValueError: - LOG.error("Bad JSON in MesoWest response") + except KeyError as exc: + LOG.error("Unexpected JSON in MesoWest response: '%s'", exc) + sys.exit(1) + except IndexError as exc: + LOG.error("Unexpected JSON in MesoWest response: '%s'", exc) + try: + LOG.error("Detailed MesoWest response: '%s'", + json['SUMMARY']['RESPONSE_MESSAGE']) + except KeyError: + pass + sys.exit(1) + except ValueError as exc: + LOG.error("Bad JSON in MesoWest response: '%s'", exc) sys.exit(1) pos = len(observations['date_time']) - 1 @@ -501,18 +540,29 @@ def switch_units_to_metric(data_map, mapping): # NOTE: to update this, use the fmap<->final_data mapping laid out # in setup_infoex_fields_mapping () - # - # NOTE: this only 'works' with MesoWest for now, as the MesoWest API - # itself handles the unit conversion; in the future, we will also - # support NRCS unit conversion, but this must be done by this - # program. + data_map[mapping['tempMaxHourUnit']] = 'C' + data_map[mapping['tempMinHourUnit']] = 'C' data_map[mapping['tempPresUnit']] = 'C' - data_map[mapping['hsUnit']] = 'm' + data_map[mapping['precipitationGaugeUnit']] = 'mm' + data_map[mapping['hsUnit']] = 'cm' data_map[mapping['windSpeedUnit']] = 'm/s' data_map[mapping['windGustSpeedNumUnit']] = 'm/s' + data_map[mapping['dewPointUnit']] = 'C' + data_map[mapping['hn24AutoUnit']] = 'cm' + data_map[mapping['hstAutoUnit']] = 'cm' return data_map +def convert_nrcs_units_to_metric(element_cd, value): + """convert NRCS values from English to metric""" + if element_cd == 'TOBS': + value = f_to_c(value) + elif element_cd == 'SNWD': + value = in_to_cm(value) + elif element_cd == 'PREC': + value = in_to_mm(value) + return value + # CSV operations def write_local_csv(path_to_file, data): """Write the specified CSV file to disk""" @@ -537,14 +587,15 @@ def upload_csv(path_to_file, infoex_data): os.remove(path_to_file) # other miscellaneous routines -def setup_time_values(provider): +def setup_time_values(station): """establish time bounds of data request(s)""" # default timezone to UTC (for MesoWest) - tz = datetime.timezone.utc + tz = pytz.utc - if provider == 'nrcs': - tz = None + # but for NRCS, use the config-specified timezone + if station['provider'] == 'nrcs': + tz = station['tz'] # floor time to nearest hour date_time = datetime.datetime.now(tz=tz) @@ -554,6 +605,18 @@ def setup_time_values(provider): begin_date = end_date - datetime.timedelta(hours=3) return (begin_date, end_date) +def f_to_c(f): + """convert Fahrenheit to Celsius""" + return (float(f) - 32) * 5.0/9.0 + +def in_to_cm(inches): + """convert inches to centimetrs""" + return float(inches) * 2.54 + +def in_to_mm(inches): + """convert inches to millimeters""" + return (float(inches) * 2.54) * 10.0 + def ms_to_mph(ms): """convert meters per second to miles per hour""" return ms * 2.236936