import os
import sys
import time
+import urllib3
+import importlib.util
from ftplib import FTP
from argparse import ArgumentParser
+import pytz
+
import requests
import zeep
import zeep.cache
import zeep.transports
-__version__ = '2.0.2'
+__version__ = '3.1.1'
LOG = logging.getLogger(__name__)
LOG.setLevel(logging.NOTSET)
+urllib3.disable_warnings()
+
def get_parser():
"""Return OptionParser for this program"""
parser = ArgumentParser()
station = dict()
station['provider'] = config['station']['type']
- if station['provider'] not in ['nrcs', 'mesowest']:
+ if station['provider'] not in ['nrcs', 'mesowest', 'python']:
print("Please specify either nrcs or mesowest as the station type.")
sys.exit(1)
station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
station['station_id'] = config['station']['station_id']
station['desired_data'] = config['station']['desired_data'].split(',')
-
- # XXX: For NRCS, we're manually overriding units for now! Once
- # unit conversion is supported for NRCS, REMOVE THIS!
- if 'units' not in station:
- station['units'] = 'imperial'
+ station['units'] = config['station']['units']
if station['provider'] == 'mesowest':
station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
'&stid=' + station['station_id'] + \
'&vars=' + station['desired_data']
+ if station['provider'] == 'python':
+ station['path'] = config['station']['path']
+
+ tz = 'America/Los_Angeles'
+
+ if 'tz' in config['station']:
+ tz = config['station']['tz']
+
+ try:
+ station['tz'] = pytz.timezone(tz)
+ except pytz.exceptions.UnknownTimeZoneError:
+ LOG.critical("%s is not a valid timezone", tz)
+ sys.exit(1)
+
except KeyError as err:
LOG.critical("%s not defined in configuration file", err)
- exit(1)
+ sys.exit(1)
# all sections/values present in config file, final sanity check
try:
raise ValueError
except ValueError:
LOG.critical("Config value '%s.%s' is empty", key, subkey)
- exit(1)
+ sys.exit(1)
return (infoex, station)
iemap = setup_infoex_counterparts_mapping(station['provider'])
# override units if user selected metric
+ if 'units' not in station:
+ if station['provider'] != 'python':
+ LOG.error("Please specify the units in the configuration "
+ "file")
+ sys.exit(1)
+
if station['units'] == 'metric':
final_data = switch_units_to_metric(final_data, fmap)
- (begin_date, end_date) = setup_time_values()
+ (begin_date, end_date) = setup_time_values(station)
- # get the data
- LOG.debug("Getting %s data from %s to %s", str(station['desired_data']),
- str(begin_date), str(end_date))
+ if station['provider'] == 'python':
+ LOG.debug("Getting custom data from external Python program")
+ else:
+ LOG.debug("Getting %s data from %s to %s (%s)",
+ str(station['desired_data']),
+ str(begin_date), str(end_date), end_date.tzinfo.zone)
time_all_elements = time.time()
elif station['provider'] == 'mesowest':
infoex['wx_data'] = get_mesowest_data(begin_date, end_date,
station)
+ elif station['provider'] == 'python':
+ try:
+ spec = importlib.util.spec_from_file_location('custom_wx',
+ station['path'])
+ mod = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(mod)
+ mod.LOG = LOG
+
+ try:
+ infoex['wx_data'] = mod.get_custom_data()
+
+ if infoex['wx_data'] is None:
+ infoex['wx_data'] = []
+ except Exception as exc:
+ LOG.error("Python program for custom Wx data failed in "
+ "execution: %s", str(exc))
+ sys.exit(1)
+
+ LOG.info("Successfully executed external Python program")
+ except ImportError:
+ LOG.error("Please upgrade to Python 3.3 or later")
+ sys.exit(1)
+ except FileNotFoundError:
+ LOG.error("Specified Python program for custom Wx data "
+ "was not found")
+ sys.exit(1)
+ except Exception as exc:
+ LOG.error("A problem was encountered when attempting to "
+ "load your custom Wx program: %s", str(exc))
+ sys.exit(1)
LOG.info("Time taken to get all data : %.3f sec", time.time() -
time_all_elements)
LOG.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
+ # timezone massaging
+ final_end_date = end_date.astimezone(station['tz'])
+
# Now we only need to add in what we want to change thanks to that
# abomination of a variable declaration earlier
final_data[fmap['Location UUID']] = infoex['location_uuid']
- final_data[fmap['obDate']] = end_date.strftime('%m/%d/%Y')
- final_data[fmap['obTime']] = end_date.strftime('%H:%M')
+ final_data[fmap['obDate']] = final_end_date.strftime('%m/%d/%Y')
+ final_data[fmap['obTime']] = final_end_date.strftime('%H:%M')
+ final_data[fmap['timeZone']] = station['tz'].zone
for element_cd in infoex['wx_data']:
if element_cd not in iemap:
LOG.warning("BAD KEY wx_data['%s']", element_cd)
continue
+ # do the conversion before the rounding
+ if station['provider'] == 'nrcs' and station['units'] == 'metric':
+ infoex['wx_data'][element_cd] = convert_nrcs_units_to_metric(element_cd, infoex['wx_data'][element_cd])
+
# Massage precision of certain values to fit InfoEx's
# expectations
#
- # 0 decimal places: wind speed, wind direction, wind gust, snow depth
+ # 0 decimal places: relative humidity, wind speed, wind
+ # direction, wind gust, snow depth
# 1 decimal place: air temp, baro
# Avoid transforming None values
if infoex['wx_data'][element_cd] is None:
continue
elif element_cd in ['wind_speed', 'WSPD', 'wind_direction',
- 'WDIR', 'wind_gust', 'SNWD', 'snow_depth']:
+ 'RHUM', 'relative_humidity', 'WDIR',
+ 'wind_gust', 'SNWD', 'snow_depth']:
infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd])
elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']:
infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1)
LOG.debug("final_data: %s", str(final_data))
- if not write_local_csv(infoex['csv_filename'], final_data):
- LOG.warning('Could not write local CSV file: %s',
- infoex['csv_filename'])
- return 1
+ if infoex['wx_data']:
+ if not write_local_csv(infoex['csv_filename'], final_data):
+ LOG.warning('Could not write local CSV file: %s',
+ infoex['csv_filename'])
+ return 1
- if not options.dry_run:
- upload_csv(infoex['csv_filename'], infoex)
+ if not options.dry_run:
+ upload_csv(infoex['csv_filename'], infoex)
LOG.debug('DONE')
return 0
if provider == 'nrcs':
iemap['PREC'] = 'precipitationGauge'
iemap['TOBS'] = 'tempPres'
+ iemap['TMAX'] = 'tempMaxHour'
+ iemap['TMIN'] = 'tempMinHour'
iemap['SNWD'] = 'hS'
iemap['PRES'] = 'baro'
iemap['RHUM'] = 'rH'
elif provider == 'mesowest':
iemap['precip_accum'] = 'precipitationGauge'
iemap['air_temp'] = 'tempPres'
+ iemap['air_temp_high_24_hour'] = 'tempMaxHour'
+ iemap['air_temp_low_24_hour'] = 'tempMinHour'
iemap['snow_depth'] = 'hS'
iemap['pressure'] = 'baro'
iemap['relative_humidity'] = 'rH'
iemap['wind_speed'] = 'windSpeedNum'
iemap['wind_direction'] = 'windDirectionNum'
iemap['wind_gust'] = 'windGustSpeedNum'
+ elif provider == 'python':
+ # we expect Python programs to use the InfoEx data type names
+ iemap['precipitationGauge'] = 'precipitationGauge'
+ iemap['tempPres'] = 'tempPres'
+ iemap['tempMaxHour'] = 'tempMaxHour'
+ iemap['tempMinHour'] = 'tempMinHour'
+ iemap['hS'] = 'hS'
+ iemap['baro'] = 'baro'
+ iemap['rH'] = 'rH'
+ iemap['windSpeedNum'] = 'windSpeedNum'
+ iemap['windDirectionNum'] = 'windDirectionNum'
+ iemap['windGustSpeedNum'] = 'windGustSpeedNum'
return iemap
def get_nrcs_data(begin, end, station):
"""get the data we're after from the NRCS WSDL"""
transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
+ transport.session.verify = False
client = zeep.Client(wsdl=station['source'], transport=transport)
remote_data = {}
+ # massage begin/end date format
+ begin_date_str = begin.strftime('%Y-%m-%d %H:%M:00')
+ end_date_str = end.strftime('%Y-%m-%d %H:%M:00')
+
for element_cd in station['desired_data']:
time_element = time.time()
stationTriplets=[station['station_id']],
elementCd=element_cd,
ordinal=1,
- beginDate=begin,
- endDate=end)
+ beginDate=begin_date_str,
+ endDate=end_date_str)
LOG.info("Time to get NRCS elementCd '%s': %.3f sec", element_cd,
time.time() - time_element)
try:
observations = json['STATION'][0]['OBSERVATIONS']
- except ValueError:
- LOG.error("Bad JSON in MesoWest response")
+ except KeyError as exc:
+ LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
+ sys.exit(1)
+ except IndexError as exc:
+ LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
+ try:
+ LOG.error("Detailed MesoWest response: '%s'",
+ json['SUMMARY']['RESPONSE_MESSAGE'])
+ except KeyError:
+ pass
+ sys.exit(1)
+ except ValueError as exc:
+ LOG.error("Bad JSON in MesoWest response: '%s'", exc)
sys.exit(1)
pos = len(observations['date_time']) - 1
# we may not have the data at all
key_name = element_cd + '_set_1'
+
if key_name in observations:
if observations[key_name][pos]:
remote_data[element_cd] = observations[key_name][pos]
+
+ # mesowest by default provides wind_speed in m/s, but
+ # we specify 'english' units in the request; either way,
+ # we want mph
+ if element_cd in ('wind_speed', 'wind_gust'):
+ remote_data[element_cd] = kn_to_mph(remote_data[element_cd])
else:
remote_data[element_cd] = None
else:
# itself handles the unit conversion; in the future, we will also
# support NRCS unit conversion, but this must be done by this
# program.
+ data_map[mapping['tempMaxHourUnit']] = 'C'
+ data_map[mapping['tempMinHourUnit']] = 'C'
data_map[mapping['tempPresUnit']] = 'C'
- data_map[mapping['hsUnit']] = 'm'
+ data_map[mapping['precipitationGaugeUnit']] = 'C'
+ data_map[mapping['hsUnit']] = 'cm'
data_map[mapping['windSpeedUnit']] = 'm/s'
data_map[mapping['windGustSpeedNumUnit']] = 'm/s'
+ data_map[mapping['dewPointUnit']] = 'C'
+ data_map[mapping['hn24AutoUnit']] = 'cm'
+ data_map[mapping['hstAutoUnit']] = 'cm'
return data_map
+def convert_nrcs_units_to_metric(element_cd, value):
+ """convert NRCS values from English to metric"""
+ if element_cd == 'TOBS':
+ value = f_to_c(value)
+ elif element_cd == 'SNWD':
+ value = in_to_cm(value)
+ elif element_cd == 'PREC':
+ value = in_to_cm(value)
+ return value
+
# CSV operations
def write_local_csv(path_to_file, data):
"""Write the specified CSV file to disk"""
os.remove(path_to_file)
# other miscellaneous routines
-def setup_time_values():
+def setup_time_values(station):
"""establish time bounds of data request(s)"""
+
+ # default timezone to UTC (for MesoWest)
+ tz = pytz.utc
+
+ # but for NRCS, use the config-specified timezone
+ if station['provider'] == 'nrcs':
+ tz = station['tz']
+
# floor time to nearest hour
- date_time = datetime.datetime.now()
+ date_time = datetime.datetime.now(tz=tz)
end_date = date_time - datetime.timedelta(minutes=date_time.minute % 60,
seconds=date_time.second,
microseconds=date_time.microsecond)
begin_date = end_date - datetime.timedelta(hours=3)
return (begin_date, end_date)
+def f_to_c(f):
+ """convert Fahrenheit to Celsius"""
+ return (float(f) - 32) * 5.0/9.0
+
+def in_to_cm(inches):
+ """convert inches to centimetrs"""
+ return float(inches) * 2.54
+
+def ms_to_mph(ms):
+ """convert meters per second to miles per hour"""
+ return ms * 2.236936
+
+def kn_to_mph(kn):
+ """convert knots to miles per hour"""
+ return kn * 1.150779
+
if __name__ == "__main__":
sys.exit(main())
wylark / infoex-autowx.git / blobdiff