import os
import sys
import time
+import urllib3
+import importlib.util
from ftplib import FTP
from argparse import ArgumentParser
+import pytz
+
import requests
import zeep
import zeep.cache
import zeep.transports
-__version__ = '3.0.1'
+__version__ = '3.2.4'
LOG = logging.getLogger(__name__)
LOG.setLevel(logging.NOTSET)
+urllib3.disable_warnings()
+
def get_parser():
"""Return OptionParser for this program"""
parser = ArgumentParser()
station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
station['station_id'] = config['station']['station_id']
station['desired_data'] = config['station']['desired_data'].split(',')
-
- # XXX: For NRCS, we're manually overriding units for now! Once
- # unit conversion is supported for NRCS, REMOVE THIS!
- if 'units' not in station:
- station['units'] = 'imperial'
+ station['units'] = config['station']['units']
if station['provider'] == 'mesowest':
station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
if station['provider'] == 'python':
station['path'] = config['station']['path']
+ tz = 'America/Los_Angeles'
+
+ if 'tz' in config['station']:
+ tz = config['station']['tz']
+
+ try:
+ station['tz'] = pytz.timezone(tz)
+ except pytz.exceptions.UnknownTimeZoneError:
+ LOG.critical("%s is not a valid timezone", tz)
+ sys.exit(1)
+
+ # By default, fetch three hours of data
+ #
+ # If user wants hn24 or wind averaging, then
+ # we need more.
+ station['num_hrs_to_fetch'] = 3
+
+ # HN24
+ if 'hn24' in config['station']:
+ if config['station']['hn24'] not in ['true', 'false']:
+ raise ValueError("hn24 must be either 'true' or 'false'")
+
+ if config['station']['hn24'] == "true":
+ station['hn24'] = True
+ station['num_hrs_to_fetch'] = 24
+ else:
+ station['hn24'] = False
+ else:
+ # default to False
+ station['hn24'] = False
+
+ # Wind mode
+ if 'wind_mode' in config['station']:
+ if config['station']['wind_mode'] not in ['normal', 'average']:
+ raise ValueError("wind_mode must be either 'normal' or 'average'")
+
+ station['wind_mode'] = config['station']['wind_mode']
+
+ if station['wind_mode'] == "average":
+ station['num_hrs_to_fetch'] = 24
+ else:
+ # default to False
+ station['wind_mode'] = "normal"
+
except KeyError as err:
LOG.critical("%s not defined in configuration file", err)
sys.exit(1)
+ except ValueError as err:
+ LOG.critical("%s", err)
+ sys.exit(1)
# all sections/values present in config file, final sanity check
try:
#LOG.addHandler(logging.handlers.SysLogHandler())
# fallback to stdout
handler = logging.StreamHandler(sys.stdout)
+ formatter = logging.Formatter('%(asctime)s.%(msecs)03d '
+ '%(levelname)s %(module)s - '
+ '%(funcName)s: %(message)s',
+ '%Y-%m-%d %H:%M:%S')
+ handler.setFormatter(formatter)
LOG.addHandler(handler)
# ugly, but passable
iemap = setup_infoex_counterparts_mapping(station['provider'])
# override units if user selected metric
- try:
- if station['units'] == 'metric':
- final_data = switch_units_to_metric(final_data, fmap)
- except KeyError:
- if station['provider'] != 'python':
- LOG.error("Please specify the units in the configuration "
- "file")
- sys.exit(1)
+ if station['provider'] != 'python' and station['units'] == 'metric':
+ final_data = switch_units_to_metric(final_data, fmap)
- (begin_date, end_date) = setup_time_values()
+ (begin_date, end_date) = setup_time_values(station)
if station['provider'] == 'python':
LOG.debug("Getting custom data from external Python program")
else:
- LOG.debug("Getting %s data from %s to %s",
+ LOG.debug("Getting %s data from %s to %s (%s)",
str(station['desired_data']),
- str(begin_date), str(end_date))
+ str(begin_date), str(end_date), end_date.tzinfo.zone)
time_all_elements = time.time()
station)
elif station['provider'] == 'python':
try:
- import importlib.util
-
spec = importlib.util.spec_from_file_location('custom_wx',
station['path'])
mod = importlib.util.module_from_spec(spec)
LOG.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
+ # timezone massaging
+ final_end_date = end_date.astimezone(station['tz'])
+
# Now we only need to add in what we want to change thanks to that
# abomination of a variable declaration earlier
final_data[fmap['Location UUID']] = infoex['location_uuid']
- final_data[fmap['obDate']] = end_date.strftime('%m/%d/%Y')
- final_data[fmap['obTime']] = end_date.strftime('%H:%M')
+ final_data[fmap['obDate']] = final_end_date.strftime('%m/%d/%Y')
+ final_data[fmap['obTime']] = final_end_date.strftime('%H:%M')
+ final_data[fmap['timeZone']] = station['tz'].zone
for element_cd in infoex['wx_data']:
if element_cd not in iemap:
LOG.warning("BAD KEY wx_data['%s']", element_cd)
continue
+ if infoex['wx_data'][element_cd] is None:
+ continue
+
+ # do the conversion before the rounding
+ if station['provider'] == 'nrcs' and station['units'] == 'metric':
+ infoex['wx_data'][element_cd] = convert_nrcs_units_to_metric(element_cd, infoex['wx_data'][element_cd])
+
# Massage precision of certain values to fit InfoEx's
# expectations
#
- # 0 decimal places: wind speed, wind direction, wind gust, snow depth
- # 1 decimal place: air temp, relative humidity, baro
+ # 0 decimal places: relative humidity, wind speed, wind
+ # direction, wind gust, snow depth
+ # 1 decimal place: air temp, baro
# Avoid transforming None values
- if infoex['wx_data'][element_cd] is None:
- continue
- elif element_cd in ['wind_speed', 'WSPD', 'wind_direction',
- 'WDIR', 'wind_gust', 'SNWD', 'snow_depth']:
+ if element_cd in ['wind_speed', 'WSPD', 'wind_direction',
+ 'RHUM', 'relative_humidity', 'WDIR',
+ 'wind_gust', 'SNWD', 'snow_depth',
+ 'hn24']:
infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd])
- elif element_cd in ['TOBS', 'air_temp', 'RHUM',
- 'relative_humidity', 'PRES', 'pressure']:
+ elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']:
infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1)
+ elif element_cd in ['PREC', 'precip_accum']:
+ infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 2)
# CONSIDER: Casting every value to Float() -- need to investigate if
# any possible elementCds we may want are any other data
iemap['wind_speed'] = 'windSpeedNum'
iemap['wind_direction'] = 'windDirectionNum'
iemap['wind_gust'] = 'windGustSpeedNum'
+
+ # NOTE: this doesn't exist in MesoWest, we create it in this
+ # program, so add it to the map here
+ iemap['hn24'] = 'hn24Auto'
elif provider == 'python':
# we expect Python programs to use the InfoEx data type names
iemap['precipitationGauge'] = 'precipitationGauge'
def get_nrcs_data(begin, end, station):
"""get the data we're after from the NRCS WSDL"""
transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
+ transport.session.verify = False
client = zeep.Client(wsdl=station['source'], transport=transport)
remote_data = {}
+ # massage begin/end date format
+ begin_date_str = begin.strftime('%Y-%m-%d %H:%M:00')
+ end_date_str = end.strftime('%Y-%m-%d %H:%M:00')
+
for element_cd in station['desired_data']:
time_element = time.time()
stationTriplets=[station['station_id']],
elementCd=element_cd,
ordinal=1,
- beginDate=begin,
- endDate=end)
+ beginDate=begin_date_str,
+ endDate=end_date_str)
LOG.info("Time to get NRCS elementCd '%s': %.3f sec", element_cd,
time.time() - time_element)
# construct final, completed API URL
api_req_url = station['source'] + '&start=' + begin_date_str + '&end=' + end_date_str
- req = requests.get(api_req_url)
+
+ try:
+ req = requests.get(api_req_url)
+ except requests.exceptions.ConnectionError:
+ LOG.error("Could not connect to '%s'", api_req_url)
+ sys.exit(1)
try:
json = req.json()
try:
observations = json['STATION'][0]['OBSERVATIONS']
- except ValueError:
- LOG.error("Bad JSON in MesoWest response")
+ except KeyError as exc:
+ LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
+ sys.exit(1)
+ except IndexError as exc:
+ LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
+ try:
+ LOG.error("Detailed MesoWest response: '%s'",
+ json['SUMMARY']['RESPONSE_MESSAGE'])
+ except KeyError:
+ pass
+ sys.exit(1)
+ except ValueError as exc:
+ LOG.error("Bad JSON in MesoWest response: '%s'", exc)
sys.exit(1)
+ # pos represents the last item in the array, aka the most recent
pos = len(observations['date_time']) - 1
+ # while these values only apply in certain cases, init them here
+ wind_speed_values = []
+ wind_gust_speed_values = []
+ wind_direction_values = []
+ hn24_values = []
+
+ # results
+ wind_speed_avg = None
+ wind_gust_speed_avg = None
+ wind_direction_avg = None
+ hn24 = None
+
for element_cd in station['desired_data'].split(','):
# sort and isolate the most recent, see note above in NRCS for how and
# why this is done
key_name = element_cd + '_set_1'
if key_name in observations:
- if observations[key_name][pos]:
- remote_data[element_cd] = observations[key_name][pos]
-
- # mesowest provides wind_speed in m/s, we want mph
+ # val is what will make it into the dataset, after
+ # conversions... it gets defined here because in certain
+ # cases we need to look at all of the data to calculate HN24
+ # or wind averages, but for the rest of the data, we only
+ # take the most recent
+ val = None
+
+ # loop through all observations for this key_name
+ # record relevant values for wind averaging or hn24, but
+ # otherwise only persist the data if it's the last datum in
+ # the set
+ for idx, _ in enumerate(observations[key_name]):
+ val = observations[key_name][idx]
+
+ # skip bunk vals
+ if val is None:
+ continue
+
+ # mesowest by default provides wind_speed in m/s, but
+ # we specify 'english' units in the request; either way,
+ # we want mph
if element_cd in ('wind_speed', 'wind_gust'):
- remote_data[element_cd] = ms_to_mph(remote_data[element_cd])
- else:
+ val = kn_to_mph(val)
+
+ # mesowest provides HS in mm, not cm; we want cm
+ if element_cd == 'snow_depth' and station['units'] == 'metric':
+ val = mm_to_cm(val)
+
+ # HN24 / wind_mode transformations, once the data has
+ # completed unit conversions
+ if station['wind_mode'] == "average":
+ if element_cd == 'wind_speed' and val is not None:
+ wind_speed_values.append(val)
+ elif element_cd == 'wind_gust' and val is not None:
+ wind_gust_speed_values.append(val)
+ elif element_cd == 'wind_direction' and val is not None:
+ wind_direction_values.append(val)
+
+ if element_cd == 'snow_depth':
+ hn24_values.append(val)
+
+ # again, only persist this datum to the final data if
+ # it's from the most recent date
+ if idx == pos:
+ remote_data[element_cd] = val
+
+ # ensure that the data is filled out
+ if not observations[key_name][pos]:
remote_data[element_cd] = None
else:
remote_data[element_cd] = None
+ if len(hn24_values) > 0:
+ hn24 = max(hn24_values) - min(hn24_values)
+
+ if len(wind_speed_values) > 0:
+ wind_speed_avg = sum(wind_speed_values) / len(wind_speed_values)
+
+ if len(wind_gust_speed_values) > 0:
+ wind_gust_speed_avg = sum(wind_gust_speed_values) / len(wind_gust_speed_values)
+
+ if len(wind_direction_values) > 0:
+ wind_direction_avg = sum(wind_direction_values) / len(wind_direction_values)
+
+ if hn24 is not None:
+ remote_data['hn24'] = hn24
+
+ # overwrite the following with the respective averages, if
+ # applicable
+ if wind_speed_avg is not None:
+ remote_data['wind_speed'] = wind_speed_avg
+
+ if wind_gust_speed_avg is not None:
+ remote_data['wind_gust'] = wind_gust_speed_avg
+
+ if wind_direction_avg is not None:
+ remote_data['wind_direction'] = wind_direction_avg
+
return remote_data
def switch_units_to_metric(data_map, mapping):
# NOTE: to update this, use the fmap<->final_data mapping laid out
# in setup_infoex_fields_mapping ()
- #
- # NOTE: this only 'works' with MesoWest for now, as the MesoWest API
- # itself handles the unit conversion; in the future, we will also
- # support NRCS unit conversion, but this must be done by this
- # program.
+ data_map[mapping['tempMaxHourUnit']] = 'C'
+ data_map[mapping['tempMinHourUnit']] = 'C'
data_map[mapping['tempPresUnit']] = 'C'
- data_map[mapping['hsUnit']] = 'm'
+ data_map[mapping['precipitationGaugeUnit']] = 'mm'
+ data_map[mapping['hsUnit']] = 'cm'
data_map[mapping['windSpeedUnit']] = 'm/s'
data_map[mapping['windGustSpeedNumUnit']] = 'm/s'
+ data_map[mapping['dewPointUnit']] = 'C'
+ data_map[mapping['hn24AutoUnit']] = 'cm'
+ data_map[mapping['hstAutoUnit']] = 'cm'
return data_map
+def convert_nrcs_units_to_metric(element_cd, value):
+ """convert NRCS values from English to metric"""
+ if element_cd == 'TOBS':
+ value = f_to_c(value)
+ elif element_cd == 'SNWD':
+ value = in_to_cm(value)
+ elif element_cd == 'PREC':
+ value = in_to_mm(value)
+ return value
+
# CSV operations
def write_local_csv(path_to_file, data):
"""Write the specified CSV file to disk"""
os.remove(path_to_file)
# other miscellaneous routines
-def setup_time_values():
+def setup_time_values(station):
"""establish time bounds of data request(s)"""
+
+ # default timezone to UTC (for MesoWest)
+ tz = pytz.utc
+
+ # but for NRCS, use the config-specified timezone
+ if station['provider'] == 'nrcs':
+ tz = station['tz']
+
# floor time to nearest hour
- date_time = datetime.datetime.now()
+ date_time = datetime.datetime.now(tz=tz)
end_date = date_time - datetime.timedelta(minutes=date_time.minute % 60,
seconds=date_time.second,
microseconds=date_time.microsecond)
- begin_date = end_date - datetime.timedelta(hours=3)
+ begin_date = end_date - datetime.timedelta(hours=station['num_hrs_to_fetch'])
return (begin_date, end_date)
+def f_to_c(f):
+ """convert Fahrenheit to Celsius"""
+ return (float(f) - 32) * 5.0/9.0
+
+def in_to_cm(inches):
+ """convert inches to centimetrs"""
+ return float(inches) * 2.54
+
+def in_to_mm(inches):
+ """convert inches to millimeters"""
+ return (float(inches) * 2.54) * 10.0
+
def ms_to_mph(ms):
"""convert meters per second to miles per hour"""
return ms * 2.236936
+def kn_to_mph(kn):
+ """convert knots to miles per hour"""
+ return kn * 1.150779
+
+def mm_to_cm(mm):
+ """convert millimeters to centimetrs"""
+ return mm / 10.0
+
if __name__ == "__main__":
sys.exit(main())
wylark / infoex-autowx.git / blobdiff