import time
from ftplib import FTP
-from optparse import OptionParser
+from argparse import ArgumentParser
import requests
import zeep.cache
import zeep.transports
-__version__ = '2.0.0'
+__version__ = '2.2.0'
LOG = logging.getLogger(__name__)
LOG.setLevel(logging.NOTSET)
def get_parser():
"""Return OptionParser for this program"""
- parser = OptionParser(version=__version__)
+ parser = ArgumentParser()
- parser.add_option("--config",
- dest="config",
- metavar="FILE",
- help="location of config file")
+ parser.add_argument("--version",
+ action="version",
+ version=__version__)
- parser.add_option("--log-level",
- dest="log_level",
- default=None,
- help="set the log level (debug, info, warning)")
+ parser.add_argument("--config",
+ dest="config",
+ metavar="FILE",
+ help="location of config file")
- parser.add_option("--dry-run",
- action="store_true",
- dest="dry_run",
- default=False,
- help="fetch data but don't upload to InfoEx")
+ parser.add_argument("--log-level",
+ dest="log_level",
+ default=None,
+ help="set the log level (debug, info, warning)")
+
+ parser.add_argument("--dry-run",
+ action="store_true",
+ dest="dry_run",
+ default=False,
+ help="fetch data but don't upload to InfoEx")
return parser
station = dict()
station['provider'] = config['station']['type']
- if station['provider'] not in ['nrcs', 'mesowest']:
+ if station['provider'] not in ['nrcs', 'mesowest', 'python']:
print("Please specify either nrcs or mesowest as the station type.")
sys.exit(1)
if station['provider'] == 'nrcs':
station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
station['station_id'] = config['station']['station_id']
-
- try:
- station['desired_data'] = config['station']['desired_data'].split(',')
- except:
- # desired_data malformed or missing, setting default
- station['desired_data'] = [
- 'TOBS', # AIR TEMPERATURE OBSERVED (degF)
- 'SNWD', # SNOW DEPTH (in)
- 'PREC' # PRECIPITATION ACCUMULATION (in)
- ]
+ station['desired_data'] = config['station']['desired_data'].split(',')
# XXX: For NRCS, we're manually overriding units for now! Once
# unit conversion is supported for NRCS, REMOVE THIS!
station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
station['station_id'] = config['station']['station_id']
station['units'] = config['station']['units']
-
- try:
- station['desired_data'] = config['station']['desired_data']
- except:
- # desired_data malformed or missing, setting default
- station['desired_data'] = 'air_temp,snow_depth'
+ station['desired_data'] = config['station']['desired_data']
# construct full API URL (sans start/end time, added later)
- station['source'] = station['source'] + '?token=' + config['station']['token'] + '&within=60&units=' + station['units'] + '&stid=' + station['station_id'] + '&vars=' + station['desired_data']
+ station['source'] = station['source'] + '?token=' + \
+ config['station']['token'] + \
+ '&within=60&units=' + station['units'] + \
+ '&stid=' + station['station_id'] + \
+ '&vars=' + station['desired_data']
+
+ if station['provider'] == 'python':
+ station['path'] = config['station']['path']
- except KeyError as e:
- LOG.critical("%s not defined in %s" % (e, options.config))
- exit(1)
- except Exception as exc:
- LOG.critical("Exception occurred in config parsing: '%s'" % (exc))
- exit(1)
+ except KeyError as err:
+ LOG.critical("%s not defined in configuration file", err)
+ sys.exit(1)
# all sections/values present in config file, final sanity check
try:
for key in config.sections():
for subkey in config[key]:
- if not len(config[key][subkey]):
- raise ValueError;
- except ValueError as exc:
- LOG.critical("Config value '%s.%s' is empty" % (key, subkey))
- exit(1)
+ if not config[key][subkey]:
+ raise ValueError
+ except ValueError:
+ LOG.critical("Config value '%s.%s' is empty", key, subkey)
+ sys.exit(1)
return (infoex, station)
try:
from systemd.journal import JournalHandler
LOG.addHandler(JournalHandler())
- except:
+ except ImportError:
## fallback to syslog
#import logging.handlers
#LOG.addHandler(logging.handlers.SysLogHandler())
def main():
"""Main routine: sort through args, decide what to do, then do it"""
parser = get_parser()
- (options, args) = parser.parse_args()
+ options = parser.parse_args()
config = configparser.ConfigParser(allow_no_value=False)
iemap = setup_infoex_counterparts_mapping(station['provider'])
# override units if user selected metric
- if station['units'] == 'metric':
- final_data = switch_units_to_metric(final_data, fmap)
+ try:
+ if station['units'] == 'metric':
+ final_data = switch_units_to_metric(final_data, fmap)
+ except KeyError:
+ if station['provider'] != 'python':
+ LOG.error("Please specify the units in the configuration "
+ "file")
+ sys.exit(1)
(begin_date, end_date) = setup_time_values()
- # get the data
- LOG.debug("Getting %s data from %s to %s" % (str(station['desired_data']),
- str(begin_date), str(end_date)))
+ if station['provider'] == 'python':
+ LOG.debug("Getting custom data from external Python program")
+ else:
+ LOG.debug("Getting %s data from %s to %s",
+ str(station['desired_data']),
+ str(begin_date), str(end_date))
time_all_elements = time.time()
infoex['wx_data'] = get_nrcs_data(begin_date, end_date, station)
elif station['provider'] == 'mesowest':
infoex['wx_data'] = get_mesowest_data(begin_date, end_date,
- station)
+ station)
+ elif station['provider'] == 'python':
+ try:
+ import importlib.util
+
+ spec = importlib.util.spec_from_file_location('custom_wx',
+ station['path'])
+ mod = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(mod)
+ mod.LOG = LOG
+
+ try:
+ infoex['wx_data'] = mod.get_custom_data()
+
+ if infoex['wx_data'] is None:
+ infoex['wx_data'] = []
+ except Exception:
+ LOG.error("Python program for custom Wx data failed in "
+ "execution")
+ sys.exit(1)
+
+ LOG.info("Successfully executed external Python program")
+ except ImportError:
+ LOG.error("Please upgrade to Python 3.3 or later")
+ sys.exit(1)
+ except FileNotFoundError:
+ LOG.error("Specified Python program for custom Wx data "
+ "was not found")
+ sys.exit(1)
- LOG.info("Time taken to get all data : %.3f sec" % (time.time() -
- time_all_elements))
+ LOG.info("Time taken to get all data : %.3f sec", time.time() -
+ time_all_elements)
LOG.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
final_data[fmap['obDate']] = end_date.strftime('%m/%d/%Y')
final_data[fmap['obTime']] = end_date.strftime('%H:%M')
- for elementCd in infoex['wx_data']:
- if elementCd not in iemap:
- LOG.warning("BAD KEY wx_data['%s']" % (elementCd))
+ for element_cd in infoex['wx_data']:
+ if element_cd not in iemap:
+ LOG.warning("BAD KEY wx_data['%s']", element_cd)
continue
+ # Massage precision of certain values to fit InfoEx's
+ # expectations
+ #
+ # 0 decimal places: wind speed, wind direction, wind gust, snow depth
+ # 1 decimal place: air temp, baro
+ # Avoid transforming None values
+ if infoex['wx_data'][element_cd] is None:
+ continue
+ elif element_cd in ['wind_speed', 'WSPD', 'wind_direction',
+ 'WDIR', 'wind_gust', 'SNWD', 'snow_depth']:
+ infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd])
+ elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']:
+ infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1)
+
# CONSIDER: Casting every value to Float() -- need to investigate if
# any possible elementCds we may want are any other data
# type than float.
# storedUnitCd. But that's pretty network-intensive and
# may not even be worth it if there's only e.g. one or two
# exceptions to any otherwise uniformly Float value set.
- final_data[fmap[iemap[elementCd]]] = infoex['wx_data'][elementCd]
+ final_data[fmap[iemap[element_cd]]] = infoex['wx_data'][element_cd]
- LOG.debug("final_data: %s" % (str(final_data)))
+ LOG.debug("final_data: %s", str(final_data))
- if not write_local_csv(infoex['csv_filename'], final_data):
- LOG.warning('Could not write local CSV file: %s',
- infoex['csv_filename'])
- return 1;
+ if len(infoex['wx_data']) > 0:
+ if not write_local_csv(infoex['csv_filename'], final_data):
+ LOG.warning('Could not write local CSV file: %s',
+ infoex['csv_filename'])
+ return 1
- if not options.dry_run:
- upload_csv(infoex['csv_filename'], infoex)
+ if not options.dry_run:
+ upload_csv(infoex['csv_filename'], infoex)
LOG.debug('DONE')
return 0
Create a mapping of InfoEx fields to the local data's indexing scheme.
INFOEX FIELDS
-
+
This won't earn style points in Python, but here we establish a couple
of helpful mappings variables. The reason this is helpful is that the
end result is simply an ordered set, the CSV file. But we still may
want to manipulate the values arbitrarily before writing that file.
-
+
Also note that the current Auto Wx InfoEx documentation shows these
keys in a graphical table with the "index" beginning at 1, but here we
sanely index beginning at 0.
"""
+ # pylint: disable=too-many-statements,multiple-statements,bad-whitespace
fmap = {} ; final_data = [None] * 29
fmap['Location UUID'] = 0 ; final_data[0] = location_uuid
fmap['obDate'] = 1 ; final_data[1] = None
if provider == 'nrcs':
iemap['PREC'] = 'precipitationGauge'
iemap['TOBS'] = 'tempPres'
+ iemap['TMAX'] = 'tempMaxHour'
+ iemap['TMIN'] = 'tempMinHour'
iemap['SNWD'] = 'hS'
iemap['PRES'] = 'baro'
iemap['RHUM'] = 'rH'
elif provider == 'mesowest':
iemap['precip_accum'] = 'precipitationGauge'
iemap['air_temp'] = 'tempPres'
+ iemap['air_temp_high_24_hour'] = 'tempMaxHour'
+ iemap['air_temp_low_24_hour'] = 'tempMinHour'
iemap['snow_depth'] = 'hS'
iemap['pressure'] = 'baro'
iemap['relative_humidity'] = 'rH'
iemap['wind_speed'] = 'windSpeedNum'
iemap['wind_direction'] = 'windDirectionNum'
iemap['wind_gust'] = 'windGustSpeedNum'
+ elif provider == 'python':
+ # we expect Python programs to use the InfoEx data type names
+ iemap['precipitationGauge'] = 'precipitationGauge'
+ iemap['tempPres'] = 'tempPres'
+ iemap['tempMaxHour'] = 'tempMaxHour'
+ iemap['tempMinHour'] = 'tempMinHour'
+ iemap['hS'] = 'hS'
+ iemap['baro'] = 'baro'
+ iemap['rH'] = 'rH'
+ iemap['windSpeedNum'] = 'windSpeedNum'
+ iemap['windDirectionNum'] = 'windDirectionNum'
+ iemap['windGustSpeedNum'] = 'windGustSpeedNum'
return iemap
client = zeep.Client(wsdl=station['source'], transport=transport)
remote_data = {}
- for elementCd in station['desired_data']:
+ for element_cd in station['desired_data']:
time_element = time.time()
- # get the last three hours of data for this elementCd
+ # get the last three hours of data for this elementCd/element_cd
tmp = client.service.getHourlyData(
- stationTriplets=[station['station_id']],
- elementCd=elementCd,
- ordinal=1,
- beginDate=begin,
- endDate=end)
+ stationTriplets=[station['station_id']],
+ elementCd=element_cd,
+ ordinal=1,
+ beginDate=begin,
+ endDate=end)
- LOG.info("Time to get elementCd '%s': %.3f sec" % (elementCd,
- time.time() - time_element))
+ LOG.info("Time to get NRCS elementCd '%s': %.3f sec", element_cd,
+ time.time() - time_element)
values = tmp[0]['values']
# so perhaps this decision will be re-evaluated in the future
if values:
ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
- remote_data[elementCd] = ordered[0]['value']
+ remote_data[element_cd] = ordered[0]['value']
else:
- remote_data[elementCd] = None
+ remote_data[element_cd] = None
return remote_data
pos = len(observations['date_time']) - 1
- for elementCd in station['desired_data'].split(','):
+ for element_cd in station['desired_data'].split(','):
# sort and isolate the most recent, see note above in NRCS for how and
# why this is done
#
# irregularities
# we may not have the data at all
- key_name = elementCd + '_set_1'
+ key_name = element_cd + '_set_1'
if key_name in observations:
if observations[key_name][pos]:
- remote_data[elementCd] = observations[key_name][pos]
+ remote_data[element_cd] = observations[key_name][pos]
else:
- remote_data[elementCd] = None
+ remote_data[element_cd] = None
else:
- remote_data[elementCd] = None
+ remote_data[element_cd] = None
return remote_data
# CSV operations
def write_local_csv(path_to_file, data):
"""Write the specified CSV file to disk"""
- with open(path_to_file, 'w') as f:
+ with open(path_to_file, 'w') as file_object:
# The requirement is that empty values are represented in the CSV
# file as "", csv.QUOTE_NONNUMERIC achieves that
- LOG.debug("writing CSV file '%s'" % (path_to_file))
- writer = csv.writer(f, quoting=csv.QUOTE_NONNUMERIC)
+ LOG.debug("writing CSV file '%s'", path_to_file)
+ writer = csv.writer(file_object, quoting=csv.QUOTE_NONNUMERIC)
writer.writerow(data)
- f.close()
+ file_object.close()
return True
def upload_csv(path_to_file, infoex_data):
"""Upload the specified CSV file to InfoEx FTP and remove the file"""
with open(path_to_file, 'rb') as file_object:
- LOG.debug("uploading FTP file '%s'" % (infoex_data['host']))
+ LOG.debug("uploading FTP file '%s'", infoex_data['host'])
ftp = FTP(infoex_data['host'], infoex_data['uuid'],
infoex_data['api_key'])
ftp.storlines('STOR ' + path_to_file, file_object)
def setup_time_values():
"""establish time bounds of data request(s)"""
# floor time to nearest hour
- dt = datetime.datetime.now()
- end_date = dt - datetime.timedelta(minutes=dt.minute % 60,
- seconds=dt.second,
- microseconds=dt.microsecond)
+ date_time = datetime.datetime.now()
+ end_date = date_time - datetime.timedelta(minutes=date_time.minute % 60,
+ seconds=date_time.second,
+ microseconds=date_time.microsecond)
begin_date = end_date - datetime.timedelta(hours=3)
return (begin_date, end_date)
wylark / infoex-autowx.git / blobdiff