X-Git-Url: https://wylark.com/src/infoex-autowx.git/blobdiff_plain/7ec3b0b74884347d7b1c38fa18b4d3879371b495..4b17ea1660c78d5b48295e805eead92f8da6ffe6:/infoex-autowx.py diff --git a/infoex-autowx.py b/infoex-autowx.py index f951c0d..f703c42 100755 --- a/infoex-autowx.py +++ b/infoex-autowx.py @@ -1,37 +1,44 @@ -#!/usr/bin/python3 +#!/usr/bin/env python +# -*- coding: utf-8 -*- -# -# InfoEx <-> NRCS Auto Wx implementation -# Alexander Vasarab -# Wylark Mountaineering LLC -# 2020-04-22 -# -# Version 0.8 -# -# This program fetches data from an NRCS SNOTEL site and pushes it to -# InfoEx using the new automated weather system implementation. -# -# It is designed to be run hourly, and it asks for the last three hours -# of data of each desired type, and selects the most recent one. This -# lends some resiliency to the process and helps ensure that we have a -# value to send, but it can lead to somewhat inconsistent/untruthful -# data if e.g. the HS is from the last hour but the tempPres is from two -# hours ago because the instrumentation had a hiccup. It's worth -# considering if this is a bug or a feature. -# +""" +InfoEx <-> NRCS Auto Wx implementation +Alexander Vasarab +Wylark Mountaineering LLC + +Version 1.0.0 + +This program fetches data from an NRCS SNOTEL site and pushes it to +InfoEx using the new automated weather system implementation. + +It is designed to be run hourly, and it asks for the last three hours +of data of each desired type, and selects the most recent one. This +lends some resiliency to the process and helps ensure that we have a +value to send, but it can lead to somewhat inconsistent/untruthful +data if e.g. the HS is from the last hour but the tempPres is from two +hours ago because the instrumentation had a hiccup. It's worth +considering if this is a bug or a feature. + +For more information, see file: README +For licensing, see file: LICENSE +""" import configparser import csv import datetime import logging +import os +import sys import time -import zeep -import zeep.cache -import zeep.transports + from collections import OrderedDict from ftplib import FTP from optparse import OptionParser +import zeep +import zeep.cache +import zeep.transports + log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -44,11 +51,24 @@ except: log.addHandler(logging.handlers.SysLogHandler()) parser = OptionParser() -parser.add_option("--config", dest="config", metavar="FILE", help="location of config file") +parser.add_option("--config", + dest="config", + metavar="FILE", + help="location of config file") +parser.add_option("--dry-run", + action="store_true", + dest="dry_run", + default=False, + help="fetch data but don't upload to InfoEx") (options, args) = parser.parse_args() config = configparser.ConfigParser(allow_no_value=False) + +if not options.config: + print("Please specify a configuration file via --config") + sys.exit(1) + config.read(options.config) log.debug('STARTING UP') @@ -57,18 +77,18 @@ wsdl = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL' try: infoex = { - 'host': config['ftp']['host'], - 'uuid': config['ftp']['uuid'], - 'api_key': config['ftp']['api_key'], - 'location_uuid': config['wxsite']['location_uuid'], - 'wx_data': {}, - 'csv_filename': config['wxsite']['csv_filename'] + 'host': config['infoex']['host'], + 'uuid': config['infoex']['uuid'], + 'api_key': config['infoex']['api_key'], + 'location_uuid': config['infoex']['location_uuid'], + 'wx_data': {}, # placeholder key, values to come later + 'csv_filename': config['infoex']['csv_filename'] } - station_triplet = config['wxsite']['station_triplet'] + station_triplet = config['nrcs']['station_triplet'] try: - desired_data = config['wxsite']['desired_data'].split(',') + desired_data = config['nrcs']['desired_data'].split(',') except: # desired_data malformed or missing, setting default desired_data = [ @@ -172,6 +192,10 @@ for elementCd in desired_data: values = tmp[0]['values'] # sort and isolate the most recent + # + # NOTE: we do this because sometimes there are gaps in hourly data + # in NRCS; yes, we may end up with slightly inaccurate data, + # so perhaps this decision will be re-evaluated in the future if values: ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True) infoex['wx_data'][elementCd] = ordered[0]['value'] @@ -183,16 +207,12 @@ log.info("Time to get all elementCds : %.3f sec" % (time.time() - log.debug("infoex[wx_data]: %s", str(infoex['wx_data'])) -# Only need to add in what we want to change thanks to that abomination -# of a variable declaration earlier +# Now we only need to add in what we want to change thanks to that +# abomination of a variable declaration earlier final_data[fmap['Location UUID']] = infoex['location_uuid'] final_data[fmap['obDate']] = end_date.strftime('%m/%d/%Y') final_data[fmap['obTime']] = end_date.strftime('%H:%M') -#final_data[fmap['tempPres']] = float(infoex['wx_data']['TOBS']) -#final_data[fmap['precipitationGauge']] = float(infoex['wx_data']['PREC']) -#final_data[fmap['hS']] = float(infoex['wx_data']['SNWD']) - for elementCd in infoex['wx_data']: if elementCd not in iemap: log.warning("BAD KEY wx_data['%s']" % (elementCd)) @@ -219,11 +239,14 @@ with open(infoex['csv_filename'], 'w') as f: writer.writerow(final_data) f.close() -#with open(infoex['csv_filename'], 'rb') as f: -# log.debug("uploading FTP file '%s'" % (infoex['host'])) -# ftp = FTP(infoex['host'], infoex['uuid'], infoex['api_key']) -# ftp.storlines('STOR ' + infoex['csv_filename'], f) -# ftp.close() -# f.close() +if not options.dry_run: + # not a dry run + with open(infoex['csv_filename'], 'rb') as f: + log.debug("uploading FTP file '%s'" % (infoex['host'])) + ftp = FTP(infoex['host'], infoex['uuid'], infoex['api_key']) + ftp.storlines('STOR ' + infoex['csv_filename'], f) + ftp.close() + f.close() + os.remove(infoex['csv_filename']) log.debug('DONE')