-#!/usr/bin/python3
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
-#
-# InfoEx <-> NRCS Auto Wx implementation
-# Alexander Vasarab
-# Wylark Mountaineering LLC
-# 2020-04-22
-#
-# Version 0.8
-#
-# This program fetches data from an NRCS SNOTEL site and pushes it to
-# InfoEx using the new automated weather system implementation.
-#
-# It is designed to be run hourly, and it asks for the last three hours
-# of data of each desired type, and selects the most recent one. This
-# lends some resiliency to the process and helps ensure that we have a
-# value to send, but it can lead to somewhat inconsistent/untruthful
-# data if e.g. the HS is from the last hour but the tempPres is from two
-# hours ago because the instrumentation had a hiccup. It's worth
-# considering if this is a bug or a feature.
-#
+"""
+InfoEx <-> NRCS/MesoWest Auto Wx implementation
+Alexander Vasarab
+Wylark Mountaineering LLC
+
+Version 2.0.0
+
+This program fetches data from either an NRCS SNOTEL site or MesoWest
+weather station and pushes it to InfoEx using the new automated weather
+system implementation.
+
+It is designed to be run hourly, and it asks for the last three hours
+of data of each desired type, and selects the most recent one. This
+lends some resiliency to the process and helps ensure that we have a
+value to send, but it can lead to somewhat inconsistent/untruthful
+data if e.g. the HS is from the last hour but the tempPres is from two
+hours ago because the instrumentation had a hiccup. It's worth
+considering if this is a bug or a feature.
+
+For more information, see file: README
+For licensing, see file: LICENSE
+"""
import configparser
import csv
import datetime
import logging
+import os
+import sys
import time
-import zeep
-import zeep.cache
-import zeep.transports
+
from collections import OrderedDict
from ftplib import FTP
from optparse import OptionParser
+import requests
+
+import zeep
+import zeep.cache
+import zeep.transports
+
+__version__ = '2.0.0'
+
log = logging.getLogger(__name__)
-log.setLevel(logging.DEBUG)
+log.setLevel(logging.NOTSET)
try:
from systemd.journal import JournalHandler
log.addHandler(JournalHandler())
except:
- # fallback to syslog
- import logging.handlers
- log.addHandler(logging.handlers.SysLogHandler())
-
-parser = OptionParser()
-parser.add_option("--config", dest="config", metavar="FILE", help="location of config file")
+ ## fallback to syslog
+ #import logging.handlers
+ #log.addHandler(logging.handlers.SysLogHandler())
+ # fallback to stdout
+ handler = logging.StreamHandler(sys.stdout)
+ log.addHandler(handler)
+
+parser = OptionParser(version=__version__)
+
+parser.add_option("--config",
+ dest="config",
+ metavar="FILE",
+ help="location of config file")
+
+parser.add_option("--log-level",
+ dest="log_level",
+ default=None,
+ help="set the log level (debug, info, warning)")
+
+parser.add_option("--dry-run",
+ action="store_true",
+ dest="dry_run",
+ default=False,
+ help="fetch data but don't upload to InfoEx")
(options, args) = parser.parse_args()
config = configparser.ConfigParser(allow_no_value=False)
+
+if not options.config:
+ parser.print_help()
+ print("\nPlease specify a configuration file via --config.")
+ sys.exit(1)
+
config.read(options.config)
-log.debug('STARTING UP')
+# ugly, but passable
+if options.log_level in [None, 'debug', 'info', 'warning']:
+ if options.log_level == 'debug':
+ log.setLevel(logging.DEBUG)
+ elif options.log_level == 'info':
+ log.setLevel(logging.INFO)
+ elif options.log_level == 'warning':
+ log.setLevel(logging.WARNING)
+ else:
+ log.setLevel(logging.NOTSET)
+else:
+ parser.print_help()
+ print("\nPlease select an appropriate log level or remove the switch (--log-level).")
+ sys.exit(1)
-wsdl = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
+log.debug('STARTING UP')
try:
infoex = {
- 'host': config['ftp']['host'],
- 'uuid': config['ftp']['uuid'],
- 'api_key': config['ftp']['api_key'],
- 'location_uuid': config['wxsite']['location_uuid'],
+ 'host': config['infoex']['host'],
+ 'uuid': config['infoex']['uuid'],
+ 'api_key': config['infoex']['api_key'],
+ 'csv_filename': config['infoex']['csv_filename'],
+ 'location_uuid': config['infoex']['location_uuid'],
'wx_data': {}, # placeholder key, values to come later
- 'csv_filename': config['wxsite']['csv_filename']
}
- station_triplet = config['wxsite']['station_triplet']
+ data = dict()
+ data['provider'] = config['station']['type']
+
+ if data['provider'] not in ['nrcs', 'mesowest']:
+ print("Please specify either nrcs or mesowest as the station type.")
+ sys.exit(1)
+
+ if data['provider'] == 'nrcs':
+ data['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
+ data['station_id'] = config['station']['station_id']
+
+ try:
+ desired_data = config['station']['desired_data'].split(',')
+ except:
+ # desired_data malformed or missing, setting default
+ desired_data = [
+ 'TOBS', # AIR TEMPERATURE OBSERVED (degF)
+ 'SNWD', # SNOW DEPTH (in)
+ 'PREC' # PRECIPITATION ACCUMULATION (in)
+ ]
+
+ # XXX: For NRCS, we're manually overriding units for now! Once
+ # unit conversion is supported for NRCS, REMOVE THIS!
+ if 'units' not in data:
+ data['units'] = 'imperial'
+
+ if data['provider'] == 'mesowest':
+ data['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
+ data['station_id'] = config['station']['station_id']
+ data['units'] = config['station']['units']
+
+ try:
+ desired_data = config['station']['desired_data']
+ except:
+ # desired_data malformed or missing, setting default
+ desired_data = 'air_temp,snow_depth'
+
+ # construct full API URL (sans start/end time, added later)
+ data['source'] = data['source'] + '?token=' + config['station']['token'] + '&within=60&units=' + data['units'] + '&stid=' + data['station_id'] + '&vars=' + desired_data
- try:
- desired_data = config['wxsite']['desired_data'].split(',')
- except:
- # desired_data malformed or missing, setting default
- desired_data = [
- 'TOBS', # AIR TEMPERATURE OBSERVED (degF)
- 'SNWD', # SNOW DEPTH (in)
- 'PREC' # PRECIPITATION ACCUMULATION (in)
- ]
except KeyError as e:
log.critical("%s not defined in %s" % (e, options.config))
exit(1)
fmap['hstAuto'] = 27 ; final_data[27] = None
fmap['hstAutoUnit'] = 28 ; final_data[28] = 'in'
-# one final mapping, the NRCS fields that this program supports to
+# one final mapping, the NRCS/MesoWest fields that this program supports to
# their InfoEx counterpart
iemap = {}
-iemap['PREC'] = 'precipitationGauge'
-iemap['TOBS'] = 'tempPres'
-iemap['SNWD'] = 'hS'
+
+if data['provider'] == 'nrcs':
+ iemap['PREC'] = 'precipitationGauge'
+ iemap['TOBS'] = 'tempPres'
+ iemap['SNWD'] = 'hS'
+ iemap['PRES'] = 'baro'
+ iemap['RHUM'] = 'rH'
+ iemap['WSPD'] = 'windSpeedNum'
+ iemap['WDIR'] = 'windDirectionNum'
+ # unsupported by NRCS:
+ # windGustSpeedNum
+elif data['provider'] == 'mesowest':
+ iemap['precip_accum'] = 'precipitationGauge'
+ iemap['air_temp'] = 'tempPres'
+ iemap['snow_depth'] = 'hS'
+ iemap['pressure'] = 'baro'
+ iemap['relative_humidity'] = 'rH'
+ iemap['wind_speed'] = 'windSpeedNum'
+ iemap['wind_direction'] = 'windDirectionNum'
+ iemap['wind_gust'] = 'windGustSpeedNum'
+
+# override units if user selected metric
+#
+# NOTE: to update this, use the fmap<->final_data mapping laid out above
+#
+# NOTE: this only 'works' with MesoWest for now, as the MesoWest API
+# itself handles the unit conversion; in the future, we will also
+# support NRCS unit conversion, but this must be done by this
+# program.
+if data['units'] == 'metric':
+ final_data[fmap['tempPresUnit']] = 'C'
+ final_data[fmap['hsUnit']] = 'm'
+ final_data[fmap['windSpeedUnit']] = 'm/s'
+ final_data[fmap['windGustSpeedNumUnit']] = 'm/s'
# floor time to nearest hour
dt = datetime.datetime.now()
microseconds=dt.microsecond)
begin_date = end_date - datetime.timedelta(hours=3)
-transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
-client = zeep.Client(wsdl=wsdl, transport=transport)
-time_all_elements = time.time()
-
+# get the data
log.debug("Getting %s data from %s to %s" % (str(desired_data),
str(begin_date), str(end_date)))
-for elementCd in desired_data:
- time_element = time.time()
-
- # get the last three hours of data for this elementCd
- tmp = client.service.getHourlyData(
- stationTriplets=[station_triplet],
- elementCd=elementCd,
- ordinal=1,
- beginDate=begin_date,
- endDate=end_date)
-
- log.info("Time to get elementCd '%s': %.3f sec" % (elementCd,
- time.time() - time_element))
+time_all_elements = time.time()
- values = tmp[0]['values']
+# NRCS-specific code
+if data['provider'] == 'nrcs':
+ transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
+ client = zeep.Client(wsdl=data['source'], transport=transport)
+
+ for elementCd in desired_data:
+ time_element = time.time()
+
+ # get the last three hours of data for this elementCd
+ tmp = client.service.getHourlyData(
+ stationTriplets=[data['station_id']],
+ elementCd=elementCd,
+ ordinal=1,
+ beginDate=begin_date,
+ endDate=end_date)
+
+ log.info("Time to get elementCd '%s': %.3f sec" % (elementCd,
+ time.time() - time_element))
+
+ values = tmp[0]['values']
+
+ # sort and isolate the most recent
+ #
+ # NOTE: we do this because sometimes there are gaps in hourly data
+ # in NRCS; yes, we may end up with slightly inaccurate data,
+ # so perhaps this decision will be re-evaluated in the future
+ if values:
+ ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
+ infoex['wx_data'][elementCd] = ordered[0]['value']
+ else:
+ infoex['wx_data'][elementCd] = None
+
+# MesoWest-specific code
+elif data['provider'] == 'mesowest':
+ # massage begin/end date format
+ begin_date_str = begin_date.strftime('%Y%m%d%H%M')
+ end_date_str = end_date.strftime('%Y%m%d%H%M')
+
+ # construct final, completed API URL
+ api_req_url = data['source'] + '&start=' + begin_date_str + '&end=' + end_date_str
+ req = requests.get(api_req_url)
- # sort and isolate the most recent
- #
- # NOTE: we do this because sometimes there are gaps in hourly data
- # in NRCS; yes, we may end up with slightly inaccurate data,
- # so perhaps this decision will be re-evaluated in the future
- if values:
- ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
- infoex['wx_data'][elementCd] = ordered[0]['value']
- else:
- infoex['wx_data'][elementCd] = None
+ try:
+ json = req.json()
+ except ValueError:
+ log.error("Bad JSON in MesoWest response")
+ sys.exit(1)
-log.info("Time to get all elementCds : %.3f sec" % (time.time() -
+ try:
+ observations = json['STATION'][0]['OBSERVATIONS']
+ except ValueError:
+ log.error("Bad JSON in MesoWest response")
+ sys.exit(1)
+
+ pos = len(observations['date_time']) - 1
+
+ for elementCd in desired_data.split(','):
+ # sort and isolate the most recent, see note above in NRCS for how and
+ # why this is done
+ #
+ # NOTE: Unlike in the NRCS case, the MesoWest API respones contains all
+ # data (whereas with NRCS, we have to make a separate request for
+ # each element we want. This is nice for network efficiency but
+ # it means we have to handle this part differently for each.
+ #
+ # NOTE: Also unlike NRCS, MesoWest provides more granular data; NRCS
+ # provides hourly data, but MesoWest can often provide data every
+ # 10 minutes -- though this provides more opportunity for
+ # irregularities
+
+ # we may not have the data at all
+ key_name = elementCd + '_set_1'
+ if key_name in observations:
+ if observations[key_name][pos]:
+ infoex['wx_data'][elementCd] = observations[key_name][pos]
+ else:
+ infoex['wx_data'][elementCd] = None
+ else:
+ infoex['wx_data'][elementCd] = None
+
+log.info("Time to get all data : %.3f sec" % (time.time() -
time_all_elements))
log.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
writer.writerow(final_data)
f.close()
-#with open(infoex['csv_filename'], 'rb') as f:
-# log.debug("uploading FTP file '%s'" % (infoex['host']))
-# ftp = FTP(infoex['host'], infoex['uuid'], infoex['api_key'])
-# ftp.storlines('STOR ' + infoex['csv_filename'], f)
-# ftp.close()
-# f.close()
+if not options.dry_run:
+ # not a dry run
+ with open(infoex['csv_filename'], 'rb') as f:
+ log.debug("uploading FTP file '%s'" % (infoex['host']))
+ ftp = FTP(infoex['host'], infoex['uuid'], infoex['api_key'])
+ ftp.storlines('STOR ' + infoex['csv_filename'], f)
+ ftp.close()
+ f.close()
+ os.remove(infoex['csv_filename'])
log.debug('DONE')
wylark / infoex-autowx.git / blobdiff