Add 'future plans' to README

[infoex-autowx.git] / infoex-autowx.py

diff --git a/infoex-autowx.py b/infoex-autowx.py
index a4e50a931f2fc9324174a3d9875441b6889833d4..78279f2672cdf0d1f0d2eec5d961a906836fb545 100755 (executable)
--- a/infoex-autowx.py
+++ b/infoex-autowx.py
@@ -2,14 +2,15 @@
 # -*- coding: utf-8 -*-
 
 """
 # -*- coding: utf-8 -*-
 
 """

-InfoEx <-> NRCS Auto Wx implementation
+InfoEx <-> NRCS/MesoWest Auto Wx implementation

 Alexander Vasarab
 Wylark Mountaineering LLC
 
 Alexander Vasarab
 Wylark Mountaineering LLC
 

-Version 1.0.0
+Version 2.0.0

 
 

-This program fetches data from an NRCS SNOTEL site and pushes it to
-InfoEx using the new automated weather system implementation.
+This program fetches data from either an NRCS SNOTEL site or MesoWest
+weather station and pushes it to InfoEx using the new automated weather
+system implementation.

 
 It is designed to be run hourly, and it asks for the last three hours
 of data of each desired type, and selects the most recent one. This
 
 It is designed to be run hourly, and it asks for the last three hours
 of data of each desired type, and selects the most recent one. This


@@ -27,12 +28,16 @@ import configparser
 import csv
 import datetime
 import logging
 import csv
 import datetime
 import logging

+import os
+import sys

 import time
 
 from collections import OrderedDict
 from ftplib import FTP
 from optparse import OptionParser
 
 import time
 
 from collections import OrderedDict
 from ftplib import FTP
 from optparse import OptionParser
 

+import requests
+

 import zeep
 import zeep.cache
 import zeep.transports
 import zeep
 import zeep.cache
 import zeep.transports


@@ -44,43 +49,83 @@ try:
     from systemd.journal import JournalHandler
     log.addHandler(JournalHandler())
 except:
     from systemd.journal import JournalHandler
     log.addHandler(JournalHandler())
 except:

-    # fallback to syslog
-    import logging.handlers
-    log.addHandler(logging.handlers.SysLogHandler())
+    ## fallback to syslog
+    #import logging.handlers
+    #log.addHandler(logging.handlers.SysLogHandler())
+    # fallback to stdout
+    handler = logging.StreamHandler(sys.stdout)
+    log.addHandler(handler)

 
 parser = OptionParser()
 
 parser = OptionParser()

-parser.add_option("--config", dest="config", metavar="FILE", help="location of config file")
+
+parser.add_option("--config",
+    dest="config",
+    metavar="FILE",
+    help="location of config file")
+
+parser.add_option("--dry-run",
+    action="store_true",
+    dest="dry_run",
+    default=False,
+    help="fetch data but don't upload to InfoEx")

 
 (options, args) = parser.parse_args()
 
 config = configparser.ConfigParser(allow_no_value=False)
 
 (options, args) = parser.parse_args()
 
 config = configparser.ConfigParser(allow_no_value=False)

+
+if not options.config:
+    print("Please specify a configuration file via --config.")
+    sys.exit(1)
+

 config.read(options.config)
 
 log.debug('STARTING UP')
 
 config.read(options.config)
 
 log.debug('STARTING UP')
 

-wsdl = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
-

 try:
     infoex = {
 try:
     infoex = {

-        'host': config['ftp']['host'],
-        'uuid': config['ftp']['uuid'],
-        'api_key': config['ftp']['api_key'],
-        'location_uuid': config['wxsite']['location_uuid'],
+        'host': config['infoex']['host'],
+        'uuid': config['infoex']['uuid'],
+        'api_key': config['infoex']['api_key'],
+        'csv_filename': config['infoex']['csv_filename'],
+        'location_uuid': config['infoex']['location_uuid'],

         'wx_data': {}, # placeholder key, values to come later
         'wx_data': {}, # placeholder key, values to come later

-        'csv_filename': config['wxsite']['csv_filename']

     }
 
     }
 

-    station_triplet = config['wxsite']['station_triplet']
+    data = dict()
+    data['provider'] = config['station']['type']
+
+    if data['provider'] not in ['nrcs', 'mesowest']:
+        print("Please specify either nrcs or mesowest as the station type.")
+        sys.exit(1)
+
+    if data['provider'] == 'nrcs':
+        data['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
+        data['station_id'] = config['station']['station_id']
+
+        try:
+            desired_data = config['station']['desired_data'].split(',')
+        except:
+            # desired_data malformed or missing, setting default
+            desired_data = [
+                    'TOBS', # AIR TEMPERATURE OBSERVED (degF)
+                    'SNWD', # SNOW DEPTH (in)
+                    'PREC'  # PRECIPITATION ACCUMULATION (in)
+                    ]
+
+    if data['provider'] == 'mesowest':
+        data['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
+        data['station_id'] = config['station']['station_id']
+        data['units'] = config['station']['units']
+
+        try:
+            desired_data = config['station']['desired_data']
+        except:
+            # desired_data malformed or missing, setting default
+            desired_data = 'air_temp,snow_depth'
+
+        # construct full API URL (sans start/end time, added later)
+        data['source'] = data['source'] + '?token=' + config['station']['token'] + '&within=60&units=' + data['units'] + '&stid=' + data['station_id'] + '&vars=' + desired_data

 
 

-    try:
-        desired_data = config['wxsite']['desired_data'].split(',')
-    except:
-        # desired_data malformed or missing, setting default
-        desired_data = [
-                'TOBS', # AIR TEMPERATURE OBSERVED (degF)
-                'SNWD', # SNOW DEPTH (in)
-                'PREC'  # PRECIPITATION ACCUMULATION (in)
-                ]

 except KeyError as e:
     log.critical("%s not defined in %s" % (e, options.config))
     exit(1)
 except KeyError as e:
     log.critical("%s not defined in %s" % (e, options.config))
     exit(1)


@@ -139,12 +184,29 @@ fmap['hn24AutoUnit'] = 26           ; final_data[26] = 'in'
 fmap['hstAuto'] = 27                ; final_data[27] = None
 fmap['hstAutoUnit'] = 28            ; final_data[28] = 'in'
 
 fmap['hstAuto'] = 27                ; final_data[27] = None
 fmap['hstAutoUnit'] = 28            ; final_data[28] = 'in'
 

-# one final mapping, the NRCS fields that this program supports to
+# one final mapping, the NRCS/MesoWest fields that this program supports to

 # their InfoEx counterpart
 iemap = {}
 # their InfoEx counterpart
 iemap = {}

-iemap['PREC'] = 'precipitationGauge'
-iemap['TOBS'] = 'tempPres'
-iemap['SNWD'] = 'hS'
+
+if data['provider'] == 'nrcs':
+    iemap['PREC'] = 'precipitationGauge'
+    iemap['TOBS'] = 'tempPres'
+    iemap['SNWD'] = 'hS'
+    iemap['PRES'] = 'baro'
+    iemap['RHUM'] = 'rH'
+    iemap['WSPD'] = 'windSpeedNum'
+    iemap['WDIR'] = 'windDirectionNum'
+    # unsupported by NRCS:
+    # windGustSpeedNum
+elif data['provider'] == 'mesowest':
+    iemap['precip_accum'] = 'precipitationGauge'
+    iemap['air_temp'] = 'tempPres'
+    iemap['snow_depth'] = 'hS'
+    iemap['pressure'] = 'baro'
+    iemap['relative_humidity'] = 'rH'
+    iemap['wind_speed'] = 'windSpeedNum'
+    iemap['wind_direction'] = 'windDirectionNum'
+    iemap['wind_gust'] = 'windGustSpeedNum'

 
 # floor time to nearest hour
 dt = datetime.datetime.now()
 
 # floor time to nearest hour
 dt = datetime.datetime.now()


@@ -153,41 +215,93 @@ end_date = dt - datetime.timedelta(minutes=dt.minute % 60,
                                    microseconds=dt.microsecond)
 begin_date = end_date - datetime.timedelta(hours=3)
 
                                    microseconds=dt.microsecond)
 begin_date = end_date - datetime.timedelta(hours=3)
 

-transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
-client = zeep.Client(wsdl=wsdl, transport=transport)
-time_all_elements = time.time()
-
+# get the data

 log.debug("Getting %s data from %s to %s" % (str(desired_data),
     str(begin_date), str(end_date)))
 
 log.debug("Getting %s data from %s to %s" % (str(desired_data),
     str(begin_date), str(end_date)))
 

-for elementCd in desired_data:
-    time_element = time.time()
-
-    # get the last three hours of data for this elementCd
-    tmp = client.service.getHourlyData(
-            stationTriplets=[station_triplet],
-            elementCd=elementCd,
-            ordinal=1,
-            beginDate=begin_date,
-            endDate=end_date)
+time_all_elements = time.time()

 
 

-    log.info("Time to get elementCd '%s': %.3f sec" % (elementCd,
-        time.time() - time_element))
+# NRCS-specific code
+if data['provider'] == 'nrcs':
+    transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
+    client = zeep.Client(wsdl=data['source'], transport=transport)
+
+    for elementCd in desired_data:
+        time_element = time.time()
+
+        # get the last three hours of data for this elementCd
+        tmp = client.service.getHourlyData(
+                stationTriplets=[data['station_id']],
+                elementCd=elementCd,
+                ordinal=1,
+                beginDate=begin_date,
+                endDate=end_date)
+
+        log.info("Time to get elementCd '%s': %.3f sec" % (elementCd,
+            time.time() - time_element))
+
+        values = tmp[0]['values']
+
+        # sort and isolate the most recent
+        #
+        # NOTE: we do this because sometimes there are gaps in hourly data
+        #       in NRCS; yes, we may end up with slightly inaccurate data,
+        #       so perhaps this decision will be re-evaluated in the future
+        if values:
+            ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
+            infoex['wx_data'][elementCd] = ordered[0]['value']
+        else:
+            infoex['wx_data'][elementCd] = None
+
+# MesoWest-specific code
+elif data['provider'] == 'mesowest':
+    # massage begin/end date format
+    begin_date_str = begin_date.strftime('%Y%m%d%H%M')
+    end_date_str = end_date.strftime('%Y%m%d%H%M')
+
+    # construct final, completed API URL
+    api_req_url = data['source'] + '&start=' + begin_date_str + '&end=' + end_date_str
+    req = requests.get(api_req_url)

 
 

-    values = tmp[0]['values']
+    try:
+        json = req.json()
+    except ValueError:
+        log.error("Bad JSON in MesoWest response")
+        sys.exit(1)

 
 

-    # sort and isolate the most recent
-    #
-    # NOTE: we do this because sometimes there are gaps in hourly data
-    #       in NRCS; yes, we may end up with slightly inaccurate data,
-    #       so perhaps this decision will be re-evaluated in the future
-    if values:
-        ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
-        infoex['wx_data'][elementCd] = ordered[0]['value']
-    else:
-        infoex['wx_data'][elementCd] = None
-
-log.info("Time to get all elementCds : %.3f sec" % (time.time() -
+    try:
+        observations = json['STATION'][0]['OBSERVATIONS']
+    except ValueError:
+        log.error("Bad JSON in MesoWest response")
+        sys.exit(1)
+
+    pos = len(observations['date_time']) - 1
+
+    for elementCd in desired_data.split(','):
+        # sort and isolate the most recent, see note above in NRCS for how and
+        # why this is done
+        #
+        # NOTE: Unlike in the NRCS case, the MesoWest API respones contains all
+        #       data (whereas with NRCS, we have to make a separate request for
+        #       each element we want. This is nice for network efficiency but
+        #       it means we have to handle this part differently for each.
+        #
+        # NOTE: Also unlike NRCS, MesoWest provides more granular data; NRCS
+        #       provides hourly data, but MesoWest can often provide data every
+        #       10 minutes -- though this provides more opportunity for
+        #       irregularities
+
+        # we may not have the data at all
+        key_name = elementCd + '_set_1'
+        if key_name in observations:
+            if observations[key_name][pos]:
+                infoex['wx_data'][elementCd] = observations[key_name][pos]
+            else:
+                infoex['wx_data'][elementCd] = None
+        else:
+            infoex['wx_data'][elementCd] = None
+
+log.info("Time to get all data : %.3f sec" % (time.time() -

     time_all_elements))
 
 log.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
     time_all_elements))
 
 log.debug("infoex[wx_data]: %s", str(infoex['wx_data']))


@@ -224,11 +338,14 @@ with open(infoex['csv_filename'], 'w') as f:
     writer.writerow(final_data)
     f.close()
 
     writer.writerow(final_data)
     f.close()
 

-#with open(infoex['csv_filename'], 'rb') as f:
-#    log.debug("uploading FTP file '%s'" % (infoex['host']))
-#    ftp = FTP(infoex['host'], infoex['uuid'], infoex['api_key'])
-#    ftp.storlines('STOR ' + infoex['csv_filename'], f)
-#    ftp.close()
-#    f.close()
+if not options.dry_run:
+    # not a dry run
+    with open(infoex['csv_filename'], 'rb') as f:
+        log.debug("uploading FTP file '%s'" % (infoex['host']))
+        ftp = FTP(infoex['host'], infoex['uuid'], infoex['api_key'])
+        ftp.storlines('STOR ' + infoex['csv_filename'], f)
+        ftp.close()
+        f.close()
+    os.remove(infoex['csv_filename'])

 
 log.debug('DONE')
 
 log.debug('DONE')