infoex-autowx.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 """
   5 InfoEx <-> NRCS Auto Wx implementation
   6 Alexander Vasarab
   7 Wylark Mountaineering LLC
   8
   9 Version 1.0.0
  10
  11 This program fetches data from an NRCS SNOTEL site and pushes it to
  12 InfoEx using the new automated weather system implementation.
  13
  14 It is designed to be run hourly, and it asks for the last three hours
  15 of data of each desired type, and selects the most recent one. This
  16 lends some resiliency to the process and helps ensure that we have a
  17 value to send, but it can lead to somewhat inconsistent/untruthful
  18 data if e.g. the HS is from the last hour but the tempPres is from two
  19 hours ago because the instrumentation had a hiccup. It's worth
  20 considering if this is a bug or a feature.
  21
  22 For more information, see file: README
  23 For licensing, see file: LICENSE
  24 """
  25
  26 import configparser
  27 import csv
  28 import datetime
  29 import logging
  30 import time
  31
  32 from collections import OrderedDict
  33 from ftplib import FTP
  34 from optparse import OptionParser
  35
  36 import zeep
  37 import zeep.cache
  38 import zeep.transports
  39
  40 log = logging.getLogger(__name__)
  41 log.setLevel(logging.DEBUG)
  42
  43 try:
  44     from systemd.journal import JournalHandler
  45     log.addHandler(JournalHandler())
  46 except:
  47     # fallback to syslog
  48     import logging.handlers
  49     log.addHandler(logging.handlers.SysLogHandler())
  50
  51 parser = OptionParser()
  52 parser.add_option("--config", dest="config", metavar="FILE", help="location of config file")
  53
  54 (options, args) = parser.parse_args()
  55
  56 config = configparser.ConfigParser(allow_no_value=False)
  57 config.read(options.config)
  58
  59 log.debug('STARTING UP')
  60
  61 wsdl = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
  62
  63 try:
  64     infoex = {
  65         'host': config['ftp']['host'],
  66         'uuid': config['ftp']['uuid'],
  67         'api_key': config['ftp']['api_key'],
  68         'location_uuid': config['wxsite']['location_uuid'],
  69         'wx_data': {}, # placeholder key, values to come later
  70         'csv_filename': config['wxsite']['csv_filename']
  71     }
  72
  73     station_triplet = config['wxsite']['station_triplet']
  74
  75     try:
  76         desired_data = config['wxsite']['desired_data'].split(',')
  77     except:
  78         # desired_data malformed or missing, setting default
  79         desired_data = [
  80                 'TOBS', # AIR TEMPERATURE OBSERVED (degF)
  81                 'SNWD', # SNOW DEPTH (in)
  82                 'PREC'  # PRECIPITATION ACCUMULATION (in)
  83                 ]
  84 except KeyError as e:
  85     log.critical("%s not defined in %s" % (e, options.config))
  86     exit(1)
  87 except Exception as exc:
  88     log.critical("Exception occurred in config parsing: '%s'" % (exc))
  89     exit(1)
  90
  91 # all sections/values present in config file, final sanity check
  92 try:
  93     for key in config.sections():
  94         for subkey in config[key]:
  95             if not len(config[key][subkey]):
  96                 raise ValueError;
  97 except ValueError as exc:
  98     log.critical("Config value '%s.%s' is empty" % (key, subkey))
  99     exit(1)
 100
 101 # INFOEX FIELDS
 102 #
 103 # This won't earn style points in Python, but here we establish a couple
 104 # of helpful mappings variables. The reason this is helpful is that the
 105 # end result is simply an ordered set, the CSV file. But we still may
 106 # want to manipulate the values arbitrarily before writing that file.
 107 #
 108 # Also note that the current Auto Wx InfoEx documentation shows these
 109 # keys in a graphical table with the "index" beginning at 1, but here we
 110 # are sanely indexing beginning at 0.
 111 fmap = {}                           ; final_data     = [None] * 29
 112 fmap['Location UUID'] = 0           ; final_data[0]  = infoex['location_uuid']
 113 fmap['obDate'] = 1                  ; final_data[1]  = None
 114 fmap['obTime'] = 2                  ; final_data[2]  = None
 115 fmap['timeZone'] = 3                ; final_data[3]  = 'Pacific'
 116 fmap['tempMaxHour'] = 4             ; final_data[4]  = None
 117 fmap['tempMaxHourUnit'] = 5         ; final_data[5]  = 'F'
 118 fmap['tempMinHour'] = 6             ; final_data[6]  = None
 119 fmap['tempMinHourUnit'] = 7         ; final_data[7]  = 'F'
 120 fmap['tempPres'] = 8                ; final_data[8]  = None
 121 fmap['tempPresUnit'] = 9            ; final_data[9]  = 'F'
 122 fmap['precipitationGauge'] = 10     ; final_data[10] = None
 123 fmap['precipitationGaugeUnit'] = 11 ; final_data[11] = 'in'
 124 fmap['windSpeedNum'] = 12           ; final_data[12] = None
 125 fmap['windSpeedUnit'] = 13          ; final_data[13] = 'mph'
 126 fmap['windDirectionNum'] = 14       ; final_data[14] = None
 127 fmap['hS'] = 15                     ; final_data[15] = None
 128 fmap['hsUnit'] = 16                 ; final_data[16] = 'in'
 129 fmap['baro'] = 17                   ; final_data[17] = None
 130 fmap['baroUnit'] = 18               ; final_data[18] = 'inHg'
 131 fmap['rH'] = 19                     ; final_data[19] = None
 132 fmap['windGustSpeedNum'] = 20       ; final_data[20] = None
 133 fmap['windGustSpeedNumUnit'] = 21   ; final_data[21] = 'mph'
 134 fmap['windGustDirNum'] = 22         ; final_data[22] = None
 135 fmap['dewPoint'] = 23               ; final_data[23] = None
 136 fmap['dewPointUnit'] = 24           ; final_data[24] = 'F'
 137 fmap['hn24Auto'] = 25               ; final_data[25] = None
 138 fmap['hn24AutoUnit'] = 26           ; final_data[26] = 'in'
 139 fmap['hstAuto'] = 27                ; final_data[27] = None
 140 fmap['hstAutoUnit'] = 28            ; final_data[28] = 'in'
 141
 142 # one final mapping, the NRCS fields that this program supports to
 143 # their InfoEx counterpart
 144 iemap = {}
 145 iemap['PREC'] = 'precipitationGauge'
 146 iemap['TOBS'] = 'tempPres'
 147 iemap['SNWD'] = 'hS'
 148
 149 # floor time to nearest hour
 150 dt = datetime.datetime.now()
 151 end_date = dt - datetime.timedelta(minutes=dt.minute % 60,
 152                                    seconds=dt.second,
 153                                    microseconds=dt.microsecond)
 154 begin_date = end_date - datetime.timedelta(hours=3)
 155
 156 transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
 157 client = zeep.Client(wsdl=wsdl, transport=transport)
 158 time_all_elements = time.time()
 159
 160 log.debug("Getting %s data from %s to %s" % (str(desired_data),
 161     str(begin_date), str(end_date)))
 162
 163 for elementCd in desired_data:
 164     time_element = time.time()
 165
 166     # get the last three hours of data for this elementCd
 167     tmp = client.service.getHourlyData(
 168             stationTriplets=[station_triplet],
 169             elementCd=elementCd,
 170             ordinal=1,
 171             beginDate=begin_date,
 172             endDate=end_date)
 173
 174     log.info("Time to get elementCd '%s': %.3f sec" % (elementCd,
 175         time.time() - time_element))
 176
 177     values = tmp[0]['values']
 178
 179     # sort and isolate the most recent
 180     #
 181     # NOTE: we do this because sometimes there are gaps in hourly data
 182     #       in NRCS; yes, we may end up with slightly inaccurate data,
 183     #       so perhaps this decision will be re-evaluated in the future
 184     if values:
 185         ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
 186         infoex['wx_data'][elementCd] = ordered[0]['value']
 187     else:
 188         infoex['wx_data'][elementCd] = None
 189
 190 log.info("Time to get all elementCds : %.3f sec" % (time.time() -
 191     time_all_elements))
 192
 193 log.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
 194
 195 # Now we only need to add in what we want to change thanks to that
 196 # abomination of a variable declaration earlier
 197 final_data[fmap['Location UUID']] = infoex['location_uuid']
 198 final_data[fmap['obDate']] = end_date.strftime('%m/%d/%Y')
 199 final_data[fmap['obTime']] = end_date.strftime('%H:%M')
 200
 201 for elementCd in infoex['wx_data']:
 202     if elementCd not in iemap:
 203         log.warning("BAD KEY wx_data['%s']" % (elementCd))
 204         continue
 205
 206     # CONSIDER: Casting every value to Float() -- need to investigate if
 207     #           any possible elementCds we may want are any other data
 208     #           type than float.
 209     #
 210     #           Another possibility is to query the API  with
 211     #           getStationElements and temporarily store the
 212     #           storedUnitCd. But that's pretty network-intensive and
 213     #           may not even be worth it if there's only e.g. one or two
 214     #           exceptions to any otherwise uniformly Float value set.
 215     final_data[fmap[iemap[elementCd]]] = infoex['wx_data'][elementCd]
 216
 217 log.debug("final_data: %s" % (str(final_data)))
 218
 219 with open(infoex['csv_filename'], 'w') as f:
 220     # The requirement is that empty values are represented in the CSV
 221     # file as "", csv.QUOTE_NONNUMERIC achieves that
 222     log.debug("writing CSV file '%s'" % (infoex['csv_filename']))
 223     writer = csv.writer(f, quoting=csv.QUOTE_NONNUMERIC)
 224     writer.writerow(final_data)
 225     f.close()
 226
 227 with open(infoex['csv_filename'], 'rb') as f:
 228     log.debug("uploading FTP file '%s'" % (infoex['host']))
 229     ftp = FTP(infoex['host'], infoex['uuid'], infoex['api_key'])
 230     ftp.storlines('STOR ' + infoex['csv_filename'], f)
 231     ftp.close()
 232     f.close()
 233
 234 log.debug('DONE')