infoex-autowx.py

   1 #!/usr/bin/python3
   2
   3 #
   4 # InfoEx <-> NRCS Auto Wx implementation
   5 # Alexander Vasarab
   6 # Wylark Mountaineering LLC
   7 #
   8 # Version 1.0.0
   9 #
  10 # This program fetches data from an NRCS SNOTEL site and pushes it to
  11 # InfoEx using the new automated weather system implementation.
  12 #
  13 # It is designed to be run hourly, and it asks for the last three hours
  14 # of data of each desired type, and selects the most recent one. This
  15 # lends some resiliency to the process and helps ensure that we have a
  16 # value to send, but it can lead to somewhat inconsistent/untruthful
  17 # data if e.g. the HS is from the last hour but the tempPres is from two
  18 # hours ago because the instrumentation had a hiccup. It's worth
  19 # considering if this is a bug or a feature.
  20 #
  21 # For more information, see file: README
  22 # For licensing, see file: LICENSE
  23 #
  24
  25 import configparser
  26 import csv
  27 import datetime
  28 import logging
  29 import time
  30 import zeep
  31 import zeep.cache
  32 import zeep.transports
  33 from collections import OrderedDict
  34 from ftplib import FTP
  35 from optparse import OptionParser
  36
  37 log = logging.getLogger(__name__)
  38 log.setLevel(logging.DEBUG)
  39
  40 try:
  41     from systemd.journal import JournalHandler
  42     log.addHandler(JournalHandler())
  43 except:
  44     # fallback to syslog
  45     import logging.handlers
  46     log.addHandler(logging.handlers.SysLogHandler())
  47
  48 parser = OptionParser()
  49 parser.add_option("--config", dest="config", metavar="FILE", help="location of config file")
  50
  51 (options, args) = parser.parse_args()
  52
  53 config = configparser.ConfigParser(allow_no_value=False)
  54 config.read(options.config)
  55
  56 log.debug('STARTING UP')
  57
  58 wsdl = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
  59
  60 try:
  61     infoex = {
  62         'host': config['ftp']['host'],
  63         'uuid': config['ftp']['uuid'],
  64         'api_key': config['ftp']['api_key'],
  65         'location_uuid': config['wxsite']['location_uuid'],
  66         'wx_data': {}, # placeholder key, values to come later
  67         'csv_filename': config['wxsite']['csv_filename']
  68     }
  69
  70     station_triplet = config['wxsite']['station_triplet']
  71
  72     try:
  73         desired_data = config['wxsite']['desired_data'].split(',')
  74     except:
  75         # desired_data malformed or missing, setting default
  76         desired_data = [
  77                 'TOBS', # AIR TEMPERATURE OBSERVED (degF)
  78                 'SNWD', # SNOW DEPTH (in)
  79                 'PREC'  # PRECIPITATION ACCUMULATION (in)
  80                 ]
  81 except KeyError as e:
  82     log.critical("%s not defined in %s" % (e, options.config))
  83     exit(1)
  84 except Exception as exc:
  85     log.critical("Exception occurred in config parsing: '%s'" % (exc))
  86     exit(1)
  87
  88 # all sections/values present in config file, final sanity check
  89 try:
  90     for key in config.sections():
  91         for subkey in config[key]:
  92             if not len(config[key][subkey]):
  93                 raise ValueError;
  94 except ValueError as exc:
  95     log.critical("Config value '%s.%s' is empty" % (key, subkey))
  96     exit(1)
  97
  98 # INFOEX FIELDS
  99 #
 100 # This won't earn style points in Python, but here we establish a couple
 101 # of helpful mappings variables. The reason this is helpful is that the
 102 # end result is simply an ordered set, the CSV file. But we still may
 103 # want to manipulate the values arbitrarily before writing that file.
 104 #
 105 # Also note that the current Auto Wx InfoEx documentation shows these
 106 # keys in a graphical table with the "index" beginning at 1, but here we
 107 # are sanely indexing beginning at 0.
 108 fmap = {}                           ; final_data     = [None] * 29
 109 fmap['Location UUID'] = 0           ; final_data[0]  = infoex['location_uuid']
 110 fmap['obDate'] = 1                  ; final_data[1]  = None
 111 fmap['obTime'] = 2                  ; final_data[2]  = None
 112 fmap['timeZone'] = 3                ; final_data[3]  = 'Pacific'
 113 fmap['tempMaxHour'] = 4             ; final_data[4]  = None
 114 fmap['tempMaxHourUnit'] = 5         ; final_data[5]  = 'F'
 115 fmap['tempMinHour'] = 6             ; final_data[6]  = None
 116 fmap['tempMinHourUnit'] = 7         ; final_data[7]  = 'F'
 117 fmap['tempPres'] = 8                ; final_data[8]  = None
 118 fmap['tempPresUnit'] = 9            ; final_data[9]  = 'F'
 119 fmap['precipitationGauge'] = 10     ; final_data[10] = None
 120 fmap['precipitationGaugeUnit'] = 11 ; final_data[11] = 'in'
 121 fmap['windSpeedNum'] = 12           ; final_data[12] = None
 122 fmap['windSpeedUnit'] = 13          ; final_data[13] = 'mph'
 123 fmap['windDirectionNum'] = 14       ; final_data[14] = None
 124 fmap['hS'] = 15                     ; final_data[15] = None
 125 fmap['hsUnit'] = 16                 ; final_data[16] = 'in'
 126 fmap['baro'] = 17                   ; final_data[17] = None
 127 fmap['baroUnit'] = 18               ; final_data[18] = 'inHg'
 128 fmap['rH'] = 19                     ; final_data[19] = None
 129 fmap['windGustSpeedNum'] = 20       ; final_data[20] = None
 130 fmap['windGustSpeedNumUnit'] = 21   ; final_data[21] = 'mph'
 131 fmap['windGustDirNum'] = 22         ; final_data[22] = None
 132 fmap['dewPoint'] = 23               ; final_data[23] = None
 133 fmap['dewPointUnit'] = 24           ; final_data[24] = 'F'
 134 fmap['hn24Auto'] = 25               ; final_data[25] = None
 135 fmap['hn24AutoUnit'] = 26           ; final_data[26] = 'in'
 136 fmap['hstAuto'] = 27                ; final_data[27] = None
 137 fmap['hstAutoUnit'] = 28            ; final_data[28] = 'in'
 138
 139 # one final mapping, the NRCS fields that this program supports to
 140 # their InfoEx counterpart
 141 iemap = {}
 142 iemap['PREC'] = 'precipitationGauge'
 143 iemap['TOBS'] = 'tempPres'
 144 iemap['SNWD'] = 'hS'
 145
 146 # floor time to nearest hour
 147 dt = datetime.datetime.now()
 148 end_date = dt - datetime.timedelta(minutes=dt.minute % 60,
 149                                    seconds=dt.second,
 150                                    microseconds=dt.microsecond)
 151 begin_date = end_date - datetime.timedelta(hours=3)
 152
 153 transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
 154 client = zeep.Client(wsdl=wsdl, transport=transport)
 155 time_all_elements = time.time()
 156
 157 log.debug("Getting %s data from %s to %s" % (str(desired_data),
 158     str(begin_date), str(end_date)))
 159
 160 for elementCd in desired_data:
 161     time_element = time.time()
 162
 163     # get the last three hours of data for this elementCd
 164     tmp = client.service.getHourlyData(
 165             stationTriplets=[station_triplet],
 166             elementCd=elementCd,
 167             ordinal=1,
 168             beginDate=begin_date,
 169             endDate=end_date)
 170
 171     log.info("Time to get elementCd '%s': %.3f sec" % (elementCd,
 172         time.time() - time_element))
 173
 174     values = tmp[0]['values']
 175
 176     # sort and isolate the most recent
 177     #
 178     # NOTE: we do this because sometimes there are gaps in hourly data
 179     #       in NRCS; yes, we may end up with slightly inaccurate data,
 180     #       so perhaps this decision will be re-evaluated in the future
 181     if values:
 182         ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
 183         infoex['wx_data'][elementCd] = ordered[0]['value']
 184     else:
 185         infoex['wx_data'][elementCd] = None
 186
 187 log.info("Time to get all elementCds : %.3f sec" % (time.time() -
 188     time_all_elements))
 189
 190 log.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
 191
 192 # Now we only need to add in what we want to change thanks to that
 193 # abomination of a variable declaration earlier
 194 final_data[fmap['Location UUID']] = infoex['location_uuid']
 195 final_data[fmap['obDate']] = end_date.strftime('%m/%d/%Y')
 196 final_data[fmap['obTime']] = end_date.strftime('%H:%M')
 197
 198 for elementCd in infoex['wx_data']:
 199     if elementCd not in iemap:
 200         log.warning("BAD KEY wx_data['%s']" % (elementCd))
 201         continue
 202
 203     # CONSIDER: Casting every value to Float() -- need to investigate if
 204     #           any possible elementCds we may want are any other data
 205     #           type than float.
 206     #
 207     #           Another possibility is to query the API  with
 208     #           getStationElements and temporarily store the
 209     #           storedUnitCd. But that's pretty network-intensive and
 210     #           may not even be worth it if there's only e.g. one or two
 211     #           exceptions to any otherwise uniformly Float value set.
 212     final_data[fmap[iemap[elementCd]]] = infoex['wx_data'][elementCd]
 213
 214 log.debug("final_data: %s" % (str(final_data)))
 215
 216 with open(infoex['csv_filename'], 'w') as f:
 217     # The requirement is that empty values are represented in the CSV
 218     # file as "", csv.QUOTE_NONNUMERIC achieves that
 219     log.debug("writing CSV file '%s'" % (infoex['csv_filename']))
 220     writer = csv.writer(f, quoting=csv.QUOTE_NONNUMERIC)
 221     writer.writerow(final_data)
 222     f.close()
 223
 224 #with open(infoex['csv_filename'], 'rb') as f:
 225 #    log.debug("uploading FTP file '%s'" % (infoex['host']))
 226 #    ftp = FTP(infoex['host'], infoex['uuid'], infoex['api_key'])
 227 #    ftp.storlines('STOR ' + infoex['csv_filename'], f)
 228 #    ftp.close()
 229 #    f.close()
 230
 231 log.debug('DONE')