infoex-autowx.py

   1 #!/usr/bin/python3
   2
   3 #
   4 # InfoEx <-> NRCS Auto Wx implementation
   5 # Alexander Vasarab
   6 # Wylark Mountaineering LLC
   7 # 2020-04-22
   8 #
   9 # Version 0.8
  10 #
  11 # This program fetches data from an NRCS SNOTEL site and pushes it to
  12 # InfoEx using the new automated weather system implementation.
  13 #
  14 # It is designed to be run hourly, and it asks for the last three hours
  15 # of data of each desired type, and selects the most recent one. This
  16 # lends some resiliency to the process and helps ensure that we have a
  17 # value to send, but it can lead to somewhat inconsistent/untruthful
  18 # data if e.g. the HS is from the last hour but the tempPres is from two
  19 # hours ago because the instrumentation had a hiccup. It's worth
  20 # considering if this is a bug or a feature.
  21 #
  22
  23 import configparser
  24 import csv
  25 import datetime
  26 import logging
  27 import time
  28 import zeep
  29 import zeep.cache
  30 import zeep.transports
  31 from collections import OrderedDict
  32 from ftplib import FTP
  33 from optparse import OptionParser
  34
  35 log = logging.getLogger(__name__)
  36 log.setLevel(logging.DEBUG)
  37
  38 try:
  39     from systemd.journal import JournalHandler
  40     log.addHandler(JournalHandler())
  41 except:
  42     # fallback to syslog
  43     import logging.handlers
  44     log.addHandler(logging.handlers.SysLogHandler())
  45
  46 parser = OptionParser()
  47 parser.add_option("--config", dest="config", metavar="FILE", help="location of config file")
  48
  49 (options, args) = parser.parse_args()
  50
  51 config = configparser.ConfigParser(allow_no_value=False)
  52 config.read(options.config)
  53
  54 log.debug('STARTING UP')
  55
  56 wsdl = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
  57
  58 try:
  59     infoex = {
  60         'host': config['ftp']['host'],
  61         'uuid': config['ftp']['uuid'],
  62         'api_key': config['ftp']['api_key'],
  63         'location_uuid': config['wxsite']['location_uuid'],
  64         'wx_data': {}, # placeholder key, values to come later
  65         'csv_filename': config['wxsite']['csv_filename']
  66     }
  67
  68     station_triplet = config['wxsite']['station_triplet']
  69
  70     try:
  71         desired_data = config['wxsite']['desired_data'].split(',')
  72     except:
  73         # desired_data malformed or missing, setting default
  74         desired_data = [
  75                 'TOBS', # AIR TEMPERATURE OBSERVED (degF)
  76                 'SNWD', # SNOW DEPTH (in)
  77                 'PREC'  # PRECIPITATION ACCUMULATION (in)
  78                 ]
  79 except KeyError as e:
  80     log.critical("%s not defined in %s" % (e, options.config))
  81     exit(1)
  82 except Exception as exc:
  83     log.critical("Exception occurred in config parsing: '%s'" % (exc))
  84     exit(1)
  85
  86 # all sections/values present in config file, final sanity check
  87 try:
  88     for key in config.sections():
  89         for subkey in config[key]:
  90             if not len(config[key][subkey]):
  91                 raise ValueError;
  92 except ValueError as exc:
  93     log.critical("Config value '%s.%s' is empty" % (key, subkey))
  94     exit(1)
  95
  96 # INFOEX FIELDS
  97 #
  98 # This won't earn style points in Python, but here we establish a couple
  99 # of helpful mappings variables. The reason this is helpful is that the
 100 # end result is simply an ordered set, the CSV file. But we still may
 101 # want to manipulate the values arbitrarily before writing that file.
 102 #
 103 # Also note that the current Auto Wx InfoEx documentation shows these
 104 # keys in a graphical table with the "index" beginning at 1, but here we
 105 # are sanely indexing beginning at 0.
 106 fmap = {}                           ; final_data     = [None] * 29
 107 fmap['Location UUID'] = 0           ; final_data[0]  = infoex['location_uuid']
 108 fmap['obDate'] = 1                  ; final_data[1]  = None
 109 fmap['obTime'] = 2                  ; final_data[2]  = None
 110 fmap['timeZone'] = 3                ; final_data[3]  = 'Pacific'
 111 fmap['tempMaxHour'] = 4             ; final_data[4]  = None
 112 fmap['tempMaxHourUnit'] = 5         ; final_data[5]  = 'F'
 113 fmap['tempMinHour'] = 6             ; final_data[6]  = None
 114 fmap['tempMinHourUnit'] = 7         ; final_data[7]  = 'F'
 115 fmap['tempPres'] = 8                ; final_data[8]  = None
 116 fmap['tempPresUnit'] = 9            ; final_data[9]  = 'F'
 117 fmap['precipitationGauge'] = 10     ; final_data[10] = None
 118 fmap['precipitationGaugeUnit'] = 11 ; final_data[11] = 'in'
 119 fmap['windSpeedNum'] = 12           ; final_data[12] = None
 120 fmap['windSpeedUnit'] = 13          ; final_data[13] = 'mph'
 121 fmap['windDirectionNum'] = 14       ; final_data[14] = None
 122 fmap['hS'] = 15                     ; final_data[15] = None
 123 fmap['hsUnit'] = 16                 ; final_data[16] = 'in'
 124 fmap['baro'] = 17                   ; final_data[17] = None
 125 fmap['baroUnit'] = 18               ; final_data[18] = 'inHg'
 126 fmap['rH'] = 19                     ; final_data[19] = None
 127 fmap['windGustSpeedNum'] = 20       ; final_data[20] = None
 128 fmap['windGustSpeedNumUnit'] = 21   ; final_data[21] = 'mph'
 129 fmap['windGustDirNum'] = 22         ; final_data[22] = None
 130 fmap['dewPoint'] = 23               ; final_data[23] = None
 131 fmap['dewPointUnit'] = 24           ; final_data[24] = 'F'
 132 fmap['hn24Auto'] = 25               ; final_data[25] = None
 133 fmap['hn24AutoUnit'] = 26           ; final_data[26] = 'in'
 134 fmap['hstAuto'] = 27                ; final_data[27] = None
 135 fmap['hstAutoUnit'] = 28            ; final_data[28] = 'in'
 136
 137 # one final mapping, the NRCS fields that this program supports to
 138 # their InfoEx counterpart
 139 iemap = {}
 140 iemap['PREC'] = 'precipitationGauge'
 141 iemap['TOBS'] = 'tempPres'
 142 iemap['SNWD'] = 'hS'
 143
 144 # floor time to nearest hour
 145 dt = datetime.datetime.now()
 146 end_date = dt - datetime.timedelta(minutes=dt.minute % 60,
 147                                    seconds=dt.second,
 148                                    microseconds=dt.microsecond)
 149 begin_date = end_date - datetime.timedelta(hours=3)
 150
 151 transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
 152 client = zeep.Client(wsdl=wsdl, transport=transport)
 153 time_all_elements = time.time()
 154
 155 log.debug("Getting %s data from %s to %s" % (str(desired_data),
 156     str(begin_date), str(end_date)))
 157
 158 for elementCd in desired_data:
 159     time_element = time.time()
 160
 161     # get the last three hours of data for this elementCd
 162     tmp = client.service.getHourlyData(
 163             stationTriplets=[station_triplet],
 164             elementCd=elementCd,
 165             ordinal=1,
 166             beginDate=begin_date,
 167             endDate=end_date)
 168
 169     log.info("Time to get elementCd '%s': %.3f sec" % (elementCd,
 170         time.time() - time_element))
 171
 172     values = tmp[0]['values']
 173
 174     # sort and isolate the most recent
 175     #
 176     # NOTE: we do this because sometimes there are gaps in hourly data
 177     #       in NRCS; yes, we may end up with slightly inaccurate data,
 178     #       so perhaps this decision will be re-evaluated in the future
 179     if values:
 180         ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
 181         infoex['wx_data'][elementCd] = ordered[0]['value']
 182     else:
 183         infoex['wx_data'][elementCd] = None
 184
 185 log.info("Time to get all elementCds : %.3f sec" % (time.time() -
 186     time_all_elements))
 187
 188 log.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
 189
 190 # Now we only need to add in what we want to change thanks to that
 191 # abomination of a variable declaration earlier
 192 final_data[fmap['Location UUID']] = infoex['location_uuid']
 193 final_data[fmap['obDate']] = end_date.strftime('%m/%d/%Y')
 194 final_data[fmap['obTime']] = end_date.strftime('%H:%M')
 195
 196 for elementCd in infoex['wx_data']:
 197     if elementCd not in iemap:
 198         log.warning("BAD KEY wx_data['%s']" % (elementCd))
 199         continue
 200
 201     # CONSIDER: Casting every value to Float() -- need to investigate if
 202     #           any possible elementCds we may want are any other data
 203     #           type than float.
 204     #
 205     #           Another possibility is to query the API  with
 206     #           getStationElements and temporarily store the
 207     #           storedUnitCd. But that's pretty network-intensive and
 208     #           may not even be worth it if there's only e.g. one or two
 209     #           exceptions to any otherwise uniformly Float value set.
 210     final_data[fmap[iemap[elementCd]]] = infoex['wx_data'][elementCd]
 211
 212 log.debug("final_data: %s" % (str(final_data)))
 213
 214 with open(infoex['csv_filename'], 'w') as f:
 215     # The requirement is that empty values are represented in the CSV
 216     # file as "", csv.QUOTE_NONNUMERIC achieves that
 217     log.debug("writing CSV file '%s'" % (infoex['csv_filename']))
 218     writer = csv.writer(f, quoting=csv.QUOTE_NONNUMERIC)
 219     writer.writerow(final_data)
 220     f.close()
 221
 222 #with open(infoex['csv_filename'], 'rb') as f:
 223 #    log.debug("uploading FTP file '%s'" % (infoex['host']))
 224 #    ftp = FTP(infoex['host'], infoex['uuid'], infoex['api_key'])
 225 #    ftp.storlines('STOR ' + infoex['csv_filename'], f)
 226 #    ftp.close()
 227 #    f.close()
 228
 229 log.debug('DONE')