infoex-autowx.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 """
   5 InfoEx <-> NRCS/MesoWest Auto Wx implementation
   6 Alexander Vasarab
   7 Wylark Mountaineering LLC
   8
   9 Version 2.0.0
  10
  11 This program fetches data from either an NRCS SNOTEL site or MesoWest
  12 weather station and pushes it to InfoEx using the new automated weather
  13 system implementation.
  14
  15 It is designed to be run hourly, and it asks for the last three hours
  16 of data of each desired type, and selects the most recent one. This
  17 lends some resiliency to the process and helps ensure that we have a
  18 value to send, but it can lead to somewhat inconsistent/untruthful
  19 data if e.g. the HS is from the last hour but the tempPres is from two
  20 hours ago because the instrumentation had a hiccup. It's worth
  21 considering if this is a bug or a feature.
  22
  23 For more information, see file: README
  24 For licensing, see file: LICENSE
  25 """
  26
  27 import configparser
  28 import csv
  29 import datetime
  30 import logging
  31 import os
  32 import sys
  33 import time
  34
  35 from collections import OrderedDict
  36 from ftplib import FTP
  37 from optparse import OptionParser
  38
  39 import requests
  40
  41 import zeep
  42 import zeep.cache
  43 import zeep.transports
  44
  45 log = logging.getLogger(__name__)
  46 log.setLevel(logging.DEBUG)
  47
  48 try:
  49     from systemd.journal import JournalHandler
  50     log.addHandler(JournalHandler())
  51 except:
  52     ## fallback to syslog
  53     #import logging.handlers
  54     #log.addHandler(logging.handlers.SysLogHandler())
  55     # fallback to stdout
  56     handler = logging.StreamHandler(sys.stdout)
  57     log.addHandler(handler)
  58
  59 parser = OptionParser()
  60
  61 parser.add_option("--config",
  62     dest="config",
  63     metavar="FILE",
  64     help="location of config file")
  65
  66 parser.add_option("--dry-run",
  67     action="store_true",
  68     dest="dry_run",
  69     default=False,
  70     help="fetch data but don't upload to InfoEx")
  71
  72 (options, args) = parser.parse_args()
  73
  74 config = configparser.ConfigParser(allow_no_value=False)
  75
  76 if not options.config:
  77     print("Please specify a configuration file via --config.")
  78     sys.exit(1)
  79
  80 config.read(options.config)
  81
  82 log.debug('STARTING UP')
  83
  84 try:
  85     infoex = {
  86         'host': config['infoex']['host'],
  87         'uuid': config['infoex']['uuid'],
  88         'api_key': config['infoex']['api_key'],
  89         'csv_filename': config['infoex']['csv_filename'],
  90         'location_uuid': config['infoex']['location_uuid'],
  91         'wx_data': {}, # placeholder key, values to come later
  92     }
  93
  94     data = dict()
  95     data['provider'] = config['station']['type']
  96
  97     if data['provider'] not in ['nrcs', 'mesowest']:
  98         print("Please specify either nrcs or mesowest as the station type.")
  99         sys.exit(1)
 100
 101     if data['provider'] == 'nrcs':
 102         data['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
 103         data['station_id'] = config['station']['station_id']
 104
 105         try:
 106             desired_data = config['station']['desired_data'].split(',')
 107         except:
 108             # desired_data malformed or missing, setting default
 109             desired_data = [
 110                     'TOBS', # AIR TEMPERATURE OBSERVED (degF)
 111                     'SNWD', # SNOW DEPTH (in)
 112                     'PREC'  # PRECIPITATION ACCUMULATION (in)
 113                     ]
 114
 115     if data['provider'] == 'mesowest':
 116         data['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
 117         data['station_id'] = config['station']['station_id']
 118         data['units'] = config['station']['units']
 119
 120         try:
 121             desired_data = config['station']['desired_data']
 122         except:
 123             # desired_data malformed or missing, setting default
 124             desired_data = 'air_temp,snow_depth'
 125
 126         # construct full API URL (sans start/end time, added later)
 127         data['source'] = data['source'] + '?token=' + config['station']['token'] + '&within=60&units=' + data['units'] + '&stid=' + data['station_id'] + '&vars=' + desired_data
 128
 129 except KeyError as e:
 130     log.critical("%s not defined in %s" % (e, options.config))
 131     exit(1)
 132 except Exception as exc:
 133     log.critical("Exception occurred in config parsing: '%s'" % (exc))
 134     exit(1)
 135
 136 # all sections/values present in config file, final sanity check
 137 try:
 138     for key in config.sections():
 139         for subkey in config[key]:
 140             if not len(config[key][subkey]):
 141                 raise ValueError;
 142 except ValueError as exc:
 143     log.critical("Config value '%s.%s' is empty" % (key, subkey))
 144     exit(1)
 145
 146 # INFOEX FIELDS
 147 #
 148 # This won't earn style points in Python, but here we establish a couple
 149 # of helpful mappings variables. The reason this is helpful is that the
 150 # end result is simply an ordered set, the CSV file. But we still may
 151 # want to manipulate the values arbitrarily before writing that file.
 152 #
 153 # Also note that the current Auto Wx InfoEx documentation shows these
 154 # keys in a graphical table with the "index" beginning at 1, but here we
 155 # are sanely indexing beginning at 0.
 156 fmap = {}                           ; final_data     = [None] * 29
 157 fmap['Location UUID'] = 0           ; final_data[0]  = infoex['location_uuid']
 158 fmap['obDate'] = 1                  ; final_data[1]  = None
 159 fmap['obTime'] = 2                  ; final_data[2]  = None
 160 fmap['timeZone'] = 3                ; final_data[3]  = 'Pacific'
 161 fmap['tempMaxHour'] = 4             ; final_data[4]  = None
 162 fmap['tempMaxHourUnit'] = 5         ; final_data[5]  = 'F'
 163 fmap['tempMinHour'] = 6             ; final_data[6]  = None
 164 fmap['tempMinHourUnit'] = 7         ; final_data[7]  = 'F'
 165 fmap['tempPres'] = 8                ; final_data[8]  = None
 166 fmap['tempPresUnit'] = 9            ; final_data[9]  = 'F'
 167 fmap['precipitationGauge'] = 10     ; final_data[10] = None
 168 fmap['precipitationGaugeUnit'] = 11 ; final_data[11] = 'in'
 169 fmap['windSpeedNum'] = 12           ; final_data[12] = None
 170 fmap['windSpeedUnit'] = 13          ; final_data[13] = 'mph'
 171 fmap['windDirectionNum'] = 14       ; final_data[14] = None
 172 fmap['hS'] = 15                     ; final_data[15] = None
 173 fmap['hsUnit'] = 16                 ; final_data[16] = 'in'
 174 fmap['baro'] = 17                   ; final_data[17] = None
 175 fmap['baroUnit'] = 18               ; final_data[18] = 'inHg'
 176 fmap['rH'] = 19                     ; final_data[19] = None
 177 fmap['windGustSpeedNum'] = 20       ; final_data[20] = None
 178 fmap['windGustSpeedNumUnit'] = 21   ; final_data[21] = 'mph'
 179 fmap['windGustDirNum'] = 22         ; final_data[22] = None
 180 fmap['dewPoint'] = 23               ; final_data[23] = None
 181 fmap['dewPointUnit'] = 24           ; final_data[24] = 'F'
 182 fmap['hn24Auto'] = 25               ; final_data[25] = None
 183 fmap['hn24AutoUnit'] = 26           ; final_data[26] = 'in'
 184 fmap['hstAuto'] = 27                ; final_data[27] = None
 185 fmap['hstAutoUnit'] = 28            ; final_data[28] = 'in'
 186
 187 # one final mapping, the NRCS/MesoWest fields that this program supports to
 188 # their InfoEx counterpart
 189 iemap = {}
 190
 191 if data['provider'] == 'nrcs':
 192     iemap['PREC'] = 'precipitationGauge'
 193     iemap['TOBS'] = 'tempPres'
 194     iemap['SNWD'] = 'hS'
 195 elif data['provider'] == 'mesowest':
 196     iemap['precip_accum'] = 'precipitationGauge'
 197     iemap['air_temp'] = 'tempPres'
 198     iemap['snow_depth'] = 'hS'
 199
 200 # floor time to nearest hour
 201 dt = datetime.datetime.now()
 202 end_date = dt - datetime.timedelta(minutes=dt.minute % 60,
 203                                    seconds=dt.second,
 204                                    microseconds=dt.microsecond)
 205 begin_date = end_date - datetime.timedelta(hours=3)
 206
 207 # get the data
 208 log.debug("Getting %s data from %s to %s" % (str(desired_data),
 209     str(begin_date), str(end_date)))
 210
 211 time_all_elements = time.time()
 212
 213 # NRCS-specific code
 214 if data['provider'] == 'nrcs':
 215     transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
 216     client = zeep.Client(wsdl=data['source'], transport=transport)
 217
 218     for elementCd in desired_data:
 219         time_element = time.time()
 220
 221         # get the last three hours of data for this elementCd
 222         tmp = client.service.getHourlyData(
 223                 stationTriplets=[data['station_id']],
 224                 elementCd=elementCd,
 225                 ordinal=1,
 226                 beginDate=begin_date,
 227                 endDate=end_date)
 228
 229         log.info("Time to get elementCd '%s': %.3f sec" % (elementCd,
 230             time.time() - time_element))
 231
 232         values = tmp[0]['values']
 233
 234         # sort and isolate the most recent
 235         #
 236         # NOTE: we do this because sometimes there are gaps in hourly data
 237         #       in NRCS; yes, we may end up with slightly inaccurate data,
 238         #       so perhaps this decision will be re-evaluated in the future
 239         if values:
 240             ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
 241             infoex['wx_data'][elementCd] = ordered[0]['value']
 242         else:
 243             infoex['wx_data'][elementCd] = None
 244
 245 # MesoWest-specific code
 246 elif data['provider'] == 'mesowest':
 247     # massage begin/end date format
 248     begin_date_str = begin_date.strftime('%Y%m%d%H%M')
 249     end_date_str = end_date.strftime('%Y%m%d%H%M')
 250
 251     # construct final, completed API URL
 252     api_req_url = data['source'] + '&start=' + begin_date_str + '&end=' + end_date_str
 253     req = requests.get(api_req_url)
 254
 255     try:
 256         json = req.json()
 257     except ValueError:
 258         log.error("Bad JSON in MesoWest response")
 259         sys.exit(1)
 260
 261     try:
 262         observations = json['STATION'][0]['OBSERVATIONS']
 263     except ValueError:
 264         log.error("Bad JSON in MesoWest response")
 265         sys.exit(1)
 266
 267     pos = len(observations['date_time']) - 1
 268
 269     for elementCd in desired_data.split(','):
 270         # sort and isolate the most recent, see note above in NRCS for how and
 271         # why this is done
 272         #
 273         # NOTE: Unlike in the NRCS case, the MesoWest API respones contains all
 274         #       data (whereas with NRCS, we have to make a separate request for
 275         #       each element we want. This is nice for network efficiency but
 276         #       it means we have to handle this part differently for each.
 277         #
 278         # NOTE: Also unlike NRCS, MesoWest provides more granular data; NRCS
 279         #       provides hourly data, but MesoWest can often provide data every
 280         #       10 minutes -- though this provides more opportunity for
 281         #       irregularities
 282
 283         # we may not have the data at all
 284         key_name = elementCd + '_set_1'
 285         if key_name in observations:
 286             if observations[key_name][pos]:
 287                 infoex['wx_data'][elementCd] = observations[key_name][pos]
 288             else:
 289                 infoex['wx_data'][elementCd] = None
 290         else:
 291             infoex['wx_data'][elementCd] = None
 292
 293 log.info("Time to get all data : %.3f sec" % (time.time() -
 294     time_all_elements))
 295
 296 log.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
 297
 298 # Now we only need to add in what we want to change thanks to that
 299 # abomination of a variable declaration earlier
 300 final_data[fmap['Location UUID']] = infoex['location_uuid']
 301 final_data[fmap['obDate']] = end_date.strftime('%m/%d/%Y')
 302 final_data[fmap['obTime']] = end_date.strftime('%H:%M')
 303
 304 for elementCd in infoex['wx_data']:
 305     if elementCd not in iemap:
 306         log.warning("BAD KEY wx_data['%s']" % (elementCd))
 307         continue
 308
 309     # CONSIDER: Casting every value to Float() -- need to investigate if
 310     #           any possible elementCds we may want are any other data
 311     #           type than float.
 312     #
 313     #           Another possibility is to query the API  with
 314     #           getStationElements and temporarily store the
 315     #           storedUnitCd. But that's pretty network-intensive and
 316     #           may not even be worth it if there's only e.g. one or two
 317     #           exceptions to any otherwise uniformly Float value set.
 318     final_data[fmap[iemap[elementCd]]] = infoex['wx_data'][elementCd]
 319
 320 log.debug("final_data: %s" % (str(final_data)))
 321
 322 with open(infoex['csv_filename'], 'w') as f:
 323     # The requirement is that empty values are represented in the CSV
 324     # file as "", csv.QUOTE_NONNUMERIC achieves that
 325     log.debug("writing CSV file '%s'" % (infoex['csv_filename']))
 326     writer = csv.writer(f, quoting=csv.QUOTE_NONNUMERIC)
 327     writer.writerow(final_data)
 328     f.close()
 329
 330 if not options.dry_run:
 331     # not a dry run
 332     with open(infoex['csv_filename'], 'rb') as f:
 333         log.debug("uploading FTP file '%s'" % (infoex['host']))
 334         ftp = FTP(infoex['host'], infoex['uuid'], infoex['api_key'])
 335         ftp.storlines('STOR ' + infoex['csv_filename'], f)
 336         ftp.close()
 337         f.close()
 338     os.remove(infoex['csv_filename'])
 339
 340 log.debug('DONE')