infoex-autowx.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 """
   5 InfoEx <-> NRCS/MesoWest Auto Wx implementation
   6 Alexander Vasarab
   7 Wylark Mountaineering LLC
   8
   9 Version 2.0.0
  10
  11 This program fetches data from either an NRCS SNOTEL site or MesoWest
  12 weather station and pushes it to InfoEx using the new automated weather
  13 system implementation.
  14
  15 It is designed to be run hourly, and it asks for the last three hours
  16 of data of each desired type, and selects the most recent one. This
  17 lends some resiliency to the process and helps ensure that we have a
  18 value to send, but it can lead to somewhat inconsistent/untruthful
  19 data if e.g. the HS is from the last hour but the tempPres is from two
  20 hours ago because the instrumentation had a hiccup. It's worth
  21 considering if this is a bug or a feature.
  22
  23 For more information, see file: README
  24 For licensing, see file: LICENSE
  25 """
  26
  27 import configparser
  28 import csv
  29 import datetime
  30 import logging
  31 import os
  32 import sys
  33 import time
  34
  35 from collections import OrderedDict
  36 from ftplib import FTP
  37 from optparse import OptionParser
  38
  39 import requests
  40
  41 import zeep
  42 import zeep.cache
  43 import zeep.transports
  44
  45 __version__ = '2.0.0'
  46
  47 log = logging.getLogger(__name__)
  48 log.setLevel(logging.DEBUG)
  49
  50 try:
  51     from systemd.journal import JournalHandler
  52     log.addHandler(JournalHandler())
  53 except:
  54     ## fallback to syslog
  55     #import logging.handlers
  56     #log.addHandler(logging.handlers.SysLogHandler())
  57     # fallback to stdout
  58     handler = logging.StreamHandler(sys.stdout)
  59     log.addHandler(handler)
  60
  61 parser = OptionParser()
  62
  63 parser.add_option("--config",
  64     dest="config",
  65     metavar="FILE",
  66     help="location of config file")
  67
  68 parser.add_option("--dry-run",
  69     action="store_true",
  70     dest="dry_run",
  71     default=False,
  72     help="fetch data but don't upload to InfoEx")
  73
  74 parser.add_option("--version",
  75     action="store_true",
  76     dest="show_version",
  77     default=False,
  78     help="show program version and exit")
  79
  80 (options, args) = parser.parse_args()
  81
  82 config = configparser.ConfigParser(allow_no_value=False)
  83
  84 if options.show_version:
  85     print("%s - %s" % (os.path.basename(__file__), __version__))
  86     sys.exit(0)
  87
  88 if not options.config:
  89     print("Please specify a configuration file via --config.")
  90     sys.exit(1)
  91
  92 config.read(options.config)
  93
  94 log.debug('STARTING UP')
  95
  96 try:
  97     infoex = {
  98         'host': config['infoex']['host'],
  99         'uuid': config['infoex']['uuid'],
 100         'api_key': config['infoex']['api_key'],
 101         'csv_filename': config['infoex']['csv_filename'],
 102         'location_uuid': config['infoex']['location_uuid'],
 103         'wx_data': {}, # placeholder key, values to come later
 104     }
 105
 106     data = dict()
 107     data['provider'] = config['station']['type']
 108
 109     if data['provider'] not in ['nrcs', 'mesowest']:
 110         print("Please specify either nrcs or mesowest as the station type.")
 111         sys.exit(1)
 112
 113     if data['provider'] == 'nrcs':
 114         data['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
 115         data['station_id'] = config['station']['station_id']
 116
 117         try:
 118             desired_data = config['station']['desired_data'].split(',')
 119         except:
 120             # desired_data malformed or missing, setting default
 121             desired_data = [
 122                     'TOBS', # AIR TEMPERATURE OBSERVED (degF)
 123                     'SNWD', # SNOW DEPTH (in)
 124                     'PREC'  # PRECIPITATION ACCUMULATION (in)
 125                     ]
 126
 127         # XXX: For NRCS, we're manually overriding units for now! Once
 128         #      unit conversion is supported for NRCS, REMOVE THIS!
 129         if 'units' not in data:
 130             data['units'] = 'imperial'
 131
 132     if data['provider'] == 'mesowest':
 133         data['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
 134         data['station_id'] = config['station']['station_id']
 135         data['units'] = config['station']['units']
 136
 137         try:
 138             desired_data = config['station']['desired_data']
 139         except:
 140             # desired_data malformed or missing, setting default
 141             desired_data = 'air_temp,snow_depth'
 142
 143         # construct full API URL (sans start/end time, added later)
 144         data['source'] = data['source'] + '?token=' + config['station']['token'] + '&within=60&units=' + data['units'] + '&stid=' + data['station_id'] + '&vars=' + desired_data
 145
 146 except KeyError as e:
 147     log.critical("%s not defined in %s" % (e, options.config))
 148     exit(1)
 149 except Exception as exc:
 150     log.critical("Exception occurred in config parsing: '%s'" % (exc))
 151     exit(1)
 152
 153 # all sections/values present in config file, final sanity check
 154 try:
 155     for key in config.sections():
 156         for subkey in config[key]:
 157             if not len(config[key][subkey]):
 158                 raise ValueError;
 159 except ValueError as exc:
 160     log.critical("Config value '%s.%s' is empty" % (key, subkey))
 161     exit(1)
 162
 163 # INFOEX FIELDS
 164 #
 165 # This won't earn style points in Python, but here we establish a couple
 166 # of helpful mappings variables. The reason this is helpful is that the
 167 # end result is simply an ordered set, the CSV file. But we still may
 168 # want to manipulate the values arbitrarily before writing that file.
 169 #
 170 # Also note that the current Auto Wx InfoEx documentation shows these
 171 # keys in a graphical table with the "index" beginning at 1, but here we
 172 # are sanely indexing beginning at 0.
 173 fmap = {}                           ; final_data     = [None] * 29
 174 fmap['Location UUID'] = 0           ; final_data[0]  = infoex['location_uuid']
 175 fmap['obDate'] = 1                  ; final_data[1]  = None
 176 fmap['obTime'] = 2                  ; final_data[2]  = None
 177 fmap['timeZone'] = 3                ; final_data[3]  = 'Pacific'
 178 fmap['tempMaxHour'] = 4             ; final_data[4]  = None
 179 fmap['tempMaxHourUnit'] = 5         ; final_data[5]  = 'F'
 180 fmap['tempMinHour'] = 6             ; final_data[6]  = None
 181 fmap['tempMinHourUnit'] = 7         ; final_data[7]  = 'F'
 182 fmap['tempPres'] = 8                ; final_data[8]  = None
 183 fmap['tempPresUnit'] = 9            ; final_data[9]  = 'F'
 184 fmap['precipitationGauge'] = 10     ; final_data[10] = None
 185 fmap['precipitationGaugeUnit'] = 11 ; final_data[11] = 'in'
 186 fmap['windSpeedNum'] = 12           ; final_data[12] = None
 187 fmap['windSpeedUnit'] = 13          ; final_data[13] = 'mph'
 188 fmap['windDirectionNum'] = 14       ; final_data[14] = None
 189 fmap['hS'] = 15                     ; final_data[15] = None
 190 fmap['hsUnit'] = 16                 ; final_data[16] = 'in'
 191 fmap['baro'] = 17                   ; final_data[17] = None
 192 fmap['baroUnit'] = 18               ; final_data[18] = 'inHg'
 193 fmap['rH'] = 19                     ; final_data[19] = None
 194 fmap['windGustSpeedNum'] = 20       ; final_data[20] = None
 195 fmap['windGustSpeedNumUnit'] = 21   ; final_data[21] = 'mph'
 196 fmap['windGustDirNum'] = 22         ; final_data[22] = None
 197 fmap['dewPoint'] = 23               ; final_data[23] = None
 198 fmap['dewPointUnit'] = 24           ; final_data[24] = 'F'
 199 fmap['hn24Auto'] = 25               ; final_data[25] = None
 200 fmap['hn24AutoUnit'] = 26           ; final_data[26] = 'in'
 201 fmap['hstAuto'] = 27                ; final_data[27] = None
 202 fmap['hstAutoUnit'] = 28            ; final_data[28] = 'in'
 203
 204 # one final mapping, the NRCS/MesoWest fields that this program supports to
 205 # their InfoEx counterpart
 206 iemap = {}
 207
 208 if data['provider'] == 'nrcs':
 209     iemap['PREC'] = 'precipitationGauge'
 210     iemap['TOBS'] = 'tempPres'
 211     iemap['SNWD'] = 'hS'
 212     iemap['PRES'] = 'baro'
 213     iemap['RHUM'] = 'rH'
 214     iemap['WSPD'] = 'windSpeedNum'
 215     iemap['WDIR'] = 'windDirectionNum'
 216     # unsupported by NRCS:
 217     # windGustSpeedNum
 218 elif data['provider'] == 'mesowest':
 219     iemap['precip_accum'] = 'precipitationGauge'
 220     iemap['air_temp'] = 'tempPres'
 221     iemap['snow_depth'] = 'hS'
 222     iemap['pressure'] = 'baro'
 223     iemap['relative_humidity'] = 'rH'
 224     iemap['wind_speed'] = 'windSpeedNum'
 225     iemap['wind_direction'] = 'windDirectionNum'
 226     iemap['wind_gust'] = 'windGustSpeedNum'
 227
 228 # override units if user selected metric
 229 #
 230 # NOTE: to update this, use the fmap<->final_data mapping laid out above
 231 #
 232 # NOTE: this only 'works' with MesoWest for now, as the MesoWest API
 233 #       itself handles the unit conversion; in the future, we will also
 234 #       support NRCS unit conversion, but this must be done by this
 235 #       program.
 236 if data['units'] == 'metric':
 237     final_data[fmap['tempPresUnit']] = 'C'
 238     final_data[fmap['hsUnit']] = 'm'
 239     final_data[fmap['windSpeedUnit']] = 'm/s'
 240     final_data[fmap['windGustSpeedNumUnit']] = 'm/s'
 241
 242 # floor time to nearest hour
 243 dt = datetime.datetime.now()
 244 end_date = dt - datetime.timedelta(minutes=dt.minute % 60,
 245                                    seconds=dt.second,
 246                                    microseconds=dt.microsecond)
 247 begin_date = end_date - datetime.timedelta(hours=3)
 248
 249 # get the data
 250 log.debug("Getting %s data from %s to %s" % (str(desired_data),
 251     str(begin_date), str(end_date)))
 252
 253 time_all_elements = time.time()
 254
 255 # NRCS-specific code
 256 if data['provider'] == 'nrcs':
 257     transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
 258     client = zeep.Client(wsdl=data['source'], transport=transport)
 259
 260     for elementCd in desired_data:
 261         time_element = time.time()
 262
 263         # get the last three hours of data for this elementCd
 264         tmp = client.service.getHourlyData(
 265                 stationTriplets=[data['station_id']],
 266                 elementCd=elementCd,
 267                 ordinal=1,
 268                 beginDate=begin_date,
 269                 endDate=end_date)
 270
 271         log.info("Time to get elementCd '%s': %.3f sec" % (elementCd,
 272             time.time() - time_element))
 273
 274         values = tmp[0]['values']
 275
 276         # sort and isolate the most recent
 277         #
 278         # NOTE: we do this because sometimes there are gaps in hourly data
 279         #       in NRCS; yes, we may end up with slightly inaccurate data,
 280         #       so perhaps this decision will be re-evaluated in the future
 281         if values:
 282             ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
 283             infoex['wx_data'][elementCd] = ordered[0]['value']
 284         else:
 285             infoex['wx_data'][elementCd] = None
 286
 287 # MesoWest-specific code
 288 elif data['provider'] == 'mesowest':
 289     # massage begin/end date format
 290     begin_date_str = begin_date.strftime('%Y%m%d%H%M')
 291     end_date_str = end_date.strftime('%Y%m%d%H%M')
 292
 293     # construct final, completed API URL
 294     api_req_url = data['source'] + '&start=' + begin_date_str + '&end=' + end_date_str
 295     req = requests.get(api_req_url)
 296
 297     try:
 298         json = req.json()
 299     except ValueError:
 300         log.error("Bad JSON in MesoWest response")
 301         sys.exit(1)
 302
 303     try:
 304         observations = json['STATION'][0]['OBSERVATIONS']
 305     except ValueError:
 306         log.error("Bad JSON in MesoWest response")
 307         sys.exit(1)
 308
 309     pos = len(observations['date_time']) - 1
 310
 311     for elementCd in desired_data.split(','):
 312         # sort and isolate the most recent, see note above in NRCS for how and
 313         # why this is done
 314         #
 315         # NOTE: Unlike in the NRCS case, the MesoWest API respones contains all
 316         #       data (whereas with NRCS, we have to make a separate request for
 317         #       each element we want. This is nice for network efficiency but
 318         #       it means we have to handle this part differently for each.
 319         #
 320         # NOTE: Also unlike NRCS, MesoWest provides more granular data; NRCS
 321         #       provides hourly data, but MesoWest can often provide data every
 322         #       10 minutes -- though this provides more opportunity for
 323         #       irregularities
 324
 325         # we may not have the data at all
 326         key_name = elementCd + '_set_1'
 327         if key_name in observations:
 328             if observations[key_name][pos]:
 329                 infoex['wx_data'][elementCd] = observations[key_name][pos]
 330             else:
 331                 infoex['wx_data'][elementCd] = None
 332         else:
 333             infoex['wx_data'][elementCd] = None
 334
 335 log.info("Time to get all data : %.3f sec" % (time.time() -
 336     time_all_elements))
 337
 338 log.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
 339
 340 # Now we only need to add in what we want to change thanks to that
 341 # abomination of a variable declaration earlier
 342 final_data[fmap['Location UUID']] = infoex['location_uuid']
 343 final_data[fmap['obDate']] = end_date.strftime('%m/%d/%Y')
 344 final_data[fmap['obTime']] = end_date.strftime('%H:%M')
 345
 346 for elementCd in infoex['wx_data']:
 347     if elementCd not in iemap:
 348         log.warning("BAD KEY wx_data['%s']" % (elementCd))
 349         continue
 350
 351     # CONSIDER: Casting every value to Float() -- need to investigate if
 352     #           any possible elementCds we may want are any other data
 353     #           type than float.
 354     #
 355     #           Another possibility is to query the API  with
 356     #           getStationElements and temporarily store the
 357     #           storedUnitCd. But that's pretty network-intensive and
 358     #           may not even be worth it if there's only e.g. one or two
 359     #           exceptions to any otherwise uniformly Float value set.
 360     final_data[fmap[iemap[elementCd]]] = infoex['wx_data'][elementCd]
 361
 362 log.debug("final_data: %s" % (str(final_data)))
 363
 364 with open(infoex['csv_filename'], 'w') as f:
 365     # The requirement is that empty values are represented in the CSV
 366     # file as "", csv.QUOTE_NONNUMERIC achieves that
 367     log.debug("writing CSV file '%s'" % (infoex['csv_filename']))
 368     writer = csv.writer(f, quoting=csv.QUOTE_NONNUMERIC)
 369     writer.writerow(final_data)
 370     f.close()
 371
 372 if not options.dry_run:
 373     # not a dry run
 374     with open(infoex['csv_filename'], 'rb') as f:
 375         log.debug("uploading FTP file '%s'" % (infoex['host']))
 376         ftp = FTP(infoex['host'], infoex['uuid'], infoex['api_key'])
 377         ftp.storlines('STOR ' + infoex['csv_filename'], f)
 378         ftp.close()
 379         f.close()
 380     os.remove(infoex['csv_filename'])
 381
 382 log.debug('DONE')