infoex-autowx.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 """
   5 InfoEx <-> NRCS/MesoWest Auto Wx implementation
   6 Alexander Vasarab
   7 Wylark Mountaineering LLC
   8
   9 This program fetches data from either an NRCS SNOTEL site or MesoWest
  10 weather station and pushes it to InfoEx using the new automated weather
  11 system implementation.
  12
  13 It is designed to be run hourly, and it asks for the last three hours
  14 of data of each desired type, and selects the most recent one. This
  15 lends some resiliency to the process and helps ensure that we have a
  16 value to send, but it can lead to somewhat inconsistent/untruthful
  17 data if e.g. the HS is from the last hour but the tempPres is from two
  18 hours ago because the instrumentation had a hiccup. It's worth
  19 considering if this is a bug or a feature.
  20
  21 For more information, see file: README
  22 For licensing, see file: LICENSE
  23 """
  24
  25 import configparser
  26 import csv
  27 import datetime
  28 import logging
  29 import os
  30 import sys
  31 import time
  32 import urllib3
  33 import importlib.util
  34
  35 from ftplib import FTP
  36 from argparse import ArgumentParser
  37
  38 import pytz
  39
  40 import requests
  41
  42 import zeep
  43 import zeep.cache
  44 import zeep.transports
  45
  46 __version__ = '3.4.4'
  47
  48 LOG = logging.getLogger(__name__)
  49 LOG.setLevel(logging.NOTSET)
  50
  51 urllib3.disable_warnings()
  52
  53 def get_parser():
  54     """Return OptionParser for this program"""
  55     parser = ArgumentParser()
  56
  57     parser.add_argument("--version",
  58                         action="version",
  59                         version=__version__)
  60
  61     parser.add_argument("--config",
  62                         dest="config",
  63                         metavar="FILE",
  64                         help="location of config file")
  65
  66     parser.add_argument("--log-level",
  67                         dest="log_level",
  68                         default=None,
  69                         help="set the log level (debug, info, warning)")
  70
  71     parser.add_argument("--dry-run",
  72                         action="store_true",
  73                         dest="dry_run",
  74                         default=False,
  75                         help="fetch data but don't upload to InfoEx")
  76
  77     return parser
  78
  79 def setup_config(config):
  80     """Setup config variable based on values specified in the ini file"""
  81     try:
  82         infoex = {
  83             'host': config['infoex']['host'],
  84             'uuid': config['infoex']['uuid'],
  85             'api_key': config['infoex']['api_key'],
  86             'csv_filename': config['infoex']['csv_filename'],
  87             'location_uuid': config['infoex']['location_uuid'],
  88             'wx_data': {}, # placeholder key, values to come later
  89         }
  90
  91         station = dict()
  92
  93         station['provider'] = config['station']['type']
  94
  95         if station['provider'] not in ['nrcs', 'mesowest', 'python']:
  96             print("Please specify either nrcs or mesowest as the station type.")
  97             sys.exit(1)
  98
  99         # massage units config items first
 100         #
 101         # NOTE: custom providers don't require units to be specified
 102         #       because they can do whatever they please with the units
 103         #       within their own program
 104         if station['provider'] != "python":
 105             station['units'] = config['station']['units']
 106
 107             if station['units'] not in ['metric', 'english', 'american']:
 108                 print("Please specify metric, english, or american for the units.")
 109                 sys.exit(1)
 110             else:
 111                 # if units are specified as "American" then we simply
 112                 # default to metric for the requests
 113                 if station['units'] == 'american':
 114                     station['units_requested'] = 'metric'
 115                 else:
 116                     station['units_requested'] = station['units']
 117
 118         # massage provider config items
 119         if station['provider'] == 'nrcs':
 120             #station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
 121             station['source'] = 'https://wcc.sc.egov.usda.gov/awdbWebService/services?WSDL'
 122
 123             station['station_id'] = config['station']['station_id']
 124             station['desired_data'] = config['station']['desired_data'].split(',')
 125
 126         if station['provider'] == 'mesowest':
 127             station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
 128             station['station_id'] = config['station']['station_id']
 129             station['desired_data'] = config['station']['desired_data']
 130
 131             # construct full API URL (sans start/end time, added later)
 132             station['source'] = station['source'] + '?token=' + \
 133                                 config['station']['token'] + \
 134                                 '&within=60&units=' + station['units_requested'] + \
 135                                 '&stid=' + station['station_id'] + \
 136                                 '&vars=' + station['desired_data']
 137
 138         if station['provider'] == 'python':
 139             station['path'] = config['station']['path']
 140
 141         tz = 'America/Los_Angeles'
 142
 143         if 'tz' in config['station']:
 144             tz = config['station']['tz']
 145
 146         try:
 147             station['tz'] = pytz.timezone(tz)
 148         except pytz.exceptions.UnknownTimeZoneError:
 149             LOG.critical("%s is not a valid timezone", tz)
 150             sys.exit(1)
 151
 152         # By default, fetch three hours of data
 153         #
 154         # If user wants hn24 or wind averaging, then
 155         # we need more.
 156         station['num_hrs_to_fetch'] = 3
 157
 158         # HN24
 159         if 'hn24' in config['station']:
 160             if config['station']['hn24'] not in ['true', 'false']:
 161                 raise ValueError("hn24 must be either 'true' or 'false'")
 162
 163             if config['station']['hn24'] == "true":
 164                 station['hn24'] = True
 165                 station['num_hrs_to_fetch'] = 24
 166             else:
 167                 station['hn24'] = False
 168         else:
 169             # default to False
 170             station['hn24'] = False
 171
 172         # Wind mode
 173         if 'wind_mode' in config['station']:
 174             if config['station']['wind_mode'] not in ['normal', 'average']:
 175                 raise ValueError("wind_mode must be either 'normal' or 'average'")
 176
 177             station['wind_mode'] = config['station']['wind_mode']
 178
 179             if station['wind_mode'] == "average":
 180                 station['num_hrs_to_fetch'] = 24
 181         else:
 182             # default to False
 183             station['wind_mode'] = "normal"
 184
 185     except KeyError as err:
 186         LOG.critical("%s not defined in configuration file", err)
 187         sys.exit(1)
 188     except ValueError as err:
 189         LOG.critical("%s", err)
 190         sys.exit(1)
 191
 192     # all sections/values present in config file, final sanity check
 193     try:
 194         for key in config.sections():
 195             for subkey in config[key]:
 196                 if not config[key][subkey]:
 197                     raise ValueError
 198     except ValueError:
 199         LOG.critical("Config value '%s.%s' is empty", key, subkey)
 200         sys.exit(1)
 201
 202     return (infoex, station)
 203
 204 def setup_logging(log_level):
 205     """Setup our logging infrastructure"""
 206     try:
 207         from systemd.journal import JournalHandler
 208         LOG.addHandler(JournalHandler())
 209     except ImportError:
 210         ## fallback to syslog
 211         #import logging.handlers
 212         #LOG.addHandler(logging.handlers.SysLogHandler())
 213         # fallback to stdout
 214         handler = logging.StreamHandler(sys.stdout)
 215         formatter = logging.Formatter('%(asctime)s.%(msecs)03d '
 216                                       '%(levelname)s %(module)s - '
 217                                       '%(funcName)s: %(message)s',
 218                                       '%Y-%m-%d %H:%M:%S')
 219         handler.setFormatter(formatter)
 220         LOG.addHandler(handler)
 221
 222     # ugly, but passable
 223     if log_level in [None, 'debug', 'info', 'warning']:
 224         if log_level == 'debug':
 225             LOG.setLevel(logging.DEBUG)
 226         elif log_level == 'info':
 227             LOG.setLevel(logging.INFO)
 228         elif log_level == 'warning':
 229             LOG.setLevel(logging.WARNING)
 230         else:
 231             LOG.setLevel(logging.NOTSET)
 232     else:
 233         return False
 234
 235     return True
 236
 237 def main():
 238     """Main routine: sort through args, decide what to do, then do it"""
 239     parser = get_parser()
 240     options = parser.parse_args()
 241
 242     config = configparser.ConfigParser(allow_no_value=False)
 243
 244     if not options.config:
 245         parser.print_help()
 246         print("\nPlease specify a configuration file via --config.")
 247         sys.exit(1)
 248
 249     config.read(options.config)
 250
 251     if not setup_logging(options.log_level):
 252         parser.print_help()
 253         print("\nPlease select an appropriate log level or remove the switch (--log-level).")
 254         sys.exit(1)
 255
 256     (infoex, station) = setup_config(config)
 257
 258     LOG.debug('Config parsed, starting up')
 259
 260     # create mappings
 261     (fmap, final_data) = setup_infoex_fields_mapping(infoex['location_uuid'])
 262     iemap = setup_infoex_counterparts_mapping(station['provider'])
 263
 264     # override units if user selected metric
 265     if station['provider'] != 'python' and station['units'] == 'metric':
 266         final_data = switch_units_to_metric(final_data, fmap)
 267
 268     # likewise for "American" units
 269     if station['provider'] != 'python' and station['units'] == 'american':
 270         final_data = switch_units_to_american(final_data, fmap)
 271
 272     (begin_date, end_date) = setup_time_values(station)
 273
 274     if station['provider'] == 'python':
 275         LOG.debug("Getting custom data from external Python program")
 276     else:
 277         LOG.debug("Getting %s data from %s to %s (%s)",
 278                   str(station['desired_data']),
 279                   str(begin_date), str(end_date), end_date.tzinfo.zone)
 280
 281     time_all_elements = time.time()
 282
 283     # get the data
 284     if station['provider'] == 'nrcs':
 285         infoex['wx_data'] = get_nrcs_data(begin_date, end_date, station)
 286     elif station['provider'] == 'mesowest':
 287         infoex['wx_data'] = get_mesowest_data(begin_date, end_date,
 288                                               station)
 289     elif station['provider'] == 'python':
 290         try:
 291             spec = importlib.util.spec_from_file_location('custom_wx',
 292                                                           station['path'])
 293             mod = importlib.util.module_from_spec(spec)
 294             spec.loader.exec_module(mod)
 295             mod.LOG = LOG
 296
 297             try:
 298                 infoex['wx_data'] = mod.get_custom_data()
 299
 300                 if infoex['wx_data'] is None:
 301                     infoex['wx_data'] = []
 302             except Exception as exc:
 303                 LOG.error("Python program for custom Wx data failed in "
 304                           "execution: %s", str(exc))
 305                 sys.exit(1)
 306
 307             LOG.info("Successfully executed external Python program")
 308         except ImportError:
 309             LOG.error("Please upgrade to Python 3.3 or later")
 310             sys.exit(1)
 311         except FileNotFoundError:
 312             LOG.error("Specified Python program for custom Wx data "
 313                       "was not found")
 314             sys.exit(1)
 315         except Exception as exc:
 316             LOG.error("A problem was encountered when attempting to "
 317                       "load your custom Wx program: %s", str(exc))
 318             sys.exit(1)
 319
 320     LOG.info("Time taken to get all data : %.3f sec", time.time() -
 321              time_all_elements)
 322
 323     LOG.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
 324
 325     # timezone massaging
 326     final_end_date = end_date.astimezone(station['tz'])
 327
 328     # Now we only need to add in what we want to change thanks to that
 329     # abomination of a variable declaration earlier
 330     final_data[fmap['Location UUID']] = infoex['location_uuid']
 331     final_data[fmap['obDate']] = final_end_date.strftime('%m/%d/%Y')
 332     final_data[fmap['obTime']] = final_end_date.strftime('%H:%M')
 333     final_data[fmap['timeZone']] = station['tz'].zone
 334
 335     for element_cd in infoex['wx_data']:
 336         if element_cd not in iemap:
 337             LOG.warning("BAD KEY wx_data['%s']", element_cd)
 338             continue
 339
 340         if infoex['wx_data'][element_cd] is None:
 341             continue
 342
 343         # do the conversion before the rounding
 344         if station['provider'] == 'nrcs' and station['units'] == 'metric':
 345             infoex['wx_data'][element_cd] = convert_nrcs_units_to_metric(element_cd, infoex['wx_data'][element_cd])
 346
 347         if station['provider'] != 'python' and station['units'] == 'american':
 348             infoex['wx_data'][element_cd] = convert_units_to_american(element_cd, infoex['wx_data'][element_cd])
 349
 350         # Massage precision of certain values to fit InfoEx's
 351         # expectations
 352         #
 353         # 0 decimal places: relative humidity, wind speed, wind
 354         #                   direction, wind gust, snow depth
 355         # 1 decimal place:  air temp, baro
 356         # Avoid transforming None values
 357         if element_cd in ['wind_speed', 'WSPD', 'wind_direction',
 358                           'RHUM', 'relative_humidity', 'WDIR',
 359                           'wind_gust', 'SNWD', 'snow_depth',
 360                           'hn24']:
 361             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd])
 362         elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']:
 363             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1)
 364         elif element_cd in ['PREC', 'precip_accum']:
 365             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 2)
 366
 367         # CONSIDER: Casting every value to Float() -- need to investigate if
 368         #           any possible elementCds we may want are any other data
 369         #           type than float.
 370         #
 371         #           Another possibility is to query the API with
 372         #           getStationElements and temporarily store the
 373         #           storedUnitCd. But that's pretty network-intensive and
 374         #           may not even be worth it if there's only e.g. one or two
 375         #           exceptions to any otherwise uniformly Float value set.
 376         final_data[fmap[iemap[element_cd]]] = infoex['wx_data'][element_cd]
 377
 378     LOG.debug("final_data: %s", str(final_data))
 379
 380     if infoex['wx_data']:
 381         if not write_local_csv(infoex['csv_filename'], final_data):
 382             LOG.warning('Could not write local CSV file: %s',
 383                         infoex['csv_filename'])
 384             return 1
 385
 386         if not options.dry_run:
 387             upload_csv(infoex['csv_filename'], infoex)
 388
 389     LOG.debug('DONE')
 390     return 0
 391
 392 # data structure operations
 393 def setup_infoex_fields_mapping(location_uuid):
 394     """
 395     Create a mapping of InfoEx fields to the local data's indexing scheme.
 396
 397     INFOEX FIELDS
 398
 399     This won't earn style points in Python, but here we establish a couple
 400     of helpful mappings variables. The reason this is helpful is that the
 401     end result is simply an ordered set, the CSV file. But we still may
 402     want to manipulate the values arbitrarily before writing that file.
 403
 404     Also note that the current Auto Wx InfoEx documentation shows these
 405     keys in a graphical table with the "index" beginning at 1, but here we
 406     sanely index beginning at 0.
 407     """
 408     # pylint: disable=too-many-statements,multiple-statements,bad-whitespace
 409     fmap = {}                           ; final_data     = [None] * 29
 410     fmap['Location UUID'] = 0           ; final_data[0]  = location_uuid
 411     fmap['obDate'] = 1                  ; final_data[1]  = None
 412     fmap['obTime'] = 2                  ; final_data[2]  = None
 413     fmap['timeZone'] = 3                ; final_data[3]  = 'Pacific'
 414     fmap['tempMaxHour'] = 4             ; final_data[4]  = None
 415     fmap['tempMaxHourUnit'] = 5         ; final_data[5]  = 'F'
 416     fmap['tempMinHour'] = 6             ; final_data[6]  = None
 417     fmap['tempMinHourUnit'] = 7         ; final_data[7]  = 'F'
 418     fmap['tempPres'] = 8                ; final_data[8]  = None
 419     fmap['tempPresUnit'] = 9            ; final_data[9]  = 'F'
 420     fmap['precipitationGauge'] = 10     ; final_data[10] = None
 421     fmap['precipitationGaugeUnit'] = 11 ; final_data[11] = 'in'
 422     fmap['windSpeedNum'] = 12           ; final_data[12] = None
 423     fmap['windSpeedUnit'] = 13          ; final_data[13] = 'mph'
 424     fmap['windDirectionNum'] = 14       ; final_data[14] = None
 425     fmap['hS'] = 15                     ; final_data[15] = None
 426     fmap['hsUnit'] = 16                 ; final_data[16] = 'in'
 427     fmap['baro'] = 17                   ; final_data[17] = None
 428     fmap['baroUnit'] = 18               ; final_data[18] = 'inHg'
 429     fmap['rH'] = 19                     ; final_data[19] = None
 430     fmap['windGustSpeedNum'] = 20       ; final_data[20] = None
 431     fmap['windGustSpeedNumUnit'] = 21   ; final_data[21] = 'mph'
 432     fmap['windGustDirNum'] = 22         ; final_data[22] = None
 433     fmap['dewPoint'] = 23               ; final_data[23] = None
 434     fmap['dewPointUnit'] = 24           ; final_data[24] = 'F'
 435     fmap['hn24Auto'] = 25               ; final_data[25] = None
 436     fmap['hn24AutoUnit'] = 26           ; final_data[26] = 'in'
 437     fmap['hstAuto'] = 27                ; final_data[27] = None
 438     fmap['hstAutoUnit'] = 28            ; final_data[28] = 'in'
 439
 440     return (fmap, final_data)
 441
 442 def setup_infoex_counterparts_mapping(provider):
 443     """
 444     Create a mapping of the NRCS/MesoWest fields that this program supports to
 445     their InfoEx counterparts
 446     """
 447     iemap = {}
 448
 449     if provider == 'nrcs':
 450         iemap['PREC'] = 'precipitationGauge'
 451         iemap['TOBS'] = 'tempPres'
 452         iemap['TMAX'] = 'tempMaxHour'
 453         iemap['TMIN'] = 'tempMinHour'
 454         iemap['SNWD'] = 'hS'
 455         iemap['PRES'] = 'baro'
 456         iemap['RHUM'] = 'rH'
 457         iemap['WSPD'] = 'windSpeedNum'
 458         iemap['WDIR'] = 'windDirectionNum'
 459         # unsupported by NRCS:
 460         # windGustSpeedNum
 461
 462         # NOTE: this doesn't exist in NRCS SNOTEL, we create it in this
 463         #       program, so add it to the map here
 464         iemap['hn24'] = 'hn24Auto'
 465     elif provider == 'mesowest':
 466         iemap['precip_accum'] = 'precipitationGauge'
 467         iemap['air_temp'] = 'tempPres'
 468         iemap['air_temp_high_24_hour'] = 'tempMaxHour'
 469         iemap['air_temp_low_24_hour'] = 'tempMinHour'
 470         iemap['snow_depth'] = 'hS'
 471         iemap['pressure'] = 'baro'
 472         iemap['relative_humidity'] = 'rH'
 473         iemap['wind_speed'] = 'windSpeedNum'
 474         iemap['wind_direction'] = 'windDirectionNum'
 475         iemap['wind_gust'] = 'windGustSpeedNum'
 476
 477         # NOTE: this doesn't exist in MesoWest, we create it in this
 478         #       program, so add it to the map here
 479         iemap['hn24'] = 'hn24Auto'
 480     elif provider == 'python':
 481         # we expect Python programs to use the InfoEx data type names
 482         iemap['precipitationGauge'] = 'precipitationGauge'
 483         iemap['precipitationGaugeUnit'] = 'precipitationGaugeUnit'
 484         iemap['tempPres'] = 'tempPres'
 485         iemap['tempPresUnit'] = 'tempPresUnit'
 486         iemap['tempMaxHour'] = 'tempMaxHour'
 487         iemap['tempMaxHourUnit'] = 'tempMaxHourUnit'
 488         iemap['tempMinHour'] = 'tempMinHour'
 489         iemap['tempMinHourUnit'] = 'tempMinHourUnit'
 490         iemap['hS'] = 'hS'
 491         iemap['hsUnit'] = 'hsUnit'
 492         iemap['baro'] = 'baro'
 493         iemap['rH'] = 'rH'
 494         iemap['windSpeedNum'] = 'windSpeedNum'
 495         iemap['windSpeedUnit'] = 'windSpeedUnit'
 496         iemap['windDirectionNum'] = 'windDirectionNum'
 497         iemap['windGustSpeedNum'] = 'windGustSpeedNum'
 498         iemap['dewPointUnit'] = 'dewPointUnit'
 499         iemap['hn24AutoUnit'] = 'hn24AutoUnit'
 500         iemap['hstAutoUnit'] = 'hstAutoUnit'
 501
 502     return iemap
 503
 504 # provider-specific operations
 505 def get_nrcs_data(begin, end, station):
 506     """get the data we're after from the NRCS WSDL"""
 507     transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
 508     transport.session.verify = False
 509     client = zeep.Client(wsdl=station['source'], transport=transport)
 510     remote_data = {}
 511
 512     # massage begin/end date format
 513     begin_date_str = begin.strftime('%Y-%m-%d %H:%M:00')
 514     end_date_str = end.strftime('%Y-%m-%d %H:%M:00')
 515
 516     for element_cd in station['desired_data']:
 517         time_element = time.time()
 518
 519         # get the last three hours of data for this elementCd/element_cd
 520         tmp = client.service.getHourlyData(
 521             stationTriplets=[station['station_id']],
 522             elementCd=element_cd,
 523             ordinal=1,
 524             beginDate=begin_date_str,
 525             endDate=end_date_str)
 526
 527         LOG.info("Time to get NRCS elementCd '%s': %.3f sec", element_cd,
 528                  time.time() - time_element)
 529
 530         values = tmp[0]['values']
 531
 532         # sort and isolate the most recent
 533         #
 534         # NOTE: we do this because sometimes there are gaps in hourly data
 535         #       in NRCS; yes, we may end up with slightly inaccurate data,
 536         #       so perhaps this decision will be re-evaluated in the future
 537         if values:
 538             ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
 539             remote_data[element_cd] = ordered[0]['value']
 540         else:
 541             remote_data[element_cd] = None
 542
 543
 544         # calc hn24, if applicable
 545         hn24 = None
 546
 547         if station['hn24']:
 548             hn24_values = []
 549
 550             if element_cd == "SNWD":
 551                 for idx, _ in enumerate(values):
 552                     val = values[idx]
 553                     if val is None:
 554                         continue
 555                     hn24_values.append(val['value'])
 556
 557                 if len(hn24_values) > 0:
 558                     # instead of taking MAX - MIN, we want the first
 559                     # value (most distant) - the last value (most
 560                     # recent)
 561                     #
 562                     # if the result is positive, then we have
 563                     # settlement; if it's not, then we have HN24
 564                     hn24 = hn24_values[0] - hn24_values[len(hn24_values)-1]
 565
 566                     if hn24 < 0.0:
 567                         hn24 = abs(hn24)
 568                     else:
 569                         # this case represents HS settlement
 570                         hn24 = 0.0
 571
 572             # finally, if user wants hn24 and it's set to None at this
 573             # point, then force it to 0.0
 574             if hn24 is None:
 575                 hn24 = 0.0
 576
 577         if hn24 is not None:
 578             if station['hn24']:
 579                 remote_data['hn24'] = hn24
 580
 581     return remote_data
 582
 583 def get_mesowest_data(begin, end, station):
 584     """get the data we're after from the MesoWest/Synoptic API"""
 585     remote_data = {}
 586
 587     # massage begin/end date format
 588     begin_date_str = begin.strftime('%Y%m%d%H%M')
 589     end_date_str = end.strftime('%Y%m%d%H%M')
 590
 591     # construct final, completed API URL
 592     api_req_url = station['source'] + '&start=' + begin_date_str + '&end=' + end_date_str
 593
 594     try:
 595         req = requests.get(api_req_url)
 596     except requests.exceptions.ConnectionError:
 597         LOG.error("Could not connect to '%s'", api_req_url)
 598         sys.exit(1)
 599
 600     try:
 601         json = req.json()
 602     except ValueError:
 603         LOG.error("Bad JSON in MesoWest response")
 604         sys.exit(1)
 605
 606     try:
 607         observations = json['STATION'][0]['OBSERVATIONS']
 608     except KeyError as exc:
 609         LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
 610         sys.exit(1)
 611     except IndexError as exc:
 612         LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
 613         try:
 614             LOG.error("Detailed MesoWest response: '%s'",
 615                       json['SUMMARY']['RESPONSE_MESSAGE'])
 616         except KeyError:
 617             pass
 618         sys.exit(1)
 619     except ValueError as exc:
 620         LOG.error("Bad JSON in MesoWest response: '%s'", exc)
 621         sys.exit(1)
 622
 623     # pos represents the last item in the array, aka the most recent
 624     pos = len(observations['date_time']) - 1
 625
 626     # while these values only apply in certain cases, init them here
 627     wind_speed_values = []
 628     wind_gust_speed_values = []
 629     wind_direction_values = []
 630     hn24_values = []
 631
 632     # results
 633     wind_speed_avg = None
 634     wind_gust_speed_avg = None
 635     wind_direction_avg = None
 636     hn24 = None
 637
 638     for element_cd in station['desired_data'].split(','):
 639         # sort and isolate the most recent, see note above in NRCS for how and
 640         # why this is done
 641         #
 642         # NOTE: Unlike in the NRCS case, the MesoWest API response contains all
 643         #       data (whereas with NRCS, we have to make a separate request for
 644         #       each element we want). This is nice for network efficiency but
 645         #       it means we have to handle this part differently for each.
 646         #
 647         # NOTE: Also unlike NRCS, MesoWest provides more granular data; NRCS
 648         #       provides hourly data, but MesoWest can often provide data every
 649         #       10 minutes -- though this provides more opportunity for
 650         #       irregularities
 651
 652         # we may not have the data at all
 653         key_name = element_cd + '_set_1'
 654
 655         if key_name in observations:
 656             # val is what will make it into the dataset, after
 657             # conversions... it gets defined here because in certain
 658             # cases we need to look at all of the data to calculate HN24
 659             # or wind averages, but for the rest of the data, we only
 660             # take the most recent
 661             val = None
 662
 663             # loop through all observations for this key_name
 664             # record relevant values for wind averaging or hn24, but
 665             # otherwise only persist the data if it's the last datum in
 666             # the set
 667             for idx, _ in enumerate(observations[key_name]):
 668                 val = observations[key_name][idx]
 669
 670                 # skip bunk vals
 671                 if val is None:
 672                     continue
 673
 674                 # mesowest by default provides wind_speed in m/s, but
 675                 # we specify 'english' units in the request; either way,
 676                 # we want mph
 677                 if element_cd in ('wind_speed', 'wind_gust'):
 678                     val = kn_to_mph(val)
 679
 680                 # mesowest provides HS in mm, not cm; we want cm
 681                 if element_cd == 'snow_depth' and station['units'] == 'metric':
 682                     val = mm_to_cm(val)
 683
 684                 # HN24 / wind_mode transformations, once the data has
 685                 # completed unit conversions
 686                 if station['wind_mode'] == "average":
 687                     if element_cd == 'wind_speed' and val is not None:
 688                         wind_speed_values.append(val)
 689                     elif element_cd == 'wind_gust' and val is not None:
 690                         wind_gust_speed_values.append(val)
 691                     elif element_cd == 'wind_direction' and val is not None:
 692                         wind_direction_values.append(val)
 693
 694                 if element_cd == 'snow_depth':
 695                     hn24_values.append(val)
 696
 697                 # again, only persist this datum to the final data if
 698                 # it's from the most recent date
 699                 if idx == pos:
 700                     remote_data[element_cd] = val
 701
 702             # ensure that the data is filled out
 703             if not observations[key_name][pos]:
 704                 remote_data[element_cd] = None
 705         else:
 706             remote_data[element_cd] = None
 707
 708     if len(hn24_values) > 0:
 709         # instead of taking MAX - MIN, we want the first value (most
 710         # distant) - the last value (most recent)
 711         #
 712         # if the result is positive, then we have settlement; if it's not,
 713         # then we have HN24
 714         hn24 = hn24_values[0] - hn24_values[len(hn24_values)-1]
 715
 716         if hn24 < 0.0:
 717             hn24 = abs(hn24)
 718         else:
 719             # this case represents HS settlement
 720             hn24 = 0.0
 721
 722
 723     # finally, if user wants hn24 and it's set to None at this
 724     # point, then force it to 0.0
 725     if station['hn24'] and hn24 is None:
 726         hn24 = 0.0
 727
 728     if len(wind_speed_values) > 0:
 729         wind_speed_avg = sum(wind_speed_values) / len(wind_speed_values)
 730
 731     if len(wind_gust_speed_values) > 0:
 732         wind_gust_speed_avg = sum(wind_gust_speed_values) / len(wind_gust_speed_values)
 733
 734     if len(wind_direction_values) > 0:
 735         wind_direction_avg = sum(wind_direction_values) / len(wind_direction_values)
 736
 737     if hn24 is not None:
 738         if station['hn24']:
 739             remote_data['hn24'] = hn24
 740
 741     # overwrite the following with the respective averages, if
 742     # applicable
 743     if wind_speed_avg is not None:
 744         remote_data['wind_speed'] = wind_speed_avg
 745
 746     if wind_gust_speed_avg is not None:
 747         remote_data['wind_gust'] = wind_gust_speed_avg
 748
 749     if wind_direction_avg is not None:
 750         remote_data['wind_direction'] = wind_direction_avg
 751
 752     return remote_data
 753
 754 def switch_units_to_metric(data_map, mapping):
 755     """replace units with metric counterparts"""
 756
 757     # NOTE: to update this, use the fmap<->final_data mapping laid out
 758     #       in setup_infoex_fields_mapping ()
 759     data_map[mapping['tempMaxHourUnit']] = 'C'
 760     data_map[mapping['tempMinHourUnit']] = 'C'
 761     data_map[mapping['tempPresUnit']] = 'C'
 762     data_map[mapping['precipitationGaugeUnit']] = 'mm'
 763     data_map[mapping['hsUnit']] = 'cm'
 764     data_map[mapping['windSpeedUnit']] = 'm/s'
 765     data_map[mapping['windGustSpeedNumUnit']] = 'm/s'
 766     data_map[mapping['dewPointUnit']] = 'C'
 767     data_map[mapping['hn24AutoUnit']] = 'cm'
 768     data_map[mapping['hstAutoUnit']] = 'cm'
 769
 770     return data_map
 771
 772 def switch_units_to_american(data_map, mapping):
 773     """
 774     replace units with the American mixture of metric and imperial
 775
 776     Precip values = metric
 777     Wind values = imperial
 778     """
 779
 780     # precip values
 781     data_map[mapping['tempMaxHourUnit']] = 'C'
 782     data_map[mapping['tempMinHourUnit']] = 'C'
 783     data_map[mapping['tempPresUnit']] = 'C'
 784     data_map[mapping['dewPointUnit']] = 'C'
 785
 786     data_map[mapping['precipitationGaugeUnit']] = 'cm'
 787     data_map[mapping['hsUnit']] = 'cm'
 788     data_map[mapping['hn24AutoUnit']] = 'cm'
 789     data_map[mapping['hstAutoUnit']] = 'cm'
 790
 791     data_map[mapping['baroUnit']] = 'inHg'
 792
 793     # wind values
 794     data_map[mapping['windSpeedUnit']] = 'mph'
 795     data_map[mapping['windGustSpeedNumUnit']] = 'mph'
 796
 797     return data_map
 798
 799 def convert_nrcs_units_to_metric(element_cd, value):
 800     """convert NRCS values from English to metric"""
 801     if element_cd == 'TOBS':
 802         value = f_to_c(value)
 803     elif element_cd == 'SNWD':
 804         value = in_to_cm(value)
 805     elif element_cd == 'PREC':
 806         value = in_to_mm(value)
 807     return value
 808
 809 def convert_units_to_american(element_cd, value):
 810     """
 811     convert value to 'American' units
 812
 813     The original unit is always metric.
 814
 815     Precip values = metric
 816     Wind values = imperial
 817     """
 818
 819     # no need to convert precip values, as they will arrive in metric
 820     # units in "American" units mode
 821     # # temp values
 822     # if element_cd in ['TMAX', 'TMIN', 'TOBS', 'air_temp', 'air_temp_high_24_hour', 'air_temp_low_24_hour']:
 823     #     value = c_to_f(value)
 824
 825     # mesowest provides HS in mm, not cm; we want cm
 826     if element_cd == 'snow_depth':
 827         value = mm_to_cm(value)
 828
 829     # baro values also arrive in metric, so convert to imperial
 830     if element_cd in ['PRES', 'pressure']:
 831         value = inhg_to_pascal(value)
 832
 833     if element_cd in ['WSPD', 'wind_speed', 'wind_gust']:
 834         value = ms_to_mph(value)
 835
 836     return value
 837
 838 # CSV operations
 839 def write_local_csv(path_to_file, data):
 840     """Write the specified CSV file to disk"""
 841     with open(path_to_file, 'w') as file_object:
 842         # The requirement is that empty values are represented in the CSV
 843         # file as "", csv.QUOTE_NONNUMERIC achieves that
 844         LOG.debug("writing CSV file '%s'", path_to_file)
 845         writer = csv.writer(file_object, quoting=csv.QUOTE_NONNUMERIC)
 846         writer.writerow(data)
 847         file_object.close()
 848     return True
 849
 850 def upload_csv(path_to_file, infoex_data):
 851     """Upload the specified CSV file to InfoEx FTP and remove the file"""
 852     with open(path_to_file, 'rb') as file_object:
 853         LOG.debug("uploading FTP file '%s'", infoex_data['host'])
 854         ftp = FTP(infoex_data['host'], infoex_data['uuid'],
 855                   infoex_data['api_key'])
 856         ftp.storlines('STOR ' + path_to_file, file_object)
 857         ftp.close()
 858         file_object.close()
 859     os.remove(path_to_file)
 860
 861 # other miscellaneous routines
 862 def setup_time_values(station):
 863     """establish time bounds of data request(s)"""
 864
 865     # default timezone to UTC (for MesoWest)
 866     tz = pytz.utc
 867
 868     # but for NRCS, use the config-specified timezone
 869     if station['provider'] == 'nrcs':
 870         tz = station['tz']
 871
 872     # floor time to nearest hour
 873     date_time = datetime.datetime.now(tz=tz)
 874     end_date = date_time - datetime.timedelta(minutes=date_time.minute % 60,
 875                                               seconds=date_time.second,
 876                                               microseconds=date_time.microsecond)
 877     begin_date = end_date - datetime.timedelta(hours=station['num_hrs_to_fetch'])
 878     return (begin_date, end_date)
 879
 880 def f_to_c(f):
 881     """convert Fahrenheit to Celsius"""
 882     return (float(f) - 32) * 5.0/9.0
 883
 884 def c_to_f(c):
 885     """convert Celsius to Fahrenheit"""
 886     return (float(c) * 1.8) + 32
 887
 888 def in_to_cm(inches):
 889     """convert inches to centimeters"""
 890     return float(inches) * 2.54
 891
 892 def cm_to_in(cms):
 893     """convert centimeters to inches"""
 894     return float(cms) / 2.54
 895
 896 def pascal_to_inhg(pa):
 897     """convert pascals to inches of mercury"""
 898     return float(pa) * 0.00029530
 899
 900 def inhg_to_pascal(inhg):
 901     """convert inches of mercury to pascals"""
 902     return float(inhg) / 0.00029530
 903
 904 def in_to_mm(inches):
 905     """convert inches to millimeters"""
 906     return (float(inches) * 2.54) * 10.0
 907
 908 def ms_to_mph(ms):
 909     """convert meters per second to miles per hour"""
 910     return ms * 2.236936
 911
 912 def kn_to_mph(kn):
 913     """convert knots to miles per hour"""
 914     return kn * 1.150779
 915
 916 def mm_to_cm(mm):
 917     """convert millimeters to centimeters"""
 918     return mm / 10.0
 919
 920 if __name__ == "__main__":
 921     sys.exit(main())