infoex-autowx.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 """
   5 InfoEx <-> NRCS/MesoWest Auto Wx implementation
   6 Alexander Vasarab
   7 Wylark Mountaineering LLC
   8
   9 This program fetches data from either an NRCS SNOTEL site or MesoWest
  10 weather station and pushes it to InfoEx using the new automated weather
  11 system implementation.
  12
  13 It is designed to be run hourly, and it asks for the last three hours
  14 of data of each desired type, and selects the most recent one. This
  15 lends some resiliency to the process and helps ensure that we have a
  16 value to send, but it can lead to somewhat inconsistent/untruthful
  17 data if e.g. the HS is from the last hour but the tempPres is from two
  18 hours ago because the instrumentation had a hiccup. It's worth
  19 considering if this is a bug or a feature.
  20
  21 For more information, see file: README
  22 For licensing, see file: LICENSE
  23 """
  24
  25 import configparser
  26 import csv
  27 import datetime
  28 import logging
  29 import os
  30 import socket
  31 import sys
  32 import time
  33 import urllib3
  34 import importlib.util
  35
  36 from ftplib import FTP
  37 from argparse import ArgumentParser
  38
  39 import pytz
  40
  41 import requests
  42
  43 import zeep
  44 import zeep.cache
  45 import zeep.transports
  46
  47 __version__ = '3.4.4'
  48
  49 LOG = logging.getLogger(__name__)
  50 LOG.setLevel(logging.NOTSET)
  51
  52 urllib3.disable_warnings()
  53
  54 def get_parser():
  55     """Return OptionParser for this program"""
  56     parser = ArgumentParser()
  57
  58     parser.add_argument("--version",
  59                         action="version",
  60                         version=__version__)
  61
  62     parser.add_argument("--config",
  63                         dest="config",
  64                         metavar="FILE",
  65                         help="location of config file")
  66
  67     parser.add_argument("--log-level",
  68                         dest="log_level",
  69                         default=None,
  70                         help="set the log level (debug, info, warning)")
  71
  72     parser.add_argument("--dry-run",
  73                         action="store_true",
  74                         dest="dry_run",
  75                         default=False,
  76                         help="fetch data but don't upload to InfoEx")
  77
  78     return parser
  79
  80 def setup_config(config):
  81     """Setup config variable based on values specified in the ini file"""
  82     try:
  83         infoex = {
  84             'host': config['infoex']['host'],
  85             'uuid': config['infoex']['uuid'],
  86             'api_key': config['infoex']['api_key'],
  87             'csv_filename': config['infoex']['csv_filename'],
  88             'location_uuid': config['infoex']['location_uuid'],
  89             'wx_data': {}, # placeholder key, values to come later
  90         }
  91
  92         station = dict()
  93
  94         station['provider'] = config['station']['type']
  95
  96         if station['provider'] not in ['nrcs', 'mesowest', 'python']:
  97             print("Please specify either nrcs or mesowest as the station type.")
  98             sys.exit(1)
  99
 100         # massage units config items first
 101         #
 102         # NOTE: custom providers don't require units to be specified
 103         #       because they can do whatever they please with the units
 104         #       within their own program
 105         if station['provider'] != "python":
 106             station['units'] = config['station']['units']
 107
 108             if station['units'] not in ['metric', 'english', 'american']:
 109                 print("Please specify metric, english, or american for the units.")
 110                 sys.exit(1)
 111             else:
 112                 # if units are specified as "American" then we simply
 113                 # default to metric for the requests
 114                 if station['units'] == 'american':
 115                     station['units_requested'] = 'metric'
 116                 else:
 117                     station['units_requested'] = station['units']
 118
 119         # massage provider config items
 120         if station['provider'] == 'nrcs':
 121             #station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
 122             station['source'] = 'https://wcc.sc.egov.usda.gov/awdbWebService/services?WSDL'
 123
 124             station['station_id'] = config['station']['station_id']
 125             station['desired_data'] = config['station']['desired_data'].split(',')
 126
 127         if station['provider'] == 'mesowest':
 128             station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
 129             station['station_id'] = config['station']['station_id']
 130             station['desired_data'] = config['station']['desired_data']
 131
 132             # construct full API URL (sans start/end time, added later)
 133             station['source'] = station['source'] + '?token=' + \
 134                                 config['station']['token'] + \
 135                                 '&within=60&units=' + station['units_requested'] + \
 136                                 '&stid=' + station['station_id'] + \
 137                                 '&vars=' + station['desired_data']
 138
 139         if station['provider'] == 'python':
 140             station['path'] = config['station']['path']
 141
 142         tz = 'America/Los_Angeles'
 143
 144         if 'tz' in config['station']:
 145             tz = config['station']['tz']
 146
 147         try:
 148             station['tz'] = pytz.timezone(tz)
 149         except pytz.exceptions.UnknownTimeZoneError:
 150             LOG.critical("%s is not a valid timezone", tz)
 151             sys.exit(1)
 152
 153         # By default, fetch three hours of data
 154         #
 155         # If user wants hn24 or wind averaging, then
 156         # we need more.
 157         station['num_hrs_to_fetch'] = 3
 158
 159         # HN24
 160         if 'hn24' in config['station']:
 161             if config['station']['hn24'] not in ['true', 'false']:
 162                 raise ValueError("hn24 must be either 'true' or 'false'")
 163
 164             if config['station']['hn24'] == "true":
 165                 station['hn24'] = True
 166                 station['num_hrs_to_fetch'] = 24
 167             else:
 168                 station['hn24'] = False
 169         else:
 170             # default to False
 171             station['hn24'] = False
 172
 173         # Wind mode
 174         if 'wind_mode' in config['station']:
 175             if config['station']['wind_mode'] not in ['normal', 'average']:
 176                 raise ValueError("wind_mode must be either 'normal' or 'average'")
 177
 178             station['wind_mode'] = config['station']['wind_mode']
 179
 180             if station['wind_mode'] == "average":
 181                 station['num_hrs_to_fetch'] = 24
 182         else:
 183             # default to False
 184             station['wind_mode'] = "normal"
 185
 186     except KeyError as err:
 187         LOG.critical("%s not defined in configuration file", err)
 188         sys.exit(1)
 189     except ValueError as err:
 190         LOG.critical("%s", err)
 191         sys.exit(1)
 192
 193     # all sections/values present in config file, final sanity check
 194     try:
 195         for key in config.sections():
 196             for subkey in config[key]:
 197                 if not config[key][subkey]:
 198                     raise ValueError
 199     except ValueError:
 200         LOG.critical("Config value '%s.%s' is empty", key, subkey)
 201         sys.exit(1)
 202
 203     return (infoex, station)
 204
 205 def setup_logging(log_level):
 206     """Setup our logging infrastructure"""
 207     try:
 208         from systemd.journal import JournalHandler
 209         LOG.addHandler(JournalHandler())
 210     except ImportError:
 211         ## fallback to syslog
 212         #import logging.handlers
 213         #LOG.addHandler(logging.handlers.SysLogHandler())
 214         # fallback to stdout
 215         handler = logging.StreamHandler(sys.stdout)
 216         formatter = logging.Formatter('%(asctime)s.%(msecs)03d '
 217                                       '%(levelname)s %(module)s - '
 218                                       '%(funcName)s: %(message)s',
 219                                       '%Y-%m-%d %H:%M:%S')
 220         handler.setFormatter(formatter)
 221         LOG.addHandler(handler)
 222
 223     # ugly, but passable
 224     if log_level in [None, 'debug', 'info', 'warning']:
 225         if log_level == 'debug':
 226             LOG.setLevel(logging.DEBUG)
 227         elif log_level == 'info':
 228             LOG.setLevel(logging.INFO)
 229         elif log_level == 'warning':
 230             LOG.setLevel(logging.WARNING)
 231         else:
 232             LOG.setLevel(logging.NOTSET)
 233     else:
 234         return False
 235
 236     return True
 237
 238 def main():
 239     """Main routine: sort through args, decide what to do, then do it"""
 240     parser = get_parser()
 241     options = parser.parse_args()
 242
 243     config = configparser.ConfigParser(allow_no_value=False)
 244
 245     if not options.config:
 246         parser.print_help()
 247         print("\nPlease specify a configuration file via --config.")
 248         sys.exit(1)
 249
 250     config.read(options.config)
 251
 252     if not setup_logging(options.log_level):
 253         parser.print_help()
 254         print("\nPlease select an appropriate log level or remove the switch (--log-level).")
 255         sys.exit(1)
 256
 257     (infoex, station) = setup_config(config)
 258
 259     LOG.debug('Config parsed, starting up')
 260
 261     # create mappings
 262     (fmap, final_data) = setup_infoex_fields_mapping(infoex['location_uuid'])
 263     iemap = setup_infoex_counterparts_mapping(station['provider'])
 264
 265     # override units if user selected metric
 266     if station['provider'] != 'python' and station['units'] == 'metric':
 267         final_data = switch_units_to_metric(final_data, fmap)
 268
 269     # likewise for "American" units
 270     if station['provider'] != 'python' and station['units'] == 'american':
 271         final_data = switch_units_to_american(final_data, fmap)
 272
 273     (begin_date, end_date) = setup_time_values(station)
 274
 275     if station['provider'] == 'python':
 276         LOG.debug("Getting custom data from external Python program")
 277     else:
 278         LOG.debug("Getting %s data from %s to %s (%s)",
 279                   str(station['desired_data']),
 280                   str(begin_date), str(end_date), end_date.tzinfo.zone)
 281
 282     time_all_elements = time.time()
 283
 284     # get the data
 285     if station['provider'] == 'nrcs':
 286         infoex['wx_data'] = get_nrcs_data(begin_date, end_date, station)
 287     elif station['provider'] == 'mesowest':
 288         infoex['wx_data'] = get_mesowest_data(begin_date, end_date,
 289                                               station)
 290     elif station['provider'] == 'python':
 291         try:
 292             spec = importlib.util.spec_from_file_location('custom_wx',
 293                                                           station['path'])
 294             mod = importlib.util.module_from_spec(spec)
 295             spec.loader.exec_module(mod)
 296             mod.LOG = LOG
 297
 298             try:
 299                 infoex['wx_data'] = mod.get_custom_data()
 300
 301                 if infoex['wx_data'] is None:
 302                     infoex['wx_data'] = []
 303             except Exception as exc:
 304                 LOG.error("Python program for custom Wx data failed in "
 305                           "execution: %s", str(exc))
 306                 sys.exit(1)
 307
 308             LOG.info("Successfully executed external Python program")
 309         except ImportError:
 310             LOG.error("Please upgrade to Python 3.3 or later")
 311             sys.exit(1)
 312         except FileNotFoundError:
 313             LOG.error("Specified Python program for custom Wx data "
 314                       "was not found")
 315             sys.exit(1)
 316         except Exception as exc:
 317             LOG.error("A problem was encountered when attempting to "
 318                       "load your custom Wx program: %s", str(exc))
 319             sys.exit(1)
 320
 321     LOG.info("Time taken to get all data : %.3f sec", time.time() -
 322              time_all_elements)
 323
 324     LOG.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
 325
 326     # timezone massaging
 327     final_end_date = end_date.astimezone(station['tz'])
 328
 329     # Now we only need to add in what we want to change thanks to that
 330     # abomination of a variable declaration earlier
 331     final_data[fmap['Location UUID']] = infoex['location_uuid']
 332     final_data[fmap['obDate']] = final_end_date.strftime('%m/%d/%Y')
 333     final_data[fmap['obTime']] = final_end_date.strftime('%H:%M')
 334     final_data[fmap['timeZone']] = station['tz'].zone
 335
 336     for element_cd in infoex['wx_data']:
 337         if element_cd not in iemap:
 338             LOG.warning("BAD KEY wx_data['%s']", element_cd)
 339             continue
 340
 341         if infoex['wx_data'][element_cd] is None:
 342             continue
 343
 344         # do the conversion before the rounding
 345         if station['provider'] == 'nrcs' and station['units'] == 'metric':
 346             infoex['wx_data'][element_cd] = convert_nrcs_units_to_metric(element_cd, infoex['wx_data'][element_cd])
 347
 348         if station['provider'] != 'python' and station['units'] == 'american':
 349             infoex['wx_data'][element_cd] = convert_units_to_american(element_cd, infoex['wx_data'][element_cd])
 350
 351         # Massage precision of certain values to fit InfoEx's
 352         # expectations
 353         #
 354         # 0 decimal places: relative humidity, wind speed, wind
 355         #                   direction, wind gust, snow depth
 356         # 1 decimal place:  air temp, baro
 357         # Avoid transforming None values
 358         if element_cd in ['wind_speed', 'WSPD', 'wind_direction',
 359                           'RHUM', 'relative_humidity', 'WDIR',
 360                           'wind_gust', 'SNWD', 'snow_depth',
 361                           'hn24']:
 362             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd])
 363         elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']:
 364             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1)
 365         elif element_cd in ['PREC', 'precip_accum']:
 366             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 2)
 367
 368         # CONSIDER: Casting every value to Float() -- need to investigate if
 369         #           any possible elementCds we may want are any other data
 370         #           type than float.
 371         #
 372         #           Another possibility is to query the API with
 373         #           getStationElements and temporarily store the
 374         #           storedUnitCd. But that's pretty network-intensive and
 375         #           may not even be worth it if there's only e.g. one or two
 376         #           exceptions to any otherwise uniformly Float value set.
 377         final_data[fmap[iemap[element_cd]]] = infoex['wx_data'][element_cd]
 378
 379     LOG.debug("final_data: %s", str(final_data))
 380
 381     if infoex['wx_data']:
 382         if not write_local_csv(infoex['csv_filename'], final_data):
 383             LOG.warning('Could not write local CSV file: %s',
 384                         infoex['csv_filename'])
 385             return 1
 386
 387         if not options.dry_run:
 388             try:
 389                 upload_csv(infoex['csv_filename'], infoex)
 390             except socket.timeout:
 391                 LOG.critical("FTP timed out")
 392
 393     LOG.debug('DONE')
 394     return 0
 395
 396 # data structure operations
 397 def setup_infoex_fields_mapping(location_uuid):
 398     """
 399     Create a mapping of InfoEx fields to the local data's indexing scheme.
 400
 401     INFOEX FIELDS
 402
 403     This won't earn style points in Python, but here we establish a couple
 404     of helpful mappings variables. The reason this is helpful is that the
 405     end result is simply an ordered set, the CSV file. But we still may
 406     want to manipulate the values arbitrarily before writing that file.
 407
 408     Also note that the current Auto Wx InfoEx documentation shows these
 409     keys in a graphical table with the "index" beginning at 1, but here we
 410     sanely index beginning at 0.
 411     """
 412     # pylint: disable=too-many-statements,multiple-statements,bad-whitespace
 413     fmap = {}                           ; final_data     = [None] * 29
 414     fmap['Location UUID'] = 0           ; final_data[0]  = location_uuid
 415     fmap['obDate'] = 1                  ; final_data[1]  = None
 416     fmap['obTime'] = 2                  ; final_data[2]  = None
 417     fmap['timeZone'] = 3                ; final_data[3]  = 'Pacific'
 418     fmap['tempMaxHour'] = 4             ; final_data[4]  = None
 419     fmap['tempMaxHourUnit'] = 5         ; final_data[5]  = 'F'
 420     fmap['tempMinHour'] = 6             ; final_data[6]  = None
 421     fmap['tempMinHourUnit'] = 7         ; final_data[7]  = 'F'
 422     fmap['tempPres'] = 8                ; final_data[8]  = None
 423     fmap['tempPresUnit'] = 9            ; final_data[9]  = 'F'
 424     fmap['precipitationGauge'] = 10     ; final_data[10] = None
 425     fmap['precipitationGaugeUnit'] = 11 ; final_data[11] = 'in'
 426     fmap['windSpeedNum'] = 12           ; final_data[12] = None
 427     fmap['windSpeedUnit'] = 13          ; final_data[13] = 'mph'
 428     fmap['windDirectionNum'] = 14       ; final_data[14] = None
 429     fmap['hS'] = 15                     ; final_data[15] = None
 430     fmap['hsUnit'] = 16                 ; final_data[16] = 'in'
 431     fmap['baro'] = 17                   ; final_data[17] = None
 432     fmap['baroUnit'] = 18               ; final_data[18] = 'inHg'
 433     fmap['rH'] = 19                     ; final_data[19] = None
 434     fmap['windGustSpeedNum'] = 20       ; final_data[20] = None
 435     fmap['windGustSpeedNumUnit'] = 21   ; final_data[21] = 'mph'
 436     fmap['windGustDirNum'] = 22         ; final_data[22] = None
 437     fmap['dewPoint'] = 23               ; final_data[23] = None
 438     fmap['dewPointUnit'] = 24           ; final_data[24] = 'F'
 439     fmap['hn24Auto'] = 25               ; final_data[25] = None
 440     fmap['hn24AutoUnit'] = 26           ; final_data[26] = 'in'
 441     fmap['hstAuto'] = 27                ; final_data[27] = None
 442     fmap['hstAutoUnit'] = 28            ; final_data[28] = 'in'
 443
 444     return (fmap, final_data)
 445
 446 def setup_infoex_counterparts_mapping(provider):
 447     """
 448     Create a mapping of the NRCS/MesoWest fields that this program supports to
 449     their InfoEx counterparts
 450     """
 451     iemap = {}
 452
 453     if provider == 'nrcs':
 454         iemap['PREC'] = 'precipitationGauge'
 455         iemap['TOBS'] = 'tempPres'
 456         iemap['TMAX'] = 'tempMaxHour'
 457         iemap['TMIN'] = 'tempMinHour'
 458         iemap['SNWD'] = 'hS'
 459         iemap['PRES'] = 'baro'
 460         iemap['RHUM'] = 'rH'
 461         iemap['WSPD'] = 'windSpeedNum'
 462         iemap['WDIR'] = 'windDirectionNum'
 463         # unsupported by NRCS:
 464         # windGustSpeedNum
 465
 466         # NOTE: this doesn't exist in NRCS SNOTEL, we create it in this
 467         #       program, so add it to the map here
 468         iemap['hn24'] = 'hn24Auto'
 469     elif provider == 'mesowest':
 470         iemap['precip_accum'] = 'precipitationGauge'
 471         iemap['air_temp'] = 'tempPres'
 472         iemap['air_temp_high_24_hour'] = 'tempMaxHour'
 473         iemap['air_temp_low_24_hour'] = 'tempMinHour'
 474         iemap['snow_depth'] = 'hS'
 475         iemap['pressure'] = 'baro'
 476         iemap['relative_humidity'] = 'rH'
 477         iemap['wind_speed'] = 'windSpeedNum'
 478         iemap['wind_direction'] = 'windDirectionNum'
 479         iemap['wind_gust'] = 'windGustSpeedNum'
 480
 481         # NOTE: this doesn't exist in MesoWest, we create it in this
 482         #       program, so add it to the map here
 483         iemap['hn24'] = 'hn24Auto'
 484     elif provider == 'python':
 485         # we expect Python programs to use the InfoEx data type names
 486         iemap['precipitationGauge'] = 'precipitationGauge'
 487         iemap['precipitationGaugeUnit'] = 'precipitationGaugeUnit'
 488         iemap['tempPres'] = 'tempPres'
 489         iemap['tempPresUnit'] = 'tempPresUnit'
 490         iemap['tempMaxHour'] = 'tempMaxHour'
 491         iemap['tempMaxHourUnit'] = 'tempMaxHourUnit'
 492         iemap['tempMinHour'] = 'tempMinHour'
 493         iemap['tempMinHourUnit'] = 'tempMinHourUnit'
 494         iemap['hS'] = 'hS'
 495         iemap['hsUnit'] = 'hsUnit'
 496         iemap['baro'] = 'baro'
 497         iemap['rH'] = 'rH'
 498         iemap['windSpeedNum'] = 'windSpeedNum'
 499         iemap['windSpeedUnit'] = 'windSpeedUnit'
 500         iemap['windDirectionNum'] = 'windDirectionNum'
 501         iemap['windGustSpeedNum'] = 'windGustSpeedNum'
 502         iemap['dewPointUnit'] = 'dewPointUnit'
 503         iemap['hn24AutoUnit'] = 'hn24AutoUnit'
 504         iemap['hstAutoUnit'] = 'hstAutoUnit'
 505
 506     return iemap
 507
 508 # provider-specific operations
 509 def get_nrcs_data(begin, end, station):
 510     """get the data we're after from the NRCS WSDL"""
 511     transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
 512     transport.session.verify = False
 513     client = zeep.Client(wsdl=station['source'], transport=transport)
 514     remote_data = {}
 515
 516     # massage begin/end date format
 517     begin_date_str = begin.strftime('%Y-%m-%d %H:%M:00')
 518     end_date_str = end.strftime('%Y-%m-%d %H:%M:00')
 519
 520     for element_cd in station['desired_data']:
 521         time_element = time.time()
 522
 523         # get the last three hours of data for this elementCd/element_cd
 524         tmp = client.service.getHourlyData(
 525             stationTriplets=[station['station_id']],
 526             elementCd=element_cd,
 527             ordinal=1,
 528             beginDate=begin_date_str,
 529             endDate=end_date_str)
 530
 531         LOG.info("Time to get NRCS elementCd '%s': %.3f sec", element_cd,
 532                  time.time() - time_element)
 533
 534         values = tmp[0]['values']
 535
 536         # sort and isolate the most recent
 537         #
 538         # NOTE: we do this because sometimes there are gaps in hourly data
 539         #       in NRCS; yes, we may end up with slightly inaccurate data,
 540         #       so perhaps this decision will be re-evaluated in the future
 541         if values:
 542             ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
 543             remote_data[element_cd] = ordered[0]['value']
 544         else:
 545             remote_data[element_cd] = None
 546
 547
 548         # calc hn24, if applicable
 549         hn24 = None
 550
 551         if station['hn24']:
 552             hn24_values = []
 553
 554             if element_cd == "SNWD":
 555                 for idx, _ in enumerate(values):
 556                     val = values[idx]
 557                     if val is None:
 558                         continue
 559                     hn24_values.append(val['value'])
 560
 561                 if len(hn24_values) > 0:
 562                     # instead of taking MAX - MIN, we want the first
 563                     # value (most distant) - the last value (most
 564                     # recent)
 565                     #
 566                     # if the result is positive, then we have
 567                     # settlement; if it's not, then we have HN24
 568                     hn24 = hn24_values[0] - hn24_values[len(hn24_values)-1]
 569
 570                     if hn24 < 0.0:
 571                         hn24 = abs(hn24)
 572                     else:
 573                         # this case represents HS settlement
 574                         hn24 = 0.0
 575
 576             # finally, if user wants hn24 and it's set to None at this
 577             # point, then force it to 0.0
 578             if hn24 is None:
 579                 hn24 = 0.0
 580
 581         if hn24 is not None:
 582             if station['hn24']:
 583                 remote_data['hn24'] = hn24
 584
 585     return remote_data
 586
 587 def get_mesowest_data(begin, end, station):
 588     """get the data we're after from the MesoWest/Synoptic API"""
 589     remote_data = {}
 590
 591     # massage begin/end date format
 592     begin_date_str = begin.strftime('%Y%m%d%H%M')
 593     end_date_str = end.strftime('%Y%m%d%H%M')
 594
 595     # construct final, completed API URL
 596     api_req_url = station['source'] + '&start=' + begin_date_str + '&end=' + end_date_str
 597
 598     try:
 599         req = requests.get(api_req_url)
 600     except requests.exceptions.ConnectionError:
 601         LOG.error("Could not connect to '%s'", api_req_url)
 602         sys.exit(1)
 603
 604     try:
 605         json = req.json()
 606     except ValueError:
 607         LOG.error("Bad JSON in MesoWest response")
 608         sys.exit(1)
 609
 610     try:
 611         observations = json['STATION'][0]['OBSERVATIONS']
 612     except KeyError as exc:
 613         LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
 614         sys.exit(1)
 615     except IndexError as exc:
 616         LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
 617         try:
 618             LOG.error("Detailed MesoWest response: '%s'",
 619                       json['SUMMARY']['RESPONSE_MESSAGE'])
 620         except KeyError:
 621             pass
 622         sys.exit(1)
 623     except ValueError as exc:
 624         LOG.error("Bad JSON in MesoWest response: '%s'", exc)
 625         sys.exit(1)
 626
 627     # pos represents the last item in the array, aka the most recent
 628     pos = len(observations['date_time']) - 1
 629
 630     # while these values only apply in certain cases, init them here
 631     wind_speed_values = []
 632     wind_gust_speed_values = []
 633     wind_direction_values = []
 634     hn24_values = []
 635
 636     # results
 637     wind_speed_avg = None
 638     wind_gust_speed_avg = None
 639     wind_direction_avg = None
 640     hn24 = None
 641
 642     for element_cd in station['desired_data'].split(','):
 643         # sort and isolate the most recent, see note above in NRCS for how and
 644         # why this is done
 645         #
 646         # NOTE: Unlike in the NRCS case, the MesoWest API response contains all
 647         #       data (whereas with NRCS, we have to make a separate request for
 648         #       each element we want). This is nice for network efficiency but
 649         #       it means we have to handle this part differently for each.
 650         #
 651         # NOTE: Also unlike NRCS, MesoWest provides more granular data; NRCS
 652         #       provides hourly data, but MesoWest can often provide data every
 653         #       10 minutes -- though this provides more opportunity for
 654         #       irregularities
 655
 656         # we may not have the data at all
 657         key_name = element_cd + '_set_1'
 658
 659         if key_name in observations:
 660             # val is what will make it into the dataset, after
 661             # conversions... it gets defined here because in certain
 662             # cases we need to look at all of the data to calculate HN24
 663             # or wind averages, but for the rest of the data, we only
 664             # take the most recent
 665             val = None
 666
 667             # loop through all observations for this key_name
 668             # record relevant values for wind averaging or hn24, but
 669             # otherwise only persist the data if it's the last datum in
 670             # the set
 671             for idx, _ in enumerate(observations[key_name]):
 672                 val = observations[key_name][idx]
 673
 674                 # skip bunk vals
 675                 if val is None:
 676                     continue
 677
 678                 # mesowest by default provides wind_speed in m/s, but
 679                 # we specify 'english' units in the request; either way,
 680                 # we want mph
 681                 if element_cd in ('wind_speed', 'wind_gust'):
 682                     val = kn_to_mph(val)
 683
 684                 # mesowest provides HS in mm, not cm; we want cm
 685                 if element_cd == 'snow_depth' and station['units'] == 'metric':
 686                     val = mm_to_cm(val)
 687
 688                 # HN24 / wind_mode transformations, once the data has
 689                 # completed unit conversions
 690                 if station['wind_mode'] == "average":
 691                     if element_cd == 'wind_speed' and val is not None:
 692                         wind_speed_values.append(val)
 693                     elif element_cd == 'wind_gust' and val is not None:
 694                         wind_gust_speed_values.append(val)
 695                     elif element_cd == 'wind_direction' and val is not None:
 696                         wind_direction_values.append(val)
 697
 698                 if element_cd == 'snow_depth':
 699                     hn24_values.append(val)
 700
 701                 # again, only persist this datum to the final data if
 702                 # it's from the most recent date
 703                 if idx == pos:
 704                     remote_data[element_cd] = val
 705
 706             # ensure that the data is filled out
 707             if not observations[key_name][pos]:
 708                 remote_data[element_cd] = None
 709         else:
 710             remote_data[element_cd] = None
 711
 712     if len(hn24_values) > 0:
 713         # instead of taking MAX - MIN, we want the first value (most
 714         # distant) - the last value (most recent)
 715         #
 716         # if the result is positive, then we have settlement; if it's not,
 717         # then we have HN24
 718         hn24 = hn24_values[0] - hn24_values[len(hn24_values)-1]
 719
 720         if hn24 < 0.0:
 721             hn24 = abs(hn24)
 722         else:
 723             # this case represents HS settlement
 724             hn24 = 0.0
 725
 726
 727     # finally, if user wants hn24 and it's set to None at this
 728     # point, then force it to 0.0
 729     if station['hn24'] and hn24 is None:
 730         hn24 = 0.0
 731
 732     if len(wind_speed_values) > 0:
 733         wind_speed_avg = sum(wind_speed_values) / len(wind_speed_values)
 734
 735     if len(wind_gust_speed_values) > 0:
 736         wind_gust_speed_avg = sum(wind_gust_speed_values) / len(wind_gust_speed_values)
 737
 738     if len(wind_direction_values) > 0:
 739         wind_direction_avg = sum(wind_direction_values) / len(wind_direction_values)
 740
 741     if hn24 is not None:
 742         if station['hn24']:
 743             remote_data['hn24'] = hn24
 744
 745     # overwrite the following with the respective averages, if
 746     # applicable
 747     if wind_speed_avg is not None:
 748         remote_data['wind_speed'] = wind_speed_avg
 749
 750     if wind_gust_speed_avg is not None:
 751         remote_data['wind_gust'] = wind_gust_speed_avg
 752
 753     if wind_direction_avg is not None:
 754         remote_data['wind_direction'] = wind_direction_avg
 755
 756     return remote_data
 757
 758 def switch_units_to_metric(data_map, mapping):
 759     """replace units with metric counterparts"""
 760
 761     # NOTE: to update this, use the fmap<->final_data mapping laid out
 762     #       in setup_infoex_fields_mapping ()
 763     data_map[mapping['tempMaxHourUnit']] = 'C'
 764     data_map[mapping['tempMinHourUnit']] = 'C'
 765     data_map[mapping['tempPresUnit']] = 'C'
 766     data_map[mapping['precipitationGaugeUnit']] = 'mm'
 767     data_map[mapping['hsUnit']] = 'cm'
 768     data_map[mapping['windSpeedUnit']] = 'm/s'
 769     data_map[mapping['windGustSpeedNumUnit']] = 'm/s'
 770     data_map[mapping['dewPointUnit']] = 'C'
 771     data_map[mapping['hn24AutoUnit']] = 'cm'
 772     data_map[mapping['hstAutoUnit']] = 'cm'
 773
 774     return data_map
 775
 776 def switch_units_to_american(data_map, mapping):
 777     """
 778     replace units with the American mixture of metric and imperial
 779
 780     Precip values = metric
 781     Wind values = imperial
 782     """
 783
 784     # precip values
 785     data_map[mapping['tempMaxHourUnit']] = 'C'
 786     data_map[mapping['tempMinHourUnit']] = 'C'
 787     data_map[mapping['tempPresUnit']] = 'C'
 788     data_map[mapping['dewPointUnit']] = 'C'
 789
 790     data_map[mapping['precipitationGaugeUnit']] = 'cm'
 791     data_map[mapping['hsUnit']] = 'cm'
 792     data_map[mapping['hn24AutoUnit']] = 'cm'
 793     data_map[mapping['hstAutoUnit']] = 'cm'
 794
 795     data_map[mapping['baroUnit']] = 'inHg'
 796
 797     # wind values
 798     data_map[mapping['windSpeedUnit']] = 'mph'
 799     data_map[mapping['windGustSpeedNumUnit']] = 'mph'
 800
 801     return data_map
 802
 803 def convert_nrcs_units_to_metric(element_cd, value):
 804     """convert NRCS values from English to metric"""
 805     if element_cd == 'TOBS':
 806         value = f_to_c(value)
 807     elif element_cd == 'SNWD':
 808         value = in_to_cm(value)
 809     elif element_cd == 'PREC':
 810         value = in_to_mm(value)
 811     return value
 812
 813 def convert_units_to_american(element_cd, value):
 814     """
 815     convert value to 'American' units
 816
 817     The original unit is always metric.
 818
 819     Precip values = metric
 820     Wind values = imperial
 821     """
 822
 823     # no need to convert precip values, as they will arrive in metric
 824     # units in "American" units mode
 825     # # temp values
 826     # if element_cd in ['TMAX', 'TMIN', 'TOBS', 'air_temp', 'air_temp_high_24_hour', 'air_temp_low_24_hour']:
 827     #     value = c_to_f(value)
 828
 829     # mesowest provides HS in mm, not cm; we want cm
 830     if element_cd == 'snow_depth':
 831         value = mm_to_cm(value)
 832
 833     # baro values also arrive in metric, so convert to imperial
 834     if element_cd in ['PRES', 'pressure']:
 835         value = inhg_to_pascal(value)
 836
 837     if element_cd in ['WSPD', 'wind_speed', 'wind_gust']:
 838         value = ms_to_mph(value)
 839
 840     return value
 841
 842 # CSV operations
 843 def write_local_csv(path_to_file, data):
 844     """Write the specified CSV file to disk"""
 845     with open(path_to_file, 'w') as file_object:
 846         # The requirement is that empty values are represented in the CSV
 847         # file as "", csv.QUOTE_NONNUMERIC achieves that
 848         LOG.debug("writing CSV file '%s'", path_to_file)
 849         writer = csv.writer(file_object, quoting=csv.QUOTE_NONNUMERIC)
 850         writer.writerow(data)
 851         file_object.close()
 852     return True
 853
 854 def upload_csv(path_to_file, infoex_data):
 855     """Upload the specified CSV file to InfoEx FTP and remove the file"""
 856     with open(path_to_file, 'rb') as file_object:
 857         LOG.debug("uploading FTP file '%s'", infoex_data['host'])
 858         ftp = FTP(infoex_data['host'], infoex_data['uuid'],
 859                   infoex_data['api_key'], timeout=10)
 860         ftp.storlines('STOR ' + path_to_file, file_object)
 861         ftp.close()
 862         file_object.close()
 863     os.remove(path_to_file)
 864
 865 # other miscellaneous routines
 866 def setup_time_values(station):
 867     """establish time bounds of data request(s)"""
 868
 869     # default timezone to UTC (for MesoWest)
 870     tz = pytz.utc
 871
 872     # but for NRCS, use the config-specified timezone
 873     if station['provider'] == 'nrcs':
 874         tz = station['tz']
 875
 876     # floor time to nearest hour
 877     date_time = datetime.datetime.now(tz=tz)
 878     end_date = date_time - datetime.timedelta(minutes=date_time.minute % 60,
 879                                               seconds=date_time.second,
 880                                               microseconds=date_time.microsecond)
 881     begin_date = end_date - datetime.timedelta(hours=station['num_hrs_to_fetch'])
 882     return (begin_date, end_date)
 883
 884 def f_to_c(f):
 885     """convert Fahrenheit to Celsius"""
 886     return (float(f) - 32) * 5.0/9.0
 887
 888 def c_to_f(c):
 889     """convert Celsius to Fahrenheit"""
 890     return (float(c) * 1.8) + 32
 891
 892 def in_to_cm(inches):
 893     """convert inches to centimeters"""
 894     return float(inches) * 2.54
 895
 896 def cm_to_in(cms):
 897     """convert centimeters to inches"""
 898     return float(cms) / 2.54
 899
 900 def pascal_to_inhg(pa):
 901     """convert pascals to inches of mercury"""
 902     return float(pa) * 0.00029530
 903
 904 def inhg_to_pascal(inhg):
 905     """convert inches of mercury to pascals"""
 906     return float(inhg) / 0.00029530
 907
 908 def in_to_mm(inches):
 909     """convert inches to millimeters"""
 910     return (float(inches) * 2.54) * 10.0
 911
 912 def ms_to_mph(ms):
 913     """convert meters per second to miles per hour"""
 914     return ms * 2.236936
 915
 916 def kn_to_mph(kn):
 917     """convert knots to miles per hour"""
 918     return kn * 1.150779
 919
 920 def mm_to_cm(mm):
 921     """convert millimeters to centimeters"""
 922     return mm / 10.0
 923
 924 if __name__ == "__main__":
 925     sys.exit(main())