infoex-autowx.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 """
   5 InfoEx <-> NRCS/MesoWest Auto Wx implementation
   6 Alexander Vasarab
   7 Wylark Mountaineering LLC
   8
   9 This program fetches data from either an NRCS SNOTEL site or MesoWest
  10 weather station and pushes it to InfoEx using the new automated weather
  11 system implementation.
  12
  13 It is designed to be run hourly, and it asks for the last three hours
  14 of data of each desired type, and selects the most recent one. This
  15 lends some resiliency to the process and helps ensure that we have a
  16 value to send, but it can lead to somewhat inconsistent/untruthful
  17 data if e.g. the HS is from the last hour but the tempPres is from two
  18 hours ago because the instrumentation had a hiccup. It's worth
  19 considering if this is a bug or a feature.
  20
  21 For more information, see file: README
  22 For licensing, see file: LICENSE
  23 """
  24
  25 import configparser
  26 import csv
  27 import datetime
  28 import logging
  29 import os
  30 import sys
  31 import time
  32 import urllib3
  33 import importlib.util
  34
  35 from ftplib import FTP
  36 from argparse import ArgumentParser
  37
  38 import pytz
  39
  40 import requests
  41
  42 import zeep
  43 import zeep.cache
  44 import zeep.transports
  45
  46 __version__ = '3.4.1'
  47
  48 LOG = logging.getLogger(__name__)
  49 LOG.setLevel(logging.NOTSET)
  50
  51 urllib3.disable_warnings()
  52
  53 def get_parser():
  54     """Return OptionParser for this program"""
  55     parser = ArgumentParser()
  56
  57     parser.add_argument("--version",
  58                         action="version",
  59                         version=__version__)
  60
  61     parser.add_argument("--config",
  62                         dest="config",
  63                         metavar="FILE",
  64                         help="location of config file")
  65
  66     parser.add_argument("--log-level",
  67                         dest="log_level",
  68                         default=None,
  69                         help="set the log level (debug, info, warning)")
  70
  71     parser.add_argument("--dry-run",
  72                         action="store_true",
  73                         dest="dry_run",
  74                         default=False,
  75                         help="fetch data but don't upload to InfoEx")
  76
  77     return parser
  78
  79 def setup_config(config):
  80     """Setup config variable based on values specified in the ini file"""
  81     try:
  82         infoex = {
  83             'host': config['infoex']['host'],
  84             'uuid': config['infoex']['uuid'],
  85             'api_key': config['infoex']['api_key'],
  86             'csv_filename': config['infoex']['csv_filename'],
  87             'location_uuid': config['infoex']['location_uuid'],
  88             'wx_data': {}, # placeholder key, values to come later
  89         }
  90
  91         station = dict()
  92
  93         station['provider'] = config['station']['type']
  94
  95         if station['provider'] not in ['nrcs', 'mesowest', 'python']:
  96             print("Please specify either nrcs or mesowest as the station type.")
  97             sys.exit(1)
  98
  99         # massage units config items first
 100         #
 101         # NOTE: custom providers don't require units to be specified
 102         #       because they can do whatever they please with the units
 103         #       within their own program
 104         if station['provider'] != "custom":
 105             station['units'] = config['station']['units']
 106
 107             if station['units'] not in ['metric', 'english', 'american']:
 108                 print("Please specify metric, english, or american for the units.")
 109                 sys.exit(1)
 110             else:
 111                 # if units are specified as "American" then we simply
 112                 # default to metric for the requests
 113                 if station['units'] == 'american':
 114                     station['units_requested'] = 'metric'
 115                 else:
 116                     station['units_requested'] = station['units']
 117
 118         # massage provider config items
 119         if station['provider'] == 'nrcs':
 120             #station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
 121             station['source'] = 'https://wcc.sc.egov.usda.gov/awdbWebService/services?WSDL'
 122
 123             station['station_id'] = config['station']['station_id']
 124             station['desired_data'] = config['station']['desired_data'].split(',')
 125
 126         if station['provider'] == 'mesowest':
 127             station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
 128             station['station_id'] = config['station']['station_id']
 129             station['desired_data'] = config['station']['desired_data']
 130
 131             # construct full API URL (sans start/end time, added later)
 132             station['source'] = station['source'] + '?token=' + \
 133                                 config['station']['token'] + \
 134                                 '&within=60&units=' + station['units_requested'] + \
 135                                 '&stid=' + station['station_id'] + \
 136                                 '&vars=' + station['desired_data']
 137
 138         if station['provider'] == 'python':
 139             station['path'] = config['station']['path']
 140
 141         tz = 'America/Los_Angeles'
 142
 143         if 'tz' in config['station']:
 144             tz = config['station']['tz']
 145
 146         try:
 147             station['tz'] = pytz.timezone(tz)
 148         except pytz.exceptions.UnknownTimeZoneError:
 149             LOG.critical("%s is not a valid timezone", tz)
 150             sys.exit(1)
 151
 152         # By default, fetch three hours of data
 153         #
 154         # If user wants hn24 or wind averaging, then
 155         # we need more.
 156         station['num_hrs_to_fetch'] = 3
 157
 158         # HN24
 159         if 'hn24' in config['station']:
 160             if config['station']['hn24'] not in ['true', 'false']:
 161                 raise ValueError("hn24 must be either 'true' or 'false'")
 162
 163             if config['station']['hn24'] == "true":
 164                 station['hn24'] = True
 165                 station['num_hrs_to_fetch'] = 24
 166             else:
 167                 station['hn24'] = False
 168         else:
 169             # default to False
 170             station['hn24'] = False
 171
 172         # Wind mode
 173         if 'wind_mode' in config['station']:
 174             if config['station']['wind_mode'] not in ['normal', 'average']:
 175                 raise ValueError("wind_mode must be either 'normal' or 'average'")
 176
 177             station['wind_mode'] = config['station']['wind_mode']
 178
 179             if station['wind_mode'] == "average":
 180                 station['num_hrs_to_fetch'] = 24
 181         else:
 182             # default to False
 183             station['wind_mode'] = "normal"
 184
 185     except KeyError as err:
 186         LOG.critical("%s not defined in configuration file", err)
 187         sys.exit(1)
 188     except ValueError as err:
 189         LOG.critical("%s", err)
 190         sys.exit(1)
 191
 192     # all sections/values present in config file, final sanity check
 193     try:
 194         for key in config.sections():
 195             for subkey in config[key]:
 196                 if not config[key][subkey]:
 197                     raise ValueError
 198     except ValueError:
 199         LOG.critical("Config value '%s.%s' is empty", key, subkey)
 200         sys.exit(1)
 201
 202     return (infoex, station)
 203
 204 def setup_logging(log_level):
 205     """Setup our logging infrastructure"""
 206     try:
 207         from systemd.journal import JournalHandler
 208         LOG.addHandler(JournalHandler())
 209     except ImportError:
 210         ## fallback to syslog
 211         #import logging.handlers
 212         #LOG.addHandler(logging.handlers.SysLogHandler())
 213         # fallback to stdout
 214         handler = logging.StreamHandler(sys.stdout)
 215         formatter = logging.Formatter('%(asctime)s.%(msecs)03d '
 216                                       '%(levelname)s %(module)s - '
 217                                       '%(funcName)s: %(message)s',
 218                                       '%Y-%m-%d %H:%M:%S')
 219         handler.setFormatter(formatter)
 220         LOG.addHandler(handler)
 221
 222     # ugly, but passable
 223     if log_level in [None, 'debug', 'info', 'warning']:
 224         if log_level == 'debug':
 225             LOG.setLevel(logging.DEBUG)
 226         elif log_level == 'info':
 227             LOG.setLevel(logging.INFO)
 228         elif log_level == 'warning':
 229             LOG.setLevel(logging.WARNING)
 230         else:
 231             LOG.setLevel(logging.NOTSET)
 232     else:
 233         return False
 234
 235     return True
 236
 237 def main():
 238     """Main routine: sort through args, decide what to do, then do it"""
 239     parser = get_parser()
 240     options = parser.parse_args()
 241
 242     config = configparser.ConfigParser(allow_no_value=False)
 243
 244     if not options.config:
 245         parser.print_help()
 246         print("\nPlease specify a configuration file via --config.")
 247         sys.exit(1)
 248
 249     config.read(options.config)
 250
 251     if not setup_logging(options.log_level):
 252         parser.print_help()
 253         print("\nPlease select an appropriate log level or remove the switch (--log-level).")
 254         sys.exit(1)
 255
 256     (infoex, station) = setup_config(config)
 257
 258     LOG.debug('Config parsed, starting up')
 259
 260     # create mappings
 261     (fmap, final_data) = setup_infoex_fields_mapping(infoex['location_uuid'])
 262     iemap = setup_infoex_counterparts_mapping(station['provider'])
 263
 264     # override units if user selected metric
 265     if station['provider'] != 'python' and station['units'] == 'metric':
 266         final_data = switch_units_to_metric(final_data, fmap)
 267
 268     # likewise for "American" units
 269     if station['provider'] != 'python' and station['units'] == 'american':
 270         final_data = switch_units_to_american(final_data, fmap)
 271
 272     (begin_date, end_date) = setup_time_values(station)
 273
 274     if station['provider'] == 'python':
 275         LOG.debug("Getting custom data from external Python program")
 276     else:
 277         LOG.debug("Getting %s data from %s to %s (%s)",
 278                   str(station['desired_data']),
 279                   str(begin_date), str(end_date), end_date.tzinfo.zone)
 280
 281     time_all_elements = time.time()
 282
 283     # get the data
 284     if station['provider'] == 'nrcs':
 285         infoex['wx_data'] = get_nrcs_data(begin_date, end_date, station)
 286     elif station['provider'] == 'mesowest':
 287         infoex['wx_data'] = get_mesowest_data(begin_date, end_date,
 288                                               station)
 289     elif station['provider'] == 'python':
 290         try:
 291             spec = importlib.util.spec_from_file_location('custom_wx',
 292                                                           station['path'])
 293             mod = importlib.util.module_from_spec(spec)
 294             spec.loader.exec_module(mod)
 295             mod.LOG = LOG
 296
 297             try:
 298                 infoex['wx_data'] = mod.get_custom_data()
 299
 300                 if infoex['wx_data'] is None:
 301                     infoex['wx_data'] = []
 302             except Exception as exc:
 303                 LOG.error("Python program for custom Wx data failed in "
 304                           "execution: %s", str(exc))
 305                 sys.exit(1)
 306
 307             LOG.info("Successfully executed external Python program")
 308         except ImportError:
 309             LOG.error("Please upgrade to Python 3.3 or later")
 310             sys.exit(1)
 311         except FileNotFoundError:
 312             LOG.error("Specified Python program for custom Wx data "
 313                       "was not found")
 314             sys.exit(1)
 315         except Exception as exc:
 316             LOG.error("A problem was encountered when attempting to "
 317                       "load your custom Wx program: %s", str(exc))
 318             sys.exit(1)
 319
 320     LOG.info("Time taken to get all data : %.3f sec", time.time() -
 321              time_all_elements)
 322
 323     LOG.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
 324
 325     # timezone massaging
 326     final_end_date = end_date.astimezone(station['tz'])
 327
 328     # Now we only need to add in what we want to change thanks to that
 329     # abomination of a variable declaration earlier
 330     final_data[fmap['Location UUID']] = infoex['location_uuid']
 331     final_data[fmap['obDate']] = final_end_date.strftime('%m/%d/%Y')
 332     final_data[fmap['obTime']] = final_end_date.strftime('%H:%M')
 333     final_data[fmap['timeZone']] = station['tz'].zone
 334
 335     for element_cd in infoex['wx_data']:
 336         if element_cd not in iemap:
 337             LOG.warning("BAD KEY wx_data['%s']", element_cd)
 338             continue
 339
 340         if infoex['wx_data'][element_cd] is None:
 341             continue
 342
 343         # do the conversion before the rounding
 344         if station['provider'] == 'nrcs' and station['units'] == 'metric':
 345             infoex['wx_data'][element_cd] = convert_nrcs_units_to_metric(element_cd, infoex['wx_data'][element_cd])
 346
 347         if station['provider'] != 'custom' and station['units'] == 'american':
 348             infoex['wx_data'][element_cd] = convert_units_to_american(element_cd, infoex['wx_data'][element_cd])
 349
 350         # Massage precision of certain values to fit InfoEx's
 351         # expectations
 352         #
 353         # 0 decimal places: relative humidity, wind speed, wind
 354         #                   direction, wind gust, snow depth
 355         # 1 decimal place:  air temp, baro
 356         # Avoid transforming None values
 357         if element_cd in ['wind_speed', 'WSPD', 'wind_direction',
 358                           'RHUM', 'relative_humidity', 'WDIR',
 359                           'wind_gust', 'SNWD', 'snow_depth',
 360                           'hn24']:
 361             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd])
 362         elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']:
 363             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1)
 364         elif element_cd in ['PREC', 'precip_accum']:
 365             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 2)
 366
 367         # CONSIDER: Casting every value to Float() -- need to investigate if
 368         #           any possible elementCds we may want are any other data
 369         #           type than float.
 370         #
 371         #           Another possibility is to query the API with
 372         #           getStationElements and temporarily store the
 373         #           storedUnitCd. But that's pretty network-intensive and
 374         #           may not even be worth it if there's only e.g. one or two
 375         #           exceptions to any otherwise uniformly Float value set.
 376         final_data[fmap[iemap[element_cd]]] = infoex['wx_data'][element_cd]
 377
 378     LOG.debug("final_data: %s", str(final_data))
 379
 380     if infoex['wx_data']:
 381         if not write_local_csv(infoex['csv_filename'], final_data):
 382             LOG.warning('Could not write local CSV file: %s',
 383                         infoex['csv_filename'])
 384             return 1
 385
 386         if not options.dry_run:
 387             upload_csv(infoex['csv_filename'], infoex)
 388
 389     LOG.debug('DONE')
 390     return 0
 391
 392 # data structure operations
 393 def setup_infoex_fields_mapping(location_uuid):
 394     """
 395     Create a mapping of InfoEx fields to the local data's indexing scheme.
 396
 397     INFOEX FIELDS
 398
 399     This won't earn style points in Python, but here we establish a couple
 400     of helpful mappings variables. The reason this is helpful is that the
 401     end result is simply an ordered set, the CSV file. But we still may
 402     want to manipulate the values arbitrarily before writing that file.
 403
 404     Also note that the current Auto Wx InfoEx documentation shows these
 405     keys in a graphical table with the "index" beginning at 1, but here we
 406     sanely index beginning at 0.
 407     """
 408     # pylint: disable=too-many-statements,multiple-statements,bad-whitespace
 409     fmap = {}                           ; final_data     = [None] * 29
 410     fmap['Location UUID'] = 0           ; final_data[0]  = location_uuid
 411     fmap['obDate'] = 1                  ; final_data[1]  = None
 412     fmap['obTime'] = 2                  ; final_data[2]  = None
 413     fmap['timeZone'] = 3                ; final_data[3]  = 'Pacific'
 414     fmap['tempMaxHour'] = 4             ; final_data[4]  = None
 415     fmap['tempMaxHourUnit'] = 5         ; final_data[5]  = 'F'
 416     fmap['tempMinHour'] = 6             ; final_data[6]  = None
 417     fmap['tempMinHourUnit'] = 7         ; final_data[7]  = 'F'
 418     fmap['tempPres'] = 8                ; final_data[8]  = None
 419     fmap['tempPresUnit'] = 9            ; final_data[9]  = 'F'
 420     fmap['precipitationGauge'] = 10     ; final_data[10] = None
 421     fmap['precipitationGaugeUnit'] = 11 ; final_data[11] = 'in'
 422     fmap['windSpeedNum'] = 12           ; final_data[12] = None
 423     fmap['windSpeedUnit'] = 13          ; final_data[13] = 'mph'
 424     fmap['windDirectionNum'] = 14       ; final_data[14] = None
 425     fmap['hS'] = 15                     ; final_data[15] = None
 426     fmap['hsUnit'] = 16                 ; final_data[16] = 'in'
 427     fmap['baro'] = 17                   ; final_data[17] = None
 428     fmap['baroUnit'] = 18               ; final_data[18] = 'inHg'
 429     fmap['rH'] = 19                     ; final_data[19] = None
 430     fmap['windGustSpeedNum'] = 20       ; final_data[20] = None
 431     fmap['windGustSpeedNumUnit'] = 21   ; final_data[21] = 'mph'
 432     fmap['windGustDirNum'] = 22         ; final_data[22] = None
 433     fmap['dewPoint'] = 23               ; final_data[23] = None
 434     fmap['dewPointUnit'] = 24           ; final_data[24] = 'F'
 435     fmap['hn24Auto'] = 25               ; final_data[25] = None
 436     fmap['hn24AutoUnit'] = 26           ; final_data[26] = 'in'
 437     fmap['hstAuto'] = 27                ; final_data[27] = None
 438     fmap['hstAutoUnit'] = 28            ; final_data[28] = 'in'
 439
 440     return (fmap, final_data)
 441
 442 def setup_infoex_counterparts_mapping(provider):
 443     """
 444     Create a mapping of the NRCS/MesoWest fields that this program supports to
 445     their InfoEx counterparts
 446     """
 447     iemap = {}
 448
 449     if provider == 'nrcs':
 450         iemap['PREC'] = 'precipitationGauge'
 451         iemap['TOBS'] = 'tempPres'
 452         iemap['TMAX'] = 'tempMaxHour'
 453         iemap['TMIN'] = 'tempMinHour'
 454         iemap['SNWD'] = 'hS'
 455         iemap['PRES'] = 'baro'
 456         iemap['RHUM'] = 'rH'
 457         iemap['WSPD'] = 'windSpeedNum'
 458         iemap['WDIR'] = 'windDirectionNum'
 459         # unsupported by NRCS:
 460         # windGustSpeedNum
 461
 462         # NOTE: this doesn't exist in NRCS SNOTEL, we create it in this
 463         #       program, so add it to the map here
 464         iemap['hn24'] = 'hn24Auto'
 465     elif provider == 'mesowest':
 466         iemap['precip_accum'] = 'precipitationGauge'
 467         iemap['air_temp'] = 'tempPres'
 468         iemap['air_temp_high_24_hour'] = 'tempMaxHour'
 469         iemap['air_temp_low_24_hour'] = 'tempMinHour'
 470         iemap['snow_depth'] = 'hS'
 471         iemap['pressure'] = 'baro'
 472         iemap['relative_humidity'] = 'rH'
 473         iemap['wind_speed'] = 'windSpeedNum'
 474         iemap['wind_direction'] = 'windDirectionNum'
 475         iemap['wind_gust'] = 'windGustSpeedNum'
 476
 477         # NOTE: this doesn't exist in MesoWest, we create it in this
 478         #       program, so add it to the map here
 479         iemap['hn24'] = 'hn24Auto'
 480     elif provider == 'python':
 481         # we expect Python programs to use the InfoEx data type names
 482         iemap['precipitationGauge'] = 'precipitationGauge'
 483         iemap['tempPres'] = 'tempPres'
 484         iemap['tempMaxHour'] = 'tempMaxHour'
 485         iemap['tempMinHour'] = 'tempMinHour'
 486         iemap['hS'] = 'hS'
 487         iemap['baro'] = 'baro'
 488         iemap['rH'] = 'rH'
 489         iemap['windSpeedNum'] = 'windSpeedNum'
 490         iemap['windDirectionNum'] = 'windDirectionNum'
 491         iemap['windGustSpeedNum'] = 'windGustSpeedNum'
 492
 493     return iemap
 494
 495 # provider-specific operations
 496 def get_nrcs_data(begin, end, station):
 497     """get the data we're after from the NRCS WSDL"""
 498     transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
 499     transport.session.verify = False
 500     client = zeep.Client(wsdl=station['source'], transport=transport)
 501     remote_data = {}
 502
 503     # massage begin/end date format
 504     begin_date_str = begin.strftime('%Y-%m-%d %H:%M:00')
 505     end_date_str = end.strftime('%Y-%m-%d %H:%M:00')
 506
 507     for element_cd in station['desired_data']:
 508         time_element = time.time()
 509
 510         # get the last three hours of data for this elementCd/element_cd
 511         tmp = client.service.getHourlyData(
 512             stationTriplets=[station['station_id']],
 513             elementCd=element_cd,
 514             ordinal=1,
 515             beginDate=begin_date_str,
 516             endDate=end_date_str)
 517
 518         LOG.info("Time to get NRCS elementCd '%s': %.3f sec", element_cd,
 519                  time.time() - time_element)
 520
 521         values = tmp[0]['values']
 522
 523         # sort and isolate the most recent
 524         #
 525         # NOTE: we do this because sometimes there are gaps in hourly data
 526         #       in NRCS; yes, we may end up with slightly inaccurate data,
 527         #       so perhaps this decision will be re-evaluated in the future
 528         if values:
 529             ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
 530             remote_data[element_cd] = ordered[0]['value']
 531         else:
 532             remote_data[element_cd] = None
 533
 534
 535         # calc hn24, if applicable
 536         hn24 = None
 537
 538         if station['hn24']:
 539             hn24_values = []
 540
 541             if element_cd == "SNWD":
 542                 for idx, _ in enumerate(values):
 543                     val = values[idx]
 544                     if val is None:
 545                         continue
 546                     hn24_values.append(val['value'])
 547
 548                 if len(hn24_values) > 0:
 549                     # instead of taking MAX - MIN, we want the first
 550                     # value (most distant) - the last value (most
 551                     # recent)
 552                     #
 553                     # if the result is positive, then we have
 554                     # settlement; if it's not, then we have HN24
 555                     hn24 = hn24_values[0] - hn24_values[len(hn24_values)-1]
 556
 557                     if hn24 < 0.0:
 558                         hn24 = abs(hn24)
 559                     else:
 560                         # this case represents HS settlement
 561                         hn24 = 0.0
 562
 563             # finally, if user wants hn24 and it's set to None at this
 564             # point, then force it to 0.0
 565             if hn24 is None:
 566                 hn24 = 0.0
 567
 568         if hn24 is not None:
 569             if station['hn24']:
 570                 remote_data['hn24'] = hn24
 571
 572     return remote_data
 573
 574 def get_mesowest_data(begin, end, station):
 575     """get the data we're after from the MesoWest/Synoptic API"""
 576     remote_data = {}
 577
 578     # massage begin/end date format
 579     begin_date_str = begin.strftime('%Y%m%d%H%M')
 580     end_date_str = end.strftime('%Y%m%d%H%M')
 581
 582     # construct final, completed API URL
 583     api_req_url = station['source'] + '&start=' + begin_date_str + '&end=' + end_date_str
 584
 585     try:
 586         req = requests.get(api_req_url)
 587     except requests.exceptions.ConnectionError:
 588         LOG.error("Could not connect to '%s'", api_req_url)
 589         sys.exit(1)
 590
 591     try:
 592         json = req.json()
 593     except ValueError:
 594         LOG.error("Bad JSON in MesoWest response")
 595         sys.exit(1)
 596
 597     try:
 598         observations = json['STATION'][0]['OBSERVATIONS']
 599     except KeyError as exc:
 600         LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
 601         sys.exit(1)
 602     except IndexError as exc:
 603         LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
 604         try:
 605             LOG.error("Detailed MesoWest response: '%s'",
 606                       json['SUMMARY']['RESPONSE_MESSAGE'])
 607         except KeyError:
 608             pass
 609         sys.exit(1)
 610     except ValueError as exc:
 611         LOG.error("Bad JSON in MesoWest response: '%s'", exc)
 612         sys.exit(1)
 613
 614     # pos represents the last item in the array, aka the most recent
 615     pos = len(observations['date_time']) - 1
 616
 617     # while these values only apply in certain cases, init them here
 618     wind_speed_values = []
 619     wind_gust_speed_values = []
 620     wind_direction_values = []
 621     hn24_values = []
 622
 623     # results
 624     wind_speed_avg = None
 625     wind_gust_speed_avg = None
 626     wind_direction_avg = None
 627     hn24 = None
 628
 629     for element_cd in station['desired_data'].split(','):
 630         # sort and isolate the most recent, see note above in NRCS for how and
 631         # why this is done
 632         #
 633         # NOTE: Unlike in the NRCS case, the MesoWest API response contains all
 634         #       data (whereas with NRCS, we have to make a separate request for
 635         #       each element we want). This is nice for network efficiency but
 636         #       it means we have to handle this part differently for each.
 637         #
 638         # NOTE: Also unlike NRCS, MesoWest provides more granular data; NRCS
 639         #       provides hourly data, but MesoWest can often provide data every
 640         #       10 minutes -- though this provides more opportunity for
 641         #       irregularities
 642
 643         # we may not have the data at all
 644         key_name = element_cd + '_set_1'
 645
 646         if key_name in observations:
 647             # val is what will make it into the dataset, after
 648             # conversions... it gets defined here because in certain
 649             # cases we need to look at all of the data to calculate HN24
 650             # or wind averages, but for the rest of the data, we only
 651             # take the most recent
 652             val = None
 653
 654             # loop through all observations for this key_name
 655             # record relevant values for wind averaging or hn24, but
 656             # otherwise only persist the data if it's the last datum in
 657             # the set
 658             for idx, _ in enumerate(observations[key_name]):
 659                 val = observations[key_name][idx]
 660
 661                 # skip bunk vals
 662                 if val is None:
 663                     continue
 664
 665                 # mesowest by default provides wind_speed in m/s, but
 666                 # we specify 'english' units in the request; either way,
 667                 # we want mph
 668                 if element_cd in ('wind_speed', 'wind_gust'):
 669                     val = kn_to_mph(val)
 670
 671                 # mesowest provides HS in mm, not cm; we want cm
 672                 if element_cd == 'snow_depth' and station['units'] == 'metric':
 673                     val = mm_to_cm(val)
 674
 675                 # HN24 / wind_mode transformations, once the data has
 676                 # completed unit conversions
 677                 if station['wind_mode'] == "average":
 678                     if element_cd == 'wind_speed' and val is not None:
 679                         wind_speed_values.append(val)
 680                     elif element_cd == 'wind_gust' and val is not None:
 681                         wind_gust_speed_values.append(val)
 682                     elif element_cd == 'wind_direction' and val is not None:
 683                         wind_direction_values.append(val)
 684
 685                 if element_cd == 'snow_depth':
 686                     hn24_values.append(val)
 687
 688                 # again, only persist this datum to the final data if
 689                 # it's from the most recent date
 690                 if idx == pos:
 691                     remote_data[element_cd] = val
 692
 693             # ensure that the data is filled out
 694             if not observations[key_name][pos]:
 695                 remote_data[element_cd] = None
 696         else:
 697             remote_data[element_cd] = None
 698
 699     if len(hn24_values) > 0:
 700         # instead of taking MAX - MIN, we want the first value (most
 701         # distant) - the last value (most recent)
 702         #
 703         # if the result is positive, then we have settlement; if it's not,
 704         # then we have HN24
 705         hn24 = hn24_values[0] - hn24_values[len(hn24_values)-1]
 706
 707         if hn24 < 0.0:
 708             hn24 = abs(hn24)
 709         else:
 710             # this case represents HS settlement
 711             hn24 = 0.0
 712
 713
 714     # finally, if user wants hn24 and it's set to None at this
 715     # point, then force it to 0.0
 716     if station['hn24'] and hn24 is None:
 717         hn24 = 0.0
 718
 719     if len(wind_speed_values) > 0:
 720         wind_speed_avg = sum(wind_speed_values) / len(wind_speed_values)
 721
 722     if len(wind_gust_speed_values) > 0:
 723         wind_gust_speed_avg = sum(wind_gust_speed_values) / len(wind_gust_speed_values)
 724
 725     if len(wind_direction_values) > 0:
 726         wind_direction_avg = sum(wind_direction_values) / len(wind_direction_values)
 727
 728     if hn24 is not None:
 729         if station['hn24']:
 730             remote_data['hn24'] = hn24
 731
 732     # overwrite the following with the respective averages, if
 733     # applicable
 734     if wind_speed_avg is not None:
 735         remote_data['wind_speed'] = wind_speed_avg
 736
 737     if wind_gust_speed_avg is not None:
 738         remote_data['wind_gust'] = wind_gust_speed_avg
 739
 740     if wind_direction_avg is not None:
 741         remote_data['wind_direction'] = wind_direction_avg
 742
 743     return remote_data
 744
 745 def switch_units_to_metric(data_map, mapping):
 746     """replace units with metric counterparts"""
 747
 748     # NOTE: to update this, use the fmap<->final_data mapping laid out
 749     #       in setup_infoex_fields_mapping ()
 750     data_map[mapping['tempMaxHourUnit']] = 'C'
 751     data_map[mapping['tempMinHourUnit']] = 'C'
 752     data_map[mapping['tempPresUnit']] = 'C'
 753     data_map[mapping['precipitationGaugeUnit']] = 'mm'
 754     data_map[mapping['hsUnit']] = 'cm'
 755     data_map[mapping['windSpeedUnit']] = 'm/s'
 756     data_map[mapping['windGustSpeedNumUnit']] = 'm/s'
 757     data_map[mapping['dewPointUnit']] = 'C'
 758     data_map[mapping['hn24AutoUnit']] = 'cm'
 759     data_map[mapping['hstAutoUnit']] = 'cm'
 760
 761     return data_map
 762
 763 def switch_units_to_american(data_map, mapping):
 764     """
 765     replace units with the American mixture of metric and imperial
 766
 767     Precip values = metric
 768     Wind values = imperial
 769     """
 770
 771     # precip values
 772     data_map[mapping['tempMaxHourUnit']] = 'C'
 773     data_map[mapping['tempMinHourUnit']] = 'C'
 774     data_map[mapping['tempPresUnit']] = 'C'
 775     data_map[mapping['dewPointUnit']] = 'C'
 776
 777     data_map[mapping['precipitationGaugeUnit']] = 'cm'
 778     data_map[mapping['hsUnit']] = 'cm'
 779     data_map[mapping['hn24AutoUnit']] = 'cm'
 780     data_map[mapping['hstAutoUnit']] = 'cm'
 781
 782     data_map[mapping['baroUnit']] = 'inHg'
 783
 784     # wind values
 785     data_map[mapping['windSpeedUnit']] = 'mph'
 786     data_map[mapping['windGustSpeedNumUnit']] = 'mph'
 787
 788     return data_map
 789
 790 def convert_nrcs_units_to_metric(element_cd, value):
 791     """convert NRCS values from English to metric"""
 792     if element_cd == 'TOBS':
 793         value = f_to_c(value)
 794     elif element_cd == 'SNWD':
 795         value = in_to_cm(value)
 796     elif element_cd == 'PREC':
 797         value = in_to_mm(value)
 798     return value
 799
 800 def convert_units_to_american(element_cd, value):
 801     """
 802     convert value to 'American' units
 803
 804     The original unit is always metric.
 805
 806     Precip values = metric
 807     Wind values = imperial
 808     """
 809
 810     # no need to convert precip values, as they will arrive in metric
 811     # units in "American" units mode
 812     # # temp values
 813     # if element_cd in ['TMAX', 'TMIN', 'TOBS', 'air_temp', 'air_temp_high_24_hour', 'air_temp_low_24_hour']:
 814     #     value = c_to_f(value)
 815
 816     # mesowest provides HS in mm, not cm; we want cm
 817     if element_cd == 'snow_depth':
 818         value = mm_to_cm(value)
 819
 820     # baro values also arrive in metric, so convert to imperial
 821     if element_cd in ['PRES', 'pressure']:
 822         value = inhg_to_pascal(value)
 823
 824     if element_cd in ['WSPD', 'wind_speed', 'wind_gust']:
 825         value = ms_to_mph(value)
 826
 827     return value
 828
 829 # CSV operations
 830 def write_local_csv(path_to_file, data):
 831     """Write the specified CSV file to disk"""
 832     with open(path_to_file, 'w') as file_object:
 833         # The requirement is that empty values are represented in the CSV
 834         # file as "", csv.QUOTE_NONNUMERIC achieves that
 835         LOG.debug("writing CSV file '%s'", path_to_file)
 836         writer = csv.writer(file_object, quoting=csv.QUOTE_NONNUMERIC)
 837         writer.writerow(data)
 838         file_object.close()
 839     return True
 840
 841 def upload_csv(path_to_file, infoex_data):
 842     """Upload the specified CSV file to InfoEx FTP and remove the file"""
 843     with open(path_to_file, 'rb') as file_object:
 844         LOG.debug("uploading FTP file '%s'", infoex_data['host'])
 845         ftp = FTP(infoex_data['host'], infoex_data['uuid'],
 846                   infoex_data['api_key'])
 847         ftp.storlines('STOR ' + path_to_file, file_object)
 848         ftp.close()
 849         file_object.close()
 850     os.remove(path_to_file)
 851
 852 # other miscellaneous routines
 853 def setup_time_values(station):
 854     """establish time bounds of data request(s)"""
 855
 856     # default timezone to UTC (for MesoWest)
 857     tz = pytz.utc
 858
 859     # but for NRCS, use the config-specified timezone
 860     if station['provider'] == 'nrcs':
 861         tz = station['tz']
 862
 863     # floor time to nearest hour
 864     date_time = datetime.datetime.now(tz=tz)
 865     end_date = date_time - datetime.timedelta(minutes=date_time.minute % 60,
 866                                               seconds=date_time.second,
 867                                               microseconds=date_time.microsecond)
 868     begin_date = end_date - datetime.timedelta(hours=station['num_hrs_to_fetch'])
 869     return (begin_date, end_date)
 870
 871 def f_to_c(f):
 872     """convert Fahrenheit to Celsius"""
 873     return (float(f) - 32) * 5.0/9.0
 874
 875 def c_to_f(c):
 876     """convert Celsius to Fahrenheit"""
 877     return (float(c) * 1.8) + 32
 878
 879 def in_to_cm(inches):
 880     """convert inches to centimeters"""
 881     return float(inches) * 2.54
 882
 883 def cm_to_in(cms):
 884     """convert centimeters to inches"""
 885     return float(cms) / 2.54
 886
 887 def pascal_to_inhg(pa):
 888     """convert pascals to inches of mercury"""
 889     return float(pa) * 0.00029530
 890
 891 def inhg_to_pascal(inhg):
 892     """convert inches of mercury to pascals"""
 893     return float(inhg) / 0.00029530
 894
 895 def in_to_mm(inches):
 896     """convert inches to millimeters"""
 897     return (float(inches) * 2.54) * 10.0
 898
 899 def ms_to_mph(ms):
 900     """convert meters per second to miles per hour"""
 901     return ms * 2.236936
 902
 903 def kn_to_mph(kn):
 904     """convert knots to miles per hour"""
 905     return kn * 1.150779
 906
 907 def mm_to_cm(mm):
 908     """convert millimeters to centimeters"""
 909     return mm / 10.0
 910
 911 if __name__ == "__main__":
 912     sys.exit(main())