infoex-autowx.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 """
   5 InfoEx <-> NRCS/MesoWest Auto Wx implementation
   6 Alexander Vasarab
   7 Wylark Mountaineering LLC
   8
   9 This program fetches data from either an NRCS SNOTEL site or MesoWest
  10 weather station and pushes it to InfoEx using the new automated weather
  11 system implementation.
  12
  13 It is designed to be run hourly, and it asks for the last three hours
  14 of data of each desired type, and selects the most recent one. This
  15 lends some resiliency to the process and helps ensure that we have a
  16 value to send, but it can lead to somewhat inconsistent/untruthful
  17 data if e.g. the HS is from the last hour but the tempPres is from two
  18 hours ago because the instrumentation had a hiccup. It's worth
  19 considering if this is a bug or a feature.
  20
  21 For more information, see file: README
  22 For licensing, see file: LICENSE
  23 """
  24
  25 import configparser
  26 import csv
  27 import datetime
  28 import logging
  29 import os
  30 import sys
  31 import time
  32 import urllib3
  33 import importlib.util
  34
  35 from ftplib import FTP
  36 from argparse import ArgumentParser
  37
  38 import pytz
  39
  40 import requests
  41
  42 import zeep
  43 import zeep.cache
  44 import zeep.transports
  45
  46 __version__ = '3.4.2'
  47
  48 LOG = logging.getLogger(__name__)
  49 LOG.setLevel(logging.NOTSET)
  50
  51 urllib3.disable_warnings()
  52
  53 def get_parser():
  54     """Return OptionParser for this program"""
  55     parser = ArgumentParser()
  56
  57     parser.add_argument("--version",
  58                         action="version",
  59                         version=__version__)
  60
  61     parser.add_argument("--config",
  62                         dest="config",
  63                         metavar="FILE",
  64                         help="location of config file")
  65
  66     parser.add_argument("--log-level",
  67                         dest="log_level",
  68                         default=None,
  69                         help="set the log level (debug, info, warning)")
  70
  71     parser.add_argument("--dry-run",
  72                         action="store_true",
  73                         dest="dry_run",
  74                         default=False,
  75                         help="fetch data but don't upload to InfoEx")
  76
  77     return parser
  78
  79 def setup_config(config):
  80     """Setup config variable based on values specified in the ini file"""
  81     try:
  82         infoex = {
  83             'host': config['infoex']['host'],
  84             'uuid': config['infoex']['uuid'],
  85             'api_key': config['infoex']['api_key'],
  86             'csv_filename': config['infoex']['csv_filename'],
  87             'location_uuid': config['infoex']['location_uuid'],
  88             'wx_data': {}, # placeholder key, values to come later
  89         }
  90
  91         station = dict()
  92
  93         station['provider'] = config['station']['type']
  94
  95         if station['provider'] not in ['nrcs', 'mesowest', 'python']:
  96             print("Please specify either nrcs or mesowest as the station type.")
  97             sys.exit(1)
  98
  99         # massage units config items first
 100         #
 101         # NOTE: custom providers don't require units to be specified
 102         #       because they can do whatever they please with the units
 103         #       within their own program
 104         if station['provider'] != "python":
 105             station['units'] = config['station']['units']
 106
 107             if station['units'] not in ['metric', 'english', 'american']:
 108                 print("Please specify metric, english, or american for the units.")
 109                 sys.exit(1)
 110             else:
 111                 # if units are specified as "American" then we simply
 112                 # default to metric for the requests
 113                 if station['units'] == 'american':
 114                     station['units_requested'] = 'metric'
 115                 else:
 116                     station['units_requested'] = station['units']
 117
 118         # massage provider config items
 119         if station['provider'] == 'nrcs':
 120             #station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
 121             station['source'] = 'https://wcc.sc.egov.usda.gov/awdbWebService/services?WSDL'
 122
 123             station['station_id'] = config['station']['station_id']
 124             station['desired_data'] = config['station']['desired_data'].split(',')
 125
 126         if station['provider'] == 'mesowest':
 127             station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
 128             station['station_id'] = config['station']['station_id']
 129             station['desired_data'] = config['station']['desired_data']
 130
 131             # construct full API URL (sans start/end time, added later)
 132             station['source'] = station['source'] + '?token=' + \
 133                                 config['station']['token'] + \
 134                                 '&within=60&units=' + station['units_requested'] + \
 135                                 '&stid=' + station['station_id'] + \
 136                                 '&vars=' + station['desired_data']
 137
 138         if station['provider'] == 'python':
 139             station['path'] = config['station']['path']
 140
 141         tz = 'America/Los_Angeles'
 142
 143         if 'tz' in config['station']:
 144             tz = config['station']['tz']
 145
 146         try:
 147             station['tz'] = pytz.timezone(tz)
 148         except pytz.exceptions.UnknownTimeZoneError:
 149             LOG.critical("%s is not a valid timezone", tz)
 150             sys.exit(1)
 151
 152         # By default, fetch three hours of data
 153         #
 154         # If user wants hn24 or wind averaging, then
 155         # we need more.
 156         station['num_hrs_to_fetch'] = 3
 157
 158         # HN24
 159         if 'hn24' in config['station']:
 160             if config['station']['hn24'] not in ['true', 'false']:
 161                 raise ValueError("hn24 must be either 'true' or 'false'")
 162
 163             if config['station']['hn24'] == "true":
 164                 station['hn24'] = True
 165                 station['num_hrs_to_fetch'] = 24
 166             else:
 167                 station['hn24'] = False
 168         else:
 169             # default to False
 170             station['hn24'] = False
 171
 172         # Wind mode
 173         if 'wind_mode' in config['station']:
 174             if config['station']['wind_mode'] not in ['normal', 'average']:
 175                 raise ValueError("wind_mode must be either 'normal' or 'average'")
 176
 177             station['wind_mode'] = config['station']['wind_mode']
 178
 179             if station['wind_mode'] == "average":
 180                 station['num_hrs_to_fetch'] = 24
 181         else:
 182             # default to False
 183             station['wind_mode'] = "normal"
 184
 185     except KeyError as err:
 186         LOG.critical("%s not defined in configuration file", err)
 187         sys.exit(1)
 188     except ValueError as err:
 189         LOG.critical("%s", err)
 190         sys.exit(1)
 191
 192     # all sections/values present in config file, final sanity check
 193     try:
 194         for key in config.sections():
 195             for subkey in config[key]:
 196                 if not config[key][subkey]:
 197                     raise ValueError
 198     except ValueError:
 199         LOG.critical("Config value '%s.%s' is empty", key, subkey)
 200         sys.exit(1)
 201
 202     return (infoex, station)
 203
 204 def setup_logging(log_level):
 205     """Setup our logging infrastructure"""
 206     try:
 207         from systemd.journal import JournalHandler
 208         LOG.addHandler(JournalHandler())
 209     except ImportError:
 210         ## fallback to syslog
 211         #import logging.handlers
 212         #LOG.addHandler(logging.handlers.SysLogHandler())
 213         # fallback to stdout
 214         handler = logging.StreamHandler(sys.stdout)
 215         formatter = logging.Formatter('%(asctime)s.%(msecs)03d '
 216                                       '%(levelname)s %(module)s - '
 217                                       '%(funcName)s: %(message)s',
 218                                       '%Y-%m-%d %H:%M:%S')
 219         handler.setFormatter(formatter)
 220         LOG.addHandler(handler)
 221
 222     # ugly, but passable
 223     if log_level in [None, 'debug', 'info', 'warning']:
 224         if log_level == 'debug':
 225             LOG.setLevel(logging.DEBUG)
 226         elif log_level == 'info':
 227             LOG.setLevel(logging.INFO)
 228         elif log_level == 'warning':
 229             LOG.setLevel(logging.WARNING)
 230         else:
 231             LOG.setLevel(logging.NOTSET)
 232     else:
 233         return False
 234
 235     return True
 236
 237 def main():
 238     """Main routine: sort through args, decide what to do, then do it"""
 239     parser = get_parser()
 240     options = parser.parse_args()
 241
 242     config = configparser.ConfigParser(allow_no_value=False)
 243
 244     if not options.config:
 245         parser.print_help()
 246         print("\nPlease specify a configuration file via --config.")
 247         sys.exit(1)
 248
 249     config.read(options.config)
 250
 251     if not setup_logging(options.log_level):
 252         parser.print_help()
 253         print("\nPlease select an appropriate log level or remove the switch (--log-level).")
 254         sys.exit(1)
 255
 256     (infoex, station) = setup_config(config)
 257
 258     LOG.debug('Config parsed, starting up')
 259
 260     # create mappings
 261     (fmap, final_data) = setup_infoex_fields_mapping(infoex['location_uuid'])
 262     iemap = setup_infoex_counterparts_mapping(station['provider'])
 263
 264     # override units if user selected metric
 265     if station['provider'] != 'python' and station['units'] == 'metric':
 266         final_data = switch_units_to_metric(final_data, fmap)
 267
 268     # likewise for "American" units
 269     if station['provider'] != 'python' and station['units'] == 'american':
 270         final_data = switch_units_to_american(final_data, fmap)
 271
 272     (begin_date, end_date) = setup_time_values(station)
 273
 274     if station['provider'] == 'python':
 275         LOG.debug("Getting custom data from external Python program")
 276     else:
 277         LOG.debug("Getting %s data from %s to %s (%s)",
 278                   str(station['desired_data']),
 279                   str(begin_date), str(end_date), end_date.tzinfo.zone)
 280
 281     time_all_elements = time.time()
 282
 283     # get the data
 284     if station['provider'] == 'nrcs':
 285         infoex['wx_data'] = get_nrcs_data(begin_date, end_date, station)
 286     elif station['provider'] == 'mesowest':
 287         infoex['wx_data'] = get_mesowest_data(begin_date, end_date,
 288                                               station)
 289     elif station['provider'] == 'python':
 290         try:
 291             spec = importlib.util.spec_from_file_location('custom_wx',
 292                                                           station['path'])
 293             mod = importlib.util.module_from_spec(spec)
 294             spec.loader.exec_module(mod)
 295             mod.LOG = LOG
 296
 297             try:
 298                 infoex['wx_data'] = mod.get_custom_data()
 299
 300                 if infoex['wx_data'] is None:
 301                     infoex['wx_data'] = []
 302             except Exception as exc:
 303                 LOG.error("Python program for custom Wx data failed in "
 304                           "execution: %s", str(exc))
 305                 sys.exit(1)
 306
 307             LOG.info("Successfully executed external Python program")
 308         except ImportError:
 309             LOG.error("Please upgrade to Python 3.3 or later")
 310             sys.exit(1)
 311         except FileNotFoundError:
 312             LOG.error("Specified Python program for custom Wx data "
 313                       "was not found")
 314             sys.exit(1)
 315         except Exception as exc:
 316             LOG.error("A problem was encountered when attempting to "
 317                       "load your custom Wx program: %s", str(exc))
 318             sys.exit(1)
 319
 320     LOG.info("Time taken to get all data : %.3f sec", time.time() -
 321              time_all_elements)
 322
 323     LOG.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
 324
 325     # timezone massaging
 326     final_end_date = end_date.astimezone(station['tz'])
 327
 328     # Now we only need to add in what we want to change thanks to that
 329     # abomination of a variable declaration earlier
 330     final_data[fmap['Location UUID']] = infoex['location_uuid']
 331     final_data[fmap['obDate']] = final_end_date.strftime('%m/%d/%Y')
 332     final_data[fmap['obTime']] = final_end_date.strftime('%H:%M')
 333     final_data[fmap['timeZone']] = station['tz'].zone
 334
 335     for element_cd in infoex['wx_data']:
 336         if element_cd not in iemap:
 337             LOG.warning("BAD KEY wx_data['%s']", element_cd)
 338             continue
 339
 340         if infoex['wx_data'][element_cd] is None:
 341             continue
 342
 343         # do the conversion before the rounding
 344         if station['provider'] == 'nrcs' and station['units'] == 'metric':
 345             infoex['wx_data'][element_cd] = convert_nrcs_units_to_metric(element_cd, infoex['wx_data'][element_cd])
 346
 347         if station['provider'] != 'python' and station['units'] == 'american':
 348             infoex['wx_data'][element_cd] = convert_units_to_american(element_cd, infoex['wx_data'][element_cd])
 349
 350         # Massage precision of certain values to fit InfoEx's
 351         # expectations
 352         #
 353         # 0 decimal places: relative humidity, wind speed, wind
 354         #                   direction, wind gust, snow depth
 355         # 1 decimal place:  air temp, baro
 356         # Avoid transforming None values
 357         if element_cd in ['wind_speed', 'WSPD', 'wind_direction',
 358                           'RHUM', 'relative_humidity', 'WDIR',
 359                           'wind_gust', 'SNWD', 'snow_depth',
 360                           'hn24']:
 361             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd])
 362         elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']:
 363             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1)
 364         elif element_cd in ['PREC', 'precip_accum']:
 365             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 2)
 366
 367         # CONSIDER: Casting every value to Float() -- need to investigate if
 368         #           any possible elementCds we may want are any other data
 369         #           type than float.
 370         #
 371         #           Another possibility is to query the API with
 372         #           getStationElements and temporarily store the
 373         #           storedUnitCd. But that's pretty network-intensive and
 374         #           may not even be worth it if there's only e.g. one or two
 375         #           exceptions to any otherwise uniformly Float value set.
 376         final_data[fmap[iemap[element_cd]]] = infoex['wx_data'][element_cd]
 377
 378     LOG.debug("final_data: %s", str(final_data))
 379
 380     if infoex['wx_data']:
 381         if not write_local_csv(infoex['csv_filename'], final_data):
 382             LOG.warning('Could not write local CSV file: %s',
 383                         infoex['csv_filename'])
 384             return 1
 385
 386         if not options.dry_run:
 387             upload_csv(infoex['csv_filename'], infoex)
 388
 389     LOG.debug('DONE')
 390     return 0
 391
 392 # data structure operations
 393 def setup_infoex_fields_mapping(location_uuid):
 394     """
 395     Create a mapping of InfoEx fields to the local data's indexing scheme.
 396
 397     INFOEX FIELDS
 398
 399     This won't earn style points in Python, but here we establish a couple
 400     of helpful mappings variables. The reason this is helpful is that the
 401     end result is simply an ordered set, the CSV file. But we still may
 402     want to manipulate the values arbitrarily before writing that file.
 403
 404     Also note that the current Auto Wx InfoEx documentation shows these
 405     keys in a graphical table with the "index" beginning at 1, but here we
 406     sanely index beginning at 0.
 407     """
 408     # pylint: disable=too-many-statements,multiple-statements,bad-whitespace
 409     fmap = {}                           ; final_data     = [None] * 29
 410     fmap['Location UUID'] = 0           ; final_data[0]  = location_uuid
 411     fmap['obDate'] = 1                  ; final_data[1]  = None
 412     fmap['obTime'] = 2                  ; final_data[2]  = None
 413     fmap['timeZone'] = 3                ; final_data[3]  = 'Pacific'
 414     fmap['tempMaxHour'] = 4             ; final_data[4]  = None
 415     fmap['tempMaxHourUnit'] = 5         ; final_data[5]  = 'F'
 416     fmap['tempMinHour'] = 6             ; final_data[6]  = None
 417     fmap['tempMinHourUnit'] = 7         ; final_data[7]  = 'F'
 418     fmap['tempPres'] = 8                ; final_data[8]  = None
 419     fmap['tempPresUnit'] = 9            ; final_data[9]  = 'F'
 420     fmap['precipitationGauge'] = 10     ; final_data[10] = None
 421     fmap['precipitationGaugeUnit'] = 11 ; final_data[11] = 'in'
 422     fmap['windSpeedNum'] = 12           ; final_data[12] = None
 423     fmap['windSpeedUnit'] = 13          ; final_data[13] = 'mph'
 424     fmap['windDirectionNum'] = 14       ; final_data[14] = None
 425     fmap['hS'] = 15                     ; final_data[15] = None
 426     fmap['hsUnit'] = 16                 ; final_data[16] = 'in'
 427     fmap['baro'] = 17                   ; final_data[17] = None
 428     fmap['baroUnit'] = 18               ; final_data[18] = 'inHg'
 429     fmap['rH'] = 19                     ; final_data[19] = None
 430     fmap['windGustSpeedNum'] = 20       ; final_data[20] = None
 431     fmap['windGustSpeedNumUnit'] = 21   ; final_data[21] = 'mph'
 432     fmap['windGustDirNum'] = 22         ; final_data[22] = None
 433     fmap['dewPoint'] = 23               ; final_data[23] = None
 434     fmap['dewPointUnit'] = 24           ; final_data[24] = 'F'
 435     fmap['hn24Auto'] = 25               ; final_data[25] = None
 436     fmap['hn24AutoUnit'] = 26           ; final_data[26] = 'in'
 437     fmap['hstAuto'] = 27                ; final_data[27] = None
 438     fmap['hstAutoUnit'] = 28            ; final_data[28] = 'in'
 439
 440     return (fmap, final_data)
 441
 442 def setup_infoex_counterparts_mapping(provider):
 443     """
 444     Create a mapping of the NRCS/MesoWest fields that this program supports to
 445     their InfoEx counterparts
 446     """
 447     iemap = {}
 448
 449     if provider == 'nrcs':
 450         iemap['PREC'] = 'precipitationGauge'
 451         iemap['TOBS'] = 'tempPres'
 452         iemap['TMAX'] = 'tempMaxHour'
 453         iemap['TMIN'] = 'tempMinHour'
 454         iemap['SNWD'] = 'hS'
 455         iemap['PRES'] = 'baro'
 456         iemap['RHUM'] = 'rH'
 457         iemap['WSPD'] = 'windSpeedNum'
 458         iemap['WDIR'] = 'windDirectionNum'
 459         # unsupported by NRCS:
 460         # windGustSpeedNum
 461
 462         # NOTE: this doesn't exist in NRCS SNOTEL, we create it in this
 463         #       program, so add it to the map here
 464         iemap['hn24'] = 'hn24Auto'
 465     elif provider == 'mesowest':
 466         iemap['precip_accum'] = 'precipitationGauge'
 467         iemap['air_temp'] = 'tempPres'
 468         iemap['air_temp_high_24_hour'] = 'tempMaxHour'
 469         iemap['air_temp_low_24_hour'] = 'tempMinHour'
 470         iemap['snow_depth'] = 'hS'
 471         iemap['pressure'] = 'baro'
 472         iemap['relative_humidity'] = 'rH'
 473         iemap['wind_speed'] = 'windSpeedNum'
 474         iemap['wind_direction'] = 'windDirectionNum'
 475         iemap['wind_gust'] = 'windGustSpeedNum'
 476
 477         # NOTE: this doesn't exist in MesoWest, we create it in this
 478         #       program, so add it to the map here
 479         iemap['hn24'] = 'hn24Auto'
 480     elif provider == 'python':
 481         # we expect Python programs to use the InfoEx data type names
 482         iemap['precipitationGauge'] = 'precipitationGauge'
 483         iemap['precipitationGaugeUnit'] = 'precipitationGaugeUnit'
 484         iemap['tempPres'] = 'tempPres'
 485         iemap['tempPresUnit'] = 'tempPresUnit'
 486         iemap['tempMaxHour'] = 'tempMaxHour'
 487         iemap['tempMinHour'] = 'tempMinHour'
 488         iemap['hS'] = 'hS'
 489         iemap['hsUnit'] = 'hsUnit'
 490         iemap['baro'] = 'baro'
 491         iemap['rH'] = 'rH'
 492         iemap['windSpeedNum'] = 'windSpeedNum'
 493         iemap['windDirectionNum'] = 'windDirectionNum'
 494         iemap['windGustSpeedNum'] = 'windGustSpeedNum'
 495
 496     return iemap
 497
 498 # provider-specific operations
 499 def get_nrcs_data(begin, end, station):
 500     """get the data we're after from the NRCS WSDL"""
 501     transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
 502     transport.session.verify = False
 503     client = zeep.Client(wsdl=station['source'], transport=transport)
 504     remote_data = {}
 505
 506     # massage begin/end date format
 507     begin_date_str = begin.strftime('%Y-%m-%d %H:%M:00')
 508     end_date_str = end.strftime('%Y-%m-%d %H:%M:00')
 509
 510     for element_cd in station['desired_data']:
 511         time_element = time.time()
 512
 513         # get the last three hours of data for this elementCd/element_cd
 514         tmp = client.service.getHourlyData(
 515             stationTriplets=[station['station_id']],
 516             elementCd=element_cd,
 517             ordinal=1,
 518             beginDate=begin_date_str,
 519             endDate=end_date_str)
 520
 521         LOG.info("Time to get NRCS elementCd '%s': %.3f sec", element_cd,
 522                  time.time() - time_element)
 523
 524         values = tmp[0]['values']
 525
 526         # sort and isolate the most recent
 527         #
 528         # NOTE: we do this because sometimes there are gaps in hourly data
 529         #       in NRCS; yes, we may end up with slightly inaccurate data,
 530         #       so perhaps this decision will be re-evaluated in the future
 531         if values:
 532             ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
 533             remote_data[element_cd] = ordered[0]['value']
 534         else:
 535             remote_data[element_cd] = None
 536
 537
 538         # calc hn24, if applicable
 539         hn24 = None
 540
 541         if station['hn24']:
 542             hn24_values = []
 543
 544             if element_cd == "SNWD":
 545                 for idx, _ in enumerate(values):
 546                     val = values[idx]
 547                     if val is None:
 548                         continue
 549                     hn24_values.append(val['value'])
 550
 551                 if len(hn24_values) > 0:
 552                     # instead of taking MAX - MIN, we want the first
 553                     # value (most distant) - the last value (most
 554                     # recent)
 555                     #
 556                     # if the result is positive, then we have
 557                     # settlement; if it's not, then we have HN24
 558                     hn24 = hn24_values[0] - hn24_values[len(hn24_values)-1]
 559
 560                     if hn24 < 0.0:
 561                         hn24 = abs(hn24)
 562                     else:
 563                         # this case represents HS settlement
 564                         hn24 = 0.0
 565
 566             # finally, if user wants hn24 and it's set to None at this
 567             # point, then force it to 0.0
 568             if hn24 is None:
 569                 hn24 = 0.0
 570
 571         if hn24 is not None:
 572             if station['hn24']:
 573                 remote_data['hn24'] = hn24
 574
 575     return remote_data
 576
 577 def get_mesowest_data(begin, end, station):
 578     """get the data we're after from the MesoWest/Synoptic API"""
 579     remote_data = {}
 580
 581     # massage begin/end date format
 582     begin_date_str = begin.strftime('%Y%m%d%H%M')
 583     end_date_str = end.strftime('%Y%m%d%H%M')
 584
 585     # construct final, completed API URL
 586     api_req_url = station['source'] + '&start=' + begin_date_str + '&end=' + end_date_str
 587
 588     try:
 589         req = requests.get(api_req_url)
 590     except requests.exceptions.ConnectionError:
 591         LOG.error("Could not connect to '%s'", api_req_url)
 592         sys.exit(1)
 593
 594     try:
 595         json = req.json()
 596     except ValueError:
 597         LOG.error("Bad JSON in MesoWest response")
 598         sys.exit(1)
 599
 600     try:
 601         observations = json['STATION'][0]['OBSERVATIONS']
 602     except KeyError as exc:
 603         LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
 604         sys.exit(1)
 605     except IndexError as exc:
 606         LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
 607         try:
 608             LOG.error("Detailed MesoWest response: '%s'",
 609                       json['SUMMARY']['RESPONSE_MESSAGE'])
 610         except KeyError:
 611             pass
 612         sys.exit(1)
 613     except ValueError as exc:
 614         LOG.error("Bad JSON in MesoWest response: '%s'", exc)
 615         sys.exit(1)
 616
 617     # pos represents the last item in the array, aka the most recent
 618     pos = len(observations['date_time']) - 1
 619
 620     # while these values only apply in certain cases, init them here
 621     wind_speed_values = []
 622     wind_gust_speed_values = []
 623     wind_direction_values = []
 624     hn24_values = []
 625
 626     # results
 627     wind_speed_avg = None
 628     wind_gust_speed_avg = None
 629     wind_direction_avg = None
 630     hn24 = None
 631
 632     for element_cd in station['desired_data'].split(','):
 633         # sort and isolate the most recent, see note above in NRCS for how and
 634         # why this is done
 635         #
 636         # NOTE: Unlike in the NRCS case, the MesoWest API response contains all
 637         #       data (whereas with NRCS, we have to make a separate request for
 638         #       each element we want). This is nice for network efficiency but
 639         #       it means we have to handle this part differently for each.
 640         #
 641         # NOTE: Also unlike NRCS, MesoWest provides more granular data; NRCS
 642         #       provides hourly data, but MesoWest can often provide data every
 643         #       10 minutes -- though this provides more opportunity for
 644         #       irregularities
 645
 646         # we may not have the data at all
 647         key_name = element_cd + '_set_1'
 648
 649         if key_name in observations:
 650             # val is what will make it into the dataset, after
 651             # conversions... it gets defined here because in certain
 652             # cases we need to look at all of the data to calculate HN24
 653             # or wind averages, but for the rest of the data, we only
 654             # take the most recent
 655             val = None
 656
 657             # loop through all observations for this key_name
 658             # record relevant values for wind averaging or hn24, but
 659             # otherwise only persist the data if it's the last datum in
 660             # the set
 661             for idx, _ in enumerate(observations[key_name]):
 662                 val = observations[key_name][idx]
 663
 664                 # skip bunk vals
 665                 if val is None:
 666                     continue
 667
 668                 # mesowest by default provides wind_speed in m/s, but
 669                 # we specify 'english' units in the request; either way,
 670                 # we want mph
 671                 if element_cd in ('wind_speed', 'wind_gust'):
 672                     val = kn_to_mph(val)
 673
 674                 # mesowest provides HS in mm, not cm; we want cm
 675                 if element_cd == 'snow_depth' and station['units'] == 'metric':
 676                     val = mm_to_cm(val)
 677
 678                 # HN24 / wind_mode transformations, once the data has
 679                 # completed unit conversions
 680                 if station['wind_mode'] == "average":
 681                     if element_cd == 'wind_speed' and val is not None:
 682                         wind_speed_values.append(val)
 683                     elif element_cd == 'wind_gust' and val is not None:
 684                         wind_gust_speed_values.append(val)
 685                     elif element_cd == 'wind_direction' and val is not None:
 686                         wind_direction_values.append(val)
 687
 688                 if element_cd == 'snow_depth':
 689                     hn24_values.append(val)
 690
 691                 # again, only persist this datum to the final data if
 692                 # it's from the most recent date
 693                 if idx == pos:
 694                     remote_data[element_cd] = val
 695
 696             # ensure that the data is filled out
 697             if not observations[key_name][pos]:
 698                 remote_data[element_cd] = None
 699         else:
 700             remote_data[element_cd] = None
 701
 702     if len(hn24_values) > 0:
 703         # instead of taking MAX - MIN, we want the first value (most
 704         # distant) - the last value (most recent)
 705         #
 706         # if the result is positive, then we have settlement; if it's not,
 707         # then we have HN24
 708         hn24 = hn24_values[0] - hn24_values[len(hn24_values)-1]
 709
 710         if hn24 < 0.0:
 711             hn24 = abs(hn24)
 712         else:
 713             # this case represents HS settlement
 714             hn24 = 0.0
 715
 716
 717     # finally, if user wants hn24 and it's set to None at this
 718     # point, then force it to 0.0
 719     if station['hn24'] and hn24 is None:
 720         hn24 = 0.0
 721
 722     if len(wind_speed_values) > 0:
 723         wind_speed_avg = sum(wind_speed_values) / len(wind_speed_values)
 724
 725     if len(wind_gust_speed_values) > 0:
 726         wind_gust_speed_avg = sum(wind_gust_speed_values) / len(wind_gust_speed_values)
 727
 728     if len(wind_direction_values) > 0:
 729         wind_direction_avg = sum(wind_direction_values) / len(wind_direction_values)
 730
 731     if hn24 is not None:
 732         if station['hn24']:
 733             remote_data['hn24'] = hn24
 734
 735     # overwrite the following with the respective averages, if
 736     # applicable
 737     if wind_speed_avg is not None:
 738         remote_data['wind_speed'] = wind_speed_avg
 739
 740     if wind_gust_speed_avg is not None:
 741         remote_data['wind_gust'] = wind_gust_speed_avg
 742
 743     if wind_direction_avg is not None:
 744         remote_data['wind_direction'] = wind_direction_avg
 745
 746     return remote_data
 747
 748 def switch_units_to_metric(data_map, mapping):
 749     """replace units with metric counterparts"""
 750
 751     # NOTE: to update this, use the fmap<->final_data mapping laid out
 752     #       in setup_infoex_fields_mapping ()
 753     data_map[mapping['tempMaxHourUnit']] = 'C'
 754     data_map[mapping['tempMinHourUnit']] = 'C'
 755     data_map[mapping['tempPresUnit']] = 'C'
 756     data_map[mapping['precipitationGaugeUnit']] = 'mm'
 757     data_map[mapping['hsUnit']] = 'cm'
 758     data_map[mapping['windSpeedUnit']] = 'm/s'
 759     data_map[mapping['windGustSpeedNumUnit']] = 'm/s'
 760     data_map[mapping['dewPointUnit']] = 'C'
 761     data_map[mapping['hn24AutoUnit']] = 'cm'
 762     data_map[mapping['hstAutoUnit']] = 'cm'
 763
 764     return data_map
 765
 766 def switch_units_to_american(data_map, mapping):
 767     """
 768     replace units with the American mixture of metric and imperial
 769
 770     Precip values = metric
 771     Wind values = imperial
 772     """
 773
 774     # precip values
 775     data_map[mapping['tempMaxHourUnit']] = 'C'
 776     data_map[mapping['tempMinHourUnit']] = 'C'
 777     data_map[mapping['tempPresUnit']] = 'C'
 778     data_map[mapping['dewPointUnit']] = 'C'
 779
 780     data_map[mapping['precipitationGaugeUnit']] = 'cm'
 781     data_map[mapping['hsUnit']] = 'cm'
 782     data_map[mapping['hn24AutoUnit']] = 'cm'
 783     data_map[mapping['hstAutoUnit']] = 'cm'
 784
 785     data_map[mapping['baroUnit']] = 'inHg'
 786
 787     # wind values
 788     data_map[mapping['windSpeedUnit']] = 'mph'
 789     data_map[mapping['windGustSpeedNumUnit']] = 'mph'
 790
 791     return data_map
 792
 793 def convert_nrcs_units_to_metric(element_cd, value):
 794     """convert NRCS values from English to metric"""
 795     if element_cd == 'TOBS':
 796         value = f_to_c(value)
 797     elif element_cd == 'SNWD':
 798         value = in_to_cm(value)
 799     elif element_cd == 'PREC':
 800         value = in_to_mm(value)
 801     return value
 802
 803 def convert_units_to_american(element_cd, value):
 804     """
 805     convert value to 'American' units
 806
 807     The original unit is always metric.
 808
 809     Precip values = metric
 810     Wind values = imperial
 811     """
 812
 813     # no need to convert precip values, as they will arrive in metric
 814     # units in "American" units mode
 815     # # temp values
 816     # if element_cd in ['TMAX', 'TMIN', 'TOBS', 'air_temp', 'air_temp_high_24_hour', 'air_temp_low_24_hour']:
 817     #     value = c_to_f(value)
 818
 819     # mesowest provides HS in mm, not cm; we want cm
 820     if element_cd == 'snow_depth':
 821         value = mm_to_cm(value)
 822
 823     # baro values also arrive in metric, so convert to imperial
 824     if element_cd in ['PRES', 'pressure']:
 825         value = inhg_to_pascal(value)
 826
 827     if element_cd in ['WSPD', 'wind_speed', 'wind_gust']:
 828         value = ms_to_mph(value)
 829
 830     return value
 831
 832 # CSV operations
 833 def write_local_csv(path_to_file, data):
 834     """Write the specified CSV file to disk"""
 835     with open(path_to_file, 'w') as file_object:
 836         # The requirement is that empty values are represented in the CSV
 837         # file as "", csv.QUOTE_NONNUMERIC achieves that
 838         LOG.debug("writing CSV file '%s'", path_to_file)
 839         writer = csv.writer(file_object, quoting=csv.QUOTE_NONNUMERIC)
 840         writer.writerow(data)
 841         file_object.close()
 842     return True
 843
 844 def upload_csv(path_to_file, infoex_data):
 845     """Upload the specified CSV file to InfoEx FTP and remove the file"""
 846     with open(path_to_file, 'rb') as file_object:
 847         LOG.debug("uploading FTP file '%s'", infoex_data['host'])
 848         ftp = FTP(infoex_data['host'], infoex_data['uuid'],
 849                   infoex_data['api_key'])
 850         ftp.storlines('STOR ' + path_to_file, file_object)
 851         ftp.close()
 852         file_object.close()
 853     os.remove(path_to_file)
 854
 855 # other miscellaneous routines
 856 def setup_time_values(station):
 857     """establish time bounds of data request(s)"""
 858
 859     # default timezone to UTC (for MesoWest)
 860     tz = pytz.utc
 861
 862     # but for NRCS, use the config-specified timezone
 863     if station['provider'] == 'nrcs':
 864         tz = station['tz']
 865
 866     # floor time to nearest hour
 867     date_time = datetime.datetime.now(tz=tz)
 868     end_date = date_time - datetime.timedelta(minutes=date_time.minute % 60,
 869                                               seconds=date_time.second,
 870                                               microseconds=date_time.microsecond)
 871     begin_date = end_date - datetime.timedelta(hours=station['num_hrs_to_fetch'])
 872     return (begin_date, end_date)
 873
 874 def f_to_c(f):
 875     """convert Fahrenheit to Celsius"""
 876     return (float(f) - 32) * 5.0/9.0
 877
 878 def c_to_f(c):
 879     """convert Celsius to Fahrenheit"""
 880     return (float(c) * 1.8) + 32
 881
 882 def in_to_cm(inches):
 883     """convert inches to centimeters"""
 884     return float(inches) * 2.54
 885
 886 def cm_to_in(cms):
 887     """convert centimeters to inches"""
 888     return float(cms) / 2.54
 889
 890 def pascal_to_inhg(pa):
 891     """convert pascals to inches of mercury"""
 892     return float(pa) * 0.00029530
 893
 894 def inhg_to_pascal(inhg):
 895     """convert inches of mercury to pascals"""
 896     return float(inhg) / 0.00029530
 897
 898 def in_to_mm(inches):
 899     """convert inches to millimeters"""
 900     return (float(inches) * 2.54) * 10.0
 901
 902 def ms_to_mph(ms):
 903     """convert meters per second to miles per hour"""
 904     return ms * 2.236936
 905
 906 def kn_to_mph(kn):
 907     """convert knots to miles per hour"""
 908     return kn * 1.150779
 909
 910 def mm_to_cm(mm):
 911     """convert millimeters to centimeters"""
 912     return mm / 10.0
 913
 914 if __name__ == "__main__":
 915     sys.exit(main())