infoex-autowx.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 """
   5 InfoEx <-> NRCS/MesoWest Auto Wx implementation
   6 Alexander Vasarab
   7 Wylark Mountaineering LLC
   8
   9 This program fetches data from either an NRCS SNOTEL site or MesoWest
  10 weather station and pushes it to InfoEx using the new automated weather
  11 system implementation.
  12
  13 It is designed to be run hourly, and it asks for the last three hours
  14 of data of each desired type, and selects the most recent one. This
  15 lends some resiliency to the process and helps ensure that we have a
  16 value to send, but it can lead to somewhat inconsistent/untruthful
  17 data if e.g. the HS is from the last hour but the tempPres is from two
  18 hours ago because the instrumentation had a hiccup. It's worth
  19 considering if this is a bug or a feature.
  20
  21 For more information, see file: README
  22 For licensing, see file: LICENSE
  23 """
  24
  25 import configparser
  26 import csv
  27 import datetime
  28 import logging
  29 import os
  30 import sys
  31 import time
  32 import urllib3
  33 import importlib.util
  34
  35 from ftplib import FTP
  36 from argparse import ArgumentParser
  37
  38 import pytz
  39
  40 import requests
  41
  42 import zeep
  43 import zeep.cache
  44 import zeep.transports
  45
  46 __version__ = '3.3.1'
  47
  48 LOG = logging.getLogger(__name__)
  49 LOG.setLevel(logging.NOTSET)
  50
  51 urllib3.disable_warnings()
  52
  53 def get_parser():
  54     """Return OptionParser for this program"""
  55     parser = ArgumentParser()
  56
  57     parser.add_argument("--version",
  58                         action="version",
  59                         version=__version__)
  60
  61     parser.add_argument("--config",
  62                         dest="config",
  63                         metavar="FILE",
  64                         help="location of config file")
  65
  66     parser.add_argument("--log-level",
  67                         dest="log_level",
  68                         default=None,
  69                         help="set the log level (debug, info, warning)")
  70
  71     parser.add_argument("--dry-run",
  72                         action="store_true",
  73                         dest="dry_run",
  74                         default=False,
  75                         help="fetch data but don't upload to InfoEx")
  76
  77     return parser
  78
  79 def setup_config(config):
  80     """Setup config variable based on values specified in the ini file"""
  81     try:
  82         infoex = {
  83             'host': config['infoex']['host'],
  84             'uuid': config['infoex']['uuid'],
  85             'api_key': config['infoex']['api_key'],
  86             'csv_filename': config['infoex']['csv_filename'],
  87             'location_uuid': config['infoex']['location_uuid'],
  88             'wx_data': {}, # placeholder key, values to come later
  89         }
  90
  91         station = dict()
  92         station['provider'] = config['station']['type']
  93
  94         if station['provider'] not in ['nrcs', 'mesowest', 'python']:
  95             print("Please specify either nrcs or mesowest as the station type.")
  96             sys.exit(1)
  97
  98         if station['provider'] == 'nrcs':
  99             station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
 100             station['station_id'] = config['station']['station_id']
 101             station['desired_data'] = config['station']['desired_data'].split(',')
 102             station['units'] = config['station']['units']
 103
 104         if station['provider'] == 'mesowest':
 105             station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
 106             station['station_id'] = config['station']['station_id']
 107             station['units'] = config['station']['units']
 108             station['desired_data'] = config['station']['desired_data']
 109
 110             # construct full API URL (sans start/end time, added later)
 111             station['source'] = station['source'] + '?token=' + \
 112                                 config['station']['token'] + \
 113                                 '&within=60&units=' + station['units'] + \
 114                                 '&stid=' + station['station_id'] + \
 115                                 '&vars=' + station['desired_data']
 116
 117         if station['provider'] == 'python':
 118             station['path'] = config['station']['path']
 119
 120         tz = 'America/Los_Angeles'
 121
 122         if 'tz' in config['station']:
 123             tz = config['station']['tz']
 124
 125         try:
 126             station['tz'] = pytz.timezone(tz)
 127         except pytz.exceptions.UnknownTimeZoneError:
 128             LOG.critical("%s is not a valid timezone", tz)
 129             sys.exit(1)
 130
 131         if station['units'] not in ['metric', 'english', 'american']:
 132             print("Please specify either metric, english, or american for the units.")
 133             sys.exit(1)
 134
 135         # By default, fetch three hours of data
 136         #
 137         # If user wants hn24 or wind averaging, then
 138         # we need more.
 139         station['num_hrs_to_fetch'] = 3
 140
 141         # HN24
 142         if 'hn24' in config['station']:
 143             if config['station']['hn24'] not in ['true', 'false']:
 144                 raise ValueError("hn24 must be either 'true' or 'false'")
 145
 146             if config['station']['hn24'] == "true":
 147                 station['hn24'] = True
 148                 station['num_hrs_to_fetch'] = 24
 149             else:
 150                 station['hn24'] = False
 151         else:
 152             # default to False
 153             station['hn24'] = False
 154
 155         # Wind mode
 156         if 'wind_mode' in config['station']:
 157             if config['station']['wind_mode'] not in ['normal', 'average']:
 158                 raise ValueError("wind_mode must be either 'normal' or 'average'")
 159
 160             station['wind_mode'] = config['station']['wind_mode']
 161
 162             if station['wind_mode'] == "average":
 163                 station['num_hrs_to_fetch'] = 24
 164         else:
 165             # default to False
 166             station['wind_mode'] = "normal"
 167
 168     except KeyError as err:
 169         LOG.critical("%s not defined in configuration file", err)
 170         sys.exit(1)
 171     except ValueError as err:
 172         LOG.critical("%s", err)
 173         sys.exit(1)
 174
 175     # all sections/values present in config file, final sanity check
 176     try:
 177         for key in config.sections():
 178             for subkey in config[key]:
 179                 if not config[key][subkey]:
 180                     raise ValueError
 181     except ValueError:
 182         LOG.critical("Config value '%s.%s' is empty", key, subkey)
 183         sys.exit(1)
 184
 185     return (infoex, station)
 186
 187 def setup_logging(log_level):
 188     """Setup our logging infrastructure"""
 189     try:
 190         from systemd.journal import JournalHandler
 191         LOG.addHandler(JournalHandler())
 192     except ImportError:
 193         ## fallback to syslog
 194         #import logging.handlers
 195         #LOG.addHandler(logging.handlers.SysLogHandler())
 196         # fallback to stdout
 197         handler = logging.StreamHandler(sys.stdout)
 198         formatter = logging.Formatter('%(asctime)s.%(msecs)03d '
 199                                       '%(levelname)s %(module)s - '
 200                                       '%(funcName)s: %(message)s',
 201                                       '%Y-%m-%d %H:%M:%S')
 202         handler.setFormatter(formatter)
 203         LOG.addHandler(handler)
 204
 205     # ugly, but passable
 206     if log_level in [None, 'debug', 'info', 'warning']:
 207         if log_level == 'debug':
 208             LOG.setLevel(logging.DEBUG)
 209         elif log_level == 'info':
 210             LOG.setLevel(logging.INFO)
 211         elif log_level == 'warning':
 212             LOG.setLevel(logging.WARNING)
 213         else:
 214             LOG.setLevel(logging.NOTSET)
 215     else:
 216         return False
 217
 218     return True
 219
 220 def main():
 221     """Main routine: sort through args, decide what to do, then do it"""
 222     parser = get_parser()
 223     options = parser.parse_args()
 224
 225     config = configparser.ConfigParser(allow_no_value=False)
 226
 227     if not options.config:
 228         parser.print_help()
 229         print("\nPlease specify a configuration file via --config.")
 230         sys.exit(1)
 231
 232     config.read(options.config)
 233
 234     if not setup_logging(options.log_level):
 235         parser.print_help()
 236         print("\nPlease select an appropriate log level or remove the switch (--log-level).")
 237         sys.exit(1)
 238
 239     (infoex, station) = setup_config(config)
 240
 241     LOG.debug('Config parsed, starting up')
 242
 243     # create mappings
 244     (fmap, final_data) = setup_infoex_fields_mapping(infoex['location_uuid'])
 245     iemap = setup_infoex_counterparts_mapping(station['provider'])
 246
 247     # override units if user selected metric
 248     if station['provider'] != 'python' and station['units'] == 'metric':
 249         final_data = switch_units_to_metric(final_data, fmap)
 250
 251     (begin_date, end_date) = setup_time_values(station)
 252
 253     if station['provider'] == 'python':
 254         LOG.debug("Getting custom data from external Python program")
 255     else:
 256         LOG.debug("Getting %s data from %s to %s (%s)",
 257                   str(station['desired_data']),
 258                   str(begin_date), str(end_date), end_date.tzinfo.zone)
 259
 260     time_all_elements = time.time()
 261
 262     # get the data
 263     if station['provider'] == 'nrcs':
 264         infoex['wx_data'] = get_nrcs_data(begin_date, end_date, station)
 265     elif station['provider'] == 'mesowest':
 266         infoex['wx_data'] = get_mesowest_data(begin_date, end_date,
 267                                               station)
 268     elif station['provider'] == 'python':
 269         try:
 270             spec = importlib.util.spec_from_file_location('custom_wx',
 271                                                           station['path'])
 272             mod = importlib.util.module_from_spec(spec)
 273             spec.loader.exec_module(mod)
 274             mod.LOG = LOG
 275
 276             try:
 277                 infoex['wx_data'] = mod.get_custom_data()
 278
 279                 if infoex['wx_data'] is None:
 280                     infoex['wx_data'] = []
 281             except Exception as exc:
 282                 LOG.error("Python program for custom Wx data failed in "
 283                           "execution: %s", str(exc))
 284                 sys.exit(1)
 285
 286             LOG.info("Successfully executed external Python program")
 287         except ImportError:
 288             LOG.error("Please upgrade to Python 3.3 or later")
 289             sys.exit(1)
 290         except FileNotFoundError:
 291             LOG.error("Specified Python program for custom Wx data "
 292                       "was not found")
 293             sys.exit(1)
 294         except Exception as exc:
 295             LOG.error("A problem was encountered when attempting to "
 296                       "load your custom Wx program: %s", str(exc))
 297             sys.exit(1)
 298
 299     LOG.info("Time taken to get all data : %.3f sec", time.time() -
 300              time_all_elements)
 301
 302     LOG.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
 303
 304     # timezone massaging
 305     final_end_date = end_date.astimezone(station['tz'])
 306
 307     # Now we only need to add in what we want to change thanks to that
 308     # abomination of a variable declaration earlier
 309     final_data[fmap['Location UUID']] = infoex['location_uuid']
 310     final_data[fmap['obDate']] = final_end_date.strftime('%m/%d/%Y')
 311     final_data[fmap['obTime']] = final_end_date.strftime('%H:%M')
 312     final_data[fmap['timeZone']] = station['tz'].zone
 313
 314     for element_cd in infoex['wx_data']:
 315         if element_cd not in iemap:
 316             LOG.warning("BAD KEY wx_data['%s']", element_cd)
 317             continue
 318
 319         if infoex['wx_data'][element_cd] is None:
 320             continue
 321
 322         # do the conversion before the rounding
 323         if station['provider'] == 'nrcs' and station['units'] == 'metric':
 324             infoex['wx_data'][element_cd] = convert_nrcs_units_to_metric(element_cd, infoex['wx_data'][element_cd])
 325
 326         # Massage precision of certain values to fit InfoEx's
 327         # expectations
 328         #
 329         # 0 decimal places: relative humidity, wind speed, wind
 330         #                   direction, wind gust, snow depth
 331         # 1 decimal place:  air temp, baro
 332         # Avoid transforming None values
 333         if element_cd in ['wind_speed', 'WSPD', 'wind_direction',
 334                           'RHUM', 'relative_humidity', 'WDIR',
 335                           'wind_gust', 'SNWD', 'snow_depth',
 336                           'hn24']:
 337             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd])
 338         elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']:
 339             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1)
 340         elif element_cd in ['PREC', 'precip_accum']:
 341             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 2)
 342
 343         # CONSIDER: Casting every value to Float() -- need to investigate if
 344         #           any possible elementCds we may want are any other data
 345         #           type than float.
 346         #
 347         #           Another possibility is to query the API with
 348         #           getStationElements and temporarily store the
 349         #           storedUnitCd. But that's pretty network-intensive and
 350         #           may not even be worth it if there's only e.g. one or two
 351         #           exceptions to any otherwise uniformly Float value set.
 352         final_data[fmap[iemap[element_cd]]] = infoex['wx_data'][element_cd]
 353
 354     LOG.debug("final_data: %s", str(final_data))
 355
 356     if infoex['wx_data']:
 357         if not write_local_csv(infoex['csv_filename'], final_data):
 358             LOG.warning('Could not write local CSV file: %s',
 359                         infoex['csv_filename'])
 360             return 1
 361
 362         if not options.dry_run:
 363             upload_csv(infoex['csv_filename'], infoex)
 364
 365     LOG.debug('DONE')
 366     return 0
 367
 368 # data structure operations
 369 def setup_infoex_fields_mapping(location_uuid):
 370     """
 371     Create a mapping of InfoEx fields to the local data's indexing scheme.
 372
 373     INFOEX FIELDS
 374
 375     This won't earn style points in Python, but here we establish a couple
 376     of helpful mappings variables. The reason this is helpful is that the
 377     end result is simply an ordered set, the CSV file. But we still may
 378     want to manipulate the values arbitrarily before writing that file.
 379
 380     Also note that the current Auto Wx InfoEx documentation shows these
 381     keys in a graphical table with the "index" beginning at 1, but here we
 382     sanely index beginning at 0.
 383     """
 384     # pylint: disable=too-many-statements,multiple-statements,bad-whitespace
 385     fmap = {}                           ; final_data     = [None] * 29
 386     fmap['Location UUID'] = 0           ; final_data[0]  = location_uuid
 387     fmap['obDate'] = 1                  ; final_data[1]  = None
 388     fmap['obTime'] = 2                  ; final_data[2]  = None
 389     fmap['timeZone'] = 3                ; final_data[3]  = 'Pacific'
 390     fmap['tempMaxHour'] = 4             ; final_data[4]  = None
 391     fmap['tempMaxHourUnit'] = 5         ; final_data[5]  = 'F'
 392     fmap['tempMinHour'] = 6             ; final_data[6]  = None
 393     fmap['tempMinHourUnit'] = 7         ; final_data[7]  = 'F'
 394     fmap['tempPres'] = 8                ; final_data[8]  = None
 395     fmap['tempPresUnit'] = 9            ; final_data[9]  = 'F'
 396     fmap['precipitationGauge'] = 10     ; final_data[10] = None
 397     fmap['precipitationGaugeUnit'] = 11 ; final_data[11] = 'in'
 398     fmap['windSpeedNum'] = 12           ; final_data[12] = None
 399     fmap['windSpeedUnit'] = 13          ; final_data[13] = 'mph'
 400     fmap['windDirectionNum'] = 14       ; final_data[14] = None
 401     fmap['hS'] = 15                     ; final_data[15] = None
 402     fmap['hsUnit'] = 16                 ; final_data[16] = 'in'
 403     fmap['baro'] = 17                   ; final_data[17] = None
 404     fmap['baroUnit'] = 18               ; final_data[18] = 'inHg'
 405     fmap['rH'] = 19                     ; final_data[19] = None
 406     fmap['windGustSpeedNum'] = 20       ; final_data[20] = None
 407     fmap['windGustSpeedNumUnit'] = 21   ; final_data[21] = 'mph'
 408     fmap['windGustDirNum'] = 22         ; final_data[22] = None
 409     fmap['dewPoint'] = 23               ; final_data[23] = None
 410     fmap['dewPointUnit'] = 24           ; final_data[24] = 'F'
 411     fmap['hn24Auto'] = 25               ; final_data[25] = None
 412     fmap['hn24AutoUnit'] = 26           ; final_data[26] = 'in'
 413     fmap['hstAuto'] = 27                ; final_data[27] = None
 414     fmap['hstAutoUnit'] = 28            ; final_data[28] = 'in'
 415
 416     return (fmap, final_data)
 417
 418 def setup_infoex_counterparts_mapping(provider):
 419     """
 420     Create a mapping of the NRCS/MesoWest fields that this program supports to
 421     their InfoEx counterparts
 422     """
 423     iemap = {}
 424
 425     if provider == 'nrcs':
 426         iemap['PREC'] = 'precipitationGauge'
 427         iemap['TOBS'] = 'tempPres'
 428         iemap['TMAX'] = 'tempMaxHour'
 429         iemap['TMIN'] = 'tempMinHour'
 430         iemap['SNWD'] = 'hS'
 431         iemap['PRES'] = 'baro'
 432         iemap['RHUM'] = 'rH'
 433         iemap['WSPD'] = 'windSpeedNum'
 434         iemap['WDIR'] = 'windDirectionNum'
 435         # unsupported by NRCS:
 436         # windGustSpeedNum
 437
 438         # NOTE: this doesn't exist in NRCS SNOTEL, we create it in this
 439         #       program, so add it to the map here
 440         iemap['hn24'] = 'hn24Auto'
 441     elif provider == 'mesowest':
 442         iemap['precip_accum'] = 'precipitationGauge'
 443         iemap['air_temp'] = 'tempPres'
 444         iemap['air_temp_high_24_hour'] = 'tempMaxHour'
 445         iemap['air_temp_low_24_hour'] = 'tempMinHour'
 446         iemap['snow_depth'] = 'hS'
 447         iemap['pressure'] = 'baro'
 448         iemap['relative_humidity'] = 'rH'
 449         iemap['wind_speed'] = 'windSpeedNum'
 450         iemap['wind_direction'] = 'windDirectionNum'
 451         iemap['wind_gust'] = 'windGustSpeedNum'
 452
 453         # NOTE: this doesn't exist in MesoWest, we create it in this
 454         #       program, so add it to the map here
 455         iemap['hn24'] = 'hn24Auto'
 456     elif provider == 'python':
 457         # we expect Python programs to use the InfoEx data type names
 458         iemap['precipitationGauge'] = 'precipitationGauge'
 459         iemap['tempPres'] = 'tempPres'
 460         iemap['tempMaxHour'] = 'tempMaxHour'
 461         iemap['tempMinHour'] = 'tempMinHour'
 462         iemap['hS'] = 'hS'
 463         iemap['baro'] = 'baro'
 464         iemap['rH'] = 'rH'
 465         iemap['windSpeedNum'] = 'windSpeedNum'
 466         iemap['windDirectionNum'] = 'windDirectionNum'
 467         iemap['windGustSpeedNum'] = 'windGustSpeedNum'
 468
 469     return iemap
 470
 471 # provider-specific operations
 472 def get_nrcs_data(begin, end, station):
 473     """get the data we're after from the NRCS WSDL"""
 474     transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
 475     transport.session.verify = False
 476     client = zeep.Client(wsdl=station['source'], transport=transport)
 477     remote_data = {}
 478
 479     # massage begin/end date format
 480     begin_date_str = begin.strftime('%Y-%m-%d %H:%M:00')
 481     end_date_str = end.strftime('%Y-%m-%d %H:%M:00')
 482
 483     for element_cd in station['desired_data']:
 484         time_element = time.time()
 485
 486         # get the last three hours of data for this elementCd/element_cd
 487         tmp = client.service.getHourlyData(
 488             stationTriplets=[station['station_id']],
 489             elementCd=element_cd,
 490             ordinal=1,
 491             beginDate=begin_date_str,
 492             endDate=end_date_str)
 493
 494         LOG.info("Time to get NRCS elementCd '%s': %.3f sec", element_cd,
 495                  time.time() - time_element)
 496
 497         values = tmp[0]['values']
 498
 499         # sort and isolate the most recent
 500         #
 501         # NOTE: we do this because sometimes there are gaps in hourly data
 502         #       in NRCS; yes, we may end up with slightly inaccurate data,
 503         #       so perhaps this decision will be re-evaluated in the future
 504         if values:
 505             ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
 506             remote_data[element_cd] = ordered[0]['value']
 507         else:
 508             remote_data[element_cd] = None
 509
 510
 511         # calc hn24, if applicable
 512         hn24 = None
 513
 514         if station['hn24']:
 515             hn24_values = []
 516
 517             if element_cd == "SNWD":
 518                 for idx, _ in enumerate(values):
 519                     val = values[idx]
 520                     if val is None:
 521                         continue
 522                     hn24_values.append(val['value'])
 523
 524                 if len(hn24_values) > 0:
 525                     # instead of taking MAX - MIN, we want the first
 526                     # value (most distant) - the last value (most
 527                     # recent)
 528                     #
 529                     # if the result is positive, then we have
 530                     # settlement; if it's not, then we have HN24
 531                     hn24 = hn24_values[0] - hn24_values[len(hn24_values)-1]
 532
 533                     if hn24 < 0.0:
 534                         hn24 = abs(hn24)
 535                     else:
 536                         # this case represents HS settlement
 537                         hn24 = 0.0
 538
 539             # finally, if user wants hn24 and it's set to None at this
 540             # point, then force it to 0.0
 541             if hn24 is None:
 542                 hn24 = 0.0
 543
 544         if hn24 is not None:
 545             if station['hn24']:
 546                 remote_data['hn24'] = hn24
 547
 548     return remote_data
 549
 550 def get_mesowest_data(begin, end, station):
 551     """get the data we're after from the MesoWest/Synoptic API"""
 552     remote_data = {}
 553
 554     # massage begin/end date format
 555     begin_date_str = begin.strftime('%Y%m%d%H%M')
 556     end_date_str = end.strftime('%Y%m%d%H%M')
 557
 558     # construct final, completed API URL
 559     api_req_url = station['source'] + '&start=' + begin_date_str + '&end=' + end_date_str
 560
 561     try:
 562         req = requests.get(api_req_url)
 563     except requests.exceptions.ConnectionError:
 564         LOG.error("Could not connect to '%s'", api_req_url)
 565         sys.exit(1)
 566
 567     try:
 568         json = req.json()
 569     except ValueError:
 570         LOG.error("Bad JSON in MesoWest response")
 571         sys.exit(1)
 572
 573     try:
 574         observations = json['STATION'][0]['OBSERVATIONS']
 575     except KeyError as exc:
 576         LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
 577         sys.exit(1)
 578     except IndexError as exc:
 579         LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
 580         try:
 581             LOG.error("Detailed MesoWest response: '%s'",
 582                       json['SUMMARY']['RESPONSE_MESSAGE'])
 583         except KeyError:
 584             pass
 585         sys.exit(1)
 586     except ValueError as exc:
 587         LOG.error("Bad JSON in MesoWest response: '%s'", exc)
 588         sys.exit(1)
 589
 590     # pos represents the last item in the array, aka the most recent
 591     pos = len(observations['date_time']) - 1
 592
 593     # while these values only apply in certain cases, init them here
 594     wind_speed_values = []
 595     wind_gust_speed_values = []
 596     wind_direction_values = []
 597     hn24_values = []
 598
 599     # results
 600     wind_speed_avg = None
 601     wind_gust_speed_avg = None
 602     wind_direction_avg = None
 603     hn24 = None
 604
 605     for element_cd in station['desired_data'].split(','):
 606         # sort and isolate the most recent, see note above in NRCS for how and
 607         # why this is done
 608         #
 609         # NOTE: Unlike in the NRCS case, the MesoWest API response contains all
 610         #       data (whereas with NRCS, we have to make a separate request for
 611         #       each element we want). This is nice for network efficiency but
 612         #       it means we have to handle this part differently for each.
 613         #
 614         # NOTE: Also unlike NRCS, MesoWest provides more granular data; NRCS
 615         #       provides hourly data, but MesoWest can often provide data every
 616         #       10 minutes -- though this provides more opportunity for
 617         #       irregularities
 618
 619         # we may not have the data at all
 620         key_name = element_cd + '_set_1'
 621
 622         if key_name in observations:
 623             # val is what will make it into the dataset, after
 624             # conversions... it gets defined here because in certain
 625             # cases we need to look at all of the data to calculate HN24
 626             # or wind averages, but for the rest of the data, we only
 627             # take the most recent
 628             val = None
 629
 630             # loop through all observations for this key_name
 631             # record relevant values for wind averaging or hn24, but
 632             # otherwise only persist the data if it's the last datum in
 633             # the set
 634             for idx, _ in enumerate(observations[key_name]):
 635                 val = observations[key_name][idx]
 636
 637                 # skip bunk vals
 638                 if val is None:
 639                     continue
 640
 641                 # mesowest by default provides wind_speed in m/s, but
 642                 # we specify 'english' units in the request; either way,
 643                 # we want mph
 644                 if element_cd in ('wind_speed', 'wind_gust'):
 645                     val = kn_to_mph(val)
 646
 647                 # mesowest provides HS in mm, not cm; we want cm
 648                 if element_cd == 'snow_depth' and station['units'] == 'metric':
 649                     val = mm_to_cm(val)
 650
 651                 # HN24 / wind_mode transformations, once the data has
 652                 # completed unit conversions
 653                 if station['wind_mode'] == "average":
 654                     if element_cd == 'wind_speed' and val is not None:
 655                         wind_speed_values.append(val)
 656                     elif element_cd == 'wind_gust' and val is not None:
 657                         wind_gust_speed_values.append(val)
 658                     elif element_cd == 'wind_direction' and val is not None:
 659                         wind_direction_values.append(val)
 660
 661                 if element_cd == 'snow_depth':
 662                     hn24_values.append(val)
 663
 664                 # again, only persist this datum to the final data if
 665                 # it's from the most recent date
 666                 if idx == pos:
 667                     remote_data[element_cd] = val
 668
 669             # ensure that the data is filled out
 670             if not observations[key_name][pos]:
 671                 remote_data[element_cd] = None
 672         else:
 673             remote_data[element_cd] = None
 674
 675     if len(hn24_values) > 0:
 676         # instead of taking MAX - MIN, we want the first value (most
 677         # distant) - the last value (most recent)
 678         #
 679         # if the result is positive, then we have settlement; if it's not,
 680         # then we have HN24
 681         hn24 = hn24_values[0] - hn24_values[len(hn24_values)-1]
 682
 683         if hn24 < 0.0:
 684             hn24 = abs(hn24)
 685         else:
 686             # this case represents HS settlement
 687             hn24 = 0.0
 688
 689
 690     # finally, if user wants hn24 and it's set to None at this
 691     # point, then force it to 0.0
 692     if station['hn24'] and hn24 is None:
 693         hn24 = 0.0
 694
 695     if len(wind_speed_values) > 0:
 696         wind_speed_avg = sum(wind_speed_values) / len(wind_speed_values)
 697
 698     if len(wind_gust_speed_values) > 0:
 699         wind_gust_speed_avg = sum(wind_gust_speed_values) / len(wind_gust_speed_values)
 700
 701     if len(wind_direction_values) > 0:
 702         wind_direction_avg = sum(wind_direction_values) / len(wind_direction_values)
 703
 704     if hn24 is not None:
 705         if station['hn24']:
 706             remote_data['hn24'] = hn24
 707
 708     # overwrite the following with the respective averages, if
 709     # applicable
 710     if wind_speed_avg is not None:
 711         remote_data['wind_speed'] = wind_speed_avg
 712
 713     if wind_gust_speed_avg is not None:
 714         remote_data['wind_gust'] = wind_gust_speed_avg
 715
 716     if wind_direction_avg is not None:
 717         remote_data['wind_direction'] = wind_direction_avg
 718
 719     return remote_data
 720
 721 def switch_units_to_metric(data_map, mapping):
 722     """replace units with metric counterparts"""
 723
 724     # NOTE: to update this, use the fmap<->final_data mapping laid out
 725     #       in setup_infoex_fields_mapping ()
 726     data_map[mapping['tempMaxHourUnit']] = 'C'
 727     data_map[mapping['tempMinHourUnit']] = 'C'
 728     data_map[mapping['tempPresUnit']] = 'C'
 729     data_map[mapping['precipitationGaugeUnit']] = 'mm'
 730     data_map[mapping['hsUnit']] = 'cm'
 731     data_map[mapping['windSpeedUnit']] = 'm/s'
 732     data_map[mapping['windGustSpeedNumUnit']] = 'm/s'
 733     data_map[mapping['dewPointUnit']] = 'C'
 734     data_map[mapping['hn24AutoUnit']] = 'cm'
 735     data_map[mapping['hstAutoUnit']] = 'cm'
 736
 737     return data_map
 738
 739 def convert_nrcs_units_to_metric(element_cd, value):
 740     """convert NRCS values from English to metric"""
 741     if element_cd == 'TOBS':
 742         value = f_to_c(value)
 743     elif element_cd == 'SNWD':
 744         value = in_to_cm(value)
 745     elif element_cd == 'PREC':
 746         value = in_to_mm(value)
 747     return value
 748
 749 # CSV operations
 750 def write_local_csv(path_to_file, data):
 751     """Write the specified CSV file to disk"""
 752     with open(path_to_file, 'w') as file_object:
 753         # The requirement is that empty values are represented in the CSV
 754         # file as "", csv.QUOTE_NONNUMERIC achieves that
 755         LOG.debug("writing CSV file '%s'", path_to_file)
 756         writer = csv.writer(file_object, quoting=csv.QUOTE_NONNUMERIC)
 757         writer.writerow(data)
 758         file_object.close()
 759     return True
 760
 761 def upload_csv(path_to_file, infoex_data):
 762     """Upload the specified CSV file to InfoEx FTP and remove the file"""
 763     with open(path_to_file, 'rb') as file_object:
 764         LOG.debug("uploading FTP file '%s'", infoex_data['host'])
 765         ftp = FTP(infoex_data['host'], infoex_data['uuid'],
 766                   infoex_data['api_key'])
 767         ftp.storlines('STOR ' + path_to_file, file_object)
 768         ftp.close()
 769         file_object.close()
 770     os.remove(path_to_file)
 771
 772 # other miscellaneous routines
 773 def setup_time_values(station):
 774     """establish time bounds of data request(s)"""
 775
 776     # default timezone to UTC (for MesoWest)
 777     tz = pytz.utc
 778
 779     # but for NRCS, use the config-specified timezone
 780     if station['provider'] == 'nrcs':
 781         tz = station['tz']
 782
 783     # floor time to nearest hour
 784     date_time = datetime.datetime.now(tz=tz)
 785     end_date = date_time - datetime.timedelta(minutes=date_time.minute % 60,
 786                                               seconds=date_time.second,
 787                                               microseconds=date_time.microsecond)
 788     begin_date = end_date - datetime.timedelta(hours=station['num_hrs_to_fetch'])
 789     return (begin_date, end_date)
 790
 791 def f_to_c(f):
 792     """convert Fahrenheit to Celsius"""
 793     return (float(f) - 32) * 5.0/9.0
 794
 795 def in_to_cm(inches):
 796     """convert inches to centimetrs"""
 797     return float(inches) * 2.54
 798
 799 def in_to_mm(inches):
 800     """convert inches to millimeters"""
 801     return (float(inches) * 2.54) * 10.0
 802
 803 def ms_to_mph(ms):
 804     """convert meters per second to miles per hour"""
 805     return ms * 2.236936
 806
 807 def kn_to_mph(kn):
 808     """convert knots to miles per hour"""
 809     return kn * 1.150779
 810
 811 def mm_to_cm(mm):
 812     """convert millimeters to centimetrs"""
 813     return mm / 10.0
 814
 815 if __name__ == "__main__":
 816     sys.exit(main())