infoex-autowx.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 """
   5 InfoEx <-> NRCS/MesoWest Auto Wx implementation
   6 Alexander Vasarab
   7 Wylark Mountaineering LLC
   8
   9 This program fetches data from either an NRCS SNOTEL site or MesoWest
  10 weather station and pushes it to InfoEx using the new automated weather
  11 system implementation.
  12
  13 It is designed to be run hourly, and it asks for the last three hours
  14 of data of each desired type, and selects the most recent one. This
  15 lends some resiliency to the process and helps ensure that we have a
  16 value to send, but it can lead to somewhat inconsistent/untruthful
  17 data if e.g. the HS is from the last hour but the tempPres is from two
  18 hours ago because the instrumentation had a hiccup. It's worth
  19 considering if this is a bug or a feature.
  20
  21 For more information, see file: README
  22 For licensing, see file: LICENSE
  23 """
  24
  25 import configparser
  26 import csv
  27 import datetime
  28 import logging
  29 import os
  30 import sys
  31 import time
  32 import urllib3
  33 import importlib.util
  34
  35 from ftplib import FTP
  36 from argparse import ArgumentParser
  37
  38 import pytz
  39
  40 import requests
  41
  42 import zeep
  43 import zeep.cache
  44 import zeep.transports
  45
  46 __version__ = '3.3.1'
  47
  48 LOG = logging.getLogger(__name__)
  49 LOG.setLevel(logging.NOTSET)
  50
  51 urllib3.disable_warnings()
  52
  53 def get_parser():
  54     """Return OptionParser for this program"""
  55     parser = ArgumentParser()
  56
  57     parser.add_argument("--version",
  58                         action="version",
  59                         version=__version__)
  60
  61     parser.add_argument("--config",
  62                         dest="config",
  63                         metavar="FILE",
  64                         help="location of config file")
  65
  66     parser.add_argument("--log-level",
  67                         dest="log_level",
  68                         default=None,
  69                         help="set the log level (debug, info, warning)")
  70
  71     parser.add_argument("--dry-run",
  72                         action="store_true",
  73                         dest="dry_run",
  74                         default=False,
  75                         help="fetch data but don't upload to InfoEx")
  76
  77     return parser
  78
  79 def setup_config(config):
  80     """Setup config variable based on values specified in the ini file"""
  81     try:
  82         infoex = {
  83             'host': config['infoex']['host'],
  84             'uuid': config['infoex']['uuid'],
  85             'api_key': config['infoex']['api_key'],
  86             'csv_filename': config['infoex']['csv_filename'],
  87             'location_uuid': config['infoex']['location_uuid'],
  88             'wx_data': {}, # placeholder key, values to come later
  89         }
  90
  91         station = dict()
  92         station['provider'] = config['station']['type']
  93
  94         if station['provider'] not in ['nrcs', 'mesowest', 'python']:
  95             print("Please specify either nrcs or mesowest as the station type.")
  96             sys.exit(1)
  97
  98         if station['provider'] == 'nrcs':
  99             station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
 100             station['station_id'] = config['station']['station_id']
 101             station['desired_data'] = config['station']['desired_data'].split(',')
 102             station['units'] = config['station']['units']
 103
 104         if station['provider'] == 'mesowest':
 105             station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
 106             station['station_id'] = config['station']['station_id']
 107             station['units'] = config['station']['units']
 108             station['desired_data'] = config['station']['desired_data']
 109
 110             # construct full API URL (sans start/end time, added later)
 111             station['source'] = station['source'] + '?token=' + \
 112                                 config['station']['token'] + \
 113                                 '&within=60&units=' + station['units'] + \
 114                                 '&stid=' + station['station_id'] + \
 115                                 '&vars=' + station['desired_data']
 116
 117         if station['provider'] == 'python':
 118             station['path'] = config['station']['path']
 119
 120         tz = 'America/Los_Angeles'
 121
 122         if 'tz' in config['station']:
 123             tz = config['station']['tz']
 124
 125         try:
 126             station['tz'] = pytz.timezone(tz)
 127         except pytz.exceptions.UnknownTimeZoneError:
 128             LOG.critical("%s is not a valid timezone", tz)
 129             sys.exit(1)
 130
 131         # By default, fetch three hours of data
 132         #
 133         # If user wants hn24 or wind averaging, then
 134         # we need more.
 135         station['num_hrs_to_fetch'] = 3
 136
 137         # HN24
 138         if 'hn24' in config['station']:
 139             if config['station']['hn24'] not in ['true', 'false']:
 140                 raise ValueError("hn24 must be either 'true' or 'false'")
 141
 142             if config['station']['hn24'] == "true":
 143                 station['hn24'] = True
 144                 station['num_hrs_to_fetch'] = 24
 145             else:
 146                 station['hn24'] = False
 147         else:
 148             # default to False
 149             station['hn24'] = False
 150
 151         # Wind mode
 152         if 'wind_mode' in config['station']:
 153             if config['station']['wind_mode'] not in ['normal', 'average']:
 154                 raise ValueError("wind_mode must be either 'normal' or 'average'")
 155
 156             station['wind_mode'] = config['station']['wind_mode']
 157
 158             if station['wind_mode'] == "average":
 159                 station['num_hrs_to_fetch'] = 24
 160         else:
 161             # default to False
 162             station['wind_mode'] = "normal"
 163
 164     except KeyError as err:
 165         LOG.critical("%s not defined in configuration file", err)
 166         sys.exit(1)
 167     except ValueError as err:
 168         LOG.critical("%s", err)
 169         sys.exit(1)
 170
 171     # all sections/values present in config file, final sanity check
 172     try:
 173         for key in config.sections():
 174             for subkey in config[key]:
 175                 if not config[key][subkey]:
 176                     raise ValueError
 177     except ValueError:
 178         LOG.critical("Config value '%s.%s' is empty", key, subkey)
 179         sys.exit(1)
 180
 181     return (infoex, station)
 182
 183 def setup_logging(log_level):
 184     """Setup our logging infrastructure"""
 185     try:
 186         from systemd.journal import JournalHandler
 187         LOG.addHandler(JournalHandler())
 188     except ImportError:
 189         ## fallback to syslog
 190         #import logging.handlers
 191         #LOG.addHandler(logging.handlers.SysLogHandler())
 192         # fallback to stdout
 193         handler = logging.StreamHandler(sys.stdout)
 194         formatter = logging.Formatter('%(asctime)s.%(msecs)03d '
 195                                       '%(levelname)s %(module)s - '
 196                                       '%(funcName)s: %(message)s',
 197                                       '%Y-%m-%d %H:%M:%S')
 198         handler.setFormatter(formatter)
 199         LOG.addHandler(handler)
 200
 201     # ugly, but passable
 202     if log_level in [None, 'debug', 'info', 'warning']:
 203         if log_level == 'debug':
 204             LOG.setLevel(logging.DEBUG)
 205         elif log_level == 'info':
 206             LOG.setLevel(logging.INFO)
 207         elif log_level == 'warning':
 208             LOG.setLevel(logging.WARNING)
 209         else:
 210             LOG.setLevel(logging.NOTSET)
 211     else:
 212         return False
 213
 214     return True
 215
 216 def main():
 217     """Main routine: sort through args, decide what to do, then do it"""
 218     parser = get_parser()
 219     options = parser.parse_args()
 220
 221     config = configparser.ConfigParser(allow_no_value=False)
 222
 223     if not options.config:
 224         parser.print_help()
 225         print("\nPlease specify a configuration file via --config.")
 226         sys.exit(1)
 227
 228     config.read(options.config)
 229
 230     if not setup_logging(options.log_level):
 231         parser.print_help()
 232         print("\nPlease select an appropriate log level or remove the switch (--log-level).")
 233         sys.exit(1)
 234
 235     (infoex, station) = setup_config(config)
 236
 237     LOG.debug('Config parsed, starting up')
 238
 239     # create mappings
 240     (fmap, final_data) = setup_infoex_fields_mapping(infoex['location_uuid'])
 241     iemap = setup_infoex_counterparts_mapping(station['provider'])
 242
 243     # override units if user selected metric
 244     if station['provider'] != 'python' and station['units'] == 'metric':
 245         final_data = switch_units_to_metric(final_data, fmap)
 246
 247     (begin_date, end_date) = setup_time_values(station)
 248
 249     if station['provider'] == 'python':
 250         LOG.debug("Getting custom data from external Python program")
 251     else:
 252         LOG.debug("Getting %s data from %s to %s (%s)",
 253                   str(station['desired_data']),
 254                   str(begin_date), str(end_date), end_date.tzinfo.zone)
 255
 256     time_all_elements = time.time()
 257
 258     # get the data
 259     if station['provider'] == 'nrcs':
 260         infoex['wx_data'] = get_nrcs_data(begin_date, end_date, station)
 261     elif station['provider'] == 'mesowest':
 262         infoex['wx_data'] = get_mesowest_data(begin_date, end_date,
 263                                               station)
 264     elif station['provider'] == 'python':
 265         try:
 266             spec = importlib.util.spec_from_file_location('custom_wx',
 267                                                           station['path'])
 268             mod = importlib.util.module_from_spec(spec)
 269             spec.loader.exec_module(mod)
 270             mod.LOG = LOG
 271
 272             try:
 273                 infoex['wx_data'] = mod.get_custom_data()
 274
 275                 if infoex['wx_data'] is None:
 276                     infoex['wx_data'] = []
 277             except Exception as exc:
 278                 LOG.error("Python program for custom Wx data failed in "
 279                           "execution: %s", str(exc))
 280                 sys.exit(1)
 281
 282             LOG.info("Successfully executed external Python program")
 283         except ImportError:
 284             LOG.error("Please upgrade to Python 3.3 or later")
 285             sys.exit(1)
 286         except FileNotFoundError:
 287             LOG.error("Specified Python program for custom Wx data "
 288                       "was not found")
 289             sys.exit(1)
 290         except Exception as exc:
 291             LOG.error("A problem was encountered when attempting to "
 292                       "load your custom Wx program: %s", str(exc))
 293             sys.exit(1)
 294
 295     LOG.info("Time taken to get all data : %.3f sec", time.time() -
 296              time_all_elements)
 297
 298     LOG.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
 299
 300     # timezone massaging
 301     final_end_date = end_date.astimezone(station['tz'])
 302
 303     # Now we only need to add in what we want to change thanks to that
 304     # abomination of a variable declaration earlier
 305     final_data[fmap['Location UUID']] = infoex['location_uuid']
 306     final_data[fmap['obDate']] = final_end_date.strftime('%m/%d/%Y')
 307     final_data[fmap['obTime']] = final_end_date.strftime('%H:%M')
 308     final_data[fmap['timeZone']] = station['tz'].zone
 309
 310     for element_cd in infoex['wx_data']:
 311         if element_cd not in iemap:
 312             LOG.warning("BAD KEY wx_data['%s']", element_cd)
 313             continue
 314
 315         if infoex['wx_data'][element_cd] is None:
 316             continue
 317
 318         # do the conversion before the rounding
 319         if station['provider'] == 'nrcs' and station['units'] == 'metric':
 320             infoex['wx_data'][element_cd] = convert_nrcs_units_to_metric(element_cd, infoex['wx_data'][element_cd])
 321
 322         # Massage precision of certain values to fit InfoEx's
 323         # expectations
 324         #
 325         # 0 decimal places: relative humidity, wind speed, wind
 326         #                   direction, wind gust, snow depth
 327         # 1 decimal place:  air temp, baro
 328         # Avoid transforming None values
 329         if element_cd in ['wind_speed', 'WSPD', 'wind_direction',
 330                           'RHUM', 'relative_humidity', 'WDIR',
 331                           'wind_gust', 'SNWD', 'snow_depth',
 332                           'hn24']:
 333             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd])
 334         elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']:
 335             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1)
 336         elif element_cd in ['PREC', 'precip_accum']:
 337             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 2)
 338
 339         # CONSIDER: Casting every value to Float() -- need to investigate if
 340         #           any possible elementCds we may want are any other data
 341         #           type than float.
 342         #
 343         #           Another possibility is to query the API with
 344         #           getStationElements and temporarily store the
 345         #           storedUnitCd. But that's pretty network-intensive and
 346         #           may not even be worth it if there's only e.g. one or two
 347         #           exceptions to any otherwise uniformly Float value set.
 348         final_data[fmap[iemap[element_cd]]] = infoex['wx_data'][element_cd]
 349
 350     LOG.debug("final_data: %s", str(final_data))
 351
 352     if infoex['wx_data']:
 353         if not write_local_csv(infoex['csv_filename'], final_data):
 354             LOG.warning('Could not write local CSV file: %s',
 355                         infoex['csv_filename'])
 356             return 1
 357
 358         if not options.dry_run:
 359             upload_csv(infoex['csv_filename'], infoex)
 360
 361     LOG.debug('DONE')
 362     return 0
 363
 364 # data structure operations
 365 def setup_infoex_fields_mapping(location_uuid):
 366     """
 367     Create a mapping of InfoEx fields to the local data's indexing scheme.
 368
 369     INFOEX FIELDS
 370
 371     This won't earn style points in Python, but here we establish a couple
 372     of helpful mappings variables. The reason this is helpful is that the
 373     end result is simply an ordered set, the CSV file. But we still may
 374     want to manipulate the values arbitrarily before writing that file.
 375
 376     Also note that the current Auto Wx InfoEx documentation shows these
 377     keys in a graphical table with the "index" beginning at 1, but here we
 378     sanely index beginning at 0.
 379     """
 380     # pylint: disable=too-many-statements,multiple-statements,bad-whitespace
 381     fmap = {}                           ; final_data     = [None] * 29
 382     fmap['Location UUID'] = 0           ; final_data[0]  = location_uuid
 383     fmap['obDate'] = 1                  ; final_data[1]  = None
 384     fmap['obTime'] = 2                  ; final_data[2]  = None
 385     fmap['timeZone'] = 3                ; final_data[3]  = 'Pacific'
 386     fmap['tempMaxHour'] = 4             ; final_data[4]  = None
 387     fmap['tempMaxHourUnit'] = 5         ; final_data[5]  = 'F'
 388     fmap['tempMinHour'] = 6             ; final_data[6]  = None
 389     fmap['tempMinHourUnit'] = 7         ; final_data[7]  = 'F'
 390     fmap['tempPres'] = 8                ; final_data[8]  = None
 391     fmap['tempPresUnit'] = 9            ; final_data[9]  = 'F'
 392     fmap['precipitationGauge'] = 10     ; final_data[10] = None
 393     fmap['precipitationGaugeUnit'] = 11 ; final_data[11] = 'in'
 394     fmap['windSpeedNum'] = 12           ; final_data[12] = None
 395     fmap['windSpeedUnit'] = 13          ; final_data[13] = 'mph'
 396     fmap['windDirectionNum'] = 14       ; final_data[14] = None
 397     fmap['hS'] = 15                     ; final_data[15] = None
 398     fmap['hsUnit'] = 16                 ; final_data[16] = 'in'
 399     fmap['baro'] = 17                   ; final_data[17] = None
 400     fmap['baroUnit'] = 18               ; final_data[18] = 'inHg'
 401     fmap['rH'] = 19                     ; final_data[19] = None
 402     fmap['windGustSpeedNum'] = 20       ; final_data[20] = None
 403     fmap['windGustSpeedNumUnit'] = 21   ; final_data[21] = 'mph'
 404     fmap['windGustDirNum'] = 22         ; final_data[22] = None
 405     fmap['dewPoint'] = 23               ; final_data[23] = None
 406     fmap['dewPointUnit'] = 24           ; final_data[24] = 'F'
 407     fmap['hn24Auto'] = 25               ; final_data[25] = None
 408     fmap['hn24AutoUnit'] = 26           ; final_data[26] = 'in'
 409     fmap['hstAuto'] = 27                ; final_data[27] = None
 410     fmap['hstAutoUnit'] = 28            ; final_data[28] = 'in'
 411
 412     return (fmap, final_data)
 413
 414 def setup_infoex_counterparts_mapping(provider):
 415     """
 416     Create a mapping of the NRCS/MesoWest fields that this program supports to
 417     their InfoEx counterparts
 418     """
 419     iemap = {}
 420
 421     if provider == 'nrcs':
 422         iemap['PREC'] = 'precipitationGauge'
 423         iemap['TOBS'] = 'tempPres'
 424         iemap['TMAX'] = 'tempMaxHour'
 425         iemap['TMIN'] = 'tempMinHour'
 426         iemap['SNWD'] = 'hS'
 427         iemap['PRES'] = 'baro'
 428         iemap['RHUM'] = 'rH'
 429         iemap['WSPD'] = 'windSpeedNum'
 430         iemap['WDIR'] = 'windDirectionNum'
 431         # unsupported by NRCS:
 432         # windGustSpeedNum
 433
 434         # NOTE: this doesn't exist in NRCS SNOTEL, we create it in this
 435         #       program, so add it to the map here
 436         iemap['hn24'] = 'hn24Auto'
 437     elif provider == 'mesowest':
 438         iemap['precip_accum'] = 'precipitationGauge'
 439         iemap['air_temp'] = 'tempPres'
 440         iemap['air_temp_high_24_hour'] = 'tempMaxHour'
 441         iemap['air_temp_low_24_hour'] = 'tempMinHour'
 442         iemap['snow_depth'] = 'hS'
 443         iemap['pressure'] = 'baro'
 444         iemap['relative_humidity'] = 'rH'
 445         iemap['wind_speed'] = 'windSpeedNum'
 446         iemap['wind_direction'] = 'windDirectionNum'
 447         iemap['wind_gust'] = 'windGustSpeedNum'
 448
 449         # NOTE: this doesn't exist in MesoWest, we create it in this
 450         #       program, so add it to the map here
 451         iemap['hn24'] = 'hn24Auto'
 452     elif provider == 'python':
 453         # we expect Python programs to use the InfoEx data type names
 454         iemap['precipitationGauge'] = 'precipitationGauge'
 455         iemap['tempPres'] = 'tempPres'
 456         iemap['tempMaxHour'] = 'tempMaxHour'
 457         iemap['tempMinHour'] = 'tempMinHour'
 458         iemap['hS'] = 'hS'
 459         iemap['baro'] = 'baro'
 460         iemap['rH'] = 'rH'
 461         iemap['windSpeedNum'] = 'windSpeedNum'
 462         iemap['windDirectionNum'] = 'windDirectionNum'
 463         iemap['windGustSpeedNum'] = 'windGustSpeedNum'
 464
 465     return iemap
 466
 467 # provider-specific operations
 468 def get_nrcs_data(begin, end, station):
 469     """get the data we're after from the NRCS WSDL"""
 470     transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
 471     transport.session.verify = False
 472     client = zeep.Client(wsdl=station['source'], transport=transport)
 473     remote_data = {}
 474
 475     # massage begin/end date format
 476     begin_date_str = begin.strftime('%Y-%m-%d %H:%M:00')
 477     end_date_str = end.strftime('%Y-%m-%d %H:%M:00')
 478
 479     for element_cd in station['desired_data']:
 480         time_element = time.time()
 481
 482         # get the last three hours of data for this elementCd/element_cd
 483         tmp = client.service.getHourlyData(
 484             stationTriplets=[station['station_id']],
 485             elementCd=element_cd,
 486             ordinal=1,
 487             beginDate=begin_date_str,
 488             endDate=end_date_str)
 489
 490         LOG.info("Time to get NRCS elementCd '%s': %.3f sec", element_cd,
 491                  time.time() - time_element)
 492
 493         values = tmp[0]['values']
 494
 495         # sort and isolate the most recent
 496         #
 497         # NOTE: we do this because sometimes there are gaps in hourly data
 498         #       in NRCS; yes, we may end up with slightly inaccurate data,
 499         #       so perhaps this decision will be re-evaluated in the future
 500         if values:
 501             ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
 502             remote_data[element_cd] = ordered[0]['value']
 503         else:
 504             remote_data[element_cd] = None
 505
 506
 507         # calc hn24, if applicable
 508         hn24 = None
 509
 510         if station['hn24']:
 511             hn24_values = []
 512
 513             if element_cd == "SNWD":
 514                 for idx, _ in enumerate(values):
 515                     val = values[idx]
 516                     if val is None:
 517                         continue
 518                     hn24_values.append(val['value'])
 519
 520                 if len(hn24_values) > 0:
 521                     # instead of taking MAX - MIN, we want the first
 522                     # value (most distant) - the last value (most
 523                     # recent)
 524                     #
 525                     # if the result is positive, then we have
 526                     # settlement; if it's not, then we have HN24
 527                     hn24 = hn24_values[0] - hn24_values[len(hn24_values)-1]
 528
 529                     if hn24 < 0.0:
 530                         hn24 = abs(hn24)
 531                     else:
 532                         # this case represents HS settlement
 533                         hn24 = 0.0
 534
 535             # finally, if user wants hn24 and it's set to None at this
 536             # point, then force it to 0.0
 537             if hn24 is None:
 538                 hn24 = 0.0
 539
 540         if hn24 is not None:
 541             remote_data['hn24'] = hn24
 542
 543     return remote_data
 544
 545 def get_mesowest_data(begin, end, station):
 546     """get the data we're after from the MesoWest/Synoptic API"""
 547     remote_data = {}
 548
 549     # massage begin/end date format
 550     begin_date_str = begin.strftime('%Y%m%d%H%M')
 551     end_date_str = end.strftime('%Y%m%d%H%M')
 552
 553     # construct final, completed API URL
 554     api_req_url = station['source'] + '&start=' + begin_date_str + '&end=' + end_date_str
 555
 556     try:
 557         req = requests.get(api_req_url)
 558     except requests.exceptions.ConnectionError:
 559         LOG.error("Could not connect to '%s'", api_req_url)
 560         sys.exit(1)
 561
 562     try:
 563         json = req.json()
 564     except ValueError:
 565         LOG.error("Bad JSON in MesoWest response")
 566         sys.exit(1)
 567
 568     try:
 569         observations = json['STATION'][0]['OBSERVATIONS']
 570     except KeyError as exc:
 571         LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
 572         sys.exit(1)
 573     except IndexError as exc:
 574         LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
 575         try:
 576             LOG.error("Detailed MesoWest response: '%s'",
 577                       json['SUMMARY']['RESPONSE_MESSAGE'])
 578         except KeyError:
 579             pass
 580         sys.exit(1)
 581     except ValueError as exc:
 582         LOG.error("Bad JSON in MesoWest response: '%s'", exc)
 583         sys.exit(1)
 584
 585     # pos represents the last item in the array, aka the most recent
 586     pos = len(observations['date_time']) - 1
 587
 588     # while these values only apply in certain cases, init them here
 589     wind_speed_values = []
 590     wind_gust_speed_values = []
 591     wind_direction_values = []
 592     hn24_values = []
 593
 594     # results
 595     wind_speed_avg = None
 596     wind_gust_speed_avg = None
 597     wind_direction_avg = None
 598     hn24 = None
 599
 600     for element_cd in station['desired_data'].split(','):
 601         # sort and isolate the most recent, see note above in NRCS for how and
 602         # why this is done
 603         #
 604         # NOTE: Unlike in the NRCS case, the MesoWest API response contains all
 605         #       data (whereas with NRCS, we have to make a separate request for
 606         #       each element we want). This is nice for network efficiency but
 607         #       it means we have to handle this part differently for each.
 608         #
 609         # NOTE: Also unlike NRCS, MesoWest provides more granular data; NRCS
 610         #       provides hourly data, but MesoWest can often provide data every
 611         #       10 minutes -- though this provides more opportunity for
 612         #       irregularities
 613
 614         # we may not have the data at all
 615         key_name = element_cd + '_set_1'
 616
 617         if key_name in observations:
 618             # val is what will make it into the dataset, after
 619             # conversions... it gets defined here because in certain
 620             # cases we need to look at all of the data to calculate HN24
 621             # or wind averages, but for the rest of the data, we only
 622             # take the most recent
 623             val = None
 624
 625             # loop through all observations for this key_name
 626             # record relevant values for wind averaging or hn24, but
 627             # otherwise only persist the data if it's the last datum in
 628             # the set
 629             for idx, _ in enumerate(observations[key_name]):
 630                 val = observations[key_name][idx]
 631
 632                 # skip bunk vals
 633                 if val is None:
 634                     continue
 635
 636                 # mesowest by default provides wind_speed in m/s, but
 637                 # we specify 'english' units in the request; either way,
 638                 # we want mph
 639                 if element_cd in ('wind_speed', 'wind_gust'):
 640                     val = kn_to_mph(val)
 641
 642                 # mesowest provides HS in mm, not cm; we want cm
 643                 if element_cd == 'snow_depth' and station['units'] == 'metric':
 644                     val = mm_to_cm(val)
 645
 646                 # HN24 / wind_mode transformations, once the data has
 647                 # completed unit conversions
 648                 if station['wind_mode'] == "average":
 649                     if element_cd == 'wind_speed' and val is not None:
 650                         wind_speed_values.append(val)
 651                     elif element_cd == 'wind_gust' and val is not None:
 652                         wind_gust_speed_values.append(val)
 653                     elif element_cd == 'wind_direction' and val is not None:
 654                         wind_direction_values.append(val)
 655
 656                 if element_cd == 'snow_depth':
 657                     hn24_values.append(val)
 658
 659                 # again, only persist this datum to the final data if
 660                 # it's from the most recent date
 661                 if idx == pos:
 662                     remote_data[element_cd] = val
 663
 664             # ensure that the data is filled out
 665             if not observations[key_name][pos]:
 666                 remote_data[element_cd] = None
 667         else:
 668             remote_data[element_cd] = None
 669
 670     if len(hn24_values) > 0:
 671         # instead of taking MAX - MIN, we want the first value (most
 672         # distant) - the last value (most recent)
 673         #
 674         # if the result is positive, then we have settlement; if it's not,
 675         # then we have HN24
 676         hn24 = hn24_values[0] - hn24_values[len(hn24_values)-1]
 677
 678         if hn24 < 0.0:
 679             hn24 = abs(hn24)
 680         else:
 681             # this case represents HS settlement
 682             hn24 = 0.0
 683
 684
 685     # finally, if user wants hn24 and it's set to None at this
 686     # point, then force it to 0.0
 687     if station['hn24'] and hn24 is None:
 688         hn24 = 0.0
 689
 690     if len(wind_speed_values) > 0:
 691         wind_speed_avg = sum(wind_speed_values) / len(wind_speed_values)
 692
 693     if len(wind_gust_speed_values) > 0:
 694         wind_gust_speed_avg = sum(wind_gust_speed_values) / len(wind_gust_speed_values)
 695
 696     if len(wind_direction_values) > 0:
 697         wind_direction_avg = sum(wind_direction_values) / len(wind_direction_values)
 698
 699     if hn24 is not None:
 700         remote_data['hn24'] = hn24
 701
 702     # overwrite the following with the respective averages, if
 703     # applicable
 704     if wind_speed_avg is not None:
 705         remote_data['wind_speed'] = wind_speed_avg
 706
 707     if wind_gust_speed_avg is not None:
 708         remote_data['wind_gust'] = wind_gust_speed_avg
 709
 710     if wind_direction_avg is not None:
 711         remote_data['wind_direction'] = wind_direction_avg
 712
 713     return remote_data
 714
 715 def switch_units_to_metric(data_map, mapping):
 716     """replace units with metric counterparts"""
 717
 718     # NOTE: to update this, use the fmap<->final_data mapping laid out
 719     #       in setup_infoex_fields_mapping ()
 720     data_map[mapping['tempMaxHourUnit']] = 'C'
 721     data_map[mapping['tempMinHourUnit']] = 'C'
 722     data_map[mapping['tempPresUnit']] = 'C'
 723     data_map[mapping['precipitationGaugeUnit']] = 'mm'
 724     data_map[mapping['hsUnit']] = 'cm'
 725     data_map[mapping['windSpeedUnit']] = 'm/s'
 726     data_map[mapping['windGustSpeedNumUnit']] = 'm/s'
 727     data_map[mapping['dewPointUnit']] = 'C'
 728     data_map[mapping['hn24AutoUnit']] = 'cm'
 729     data_map[mapping['hstAutoUnit']] = 'cm'
 730
 731     return data_map
 732
 733 def convert_nrcs_units_to_metric(element_cd, value):
 734     """convert NRCS values from English to metric"""
 735     if element_cd == 'TOBS':
 736         value = f_to_c(value)
 737     elif element_cd == 'SNWD':
 738         value = in_to_cm(value)
 739     elif element_cd == 'PREC':
 740         value = in_to_mm(value)
 741     return value
 742
 743 # CSV operations
 744 def write_local_csv(path_to_file, data):
 745     """Write the specified CSV file to disk"""
 746     with open(path_to_file, 'w') as file_object:
 747         # The requirement is that empty values are represented in the CSV
 748         # file as "", csv.QUOTE_NONNUMERIC achieves that
 749         LOG.debug("writing CSV file '%s'", path_to_file)
 750         writer = csv.writer(file_object, quoting=csv.QUOTE_NONNUMERIC)
 751         writer.writerow(data)
 752         file_object.close()
 753     return True
 754
 755 def upload_csv(path_to_file, infoex_data):
 756     """Upload the specified CSV file to InfoEx FTP and remove the file"""
 757     with open(path_to_file, 'rb') as file_object:
 758         LOG.debug("uploading FTP file '%s'", infoex_data['host'])
 759         ftp = FTP(infoex_data['host'], infoex_data['uuid'],
 760                   infoex_data['api_key'])
 761         ftp.storlines('STOR ' + path_to_file, file_object)
 762         ftp.close()
 763         file_object.close()
 764     os.remove(path_to_file)
 765
 766 # other miscellaneous routines
 767 def setup_time_values(station):
 768     """establish time bounds of data request(s)"""
 769
 770     # default timezone to UTC (for MesoWest)
 771     tz = pytz.utc
 772
 773     # but for NRCS, use the config-specified timezone
 774     if station['provider'] == 'nrcs':
 775         tz = station['tz']
 776
 777     # floor time to nearest hour
 778     date_time = datetime.datetime.now(tz=tz)
 779     end_date = date_time - datetime.timedelta(minutes=date_time.minute % 60,
 780                                               seconds=date_time.second,
 781                                               microseconds=date_time.microsecond)
 782     begin_date = end_date - datetime.timedelta(hours=station['num_hrs_to_fetch'])
 783     return (begin_date, end_date)
 784
 785 def f_to_c(f):
 786     """convert Fahrenheit to Celsius"""
 787     return (float(f) - 32) * 5.0/9.0
 788
 789 def in_to_cm(inches):
 790     """convert inches to centimetrs"""
 791     return float(inches) * 2.54
 792
 793 def in_to_mm(inches):
 794     """convert inches to millimeters"""
 795     return (float(inches) * 2.54) * 10.0
 796
 797 def ms_to_mph(ms):
 798     """convert meters per second to miles per hour"""
 799     return ms * 2.236936
 800
 801 def kn_to_mph(kn):
 802     """convert knots to miles per hour"""
 803     return kn * 1.150779
 804
 805 def mm_to_cm(mm):
 806     """convert millimeters to centimetrs"""
 807     return mm / 10.0
 808
 809 if __name__ == "__main__":
 810     sys.exit(main())