infoex-autowx.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 """
   5 InfoEx <-> NRCS/MesoWest Auto Wx implementation
   6 Alexander Vasarab
   7 Wylark Mountaineering LLC
   8
   9 This program fetches data from either an NRCS SNOTEL site or MesoWest
  10 weather station and pushes it to InfoEx using the new automated weather
  11 system implementation.
  12
  13 It is designed to be run hourly, and it asks for the last three hours
  14 of data of each desired type, and selects the most recent one. This
  15 lends some resiliency to the process and helps ensure that we have a
  16 value to send, but it can lead to somewhat inconsistent/untruthful
  17 data if e.g. the HS is from the last hour but the tempPres is from two
  18 hours ago because the instrumentation had a hiccup. It's worth
  19 considering if this is a bug or a feature.
  20
  21 For more information, see file: README
  22 For licensing, see file: LICENSE
  23 """
  24
  25 import configparser
  26 import csv
  27 import datetime
  28 import logging
  29 import os
  30 import sys
  31 import time
  32 import urllib3
  33 import importlib.util
  34
  35 from ftplib import FTP
  36 from argparse import ArgumentParser
  37
  38 import pytz
  39
  40 import requests
  41
  42 import zeep
  43 import zeep.cache
  44 import zeep.transports
  45
  46 __version__ = '3.2.4'
  47
  48 LOG = logging.getLogger(__name__)
  49 LOG.setLevel(logging.NOTSET)
  50
  51 urllib3.disable_warnings()
  52
  53 def get_parser():
  54     """Return OptionParser for this program"""
  55     parser = ArgumentParser()
  56
  57     parser.add_argument("--version",
  58                         action="version",
  59                         version=__version__)
  60
  61     parser.add_argument("--config",
  62                         dest="config",
  63                         metavar="FILE",
  64                         help="location of config file")
  65
  66     parser.add_argument("--log-level",
  67                         dest="log_level",
  68                         default=None,
  69                         help="set the log level (debug, info, warning)")
  70
  71     parser.add_argument("--dry-run",
  72                         action="store_true",
  73                         dest="dry_run",
  74                         default=False,
  75                         help="fetch data but don't upload to InfoEx")
  76
  77     return parser
  78
  79 def setup_config(config):
  80     """Setup config variable based on values specified in the ini file"""
  81     try:
  82         infoex = {
  83             'host': config['infoex']['host'],
  84             'uuid': config['infoex']['uuid'],
  85             'api_key': config['infoex']['api_key'],
  86             'csv_filename': config['infoex']['csv_filename'],
  87             'location_uuid': config['infoex']['location_uuid'],
  88             'wx_data': {}, # placeholder key, values to come later
  89         }
  90
  91         station = dict()
  92         station['provider'] = config['station']['type']
  93
  94         if station['provider'] not in ['nrcs', 'mesowest', 'python']:
  95             print("Please specify either nrcs or mesowest as the station type.")
  96             sys.exit(1)
  97
  98         if station['provider'] == 'nrcs':
  99             station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
 100             station['station_id'] = config['station']['station_id']
 101             station['desired_data'] = config['station']['desired_data'].split(',')
 102             station['units'] = config['station']['units']
 103
 104         if station['provider'] == 'mesowest':
 105             station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
 106             station['station_id'] = config['station']['station_id']
 107             station['units'] = config['station']['units']
 108             station['desired_data'] = config['station']['desired_data']
 109
 110             # construct full API URL (sans start/end time, added later)
 111             station['source'] = station['source'] + '?token=' + \
 112                                 config['station']['token'] + \
 113                                 '&within=60&units=' + station['units'] + \
 114                                 '&stid=' + station['station_id'] + \
 115                                 '&vars=' + station['desired_data']
 116
 117         if station['provider'] == 'python':
 118             station['path'] = config['station']['path']
 119
 120         tz = 'America/Los_Angeles'
 121
 122         if 'tz' in config['station']:
 123             tz = config['station']['tz']
 124
 125         try:
 126             station['tz'] = pytz.timezone(tz)
 127         except pytz.exceptions.UnknownTimeZoneError:
 128             LOG.critical("%s is not a valid timezone", tz)
 129             sys.exit(1)
 130
 131         # By default, fetch three hours of data
 132         #
 133         # If user wants hn24 or wind averaging, then
 134         # we need more.
 135         station['num_hrs_to_fetch'] = 3
 136
 137         # HN24
 138         if 'hn24' in config['station']:
 139             if config['station']['hn24'] not in ['true', 'false']:
 140                 raise ValueError("hn24 must be either 'true' or 'false'")
 141
 142             if config['station']['hn24'] == "true":
 143                 station['hn24'] = True
 144                 station['num_hrs_to_fetch'] = 24
 145             else:
 146                 station['hn24'] = False
 147         else:
 148             # default to False
 149             station['hn24'] = False
 150
 151         # Wind mode
 152         if 'wind_mode' in config['station']:
 153             if config['station']['wind_mode'] not in ['normal', 'average']:
 154                 raise ValueError("wind_mode must be either 'normal' or 'average'")
 155
 156             station['wind_mode'] = config['station']['wind_mode']
 157
 158             if station['wind_mode'] == "average":
 159                 station['num_hrs_to_fetch'] = 24
 160         else:
 161             # default to False
 162             station['wind_mode'] = "normal"
 163
 164     except KeyError as err:
 165         LOG.critical("%s not defined in configuration file", err)
 166         sys.exit(1)
 167     except ValueError as err:
 168         LOG.critical("%s", err)
 169         sys.exit(1)
 170
 171     # all sections/values present in config file, final sanity check
 172     try:
 173         for key in config.sections():
 174             for subkey in config[key]:
 175                 if not config[key][subkey]:
 176                     raise ValueError
 177     except ValueError:
 178         LOG.critical("Config value '%s.%s' is empty", key, subkey)
 179         sys.exit(1)
 180
 181     return (infoex, station)
 182
 183 def setup_logging(log_level):
 184     """Setup our logging infrastructure"""
 185     try:
 186         from systemd.journal import JournalHandler
 187         LOG.addHandler(JournalHandler())
 188     except ImportError:
 189         ## fallback to syslog
 190         #import logging.handlers
 191         #LOG.addHandler(logging.handlers.SysLogHandler())
 192         # fallback to stdout
 193         handler = logging.StreamHandler(sys.stdout)
 194         formatter = logging.Formatter('%(asctime)s.%(msecs)03d '
 195                                       '%(levelname)s %(module)s - '
 196                                       '%(funcName)s: %(message)s',
 197                                       '%Y-%m-%d %H:%M:%S')
 198         handler.setFormatter(formatter)
 199         LOG.addHandler(handler)
 200
 201     # ugly, but passable
 202     if log_level in [None, 'debug', 'info', 'warning']:
 203         if log_level == 'debug':
 204             LOG.setLevel(logging.DEBUG)
 205         elif log_level == 'info':
 206             LOG.setLevel(logging.INFO)
 207         elif log_level == 'warning':
 208             LOG.setLevel(logging.WARNING)
 209         else:
 210             LOG.setLevel(logging.NOTSET)
 211     else:
 212         return False
 213
 214     return True
 215
 216 def main():
 217     """Main routine: sort through args, decide what to do, then do it"""
 218     parser = get_parser()
 219     options = parser.parse_args()
 220
 221     config = configparser.ConfigParser(allow_no_value=False)
 222
 223     if not options.config:
 224         parser.print_help()
 225         print("\nPlease specify a configuration file via --config.")
 226         sys.exit(1)
 227
 228     config.read(options.config)
 229
 230     if not setup_logging(options.log_level):
 231         parser.print_help()
 232         print("\nPlease select an appropriate log level or remove the switch (--log-level).")
 233         sys.exit(1)
 234
 235     (infoex, station) = setup_config(config)
 236
 237     LOG.debug('Config parsed, starting up')
 238
 239     # create mappings
 240     (fmap, final_data) = setup_infoex_fields_mapping(infoex['location_uuid'])
 241     iemap = setup_infoex_counterparts_mapping(station['provider'])
 242
 243     # override units if user selected metric
 244     if station['provider'] != 'python' and station['units'] == 'metric':
 245         final_data = switch_units_to_metric(final_data, fmap)
 246
 247     (begin_date, end_date) = setup_time_values(station)
 248
 249     if station['provider'] == 'python':
 250         LOG.debug("Getting custom data from external Python program")
 251     else:
 252         LOG.debug("Getting %s data from %s to %s (%s)",
 253                   str(station['desired_data']),
 254                   str(begin_date), str(end_date), end_date.tzinfo.zone)
 255
 256     time_all_elements = time.time()
 257
 258     # get the data
 259     if station['provider'] == 'nrcs':
 260         infoex['wx_data'] = get_nrcs_data(begin_date, end_date, station)
 261     elif station['provider'] == 'mesowest':
 262         infoex['wx_data'] = get_mesowest_data(begin_date, end_date,
 263                                               station)
 264     elif station['provider'] == 'python':
 265         try:
 266             spec = importlib.util.spec_from_file_location('custom_wx',
 267                                                           station['path'])
 268             mod = importlib.util.module_from_spec(spec)
 269             spec.loader.exec_module(mod)
 270             mod.LOG = LOG
 271
 272             try:
 273                 infoex['wx_data'] = mod.get_custom_data()
 274
 275                 if infoex['wx_data'] is None:
 276                     infoex['wx_data'] = []
 277             except Exception as exc:
 278                 LOG.error("Python program for custom Wx data failed in "
 279                           "execution: %s", str(exc))
 280                 sys.exit(1)
 281
 282             LOG.info("Successfully executed external Python program")
 283         except ImportError:
 284             LOG.error("Please upgrade to Python 3.3 or later")
 285             sys.exit(1)
 286         except FileNotFoundError:
 287             LOG.error("Specified Python program for custom Wx data "
 288                       "was not found")
 289             sys.exit(1)
 290         except Exception as exc:
 291             LOG.error("A problem was encountered when attempting to "
 292                       "load your custom Wx program: %s", str(exc))
 293             sys.exit(1)
 294
 295     LOG.info("Time taken to get all data : %.3f sec", time.time() -
 296              time_all_elements)
 297
 298     LOG.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
 299
 300     # timezone massaging
 301     final_end_date = end_date.astimezone(station['tz'])
 302
 303     # Now we only need to add in what we want to change thanks to that
 304     # abomination of a variable declaration earlier
 305     final_data[fmap['Location UUID']] = infoex['location_uuid']
 306     final_data[fmap['obDate']] = final_end_date.strftime('%m/%d/%Y')
 307     final_data[fmap['obTime']] = final_end_date.strftime('%H:%M')
 308     final_data[fmap['timeZone']] = station['tz'].zone
 309
 310     for element_cd in infoex['wx_data']:
 311         if element_cd not in iemap:
 312             LOG.warning("BAD KEY wx_data['%s']", element_cd)
 313             continue
 314
 315         if infoex['wx_data'][element_cd] is None:
 316             continue
 317
 318         # do the conversion before the rounding
 319         if station['provider'] == 'nrcs' and station['units'] == 'metric':
 320             infoex['wx_data'][element_cd] = convert_nrcs_units_to_metric(element_cd, infoex['wx_data'][element_cd])
 321
 322         # Massage precision of certain values to fit InfoEx's
 323         # expectations
 324         #
 325         # 0 decimal places: relative humidity, wind speed, wind
 326         #                   direction, wind gust, snow depth
 327         # 1 decimal place:  air temp, baro
 328         # Avoid transforming None values
 329         if element_cd in ['wind_speed', 'WSPD', 'wind_direction',
 330                           'RHUM', 'relative_humidity', 'WDIR',
 331                           'wind_gust', 'SNWD', 'snow_depth',
 332                           'hn24']:
 333             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd])
 334         elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']:
 335             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1)
 336         elif element_cd in ['PREC', 'precip_accum']:
 337             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 2)
 338
 339         # CONSIDER: Casting every value to Float() -- need to investigate if
 340         #           any possible elementCds we may want are any other data
 341         #           type than float.
 342         #
 343         #           Another possibility is to query the API with
 344         #           getStationElements and temporarily store the
 345         #           storedUnitCd. But that's pretty network-intensive and
 346         #           may not even be worth it if there's only e.g. one or two
 347         #           exceptions to any otherwise uniformly Float value set.
 348         final_data[fmap[iemap[element_cd]]] = infoex['wx_data'][element_cd]
 349
 350     LOG.debug("final_data: %s", str(final_data))
 351
 352     if infoex['wx_data']:
 353         if not write_local_csv(infoex['csv_filename'], final_data):
 354             LOG.warning('Could not write local CSV file: %s',
 355                         infoex['csv_filename'])
 356             return 1
 357
 358         if not options.dry_run:
 359             upload_csv(infoex['csv_filename'], infoex)
 360
 361     LOG.debug('DONE')
 362     return 0
 363
 364 # data structure operations
 365 def setup_infoex_fields_mapping(location_uuid):
 366     """
 367     Create a mapping of InfoEx fields to the local data's indexing scheme.
 368
 369     INFOEX FIELDS
 370
 371     This won't earn style points in Python, but here we establish a couple
 372     of helpful mappings variables. The reason this is helpful is that the
 373     end result is simply an ordered set, the CSV file. But we still may
 374     want to manipulate the values arbitrarily before writing that file.
 375
 376     Also note that the current Auto Wx InfoEx documentation shows these
 377     keys in a graphical table with the "index" beginning at 1, but here we
 378     sanely index beginning at 0.
 379     """
 380     # pylint: disable=too-many-statements,multiple-statements,bad-whitespace
 381     fmap = {}                           ; final_data     = [None] * 29
 382     fmap['Location UUID'] = 0           ; final_data[0]  = location_uuid
 383     fmap['obDate'] = 1                  ; final_data[1]  = None
 384     fmap['obTime'] = 2                  ; final_data[2]  = None
 385     fmap['timeZone'] = 3                ; final_data[3]  = 'Pacific'
 386     fmap['tempMaxHour'] = 4             ; final_data[4]  = None
 387     fmap['tempMaxHourUnit'] = 5         ; final_data[5]  = 'F'
 388     fmap['tempMinHour'] = 6             ; final_data[6]  = None
 389     fmap['tempMinHourUnit'] = 7         ; final_data[7]  = 'F'
 390     fmap['tempPres'] = 8                ; final_data[8]  = None
 391     fmap['tempPresUnit'] = 9            ; final_data[9]  = 'F'
 392     fmap['precipitationGauge'] = 10     ; final_data[10] = None
 393     fmap['precipitationGaugeUnit'] = 11 ; final_data[11] = 'in'
 394     fmap['windSpeedNum'] = 12           ; final_data[12] = None
 395     fmap['windSpeedUnit'] = 13          ; final_data[13] = 'mph'
 396     fmap['windDirectionNum'] = 14       ; final_data[14] = None
 397     fmap['hS'] = 15                     ; final_data[15] = None
 398     fmap['hsUnit'] = 16                 ; final_data[16] = 'in'
 399     fmap['baro'] = 17                   ; final_data[17] = None
 400     fmap['baroUnit'] = 18               ; final_data[18] = 'inHg'
 401     fmap['rH'] = 19                     ; final_data[19] = None
 402     fmap['windGustSpeedNum'] = 20       ; final_data[20] = None
 403     fmap['windGustSpeedNumUnit'] = 21   ; final_data[21] = 'mph'
 404     fmap['windGustDirNum'] = 22         ; final_data[22] = None
 405     fmap['dewPoint'] = 23               ; final_data[23] = None
 406     fmap['dewPointUnit'] = 24           ; final_data[24] = 'F'
 407     fmap['hn24Auto'] = 25               ; final_data[25] = None
 408     fmap['hn24AutoUnit'] = 26           ; final_data[26] = 'in'
 409     fmap['hstAuto'] = 27                ; final_data[27] = None
 410     fmap['hstAutoUnit'] = 28            ; final_data[28] = 'in'
 411
 412     return (fmap, final_data)
 413
 414 def setup_infoex_counterparts_mapping(provider):
 415     """
 416     Create a mapping of the NRCS/MesoWest fields that this program supports to
 417     their InfoEx counterparts
 418     """
 419     iemap = {}
 420
 421     if provider == 'nrcs':
 422         iemap['PREC'] = 'precipitationGauge'
 423         iemap['TOBS'] = 'tempPres'
 424         iemap['TMAX'] = 'tempMaxHour'
 425         iemap['TMIN'] = 'tempMinHour'
 426         iemap['SNWD'] = 'hS'
 427         iemap['PRES'] = 'baro'
 428         iemap['RHUM'] = 'rH'
 429         iemap['WSPD'] = 'windSpeedNum'
 430         iemap['WDIR'] = 'windDirectionNum'
 431         # unsupported by NRCS:
 432         # windGustSpeedNum
 433     elif provider == 'mesowest':
 434         iemap['precip_accum'] = 'precipitationGauge'
 435         iemap['air_temp'] = 'tempPres'
 436         iemap['air_temp_high_24_hour'] = 'tempMaxHour'
 437         iemap['air_temp_low_24_hour'] = 'tempMinHour'
 438         iemap['snow_depth'] = 'hS'
 439         iemap['pressure'] = 'baro'
 440         iemap['relative_humidity'] = 'rH'
 441         iemap['wind_speed'] = 'windSpeedNum'
 442         iemap['wind_direction'] = 'windDirectionNum'
 443         iemap['wind_gust'] = 'windGustSpeedNum'
 444
 445         # NOTE: this doesn't exist in MesoWest, we create it in this
 446         #       program, so add it to the map here
 447         iemap['hn24'] = 'hn24Auto'
 448     elif provider == 'python':
 449         # we expect Python programs to use the InfoEx data type names
 450         iemap['precipitationGauge'] = 'precipitationGauge'
 451         iemap['tempPres'] = 'tempPres'
 452         iemap['tempMaxHour'] = 'tempMaxHour'
 453         iemap['tempMinHour'] = 'tempMinHour'
 454         iemap['hS'] = 'hS'
 455         iemap['baro'] = 'baro'
 456         iemap['rH'] = 'rH'
 457         iemap['windSpeedNum'] = 'windSpeedNum'
 458         iemap['windDirectionNum'] = 'windDirectionNum'
 459         iemap['windGustSpeedNum'] = 'windGustSpeedNum'
 460
 461     return iemap
 462
 463 # provider-specific operations
 464 def get_nrcs_data(begin, end, station):
 465     """get the data we're after from the NRCS WSDL"""
 466     transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
 467     transport.session.verify = False
 468     client = zeep.Client(wsdl=station['source'], transport=transport)
 469     remote_data = {}
 470
 471     # massage begin/end date format
 472     begin_date_str = begin.strftime('%Y-%m-%d %H:%M:00')
 473     end_date_str = end.strftime('%Y-%m-%d %H:%M:00')
 474
 475     for element_cd in station['desired_data']:
 476         time_element = time.time()
 477
 478         # get the last three hours of data for this elementCd/element_cd
 479         tmp = client.service.getHourlyData(
 480             stationTriplets=[station['station_id']],
 481             elementCd=element_cd,
 482             ordinal=1,
 483             beginDate=begin_date_str,
 484             endDate=end_date_str)
 485
 486         LOG.info("Time to get NRCS elementCd '%s': %.3f sec", element_cd,
 487                  time.time() - time_element)
 488
 489         values = tmp[0]['values']
 490
 491         # sort and isolate the most recent
 492         #
 493         # NOTE: we do this because sometimes there are gaps in hourly data
 494         #       in NRCS; yes, we may end up with slightly inaccurate data,
 495         #       so perhaps this decision will be re-evaluated in the future
 496         if values:
 497             ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
 498             remote_data[element_cd] = ordered[0]['value']
 499         else:
 500             remote_data[element_cd] = None
 501
 502     return remote_data
 503
 504 def get_mesowest_data(begin, end, station):
 505     """get the data we're after from the MesoWest/Synoptic API"""
 506     remote_data = {}
 507
 508     # massage begin/end date format
 509     begin_date_str = begin.strftime('%Y%m%d%H%M')
 510     end_date_str = end.strftime('%Y%m%d%H%M')
 511
 512     # construct final, completed API URL
 513     api_req_url = station['source'] + '&start=' + begin_date_str + '&end=' + end_date_str
 514
 515     try:
 516         req = requests.get(api_req_url)
 517     except requests.exceptions.ConnectionError:
 518         LOG.error("Could not connect to '%s'", api_req_url)
 519         sys.exit(1)
 520
 521     try:
 522         json = req.json()
 523     except ValueError:
 524         LOG.error("Bad JSON in MesoWest response")
 525         sys.exit(1)
 526
 527     try:
 528         observations = json['STATION'][0]['OBSERVATIONS']
 529     except KeyError as exc:
 530         LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
 531         sys.exit(1)
 532     except IndexError as exc:
 533         LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
 534         try:
 535             LOG.error("Detailed MesoWest response: '%s'",
 536                       json['SUMMARY']['RESPONSE_MESSAGE'])
 537         except KeyError:
 538             pass
 539         sys.exit(1)
 540     except ValueError as exc:
 541         LOG.error("Bad JSON in MesoWest response: '%s'", exc)
 542         sys.exit(1)
 543
 544     # pos represents the last item in the array, aka the most recent
 545     pos = len(observations['date_time']) - 1
 546
 547     # while these values only apply in certain cases, init them here
 548     wind_speed_values = []
 549     wind_gust_speed_values = []
 550     wind_direction_values = []
 551     hn24_values = []
 552
 553     # results
 554     wind_speed_avg = None
 555     wind_gust_speed_avg = None
 556     wind_direction_avg = None
 557     hn24 = None
 558
 559     for element_cd in station['desired_data'].split(','):
 560         # sort and isolate the most recent, see note above in NRCS for how and
 561         # why this is done
 562         #
 563         # NOTE: Unlike in the NRCS case, the MesoWest API response contains all
 564         #       data (whereas with NRCS, we have to make a separate request for
 565         #       each element we want). This is nice for network efficiency but
 566         #       it means we have to handle this part differently for each.
 567         #
 568         # NOTE: Also unlike NRCS, MesoWest provides more granular data; NRCS
 569         #       provides hourly data, but MesoWest can often provide data every
 570         #       10 minutes -- though this provides more opportunity for
 571         #       irregularities
 572
 573         # we may not have the data at all
 574         key_name = element_cd + '_set_1'
 575
 576         if key_name in observations:
 577             # val is what will make it into the dataset, after
 578             # conversions... it gets defined here because in certain
 579             # cases we need to look at all of the data to calculate HN24
 580             # or wind averages, but for the rest of the data, we only
 581             # take the most recent
 582             val = None
 583
 584             # loop through all observations for this key_name
 585             # record relevant values for wind averaging or hn24, but
 586             # otherwise only persist the data if it's the last datum in
 587             # the set
 588             for idx, _ in enumerate(observations[key_name]):
 589                 val = observations[key_name][idx]
 590
 591                 # skip bunk vals
 592                 if val is None:
 593                     continue
 594
 595                 # mesowest by default provides wind_speed in m/s, but
 596                 # we specify 'english' units in the request; either way,
 597                 # we want mph
 598                 if element_cd in ('wind_speed', 'wind_gust'):
 599                     val = kn_to_mph(val)
 600
 601                 # mesowest provides HS in mm, not cm; we want cm
 602                 if element_cd == 'snow_depth' and station['units'] == 'metric':
 603                     val = mm_to_cm(val)
 604
 605                 # HN24 / wind_mode transformations, once the data has
 606                 # completed unit conversions
 607                 if station['wind_mode'] == "average":
 608                     if element_cd == 'wind_speed' and val is not None:
 609                         wind_speed_values.append(val)
 610                     elif element_cd == 'wind_gust' and val is not None:
 611                         wind_gust_speed_values.append(val)
 612                     elif element_cd == 'wind_direction' and val is not None:
 613                         wind_direction_values.append(val)
 614
 615                 if element_cd == 'snow_depth':
 616                     hn24_values.append(val)
 617
 618                 # again, only persist this datum to the final data if
 619                 # it's from the most recent date
 620                 if idx == pos:
 621                     remote_data[element_cd] = val
 622
 623             # ensure that the data is filled out
 624             if not observations[key_name][pos]:
 625                 remote_data[element_cd] = None
 626         else:
 627             remote_data[element_cd] = None
 628
 629     if len(hn24_values) > 0:
 630         hn24 = max(hn24_values) - min(hn24_values)
 631
 632     if len(wind_speed_values) > 0:
 633         wind_speed_avg = sum(wind_speed_values) / len(wind_speed_values)
 634
 635     if len(wind_gust_speed_values) > 0:
 636         wind_gust_speed_avg = sum(wind_gust_speed_values) / len(wind_gust_speed_values)
 637
 638     if len(wind_direction_values) > 0:
 639         wind_direction_avg = sum(wind_direction_values) / len(wind_direction_values)
 640
 641     if hn24 is not None:
 642         remote_data['hn24'] = hn24
 643
 644     # overwrite the following with the respective averages, if
 645     # applicable
 646     if wind_speed_avg is not None:
 647         remote_data['wind_speed'] = wind_speed_avg
 648
 649     if wind_gust_speed_avg is not None:
 650         remote_data['wind_gust'] = wind_gust_speed_avg
 651
 652     if wind_direction_avg is not None:
 653         remote_data['wind_direction'] = wind_direction_avg
 654
 655     return remote_data
 656
 657 def switch_units_to_metric(data_map, mapping):
 658     """replace units with metric counterparts"""
 659
 660     # NOTE: to update this, use the fmap<->final_data mapping laid out
 661     #       in setup_infoex_fields_mapping ()
 662     data_map[mapping['tempMaxHourUnit']] = 'C'
 663     data_map[mapping['tempMinHourUnit']] = 'C'
 664     data_map[mapping['tempPresUnit']] = 'C'
 665     data_map[mapping['precipitationGaugeUnit']] = 'mm'
 666     data_map[mapping['hsUnit']] = 'cm'
 667     data_map[mapping['windSpeedUnit']] = 'm/s'
 668     data_map[mapping['windGustSpeedNumUnit']] = 'm/s'
 669     data_map[mapping['dewPointUnit']] = 'C'
 670     data_map[mapping['hn24AutoUnit']] = 'cm'
 671     data_map[mapping['hstAutoUnit']] = 'cm'
 672
 673     return data_map
 674
 675 def convert_nrcs_units_to_metric(element_cd, value):
 676     """convert NRCS values from English to metric"""
 677     if element_cd == 'TOBS':
 678         value = f_to_c(value)
 679     elif element_cd == 'SNWD':
 680         value = in_to_cm(value)
 681     elif element_cd == 'PREC':
 682         value = in_to_mm(value)
 683     return value
 684
 685 # CSV operations
 686 def write_local_csv(path_to_file, data):
 687     """Write the specified CSV file to disk"""
 688     with open(path_to_file, 'w') as file_object:
 689         # The requirement is that empty values are represented in the CSV
 690         # file as "", csv.QUOTE_NONNUMERIC achieves that
 691         LOG.debug("writing CSV file '%s'", path_to_file)
 692         writer = csv.writer(file_object, quoting=csv.QUOTE_NONNUMERIC)
 693         writer.writerow(data)
 694         file_object.close()
 695     return True
 696
 697 def upload_csv(path_to_file, infoex_data):
 698     """Upload the specified CSV file to InfoEx FTP and remove the file"""
 699     with open(path_to_file, 'rb') as file_object:
 700         LOG.debug("uploading FTP file '%s'", infoex_data['host'])
 701         ftp = FTP(infoex_data['host'], infoex_data['uuid'],
 702                   infoex_data['api_key'])
 703         ftp.storlines('STOR ' + path_to_file, file_object)
 704         ftp.close()
 705         file_object.close()
 706     os.remove(path_to_file)
 707
 708 # other miscellaneous routines
 709 def setup_time_values(station):
 710     """establish time bounds of data request(s)"""
 711
 712     # default timezone to UTC (for MesoWest)
 713     tz = pytz.utc
 714
 715     # but for NRCS, use the config-specified timezone
 716     if station['provider'] == 'nrcs':
 717         tz = station['tz']
 718
 719     # floor time to nearest hour
 720     date_time = datetime.datetime.now(tz=tz)
 721     end_date = date_time - datetime.timedelta(minutes=date_time.minute % 60,
 722                                               seconds=date_time.second,
 723                                               microseconds=date_time.microsecond)
 724     begin_date = end_date - datetime.timedelta(hours=station['num_hrs_to_fetch'])
 725     return (begin_date, end_date)
 726
 727 def f_to_c(f):
 728     """convert Fahrenheit to Celsius"""
 729     return (float(f) - 32) * 5.0/9.0
 730
 731 def in_to_cm(inches):
 732     """convert inches to centimetrs"""
 733     return float(inches) * 2.54
 734
 735 def in_to_mm(inches):
 736     """convert inches to millimeters"""
 737     return (float(inches) * 2.54) * 10.0
 738
 739 def ms_to_mph(ms):
 740     """convert meters per second to miles per hour"""
 741     return ms * 2.236936
 742
 743 def kn_to_mph(kn):
 744     """convert knots to miles per hour"""
 745     return kn * 1.150779
 746
 747 def mm_to_cm(mm):
 748     """convert millimeters to centimetrs"""
 749     return mm / 10.0
 750
 751 if __name__ == "__main__":
 752     sys.exit(main())