infoex-autowx.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 """
   5 InfoEx <-> NRCS/MesoWest Auto Wx implementation
   6 Alexander Vasarab
   7 Wylark Mountaineering LLC
   8
   9 This program fetches data from either an NRCS SNOTEL site or MesoWest
  10 weather station and pushes it to InfoEx using the new automated weather
  11 system implementation.
  12
  13 It is designed to be run hourly, and it asks for the last three hours
  14 of data of each desired type, and selects the most recent one. This
  15 lends some resiliency to the process and helps ensure that we have a
  16 value to send, but it can lead to somewhat inconsistent/untruthful
  17 data if e.g. the HS is from the last hour but the tempPres is from two
  18 hours ago because the instrumentation had a hiccup. It's worth
  19 considering if this is a bug or a feature.
  20
  21 For more information, see file: README
  22 For licensing, see file: LICENSE
  23 """
  24
  25 import configparser
  26 import csv
  27 import datetime
  28 import logging
  29 import os
  30 import sys
  31 import time
  32 import urllib3
  33 import importlib.util
  34
  35 from ftplib import FTP
  36 from argparse import ArgumentParser
  37
  38 import pytz
  39
  40 import requests
  41
  42 import zeep
  43 import zeep.cache
  44 import zeep.transports
  45
  46 __version__ = '3.4.0'
  47
  48 LOG = logging.getLogger(__name__)
  49 LOG.setLevel(logging.NOTSET)
  50
  51 urllib3.disable_warnings()
  52
  53 def get_parser():
  54     """Return OptionParser for this program"""
  55     parser = ArgumentParser()
  56
  57     parser.add_argument("--version",
  58                         action="version",
  59                         version=__version__)
  60
  61     parser.add_argument("--config",
  62                         dest="config",
  63                         metavar="FILE",
  64                         help="location of config file")
  65
  66     parser.add_argument("--log-level",
  67                         dest="log_level",
  68                         default=None,
  69                         help="set the log level (debug, info, warning)")
  70
  71     parser.add_argument("--dry-run",
  72                         action="store_true",
  73                         dest="dry_run",
  74                         default=False,
  75                         help="fetch data but don't upload to InfoEx")
  76
  77     return parser
  78
  79 def setup_config(config):
  80     """Setup config variable based on values specified in the ini file"""
  81     try:
  82         infoex = {
  83             'host': config['infoex']['host'],
  84             'uuid': config['infoex']['uuid'],
  85             'api_key': config['infoex']['api_key'],
  86             'csv_filename': config['infoex']['csv_filename'],
  87             'location_uuid': config['infoex']['location_uuid'],
  88             'wx_data': {}, # placeholder key, values to come later
  89         }
  90
  91         station = dict()
  92
  93         station['provider'] = config['station']['type']
  94
  95         if station['provider'] not in ['nrcs', 'mesowest', 'python']:
  96             print("Please specify either nrcs or mesowest as the station type.")
  97             sys.exit(1)
  98
  99         # massage units config items first
 100         #
 101         # NOTE: custom providers don't require units to be specified
 102         #       because they can do whatever they please with the units
 103         #       within their own program
 104         if station['provider'] != "custom":
 105             station['units'] = config['station']['units']
 106
 107             if station['units'] not in ['metric', 'english', 'american']:
 108                 print("Please specify either metric, english, or american for the units.")
 109                 sys.exit(1)
 110             else:
 111                 # if units are specified as "American" then we simply
 112                 # default to metric for the requests
 113                 if station['units'] == 'american':
 114                     station['units_requested'] = 'metric'
 115                 else:
 116                     station['units_requested'] = station['units']
 117
 118         # massage provider config items
 119         if station['provider'] == 'nrcs':
 120             station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
 121             station['station_id'] = config['station']['station_id']
 122             station['desired_data'] = config['station']['desired_data'].split(',')
 123
 124         if station['provider'] == 'mesowest':
 125             station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
 126             station['station_id'] = config['station']['station_id']
 127             station['desired_data'] = config['station']['desired_data']
 128
 129             # construct full API URL (sans start/end time, added later)
 130             station['source'] = station['source'] + '?token=' + \
 131                                 config['station']['token'] + \
 132                                 '&within=60&units=' + station['units_requested'] + \
 133                                 '&stid=' + station['station_id'] + \
 134                                 '&vars=' + station['desired_data']
 135
 136         if station['provider'] == 'python':
 137             station['path'] = config['station']['path']
 138
 139         tz = 'America/Los_Angeles'
 140
 141         if 'tz' in config['station']:
 142             tz = config['station']['tz']
 143
 144         try:
 145             station['tz'] = pytz.timezone(tz)
 146         except pytz.exceptions.UnknownTimeZoneError:
 147             LOG.critical("%s is not a valid timezone", tz)
 148             sys.exit(1)
 149
 150         # By default, fetch three hours of data
 151         #
 152         # If user wants hn24 or wind averaging, then
 153         # we need more.
 154         station['num_hrs_to_fetch'] = 3
 155
 156         # HN24
 157         if 'hn24' in config['station']:
 158             if config['station']['hn24'] not in ['true', 'false']:
 159                 raise ValueError("hn24 must be either 'true' or 'false'")
 160
 161             if config['station']['hn24'] == "true":
 162                 station['hn24'] = True
 163                 station['num_hrs_to_fetch'] = 24
 164             else:
 165                 station['hn24'] = False
 166         else:
 167             # default to False
 168             station['hn24'] = False
 169
 170         # Wind mode
 171         if 'wind_mode' in config['station']:
 172             if config['station']['wind_mode'] not in ['normal', 'average']:
 173                 raise ValueError("wind_mode must be either 'normal' or 'average'")
 174
 175             station['wind_mode'] = config['station']['wind_mode']
 176
 177             if station['wind_mode'] == "average":
 178                 station['num_hrs_to_fetch'] = 24
 179         else:
 180             # default to False
 181             station['wind_mode'] = "normal"
 182
 183     except KeyError as err:
 184         LOG.critical("%s not defined in configuration file", err)
 185         sys.exit(1)
 186     except ValueError as err:
 187         LOG.critical("%s", err)
 188         sys.exit(1)
 189
 190     # all sections/values present in config file, final sanity check
 191     try:
 192         for key in config.sections():
 193             for subkey in config[key]:
 194                 if not config[key][subkey]:
 195                     raise ValueError
 196     except ValueError:
 197         LOG.critical("Config value '%s.%s' is empty", key, subkey)
 198         sys.exit(1)
 199
 200     return (infoex, station)
 201
 202 def setup_logging(log_level):
 203     """Setup our logging infrastructure"""
 204     try:
 205         from systemd.journal import JournalHandler
 206         LOG.addHandler(JournalHandler())
 207     except ImportError:
 208         ## fallback to syslog
 209         #import logging.handlers
 210         #LOG.addHandler(logging.handlers.SysLogHandler())
 211         # fallback to stdout
 212         handler = logging.StreamHandler(sys.stdout)
 213         formatter = logging.Formatter('%(asctime)s.%(msecs)03d '
 214                                       '%(levelname)s %(module)s - '
 215                                       '%(funcName)s: %(message)s',
 216                                       '%Y-%m-%d %H:%M:%S')
 217         handler.setFormatter(formatter)
 218         LOG.addHandler(handler)
 219
 220     # ugly, but passable
 221     if log_level in [None, 'debug', 'info', 'warning']:
 222         if log_level == 'debug':
 223             LOG.setLevel(logging.DEBUG)
 224         elif log_level == 'info':
 225             LOG.setLevel(logging.INFO)
 226         elif log_level == 'warning':
 227             LOG.setLevel(logging.WARNING)
 228         else:
 229             LOG.setLevel(logging.NOTSET)
 230     else:
 231         return False
 232
 233     return True
 234
 235 def main():
 236     """Main routine: sort through args, decide what to do, then do it"""
 237     parser = get_parser()
 238     options = parser.parse_args()
 239
 240     config = configparser.ConfigParser(allow_no_value=False)
 241
 242     if not options.config:
 243         parser.print_help()
 244         print("\nPlease specify a configuration file via --config.")
 245         sys.exit(1)
 246
 247     config.read(options.config)
 248
 249     if not setup_logging(options.log_level):
 250         parser.print_help()
 251         print("\nPlease select an appropriate log level or remove the switch (--log-level).")
 252         sys.exit(1)
 253
 254     (infoex, station) = setup_config(config)
 255
 256     LOG.debug('Config parsed, starting up')
 257
 258     # create mappings
 259     (fmap, final_data) = setup_infoex_fields_mapping(infoex['location_uuid'])
 260     iemap = setup_infoex_counterparts_mapping(station['provider'])
 261
 262     # override units if user selected metric
 263     if station['provider'] != 'python' and station['units'] == 'metric':
 264         final_data = switch_units_to_metric(final_data, fmap)
 265
 266     # likewise for "American" units
 267     if station['provider'] != 'python' and station['units'] == 'american':
 268         final_data = switch_units_to_american(final_data, fmap)
 269
 270     (begin_date, end_date) = setup_time_values(station)
 271
 272     if station['provider'] == 'python':
 273         LOG.debug("Getting custom data from external Python program")
 274     else:
 275         LOG.debug("Getting %s data from %s to %s (%s)",
 276                   str(station['desired_data']),
 277                   str(begin_date), str(end_date), end_date.tzinfo.zone)
 278
 279     time_all_elements = time.time()
 280
 281     # get the data
 282     if station['provider'] == 'nrcs':
 283         infoex['wx_data'] = get_nrcs_data(begin_date, end_date, station)
 284     elif station['provider'] == 'mesowest':
 285         infoex['wx_data'] = get_mesowest_data(begin_date, end_date,
 286                                               station)
 287     elif station['provider'] == 'python':
 288         try:
 289             spec = importlib.util.spec_from_file_location('custom_wx',
 290                                                           station['path'])
 291             mod = importlib.util.module_from_spec(spec)
 292             spec.loader.exec_module(mod)
 293             mod.LOG = LOG
 294
 295             try:
 296                 infoex['wx_data'] = mod.get_custom_data()
 297
 298                 if infoex['wx_data'] is None:
 299                     infoex['wx_data'] = []
 300             except Exception as exc:
 301                 LOG.error("Python program for custom Wx data failed in "
 302                           "execution: %s", str(exc))
 303                 sys.exit(1)
 304
 305             LOG.info("Successfully executed external Python program")
 306         except ImportError:
 307             LOG.error("Please upgrade to Python 3.3 or later")
 308             sys.exit(1)
 309         except FileNotFoundError:
 310             LOG.error("Specified Python program for custom Wx data "
 311                       "was not found")
 312             sys.exit(1)
 313         except Exception as exc:
 314             LOG.error("A problem was encountered when attempting to "
 315                       "load your custom Wx program: %s", str(exc))
 316             sys.exit(1)
 317
 318     LOG.info("Time taken to get all data : %.3f sec", time.time() -
 319              time_all_elements)
 320
 321     LOG.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
 322
 323     # timezone massaging
 324     final_end_date = end_date.astimezone(station['tz'])
 325
 326     # Now we only need to add in what we want to change thanks to that
 327     # abomination of a variable declaration earlier
 328     final_data[fmap['Location UUID']] = infoex['location_uuid']
 329     final_data[fmap['obDate']] = final_end_date.strftime('%m/%d/%Y')
 330     final_data[fmap['obTime']] = final_end_date.strftime('%H:%M')
 331     final_data[fmap['timeZone']] = station['tz'].zone
 332
 333     for element_cd in infoex['wx_data']:
 334         if element_cd not in iemap:
 335             LOG.warning("BAD KEY wx_data['%s']", element_cd)
 336             continue
 337
 338         if infoex['wx_data'][element_cd] is None:
 339             continue
 340
 341         # do the conversion before the rounding
 342         if station['provider'] == 'nrcs' and station['units'] == 'metric':
 343             infoex['wx_data'][element_cd] = convert_nrcs_units_to_metric(element_cd, infoex['wx_data'][element_cd])
 344
 345         if station['provider'] != 'custom' and station['units'] == 'american':
 346             infoex['wx_data'][element_cd] = convert_units_to_american(element_cd, infoex['wx_data'][element_cd])
 347
 348         # Massage precision of certain values to fit InfoEx's
 349         # expectations
 350         #
 351         # 0 decimal places: relative humidity, wind speed, wind
 352         #                   direction, wind gust, snow depth
 353         # 1 decimal place:  air temp, baro
 354         # Avoid transforming None values
 355         if element_cd in ['wind_speed', 'WSPD', 'wind_direction',
 356                           'RHUM', 'relative_humidity', 'WDIR',
 357                           'wind_gust', 'SNWD', 'snow_depth',
 358                           'hn24']:
 359             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd])
 360         elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']:
 361             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1)
 362         elif element_cd in ['PREC', 'precip_accum']:
 363             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 2)
 364
 365         # CONSIDER: Casting every value to Float() -- need to investigate if
 366         #           any possible elementCds we may want are any other data
 367         #           type than float.
 368         #
 369         #           Another possibility is to query the API with
 370         #           getStationElements and temporarily store the
 371         #           storedUnitCd. But that's pretty network-intensive and
 372         #           may not even be worth it if there's only e.g. one or two
 373         #           exceptions to any otherwise uniformly Float value set.
 374         final_data[fmap[iemap[element_cd]]] = infoex['wx_data'][element_cd]
 375
 376     LOG.debug("final_data: %s", str(final_data))
 377
 378     if infoex['wx_data']:
 379         if not write_local_csv(infoex['csv_filename'], final_data):
 380             LOG.warning('Could not write local CSV file: %s',
 381                         infoex['csv_filename'])
 382             return 1
 383
 384         if not options.dry_run:
 385             upload_csv(infoex['csv_filename'], infoex)
 386
 387     LOG.debug('DONE')
 388     return 0
 389
 390 # data structure operations
 391 def setup_infoex_fields_mapping(location_uuid):
 392     """
 393     Create a mapping of InfoEx fields to the local data's indexing scheme.
 394
 395     INFOEX FIELDS
 396
 397     This won't earn style points in Python, but here we establish a couple
 398     of helpful mappings variables. The reason this is helpful is that the
 399     end result is simply an ordered set, the CSV file. But we still may
 400     want to manipulate the values arbitrarily before writing that file.
 401
 402     Also note that the current Auto Wx InfoEx documentation shows these
 403     keys in a graphical table with the "index" beginning at 1, but here we
 404     sanely index beginning at 0.
 405     """
 406     # pylint: disable=too-many-statements,multiple-statements,bad-whitespace
 407     fmap = {}                           ; final_data     = [None] * 29
 408     fmap['Location UUID'] = 0           ; final_data[0]  = location_uuid
 409     fmap['obDate'] = 1                  ; final_data[1]  = None
 410     fmap['obTime'] = 2                  ; final_data[2]  = None
 411     fmap['timeZone'] = 3                ; final_data[3]  = 'Pacific'
 412     fmap['tempMaxHour'] = 4             ; final_data[4]  = None
 413     fmap['tempMaxHourUnit'] = 5         ; final_data[5]  = 'F'
 414     fmap['tempMinHour'] = 6             ; final_data[6]  = None
 415     fmap['tempMinHourUnit'] = 7         ; final_data[7]  = 'F'
 416     fmap['tempPres'] = 8                ; final_data[8]  = None
 417     fmap['tempPresUnit'] = 9            ; final_data[9]  = 'F'
 418     fmap['precipitationGauge'] = 10     ; final_data[10] = None
 419     fmap['precipitationGaugeUnit'] = 11 ; final_data[11] = 'in'
 420     fmap['windSpeedNum'] = 12           ; final_data[12] = None
 421     fmap['windSpeedUnit'] = 13          ; final_data[13] = 'mph'
 422     fmap['windDirectionNum'] = 14       ; final_data[14] = None
 423     fmap['hS'] = 15                     ; final_data[15] = None
 424     fmap['hsUnit'] = 16                 ; final_data[16] = 'in'
 425     fmap['baro'] = 17                   ; final_data[17] = None
 426     fmap['baroUnit'] = 18               ; final_data[18] = 'inHg'
 427     fmap['rH'] = 19                     ; final_data[19] = None
 428     fmap['windGustSpeedNum'] = 20       ; final_data[20] = None
 429     fmap['windGustSpeedNumUnit'] = 21   ; final_data[21] = 'mph'
 430     fmap['windGustDirNum'] = 22         ; final_data[22] = None
 431     fmap['dewPoint'] = 23               ; final_data[23] = None
 432     fmap['dewPointUnit'] = 24           ; final_data[24] = 'F'
 433     fmap['hn24Auto'] = 25               ; final_data[25] = None
 434     fmap['hn24AutoUnit'] = 26           ; final_data[26] = 'in'
 435     fmap['hstAuto'] = 27                ; final_data[27] = None
 436     fmap['hstAutoUnit'] = 28            ; final_data[28] = 'in'
 437
 438     return (fmap, final_data)
 439
 440 def setup_infoex_counterparts_mapping(provider):
 441     """
 442     Create a mapping of the NRCS/MesoWest fields that this program supports to
 443     their InfoEx counterparts
 444     """
 445     iemap = {}
 446
 447     if provider == 'nrcs':
 448         iemap['PREC'] = 'precipitationGauge'
 449         iemap['TOBS'] = 'tempPres'
 450         iemap['TMAX'] = 'tempMaxHour'
 451         iemap['TMIN'] = 'tempMinHour'
 452         iemap['SNWD'] = 'hS'
 453         iemap['PRES'] = 'baro'
 454         iemap['RHUM'] = 'rH'
 455         iemap['WSPD'] = 'windSpeedNum'
 456         iemap['WDIR'] = 'windDirectionNum'
 457         # unsupported by NRCS:
 458         # windGustSpeedNum
 459
 460         # NOTE: this doesn't exist in NRCS SNOTEL, we create it in this
 461         #       program, so add it to the map here
 462         iemap['hn24'] = 'hn24Auto'
 463     elif provider == 'mesowest':
 464         iemap['precip_accum'] = 'precipitationGauge'
 465         iemap['air_temp'] = 'tempPres'
 466         iemap['air_temp_high_24_hour'] = 'tempMaxHour'
 467         iemap['air_temp_low_24_hour'] = 'tempMinHour'
 468         iemap['snow_depth'] = 'hS'
 469         iemap['pressure'] = 'baro'
 470         iemap['relative_humidity'] = 'rH'
 471         iemap['wind_speed'] = 'windSpeedNum'
 472         iemap['wind_direction'] = 'windDirectionNum'
 473         iemap['wind_gust'] = 'windGustSpeedNum'
 474
 475         # NOTE: this doesn't exist in MesoWest, we create it in this
 476         #       program, so add it to the map here
 477         iemap['hn24'] = 'hn24Auto'
 478     elif provider == 'python':
 479         # we expect Python programs to use the InfoEx data type names
 480         iemap['precipitationGauge'] = 'precipitationGauge'
 481         iemap['tempPres'] = 'tempPres'
 482         iemap['tempMaxHour'] = 'tempMaxHour'
 483         iemap['tempMinHour'] = 'tempMinHour'
 484         iemap['hS'] = 'hS'
 485         iemap['baro'] = 'baro'
 486         iemap['rH'] = 'rH'
 487         iemap['windSpeedNum'] = 'windSpeedNum'
 488         iemap['windDirectionNum'] = 'windDirectionNum'
 489         iemap['windGustSpeedNum'] = 'windGustSpeedNum'
 490
 491     return iemap
 492
 493 # provider-specific operations
 494 def get_nrcs_data(begin, end, station):
 495     """get the data we're after from the NRCS WSDL"""
 496     transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
 497     transport.session.verify = False
 498     client = zeep.Client(wsdl=station['source'], transport=transport)
 499     remote_data = {}
 500
 501     # massage begin/end date format
 502     begin_date_str = begin.strftime('%Y-%m-%d %H:%M:00')
 503     end_date_str = end.strftime('%Y-%m-%d %H:%M:00')
 504
 505     for element_cd in station['desired_data']:
 506         time_element = time.time()
 507
 508         # get the last three hours of data for this elementCd/element_cd
 509         tmp = client.service.getHourlyData(
 510             stationTriplets=[station['station_id']],
 511             elementCd=element_cd,
 512             ordinal=1,
 513             beginDate=begin_date_str,
 514             endDate=end_date_str)
 515
 516         LOG.info("Time to get NRCS elementCd '%s': %.3f sec", element_cd,
 517                  time.time() - time_element)
 518
 519         values = tmp[0]['values']
 520
 521         # sort and isolate the most recent
 522         #
 523         # NOTE: we do this because sometimes there are gaps in hourly data
 524         #       in NRCS; yes, we may end up with slightly inaccurate data,
 525         #       so perhaps this decision will be re-evaluated in the future
 526         if values:
 527             ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
 528             remote_data[element_cd] = ordered[0]['value']
 529         else:
 530             remote_data[element_cd] = None
 531
 532
 533         # calc hn24, if applicable
 534         hn24 = None
 535
 536         if station['hn24']:
 537             hn24_values = []
 538
 539             if element_cd == "SNWD":
 540                 for idx, _ in enumerate(values):
 541                     val = values[idx]
 542                     if val is None:
 543                         continue
 544                     hn24_values.append(val['value'])
 545
 546                 if len(hn24_values) > 0:
 547                     # instead of taking MAX - MIN, we want the first
 548                     # value (most distant) - the last value (most
 549                     # recent)
 550                     #
 551                     # if the result is positive, then we have
 552                     # settlement; if it's not, then we have HN24
 553                     hn24 = hn24_values[0] - hn24_values[len(hn24_values)-1]
 554
 555                     if hn24 < 0.0:
 556                         hn24 = abs(hn24)
 557                     else:
 558                         # this case represents HS settlement
 559                         hn24 = 0.0
 560
 561             # finally, if user wants hn24 and it's set to None at this
 562             # point, then force it to 0.0
 563             if hn24 is None:
 564                 hn24 = 0.0
 565
 566         if hn24 is not None:
 567             if station['hn24']:
 568                 remote_data['hn24'] = hn24
 569
 570     return remote_data
 571
 572 def get_mesowest_data(begin, end, station):
 573     """get the data we're after from the MesoWest/Synoptic API"""
 574     remote_data = {}
 575
 576     # massage begin/end date format
 577     begin_date_str = begin.strftime('%Y%m%d%H%M')
 578     end_date_str = end.strftime('%Y%m%d%H%M')
 579
 580     # construct final, completed API URL
 581     api_req_url = station['source'] + '&start=' + begin_date_str + '&end=' + end_date_str
 582
 583     try:
 584         req = requests.get(api_req_url)
 585     except requests.exceptions.ConnectionError:
 586         LOG.error("Could not connect to '%s'", api_req_url)
 587         sys.exit(1)
 588
 589     try:
 590         json = req.json()
 591     except ValueError:
 592         LOG.error("Bad JSON in MesoWest response")
 593         sys.exit(1)
 594
 595     try:
 596         observations = json['STATION'][0]['OBSERVATIONS']
 597     except KeyError as exc:
 598         LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
 599         sys.exit(1)
 600     except IndexError as exc:
 601         LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
 602         try:
 603             LOG.error("Detailed MesoWest response: '%s'",
 604                       json['SUMMARY']['RESPONSE_MESSAGE'])
 605         except KeyError:
 606             pass
 607         sys.exit(1)
 608     except ValueError as exc:
 609         LOG.error("Bad JSON in MesoWest response: '%s'", exc)
 610         sys.exit(1)
 611
 612     # pos represents the last item in the array, aka the most recent
 613     pos = len(observations['date_time']) - 1
 614
 615     # while these values only apply in certain cases, init them here
 616     wind_speed_values = []
 617     wind_gust_speed_values = []
 618     wind_direction_values = []
 619     hn24_values = []
 620
 621     # results
 622     wind_speed_avg = None
 623     wind_gust_speed_avg = None
 624     wind_direction_avg = None
 625     hn24 = None
 626
 627     for element_cd in station['desired_data'].split(','):
 628         # sort and isolate the most recent, see note above in NRCS for how and
 629         # why this is done
 630         #
 631         # NOTE: Unlike in the NRCS case, the MesoWest API response contains all
 632         #       data (whereas with NRCS, we have to make a separate request for
 633         #       each element we want). This is nice for network efficiency but
 634         #       it means we have to handle this part differently for each.
 635         #
 636         # NOTE: Also unlike NRCS, MesoWest provides more granular data; NRCS
 637         #       provides hourly data, but MesoWest can often provide data every
 638         #       10 minutes -- though this provides more opportunity for
 639         #       irregularities
 640
 641         # we may not have the data at all
 642         key_name = element_cd + '_set_1'
 643
 644         if key_name in observations:
 645             # val is what will make it into the dataset, after
 646             # conversions... it gets defined here because in certain
 647             # cases we need to look at all of the data to calculate HN24
 648             # or wind averages, but for the rest of the data, we only
 649             # take the most recent
 650             val = None
 651
 652             # loop through all observations for this key_name
 653             # record relevant values for wind averaging or hn24, but
 654             # otherwise only persist the data if it's the last datum in
 655             # the set
 656             for idx, _ in enumerate(observations[key_name]):
 657                 val = observations[key_name][idx]
 658
 659                 # skip bunk vals
 660                 if val is None:
 661                     continue
 662
 663                 # mesowest by default provides wind_speed in m/s, but
 664                 # we specify 'english' units in the request; either way,
 665                 # we want mph
 666                 if element_cd in ('wind_speed', 'wind_gust'):
 667                     val = kn_to_mph(val)
 668
 669                 # mesowest provides HS in mm, not cm; we want cm
 670                 if element_cd == 'snow_depth' and station['units'] == 'metric':
 671                     val = mm_to_cm(val)
 672
 673                 # HN24 / wind_mode transformations, once the data has
 674                 # completed unit conversions
 675                 if station['wind_mode'] == "average":
 676                     if element_cd == 'wind_speed' and val is not None:
 677                         wind_speed_values.append(val)
 678                     elif element_cd == 'wind_gust' and val is not None:
 679                         wind_gust_speed_values.append(val)
 680                     elif element_cd == 'wind_direction' and val is not None:
 681                         wind_direction_values.append(val)
 682
 683                 if element_cd == 'snow_depth':
 684                     hn24_values.append(val)
 685
 686                 # again, only persist this datum to the final data if
 687                 # it's from the most recent date
 688                 if idx == pos:
 689                     remote_data[element_cd] = val
 690
 691             # ensure that the data is filled out
 692             if not observations[key_name][pos]:
 693                 remote_data[element_cd] = None
 694         else:
 695             remote_data[element_cd] = None
 696
 697     if len(hn24_values) > 0:
 698         # instead of taking MAX - MIN, we want the first value (most
 699         # distant) - the last value (most recent)
 700         #
 701         # if the result is positive, then we have settlement; if it's not,
 702         # then we have HN24
 703         hn24 = hn24_values[0] - hn24_values[len(hn24_values)-1]
 704
 705         if hn24 < 0.0:
 706             hn24 = abs(hn24)
 707         else:
 708             # this case represents HS settlement
 709             hn24 = 0.0
 710
 711
 712     # finally, if user wants hn24 and it's set to None at this
 713     # point, then force it to 0.0
 714     if station['hn24'] and hn24 is None:
 715         hn24 = 0.0
 716
 717     if len(wind_speed_values) > 0:
 718         wind_speed_avg = sum(wind_speed_values) / len(wind_speed_values)
 719
 720     if len(wind_gust_speed_values) > 0:
 721         wind_gust_speed_avg = sum(wind_gust_speed_values) / len(wind_gust_speed_values)
 722
 723     if len(wind_direction_values) > 0:
 724         wind_direction_avg = sum(wind_direction_values) / len(wind_direction_values)
 725
 726     if hn24 is not None:
 727         if station['hn24']:
 728             remote_data['hn24'] = hn24
 729
 730     # overwrite the following with the respective averages, if
 731     # applicable
 732     if wind_speed_avg is not None:
 733         remote_data['wind_speed'] = wind_speed_avg
 734
 735     if wind_gust_speed_avg is not None:
 736         remote_data['wind_gust'] = wind_gust_speed_avg
 737
 738     if wind_direction_avg is not None:
 739         remote_data['wind_direction'] = wind_direction_avg
 740
 741     return remote_data
 742
 743 def switch_units_to_metric(data_map, mapping):
 744     """replace units with metric counterparts"""
 745
 746     # NOTE: to update this, use the fmap<->final_data mapping laid out
 747     #       in setup_infoex_fields_mapping ()
 748     data_map[mapping['tempMaxHourUnit']] = 'C'
 749     data_map[mapping['tempMinHourUnit']] = 'C'
 750     data_map[mapping['tempPresUnit']] = 'C'
 751     data_map[mapping['precipitationGaugeUnit']] = 'mm'
 752     data_map[mapping['hsUnit']] = 'cm'
 753     data_map[mapping['windSpeedUnit']] = 'm/s'
 754     data_map[mapping['windGustSpeedNumUnit']] = 'm/s'
 755     data_map[mapping['dewPointUnit']] = 'C'
 756     data_map[mapping['hn24AutoUnit']] = 'cm'
 757     data_map[mapping['hstAutoUnit']] = 'cm'
 758
 759     return data_map
 760
 761 def switch_units_to_american(data_map, mapping):
 762     """
 763     replace units with the American mixture of metric and imperial
 764
 765     Precip values = imperial
 766     Wind values = metric
 767     """
 768
 769     # precip values
 770     data_map[mapping['tempMaxHourUnit']] = 'F'
 771     data_map[mapping['tempMinHourUnit']] = 'F'
 772     data_map[mapping['tempPresUnit']] = 'F'
 773     data_map[mapping['dewPointUnit']] = 'F'
 774
 775     data_map[mapping['precipitationGaugeUnit']] = 'in'
 776     data_map[mapping['hsUnit']] = 'in'
 777     data_map[mapping['hn24AutoUnit']] = 'in'
 778     data_map[mapping['hstAutoUnit']] = 'in'
 779
 780     data_map[mapping['baroUnit']] = 'inHg'
 781
 782     # wind values
 783     data_map[mapping['windSpeedUnit']] = 'm/s'
 784     data_map[mapping['windGustSpeedNumUnit']] = 'm/s'
 785
 786     return data_map
 787
 788 def convert_nrcs_units_to_metric(element_cd, value):
 789     """convert NRCS values from English to metric"""
 790     if element_cd == 'TOBS':
 791         value = f_to_c(value)
 792     elif element_cd == 'SNWD':
 793         value = in_to_cm(value)
 794     elif element_cd == 'PREC':
 795         value = in_to_mm(value)
 796     return value
 797
 798 def convert_units_to_american(element_cd, value):
 799     """
 800     convert value to 'American' units
 801
 802     The original unit is always metric.
 803
 804     Precip values = imperial
 805     Wind values = metric
 806     """
 807
 808     # temp values
 809     if element_cd in ['TMAX', 'TMIN', 'TOBS', 'air_temp', 'air_temp_high_24_hour', 'air_temp_low_24_hour']:
 810         value = c_to_f(value)
 811
 812     # snow values
 813     if element_cd in ['SNWD', 'snow_depth']:
 814         value = cm_to_in(mm_to_cm(value))
 815
 816     # likewise for baro values
 817     if element_cd in ['PRES', 'pressure']:
 818         value = pascal_to_inhg(value)
 819
 820     # no need to convert wind values, as they will arrive in metric
 821     # units in "American" units mode
 822     # [
 823     # 'WSPD',
 824     # 'wind_speed', 'wind_gust'
 825     # ]
 826     # in_to_mm(value)
 827
 828     return value
 829
 830 # CSV operations
 831 def write_local_csv(path_to_file, data):
 832     """Write the specified CSV file to disk"""
 833     with open(path_to_file, 'w') as file_object:
 834         # The requirement is that empty values are represented in the CSV
 835         # file as "", csv.QUOTE_NONNUMERIC achieves that
 836         LOG.debug("writing CSV file '%s'", path_to_file)
 837         writer = csv.writer(file_object, quoting=csv.QUOTE_NONNUMERIC)
 838         writer.writerow(data)
 839         file_object.close()
 840     return True
 841
 842 def upload_csv(path_to_file, infoex_data):
 843     """Upload the specified CSV file to InfoEx FTP and remove the file"""
 844     with open(path_to_file, 'rb') as file_object:
 845         LOG.debug("uploading FTP file '%s'", infoex_data['host'])
 846         ftp = FTP(infoex_data['host'], infoex_data['uuid'],
 847                   infoex_data['api_key'])
 848         ftp.storlines('STOR ' + path_to_file, file_object)
 849         ftp.close()
 850         file_object.close()
 851     os.remove(path_to_file)
 852
 853 # other miscellaneous routines
 854 def setup_time_values(station):
 855     """establish time bounds of data request(s)"""
 856
 857     # default timezone to UTC (for MesoWest)
 858     tz = pytz.utc
 859
 860     # but for NRCS, use the config-specified timezone
 861     if station['provider'] == 'nrcs':
 862         tz = station['tz']
 863
 864     # floor time to nearest hour
 865     date_time = datetime.datetime.now(tz=tz)
 866     end_date = date_time - datetime.timedelta(minutes=date_time.minute % 60,
 867                                               seconds=date_time.second,
 868                                               microseconds=date_time.microsecond)
 869     begin_date = end_date - datetime.timedelta(hours=station['num_hrs_to_fetch'])
 870     return (begin_date, end_date)
 871
 872 def f_to_c(f):
 873     """convert Fahrenheit to Celsius"""
 874     return (float(f) - 32) * 5.0/9.0
 875
 876 def c_to_f(c):
 877     """convert Celsius to Fahrenheit"""
 878     return (float(c) * 1.8) + 32
 879
 880 def in_to_cm(inches):
 881     """convert inches to centimeters"""
 882     return float(inches) * 2.54
 883
 884 def cm_to_in(cms):
 885     """convert centimeters to inches"""
 886     return float(cms) / 2.54
 887
 888 def pascal_to_inhg(pa):
 889     """convert pascals to inches of mercury"""
 890     return float(pa) * 0.00029530
 891
 892 def in_to_mm(inches):
 893     """convert inches to millimeters"""
 894     return (float(inches) * 2.54) * 10.0
 895
 896 def ms_to_mph(ms):
 897     """convert meters per second to miles per hour"""
 898     return ms * 2.236936
 899
 900 def kn_to_mph(kn):
 901     """convert knots to miles per hour"""
 902     return kn * 1.150779
 903
 904 def mm_to_cm(mm):
 905     """convert millimeters to centimeters"""
 906     return mm / 10.0
 907
 908 if __name__ == "__main__":
 909     sys.exit(main())