Version bump

[infoex-autowx.git] / infoex-autowx.py

diff --git a/infoex-autowx.py b/infoex-autowx.py
index ea23bc3e9567b47ac3a710ed7f3938fc4fc71a27..9df112c6d96c149786fc8749c287e01437730d74 100755 (executable)
--- a/infoex-autowx.py
+++ b/infoex-autowx.py
@@ -27,23 +27,30 @@ import csv
 import datetime
 import logging
 import os
 import datetime
 import logging
 import os

+import socket

 import sys
 import time
 import sys
 import time

+import urllib3
+import importlib.util

 
 from ftplib import FTP
 from argparse import ArgumentParser
 
 
 from ftplib import FTP
 from argparse import ArgumentParser
 

+import pytz
+

 import requests
 
 import zeep
 import zeep.cache
 import zeep.transports
 
 import requests
 
 import zeep
 import zeep.cache
 import zeep.transports
 

-__version__ = '2.2.0'
+__version__ = '3.4.5'

 
 LOG = logging.getLogger(__name__)
 LOG.setLevel(logging.NOTSET)
 
 
 LOG = logging.getLogger(__name__)
 LOG.setLevel(logging.NOTSET)
 

+urllib3.disable_warnings()
+

 def get_parser():
     """Return OptionParser for this program"""
     parser = ArgumentParser()
 def get_parser():
     """Return OptionParser for this program"""
     parser = ArgumentParser()


@@ -83,38 +90,105 @@ def setup_config(config):
         }
 
         station = dict()
         }
 
         station = dict()

+

         station['provider'] = config['station']['type']
 
         station['provider'] = config['station']['type']
 

-        if station['provider'] not in ['nrcs', 'mesowest']:
+        if station['provider'] not in ['nrcs', 'mesowest', 'python']:

             print("Please specify either nrcs or mesowest as the station type.")
             sys.exit(1)
 
             print("Please specify either nrcs or mesowest as the station type.")
             sys.exit(1)
 

+        # massage units config items first
+        #
+        # NOTE: custom providers don't require units to be specified
+        #       because they can do whatever they please with the units
+        #       within their own program
+        if station['provider'] != "python":
+            station['units'] = config['station']['units']
+
+            if station['units'] not in ['metric', 'english', 'american']:
+                print("Please specify metric, english, or american for the units.")
+                sys.exit(1)
+            else:
+                # if units are specified as "American" then we simply
+                # default to metric for the requests
+                if station['units'] == 'american':
+                    station['units_requested'] = 'metric'
+                else:
+                    station['units_requested'] = station['units']
+
+        # massage provider config items

         if station['provider'] == 'nrcs':
         if station['provider'] == 'nrcs':

-            station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
+            #station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
+            station['source'] = 'https://wcc.sc.egov.usda.gov/awdbWebService/services?WSDL'
+

             station['station_id'] = config['station']['station_id']
             station['desired_data'] = config['station']['desired_data'].split(',')
 
             station['station_id'] = config['station']['station_id']
             station['desired_data'] = config['station']['desired_data'].split(',')
 

-            # XXX: For NRCS, we're manually overriding units for now! Once
-            #      unit conversion is supported for NRCS, REMOVE THIS!
-            if 'units' not in station:
-                station['units'] = 'imperial'
-

         if station['provider'] == 'mesowest':
             station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
             station['station_id'] = config['station']['station_id']
         if station['provider'] == 'mesowest':
             station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
             station['station_id'] = config['station']['station_id']

-            station['units'] = config['station']['units']

             station['desired_data'] = config['station']['desired_data']
 
             # construct full API URL (sans start/end time, added later)
             station['source'] = station['source'] + '?token=' + \
                                 config['station']['token'] + \
             station['desired_data'] = config['station']['desired_data']
 
             # construct full API URL (sans start/end time, added later)
             station['source'] = station['source'] + '?token=' + \
                                 config['station']['token'] + \

-                                '&within=60&units=' + station['units'] + \
+                                '&within=60&units=' + station['units_requested'] + \

                                 '&stid=' + station['station_id'] + \
                                 '&vars=' + station['desired_data']
 
                                 '&stid=' + station['station_id'] + \
                                 '&vars=' + station['desired_data']
 

+        if station['provider'] == 'python':
+            station['path'] = config['station']['path']
+
+        tz = 'America/Los_Angeles'
+
+        if 'tz' in config['station']:
+            tz = config['station']['tz']
+
+        try:
+            station['tz'] = pytz.timezone(tz)
+        except pytz.exceptions.UnknownTimeZoneError:
+            LOG.critical("%s is not a valid timezone", tz)
+            sys.exit(1)
+
+        # By default, fetch three hours of data
+        #
+        # If user wants hn24 or wind averaging, then
+        # we need more.
+        station['num_hrs_to_fetch'] = 3
+
+        # HN24
+        if 'hn24' in config['station']:
+            if config['station']['hn24'] not in ['true', 'false']:
+                raise ValueError("hn24 must be either 'true' or 'false'")
+
+            if config['station']['hn24'] == "true":
+                station['hn24'] = True
+                station['num_hrs_to_fetch'] = 24
+            else:
+                station['hn24'] = False
+        else:
+            # default to False
+            station['hn24'] = False
+
+        # Wind mode
+        if 'wind_mode' in config['station']:
+            if config['station']['wind_mode'] not in ['normal', 'average']:
+                raise ValueError("wind_mode must be either 'normal' or 'average'")
+
+            station['wind_mode'] = config['station']['wind_mode']
+
+            if station['wind_mode'] == "average":
+                station['num_hrs_to_fetch'] = 24
+        else:
+            # default to False
+            station['wind_mode'] = "normal"
+

     except KeyError as err:
         LOG.critical("%s not defined in configuration file", err)
     except KeyError as err:
         LOG.critical("%s not defined in configuration file", err)

-        exit(1)
+        sys.exit(1)
+    except ValueError as err:
+        LOG.critical("%s", err)
+        sys.exit(1)

 
     # all sections/values present in config file, final sanity check
     try:
 
     # all sections/values present in config file, final sanity check
     try:


@@ -124,7 +198,7 @@ def setup_config(config):
                     raise ValueError
     except ValueError:
         LOG.critical("Config value '%s.%s' is empty", key, subkey)
                     raise ValueError
     except ValueError:
         LOG.critical("Config value '%s.%s' is empty", key, subkey)

-        exit(1)
+        sys.exit(1)

 
     return (infoex, station)
 
 
     return (infoex, station)
 


@@ -139,6 +213,11 @@ def setup_logging(log_level):
         #LOG.addHandler(logging.handlers.SysLogHandler())
         # fallback to stdout
         handler = logging.StreamHandler(sys.stdout)
         #LOG.addHandler(logging.handlers.SysLogHandler())
         # fallback to stdout
         handler = logging.StreamHandler(sys.stdout)

+        formatter = logging.Formatter('%(asctime)s.%(msecs)03d '
+                                      '%(levelname)s %(module)s - '
+                                      '%(funcName)s: %(message)s',
+                                      '%Y-%m-%d %H:%M:%S')
+        handler.setFormatter(formatter)

         LOG.addHandler(handler)
 
     # ugly, but passable
         LOG.addHandler(handler)
 
     # ugly, but passable


@@ -184,14 +263,21 @@ def main():
     iemap = setup_infoex_counterparts_mapping(station['provider'])
 
     # override units if user selected metric
     iemap = setup_infoex_counterparts_mapping(station['provider'])
 
     # override units if user selected metric

-    if station['units'] == 'metric':
+    if station['provider'] != 'python' and station['units'] == 'metric':

         final_data = switch_units_to_metric(final_data, fmap)
 
         final_data = switch_units_to_metric(final_data, fmap)
 

-    (begin_date, end_date) = setup_time_values()
+    # likewise for "American" units
+    if station['provider'] != 'python' and station['units'] == 'american':
+        final_data = switch_units_to_american(final_data, fmap)

 
 

-    # get the data
-    LOG.debug("Getting %s data from %s to %s", str(station['desired_data']),
-              str(begin_date), str(end_date))
+    (begin_date, end_date) = setup_time_values(station)
+
+    if station['provider'] == 'python':
+        LOG.debug("Getting custom data from external Python program")
+    else:
+        LOG.debug("Getting %s data from %s to %s (%s)",
+                  str(station['desired_data']),
+                  str(begin_date), str(end_date), end_date.tzinfo.zone)

 
     time_all_elements = time.time()
 
 
     time_all_elements = time.time()
 


@@ -201,36 +287,83 @@ def main():
     elif station['provider'] == 'mesowest':
         infoex['wx_data'] = get_mesowest_data(begin_date, end_date,
                                               station)
     elif station['provider'] == 'mesowest':
         infoex['wx_data'] = get_mesowest_data(begin_date, end_date,
                                               station)

+    elif station['provider'] == 'python':
+        try:
+            spec = importlib.util.spec_from_file_location('custom_wx',
+                                                          station['path'])
+            mod = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(mod)
+            mod.LOG = LOG
+
+            try:
+                infoex['wx_data'] = mod.get_custom_data()
+
+                if infoex['wx_data'] is None:
+                    infoex['wx_data'] = []
+            except Exception as exc:
+                LOG.error("Python program for custom Wx data failed in "
+                          "execution: %s", str(exc))
+                sys.exit(1)
+
+            LOG.info("Successfully executed external Python program")
+        except ImportError:
+            LOG.error("Please upgrade to Python 3.3 or later")
+            sys.exit(1)
+        except FileNotFoundError:
+            LOG.error("Specified Python program for custom Wx data "
+                      "was not found")
+            sys.exit(1)
+        except Exception as exc:
+            LOG.error("A problem was encountered when attempting to "
+                      "load your custom Wx program: %s", str(exc))
+            sys.exit(1)

 
     LOG.info("Time taken to get all data : %.3f sec", time.time() -
              time_all_elements)
 
     LOG.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
 
 
     LOG.info("Time taken to get all data : %.3f sec", time.time() -
              time_all_elements)
 
     LOG.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
 

+    # timezone massaging
+    final_end_date = end_date.astimezone(station['tz'])
+

     # Now we only need to add in what we want to change thanks to that
     # abomination of a variable declaration earlier
     final_data[fmap['Location UUID']] = infoex['location_uuid']
     # Now we only need to add in what we want to change thanks to that
     # abomination of a variable declaration earlier
     final_data[fmap['Location UUID']] = infoex['location_uuid']

-    final_data[fmap['obDate']] = end_date.strftime('%m/%d/%Y')
-    final_data[fmap['obTime']] = end_date.strftime('%H:%M')
+    final_data[fmap['obDate']] = final_end_date.strftime('%m/%d/%Y')
+    final_data[fmap['obTime']] = final_end_date.strftime('%H:%M')
+    final_data[fmap['timeZone']] = station['tz'].zone

 
     for element_cd in infoex['wx_data']:
         if element_cd not in iemap:
             LOG.warning("BAD KEY wx_data['%s']", element_cd)
             continue
 
 
     for element_cd in infoex['wx_data']:
         if element_cd not in iemap:
             LOG.warning("BAD KEY wx_data['%s']", element_cd)
             continue
 

+        if infoex['wx_data'][element_cd] is None:
+            continue
+
+        # do the conversion before the rounding
+        if station['provider'] == 'nrcs' and station['units'] == 'metric':
+            infoex['wx_data'][element_cd] = convert_nrcs_units_to_metric(element_cd, infoex['wx_data'][element_cd])
+
+        if station['provider'] != 'python' and station['units'] == 'american':
+            infoex['wx_data'][element_cd] = convert_units_to_american(element_cd, infoex['wx_data'][element_cd])
+

         # Massage precision of certain values to fit InfoEx's
         # expectations
         #
         # Massage precision of certain values to fit InfoEx's
         # expectations
         #

-        # 0 decimal places: wind speed, wind direction, wind gust, snow depth
+        # 0 decimal places: relative humidity, wind speed, wind
+        #                   direction, wind gust, snow depth

         # 1 decimal place:  air temp, baro
         # Avoid transforming None values
         # 1 decimal place:  air temp, baro
         # Avoid transforming None values

-        if infoex['wx_data'][element_cd] is None:
-            continue
-        elif element_cd in ['wind_speed', 'WSPD', 'wind_direction',
-                            'WDIR', 'wind_gust', 'SNWD', 'snow_depth']:
+        if element_cd in ['wind_speed', 'WSPD', 'wind_direction',
+                          'RHUM', 'relative_humidity', 'WDIR',
+                          'wind_gust', 'SNWD', 'snow_depth',
+                          'hn24']:

             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd])
         elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']:
             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1)
             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd])
         elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']:
             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1)

+        elif element_cd in ['PREC', 'precip_accum']:
+            infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 2)

 
         # CONSIDER: Casting every value to Float() -- need to investigate if
         #           any possible elementCds we may want are any other data
 
         # CONSIDER: Casting every value to Float() -- need to investigate if
         #           any possible elementCds we may want are any other data


@@ -245,13 +378,17 @@ def main():
 
     LOG.debug("final_data: %s", str(final_data))
 
 
     LOG.debug("final_data: %s", str(final_data))
 

-    if not write_local_csv(infoex['csv_filename'], final_data):
-        LOG.warning('Could not write local CSV file: %s',
-                    infoex['csv_filename'])
-        return 1
+    if infoex['wx_data']:
+        if not write_local_csv(infoex['csv_filename'], final_data):
+            LOG.warning('Could not write local CSV file: %s',
+                        infoex['csv_filename'])
+            return 1

 
 

-    if not options.dry_run:
-        upload_csv(infoex['csv_filename'], infoex)
+        if not options.dry_run:
+            try:
+                upload_csv(infoex['csv_filename'], infoex)
+            except socket.timeout:
+                LOG.critical("FTP timed out")

 
     LOG.debug('DONE')
     return 0
 
     LOG.debug('DONE')
     return 0


@@ -325,6 +462,10 @@ def setup_infoex_counterparts_mapping(provider):
         iemap['WDIR'] = 'windDirectionNum'
         # unsupported by NRCS:
         # windGustSpeedNum
         iemap['WDIR'] = 'windDirectionNum'
         # unsupported by NRCS:
         # windGustSpeedNum

+
+        # NOTE: this doesn't exist in NRCS SNOTEL, we create it in this
+        #       program, so add it to the map here
+        iemap['hn24'] = 'hn24Auto'

     elif provider == 'mesowest':
         iemap['precip_accum'] = 'precipitationGauge'
         iemap['air_temp'] = 'tempPres'
     elif provider == 'mesowest':
         iemap['precip_accum'] = 'precipitationGauge'
         iemap['air_temp'] = 'tempPres'


@@ -337,15 +478,45 @@ def setup_infoex_counterparts_mapping(provider):
         iemap['wind_direction'] = 'windDirectionNum'
         iemap['wind_gust'] = 'windGustSpeedNum'
 
         iemap['wind_direction'] = 'windDirectionNum'
         iemap['wind_gust'] = 'windGustSpeedNum'
 

+        # NOTE: this doesn't exist in MesoWest, we create it in this
+        #       program, so add it to the map here
+        iemap['hn24'] = 'hn24Auto'
+    elif provider == 'python':
+        # we expect Python programs to use the InfoEx data type names
+        iemap['precipitationGauge'] = 'precipitationGauge'
+        iemap['precipitationGaugeUnit'] = 'precipitationGaugeUnit'
+        iemap['tempPres'] = 'tempPres'
+        iemap['tempPresUnit'] = 'tempPresUnit'
+        iemap['tempMaxHour'] = 'tempMaxHour'
+        iemap['tempMaxHourUnit'] = 'tempMaxHourUnit'
+        iemap['tempMinHour'] = 'tempMinHour'
+        iemap['tempMinHourUnit'] = 'tempMinHourUnit'
+        iemap['hS'] = 'hS'
+        iemap['hsUnit'] = 'hsUnit'
+        iemap['baro'] = 'baro'
+        iemap['rH'] = 'rH'
+        iemap['windSpeedNum'] = 'windSpeedNum'
+        iemap['windSpeedUnit'] = 'windSpeedUnit'
+        iemap['windDirectionNum'] = 'windDirectionNum'
+        iemap['windGustSpeedNum'] = 'windGustSpeedNum'
+        iemap['dewPointUnit'] = 'dewPointUnit'
+        iemap['hn24AutoUnit'] = 'hn24AutoUnit'
+        iemap['hstAutoUnit'] = 'hstAutoUnit'
+

     return iemap
 
 # provider-specific operations
 def get_nrcs_data(begin, end, station):
     """get the data we're after from the NRCS WSDL"""
     transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
     return iemap
 
 # provider-specific operations
 def get_nrcs_data(begin, end, station):
     """get the data we're after from the NRCS WSDL"""
     transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())

+    transport.session.verify = False

     client = zeep.Client(wsdl=station['source'], transport=transport)
     remote_data = {}
 
     client = zeep.Client(wsdl=station['source'], transport=transport)
     remote_data = {}
 

+    # massage begin/end date format
+    begin_date_str = begin.strftime('%Y-%m-%d %H:%M:00')
+    end_date_str = end.strftime('%Y-%m-%d %H:%M:00')
+

     for element_cd in station['desired_data']:
         time_element = time.time()
 
     for element_cd in station['desired_data']:
         time_element = time.time()
 


@@ -354,8 +525,8 @@ def get_nrcs_data(begin, end, station):
             stationTriplets=[station['station_id']],
             elementCd=element_cd,
             ordinal=1,
             stationTriplets=[station['station_id']],
             elementCd=element_cd,
             ordinal=1,

-            beginDate=begin,
-            endDate=end)
+            beginDate=begin_date_str,
+            endDate=end_date_str)

 
         LOG.info("Time to get NRCS elementCd '%s': %.3f sec", element_cd,
                  time.time() - time_element)
 
         LOG.info("Time to get NRCS elementCd '%s': %.3f sec", element_cd,
                  time.time() - time_element)


@@ -373,6 +544,44 @@ def get_nrcs_data(begin, end, station):
         else:
             remote_data[element_cd] = None
 
         else:
             remote_data[element_cd] = None
 

+
+        # calc hn24, if applicable
+        hn24 = None
+
+        if station['hn24']:
+            hn24_values = []
+
+            if element_cd == "SNWD":
+                for idx, _ in enumerate(values):
+                    val = values[idx]
+                    if val is None:
+                        continue
+                    hn24_values.append(val['value'])
+
+                if len(hn24_values) > 0:
+                    # instead of taking MAX - MIN, we want the first
+                    # value (most distant) - the last value (most
+                    # recent)
+                    #
+                    # if the result is positive, then we have
+                    # settlement; if it's not, then we have HN24
+                    hn24 = hn24_values[0] - hn24_values[len(hn24_values)-1]
+
+                    if hn24 < 0.0:
+                        hn24 = abs(hn24)
+                    else:
+                        # this case represents HS settlement
+                        hn24 = 0.0
+
+            # finally, if user wants hn24 and it's set to None at this
+            # point, then force it to 0.0
+            if hn24 is None:
+                hn24 = 0.0
+
+        if hn24 is not None:
+            if station['hn24']:
+                remote_data['hn24'] = hn24
+

     return remote_data
 
 def get_mesowest_data(begin, end, station):
     return remote_data
 
 def get_mesowest_data(begin, end, station):


@@ -385,7 +594,12 @@ def get_mesowest_data(begin, end, station):
 
     # construct final, completed API URL
     api_req_url = station['source'] + '&start=' + begin_date_str + '&end=' + end_date_str
 
     # construct final, completed API URL
     api_req_url = station['source'] + '&start=' + begin_date_str + '&end=' + end_date_str

-    req = requests.get(api_req_url)
+
+    try:
+        req = requests.get(api_req_url)
+    except requests.exceptions.ConnectionError:
+        LOG.error("Could not connect to '%s'", api_req_url)
+        sys.exit(1)

 
     try:
         json = req.json()
 
     try:
         json = req.json()


@@ -395,12 +609,36 @@ def get_mesowest_data(begin, end, station):
 
     try:
         observations = json['STATION'][0]['OBSERVATIONS']
 
     try:
         observations = json['STATION'][0]['OBSERVATIONS']

-    except ValueError:
-        LOG.error("Bad JSON in MesoWest response")
+    except KeyError as exc:
+        LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
+        sys.exit(1)
+    except IndexError as exc:
+        LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
+        try:
+            LOG.error("Detailed MesoWest response: '%s'",
+                      json['SUMMARY']['RESPONSE_MESSAGE'])
+        except KeyError:
+            pass
+        sys.exit(1)
+    except ValueError as exc:
+        LOG.error("Bad JSON in MesoWest response: '%s'", exc)

         sys.exit(1)
 
         sys.exit(1)
 

+    # pos represents the last item in the array, aka the most recent

     pos = len(observations['date_time']) - 1
 
     pos = len(observations['date_time']) - 1
 

+    # while these values only apply in certain cases, init them here
+    wind_speed_values = []
+    wind_gust_speed_values = []
+    wind_direction_values = []
+    hn24_values = []
+
+    # results
+    wind_speed_avg = None
+    wind_gust_speed_avg = None
+    wind_direction_avg = None
+    hn24 = None
+

     for element_cd in station['desired_data'].split(','):
         # sort and isolate the most recent, see note above in NRCS for how and
         # why this is done
     for element_cd in station['desired_data'].split(','):
         # sort and isolate the most recent, see note above in NRCS for how and
         # why this is done


@@ -417,14 +655,104 @@ def get_mesowest_data(begin, end, station):
 
         # we may not have the data at all
         key_name = element_cd + '_set_1'
 
         # we may not have the data at all
         key_name = element_cd + '_set_1'

+

         if key_name in observations:
         if key_name in observations:

-            if observations[key_name][pos]:
-                remote_data[element_cd] = observations[key_name][pos]
-            else:
+            # val is what will make it into the dataset, after
+            # conversions... it gets defined here because in certain
+            # cases we need to look at all of the data to calculate HN24
+            # or wind averages, but for the rest of the data, we only
+            # take the most recent
+            val = None
+
+            # loop through all observations for this key_name
+            # record relevant values for wind averaging or hn24, but
+            # otherwise only persist the data if it's the last datum in
+            # the set
+            for idx, _ in enumerate(observations[key_name]):
+                val = observations[key_name][idx]
+
+                # skip bunk vals
+                if val is None:
+                    continue
+
+                # mesowest by default provides wind_speed in m/s, but
+                # we specify 'english' units in the request; either way,
+                # we want mph
+                if element_cd in ('wind_speed', 'wind_gust'):
+                    val = kn_to_mph(val)
+
+                # mesowest provides HS in mm, not cm; we want cm
+                if element_cd == 'snow_depth' and station['units'] == 'metric':
+                    val = mm_to_cm(val)
+
+                # HN24 / wind_mode transformations, once the data has
+                # completed unit conversions
+                if station['wind_mode'] == "average":
+                    if element_cd == 'wind_speed' and val is not None:
+                        wind_speed_values.append(val)
+                    elif element_cd == 'wind_gust' and val is not None:
+                        wind_gust_speed_values.append(val)
+                    elif element_cd == 'wind_direction' and val is not None:
+                        wind_direction_values.append(val)
+
+                if element_cd == 'snow_depth':
+                    hn24_values.append(val)
+
+                # again, only persist this datum to the final data if
+                # it's from the most recent date
+                if idx == pos:
+                    remote_data[element_cd] = val
+
+            # ensure that the data is filled out
+            if not observations[key_name][pos]:

                 remote_data[element_cd] = None
         else:
             remote_data[element_cd] = None
 
                 remote_data[element_cd] = None
         else:
             remote_data[element_cd] = None
 

+    if len(hn24_values) > 0:
+        # instead of taking MAX - MIN, we want the first value (most
+        # distant) - the last value (most recent)
+        #
+        # if the result is positive, then we have settlement; if it's not,
+        # then we have HN24
+        hn24 = hn24_values[0] - hn24_values[len(hn24_values)-1]
+
+        if hn24 < 0.0:
+            hn24 = abs(hn24)
+        else:
+            # this case represents HS settlement
+            hn24 = 0.0
+
+
+    # finally, if user wants hn24 and it's set to None at this
+    # point, then force it to 0.0
+    if station['hn24'] and hn24 is None:
+        hn24 = 0.0
+
+    if len(wind_speed_values) > 0:
+        wind_speed_avg = sum(wind_speed_values) / len(wind_speed_values) 
+
+    if len(wind_gust_speed_values) > 0:
+        wind_gust_speed_avg = sum(wind_gust_speed_values) / len(wind_gust_speed_values) 
+
+    if len(wind_direction_values) > 0:
+        wind_direction_avg = sum(wind_direction_values) / len(wind_direction_values) 
+
+    if hn24 is not None:
+        if station['hn24']:
+            remote_data['hn24'] = hn24
+
+    # overwrite the following with the respective averages, if
+    # applicable
+    if wind_speed_avg is not None:
+        remote_data['wind_speed'] = wind_speed_avg
+
+    if wind_gust_speed_avg is not None:
+        remote_data['wind_gust'] = wind_gust_speed_avg
+
+    if wind_direction_avg is not None:
+        remote_data['wind_direction'] = wind_direction_avg
+

     return remote_data
 
 def switch_units_to_metric(data_map, mapping):
     return remote_data
 
 def switch_units_to_metric(data_map, mapping):


@@ -432,18 +760,85 @@ def switch_units_to_metric(data_map, mapping):
 
     # NOTE: to update this, use the fmap<->final_data mapping laid out
     #       in setup_infoex_fields_mapping ()
 
     # NOTE: to update this, use the fmap<->final_data mapping laid out
     #       in setup_infoex_fields_mapping ()

-    #
-    # NOTE: this only 'works' with MesoWest for now, as the MesoWest API
-    #       itself handles the unit conversion; in the future, we will also
-    #       support NRCS unit conversion, but this must be done by this
-    #       program.
+    data_map[mapping['tempMaxHourUnit']] = 'C'
+    data_map[mapping['tempMinHourUnit']] = 'C'

     data_map[mapping['tempPresUnit']] = 'C'
     data_map[mapping['tempPresUnit']] = 'C'

-    data_map[mapping['hsUnit']] = 'm'
+    data_map[mapping['precipitationGaugeUnit']] = 'mm'
+    data_map[mapping['hsUnit']] = 'cm'

     data_map[mapping['windSpeedUnit']] = 'm/s'
     data_map[mapping['windGustSpeedNumUnit']] = 'm/s'
     data_map[mapping['windSpeedUnit']] = 'm/s'
     data_map[mapping['windGustSpeedNumUnit']] = 'm/s'

+    data_map[mapping['dewPointUnit']] = 'C'
+    data_map[mapping['hn24AutoUnit']] = 'cm'
+    data_map[mapping['hstAutoUnit']] = 'cm'

 
     return data_map
 
 
     return data_map
 

+def switch_units_to_american(data_map, mapping):
+    """
+    replace units with the American mixture of metric and imperial
+
+    Precip values = metric
+    Wind values = imperial
+    """
+
+    # precip values
+    data_map[mapping['tempMaxHourUnit']] = 'C'
+    data_map[mapping['tempMinHourUnit']] = 'C'
+    data_map[mapping['tempPresUnit']] = 'C'
+    data_map[mapping['dewPointUnit']] = 'C'
+
+    data_map[mapping['precipitationGaugeUnit']] = 'cm'
+    data_map[mapping['hsUnit']] = 'cm'
+    data_map[mapping['hn24AutoUnit']] = 'cm'
+    data_map[mapping['hstAutoUnit']] = 'cm'
+
+    data_map[mapping['baroUnit']] = 'inHg'
+
+    # wind values
+    data_map[mapping['windSpeedUnit']] = 'mph'
+    data_map[mapping['windGustSpeedNumUnit']] = 'mph'
+
+    return data_map
+
+def convert_nrcs_units_to_metric(element_cd, value):
+    """convert NRCS values from English to metric"""
+    if element_cd == 'TOBS':
+        value = f_to_c(value)
+    elif element_cd == 'SNWD':
+        value = in_to_cm(value)
+    elif element_cd == 'PREC':
+        value = in_to_mm(value)
+    return value
+
+def convert_units_to_american(element_cd, value):
+    """
+    convert value to 'American' units
+
+    The original unit is always metric.
+
+    Precip values = metric
+    Wind values = imperial
+    """
+
+    # no need to convert precip values, as they will arrive in metric
+    # units in "American" units mode
+    # # temp values
+    # if element_cd in ['TMAX', 'TMIN', 'TOBS', 'air_temp', 'air_temp_high_24_hour', 'air_temp_low_24_hour']:
+    #     value = c_to_f(value)
+
+    # mesowest provides HS in mm, not cm; we want cm
+    if element_cd == 'snow_depth':
+        value = mm_to_cm(value)
+
+    # baro values also arrive in metric, so convert to imperial
+    if element_cd in ['PRES', 'pressure']:
+        value = inhg_to_pascal(value)
+
+    if element_cd in ['WSPD', 'wind_speed', 'wind_gust']:
+        value = ms_to_mph(value)
+
+    return value
+

 # CSV operations
 def write_local_csv(path_to_file, data):
     """Write the specified CSV file to disk"""
 # CSV operations
 def write_local_csv(path_to_file, data):
     """Write the specified CSV file to disk"""


@@ -461,22 +856,70 @@ def upload_csv(path_to_file, infoex_data):
     with open(path_to_file, 'rb') as file_object:
         LOG.debug("uploading FTP file '%s'", infoex_data['host'])
         ftp = FTP(infoex_data['host'], infoex_data['uuid'],
     with open(path_to_file, 'rb') as file_object:
         LOG.debug("uploading FTP file '%s'", infoex_data['host'])
         ftp = FTP(infoex_data['host'], infoex_data['uuid'],

-                  infoex_data['api_key'])
+                  infoex_data['api_key'], timeout=10)

         ftp.storlines('STOR ' + path_to_file, file_object)
         ftp.close()
         file_object.close()
     os.remove(path_to_file)
 
 # other miscellaneous routines
         ftp.storlines('STOR ' + path_to_file, file_object)
         ftp.close()
         file_object.close()
     os.remove(path_to_file)
 
 # other miscellaneous routines

-def setup_time_values():
+def setup_time_values(station):

     """establish time bounds of data request(s)"""
     """establish time bounds of data request(s)"""

+
+    # default timezone to UTC (for MesoWest)
+    tz = pytz.utc
+
+    # but for NRCS, use the config-specified timezone
+    if station['provider'] == 'nrcs':
+        tz = station['tz']
+

     # floor time to nearest hour
     # floor time to nearest hour

-    date_time = datetime.datetime.now()
+    date_time = datetime.datetime.now(tz=tz)

     end_date = date_time - datetime.timedelta(minutes=date_time.minute % 60,
                                               seconds=date_time.second,
                                               microseconds=date_time.microsecond)
     end_date = date_time - datetime.timedelta(minutes=date_time.minute % 60,
                                               seconds=date_time.second,
                                               microseconds=date_time.microsecond)

-    begin_date = end_date - datetime.timedelta(hours=3)
+    begin_date = end_date - datetime.timedelta(hours=station['num_hrs_to_fetch'])

     return (begin_date, end_date)
 
     return (begin_date, end_date)
 

+def f_to_c(f):
+    """convert Fahrenheit to Celsius"""
+    return (float(f) - 32) * 5.0/9.0
+
+def c_to_f(c):
+    """convert Celsius to Fahrenheit"""
+    return (float(c) * 1.8) + 32
+
+def in_to_cm(inches):
+    """convert inches to centimeters"""
+    return float(inches) * 2.54
+
+def cm_to_in(cms):
+    """convert centimeters to inches"""
+    return float(cms) / 2.54
+
+def pascal_to_inhg(pa):
+    """convert pascals to inches of mercury"""
+    return float(pa) * 0.00029530
+
+def inhg_to_pascal(inhg):
+    """convert inches of mercury to pascals"""
+    return float(inhg) / 0.00029530
+
+def in_to_mm(inches):
+    """convert inches to millimeters"""
+    return (float(inches) * 2.54) * 10.0
+
+def ms_to_mph(ms):
+    """convert meters per second to miles per hour"""
+    return ms * 2.236936
+
+def kn_to_mph(kn):
+    """convert knots to miles per hour"""
+    return kn * 1.150779
+
+def mm_to_cm(mm):
+    """convert millimeters to centimeters"""
+    return mm / 10.0
+

 if __name__ == "__main__":
     sys.exit(main())
 if __name__ == "__main__":
     sys.exit(main())