Merge branch 'nrcs-unit-conversion' into develop

[infoex-autowx.git] / infoex-autowx.py

diff --git a/infoex-autowx.py b/infoex-autowx.py
index 439ff19d0dd05fc4777d8c5ab2e4aab6698e8609..1cdc51396862904957d1e803fe60b4db404d1bfa 100755 (executable)
--- a/infoex-autowx.py
+++ b/infoex-autowx.py
@@ -29,21 +29,27 @@ import logging
 import os
 import sys
 import time
 import os
 import sys
 import time

+import urllib3
+import importlib.util

 
 from ftplib import FTP
 from argparse import ArgumentParser
 
 
 from ftplib import FTP
 from argparse import ArgumentParser
 

+import pytz
+

 import requests
 
 import zeep
 import zeep.cache
 import zeep.transports
 
 import requests
 
 import zeep
 import zeep.cache
 import zeep.transports
 

-__version__ = '3.0.1'
+__version__ = '3.1.1'

 
 LOG = logging.getLogger(__name__)
 LOG.setLevel(logging.NOTSET)
 
 
 LOG = logging.getLogger(__name__)
 LOG.setLevel(logging.NOTSET)
 

+urllib3.disable_warnings()
+

 def get_parser():
     """Return OptionParser for this program"""
     parser = ArgumentParser()
 def get_parser():
     """Return OptionParser for this program"""
     parser = ArgumentParser()


@@ -93,11 +99,7 @@ def setup_config(config):
             station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
             station['station_id'] = config['station']['station_id']
             station['desired_data'] = config['station']['desired_data'].split(',')
             station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
             station['station_id'] = config['station']['station_id']
             station['desired_data'] = config['station']['desired_data'].split(',')

-
-            # XXX: For NRCS, we're manually overriding units for now! Once
-            #      unit conversion is supported for NRCS, REMOVE THIS!
-            if 'units' not in station:
-                station['units'] = 'imperial'
+            station['units'] = config['station']['units']

 
         if station['provider'] == 'mesowest':
             station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
 
         if station['provider'] == 'mesowest':
             station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'


@@ -115,6 +117,17 @@ def setup_config(config):
         if station['provider'] == 'python':
             station['path'] = config['station']['path']
 
         if station['provider'] == 'python':
             station['path'] = config['station']['path']
 

+        tz = 'America/Los_Angeles'
+
+        if 'tz' in config['station']:
+            tz = config['station']['tz']
+
+        try:
+            station['tz'] = pytz.timezone(tz)
+        except pytz.exceptions.UnknownTimeZoneError:
+            LOG.critical("%s is not a valid timezone", tz)
+            sys.exit(1)
+

     except KeyError as err:
         LOG.critical("%s not defined in configuration file", err)
         sys.exit(1)
     except KeyError as err:
         LOG.critical("%s not defined in configuration file", err)
         sys.exit(1)


@@ -187,23 +200,23 @@ def main():
     iemap = setup_infoex_counterparts_mapping(station['provider'])
 
     # override units if user selected metric
     iemap = setup_infoex_counterparts_mapping(station['provider'])
 
     # override units if user selected metric

-    try:
-        if station['units'] == 'metric':
-            final_data = switch_units_to_metric(final_data, fmap)
-    except KeyError:
+    if 'units' not in station:

         if station['provider'] != 'python':
             LOG.error("Please specify the units in the configuration "
                       "file")
             sys.exit(1)
 
         if station['provider'] != 'python':
             LOG.error("Please specify the units in the configuration "
                       "file")
             sys.exit(1)
 

-    (begin_date, end_date) = setup_time_values(station['provider'])
+    if station['units'] == 'metric':
+        final_data = switch_units_to_metric(final_data, fmap)
+
+    (begin_date, end_date) = setup_time_values(station)

 
     if station['provider'] == 'python':
         LOG.debug("Getting custom data from external Python program")
     else:
 
     if station['provider'] == 'python':
         LOG.debug("Getting custom data from external Python program")
     else:

-        LOG.debug("Getting %s data from %s to %s",
+        LOG.debug("Getting %s data from %s to %s (%s)",

                   str(station['desired_data']),
                   str(station['desired_data']),

-                  str(begin_date), str(end_date))
+                  str(begin_date), str(end_date), end_date.tzinfo.zone)

 
     time_all_elements = time.time()
 
 
     time_all_elements = time.time()
 


@@ -215,8 +228,6 @@ def main():
                                               station)
     elif station['provider'] == 'python':
         try:
                                               station)
     elif station['provider'] == 'python':
         try:

-            import importlib.util
-

             spec = importlib.util.spec_from_file_location('custom_wx',
                                                           station['path'])
             mod = importlib.util.module_from_spec(spec)
             spec = importlib.util.spec_from_file_location('custom_wx',
                                                           station['path'])
             mod = importlib.util.module_from_spec(spec)


@@ -251,30 +262,39 @@ def main():
 
     LOG.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
 
 
     LOG.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
 

+    # timezone massaging
+    final_end_date = end_date.astimezone(station['tz'])
+

     # Now we only need to add in what we want to change thanks to that
     # abomination of a variable declaration earlier
     final_data[fmap['Location UUID']] = infoex['location_uuid']
     # Now we only need to add in what we want to change thanks to that
     # abomination of a variable declaration earlier
     final_data[fmap['Location UUID']] = infoex['location_uuid']

-    final_data[fmap['obDate']] = end_date.strftime('%m/%d/%Y')
-    final_data[fmap['obTime']] = end_date.strftime('%H:%M')
+    final_data[fmap['obDate']] = final_end_date.strftime('%m/%d/%Y')
+    final_data[fmap['obTime']] = final_end_date.strftime('%H:%M')
+    final_data[fmap['timeZone']] = station['tz'].zone

 
     for element_cd in infoex['wx_data']:
         if element_cd not in iemap:
             LOG.warning("BAD KEY wx_data['%s']", element_cd)
             continue
 
 
     for element_cd in infoex['wx_data']:
         if element_cd not in iemap:
             LOG.warning("BAD KEY wx_data['%s']", element_cd)
             continue
 

+        # do the conversion before the rounding
+        if station['provider'] == 'nrcs' and station['units'] == 'metric':
+            infoex['wx_data'][element_cd] = convert_nrcs_units_to_metric(element_cd, infoex['wx_data'][element_cd])
+

         # Massage precision of certain values to fit InfoEx's
         # expectations
         #
         # Massage precision of certain values to fit InfoEx's
         # expectations
         #

-        # 0 decimal places: wind speed, wind direction, wind gust, snow depth
-        # 1 decimal place:  air temp, relative humidity, baro
+        # 0 decimal places: relative humidity, wind speed, wind
+        #                   direction, wind gust, snow depth
+        # 1 decimal place:  air temp, baro

         # Avoid transforming None values
         if infoex['wx_data'][element_cd] is None:
             continue
         elif element_cd in ['wind_speed', 'WSPD', 'wind_direction',
         # Avoid transforming None values
         if infoex['wx_data'][element_cd] is None:
             continue
         elif element_cd in ['wind_speed', 'WSPD', 'wind_direction',

-                            'WDIR', 'wind_gust', 'SNWD', 'snow_depth']:
+                            'RHUM', 'relative_humidity', 'WDIR',
+                            'wind_gust', 'SNWD', 'snow_depth']:

             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd])
             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd])

-        elif element_cd in ['TOBS', 'air_temp', 'RHUM',
-                            'relative_humidity', 'PRES', 'pressure']:
+        elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']:

             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1)
 
         # CONSIDER: Casting every value to Float() -- need to investigate if
             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1)
 
         # CONSIDER: Casting every value to Float() -- need to investigate if


@@ -401,6 +421,7 @@ def setup_infoex_counterparts_mapping(provider):
 def get_nrcs_data(begin, end, station):
     """get the data we're after from the NRCS WSDL"""
     transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
 def get_nrcs_data(begin, end, station):
     """get the data we're after from the NRCS WSDL"""
     transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())

+    transport.session.verify = False

     client = zeep.Client(wsdl=station['source'], transport=transport)
     remote_data = {}
 
     client = zeep.Client(wsdl=station['source'], transport=transport)
     remote_data = {}
 


@@ -457,8 +478,19 @@ def get_mesowest_data(begin, end, station):
 
     try:
         observations = json['STATION'][0]['OBSERVATIONS']
 
     try:
         observations = json['STATION'][0]['OBSERVATIONS']

-    except ValueError:
-        LOG.error("Bad JSON in MesoWest response")
+    except KeyError as exc:
+        LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
+        sys.exit(1)
+    except IndexError as exc:
+        LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
+        try:
+            LOG.error("Detailed MesoWest response: '%s'",
+                      json['SUMMARY']['RESPONSE_MESSAGE'])
+        except KeyError:
+            pass
+        sys.exit(1)
+    except ValueError as exc:
+        LOG.error("Bad JSON in MesoWest response: '%s'", exc)

         sys.exit(1)
 
     pos = len(observations['date_time']) - 1
         sys.exit(1)
 
     pos = len(observations['date_time']) - 1


@@ -501,18 +533,29 @@ def switch_units_to_metric(data_map, mapping):
 
     # NOTE: to update this, use the fmap<->final_data mapping laid out
     #       in setup_infoex_fields_mapping ()
 
     # NOTE: to update this, use the fmap<->final_data mapping laid out
     #       in setup_infoex_fields_mapping ()

-    #
-    # NOTE: this only 'works' with MesoWest for now, as the MesoWest API
-    #       itself handles the unit conversion; in the future, we will also
-    #       support NRCS unit conversion, but this must be done by this
-    #       program.
+    data_map[mapping['tempMaxHourUnit']] = 'C'
+    data_map[mapping['tempMinHourUnit']] = 'C'

     data_map[mapping['tempPresUnit']] = 'C'
     data_map[mapping['tempPresUnit']] = 'C'

-    data_map[mapping['hsUnit']] = 'm'
+    data_map[mapping['precipitationGaugeUnit']] = 'mm'
+    data_map[mapping['hsUnit']] = 'cm'

     data_map[mapping['windSpeedUnit']] = 'm/s'
     data_map[mapping['windGustSpeedNumUnit']] = 'm/s'
     data_map[mapping['windSpeedUnit']] = 'm/s'
     data_map[mapping['windGustSpeedNumUnit']] = 'm/s'

+    data_map[mapping['dewPointUnit']] = 'C'
+    data_map[mapping['hn24AutoUnit']] = 'cm'
+    data_map[mapping['hstAutoUnit']] = 'cm'

 
     return data_map
 
 
     return data_map
 

+def convert_nrcs_units_to_metric(element_cd, value):
+    """convert NRCS values from English to metric"""
+    if element_cd == 'TOBS':
+        value = f_to_c(value)
+    elif element_cd == 'SNWD':
+        value = in_to_cm(value)
+    elif element_cd == 'PREC':
+        value = in_to_mm(value)
+    return value
+

 # CSV operations
 def write_local_csv(path_to_file, data):
     """Write the specified CSV file to disk"""
 # CSV operations
 def write_local_csv(path_to_file, data):
     """Write the specified CSV file to disk"""


@@ -537,14 +580,15 @@ def upload_csv(path_to_file, infoex_data):
     os.remove(path_to_file)
 
 # other miscellaneous routines
     os.remove(path_to_file)
 
 # other miscellaneous routines

-def setup_time_values(provider):
+def setup_time_values(station):

     """establish time bounds of data request(s)"""
 
     # default timezone to UTC (for MesoWest)
     """establish time bounds of data request(s)"""
 
     # default timezone to UTC (for MesoWest)

-    tz = datetime.timezone.utc
+    tz = pytz.utc

 
 

-    if provider == 'nrcs':
-        tz = None
+    # but for NRCS, use the config-specified timezone
+    if station['provider'] == 'nrcs':
+        tz = station['tz']

 
     # floor time to nearest hour
     date_time = datetime.datetime.now(tz=tz)
 
     # floor time to nearest hour
     date_time = datetime.datetime.now(tz=tz)


@@ -554,6 +598,18 @@ def setup_time_values(provider):
     begin_date = end_date - datetime.timedelta(hours=3)
     return (begin_date, end_date)
 
     begin_date = end_date - datetime.timedelta(hours=3)
     return (begin_date, end_date)
 

+def f_to_c(f):
+    """convert Fahrenheit to Celsius"""
+    return (float(f) - 32) * 5.0/9.0
+
+def in_to_cm(inches):
+    """convert inches to centimetrs"""
+    return float(inches) * 2.54
+
+def in_to_mm(inches):
+    """convert inches to millimeters"""
+    return (float(inches) * 2.54) * 10.0
+

 def ms_to_mph(ms):
     """convert meters per second to miles per hour"""
     return ms * 2.236936
 def ms_to_mph(ms):
     """convert meters per second to miles per hour"""
     return ms * 2.236936