Merge branch 'various-small-chores' into develop

[infoex-autowx.git] / infoex-autowx.py

diff --git a/infoex-autowx.py b/infoex-autowx.py
index af1ae31344a9c2074fe040b550356cadb182d123..1844d4574682202fa00307d06c13510aab1a6e2f 100755 (executable)
--- a/infoex-autowx.py
+++ b/infoex-autowx.py
@@ -43,7 +43,7 @@ import zeep
 import zeep.cache
 import zeep.transports
 
 import zeep.cache
 import zeep.transports
 

-__version__ = '3.1.1'
+__version__ = '3.2.1'

 
 LOG = logging.getLogger(__name__)
 LOG.setLevel(logging.NOTSET)
 
 LOG = logging.getLogger(__name__)
 LOG.setLevel(logging.NOTSET)


@@ -99,11 +99,7 @@ def setup_config(config):
             station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
             station['station_id'] = config['station']['station_id']
             station['desired_data'] = config['station']['desired_data'].split(',')
             station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
             station['station_id'] = config['station']['station_id']
             station['desired_data'] = config['station']['desired_data'].split(',')

-
-            # XXX: For NRCS, we're manually overriding units for now! Once
-            #      unit conversion is supported for NRCS, REMOVE THIS!
-            if 'units' not in station:
-                station['units'] = 'imperial'
+            station['units'] = config['station']['units']

 
         if station['provider'] == 'mesowest':
             station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
 
         if station['provider'] == 'mesowest':
             station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'


@@ -159,6 +155,10 @@ def setup_logging(log_level):
         #LOG.addHandler(logging.handlers.SysLogHandler())
         # fallback to stdout
         handler = logging.StreamHandler(sys.stdout)
         #LOG.addHandler(logging.handlers.SysLogHandler())
         # fallback to stdout
         handler = logging.StreamHandler(sys.stdout)

+        formatter = logging.Formatter('%(asctime)s.%(msecs)03d '
+            '%(levelname)s %(module)s - %(funcName)s: %(message)s',
+            '%Y-%m-%d %H:%M:%S')
+        handler.setFormatter(formatter)

         LOG.addHandler(handler)
 
     # ugly, but passable
         LOG.addHandler(handler)
 
     # ugly, but passable


@@ -204,14 +204,8 @@ def main():
     iemap = setup_infoex_counterparts_mapping(station['provider'])
 
     # override units if user selected metric
     iemap = setup_infoex_counterparts_mapping(station['provider'])
 
     # override units if user selected metric

-    try:
-        if station['units'] == 'metric':
-            final_data = switch_units_to_metric(final_data, fmap)
-    except KeyError:
-        if station['provider'] != 'python':
-            LOG.error("Please specify the units in the configuration "
-                      "file")
-            sys.exit(1)
+    if station['provider'] != 'python' and station['units'] == 'metric':
+        final_data = switch_units_to_metric(final_data, fmap)

 
     (begin_date, end_date) = setup_time_values(station)
 
 
     (begin_date, end_date) = setup_time_values(station)
 


@@ -274,12 +268,17 @@ def main():
     final_data[fmap['Location UUID']] = infoex['location_uuid']
     final_data[fmap['obDate']] = final_end_date.strftime('%m/%d/%Y')
     final_data[fmap['obTime']] = final_end_date.strftime('%H:%M')
     final_data[fmap['Location UUID']] = infoex['location_uuid']
     final_data[fmap['obDate']] = final_end_date.strftime('%m/%d/%Y')
     final_data[fmap['obTime']] = final_end_date.strftime('%H:%M')

+    final_data[fmap['timeZone']] = station['tz'].zone

 
     for element_cd in infoex['wx_data']:
         if element_cd not in iemap:
             LOG.warning("BAD KEY wx_data['%s']", element_cd)
             continue
 
 
     for element_cd in infoex['wx_data']:
         if element_cd not in iemap:
             LOG.warning("BAD KEY wx_data['%s']", element_cd)
             continue
 

+        # do the conversion before the rounding
+        if station['provider'] == 'nrcs' and station['units'] == 'metric':
+            infoex['wx_data'][element_cd] = convert_nrcs_units_to_metric(element_cd, infoex['wx_data'][element_cd])
+

         # Massage precision of certain values to fit InfoEx's
         # expectations
         #
         # Massage precision of certain values to fit InfoEx's
         # expectations
         #


@@ -295,6 +294,8 @@ def main():
             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd])
         elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']:
             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1)
             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd])
         elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']:
             infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1)

+        elif element_cd in ['PREC', 'precip_accum']:
+            infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 2)

 
         # CONSIDER: Casting every value to Float() -- need to investigate if
         #           any possible elementCds we may want are any other data
 
         # CONSIDER: Casting every value to Float() -- need to investigate if
         #           any possible elementCds we may want are any other data


@@ -467,7 +468,12 @@ def get_mesowest_data(begin, end, station):
 
     # construct final, completed API URL
     api_req_url = station['source'] + '&start=' + begin_date_str + '&end=' + end_date_str
 
     # construct final, completed API URL
     api_req_url = station['source'] + '&start=' + begin_date_str + '&end=' + end_date_str

-    req = requests.get(api_req_url)
+
+    try:
+        req = requests.get(api_req_url)
+    except requests.exceptions.ConnectionError:
+        LOG.error("Could not connect to '%s'", api_req_url)
+        sys.exit(1)

 
     try:
         json = req.json()
 
     try:
         json = req.json()


@@ -532,18 +538,29 @@ def switch_units_to_metric(data_map, mapping):
 
     # NOTE: to update this, use the fmap<->final_data mapping laid out
     #       in setup_infoex_fields_mapping ()
 
     # NOTE: to update this, use the fmap<->final_data mapping laid out
     #       in setup_infoex_fields_mapping ()

-    #
-    # NOTE: this only 'works' with MesoWest for now, as the MesoWest API
-    #       itself handles the unit conversion; in the future, we will also
-    #       support NRCS unit conversion, but this must be done by this
-    #       program.
+    data_map[mapping['tempMaxHourUnit']] = 'C'
+    data_map[mapping['tempMinHourUnit']] = 'C'

     data_map[mapping['tempPresUnit']] = 'C'
     data_map[mapping['tempPresUnit']] = 'C'

-    data_map[mapping['hsUnit']] = 'm'
+    data_map[mapping['precipitationGaugeUnit']] = 'mm'
+    data_map[mapping['hsUnit']] = 'cm'

     data_map[mapping['windSpeedUnit']] = 'm/s'
     data_map[mapping['windGustSpeedNumUnit']] = 'm/s'
     data_map[mapping['windSpeedUnit']] = 'm/s'
     data_map[mapping['windGustSpeedNumUnit']] = 'm/s'

+    data_map[mapping['dewPointUnit']] = 'C'
+    data_map[mapping['hn24AutoUnit']] = 'cm'
+    data_map[mapping['hstAutoUnit']] = 'cm'

 
     return data_map
 
 
     return data_map
 

+def convert_nrcs_units_to_metric(element_cd, value):
+    """convert NRCS values from English to metric"""
+    if element_cd == 'TOBS':
+        value = f_to_c(value)
+    elif element_cd == 'SNWD':
+        value = in_to_cm(value)
+    elif element_cd == 'PREC':
+        value = in_to_mm(value)
+    return value
+

 # CSV operations
 def write_local_csv(path_to_file, data):
     """Write the specified CSV file to disk"""
 # CSV operations
 def write_local_csv(path_to_file, data):
     """Write the specified CSV file to disk"""


@@ -586,6 +603,18 @@ def setup_time_values(station):
     begin_date = end_date - datetime.timedelta(hours=3)
     return (begin_date, end_date)
 
     begin_date = end_date - datetime.timedelta(hours=3)
     return (begin_date, end_date)
 

+def f_to_c(f):
+    """convert Fahrenheit to Celsius"""
+    return (float(f) - 32) * 5.0/9.0
+
+def in_to_cm(inches):
+    """convert inches to centimetrs"""
+    return float(inches) * 2.54
+
+def in_to_mm(inches):
+    """convert inches to millimeters"""
+    return (float(inches) * 2.54) * 10.0
+

 def ms_to_mph(ms):
     """convert meters per second to miles per hour"""
     return ms * 2.236936
 def ms_to_mph(ms):
     """convert meters per second to miles per hour"""
     return ms * 2.236936