X-Git-Url: https://wylark.com/src/infoex-autowx.git/blobdiff_plain/1b6860bd5fcb34a7cf90347983af8c685f23e8ee..93fc527eae305d5a32c30d7a38272668894b2812:/infoex-autowx.py diff --git a/infoex-autowx.py b/infoex-autowx.py index 05e1d46..9df112c 100755 --- a/infoex-autowx.py +++ b/infoex-autowx.py @@ -27,6 +27,7 @@ import csv import datetime import logging import os +import socket import sys import time import urllib3 @@ -43,7 +44,7 @@ import zeep import zeep.cache import zeep.transports -__version__ = '3.3.1' +__version__ = '3.4.5' LOG = logging.getLogger(__name__) LOG.setLevel(logging.NOTSET) @@ -89,28 +90,49 @@ def setup_config(config): } station = dict() + station['provider'] = config['station']['type'] if station['provider'] not in ['nrcs', 'mesowest', 'python']: print("Please specify either nrcs or mesowest as the station type.") sys.exit(1) + # massage units config items first + # + # NOTE: custom providers don't require units to be specified + # because they can do whatever they please with the units + # within their own program + if station['provider'] != "python": + station['units'] = config['station']['units'] + + if station['units'] not in ['metric', 'english', 'american']: + print("Please specify metric, english, or american for the units.") + sys.exit(1) + else: + # if units are specified as "American" then we simply + # default to metric for the requests + if station['units'] == 'american': + station['units_requested'] = 'metric' + else: + station['units_requested'] = station['units'] + + # massage provider config items if station['provider'] == 'nrcs': - station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL' + #station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL' + station['source'] = 'https://wcc.sc.egov.usda.gov/awdbWebService/services?WSDL' + station['station_id'] = config['station']['station_id'] station['desired_data'] = config['station']['desired_data'].split(',') - station['units'] = config['station']['units'] if station['provider'] == 'mesowest': station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries' station['station_id'] = config['station']['station_id'] - station['units'] = config['station']['units'] station['desired_data'] = config['station']['desired_data'] # construct full API URL (sans start/end time, added later) station['source'] = station['source'] + '?token=' + \ config['station']['token'] + \ - '&within=60&units=' + station['units'] + \ + '&within=60&units=' + station['units_requested'] + \ '&stid=' + station['station_id'] + \ '&vars=' + station['desired_data'] @@ -244,6 +266,10 @@ def main(): if station['provider'] != 'python' and station['units'] == 'metric': final_data = switch_units_to_metric(final_data, fmap) + # likewise for "American" units + if station['provider'] != 'python' and station['units'] == 'american': + final_data = switch_units_to_american(final_data, fmap) + (begin_date, end_date) = setup_time_values(station) if station['provider'] == 'python': @@ -319,6 +345,9 @@ def main(): if station['provider'] == 'nrcs' and station['units'] == 'metric': infoex['wx_data'][element_cd] = convert_nrcs_units_to_metric(element_cd, infoex['wx_data'][element_cd]) + if station['provider'] != 'python' and station['units'] == 'american': + infoex['wx_data'][element_cd] = convert_units_to_american(element_cd, infoex['wx_data'][element_cd]) + # Massage precision of certain values to fit InfoEx's # expectations # @@ -356,7 +385,10 @@ def main(): return 1 if not options.dry_run: - upload_csv(infoex['csv_filename'], infoex) + try: + upload_csv(infoex['csv_filename'], infoex) + except socket.timeout: + LOG.critical("FTP timed out") LOG.debug('DONE') return 0 @@ -452,15 +484,24 @@ def setup_infoex_counterparts_mapping(provider): elif provider == 'python': # we expect Python programs to use the InfoEx data type names iemap['precipitationGauge'] = 'precipitationGauge' + iemap['precipitationGaugeUnit'] = 'precipitationGaugeUnit' iemap['tempPres'] = 'tempPres' + iemap['tempPresUnit'] = 'tempPresUnit' iemap['tempMaxHour'] = 'tempMaxHour' + iemap['tempMaxHourUnit'] = 'tempMaxHourUnit' iemap['tempMinHour'] = 'tempMinHour' + iemap['tempMinHourUnit'] = 'tempMinHourUnit' iemap['hS'] = 'hS' + iemap['hsUnit'] = 'hsUnit' iemap['baro'] = 'baro' iemap['rH'] = 'rH' iemap['windSpeedNum'] = 'windSpeedNum' + iemap['windSpeedUnit'] = 'windSpeedUnit' iemap['windDirectionNum'] = 'windDirectionNum' iemap['windGustSpeedNum'] = 'windGustSpeedNum' + iemap['dewPointUnit'] = 'dewPointUnit' + iemap['hn24AutoUnit'] = 'hn24AutoUnit' + iemap['hstAutoUnit'] = 'hstAutoUnit' return iemap @@ -538,7 +579,8 @@ def get_nrcs_data(begin, end, station): hn24 = 0.0 if hn24 is not None: - remote_data['hn24'] = hn24 + if station['hn24']: + remote_data['hn24'] = hn24 return remote_data @@ -697,7 +739,8 @@ def get_mesowest_data(begin, end, station): wind_direction_avg = sum(wind_direction_values) / len(wind_direction_values) if hn24 is not None: - remote_data['hn24'] = hn24 + if station['hn24']: + remote_data['hn24'] = hn24 # overwrite the following with the respective averages, if # applicable @@ -730,6 +773,33 @@ def switch_units_to_metric(data_map, mapping): return data_map +def switch_units_to_american(data_map, mapping): + """ + replace units with the American mixture of metric and imperial + + Precip values = metric + Wind values = imperial + """ + + # precip values + data_map[mapping['tempMaxHourUnit']] = 'C' + data_map[mapping['tempMinHourUnit']] = 'C' + data_map[mapping['tempPresUnit']] = 'C' + data_map[mapping['dewPointUnit']] = 'C' + + data_map[mapping['precipitationGaugeUnit']] = 'cm' + data_map[mapping['hsUnit']] = 'cm' + data_map[mapping['hn24AutoUnit']] = 'cm' + data_map[mapping['hstAutoUnit']] = 'cm' + + data_map[mapping['baroUnit']] = 'inHg' + + # wind values + data_map[mapping['windSpeedUnit']] = 'mph' + data_map[mapping['windGustSpeedNumUnit']] = 'mph' + + return data_map + def convert_nrcs_units_to_metric(element_cd, value): """convert NRCS values from English to metric""" if element_cd == 'TOBS': @@ -740,6 +810,35 @@ def convert_nrcs_units_to_metric(element_cd, value): value = in_to_mm(value) return value +def convert_units_to_american(element_cd, value): + """ + convert value to 'American' units + + The original unit is always metric. + + Precip values = metric + Wind values = imperial + """ + + # no need to convert precip values, as they will arrive in metric + # units in "American" units mode + # # temp values + # if element_cd in ['TMAX', 'TMIN', 'TOBS', 'air_temp', 'air_temp_high_24_hour', 'air_temp_low_24_hour']: + # value = c_to_f(value) + + # mesowest provides HS in mm, not cm; we want cm + if element_cd == 'snow_depth': + value = mm_to_cm(value) + + # baro values also arrive in metric, so convert to imperial + if element_cd in ['PRES', 'pressure']: + value = inhg_to_pascal(value) + + if element_cd in ['WSPD', 'wind_speed', 'wind_gust']: + value = ms_to_mph(value) + + return value + # CSV operations def write_local_csv(path_to_file, data): """Write the specified CSV file to disk""" @@ -757,7 +856,7 @@ def upload_csv(path_to_file, infoex_data): with open(path_to_file, 'rb') as file_object: LOG.debug("uploading FTP file '%s'", infoex_data['host']) ftp = FTP(infoex_data['host'], infoex_data['uuid'], - infoex_data['api_key']) + infoex_data['api_key'], timeout=10) ftp.storlines('STOR ' + path_to_file, file_object) ftp.close() file_object.close() @@ -786,10 +885,26 @@ def f_to_c(f): """convert Fahrenheit to Celsius""" return (float(f) - 32) * 5.0/9.0 +def c_to_f(c): + """convert Celsius to Fahrenheit""" + return (float(c) * 1.8) + 32 + def in_to_cm(inches): - """convert inches to centimetrs""" + """convert inches to centimeters""" return float(inches) * 2.54 +def cm_to_in(cms): + """convert centimeters to inches""" + return float(cms) / 2.54 + +def pascal_to_inhg(pa): + """convert pascals to inches of mercury""" + return float(pa) * 0.00029530 + +def inhg_to_pascal(inhg): + """convert inches of mercury to pascals""" + return float(inhg) / 0.00029530 + def in_to_mm(inches): """convert inches to millimeters""" return (float(inches) * 2.54) * 10.0 @@ -803,7 +918,7 @@ def kn_to_mph(kn): return kn * 1.150779 def mm_to_cm(mm): - """convert millimeters to centimetrs""" + """convert millimeters to centimeters""" return mm / 10.0 if __name__ == "__main__":