From: Alexander Vasarab Date: Mon, 30 Nov 2020 01:38:35 +0000 (-0800) Subject: Merge branch 'release-3.0.0' X-Git-Tag: v3.0.0^0 X-Git-Url: https://wylark.com/src/infoex-autowx.git/commitdiff_plain/9e32afeb775aa2a1f45609d3796a3f8a6d1b8146?hp=2a76db6a9d50678b0dcf1a2bbce767542ac3e0d2 Merge branch 'release-3.0.0' --- diff --git a/.gitignore b/.gitignore index bc543d3..4497b41 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ env *.CSV scratch/ configs/ +__pycache__/ diff --git a/README.md b/README.md index b2af012..8d9da91 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ InfoEx AutoWx (IEAW) ============= -This program fetches data from an NRCS SNOTEL or MesoWest station and -pushes it into the InfoEx system using the new automated weather system -implementation. +This program fetches data from an NRCS SNOTEL or MesoWest station, or +your own custom data source, and pushes it into the InfoEx system using +the new automated weather system implementation. Licensed under the ISC license (see file: LICENSE). @@ -78,11 +78,12 @@ weather station FTP server and other InfoEx-related configuration options. `[station]` -`type = # either mesowest or nrcs #` -`token = # MesoWest API token -- ignored when type is nrcs #` +`type = # mesowest, nrcs, or python #` +`token = # MesoWest API token -- only applies when type is mesowest #` `station_id = # the NRCS/MesoWest identifier for a particular station #` `desired_data = # a comma-delimited list of fields you're interested in #` -`units = # either english or metric -- ignored when type is nrcs #` +`units = # either english or metric -- only applies when type is mesowest #` +`path = # the filesystem path to the Python program -- only applies when type is python #` `[infoex]` `host = # InfoEx FTP host address #` @@ -191,6 +192,20 @@ indicates that I'd like to import "Temperature" and "Precipitation accumulated" from the MesoWest station at Santiam Pass, OR, into InfoEx, and that I want that data in imperial units. +Custom weather station support +------------------------------ + +This program supports custom weather station data by allowing the user +to specify the path to an external Python program. The external Python +program should emit its data in the form expected by infoex-autowx. + +This is a powerful feature which enables the user to upload data from +any source imaginable into InfoEx. Common examples are a local database +or a remote web page which requires some custom parsing. + +Please see the program located at examples/custom-wx.example.py for a +complete description of what's required. + A note on supported measurements -------------------------------- @@ -200,7 +215,6 @@ weather station may not record them. In this case, the data will simply be ignored (i.e. it will NOT log "0" when there's no measurement available). - InfoEx provides a mechanism for inspecting your automated weather station data, so use that after setting this program up and compare it with the data you see in your web browser. @@ -226,15 +240,39 @@ wind\_speed wind\_direction wind\_gust +**Custom Wx program** +*infoex-autowx expects a custom Wx data provider to provide at least one +of the following:* +precipitationGauge +tempPres +tempMaxHour +tempMinHour +hS +baro +rH +windSpeedNum +windDirectionNum +windGustSpeedNum + Future plans ------------ -- Improve the documentation - Implement unit conversion for NRCS stations Version history --------------- +- 3.0.0 (Nov 2020) + + Implement Custom Wx data providers. + + This release enables the user to write their own Python programs and + specify them to infoex-autowx as a data provider. + + This in turn enables the user to pull data from e.g. a local database + or an HTML page and push it into their InfoEx auto station data, + limited only by the imagination. + - 2.2.0 (Nov 2020) Add support for Tmin/Tmax values (directly from MesoWest/NRCS). diff --git a/config.ini.example b/config.ini.example deleted file mode 100644 index 17544ae..0000000 --- a/config.ini.example +++ /dev/null @@ -1,14 +0,0 @@ -[station] -type = # (mesowest|nrcs) # -token = # MesoWest API token (ignored for NRCS) # -station_id = # NRCS/MesoWest Station ID # -desired_data = # Comma-separated list of values # -units = # (english|metric) (ignored for NRCS) # - -[infoex] -host = # InfoEx FTP host address # -uuid = # InfoEx-supplied UUID # -api_key = # InfoEx-supplied API Key # -csv_filename = # Name of file to upload to InfoEx FTP # -location_uuid = # InfoEx-supplied Location UUID # - diff --git a/examples/config.example.ini b/examples/config.example.ini new file mode 100644 index 0000000..17544ae --- /dev/null +++ b/examples/config.example.ini @@ -0,0 +1,14 @@ +[station] +type = # (mesowest|nrcs) # +token = # MesoWest API token (ignored for NRCS) # +station_id = # NRCS/MesoWest Station ID # +desired_data = # Comma-separated list of values # +units = # (english|metric) (ignored for NRCS) # + +[infoex] +host = # InfoEx FTP host address # +uuid = # InfoEx-supplied UUID # +api_key = # InfoEx-supplied API Key # +csv_filename = # Name of file to upload to InfoEx FTP # +location_uuid = # InfoEx-supplied Location UUID # + diff --git a/examples/custom-wx.example.py b/examples/custom-wx.example.py new file mode 100644 index 0000000..0b2cc42 --- /dev/null +++ b/examples/custom-wx.example.py @@ -0,0 +1,35 @@ +# reference implementation for an infoex-autowx custom Wx data provider + +# global variable which will hold the Wx data to be uploaded to InfoEx +wx_data = {} + +# the following data types are supported by infoex-autowx +wx_data['precipitationGauge'] = None +wx_data['tempPres'] = None +wx_data['tempMaxHour'] = None +wx_data['tempMinHour'] = None +wx_data['hS'] = None +wx_data['baro'] = None +wx_data['rH'] = None +wx_data['windSpeedNum'] = None +wx_data['windDirectionNum'] = None +wx_data['windGustSpeedNum'] = None + +def get_custom_data(): + # This function will be called by infoex-autowx, and the `wx_data` + # variable (a global variable within this program) will be returned. + # + # For example, maybe you will `import psycopg2` and grab your data + # from a local PostgreSQL database. Or maybe you will use the + # requests library to fetch a remote web page and parse out the data + # that's meaningful to your operation. + # + # Whatever your program needs to do to get its data can be done + # either here in this function directly, or elsewhere with + # modification to the global variable `wx_data`. + # + # NOTE: The LOG class from infoex-autowx is available, so you may + # issue e.g. LOG.info('some helpful information') in your + # program. + + return wx_data diff --git a/infoex-autowx.py b/infoex-autowx.py index ea23bc3..3fe07fc 100755 --- a/infoex-autowx.py +++ b/infoex-autowx.py @@ -39,7 +39,7 @@ import zeep import zeep.cache import zeep.transports -__version__ = '2.2.0' +__version__ = '3.0.0' LOG = logging.getLogger(__name__) LOG.setLevel(logging.NOTSET) @@ -85,7 +85,7 @@ def setup_config(config): station = dict() station['provider'] = config['station']['type'] - if station['provider'] not in ['nrcs', 'mesowest']: + if station['provider'] not in ['nrcs', 'mesowest', 'python']: print("Please specify either nrcs or mesowest as the station type.") sys.exit(1) @@ -112,9 +112,12 @@ def setup_config(config): '&stid=' + station['station_id'] + \ '&vars=' + station['desired_data'] + if station['provider'] == 'python': + station['path'] = config['station']['path'] + except KeyError as err: LOG.critical("%s not defined in configuration file", err) - exit(1) + sys.exit(1) # all sections/values present in config file, final sanity check try: @@ -124,7 +127,7 @@ def setup_config(config): raise ValueError except ValueError: LOG.critical("Config value '%s.%s' is empty", key, subkey) - exit(1) + sys.exit(1) return (infoex, station) @@ -184,14 +187,23 @@ def main(): iemap = setup_infoex_counterparts_mapping(station['provider']) # override units if user selected metric - if station['units'] == 'metric': - final_data = switch_units_to_metric(final_data, fmap) + try: + if station['units'] == 'metric': + final_data = switch_units_to_metric(final_data, fmap) + except KeyError: + if station['provider'] != 'python': + LOG.error("Please specify the units in the configuration " + "file") + sys.exit(1) (begin_date, end_date) = setup_time_values() - # get the data - LOG.debug("Getting %s data from %s to %s", str(station['desired_data']), - str(begin_date), str(end_date)) + if station['provider'] == 'python': + LOG.debug("Getting custom data from external Python program") + else: + LOG.debug("Getting %s data from %s to %s", + str(station['desired_data']), + str(begin_date), str(end_date)) time_all_elements = time.time() @@ -201,6 +213,38 @@ def main(): elif station['provider'] == 'mesowest': infoex['wx_data'] = get_mesowest_data(begin_date, end_date, station) + elif station['provider'] == 'python': + try: + import importlib.util + + spec = importlib.util.spec_from_file_location('custom_wx', + station['path']) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + mod.LOG = LOG + + try: + infoex['wx_data'] = mod.get_custom_data() + + if infoex['wx_data'] is None: + infoex['wx_data'] = [] + except Exception as exc: + LOG.error("Python program for custom Wx data failed in " + "execution: " + str(exc)) + sys.exit(1) + + LOG.info("Successfully executed external Python program") + except ImportError: + LOG.error("Please upgrade to Python 3.3 or later") + sys.exit(1) + except FileNotFoundError: + LOG.error("Specified Python program for custom Wx data " + "was not found") + sys.exit(1) + except Exception as exc: + LOG.error("A problem was encountered when attempting to " + "load your custom Wx program: " + str(exc)) + sys.exit(1) LOG.info("Time taken to get all data : %.3f sec", time.time() - time_all_elements) @@ -245,13 +289,14 @@ def main(): LOG.debug("final_data: %s", str(final_data)) - if not write_local_csv(infoex['csv_filename'], final_data): - LOG.warning('Could not write local CSV file: %s', - infoex['csv_filename']) - return 1 + if len(infoex['wx_data']) > 0: + if not write_local_csv(infoex['csv_filename'], final_data): + LOG.warning('Could not write local CSV file: %s', + infoex['csv_filename']) + return 1 - if not options.dry_run: - upload_csv(infoex['csv_filename'], infoex) + if not options.dry_run: + upload_csv(infoex['csv_filename'], infoex) LOG.debug('DONE') return 0 @@ -336,6 +381,18 @@ def setup_infoex_counterparts_mapping(provider): iemap['wind_speed'] = 'windSpeedNum' iemap['wind_direction'] = 'windDirectionNum' iemap['wind_gust'] = 'windGustSpeedNum' + elif provider == 'python': + # we expect Python programs to use the InfoEx data type names + iemap['precipitationGauge'] = 'precipitationGauge' + iemap['tempPres'] = 'tempPres' + iemap['tempMaxHour'] = 'tempMaxHour' + iemap['tempMinHour'] = 'tempMinHour' + iemap['hS'] = 'hS' + iemap['baro'] = 'baro' + iemap['rH'] = 'rH' + iemap['windSpeedNum'] = 'windSpeedNum' + iemap['windDirectionNum'] = 'windDirectionNum' + iemap['windGustSpeedNum'] = 'windGustSpeedNum' return iemap