From: Alexander Vasarab Date: Sat, 20 Jun 2020 21:32:01 +0000 (-0700) Subject: Merge branch 'v1.0.0' X-Git-Tag: v1.0.0^0 X-Git-Url: https://wylark.com/src/infoex-autowx.git/commitdiff_plain/a585efb4e491a6a2dab7df5e858b34fe5235f662?hp=7ec3b0b74884347d7b1c38fa18b4d3879371b495 Merge branch 'v1.0.0' --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0a764a4 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +env diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0af8c0a --- /dev/null +++ b/LICENSE @@ -0,0 +1,15 @@ +ISC License + +Copyright (c) 2020, WYLARK MOUNTAINEERING LLC + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..4433828 --- /dev/null +++ b/README.md @@ -0,0 +1,144 @@ +InfoEx AutoWx (IEAW) +============= + +This program fetches data from an NRCS SNOTEL site and pushes it into +the InfoEx system using the new automated weather system implementation. + +License under the MIT license (see file: LICENSE). + +Disclaimer +---------- + +Your usage of the NRCS and InfoEx systems is bound by their respective +terms and this software makes no attempt or claim to abide by any such +terms. + +Installation +------------ + +It's recommended to use venv and pip with this program. Here's a typical +sequence of commands for a new setup: + +`$ cd /path/to/src` +`$ python3 -m venv env` +`$ . env/bin/activate` +`$ pip install -r requirements.txt` + +How to use it +------------- + +This program is designed to be run from the command line (or via +cron(8)) and administered via a simple, concise configuration file. + +This design allows you to run a separate program instance for each NRCS +weather station from which you'd like to automate the importation of +data into your InfoEx subscriber account. + +To run ad-hoc (be sure to activate the virtual environment, as detailed in the +Installation section): + +`./infoex-autowx.py --config [path/to/config-file.ini] [--dry-run]` + +**NOTE: Specifying --dry-run will also not clean up the generated CSV +file.** This is so that you can debug any issues more easily. + +Here's an example of a crontab(5) with two SNOTEL sites, each of which +will run once per hour (note that this will activate the virtual environment +created earlier): + +`2 * * * * /usr/bin/env bash -c 'cd /home/user/infoex-autowx && source env/bin/activate && ./infoex-autowx.py --config laurance-lake.ini'` +`4 * * * * /usr/bin/env bash -c 'cd /home/user/infoex-autowx && source env/bin/activate && ./infoex-autowx.py --config mud-ridge.ini'` + +Configuration File +------------------ + +The configuration file is separated into two parts, the [wxsite] +portion, and the [ftp] portion. + +The [wxsite] values describe which NRCS SNOTEL site's data you're after. +See the next section in this README for instructions on obtaining these +values. + +The [ftp] values describe your credentials for the InfoEx automated +weather station FTP server. + +`[wxsite]` +`station_triplet = [The NRCS identifier for a particular SNOTEL site]` +`desired_data = [A comma-delimited list of NRCS elements you're interested in]` +`location_uuid = [The UUID used by InfoEx to identify your automated Wx site]` +`csv_filename = [Arbitrary name of the file that will be uploaded to InfoEx]` + +`[ftp]` +`host = [InfoEx FTP host address]` +`uuid = [InfoEx-supplied UUID]` +`api_key = [InfoEx-supplied API Key]` + +Finding Your WXSITE values +-------------------------- + +To complete the [wxsite] configuration section, you must fill in the +attributes of the NRCS SNOTEL site from which you want to import data. +Here are the steps to do that: + +1. Find your station by clicking through this website: + + https://www.wcc.nrcs.usda.gov/snow/sntllist.html + + Or by using the interactive map: + + https://www.nrcs.usda.gov/wps/portal/wcc/home/quicklinks/imap + +2. Once you've found your station, look for the "Station ID" (a 3- or + 4-digit number). + +3. Combine your Station ID, state abbreviation, and the network type + "SNTL" to get your station triplet (`station_triplet`, in the + configuration file). For example: + + 655:OR:SNTL + + would represent the Mud Ridge station (Station ID 655) in the state + of Oregon (OR). SNTL just represents that the station is in the + SNOTEL network and is used internally by NRCS. + +Once you have your station triplet, fill in the field in your +configuration file. Now you must select which data you'd like to pull +from NRCS to push into InfoEx. + +For that, visit the NRCS web service: + +https://wcc.sc.egov.usda.gov/awdbWebService/webservice/testwebservice.jsf?webserviceName=/awdbWebService + +Click "getElements" on the left, and then click, "Test Operation." This +will return a long list of elements to your web browser from which you +can choose. Each returned element has its identifier and a description. + +Once you've chosen your elements, combine all of their respective +"elementCd" values into a comma-delimited string and put that into your +configuration file as the `desired_data` value. + +For example: + +`station_triplet = 655:OR:SNTL` +`desired_data = TOBS,PREC` + +indicates that I'd like to import "AIR TEMPERATURE OBSERVED" and +"PRECIPITATION ACCUMULATION" from the NRCS SNOTEL site at Mud Ridge, OR, +into InfoEx. + +Version History +--------------- + +- 1.0.0 (Jun 2020) + + First released version. Cleaned up the program and design. + Implemented configuration file, added LICENSE, README, docs, etc. + +- 0.8.0 (Apr 2020) + + First finished (unreleased) version. + +- pre-0.8.0 (Apr 2020) + + First (private) finished implementation with successful importation of + NRCS data into InfoEx. diff --git a/config.ini.example b/config.ini.example index 3d87250..018a6fc 100644 --- a/config.ini.example +++ b/config.ini.example @@ -1,10 +1,10 @@ [wxsite] -station_triplet = 655:OR:SNTL -desired_data = TOBS,PREC,SNWD,XYZH -location_uuid = a5bb872b-14c1-4367-bd81-177103acaef3 -csv_filename = INFOEX-MUDRIDGEAUTO.CSV +station_triplet = +desired_data = +location_uuid = +csv_filename = [ftp] -host = weather.infoex.ca +host = uuid = api_key = diff --git a/infoex-autowx.py b/infoex-autowx.py index f951c0d..ea91bef 100755 --- a/infoex-autowx.py +++ b/infoex-autowx.py @@ -1,37 +1,44 @@ -#!/usr/bin/python3 +#!/usr/bin/env python +# -*- coding: utf-8 -*- -# -# InfoEx <-> NRCS Auto Wx implementation -# Alexander Vasarab -# Wylark Mountaineering LLC -# 2020-04-22 -# -# Version 0.8 -# -# This program fetches data from an NRCS SNOTEL site and pushes it to -# InfoEx using the new automated weather system implementation. -# -# It is designed to be run hourly, and it asks for the last three hours -# of data of each desired type, and selects the most recent one. This -# lends some resiliency to the process and helps ensure that we have a -# value to send, but it can lead to somewhat inconsistent/untruthful -# data if e.g. the HS is from the last hour but the tempPres is from two -# hours ago because the instrumentation had a hiccup. It's worth -# considering if this is a bug or a feature. -# +""" +InfoEx <-> NRCS Auto Wx implementation +Alexander Vasarab +Wylark Mountaineering LLC + +Version 1.0.0 + +This program fetches data from an NRCS SNOTEL site and pushes it to +InfoEx using the new automated weather system implementation. + +It is designed to be run hourly, and it asks for the last three hours +of data of each desired type, and selects the most recent one. This +lends some resiliency to the process and helps ensure that we have a +value to send, but it can lead to somewhat inconsistent/untruthful +data if e.g. the HS is from the last hour but the tempPres is from two +hours ago because the instrumentation had a hiccup. It's worth +considering if this is a bug or a feature. + +For more information, see file: README +For licensing, see file: LICENSE +""" import configparser import csv import datetime import logging +import os +import sys import time -import zeep -import zeep.cache -import zeep.transports + from collections import OrderedDict from ftplib import FTP from optparse import OptionParser +import zeep +import zeep.cache +import zeep.transports + log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -44,11 +51,24 @@ except: log.addHandler(logging.handlers.SysLogHandler()) parser = OptionParser() -parser.add_option("--config", dest="config", metavar="FILE", help="location of config file") +parser.add_option("--config", + dest="config", + metavar="FILE", + help="location of config file") +parser.add_option("--dry-run", + action="store_true", + dest="dry_run", + default=False, + help="fetch data but don't upload to InfoEx") (options, args) = parser.parse_args() config = configparser.ConfigParser(allow_no_value=False) + +if not options.config: + print("Please specify a configuration file via --config") + sys.exit(1) + config.read(options.config) log.debug('STARTING UP') @@ -61,7 +81,7 @@ try: 'uuid': config['ftp']['uuid'], 'api_key': config['ftp']['api_key'], 'location_uuid': config['wxsite']['location_uuid'], - 'wx_data': {}, + 'wx_data': {}, # placeholder key, values to come later 'csv_filename': config['wxsite']['csv_filename'] } @@ -172,6 +192,10 @@ for elementCd in desired_data: values = tmp[0]['values'] # sort and isolate the most recent + # + # NOTE: we do this because sometimes there are gaps in hourly data + # in NRCS; yes, we may end up with slightly inaccurate data, + # so perhaps this decision will be re-evaluated in the future if values: ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True) infoex['wx_data'][elementCd] = ordered[0]['value'] @@ -183,16 +207,12 @@ log.info("Time to get all elementCds : %.3f sec" % (time.time() - log.debug("infoex[wx_data]: %s", str(infoex['wx_data'])) -# Only need to add in what we want to change thanks to that abomination -# of a variable declaration earlier +# Now we only need to add in what we want to change thanks to that +# abomination of a variable declaration earlier final_data[fmap['Location UUID']] = infoex['location_uuid'] final_data[fmap['obDate']] = end_date.strftime('%m/%d/%Y') final_data[fmap['obTime']] = end_date.strftime('%H:%M') -#final_data[fmap['tempPres']] = float(infoex['wx_data']['TOBS']) -#final_data[fmap['precipitationGauge']] = float(infoex['wx_data']['PREC']) -#final_data[fmap['hS']] = float(infoex['wx_data']['SNWD']) - for elementCd in infoex['wx_data']: if elementCd not in iemap: log.warning("BAD KEY wx_data['%s']" % (elementCd)) @@ -219,11 +239,14 @@ with open(infoex['csv_filename'], 'w') as f: writer.writerow(final_data) f.close() -#with open(infoex['csv_filename'], 'rb') as f: -# log.debug("uploading FTP file '%s'" % (infoex['host'])) -# ftp = FTP(infoex['host'], infoex['uuid'], infoex['api_key']) -# ftp.storlines('STOR ' + infoex['csv_filename'], f) -# ftp.close() -# f.close() +if not options.dry_run: + # not a dry run + with open(infoex['csv_filename'], 'rb') as f: + log.debug("uploading FTP file '%s'" % (infoex['host'])) + ftp = FTP(infoex['host'], infoex['uuid'], infoex['api_key']) + ftp.storlines('STOR ' + infoex['csv_filename'], f) + ftp.close() + f.close() + os.remove(infoex['csv_filename']) log.debug('DONE') diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..739dc36 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +zeep>=3.4.0