--- /dev/null
+ISC License
+
+Copyright (c) 2020, WYLARK MOUNTAINEERING LLC
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
--- /dev/null
+InfoEx AutoWx (IEAW)
+=============
+
+This program fetches data from an NRCS SNOTEL site and pushes it into
+the InfoEx system using the new automated weather system implementation.
+
+License under the MIT license (see file: LICENSE).
+
+Disclaimer
+----------
+
+Your usage of the NRCS and InfoEx systems is bound by their respective
+terms and this software makes no attempt or claim to abide by any such
+terms.
+
+Installation
+------------
+
+It's recommended to use venv and pip with this program. Here's a typical
+sequence of commands for a new setup:
+
+`$ cd /path/to/src`
+`$ python3 -m venv env`
+`$ . env/bin/activate`
+`$ pip install -r requirements.txt`
+
+How to use it
+-------------
+
+This program is designed to be run from the command line (or via
+cron(8)) and administered via a simple, concise configuration file.
+
+This design allows you to run a separate program instance for each NRCS
+weather station from which you'd like to automate the importation of
+data into your InfoEx subscriber account.
+
+To run ad-hoc (be sure to activate the virtual environment, as detailed in the
+Installation section):
+
+`./infoex-autowx.py --config [path/to/config-file.ini] [--dry-run]`
+
+**NOTE: Specifying --dry-run will also not clean up the generated CSV
+file.** This is so that you can debug any issues more easily.
+
+Here's an example of a crontab(5) with two SNOTEL sites, each of which
+will run once per hour (note that this will activate the virtual environment
+created earlier):
+
+`2 * * * * /usr/bin/env bash -c 'cd /home/user/infoex-autowx && source env/bin/activate && ./infoex-autowx.py --config laurance-lake.ini'`
+`4 * * * * /usr/bin/env bash -c 'cd /home/user/infoex-autowx && source env/bin/activate && ./infoex-autowx.py --config mud-ridge.ini'`
+
+Configuration File
+------------------
+
+The configuration file is separated into two parts, the [wxsite]
+portion, and the [ftp] portion.
+
+The [wxsite] values describe which NRCS SNOTEL site's data you're after.
+See the next section in this README for instructions on obtaining these
+values.
+
+The [ftp] values describe your credentials for the InfoEx automated
+weather station FTP server.
+
+`[wxsite]`
+`station_triplet = [The NRCS identifier for a particular SNOTEL site]`
+`desired_data = [A comma-delimited list of NRCS elements you're interested in]`
+`location_uuid = [The UUID used by InfoEx to identify your automated Wx site]`
+`csv_filename = [Arbitrary name of the file that will be uploaded to InfoEx]`
+
+`[ftp]`
+`host = [InfoEx FTP host address]`
+`uuid = [InfoEx-supplied UUID]`
+`api_key = [InfoEx-supplied API Key]`
+
+Finding Your WXSITE values
+--------------------------
+
+To complete the [wxsite] configuration section, you must fill in the
+attributes of the NRCS SNOTEL site from which you want to import data.
+Here are the steps to do that:
+
+1. Find your station by clicking through this website:
+
+ https://www.wcc.nrcs.usda.gov/snow/sntllist.html
+
+ Or by using the interactive map:
+
+ https://www.nrcs.usda.gov/wps/portal/wcc/home/quicklinks/imap
+
+2. Once you've found your station, look for the "Station ID" (a 3- or
+ 4-digit number).
+
+3. Combine your Station ID, state abbreviation, and the network type
+ "SNTL" to get your station triplet (`station_triplet`, in the
+ configuration file). For example:
+
+ 655:OR:SNTL
+
+ would represent the Mud Ridge station (Station ID 655) in the state
+ of Oregon (OR). SNTL just represents that the station is in the
+ SNOTEL network and is used internally by NRCS.
+
+Once you have your station triplet, fill in the field in your
+configuration file. Now you must select which data you'd like to pull
+from NRCS to push into InfoEx.
+
+For that, visit the NRCS web service:
+
+https://wcc.sc.egov.usda.gov/awdbWebService/webservice/testwebservice.jsf?webserviceName=/awdbWebService
+
+Click "getElements" on the left, and then click, "Test Operation." This
+will return a long list of elements to your web browser from which you
+can choose. Each returned element has its identifier and a description.
+
+Once you've chosen your elements, combine all of their respective
+"elementCd" values into a comma-delimited string and put that into your
+configuration file as the `desired_data` value.
+
+For example:
+
+`station_triplet = 655:OR:SNTL`
+`desired_data = TOBS,PREC`
+
+indicates that I'd like to import "AIR TEMPERATURE OBSERVED" and
+"PRECIPITATION ACCUMULATION" from the NRCS SNOTEL site at Mud Ridge, OR,
+into InfoEx.
+
+Version History
+---------------
+
+- 1.0.0 (Jun 2020)
+
+ First released version. Cleaned up the program and design.
+ Implemented configuration file, added LICENSE, README, docs, etc.
+
+- 0.8.0 (Apr 2020)
+
+ First finished (unreleased) version.
+
+- pre-0.8.0 (Apr 2020)
+
+ First (private) finished implementation with successful importation of
+ NRCS data into InfoEx.
[wxsite]
-station_triplet = 655:OR:SNTL
-desired_data = TOBS,PREC,SNWD,XYZH
-location_uuid = a5bb872b-14c1-4367-bd81-177103acaef3
-csv_filename = INFOEX-MUDRIDGEAUTO.CSV
+station_triplet = <NRCS Station ID>
+desired_data = <Comma-separated list of NRCS elementCd values>
+location_uuid = <InfoEx-supplied Location UUID>
+csv_filename = <Name of file to upload to InfoEx FTP>
[ftp]
-host = weather.infoex.ca
+host = <InfoEx FTP host address>
uuid = <InfoEx-supplied UUID>
api_key = <InfoEx-supplied API Key>
-#!/usr/bin/python3
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
-#
-# InfoEx <-> NRCS Auto Wx implementation
-# Alexander Vasarab
-# Wylark Mountaineering LLC
-# 2020-04-22
-#
-# Version 0.8
-#
-# This program fetches data from an NRCS SNOTEL site and pushes it to
-# InfoEx using the new automated weather system implementation.
-#
-# It is designed to be run hourly, and it asks for the last three hours
-# of data of each desired type, and selects the most recent one. This
-# lends some resiliency to the process and helps ensure that we have a
-# value to send, but it can lead to somewhat inconsistent/untruthful
-# data if e.g. the HS is from the last hour but the tempPres is from two
-# hours ago because the instrumentation had a hiccup. It's worth
-# considering if this is a bug or a feature.
-#
+"""
+InfoEx <-> NRCS Auto Wx implementation
+Alexander Vasarab
+Wylark Mountaineering LLC
+
+Version 1.0.0
+
+This program fetches data from an NRCS SNOTEL site and pushes it to
+InfoEx using the new automated weather system implementation.
+
+It is designed to be run hourly, and it asks for the last three hours
+of data of each desired type, and selects the most recent one. This
+lends some resiliency to the process and helps ensure that we have a
+value to send, but it can lead to somewhat inconsistent/untruthful
+data if e.g. the HS is from the last hour but the tempPres is from two
+hours ago because the instrumentation had a hiccup. It's worth
+considering if this is a bug or a feature.
+
+For more information, see file: README
+For licensing, see file: LICENSE
+"""
import configparser
import csv
import datetime
import logging
+import os
+import sys
import time
-import zeep
-import zeep.cache
-import zeep.transports
+
from collections import OrderedDict
from ftplib import FTP
from optparse import OptionParser
+import zeep
+import zeep.cache
+import zeep.transports
+
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
log.addHandler(logging.handlers.SysLogHandler())
parser = OptionParser()
-parser.add_option("--config", dest="config", metavar="FILE", help="location of config file")
+parser.add_option("--config",
+ dest="config",
+ metavar="FILE",
+ help="location of config file")
+parser.add_option("--dry-run",
+ action="store_true",
+ dest="dry_run",
+ default=False,
+ help="fetch data but don't upload to InfoEx")
(options, args) = parser.parse_args()
config = configparser.ConfigParser(allow_no_value=False)
+
+if not options.config:
+ print("Please specify a configuration file via --config")
+ sys.exit(1)
+
config.read(options.config)
log.debug('STARTING UP')
'uuid': config['ftp']['uuid'],
'api_key': config['ftp']['api_key'],
'location_uuid': config['wxsite']['location_uuid'],
- 'wx_data': {},
+ 'wx_data': {}, # placeholder key, values to come later
'csv_filename': config['wxsite']['csv_filename']
}
values = tmp[0]['values']
# sort and isolate the most recent
+ #
+ # NOTE: we do this because sometimes there are gaps in hourly data
+ # in NRCS; yes, we may end up with slightly inaccurate data,
+ # so perhaps this decision will be re-evaluated in the future
if values:
ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
infoex['wx_data'][elementCd] = ordered[0]['value']
log.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
-# Only need to add in what we want to change thanks to that abomination
-# of a variable declaration earlier
+# Now we only need to add in what we want to change thanks to that
+# abomination of a variable declaration earlier
final_data[fmap['Location UUID']] = infoex['location_uuid']
final_data[fmap['obDate']] = end_date.strftime('%m/%d/%Y')
final_data[fmap['obTime']] = end_date.strftime('%H:%M')
-#final_data[fmap['tempPres']] = float(infoex['wx_data']['TOBS'])
-#final_data[fmap['precipitationGauge']] = float(infoex['wx_data']['PREC'])
-#final_data[fmap['hS']] = float(infoex['wx_data']['SNWD'])
-
for elementCd in infoex['wx_data']:
if elementCd not in iemap:
log.warning("BAD KEY wx_data['%s']" % (elementCd))
writer.writerow(final_data)
f.close()
-#with open(infoex['csv_filename'], 'rb') as f:
-# log.debug("uploading FTP file '%s'" % (infoex['host']))
-# ftp = FTP(infoex['host'], infoex['uuid'], infoex['api_key'])
-# ftp.storlines('STOR ' + infoex['csv_filename'], f)
-# ftp.close()
-# f.close()
+if not options.dry_run:
+ # not a dry run
+ with open(infoex['csv_filename'], 'rb') as f:
+ log.debug("uploading FTP file '%s'" % (infoex['host']))
+ ftp = FTP(infoex['host'], infoex['uuid'], infoex['api_key'])
+ ftp.storlines('STOR ' + infoex['csv_filename'], f)
+ ftp.close()
+ f.close()
+ os.remove(infoex['csv_filename'])
log.debug('DONE')
--- /dev/null
+zeep>=3.4.0
wylark / infoex-autowx.git / commitdiff