Merge branch 'hotfix-3.2.1'
[infoex-autowx.git] / infoex-autowx.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 """
5 InfoEx <-> NRCS/MesoWest Auto Wx implementation
6 Alexander Vasarab
7 Wylark Mountaineering LLC
8
9 This program fetches data from either an NRCS SNOTEL site or MesoWest
10 weather station and pushes it to InfoEx using the new automated weather
11 system implementation.
12
13 It is designed to be run hourly, and it asks for the last three hours
14 of data of each desired type, and selects the most recent one. This
15 lends some resiliency to the process and helps ensure that we have a
16 value to send, but it can lead to somewhat inconsistent/untruthful
17 data if e.g. the HS is from the last hour but the tempPres is from two
18 hours ago because the instrumentation had a hiccup. It's worth
19 considering if this is a bug or a feature.
20
21 For more information, see file: README
22 For licensing, see file: LICENSE
23 """
24
25 import configparser
26 import csv
27 import datetime
28 import logging
29 import os
30 import sys
31 import time
32 import urllib3
33 import importlib.util
34
35 from ftplib import FTP
36 from argparse import ArgumentParser
37
38 import pytz
39
40 import requests
41
42 import zeep
43 import zeep.cache
44 import zeep.transports
45
46 __version__ = '3.2.1'
47
48 LOG = logging.getLogger(__name__)
49 LOG.setLevel(logging.NOTSET)
50
51 urllib3.disable_warnings()
52
53 def get_parser():
54 """Return OptionParser for this program"""
55 parser = ArgumentParser()
56
57 parser.add_argument("--version",
58 action="version",
59 version=__version__)
60
61 parser.add_argument("--config",
62 dest="config",
63 metavar="FILE",
64 help="location of config file")
65
66 parser.add_argument("--log-level",
67 dest="log_level",
68 default=None,
69 help="set the log level (debug, info, warning)")
70
71 parser.add_argument("--dry-run",
72 action="store_true",
73 dest="dry_run",
74 default=False,
75 help="fetch data but don't upload to InfoEx")
76
77 return parser
78
79 def setup_config(config):
80 """Setup config variable based on values specified in the ini file"""
81 try:
82 infoex = {
83 'host': config['infoex']['host'],
84 'uuid': config['infoex']['uuid'],
85 'api_key': config['infoex']['api_key'],
86 'csv_filename': config['infoex']['csv_filename'],
87 'location_uuid': config['infoex']['location_uuid'],
88 'wx_data': {}, # placeholder key, values to come later
89 }
90
91 station = dict()
92 station['provider'] = config['station']['type']
93
94 if station['provider'] not in ['nrcs', 'mesowest', 'python']:
95 print("Please specify either nrcs or mesowest as the station type.")
96 sys.exit(1)
97
98 if station['provider'] == 'nrcs':
99 station['source'] = 'https://www.wcc.nrcs.usda.gov/awdbWebService/services?WSDL'
100 station['station_id'] = config['station']['station_id']
101 station['desired_data'] = config['station']['desired_data'].split(',')
102 station['units'] = config['station']['units']
103
104 if station['provider'] == 'mesowest':
105 station['source'] = 'https://api.synopticdata.com/v2/stations/timeseries'
106 station['station_id'] = config['station']['station_id']
107 station['units'] = config['station']['units']
108 station['desired_data'] = config['station']['desired_data']
109
110 # construct full API URL (sans start/end time, added later)
111 station['source'] = station['source'] + '?token=' + \
112 config['station']['token'] + \
113 '&within=60&units=' + station['units'] + \
114 '&stid=' + station['station_id'] + \
115 '&vars=' + station['desired_data']
116
117 if station['provider'] == 'python':
118 station['path'] = config['station']['path']
119
120 tz = 'America/Los_Angeles'
121
122 if 'tz' in config['station']:
123 tz = config['station']['tz']
124
125 try:
126 station['tz'] = pytz.timezone(tz)
127 except pytz.exceptions.UnknownTimeZoneError:
128 LOG.critical("%s is not a valid timezone", tz)
129 sys.exit(1)
130
131 except KeyError as err:
132 LOG.critical("%s not defined in configuration file", err)
133 sys.exit(1)
134
135 # all sections/values present in config file, final sanity check
136 try:
137 for key in config.sections():
138 for subkey in config[key]:
139 if not config[key][subkey]:
140 raise ValueError
141 except ValueError:
142 LOG.critical("Config value '%s.%s' is empty", key, subkey)
143 sys.exit(1)
144
145 return (infoex, station)
146
147 def setup_logging(log_level):
148 """Setup our logging infrastructure"""
149 try:
150 from systemd.journal import JournalHandler
151 LOG.addHandler(JournalHandler())
152 except ImportError:
153 ## fallback to syslog
154 #import logging.handlers
155 #LOG.addHandler(logging.handlers.SysLogHandler())
156 # fallback to stdout
157 handler = logging.StreamHandler(sys.stdout)
158 LOG.addHandler(handler)
159
160 # ugly, but passable
161 if log_level in [None, 'debug', 'info', 'warning']:
162 if log_level == 'debug':
163 LOG.setLevel(logging.DEBUG)
164 elif log_level == 'info':
165 LOG.setLevel(logging.INFO)
166 elif log_level == 'warning':
167 LOG.setLevel(logging.WARNING)
168 else:
169 LOG.setLevel(logging.NOTSET)
170 else:
171 return False
172
173 return True
174
175 def main():
176 """Main routine: sort through args, decide what to do, then do it"""
177 parser = get_parser()
178 options = parser.parse_args()
179
180 config = configparser.ConfigParser(allow_no_value=False)
181
182 if not options.config:
183 parser.print_help()
184 print("\nPlease specify a configuration file via --config.")
185 sys.exit(1)
186
187 config.read(options.config)
188
189 if not setup_logging(options.log_level):
190 parser.print_help()
191 print("\nPlease select an appropriate log level or remove the switch (--log-level).")
192 sys.exit(1)
193
194 (infoex, station) = setup_config(config)
195
196 LOG.debug('Config parsed, starting up')
197
198 # create mappings
199 (fmap, final_data) = setup_infoex_fields_mapping(infoex['location_uuid'])
200 iemap = setup_infoex_counterparts_mapping(station['provider'])
201
202 # override units if user selected metric
203 if station['provider'] != 'python' and station['units'] == 'metric':
204 final_data = switch_units_to_metric(final_data, fmap)
205
206 (begin_date, end_date) = setup_time_values(station)
207
208 if station['provider'] == 'python':
209 LOG.debug("Getting custom data from external Python program")
210 else:
211 LOG.debug("Getting %s data from %s to %s (%s)",
212 str(station['desired_data']),
213 str(begin_date), str(end_date), end_date.tzinfo.zone)
214
215 time_all_elements = time.time()
216
217 # get the data
218 if station['provider'] == 'nrcs':
219 infoex['wx_data'] = get_nrcs_data(begin_date, end_date, station)
220 elif station['provider'] == 'mesowest':
221 infoex['wx_data'] = get_mesowest_data(begin_date, end_date,
222 station)
223 elif station['provider'] == 'python':
224 try:
225 spec = importlib.util.spec_from_file_location('custom_wx',
226 station['path'])
227 mod = importlib.util.module_from_spec(spec)
228 spec.loader.exec_module(mod)
229 mod.LOG = LOG
230
231 try:
232 infoex['wx_data'] = mod.get_custom_data()
233
234 if infoex['wx_data'] is None:
235 infoex['wx_data'] = []
236 except Exception as exc:
237 LOG.error("Python program for custom Wx data failed in "
238 "execution: %s", str(exc))
239 sys.exit(1)
240
241 LOG.info("Successfully executed external Python program")
242 except ImportError:
243 LOG.error("Please upgrade to Python 3.3 or later")
244 sys.exit(1)
245 except FileNotFoundError:
246 LOG.error("Specified Python program for custom Wx data "
247 "was not found")
248 sys.exit(1)
249 except Exception as exc:
250 LOG.error("A problem was encountered when attempting to "
251 "load your custom Wx program: %s", str(exc))
252 sys.exit(1)
253
254 LOG.info("Time taken to get all data : %.3f sec", time.time() -
255 time_all_elements)
256
257 LOG.debug("infoex[wx_data]: %s", str(infoex['wx_data']))
258
259 # timezone massaging
260 final_end_date = end_date.astimezone(station['tz'])
261
262 # Now we only need to add in what we want to change thanks to that
263 # abomination of a variable declaration earlier
264 final_data[fmap['Location UUID']] = infoex['location_uuid']
265 final_data[fmap['obDate']] = final_end_date.strftime('%m/%d/%Y')
266 final_data[fmap['obTime']] = final_end_date.strftime('%H:%M')
267 final_data[fmap['timeZone']] = station['tz'].zone
268
269 for element_cd in infoex['wx_data']:
270 if element_cd not in iemap:
271 LOG.warning("BAD KEY wx_data['%s']", element_cd)
272 continue
273
274 # do the conversion before the rounding
275 if station['provider'] == 'nrcs' and station['units'] == 'metric':
276 infoex['wx_data'][element_cd] = convert_nrcs_units_to_metric(element_cd, infoex['wx_data'][element_cd])
277
278 # Massage precision of certain values to fit InfoEx's
279 # expectations
280 #
281 # 0 decimal places: relative humidity, wind speed, wind
282 # direction, wind gust, snow depth
283 # 1 decimal place: air temp, baro
284 # Avoid transforming None values
285 if infoex['wx_data'][element_cd] is None:
286 continue
287 elif element_cd in ['wind_speed', 'WSPD', 'wind_direction',
288 'RHUM', 'relative_humidity', 'WDIR',
289 'wind_gust', 'SNWD', 'snow_depth']:
290 infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd])
291 elif element_cd in ['TOBS', 'air_temp', 'PRES', 'pressure']:
292 infoex['wx_data'][element_cd] = round(infoex['wx_data'][element_cd], 1)
293
294 # CONSIDER: Casting every value to Float() -- need to investigate if
295 # any possible elementCds we may want are any other data
296 # type than float.
297 #
298 # Another possibility is to query the API with
299 # getStationElements and temporarily store the
300 # storedUnitCd. But that's pretty network-intensive and
301 # may not even be worth it if there's only e.g. one or two
302 # exceptions to any otherwise uniformly Float value set.
303 final_data[fmap[iemap[element_cd]]] = infoex['wx_data'][element_cd]
304
305 LOG.debug("final_data: %s", str(final_data))
306
307 if infoex['wx_data']:
308 if not write_local_csv(infoex['csv_filename'], final_data):
309 LOG.warning('Could not write local CSV file: %s',
310 infoex['csv_filename'])
311 return 1
312
313 if not options.dry_run:
314 upload_csv(infoex['csv_filename'], infoex)
315
316 LOG.debug('DONE')
317 return 0
318
319 # data structure operations
320 def setup_infoex_fields_mapping(location_uuid):
321 """
322 Create a mapping of InfoEx fields to the local data's indexing scheme.
323
324 INFOEX FIELDS
325
326 This won't earn style points in Python, but here we establish a couple
327 of helpful mappings variables. The reason this is helpful is that the
328 end result is simply an ordered set, the CSV file. But we still may
329 want to manipulate the values arbitrarily before writing that file.
330
331 Also note that the current Auto Wx InfoEx documentation shows these
332 keys in a graphical table with the "index" beginning at 1, but here we
333 sanely index beginning at 0.
334 """
335 # pylint: disable=too-many-statements,multiple-statements,bad-whitespace
336 fmap = {} ; final_data = [None] * 29
337 fmap['Location UUID'] = 0 ; final_data[0] = location_uuid
338 fmap['obDate'] = 1 ; final_data[1] = None
339 fmap['obTime'] = 2 ; final_data[2] = None
340 fmap['timeZone'] = 3 ; final_data[3] = 'Pacific'
341 fmap['tempMaxHour'] = 4 ; final_data[4] = None
342 fmap['tempMaxHourUnit'] = 5 ; final_data[5] = 'F'
343 fmap['tempMinHour'] = 6 ; final_data[6] = None
344 fmap['tempMinHourUnit'] = 7 ; final_data[7] = 'F'
345 fmap['tempPres'] = 8 ; final_data[8] = None
346 fmap['tempPresUnit'] = 9 ; final_data[9] = 'F'
347 fmap['precipitationGauge'] = 10 ; final_data[10] = None
348 fmap['precipitationGaugeUnit'] = 11 ; final_data[11] = 'in'
349 fmap['windSpeedNum'] = 12 ; final_data[12] = None
350 fmap['windSpeedUnit'] = 13 ; final_data[13] = 'mph'
351 fmap['windDirectionNum'] = 14 ; final_data[14] = None
352 fmap['hS'] = 15 ; final_data[15] = None
353 fmap['hsUnit'] = 16 ; final_data[16] = 'in'
354 fmap['baro'] = 17 ; final_data[17] = None
355 fmap['baroUnit'] = 18 ; final_data[18] = 'inHg'
356 fmap['rH'] = 19 ; final_data[19] = None
357 fmap['windGustSpeedNum'] = 20 ; final_data[20] = None
358 fmap['windGustSpeedNumUnit'] = 21 ; final_data[21] = 'mph'
359 fmap['windGustDirNum'] = 22 ; final_data[22] = None
360 fmap['dewPoint'] = 23 ; final_data[23] = None
361 fmap['dewPointUnit'] = 24 ; final_data[24] = 'F'
362 fmap['hn24Auto'] = 25 ; final_data[25] = None
363 fmap['hn24AutoUnit'] = 26 ; final_data[26] = 'in'
364 fmap['hstAuto'] = 27 ; final_data[27] = None
365 fmap['hstAutoUnit'] = 28 ; final_data[28] = 'in'
366
367 return (fmap, final_data)
368
369 def setup_infoex_counterparts_mapping(provider):
370 """
371 Create a mapping of the NRCS/MesoWest fields that this program supports to
372 their InfoEx counterparts
373 """
374 iemap = {}
375
376 if provider == 'nrcs':
377 iemap['PREC'] = 'precipitationGauge'
378 iemap['TOBS'] = 'tempPres'
379 iemap['TMAX'] = 'tempMaxHour'
380 iemap['TMIN'] = 'tempMinHour'
381 iemap['SNWD'] = 'hS'
382 iemap['PRES'] = 'baro'
383 iemap['RHUM'] = 'rH'
384 iemap['WSPD'] = 'windSpeedNum'
385 iemap['WDIR'] = 'windDirectionNum'
386 # unsupported by NRCS:
387 # windGustSpeedNum
388 elif provider == 'mesowest':
389 iemap['precip_accum'] = 'precipitationGauge'
390 iemap['air_temp'] = 'tempPres'
391 iemap['air_temp_high_24_hour'] = 'tempMaxHour'
392 iemap['air_temp_low_24_hour'] = 'tempMinHour'
393 iemap['snow_depth'] = 'hS'
394 iemap['pressure'] = 'baro'
395 iemap['relative_humidity'] = 'rH'
396 iemap['wind_speed'] = 'windSpeedNum'
397 iemap['wind_direction'] = 'windDirectionNum'
398 iemap['wind_gust'] = 'windGustSpeedNum'
399 elif provider == 'python':
400 # we expect Python programs to use the InfoEx data type names
401 iemap['precipitationGauge'] = 'precipitationGauge'
402 iemap['tempPres'] = 'tempPres'
403 iemap['tempMaxHour'] = 'tempMaxHour'
404 iemap['tempMinHour'] = 'tempMinHour'
405 iemap['hS'] = 'hS'
406 iemap['baro'] = 'baro'
407 iemap['rH'] = 'rH'
408 iemap['windSpeedNum'] = 'windSpeedNum'
409 iemap['windDirectionNum'] = 'windDirectionNum'
410 iemap['windGustSpeedNum'] = 'windGustSpeedNum'
411
412 return iemap
413
414 # provider-specific operations
415 def get_nrcs_data(begin, end, station):
416 """get the data we're after from the NRCS WSDL"""
417 transport = zeep.transports.Transport(cache=zeep.cache.SqliteCache())
418 transport.session.verify = False
419 client = zeep.Client(wsdl=station['source'], transport=transport)
420 remote_data = {}
421
422 # massage begin/end date format
423 begin_date_str = begin.strftime('%Y-%m-%d %H:%M:00')
424 end_date_str = end.strftime('%Y-%m-%d %H:%M:00')
425
426 for element_cd in station['desired_data']:
427 time_element = time.time()
428
429 # get the last three hours of data for this elementCd/element_cd
430 tmp = client.service.getHourlyData(
431 stationTriplets=[station['station_id']],
432 elementCd=element_cd,
433 ordinal=1,
434 beginDate=begin_date_str,
435 endDate=end_date_str)
436
437 LOG.info("Time to get NRCS elementCd '%s': %.3f sec", element_cd,
438 time.time() - time_element)
439
440 values = tmp[0]['values']
441
442 # sort and isolate the most recent
443 #
444 # NOTE: we do this because sometimes there are gaps in hourly data
445 # in NRCS; yes, we may end up with slightly inaccurate data,
446 # so perhaps this decision will be re-evaluated in the future
447 if values:
448 ordered = sorted(values, key=lambda t: t['dateTime'], reverse=True)
449 remote_data[element_cd] = ordered[0]['value']
450 else:
451 remote_data[element_cd] = None
452
453 return remote_data
454
455 def get_mesowest_data(begin, end, station):
456 """get the data we're after from the MesoWest/Synoptic API"""
457 remote_data = {}
458
459 # massage begin/end date format
460 begin_date_str = begin.strftime('%Y%m%d%H%M')
461 end_date_str = end.strftime('%Y%m%d%H%M')
462
463 # construct final, completed API URL
464 api_req_url = station['source'] + '&start=' + begin_date_str + '&end=' + end_date_str
465 req = requests.get(api_req_url)
466
467 try:
468 json = req.json()
469 except ValueError:
470 LOG.error("Bad JSON in MesoWest response")
471 sys.exit(1)
472
473 try:
474 observations = json['STATION'][0]['OBSERVATIONS']
475 except KeyError as exc:
476 LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
477 sys.exit(1)
478 except IndexError as exc:
479 LOG.error("Unexpected JSON in MesoWest response: '%s'", exc)
480 try:
481 LOG.error("Detailed MesoWest response: '%s'",
482 json['SUMMARY']['RESPONSE_MESSAGE'])
483 except KeyError:
484 pass
485 sys.exit(1)
486 except ValueError as exc:
487 LOG.error("Bad JSON in MesoWest response: '%s'", exc)
488 sys.exit(1)
489
490 pos = len(observations['date_time']) - 1
491
492 for element_cd in station['desired_data'].split(','):
493 # sort and isolate the most recent, see note above in NRCS for how and
494 # why this is done
495 #
496 # NOTE: Unlike in the NRCS case, the MesoWest API response contains all
497 # data (whereas with NRCS, we have to make a separate request for
498 # each element we want). This is nice for network efficiency but
499 # it means we have to handle this part differently for each.
500 #
501 # NOTE: Also unlike NRCS, MesoWest provides more granular data; NRCS
502 # provides hourly data, but MesoWest can often provide data every
503 # 10 minutes -- though this provides more opportunity for
504 # irregularities
505
506 # we may not have the data at all
507 key_name = element_cd + '_set_1'
508
509 if key_name in observations:
510 if observations[key_name][pos]:
511 remote_data[element_cd] = observations[key_name][pos]
512
513 # mesowest by default provides wind_speed in m/s, but
514 # we specify 'english' units in the request; either way,
515 # we want mph
516 if element_cd in ('wind_speed', 'wind_gust'):
517 remote_data[element_cd] = kn_to_mph(remote_data[element_cd])
518 else:
519 remote_data[element_cd] = None
520 else:
521 remote_data[element_cd] = None
522
523 return remote_data
524
525 def switch_units_to_metric(data_map, mapping):
526 """replace units with metric counterparts"""
527
528 # NOTE: to update this, use the fmap<->final_data mapping laid out
529 # in setup_infoex_fields_mapping ()
530 data_map[mapping['tempMaxHourUnit']] = 'C'
531 data_map[mapping['tempMinHourUnit']] = 'C'
532 data_map[mapping['tempPresUnit']] = 'C'
533 data_map[mapping['precipitationGaugeUnit']] = 'mm'
534 data_map[mapping['hsUnit']] = 'cm'
535 data_map[mapping['windSpeedUnit']] = 'm/s'
536 data_map[mapping['windGustSpeedNumUnit']] = 'm/s'
537 data_map[mapping['dewPointUnit']] = 'C'
538 data_map[mapping['hn24AutoUnit']] = 'cm'
539 data_map[mapping['hstAutoUnit']] = 'cm'
540
541 return data_map
542
543 def convert_nrcs_units_to_metric(element_cd, value):
544 """convert NRCS values from English to metric"""
545 if element_cd == 'TOBS':
546 value = f_to_c(value)
547 elif element_cd == 'SNWD':
548 value = in_to_cm(value)
549 elif element_cd == 'PREC':
550 value = in_to_mm(value)
551 return value
552
553 # CSV operations
554 def write_local_csv(path_to_file, data):
555 """Write the specified CSV file to disk"""
556 with open(path_to_file, 'w') as file_object:
557 # The requirement is that empty values are represented in the CSV
558 # file as "", csv.QUOTE_NONNUMERIC achieves that
559 LOG.debug("writing CSV file '%s'", path_to_file)
560 writer = csv.writer(file_object, quoting=csv.QUOTE_NONNUMERIC)
561 writer.writerow(data)
562 file_object.close()
563 return True
564
565 def upload_csv(path_to_file, infoex_data):
566 """Upload the specified CSV file to InfoEx FTP and remove the file"""
567 with open(path_to_file, 'rb') as file_object:
568 LOG.debug("uploading FTP file '%s'", infoex_data['host'])
569 ftp = FTP(infoex_data['host'], infoex_data['uuid'],
570 infoex_data['api_key'])
571 ftp.storlines('STOR ' + path_to_file, file_object)
572 ftp.close()
573 file_object.close()
574 os.remove(path_to_file)
575
576 # other miscellaneous routines
577 def setup_time_values(station):
578 """establish time bounds of data request(s)"""
579
580 # default timezone to UTC (for MesoWest)
581 tz = pytz.utc
582
583 # but for NRCS, use the config-specified timezone
584 if station['provider'] == 'nrcs':
585 tz = station['tz']
586
587 # floor time to nearest hour
588 date_time = datetime.datetime.now(tz=tz)
589 end_date = date_time - datetime.timedelta(minutes=date_time.minute % 60,
590 seconds=date_time.second,
591 microseconds=date_time.microsecond)
592 begin_date = end_date - datetime.timedelta(hours=3)
593 return (begin_date, end_date)
594
595 def f_to_c(f):
596 """convert Fahrenheit to Celsius"""
597 return (float(f) - 32) * 5.0/9.0
598
599 def in_to_cm(inches):
600 """convert inches to centimetrs"""
601 return float(inches) * 2.54
602
603 def in_to_mm(inches):
604 """convert inches to millimeters"""
605 return (float(inches) * 2.54) * 10.0
606
607 def ms_to_mph(ms):
608 """convert meters per second to miles per hour"""
609 return ms * 2.236936
610
611 def kn_to_mph(kn):
612 """convert knots to miles per hour"""
613 return kn * 1.150779
614
615 if __name__ == "__main__":
616 sys.exit(main())