X-Git-Url: https://git.toastfreeware.priv.at/chrisu/seepark.git/blobdiff_plain/7157d90050101aa4149f126cccf275622d7a4e17..d782d33862c95ae0fd1c2b0f8cd75a9cbc2af95b:/web/seepark_web.py diff --git a/web/seepark_web.py b/web/seepark_web.py index b132960..c6ceffb 100644 --- a/web/seepark_web.py +++ b/web/seepark_web.py @@ -1,19 +1,45 @@ -from random import uniform +import collections import datetime +import itertools import time import configparser import os import sys from collections import defaultdict -from flask import Flask, render_template, jsonify, request, abort, Response +import io +import numpy as np +import matplotlib +matplotlib.use('pdf') +import matplotlib.pyplot as plt +from matplotlib.backends.backend_pdf import PdfPages + +from flask import Flask, render_template, jsonify, request, abort, Response, make_response import flask.json from flask_sqlalchemy import SQLAlchemy, inspect - - -app_path = os.path.dirname(os.path.realpath(__file__)) -lib_path = os.path.join(app_path, '..') -sys.path.append(lib_path) -from seeparklib.openweathermap import openweathermap_json, OpenWeatherMapError +from sqlalchemy import func + +MONTH_DE = [ + 'Jänner', + 'Februar', + 'März', + 'April', + 'Mai', + 'Juni', + 'Juli', + 'August', + 'September', + 'Oktober', + 'November', + 'Dezember'] + +DAY_OF_WEEK_DE = [ + 'Montag', + 'Dienstag', + 'Mittwoch', + 'Donnerstag', + 'Freitag', + 'Samstag', + 'Sonntag'] # https://stackoverflow.com/a/37350445 @@ -35,6 +61,11 @@ def parse_datetime(date_str): return datetime.datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S') +def ntimes(it, n): + for v in it: + yield from itertools.repeat(v, n) + + def get_sqlalchemy_database_uri(config): user = config.get('database', 'user') pwd = config.get('database', 'password') @@ -47,7 +78,7 @@ config = configparser.ConfigParser() config.read(os.environ['SEEPARKINI']) apikey = config.get('openweathermap', 'apikey') cityid = config.get('openweathermap', 'cityid') -mainsensor = config.get('temperature', 'mainsensor') +mainsensor = config.get('webapp', 'mainsensor') app = Flask(__name__) app.json_encoder = JSONEncoder @@ -65,7 +96,30 @@ class OpenWeatherMap(db.Model): __tablename__ = 'openweathermap' -def select_sensordata(sensor_id, sensor_type, begin, end, mode): +def calc_grouping_resolution(begin, end): + """How many data points should be between the timestamps begin and end?""" + # copied from munin/master/_bin/munin-cgi-graph.in + # except day: 300 -> 600 + resolutions = dict( + day = 600, + week = 1800, + month = 7200, + year = 86400, + ) + duration = (end - begin).total_seconds() + day = 60 * 60 * 24 + if duration <= day: + resolution = resolutions['day'] + elif duration <= 7 * day: + resolution = resolutions['week'] + elif duration <= 31 * day: + resolution = resolutions['month'] + else: + resolution = resolutions['year'] + return resolution + + +def select_sensordata(sensor_id, sensor_type, begin, end): query = Sensors.query if sensor_id is not None: query = query.filter(Sensors.sensor_id == sensor_id) @@ -75,66 +129,99 @@ def select_sensordata(sensor_id, sensor_type, begin, end, mode): query = query.filter(Sensors.timestamp >= begin) if end is not None: query = query.filter(Sensors.timestamp <= end) - if mode == 'consolidated' and begin is not None and end is not None: - # copied from munin/master/_bin/munin-cgi-graph.in - # interval in seconds for data points - resolutions = dict( - day = 300, - week = 1800, - month = 7200, - year = 86400, - ) - duration = (end - begin).total_seconds() - day = 60 * 60 * 24 - if duration < day: - resolution = resolutions['day'] - elif duration < 7 * day: - resolution = resolutions['week'] - elif duration < 31 * day: - resolution = resolutions['month'] - else: - resolution = resolutions['year'] - # TODO: filter out samples from 'result' - # something like - # select to_seconds(datetime) DIV (60*60*24) as interval_id, min(datetime), max(datetime), min(temp), avg(temp), max(temp), count(temp) from openweathermap group by interval_id order by interval_id; return query.all() -def select_openweatherdata(cityid, begin, end, mode): +def sensordata_to_xy(sensordata): + sensordata = list(sensordata) + x = np.array([d.timestamp for d in sensordata]) + y = np.array([d.value for d in sensordata]) + return x, y + + +def select_sensordata_grouped(sensor_id, sensor_type, begin, end): + # determine resolution (interval in seconds for data points) + resolution = calc_grouping_resolution(begin, end) + + # Let the database do the grouping. Example in SQL (MySQL): + # select to_seconds(datetime) DIV (60*60*24) as interval_id, min(datetime), max(datetime), min(temp), avg(temp), max(temp), count(temp) from openweathermap group by interval_id order by interval_id; + query = db.session.query(func.to_seconds(Sensors.timestamp).op('div')(resolution).label('g'), + func.from_unixtime(func.avg(func.unix_timestamp(Sensors.timestamp))).label('timestamp'), + func.avg(Sensors.value).label('value'), + Sensors.sensor_id, Sensors.value_type, Sensors.sensor_name) + if sensor_id is not None: + query = query.filter(Sensors.sensor_id == sensor_id) + if sensor_type is not None: + query = query.filter(Sensors.value_type == sensor_type) + query = query.filter(Sensors.timestamp >= begin) + query = query.filter(Sensors.timestamp <= end) + query = query.group_by('g', Sensors.sensor_id, Sensors.value_type, Sensors.sensor_name) + return query.all() + + +def select_openweatherdata(cityid, begin, end): query = OpenWeatherMap.query.filter(OpenWeatherMap.cityid == cityid) if begin is not None: query = query.filter(OpenWeatherMap.datetime >= begin) if end is not None: query = query.filter(OpenWeatherMap.datetime <= end) - if mode == 'consolidated' and begin is not None and end is not None: - # copied from munin/master/_bin/munin-cgi-graph.in - # interval in seconds for data points - resolutions = dict( - day = 300, - week = 1800, - month = 7200, - year = 86400, - ) - duration = (end - begin).total_seconds() - day = 60 * 60 * 24 - if duration < day: - resolution = resolutions['day'] - elif duration < 7 * day: - resolution = resolutions['week'] - elif duration < 31 * day: - resolution = resolutions['month'] - else: - resolution = resolutions['year'] - # TODO: filter out samples from 'result' - # something like - # select to_seconds(datetime) DIV (60*60*24) as interval_id, min(datetime), max(datetime), min(temp), avg(temp), max(temp), count(temp) from openweathermap group by interval_id order by interval_id; return query.all() +def openweatherdata_to_xy(openweatherdata): + openweatherdata = list(openweatherdata) + x = np.array([d.datetime for d in openweatherdata]) + y = np.array([d.temp for d in openweatherdata]) + return x, y + + +def select_openweatherdata_grouped(cityid, begin, end): + # determine resolution (interval in seconds for data points) + resolution = calc_grouping_resolution(begin, end) + + # Let the database do the grouping. Example in SQL (MySQL): + # select to_seconds(datetime) DIV (60*60*24) as interval_id, min(datetime), max(datetime), min(temp), avg(temp), max(temp), count(temp) from openweathermap group by interval_id order by interval_id; + query = db.session.query(func.to_seconds(OpenWeatherMap.datetime).op('div')(resolution).label('g'), + func.from_unixtime(func.avg(func.unix_timestamp(OpenWeatherMap.datetime))).label('datetime'), + func.avg(OpenWeatherMap.temp).label('temp'), + OpenWeatherMap.cityid) + OpenWeatherMap.query.filter(OpenWeatherMap.cityid == cityid) + query = query.filter(OpenWeatherMap.datetime >= begin) + query = query.filter(OpenWeatherMap.datetime <= end) + query = query.group_by('g', OpenWeatherMap.cityid) + return query.all() + + +def estimate_swimmer_count(date): + return date.day + + +def select_swimmerdata(begin, end): + def report_times(begin, end): + d = begin + while d < end: + for t in [10, 15]: + a = datetime.datetime.combine(d.date(), datetime.time(t)) + if a >= d: + yield a + d += datetime.timedelta(days=1) + SwimmerData = collections.namedtuple('SwimmerData', ['datetime', 'count']) + for d in report_times(begin, end): + count = estimate_swimmer_count(d) + yield SwimmerData(d, count) + + +def swimmerdata_to_xy(swimmerdata): + swimmerdata = list(swimmerdata) + x = np.array([d.datetime for d in swimmerdata]) + y = np.array([d.count for d in swimmerdata]) + return x, y + + def convert_to_c3(result, id, field_x, field_y): c3result = defaultdict(list) for row in result: - c3result[getattr(row, id)].append(getattr(row, field_y)) + c3result[str(getattr(row, id))].append(getattr(row, field_y)) dt = getattr(row, field_x).strftime('%Y-%m-%d %H:%M:%S') c3result[str(getattr(row, id)) + '_x'].append(dt) return c3result @@ -160,7 +247,14 @@ def sensordata(sensor_id=None, sensor_type=None): mode = request.args.get('mode', 'full') format = request.args.get('format', 'default') - result = select_sensordata(sensor_id, sensor_type, begin, end, mode) + if mode == 'full': + result = select_sensordata(sensor_id, sensor_type, begin, end) + elif mode == 'consolidated': + if begin is None or end is None: + abort(Response('begin and end have to be set for mode==consolidated', 400)) + result = select_sensordata_grouped(sensor_id, sensor_type, begin, end) + else: + abort(Response('unknown value for mode', 400)) if format == 'c3': return convert_to_c3(result, 'sensor_id', 'timestamp', 'value') @@ -173,7 +267,14 @@ def openweathermapdata(cityid): mode = request.args.get('mode', 'full') format = request.args.get('format', 'default') - result = select_openweatherdata(cityid, begin, end, mode) + if mode == 'full': + result = select_openweatherdata(cityid, begin, end) + elif mode == 'consolidated': + if begin is None or end is None: + abort(Response('begin and end have to be set for mode==consolidated', 400)) + result = select_openweatherdata_grouped(cityid, begin, end) + else: + abort(Response('unknown value for mode', 400)) if format == 'c3': return convert_to_c3(result, 'cityid', 'datetime', 'temp') @@ -190,6 +291,17 @@ def currentwatertemperature(sensorid): return result.value, result.timestamp +def first_of_month(date, month): + date = date.replace(day=1) + if month == 0: + return date + if month == 1: + return (date + datetime.timedelta(days=42)).replace(day=1) + if month == -1: + return (date - datetime.timedelta(days=1)).replace(day=1) + assert False + + @app.route('/api//sensors/') def sensors(version): """List all sensors found in the database""" @@ -202,10 +314,11 @@ def sensorid(version, sensor_id): """Return all data for a specific sensor URL parameters: - begin=, optional, format like "2018-05-19T21:07:53" - end=, optional, format like "2018-05-19T21:07:53" - mode=, optional. return all rows (default) or with lower resolution (for charts) - format=, optional. return result as returned by sqlalchemy (default) or formatted for c3.js + + * ``begin=``, optional, format like ``2018-05-19T21:07:53`` + * ``end=``, optional, format like ``2018-05-19T21:07:53`` + * ``mode=``, optional. return all rows (default) or with lower resolution (for charts) + * ``format=``, optional. return result as returned by sqlalchemy (default) or formatted for c3.js """ result = sensordata(sensor_id=sensor_id) return jsonify(result) @@ -216,10 +329,11 @@ def sensortype(version, sensor_type): """Return all data for a specific sensor type URL parameters: - begin=, optional, format like "2018-05-19T21:07:53" - end=, optional, format like "2018-05-19T21:07:53" - mode=, optional. return all rows (default) or with lower resolution (for charts) - format=, optional. return result as returned by sqlalchemy (default) or formatted for c3.js + + * ``begin=``, optional, format like ``2018-05-19T21:07:53`` + * ``end=``, optional, format like ``2018-05-19T21:07:53`` + * ``mode=``, optional. return all rows (default) or with lower resolution (for charts) + * ``format=``, optional. return result as returned by sqlalchemy (default) or formatted for c3.js """ result = sensordata(sensor_type=sensor_type) return jsonify(result) @@ -239,40 +353,122 @@ def openweathermap_city(version, cityid): return jsonify(result) -@app.route('/data/', defaults={'timespan': 1}) -@app.route("/data/", methods=['GET']) -def data(timespan): - granularity = 5 * timespan # (every) minute(s) per day - samples = 60/granularity * 24 * timespan # per hour over whole timespan - s4m = [] - s4m_x = [] - s5m = [] - s5m_x = [] - end = time.time() - start = end - samples * granularity * 60 - - for i in range(int(samples)): - s4m.append(uniform(-10,30)) - s5m.append(uniform(-10,30)) - s4mt = uniform(start, end) - s5mt = uniform(start, end) - s4m_x.append(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(s4mt))) - s5m_x.append(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(s5mt))) - - data = { - '0316a2193bff': s4m, - '0316a2193bff_x': s4m_x, - '0316a21383ff': s5m, - '0316a21383ff_x': s5m_x, - } - - return jsonify(data) +@app.route('/api//currentairtemperature') +def currentair(version): + value, timestamp = currentairtemperature(cityid) + return jsonify({"value": value, "timestamp": timestamp}) + + +@app.route('/api//currentwatertemperature') +def currentwater(version): + value, timestamp = currentwatertemperature(mainsensor) + return jsonify({"value": value, "timestamp": timestamp}) + + +@app.route('/report//') +def report(year, month): + """Report for given year (4 digits) and month (2 digits) + """ + paper_size = (29.7 / 2.54, 21. / 2.54) # A4 + + begin = datetime.datetime(year, month, 1) + end = first_of_month(begin, 1) + + water_data = sensordata_to_xy(select_sensordata(mainsensor, 'Wassertemperatur', begin, end)) + air_data = openweatherdata_to_xy(select_openweatherdata(cityid, begin, end)) + swimmer_data = swimmerdata_to_xy(select_swimmerdata(begin, end)) + + report_times = [datetime.time(10), datetime.time(15)] + report_data = {'Wasser': water_data, 'Luft': air_data} + + days_datetime = [] + d = begin + while d < end: + days_datetime.append(d) + d = d + datetime.timedelta(1) + + binary_pdf = io.BytesIO() + with PdfPages(binary_pdf) as pdf: + title = 'Seepark Obsteig {} {}'.format(MONTH_DE[begin.month-1], begin.year) + + # graphic + plt.figure(figsize=paper_size) + report_colors = [] + for label, data in sorted(report_data.items(), reverse=True): + x, y = data + lines = plt.plot(x, y, label=label) + report_colors.append(lines[0].get_color()) + plt.xticks(days_datetime, [''] * len(days_datetime)) + plt.ylabel('Temperatur in °C') + plt.axis(xmin=begin, xmax=end) + plt.legend() + plt.grid() + plt.title(title) + + # table + columns = [] + for d in days_datetime: + columns.append('{}.'.format(d.day)) + rows = [] + for label in sorted(report_data.keys(), reverse=True): + for t in report_times: + rows.append('{:02d}:{:02d} {} °C'.format(t.hour, t.minute, label)) + for t in report_times: + rows.append('{:02d}:{:02d} Badende'.format(t.hour, t.minute)) + cells = [] + for label, data in sorted(report_data.items(), reverse=True): + for t in report_times: + row_cells = [] + x, y = data + for d in days_datetime: + report_datetime = datetime.datetime.combine(d.date(), t) + closest_index = np.argmin(np.abs(x - report_datetime)) + if abs(x[closest_index] - report_datetime) > datetime.timedelta(hours=1): + cell = 'N/A' + else: + value = y[closest_index] + cell = '{:.1f}'.format(value) + row_cells.append(cell) + cells.append(row_cells) + for t in report_times: + row_cells = [] + x, y = swimmer_data + for d in days_datetime: + report_datetime = datetime.datetime.combine(d.date(), t) + closest_index = np.argmin(np.abs(x - report_datetime)) + if abs(x[closest_index] - report_datetime) > datetime.timedelta(hours=1): + cell = 'N/A' + else: + cell = y[closest_index] + row_cells.append(cell) + cells.append(row_cells) + row_colors = list(ntimes(report_colors + ['w'], len(report_times))) + table = plt.table(cellText=cells, colLabels=columns, rowLabels=rows, rowColours=row_colors, loc='bottom') + table.scale(xscale=1, yscale=2) + plt.title(title) + plt.subplots_adjust(left=0.15, right=0.97, bottom=0.3) # do not cut row labels + pdf.savefig() + + pdf_info = pdf.infodict() + pdf_info['Title'] = title + pdf_info['Author'] = 'Chrisu Jähnl' + pdf_info['Subject'] = 'Temperaturen' + pdf_info['Keywords'] = 'Seepark Obsteig' + pdf_info['CreationDate'] = datetime.datetime.now() + pdf_info['ModDate'] = datetime.datetime.today() + + response = make_response(binary_pdf.getvalue()) + response.headers['Content-Type'] = 'application/pdf' + response.headers['Content-Disposition'] = 'attachment; filename=seepark_{:04d}-{:02d}.pdf'.format(year, month) + return response @app.route("/") def index(): airvalue, airtime = currentairtemperature(cityid) watervalue, watertime = currentwatertemperature(mainsensor) + this_month = first_of_month(datetime.date.today(), 0) + last_month = first_of_month(this_month, -1) return render_template( 'seepark_web.html', @@ -281,4 +477,6 @@ def index(): watertime=watertime, airvalue=airvalue, airtime=airtime, + this_month=this_month, + last_month=last_month )