Ignore warning about duplicate key when inserting openweathermap data.

[chrisu/seepark.git] / web / seepark_web.py

diff --git a/web/seepark_web.py b/web/seepark_web.py
index 065e3f2a6365535dfae30cff3146d3e43c7cc9a0..e8b9da11dc6f0b4b20ca6c44841cc22825acb213 100644 (file)
--- a/web/seepark_web.py
+++ b/web/seepark_web.py
@@ -7,17 +7,27 @@ import sys
 from collections import defaultdict
 from flask import Flask, render_template, jsonify, request
 import flask.json
 from collections import defaultdict
 from flask import Flask, render_template, jsonify, request
 import flask.json

-from flask_sqlalchemy import SQLAlchemy
+from flask_sqlalchemy import SQLAlchemy, inspect

 
 
 
 

-sys.path.append('..')
+app_path = os.path.dirname(os.path.realpath(__file__))
+lib_path = os.path.join(app_path, '..')
+sys.path.append(lib_path)

 from seeparklib.openweathermap import openweathermap_json, OpenWeatherMapError
 
 
 from seeparklib.openweathermap import openweathermap_json, OpenWeatherMapError
 
 

+# https://stackoverflow.com/a/37350445
+def sqlalchemy_model_to_dict(model):
+    return {c.key: getattr(model, c.key)
+        for c in inspect(model).mapper.column_attrs}
+
+

 class JSONEncoder(flask.json.JSONEncoder):
     def default(self, object):
         if isinstance(object, datetime.datetime):
             return object.isoformat()
 class JSONEncoder(flask.json.JSONEncoder):
     def default(self, object):
         if isinstance(object, datetime.datetime):
             return object.isoformat()

+        elif isinstance(object, db.Model):
+            return sqlalchemy_model_to_dict(object)

         return super().default(object)
 
 
         return super().default(object)
 
 


@@ -37,6 +47,7 @@ config = configparser.ConfigParser()
 config.read(os.environ['SEEPARKINI'])
 apikey = config.get('openweathermap', 'apikey')
 cityid = config.get('openweathermap', 'cityid')
 config.read(os.environ['SEEPARKINI'])
 apikey = config.get('openweathermap', 'apikey')
 cityid = config.get('openweathermap', 'cityid')

+mainsensor = config.get('temperature', 'mainsensor')

 
 app = Flask(__name__)
 app.json_encoder = JSONEncoder
 
 app = Flask(__name__)
 app.json_encoder = JSONEncoder


@@ -50,56 +61,78 @@ class Sensors(db.Model):
     __tablename__ = 'sensors'
 
 
     __tablename__ = 'sensors'
 
 

-def select_sensordata(initial_where):
-    query = Sensors.query.filter(initial_where)
-    begin = request.args.get('begin', None, parse_datetime)
-    end = request.args.get('end', None, parse_datetime)
+def select_sensordata(sensor_id, sensor_type, begin, end, mode):
+    query = Sensors.query
+    if sensor_id is not None:
+        query = query.filter(Sensors.sensor_id == sensor_id)
+    if sensor_type is not None:
+        query = query.filter(Sensors.value_type == sensor_type)

     if begin is not None:
         query = query.filter(Sensors.timestamp >= begin)
     if end is not None:
         query = query.filter(Sensors.timestamp <= end)
     if begin is not None:
         query = query.filter(Sensors.timestamp >= begin)
     if end is not None:
         query = query.filter(Sensors.timestamp <= end)

-    result = query.all()
-
-    mode = request.args.get('mode', 'full')
-    if mode == 'consolidated':
-        if begin is None or end is None:
-            pass
+    if mode == 'consolidated' and begin is None and end is None:
+        # copied from munin/master/_bin/munin-cgi-graph.in
+        # interval in seconds for data points
+        resolutions = dict(
+            day   =   300,
+            week  =  1800,
+            month =  7200,
+            year  = 86400,
+        )
+        duration = (end - begin).total_seconds()
+        day = 60 * 60 * 24
+        if duration < day:
+            resolution = resolutions['day']
+        elif duration < 7 * day:
+            resolution = resolutions['week']
+        elif duration < 31 * day:
+            resolution = resolutions['month']

         else:
         else:

-            # copied from munin/master/_bin/munin-cgi-graph.in
-            resolutions = dict(
-                day   =   300,
-                week  =  1800,
-                month =  7200,
-                year  = 86400,
-            )
-            duration = (end - begin).total_seconds()
-            day = 60 * 60 * 24
-            if duration < day:
-                resolution = resolutions['day']
-            elif duration < 7 * day:
-                resolution = resolutions['week']
-            elif duration < 31 * day:
-                resolution = resolutions['month']
-            else:
-                resolution = resolutions['year']
-            # TODO: filter out samples from 'result'
-            # like loop over results and skip if timestamp(n+1)-timestamp(n)<resolution
-
+            resolution = resolutions['year']
+        # TODO: filter out samples from 'result'
+        # something like 
+        # select mean(temperature) from sensors where ... group by mod(timestamp, resolution)
+        # func.avg(...)
+        #
+        # from https://stackoverflow.com/questions/4342370/grouping-into-interval-of-5-minutes-within-a-time-range
+        # SELECT
+        #    timestamp,  -- not sure about that
+        #        name,
+        #            count(b.name)
+        #            FROM time a, id 
+        #            WHERE …
+        #            GROUP BY 
+        #            UNIX_TIMESTAMP(timestamp) DIV 300, name
+    return query.all()
+
+
+def convert_to_c3(result):
+    c3result = defaultdict(list)
+    for row in result:
+        c3result[row.sensor_id].append(row.value)
+        dt = row.timestamp.strftime('%Y-%m-%d %H:%M:%S')
+        c3result[row.sensor_id + '_x'].append(dt)
+    return c3result
+
+
+def sensordata(sensor_id=None, sensor_type=None):
+    begin = request.args.get('begin', None, parse_datetime)
+    end = request.args.get('end', None, parse_datetime)
+    mode = request.args.get('mode', 'full')

     format = request.args.get('format', 'default')
     format = request.args.get('format', 'default')

+
+    result = select_sensordata(sensor_id, sensor_type, begin, end, mode)
+

     if format == 'c3':
     if format == 'c3':

-        c3result = defaultdict(list)
-        for row in result:
-            c3result[row.sensor_id].append(row.value)
-            dt = row.timestamp.strftime('%Y-%m-%d %H:%M:%S')
-            c3result[row.sensor_id + '_x'].append(dt)
-        result = c3result
+        return convert_to_c3(result)

     return result
 
 
 def currentairtemperature(apikey, cityid):
     """Retruns the tuple temperature, datetime (as float, datetime) in case of success, otherwise None, None."""
     try:
     return result
 
 
 def currentairtemperature(apikey, cityid):
     """Retruns the tuple temperature, datetime (as float, datetime) in case of success, otherwise None, None."""
     try:

-        weatherdata = openweathermap_json(apikey, cityid)
+        url, weatherdata = openweathermap_json(apikey, cityid)

         return weatherdata['main']['temp'], datetime.datetime.fromtimestamp(weatherdata['dt'])
     except OpenWeatherMapError:
         return None, None
         return weatherdata['main']['temp'], datetime.datetime.fromtimestamp(weatherdata['dt'])
     except OpenWeatherMapError:
         return None, None


@@ -127,8 +160,8 @@ def sensorid(version, sensor_id):
     mode=<full|consolidated>, optional. return all rows (default) or with lower resolution (for charts)
     format=<default|c3>, optional. return result as returned by sqlalchemy (default) or formatted for c3.js
     """
     mode=<full|consolidated>, optional. return all rows (default) or with lower resolution (for charts)
     format=<default|c3>, optional. return result as returned by sqlalchemy (default) or formatted for c3.js
     """

-    result = select_sensordata(Sensors.sensor_id == sensor_id)
-    return jsonify([row._asdict() for row in result])
+    result = sensordata(sensor_id=sensor_id)
+    return jsonify(result)

 
 
 @app.route('/api/<version>/sensor/type/<sensor_type>')
 
 
 @app.route('/api/<version>/sensor/type/<sensor_type>')


@@ -141,7 +174,7 @@ def sensortype(version, sensor_type):
     mode=<full|consolidated>, optional. return all rows (default) or with lower resolution (for charts)
     format=<default|c3>, optional. return result as returned by sqlalchemy (default) or formatted for c3.js
     """
     mode=<full|consolidated>, optional. return all rows (default) or with lower resolution (for charts)
     format=<default|c3>, optional. return result as returned by sqlalchemy (default) or formatted for c3.js
     """

-    result = select_sensordata(Sensors.value_type == sensor_type)
+    result = sensordata(sensor_type=sensor_type)

     return jsonify(result)
 
 
     return jsonify(result)
 
 


@@ -178,7 +211,7 @@ def data(timespan):
 @app.route("/")
 def index():
     airvalue, airtime     = currentairtemperature(apikey, cityid)
 @app.route("/")
 def index():
     airvalue, airtime     = currentairtemperature(apikey, cityid)

-    watervalue, watertime = currentwatertemperature('0316a21383ff') # config? mainwatertemp?
+    watervalue, watertime = currentwatertemperature(mainsensor)

 
     return render_template(
         'seepark_web.html',
 
     return render_template(
         'seepark_web.html',