scripts/update_wrgeojson_to_highres_batch.py

   1 #!/usr/bin/python3
   2 import argparse
   3 import configparser
   4 import json
   5 import sys
   6 from typing import List, Tuple, Iterable, NamedTuple
   7 from xml.etree.ElementTree import ElementTree
   8
   9 import geojson
  10 import shapely
  11 import sqlalchemy
  12 from geojson import Feature, GeoJSON
  13 from shapely.geometry import shape
  14 from sqlalchemy import create_engine
  15 from termcolor import cprint
  16
  17 from wrpylib.cli_tools import unified_diff, input_yes_no_quit, Choice
  18 from wrpylib.json_tools import format_json
  19 from wrpylib.mwapi import WikiSite
  20 from wrpylib.wrgeojson import join_wrgeojson_ways, WrLineStringFeatureType, get_geometry_center, \
  21     LanToToMetricTransformer, DEFAULT_MAX_DIST_ERROR_M, get_geometry_envelope, get_geometry_points
  22 from wrpylib.wrosm import find_sledrun_relations, convert_osm_to_geojson, DeRefError, tags
  23
  24
  25 def filter_way(feature: dict, way_type: str) -> bool:
  26     return feature['geometry']['type'] == 'LineString' and feature.get('properties', {}).get('type') == way_type
  27
  28
  29 def feature_distance(a: Feature, b: Feature) -> float:
  30     return shape(a['geometry']).hausdorff_distance(shape(b['geometry']))
  31
  32
  33 def geojson_distance(a: GeoJSON, b: GeoJSON) -> float:
  34     return get_geometry_points(a).hausdorff_distance(get_geometry_points(b))
  35
  36
  37 def geojson_way_sort_key(feature: Feature, reference_feature: Feature) -> Tuple[str, float]:
  38     return feature.get('properties', {}).get('type', ''), feature_distance(feature, reference_feature)
  39
  40
  41 def get_db_engine(config: configparser.ConfigParser) -> sqlalchemy.engine.Engine:
  42     host = config.get('mysql', 'host')
  43     dbname = config.get('mysql', 'dbname')
  44     user = config.get('mysql', 'user_name')
  45     passwd = config.get('mysql', 'user_pass')
  46
  47     return create_engine(f'mysql://{user}@{host}:3306/{dbname}?passwd={passwd}&charset=utf8mb4')
  48
  49
  50 class SimpleSledrunMap(NamedTuple):
  51     map_title: str
  52     map_json: GeoJSON
  53     map_bbox: shapely.geometry.Polygon
  54
  55
  56 def get_simplified_wrgeojson_from_db(connection: sqlalchemy.engine.Connection) -> Iterable[SimpleSledrunMap]:
  57     cursor = connection.execute("select page_title, map_json from wrsledrunjsoncache "
  58                                 "left join page on page_id = map_json_page_id "
  59                                 "where json_extract(map_json, '$.wr_properties.simplified') order by page_title")
  60     for row in cursor:
  61         map_title = row['page_title'].decode('utf-8')
  62         wrgeojson = geojson.loads(row['map_json'])
  63         map_bbox = get_geometry_envelope(wrgeojson)
  64         yield SimpleSledrunMap(map_title, wrgeojson, map_bbox)
  65
  66
  67 class OsmWrgeojson(NamedTuple):
  68     name: str
  69     bbox: shapely.geometry.Polygon
  70     wrgeojson: GeoJSON
  71
  72
  73 def get_osm_wrgeojson_from_xml(osm_tree: ElementTree) -> Iterable[OsmWrgeojson]:
  74     for osm_sledrun in find_sledrun_relations(osm_tree):
  75         name = tags(osm_sledrun).get('name')
  76         print(f'Calculate bbox of {name}')
  77         try:
  78             wrgeojson = convert_osm_to_geojson(osm_tree, osm_sledrun)
  79             join_wrgeojson_ways(wrgeojson)
  80             map_bbox = get_geometry_envelope(wrgeojson)
  81             yield OsmWrgeojson(name, map_bbox, wrgeojson)
  82         except DeRefError:
  83             print('Error: Incomplete XML - please load larger region.')
  84         except ValueError as e:
  85             print(e)
  86
  87
  88 def intersecting_wrgeojson(bbox: shapely.geometry.Polygon, candidates: Iterable[OsmWrgeojson]) \
  89         -> Iterable[OsmWrgeojson]:
  90     for osm_wrgeojson in candidates:
  91         if bbox.intersects(osm_wrgeojson.bbox):
  92             yield osm_wrgeojson
  93
  94
  95 def update_sledrun_wrgeojson_to_highres(simple_sledrun_map: SimpleSledrunMap, osm_sledrun_map: OsmWrgeojson,
  96                                         max_dist_m: float, site: WikiSite) -> bool:
  97     wrgeojson_page = site.query_page(simple_sledrun_map.map_title)
  98     wrgeojson_page_content = wrgeojson_page['revisions'][0]['slots']['main']['content']
  99     wr_wrgeojson = geojson.loads(wrgeojson_page_content)
 100
 101     osm_wrgeojson = osm_sledrun_map.wrgeojson
 102
 103     center = get_geometry_center(wr_wrgeojson)
 104     transformer = LanToToMetricTransformer(center.coordinates[1])
 105
 106     for way_type in WrLineStringFeatureType:
 107         wr_ways = [f for f in wr_wrgeojson['features'] if filter_way(f, way_type.value)]
 108         osm_ways = [f for f in osm_wrgeojson['features'] if filter_way(f, way_type.value)]
 109         if len(wr_ways) != len(osm_ways):
 110             cprint(f'{simple_sledrun_map.map_title} does not have the same number of {way_type.value} features', 'red')
 111             return False
 112
 113     for reference_feature in wr_wrgeojson['features']:
 114         reference_feature_type: str = reference_feature.get('properties', {}).get('type', '')
 115         if reference_feature_type not in [v.value for v in WrLineStringFeatureType]:
 116             continue
 117
 118         osm_features: List[Feature] = [f for f in osm_wrgeojson['features'] if filter_way(f, reference_feature_type)]
 119         osm_features = sorted(osm_features, key=lambda f: geojson_way_sort_key(f, reference_feature))
 120         osm_feature = osm_features[0]
 121         osm_feature_m = transformer.geojson_lon_lat_to_metric(osm_feature)
 122         reference_feature_m = transformer.geojson_lon_lat_to_metric(reference_feature)
 123         assert isinstance(osm_feature_m, Feature)
 124         assert isinstance(reference_feature_m, Feature)
 125
 126         dist_m = feature_distance(osm_feature_m, reference_feature_m)
 127         if dist_m > max_dist_m:
 128             cprint(f'Distance of ways "{reference_feature_type}" too big ({dist_m} m > {max_dist_m}).', 'red')
 129             return False
 130
 131         reference_feature['geometry'] = osm_feature['geometry']
 132
 133     wr_wrgeojson_old = json.loads(wrgeojson_page_content)
 134     if wr_wrgeojson_old == wr_wrgeojson:
 135         cprint('No changes detected', 'red')
 136         return False
 137
 138     wr_wrgeojson['wr_properties']['simplified'] = False
 139
 140     wr_wrgeojson_old_str = format_json(wr_wrgeojson_old)
 141     wr_wrgeojson_str = format_json(wr_wrgeojson)
 142
 143     unified_diff(wr_wrgeojson_old_str, wr_wrgeojson_str)
 144     choice = input_yes_no_quit('Do you accept the changes [yes, no, quit]? ', None)
 145     if choice == Choice.no:
 146         return False
 147     elif choice == Choice.quit:
 148         sys.exit(0)
 149
 150     site(
 151         'edit',
 152         pageid=wrgeojson_page['pageid'],
 153         text=wr_wrgeojson_str,
 154         summary='Höhere Auflösung der Wege erstellt.',
 155         minor=1,
 156         bot=1,
 157         baserevid=wrgeojson_page['revisions'][0]['revid'],
 158         nocreate=1,
 159         token=site.token(),
 160     )
 161     return True
 162
 163
 164 def update_wrgeojson_to_highres_batch(osm_file: str, ini_files: List[str]):
 165     config = configparser.ConfigParser()
 166     config.read(ini_files)
 167
 168     engine = get_db_engine(config)
 169     with engine.connect() as connection:
 170         simplified_wrgeojson_from_db = list(get_simplified_wrgeojson_from_db(connection))
 171
 172     osm_tree = ElementTree()
 173     osm_tree.parse(osm_file)
 174     osm_wrgeojson_list = list(get_osm_wrgeojson_from_xml(osm_tree))
 175
 176     site = WikiSite(ini_files)
 177
 178     max_dist = DEFAULT_MAX_DIST_ERROR_M * 1.5
 179     for simple_sledrun_map in simplified_wrgeojson_from_db:
 180         cprint(f'Processing {simple_sledrun_map.map_title}', 'green')
 181         candidates = list(intersecting_wrgeojson(simple_sledrun_map.map_bbox, osm_wrgeojson_list))
 182         if len(candidates) == 0:
 183             print('No candidates, skipping')
 184         else:
 185             print(f'{len(candidates)} candidate(s)')
 186             for candidate in sorted(candidates,
 187                                     key=lambda c: geojson_distance(c.wrgeojsonyes, simple_sledrun_map.map_json)):
 188                 if update_sledrun_wrgeojson_to_highres(simple_sledrun_map, candidate, max_dist, site):
 189                     break
 190
 191
 192 def main():
 193     parser = argparse.ArgumentParser(
 194         description='Updates sledrun wrgeojson maps with higher resolution ways from OSM files.')
 195     parser.add_argument('osm_file', help='OSM file containing many sledruns with high resolution ways.')
 196     parser.add_argument('ini_file', nargs='+',
 197                         help='inifile.ini, see: https://www.winterrodeln.org/trac/wiki/ConfigIni')
 198     args = parser.parse_args()
 199     update_wrgeojson_to_highres_batch(args.osm_file, args.ini_file)
 200
 201
 202 if __name__ == '__main__':
 203     main()