bots/sledrun_wikitext_to_json.py

   1 #!/usr/bin/python
   2 """
   3 User script for pywikibot (https://gerrit.wikimedia.org/r/pywikibot/core.git), tested with version 6.6.1.
   4 Put it in directory scripts/userscripts.
   5
   6 Create a sledrun JSON page from a sledrun wikitext page (including map).
   7
   8 The following generators and filters are supported:
   9
  10 &params;
  11 """
  12 import io
  13 import json
  14 import re
  15 from typing import Any, Optional
  16
  17 import mwparserfromhell
  18 import pywikibot
  19 from mwparserfromhell.nodes import Tag, Text, ExternalLink, Template
  20 from mwparserfromhell.wikicode import Wikicode
  21 from pywikibot import pagegenerators, Page
  22 from pywikibot.bot import (
  23     AutomaticTWSummaryBot,
  24     ConfigParserBot,
  25     ExistingPageBot,
  26     NoRedirectPageBot,
  27     SingleSiteBot,
  28 )
  29 from pywikibot.logging import warning
  30 from pywikibot.site._namespace import BuiltinNamespace
  31
  32 from wrpylib.wrmwmarkup import create_sledrun_wiki, lonlat_to_json, lonlat_ele_to_json, parse_wrmap
  33 from wrpylib.wrvalidators import rodelbahnbox_from_template, tristate_german_to_str, difficulty_german_to_str, \
  34     avalanches_german_to_str, public_transport_german_to_str, opt_str_opt_comment_enum_to_str, opt_lonlat_from_str, \
  35     opt_uint_from_str
  36
  37 from pywikibot.site import Namespace
  38
  39 docuReplacements = {'&params;': pagegenerators.parameterHelp}
  40
  41
  42 def str_or_none(value: Any) -> Optional[str]:
  43     if value is not None:
  44         return str(value)
  45     return None
  46
  47
  48 def template_to_json(value: Template) -> dict:
  49     parameter = []
  50     for p in value.params:
  51         parameter.append({'value': str(p)})
  52     return {
  53         'name': str(value.name),
  54         'parameter': parameter
  55     }
  56
  57
  58 class SledrunWikiTextToJsonBot(
  59     SingleSiteBot,
  60     ConfigParserBot,
  61     ExistingPageBot,
  62     NoRedirectPageBot,
  63     AutomaticTWSummaryBot,
  64 ):
  65     def treat_page(self) -> None:
  66         """Load the given page, do some changes, and save it."""
  67         wikitext_content_model = 'wikitext'
  68         if self.current_page.content_model != wikitext_content_model:
  69             warning(f"The content model of {self.current_page.title()} is {self.current_page.content_model} "
  70                     f"instead of {wikitext_content_model}.")
  71             return
  72
  73         wikicode = mwparserfromhell.parse(self.current_page.text)
  74         wikilink_list = wikicode.filter_wikilinks()
  75         category_sledrun = 'Kategorie:Rodelbahn'
  76         if sum(1 for c in wikilink_list if c.title == category_sledrun) == 0:
  77             warning(f'The page {self.current_page.title()} does not have category {category_sledrun}.')
  78             return
  79
  80         sledrun_json_page = Page(self.site, self.current_page.title() + '/Rodelbahn.json')
  81         if sledrun_json_page.exists():
  82             warning(f"{sledrun_json_page.title()} already exists, skipping {self.current_page.title()}.")
  83             return
  84
  85         map_json_page = Page(self.site, self.current_page.title() + '/Landkarte.json')
  86         if map_json_page.exists():
  87             warning(f"{map_json_page.title()} already exists, skipping {self.current_page.title()}.")
  88             return
  89
  90         map_json = None
  91         v = wikicode.filter_tags(matches='wrmap')
  92         if len(v) > 0:
  93             map_json = parse_wrmap(str(v[0]))
  94
  95         sledrun_json = {
  96             "name": self.current_page.title(),
  97             "aliases": [],
  98             "entry_under_construction": sum(1 for c in wikilink_list if c.text == 'Kategorie:In Arbeit') > 0,
  99         }
 100
 101         for v in wikicode.get_sections(levels=[2], matches='Allgemeines'):
 102             for w in v.ifilter_text(recursive=False):
 103                 x = w.strip()
 104                 if x:
 105                     sledrun_json["description"] = str(x)
 106                     break
 107             break
 108
 109         rbb_list = wikicode.filter_templates(recursive=False, matches=lambda t: t.name.strip() == 'Rodelbahnbox')
 110         if len(rbb_list) == 1:
 111             rbb = rodelbahnbox_from_template(rbb_list[0])
 112             v = rbb['Bild']
 113             if v is not None:
 114                 image_page = Page(self.site, v, ns=BuiltinNamespace.FILE)
 115                 if not image_page.exists():
 116                     warning(f"{image_page.title()} does not exist.")
 117                 sledrun_json['image'] = v
 118
 119             v = rbb['Länge']
 120             if v is not None:
 121                 sledrun_json['length'] = v
 122
 123             v = rbb['Schwierigkeit']
 124             if v is not None:
 125                 sledrun_json['difficulty'] = difficulty_german_to_str(v)
 126
 127             v = rbb['Lawinen']
 128             if v is not None:
 129                 sledrun_json['avalanches'] = avalanches_german_to_str(v)
 130
 131             v, w = rbb['Betreiber']
 132             if v is not None:
 133                 sledrun_json['has_operator'] = v
 134             if w is not None:
 135                 sledrun_json['operator'] = w
 136
 137             v = rbb['Aufstieg möglich']
 138             if v is not None:
 139                 sledrun_json['walkup_possible'] = v
 140
 141             v, w = rbb['Aufstieg getrennt']
 142             if v is not None:
 143                 sledrun_json['walkup_separate'] = tristate_german_to_str(v)
 144             if w is not None:
 145                 sledrun_json['walkup_comment'] = w  # TODO
 146
 147             v = rbb['Gehzeit']
 148             if v is not None:
 149                 sledrun_json['walkup_time'] = v
 150
 151             v, w = rbb['Beleuchtungsanlage']
 152             if v is not None:
 153                 sledrun_json['nightlight_possible'] = tristate_german_to_str(v)
 154             if w is not None:
 155                 sledrun_json['nightlight_description'] = w
 156
 157             v = rbb['Rodelverleih']
 158             if v is not None:
 159                 sledrun_json['sled_rental_direct'] = v != []
 160                 sledrun_json['sled_rental_description'] = opt_str_opt_comment_enum_to_str(v)
 161
 162             v = rbb['In Übersichtskarte']
 163             if v is not None:
 164                 sledrun_json['show_in_overview'] = v
 165
 166             v = rbb['Forumid']
 167             if v is not None:
 168                 sledrun_json['forum_id'] = v
 169
 170             v = rbb['Position']
 171             if v is not None:
 172                 sledrun_json['position'] = lonlat_to_json(v)
 173
 174             v = lonlat_ele_to_json(rbb['Position oben'], rbb['Höhe oben'])
 175             if v != {}:
 176                 sledrun_json['top'] = v
 177
 178             v = lonlat_ele_to_json(rbb['Position unten'], rbb['Höhe unten'])
 179             if v != {}:
 180                 sledrun_json['bottom'] = v
 181
 182             v = rbb['Telefonauskunft']
 183             if v is not None:
 184                 sledrun_json['info_phone'] = [{'phone': p, 'name': n} for p, n in v]
 185
 186             v = rbb['Öffentliche Anreise']
 187             if v is not None:
 188                 sledrun_json['public_transport'] = public_transport_german_to_str(v)
 189
 190             for v in wikicode.get_sections(levels=[2], matches='Anreise mit öffentlichen Verkehrsmitteln',
 191                                            include_headings=False):
 192                 w = next((w for w in v.nodes if isinstance(w, Tag) and w.wiki_markup == '*'), None)
 193                 if w is not None:
 194                     x = str(Wikicode(v.nodes[:v.nodes.index(w)])).strip()
 195                     if x:
 196                         sledrun_json["public_transport_description"] = str(x)
 197
 198                 public_transport_stops = []
 199                 public_transport_lines = []
 200                 ya = None
 201                 for w in v.nodes:
 202                     if isinstance(w, Template):
 203                         if w.name == 'Haltestelle':
 204                             if ya is not None:
 205                                 public_transport_stops.append(ya)
 206                             ya = {}
 207                             z = w.get(1, None)
 208                             if z is not None:
 209                                 ya['municipality'] = str(z)
 210                             z = w.get(2, None)
 211                             if z is not None:
 212                                 ya['name_local'] = str(z)
 213                             za = str_or_none(w.get(3, None))
 214                             zb = str_or_none(w.get(4, None))
 215                             z = lonlat_ele_to_json(opt_lonlat_from_str(za), opt_uint_from_str(zb))
 216                             if len(z) > 0:
 217                                 ya['position'] = z
 218                         elif w.name in ["Fahrplan Abfahrtsmonitor VVT"]:
 219                             ya['monitor_template'] = template_to_json(w)
 220                         elif w.name in ["Fahrplan Hinfahrt VVT"]:
 221                             ya['route_arrival_template'] = template_to_json(w)
 222                         elif w.name in ["Fahrplan Rückfahrt VVT"]:
 223                             ya['route_departure_template'] = template_to_json(w)
 224                         elif w.name in ["Fahrplan Linie VVT"]:
 225                             if ya is not None:
 226                                 public_transport_stops.append(ya)
 227                                 ya = None
 228                             y = {
 229                                 'timetable_template': template_to_json(w),
 230                             }
 231                             public_transport_lines.append(y)
 232                 if ya is not None:
 233                     public_transport_stops.append(ya)
 234                 if len(public_transport_stops) > 0:
 235                     sledrun_json['public_transport_stops'] = public_transport_stops
 236                 if len(public_transport_lines) > 0:
 237                     sledrun_json['public_transport_lines'] = public_transport_lines
 238                 break
 239
 240             for v in wikicode.get_sections(levels=[2], matches='Anreise mit dem Auto'):
 241                 for w in v.ifilter_text(recursive=False):
 242                     x = w.strip()
 243                     if x:
 244                         sledrun_json["car_description"] = str(x)
 245                         break
 246                 x = []
 247                 for w in v.ifilter_templates(matches='Parkplatz'):
 248                     za = str_or_none(w.get(1, None))
 249                     zb = str_or_none(w.get(2, None))
 250                     z = lonlat_ele_to_json(opt_lonlat_from_str(za), opt_uint_from_str(zb))
 251                     if len(z) > 0:
 252                         x.append({'position': z})
 253                 if len(x) > 0:
 254                     sledrun_json['car_parking'] = x
 255
 256                 x = []
 257                 for w in io.StringIO(str(v)):
 258                     match = re.match(r"\*\* von \'\'\'(.+)\'\'\'(.*): ([\d.,]+) km", w.rstrip())
 259                     if match:
 260                         ya, yb, yc = match.groups()
 261
 262                         yc = float(yc.replace(',', '.'))
 263                         x.append({
 264                             'km': yc,
 265                             'route': (ya.strip() + ' ' + yb.strip()).strip(),
 266                         })
 267                 if len(x) > 0:
 268                     sledrun_json['car_distances'] = x
 269
 270             x = []
 271             for v in wikicode.get_sections(levels=[2], matches='Allgemeines'):
 272                 i = iter(v.nodes)
 273                 w = next(i, None)
 274                 while w is not None:
 275                     if isinstance(w, Tag) and str(w) == "'''Siehe auch'''":
 276                         w = next(i, None)
 277                         break
 278                     w = next(i, None)
 279                 while w is not None:
 280                     if isinstance(w, ExternalLink):
 281                         link = {'url': w.url}
 282                         if w.title is not None:
 283                             link['text'] = w.title
 284                         x.append(link)
 285                     elif isinstance(w, (Text, Tag)) and str(w).strip() in ['', '*', ':']:
 286                         pass
 287                     else:
 288                         break
 289                     w = next(i, None)
 290             if len(x) > 0:
 291                 sledrun_json['see_also'] = x
 292
 293             sledrun_json['allow_reports'] = True
 294
 295         text = create_sledrun_wiki(sledrun_json, map_json)
 296         summary = 'Rodelbahnbeschreibung nach Konvertierung nach und von JSON.'
 297         self.put_current(text, summary=summary)
 298
 299
 300 def main(*args: str) -> None:
 301     local_args = pywikibot.handle_args(args)
 302     gen_factory = pagegenerators.GeneratorFactory()
 303     gen_factory.handle_args(local_args)
 304     gen = gen_factory.getCombinedGenerator(preload=True)
 305     if gen:
 306         bot = SledrunWikiTextToJsonBot(generator=gen)
 307         bot.run()
 308     else:
 309         pywikibot.bot.suggest_help(missing_generator=True)
 310
 311
 312 if __name__ == '__main__':
 313     main()