bots/sledrun_wikitext_to_json.py

   1 #!/usr/bin/python
   2 """
   3 User script for pywikibot (https://gerrit.wikimedia.org/r/pywikibot/core.git), tested with version 6.6.1.
   4 Put it in directory scripts/userscripts.
   5
   6 Create a sledrun JSON page from a sledrun wikitext page (including map).
   7
   8 The following generators and filters are supported:
   9
  10 &params;
  11 """
  12 import io
  13 import json
  14 import re
  15 from typing import Any, Optional
  16
  17 import mwparserfromhell
  18 import pywikibot
  19 from mwparserfromhell.nodes import Tag, Text, ExternalLink, Template, Wikilink
  20 from mwparserfromhell.wikicode import Wikicode
  21 from pywikibot import pagegenerators, Page
  22 from pywikibot.bot import (
  23     AutomaticTWSummaryBot,
  24     ConfigParserBot,
  25     ExistingPageBot,
  26     NoRedirectPageBot,
  27     SingleSiteBot,
  28 )
  29 from pywikibot.logging import warning
  30 from pywikibot.site._namespace import BuiltinNamespace
  31
  32 from wrpylib.wrmwmarkup import create_sledrun_wiki, lonlat_to_json, lonlat_ele_to_json, parse_wrmap
  33 from wrpylib.wrvalidators import rodelbahnbox_from_template, tristate_german_to_str, difficulty_german_to_str, \
  34     avalanches_german_to_str, public_transport_german_to_str, opt_str_opt_comment_enum_to_str, opt_lonlat_from_str, \
  35     opt_uint_from_str
  36
  37 from pywikibot.site import Namespace
  38
  39 docuReplacements = {'&params;': pagegenerators.parameterHelp}
  40
  41
  42 def str_or_none(value: Any) -> Optional[str]:
  43     if value is not None:
  44         return str(value)
  45     return None
  46
  47
  48 def template_to_json(value: Template) -> dict:
  49     parameter = []
  50     for p in value.params:
  51         parameter.append({'value': str(p)})
  52     return {
  53         'name': str(value.name),
  54         'parameter': parameter
  55     }
  56
  57
  58 class SledrunWikiTextToJsonBot(
  59     SingleSiteBot,
  60     ConfigParserBot,
  61     ExistingPageBot,
  62     NoRedirectPageBot,
  63     AutomaticTWSummaryBot,
  64 ):
  65     def treat_page(self) -> None:
  66         """Load the given page, do some changes, and save it."""
  67         wikitext_content_model = 'wikitext'
  68         if self.current_page.content_model != wikitext_content_model:
  69             warning(f"The content model of {self.current_page.title()} is {self.current_page.content_model} "
  70                     f"instead of {wikitext_content_model}.")
  71             return
  72
  73         wikicode = mwparserfromhell.parse(self.current_page.text)
  74         wikilink_list = wikicode.filter_wikilinks()
  75         category_sledrun = 'Kategorie:Rodelbahn'
  76         if sum(1 for c in wikilink_list if c.title == category_sledrun) == 0:
  77             warning(f'The page {self.current_page.title()} does not have category {category_sledrun}.')
  78             return
  79
  80         sledrun_json_page = Page(self.site, self.current_page.title() + '/Rodelbahn.json')
  81         if sledrun_json_page.exists():
  82             warning(f"{sledrun_json_page.title()} already exists, skipping {self.current_page.title()}.")
  83             return
  84
  85         map_json_page = Page(self.site, self.current_page.title() + '/Landkarte.json')
  86         if map_json_page.exists():
  87             warning(f"{map_json_page.title()} already exists, skipping {self.current_page.title()}.")
  88             return
  89
  90         map_json = None
  91         v = wikicode.filter_tags(matches='wrmap')
  92         if len(v) > 0:
  93             map_json = parse_wrmap(str(v[0]))
  94
  95         sledrun_json = {
  96             "name": self.current_page.title(),
  97             "aliases": [],
  98             "entry_under_construction": sum(1 for c in wikilink_list if c.text == 'Kategorie:In Arbeit') > 0,
  99         }
 100
 101         for v in wikicode.get_sections(levels=[2], matches='Allgemeines'):
 102             for w in v.ifilter_text(recursive=False):
 103                 x = w.strip()
 104                 if x:
 105                     sledrun_json["description"] = str(x)
 106                     break
 107             break
 108
 109         rbb_list = wikicode.filter_templates(recursive=False, matches=lambda t: t.name.strip() == 'Rodelbahnbox')
 110         if len(rbb_list) == 1:
 111             rbb = rodelbahnbox_from_template(rbb_list[0])
 112             v = rbb['Bild']
 113             if v is not None:
 114                 image_page = Page(self.site, v, ns=BuiltinNamespace.FILE)
 115                 if not image_page.exists():
 116                     warning(f"{image_page.title()} does not exist.")
 117                 sledrun_json['image'] = v
 118
 119             v = rbb['Länge']
 120             if v is not None:
 121                 sledrun_json['length'] = v
 122
 123             v = rbb['Schwierigkeit']
 124             if v is not None:
 125                 sledrun_json['difficulty'] = difficulty_german_to_str(v)
 126
 127             v = rbb['Lawinen']
 128             if v is not None:
 129                 sledrun_json['avalanches'] = avalanches_german_to_str(v)
 130
 131             v, w = rbb['Betreiber']
 132             if v is not None:
 133                 sledrun_json['has_operator'] = v
 134             if w is not None:
 135                 sledrun_json['operator'] = w
 136
 137             v = rbb['Aufstieg möglich']
 138             if v is not None:
 139                 sledrun_json['walkup_possible'] = v
 140
 141             v, w = rbb['Aufstieg getrennt']
 142             if v is not None:
 143                 sledrun_json['walkup_separate'] = tristate_german_to_str(v)
 144             if w is not None:
 145                 sledrun_json['walkup_comment'] = w  # TODO
 146
 147             v = rbb['Gehzeit']
 148             if v is not None:
 149                 sledrun_json['walkup_time'] = v
 150
 151             v, w = rbb['Beleuchtungsanlage']
 152             if v is not None:
 153                 sledrun_json['nightlight_possible'] = tristate_german_to_str(v)
 154             if w is not None:
 155                 sledrun_json['nightlight_description'] = w
 156
 157             v = rbb['Rodelverleih']
 158             if v is not None:
 159                 sledrun_json['sled_rental_direct'] = v != []
 160                 sledrun_json['sled_rental_description'] = opt_str_opt_comment_enum_to_str(v)
 161
 162             v = rbb['In Übersichtskarte']
 163             if v is not None:
 164                 sledrun_json['show_in_overview'] = v
 165
 166             v = rbb['Forumid']
 167             if v is not None:
 168                 sledrun_json['forum_id'] = v
 169
 170             v = rbb['Position']
 171             if v is not None:
 172                 sledrun_json['position'] = lonlat_to_json(v)
 173
 174             v = lonlat_ele_to_json(rbb['Position oben'], rbb['Höhe oben'])
 175             if v != {}:
 176                 sledrun_json['top'] = v
 177
 178             v = lonlat_ele_to_json(rbb['Position unten'], rbb['Höhe unten'])
 179             if v != {}:
 180                 sledrun_json['bottom'] = v
 181
 182             v = rbb['Telefonauskunft']
 183             if v is not None:
 184                 sledrun_json['info_phone'] = [{'phone': p, 'name': n} for p, n in v]
 185
 186             v = rbb['Öffentliche Anreise']
 187             if v is not None:
 188                 sledrun_json['public_transport'] = public_transport_german_to_str(v)
 189
 190         for v in wikicode.get_sections(levels=[2], matches='Anreise mit öffentlichen Verkehrsmitteln',
 191                                        include_headings=False):
 192             w = next((w for w in v.nodes if isinstance(w, Tag) and w.wiki_markup == '*'), None)
 193             if w is not None:
 194                 x = str(Wikicode(v.nodes[:v.nodes.index(w)])).strip()
 195                 if x:
 196                     sledrun_json["public_transport_description"] = str(x)
 197
 198             public_transport_stops = []
 199             public_transport_lines = []
 200             ya = None
 201             for w in v.nodes:
 202                 if isinstance(w, Template):
 203                     if w.name == 'Haltestelle':
 204                         if ya is not None:
 205                             public_transport_stops.append(ya)
 206                         ya = {}
 207                         z = w.get(1, None)
 208                         if z is not None:
 209                             ya['municipality'] = str(z)
 210                         z = w.get(2, None)
 211                         if z is not None:
 212                             ya['name_local'] = str(z)
 213                         za = str_or_none(w.get(3, None))
 214                         zb = str_or_none(w.get(4, None))
 215                         z = lonlat_ele_to_json(opt_lonlat_from_str(za), opt_uint_from_str(zb))
 216                         if len(z) > 0:
 217                             ya['position'] = z
 218                     elif w.name in ["Fahrplan Abfahrtsmonitor VVT"]:
 219                         ya['monitor_template'] = template_to_json(w)
 220                     elif w.name in ["Fahrplan Hinfahrt VVT"]:
 221                         ya['route_arrival_template'] = template_to_json(w)
 222                     elif w.name in ["Fahrplan Rückfahrt VVT"]:
 223                         ya['route_departure_template'] = template_to_json(w)
 224                     elif w.name in ["Fahrplan Linie VVT"]:
 225                         if ya is not None:
 226                             public_transport_stops.append(ya)
 227                             ya = None
 228                         y = {
 229                             'timetable_template': template_to_json(w),
 230                         }
 231                         public_transport_lines.append(y)
 232             if ya is not None:
 233                 public_transport_stops.append(ya)
 234             if len(public_transport_stops) > 0:
 235                 sledrun_json['public_transport_stops'] = public_transport_stops
 236             if len(public_transport_lines) > 0:
 237                 sledrun_json['public_transport_lines'] = public_transport_lines
 238             break
 239
 240         for v in wikicode.get_sections(levels=[2], matches='Anreise mit dem Auto'):
 241             for w in v.ifilter_text(recursive=False):
 242                 x = w.strip()
 243                 if x:
 244                     sledrun_json["car_description"] = str(x)
 245                     break
 246             x = []
 247             for w in v.ifilter_templates(matches='Parkplatz'):
 248                 za = str_or_none(w.get(1, None))
 249                 zb = str_or_none(w.get(2, None))
 250                 z = lonlat_ele_to_json(opt_lonlat_from_str(za), opt_uint_from_str(zb))
 251                 if len(z) > 0:
 252                     x.append({'position': z})
 253             if len(x) > 0:
 254                 sledrun_json['car_parking'] = x
 255
 256             x = []
 257             for w in io.StringIO(str(v)):
 258                 match = re.match(r"\*\* von \'\'\'(.+)\'\'\'(.*): ([\d.,]+) km", w.rstrip())
 259                 if match:
 260                     ya, yb, yc = match.groups()
 261                     yc = float(yc.replace(',', '.'))
 262                     x.append({
 263                         'km': yc,
 264                         'route': (ya.strip() + ' ' + yb.strip()).strip(),
 265                     })
 266             if len(x) > 0:
 267                 sledrun_json['car_distances'] = x
 268
 269             x = []
 270             for v in wikicode.get_sections(levels=[2], matches='Allgemeines'):
 271                 def _gastronomy(value: str):
 272                     gastronomy = []
 273                     line_iter = io.StringIO(value)
 274                     line = next(line_iter, None)
 275                     while line is not None and line.rstrip() != "* '''Hütten''':":
 276                         line = next(line_iter, None)
 277                     if line is None:
 278                         return gastronomy
 279                     while line is not None:
 280                         line = next(line_iter, None)
 281                         if line is not None:
 282                             if line.startswith('** '):
 283                                 g = {}
 284                                 wiki = mwparserfromhell.parse(line)
 285                                 wiki_link = next(wiki.ifilter_wikilinks(), None)
 286                                 if isinstance(wiki_link, Wikilink):
 287                                     wl = {
 288                                         'title': str(wiki_link.title),
 289                                     }
 290                                     text = str_or_none(wiki_link.text)
 291                                     if text is not None:
 292                                         wl['text'] = text
 293                                     g['wr_page'] = wl
 294                                 ext_link = next(wiki.ifilter_external_links(), None)
 295                                 if isinstance(ext_link, ExternalLink):
 296                                     el = {
 297                                         'url': str(ext_link.url),
 298                                         'text': str(ext_link.title)
 299                                     }
 300                                     g['weblink'] = el
 301                                 remaining = str(Wikicode(n for n in wiki.nodes
 302                                                          if isinstance(n, (Text, Tag)) and str(n).strip() is not '*')).\
 303                                                 strip()
 304                                 match = re.match(r'\((.+)\)', remaining)
 305                                 if match:
 306                                     remaining = match.group(1)
 307                                 if len(remaining) > 0:
 308                                     g['note'] = remaining
 309                                 gastronomy.append(g)
 310                             else:
 311                                 break
 312                     return gastronomy
 313                 w = _gastronomy(str(v))
 314                 if len(w) > 0:
 315                     sledrun_json['gastronomy'] = w
 316
 317                 i = iter(v.nodes)
 318                 w = next(i, None)
 319                 while w is not None:
 320                     if isinstance(w, Tag) and str(w) == "'''Siehe auch'''":
 321                         w = next(i, None)
 322                         break
 323                     w = next(i, None)
 324                 while w is not None:
 325                     if isinstance(w, ExternalLink):
 326                         link = {'url': w.url}
 327                         if w.title is not None:
 328                             link['text'] = w.title
 329                         x.append(link)
 330                     elif isinstance(w, (Text, Tag)) and str(w).strip() in ['', '*', ':']:
 331                         pass
 332                     else:
 333                         break
 334                     w = next(i, None)
 335             if len(x) > 0:
 336                 sledrun_json['see_also'] = x
 337
 338             sledrun_json['allow_reports'] = True
 339
 340         text = create_sledrun_wiki(sledrun_json, map_json)
 341         summary = 'Rodelbahnbeschreibung nach Konvertierung nach und von JSON.'
 342         self.put_current(text, summary=summary)
 343
 344
 345 def main(*args: str) -> None:
 346     local_args = pywikibot.handle_args(args)
 347     gen_factory = pagegenerators.GeneratorFactory()
 348     gen_factory.handle_args(local_args)
 349     gen = gen_factory.getCombinedGenerator(preload=True)
 350     if gen:
 351         bot = SledrunWikiTextToJsonBot(generator=gen)
 352         bot.run()
 353     else:
 354         pywikibot.bot.suggest_help(missing_generator=True)
 355
 356
 357 if __name__ == '__main__':
 358     main()