bots/sledrun_wikitext_to_json.py

   1 #!/usr/bin/python
   2 """
   3 User script for pywikibot (https://gerrit.wikimedia.org/r/pywikibot/core.git), tested with version 6.6.1.
   4 Put it in directory scripts/userscripts.
   5
   6 Create a sledrun JSON page from a sledrun wikitext page (including map).
   7
   8 The following generators and filters are supported:
   9
  10 &params;
  11 """
  12 import json
  13 from typing import Any, Optional
  14
  15 import mwparserfromhell
  16 import pywikibot
  17 from mwparserfromhell.nodes import Tag, Text, ExternalLink, Template
  18 from mwparserfromhell.wikicode import Wikicode
  19 from pywikibot import pagegenerators, Page
  20 from pywikibot.bot import (
  21     AutomaticTWSummaryBot,
  22     ConfigParserBot,
  23     ExistingPageBot,
  24     NoRedirectPageBot,
  25     SingleSiteBot,
  26 )
  27 from pywikibot.logging import warning
  28 from pywikibot.site._namespace import BuiltinNamespace
  29
  30 from wrpylib.wrmwmarkup import create_sledrun_wiki, lonlat_to_json, lonlat_ele_to_json, parse_wrmap
  31 from wrpylib.wrvalidators import rodelbahnbox_from_template, tristate_german_to_str, difficulty_german_to_str, \
  32     avalanches_german_to_str, public_transport_german_to_str, opt_str_opt_comment_enum_to_str, opt_lonlat_from_str, \
  33     opt_uint_from_str
  34
  35 from pywikibot.site import Namespace
  36
  37 docuReplacements = {'&params;': pagegenerators.parameterHelp}
  38
  39
  40 def str_or_none(value: Any) -> Optional[str]:
  41     if value is not None:
  42         return str(value)
  43     return None
  44
  45
  46 def template_to_json(value: Template) -> dict:
  47     parameter = []
  48     for p in value.params:
  49         parameter.append({'value': str(p)})
  50     return {
  51         'name': str(value.name),
  52         'parameter': parameter
  53     }
  54
  55
  56 class SledrunWikiTextToJsonBot(
  57     SingleSiteBot,
  58     ConfigParserBot,
  59     ExistingPageBot,
  60     NoRedirectPageBot,
  61     AutomaticTWSummaryBot,
  62 ):
  63     def treat_page(self) -> None:
  64         """Load the given page, do some changes, and save it."""
  65         wikitext_content_model = 'wikitext'
  66         if self.current_page.content_model != wikitext_content_model:
  67             warning(f"The content model of {self.current_page.title()} is {self.current_page.content_model} "
  68                     f"instead of {wikitext_content_model}.")
  69             return
  70
  71         wikicode = mwparserfromhell.parse(self.current_page.text)
  72         wikilink_list = wikicode.filter_wikilinks()
  73         category_sledrun = 'Kategorie:Rodelbahn'
  74         if sum(1 for c in wikilink_list if c.title == category_sledrun) == 0:
  75             warning(f'The page {self.current_page.title()} does not have category {category_sledrun}.')
  76             return
  77
  78         sledrun_json_page = Page(self.site, self.current_page.title() + '/Rodelbahn.json')
  79         if sledrun_json_page.exists():
  80             warning(f"{sledrun_json_page.title()} already exists, skipping {self.current_page.title()}.")
  81             return
  82
  83         map_json_page = Page(self.site, self.current_page.title() + '/Landkarte.json')
  84         if map_json_page.exists():
  85             warning(f"{map_json_page.title()} already exists, skipping {self.current_page.title()}.")
  86             return
  87
  88         map_json = None
  89         v = wikicode.filter_tags(matches='wrmap')
  90         if len(v) > 0:
  91             map_json = parse_wrmap(str(v[0]))
  92
  93         sledrun_json = {
  94             "name": self.current_page.title(),
  95             "aliases": [],
  96             "entry_under_construction": sum(1 for c in wikilink_list if c.text == 'Kategorie:In Arbeit') > 0,
  97         }
  98
  99         for v in wikicode.get_sections(levels=[2], matches='Allgemeines'):
 100             for w in v.ifilter_text(recursive=False):
 101                 x = w.strip()
 102                 if x:
 103                     sledrun_json["description"] = str(x)
 104                     break
 105             break
 106
 107         rbb_list = wikicode.filter_templates(recursive=False, matches=lambda t: t.name.strip() == 'Rodelbahnbox')
 108         if len(rbb_list) == 1:
 109             rbb = rodelbahnbox_from_template(rbb_list[0])
 110             v = rbb['Bild']
 111             if v is not None:
 112                 image_page = Page(self.site, v, ns=BuiltinNamespace.FILE)
 113                 if not image_page.exists():
 114                     warning(f"{image_page.title()} does not exist.")
 115                 sledrun_json['image'] = v
 116
 117             v = rbb['Länge']
 118             if v is not None:
 119                 sledrun_json['length'] = v
 120
 121             v = rbb['Schwierigkeit']
 122             if v is not None:
 123                 sledrun_json['difficulty'] = difficulty_german_to_str(v)
 124
 125             v = rbb['Lawinen']
 126             if v is not None:
 127                 sledrun_json['avalanches'] = avalanches_german_to_str(v)
 128
 129             v, w = rbb['Betreiber']
 130             if v is not None:
 131                 sledrun_json['has_operator'] = v
 132             if w is not None:
 133                 sledrun_json['operator'] = w
 134
 135             v = rbb['Aufstieg möglich']
 136             if v is not None:
 137                 sledrun_json['walkup_possible'] = v
 138
 139             v, w = rbb['Aufstieg getrennt']
 140             if v is not None:
 141                 sledrun_json['walkup_separate'] = tristate_german_to_str(v)
 142             if w is not None:
 143                 sledrun_json['walkup_comment'] = w  # TODO
 144
 145             v = rbb['Gehzeit']
 146             if v is not None:
 147                 sledrun_json['walkup_time'] = v
 148
 149             v, w = rbb['Beleuchtungsanlage']
 150             if v is not None:
 151                 sledrun_json['nightlight_possible'] = tristate_german_to_str(v)
 152             if w is not None:
 153                 sledrun_json['nightlight_description'] = w
 154
 155             v = rbb['Rodelverleih']
 156             if v is not None:
 157                 sledrun_json['sled_rental_direct'] = v != []
 158                 sledrun_json['sled_rental_description'] = opt_str_opt_comment_enum_to_str(v)
 159
 160             v = rbb['In Übersichtskarte']
 161             if v is not None:
 162                 sledrun_json['show_in_overview'] = v
 163
 164             v = rbb['Forumid']
 165             if v is not None:
 166                 sledrun_json['forum_id'] = v
 167
 168             v = rbb['Position']
 169             if v is not None:
 170                 sledrun_json['position'] = lonlat_to_json(v)
 171
 172             v = lonlat_ele_to_json(rbb['Position oben'], rbb['Höhe oben'])
 173             if v != {}:
 174                 sledrun_json['top'] = v
 175
 176             v = lonlat_ele_to_json(rbb['Position unten'], rbb['Höhe unten'])
 177             if v != {}:
 178                 sledrun_json['bottom'] = v
 179
 180             v = rbb['Telefonauskunft']
 181             if v is not None:
 182                 sledrun_json['info_phone'] = [{'phone': p, 'name': n} for p, n in v]
 183
 184             v = rbb['Öffentliche Anreise']
 185             if v is not None:
 186                 sledrun_json['public_transport'] = public_transport_german_to_str(v)
 187
 188             for v in wikicode.get_sections(levels=[2], matches='Anreise mit öffentlichen Verkehrsmitteln',
 189                                            include_headings=False):
 190                 w = next((w for w in v.nodes if isinstance(w, Tag) and w.wiki_markup == '*'), None)
 191                 if w is not None:
 192                     x = str(Wikicode(v.nodes[:v.nodes.index(w)])).strip()
 193                     if x:
 194                         sledrun_json["public_transport_description"] = str(x)
 195
 196                 public_transport_stops = []
 197                 public_transport_lines = []
 198                 ya = None
 199                 for w in v.nodes:
 200                     if isinstance(w, Template):
 201                         if w.name == 'Haltestelle':
 202                             if ya is not None:
 203                                 public_transport_stops.append(ya)
 204                             ya = {}
 205                             z = w.get(1, None)
 206                             if z is not None:
 207                                 ya['municipality'] = str(z)
 208                             z = w.get(2, None)
 209                             if z is not None:
 210                                 ya['name_local'] = str(z)
 211                             za = str_or_none(w.get(3, None))
 212                             zb = str_or_none(w.get(4, None))
 213                             z = lonlat_ele_to_json(opt_lonlat_from_str(za), opt_uint_from_str(zb))
 214                             if len(z) > 0:
 215                                 ya['position'] = z
 216                         elif w.name in ["Fahrplan Abfahrtsmonitor VVT"]:
 217                             ya['monitor_template'] = template_to_json(w)
 218                         elif w.name in ["Fahrplan Hinfahrt VVT"]:
 219                             ya['route_arrival_template'] = template_to_json(w)
 220                         elif w.name in ["Fahrplan Rückfahrt VVT"]:
 221                             ya['route_departure_template'] = template_to_json(w)
 222                         elif w.name in ["Fahrplan Linie VVT"]:
 223                             if ya is not None:
 224                                 public_transport_stops.append(ya)
 225                                 ya = None
 226                             y = {
 227                                 'timetable_template': template_to_json(w),
 228                             }
 229                             public_transport_lines.append(y)
 230                 if ya is not None:
 231                     public_transport_stops.append(ya)
 232                 if len(public_transport_stops) > 0:
 233                     sledrun_json['public_transport_stops'] = public_transport_stops
 234                 if len(public_transport_lines) > 0:
 235                     sledrun_json['public_transport_lines'] = public_transport_lines
 236                 break
 237
 238             for v in wikicode.get_sections(levels=[2], matches='Anreise mit dem Auto'):
 239                 for w in v.ifilter_text(recursive=False):
 240                     x = w.strip()
 241                     if x:
 242                         sledrun_json["car_description"] = str(x)
 243                         break
 244                 x = []
 245                 for w in v.ifilter_templates(matches='Parkplatz'):
 246                     za = str_or_none(w.get(1, None))
 247                     zb = str_or_none(w.get(2, None))
 248                     z = lonlat_ele_to_json(opt_lonlat_from_str(za), opt_uint_from_str(zb))
 249                     if len(z) > 0:
 250                         x.append({'position': z})
 251                 if len(x) > 0:
 252                     sledrun_json['car_parking'] = x
 253
 254             x = []
 255             for v in wikicode.get_sections(levels=[2], matches='Allgemeines'):
 256                 i = iter(v.nodes)
 257                 w = next(i, None)
 258                 while w is not None:
 259                     if isinstance(w, Tag) and str(w) == "'''Siehe auch'''":
 260                         w = next(i, None)
 261                         break
 262                     w = next(i, None)
 263                 while w is not None:
 264                     if isinstance(w, ExternalLink):
 265                         link = {'url': w.url}
 266                         if w.title is not None:
 267                             link['text'] = w.title
 268                         x.append(link)
 269                     elif isinstance(w, (Text, Tag)) and str(w).strip() in ['', '*', ':']:
 270                         pass
 271                     else:
 272                         break
 273                     w = next(i, None)
 274             if len(x) > 0:
 275                 sledrun_json['see_also'] = x
 276
 277             sledrun_json['allow_reports'] = True
 278
 279         text = create_sledrun_wiki(sledrun_json, map_json)
 280         summary = 'Rodelbahnbeschreibung nach Konvertierung nach und von JSON.'
 281         self.put_current(text, summary=summary)
 282
 283
 284 def main(*args: str) -> None:
 285     local_args = pywikibot.handle_args(args)
 286     gen_factory = pagegenerators.GeneratorFactory()
 287     gen_factory.handle_args(local_args)
 288     gen = gen_factory.getCombinedGenerator(preload=True)
 289     if gen:
 290         bot = SledrunWikiTextToJsonBot(generator=gen)
 291         bot.run()
 292     else:
 293         pywikibot.bot.suggest_help(missing_generator=True)
 294
 295
 296 if __name__ == '__main__':
 297     main()