dotfiles/scripts/mediawiki-preview

#!/usr/bin/env python3

# This script is based on <https://github.com/m-pilia/vim-mediawiki/blob/266d7ab9c7d4d7924e16f527c109c65851da0164/scripts/preview.py>

# Useful resources:
# <https://www.mediawiki.org/wiki/API:Parsing_wikitext#parse>
# <https://www.mediawiki.org/wiki/API:Styling_content>
# <https://www.mediawiki.org/wiki/ResourceLoader/Developing_with_ResourceLoader>
# <https://github.com/wikimedia/mediawiki>
# <https://github.com/wikimedia/Vector>
# and, most importantly: `view-source:` for any MediaWiki page

# Skin-specific notes:

#   citizen
#
#     - Fonts are not loaded due to CSP. One possible mitigation is running a
#       local proxy and setting `Access-Control-Allow-Origin` to `*`. An example
#       nginx configuration:
#
#           server {
#               listen 80;
#               server_name 127.155.44.48;  # This address was chosen randomly
#               location / {
#                   proxy_pass https://wiki.c2dl.info;  # Replace with your wiki
#                   add_header Access-Control-Allow-Origin *;
#               }
#           }
#
#       By modifying `/etc/hosts` this can be made into its own "domain":
#
#           127.155.44.48  local.wiki.c2dl.info
#
#     - The module `skins.citizen.scripts.toc` is broken because to find and
#       highlight the current section header in the table of contents it uses a
#       CSS selector incompatible with the URL rewriting applied to anchor links.
#
#     - The module `skins.citizen.scripts` references search inputs which aren't
#       created by this script.


import argparse
import mwclient
import json
from urllib.parse import urlencode
import html
import re


LANG = "en"
LANG_TEXT_DIRECTION = "ltr"

MODULES_POST_LOAD = {
    "vector": [
        "site",
        "mediawiki.page.startup",
        "mediawiki.page.ready",
        "mediawiki.toc",
        # "mediawiki.searchSuggest",
        # "mediawiki.page.watch.ajax",
        "skins.vector.js",
    ],
    "citizen": [
        # "site",
        # "mediawiki.page.startup",
        # "mediawiki.page.ready",
        # "mediawiki.toc",
        # "skins.citizen.scripts.toc",
        # "skins.citizen.scripts.search",
        # "skins.citizen.styles.search",
        # "skins.citizen.icons.search",
        # "skins.citizen.scripts",
    ],
}

MODULES_POST_LOAD_BLOCKED = {
    "citizen": [
        "skins.citizen.scripts.toc",
        "skins.citizen.scripts.search",
        "skins.citizen.styles.search",
        "skins.citizen.icons.search",
    ],
}

MODULES_PRELOAD_STYLES = {
    "vector": [
        "mediawiki.legacy.commonPrint",
        "mediawiki.legacy.shared",
        "mediawiki.skinning.interface",
        "mediawiki.toc.styles",
        "skins.vector.styles",
        "site.styles",
    ],
    "citizen": [
        # "mediawiki.legacy.commonPrint",
        # "mediawiki.legacy.shared",
        "mediawiki.skinning.content.externallinks",
        # "mediawiki.toc.styles",
        "skins.citizen.icons",
        "skins.citizen.styles",
        "skins.citizen.icons.ca",
        "skins.citizen.icons.es",
        "skins.citizen.icons.footer",
        "skins.citizen.icons.n",
        "skins.citizen.icons.pt",
        "skins.citizen.icons.t",
        "skins.citizen.styles.fonts",
        "skins.citizen.styles.toc",
        "site.styles",
    ],
}

MODULES_PRELOAD_SCRIPTS = {
    "vector": ["startup"],
    "citizen": ["startup"],
}


# ported from <https://github.com/wikimedia/mediawiki/blob/c15ded31a6ca79fa65c00d151a7220632ad90b6d/includes/parser/Sanitizer.php#L1205-L1222>
def escape_css_class(class_str):
    class_str = re.sub(
        r"""(^[0-9\-])|[\x00-\x20!"#$%&'()*+,.\/:;<=>?@[\]^`{|}~]|\xA0""",
        "_",
        class_str,
    )
    class_str = re.sub(r"_+", "_", class_str)
    class_str = class_str.rstrip("_")
    return class_str


def json_dumps_compact(data):
    return json.dumps(data, indent=None, separators=(",", ":"))


parser = argparse.ArgumentParser()
parser.add_argument("--site", type=str, required=True)
parser.add_argument("--scheme", type=str, default="https")
parser.add_argument("--skin", type=str, default="vector")
parser.add_argument(
    "--input", type=str, required=True,
)
parser.add_argument("--title", type=str)
parser.add_argument("--output", type=str, required=True)
cli_args = parser.parse_args()


site = mwclient.Site(cli_args.site, scheme=cli_args.scheme)


def get_load_script_url(**args):
    return "{path}load{ext}?{args}".format(
        path=site.path,
        ext=site.ext,
        args=urlencode({"lang": LANG, "skin": cli_args.skin, **args}),
    )


with open(cli_args.input, "r") as f:
    wikitext_str = f.read()

result = site.post(
    "parse",
    title=cli_args.title,
    text=wikitext_str,
    contentmodel="wikitext",
    prop="text|indicators|displaytitle|modules|jsconfigvars|categorieshtml",
    preview=True,
    pst=True,  # pre-save transforms
    sectionpreview=False,
    disableeditsection=True,  # disables "[edit]" links next to headers
    useskin=cli_args.skin,
    uselang=LANG,
)["parse"]


def get_modules(page_modules, added_modules_dict, blocked_modules_dict={}):
    modules = page_modules + added_modules_dict[cli_args.skin]
    for blocked_module in blocked_modules_dict.get(cli_args.skin, []):
        try:
            modules.remove(blocked_module)
        except ValueError:
            pass
    return modules


rendered_html = """\
<!DOCTYPE html>
<html class="client-nojs" lang="{lang}" dir="{text_dir}">

<head>
    <meta charset="UTF-8"/>
    <base href="{base_url}"/>
    <script>document.documentElement.className="client-js";RLSTATE={page_modules_state_json};RLCONF={page_config_json};RLPAGEMODULES={page_modules_json};</script>
    <script>(RLQ=window.RLQ||[]).push(function(){{mw.loader.implement("user.tokens",function(){{mw.user.tokens.set({{"editToken":"+\\\\","patrolToken":"+\\\\","watchToken":"+\\\\","csrfToken":"+\\\\"}});}});}});</script>
    <link rel="stylesheet" href="{style_url}"/>
    <script async="" src="{script_url}"></script>
</head>

<body class="mediawiki {text_dir} sitedir-{text_dir} mw-hide-empty-elt page-{page_class} skin-{skin} action-view">
    <div id="mw-page-base" class="noprint"></div>
    <div id="mw-head-base" class="noprint"></div>

    <div id="content" class="mw-body" role="main">
        <a id="top"></a>

        <div class="mw-indicators mw-body-content">
{indicators_html}
        </div>

        <h1 id="firstHeading" class="firstHeading" lang="{lang}">{title}</h1>

        <div id="bodyContent" class="mw-body-content">
            <div id="contentSub"></div>
            <div id="jump-to-nav"></div>

            <div id="mw-content-text" lang="{lang}" dir="{text_dir}" class="mw-content-{text_dir}">
{content_html}
            </div>

{categories_html}
            <div class="visualClear"></div>
        </div>
    </div>

    <div id="footer" role="contentinfo"></div>

    <script>(function(){{
        var anchorHrefPrefix = window.location.href.replace(/#.+$/, "")

        var links = document.getElementsByTagName("a")
        for (var i = 0, len = links.length; i < len; i++) {{
            var link = links[i]
            var href = link.getAttribute("href")
            if (typeof href === "string" && href[0] === "#") {{
                link.setAttribute("href", anchorHrefPrefix + href)
            }}
        }}
    }}())</script>
</body>

</html>
""".format(
    lang=html.escape(LANG),
    text_dir=html.escape(LANG_TEXT_DIRECTION),
    base_url=html.escape("{}://{}".format(site.scheme, site.host)),
    page_modules_state_json=json_dumps_compact(
        {
            "noscript": "ready",
            "user.options": "ready",
            "user.tokens": "loading",
            **{name: "ready" for name in MODULES_PRELOAD_STYLES[cli_args.skin]},
        }
    ),
    page_config_json=json_dumps_compact(result["jsconfigvars"]),
    page_modules_json=json_dumps_compact(
        get_modules(result["modules"], MODULES_POST_LOAD, MODULES_POST_LOAD_BLOCKED)
    ),
    style_url=html.escape(
        get_load_script_url(
            only="styles",
            modules="|".join(
                get_modules(result["modulestyles"], MODULES_PRELOAD_STYLES)
            ),
        )
    ),
    script_url=html.escape(
        get_load_script_url(
            only="scripts",
            modules="|".join(
                get_modules(result["modulescripts"], MODULES_PRELOAD_SCRIPTS)
            ),
            raw="1",
        )
    ),
    skin=html.escape(cli_args.skin),
    page_class=html.escape(escape_css_class(result["displaytitle"])),
    title=html.escape(result["displaytitle"]),
    indicators_html="\n".join(
        [
            '<div id="mw-indicator-{}" class="mw-indicator">{}</div>'.format(
                indicator["name"], indicator["*"]
            )
            for indicator in result["indicators"]
        ]
    ),
    content_html=result["text"]["*"],
    categories_html=result["categorieshtml"]["*"],
)


with open(cli_args.output, "w") as f:
    f.write(rendered_html)