peertube-plugin-livechat/support/forbidden_words/import-wikimedia.mjs

69 lines
2.2 KiB
JavaScript
Raw Normal View History

2024-05-23 09:42:14 +00:00
// SPDX-FileCopyrightText: 2024 John Livingston <https://www.john-livingston.fr/>
2023-09-25 15:18:42 +00:00
//
// SPDX-License-Identifier: AGPL-3.0-only
2023-09-25 15:12:10 +00:00
import { Command } from 'commander'
const program = new Command()
program
.usage('[command] [options]')
.showHelpAfterError()
const runCommand = program.command('category')
runCommand.description('Loads data from wikimedia services (wiktionary, ...), and prints it on the standard output.')
runCommand.requiredOption('-c, --category <category>', 'The category to request.')
runCommand.option(
'-s, --service <service>',
'The service to query. Possible values: "wiktionary", "wikipedia". Default: "wiktionary".',
'wiktionary'
)
runCommand.option(
'-l, --lang <lang>',
'The code lang for the requested service. For example "en", "fr", ... (as it appears in the wikimedia site url). Default: "en"',
'en'
)
runCommand.action(async (options) => {
const service = options.service ?? 'wiktionary'
if (service !== 'wiktionary' && service !== 'wikipedia') {
throw new Error('Invalid service ' + service)
}
const lang = options.lang
if (!/^[a-z-]+$/.test(lang)) {
throw new Error('Invalid lang ' + lang)
}
const category = options.category
const url = new URL(`https://${lang}.${service}.org/w/api.php?action=query&format=json&formatversion=2&list=categorymembers&cmlimit=200`)
url.searchParams.append('cmtitle', 'Category:' + category)
while (true) {
const r = await fetch(url.toString())
if (!r.ok) {
throw new Error('Failed requesting ' + url)
}
const json = await r.json()
if (!json) {
throw new Error('Invalid JSON content')
}
const list = json.query.categorymembers
for (const entry of list) {
// Note: at the end, there might be some links to other categories.
// These links have title like "Category:xxx", or "Catégorie:xxx".
// We don't want to test every translation of "Category"...
// So we just ignore titles with ":"
if (entry.title.includes(':')) { continue }
console.log(entry.title)
}
if (json.continue) {
for (const k in json.continue) {
url.searchParams.set(k, json.continue[k])
}
continue
}
break
}
})
program.parse(process.argv)