From f97e54d4992d74ed24225bbe00c9b47750f5abbe Mon Sep 17 00:00:00 2001 From: John Livingston Date: Mon, 18 Sep 2023 18:53:07 +0200 Subject: [PATCH] Moderation Bot integration WIP: * Start and stop the bot WIP * Prosody: removing the BOSH module from the global scope (must only be present on relevant virtualhosts) * Some refactoring --- CHANGELOG.md | 3 +- client/common/configuration/register.ts | 2 +- languages/en.yml | 2 +- package-lock.json | 14 +- package.json | 2 +- server/lib/bots/ctl.ts | 182 ++++++++++++++++++ server/lib/configuration/bot.ts | 80 +++++++- .../configuration/configuration.ts | 4 +- server/lib/prosody/config.ts | 18 +- server/lib/prosody/config/content.ts | 18 +- server/lib/room-channel/room-channel-class.ts | 2 +- server/lib/settings.ts | 15 +- server/main.ts | 21 +- .../en/documentation/admin/settings.md | 2 +- .../content/en/technical/data/_index.md | 9 +- 15 files changed, 348 insertions(+), 26 deletions(-) create mode 100644 server/lib/bots/ctl.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index b6eb678a..36d129eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## XXX (Unreleased Yet) +## 8.0.0 (Unreleased Yet) ### New features @@ -13,6 +13,7 @@ * Links to documentation are now using the front-end language to point to the translated documentation page (except for some links generated from the backend, in the diagnostic tool for example). * Some code refactoring. * You can now configure on which network interfaces Prosody will listen for external components. +* Prosody: removing the BOSH module from the global scope (must only be present on relevant virtualhosts). ## 7.2.2 diff --git a/client/common/configuration/register.ts b/client/common/configuration/register.ts index b7fd2ed4..51573a9b 100644 --- a/client/common/configuration/register.ts +++ b/client/common/configuration/register.ts @@ -11,7 +11,7 @@ async function registerConfiguration (clientOptions: RegisterClientOptions): Pro const { peertubeHelpers, registerClientRoute, registerHook } = clientOptions const settings = await peertubeHelpers.getSettings() - if (settings['disable-configuration']) { return } + if (settings['disable-channel-configuration']) { return } registerClientRoute({ route: 'livechat/configuration', diff --git a/languages/en.yml b/languages/en.yml index db8dd7d5..8dc0cb62 100644 --- a/languages/en.yml +++ b/languages/en.yml @@ -302,7 +302,7 @@ configuration_description: | users will be able to add some customization on their channels, activate the moderation bot, ... -disable_configuration_label: "Disable the advanced channel configuration and the chatbot" +disable_channel_configuration_label: "Disable the advanced channel configuration and the chatbot" save: "Save" cancel: "Cancel" diff --git a/package-lock.json b/package-lock.json index b4473cb5..3aa58db4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,7 @@ "http-proxy": "^1.18.1", "log-rotate": "^0.2.8", "validate-color": "^2.2.1", - "xmppjs-chat-bot": "^0.2.1" + "xmppjs-chat-bot": "^0.2.2" }, "devDependencies": { "@peertube/feed": "^5.1.0", @@ -12069,9 +12069,9 @@ } }, "node_modules/xmppjs-chat-bot": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/xmppjs-chat-bot/-/xmppjs-chat-bot-0.2.1.tgz", - "integrity": "sha512-O2uiVBlgio/5psb+DhCYjBU2HfW0pnQwTuKjutBTLoaTtfG+A+oxFmwarZKFDUjiQK6DrytW+LL+M8792rBaxw==", + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/xmppjs-chat-bot/-/xmppjs-chat-bot-0.2.2.tgz", + "integrity": "sha512-/t1L2fSW04M/5zEGYQzXqgTa7CVaE1dAT9kO1C5iSOrd4HzksQ6vVsk0PlAbEZere08pbea8Kw8uxBSwGlTiXw==", "funding": [ "https://paypal.me/JohnXLivingston", "https://liberapay.com/JohnLivingston/" @@ -21291,9 +21291,9 @@ } }, "xmppjs-chat-bot": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/xmppjs-chat-bot/-/xmppjs-chat-bot-0.2.1.tgz", - "integrity": "sha512-O2uiVBlgio/5psb+DhCYjBU2HfW0pnQwTuKjutBTLoaTtfG+A+oxFmwarZKFDUjiQK6DrytW+LL+M8792rBaxw==", + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/xmppjs-chat-bot/-/xmppjs-chat-bot-0.2.2.tgz", + "integrity": "sha512-/t1L2fSW04M/5zEGYQzXqgTa7CVaE1dAT9kO1C5iSOrd4HzksQ6vVsk0PlAbEZere08pbea8Kw8uxBSwGlTiXw==", "requires": { "@xmpp/client": "^0.13.1", "@xmpp/component": "^0.13.1", diff --git a/package.json b/package.json index a6c1be78..8ca88e53 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "http-proxy": "^1.18.1", "log-rotate": "^0.2.8", "validate-color": "^2.2.1", - "xmppjs-chat-bot": "^0.2.1" + "xmppjs-chat-bot": "^0.2.2" }, "devDependencies": { "@peertube/feed": "^5.1.0", diff --git a/server/lib/bots/ctl.ts b/server/lib/bots/ctl.ts new file mode 100644 index 00000000..2fc69b3b --- /dev/null +++ b/server/lib/bots/ctl.ts @@ -0,0 +1,182 @@ +import type { RegisterServerOptions } from '@peertube/peertube-types' +import type { Config as XMPPChatBotConfig } from 'xmppjs-chat-bot' +import { BotConfiguration } from '../configuration/bot' +import * as child_process from 'child_process' + +let singleton: BotsCtl | undefined + +/** + * This class is to control the plugin bots. + * For now there is only one, the Moderation bot. + * But all public methods are made as if there was several bots, so it will be easier to add bots. + */ +class BotsCtl { + protected readonly options: RegisterServerOptions + protected readonly moderationGlobalConf: XMPPChatBotConfig + protected readonly logger: { + debug: (s: string) => void + info: (s: string) => void + warn: (s: string) => void + error: (s: string) => void + } + + protected moderationBotProcess: ReturnType | undefined + + constructor (params: { + options: RegisterServerOptions + moderationGlobalConf: XMPPChatBotConfig + }) { + this.options = params.options + this.moderationGlobalConf = params.moderationGlobalConf + + const logger = params.options.peertubeHelpers.logger + this.logger = { + debug: (s) => logger.debug('[Bots] ' + s), + info: (s) => logger.info('[Bots] ' + s), + warn: (s) => logger.warn('[Bots] ' + s), + error: (s) => logger.error('[Bots] ' + s) + } + } + + /** + * Starts all the required bots. + * If bots are already running, does nothing. + */ + public async start (): Promise { + if (await this.options.settingsManager.getSetting('disable-channel-configuration')) { + this.logger.info('Advanced channel configuration is disabled, no bot to start') + return + } + + this.logger.info('Starting moderation bot...') + + if (this.moderationBotProcess?.exitCode === null) { + this.logger.info('Moderation bot still running, nothing to do') + return + } + + const paths = BotConfiguration.singleton().configurationPaths() + + // We will run: npm exec -- xmppjs-chat-bot [...] + const execArgs = [ + 'exec', + '--', + 'xmppjs-chat-bot', + 'run', + '-f', + paths.moderation.globalFile, + '--room-conf-dir', + paths.moderation.roomConfDir + ] + const moderationBotProcess = child_process.spawn('npm', execArgs, { + cwd: __dirname, // must be in the livechat plugin tree, so that npm can found the package. + env: { + ...process.env // will include NODE_ENV and co + } + }) + moderationBotProcess.stdout?.on('data', (data) => { + this.logger.debug(`ModerationBot stdout: ${data as string}`) + }) + moderationBotProcess.stderr?.on('data', (data) => { + this.logger.error(`ModerationBot stderr: ${data as string}`) + }) + moderationBotProcess.on('error', (error) => { + this.logger.error(`ModerationBot exec error: ${JSON.stringify(error)}`) + }) + moderationBotProcess.on('exit', (code) => { + this.logger.info(`ModerationBot process exited with code ${code ?? 'null'}`) + }) + moderationBotProcess.on('close', (code) => { + this.logger.info(`ModerationBot process closed all stdio with code ${code ?? 'null'}`) + }) + + this.moderationBotProcess = moderationBotProcess + } + + /** + * Stops all the bots + */ + public async stop (): Promise { + this.logger.info('Stopping bots...') + + if (!this.moderationBotProcess) { + this.logger.info('moderationBot was never running, everything is fine.') + return + } + if (this.moderationBotProcess.exitCode !== null) { + this.logger.info('The moderation bot has an exitCode, already stopped.') + return + } + const p = new Promise((resolve, reject) => { + try { + if (!this.moderationBotProcess) { resolve() } + const moderationBotProcess: ReturnType = + this.moderationBotProcess as ReturnType + + let resolved = false + // Trying to kill, and force kill if it takes more than 2 seconds + const timeout = setTimeout(() => { + this.logger.error('Moderation bot was not killed within 2 seconds, force killing') + moderationBotProcess.kill('SIGKILL') + resolved = true + resolve() + }, 2000) + + moderationBotProcess.on('exit', () => { + if (resolved) { return } + resolved = true + if (timeout) { clearTimeout(timeout) } + resolve() + }) + moderationBotProcess.on('close', () => { + if (resolved) { return } + resolved = true + if (timeout) { clearTimeout(timeout) } + resolve() + }) + moderationBotProcess.kill() + } catch (err) { + this.logger.error(err as string) + reject(err) + } + }) + + return p + } + + /** + * Instanciate a new singleton + * @param options server options + */ + public static async initSingleton (options: RegisterServerOptions): Promise { + // forceRefresh the bot global configuration file: + const moderationGlobalConf = await BotConfiguration.singleton().getModerationBotGlobalConf(true) + + singleton = new BotsCtl({ + options, + moderationGlobalConf + }) + return singleton + } + + /** + * Returns the singleton, of thrown an exception if it is not initialized yet. + * @returns the singleton + */ + public static singleton (): BotsCtl { + if (!singleton) { + throw new Error('Bots singleton not initialized yet') + } + return singleton + } + + public static async destroySingleton (): Promise { + if (!singleton) { return } + await singleton.stop() + singleton = undefined + } +} + +export { + BotsCtl +} diff --git a/server/lib/configuration/bot.ts b/server/lib/configuration/bot.ts index 9be8dcde..ede246d8 100644 --- a/server/lib/configuration/bot.ts +++ b/server/lib/configuration/bot.ts @@ -1,6 +1,7 @@ import type { RegisterServerOptions } from '@peertube/peertube-types' -import type { RoomConf } from 'xmppjs-chat-bot' +import type { RoomConf, Config } from 'xmppjs-chat-bot' import { getProsodyDomain } from '../prosody/config/domain' +import { isDebugMode } from '../debug' import * as path from 'path' import * as fs from 'fs' @@ -18,8 +19,10 @@ type ChannelCommonRoomConf = Omit class BotConfiguration { protected readonly options: RegisterServerOptions protected readonly mucDomain: string + protected readonly botsDomain: string protected readonly confDir: string protected readonly roomConfDir: string + protected readonly moderationBotGlobalConf: string protected readonly logger: { debug: (s: string) => void info: (s: string) => void @@ -32,13 +35,17 @@ class BotConfiguration { constructor (params: { options: RegisterServerOptions mucDomain: string + botsDomain: string confDir: string roomConfDir: string + moderationBotGlobalConf: string }) { this.options = params.options this.mucDomain = params.mucDomain + this.botsDomain = params.botsDomain this.confDir = params.confDir this.roomConfDir = params.roomConfDir + this.moderationBotGlobalConf = params.moderationBotGlobalConf const logger = params.options.peertubeHelpers.logger this.logger = { @@ -55,15 +62,21 @@ class BotConfiguration { public static async initSingleton (options: RegisterServerOptions): Promise { const prosodyDomain = await getProsodyDomain(options) const mucDomain = 'room.' + prosodyDomain + const botsDomain = 'bot.' + prosodyDomain const confDir = path.resolve( options.peertubeHelpers.plugin.getDataDirectoryPath(), 'bot', mucDomain ) + const roomConfDir = path.resolve( confDir, 'rooms' ) + const moderationBotGlobalConf = path.resolve( + confDir, + 'moderation.json' + ) await fs.promises.mkdir(confDir, { recursive: true }) await fs.promises.mkdir(roomConfDir, { recursive: true }) @@ -71,8 +84,10 @@ class BotConfiguration { singleton = new BotConfiguration({ options, mucDomain, + botsDomain, confDir, - roomConfDir + roomConfDir, + moderationBotGlobalConf }) return singleton @@ -93,7 +108,7 @@ class BotConfiguration { * @param roomJIDParam Room full or local JID * @param conf Configuration to write */ - public async update (roomJIDParam: string, conf: ChannelCommonRoomConf): Promise { + public async updateRoom (roomJIDParam: string, conf: ChannelCommonRoomConf): Promise { const roomJID = this._canonicJID(roomJIDParam) if (!roomJID) { this.logger.error('Invalid room JID') @@ -141,6 +156,65 @@ class BotConfiguration { await this._writeRoomConf(roomJID) } + /** + * Returns the moderation bot global configuration. + * It it does not exists, creates it. + * @param forceRefresh if true, regenerates the configuration file, even if exists. + */ + public async getModerationBotGlobalConf (forceRefresh?: boolean): Promise { + let config: Config | undefined + if (!forceRefresh) { + try { + const content = (await fs.promises.readFile(this.moderationBotGlobalConf, { + encoding: 'utf-8' + })).toString() + + config = JSON.parse(content) + } catch (err) { + this.logger.info('Error reading the moderation bot global configuration file, assuming it does not exists.') + config = undefined + } + } + + if (!config) { + // FIXME: use existing lib to get the port, dont hardcode default value here. + const portSetting = await this.options.settingsManager.getSetting('prosody-port') + const port = (portSetting as string) || '52800' + + config = { + type: 'client', + connection: { + username: 'moderator', + password: Math.random().toString(36).slice(2, 12), + domain: this.botsDomain, + // Note: using localhost, and not currentProsody.host, because it does not always resolve correctly + service: 'xmpp://localhost:' + port + }, + name: 'Moderator', + logger: 'ConsoleLogger', + log_level: isDebugMode(this.options) ? 'debug' : 'info' + } + await fs.promises.writeFile(this.moderationBotGlobalConf, JSON.stringify(config), { + encoding: 'utf-8' + }) + } + return config + } + + public configurationPaths (): { + moderation: { + globalFile: string + roomConfDir: string + } + } { + return { + moderation: { + globalFile: this.moderationBotGlobalConf, + roomConfDir: this.roomConfDir + } + } + } + /** * frees the singleton */ diff --git a/server/lib/middlewares/configuration/configuration.ts b/server/lib/middlewares/configuration/configuration.ts index 96cc18af..45924cf1 100644 --- a/server/lib/middlewares/configuration/configuration.ts +++ b/server/lib/middlewares/configuration/configuration.ts @@ -10,9 +10,9 @@ import type { RequestPromiseHandler } from '../async' function checkConfigurationEnabledMiddleware (options: RegisterServerOptions): RequestPromiseHandler { return async (req: Request, res: Response, next: NextFunction) => { const settings = await options.settingsManager.getSettings([ - 'disable-configuration' + 'disable-channel-configuration' ]) - if (!settings['disable-configuration']) { + if (!settings['disable-channel-configuration']) { next() return } diff --git a/server/lib/prosody/config.ts b/server/lib/prosody/config.ts index b04f7451..040f8b31 100644 --- a/server/lib/prosody/config.ts +++ b/server/lib/prosody/config.ts @@ -1,4 +1,5 @@ import type { RegisterServerOptions } from '@peertube/peertube-types' +import type { Config as XMPPBotConfig } from 'xmppjs-chat-bot' import type { ProsodyLogLevel } from './config/content' import * as fs from 'fs' import * as path from 'path' @@ -9,6 +10,7 @@ import { getProsodyDomain } from './config/domain' import { getAPIKey } from '../apikey' import { parseExternalComponents } from './config/components' import { getRemoteServerInfosDir } from '../federation/storage' +import { BotConfiguration } from '../configuration/bot' async function getWorkingDir (options: RegisterServerOptions): Promise { const peertubeHelpers = options.peertubeHelpers @@ -125,6 +127,9 @@ interface ProsodyConfig { logExpiration: ConfigLogExpiration valuesToHideInDiagnostic: Map certificates: ProsodyConfigCertificates + bots: { + moderation?: XMPPBotConfig + } } async function getProsodyConfig (options: RegisterServerOptionsV5): Promise { const logger = options.peertubeHelpers.logger @@ -147,7 +152,8 @@ async function getProsodyConfig (options: RegisterServerOptionsV5): Promise() @@ -168,6 +174,7 @@ async function getProsodyConfig (options: RegisterServerOptionsV5): Promise { @@ -94,9 +95,9 @@ Please read descriptionHTML: loc('experimental_warning') }) registerSetting({ - name: 'disable-configuration', - label: loc('disable_configuration_label'), - // descriptionHTML: loc('disable_configuration_description'), + name: 'disable-channel-configuration', + label: loc('disable_channel_configuration_label'), + // descriptionHTML: loc('disable_channel_configuration_description'), type: 'input-checkbox', default: false, private: false @@ -401,8 +402,16 @@ Please read // ********** settings changes management settingsManager.onSettingsChange(async (settings: any) => { + // In case the Prosody port has changed, we must rewrite the Bot configuration file. + // To avoid race condition, we will just stop and start the bots at every settings saving. + await BotsCtl.destroySingleton() + await BotsCtl.initSingleton(options) + peertubeHelpers.logger.info('Saving settings, ensuring prosody is running') await ensureProsodyRunning(options) + + await BotsCtl.singleton().start() + // In case prosody-room-type changed, we must rebuild room-channel links. if (settings['prosody-room-type'] !== currentProsodyRoomtype) { peertubeHelpers.logger.info('Setting prosody-room-type has changed value, must rebuild room-channel infos') diff --git a/server/main.ts b/server/main.ts index f1b87b7b..5df544c7 100644 --- a/server/main.ts +++ b/server/main.ts @@ -11,6 +11,7 @@ import { unloadDebugMode } from './lib/debug' import { loadLoc } from './lib/loc' import { RoomChannel } from './lib/room-channel' import { BotConfiguration } from './lib/configuration/bot' +import { BotsCtl } from './lib/bots/ctl' import decache from 'decache' // FIXME: Peertube unregister don't have any parameter. @@ -35,6 +36,9 @@ async function register (options: RegisterServerOptions): Promise { // roomChannelNeedsDataInit: if true, means that the data file does not exist (or is invalid), so we must initiate it const roomChannelNeedsDataInit = !await roomChannelSingleton.readData() + // BotsCtl.initSingleton() will force reload the bots conf files, so must be done before generating Prosody Conf. + await BotsCtl.initSingleton(options) + await migrateSettings(options) await initSettings(options) @@ -48,20 +52,35 @@ async function register (options: RegisterServerOptions): Promise { await prepareProsody(options) await ensureProsodyRunning(options) + let preBotPromise: Promise if (roomChannelNeedsDataInit) { logger.info('The RoomChannel singleton has not found any data, we must rebuild') // no need to wait here, can be done without await. - roomChannelSingleton.rebuildData().then( + preBotPromise = roomChannelSingleton.rebuildData().then( () => { logger.info('RoomChannel singleton rebuild done') }, (reason) => { logger.error('RoomChannel singleton rebuild failed: ' + (reason as string)) } ) + } else { + preBotPromise = Promise.resolve() } + + // Don't need to wait for the bot to start. + preBotPromise.then( + async () => { + await BotsCtl.singleton().start() + }, + () => {} + ) } catch (error) { options.peertubeHelpers.logger.error('Error when launching Prosody: ' + (error as string)) } } async function unregister (): Promise { + try { + await BotsCtl.destroySingleton() + } catch (_error) {} // BotsCtl will log errors. + if (OPTIONS) { try { await ensureProsodyNotRunning(OPTIONS) diff --git a/support/documentation/content/en/documentation/admin/settings.md b/support/documentation/content/en/documentation/admin/settings.md index 1b09f5d4..2672d3ed 100644 --- a/support/documentation/content/en/documentation/admin/settings.md +++ b/support/documentation/content/en/documentation/admin/settings.md @@ -29,7 +29,7 @@ Following settings concern the federation with other Peertube instances, and oth Following settings concern the advanced channel options: users will be able to add some customization on their channels, activate the moderation bot, ... -### {{% livechat_label disable_configuration_label %}} +### {{% livechat_label disable_channel_configuration_label %}} If you encounter any issue with this feature, you can disable it. diff --git a/support/documentation/content/en/technical/data/_index.md b/support/documentation/content/en/technical/data/_index.md index b640d5e8..07b81474 100644 --- a/support/documentation/content/en/technical/data/_index.md +++ b/support/documentation/content/en/technical/data/_index.md @@ -93,9 +93,16 @@ There will be some cleaning batch, to delete deprecated files. The `bot/muc_domain` (where muc_domain is the current MUC domain) folder contains configuration files that are read by the moderation bot. This bot uses the [xmppjs-chat-bot](https://github.com/JohnXLivingston/xmppjs-chat-bot) package. -Note: we include the MUC domain (`room.instance.tld`) in the filename in case the instance domain changes. +Note: we include the MUC domain (`room.instance.tld`) in the dirname in case the instance domain changes. In such case, existing rooms could get lost, and we want a way to ignore them to avoid gettings errors. +## bot/muc_domain/moderation.json + +The `bot/muc_domain/moderation.json` file contains the moderation bot global configuration. +This bot uses the [xmppjs-chat-bot](https://github.com/JohnXLivingston/xmppjs-chat-bot) package, see it's README file for more information. + +Note: this includes the bot username and password. Don't let it leak. + ### bot/muc_domain/rooms The `bot/muc_domain/rooms` folder contains room configuration files.