Chat federation: storing chat information

server/lib/federation/incoming.ts

@@ -1,19 +1,24 @@
 import type { RegisterServerOptions } from '@peertube/peertube-types'
 import type { RemoteVideoHandlerParams } from './types'
+import { storeVideoLiveChatInfos } from './storage'
+import { sanitizePeertubeLiveChatInfos } from './sanitize'
 
+/**
+ * This function reads incoming ActivityPub data, to detect LiveChat informations.
+ * @param options server options
+ * @param param1 handler parameters
+ * @returns void
+ */
 async function readIncomingAPVideo (
   options: RegisterServerOptions,
   { video, videoAPObject }: RemoteVideoHandlerParams
 ): Promise<void> {
-  if (!('peertubeLiveChat' in videoAPObject)) {
-    return
-  }
-  const logger = options.peertubeHelpers.logger
-  // TODO: save the information.
-  logger.debug(
-    `Remote video uuid=${video.uuid} has a peertubeLiveChat attribute: ` +
-    JSON.stringify(videoAPObject.peertubeLiveChat)
-  )
+  let peertubeLiveChat = ('peertubeLiveChat' in videoAPObject) ? videoAPObject.peertubeLiveChat : false
+
+  // We must sanitize peertubeLiveChat, as it comes for the outer world.
+  peertubeLiveChat = sanitizePeertubeLiveChatInfos(peertubeLiveChat)
+
+  await storeVideoLiveChatInfos(options, video, peertubeLiveChat)
 }
 
 export {

server/lib/federation/outgoing.ts

@@ -1,10 +1,18 @@
 import type { RegisterServerOptions, VideoObject } from '@peertube/peertube-types'
-import type { LiveChatVideoObject, VideoBuildResultContext, LiveChatJSONLDLink } from './types'
+import type { LiveChatVideoObject, VideoBuildResultContext, LiveChatJSONLDLink, LiveChatJSONLDAttribute } from './types'
+import { storeVideoLiveChatInfos } from './storage'
 import { videoHasWebchat } from '../../../shared/lib/video'
 import { getBoshUri, getWSUri } from '../uri/webchat'
 import { canonicalizePluginUri } from '../uri/canonicalize'
 import { getProsodyDomain } from '../prosody/config/domain'
 
+/**
+ * This function adds LiveChat information on video ActivityPub data if relevant.
+ * @param options server options
+ * @param jsonld JSON-LD video data to fill
+ * @param context handler context
+ * @returns void
+ */
 async function videoBuildJSONLD (
   options: RegisterServerOptions,
   jsonld: VideoObject,
@@ -36,8 +44,12 @@ async function videoBuildJSONLD (
   }, video)
 
   if (!hasChat) {
-    // logger.debug(`Video uuid=${video.uuid} has not livechat, adding peertubeLiveChat=false.`)
-    (jsonld as LiveChatVideoObject).peertubeLiveChat = false
+    logger.debug(`Video uuid=${video.uuid} has not livechat, adding peertubeLiveChat=false.`)
+    // Note: we store also outgoing data. Could help for migration/cleanup scripts, for example.
+    await storeVideoLiveChatInfos(options, video, false)
+    Object.assign(jsonld, {
+      peertubeLiveChat: false
+    })
     return jsonld
   }
 
@@ -71,11 +83,16 @@ async function videoBuildJSONLD (
     }
   }
 
-  (jsonld as LiveChatVideoObject).peertubeLiveChat = {
+  const peertubeLiveChat: LiveChatJSONLDAttribute = {
     type: 'xmpp',
     jid: roomJID,
     links
   }
+  Object.assign(jsonld, {
+    peertubeLiveChat
+  })
+  // Note: we store also outgoing data. Could help for migration/cleanup scripts, for example.
+  await storeVideoLiveChatInfos(options, video, peertubeLiveChat)
   return jsonld
 }
 

server/lib/federation/sanitize.ts

@@ -0,0 +1,81 @@
+import type { LiveChatJSONLDInfos, LiveChatJSONLDAttribute } from './types'
+import { URL } from 'url'
+
+function sanitizePeertubeLiveChatInfos (chatInfos: any): LiveChatJSONLDAttribute {
+  if (chatInfos === false) { return false }
+  if (typeof chatInfos !== 'object') { return false }
+
+  if (chatInfos.type !== 'xmpp') { return false }
+  if ((typeof chatInfos.jid) !== 'string') { return false }
+  if (!Array.isArray(chatInfos.links)) { return false }
+
+  const r: LiveChatJSONLDInfos = {
+    type: chatInfos.type,
+    jid: chatInfos.jid,
+    links: []
+  }
+
+  for (const link of chatInfos.links) {
+    if ((typeof link) !== 'object') { continue }
+    if (['xmpp-bosh-anonymous', 'xmpp-websocket-anonymous'].includes(link.type)) {
+      if ((typeof link.jid) !== 'string') { continue }
+      if ((typeof link.url) !== 'string') { continue }
+
+      if (
+        !_validUrl(link.url, {
+          noSearchParams: true,
+          protocol: link.type === 'xmpp-websocket-anonymous' ? 'ws.' : 'http.'
+        })
+      ) {
+        continue
+      }
+
+      r.links.push({
+        type: link.type,
+        jid: link.jid,
+        url: link.url
+      })
+    }
+  }
+  return r
+}
+
+interface URLConstraints {
+  protocol: 'http.' | 'ws.'
+  noSearchParams: boolean
+}
+
+function _validUrl (s: string, constraints: URLConstraints): boolean {
+  if ((typeof s) !== 'string') { return false }
+  if (s === '') { return false }
+  let url: URL
+  try {
+    url = new URL(s)
+  } catch (_err) {
+    return false
+  }
+
+  if (constraints.protocol) {
+    if (constraints.protocol === 'http.') {
+      if (url.protocol !== 'https' && url.protocol !== 'http') {
+        return false
+      }
+    } else if (constraints.protocol === 'ws.') {
+      if (url.protocol !== 'wss' && url.protocol !== 'ws') {
+        return false
+      }
+    }
+  }
+
+  if (constraints.noSearchParams) {
+    if (url.search !== '') {
+      return false
+    }
+  }
+
+  return true
+}
+
+export {
+  sanitizePeertubeLiveChatInfos
+}

server/lib/federation/storage.ts

@@ -0,0 +1,128 @@
+import type { RegisterServerOptions, MVideoFullLight, MVideoAP } from '@peertube/peertube-types'
+import type { LiveChatJSONLDAttribute } from './types'
+import { URL } from 'url'
+import * as fs from 'fs'
+import * as path from 'path'
+
+/*
+Important Note: we could store these data in database. For example by using storageManager.storeData.
+But I'm afraid there might be write concurrency issues, or performance issues.
+Indeed, Peertube storageManager stores everything in one JSON attribute on the plugin tuple.
+
+Moreover, remote instance can create many data, that will not be cleaned.
+
+So, instead of using storageManager, or using a custom DB table, we will store data in files.
+If a file exists, it means the video has a chat.
+The file itself contains the JSON LiveChatInfos object.
+*/
+
+/**
+ * This function stores remote LiveChat infos that are contained in ActivityPub objects.
+ * We store these data for remotes videos.
+ *
+ * But we also store the data for local videos, so we can know what data we send to other Peertube instances.
+ * This is not really used for now, but can help if we want, one day, to know which videos we must refresh on remote
+ * instances.
+ * @param options server options
+ * @param video video object
+ * @param liveChatInfos video ActivityPub data to read
+ * @returns void
+ */
+async function storeVideoLiveChatInfos (
+  options: RegisterServerOptions,
+  video: MVideoFullLight | MVideoAP,
+  liveChatInfos: LiveChatJSONLDAttribute
+): Promise<void> {
+  const logger = options.peertubeHelpers.logger
+
+  const remote = video.remote
+  const filePath = await _getFilePath(options, remote, video.uuid, video.url)
+  if (!filePath) {
+    logger.error('Cant compute the file path for storing liveChat infos for video ' + video.uuid)
+    return
+  }
+
+  logger.debug(`Video ${video.uuid} data should be stored in ${filePath}`)
+
+  if (!liveChatInfos) {
+    logger.debug(`${remote ? 'Remote' : 'Local'} video ${video.uuid} has no chat infos, removing if necessary`)
+    await _del(options, filePath)
+    return
+  }
+
+  logger.debug(`${remote ? 'Remote' : 'Local'} video ${video.uuid} has caht infos to store`)
+  await _store(options, filePath, liveChatInfos)
+}
+
+async function _getFilePath (
+  options: RegisterServerOptions,
+  remote: boolean,
+  uuid: string,
+  videoUrl: string
+): Promise<string | null> {
+  const logger = options.peertubeHelpers.logger
+  try {
+    // some sanitization, just in case...
+    if (!/^(\w|-)+$/.test(uuid)) {
+      logger.error(`Video uuid seems not correct: ${uuid}`)
+      return null
+    }
+
+    let subFolders: string[]
+    if (remote) {
+      const u = new URL(videoUrl)
+      const host = u.hostname
+
+      if (host.includes('..')) {
+        // to prevent exploits that could go outside the plugin data dir.
+        logger.error(`Video host seems not correct, contains ..: ${host}`)
+        return null
+      }
+      subFolders = ['remote', host]
+    } else {
+      subFolders = ['local']
+    }
+
+    return path.resolve(
+      options.peertubeHelpers.plugin.getDataDirectoryPath(),
+      'videoInfos',
+      ...subFolders,
+      uuid + '.json'
+    )
+  } catch (err) {
+    logger.error(err)
+    return null
+  }
+}
+
+async function _del (options: RegisterServerOptions, filePath: string): Promise<void> {
+  const logger = options.peertubeHelpers.logger
+  try {
+    if (!fs.existsSync(filePath)) { return }
+    logger.info('Deleting file ' + filePath)
+    fs.rmSync(filePath)
+  } catch (err) {
+    logger.error(err)
+  }
+}
+
+async function _store (options: RegisterServerOptions, filePath: string, content: any): Promise<void> {
+  const logger = options.peertubeHelpers.logger
+  try {
+    if (!fs.existsSync(filePath)) {
+      const dir = path.dirname(filePath)
+      if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true })
+      }
+    }
+    await fs.promises.writeFile(filePath, JSON.stringify(content), {
+      encoding: 'utf-8'
+    })
+  } catch (err) {
+    logger.error(err)
+  }
+}
+
+export {
+  storeVideoLiveChatInfos
+}