Fix #436: Saving emojis per batch, to avoid hitting max payload limit.

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 10.1.1 (Not released yet)
+
+* #436: Saving emojis per batch, to avoid hitting max payload limit.
+
 ## 10.1.0
 
 ### New features

client/common/configuration/elements/channel-emojis.ts

@@ -102,9 +102,13 @@ export class ChannelEmojisElement extends LivechatElement {
 
     try {
       this.actionDisabled = true
-      await this._channelDetailsService.saveEmojisConfiguration(this.channelId, this.channelEmojisConfiguration.emojis)
+      this.channelEmojisConfiguration = await this._channelDetailsService.saveEmojisConfiguration(
+        this.channelId,
+        this.channelEmojisConfiguration.emojis
+      )
       this.validationError = undefined
       this.ptNotifier.info(await this.ptTranslate(LOC_SUCCESSFULLY_SAVED))
+      this.requestUpdate('channelEmojisConfiguration')
       this.requestUpdate('_validationError')
     } catch (error) {
       this.validationError = undefined

client/common/configuration/services/channel-details.ts

@@ -5,10 +5,12 @@
 
 import type { RegisterClientOptions } from '@peertube/peertube-types/client'
 import type {
-  ChannelLiveChatInfos, ChannelConfiguration, ChannelConfigurationOptions, ChannelEmojisConfiguration, ChannelEmojis
+  ChannelLiveChatInfos, ChannelConfiguration, ChannelConfigurationOptions, ChannelEmojisConfiguration, ChannelEmojis,
+  CustomEmojiDefinition
 } from 'shared/lib/types'
 import { ValidationError, ValidationErrorType } from '../../lib/models/validation'
 import { getBaseRoute } from '../../../utils/uri'
+import { maxEmojisPerChannel } from 'shared/lib/emojis'
 
 export class ChannelDetailsService {
   public _registerClientOptions: RegisterClientOptions
@@ -213,11 +215,57 @@ export class ChannelDetailsService {
   public async saveEmojisConfiguration (
     channelId: number,
     channelEmojis: ChannelEmojis
-  ): Promise<void> {
+  ): Promise<ChannelEmojisConfiguration> {
     if (!await this.validateEmojisConfiguration(channelEmojis)) {
       throw new Error('Invalid form data')
     }
 
+    // Note: API request body size is limited to 100Kb (expressjs body-parser defaut limit, and Peertube nginx config).
+    // So we must send new emojis 1 by 1, to be sure to not reach the limit.
+    if (!channelEmojis.customEmojis.find(e => e.url.startsWith('data:'))) {
+      // No new emojis, just saving.
+      return this._saveEmojisConfiguration(channelId, channelEmojis)
+    }
+
+    let lastResult: ChannelEmojisConfiguration | undefined
+    let customEmojis: CustomEmojiDefinition[] = [...channelEmojis.customEmojis] // copy the original array
+    let i = customEmojis.findIndex(e => e.url.startsWith('data:'))
+    let watchDog = 0
+    while (i >= 0) {
+      watchDog++
+      if (watchDog > maxEmojisPerChannel + 10) { // just to avoid infinite loop
+        throw new Error('Seems we have sent too many emojis, this was not expected')
+      }
+      const data: CustomEmojiDefinition[] = customEmojis.slice(0, i + 1) // all elements until first new file
+      data.push(
+        // all remaining elements that where already uploaded (to not loose them):
+        ...customEmojis.slice(i + 1).filter((e) => !e.url.startsWith('data:'))
+      )
+      lastResult = await this._saveEmojisConfiguration(channelId, {
+        customEmojis: data
+      })
+
+      // Must inject the result in customEmojis
+      const temp = lastResult.emojis.customEmojis.slice(0, i + 1) // last element should have been replace by a http url
+      temp.push(
+        ...customEmojis.slice(i + 1) // remaining elements in the previous array
+      )
+      customEmojis = temp
+
+      // and searching again next new emojis
+      i = customEmojis.findIndex(e => e.url.startsWith('data:'))
+    }
+    if (!lastResult) {
+      // This should not happen...
+      throw new Error('Unexpected: no last result')
+    }
+    return lastResult
+  }
+
+  private async _saveEmojisConfiguration (
+    channelId: number,
+    channelEmojis: ChannelEmojis
+  ): Promise<ChannelEmojisConfiguration> {
     const response = await fetch(
       getBaseRoute(this._registerClientOptions) +
         '/api/configuration/channel/emojis/' +
@@ -230,10 +278,9 @@ export class ChannelDetailsService {
     )
 
     if (!response.ok) {
-      if (response.status === 404) {
-        // File does not exist yet, that is a normal use case.
-      }
       throw new Error('Can\'t get channel emojis options.')
     }
+
+    return response.json()
   }
 }

server/lib/routers/api/configuration.ts

@@ -168,7 +168,16 @@ async function initConfigurationApiRouter (options: RegisterServerOptions, route
 
         await emojis.saveChannelDefinition(channelInfos.id, emojisDefinitionSanitized, bufferInfos)
 
-        res.sendStatus(200)
+        // Reloading data, to send them back to front:
+        const channelEmojis =
+          (await emojis.channelCustomEmojisDefinition(channelInfos.id)) ??
+          emojis.emptyChannelDefinition()
+        const result: ChannelEmojisConfiguration = {
+          channel: channelInfos,
+          emojis: channelEmojis
+        }
+        res.status(200)
+        res.json(result)
       } catch (err) {
         logger.error(err)
         res.sendStatus(500)

shared/lib/emojis.ts

@@ -2,6 +2,9 @@
 //
 // SPDX-License-Identifier: AGPL-3.0-only
 
+// Note: API request body size is limited to 100Kb (expressjs body-parser defaut limit, and Peertube nginx config).
+// So we must be sure to never send more than 100Kb. The front end sends new emojis by batch, but maxSize must remain
+// as little as possible, so that we never reach 100Kb in JSON/base64 format.
 export const maxSize: number = 30 * 1024
 export const allowedExtensions = ['png', 'jpg', 'jpeg', 'gif']
 export const inputFileAccept = ['image/jpg', 'image/png', 'image/gif']