More image formats

config/default.yaml

@@ -79,6 +79,15 @@ stickers:
   # Whether or not to allow people to add custom sticker packs
   enabled: true
 
+  # Whether to resize images that are not 512x512 during import. This might slow down the import of animated Stickers.
+  resize: true
+
+  # Whether to allow animated stickers. If set to false animations will be stripped from images.
+  allowAnimated: true
+
+  # Whether to skip over images that are not 512x512. Ignored when resize is true.
+  verifyImageSize: true
+
   # The sticker manager bot to promote
   stickerBot: "@stickers:t2bot.io"
 

package.json

@@ -71,7 +71,7 @@
     "semver": "^7.3.5",
     "sequelize": "^6.12.0-alpha.1",
     "sequelize-typescript": "^2.1.1",
-    "sharp": "^0.29.0",
+    "sharp": "^0.30.7",
     "split-host": "^0.1.1",
     "spotify-uri": "^2.2.0",
     "sqlite3": "^5.0.2",

src/config.ts

@@ -36,6 +36,9 @@ export interface DimensionConfig {
     };
     stickers: {
         enabled: boolean;
+        resize: boolean;
+        allowAnimated: boolean;
+        verifyImageSize: boolean;
         stickerBot: string;
         managerUrl: string;
     };

src/matrix/MatrixLiteClient.ts

@@ -36,6 +36,15 @@ export class MatrixLiteClient {
         return baseUrl + `/_matrix/media/r0/thumbnail/${serverName}/${contentId}?width=${width}&height=${height}&method=${method}&animated=${isAnimated}`;
     }
 
+    public async getMediaUrl(serverName: string, contentId: string): Promise<string> {
+        let baseUrl = config.homeserver.mediaUrl;
+        if (!baseUrl) baseUrl = config.homeserver.clientServerUrl;
+        if (baseUrl.endsWith("/")) baseUrl = baseUrl.substring(0, baseUrl.length - 1);
+
+        // DO NOT RETURN THE ACCESS TOKEN.
+        return baseUrl + `/_matrix/media/r0/download/${serverName}/${contentId}`;
+    }
+
     public async whoAmI(): Promise<string> {
         const response = await doClientApiCall(
             "GET",
@@ -106,6 +115,7 @@ export class MatrixLiteClient {
     }
 
     public async upload(content: Buffer, contentType: string): Promise<string> {
+        LogService.info("MatrixLiteClient", "Uploading file (type:" + contentType + ")");
         return doClientApiCall(
             "POST",
             "/_matrix/media/r0/upload",
@@ -126,6 +136,7 @@ export class MatrixLiteClient {
                 method: "GET",
                 url: url,
                 encoding: null,
+                headers: {},
             }, (err, res, _body) => {
                 if (err) {
                     LogService.error("MatrixLiteClient", "Error downloading file from " + url);
@@ -140,4 +151,50 @@ export class MatrixLiteClient {
             });
         });
     }
+
+    public async parseMediaMIME(url: string): Promise<any> {
+        return new Promise((resolve, reject) => {
+            request({
+                method: "GET",
+                url: url,
+                encoding: null,
+                headers: {
+                    'Range': 'bytes=0-32'
+                },
+            }, (err, res, _body) => {
+                if (err) {
+                    LogService.error("MatrixLiteClient", "Error downloading file from " + url);
+                    LogService.error("MatrixLiteClient", err);
+                    reject(err);
+                } else if (res.statusCode !== 200) {
+                    if (res.statusCode !== 206) {
+                        LogService.error("MatrixLiteClient", "Got status code " + res.statusCode + " while calling url " + url);
+                        reject(new Error("Error in request: invalid status code"));
+                    }
+                } else {
+                    return this.parseFileHeaderMIME(res.body);
+                }
+            });
+        });
+    }
+
+    public parseFileHeaderMIME(data: Buffer): string {
+        const s = data.slice(0,32);
+        if (s.slice(0,8).includes(Buffer.from("89504E470D0A1A0A", "hex"))) {
+            return("image/png");
+        } else if (s.slice(0,3).includes(Buffer.from("474946", "hex"))) {
+            return("image/gif");
+        } else if (s.slice(0,3).includes(Buffer.from("FFD8FF", "hex"))) {
+            return("image/jpeg");
+        } else if (s.slice(0,3).includes(Buffer.from("000000", "hex")) && s.slice(4,8).includes(Buffer.from("66747970", "hex"))) {
+            if (s.slice(16,28).includes(Buffer.from("61766973", "hex"))) {
+                return("image/avif-sequence");
+            } else if (s.slice(16,28).includes(Buffer.from("61766966", "hex"))) {
+                return("image/avif");
+            }
+        } else if (s.slice(0,4).includes(Buffer.from("52494646", "hex")) && s.slice(8,12).includes(Buffer.from("57454250", "hex"))) {
+            return("image/webp");
+        }
+        return;
+    }
 }

src/matrix/MatrixStickerBot.ts

@@ -12,6 +12,7 @@ import { MatrixLiteClient } from "./MatrixLiteClient";
 import { Cache, CACHE_STICKERS } from "../MemoryCache";
 import { LicenseMap } from "../utils/LicenseMap";
 import { OpenId } from "../models/OpenId";
+import * as sharp from "sharp";
 
 class _MatrixStickerBot {
 
@@ -113,7 +114,119 @@ class _MatrixStickerBot {
 
                 const serverName = mxc.substring("mxc://".length).split("/")[0];
                 const contentId = mxc.substring("mxc://".length).split("/")[1];
-                stickerEvent.thumbMxc = await mx.uploadFromUrl(await mx.getThumbnailUrl(serverName, contentId, 512, 512, "scale", false), "image/png");
+
+                const url = await mx.getMediaUrl(serverName, contentId);
+                const downImage = await mx.downloadFromUrl(url);
+
+                var mime = mx.parseFileHeaderMIME(downImage);
+                if (!mime) continue;
+
+                const origImage = await sharp(downImage, {animated: config.stickers.allowAnimated});
+                var resizedImage:any;
+                var size;
+                if (config.stickers.resize) {
+                    const metadata = await origImage.metadata();
+                    size = metadata.height;
+                    if (metadata.width > metadata.height) {
+                        metadata.width;
+                    }
+                    if (size > 512) size = 512;
+                    resizedImage = await origImage.resize({
+                        width: size,
+                        height: size,
+                        fit: 'contain',
+                        background: 'rgba(0,0,0,0)',
+                    });
+                } else {
+                    if (config.stickers.verifyImageSize) {
+                        const metadata = await origImage.metadata();
+                        if (metadata.width !== metadata.height || metadata.width !== 512) {
+                            LogService.info("MatrixStickerBot", `Sticker ${stickerId} has an invalid size. Skipping...`);
+                            continue;
+                        }
+                    }
+                    resizedImage = origImage;
+                }
+                var imageUpload;
+                var thumbUpload;
+                size = 512;
+                if (mime === "image/png") {
+                    if (config.stickers.resize) {
+                        imageUpload = await resizedImage.png().toBuffer();
+                        thumbUpload = imageUpload;
+                    } else {
+                        thumbUpload = await resizedImage.resize({
+                            width: size,
+                            height: size,
+                            fit: 'contain',
+                            background: 'rgba(0,0,0,0)',
+                        }).png().toBuffer();
+                    }
+                }
+                if (mime === "image/gif" || mime === "image/webp" || mime === "image/avif-sequence") {
+                    if (config.stickers.allowAnimated) {
+                        if (config.stickers.resize){
+                            imageUpload = await resizedImage.webp({quality: 60, effort: 3}).toBuffer();
+                            mime = "image/webp";
+                            thumbUpload = await sharp(downImage, {animated: false}).webp({quality: 50}).toBuffer();
+                        } else {
+                            resizedImage = await sharp(downImage, {animated: false}).resize({
+                                width: size,
+                                height: size,
+                                fit: 'contain',
+                                background: 'rgba(0,0,0,0)',
+                            });
+                            if (mime === "image/gif") {
+                                thumbUpload = await resizedImage.gif().toBuffer();
+                            } else if (mime === "image/avif-sequence") {
+                                thumbUpload = await resizedImage.avif().toBuffer();
+                            } else {
+                                thumbUpload = await resizedImage.webp({quality: 50}).toBuffer();
+                            }
+                        }
+
+                    } else {
+                        imageUpload = await resizedImage.clone().webp({quality: 60, effort: 3}).toBuffer();
+                        thumbUpload = await resizedImage.webp({quality: 50}).toBuffer();
+                        mime = "image/webp";
+                    }
+
+                }
+                if (mime === "image/avif") {
+                    if (config.stickers.resize) {
+                        imageUpload = await resizedImage.clone().avif({quality: 70}).toBuffer();
+                        thumbUpload = await resizedImage.avif({quality: 50, chromaSubsampling: '4:2:0'}).toBuffer();
+                    } else {
+                        thumbUpload = await resizedImage.resize({
+                            width: size,
+                            height: size,
+                            fit: 'contain',
+                            background: 'rgba(0,0,0,0)',
+                        }).avif({quality: 50, chromaSubsampling: '4:2:0'}).toBuffer();
+                    }
+                }
+                if (mime === "image/jpeg") {
+                    if (config.stickers.resize) {
+                        imageUpload = await resizedImage.clone().jpeg({quality: 80, chromaSubsampling: '4:4:4'}).toBuffer();
+                        thumbUpload = await resizedImage.jpeg({quality: 60, chromaSubsampling: '4:2:0'}).toBuffer();
+                    } else {
+                        thumbUpload = await resizedImage.resize({
+                            width: size,
+                            height: size,
+                            fit: 'contain',
+                            background: 'rgba(0,0,0,0)',
+                        }).jpeg({quality: 60, chromaSubsampling: '4:2:0'}).toBuffer();
+                    }
+                }
+                if (imageUpload) {
+                    stickerEvent.contentUri = await mx.upload(imageUpload, mime);
+                }
+                stickerEvent.mimetype = mime;
+                if (thumbUpload) {
+                    stickerEvent.thumbMxc = await mx.upload(thumbUpload, mime);
+                } else {
+                    continue;
+                }
 
                 stickerEvents.push(stickerEvent);
             }
@@ -142,8 +255,13 @@ class _MatrixStickerBot {
                 pack.description = "Matrix sticker pack created by " + authorDisplayName;
                 pack.license = license.name;
                 pack.licensePath = license.url;
-                if (stickerEvents.length > 0) pack.avatarUrl = stickerEvents[0].contentUri;
-                await pack.save();
+                if (stickerEvents.length > 0) {
+                    pack.avatarUrl = stickerEvents[0].thumbMxc;
+                    await pack.save();
+                } else {
+                    LogService.error("MatrixStickerBot", `No stickers in pack ${pack.name}. Removing...`);
+                    pack.destroy();
+                }
 
                 const existingStickers = await Sticker.findAll({where: {packId: pack.id}});
                 for (const sticker of existingStickers) await sticker.destroy();
@@ -157,7 +275,7 @@ class _MatrixStickerBot {
                         thumbnailMxc: stickerEvent.thumbMxc,
                         thumbnailWidth: 512,
                         thumbnailHeight: 512,
-                        mimetype: "image/png",
+                        mimetype: stickerEvent.mimetype,
                     });
                 }
             }

web/app/shared/services/scalar/scalar-widget.api.ts

@@ -45,7 +45,7 @@ export class ScalarWidgetApi {
                 // Element Android requires content.body to contain the sticker description, otherwise
                 // you will not be able to send any stickers
                 body: sticker.description,
-                url: sticker.thumbnail.mxc,
+                url: sticker.image.mxc,
                 info: {
                     mimetype: sticker.image.mimetype,
                     w: Math.round(sticker.thumbnail.width / 2),