Merge pull request #4268 from owid/cloudflare-images-hash

🎉 Cloudflare images hash column
owid · Dec 6, 2024 · f559b0c · f559b0c
2 parents c7193b2 + 515c084
commit f559b0c
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 8 deletions.
diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts
@@ -3124,7 +3124,21 @@ postRouteWithRWTransaction(apiRouter, "/images", async (req, res, trx) => {
         }
     }
 
-    const { asBlob, dimensions } = await processImageContent(content, type)
+    const { asBlob, dimensions, hash } = await processImageContent(
+        content,
+        type
+    )
+
+    const collision = await trx<DbEnrichedImage>("images")
+        .where("hash", "=", hash)
+        .first()
+
+    if (collision) {
+        return {
+            success: false,
+            error: `An image with this content already exists (filename: ${collision.filename})`,
+        }
+    }
 
     const cloudflareId = await uploadToCloudflare(filename, asBlob)
 
@@ -3140,10 +3154,9 @@ postRouteWithRWTransaction(apiRouter, "/images", async (req, res, trx) => {
         originalWidth: dimensions.width,
         originalHeight: dimensions.height,
         cloudflareId,
-        // TODO: make defaultAlt nullable
-        defaultAlt: "Default alt text",
         updatedAt: new Date().getTime(),
         userId: res.locals.user.id,
+        hash,
     })
 
     const image = await db.getCloudflareImage(trx, filename)
@@ -3180,10 +3193,23 @@ putRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => {
         )
     }
 
-    await deleteFromCloudflare(originalCloudflareId)
-
     const { type, content } = validateImagePayload(req.body)
-    const { asBlob, dimensions } = await processImageContent(content, type)
+    const { asBlob, dimensions, hash } = await processImageContent(
+        content,
+        type
+    )
+    const collision = await trx<DbEnrichedImage>("images")
+        .where("hash", "=", hash)
+        .first()
+
+    if (collision) {
+        return {
+            success: false,
+            error: `An image with this content already exists (filename: ${collision.filename})`,
+        }
+    }
+
+    await deleteFromCloudflare(originalCloudflareId)
     const newCloudflareId = await uploadToCloudflare(originalFilename, asBlob)
 
     if (!newCloudflareId) {
@@ -3194,6 +3220,7 @@ putRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => {
         originalWidth: dimensions.width,
         originalHeight: dimensions.height,
         updatedAt: new Date().getTime(),
+        hash,
     })
 
     const updated = await db.getCloudflareImage(trx, originalFilename)

diff --git a/adminSiteServer/imagesHelpers.ts b/adminSiteServer/imagesHelpers.ts
@@ -1,3 +1,4 @@
+import crypto from "crypto"
 import { JsonError } from "@ourworldindata/types"
 import sharp from "sharp"
 import {
@@ -32,9 +33,11 @@ export async function processImageContent(
 ): Promise<{
     asBlob: Blob
     dimensions: { width: number; height: number }
+    hash: string
 }> {
     const stripped = content.slice(content.indexOf(",") + 1)
     const asBuffer = Buffer.from(stripped, "base64")
+    const hash = crypto.createHash("sha256").update(asBuffer).digest("hex")
     const asBlob = new Blob([asBuffer], { type })
     const { width, height } = await sharp(asBuffer)
         .metadata()
@@ -50,6 +53,7 @@ export async function processImageContent(
             width,
             height,
         },
+        hash,
     }
 }
 

diff --git a/db/migration/1731360326761-CloudflareImages.ts b/db/migration/1731360326761-CloudflareImages.ts
@@ -5,6 +5,8 @@ export class CloudflareImages1731360326761 implements MigrationInterface {
         await queryRunner.query(`-- sql
             ALTER TABLE images
             ADD COLUMN cloudflareId VARCHAR(255) NULL,
+            ADD CONSTRAINT images_cloudflareId_unique UNIQUE (cloudflareId),
+            ADD COLUMN hash VARCHAR(255) NULL,
             MODIFY COLUMN googleId VARCHAR(255) NULL,
             MODIFY COLUMN defaultAlt VARCHAR(1600) NULL;`)
 
@@ -19,7 +21,8 @@ export class CloudflareImages1731360326761 implements MigrationInterface {
     public async down(queryRunner: QueryRunner): Promise<void> {
         await queryRunner.query(`-- sql
             ALTER TABLE images
-            DROP COLUMN cloudflareId
+            DROP COLUMN cloudflareId,
+            DROP COLUMN hash
         `)
 
         await queryRunner.query(`-- sql

diff --git a/packages/@ourworldindata/types/src/dbTypes/Images.ts b/packages/@ourworldindata/types/src/dbTypes/Images.ts
@@ -2,14 +2,15 @@ import { DbPlainUser } from "./Users.js"
 
 export const ImagesTableName = "images"
 export interface DbInsertImage {
-    googleId: string
+    googleId: string | null
     defaultAlt: string
     filename: string
     id?: number
     originalWidth?: number | null
     originalHeight?: number | null
     updatedAt?: string | null // MySQL Date objects round to the nearest second, whereas Google includes milliseconds so we store as an epoch of type bigint to avoid any conversion issues
     cloudflareId?: string | null
+    hash?: string | null
     userId?: number | null
 }
 export type DbRawImage = Required<DbInsertImage>