Merge pull request #4261 from owid/cloudflare-images-alt-text

🎉 Automatic alt text for cloudflare images

adminSiteClient/ImagesIndexPage.tsx

@@ -13,17 +13,23 @@ import { DbEnrichedImageWithUserId, DbPlainUser } from "@ourworldindata/types"
 import { Timeago } from "./Forms.js"
 import { ColumnsType } from "antd/es/table/InternalTable.js"
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"
-import { faClose, faUpload } from "@fortawesome/free-solid-svg-icons"
+import {
+    faClose,
+    faRobot,
+    faSave,
+    faUpload,
+} from "@fortawesome/free-solid-svg-icons"
 import { RcFile } from "antd/es/upload/interface.js"
-import TextArea from "antd/es/input/TextArea.js"
 import { CLOUDFLARE_IMAGES_URL } from "../settings/clientSettings.js"
 import { keyBy } from "lodash"
+import cx from "classnames"
 
 type ImageMap = Record<string, DbEnrichedImageWithUserId>
 
 type UserMap = Record<string, DbPlainUser>
 
 type ImageEditorApi = {
+    getAltText: (id: number) => Promise<{ altText: string; success: boolean }>
     patchImage: (
         image: DbEnrichedImageWithUserId,
         patch: Partial<DbEnrichedImageWithUserId>
@@ -55,30 +61,48 @@ function AltTextEditor({
     image,
     text,
     patchImage,
+    getAltText,
 }: {
     image: DbEnrichedImageWithUserId
     text: string
     patchImage: ImageEditorApi["patchImage"]
+    getAltText: ImageEditorApi["getAltText"]
 }) {
     const [value, setValue] = useState(text)
+    const [shouldAutosize, setShouldAutosize] = useState(false)
 
-    const handleBlur = useCallback(
-        (e: React.FocusEvent<HTMLTextAreaElement>) => {
-            const trimmed = e.target.value.trim()
-            setValue(trimmed)
-            if (trimmed !== text) {
-                patchImage(image, { defaultAlt: trimmed })
-            }
-        },
-        [image, text, patchImage]
-    )
+    const saveAltText = useCallback(() => {
+        const trimmed = value.trim()
+        patchImage(image, { defaultAlt: trimmed })
+    }, [image, patchImage, value])
+
+    const handleGetAltText = useCallback(async () => {
+        const response = await getAltText(image.id)
+        setValue(response.altText)
+        // Only autoexpand the textarea if the user generates alt text
+        setShouldAutosize(true)
+    }, [image.id, getAltText])
 
     return (
-        <TextArea
-            value={value}
-            onChange={(e) => setValue(e.target.value)}
-            onBlur={handleBlur}
-        />
+        <div className="ImageIndexPage__alt-text-editor">
+            <textarea
+                className={cx({
+                    "ImageIndexPage__alt-text-editor--should-autosize":
+                        shouldAutosize,
+                })}
+                value={value}
+                onChange={(e) => setValue(e.target.value)}
+            />
+            <Button onClick={handleGetAltText} type="text">
+                <FontAwesomeIcon icon={faRobot} />
+            </Button>
+            <Button type="text" onClick={saveAltText} disabled={value === text}>
+                <FontAwesomeIcon icon={faSave} />
+            </Button>
+            {value !== text && (
+                <span className="ImageIndexPage__unsaved-chip">Unsaved</span>
+            )}
+        </div>
     )
 }
 
@@ -183,7 +207,7 @@ function ImgWithRefresh({
                     console.log("Something went wrong refreshing the image", e)
                 })
         }
-    })
+    }, [src, updatedAt])
     return <img ref={ref} src={src} style={{ maxHeight: 100, maxWidth: 100 }} />
 }
 
@@ -241,6 +265,7 @@ function createColumns({
                     text={text}
                     image={image}
                     patchImage={api.patchImage}
+                    getAltText={api.getAltText}
                 />
             ),
         },
@@ -417,6 +442,12 @@ export function ImageIndexPage() {
 
     const api = useMemo(
         (): ImageEditorApi => ({
+            getAltText: (id) => {
+                return admin.requestJSON<{
+                    success: true
+                    altText: string
+                }>(`/api/gpt/suggest-alt-text/${id}`, {}, "GET")
+            },
             deleteImage: async (image) => {
                 await admin.requestJSON(`/api/images/${image.id}`, {}, "DELETE")
                 setImages((prevMap) => {

adminSiteClient/admin.scss

@@ -1224,4 +1224,27 @@ main:not(.ChartEditorPage):not(.GdocsEditPage) {
         color: #007bff;
         margin-bottom: 8px;
     }
+
+    .ImageIndexPage__alt-text-editor {
+        textarea {
+            border-radius: 6px;
+            position: relative;
+            border: 1px solid #ccc;
+            width: 100%;
+            &.ImageIndexPage__alt-text-editor--should-autosize {
+                field-sizing: content;
+            }
+        }
+    }
+    .ImageIndexPage__unsaved-chip {
+        color: gray;
+        background-color: lightgray;
+        padding: 2px 4px;
+        border-radius: 3px;
+        font-size: 10px;
+        text-transform: uppercase;
+        display: inline-block;
+        top: -2px;
+        position: relative;
+    }
 }

adminSiteServer/apiRouter.ts

@@ -10,7 +10,10 @@ import {
     DATA_API_URL,
     FEATURE_FLAGS,
 } from "../settings/serverSettings.js"
-import { FeatureFlagFeature } from "../settings/clientSettings.js"
+import {
+    CLOUDFLARE_IMAGES_URL,
+    FeatureFlagFeature,
+} from "../settings/clientSettings.js"
 import { expectInt, isValidSlug } from "../serverUtils/serverUtil.js"
 import {
     OldChartFieldList,
@@ -197,6 +200,7 @@ import {
 import { CHART_VIEW_PROPS_TO_PERSIST } from "../db/model/ChartView.js"
 import {
     deleteFromCloudflare,
+    fetchGptGeneratedAltText,
     processImageContent,
     uploadToCloudflare,
     validateImagePayload,
@@ -3283,6 +3287,44 @@ getRouteWithROTransaction(
     }
 )
 
+getRouteWithROTransaction(
+    apiRouter,
+    `/gpt/suggest-alt-text/:imageId`,
+    async (
+        req: Request,
+        res,
+        trx
+    ): Promise<{
+        success: boolean
+        altText: string | null
+    }> => {
+        const imageId = parseIntOrUndefined(req.params.imageId)
+        if (!imageId) throw new JsonError(`Invalid image ID`, 400)
+        const image = await trx<DbEnrichedImage>("images")
+            .where("id", imageId)
+            .first()
+        if (!image) throw new JsonError(`No image found for ID ${imageId}`, 404)
+
+        const src = `${CLOUDFLARE_IMAGES_URL}/${image.cloudflareId}/public`
+        let altText: string | null = ""
+        try {
+            altText = await fetchGptGeneratedAltText(src)
+        } catch (error) {
+            console.error(
+                `Error fetching GPT alt text for image ${imageId}`,
+                error
+            )
+            throw new JsonError(`Error fetching GPT alt text: ${error}`, 500)
+        }
+
+        if (!altText) {
+            throw new JsonError(`Unable to generate alt text for image`, 404)
+        }
+
+        return { success: true, altText }
+    }
+)
+
 postRouteWithRWTransaction(
     apiRouter,
     "/explorer/:slug/tags",

adminSiteServer/imagesHelpers.ts

@@ -3,7 +3,9 @@ import sharp from "sharp"
 import {
     CLOUDFLARE_IMAGES_ACCOUNT_ID,
     CLOUDFLARE_IMAGES_API_KEY,
+    OPENAI_API_KEY,
 } from "../settings/serverSettings.js"
+import { OpenAI } from "openai"
 
 export function validateImagePayload(body: any): {
     filename: string
@@ -101,3 +103,35 @@ export async function deleteFromCloudflare(cloudflareId: string) {
         throw new JsonError(JSON.stringify(response.errors))
     }
 }
+
+export async function fetchGptGeneratedAltText(url: string) {
+    const openai = new OpenAI({
+        apiKey: OPENAI_API_KEY,
+    })
+
+    const completion = await openai.chat.completions.create({
+        messages: [
+            {
+                role: "user",
+                content: `Generate alt text for this image, describing it for a vision impaired person using a screen reader.
+Do not say "alt text:".
+Do not say "The image...".
+If the image is a data visualization and there are data sources in the footer, describe them exhaustively.`,
+            },
+            {
+                role: "user",
+                content: [
+                    {
+                        type: "image_url",
+                        image_url: {
+                            url,
+                        },
+                    },
+                ],
+            },
+        ],
+        model: "gpt-4o-mini",
+    })
+
+    return completion.choices[0].message.content
+}

db/migration/1731360326761-CloudflareImages.ts

@@ -3,17 +3,34 @@ import { MigrationInterface, QueryRunner } from "typeorm"
 export class CloudflareImages1731360326761 implements MigrationInterface {
     public async up(queryRunner: QueryRunner): Promise<void> {
         await queryRunner.query(`-- sql
-            ALTER TABLE images 
-                ADD COLUMN cloudflareId VARCHAR(255) NULL,
-                MODIFY COLUMN googleId VARCHAR(255) NULL
+            ALTER TABLE images
+            ADD COLUMN cloudflareId VARCHAR(255) NULL,
+            MODIFY COLUMN googleId VARCHAR(255) NULL,
+            MODIFY COLUMN defaultAlt VARCHAR(1600) NULL;`)
+
+        // One-way migration 👋
+        await queryRunner.query(`-- sql
+            UPDATE images
+            SET defaultAlt = NULL
+            WHERE defaultAlt = 'legacy-wordpress-upload';
         `)
     }
 
     public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`-- sql
+            ALTER TABLE images
+            DROP COLUMN cloudflareId
+        `)
+
+        await queryRunner.query(`-- sql
+            UPDATE images 
+            SET googleId = 'cloudflare_image' 
+            WHERE googleId IS NULL
+        `)
+
         await queryRunner.query(`-- sql
             ALTER TABLE images 
-                DROP COLUMN cloudflareId,
-                MODIFY COLUMN googleId VARCHAR(255) NOT NULL
+            MODIFY COLUMN googleId VARCHAR(255) NOT NULL
         `)
     }
 }

db/migration/1732994843041-CloudflareImagesAddUserId.ts

@@ -13,6 +13,9 @@ export class CloudflareImagesAddUserId1732994843041
         `)
     }
     public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`-- sql
+            ALTER TABLE images DROP CONSTRAINT fk_user_images;
+        `)
         await queryRunner.query(`-- sql
             ALTER TABLE images DROP COLUMN userId;
         `)