From 5fce282d4029771745bc33211af8c0fc0037561b Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Wed, 11 Dec 2024 17:22:47 +0100 Subject: [PATCH 01/20] feat: pre-fetch chart views metadata in gdocs --- baker/SiteBaker.tsx | 10 ++++++- baker/siteRenderers.tsx | 1 + db/model/ChartView.ts | 29 +++++++++++++++++++ .../types/src/gdocTypes/Gdoc.ts | 10 +++++++ site/gdocs/OwidGdoc.tsx | 3 ++ 5 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 db/model/ChartView.ts diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index c422adcaa4e..e6a7495b3fa 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -109,6 +109,7 @@ import { getTombstones } from "../db/model/GdocTombstone.js" import { bakeAllMultiDimDataPages } from "./MultiDimBaker.js" import { getAllLinkedPublishedMultiDimDataPages } from "../db/model/MultiDimDataPage.js" import { getPublicDonorNames } from "../db/model/Donor.js" +import { getAllChartViewsMetadata } from "../db/model/ChartView.js" type PrefetchedAttachments = { donors: string[] @@ -176,7 +177,7 @@ function getProgressBarTotal(bakeSteps: BakeStepConfig): number { bakeSteps.has("dataInsights") || bakeSteps.has("authors") ) { - total += 8 + total += 9 } return total } @@ -459,6 +460,12 @@ export class SiteBaker { name: `✅ Prefetched ${publishedAuthors.length} authors`, }) + const chartViewMetadata = await getAllChartViewsMetadata(knex) + const chartViewMetadataByName = keyBy(chartViewMetadata, "name") + this.progressBar.tick({ + name: `✅ Prefetched ${chartViewMetadata.length} chart views`, + }) + const prefetchedAttachments = { donors, linkedAuthors: publishedAuthors, @@ -469,6 +476,7 @@ export class SiteBaker { graphers: publishedChartsBySlug, }, linkedIndicators: datapageIndicatorsById, + chartViewMetadata: chartViewMetadataByName, } this.progressBar.tick({ name: "✅ Prefetched attachments" }) this._prefetchedAttachmentsCache = prefetchedAttachments diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index de8bd8deaa4..7b310471018 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -439,6 +439,7 @@ ${dataInsights latestDataInsights: get(post, "latestDataInsights", []), homepageMetadata: get(post, "homepageMetadata", {}), latestWorkLinks: get(post, "latestWorkLinks", []), + chartViewMetadata: get(post, "chartViewMetadata", {}), }} > diff --git a/db/model/ChartView.ts b/db/model/ChartView.ts new file mode 100644 index 00000000000..6cdce8e4698 --- /dev/null +++ b/db/model/ChartView.ts @@ -0,0 +1,29 @@ +import { ChartViewMetadata, JsonString } from "@ourworldindata/types" +import * as db from "../db.js" + +export const getAllChartViewsMetadata = async ( + knex: db.KnexReadonlyTransaction +): Promise => { + type RawRow = Omit & { + queryParamsForParentChart: JsonString + } + const rows: RawRow[] = await db.knexRaw( + knex, + `-- sql +SELECT cv.name, + cc.full ->> "$.title" as title, + chartConfigId, + pcc.slug as parentChartSlug, + cv.queryParamsForParentChart +FROM chart_views cv +JOIN chart_configs cc on cc.id = cv.chartConfigId +JOIN charts pc on cv.parentChartId = pc.id +JOIN chart_configs pcc on pc.configId = pcc.id + ` + ) + + return rows.map((row) => ({ + ...row, + queryParamsForParentChart: JSON.parse(row.queryParamsForParentChart), + })) +} diff --git a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts index a6f35022ea3..9731db1b709 100644 --- a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts +++ b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts @@ -13,6 +13,7 @@ import { } from "./ArchieMlComponents.js" import { MinimalTag } from "../dbTypes/Tags.js" import { DbEnrichedLatestWork } from "../domainTypes/Author.js" +import { QueryParams } from "../domainTypes/Various.js" export enum OwidGdocPublicationContext { unlisted = "unlisted", @@ -53,6 +54,15 @@ export interface LinkedChart { indicatorId?: number // in case of a datapage } +// An object containing metadata needed for embedded narrative charts +export interface ChartViewMetadata { + name: string + title: string + chartConfigId: string + parentChartSlug: string + queryParamsForParentChart: QueryParams +} + /** * A linked indicator is derived from a linked grapher's config (see: getVariableOfDatapageIfApplicable) * e.g. https://ourworldindata.org/grapher/tomato-production -> config for grapher with { slug: "tomato-production" } -> indicator metadata diff --git a/site/gdocs/OwidGdoc.tsx b/site/gdocs/OwidGdoc.tsx index 0380c0cf0c8..00c8c00f8a7 100644 --- a/site/gdocs/OwidGdoc.tsx +++ b/site/gdocs/OwidGdoc.tsx @@ -13,6 +13,7 @@ import { OwidGdocMinimalPostInterface, OwidGdocHomepageMetadata, DbEnrichedLatestWork, + ChartViewMetadata, } from "@ourworldindata/types" import { get, getOwidGdocFromJSON } from "@ourworldindata/utils" import { DebugProvider } from "./DebugContext.js" @@ -35,6 +36,7 @@ export type Attachments = { latestDataInsights?: LatestDataInsight[] homepageMetadata?: OwidGdocHomepageMetadata latestWorkLinks?: DbEnrichedLatestWork[] + chartViewMetadata?: Record } export const AttachmentsContext = createContext({ @@ -131,6 +133,7 @@ export function OwidGdoc({ latestDataInsights: get(props, "latestDataInsights", []), homepageMetadata: get(props, "homepageMetadata", {}), latestWorkLinks: get(props, "latestWorkLinks", []), + chartViewMetadata: get(props, "chartViewMetadata", {}), }} > From 7ec093cdae576625f7ba7c34a8ff9de477fd8866 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Wed, 11 Dec 2024 17:23:22 +0100 Subject: [PATCH 02/20] feat: NarrativeChart component --- db/model/Gdoc/GdocBase.ts | 4 ++ db/model/Gdoc/enrichedToMarkdown.ts | 11 +++ db/model/Gdoc/enrichedToRaw.ts | 15 ++++ db/model/Gdoc/exampleEnrichedBlocks.ts | 10 +++ db/model/Gdoc/gdocUtils.ts | 1 + db/model/Gdoc/rawToArchie.ts | 20 ++++++ db/model/Gdoc/rawToEnriched.ts | 64 +++++++++++++++++ .../types/src/gdocTypes/ArchieMlComponents.ts | 27 ++++++++ packages/@ourworldindata/types/src/index.ts | 3 + packages/@ourworldindata/utils/src/Util.ts | 1 + site/gdocs/components/ArticleBlock.tsx | 10 +++ site/gdocs/components/NarrativeChart.tsx | 68 +++++++++++++++++++ 12 files changed, 234 insertions(+) create mode 100644 site/gdocs/components/NarrativeChart.tsx diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index 69fe5c49611..532c4722f25 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -567,6 +567,10 @@ export class GdocBase implements OwidGdocBaseInterface { "key-indicator-collection", "list", "missing-data", + + // Open question: there's not a direct link to a chart here, but there is a chart and also a parent chart + "narrative-chart", + "numbered-list", "people", "people-rows", diff --git a/db/model/Gdoc/enrichedToMarkdown.ts b/db/model/Gdoc/enrichedToMarkdown.ts index 2556f74ef2b..794700f30a3 100644 --- a/db/model/Gdoc/enrichedToMarkdown.ts +++ b/db/model/Gdoc/enrichedToMarkdown.ts @@ -127,6 +127,17 @@ ${items} exportComponents ) ) + .with({ type: "narrative-chart" }, (b): string | undefined => + markdownComponent( + "NarrativeChart", + { + name: b.name, + caption: b.caption ? spansToMarkdown(b.caption) : undefined, + // Note: truncated + }, + exportComponents + ) + ) .with({ type: "code" }, (b): string | undefined => { return ( "```\n" + diff --git a/db/model/Gdoc/enrichedToRaw.ts b/db/model/Gdoc/enrichedToRaw.ts index bc27e3356ed..08e8e1fa288 100644 --- a/db/model/Gdoc/enrichedToRaw.ts +++ b/db/model/Gdoc/enrichedToRaw.ts @@ -48,6 +48,7 @@ import { RawBlockPeople, RawBlockPeopleRows, RawBlockPerson, + RawBlockNarrativeChart, RawBlockCode, } from "@ourworldindata/types" import { spanToHtmlString } from "./gdocUtils.js" @@ -123,6 +124,20 @@ export function enrichedBlockToRawBlock( }, }) ) + .with( + { type: "narrative-chart" }, + (b): RawBlockNarrativeChart => ({ + type: b.type, + value: { + name: b.name, + height: b.height, + row: b.row, + column: b.column, + position: b.position, + caption: b.caption ? spansToHtmlText(b.caption) : undefined, + }, + }) + ) .with( { type: "code" }, (b): RawBlockCode => ({ diff --git a/db/model/Gdoc/exampleEnrichedBlocks.ts b/db/model/Gdoc/exampleEnrichedBlocks.ts index 7e37a16bc27..74b2a0f8842 100644 --- a/db/model/Gdoc/exampleEnrichedBlocks.ts +++ b/db/model/Gdoc/exampleEnrichedBlocks.ts @@ -121,6 +121,16 @@ export const enrichedBlockExamples: Record< caption: boldLinkExampleText, parseErrors: [], }, + "narrative-chart": { + type: "narrative-chart", + name: "world-has-become-less-democratic", + height: "400", + row: "1", + column: "1", + position: "featured", + caption: boldLinkExampleText, + parseErrors: [], + }, code: { type: "code", text: [ diff --git a/db/model/Gdoc/gdocUtils.ts b/db/model/Gdoc/gdocUtils.ts index f08dfd85786..d53f3f41a5d 100644 --- a/db/model/Gdoc/gdocUtils.ts +++ b/db/model/Gdoc/gdocUtils.ts @@ -237,6 +237,7 @@ export function extractFilenamesFromBlock( "latest-data-insights", "list", "missing-data", + "narrative-chart", "numbered-list", "people", "people-rows", diff --git a/db/model/Gdoc/rawToArchie.ts b/db/model/Gdoc/rawToArchie.ts index c2c9b50803d..b1b344a4491 100644 --- a/db/model/Gdoc/rawToArchie.ts +++ b/db/model/Gdoc/rawToArchie.ts @@ -47,6 +47,7 @@ import { RawBlockPeople, RawBlockPeopleRows, RawBlockPerson, + RawBlockNarrativeChart, RawBlockCode, } from "@ourworldindata/types" import { isArray } from "@ourworldindata/utils" @@ -128,6 +129,21 @@ function* rawBlockChartToArchieMLString( yield "{}" } +function* rawBlockNarrativeChartToArchieMLString( + block: RawBlockNarrativeChart +): Generator { + yield "{.narrative-chart}" + if (typeof block.value !== "string") { + yield* propertyToArchieMLString("name", block.value) + yield* propertyToArchieMLString("height", block.value) + yield* propertyToArchieMLString("row", block.value) + yield* propertyToArchieMLString("column", block.value) + yield* propertyToArchieMLString("position", block.value) + yield* propertyToArchieMLString("caption", block.value) + } + yield "{}" +} + function* rawBlockCodeToArchieMLString( block: RawBlockCode ): Generator { @@ -840,6 +856,10 @@ export function* OwidRawGdocBlockToArchieMLStringGenerator( .with({ type: "all-charts" }, rawBlockAllChartsToArchieMLString) .with({ type: "aside" }, rawBlockAsideToArchieMLString) .with({ type: "chart" }, rawBlockChartToArchieMLString) + .with( + { type: "narrative-chart" }, + rawBlockNarrativeChartToArchieMLString + ) .with({ type: "code" }, rawBlockCodeToArchieMLString) .with({ type: "donors" }, rawBlockDonorListToArchieMLString) .with({ type: "scroller" }, rawBlockScrollerToArchieMLString) diff --git a/db/model/Gdoc/rawToEnriched.ts b/db/model/Gdoc/rawToEnriched.ts index a60bd1f4e79..209fcc59847 100644 --- a/db/model/Gdoc/rawToEnriched.ts +++ b/db/model/Gdoc/rawToEnriched.ts @@ -129,6 +129,8 @@ import { EnrichedBlockPerson, RawBlockPeopleRows, EnrichedBlockPeopleRows, + RawBlockNarrativeChart, + EnrichedBlockNarrativeChart, RawBlockCode, EnrichedBlockCode, } from "@ourworldindata/types" @@ -172,6 +174,7 @@ export function parseRawBlocksToEnrichedBlocks( .with({ type: "blockquote" }, parseBlockquote) .with({ type: "callout" }, parseCallout) .with({ type: "chart" }, parseChart) + .with({ type: "narrative-chart" }, parseNarrativeChart) .with({ type: "code" }, parseCode) .with({ type: "donors" }, parseDonorList) .with({ type: "scroller" }, parseScroller) @@ -496,6 +499,67 @@ const parseChart = (raw: RawBlockChart): EnrichedBlockChart => { } } +const parseNarrativeChart = ( + raw: RawBlockNarrativeChart +): EnrichedBlockNarrativeChart => { + const createError = ( + error: ParseError, + name: string, + caption: Span[] = [] + ): EnrichedBlockNarrativeChart => ({ + type: "narrative-chart", + name, + caption, + parseErrors: [error], + }) + + const val = raw.value + + if (typeof val === "string") { + return { + type: "narrative-chart", + name: val, + parseErrors: [], + } + } else { + if (!val.name) + return createError( + { + message: "name property is missing", + }, + "" + ) + + const warnings: ParseError[] = [] + + const height = val.height + const row = val.row + const column = val.column + // This property is currently unused, a holdover from @mathisonian's gdocs demo. + // We will decide soon™️ if we want to use it for something + let position: ChartPositionChoice | undefined = undefined + if (val.position) + if (val.position === "featured") position = val.position + else { + warnings.push({ + message: "position must be 'featured' or unset", + }) + } + const caption = val.caption ? htmlToSpans(val.caption) : [] + + return omitUndefinedValues({ + type: "narrative-chart", + name: val.name, + height, + row, + column, + position, + caption: caption.length > 0 ? caption : undefined, + parseErrors: [], + }) as EnrichedBlockNarrativeChart + } +} + const parseCode = (raw: RawBlockCode): EnrichedBlockCode => { return { type: "code", diff --git a/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts b/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts index cbb50d10c51..06c200041d1 100644 --- a/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts +++ b/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts @@ -86,6 +86,31 @@ export type EnrichedBlockChart = { tabs?: ChartTabKeyword[] } & EnrichedBlockWithParseErrors +export type RawBlockNarrativeChartValue = { + name?: string + height?: string + row?: string + column?: string + // TODO: position is used as a classname apparently? Should be renamed or split + position?: string + caption?: string +} + +export type RawBlockNarrativeChart = { + type: "narrative-chart" + value: RawBlockNarrativeChartValue | string +} + +export type EnrichedBlockNarrativeChart = { + type: "narrative-chart" + name: string + height?: string + row?: string + column?: string + position?: ChartPositionChoice + caption?: Span[] +} & EnrichedBlockWithParseErrors + export type RawBlockCode = { type: "code" value: RawBlockText[] @@ -950,6 +975,7 @@ export type OwidRawGdocBlock = | RawBlockAside | RawBlockCallout | RawBlockChart + | RawBlockNarrativeChart | RawBlockCode | RawBlockDonorList | RawBlockScroller @@ -1001,6 +1027,7 @@ export type OwidEnrichedGdocBlock = | EnrichedBlockAside | EnrichedBlockCallout | EnrichedBlockChart + | EnrichedBlockNarrativeChart | EnrichedBlockCode | EnrichedBlockDonorList | EnrichedBlockScroller diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts index f5fd8971661..3be6749e83e 100644 --- a/packages/@ourworldindata/types/src/index.ts +++ b/packages/@ourworldindata/types/src/index.ts @@ -287,6 +287,8 @@ export { SocialLinkType, type RawSocialLink, type EnrichedSocialLink, + type RawBlockNarrativeChart, + type EnrichedBlockNarrativeChart, } from "./gdocTypes/ArchieMlComponents.js" export { ChartConfigType, @@ -330,6 +332,7 @@ export { type OwidGdocContent, type OwidGdocIndexItem, extractGdocIndexItem, + type ChartViewMetadata, } from "./gdocTypes/Gdoc.js" export { diff --git a/packages/@ourworldindata/utils/src/Util.ts b/packages/@ourworldindata/utils/src/Util.ts index 9908fd28753..32f55f7478c 100644 --- a/packages/@ourworldindata/utils/src/Util.ts +++ b/packages/@ourworldindata/utils/src/Util.ts @@ -1708,6 +1708,7 @@ export function traverseEnrichedBlock( type: P.union( "chart-story", "chart", + "narrative-chart", "code", "donors", "horizontal-rule", diff --git a/site/gdocs/components/ArticleBlock.tsx b/site/gdocs/components/ArticleBlock.tsx index b4998618028..c6d1aefc1f3 100644 --- a/site/gdocs/components/ArticleBlock.tsx +++ b/site/gdocs/components/ArticleBlock.tsx @@ -46,6 +46,7 @@ import { HomepageSearch } from "./HomepageSearch.js" import LatestDataInsightsBlock from "./LatestDataInsightsBlock.js" import { Socials } from "./Socials.js" import Person from "./Person.js" +import NarrativeChart from "./NarrativeChart.js" export type Container = | "default" @@ -246,6 +247,15 @@ export default function ArticleBlock({ /> ) }) + .with({ type: "narrative-chart" }, (block) => { + return ( + + ) + }) .with({ type: "code" }, (block) => ( (null) + useEmbedChart(0, refChartContainer) + + const attachments = useContext(AttachmentsContext) + + const viewMetadata = attachments.chartViewMetadata?.[d.name] + + if (!viewMetadata) + return ( + + ) + + const metadataStringified = JSON.stringify(viewMetadata) + + return ( +
+
+ {/* + + + */} +
+ {d.caption ? ( +
{renderSpans(d.caption)}
+ ) : null} +
+ ) +} From 523c203eed43f070866c5f740bbfe04b189ffda6 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Thu, 12 Dec 2024 10:33:40 +0100 Subject: [PATCH 03/20] refactor: properly attach gdocs attachments --- baker/SiteBaker.tsx | 5 +++++ db/model/Gdoc/GdocBase.ts | 12 ++++++++++++ site/gdocs/OwidGdoc.tsx | 1 + site/gdocs/components/NarrativeChart.tsx | 9 +++------ site/gdocs/utils.tsx | 5 +++++ 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index e6a7495b3fa..a4ce0036a98 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -56,6 +56,7 @@ import { grabMetadataForGdocLinkedIndicator, TombstonePageData, gdocUrlRegex, + ChartViewMetadata, } from "@ourworldindata/utils" import { execWrapper } from "../db/execWrapper.js" import { countryProfileSpecs } from "../site/countryProfileProjects.js" @@ -121,6 +122,7 @@ type PrefetchedAttachments = { explorers: Record } linkedIndicators: Record + chartViewMetadata: Record } // These aren't all "wordpress" steps @@ -536,6 +538,8 @@ export class SiteBaker { this._prefetchedAttachmentsCache.linkedAuthors.filter( (author) => authorNames.includes(author.name) ), + chartViewMetadata: + this._prefetchedAttachmentsCache.chartViewMetadata, // TODO: Filter } } return this._prefetchedAttachmentsCache @@ -637,6 +641,7 @@ export class SiteBaker { ...attachments.linkedCharts.explorers, } publishedGdoc.linkedIndicators = attachments.linkedIndicators + publishedGdoc.chartViewMetadata = attachments.chartViewMetadata // this is a no-op if the gdoc doesn't have an all-chart block if ("loadRelatedCharts" in publishedGdoc) { diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index 532c4722f25..85754e328f4 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -56,6 +56,7 @@ import { import { ARCHVED_THUMBNAIL_FILENAME, ChartConfigType, + ChartViewMetadata, DEFAULT_THUMBNAIL_FILENAME, GrapherInterface, LatestDataInsight, @@ -66,6 +67,7 @@ import { OwidGdocLinkType, OwidGdocType, } from "@ourworldindata/types" +import { getAllChartViewsMetadata } from "../ChartView.js" export class GdocBase implements OwidGdocBaseInterface { id!: string @@ -89,6 +91,7 @@ export class GdocBase implements OwidGdocBaseInterface { linkedIndicators: Record = {} linkedDocuments: Record = {} latestDataInsights: LatestDataInsight[] = [] + chartViewMetadata?: Record = {} _omittableFields: string[] = [] constructor(id?: string) { @@ -704,6 +707,14 @@ export class GdocBase implements OwidGdocBaseInterface { } } + async loadChartViewMetadata( + knex: db.KnexReadonlyTransaction + ): Promise { + // TODO: Filter down to only those that are used in the Gdoc + const result = await getAllChartViewsMetadata(knex) + this.chartViewMetadata = keyBy(result, "name") + } + async fetchAndEnrichGdoc(): Promise { const docsClient = google.docs({ version: "v1", @@ -849,6 +860,7 @@ export class GdocBase implements OwidGdocBaseInterface { await this.loadImageMetadataFromDB(knex) await this.loadLinkedCharts(knex) await this.loadLinkedIndicators() // depends on linked charts + await this.loadChartViewMetadata(knex) await this._loadSubclassAttachments(knex) await this.validate(knex) } diff --git a/site/gdocs/OwidGdoc.tsx b/site/gdocs/OwidGdoc.tsx index 00c8c00f8a7..9832d537911 100644 --- a/site/gdocs/OwidGdoc.tsx +++ b/site/gdocs/OwidGdoc.tsx @@ -49,6 +49,7 @@ export const AttachmentsContext = createContext({ latestDataInsights: [], homepageMetadata: {}, latestWorkLinks: [], + chartViewMetadata: {}, }) export const DocumentContext = createContext<{ isPreviewing: boolean }>({ diff --git a/site/gdocs/components/NarrativeChart.tsx b/site/gdocs/components/NarrativeChart.tsx index 096d9f41ae3..2f1cc74ddab 100644 --- a/site/gdocs/components/NarrativeChart.tsx +++ b/site/gdocs/components/NarrativeChart.tsx @@ -1,10 +1,9 @@ -import React, { useContext, useRef } from "react" +import React, { useRef } from "react" import { useEmbedChart } from "../../hooks.js" import { EnrichedBlockNarrativeChart } from "@ourworldindata/types" -import { renderSpans } from "../utils.js" +import { renderSpans, useChartViewMetadata } from "../utils.js" import cx from "classnames" import { GRAPHER_PREVIEW_CLASS } from "../../SiteConstants.js" -import { AttachmentsContext } from "../OwidGdoc.js" import { BlockErrorFallback } from "./BlockErrorBoundary.js" export default function NarrativeChart({ @@ -19,9 +18,7 @@ export default function NarrativeChart({ const refChartContainer = useRef(null) useEmbedChart(0, refChartContainer) - const attachments = useContext(AttachmentsContext) - - const viewMetadata = attachments.chartViewMetadata?.[d.name] + const viewMetadata = useChartViewMetadata(d.name) if (!viewMetadata) return ( diff --git a/site/gdocs/utils.tsx b/site/gdocs/utils.tsx index cbeb58774ef..e3a39c703a8 100644 --- a/site/gdocs/utils.tsx +++ b/site/gdocs/utils.tsx @@ -147,6 +147,11 @@ export function useDonors(): string[] | undefined { return donors } +export const useChartViewMetadata = (name: string) => { + const { chartViewMetadata } = useContext(AttachmentsContext) + return chartViewMetadata?.[name] +} + const LinkedA = ({ span }: { span: SpanLink }): React.ReactElement => { const linkType = getLinkType(span.url) const { linkedDocument } = useLinkedDocument(span.url) From a6da14d3ad198cd732de543f807f31d1a9fdf142 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Thu, 12 Dec 2024 10:38:53 +0100 Subject: [PATCH 04/20] enhance: ability to filter `chartViewMetadata` --- baker/SiteBaker.tsx | 4 ++-- db/model/ChartView.ts | 17 +++++++++++------ db/model/Gdoc/GdocBase.ts | 4 ++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index a4ce0036a98..a62a4b1f40b 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -110,7 +110,7 @@ import { getTombstones } from "../db/model/GdocTombstone.js" import { bakeAllMultiDimDataPages } from "./MultiDimBaker.js" import { getAllLinkedPublishedMultiDimDataPages } from "../db/model/MultiDimDataPage.js" import { getPublicDonorNames } from "../db/model/Donor.js" -import { getAllChartViewsMetadata } from "../db/model/ChartView.js" +import { getChartViewsMetadata } from "../db/model/ChartView.js" type PrefetchedAttachments = { donors: string[] @@ -462,7 +462,7 @@ export class SiteBaker { name: `✅ Prefetched ${publishedAuthors.length} authors`, }) - const chartViewMetadata = await getAllChartViewsMetadata(knex) + const chartViewMetadata = await getChartViewsMetadata(knex) const chartViewMetadataByName = keyBy(chartViewMetadata, "name") this.progressBar.tick({ name: `✅ Prefetched ${chartViewMetadata.length} chart views`, diff --git a/db/model/ChartView.ts b/db/model/ChartView.ts index 6cdce8e4698..2113aab0d84 100644 --- a/db/model/ChartView.ts +++ b/db/model/ChartView.ts @@ -1,15 +1,16 @@ import { ChartViewMetadata, JsonString } from "@ourworldindata/types" import * as db from "../db.js" -export const getAllChartViewsMetadata = async ( - knex: db.KnexReadonlyTransaction +export const getChartViewsMetadata = async ( + knex: db.KnexReadonlyTransaction, + names?: string[] ): Promise => { type RawRow = Omit & { queryParamsForParentChart: JsonString } - const rows: RawRow[] = await db.knexRaw( - knex, - `-- sql + let rows: RawRow[] + + const query = `-- sql SELECT cv.name, cc.full ->> "$.title" as title, chartConfigId, @@ -20,7 +21,11 @@ JOIN chart_configs cc on cc.id = cv.chartConfigId JOIN charts pc on cv.parentChartId = pc.id JOIN chart_configs pcc on pc.configId = pcc.id ` - ) + + if (names) { + if (names.length === 0) return [] + rows = await db.knexRaw(knex, `${query} WHERE cv.name IN (?)`, [names]) + } else rows = await db.knexRaw(knex, query) return rows.map((row) => ({ ...row, diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index 85754e328f4..23ddfd187da 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -67,7 +67,7 @@ import { OwidGdocLinkType, OwidGdocType, } from "@ourworldindata/types" -import { getAllChartViewsMetadata } from "../ChartView.js" +import { getChartViewsMetadata } from "../ChartView.js" export class GdocBase implements OwidGdocBaseInterface { id!: string @@ -711,7 +711,7 @@ export class GdocBase implements OwidGdocBaseInterface { knex: db.KnexReadonlyTransaction ): Promise { // TODO: Filter down to only those that are used in the Gdoc - const result = await getAllChartViewsMetadata(knex) + const result = await getChartViewsMetadata(knex) this.chartViewMetadata = keyBy(result, "name") } From 49a2c87c6af9256a97ae7a97dcec94d0e2cfa44c Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Mon, 16 Dec 2024 23:07:24 +0100 Subject: [PATCH 05/20] refactor: add narrative-chart to `extractGdocComponentInfo` --- db/model/Gdoc/extractGdocComponentInfo.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/model/Gdoc/extractGdocComponentInfo.ts b/db/model/Gdoc/extractGdocComponentInfo.ts index 66b4c3c608a..ae610cf6fcb 100644 --- a/db/model/Gdoc/extractGdocComponentInfo.ts +++ b/db/model/Gdoc/extractGdocComponentInfo.ts @@ -327,6 +327,7 @@ export function enumerateGdocComponentsWithoutChildren( type: P.union( "chart-story", "chart", + "narrative-chart", "horizontal-rule", "html", "image", @@ -354,7 +355,6 @@ export function enumerateGdocComponentsWithoutChildren( "simple-text", "donors", "socials" - // "narrative-chart" should go here once it's done ), }, (c) => handleComponent(c, [], parentPath, path) From 06a8e5a89ebb485623d7e80249f674870e94e538 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Tue, 17 Dec 2024 15:33:53 +0100 Subject: [PATCH 06/20] refactor: chartViewMetadata -> narrativeViewInfo --- baker/SiteBaker.tsx | 20 +++++++++---------- baker/siteRenderers.tsx | 2 +- db/model/ChartView.ts | 8 ++++---- db/model/Gdoc/GdocBase.ts | 14 ++++++------- .../types/src/gdocTypes/Gdoc.ts | 2 +- packages/@ourworldindata/types/src/index.ts | 2 +- site/gdocs/OwidGdoc.tsx | 8 ++++---- site/gdocs/components/NarrativeChart.tsx | 4 ++-- site/gdocs/utils.tsx | 6 +++--- 9 files changed, 33 insertions(+), 33 deletions(-) diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index a62a4b1f40b..24ca2315cc1 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -56,7 +56,7 @@ import { grabMetadataForGdocLinkedIndicator, TombstonePageData, gdocUrlRegex, - ChartViewMetadata, + NarrativeViewInfo, } from "@ourworldindata/utils" import { execWrapper } from "../db/execWrapper.js" import { countryProfileSpecs } from "../site/countryProfileProjects.js" @@ -110,7 +110,7 @@ import { getTombstones } from "../db/model/GdocTombstone.js" import { bakeAllMultiDimDataPages } from "./MultiDimBaker.js" import { getAllLinkedPublishedMultiDimDataPages } from "../db/model/MultiDimDataPage.js" import { getPublicDonorNames } from "../db/model/Donor.js" -import { getChartViewsMetadata } from "../db/model/ChartView.js" +import { getNarrativeViewsInfo } from "../db/model/ChartView.js" type PrefetchedAttachments = { donors: string[] @@ -122,7 +122,7 @@ type PrefetchedAttachments = { explorers: Record } linkedIndicators: Record - chartViewMetadata: Record + narrativeViewsInfo: Record } // These aren't all "wordpress" steps @@ -462,10 +462,10 @@ export class SiteBaker { name: `✅ Prefetched ${publishedAuthors.length} authors`, }) - const chartViewMetadata = await getChartViewsMetadata(knex) - const chartViewMetadataByName = keyBy(chartViewMetadata, "name") + const narrativeViewsInfo = await getNarrativeViewsInfo(knex) + const narrativeViewsInfoByName = keyBy(narrativeViewsInfo, "name") this.progressBar.tick({ - name: `✅ Prefetched ${chartViewMetadata.length} chart views`, + name: `✅ Prefetched ${narrativeViewsInfo.length} chart views`, }) const prefetchedAttachments = { @@ -478,7 +478,7 @@ export class SiteBaker { graphers: publishedChartsBySlug, }, linkedIndicators: datapageIndicatorsById, - chartViewMetadata: chartViewMetadataByName, + narrativeViewsInfo: narrativeViewsInfoByName, } this.progressBar.tick({ name: "✅ Prefetched attachments" }) this._prefetchedAttachmentsCache = prefetchedAttachments @@ -538,8 +538,8 @@ export class SiteBaker { this._prefetchedAttachmentsCache.linkedAuthors.filter( (author) => authorNames.includes(author.name) ), - chartViewMetadata: - this._prefetchedAttachmentsCache.chartViewMetadata, // TODO: Filter + narrativeViewsInfo: + this._prefetchedAttachmentsCache.narrativeViewsInfo, // TODO: Filter } } return this._prefetchedAttachmentsCache @@ -641,7 +641,7 @@ export class SiteBaker { ...attachments.linkedCharts.explorers, } publishedGdoc.linkedIndicators = attachments.linkedIndicators - publishedGdoc.chartViewMetadata = attachments.chartViewMetadata + publishedGdoc.narrativeViewsInfo = attachments.narrativeViewsInfo // this is a no-op if the gdoc doesn't have an all-chart block if ("loadRelatedCharts" in publishedGdoc) { diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index 7b310471018..52856e0b6ed 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -439,7 +439,7 @@ ${dataInsights latestDataInsights: get(post, "latestDataInsights", []), homepageMetadata: get(post, "homepageMetadata", {}), latestWorkLinks: get(post, "latestWorkLinks", []), - chartViewMetadata: get(post, "chartViewMetadata", {}), + narrativeViewsInfo: get(post, "narrativeViewsInfo", {}), }} > diff --git a/db/model/ChartView.ts b/db/model/ChartView.ts index 2113aab0d84..5a7e609694f 100644 --- a/db/model/ChartView.ts +++ b/db/model/ChartView.ts @@ -1,11 +1,11 @@ -import { ChartViewMetadata, JsonString } from "@ourworldindata/types" +import { NarrativeViewInfo, JsonString } from "@ourworldindata/types" import * as db from "../db.js" -export const getChartViewsMetadata = async ( +export const getNarrativeViewsInfo = async ( knex: db.KnexReadonlyTransaction, names?: string[] -): Promise => { - type RawRow = Omit & { +): Promise => { + type RawRow = Omit & { queryParamsForParentChart: JsonString } let rows: RawRow[] diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index 23ddfd187da..da815341f57 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -56,7 +56,7 @@ import { import { ARCHVED_THUMBNAIL_FILENAME, ChartConfigType, - ChartViewMetadata, + NarrativeViewInfo, DEFAULT_THUMBNAIL_FILENAME, GrapherInterface, LatestDataInsight, @@ -67,7 +67,7 @@ import { OwidGdocLinkType, OwidGdocType, } from "@ourworldindata/types" -import { getChartViewsMetadata } from "../ChartView.js" +import { getNarrativeViewsInfo } from "../ChartView.js" export class GdocBase implements OwidGdocBaseInterface { id!: string @@ -91,7 +91,7 @@ export class GdocBase implements OwidGdocBaseInterface { linkedIndicators: Record = {} linkedDocuments: Record = {} latestDataInsights: LatestDataInsight[] = [] - chartViewMetadata?: Record = {} + narrativeViewsInfo?: Record = {} _omittableFields: string[] = [] constructor(id?: string) { @@ -707,12 +707,12 @@ export class GdocBase implements OwidGdocBaseInterface { } } - async loadChartViewMetadata( + async loadNarrativeViewsInfo( knex: db.KnexReadonlyTransaction ): Promise { // TODO: Filter down to only those that are used in the Gdoc - const result = await getChartViewsMetadata(knex) - this.chartViewMetadata = keyBy(result, "name") + const result = await getNarrativeViewsInfo(knex) + this.narrativeViewsInfo = keyBy(result, "name") } async fetchAndEnrichGdoc(): Promise { @@ -860,7 +860,7 @@ export class GdocBase implements OwidGdocBaseInterface { await this.loadImageMetadataFromDB(knex) await this.loadLinkedCharts(knex) await this.loadLinkedIndicators() // depends on linked charts - await this.loadChartViewMetadata(knex) + await this.loadNarrativeViewsInfo(knex) await this._loadSubclassAttachments(knex) await this.validate(knex) } diff --git a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts index 9731db1b709..35187a30f3c 100644 --- a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts +++ b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts @@ -55,7 +55,7 @@ export interface LinkedChart { } // An object containing metadata needed for embedded narrative charts -export interface ChartViewMetadata { +export interface NarrativeViewInfo { name: string title: string chartConfigId: string diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts index 3be6749e83e..9e94a05ee01 100644 --- a/packages/@ourworldindata/types/src/index.ts +++ b/packages/@ourworldindata/types/src/index.ts @@ -332,7 +332,7 @@ export { type OwidGdocContent, type OwidGdocIndexItem, extractGdocIndexItem, - type ChartViewMetadata, + type NarrativeViewInfo, } from "./gdocTypes/Gdoc.js" export { diff --git a/site/gdocs/OwidGdoc.tsx b/site/gdocs/OwidGdoc.tsx index 9832d537911..f12ac4a0387 100644 --- a/site/gdocs/OwidGdoc.tsx +++ b/site/gdocs/OwidGdoc.tsx @@ -13,7 +13,7 @@ import { OwidGdocMinimalPostInterface, OwidGdocHomepageMetadata, DbEnrichedLatestWork, - ChartViewMetadata, + NarrativeViewInfo, } from "@ourworldindata/types" import { get, getOwidGdocFromJSON } from "@ourworldindata/utils" import { DebugProvider } from "./DebugContext.js" @@ -36,7 +36,7 @@ export type Attachments = { latestDataInsights?: LatestDataInsight[] homepageMetadata?: OwidGdocHomepageMetadata latestWorkLinks?: DbEnrichedLatestWork[] - chartViewMetadata?: Record + narrativeViewsInfo?: Record } export const AttachmentsContext = createContext({ @@ -49,7 +49,7 @@ export const AttachmentsContext = createContext({ latestDataInsights: [], homepageMetadata: {}, latestWorkLinks: [], - chartViewMetadata: {}, + narrativeViewsInfo: {}, }) export const DocumentContext = createContext<{ isPreviewing: boolean }>({ @@ -134,7 +134,7 @@ export function OwidGdoc({ latestDataInsights: get(props, "latestDataInsights", []), homepageMetadata: get(props, "homepageMetadata", {}), latestWorkLinks: get(props, "latestWorkLinks", []), - chartViewMetadata: get(props, "chartViewMetadata", {}), + narrativeViewsInfo: get(props, "narrativeViewsInfo", {}), }} > diff --git a/site/gdocs/components/NarrativeChart.tsx b/site/gdocs/components/NarrativeChart.tsx index 2f1cc74ddab..482360b68b0 100644 --- a/site/gdocs/components/NarrativeChart.tsx +++ b/site/gdocs/components/NarrativeChart.tsx @@ -1,7 +1,7 @@ import React, { useRef } from "react" import { useEmbedChart } from "../../hooks.js" import { EnrichedBlockNarrativeChart } from "@ourworldindata/types" -import { renderSpans, useChartViewMetadata } from "../utils.js" +import { renderSpans, useNarrativeViewsInfo } from "../utils.js" import cx from "classnames" import { GRAPHER_PREVIEW_CLASS } from "../../SiteConstants.js" import { BlockErrorFallback } from "./BlockErrorBoundary.js" @@ -18,7 +18,7 @@ export default function NarrativeChart({ const refChartContainer = useRef(null) useEmbedChart(0, refChartContainer) - const viewMetadata = useChartViewMetadata(d.name) + const viewMetadata = useNarrativeViewsInfo(d.name) if (!viewMetadata) return ( diff --git a/site/gdocs/utils.tsx b/site/gdocs/utils.tsx index e3a39c703a8..afb6af58134 100644 --- a/site/gdocs/utils.tsx +++ b/site/gdocs/utils.tsx @@ -147,9 +147,9 @@ export function useDonors(): string[] | undefined { return donors } -export const useChartViewMetadata = (name: string) => { - const { chartViewMetadata } = useContext(AttachmentsContext) - return chartViewMetadata?.[name] +export const useNarrativeViewsInfo = (name: string) => { + const { narrativeViewsInfo } = useContext(AttachmentsContext) + return narrativeViewsInfo?.[name] } const LinkedA = ({ span }: { span: SpanLink }): React.ReactElement => { From 4efe82c9971f8729a2f917ae340895da031a3b4d Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Tue, 17 Dec 2024 16:28:12 +0100 Subject: [PATCH 07/20] enhance: narrative views are reflected as links in gdocs --- db/model/Gdoc/GdocBase.ts | 13 +++++++----- db/model/Link.ts | 20 +++++++++++++++++++ .../types/src/gdocTypes/Gdoc.ts | 1 + 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index da815341f57..b7dbd69ab5f 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -48,7 +48,7 @@ import { getVariableMetadata, getVariableOfDatapageIfApplicable, } from "../Variable.js" -import { createLinkFromUrl } from "../Link.js" +import { createLinkForNarrativeChart, createLinkFromUrl } from "../Link.js" import { getMultiDimDataPageBySlug, isMultiDimDataPagePublished, @@ -341,6 +341,13 @@ export class GdocBase implements OwidGdocBaseInterface { componentType: block.type, }), ]) + .with({ type: "narrative-chart" }, (block) => [ + createLinkForNarrativeChart({ + name: block.name, + source: this, + componentType: block.type, + }), + ]) .with({ type: "all-charts" }, (block) => block.top.map((item) => createLinkFromUrl({ @@ -570,10 +577,6 @@ export class GdocBase implements OwidGdocBaseInterface { "key-indicator-collection", "list", "missing-data", - - // Open question: there's not a direct link to a chart here, but there is a chart and also a parent chart - "narrative-chart", - "numbered-list", "people", "people-rows", diff --git a/db/model/Link.ts b/db/model/Link.ts index 4468e6832dd..c2a62a16cc0 100644 --- a/db/model/Link.ts +++ b/db/model/Link.ts @@ -62,3 +62,23 @@ export function createLinkFromUrl({ sourceId: source.id, } satisfies DbInsertPostGdocLink } + +export function createLinkForNarrativeChart({ + name, + source, + componentType, +}: { + name: string + source: GdocBase + componentType: string +}): DbInsertPostGdocLink { + return { + target: name, + linkType: OwidGdocLinkType.NarrativeChart, + queryString: "", + hash: "", + text: "", + componentType, + sourceId: source.id, + } satisfies DbInsertPostGdocLink +} diff --git a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts index 35187a30f3c..9f75af5e70d 100644 --- a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts +++ b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts @@ -281,6 +281,7 @@ export enum OwidGdocLinkType { Url = "url", Grapher = "grapher", Explorer = "explorer", + NarrativeChart = "narrative-chart", } export interface OwidGdocLinkJSON { From 41867abb90b047b0befbe1c47b2f7e5569e83676 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Tue, 17 Dec 2024 16:38:01 +0100 Subject: [PATCH 08/20] refactor: filter down `narrativeViewsInfo` --- baker/SiteBaker.tsx | 12 +++++++++--- db/model/Gdoc/GdocBase.ts | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index 24ca2315cc1..29dc2c2c934 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -348,7 +348,7 @@ export class SiteBaker { _prefetchedAttachmentsCache: PrefetchedAttachments | undefined = undefined private async getPrefetchedGdocAttachments( knex: db.KnexReadonlyTransaction, - picks?: [string[], string[], string[], string[], string[]] + picks?: [string[], string[], string[], string[], string[], string[]] ): Promise { if (!this._prefetchedAttachmentsCache) { console.log("Prefetching attachments...") @@ -490,6 +490,7 @@ export class SiteBaker { imageFilenames, linkedGrapherSlugs, linkedExplorerSlugs, + linkedNarrativeChartNames, ] = picks const linkedDocuments = pick( this._prefetchedAttachmentsCache.linkedDocuments, @@ -538,8 +539,10 @@ export class SiteBaker { this._prefetchedAttachmentsCache.linkedAuthors.filter( (author) => authorNames.includes(author.name) ), - narrativeViewsInfo: - this._prefetchedAttachmentsCache.narrativeViewsInfo, // TODO: Filter + narrativeViewsInfo: pick( + this._prefetchedAttachmentsCache.narrativeViewsInfo, + linkedNarrativeChartNames + ), } } return this._prefetchedAttachmentsCache @@ -631,6 +634,7 @@ export class SiteBaker { publishedGdoc.linkedImageFilenames, publishedGdoc.linkedChartSlugs.grapher, publishedGdoc.linkedChartSlugs.explorer, + publishedGdoc.linkedNarrativeChartNames, ]) publishedGdoc.donors = attachments.donors publishedGdoc.linkedAuthors = attachments.linkedAuthors @@ -889,6 +893,7 @@ export class SiteBaker { dataInsight.linkedImageFilenames, dataInsight.linkedChartSlugs.grapher, dataInsight.linkedChartSlugs.explorer, + dataInsight.linkedNarrativeChartNames, ]) dataInsight.linkedDocuments = attachments.linkedDocuments dataInsight.imageMetadata = { @@ -962,6 +967,7 @@ export class SiteBaker { publishedAuthor.linkedImageFilenames, publishedAuthor.linkedChartSlugs.grapher, publishedAuthor.linkedChartSlugs.explorer, + publishedAuthor.linkedNarrativeChartNames, ]) // We don't need these to be attached to the gdoc in the current diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index b7dbd69ab5f..9d1ebf9cf7e 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -295,6 +295,14 @@ export class GdocBase implements OwidGdocBaseInterface { return { grapher: [...grapher], explorer: [...explorer] } } + get linkedNarrativeChartNames(): string[] { + const filteredLinks = this.links + .filter((link) => link.linkType === "narrative-chart") + .map((link) => link.target) + + return filteredLinks + } + get hasAllChartsBlock(): boolean { let hasAllChartsBlock = false for (const enrichedBlockSource of this.enrichedBlockSources) { @@ -713,8 +721,10 @@ export class GdocBase implements OwidGdocBaseInterface { async loadNarrativeViewsInfo( knex: db.KnexReadonlyTransaction ): Promise { - // TODO: Filter down to only those that are used in the Gdoc - const result = await getNarrativeViewsInfo(knex) + const result = await getNarrativeViewsInfo( + knex, + this.linkedNarrativeChartNames + ) this.narrativeViewsInfo = keyBy(result, "name") } From 242bd2f4d7ee0ff7cd8a93968e2c9167114b617b Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Tue, 17 Dec 2024 18:06:54 +0100 Subject: [PATCH 09/20] refactor: change `linkType` enum --- ...4799588-PostsGdocsLinksAddNarrativeCharts.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 db/migration/1734454799588-PostsGdocsLinksAddNarrativeCharts.ts diff --git a/db/migration/1734454799588-PostsGdocsLinksAddNarrativeCharts.ts b/db/migration/1734454799588-PostsGdocsLinksAddNarrativeCharts.ts new file mode 100644 index 00000000000..cb1d710cf9c --- /dev/null +++ b/db/migration/1734454799588-PostsGdocsLinksAddNarrativeCharts.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from "typeorm" + +export class PostsGdocsLinksAddNarrativeCharts1734454799588 + implements MigrationInterface +{ + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE posts_gdocs_links + MODIFY linkType ENUM ('gdoc', 'url', 'grapher', 'explorer', 'narrative-chart') NULL`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE posts_gdocs_links + MODIFY linkType ENUM ('gdoc', 'url', 'grapher', 'explorer') NULL`) + } +} From c7ed3a1c9012968c1e25e8e10a9cf4ecebeb13aa Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Wed, 18 Dec 2024 20:28:11 +0100 Subject: [PATCH 10/20] fix: fix error when publishing NarrativeChart with error --- site/gdocs/components/NarrativeChart.tsx | 29 ++++++++++++++---------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/site/gdocs/components/NarrativeChart.tsx b/site/gdocs/components/NarrativeChart.tsx index 482360b68b0..7cb3311515c 100644 --- a/site/gdocs/components/NarrativeChart.tsx +++ b/site/gdocs/components/NarrativeChart.tsx @@ -1,10 +1,11 @@ -import React, { useRef } from "react" +import React, { useContext, useRef } from "react" import { useEmbedChart } from "../../hooks.js" import { EnrichedBlockNarrativeChart } from "@ourworldindata/types" import { renderSpans, useNarrativeViewsInfo } from "../utils.js" import cx from "classnames" import { GRAPHER_PREVIEW_CLASS } from "../../SiteConstants.js" import { BlockErrorFallback } from "./BlockErrorBoundary.js" +import { DocumentContext } from "../OwidGdoc.js" export default function NarrativeChart({ d, @@ -20,16 +21,21 @@ export default function NarrativeChart({ const viewMetadata = useNarrativeViewsInfo(d.name) - if (!viewMetadata) - return ( - - ) + const { isPreviewing } = useContext(DocumentContext) + + if (!viewMetadata) { + if (isPreviewing) { + return ( + + ) + } else return null // If not previewing, just don't render anything + } const metadataStringified = JSON.stringify(viewMetadata) @@ -45,7 +51,6 @@ export default function NarrativeChart({ key={metadataStringified} className={cx(GRAPHER_PREVIEW_CLASS, "chart")} data-grapher-view-config={metadataStringified} - // data-grapher-src={isExplorer ? undefined : resolvedUrl} style={{ width: "100%", border: "0px none", From b5b24a68068e3c8e8a660f8314f661def6b78c11 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Wed, 18 Dec 2024 22:09:56 +0100 Subject: [PATCH 11/20] enhance: add narrative chart support to MultiEmbedder --- .../grapher/src/core/GrapherConstants.ts | 3 + packages/@ourworldindata/grapher/src/index.ts | 1 + site/multiembedder/MultiEmbedder.tsx | 283 +++++++++++------- 3 files changed, 180 insertions(+), 107 deletions(-) diff --git a/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts b/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts index 1601f36f65e..a4f8a896827 100644 --- a/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts +++ b/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts @@ -5,6 +5,9 @@ import type { GrapherProgrammaticInterface } from "./Grapher" export const GRAPHER_EMBEDDED_FIGURE_ATTR = "data-grapher-src" export const GRAPHER_EMBEDDED_FIGURE_CONFIG_ATTR = "data-grapher-config" +export const GRAPHER_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR = + "data-grapher-view-config" + export const GRAPHER_PAGE_BODY_CLASS = "StandaloneGrapherOrExplorerPage" export const GRAPHER_IS_IN_IFRAME_CLASS = "IsInIframe" export const GRAPHER_TIMELINE_CLASS = "timeline-component" diff --git a/packages/@ourworldindata/grapher/src/index.ts b/packages/@ourworldindata/grapher/src/index.ts index b9b100cdf1e..8815ad4a847 100644 --- a/packages/@ourworldindata/grapher/src/index.ts +++ b/packages/@ourworldindata/grapher/src/index.ts @@ -10,6 +10,7 @@ export { ChartDimension } from "./chart/ChartDimension" export { GRAPHER_EMBEDDED_FIGURE_ATTR, GRAPHER_EMBEDDED_FIGURE_CONFIG_ATTR, + GRAPHER_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR, GRAPHER_PAGE_BODY_CLASS, GRAPHER_IS_IN_IFRAME_CLASS, DEFAULT_GRAPHER_WIDTH, diff --git a/site/multiembedder/MultiEmbedder.tsx b/site/multiembedder/MultiEmbedder.tsx index 3abc824771e..b1960ca5a2d 100644 --- a/site/multiembedder/MultiEmbedder.tsx +++ b/site/multiembedder/MultiEmbedder.tsx @@ -10,6 +10,7 @@ import { migrateSelectedEntityNamesParam, SelectionArray, migrateGrapherConfigToLatestVersion, + GRAPHER_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR, } from "@ourworldindata/grapher" import { fetchText, @@ -21,6 +22,7 @@ import { MultiDimDataPageConfig, extractMultiDimChoicesFromQueryStr, fetchWithRetry, + NarrativeViewInfo, } from "@ourworldindata/utils" import { action } from "mobx" import React from "react" @@ -42,6 +44,9 @@ import { } from "../../settings/clientSettings.js" import Bugsnag from "@bugsnag/js" import { embedDynamicCollectionGrapher } from "../collections/DynamicCollection.js" +import { match } from "ts-pattern" + +type EmbedType = "grapher" | "explorer" | "multiDim" | "grapherView" const figuresFromDOM = ( container: HTMLElement | Document = document, @@ -110,10 +115,16 @@ class MultiEmbedder { * Use this when you programmatically create/replace charts. */ observeFigures(container: HTMLElement | Document = document) { - const figures = figuresFromDOM( - container, - GRAPHER_EMBEDDED_FIGURE_ATTR - ).concat(figuresFromDOM(container, EXPLORER_EMBEDDED_FIGURE_SELECTOR)) + const figures = figuresFromDOM(container, GRAPHER_EMBEDDED_FIGURE_ATTR) + .concat( + figuresFromDOM(container, EXPLORER_EMBEDDED_FIGURE_SELECTOR) + ) + .concat( + figuresFromDOM( + container, + GRAPHER_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR + ) + ) figures.forEach((figure) => { this.figuresObserver?.observe(figure) @@ -128,33 +139,41 @@ class MultiEmbedder { }) } - @action.bound - async renderInteractiveFigure(figure: Element) { - const isExplorer = figure.hasAttribute( + async renderExplorerIntoFigure(figure: Element) { + const explorerUrl = figure.getAttribute( EXPLORER_EMBEDDED_FIGURE_SELECTOR ) - const isMultiDim = figure.hasAttribute("data-is-multi-dim") - const dataSrc = figure.getAttribute( - isExplorer - ? EXPLORER_EMBEDDED_FIGURE_SELECTOR - : GRAPHER_EMBEDDED_FIGURE_ATTR - ) + if (!explorerUrl) return - if (!dataSrc) return + const { fullUrl, queryStr } = Url.fromURL(explorerUrl) - const hasPreview = isExplorer ? false : !!figure.querySelector("img") - if (!shouldProgressiveEmbed() && hasPreview) return - - // Stop observing visibility as soon as possible, that is not before - // shouldProgressiveEmbed gets a chance to reevaluate a possible change - // in screen size on mobile (i.e. after a rotation). Stopping before - // shouldProgressiveEmbed would prevent rendering interactive charts - // when going from portrait to landscape mode (without page reload). - this.figuresObserver?.unobserve(figure) + const html = await fetchText(fullUrl) + const props: ExplorerProps = await buildExplorerProps( + html, + queryStr, + this.selection + ) + if (props.selection) + this.graphersAndExplorersToUpdate.add(props.selection) + ReactDOM.render(, figure) + } - const { fullUrl, queryStr, queryParams } = Url.fromURL(dataSrc) + private async _renderGrapherComponentIntoFigure( + figure: Element, + { + configUrl, + embedUrl, + additionalConfig, + }: { + configUrl: string + embedUrl?: Url + additionalConfig?: Partial + } + ) { + const { queryStr, queryParams } = embedUrl ?? {} + figure.classList.remove(GRAPHER_PREVIEW_CLASS) const common: GrapherProgrammaticInterface = { isEmbeddedInAnOwidPage: true, queryStr, @@ -163,95 +182,145 @@ class MultiEmbedder { dataApiUrl: DATA_API_URL, } - if (isExplorer) { - const html = await fetchText(fullUrl) - const props: ExplorerProps = await buildExplorerProps( - html, - queryStr, - this.selection - ) - if (props.selection) - this.graphersAndExplorersToUpdate.add(props.selection) - ReactDOM.render(, figure) - } else { - figure.classList.remove(GRAPHER_PREVIEW_CLASS) - const url = new URL(fullUrl) - const slug = url.pathname.split("/").pop() - let configUrl - if (isMultiDim) { - const mdimConfigUrl = `${MULTI_DIM_DYNAMIC_CONFIG_URL}/${slug}.json` - const mdimJsonConfig = await fetchWithRetry(mdimConfigUrl).then( - (res) => res.json() - ) - const mdimConfig = - MultiDimDataPageConfig.fromObject(mdimJsonConfig) - const dimensions = extractMultiDimChoicesFromQueryStr( - url.search, - mdimConfig - ) - const view = mdimConfig.findViewByDimensions(dimensions) - if (!view) { - throw new Error( - `No view found for dimensions ${JSON.stringify( - dimensions - )}` - ) - } - configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/by-uuid/${view.fullConfigId}.config.json` - } else { - configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/${slug}.config.json` - } - const fetchedGrapherPageConfig = await fetchWithRetry( - configUrl - ).then((res) => res.json()) - const grapherPageConfig = migrateGrapherConfigToLatestVersion( - fetchedGrapherPageConfig - ) + const fetchedGrapherPageConfig = await fetchWithRetry(configUrl).then( + (res) => res.json() + ) + const grapherPageConfig = migrateGrapherConfigToLatestVersion( + fetchedGrapherPageConfig + ) - const figureConfigAttr = figure.getAttribute( - GRAPHER_EMBEDDED_FIGURE_CONFIG_ATTR - ) - const localConfig = figureConfigAttr - ? JSON.parse(figureConfigAttr) - : {} - - // make sure the tab of the active pane is visible - if (figureConfigAttr && !isEmpty(localConfig)) { - const activeTab = queryParams.tab || grapherPageConfig.tab - if (activeTab === GRAPHER_TAB_OPTIONS.chart) - localConfig.hideChartTabs = false - if (activeTab === GRAPHER_TAB_OPTIONS.map) - localConfig.hasMapTab = true - if (activeTab === GRAPHER_TAB_OPTIONS.table) - localConfig.hasTableTab = true + const figureConfigAttr = figure.getAttribute( + GRAPHER_EMBEDDED_FIGURE_CONFIG_ATTR + ) + const localConfig = figureConfigAttr ? JSON.parse(figureConfigAttr) : {} + + // make sure the tab of the active pane is visible + if (figureConfigAttr && !isEmpty(localConfig)) { + const activeTab = queryParams?.tab || grapherPageConfig.tab + if (activeTab === GRAPHER_TAB_OPTIONS.chart) + localConfig.hideChartTabs = false + if (activeTab === GRAPHER_TAB_OPTIONS.map) + localConfig.hasMapTab = true + if (activeTab === GRAPHER_TAB_OPTIONS.table) + localConfig.hasTableTab = true + } + + const config = merge( + {}, // merge mutates the first argument + grapherPageConfig, + common, + additionalConfig, + localConfig, + { + manager: { + selection: new SelectionArray( + this.selection.selectedEntityNames + ), + }, } + ) + if (config.manager?.selection) + this.graphersAndExplorersToUpdate.add(config.manager.selection) - const config = merge( - {}, // merge mutates the first argument - grapherPageConfig, - common, - localConfig, - { - manager: { - selection: new SelectionArray( - this.selection.selectedEntityNames - ), - }, - } - ) - if (config.manager?.selection) - this.graphersAndExplorersToUpdate.add(config.manager.selection) + const grapherRef = Grapher.renderGrapherIntoContainer(config, figure) - const grapherRef = Grapher.renderGrapherIntoContainer( - config, - figure - ) + // Special handling for shared collections + if (window.location.pathname.startsWith("/collection/custom")) { + embedDynamicCollectionGrapher(grapherRef, figure) + } + } + async renderGrapherIntoFigure(figure: Element) { + const embedUrlRaw = figure.getAttribute(GRAPHER_EMBEDDED_FIGURE_ATTR) + if (!embedUrlRaw) return + const embedUrl = Url.fromURL(embedUrlRaw) - // Special handling for shared collections - if (window.location.pathname.startsWith("/collection/custom")) { - embedDynamicCollectionGrapher(grapherRef, figure) - } + const configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/${embedUrl.slug}.config.json` + + await this._renderGrapherComponentIntoFigure(figure, { + configUrl, + embedUrl, + }) + } + async renderMultiDimIntoFigure(figure: Element) { + const embedUrlRaw = figure.getAttribute(GRAPHER_EMBEDDED_FIGURE_ATTR) + if (!embedUrlRaw) return + const embedUrl = Url.fromURL(embedUrlRaw) + + const { queryStr, slug } = embedUrl + + const mdimConfigUrl = `${MULTI_DIM_DYNAMIC_CONFIG_URL}/${slug}.json` + const mdimJsonConfig = await fetchWithRetry(mdimConfigUrl).then((res) => + res.json() + ) + const mdimConfig = MultiDimDataPageConfig.fromObject(mdimJsonConfig) + const dimensions = extractMultiDimChoicesFromQueryStr( + queryStr, + mdimConfig + ) + const view = mdimConfig.findViewByDimensions(dimensions) + if (!view) { + throw new Error( + `No view found for dimensions ${JSON.stringify(dimensions)}` + ) } + + const configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/by-uuid/${view.fullConfigId}.config.json` + + await this._renderGrapherComponentIntoFigure(figure, { + configUrl, + embedUrl, + }) + } + async renderGrapherViewIntoFigure(figure: Element) { + const viewConfigRaw = figure.getAttribute( + GRAPHER_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR + ) + if (!viewConfigRaw) return + const viewConfig: NarrativeViewInfo = JSON.parse(viewConfigRaw) + if (!viewConfig) return + + const configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/by-uuid/${viewConfig.chartConfigId}.config.json` + + await this._renderGrapherComponentIntoFigure(figure, { + configUrl, + additionalConfig: {}, + }) + } + + @action.bound + async renderInteractiveFigure(figure: Element) { + const isExplorer = figure.hasAttribute( + EXPLORER_EMBEDDED_FIGURE_SELECTOR + ) + const isMultiDim = figure.hasAttribute("data-is-multi-dim") + const isGrapherView = figure.hasAttribute( + GRAPHER_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR + ) + + const embedType: EmbedType = isExplorer + ? "explorer" + : isMultiDim + ? "multiDim" + : isGrapherView + ? "grapherView" + : "grapher" + + const hasPreview = isExplorer ? false : !!figure.querySelector("img") + if (!shouldProgressiveEmbed() && hasPreview) return + + // Stop observing visibility as soon as possible, that is not before + // shouldProgressiveEmbed gets a chance to reevaluate a possible change + // in screen size on mobile (i.e. after a rotation). Stopping before + // shouldProgressiveEmbed would prevent rendering interactive charts + // when going from portrait to landscape mode (without page reload). + this.figuresObserver?.unobserve(figure) + + await match(embedType) + .with("explorer", () => this.renderExplorerIntoFigure(figure)) + .with("multiDim", () => this.renderMultiDimIntoFigure(figure)) + .with("grapherView", () => this.renderGrapherViewIntoFigure(figure)) + .with("grapher", () => this.renderGrapherIntoFigure(figure)) + .exhaustive() } setUpGlobalEntitySelectorForEmbeds() { From cb37196e0af9c14f78ef7bdf3dabee6070672fed Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Wed, 18 Dec 2024 22:35:40 +0100 Subject: [PATCH 12/20] enhance: basic config to hide some grapher elements --- site/multiembedder/MultiEmbedder.tsx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/site/multiembedder/MultiEmbedder.tsx b/site/multiembedder/MultiEmbedder.tsx index b1960ca5a2d..bf7bf2e7cc2 100644 --- a/site/multiembedder/MultiEmbedder.tsx +++ b/site/multiembedder/MultiEmbedder.tsx @@ -23,6 +23,7 @@ import { extractMultiDimChoicesFromQueryStr, fetchWithRetry, NarrativeViewInfo, + queryParamsToStr, } from "@ourworldindata/utils" import { action } from "mobx" import React from "react" @@ -281,9 +282,18 @@ class MultiEmbedder { const configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/by-uuid/${viewConfig.chartConfigId}.config.json` + const queryStr = queryParamsToStr(viewConfig.queryParamsForParentChart) + await this._renderGrapherComponentIntoFigure(figure, { configUrl, - additionalConfig: {}, + additionalConfig: { + hideRelatedQuestion: true, + hideShareButton: true, // always hidden since the original chart would be shared, not the customized one + hideExploreTheDataButton: false, + manager: { + canonicalUrl: `${BAKED_GRAPHER_URL}/${viewConfig.parentChartSlug}${queryStr}`, + }, + }, }) } From afa899c881e66d4994cf159d6cafffa49cfac4b6 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Wed, 18 Dec 2024 22:45:01 +0100 Subject: [PATCH 13/20] fix: correctly generate narrative view query params --- adminSiteServer/apiRouter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index 945d6b66700..109e31e3bd8 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -3637,7 +3637,7 @@ const createPatchConfigAndQueryParamsForChartView = async ( ...pick(fullConfigIncludingDefaults, CHART_VIEW_PROPS_TO_PERSIST), } - const queryParams = grapherConfigToQueryParams(config) + const queryParams = grapherConfigToQueryParams(patchConfigToSave) const fullConfig = mergeGrapherConfigs(parentChartConfig, patchConfigToSave) return { patchConfig: patchConfigToSave, fullConfig, queryParams } From 5a834f87d8abd051582927eb4d453585947901e9 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Thu, 19 Dec 2024 18:18:10 +0100 Subject: [PATCH 14/20] refactor: use consistent names -- chart views & narrative charts --- adminSiteClient/AdminSidebar.tsx | 2 +- adminSiteClient/ChartEditor.ts | 4 +-- adminSiteClient/ChartViewIndexPage.tsx | 2 +- adminSiteClient/EditorReferencesTab.tsx | 2 +- adminSiteClient/SaveButtons.tsx | 8 ++--- baker/SiteBaker.tsx | 30 +++++++++---------- baker/siteRenderers.tsx | 2 +- ...454799588-PostsGdocsLinksAddChartViews.ts} | 4 +-- db/model/ChartView.ts | 8 ++--- db/model/Gdoc/GdocBase.ts | 27 +++++++---------- db/model/Link.ts | 4 +-- .../grapher/src/core/GrapherConstants.ts | 4 +-- packages/@ourworldindata/grapher/src/index.ts | 2 +- .../types/src/gdocTypes/Gdoc.ts | 4 +-- packages/@ourworldindata/types/src/index.ts | 2 +- site/gdocs/OwidGdoc.tsx | 8 ++--- site/gdocs/components/NarrativeChart.tsx | 14 +++++---- site/gdocs/utils.tsx | 6 ++-- site/multiembedder/MultiEmbedder.tsx | 24 +++++++-------- 19 files changed, 78 insertions(+), 79 deletions(-) rename db/migration/{1734454799588-PostsGdocsLinksAddNarrativeCharts.ts => 1734454799588-PostsGdocsLinksAddChartViews.ts} (85%) diff --git a/adminSiteClient/AdminSidebar.tsx b/adminSiteClient/AdminSidebar.tsx index cb5e5eecf53..96eccd26fa4 100644 --- a/adminSiteClient/AdminSidebar.tsx +++ b/adminSiteClient/AdminSidebar.tsx @@ -37,7 +37,7 @@ export const AdminSidebar = (): React.ReactElement => ( {chartViewsFeatureEnabled && (
  • - Narrative views + Narrative charts
    • )} diff --git a/adminSiteClient/ChartEditor.ts b/adminSiteClient/ChartEditor.ts index 1a4747fbb39..25b10df424d 100644 --- a/adminSiteClient/ChartEditor.ts +++ b/adminSiteClient/ChartEditor.ts @@ -200,7 +200,7 @@ export class ChartEditor extends AbstractChartEditor { ) } - async saveAsNarrativeView(): Promise { + async saveAsChartView(): Promise { const { patchConfig, grapher } = this const chartJson = omit(patchConfig, CHART_VIEW_PROPS_TO_OMIT) @@ -208,7 +208,7 @@ export class ChartEditor extends AbstractChartEditor { const suggestedName = grapher.title ? slugify(grapher.title) : undefined const name = prompt( - "Please enter a programmatic name for the narrative view. Note that this name cannot be changed later.", + "Please enter a programmatic name for the narrative chart. Note that this name cannot be changed later.", suggestedName ) diff --git a/adminSiteClient/ChartViewIndexPage.tsx b/adminSiteClient/ChartViewIndexPage.tsx index fca45f0728d..267bd50c10b 100644 --- a/adminSiteClient/ChartViewIndexPage.tsx +++ b/adminSiteClient/ChartViewIndexPage.tsx @@ -134,7 +134,7 @@ export function ChartViewIndexPage() { }, [admin]) return ( - +
    -

    Narrative views based on this chart

    +

    Narrative charts based on this chart