From 1b1628d4f1bee680bfa7f4c071d6d8096f1f8af0 Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Fri, 20 Dec 2024 15:33:00 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A8=20refactor=20request=20handler=20l?= =?UTF-8?q?ambdas=20to=20named=20functions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- adminSiteServer/apiRoutes/bulkUpdates.ts | 370 +++++----- adminSiteServer/apiRoutes/chartViews.ts | 238 +++---- adminSiteServer/apiRoutes/charts.ts | 399 ++++++----- adminSiteServer/apiRoutes/datasets.ts | 720 ++++++++++---------- adminSiteServer/apiRoutes/explorer.ts | 55 +- adminSiteServer/apiRoutes/gdocs.ts | 110 +-- adminSiteServer/apiRoutes/images.ts | 91 ++- adminSiteServer/apiRoutes/mdims.ts | 45 +- adminSiteServer/apiRoutes/misc.ts | 110 +-- adminSiteServer/apiRoutes/posts.ts | 250 +++---- adminSiteServer/apiRoutes/redirects.ts | 239 ++++--- adminSiteServer/apiRoutes/suggest.ts | 104 +-- adminSiteServer/apiRoutes/tagGraph.ts | 35 +- adminSiteServer/apiRoutes/tags.ts | 343 +++++----- adminSiteServer/apiRoutes/users.ts | 214 +++--- adminSiteServer/apiRoutes/variables.ts | 826 +++++++++++++---------- 16 files changed, 2267 insertions(+), 1882 deletions(-) diff --git a/adminSiteServer/apiRoutes/bulkUpdates.ts b/adminSiteServer/apiRoutes/bulkUpdates.ts index 4dbb3cc902..364146238c 100644 --- a/adminSiteServer/apiRoutes/bulkUpdates.ts +++ b/adminSiteServer/apiRoutes/bulkUpdates.ts @@ -30,35 +30,34 @@ import { saveGrapher } from "./charts.js" import * as db from "../../db/db.js" import * as lodash from "lodash" import { apiRouter } from "../apiRouter.js" +import { Request } from "../authentication.js" +import e from "express" -getRouteWithROTransaction( - apiRouter, - "/chart-bulk-update", - async ( - req, - res, - trx - ): Promise> => { - const context: OperationContext = { - grapherConfigFieldName: "chart_configs.full", - whitelistedColumnNamesAndTypes: - chartBulkUpdateAllowedColumnNamesAndTypes, - } - const filterSExpr = - req.query.filter !== undefined - ? parseToOperation(req.query.filter as string, context) - : undefined +export async function getChartBulkUpdate( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +): Promise> { + const context: OperationContext = { + grapherConfigFieldName: "chart_configs.full", + whitelistedColumnNamesAndTypes: + chartBulkUpdateAllowedColumnNamesAndTypes, + } + const filterSExpr = + req.query.filter !== undefined + ? parseToOperation(req.query.filter as string, context) + : undefined - const offset = parseIntOrUndefined(req.query.offset as string) ?? 0 + const offset = parseIntOrUndefined(req.query.offset as string) ?? 0 - // Note that our DSL generates sql here that we splice directly into the SQL as text - // This is a potential for a SQL injection attack but we control the DSL and are - // careful there to only allow carefully guarded vocabularies from being used, not - // arbitrary user input - const whereClause = filterSExpr?.toSql() ?? "true" - const resultsWithStringGrapherConfigs = await db.knexRaw( - trx, - `-- sql + // Note that our DSL generates sql here that we splice directly into the SQL as text + // This is a potential for a SQL injection attack but we control the DSL and are + // careful there to only allow carefully guarded vocabularies from being used, not + // arbitrary user input + const whereClause = filterSExpr?.toSql() ?? "true" + const resultsWithStringGrapherConfigs = await db.knexRaw( + trx, + `-- sql SELECT charts.id as id, chart_configs.full as config, @@ -77,180 +76,191 @@ getRouteWithROTransaction( LIMIT 50 OFFSET ${offset.toString()} ` - ) + ) - const results = resultsWithStringGrapherConfigs.map((row: any) => ({ - ...row, - config: lodash.isNil(row.config) ? null : JSON.parse(row.config), - })) - const resultCount = await db.knexRaw<{ count: number }>( - trx, - `-- sql + const results = resultsWithStringGrapherConfigs.map((row: any) => ({ + ...row, + config: lodash.isNil(row.config) ? null : JSON.parse(row.config), + })) + const resultCount = await db.knexRaw<{ count: number }>( + trx, + `-- sql SELECT count(*) as count FROM charts JOIN chart_configs ON chart_configs.id = charts.configId WHERE ${whereClause} ` - ) - return { rows: results, numTotalRows: resultCount[0].count } - } -) + ) + return { rows: results, numTotalRows: resultCount[0].count } +} -patchRouteWithRWTransaction( - apiRouter, - "/chart-bulk-update", - async (req, res, trx) => { - const patchesList = req.body as GrapherConfigPatch[] - const chartIds = new Set(patchesList.map((patch) => patch.id)) +export async function updateBulkChartConfigs( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const patchesList = req.body as GrapherConfigPatch[] + const chartIds = new Set(patchesList.map((patch) => patch.id)) - const configsAndIds = await db.knexRaw< - Pick & { config: DbRawChartConfig["full"] } - >( - trx, - `-- sql - SELECT c.id, cc.full as config - FROM charts c - JOIN chart_configs cc ON cc.id = c.configId - WHERE c.id IN (?) - `, - [[...chartIds.values()]] - ) - const configMap = new Map( - configsAndIds.map((item: any) => [ - item.id, - // make sure that the id is set, otherwise the update behaviour is weird - // TODO: discuss if this has unintended side effects - item.config ? { ...JSON.parse(item.config), id: item.id } : {}, - ]) - ) - const oldValuesConfigMap = new Map(configMap) - // console.log("ids", configsAndIds.map((item : any) => item.id)) - for (const patchSet of patchesList) { - const config = configMap.get(patchSet.id) - configMap.set(patchSet.id, applyPatch(patchSet, config)) - } + const configsAndIds = await db.knexRaw< + Pick & { config: DbRawChartConfig["full"] } + >( + trx, + `-- sql + SELECT c.id, cc.full as config + FROM charts c + JOIN chart_configs cc ON cc.id = c.configId + WHERE c.id IN (?) + `, + [[...chartIds.values()]] + ) + const configMap = new Map( + configsAndIds.map((item: any) => [ + item.id, + // make sure that the id is set, otherwise the update behaviour is weird + // TODO: discuss if this has unintended side effects + item.config ? { ...JSON.parse(item.config), id: item.id } : {}, + ]) + ) + const oldValuesConfigMap = new Map(configMap) + // console.log("ids", configsAndIds.map((item : any) => item.id)) + for (const patchSet of patchesList) { + const config = configMap.get(patchSet.id) + configMap.set(patchSet.id, applyPatch(patchSet, config)) + } - for (const [id, newConfig] of configMap.entries()) { - await saveGrapher(trx, { - user: res.locals.user, - newConfig, - existingConfig: oldValuesConfigMap.get(id), - referencedVariablesMightChange: false, - }) - } + for (const [id, newConfig] of configMap.entries()) { + await saveGrapher(trx, { + user: res.locals.user, + newConfig, + existingConfig: oldValuesConfigMap.get(id), + referencedVariablesMightChange: false, + }) + } + + return { success: true } +} - return { success: true } +export async function getVariableAnnotations( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +): Promise> { + const context: OperationContext = { + grapherConfigFieldName: "grapherConfigAdmin", + whitelistedColumnNamesAndTypes: + variableAnnotationAllowedColumnNamesAndTypes, } -) + const filterSExpr = + req.query.filter !== undefined + ? parseToOperation(req.query.filter as string, context) + : undefined -getRouteWithROTransaction( - apiRouter, - "/variable-annotations", - async ( - req, - res, - trx - ): Promise> => { - const context: OperationContext = { - grapherConfigFieldName: "grapherConfigAdmin", - whitelistedColumnNamesAndTypes: - variableAnnotationAllowedColumnNamesAndTypes, - } - const filterSExpr = - req.query.filter !== undefined - ? parseToOperation(req.query.filter as string, context) - : undefined + const offset = parseIntOrUndefined(req.query.offset as string) ?? 0 - const offset = parseIntOrUndefined(req.query.offset as string) ?? 0 + // Note that our DSL generates sql here that we splice directly into the SQL as text + // This is a potential for a SQL injection attack but we control the DSL and are + // careful there to only allow carefully guarded vocabularies from being used, not + // arbitrary user input + const whereClause = filterSExpr?.toSql() ?? "true" + const resultsWithStringGrapherConfigs = await db.knexRaw( + trx, + `-- sql + SELECT + variables.id as id, + variables.name as name, + chart_configs.patch as config, + d.name as datasetname, + namespaces.name as namespacename, + variables.createdAt as createdAt, + variables.updatedAt as updatedAt, + variables.description as description + FROM variables + LEFT JOIN active_datasets as d on variables.datasetId = d.id + LEFT JOIN namespaces on d.namespace = namespaces.name + LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id + WHERE ${whereClause} + ORDER BY variables.id DESC + LIMIT 50 + OFFSET ${offset.toString()} + ` + ) - // Note that our DSL generates sql here that we splice directly into the SQL as text - // This is a potential for a SQL injection attack but we control the DSL and are - // careful there to only allow carefully guarded vocabularies from being used, not - // arbitrary user input - const whereClause = filterSExpr?.toSql() ?? "true" - const resultsWithStringGrapherConfigs = await db.knexRaw( - trx, - `-- sql - SELECT - variables.id as id, - variables.name as name, - chart_configs.patch as config, - d.name as datasetname, - namespaces.name as namespacename, - variables.createdAt as createdAt, - variables.updatedAt as updatedAt, - variables.description as description - FROM variables - LEFT JOIN active_datasets as d on variables.datasetId = d.id - LEFT JOIN namespaces on d.namespace = namespaces.name - LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id - WHERE ${whereClause} - ORDER BY variables.id DESC - LIMIT 50 - OFFSET ${offset.toString()} - ` - ) + const results = resultsWithStringGrapherConfigs.map((row: any) => ({ + ...row, + config: lodash.isNil(row.config) ? null : JSON.parse(row.config), + })) + const resultCount = await db.knexRaw<{ count: number }>( + trx, + `-- sql + SELECT count(*) as count + FROM variables + LEFT JOIN active_datasets as d on variables.datasetId = d.id + LEFT JOIN namespaces on d.namespace = namespaces.name + LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id + WHERE ${whereClause} + ` + ) + return { rows: results, numTotalRows: resultCount[0].count } +} - const results = resultsWithStringGrapherConfigs.map((row: any) => ({ - ...row, - config: lodash.isNil(row.config) ? null : JSON.parse(row.config), - })) - const resultCount = await db.knexRaw<{ count: number }>( - trx, - `-- sql - SELECT count(*) as count - FROM variables - LEFT JOIN active_datasets as d on variables.datasetId = d.id - LEFT JOIN namespaces on d.namespace = namespaces.name - LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id - WHERE ${whereClause} - ` - ) - return { rows: results, numTotalRows: resultCount[0].count } +export async function updateVariableAnnotations( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const patchesList = req.body as GrapherConfigPatch[] + const variableIds = new Set(patchesList.map((patch) => patch.id)) + + const configsAndIds = await db.knexRaw< + Pick & { + grapherConfigAdmin: DbRawChartConfig["patch"] + } + >( + trx, + `-- sql + SELECT v.id, cc.patch AS grapherConfigAdmin + FROM variables v + LEFT JOIN chart_configs cc ON v.grapherConfigIdAdmin = cc.id + WHERE v.id IN (?)`, + [[...variableIds.values()]] + ) + const configMap = new Map( + configsAndIds.map((item: any) => [ + item.id, + item.grapherConfigAdmin ? JSON.parse(item.grapherConfigAdmin) : {}, + ]) + ) + // console.log("ids", configsAndIds.map((item : any) => item.id)) + for (const patchSet of patchesList) { + const config = configMap.get(patchSet.id) + configMap.set(patchSet.id, applyPatch(patchSet, config)) } -) + + for (const [variableId, newConfig] of configMap.entries()) { + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) continue + await updateGrapherConfigAdminOfVariable(trx, variable, newConfig) + } + + return { success: true } +} patchRouteWithRWTransaction( apiRouter, "/variable-annotations", - async (req, res, trx) => { - const patchesList = req.body as GrapherConfigPatch[] - const variableIds = new Set(patchesList.map((patch) => patch.id)) - - const configsAndIds = await db.knexRaw< - Pick & { - grapherConfigAdmin: DbRawChartConfig["patch"] - } - >( - trx, - `-- sql - SELECT v.id, cc.patch AS grapherConfigAdmin - FROM variables v - LEFT JOIN chart_configs cc ON v.grapherConfigIdAdmin = cc.id - WHERE v.id IN (?) - `, - [[...variableIds.values()]] - ) - const configMap = new Map( - configsAndIds.map((item: any) => [ - item.id, - item.grapherConfigAdmin - ? JSON.parse(item.grapherConfigAdmin) - : {}, - ]) - ) - // console.log("ids", configsAndIds.map((item : any) => item.id)) - for (const patchSet of patchesList) { - const config = configMap.get(patchSet.id) - configMap.set(patchSet.id, applyPatch(patchSet, config)) - } + updateVariableAnnotations +) - for (const [variableId, newConfig] of configMap.entries()) { - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) continue - await updateGrapherConfigAdminOfVariable(trx, variable, newConfig) - } +getRouteWithROTransaction(apiRouter, "/chart-bulk-update", getChartBulkUpdate) - return { success: true } - } +patchRouteWithRWTransaction( + apiRouter, + "/chart-bulk-update", + updateBulkChartConfigs +) +getRouteWithROTransaction( + apiRouter, + "/variable-annotations", + getVariableAnnotations ) diff --git a/adminSiteServer/apiRoutes/chartViews.ts b/adminSiteServer/apiRoutes/chartViews.ts index c0013b57ef..4eda8ff3aa 100644 --- a/adminSiteServer/apiRoutes/chartViews.ts +++ b/adminSiteServer/apiRoutes/chartViews.ts @@ -34,6 +34,8 @@ import { import * as db from "../../db/db.js" import { expectChartById } from "./charts.js" +import { Request } from "../authentication.js" +import e from "express" const createPatchConfigAndQueryParamsForChartView = async ( knex: db.KnexReadonlyTransaction, parentChartId: number, @@ -65,7 +67,11 @@ const createPatchConfigAndQueryParamsForChartView = async ( return { patchConfig: patchConfigToSave, fullConfig, queryParams } } -getRouteWithROTransaction(apiRouter, "/chartViews", async (req, res, trx) => { +export async function getChartViews( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { type ChartViewRow = Pick & { lastEditedByUser: string chartConfigId: string @@ -109,30 +115,28 @@ getRouteWithROTransaction(apiRouter, "/chartViews", async (req, res, trx) => { })) return { chartViews } -}) - -getRouteWithROTransaction( - apiRouter, - "/chartViews/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) - - type ChartViewRow = Pick< - DbPlainChartView, - "id" | "name" | "updatedAt" - > & { - lastEditedByUser: string - chartConfigId: string - configFull: JsonString - configPatch: JsonString - parentChartId: number - parentConfigFull: JsonString - queryParamsForParentChart: JsonString - } - - const row = await db.knexRawFirst( - trx, - `-- sql +} + +export async function getChartViewById( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const id = expectInt(req.params.id) + + type ChartViewRow = Pick & { + lastEditedByUser: string + chartConfigId: string + configFull: JsonString + configPatch: JsonString + parentChartId: number + parentConfigFull: JsonString + queryParamsForParentChart: JsonString + } + + const row = await db.knexRawFirst( + trx, + `-- sql SELECT cv.id, cv.name, @@ -151,28 +155,29 @@ getRouteWithROTransaction( JOIN users u ON cv.lastEditedByUserId = u.id WHERE cv.id = ? `, - [id] - ) + [id] + ) - if (!row) { - throw new JsonError(`No chart view found for id ${id}`, 404) - } - - const chartView = { - ...row, - configFull: parseChartConfig(row.configFull), - configPatch: parseChartConfig(row.configPatch), - parentConfigFull: parseChartConfig(row.parentConfigFull), - queryParamsForParentChart: JSON.parse( - row.queryParamsForParentChart - ), - } - - return chartView + if (!row) { + throw new JsonError(`No chart view found for id ${id}`, 404) } -) -postRouteWithRWTransaction(apiRouter, "/chartViews", async (req, res, trx) => { + const chartView = { + ...row, + configFull: parseChartConfig(row.configFull), + configPatch: parseChartConfig(row.configPatch), + parentConfigFull: parseChartConfig(row.parentConfigFull), + queryParamsForParentChart: JSON.parse(row.queryParamsForParentChart), + } + + return chartView +} + +export async function createChartView( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { const { name, parentChartId } = req.body as Pick< DbPlainChartView, "name" | "parentChartId" @@ -195,7 +200,6 @@ postRouteWithRWTransaction(apiRouter, "/chartViews", async (req, res, trx) => { patchConfig, fullConfig ) - // insert into chart_views const insertRow: DbInsertChartView = { name, @@ -208,83 +212,89 @@ postRouteWithRWTransaction(apiRouter, "/chartViews", async (req, res, trx) => { const [resultId] = result return { chartViewId: resultId, success: true } -}) - -putRouteWithRWTransaction( - apiRouter, - "/chartViews/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) - const rawConfig = req.body.config as GrapherInterface - if (!rawConfig) { - throw new JsonError("Invalid request", 400) - } - - const existingRow: Pick< - DbPlainChartView, - "chartConfigId" | "parentChartId" - > = await trx(ChartViewsTableName) - .select("parentChartId", "chartConfigId") - .where({ id }) - .first() - - if (!existingRow) { - throw new JsonError(`No chart view found for id ${id}`, 404) - } - - const { patchConfig, fullConfig, queryParams } = - await createPatchConfigAndQueryParamsForChartView( - trx, - existingRow.parentChartId, - rawConfig - ) - - await updateChartConfigInDbAndR2( +} + +export async function updateChartView( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const id = expectInt(req.params.id) + const rawConfig = req.body.config as GrapherInterface + if (!rawConfig) { + throw new JsonError("Invalid request", 400) + } + + const existingRow: Pick< + DbPlainChartView, + "chartConfigId" | "parentChartId" + > = await trx(ChartViewsTableName) + .select("parentChartId", "chartConfigId") + .where({ id }) + .first() + + if (!existingRow) { + throw new JsonError(`No chart view found for id ${id}`, 404) + } + + const { patchConfig, fullConfig, queryParams } = + await createPatchConfigAndQueryParamsForChartView( trx, - existingRow.chartConfigId as Base64String, - patchConfig, - fullConfig + existingRow.parentChartId, + rawConfig ) - // update chart_views - await trx - .table(ChartViewsTableName) - .where({ id }) - .update({ - updatedAt: new Date(), - lastEditedByUserId: res.locals.user.id, - queryParamsForParentChart: JSON.stringify(queryParams), - }) - - return { success: true } + await updateChartConfigInDbAndR2( + trx, + existingRow.chartConfigId as Base64String, + patchConfig, + fullConfig + ) + + await trx + .table(ChartViewsTableName) + .where({ id }) + .update({ + updatedAt: new Date(), + lastEditedByUserId: res.locals.user.id, + queryParamsForParentChart: JSON.stringify(queryParams), + }) + + return { success: true } +} + +export async function deleteChartView( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const id = expectInt(req.params.id) + + const chartConfigId: string | undefined = await trx(ChartViewsTableName) + .select("chartConfigId") + .where({ id }) + .first() + .then((row) => row?.chartConfigId) + + if (!chartConfigId) { + throw new JsonError(`No chart view found for id ${id}`, 404) } -) -deleteRouteWithRWTransaction( - apiRouter, - "/chartViews/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) + await trx.table(ChartViewsTableName).where({ id }).delete() - const chartConfigId: string | undefined = await trx(ChartViewsTableName) - .select("chartConfigId") - .where({ id }) - .first() - .then((row) => row?.chartConfigId) + await deleteGrapherConfigFromR2ByUUID(chartConfigId) - if (!chartConfigId) { - throw new JsonError(`No chart view found for id ${id}`, 404) - } + await trx.table(ChartConfigsTableName).where({ id: chartConfigId }).delete() - await trx.table(ChartViewsTableName).where({ id }).delete() + return { success: true } +} - await deleteGrapherConfigFromR2ByUUID(chartConfigId) +getRouteWithROTransaction(apiRouter, "/chartViews", getChartViews) - await trx - .table(ChartConfigsTableName) - .where({ id: chartConfigId }) - .delete() +getRouteWithROTransaction(apiRouter, "/chartViews/:id", getChartViewById) - return { success: true } - } -) +postRouteWithRWTransaction(apiRouter, "/chartViews", createChartView) + +putRouteWithRWTransaction(apiRouter, "/chartViews/:id", updateChartView) + +deleteRouteWithRWTransaction(apiRouter, "/chartViews/:id", deleteChartView) diff --git a/adminSiteServer/apiRoutes/charts.ts b/adminSiteServer/apiRoutes/charts.ts index ae295c11fe..0ab2670cdd 100644 --- a/adminSiteServer/apiRoutes/charts.ts +++ b/adminSiteServer/apiRoutes/charts.ts @@ -66,6 +66,8 @@ import * as db from "../../db/db.js" import { getLogsByChartId } from "../getLogsByChartId.js" import { getPublishedLinksTo } from "../../db/model/Link.js" +import { Request } from "../authentication.js" +import e from "express" export const getReferencesByChartId = async ( chartId: number, knex: db.KnexReadonlyTransaction @@ -501,7 +503,11 @@ export async function updateGrapherConfigsInR2( } } -getRouteWithROTransaction(apiRouter, "/charts.json", async (req, res, trx) => { +async function getChartsJson( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000 const charts = await db.knexRaw( trx, @@ -521,9 +527,13 @@ getRouteWithROTransaction(apiRouter, "/charts.json", async (req, res, trx) => { await assignTagsForCharts(trx, charts) return { charts } -}) +} -getRouteWithROTransaction(apiRouter, "/charts.csv", async (req, res, trx) => { +async function getChartsCsv( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000 // note: this query is extended from OldChart.listFields. @@ -577,106 +587,116 @@ getRouteWithROTransaction(apiRouter, "/charts.csv", async (req, res, trx) => { res.setHeader("content-type", "text/csv") const csv = Papa.unparse(charts) return csv -}) +} -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.config.json", - async (req, res, trx) => expectChartById(trx, req.params.chartId) -) +async function getChartConfigJson( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + return expectChartById(trx, req.params.chartId) +} -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.parent.json", - async (req, res, trx) => { - const chartId = expectInt(req.params.chartId) - const parent = await getParentByChartId(trx, chartId) - const isInheritanceEnabled = await isInheritanceEnabledForChart( - trx, - chartId - ) - return omitUndefinedValues({ - variableId: parent?.variableId, - config: parent?.config, - isActive: isInheritanceEnabled, - }) - } -) +async function getChartParentJson( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const chartId = expectInt(req.params.chartId) + const parent = await getParentByChartId(trx, chartId) + const isInheritanceEnabled = await isInheritanceEnabledForChart( + trx, + chartId + ) + return omitUndefinedValues({ + variableId: parent?.variableId, + config: parent?.config, + isActive: isInheritanceEnabled, + }) +} -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.patchConfig.json", - async (req, res, trx) => { - const chartId = expectInt(req.params.chartId) - const config = await expectPatchConfigByChartId(trx, chartId) - return config - } -) +async function getChartPatchConfigJson( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const chartId = expectInt(req.params.chartId) + const config = await expectPatchConfigByChartId(trx, chartId) + return config +} -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.logs.json", - async (req, res, trx) => ({ +async function getChartLogsJson( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + return { logs: await getLogsByChartId( trx, parseInt(req.params.chartId as string) ), - }) -) + } +} -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.references.json", - async (req, res, trx) => { - const references = { - references: await getReferencesByChartId( - parseInt(req.params.chartId as string), - trx - ), - } - return references +async function getChartReferencesJson( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const references = { + references: await getReferencesByChartId( + parseInt(req.params.chartId as string), + trx + ), } -) + return references +} -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.redirects.json", - async (req, res, trx) => ({ +async function getChartRedirectsJson( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + return { redirects: await getRedirectsByChartId( trx, parseInt(req.params.chartId as string) ), - }) -) + } +} -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.pageviews.json", - async (req, res, trx) => { - const slug = await getChartSlugById( - trx, - parseInt(req.params.chartId as string) - ) - if (!slug) return {} +async function getChartPageviewsJson( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const slug = await getChartSlugById( + trx, + parseInt(req.params.chartId as string) + ) + if (!slug) return {} - const pageviewsByUrl = await db.knexRawFirst( - trx, - `-- sql - SELECT * - FROM - analytics_pageviews - WHERE - url = ?`, - [`https://ourworldindata.org/grapher/${slug}`] - ) + const pageviewsByUrl = await db.knexRawFirst( + trx, + `-- sql + SELECT * + FROM + analytics_pageviews + WHERE + url = ?`, + [`https://ourworldindata.org/grapher/${slug}`] + ) - return { - pageviews: pageviewsByUrl ?? undefined, - } + return { + pageviews: pageviewsByUrl ?? undefined, } -) +} -postRouteWithRWTransaction(apiRouter, "/charts", async (req, res, trx) => { +async function createChart( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { let shouldInherit: boolean | undefined if (req.query.inheritance) { shouldInherit = req.query.inheritance === "enable" @@ -693,109 +713,150 @@ postRouteWithRWTransaction(apiRouter, "/charts", async (req, res, trx) => { } catch (err) { return { success: false, error: String(err) } } -}) +} -postRouteWithRWTransaction( - apiRouter, - "/charts/:chartId/setTags", - async (req, res, trx) => { - const chartId = expectInt(req.params.chartId) +async function setChartTagsHandler( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const chartId = expectInt(req.params.chartId) + + await setChartTags(trx, chartId, req.body.tags) - await setChartTags(trx, chartId, req.body.tags) + return { success: true } +} - return { success: true } +async function updateChart( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + let shouldInherit: boolean | undefined + if (req.query.inheritance) { + shouldInherit = req.query.inheritance === "enable" } -) -putRouteWithRWTransaction( - apiRouter, - "/charts/:chartId", - async (req, res, trx) => { - let shouldInherit: boolean | undefined - if (req.query.inheritance) { - shouldInherit = req.query.inheritance === "enable" - } + const existingConfig = await expectChartById(trx, req.params.chartId) - const existingConfig = await expectChartById(trx, req.params.chartId) + try { + const { chartId, savedPatch } = await saveGrapher(trx, { + user: res.locals.user, + newConfig: req.body, + existingConfig, + shouldInherit, + }) - try { - const { chartId, savedPatch } = await saveGrapher(trx, { - user: res.locals.user, - newConfig: req.body, - existingConfig, - shouldInherit, - }) + const logs = await getLogsByChartId(trx, existingConfig.id as number) + return { + success: true, + chartId, + savedPatch, + newLog: logs[0], + } + } catch (err) { + return { + success: false, + error: String(err), + } + } +} - const logs = await getLogsByChartId( - trx, - existingConfig.id as number +async function deleteChart( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const chart = await expectChartById(trx, req.params.chartId) + if (chart.slug) { + const links = await getPublishedLinksTo(trx, [chart.slug]) + if (links.length) { + const sources = links.map((link) => link.sourceSlug).join(", ") + throw new Error( + `Cannot delete chart in-use in the following published documents: ${sources}` ) - return { - success: true, - chartId, - savedPatch, - newLog: logs[0], - } - } catch (err) { - return { - success: false, - error: String(err), - } } } -) -deleteRouteWithRWTransaction( - apiRouter, - "/charts/:chartId", - async (req, res, trx) => { - const chart = await expectChartById(trx, req.params.chartId) - if (chart.slug) { - const links = await getPublishedLinksTo(trx, [chart.slug]) - if (links.length) { - const sources = links.map((link) => link.sourceSlug).join(", ") - throw new Error( - `Cannot delete chart in-use in the following published documents: ${sources}` - ) - } - } + await db.knexRaw(trx, `DELETE FROM chart_dimensions WHERE chartId=?`, [ + chart.id, + ]) + await db.knexRaw(trx, `DELETE FROM chart_slug_redirects WHERE chart_id=?`, [ + chart.id, + ]) - await db.knexRaw(trx, `DELETE FROM chart_dimensions WHERE chartId=?`, [ - chart.id, + const row = await db.knexRawFirst>( + trx, + `SELECT configId FROM charts WHERE id = ?`, + [chart.id] + ) + if (!row || !row.configId) + throw new JsonError(`No chart config found for id ${chart.id}`, 404) + if (row) { + await db.knexRaw(trx, `DELETE FROM charts WHERE id=?`, [chart.id]) + await db.knexRaw(trx, `DELETE FROM chart_configs WHERE id=?`, [ + row.configId, ]) - await db.knexRaw( - trx, - `DELETE FROM chart_slug_redirects WHERE chart_id=?`, - [chart.id] - ) + } - const row = await db.knexRawFirst>( - trx, - `SELECT configId FROM charts WHERE id = ?`, - [chart.id] + if (chart.isPublished) + await triggerStaticBuild( + res.locals.user, + `Deleting chart ${chart.slug}` ) - if (!row || !row.configId) - throw new JsonError(`No chart config found for id ${chart.id}`, 404) - if (row) { - await db.knexRaw(trx, `DELETE FROM charts WHERE id=?`, [chart.id]) - await db.knexRaw(trx, `DELETE FROM chart_configs WHERE id=?`, [ - row.configId, - ]) - } - if (chart.isPublished) - await triggerStaticBuild( - res.locals.user, - `Deleting chart ${chart.slug}` - ) + await deleteGrapherConfigFromR2ByUUID(row.configId) + if (chart.isPublished) + await deleteGrapherConfigFromR2( + R2GrapherConfigDirectory.publishedGrapherBySlug, + `${chart.slug}.json` + ) - await deleteGrapherConfigFromR2ByUUID(row.configId) - if (chart.isPublished) - await deleteGrapherConfigFromR2( - R2GrapherConfigDirectory.publishedGrapherBySlug, - `${chart.slug}.json` - ) + return { success: true } +} - return { success: true } - } +getRouteWithROTransaction(apiRouter, "/charts.json", getChartsJson) +getRouteWithROTransaction(apiRouter, "/charts.csv", getChartsCsv) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.config.json", + getChartConfigJson +) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.parent.json", + getChartParentJson +) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.patchConfig.json", + getChartPatchConfigJson +) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.logs.json", + getChartLogsJson +) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.references.json", + getChartReferencesJson +) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.redirects.json", + getChartRedirectsJson +) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.pageviews.json", + getChartPageviewsJson +) +postRouteWithRWTransaction(apiRouter, "/charts", createChart) +postRouteWithRWTransaction( + apiRouter, + "/charts/:chartId/setTags", + setChartTagsHandler ) +putRouteWithRWTransaction(apiRouter, "/charts/:chartId", updateChart) +deleteRouteWithRWTransaction(apiRouter, "/charts/:chartId", deleteChart) diff --git a/adminSiteServer/apiRoutes/datasets.ts b/adminSiteServer/apiRoutes/datasets.ts index 365f00be51..d6bac477a2 100644 --- a/adminSiteServer/apiRoutes/datasets.ts +++ b/adminSiteServer/apiRoutes/datasets.ts @@ -28,390 +28,404 @@ import { import { triggerStaticBuild } from "./routeUtils.js" import * as db from "../../db/db.js" import * as lodash from "lodash" - -getRouteWithROTransaction( - apiRouter, - "/datasets.json", - async (req, res, trx) => { - const datasets = await db.knexRaw>( - trx, - `-- sql - WITH variable_counts AS ( - SELECT - v.datasetId, - COUNT(DISTINCT cd.chartId) as numCharts - FROM chart_dimensions cd - JOIN variables v ON cd.variableId = v.id - GROUP BY v.datasetId - ) +import { Request } from "express" +import * as e from "express" + +export async function getDatasets( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const datasets = await db.knexRaw>( + trx, + `-- sql + WITH variable_counts AS ( SELECT - ad.id, - ad.namespace, - ad.name, - d.shortName, - ad.description, - ad.dataEditedAt, - du.fullName AS dataEditedByUserName, - ad.metadataEditedAt, - mu.fullName AS metadataEditedByUserName, - ad.isPrivate, - ad.nonRedistributable, - d.version, - vc.numCharts - FROM active_datasets ad - LEFT JOIN variable_counts vc ON ad.id = vc.datasetId - JOIN users du ON du.id=ad.dataEditedByUserId - JOIN users mu ON mu.id=ad.metadataEditedByUserId - JOIN datasets d ON d.id=ad.id - ORDER BY ad.dataEditedAt DESC + v.datasetId, + COUNT(DISTINCT cd.chartId) as numCharts + FROM chart_dimensions cd + JOIN variables v ON cd.variableId = v.id + GROUP BY v.datasetId + ) + SELECT + ad.id, + ad.namespace, + ad.name, + d.shortName, + ad.description, + ad.dataEditedAt, + du.fullName AS dataEditedByUserName, + ad.metadataEditedAt, + mu.fullName AS metadataEditedByUserName, + ad.isPrivate, + ad.nonRedistributable, + d.version, + vc.numCharts + FROM active_datasets ad + LEFT JOIN variable_counts vc ON ad.id = vc.datasetId + JOIN users du ON du.id=ad.dataEditedByUserId + JOIN users mu ON mu.id=ad.metadataEditedByUserId + JOIN datasets d ON d.id=ad.id + ORDER BY ad.dataEditedAt DESC ` - ) - - const tags = await db.knexRaw< - Pick & - Pick - >( - trx, - `-- sql - SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt - JOIN tags t ON dt.tagId = t.id + ) + + const tags = await db.knexRaw< + Pick & Pick + >( + trx, + `-- sql + SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt + JOIN tags t ON dt.tagId = t.id ` + ) + const tagsByDatasetId = lodash.groupBy(tags, (t) => t.datasetId) + for (const dataset of datasets) { + dataset.tags = (tagsByDatasetId[dataset.id] || []).map((t) => + lodash.omit(t, "datasetId") ) - const tagsByDatasetId = lodash.groupBy(tags, (t) => t.datasetId) - for (const dataset of datasets) { - dataset.tags = (tagsByDatasetId[dataset.id] || []).map((t) => - lodash.omit(t, "datasetId") - ) - } - /*LEFT JOIN variables AS v ON v.datasetId=d.id - GROUP BY d.id*/ - - return { datasets: datasets } } -) - -getRouteWithROTransaction( - apiRouter, - "/datasets/:datasetId.json", - async (req, res, trx) => { - const datasetId = expectInt(req.params.datasetId) + /*LEFT JOIN variables AS v ON v.datasetId=d.id + GROUP BY d.id*/ - const dataset = await db.knexRawFirst>( - trx, - `-- sql - SELECT d.id, - d.namespace, - d.name, - d.shortName, - d.version, - d.description, - d.updatedAt, - d.dataEditedAt, - d.dataEditedByUserId, - du.fullName AS dataEditedByUserName, - d.metadataEditedAt, - d.metadataEditedByUserId, - mu.fullName AS metadataEditedByUserName, - d.isPrivate, - d.isArchived, - d.nonRedistributable, - d.updatePeriodDays - FROM datasets AS d - JOIN users du ON du.id=d.dataEditedByUserId - JOIN users mu ON mu.id=d.metadataEditedByUserId - WHERE d.id = ? + return { datasets: datasets } +} + +export async function getDataset( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const datasetId = expectInt(req.params.datasetId) + + const dataset = await db.knexRawFirst>( + trx, + `-- sql + SELECT d.id, + d.namespace, + d.name, + d.shortName, + d.version, + d.description, + d.updatedAt, + d.dataEditedAt, + d.dataEditedByUserId, + du.fullName AS dataEditedByUserName, + d.metadataEditedAt, + d.metadataEditedByUserId, + mu.fullName AS metadataEditedByUserName, + d.isPrivate, + d.isArchived, + d.nonRedistributable, + d.updatePeriodDays + FROM datasets AS d + JOIN users du ON du.id=d.dataEditedByUserId + JOIN users mu ON mu.id=d.metadataEditedByUserId + WHERE d.id = ? `, - [datasetId] - ) - - if (!dataset) - throw new JsonError(`No dataset by id '${datasetId}'`, 404) - - const zipFile = await db.knexRawFirst<{ filename: string }>( - trx, - `SELECT filename FROM dataset_files WHERE datasetId=?`, - [datasetId] - ) - if (zipFile) dataset.zipFile = zipFile - - const variables = await db.knexRaw< - Pick< - DbRawVariable, - "id" | "name" | "description" | "display" | "catalogPath" - > - >( - trx, - `-- sql - SELECT - v.id, - v.name, - v.description, - v.display, - v.catalogPath - FROM - variables AS v - WHERE - v.datasetId = ? + [datasetId] + ) + + if (!dataset) throw new JsonError(`No dataset by id '${datasetId}'`, 404) + + const zipFile = await db.knexRawFirst<{ filename: string }>( + trx, + `SELECT filename FROM dataset_files WHERE datasetId=?`, + [datasetId] + ) + if (zipFile) dataset.zipFile = zipFile + + const variables = await db.knexRaw< + Pick< + DbRawVariable, + "id" | "name" | "description" | "display" | "catalogPath" + > + >( + trx, + `-- sql + SELECT + v.id, + v.name, + v.description, + v.display, + v.catalogPath + FROM + variables AS v + WHERE + v.datasetId = ? `, - [datasetId] - ) + [datasetId] + ) - for (const v of variables) { - v.display = JSON.parse(v.display) - } - - dataset.variables = variables + for (const v of variables) { + v.display = JSON.parse(v.display) + } - // add all origins - const origins: DbRawOrigin[] = await db.knexRaw( - trx, - `-- sql - SELECT DISTINCT - o.* - FROM - origins_variables AS ov - JOIN origins AS o ON ov.originId = o.id - JOIN variables AS v ON ov.variableId = v.id - WHERE - v.datasetId = ? + dataset.variables = variables + + // add all origins + const origins: DbRawOrigin[] = await db.knexRaw( + trx, + `-- sql + SELECT DISTINCT + o.* + FROM + origins_variables AS ov + JOIN origins AS o ON ov.originId = o.id + JOIN variables AS v ON ov.variableId = v.id + WHERE + v.datasetId = ? `, - [datasetId] - ) - - const parsedOrigins = origins.map(parseOriginsRow) - - dataset.origins = parsedOrigins - - const sources = await db.knexRaw<{ - id: number - name: string - description: string - }>( - trx, - ` - SELECT s.id, s.name, s.description - FROM sources AS s - WHERE s.datasetId = ? - ORDER BY s.id ASC + [datasetId] + ) + + const parsedOrigins = origins.map(parseOriginsRow) + + dataset.origins = parsedOrigins + + const sources = await db.knexRaw<{ + id: number + name: string + description: string + }>( + trx, + ` + SELECT s.id, s.name, s.description + FROM sources AS s + WHERE s.datasetId = ? + ORDER BY s.id ASC `, - [datasetId] - ) - - // expand description of sources and add to dataset as variableSources - dataset.variableSources = sources.map((s: any) => { - return { - id: s.id, - name: s.name, - ...JSON.parse(s.description), - } - }) - - const charts = await db.knexRaw( - trx, - `-- sql - SELECT ${oldChartFieldList} - FROM charts - JOIN chart_configs ON chart_configs.id = charts.configId - JOIN chart_dimensions AS cd ON cd.chartId = charts.id - JOIN variables AS v ON cd.variableId = v.id - JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId - LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId - WHERE v.datasetId = ? - GROUP BY charts.id - `, - [datasetId] - ) - - dataset.charts = charts - - await assignTagsForCharts(trx, charts) - - const tags = await db.knexRaw<{ id: number; name: string }>( - trx, - ` - SELECT t.id, t.name - FROM tags t - JOIN dataset_tags dt ON dt.tagId = t.id - WHERE dt.datasetId = ? + [datasetId] + ) + + // expand description of sources and add to dataset as variableSources + dataset.variableSources = sources.map((s: any) => { + return { + id: s.id, + name: s.name, + ...JSON.parse(s.description), + } + }) + + const charts = await db.knexRaw( + trx, + `-- sql + SELECT ${oldChartFieldList} + FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + JOIN chart_dimensions AS cd ON cd.chartId = charts.id + JOIN variables AS v ON cd.variableId = v.id + JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId + LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId + WHERE v.datasetId = ? + GROUP BY charts.id + `, + [datasetId] + ) + + dataset.charts = charts + + await assignTagsForCharts(trx, charts) + + const tags = await db.knexRaw<{ id: number; name: string }>( + trx, + ` + SELECT t.id, t.name + FROM tags t + JOIN dataset_tags dt ON dt.tagId = t.id + WHERE dt.datasetId = ? `, - [datasetId] - ) - dataset.tags = tags - - const availableTags = await db.knexRaw<{ - id: number - name: string - parentName: string - }>( - trx, - ` - SELECT t.id, t.name, p.name AS parentName - FROM tags AS t - JOIN tags AS p ON t.parentId=p.id + [datasetId] + ) + dataset.tags = tags + + const availableTags = await db.knexRaw<{ + id: number + name: string + parentName: string + }>( + trx, + ` + SELECT t.id, t.name, p.name AS parentName + FROM tags AS t + JOIN tags AS p ON t.parentId=p.id ` - ) - dataset.availableTags = availableTags - - return { dataset: dataset } - } -) - -putRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId", - async (req, res, trx) => { - // Only updates `nonRedistributable` and `tags`, other fields come from ETL - // and are not editable - const datasetId = expectInt(req.params.datasetId) - const dataset = await getDatasetById(trx, datasetId) - if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) - - const newDataset = (req.body as { dataset: any }).dataset - await db.knexRaw( - trx, - ` - UPDATE datasets - SET - nonRedistributable=?, - metadataEditedAt=?, - metadataEditedByUserId=? - WHERE id=? - `, - [ - newDataset.nonRedistributable, - new Date(), - res.locals.user.id, - datasetId, - ] - ) - - const tagRows = newDataset.tags.map((tag: any) => [tag.id, datasetId]) - await db.knexRaw(trx, `DELETE FROM dataset_tags WHERE datasetId=?`, [ + ) + dataset.availableTags = availableTags + + return { dataset: dataset } +} + +export async function updateDataset( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + // Only updates `nonRedistributable` and `tags`, other fields come from ETL + // and are not editable + const datasetId = expectInt(req.params.datasetId) + const dataset = await getDatasetById(trx, datasetId) + if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) + + const newDataset = (req.body as { dataset: any }).dataset + await db.knexRaw( + trx, + ` + UPDATE datasets + SET + nonRedistributable=?, + metadataEditedAt=?, + metadataEditedByUserId=? + WHERE id=? + `, + [ + newDataset.nonRedistributable, + new Date(), + _res.locals.user.id, datasetId, - ]) - if (tagRows.length) - for (const tagRow of tagRows) { - await db.knexRaw( - trx, - `INSERT INTO dataset_tags (tagId, datasetId) VALUES (?, ?)`, - tagRow - ) - } - - try { - await syncDatasetToGitRepo(trx, datasetId, { - oldDatasetName: dataset.name, - commitName: res.locals.user.fullName, - commitEmail: res.locals.user.email, - }) - } catch (err) { - await logErrorAndMaybeSendToBugsnag(err, req) - // Continue + ] + ) + + const tagRows = newDataset.tags.map((tag: any) => [tag.id, datasetId]) + await db.knexRaw(trx, `DELETE FROM dataset_tags WHERE datasetId=?`, [ + datasetId, + ]) + if (tagRows.length) + for (const tagRow of tagRows) { + await db.knexRaw( + trx, + `INSERT INTO dataset_tags (tagId, datasetId) VALUES (?, ?)`, + tagRow + ) } - return { success: true } + try { + await syncDatasetToGitRepo(trx, datasetId, { + oldDatasetName: dataset.name, + commitName: _res.locals.user.fullName, + commitEmail: _res.locals.user.email, + }) + } catch (err) { + await logErrorAndMaybeSendToBugsnag(err, req) + // Continue } -) - -postRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId/setArchived", - async (req, res, trx) => { - const datasetId = expectInt(req.params.datasetId) - const dataset = await getDatasetById(trx, datasetId) - if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) - await db.knexRaw(trx, `UPDATE datasets SET isArchived = 1 WHERE id=?`, [ - datasetId, - ]) - return { success: true } + return { success: true } +} + +export async function setArchived( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const datasetId = expectInt(req.params.datasetId) + const dataset = await getDatasetById(trx, datasetId) + if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) + + await db.knexRaw(trx, `UPDATE datasets SET isArchived = 1 WHERE id=?`, [ + datasetId, + ]) + return { success: true } +} + +export async function setTags( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const datasetId = expectInt(req.params.datasetId) + + await setTagsForDataset(trx, datasetId, req.body.tagIds) + + return { success: true } +} + +export async function deleteDataset( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const datasetId = expectInt(req.params.datasetId) + + const dataset = await getDatasetById(trx, datasetId) + if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) + + await db.knexRaw( + trx, + `DELETE d FROM country_latest_data AS d JOIN variables AS v ON d.variable_id=v.id WHERE v.datasetId=?`, + [datasetId] + ) + await db.knexRaw(trx, `DELETE FROM dataset_files WHERE datasetId=?`, [ + datasetId, + ]) + await db.knexRaw(trx, `DELETE FROM variables WHERE datasetId=?`, [ + datasetId, + ]) + await db.knexRaw(trx, `DELETE FROM sources WHERE datasetId=?`, [datasetId]) + await db.knexRaw(trx, `DELETE FROM datasets WHERE id=?`, [datasetId]) + + try { + await removeDatasetFromGitRepo(dataset.name, dataset.namespace, { + commitName: _res.locals.user.fullName, + commitEmail: _res.locals.user.email, + }) + } catch (err: any) { + await logErrorAndMaybeSendToBugsnag(err, req) + // Continue } -) - -postRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId/setTags", - async (req, res, trx) => { - const datasetId = expectInt(req.params.datasetId) - await setTagsForDataset(trx, datasetId, req.body.tagIds) + return { success: true } +} - return { success: true } - } -) +export async function republishCharts( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const datasetId = expectInt(req.params.datasetId) -deleteRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId", - async (req, res, trx) => { - const datasetId = expectInt(req.params.datasetId) - - const dataset = await getDatasetById(trx, datasetId) - if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) + const dataset = await getDatasetById(trx, datasetId) + if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) + if (req.body.republish) { await db.knexRaw( trx, - `DELETE d FROM country_latest_data AS d JOIN variables AS v ON d.variable_id=v.id WHERE v.datasetId=?`, + `-- sql + UPDATE chart_configs cc + JOIN charts c ON c.configId = cc.id + SET + cc.patch = JSON_SET(cc.patch, "$.version", cc.patch->"$.version" + 1), + cc.full = JSON_SET(cc.full, "$.version", cc.full->"$.version" + 1) + WHERE c.id IN ( + SELECT DISTINCT chart_dimensions.chartId + FROM chart_dimensions + JOIN variables ON variables.id = chart_dimensions.variableId + WHERE variables.datasetId = ? + )`, [datasetId] ) - await db.knexRaw(trx, `DELETE FROM dataset_files WHERE datasetId=?`, [ - datasetId, - ]) - await db.knexRaw(trx, `DELETE FROM variables WHERE datasetId=?`, [ - datasetId, - ]) - await db.knexRaw(trx, `DELETE FROM sources WHERE datasetId=?`, [ - datasetId, - ]) - await db.knexRaw(trx, `DELETE FROM datasets WHERE id=?`, [datasetId]) - - try { - await removeDatasetFromGitRepo(dataset.name, dataset.namespace, { - commitName: res.locals.user.fullName, - commitEmail: res.locals.user.email, - }) - } catch (err: any) { - await logErrorAndMaybeSendToBugsnag(err, req) - // Continue - } - - return { success: true } } -) + await triggerStaticBuild( + _res.locals.user, + `Republishing all charts in dataset ${dataset.name} (${dataset.id})` + ) + + return { success: true } +} + +getRouteWithROTransaction(apiRouter, "/datasets.json", getDatasets) +getRouteWithROTransaction(apiRouter, "/datasets/:datasetId.json", getDataset) +putRouteWithRWTransaction(apiRouter, "/datasets/:datasetId", updateDataset) +postRouteWithRWTransaction( + apiRouter, + "/datasets/:datasetId/setArchived", + setArchived +) +postRouteWithRWTransaction(apiRouter, "/datasets/:datasetId/setTags", setTags) +deleteRouteWithRWTransaction(apiRouter, "/datasets/:datasetId", deleteDataset) postRouteWithRWTransaction( apiRouter, "/datasets/:datasetId/charts", - async (req, res, trx) => { - const datasetId = expectInt(req.params.datasetId) - - const dataset = await getDatasetById(trx, datasetId) - if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) - - if (req.body.republish) { - await db.knexRaw( - trx, - `-- sql - UPDATE chart_configs cc - JOIN charts c ON c.configId = cc.id - SET - cc.patch = JSON_SET(cc.patch, "$.version", cc.patch->"$.version" + 1), - cc.full = JSON_SET(cc.full, "$.version", cc.full->"$.version" + 1) - WHERE c.id IN ( - SELECT DISTINCT chart_dimensions.chartId - FROM chart_dimensions - JOIN variables ON variables.id = chart_dimensions.variableId - WHERE variables.datasetId = ? - )`, - [datasetId] - ) - } - - await triggerStaticBuild( - res.locals.user, - `Republishing all charts in dataset ${dataset.name} (${dataset.id})` - ) - - return { success: true } - } + republishCharts ) diff --git a/adminSiteServer/apiRoutes/explorer.ts b/adminSiteServer/apiRoutes/explorer.ts index eb184e2bef..f0228fafff 100644 --- a/adminSiteServer/apiRoutes/explorer.ts +++ b/adminSiteServer/apiRoutes/explorer.ts @@ -4,34 +4,43 @@ import { postRouteWithRWTransaction, deleteRouteWithRWTransaction, } from "../functionalRouterHelpers.js" +import { Request } from "express" +import * as e from "express" -postRouteWithRWTransaction( - apiRouter, - "/explorer/:slug/tags", - async (req, res, trx) => { - const { slug } = req.params - const { tagIds } = req.body - const explorer = await trx.table("explorers").where({ slug }).first() - if (!explorer) - throw new JsonError(`No explorer found for slug ${slug}`, 404) +import * as db from "../../db/db.js" +export async function addExplorerTags( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const { slug } = req.params + const { tagIds } = req.body + const explorer = await trx.table("explorers").where({ slug }).first() + if (!explorer) + throw new JsonError(`No explorer found for slug ${slug}`, 404) - await trx.table("explorer_tags").where({ explorerSlug: slug }).delete() - for (const tagId of tagIds) { - await trx - .table("explorer_tags") - .insert({ explorerSlug: slug, tagId }) - } - - return { success: true } + await trx.table("explorer_tags").where({ explorerSlug: slug }).delete() + for (const tagId of tagIds) { + await trx.table("explorer_tags").insert({ explorerSlug: slug, tagId }) } -) + + return { success: true } +} + +export async function deleteExplorerTags( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const { slug } = req.params + await trx.table("explorer_tags").where({ explorerSlug: slug }).delete() + return { success: true } +} + +postRouteWithRWTransaction(apiRouter, "/explorer/:slug/tags", addExplorerTags) deleteRouteWithRWTransaction( apiRouter, "/explorer/:slug/tags", - async (req, res, trx) => { - const { slug } = req.params - await trx.table("explorer_tags").where({ explorerSlug: slug }).delete() - return { success: true } - } + deleteExplorerTags ) diff --git a/adminSiteServer/apiRoutes/gdocs.ts b/adminSiteServer/apiRoutes/gdocs.ts index 0bd40226c5..ed96cb2417 100644 --- a/adminSiteServer/apiRoutes/gdocs.ts +++ b/adminSiteServer/apiRoutes/gdocs.ts @@ -53,38 +53,44 @@ import { import { triggerStaticBuild, enqueueLightningChange } from "./routeUtils.js" import * as db from "../../db/db.js" import * as lodash from "lodash" +import { Request } from "../authentication.js" +import e from "express" -getRouteWithROTransaction(apiRouter, "/gdocs", (req, res, trx) => { +export async function getAllGdocIndexItems( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { return getAllGdocIndexItemsOrderedByUpdatedAt(trx) -}) - -getRouteNonIdempotentWithRWTransaction( - apiRouter, - "/gdocs/:id", - async (req, res, trx) => { - const id = req.params.id - const contentSource = req.query.contentSource as - | GdocsContentSource - | undefined +} - try { - // Beware: if contentSource=gdocs this will update images in the DB+S3 even if the gdoc is published - const gdoc = await getAndLoadGdocById(trx, id, contentSource) +export async function getIndividualGdoc( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const id = req.params.id + const contentSource = req.query.contentSource as + | GdocsContentSource + | undefined - if (!gdoc.published) { - await updateGdocContentOnly(trx, id, gdoc) - } + try { + // Beware: if contentSource=gdocs this will update images in the DB+S3 even if the gdoc is published + const gdoc = await getAndLoadGdocById(trx, id, contentSource) - res.set("Cache-Control", "no-store") - res.send(gdoc) - } catch (error) { - console.error("Error fetching gdoc", error) - res.status(500).json({ - error: { message: String(error), status: 500 }, - }) + if (!gdoc.published) { + await updateGdocContentOnly(trx, id, gdoc) } + + res.set("Cache-Control", "no-store") + res.send(gdoc) + } catch (error) { + console.error("Error fetching gdoc", error) + res.status(500).json({ + error: { message: String(error), status: 500 }, + }) } -) +} /** * Handles all four `GdocPublishingAction` cases @@ -152,7 +158,11 @@ async function indexAndBakeGdocIfNeccesary( * support creating a new Gdoc from an existing one. Relevant updates will * trigger a deploy. */ -putRouteWithRWTransaction(apiRouter, "/gdocs/:id", async (req, res, trx) => { +export async function createOrUpdateGdoc( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { const { id } = req.params if (isEmpty(req.body)) { @@ -181,7 +191,7 @@ putRouteWithRWTransaction(apiRouter, "/gdocs/:id", async (req, res, trx) => { await indexAndBakeGdocIfNeccesary(trx, res.locals.user, prevGdoc, nextGdoc) return nextGdoc -}) +} async function validateTombstoneRelatedLinkUrl( trx: db.KnexReadonlyTransaction, @@ -201,7 +211,11 @@ async function validateTombstoneRelatedLinkUrl( } } -deleteRouteWithRWTransaction(apiRouter, "/gdocs/:id", async (req, res, trx) => { +export async function deleteGdoc( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { const { id } = req.params const gdoc = await getGdocBaseObjectById(trx, id, false) @@ -264,20 +278,34 @@ deleteRouteWithRWTransaction(apiRouter, "/gdocs/:id", async (req, res, trx) => { await triggerStaticBuild(res.locals.user, `Deleting ${gdocSlug}`) } return {} -}) +} -postRouteWithRWTransaction( - apiRouter, - "/gdocs/:gdocId/setTags", - async (req, res, trx) => { - const { gdocId } = req.params - const { tagIds } = req.body - const tagIdsAsObjects: { id: number }[] = tagIds.map((id: number) => ({ - id: id, - })) +export async function setGdocTags( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const { gdocId } = req.params + const { tagIds } = req.body + const tagIdsAsObjects: { id: number }[] = tagIds.map((id: number) => ({ + id: id, + })) - await setTagsForGdoc(trx, gdocId, tagIdsAsObjects) + await setTagsForGdoc(trx, gdocId, tagIdsAsObjects) - return { success: true } - } + return { success: true } +} + +getRouteWithROTransaction(apiRouter, "/gdocs", getAllGdocIndexItems) + +getRouteNonIdempotentWithRWTransaction( + apiRouter, + "/gdocs/:id", + getIndividualGdoc ) + +putRouteWithRWTransaction(apiRouter, "/gdocs/:id", createOrUpdateGdoc) + +deleteRouteWithRWTransaction(apiRouter, "/gdocs/:id", deleteGdoc) + +postRouteWithRWTransaction(apiRouter, "/gdocs/:gdocId/setTags", setGdocTags) diff --git a/adminSiteServer/apiRoutes/images.ts b/adminSiteServer/apiRoutes/images.ts index 4c91ac15ed..481415858f 100644 --- a/adminSiteServer/apiRoutes/images.ts +++ b/adminSiteServer/apiRoutes/images.ts @@ -19,24 +19,30 @@ import { triggerStaticBuild } from "./routeUtils.js" import * as db from "../../db/db.js" import * as lodash from "lodash" -getRouteNonIdempotentWithRWTransaction( - apiRouter, - "/images.json", - async (_, res, trx) => { - try { - const images = await db.getCloudflareImages(trx) - res.set("Cache-Control", "no-store") - res.send({ images }) - } catch (error) { - console.error("Error fetching images", error) - res.status(500).json({ - error: { message: String(error), status: 500 }, - }) - } +import { Request } from "../authentication.js" +import e from "express" +export async function getImagesHandler( + _: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + try { + const images = await db.getCloudflareImages(trx) + res.set("Cache-Control", "no-store") + res.send({ images }) + } catch (error) { + console.error("Error fetching images", error) + res.status(500).json({ + error: { message: String(error), status: 500 }, + }) } -) +} -postRouteWithRWTransaction(apiRouter, "/images", async (req, res, trx) => { +export async function postImageHandler( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { const { filename, type, content } = validateImagePayload(req.body) const { asBlob, dimensions, hash } = await processImageContent( @@ -94,14 +100,17 @@ postRouteWithRWTransaction(apiRouter, "/images", async (req, res, trx) => { success: true, image, } -}) - +} /** * Similar to the POST route, but for updating an existing image. * Creates a new image entry in the database and uploads the new image to Cloudflare. * The old image is marked as replaced by the new image. */ -putRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => { +export async function putImageHandler( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { const { type, content } = validateImagePayload(req.body) const { asBlob, dimensions, hash } = await processImageContent( content, @@ -173,10 +182,13 @@ putRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => { success: true, image: updated, } -}) - +} // Update alt text via patch -patchRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => { +export async function patchImageHandler( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { const { id } = req.params const image = await trx("images") @@ -204,9 +216,13 @@ patchRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => { success: true, image: updated, } -}) +} -deleteRouteWithRWTransaction(apiRouter, "/images/:id", async (req, _, trx) => { +export async function deleteImageHandler( + req: Request, + _: e.Response>, + trx: db.KnexReadonlyTransaction +) { const { id } = req.params const image = await trx("images") @@ -238,13 +254,34 @@ deleteRouteWithRWTransaction(apiRouter, "/images/:id", async (req, _, trx) => { return { success: true, } -}) +} -getRouteWithROTransaction(apiRouter, "/images/usage", async (_, __, trx) => { +export async function getImageUsageHandler( + _: Request, + __: e.Response>, + trx: db.KnexReadonlyTransaction +) { const usage = await db.getImageUsage(trx) return { success: true, usage, } -}) +} + +getRouteNonIdempotentWithRWTransaction( + apiRouter, + "/images.json", + getImagesHandler +) + +postRouteWithRWTransaction(apiRouter, "/images", postImageHandler) + +putRouteWithRWTransaction(apiRouter, "/images/:id", putImageHandler) + +// Update alt text via patch +patchRouteWithRWTransaction(apiRouter, "/images/:id", patchImageHandler) + +deleteRouteWithRWTransaction(apiRouter, "/images/:id", deleteImageHandler) + +getRouteWithROTransaction(apiRouter, "/images/usage", getImageUsageHandler) diff --git a/adminSiteServer/apiRoutes/mdims.ts b/adminSiteServer/apiRoutes/mdims.ts index a26116472e..34a05595d2 100644 --- a/adminSiteServer/apiRoutes/mdims.ts +++ b/adminSiteServer/apiRoutes/mdims.ts @@ -9,26 +9,35 @@ import { apiRouter } from "../apiRouter.js" import { putRouteWithRWTransaction } from "../functionalRouterHelpers.js" import { createMultiDimConfig } from "../multiDim.js" import { triggerStaticBuild } from "./routeUtils.js" +import { Request } from "../authentication.js" +import * as db from "../../db/db.js" +import e from "express" + +export async function handleMultiDimDataPageRequest( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const { slug } = req.params + if (!isValidSlug(slug)) { + throw new JsonError(`Invalid multi-dim slug ${slug}`) + } + const rawConfig = req.body as MultiDimDataPageConfigRaw + const id = await createMultiDimConfig(trx, slug, rawConfig) + if ( + FEATURE_FLAGS.has(FeatureFlagFeature.MultiDimDataPage) && + (await isMultiDimDataPagePublished(trx, slug)) + ) { + await triggerStaticBuild( + res.locals.user, + `Publishing multidimensional chart ${slug}` + ) + } + return { success: true, id } +} putRouteWithRWTransaction( apiRouter, "/multi-dim/:slug", - async (req, res, trx) => { - const { slug } = req.params - if (!isValidSlug(slug)) { - throw new JsonError(`Invalid multi-dim slug ${slug}`) - } - const rawConfig = req.body as MultiDimDataPageConfigRaw - const id = await createMultiDimConfig(trx, slug, rawConfig) - if ( - FEATURE_FLAGS.has(FeatureFlagFeature.MultiDimDataPage) && - (await isMultiDimDataPagePublished(trx, slug)) - ) { - await triggerStaticBuild( - res.locals.user, - `Publishing multidimensional chart ${slug}` - ) - } - return { success: true, id } - } + handleMultiDimDataPageRequest ) diff --git a/adminSiteServer/apiRoutes/misc.ts b/adminSiteServer/apiRoutes/misc.ts index 7a0f0c0dcd..a5ade731c0 100644 --- a/adminSiteServer/apiRoutes/misc.ts +++ b/adminSiteServer/apiRoutes/misc.ts @@ -13,8 +13,14 @@ import path from "path" import { DeployQueueServer } from "../../baker/DeployQueueServer.js" import { expectInt } from "../../serverUtils/serverUtil.js" import { triggerStaticBuild } from "./routeUtils.js" +import { Request } from "../authentication.js" +import e from "express" // using the alternate template, which highlights topics rather than articles. -getRouteWithROTransaction(apiRouter, "/all-work", async (req, res, trx) => { +export async function fetchAllWork( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { type WordpressPageRecord = { isWordpressPage: number } & Record< @@ -117,62 +123,62 @@ getRouteWithROTransaction(apiRouter, "/all-work", async (req, res, trx) => { res.type("text/plain") return [...generateAllWorkArchieMl()].join("") -}) - -getRouteWithROTransaction( - apiRouter, - "/editorData/namespaces.json", - async (req, res, trx) => { - const rows = await db.knexRaw<{ - name: string - description?: string - isArchived: boolean - }>( - trx, - `SELECT DISTINCT - namespace AS name, - namespaces.description AS description, - namespaces.isArchived AS isArchived - FROM active_datasets - JOIN namespaces ON namespaces.name = active_datasets.namespace` - ) +} + +export async function fetchNamespaces( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const rows = await db.knexRaw<{ + name: string + description?: string + isArchived: boolean + }>( + trx, + `SELECT DISTINCT + namespace AS name, + namespaces.description AS description, + namespaces.isArchived AS isArchived + FROM active_datasets + JOIN namespaces ON namespaces.name = active_datasets.namespace` + ) - return { - namespaces: lodash - .sortBy(rows, (row) => row.description) - .map((namespace) => ({ - ...namespace, - isArchived: !!namespace.isArchived, - })), - } + return { + namespaces: lodash + .sortBy(rows, (row) => row.description) + .map((namespace) => ({ + ...namespace, + isArchived: !!namespace.isArchived, + })), } -) +} -getRouteWithROTransaction( - apiRouter, - "/sources/:sourceId.json", - async (req, res, trx) => { - const sourceId = expectInt(req.params.sourceId) +export async function fetchSourceById( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const sourceId = expectInt(req.params.sourceId) - const source = await db.knexRawFirst>( - trx, - ` + const source = await db.knexRawFirst>( + trx, + ` SELECT s.id, s.name, s.description, s.createdAt, s.updatedAt, d.namespace FROM sources AS s JOIN active_datasets AS d ON d.id=s.datasetId WHERE s.id=?`, - [sourceId] - ) - if (!source) throw new JsonError(`No source by id '${sourceId}'`, 404) - source.variables = await db.knexRaw( - trx, - `SELECT id, name, updatedAt FROM variables WHERE variables.sourceId=?`, - [sourceId] - ) + [sourceId] + ) + if (!source) throw new JsonError(`No source by id '${sourceId}'`, 404) + source.variables = await db.knexRaw( + trx, + `SELECT id, name, updatedAt FROM variables WHERE variables.sourceId=?`, + [sourceId] + ) - return { source: source } - } -) + return { source: source } +} apiRouter.get("/deploys.json", async () => ({ deploys: await new DeployQueueServer().getDeploys(), @@ -181,3 +187,11 @@ apiRouter.get("/deploys.json", async () => ({ apiRouter.put("/deploy", async (req, res) => { return triggerStaticBuild(res.locals.user, "Manually triggered deploy") }) + +getRouteWithROTransaction(apiRouter, "/all-work", fetchAllWork) +getRouteWithROTransaction( + apiRouter, + "/editorData/namespaces.json", + fetchNamespaces +) +getRouteWithROTransaction(apiRouter, "/sources/:sourceId.json", fetchSourceById) diff --git a/adminSiteServer/apiRoutes/posts.ts b/adminSiteServer/apiRoutes/posts.ts index 4f36fd0a12..efd31d99db 100644 --- a/adminSiteServer/apiRoutes/posts.ts +++ b/adminSiteServer/apiRoutes/posts.ts @@ -19,8 +19,13 @@ import { postRouteWithRWTransaction, } from "../functionalRouterHelpers.js" import * as db from "../../db/db.js" - -getRouteWithROTransaction(apiRouter, "/posts.json", async (req, res, trx) => { +import { Request } from "../authentication.js" +import e from "express" +export async function handleGetPostsJson( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { const raw_rows = await db.knexRaw( trx, `-- sql @@ -88,133 +93,150 @@ getRouteWithROTransaction(apiRouter, "/posts.json", async (req, res, trx) => { })) return { posts: rows } -}) +} -postRouteWithRWTransaction( - apiRouter, - "/posts/:postId/setTags", - async (req, res, trx) => { - const postId = expectInt(req.params.postId) +export async function handleSetTagsForPost( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const postId = expectInt(req.params.postId) + await setTagsForPost(trx, postId, req.body.tagIds) + return { success: true } +} - await setTagsForPost(trx, postId, req.body.tagIds) +export async function handleGetPostById( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const postId = expectInt(req.params.postId) + const post = (await trx + .table(PostsTableName) + .where({ id: postId }) + .select("*") + .first()) as DbRawPost | undefined + return camelCaseProperties({ ...post }) +} - return { success: true } - } -) +export async function handleCreateGdoc( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const postId = expectInt(req.params.postId) + const allowRecreate = !!req.body.allowRecreate + const post = (await trx + .table("posts_with_gdoc_publish_status") + .where({ id: postId }) + .select("*") + .first()) as DbRawPostWithGdocPublishStatus | undefined -getRouteWithROTransaction( - apiRouter, - "/posts/:postId.json", - async (req, res, trx) => { - const postId = expectInt(req.params.postId) - const post = (await trx + if (!post) throw new JsonError(`No post found for id ${postId}`, 404) + const existingGdocId = post.gdocSuccessorId + if (!allowRecreate && existingGdocId) + throw new JsonError("A gdoc already exists for this post", 400) + if (allowRecreate && existingGdocId && post.isGdocPublished) { + throw new JsonError( + "A gdoc already exists for this post and it is already published", + 400 + ) + } + if (post.archieml === null) + throw new JsonError( + `ArchieML was not present for post with id ${postId}`, + 500 + ) + const tagsByPostId = await getTagsByPostId(trx) + const tags = tagsByPostId.get(postId) || [] + const archieMl = JSON.parse( + // Google Docs interprets ®ion in grapher URLS as ®ion + // So we escape them here + post.archieml.replaceAll("&", "&") + ) as OwidGdocPostInterface + const gdocId = await createGdocAndInsertOwidGdocPostContent( + archieMl.content, + post.gdocSuccessorId + ) + // If we did not yet have a gdoc associated with this post, we need to register + // the gdocSuccessorId and create an entry in the posts_gdocs table. Otherwise + // we don't need to make changes to the DB (only the gdoc regeneration was required) + if (!existingGdocId) { + post.gdocSuccessorId = gdocId + // This is not ideal - we are using knex for on thing and typeorm for another + // which means that we can't wrap this in a transaction. We should probably + // move posts to use typeorm as well or at least have a typeorm alternative for it + await trx .table(PostsTableName) .where({ id: postId }) - .select("*") - .first()) as DbRawPost | undefined - return camelCaseProperties({ ...post }) + .update("gdocSuccessorId", gdocId) + + const gdoc = new GdocPost(gdocId) + gdoc.slug = post.slug + gdoc.content.title = post.title + gdoc.content.type = archieMl.content.type || OwidGdocType.Article + gdoc.published = false + gdoc.createdAt = new Date() + gdoc.publishedAt = post.published_at + await upsertGdoc(trx, gdoc) + await setTagsForGdoc(trx, gdocId, tags) } -) + return { googleDocsId: gdocId } +} -postRouteWithRWTransaction( - apiRouter, - "/posts/:postId/createGdoc", - async (req, res, trx) => { - const postId = expectInt(req.params.postId) - const allowRecreate = !!req.body.allowRecreate - const post = (await trx - .table("posts_with_gdoc_publish_status") - .where({ id: postId }) - .select("*") - .first()) as DbRawPostWithGdocPublishStatus | undefined +export async function handleUnlinkGdoc( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const postId = expectInt(req.params.postId) + const post = (await trx + .table("posts_with_gdoc_publish_status") + .where({ id: postId }) + .select("*") + .first()) as DbRawPostWithGdocPublishStatus | undefined - if (!post) throw new JsonError(`No post found for id ${postId}`, 404) - const existingGdocId = post.gdocSuccessorId - if (!allowRecreate && existingGdocId) - throw new JsonError("A gdoc already exists for this post", 400) - if (allowRecreate && existingGdocId && post.isGdocPublished) { - throw new JsonError( - "A gdoc already exists for this post and it is already published", - 400 - ) - } - if (post.archieml === null) - throw new JsonError( - `ArchieML was not present for post with id ${postId}`, - 500 - ) - const tagsByPostId = await getTagsByPostId(trx) - const tags = tagsByPostId.get(postId) || [] - const archieMl = JSON.parse( - // Google Docs interprets ®ion in grapher URLS as ®ion - // So we escape them here - post.archieml.replaceAll("&", "&") - ) as OwidGdocPostInterface - const gdocId = await createGdocAndInsertOwidGdocPostContent( - archieMl.content, - post.gdocSuccessorId + if (!post) throw new JsonError(`No post found for id ${postId}`, 404) + const existingGdocId = post.gdocSuccessorId + if (!existingGdocId) + throw new JsonError("No gdoc exists for this post", 400) + if (existingGdocId && post.isGdocPublished) { + throw new JsonError( + "The GDoc is already published - you can't unlink it", + 400 ) - // If we did not yet have a gdoc associated with this post, we need to register - // the gdocSuccessorId and create an entry in the posts_gdocs table. Otherwise - // we don't need to make changes to the DB (only the gdoc regeneration was required) - if (!existingGdocId) { - post.gdocSuccessorId = gdocId - // This is not ideal - we are using knex for on thing and typeorm for another - // which means that we can't wrap this in a transaction. We should probably - // move posts to use typeorm as well or at least have a typeorm alternative for it - await trx - .table(PostsTableName) - .where({ id: postId }) - .update("gdocSuccessorId", gdocId) - - const gdoc = new GdocPost(gdocId) - gdoc.slug = post.slug - gdoc.content.title = post.title - gdoc.content.type = archieMl.content.type || OwidGdocType.Article - gdoc.published = false - gdoc.createdAt = new Date() - gdoc.publishedAt = post.published_at - await upsertGdoc(trx, gdoc) - await setTagsForGdoc(trx, gdocId, tags) - } - return { googleDocsId: gdocId } } -) + // This is not ideal - we are using knex for on thing and typeorm for another + // which means that we can't wrap this in a transaction. We should probably + // move posts to use typeorm as well or at least have a typeorm alternative for it + await trx + .table(PostsTableName) + .where({ id: postId }) + .update("gdocSuccessorId", null) + + await trx.table(PostsGdocsTableName).where({ id: existingGdocId }).delete() + + return { success: true } +} + +getRouteWithROTransaction(apiRouter, "/posts.json", handleGetPostsJson) postRouteWithRWTransaction( apiRouter, - "/posts/:postId/unlinkGdoc", - async (req, res, trx) => { - const postId = expectInt(req.params.postId) - const post = (await trx - .table("posts_with_gdoc_publish_status") - .where({ id: postId }) - .select("*") - .first()) as DbRawPostWithGdocPublishStatus | undefined + "/posts/:postId/setTags", + handleSetTagsForPost +) - if (!post) throw new JsonError(`No post found for id ${postId}`, 404) - const existingGdocId = post.gdocSuccessorId - if (!existingGdocId) - throw new JsonError("No gdoc exists for this post", 400) - if (existingGdocId && post.isGdocPublished) { - throw new JsonError( - "The GDoc is already published - you can't unlink it", - 400 - ) - } - // This is not ideal - we are using knex for on thing and typeorm for another - // which means that we can't wrap this in a transaction. We should probably - // move posts to use typeorm as well or at least have a typeorm alternative for it - await trx - .table(PostsTableName) - .where({ id: postId }) - .update("gdocSuccessorId", null) +getRouteWithROTransaction(apiRouter, "/posts/:postId.json", handleGetPostById) - await trx - .table(PostsGdocsTableName) - .where({ id: existingGdocId }) - .delete() +postRouteWithRWTransaction( + apiRouter, + "/posts/:postId/createGdoc", + handleCreateGdoc +) - return { success: true } - } +postRouteWithRWTransaction( + apiRouter, + "/posts/:postId/unlinkGdoc", + handleUnlinkGdoc ) diff --git a/adminSiteServer/apiRoutes/redirects.ts b/adminSiteServer/apiRoutes/redirects.ts index 0752c4ece1..00f8971b07 100644 --- a/adminSiteServer/apiRoutes/redirects.ts +++ b/adminSiteServer/apiRoutes/redirects.ts @@ -14,78 +14,82 @@ import { } from "../functionalRouterHelpers.js" import { triggerStaticBuild } from "./routeUtils.js" import * as db from "../../db/db.js" +import { Request } from "../authentication.js" +import e from "express" +export async function handleGetSiteRedirects( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + return { redirects: await getRedirects(trx) } +} -getRouteWithROTransaction( - apiRouter, - "/site-redirects.json", - async (req, res, trx) => ({ redirects: await getRedirects(trx) }) -) - -postRouteWithRWTransaction( - apiRouter, - "/site-redirects/new", - async (req, res, trx) => { - const { source, target } = req.body - const sourceAsUrl = new URL(source, "https://ourworldindata.org") - if (sourceAsUrl.pathname === "/") - throw new JsonError("Cannot redirect from /", 400) - if (await redirectWithSourceExists(trx, source)) { - throw new JsonError( - `Redirect with source ${source} already exists`, - 400 - ) - } - const chainedRedirect = await getChainedRedirect(trx, source, target) - if (chainedRedirect) { - throw new JsonError( - "Creating this redirect would create a chain, redirect from " + - `${chainedRedirect.source} to ${chainedRedirect.target} ` + - "already exists. " + - (target === chainedRedirect.source - ? `Please create the redirect from ${source} to ` + - `${chainedRedirect.target} directly instead.` - : `Please delete the existing redirect and create a ` + - `new redirect from ${chainedRedirect.source} to ` + - `${target} instead.`), - 400 - ) - } - const { insertId: id } = await db.knexRawInsert( - trx, - `INSERT INTO redirects (source, target) VALUES (?, ?)`, - [source, target] +export async function handlePostNewSiteRedirect( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const { source, target } = req.body + const sourceAsUrl = new URL(source, "https://ourworldindata.org") + if (sourceAsUrl.pathname === "/") + throw new JsonError("Cannot redirect from /", 400) + if (await redirectWithSourceExists(trx, source)) { + throw new JsonError( + `Redirect with source ${source} already exists`, + 400 ) - await triggerStaticBuild( - res.locals.user, - `Creating redirect id=${id} source=${source} target=${target}` + } + const chainedRedirect = await getChainedRedirect(trx, source, target) + if (chainedRedirect) { + throw new JsonError( + "Creating this redirect would create a chain, redirect from " + + `${chainedRedirect.source} to ${chainedRedirect.target} ` + + "already exists. " + + (target === chainedRedirect.source + ? `Please create the redirect from ${source} to ` + + `${chainedRedirect.target} directly instead.` + : `Please delete the existing redirect and create a ` + + `new redirect from ${chainedRedirect.source} to ` + + `${target} instead.`), + 400 ) - return { success: true, redirect: { id, source, target } } } -) + const { insertId: id } = await db.knexRawInsert( + trx, + `INSERT INTO redirects (source, target) VALUES (?, ?)`, + [source, target] + ) + await triggerStaticBuild( + res.locals.user, + `Creating redirect id=${id} source=${source} target=${target}` + ) + return { success: true, redirect: { id, source, target } } +} -deleteRouteWithRWTransaction( - apiRouter, - "/site-redirects/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) - const redirect = await getRedirectById(trx, id) - if (!redirect) { - throw new JsonError(`No redirect found for id ${id}`, 404) - } - await db.knexRaw(trx, `DELETE FROM redirects WHERE id=?`, [id]) - await triggerStaticBuild( - res.locals.user, - `Deleting redirect id=${id} source=${redirect.source} target=${redirect.target}` - ) - return { success: true } +export async function handleDeleteSiteRedirect( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const id = expectInt(req.params.id) + const redirect = await getRedirectById(trx, id) + if (!redirect) { + throw new JsonError(`No redirect found for id ${id}`, 404) } -) + await db.knexRaw(trx, `DELETE FROM redirects WHERE id=?`, [id]) + await triggerStaticBuild( + res.locals.user, + `Deleting redirect id=${id} source=${redirect.source} target=${redirect.target}` + ) + return { success: true } +} -// Get a list of redirects that map old slugs to charts -getRouteWithROTransaction( - apiRouter, - "/redirects.json", - async (req, res, trx) => ({ +export async function handleGetRedirects( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + return { redirects: await db.knexRaw( trx, `-- sql @@ -100,53 +104,82 @@ getRouteWithROTransaction( ORDER BY r.id DESC ` ), - }) + } +} + +export async function handlePostNewChartRedirect( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const chartId = expectInt(req.params.chartId) + const fields = req.body as { slug: string } + const result = await db.knexRawInsert( + trx, + `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`, + [chartId, fields.slug] + ) + const redirectId = result.insertId + const redirect = await db.knexRaw( + trx, + `SELECT * FROM chart_slug_redirects WHERE id = ?`, + [redirectId] + ) + return { success: true, redirect: redirect } +} + +export async function handleDeleteChartRedirect( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const id = expectInt(req.params.id) + + const redirect = await db.knexRawFirst( + trx, + `SELECT * FROM chart_slug_redirects WHERE id = ?`, + [id] + ) + + if (!redirect) throw new JsonError(`No redirect found for id ${id}`, 404) + + await db.knexRaw(trx, `DELETE FROM chart_slug_redirects WHERE id=?`, [id]) + await triggerStaticBuild( + res.locals.user, + `Deleting redirect from ${redirect.slug}` + ) + + return { success: true } +} + +getRouteWithROTransaction( + apiRouter, + "/site-redirects.json", + handleGetSiteRedirects ) postRouteWithRWTransaction( apiRouter, - "/charts/:chartId/redirects/new", - async (req, res, trx) => { - const chartId = expectInt(req.params.chartId) - const fields = req.body as { slug: string } - const result = await db.knexRawInsert( - trx, - `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`, - [chartId, fields.slug] - ) - const redirectId = result.insertId - const redirect = await db.knexRaw( - trx, - `SELECT * FROM chart_slug_redirects WHERE id = ?`, - [redirectId] - ) - return { success: true, redirect: redirect } - } + "/site-redirects/new", + handlePostNewSiteRedirect ) deleteRouteWithRWTransaction( apiRouter, - "/redirects/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) - - const redirect = await db.knexRawFirst( - trx, - `SELECT * FROM chart_slug_redirects WHERE id = ?`, - [id] - ) + "/site-redirects/:id", + handleDeleteSiteRedirect +) - if (!redirect) - throw new JsonError(`No redirect found for id ${id}`, 404) +getRouteWithROTransaction(apiRouter, "/redirects.json", handleGetRedirects) - await db.knexRaw(trx, `DELETE FROM chart_slug_redirects WHERE id=?`, [ - id, - ]) - await triggerStaticBuild( - res.locals.user, - `Deleting redirect from ${redirect.slug}` - ) +postRouteWithRWTransaction( + apiRouter, + "/charts/:chartId/redirects/new", + handlePostNewChartRedirect +) - return { success: true } - } +deleteRouteWithRWTransaction( + apiRouter, + "/redirects/:id", + handleDeleteChartRedirect ) diff --git a/adminSiteServer/apiRoutes/suggest.ts b/adminSiteServer/apiRoutes/suggest.ts index 2b9e3303fa..657d0b6b1f 100644 --- a/adminSiteServer/apiRoutes/suggest.ts +++ b/adminSiteServer/apiRoutes/suggest.ts @@ -10,62 +10,70 @@ import { CLOUDFLARE_IMAGES_URL } from "../../settings/clientSettings.js" import { apiRouter } from "../apiRouter.js" import { getRouteWithROTransaction } from "../functionalRouterHelpers.js" import { fetchGptGeneratedAltText } from "../imagesHelpers.js" +import * as db from "../../db/db.js" +import e from "express" +import { Request } from "../authentication.js" -getRouteWithROTransaction( - apiRouter, - `/gpt/suggest-topics/${TaggableType.Charts}/:chartId.json`, - async (req, res, trx): Promise> => { - const chartId = parseIntOrUndefined(req.params.chartId) - if (!chartId) throw new JsonError(`Invalid chart ID`, 400) +export async function suggestGptTopics( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +): Promise> { + const chartId = parseIntOrUndefined(req.params.chartId) + if (!chartId) throw new JsonError(`Invalid chart ID`, 400) - const topics = await getGptTopicSuggestions(trx, chartId) + const topics = await getGptTopicSuggestions(trx, chartId) - if (!topics.length) - throw new JsonError( - `No GPT topic suggestions found for chart ${chartId}`, - 404 - ) + if (!topics.length) + throw new JsonError( + `No GPT topic suggestions found for chart ${chartId}`, + 404 + ) - return { - topics, - } + return { + topics, } +} + +export async function suggestGptAltText( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +): Promise<{ + success: boolean + altText: string | null +}> { + const imageId = parseIntOrUndefined(req.params.imageId) + if (!imageId) throw new JsonError(`Invalid image ID`, 400) + const image = await trx("images") + .where("id", imageId) + .first() + if (!image) throw new JsonError(`No image found for ID ${imageId}`, 404) + + const src = `${CLOUDFLARE_IMAGES_URL}/${image.cloudflareId}/public` + let altText: string | null = "" + try { + altText = await fetchGptGeneratedAltText(src) + } catch (error) { + console.error(`Error fetching GPT alt text for image ${imageId}`, error) + throw new JsonError(`Error fetching GPT alt text: ${error}`, 500) + } + + if (!altText) { + throw new JsonError(`Unable to generate alt text for image`, 404) + } + + return { success: true, altText } +} + +getRouteWithROTransaction( + apiRouter, + `/gpt/suggest-topics/${TaggableType.Charts}/:chartId.json`, + suggestGptTopics ) getRouteWithROTransaction( apiRouter, `/gpt/suggest-alt-text/:imageId`, - async ( - req, - res, - trx - ): Promise<{ - success: boolean - altText: string | null - }> => { - const imageId = parseIntOrUndefined(req.params.imageId) - if (!imageId) throw new JsonError(`Invalid image ID`, 400) - const image = await trx("images") - .where("id", imageId) - .first() - if (!image) throw new JsonError(`No image found for ID ${imageId}`, 404) - - const src = `${CLOUDFLARE_IMAGES_URL}/${image.cloudflareId}/public` - let altText: string | null = "" - try { - altText = await fetchGptGeneratedAltText(src) - } catch (error) { - console.error( - `Error fetching GPT alt text for image ${imageId}`, - error - ) - throw new JsonError(`Error fetching GPT alt text: ${error}`, 500) - } - - if (!altText) { - throw new JsonError(`Unable to generate alt text for image`, 404) - } - - return { success: true, altText } - } + suggestGptAltText ) diff --git a/adminSiteServer/apiRoutes/tagGraph.ts b/adminSiteServer/apiRoutes/tagGraph.ts index 3690e2e541..f4dfc8b7b2 100644 --- a/adminSiteServer/apiRoutes/tagGraph.ts +++ b/adminSiteServer/apiRoutes/tagGraph.ts @@ -7,17 +7,23 @@ import { } from "../functionalRouterHelpers.js" import * as db from "../../db/db.js" import * as lodash from "lodash" +import { Request } from "../authentication.js" +import e from "express" -getRouteWithROTransaction( - apiRouter, - "/flatTagGraph.json", - async (req, res, trx) => { - const flatTagGraph = await db.getFlatTagGraph(trx) - return flatTagGraph - } -) +export async function handleGetFlatTagGraph( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const flatTagGraph = await db.getFlatTagGraph(trx) + return flatTagGraph +} -postRouteWithRWTransaction(apiRouter, "/tagGraph", async (req, res, trx) => { +export async function handlePostTagGraph( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { const tagGraph = req.body?.tagGraph as unknown if (!tagGraph) { throw new JsonError("No tagGraph provided", 400) @@ -51,10 +57,19 @@ postRouteWithRWTransaction(apiRouter, "/tagGraph", async (req, res, trx) => { return true } + const isValid = validateFlatTagGraph(tagGraph) if (!isValid) { throw new JsonError("Invalid tag graph provided", 400) } await db.updateTagGraph(trx, tagGraph) res.send({ success: true }) -}) +} + +getRouteWithROTransaction( + apiRouter, + "/flatTagGraph.json", + handleGetFlatTagGraph +) + +postRouteWithRWTransaction(apiRouter, "/tagGraph", handlePostTagGraph) diff --git a/adminSiteServer/apiRoutes/tags.ts b/adminSiteServer/apiRoutes/tags.ts index 0e698df454..578209cfe2 100644 --- a/adminSiteServer/apiRoutes/tags.ts +++ b/adminSiteServer/apiRoutes/tags.ts @@ -21,57 +21,54 @@ import { } from "../functionalRouterHelpers.js" import * as db from "../../db/db.js" import * as lodash from "lodash" +import e from "express" import { Request } from "../authentication.js" -getRouteWithROTransaction( - apiRouter, - "/tags/:tagId.json", - async (req, res, trx) => { - const tagId = expectInt(req.params.tagId) as number | null +export async function getTagById( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const tagId = expectInt(req.params.tagId) as number | null - // NOTE (Mispy): The "uncategorized" tag is special -- it represents all untagged stuff - // Bit fiddly to handle here but more true to normalized schema than having to remember to add the special tag - // every time we create a new chart etcs - const uncategorized = tagId === UNCATEGORIZED_TAG_ID + // NOTE (Mispy): The "uncategorized" tag is special -- it represents all untagged stuff + // Bit fiddly to handle here but more true to normalized schema than having to remember to add the special tag + // every time we create a new chart etcs + const uncategorized = tagId === UNCATEGORIZED_TAG_ID - // TODO: when we have types for our endpoints, make tag of that type instead of any - const tag: any = await db.knexRawFirst< - Pick< - DbPlainTag, - | "id" - | "name" - | "specialType" - | "updatedAt" - | "parentId" - | "slug" - > - >( - trx, - `-- sql + // TODO: when we have types for our endpoints, make tag of that type instead of any + const tag: any = await db.knexRawFirst< + Pick< + DbPlainTag, + "id" | "name" | "specialType" | "updatedAt" | "parentId" | "slug" + > + >( + trx, + `-- sql SELECT t.id, t.name, t.specialType, t.updatedAt, t.parentId, t.slug FROM tags t LEFT JOIN tags p ON t.parentId=p.id WHERE t.id = ? `, - [tagId] - ) + [tagId] + ) - // Datasets tagged with this tag - const datasets = await db.knexRaw< - Pick< - DbPlainDataset, - | "id" - | "namespace" - | "name" - | "description" - | "createdAt" - | "updatedAt" - | "dataEditedAt" - | "isPrivate" - | "nonRedistributable" - > & { dataEditedByUserName: string } - >( - trx, - `-- sql + // Datasets tagged with this tag + const datasets = await db.knexRaw< + Pick< + DbPlainDataset, + | "id" + | "namespace" + | "name" + | "description" + | "createdAt" + | "updatedAt" + | "dataEditedAt" + | "isPrivate" + | "nonRedistributable" + > & { dataEditedByUserName: string } + >( + trx, + `-- sql SELECT d.id, d.namespace, @@ -89,44 +86,44 @@ getRouteWithROTransaction( WHERE dt.tagId ${uncategorized ? "IS NULL" : "= ?"} ORDER BY d.dataEditedAt DESC `, - uncategorized ? [] : [tagId] - ) - tag.datasets = datasets + uncategorized ? [] : [tagId] + ) + tag.datasets = datasets - // The other tags for those datasets - if (tag.datasets.length) { - if (uncategorized) { - for (const dataset of tag.datasets) dataset.tags = [] - } else { - const datasetTags = await db.knexRaw<{ - datasetId: number - id: number - name: string - }>( - trx, - `-- sql + // The other tags for those datasets + if (tag.datasets.length) { + if (uncategorized) { + for (const dataset of tag.datasets) dataset.tags = [] + } else { + const datasetTags = await db.knexRaw<{ + datasetId: number + id: number + name: string + }>( + trx, + `-- sql SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt JOIN tags t ON dt.tagId = t.id WHERE dt.datasetId IN (?) `, - [tag.datasets.map((d: any) => d.id)] - ) - const tagsByDatasetId = lodash.groupBy( - datasetTags, - (t) => t.datasetId + [tag.datasets.map((d: any) => d.id)] + ) + const tagsByDatasetId = lodash.groupBy( + datasetTags, + (t) => t.datasetId + ) + for (const dataset of tag.datasets) { + dataset.tags = tagsByDatasetId[dataset.id].map((t) => + lodash.omit(t, "datasetId") ) - for (const dataset of tag.datasets) { - dataset.tags = tagsByDatasetId[dataset.id].map((t) => - lodash.omit(t, "datasetId") - ) - } } } + } - // Charts using datasets under this tag - const charts = await db.knexRaw( - trx, - `-- sql + // Charts using datasets under this tag + const charts = await db.knexRaw( + trx, + `-- sql SELECT ${oldChartFieldList} FROM charts JOIN chart_configs ON chart_configs.id = charts.configId LEFT JOIN chart_tags ct ON ct.chartId=charts.id @@ -136,134 +133,142 @@ getRouteWithROTransaction( GROUP BY charts.id ORDER BY charts.updatedAt DESC `, - uncategorized ? [] : [tagId] - ) - tag.charts = charts + uncategorized ? [] : [tagId] + ) + tag.charts = charts - await assignTagsForCharts(trx, charts) + await assignTagsForCharts(trx, charts) - // Subcategories - const children = await db.knexRaw<{ id: number; name: string }>( - trx, - `-- sql + // Subcategories + const children = await db.knexRaw<{ id: number; name: string }>( + trx, + `-- sql SELECT t.id, t.name FROM tags t WHERE t.parentId = ? `, - [tag.id] - ) - tag.children = children + [tag.id] + ) + tag.children = children - // Possible parents to choose from - const possibleParents = await db.knexRaw<{ id: number; name: string }>( - trx, - `-- sql + const possibleParents = await db.knexRaw<{ id: number; name: string }>( + trx, + `-- sql SELECT t.id, t.name FROM tags t WHERE t.parentId IS NULL ` - ) - tag.possibleParents = possibleParents + ) + tag.possibleParents = possibleParents - return { - tag, - } + return { + tag, } -) +} -putRouteWithRWTransaction( - apiRouter, - "/tags/:tagId", - async (req: Request, res, trx) => { - const tagId = expectInt(req.params.tagId) - const tag = (req.body as { tag: any }).tag - await db.knexRaw( +export async function updateTag( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const tagId = expectInt(req.params.tagId) + const tag = (req.body as { tag: any }).tag + await db.knexRaw( + trx, + `UPDATE tags SET name=?, updatedAt=?, slug=? WHERE id=?`, + [tag.name, new Date(), tag.slug, tagId] + ) + if (tag.slug) { + // See if there's a published gdoc with a matching slug. + // We're not enforcing that the gdoc be a topic page, as there are cases like /human-development-index, + // where the page for the topic is just an article. + const gdoc = await db.knexRaw>( trx, - `UPDATE tags SET name=?, updatedAt=?, slug=? WHERE id=?`, - [tag.name, new Date(), tag.slug, tagId] - ) - if (tag.slug) { - // See if there's a published gdoc with a matching slug. - // We're not enforcing that the gdoc be a topic page, as there are cases like /human-development-index, - // where the page for the topic is just an article. - const gdoc = await db.knexRaw>( - trx, - `-- sql + `-- sql SELECT slug FROM posts_gdocs pg WHERE EXISTS ( SELECT 1 FROM posts_gdocs_x_tags gt WHERE pg.id = gt.gdocId AND gt.tagId = ? ) AND pg.published = TRUE AND pg.slug = ?`, - [tagId, tag.slug] - ) - if (!gdoc.length) { - return { - success: true, - tagUpdateWarning: `The tag's slug has been updated, but there isn't a published Gdoc page with the same slug. + [tagId, tag.slug] + ) + if (!gdoc.length) { + return { + success: true, + tagUpdateWarning: `The tag's slug has been updated, but there isn't a published Gdoc page with the same slug. Are you sure you haven't made a typo?`, - } } } - return { success: true } } -) + return { success: true } +} -postRouteWithRWTransaction( - apiRouter, - "/tags/new", - async (req: Request, res, trx) => { - const tag = req.body - function validateTag( - tag: unknown - ): tag is { name: string; slug: string | null } { - return ( - checkIsPlainObjectWithGuard(tag) && - typeof tag.name === "string" && - (tag.slug === null || - (typeof tag.slug === "string" && tag.slug !== "")) - ) - } - if (!validateTag(tag)) throw new JsonError("Invalid tag", 400) - - const conflictingTag = await db.knexRawFirst<{ - name: string - slug: string | null - }>( - trx, - `SELECT name, slug FROM tags WHERE name = ? OR (slug IS NOT NULL AND slug = ?)`, - [tag.name, tag.slug] +export async function createTag( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const tag = req.body + function validateTag( + tag: unknown + ): tag is { name: string; slug: string | null } { + return ( + checkIsPlainObjectWithGuard(tag) && + typeof tag.name === "string" && + (tag.slug === null || + (typeof tag.slug === "string" && tag.slug !== "")) ) - if (conflictingTag) - throw new JsonError( - conflictingTag.name === tag.name - ? `Tag with name ${tag.name} already exists` - : `Tag with slug ${tag.slug} already exists`, - 400 - ) + } + if (!validateTag(tag)) throw new JsonError("Invalid tag", 400) - const now = new Date() - const result = await db.knexRawInsert( - trx, - `INSERT INTO tags (name, slug, createdAt, updatedAt) VALUES (?, ?, ?, ?)`, - // parentId will be deprecated soon once we migrate fully to the tag graph - [tag.name, tag.slug, now, now] + const conflictingTag = await db.knexRawFirst<{ + name: string + slug: string | null + }>( + trx, + `SELECT name, slug FROM tags WHERE name = ? OR (slug IS NOT NULL AND slug = ?)`, + [tag.name, tag.slug] + ) + if (conflictingTag) + throw new JsonError( + conflictingTag.name === tag.name + ? `Tag with name ${tag.name} already exists` + : `Tag with slug ${tag.slug} already exists`, + 400 ) - return { success: true, tagId: result.insertId } - } -) -getRouteWithROTransaction(apiRouter, "/tags.json", async (req, res, trx) => { + const now = new Date() + const result = await db.knexRawInsert( + trx, + `INSERT INTO tags (name, slug, createdAt, updatedAt) VALUES (?, ?, ?, ?)`, + // parentId will be deprecated soon once we migrate fully to the tag graph + [tag.name, tag.slug, now, now] + ) + return { success: true, tagId: result.insertId } +} + +export async function getAllTags( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { return { tags: await db.getMinimalTagsWithIsTopic(trx) } -}) +} -deleteRouteWithRWTransaction( - apiRouter, - "/tags/:tagId/delete", - async (req, res, trx) => { - const tagId = expectInt(req.params.tagId) +export async function deleteTag( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const tagId = expectInt(req.params.tagId) - await db.knexRaw(trx, `DELETE FROM tags WHERE id=?`, [tagId]) + await db.knexRaw(trx, `DELETE FROM tags WHERE id=?`, [tagId]) - return { success: true } - } -) + return { success: true } +} + +getRouteWithROTransaction(apiRouter, "/tags/:tagId.json", getTagById) +putRouteWithRWTransaction(apiRouter, "/tags/:tagId", updateTag) +postRouteWithRWTransaction(apiRouter, "/tags/new", createTag) +getRouteWithROTransaction(apiRouter, "/tags.json", getAllTags) +deleteRouteWithRWTransaction(apiRouter, "/tags/:tagId/delete", deleteTag) diff --git a/adminSiteServer/apiRoutes/users.ts b/adminSiteServer/apiRoutes/users.ts index 256ad22995..ea2016608e 100644 --- a/adminSiteServer/apiRoutes/users.ts +++ b/adminSiteServer/apiRoutes/users.ts @@ -11,108 +11,134 @@ import { postRouteWithRWTransaction, } from "../functionalRouterHelpers.js" import * as db from "../../db/db.js" - -getRouteWithROTransaction(apiRouter, "/users.json", async (req, res, trx) => ({ - users: await trx - .select( - "id" satisfies keyof DbPlainUser, - "email" satisfies keyof DbPlainUser, - "fullName" satisfies keyof DbPlainUser, - "isActive" satisfies keyof DbPlainUser, - "isSuperuser" satisfies keyof DbPlainUser, - "createdAt" satisfies keyof DbPlainUser, - "updatedAt" satisfies keyof DbPlainUser, - "lastLogin" satisfies keyof DbPlainUser, - "lastSeen" satisfies keyof DbPlainUser - ) - .from(UsersTableName) - .orderBy("lastSeen", "desc"), -})) - -getRouteWithROTransaction( - apiRouter, - "/users/:userId.json", - async (req, res, trx) => { - const id = parseIntOrUndefined(req.params.userId) - if (!id) throw new JsonError("No user id given") - const user = await getUserById(trx, id) - return { user } - } -) - -deleteRouteWithRWTransaction( - apiRouter, - "/users/:userId", - async (req, res, trx) => { - if (!res.locals.user.isSuperuser) - throw new JsonError("Permission denied", 403) - - const userId = expectInt(req.params.userId) - await db.knexRaw(trx, `DELETE FROM users WHERE id=?`, [userId]) - - return { success: true } - } -) - -putRouteWithRWTransaction( - apiRouter, - "/users/:userId", - async (req, res, trx: db.KnexReadWriteTransaction) => { - if (!res.locals.user.isSuperuser) - throw new JsonError("Permission denied", 403) - - const userId = parseIntOrUndefined(req.params.userId) - const user = - userId !== undefined ? await getUserById(trx, userId) : null - if (!user) throw new JsonError("No such user", 404) - - user.fullName = req.body.fullName - user.isActive = req.body.isActive - - await updateUser(trx, userId!, pick(user, ["fullName", "isActive"])) - - return { success: true } - } -) - -postRouteWithRWTransaction( - apiRouter, - "/users/add", - async (req, res, trx: db.KnexReadWriteTransaction) => { - if (!res.locals.user.isSuperuser) - throw new JsonError("Permission denied", 403) - - const { email, fullName } = req.body - - await insertUser(trx, { - email, - fullName, - }) - - return { success: true } +import { Request } from "../authentication.js" +import e from "express" +export async function getUsers( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + return { + users: await trx + .select( + "id" satisfies keyof DbPlainUser, + "email" satisfies keyof DbPlainUser, + "fullName" satisfies keyof DbPlainUser, + "isActive" satisfies keyof DbPlainUser, + "isSuperuser" satisfies keyof DbPlainUser, + "createdAt" satisfies keyof DbPlainUser, + "updatedAt" satisfies keyof DbPlainUser, + "lastLogin" satisfies keyof DbPlainUser, + "lastSeen" satisfies keyof DbPlainUser + ) + .from(UsersTableName) + .orderBy("lastSeen", "desc"), } -) +} + +export async function getUserByIdHandler( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const id = parseIntOrUndefined(req.params.userId) + if (!id) throw new JsonError("No user id given") + const user = await getUserById(trx, id) + return { user } +} + +export async function deleteUser( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + if (!res.locals.user.isSuperuser) + throw new JsonError("Permission denied", 403) + + const userId = expectInt(req.params.userId) + await db.knexRaw(trx, `DELETE FROM users WHERE id=?`, [userId]) + + return { success: true } +} + +export async function updateUserHandler( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + if (!res.locals.user.isSuperuser) + throw new JsonError("Permission denied", 403) + + const userId = parseIntOrUndefined(req.params.userId) + const user = userId !== undefined ? await getUserById(trx, userId) : null + if (!user) throw new JsonError("No such user", 404) + + user.fullName = req.body.fullName + user.isActive = req.body.isActive + + await updateUser(trx, userId!, pick(user, ["fullName", "isActive"])) + + return { success: true } +} + +export async function addUser( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + if (!res.locals.user.isSuperuser) + throw new JsonError("Permission denied", 403) + + const { email, fullName } = req.body + + await insertUser(trx, { + email, + fullName, + }) + + return { success: true } +} + +export async function addImageToUser( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const userId = expectInt(req.params.userId) + const imageId = expectInt(req.params.imageId) + await trx("images").where({ id: imageId }).update({ userId }) + return { success: true } +} + +export async function removeUserImage( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const userId = expectInt(req.params.userId) + const imageId = expectInt(req.params.imageId) + await trx("images").where({ id: imageId, userId }).update({ userId: null }) + return { success: true } +} + +getRouteWithROTransaction(apiRouter, "/users.json", getUsers) + +getRouteWithROTransaction(apiRouter, "/users/:userId.json", getUserByIdHandler) + +deleteRouteWithRWTransaction(apiRouter, "/users/:userId", deleteUser) + +putRouteWithRWTransaction(apiRouter, "/users/:userId", updateUserHandler) + +postRouteWithRWTransaction(apiRouter, "/users/add", addUser) postRouteWithRWTransaction( apiRouter, "/users/:userId/images/:imageId", - async (req, res, trx) => { - const userId = expectInt(req.params.userId) - const imageId = expectInt(req.params.imageId) - await trx("images").where({ id: imageId }).update({ userId }) - return { success: true } - } + addImageToUser ) deleteRouteWithRWTransaction( apiRouter, "/users/:userId/images/:imageId", - async (req, res, trx) => { - const userId = expectInt(req.params.userId) - const imageId = expectInt(req.params.imageId) - await trx("images") - .where({ id: imageId, userId }) - .update({ userId: null }) - return { success: true } - } + removeUserImage ) diff --git a/adminSiteServer/apiRoutes/variables.ts b/adminSiteServer/apiRoutes/variables.ts index 0853e92934..f8f21a65ab 100644 --- a/adminSiteServer/apiRoutes/variables.ts +++ b/adminSiteServer/apiRoutes/variables.ts @@ -48,24 +48,27 @@ import { expectInt } from "../../serverUtils/serverUtil.js" import { triggerStaticBuild } from "./routeUtils.js" import * as lodash from "lodash" import { updateGrapherConfigsInR2 } from "./charts.js" - -getRouteWithROTransaction( - apiRouter, - "/editorData/variables.json", - async (req, res, trx) => { - const datasets = [] - const rows = await db.knexRaw< - Pick & { - datasetId: number - datasetName: string - datasetVersion: string - } & Pick< - DbPlainDataset, - "namespace" | "isPrivate" | "nonRedistributable" - > - >( - trx, - `-- sql +import { Request } from "../authentication.js" +import e from "express" + +export async function getEditorVariablesJson( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const datasets = [] + const rows = await db.knexRaw< + Pick & { + datasetId: number + datasetName: string + datasetVersion: string + } & Pick< + DbPlainDataset, + "namespace" | "isPrivate" | "nonRedistributable" + > + >( + trx, + `-- sql SELECT v.name, v.id, @@ -78,47 +81,50 @@ getRouteWithROTransaction( FROM variables as v JOIN active_datasets as d ON v.datasetId = d.id ORDER BY d.updatedAt DESC ` - ) + ) - let dataset: - | { - id: number - name: string - version: string - namespace: string - isPrivate: boolean - nonRedistributable: boolean - variables: { id: number; name: string }[] - } - | undefined - for (const row of rows) { - if (!dataset || row.datasetName !== dataset.name) { - if (dataset) datasets.push(dataset) - - dataset = { - id: row.datasetId, - name: row.datasetName, - version: row.datasetVersion, - namespace: row.namespace, - isPrivate: !!row.isPrivate, - nonRedistributable: !!row.nonRedistributable, - variables: [], - } + let dataset: + | { + id: number + name: string + version: string + namespace: string + isPrivate: boolean + nonRedistributable: boolean + variables: { id: number; name: string }[] + } + | undefined + for (const row of rows) { + if (!dataset || row.datasetName !== dataset.name) { + if (dataset) datasets.push(dataset) + + dataset = { + id: row.datasetId, + name: row.datasetName, + version: row.datasetVersion, + namespace: row.namespace, + isPrivate: !!row.isPrivate, + nonRedistributable: !!row.nonRedistributable, + variables: [], } - - dataset.variables.push({ - id: row.id, - name: row.name ?? "", - }) } - if (dataset) datasets.push(dataset) - - return { datasets: datasets } + dataset.variables.push({ + id: row.id, + name: row.name ?? "", + }) } -) -apiRouter.get("/data/variables/data/:variableStr.json", async (req, res) => { + if (dataset) datasets.push(dataset) + + return { datasets: datasets } +} + +export async function getVariableDataJson( + req: Request, + _res: e.Response>, + _trx: db.KnexReadonlyTransaction +) { const variableStr = req.params.variableStr as string if (!variableStr) throw new JsonError("No variable id given") if (variableStr.includes("+")) @@ -130,40 +136,42 @@ apiRouter.get("/data/variables/data/:variableStr.json", async (req, res) => { return await fetchS3DataValuesByPath( getVariableDataRoute(DATA_API_URL, variableId) + "?nocache" ) -}) +} -apiRouter.get( - "/data/variables/metadata/:variableStr.json", - async (req, res) => { - const variableStr = req.params.variableStr as string - if (!variableStr) throw new JsonError("No variable id given") - if (variableStr.includes("+")) - throw new JsonError( - "Requesting multiple variables at the same time is no longer supported" - ) - const variableId = parseInt(variableStr) - if (isNaN(variableId)) throw new JsonError("Invalid variable id") - return await fetchS3MetadataByPath( - getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache" +export async function getVariableMetadataJson( + req: Request, + _res: e.Response>, + _trx: db.KnexReadonlyTransaction +) { + const variableStr = req.params.variableStr as string + if (!variableStr) throw new JsonError("No variable id given") + if (variableStr.includes("+")) + throw new JsonError( + "Requesting multiple variables at the same time is no longer supported" ) - } -) - -getRouteWithROTransaction( - apiRouter, - "/variables.json", - async (req, res, trx) => { - const limit = parseIntOrUndefined(req.query.limit as string) ?? 50 - const query = req.query.search as string - return await searchVariables(query, limit, trx) - } -) - -getRouteWithROTransaction( - apiRouter, - "/variables.usages.json", - async (req, res, trx) => { - const query = `-- sql + const variableId = parseInt(variableStr) + if (isNaN(variableId)) throw new JsonError("Invalid variable id") + return await fetchS3MetadataByPath( + getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache" + ) +} + +export async function getVariablesJson( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const limit = parseIntOrUndefined(req.query.limit as string) ?? 50 + const query = req.query.search as string + return await searchVariables(query, limit, trx) +} + +export async function getVariablesUsagesJson( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const query = `-- sql SELECT variableId, COUNT(DISTINCT chartId) AS usageCount @@ -174,74 +182,73 @@ getRouteWithROTransaction( ORDER BY usageCount DESC` - const rows = await db.knexRaw(trx, query) + const rows = await db.knexRaw(trx, query) - return rows - } -) + return rows +} -getRouteWithROTransaction( - apiRouter, - "/variables/grapherConfigETL/:variableId.patchConfig.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } - return variable.etl?.patchConfig ?? {} +export async function getVariablesGrapherConfigETLPatchConfigJson( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const variableId = expectInt(req.params.variableId) + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) } -) - -getRouteWithROTransaction( - apiRouter, - "/variables/grapherConfigAdmin/:variableId.patchConfig.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } - return variable.admin?.patchConfig ?? {} + return variable.etl?.patchConfig ?? {} +} + +export async function getVariablesGrapherConfigAdminPatchConfigJson( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const variableId = expectInt(req.params.variableId) + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) } -) + return variable.admin?.patchConfig ?? {} +} + +export async function getVariablesMergedGrapherConfigJson( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const variableId = expectInt(req.params.variableId) + const config = await getMergedGrapherConfigForVariable(trx, variableId) + return config ?? {} +} + +export async function getVariablesVariableIdJson( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const variableId = expectInt(req.params.variableId) + + const variable = await fetchS3MetadataByPath( + getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache" + ) -getRouteWithROTransaction( - apiRouter, - "/variables/mergedGrapherConfig/:variableId.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - const config = await getMergedGrapherConfigForVariable(trx, variableId) - return config ?? {} + // XXX: Patch shortName onto the end of catalogPath when it's missing, + // a temporary hack since our S3 metadata is out of date with our DB. + // See: https://github.com/owid/etl/issues/2135 + if (variable.catalogPath && !variable.catalogPath.includes("#")) { + variable.catalogPath += `#${variable.shortName}` } -) -// Used in VariableEditPage -getRouteWithROTransaction( - apiRouter, - "/variables/:variableId.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - - const variable = await fetchS3MetadataByPath( - getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache" - ) - - // XXX: Patch shortName onto the end of catalogPath when it's missing, - // a temporary hack since our S3 metadata is out of date with our DB. - // See: https://github.com/owid/etl/issues/2135 - if (variable.catalogPath && !variable.catalogPath.includes("#")) { - variable.catalogPath += `#${variable.shortName}` + const rawCharts = await db.knexRaw< + OldChartFieldList & { + isInheritanceEnabled: DbPlainChart["isInheritanceEnabled"] + config: DbRawChartConfig["full"] } - - const rawCharts = await db.knexRaw< - OldChartFieldList & { - isInheritanceEnabled: DbPlainChart["isInheritanceEnabled"] - config: DbRawChartConfig["full"] - } - >( - trx, - `-- sql + >( + trx, + `-- sql SELECT ${oldChartFieldList}, charts.isInheritanceEnabled, chart_configs.full AS config FROM charts JOIN chart_configs ON chart_configs.id = charts.configId @@ -251,297 +258,374 @@ getRouteWithROTransaction( WHERE cd.variableId = ? GROUP BY charts.id `, - [variableId] - ) - - // check for parent indicators - const charts = rawCharts.map((chart) => { - const parentIndicatorId = getParentVariableIdFromChartConfig( - parseChartConfig(chart.config) - ) - const hasParentIndicator = parentIndicatorId !== undefined - return omit({ ...chart, hasParentIndicator }, "config") - }) - - await assignTagsForCharts(trx, charts) + [variableId] + ) - const variableWithConfigs = await getGrapherConfigsForVariable( - trx, - variableId + // check for parent indicators + const charts = rawCharts.map((chart) => { + const parentIndicatorId = getParentVariableIdFromChartConfig( + parseChartConfig(chart.config) ) - const grapherConfigETL = variableWithConfigs?.etl?.patchConfig - const grapherConfigAdmin = variableWithConfigs?.admin?.patchConfig - const mergedGrapherConfig = - variableWithConfigs?.admin?.fullConfig ?? - variableWithConfigs?.etl?.fullConfig - - // add the variable's display field to the merged grapher config - if (mergedGrapherConfig) { - const [varDims, otherDims] = lodash.partition( - mergedGrapherConfig.dimensions ?? [], - (dim) => dim.variableId === variableId - ) - const varDimsWithDisplay = varDims.map((dim) => ({ - display: variable.display, - ...dim, - })) - mergedGrapherConfig.dimensions = [ - ...varDimsWithDisplay, - ...otherDims, - ] - } + const hasParentIndicator = parentIndicatorId !== undefined + return omit({ ...chart, hasParentIndicator }, "config") + }) - const variableWithCharts: OwidVariableWithSource & { - charts: Record - grapherConfig: GrapherInterface | undefined - grapherConfigETL: GrapherInterface | undefined - grapherConfigAdmin: GrapherInterface | undefined - } = { - ...variable, - charts, - grapherConfig: mergedGrapherConfig, - grapherConfigETL, - grapherConfigAdmin, - } + await assignTagsForCharts(trx, charts) - return { - variable: variableWithCharts, - } /*, vardata: await getVariableData([variableId]) }*/ + const variableWithConfigs = await getGrapherConfigsForVariable( + trx, + variableId + ) + const grapherConfigETL = variableWithConfigs?.etl?.patchConfig + const grapherConfigAdmin = variableWithConfigs?.admin?.patchConfig + const mergedGrapherConfig = + variableWithConfigs?.admin?.fullConfig ?? + variableWithConfigs?.etl?.fullConfig + + // add the variable's display field to the merged grapher config + if (mergedGrapherConfig) { + const [varDims, otherDims] = lodash.partition( + mergedGrapherConfig.dimensions ?? [], + (dim) => dim.variableId === variableId + ) + const varDimsWithDisplay = varDims.map((dim) => ({ + display: variable.display, + ...dim, + })) + mergedGrapherConfig.dimensions = [...varDimsWithDisplay, ...otherDims] } -) -// inserts a new config or updates an existing one -putRouteWithRWTransaction( - apiRouter, - "/variables/:variableId/grapherConfigETL", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - - let validConfig: GrapherInterface - try { - validConfig = migrateGrapherConfigToLatestVersion(req.body) - } catch (err) { - return { - success: false, - error: String(err), - } - } + const variableWithCharts: OwidVariableWithSource & { + charts: Record + grapherConfig: GrapherInterface | undefined + grapherConfigETL: GrapherInterface | undefined + grapherConfigAdmin: GrapherInterface | undefined + } = { + ...variable, + charts, + grapherConfig: mergedGrapherConfig, + grapherConfigETL, + grapherConfigAdmin, + } - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) + return { + variable: variableWithCharts, + } /*, vardata: await getVariableData([variableId]) }*/ +} + +export async function putVariablesVariableIdGrapherConfigETL( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const variableId = expectInt(req.params.variableId) + + let validConfig: GrapherInterface + try { + validConfig = migrateGrapherConfigToLatestVersion(req.body) + } catch (err) { + return { + success: false, + error: String(err), } + } - const { savedPatch, updatedCharts, updatedMultiDimViews } = - await updateGrapherConfigETLOfVariable(trx, variable, validConfig) + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } - await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) - const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] + const { savedPatch, updatedCharts, updatedMultiDimViews } = + await updateGrapherConfigETLOfVariable(trx, variable, validConfig) - if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { - await triggerStaticBuild( - res.locals.user, - `Updating ETL config for variable ${variableId}` - ) - } + await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) + const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - return { success: true, savedPatch } + if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { + await triggerStaticBuild( + res.locals.user, + `Updating ETL config for variable ${variableId}` + ) } -) -deleteRouteWithRWTransaction( - apiRouter, - "/variables/:variableId/grapherConfigETL", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) + return { success: true, savedPatch } +} - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } +export async function deleteVariablesVariableIdGrapherConfigETL( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const variableId = expectInt(req.params.variableId) - // no-op if the variable doesn't have an ETL config - if (!variable.etl) return { success: true } + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } - const now = new Date() + // no-op if the variable doesn't have an ETL config + if (!variable.etl) return { success: true } - // remove reference in the variables table - await db.knexRaw( - trx, - `-- sql + const now = new Date() + + // remove reference in the variables table + await db.knexRaw( + trx, + `-- sql UPDATE variables SET grapherConfigIdETL = NULL WHERE id = ? `, - [variableId] - ) + [variableId] + ) - // delete row in the chart_configs table - await db.knexRaw( - trx, - `-- sql + // delete row in the chart_configs table + await db.knexRaw( + trx, + `-- sql DELETE FROM chart_configs WHERE id = ? `, - [variable.etl.configId] - ) - - // update admin config if there is one - if (variable.admin) { - await updateExistingFullConfig(trx, { - configId: variable.admin.configId, - config: variable.admin.patchConfig, - updatedAt: now, - }) - } + [variable.etl.configId] + ) - const updates = { - patchConfigAdmin: variable.admin?.patchConfig, + // update admin config if there is one + if (variable.admin) { + await updateExistingFullConfig(trx, { + configId: variable.admin.configId, + config: variable.admin.patchConfig, updatedAt: now, - } - const updatedCharts = await updateAllChartsThatInheritFromIndicator( + }) + } + + const updates = { + patchConfigAdmin: variable.admin?.patchConfig, + updatedAt: now, + } + const updatedCharts = await updateAllChartsThatInheritFromIndicator( + trx, + variableId, + updates + ) + const updatedMultiDimViews = + await updateAllMultiDimViewsThatInheritFromIndicator( trx, variableId, updates ) - const updatedMultiDimViews = - await updateAllMultiDimViewsThatInheritFromIndicator( - trx, - variableId, - updates - ) - await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) - const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - - if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { - await triggerStaticBuild( - res.locals.user, - `Updating ETL config for variable ${variableId}` - ) - } + await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) + const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - return { success: true } + if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { + await triggerStaticBuild( + res.locals.user, + `Updating ETL config for variable ${variableId}` + ) } -) -// inserts a new config or updates an existing one -putRouteWithRWTransaction( - apiRouter, - "/variables/:variableId/grapherConfigAdmin", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - - let validConfig: GrapherInterface - try { - validConfig = migrateGrapherConfigToLatestVersion(req.body) - } catch (err) { - return { - success: false, - error: String(err), - } - } + return { success: true } +} - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) +export async function putVariablesVariableIdGrapherConfigAdmin( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const variableId = expectInt(req.params.variableId) + + let validConfig: GrapherInterface + try { + validConfig = migrateGrapherConfigToLatestVersion(req.body) + } catch (err) { + return { + success: false, + error: String(err), } + } - const { savedPatch, updatedCharts, updatedMultiDimViews } = - await updateGrapherConfigAdminOfVariable(trx, variable, validConfig) + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } - await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) - const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] + const { savedPatch, updatedCharts, updatedMultiDimViews } = + await updateGrapherConfigAdminOfVariable(trx, variable, validConfig) - if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { - await triggerStaticBuild( - res.locals.user, - `Updating admin-authored config for variable ${variableId}` - ) - } + await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) + const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - return { success: true, savedPatch } + if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { + await triggerStaticBuild( + res.locals.user, + `Updating admin-authored config for variable ${variableId}` + ) } -) -deleteRouteWithRWTransaction( - apiRouter, - "/variables/:variableId/grapherConfigAdmin", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) + return { success: true, savedPatch } +} - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } +export async function deleteVariablesVariableIdGrapherConfigAdmin( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const variableId = expectInt(req.params.variableId) - // no-op if the variable doesn't have an admin-authored config - if (!variable.admin) return { success: true } + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } - const now = new Date() + // no-op if the variable doesn't have an admin-authored config + if (!variable.admin) return { success: true } - // remove reference in the variables table - await db.knexRaw( - trx, - `-- sql + const now = new Date() + + // remove reference in the variables table + await db.knexRaw( + trx, + `-- sql UPDATE variables SET grapherConfigIdAdmin = NULL WHERE id = ? `, - [variableId] - ) + [variableId] + ) - // delete row in the chart_configs table - await db.knexRaw( - trx, - `-- sql + // delete row in the chart_configs table + await db.knexRaw( + trx, + `-- sql DELETE FROM chart_configs WHERE id = ? `, - [variable.admin.configId] - ) + [variable.admin.configId] + ) - const updates = { - patchConfigETL: variable.etl?.patchConfig, - updatedAt: now, - } - const updatedCharts = await updateAllChartsThatInheritFromIndicator( + const updates = { + patchConfigETL: variable.etl?.patchConfig, + updatedAt: now, + } + const updatedCharts = await updateAllChartsThatInheritFromIndicator( + trx, + variableId, + updates + ) + const updatedMultiDimViews = + await updateAllMultiDimViewsThatInheritFromIndicator( trx, variableId, updates ) - const updatedMultiDimViews = - await updateAllMultiDimViewsThatInheritFromIndicator( - trx, - variableId, - updates - ) - await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) - const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - - if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { - await triggerStaticBuild( - res.locals.user, - `Updating admin-authored config for variable ${variableId}` - ) - } + await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) + const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - return { success: true } + if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { + await triggerStaticBuild( + res.locals.user, + `Updating admin-authored config for variable ${variableId}` + ) } + + return { success: true } +} + +export async function getVariablesVariableIdChartsJson( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const variableId = expectInt(req.params.variableId) + const charts = await getAllChartsForIndicator(trx, variableId) + return charts.map((chart) => ({ + id: chart.chartId, + title: chart.config.title, + variantName: chart.config.variantName, + isChild: chart.isChild, + isInheritanceEnabled: chart.isInheritanceEnabled, + isPublished: chart.isPublished, + })) +} + +getRouteWithROTransaction( + apiRouter, + "/editorData/variables.json", + getEditorVariablesJson +) + +getRouteWithROTransaction( + apiRouter, + "/data/variables/data/:variableStr.json", + getVariableDataJson +) + +getRouteWithROTransaction( + apiRouter, + "/data/variables/metadata/:variableStr.json", + getVariableMetadataJson +) + +getRouteWithROTransaction(apiRouter, "/variables.json", getVariablesJson) + +getRouteWithROTransaction( + apiRouter, + "/variables.usages.json", + getVariablesUsagesJson +) + +getRouteWithROTransaction( + apiRouter, + "/variables/grapherConfigETL/:variableId.patchConfig.json", + getVariablesGrapherConfigETLPatchConfigJson +) + +getRouteWithROTransaction( + apiRouter, + "/variables/grapherConfigAdmin/:variableId.patchConfig.json", + getVariablesGrapherConfigAdminPatchConfigJson +) + +getRouteWithROTransaction( + apiRouter, + "/variables/mergedGrapherConfig/:variableId.json", + getVariablesMergedGrapherConfigJson +) + +// Used in VariableEditPage +getRouteWithROTransaction( + apiRouter, + "/variables/:variableId.json", + getVariablesVariableIdJson +) + +// inserts a new config or updates an existing one +putRouteWithRWTransaction( + apiRouter, + "/variables/:variableId/grapherConfigETL", + putVariablesVariableIdGrapherConfigETL +) + +deleteRouteWithRWTransaction( + apiRouter, + "/variables/:variableId/grapherConfigETL", + deleteVariablesVariableIdGrapherConfigETL +) + +// inserts a new config or updates an existing one +putRouteWithRWTransaction( + apiRouter, + "/variables/:variableId/grapherConfigAdmin", + putVariablesVariableIdGrapherConfigAdmin +) + +deleteRouteWithRWTransaction( + apiRouter, + "/variables/:variableId/grapherConfigAdmin", + deleteVariablesVariableIdGrapherConfigAdmin ) getRouteWithROTransaction( apiRouter, "/variables/:variableId/charts.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - const charts = await getAllChartsForIndicator(trx, variableId) - return charts.map((chart) => ({ - id: chart.chartId, - title: chart.config.title, - variantName: chart.config.variantName, - isChild: chart.isChild, - isInheritanceEnabled: chart.isInheritanceEnabled, - isPublished: chart.isPublished, - })) - } + getVariablesVariableIdChartsJson )