From 69f2ec2826e5c4ebac14d6b00130832acbdb0f3a Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Fri, 8 Nov 2024 12:00:34 -0500 Subject: [PATCH 01/39] Add routes for CreateExternalSource, CreateExternalSourceType, and CreateExternalEventType --- src/main.ts | 4 + src/packages/external-event/external-event.ts | 124 +++++++ src/packages/external-event/gql.ts | 11 + .../external-source/external-source.ts | 345 ++++++++++++++++++ src/packages/external-source/gql.ts | 49 +++ src/types/external-event.ts | 20 + src/types/external-source.ts | 27 ++ 7 files changed, 580 insertions(+) create mode 100644 src/packages/external-event/external-event.ts create mode 100644 src/packages/external-event/gql.ts create mode 100644 src/packages/external-source/external-source.ts create mode 100644 src/packages/external-source/gql.ts create mode 100644 src/types/external-event.ts create mode 100644 src/types/external-source.ts diff --git a/src/main.ts b/src/main.ts index fd27de5..e26168a 100644 --- a/src/main.ts +++ b/src/main.ts @@ -11,6 +11,8 @@ import initHasuraRoutes from './packages/hasura/hasura-events.js'; import initHealthRoutes from './packages/health/health.js'; import initPlanRoutes from './packages/plan/plan.js'; import initSwaggerRoutes from './packages/swagger/swagger.js'; +import initExternalSourceRoutes from './packages/external-source/external-source.js'; +import initExternalEventRoutes from './packages/external-event/external-event.js'; import cookieParser from 'cookie-parser'; import { AuthAdapter } from './types/auth.js'; import { NoAuthAdapter } from './packages/auth/adapters/NoAuthAdapter.js'; @@ -48,6 +50,8 @@ async function main(): Promise { initHealthRoutes(app); initHasuraRoutes(app); initPlanRoutes(app); + initExternalSourceRoutes(app); + initExternalEventRoutes(app); initSwaggerRoutes(app); app.listen(PORT, () => { diff --git a/src/packages/external-event/external-event.ts b/src/packages/external-event/external-event.ts new file mode 100644 index 0000000..a6512ea --- /dev/null +++ b/src/packages/external-event/external-event.ts @@ -0,0 +1,124 @@ +import type { Express, Request, Response } from 'express'; +import type { ExternalEventTypeInsertInput } from '../../types/external-event.js'; +import Ajv from 'ajv'; +import { getEnv } from '../../env.js'; +import getLogger from '../../logger.js'; +import gql from './gql.js'; +import { HasuraError } from '../../types/hasura.js'; + +const logger = getLogger('packages/external-event/external-event'); +const { HASURA_API_URL } = getEnv(); +const GQL_API_URL = `${HASURA_API_URL}/v1/graphql`; + +const ajv = new Ajv(); + +async function uploadExternalEventType(req: Request, res: Response) { + logger.info(`POST /uploadExternalEventType: Entering function...`); + const authorizationHeader = req.get('authorization'); + logger.info(`POST /uploadExternalEventType: ${authorizationHeader}`); + + const { + headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, + } = req; + + const { body } = req; + const { external_event_type_name, attribute_schema } = body; + + const headers: HeadersInit = { + Authorization: authorizationHeader ?? '', + 'Content-Type': 'application/json', + 'x-hasura-role': roleHeader ? `${roleHeader}` : '', + 'x-hasura-user-id': userHeader ? `${userHeader}` : '', + + }; + + // Validate schema is valid JSON Schema + try { + const schemaIsValid: boolean = ajv.validateSchema(attribute_schema); + if (!schemaIsValid) { + throw new Error("Schema was not a valid JSON Schema."); + } + } catch (e) { + logger.error(`POST /uploadExternalEventType: ${(e as Error).message}`); + res.json({ created: false }); + return; + } + + logger.info(`POST /uploadExternalEventType: Attribute schema was VALID! Calling Hasura mutation...`); + + // Run the Hasura migration for creating an external event + const externalEventTypeInsertInput: ExternalEventTypeInsertInput = { + attribute_schema: attribute_schema, + name: external_event_type_name, + } + + + const response = await fetch(GQL_API_URL, { + body: JSON.stringify({ + query: gql.CREATE_EXTERNAL_EVENT_TYPE, + variables: { eventType: externalEventTypeInsertInput }, + }), + headers, + method: 'POST', + }); + + type CreateExternalEventTypeResponse = { data: { createExternalEventType: { attribute_schema: object, name: string } | null } }; + const jsonResponse = await response.json(); + const createExternalEventTypeResponse = jsonResponse as CreateExternalEventTypeResponse | HasuraError; + + res.json(createExternalEventTypeResponse); +} + +export default (app: Express) => { + /** + * @swagger + * /uploadExternalEventType: + * post: + * security: + * - bearerAuth: [] + * consumes: + * - application/json + * produces: + * - application/json + * parameters: + * - in: header + * name: x-hasura-role + * schema: + * type: string + * required: false + * requestBody: + * content: + * application/json: + * schema: + * type: object + * properties: + * attribute_schema: + * type: object + * external_event_type_name: + * type: string + * required: + * - external_event_type_name + * attribute_schema + * responses: + * 200: + * description: Created External Event Type + * content: + * application/json: + * schema: + * properties: + * attribute_schema: + * description: JSON Schema for the created External Event Type's attributes + * type: object + * name: + * description: Name of the created External Event Type + * type: string + * 403: + * description: Unauthorized error + * 401: + * description: Unauthenticated error + * summary: Uploads an External Event Type definition (containing name & attributes schema) to Hasura. + * tags: + * - Hasura + */ + app.post('/uploadExternalEventType', uploadExternalEventType); +}; diff --git a/src/packages/external-event/gql.ts b/src/packages/external-event/gql.ts new file mode 100644 index 0000000..60f7421 --- /dev/null +++ b/src/packages/external-event/gql.ts @@ -0,0 +1,11 @@ +export default { + CREATE_EXTERNAL_EVENT_TYPE: `#graphql + mutation CreateExternalEventType($eventType: external_event_type_insert_input!) + { + createExternalEventType: insert_external_event_type_one(object: $eventType) { + attribute_schema + name + } + } + ` +} diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts new file mode 100644 index 0000000..1304326 --- /dev/null +++ b/src/packages/external-source/external-source.ts @@ -0,0 +1,345 @@ +import type { Express, Request, Response } from 'express'; +import type { DerivationGroupInsertInput, ExternalSourceInsertInput, ExternalSourceTypeInsertInput } from '../../types/external-source.js'; +import type { ExternalEvent, ExternalEventInsertInput } from '../../types/external-event.js'; +import Ajv from 'ajv'; +import { getEnv } from '../../env.js'; +import getLogger from '../../logger.js'; +import gql from './gql.js'; +//import { gql as externalEventGQL } from '../external-event/gql.js'; +import { HasuraError } from '../../types/hasura.js'; + +type CreateExternalSourceResponse = { data: { createExternalSource: { name: string } | null } }; +type CreateExternalSourceTypeResponse = { data: { createExternalSourceType: { attribute_schema: object, name: string } | null } }; +type GetExternalSourceTypeAttributeSchemaResponse = { data: { external_source_type_by_pk: { attribute_schema: object } | null } }; +type GetExternalEventTypeAttributeSchemaResponse = { data: { external_event_type_by_pk: { attribute_schema: object } | null } }; + +const logger = getLogger('packages/external-source/external-source'); +const { HASURA_API_URL } = getEnv(); +const GQL_API_URL = `${HASURA_API_URL}/v1/graphql`; +const ajv = new Ajv(); + +async function uploadExternalSourceType(req: Request, res: Response) { + logger.info(`POST /uploadExternalSourceType: Entering function...`); + const authorizationHeader = req.get('authorization'); + + const { + headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, + } = req; + + const { body } = req; + const { external_source_type_name, attribute_schema } = body; + + const headers: HeadersInit = { + Authorization: authorizationHeader ?? '', + 'Content-Type': 'application/json', + 'x-hasura-role': roleHeader ? `${roleHeader}` : '', + 'x-hasura-user-id': userHeader ? `${userHeader}` : '', + + }; + + // Validate schema is valid JSON Schema + try { + const schemaIsValid: boolean = ajv.validateSchema(attribute_schema); + if (!schemaIsValid) { + throw new Error("Schema was not a valid JSON Schema."); + } + } catch (e) { + logger.error(`POST /uploadExternalSourceType: ${(e as Error).message}`); + res.status(500); + res.send((e as Error).message); + return; + } + + logger.info(`POST /uploadExternalSourceType: Attribute schema was VALID! Calling Hasura mutation...`); + + // Run the Hasura migration for creating an external source + const externalSourceTypeInput: ExternalSourceTypeInsertInput = { + attribute_schema: attribute_schema, + name: external_source_type_name, + } + + + const response = await fetch(GQL_API_URL, { + body: JSON.stringify({ + query: gql.CREATE_EXTERNAL_SOURCE_TYPE, + variables: { sourceType: externalSourceTypeInput }, + }), + headers, + method: 'POST', + }); + + const jsonResponse = await response.json(); + const createExternalSourceTypeResponse = jsonResponse as CreateExternalSourceTypeResponse | HasuraError; + + res.json(createExternalSourceTypeResponse); +} + +async function uploadExternalSource(req: Request, res: Response) { + logger.info(`POST /uploadExternalSource: Entering function...`); + const authorizationHeader = req.get('authorization'); + const { + headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, + } = req; + const { body } = req; + const { + attributes, + derivation_group_name, + end_time, + external_events, + key, + source_type_name, + start_time, + valid_at + } = body; + const headers: HeadersInit = { + Authorization: authorizationHeader ?? '', + 'Content-Type': 'application/json', + 'x-hasura-role': roleHeader ? `${roleHeader}` : '', + 'x-hasura-user-id': userHeader ? `${userHeader}` : '', + + }; + + // Get the attribute schema for the source's external source type + const sourceAttributeSchema = await fetch(GQL_API_URL, { + body: JSON.stringify({ + query: gql.GET_EXTERNAL_SOURCE_TYPE_ATTRIBUTE_SCHEMA, + variables: { + name: source_type_name + } + }), + headers, + method: 'POST' + }); + + // Validate the attributes on the External Source + let sourceAttributesAreValid: boolean = false; + const sourceTypeResponseJSON = await sourceAttributeSchema.json(); + const getExternalSourceTypeAttributeSchemaResponse = sourceTypeResponseJSON as GetExternalSourceTypeAttributeSchemaResponse | HasuraError; + if ((getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse).data?.external_source_type_by_pk?.attribute_schema !== null) { + const { data: { external_source_type_by_pk: sourceAttributeSchema } } = getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse; + if (sourceAttributeSchema !== undefined && sourceAttributeSchema !== null) { + const sourceSchema = ajv.compile(sourceAttributeSchema.attribute_schema); + sourceAttributesAreValid = await sourceSchema(attributes); + } + } + if (sourceAttributesAreValid) { + logger.info(`POST /uploadExternalSource: Source's attributes are valid`); + } else { + logger.error(`POST /uploadExternalSource: Source's attributes are invalid`); + } + + // Get the attribute schema(s) for all external event types used by the source's events + const usedExternalEventTypes = external_events.data.map((externalEvent: ExternalEventInsertInput) => externalEvent.event_type_name).reduce( + (acc: string[], externalEventType: string) => { + if (!acc.includes(externalEventType)) { + acc.push(externalEventType) + }; + return acc; + }, []); + + const usedExternalEventTypesAttributesSchemas = await usedExternalEventTypes.reduce(async (acc: Record, eventType: string) => { + const eventAttributeSchema = await fetch(GQL_API_URL, { + body: JSON.stringify({ + query: gql.GET_EXTERNAL_EVENT_TYPE_ATTRIBUTE_SCHEMA, + variables: { + name: eventType + } + }), + headers, + method: 'POST' + }); + const eventTypeJSONResponse = await eventAttributeSchema.json(); + const getExternalEventTypeAttributeSchemaResponse = eventTypeJSONResponse as GetExternalEventTypeAttributeSchemaResponse | HasuraError; + + if ((getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse).data?.external_event_type_by_pk?.attribute_schema !== null) { + console.log(getExternalEventTypeAttributeSchemaResponse); + const { data: { external_event_type_by_pk: eventAttributeSchema } } = getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse; + if (eventAttributeSchema !== undefined && eventAttributeSchema !== null) { + acc[eventType] = ajv.compile(eventAttributeSchema); + } + } + + return acc; + }, {} as Record) + + // Validate all the event's attributes + try { + external_events.data.forEach(async (externalEvent: ExternalEvent) => { + const currentEventType = externalEvent.event_type_name; + const currentEventSchema = usedExternalEventTypesAttributesSchemas[currentEventType]; + const eventAttributesAreValid = await currentEventSchema(externalEvent.attributes); + if (!eventAttributesAreValid) { + throw new Error(`External Event '${externalEvent.key}' does not have a valid set of attributes, per it's type's schema.`); + } + }); + } catch (e) { + logger.error(`POST /uploadExternalSource: ${(e as Error).message}`); + res.status(500); + res.send((e as Error).message); + return; + } + + console.log("VALID!"); + + // Run the Hasura migration for creating an external source + const derivationGroupInsert: DerivationGroupInsertInput = { + name: derivation_group_name, + source_type_name: source_type_name + } + + const externalSourceInsert: ExternalSourceInsertInput = { + attributes: attributes, + derivation_group_name: derivation_group_name, + end_time: end_time, + external_events: external_events, + key: key, + source_type_name: source_type_name, + start_time: start_time, + valid_at: valid_at + } + + const response = await fetch(GQL_API_URL, { + body: JSON.stringify({ + query: gql.CREATE_EXTERNAL_SOURCE, + variables: { + derivation_group: derivationGroupInsert, + source: externalSourceInsert, + }, + }), + headers, + method: 'POST', + }); + + const jsonResponse = await response.json(); + console.log(jsonResponse); + const createExternalSourceResponse = jsonResponse as CreateExternalSourceResponse | HasuraError; + + + res.json(createExternalSourceResponse); +} + +export default (app: Express) => { + /** + * @swagger + * /uploadExternalSourceType: + * post: + * security: + * - bearerAuth: [] + * consumes: + * - application/json + * produces: + * - application/json + * parameters: + * - in: header + * name: x-hasura-role + * schema: + * type: string + * required: false + * requestBody: + * content: + * application/json: + * schema: + * type: object + * properties: + * attribute_schema: + * type: object + * external_source_type_name: + * type: string + * required: + * - external_source_type_name + * attribute_schema + * responses: + * 200: + * description: Created External Source Type + * content: + * application/json: + * schema: + * properties: + * attribute_schema: + * description: JSON Schema for the created External Source Type's attributes + * type: object + * name: + * description: Name of the created External Source Type + * type: string + * 403: + * description: Unauthorized error + * 401: + * description: Unauthenticated error + * summary: Uploads an External Source Type definition (containing name & attributes schema) to Hasura. + * tags: + * - Hasura + */ + app.post('/uploadExternalSourceType', uploadExternalSourceType); + + /** + * @swagger + * /uploadExternalSource: + * post: + * security: + * - bearerAuth: [] + * consumes: + * - application/json + * produces: + * - application/json + * parameters: + * - in: header + * name: x-hasura-role + * schema: + * type: string + * required: false + * requestBody: + * content: + * application/json: + * schema: + * type: object + * properties: + * attributes: + * type: object + * derivation_group_name: + * type: string + * end_time: + * type: string + * external_events: + * type: object + * properties: + * data: + * type: array + * required: + * - data + * key: + * type: string + * source_type_name: + * type: string + * start_time: + * type: string + * valid_at: + * type: string + * required: + * - attributes + * derivation_group_name + * end_time + * external_events + * key + * source_type_name + * start_time + * valid_at + * responses: + * 200: + * description: Created External Source + * content: + * application/json: + * schema: + * properties: + * name: + * description: Name of the created External Source + * type: string + * 403: + * description: Unauthorized error + * 401: + * description: Unauthenticated error + * summary: Uploads an External Source to Hasura. + * tags: + * - Hasura + */ + app.post('/uploadExternalSource', uploadExternalSource); +}; diff --git a/src/packages/external-source/gql.ts b/src/packages/external-source/gql.ts new file mode 100644 index 0000000..c119075 --- /dev/null +++ b/src/packages/external-source/gql.ts @@ -0,0 +1,49 @@ +export default { + CREATE_EXTERNAL_SOURCE: `#graphql + mutation CreateExternalSource( + $derivation_group: derivation_group_insert_input!, + $source: external_source_insert_input!, + ) { + upsertDerivationGroup: insert_derivation_group_one ( + object: $derivation_group, + on_conflict: { + constraint: derivation_group_pkey + } + ) { + name + } + createExternalSource: insert_external_source_one ( + object: $source + ) { + derivation_group_name, + end_time, + key, + source_type_name, + start_time, + valid_at, + } + } + `, + CREATE_EXTERNAL_SOURCE_TYPE: `#graphql + mutation CreateExternalSourceType($sourceType: external_source_type_insert_input!) { + createExternalSourceType: insert_external_source_type_one(object: $sourceType) { + name + attribute_schema + } + } + `, + GET_EXTERNAL_EVENT_TYPE_ATTRIBUTE_SCHEMA: `#graphql + query GetExternalEventTypeAttributeSchema($name: String!) { + external_event_type_by_pk(name: $name) { + attribute_schema + } + } + `, + GET_EXTERNAL_SOURCE_TYPE_ATTRIBUTE_SCHEMA: `#graphql + query GetExternalSourceTypeAttributeSchema($name: String!) { + external_source_type_by_pk(name: $name) { + attribute_schema + } + } + ` +} diff --git a/src/types/external-event.ts b/src/types/external-event.ts new file mode 100644 index 0000000..224d5ce --- /dev/null +++ b/src/types/external-event.ts @@ -0,0 +1,20 @@ +export type ExternalEventInsertInput = { + attributes: object; + start_time: string; + duration: string; + event_type_name: string; + key: string; +} + +export type ExternalEventTypeInsertInput = { + name: string; + attribute_schema: object; +} + +export type ExternalEvent = { + key: string; + event_type_name: string; + start_time: string; + duration: string; + attributes: object; +} diff --git a/src/types/external-source.ts b/src/types/external-source.ts new file mode 100644 index 0000000..34e9e5c --- /dev/null +++ b/src/types/external-source.ts @@ -0,0 +1,27 @@ +export type DerivationGroupInsertInput = { + name: string; + source_type_name: string; +} + +export type ExternalSourceTypeInsertInput = { + name: string; + attribute_schema: object; +} + +export type ExternalSourceInsertInput = { + attributes: object; + derivation_group_name: string; + end_time: string; + external_events: { + data: { + start_time: string; + duration: string; + event_type_name: string; + key: string; + }[]; + } + key: string; + source_type_name: string; + start_time: string; + valid_at: string; +} From b9839d3d4615801a59547e5142370b71ebaea12c Mon Sep 17 00:00:00 2001 From: psubram3 Date: Wed, 13 Nov 2024 14:21:28 -0800 Subject: [PATCH 02/39] require source type to specify associated event types --- src/packages/external-event/external-event.ts | 2 +- .../external-source/external-source.ts | 73 +++++++++++++++++-- src/packages/external-source/gql.ts | 22 +++++- 3 files changed, 87 insertions(+), 10 deletions(-) diff --git a/src/packages/external-event/external-event.ts b/src/packages/external-event/external-event.ts index a6512ea..c84a671 100644 --- a/src/packages/external-event/external-event.ts +++ b/src/packages/external-event/external-event.ts @@ -27,9 +27,9 @@ async function uploadExternalEventType(req: Request, res: Response) { const headers: HeadersInit = { Authorization: authorizationHeader ?? '', 'Content-Type': 'application/json', + 'x-hasura-admin-secret': 'aerie', // HACK, TODO: FIX 'x-hasura-role': roleHeader ? `${roleHeader}` : '', 'x-hasura-user-id': userHeader ? `${userHeader}` : '', - }; // Validate schema is valid JSON Schema diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 1304326..91074f7 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -10,6 +10,7 @@ import { HasuraError } from '../../types/hasura.js'; type CreateExternalSourceResponse = { data: { createExternalSource: { name: string } | null } }; type CreateExternalSourceTypeResponse = { data: { createExternalSourceType: { attribute_schema: object, name: string } | null } }; +type ExistingTypesResponse = { data: {existingEventTypes: { name: string }[]}}; type GetExternalSourceTypeAttributeSchemaResponse = { data: { external_source_type_by_pk: { attribute_schema: object } | null } }; type GetExternalEventTypeAttributeSchemaResponse = { data: { external_event_type_by_pk: { attribute_schema: object } | null } }; @@ -27,14 +28,18 @@ async function uploadExternalSourceType(req: Request, res: Response) { } = req; const { body } = req; - const { external_source_type_name, attribute_schema } = body; + const { external_source_type_name, attribute_schema, allowed_event_types } = body; + + console.log(body) + + const allowed_event_types_parsed = allowed_event_types as string[]; const headers: HeadersInit = { Authorization: authorizationHeader ?? '', 'Content-Type': 'application/json', + 'x-hasura-admin-secret': 'aerie', 'x-hasura-role': roleHeader ? `${roleHeader}` : '', 'x-hasura-user-id': userHeader ? `${userHeader}` : '', - }; // Validate schema is valid JSON Schema @@ -50,19 +55,46 @@ async function uploadExternalSourceType(req: Request, res: Response) { return; } - logger.info(`POST /uploadExternalSourceType: Attribute schema was VALID! Calling Hasura mutation...`); + logger.info(`POST /uploadExternalSourceType: Attribute schema was VALID!`); - // Run the Hasura migration for creating an external source + // TODO: Check the list of allowed event types are all defined + // QUESTION: only do this check in the UI? The database ultimately checks these things. + const existingTypesResponse = await fetch(GQL_API_URL, { + body: JSON.stringify({ + query: gql.GET_EXTERNAL_EVENT_TYPES, + variables: {} + }), + headers, + method: 'POST' + }); + const existingTypesStruct: ExistingTypesResponse = await existingTypesResponse.json(); + const existingTypes: string[] = existingTypesStruct.data.existingEventTypes.map(entry => entry.name); + for (const event_type of allowed_event_types_parsed) { + if (!existingTypes.includes(event_type)) { + logger.error(`POST /uploadExternalSourceType: Event type ${event_type} is not defined.`); + res.status(500); + res.send(`POST /uploadExternalSourceType: Event type ${event_type} is not defined.`); + return; + } + } + + logger.info(`POST /uploadExternalSourceType: Successfully checked event types valid! Calling Hasura mutation...`); + + // Run the Hasura migration for creating an external source type (and inserting allowed event types) const externalSourceTypeInput: ExternalSourceTypeInsertInput = { attribute_schema: attribute_schema, name: external_source_type_name, } + const allowedTypes: { external_event_type: string, external_source_type: string }[] = allowed_event_types_parsed.map(external_event_type => { + return { external_event_type, external_source_type: external_source_type_name }; + }) - const response = await fetch(GQL_API_URL, { + + const response = await fetch(GQL_API_URL, { // TODO: update body: JSON.stringify({ query: gql.CREATE_EXTERNAL_SOURCE_TYPE, - variables: { sourceType: externalSourceTypeInput }, + variables: { allowedTypes, sourceType: externalSourceTypeInput }, }), headers, method: 'POST', @@ -71,6 +103,8 @@ async function uploadExternalSourceType(req: Request, res: Response) { const jsonResponse = await response.json(); const createExternalSourceTypeResponse = jsonResponse as CreateExternalSourceTypeResponse | HasuraError; + logger.info(`POST /uploadExternalSourceType: Successfully uploaded new type and event type associations!`); + res.json(createExternalSourceTypeResponse); } @@ -94,9 +128,9 @@ async function uploadExternalSource(req: Request, res: Response) { const headers: HeadersInit = { Authorization: authorizationHeader ?? '', 'Content-Type': 'application/json', + 'x-hasura-admin-secret': 'aerie', // HACK, TODO: FIX 'x-hasura-role': roleHeader ? `${roleHeader}` : '', 'x-hasura-user-id': userHeader ? `${userHeader}` : '', - }; // Get the attribute schema for the source's external source type @@ -128,7 +162,8 @@ async function uploadExternalSource(req: Request, res: Response) { logger.error(`POST /uploadExternalSource: Source's attributes are invalid`); } - // Get the attribute schema(s) for all external event types used by the source's events + // TODO: verify events are all of allowed type + // get list of all used event types const usedExternalEventTypes = external_events.data.map((externalEvent: ExternalEventInsertInput) => externalEvent.event_type_name).reduce( (acc: string[], externalEventType: string) => { if (!acc.includes(externalEventType)) { @@ -137,6 +172,28 @@ async function uploadExternalSource(req: Request, res: Response) { return acc; }, []); + // get allowed event types + const allowedExternalEventTypes = await fetch(GQL_API_URL, { + body: JSON.stringify({ + query: gql.GET_EXTERNAL_EVENT_TYPES_FOR_SOURCE_TYPE, + variables: { sourceType: source_type_name } + }), + headers, + method: 'POST' + }); + + // check + const allowedEventTypes = await allowedExternalEventTypes.json() as ExistingTypesResponse; + for (const event_type of usedExternalEventTypes) { + if (!allowedEventTypes.data.existingEventTypes.includes(event_type)) { + logger.error(`POST /uploadExternalSourceType: An event uses event type ${event_type}, which is not defined for source type ${source_type_name}.`); + res.status(500); + res.send(`POST /uploadExternalSourceType: An event uses event type ${event_type}, which is not defined for source type ${source_type_name}.`); + return; + } + } + + // Get the attribute schema(s) for all external event types used by the source's events const usedExternalEventTypesAttributesSchemas = await usedExternalEventTypes.reduce(async (acc: Record, eventType: string) => { const eventAttributeSchema = await fetch(GQL_API_URL, { body: JSON.stringify({ diff --git a/src/packages/external-source/gql.ts b/src/packages/external-source/gql.ts index c119075..ad0a70b 100644 --- a/src/packages/external-source/gql.ts +++ b/src/packages/external-source/gql.ts @@ -25,11 +25,31 @@ export default { } `, CREATE_EXTERNAL_SOURCE_TYPE: `#graphql - mutation CreateExternalSourceType($sourceType: external_source_type_insert_input!) { + mutation CreateExternalSourceType($sourceType: external_source_type_insert_input!, $allowedTypes: [external_source_type_allowed_event_types_insert_input!]!) { createExternalSourceType: insert_external_source_type_one(object: $sourceType) { name attribute_schema } + defineAllowedTypes: insert_external_source_type_allowed_event_types(objects: $allowedTypes) { + returning { + external_source_type + external_event_type + } + } + } + `, + GET_EXTERNAL_EVENT_TYPES: `#graphql + query ExistingEventTypes { + existingEventTypes: external_event_type { + name + } + } + `, + GET_EXTERNAL_EVENT_TYPES_FOR_SOURCE_TYPE: `#graphql + query ExistingEventTypesForSourceType($sourceType: string!) { + existingEventTypesForSourceType: external_source_type_allowed_event_types(where: {name: {_eq: $sourceType}}) { + external_event_type + } } `, GET_EXTERNAL_EVENT_TYPE_ATTRIBUTE_SCHEMA: `#graphql From f77f9e0f157c2ed4427257c99114f8be3c338ed1 Mon Sep 17 00:00:00 2001 From: psubram3 Date: Wed, 13 Nov 2024 14:29:38 -0800 Subject: [PATCH 03/39] check event types in external source --- src/packages/external-source/external-source.ts | 11 +++++++---- src/packages/external-source/gql.ts | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 91074f7..aef0b69 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -11,6 +11,7 @@ import { HasuraError } from '../../types/hasura.js'; type CreateExternalSourceResponse = { data: { createExternalSource: { name: string } | null } }; type CreateExternalSourceTypeResponse = { data: { createExternalSourceType: { attribute_schema: object, name: string } | null } }; type ExistingTypesResponse = { data: {existingEventTypes: { name: string }[]}}; +type AssociatedTypesResponse = { data: {existingEventTypes: { external_event_type: string }[]}}; type GetExternalSourceTypeAttributeSchemaResponse = { data: { external_source_type_by_pk: { attribute_schema: object } | null } }; type GetExternalEventTypeAttributeSchemaResponse = { data: { external_event_type_by_pk: { attribute_schema: object } | null } }; @@ -30,8 +31,6 @@ async function uploadExternalSourceType(req: Request, res: Response) { const { body } = req; const { external_source_type_name, attribute_schema, allowed_event_types } = body; - console.log(body) - const allowed_event_types_parsed = allowed_event_types as string[]; const headers: HeadersInit = { @@ -183,9 +182,11 @@ async function uploadExternalSource(req: Request, res: Response) { }); // check - const allowedEventTypes = await allowedExternalEventTypes.json() as ExistingTypesResponse; + const allowedEventTypesStruct: AssociatedTypesResponse = await allowedExternalEventTypes.json(); + const allowedEventTypes = allowedEventTypesStruct.data.existingEventTypes.map(eventType => eventType.external_event_type); + for (const event_type of usedExternalEventTypes) { - if (!allowedEventTypes.data.existingEventTypes.includes(event_type)) { + if (!allowedEventTypes.includes(event_type)) { logger.error(`POST /uploadExternalSourceType: An event uses event type ${event_type}, which is not defined for source type ${source_type_name}.`); res.status(500); res.send(`POST /uploadExternalSourceType: An event uses event type ${event_type}, which is not defined for source type ${source_type_name}.`); @@ -193,6 +194,8 @@ async function uploadExternalSource(req: Request, res: Response) { } } + logger.info(`POST /uploadExternalSource: Source's included events' types are valid.`); + // Get the attribute schema(s) for all external event types used by the source's events const usedExternalEventTypesAttributesSchemas = await usedExternalEventTypes.reduce(async (acc: Record, eventType: string) => { const eventAttributeSchema = await fetch(GQL_API_URL, { diff --git a/src/packages/external-source/gql.ts b/src/packages/external-source/gql.ts index ad0a70b..619c83e 100644 --- a/src/packages/external-source/gql.ts +++ b/src/packages/external-source/gql.ts @@ -46,8 +46,8 @@ export default { } `, GET_EXTERNAL_EVENT_TYPES_FOR_SOURCE_TYPE: `#graphql - query ExistingEventTypesForSourceType($sourceType: string!) { - existingEventTypesForSourceType: external_source_type_allowed_event_types(where: {name: {_eq: $sourceType}}) { + query ExistingEventTypesForSourceType($sourceType: String!) { + existingEventTypes: external_source_type_allowed_event_types(where: {external_source_type: {_eq: $sourceType}}) { external_event_type } } From 78c94a802f502197025bda5e9a8f0ca451e57f4f Mon Sep 17 00:00:00 2001 From: psubram3 Date: Thu, 14 Nov 2024 14:12:04 -0800 Subject: [PATCH 04/39] ensure external event attributes actually checked --- src/packages/external-event/external-event.ts | 2 ++ .../external-source/external-source.ts | 31 ++++++++++--------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/packages/external-event/external-event.ts b/src/packages/external-event/external-event.ts index c84a671..e20d6a0 100644 --- a/src/packages/external-event/external-event.ts +++ b/src/packages/external-event/external-event.ts @@ -32,6 +32,8 @@ async function uploadExternalEventType(req: Request, res: Response) { 'x-hasura-user-id': userHeader ? `${userHeader}` : '', }; + console.log("\n\n", JSON.stringify(attribute_schema), "\n\n") + // Validate schema is valid JSON Schema try { const schemaIsValid: boolean = ajv.validateSchema(attribute_schema); diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index aef0b69..f756a47 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -1,6 +1,6 @@ import type { Express, Request, Response } from 'express'; import type { DerivationGroupInsertInput, ExternalSourceInsertInput, ExternalSourceTypeInsertInput } from '../../types/external-source.js'; -import type { ExternalEvent, ExternalEventInsertInput } from '../../types/external-event.js'; +import type { ExternalEventInsertInput } from '../../types/external-event.js'; import Ajv from 'ajv'; import { getEnv } from '../../env.js'; import getLogger from '../../logger.js'; @@ -151,7 +151,7 @@ async function uploadExternalSource(req: Request, res: Response) { if ((getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse).data?.external_source_type_by_pk?.attribute_schema !== null) { const { data: { external_source_type_by_pk: sourceAttributeSchema } } = getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse; if (sourceAttributeSchema !== undefined && sourceAttributeSchema !== null) { - const sourceSchema = ajv.compile(sourceAttributeSchema.attribute_schema); + const sourceSchema: Ajv.ValidateFunction = ajv.compile(sourceAttributeSchema.attribute_schema); sourceAttributesAreValid = await sourceSchema(attributes); } } @@ -159,6 +159,9 @@ async function uploadExternalSource(req: Request, res: Response) { logger.info(`POST /uploadExternalSource: Source's attributes are valid`); } else { logger.error(`POST /uploadExternalSource: Source's attributes are invalid`); + res.status(500); + res.send(`POST /uploadExternalSource: Source's attributes are invalid:\n${JSON.stringify(ajv.errors)}`); + return; } // TODO: verify events are all of allowed type @@ -197,7 +200,7 @@ async function uploadExternalSource(req: Request, res: Response) { logger.info(`POST /uploadExternalSource: Source's included events' types are valid.`); // Get the attribute schema(s) for all external event types used by the source's events - const usedExternalEventTypesAttributesSchemas = await usedExternalEventTypes.reduce(async (acc: Record, eventType: string) => { + const usedExternalEventTypesAttributesSchemas: Record = await usedExternalEventTypes.reduce(async (acc: Record, eventType: string) => { const eventAttributeSchema = await fetch(GQL_API_URL, { body: JSON.stringify({ query: gql.GET_EXTERNAL_EVENT_TYPE_ATTRIBUTE_SCHEMA, @@ -212,31 +215,29 @@ async function uploadExternalSource(req: Request, res: Response) { const getExternalEventTypeAttributeSchemaResponse = eventTypeJSONResponse as GetExternalEventTypeAttributeSchemaResponse | HasuraError; if ((getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse).data?.external_event_type_by_pk?.attribute_schema !== null) { - console.log(getExternalEventTypeAttributeSchemaResponse); const { data: { external_event_type_by_pk: eventAttributeSchema } } = getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse; if (eventAttributeSchema !== undefined && eventAttributeSchema !== null) { - acc[eventType] = ajv.compile(eventAttributeSchema); + acc[eventType] = ajv.compile(eventAttributeSchema.attribute_schema); } } return acc; }, {} as Record) - // Validate all the event's attributes - try { - external_events.data.forEach(async (externalEvent: ExternalEvent) => { + for (const externalEvent of external_events.data) { + try { const currentEventType = externalEvent.event_type_name; - const currentEventSchema = usedExternalEventTypesAttributesSchemas[currentEventType]; + const currentEventSchema: Ajv.ValidateFunction = usedExternalEventTypesAttributesSchemas[currentEventType]; const eventAttributesAreValid = await currentEventSchema(externalEvent.attributes); if (!eventAttributesAreValid) { throw new Error(`External Event '${externalEvent.key}' does not have a valid set of attributes, per it's type's schema.`); } - }); - } catch (e) { - logger.error(`POST /uploadExternalSource: ${(e as Error).message}`); - res.status(500); - res.send((e as Error).message); - return; + } catch (e) { + logger.error(`POST /uploadExternalSource: ${(e as Error).message}`); + res.status(500); + res.send((e as Error).message); + return; + } } console.log("VALID!"); From ca07a178ddf8a7e643867831338feeb201822e63 Mon Sep 17 00:00:00 2001 From: psubram3 Date: Fri, 15 Nov 2024 11:12:05 -0800 Subject: [PATCH 05/39] add external source validation to gateway --- .../external-source/external-source.ts | 43 ++++-- src/packages/external-source/gql.ts | 2 +- .../external-event-validation-schemata.ts | 140 ++++++++++++++++++ 3 files changed, 175 insertions(+), 10 deletions(-) create mode 100644 src/packages/schemas/external-event-validation-schemata.ts diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index f756a47..eda3613 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -5,7 +5,7 @@ import Ajv from 'ajv'; import { getEnv } from '../../env.js'; import getLogger from '../../logger.js'; import gql from './gql.js'; -//import { gql as externalEventGQL } from '../external-event/gql.js'; +import { externalSourceSchema } from '../schemas/external-event-validation-schemata.js'; import { HasuraError } from '../../types/hasura.js'; type CreateExternalSourceResponse = { data: { createExternalSource: { name: string } | null } }; @@ -19,6 +19,7 @@ const logger = getLogger('packages/external-source/external-source'); const { HASURA_API_URL } = getEnv(); const GQL_API_URL = `${HASURA_API_URL}/v1/graphql`; const ajv = new Ajv(); +const compiledExternalSourceSchema = ajv.compile(externalSourceSchema); async function uploadExternalSourceType(req: Request, res: Response) { logger.info(`POST /uploadExternalSourceType: Entering function...`); @@ -114,16 +115,22 @@ async function uploadExternalSource(req: Request, res: Response) { headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, } = req; const { body } = req; + const { + external_events, + source + } = body; const { attributes, derivation_group_name, - end_time, - external_events, key, source_type_name, - start_time, + period, valid_at - } = body; + } = source; + const { + end_time, + start_time + } = period; const headers: HeadersInit = { Authorization: authorizationHeader ?? '', 'Content-Type': 'application/json', @@ -132,6 +139,21 @@ async function uploadExternalSource(req: Request, res: Response) { 'x-hasura-user-id': userHeader ? `${userHeader}` : '', }; + + console.log("\n\n", body, "\n\n"); + + // Verify that this is a valid external source! + let sourceIsValid: boolean = false; + sourceIsValid = await compiledExternalSourceSchema(body); + if (sourceIsValid) { + logger.info(`POST /uploadExternalSource: Source's formatting is valid per basic schema validation.`); + } else { + logger.error(`POST /uploadExternalSource: Source's formatting is invalid per basic schema validation:\n${JSON.stringify(compiledExternalSourceSchema.errors)}`); + res.status(500); + res.send(`POST /uploadExternalSource: Source's formatting is invalid per basic schema validation:\n${JSON.stringify(compiledExternalSourceSchema.errors)}`); + return; + } + // Get the attribute schema for the source's external source type const sourceAttributeSchema = await fetch(GQL_API_URL, { body: JSON.stringify({ @@ -149,6 +171,7 @@ async function uploadExternalSource(req: Request, res: Response) { const sourceTypeResponseJSON = await sourceAttributeSchema.json(); const getExternalSourceTypeAttributeSchemaResponse = sourceTypeResponseJSON as GetExternalSourceTypeAttributeSchemaResponse | HasuraError; if ((getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse).data?.external_source_type_by_pk?.attribute_schema !== null) { + console.log(sourceTypeResponseJSON, source_type_name) const { data: { external_source_type_by_pk: sourceAttributeSchema } } = getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse; if (sourceAttributeSchema !== undefined && sourceAttributeSchema !== null) { const sourceSchema: Ajv.ValidateFunction = ajv.compile(sourceAttributeSchema.attribute_schema); @@ -166,7 +189,7 @@ async function uploadExternalSource(req: Request, res: Response) { // TODO: verify events are all of allowed type // get list of all used event types - const usedExternalEventTypes = external_events.data.map((externalEvent: ExternalEventInsertInput) => externalEvent.event_type_name).reduce( + const usedExternalEventTypes = external_events.map((externalEvent: ExternalEventInsertInput) => externalEvent.event_type_name).reduce( (acc: string[], externalEventType: string) => { if (!acc.includes(externalEventType)) { acc.push(externalEventType) @@ -224,13 +247,13 @@ async function uploadExternalSource(req: Request, res: Response) { return acc; }, {} as Record) - for (const externalEvent of external_events.data) { + for (const externalEvent of external_events) { try { const currentEventType = externalEvent.event_type_name; const currentEventSchema: Ajv.ValidateFunction = usedExternalEventTypesAttributesSchemas[currentEventType]; const eventAttributesAreValid = await currentEventSchema(externalEvent.attributes); if (!eventAttributesAreValid) { - throw new Error(`External Event '${externalEvent.key}' does not have a valid set of attributes, per it's type's schema.`); + throw new Error(`External Event '${externalEvent.key}' does not have a valid set of attributes, per it's type's schema:\n${JSON.stringify(currentEventSchema.errors)}`); } } catch (e) { logger.error(`POST /uploadExternalSource: ${(e as Error).message}`); @@ -252,7 +275,9 @@ async function uploadExternalSource(req: Request, res: Response) { attributes: attributes, derivation_group_name: derivation_group_name, end_time: end_time, - external_events: external_events, + external_events: { + data: external_events + }, key: key, source_type_name: source_type_name, start_time: start_time, diff --git a/src/packages/external-source/gql.ts b/src/packages/external-source/gql.ts index 619c83e..b2cc75b 100644 --- a/src/packages/external-source/gql.ts +++ b/src/packages/external-source/gql.ts @@ -1,4 +1,4 @@ -export default { +export default { // TODO: discuss upset for derivation group CREATE_EXTERNAL_SOURCE: `#graphql mutation CreateExternalSource( $derivation_group: derivation_group_insert_input!, diff --git a/src/packages/schemas/external-event-validation-schemata.ts b/src/packages/schemas/external-event-validation-schemata.ts new file mode 100644 index 0000000..5fcabd8 --- /dev/null +++ b/src/packages/schemas/external-event-validation-schemata.ts @@ -0,0 +1,140 @@ +// TODO: Discuss external event/source type schemas. Do we want to validate these with a (meta)schema too? Or just allow any plain JSON Schema? +// Currently, we do the latter but the former doesn't seem like a bad idea! +// The main argument against the former is what we have works and introducing new schemas could be a rabbit hole. + + +// export const externalEventTypeSchema = { +// additionalProperties: false, +// properties: { +// entries: { +// items: { +// additionalProperties: false, +// properties: { +// metadata: { +// items: { +// additionalProperties: false, +// properties: { +// isRequired: { type: 'boolean' }, +// name: { type: 'string' }, +// schema: { +// additionalProperties: false, +// properties: { type: { type: 'string' } }, +// required: ['type'], +// type: 'object', +// }, +// }, +// required: ['name', 'isRequired', 'schema'], +// type: 'object', +// }, +// type: 'array', +// }, +// name: { type: 'string' }, +// }, +// required: ['name', 'metadata'], +// type: 'object', +// }, +// type: 'array', +// }, +// }, +// required: ['entries'], +// type: 'object', +// }; +// export const externalSourceTypeSchema = { +// additionalProperties: false, +// properties: { +// entries: { +// items: { +// additionalProperties: false, +// properties: { +// metadata: { +// items: { +// additionalProperties: false, +// properties: { +// isRequired: { type: 'boolean' }, +// name: { type: 'string' }, +// schema: { +// additionalProperties: false, +// properties: { type: { type: 'string' } }, +// required: ['type'], +// type: 'object', +// }, +// }, +// required: ['name', 'isRequired', 'schema'], +// type: 'object', +// }, +// type: 'array', +// }, +// name: { type: 'string' }, +// }, +// required: ['name', 'metadata'], +// type: 'object', +// }, +// type: 'array', +// }, +// }, +// required: ['entries'], +// type: 'object', +// }; + +export const externalSourceSchema = { + additionalProperties: false, + properties: { + external_events: { + items: { + additionalProperties: false, + properties: { + attributes: { + additionalProperties: true, + properties: {}, + required: [], + type: 'object' + }, + duration: { type: 'string' }, + event_type_name: { type: 'string' }, + key: { type: 'string' }, + start_time: { type: 'string' }, + }, + required: ['duration', 'event_type_name', 'key', 'attributes', 'start_time'], + type: 'object' + }, + type: 'array' + }, + source: { + additionalProperties: false, + properties: { + attributes: { + additionalProperties: true, + properties: {}, // constrained by type, checked by DB trigger on upload. TODO: CHECK LOCALLY? + required: [], + type: 'object' + }, + derivation_group_name: { type: 'string' }, + key: { type: 'string' }, + period: { + additionalProperties: false, + properties: { + end_time: { + pattern: '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', + type: 'string' + }, + start_time: { + pattern: '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', + type: 'string' + } + }, + required: ['start_time', 'end_time'], + type: 'object' + }, + source_type_name: { type: "string" }, + valid_at: { + pattern: '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', + type: "string" + } + }, + required: ["key", "source_type_name", "valid_at", "period", "attributes"], + type: 'object' + } + }, + required: ['source', 'external_events'], + type: 'object' +} \ No newline at end of file From 6995e660da866d8316b8464aad8e69d3f889022a Mon Sep 17 00:00:00 2001 From: psubram3 Date: Fri, 15 Nov 2024 14:11:54 -0800 Subject: [PATCH 06/39] add validation tests --- package-lock.json | 683 +++++++++++++----------- package.json | 2 +- test/external_source.validation.test.ts | 186 +++++++ 3 files changed, 556 insertions(+), 315 deletions(-) create mode 100644 test/external_source.validation.test.ts diff --git a/package-lock.json b/package-lock.json index b05c7e3..9d4a3e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,7 +43,7 @@ "nodemon": "^3.1.4", "prettier": "^3.0.0", "typescript": "^5.1.6", - "vitest": "^1.2.2" + "vitest": "^1.4.0" }, "engines": { "node": ">=20.0.0", @@ -587,9 +587,9 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.4.15", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", - "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", "dev": true }, "node_modules/@jsdevtools/ono": { @@ -633,9 +633,9 @@ } }, "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.18.0.tgz", - "integrity": "sha512-Tya6xypR10giZV1XzxmH5wr25VcZSncG0pZIjfePT0OVBvqNEurzValetGNarVrGiq66EBVAFn15iYX4w6FKgQ==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.27.2.tgz", + "integrity": "sha512-Tj+j7Pyzd15wAdSJswvs5CJzJNV+qqSUcr/aCD+jpQSBtXvGnV0pnrjoc8zFTe9fcKCatkpFpOO7yAzpO998HA==", "cpu": [ "arm" ], @@ -646,9 +646,9 @@ ] }, "node_modules/@rollup/rollup-android-arm64": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.18.0.tgz", - "integrity": "sha512-avCea0RAP03lTsDhEyfy+hpfr85KfyTctMADqHVhLAF3MlIkq83CP8UfAHUssgXTYd+6er6PaAhx/QGv4L1EiA==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.27.2.tgz", + "integrity": "sha512-xsPeJgh2ThBpUqlLgRfiVYBEf/P1nWlWvReG+aBWfNv3XEBpa6ZCmxSVnxJgLgkNz4IbxpLy64h2gCmAAQLneQ==", "cpu": [ "arm64" ], @@ -659,9 +659,9 @@ ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.18.0.tgz", - "integrity": "sha512-IWfdwU7KDSm07Ty0PuA/W2JYoZ4iTj3TUQjkVsO/6U+4I1jN5lcR71ZEvRh52sDOERdnNhhHU57UITXz5jC1/w==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.27.2.tgz", + "integrity": "sha512-KnXU4m9MywuZFedL35Z3PuwiTSn/yqRIhrEA9j+7OSkji39NzVkgxuxTYg5F8ryGysq4iFADaU5osSizMXhU2A==", "cpu": [ "arm64" ], @@ -672,9 +672,9 @@ ] }, "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.18.0.tgz", - "integrity": "sha512-n2LMsUz7Ynu7DoQrSQkBf8iNrjOGyPLrdSg802vk6XT3FtsgX6JbE8IHRvposskFm9SNxzkLYGSq9QdpLYpRNA==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.27.2.tgz", + "integrity": "sha512-Hj77A3yTvUeCIx/Vi+4d4IbYhyTwtHj07lVzUgpUq9YpJSEiGJj4vXMKwzJ3w5zp5v3PFvpJNgc/J31smZey6g==", "cpu": [ "x64" ], @@ -684,10 +684,36 @@ "darwin" ] }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.27.2.tgz", + "integrity": "sha512-RjgKf5C3xbn8gxvCm5VgKZ4nn0pRAIe90J0/fdHUsgztd3+Zesb2lm2+r6uX4prV2eUByuxJNdt647/1KPRq5g==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.27.2.tgz", + "integrity": "sha512-duq21FoXwQtuws+V9H6UZ+eCBc7fxSpMK1GQINKn3fAyd9DFYKPJNcUhdIKOrMFjLEJgQskoMoiuizMt+dl20g==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ] + }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.18.0.tgz", - "integrity": "sha512-C/zbRYRXFjWvz9Z4haRxcTdnkPt1BtCkz+7RtBSuNmKzMzp3ZxdM28Mpccn6pt28/UWUCTXa+b0Mx1k3g6NOMA==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.27.2.tgz", + "integrity": "sha512-6npqOKEPRZkLrMcvyC/32OzJ2srdPzCylJjiTJT2c0bwwSGm7nz2F9mNQ1WrAqCBZROcQn91Fno+khFhVijmFA==", "cpu": [ "arm" ], @@ -698,9 +724,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.18.0.tgz", - "integrity": "sha512-l3m9ewPgjQSXrUMHg93vt0hYCGnrMOcUpTz6FLtbwljo2HluS4zTXFy2571YQbisTnfTKPZ01u/ukJdQTLGh9A==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.27.2.tgz", + "integrity": "sha512-V9Xg6eXtgBtHq2jnuQwM/jr2mwe2EycnopO8cbOvpzFuySCGtKlPCI3Hj9xup/pJK5Q0388qfZZy2DqV2J8ftw==", "cpu": [ "arm" ], @@ -711,9 +737,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.18.0.tgz", - "integrity": "sha512-rJ5D47d8WD7J+7STKdCUAgmQk49xuFrRi9pZkWoRD1UeSMakbcepWXPF8ycChBoAqs1pb2wzvbY6Q33WmN2ftw==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.27.2.tgz", + "integrity": "sha512-uCFX9gtZJoQl2xDTpRdseYuNqyKkuMDtH6zSrBTA28yTfKyjN9hQ2B04N5ynR8ILCoSDOrG/Eg+J2TtJ1e/CSA==", "cpu": [ "arm64" ], @@ -724,9 +750,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.18.0.tgz", - "integrity": "sha512-be6Yx37b24ZwxQ+wOQXXLZqpq4jTckJhtGlWGZs68TgdKXJgw54lUUoFYrg6Zs/kjzAQwEwYbp8JxZVzZLRepQ==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.27.2.tgz", + "integrity": "sha512-/PU9P+7Rkz8JFYDHIi+xzHabOu9qEWR07L5nWLIUsvserrxegZExKCi2jhMZRd0ATdboKylu/K5yAXbp7fYFvA==", "cpu": [ "arm64" ], @@ -737,9 +763,9 @@ ] }, "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.18.0.tgz", - "integrity": "sha512-hNVMQK+qrA9Todu9+wqrXOHxFiD5YmdEi3paj6vP02Kx1hjd2LLYR2eaN7DsEshg09+9uzWi2W18MJDlG0cxJA==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.27.2.tgz", + "integrity": "sha512-eCHmol/dT5odMYi/N0R0HC8V8QE40rEpkyje/ZAXJYNNoSfrObOvG/Mn+s1F/FJyB7co7UQZZf6FuWnN6a7f4g==", "cpu": [ "ppc64" ], @@ -750,9 +776,9 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.18.0.tgz", - "integrity": "sha512-ROCM7i+m1NfdrsmvwSzoxp9HFtmKGHEqu5NNDiZWQtXLA8S5HBCkVvKAxJ8U+CVctHwV2Gb5VUaK7UAkzhDjlg==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.27.2.tgz", + "integrity": "sha512-DEP3Njr9/ADDln3kNi76PXonLMSSMiCir0VHXxmGSHxCxDfQ70oWjHcJGfiBugzaqmYdTC7Y+8Int6qbnxPBIQ==", "cpu": [ "riscv64" ], @@ -763,9 +789,9 @@ ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.18.0.tgz", - "integrity": "sha512-0UyyRHyDN42QL+NbqevXIIUnKA47A+45WyasO+y2bGJ1mhQrfrtXUpTxCOrfxCR4esV3/RLYyucGVPiUsO8xjg==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.27.2.tgz", + "integrity": "sha512-NHGo5i6IE/PtEPh5m0yw5OmPMpesFnzMIS/lzvN5vknnC1sXM5Z/id5VgcNPgpD+wHmIcuYYgW+Q53v+9s96lQ==", "cpu": [ "s390x" ], @@ -776,9 +802,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.18.0.tgz", - "integrity": "sha512-xuglR2rBVHA5UsI8h8UbX4VJ470PtGCf5Vpswh7p2ukaqBGFTnsfzxUBetoWBWymHMxbIG0Cmx7Y9qDZzr648w==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.27.2.tgz", + "integrity": "sha512-PaW2DY5Tan+IFvNJGHDmUrORadbe/Ceh8tQxi8cmdQVCCYsLoQo2cuaSj+AU+YRX8M4ivS2vJ9UGaxfuNN7gmg==", "cpu": [ "x64" ], @@ -789,9 +815,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.18.0.tgz", - "integrity": "sha512-LKaqQL9osY/ir2geuLVvRRs+utWUNilzdE90TpyoX0eNqPzWjRm14oMEE+YLve4k/NAqCdPkGYDaDF5Sw+xBfg==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.27.2.tgz", + "integrity": "sha512-dOlWEMg2gI91Qx5I/HYqOD6iqlJspxLcS4Zlg3vjk1srE67z5T2Uz91yg/qA8sY0XcwQrFzWWiZhMNERylLrpQ==", "cpu": [ "x64" ], @@ -802,9 +828,9 @@ ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.18.0.tgz", - "integrity": "sha512-7J6TkZQFGo9qBKH0pk2cEVSRhJbL6MtfWxth7Y5YmZs57Pi+4x6c2dStAUvaQkHQLnEQv1jzBUW43GvZW8OFqA==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.27.2.tgz", + "integrity": "sha512-euMIv/4x5Y2/ImlbGl88mwKNXDsvzbWUlT7DFky76z2keajCtcbAsN9LUdmk31hAoVmJJYSThgdA0EsPeTr1+w==", "cpu": [ "arm64" ], @@ -815,9 +841,9 @@ ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.18.0.tgz", - "integrity": "sha512-Txjh+IxBPbkUB9+SXZMpv+b/vnTEtFyfWZgJ6iyCmt2tdx0OF5WhFowLmnh8ENGNpfUlUZkdI//4IEmhwPieNg==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.27.2.tgz", + "integrity": "sha512-RsnE6LQkUHlkC10RKngtHNLxb7scFykEbEwOFDjr3CeCMG+Rr+cKqlkKc2/wJ1u4u990urRHCbjz31x84PBrSQ==", "cpu": [ "ia32" ], @@ -828,9 +854,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.18.0.tgz", - "integrity": "sha512-UOo5FdvOL0+eIVTgS4tIdbW+TtnBLWg1YBCcU2KWM7nuNwRz9bksDX1bekJJCpu25N1DVWaCwnT39dVQxzqS8g==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.27.2.tgz", + "integrity": "sha512-foJM5vv+z2KQmn7emYdDLyTbkoO5bkHZE1oth2tWbQNGW7mX32d46Hz6T0MqXdWS2vBZhaEtHqdy9WYwGfiliA==", "cpu": [ "x64" ], @@ -889,9 +915,9 @@ } }, "node_modules/@types/estree": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz", - "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", + "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", "dev": true }, "node_modules/@types/express": { @@ -1219,13 +1245,13 @@ } }, "node_modules/@vitest/expect": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-1.2.2.tgz", - "integrity": "sha512-3jpcdPAD7LwHUUiT2pZTj2U82I2Tcgg2oVPvKxhn6mDI2On6tfvPQTjAI4628GUGDZrCm4Zna9iQHm5cEexOAg==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-1.6.0.tgz", + "integrity": "sha512-ixEvFVQjycy/oNgHjqsL6AZCDduC+tflRluaHIzKIsdbzkLn2U/iBnVeJwB6HsIjQBdfMR8Z0tRxKUsvFJEeWQ==", "dev": true, "dependencies": { - "@vitest/spy": "1.2.2", - "@vitest/utils": "1.2.2", + "@vitest/spy": "1.6.0", + "@vitest/utils": "1.6.0", "chai": "^4.3.10" }, "funding": { @@ -1233,12 +1259,12 @@ } }, "node_modules/@vitest/runner": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-1.2.2.tgz", - "integrity": "sha512-JctG7QZ4LSDXr5CsUweFgcpEvrcxOV1Gft7uHrvkQ+fsAVylmWQvnaAr/HDp3LAH1fztGMQZugIheTWjaGzYIg==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-1.6.0.tgz", + "integrity": "sha512-P4xgwPjwesuBiHisAVz/LSSZtDjOTPYZVmNAnpHHSR6ONrf8eCJOFRvUwdHn30F5M1fxhqtl7QZQUk2dprIXAg==", "dev": true, "dependencies": { - "@vitest/utils": "1.2.2", + "@vitest/utils": "1.6.0", "p-limit": "^5.0.0", "pathe": "^1.1.1" }, @@ -1262,9 +1288,9 @@ } }, "node_modules/@vitest/runner/node_modules/yocto-queue": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", - "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.1.1.tgz", + "integrity": "sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==", "dev": true, "engines": { "node": ">=12.20" @@ -1274,9 +1300,9 @@ } }, "node_modules/@vitest/snapshot": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-1.2.2.tgz", - "integrity": "sha512-SmGY4saEw1+bwE1th6S/cZmPxz/Q4JWsl7LvbQIky2tKE35US4gd0Mjzqfr84/4OD0tikGWaWdMja/nWL5NIPA==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-1.6.0.tgz", + "integrity": "sha512-+Hx43f8Chus+DCmygqqfetcAZrDJwvTj0ymqjQq4CvmpKFSTVteEOBzCusu1x2tt4OJcvBflyHUE0DZSLgEMtQ==", "dev": true, "dependencies": { "magic-string": "^0.30.5", @@ -1288,9 +1314,9 @@ } }, "node_modules/@vitest/spy": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-1.2.2.tgz", - "integrity": "sha512-k9Gcahssw8d7X3pSLq3e3XEu/0L78mUkCjivUqCQeXJm9clfXR/Td8+AP+VC1O6fKPIDLcHDTAmBOINVuv6+7g==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-1.6.0.tgz", + "integrity": "sha512-leUTap6B/cqi/bQkXUu6bQV5TZPx7pmMBKBQiI0rJA8c3pB56ZsaTbREnF7CJfmvAS4V2cXIBAh/3rVwrrCYgw==", "dev": true, "dependencies": { "tinyspy": "^2.2.0" @@ -1300,9 +1326,9 @@ } }, "node_modules/@vitest/utils": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-1.2.2.tgz", - "integrity": "sha512-WKITBHLsBHlpjnDQahr+XK6RE7MiAsgrIkr0pGhQ9ygoxBfUeG0lUG5iLlzqjmKSlBv3+j5EGsriBzh+C3Tq9g==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-1.6.0.tgz", + "integrity": "sha512-21cPiuGMoMZwiOHa2i4LXkMkMkCGzA+MVFV70jRwHo95dL4x/ts5GZhML1QWuy7yfp3WzK3lRvZi3JnXTYqrBw==", "dev": true, "dependencies": { "diff-sequences": "^29.6.3", @@ -1628,9 +1654,9 @@ } }, "node_modules/chai": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/chai/-/chai-4.4.1.tgz", - "integrity": "sha512-13sOfMv2+DWduEU+/xbun3LScLoqN17nBeTLUsmDfKdoiC1fr0n9PU4guu4AhRcOVFk/sW8LyZWHuhWtQZiF+g==", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/chai/-/chai-4.5.0.tgz", + "integrity": "sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==", "dev": true, "dependencies": { "assertion-error": "^1.1.0", @@ -1639,7 +1665,7 @@ "get-func-name": "^2.0.2", "loupe": "^2.3.6", "pathval": "^1.1.1", - "type-detect": "^4.0.8" + "type-detect": "^4.1.0" }, "engines": { "node": ">=4" @@ -1905,9 +1931,9 @@ } }, "node_modules/deep-eql": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.3.tgz", - "integrity": "sha512-WaEtAOpRA1MQ0eohqZjpGD8zdI0Ovsm8mmFhaDN8dvDZzyoUMcYDnf5Y6iu7HTXxf8JDS23qWa4a+hKCDyOPzw==", + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.4.tgz", + "integrity": "sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg==", "dev": true, "dependencies": { "type-detect": "^4.0.0" @@ -2999,6 +3025,12 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true }, + "node_modules/js-tokens": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.0.tgz", + "integrity": "sha512-WriZw1luRMlmV3LGJaR6QOJjWwgLUTf89OwT2lUOyjX2dJGBwgmIkbcz+7WFZjrZM635JOIR517++e/67CP9dQ==", + "dev": true + }, "node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -3171,15 +3203,12 @@ } }, "node_modules/magic-string": { - "version": "0.30.7", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.7.tgz", - "integrity": "sha512-8vBuFF/I/+OSLRmdf2wwFCJCz+nSn0m6DPvGH1fS/KiQoSaR+sETbov0eIk9KhEKy8CYqIkIAnbohxT/4H0kuA==", + "version": "0.30.12", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.12.tgz", + "integrity": "sha512-Ea8I3sQMVXr8JhN4z+H/d8zwo+tYDgHE9+5G4Wnrwhs0gaK9fXTKx0Tw5Xwsd/bCPTTZNRAdpyzvoeORe9LYpw==", "dev": true, "dependencies": { - "@jridgewell/sourcemap-codec": "^1.4.15" - }, - "engines": { - "node": ">=12" + "@jridgewell/sourcemap-codec": "^1.5.0" } }, "node_modules/media-typer": { @@ -3837,9 +3866,9 @@ } }, "node_modules/picocolors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", - "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", "dev": true }, "node_modules/picomatch": { @@ -3866,9 +3895,9 @@ } }, "node_modules/postcss": { - "version": "8.4.38", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.38.tgz", - "integrity": "sha512-Wglpdk03BSfXkHoQa3b/oulrotAkwrlLDRSOb9D0bN86FdRyE9lppSp33aHNPgBa0JKCoB+drFLZkQoRRYae5A==", + "version": "8.4.49", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.49.tgz", + "integrity": "sha512-OCVPnIObs4N29kxTjzLfUryOkvZEq+pf8jTF0lg8E7uETuWHA+v7j3c/xJmiqpX450191LlmZfUKkXxkTry7nA==", "dev": true, "funding": [ { @@ -3886,8 +3915,8 @@ ], "dependencies": { "nanoid": "^3.3.7", - "picocolors": "^1.0.0", - "source-map-js": "^1.2.0" + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" }, "engines": { "node": "^10 || ^12 || >=14" @@ -4095,9 +4124,9 @@ } }, "node_modules/react-is": { - "version": "18.2.0", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz", - "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==", + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", "dev": true }, "node_modules/readable-stream": { @@ -4166,12 +4195,12 @@ } }, "node_modules/rollup": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.18.0.tgz", - "integrity": "sha512-QmJz14PX3rzbJCN1SG4Xe/bAAX2a6NpCP8ab2vfu2GiUr8AQcr2nCV/oEO3yneFarB67zk8ShlIyWb2LGTb3Sg==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.27.2.tgz", + "integrity": "sha512-KreA+PzWmk2yaFmZVwe6GB2uBD86nXl86OsDkt1bJS9p3vqWuEQ6HnJJ+j/mZi/q0920P99/MVRlB4L3crpF5w==", "dev": true, "dependencies": { - "@types/estree": "1.0.5" + "@types/estree": "1.0.6" }, "bin": { "rollup": "dist/bin/rollup" @@ -4181,22 +4210,24 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.18.0", - "@rollup/rollup-android-arm64": "4.18.0", - "@rollup/rollup-darwin-arm64": "4.18.0", - "@rollup/rollup-darwin-x64": "4.18.0", - "@rollup/rollup-linux-arm-gnueabihf": "4.18.0", - "@rollup/rollup-linux-arm-musleabihf": "4.18.0", - "@rollup/rollup-linux-arm64-gnu": "4.18.0", - "@rollup/rollup-linux-arm64-musl": "4.18.0", - "@rollup/rollup-linux-powerpc64le-gnu": "4.18.0", - "@rollup/rollup-linux-riscv64-gnu": "4.18.0", - "@rollup/rollup-linux-s390x-gnu": "4.18.0", - "@rollup/rollup-linux-x64-gnu": "4.18.0", - "@rollup/rollup-linux-x64-musl": "4.18.0", - "@rollup/rollup-win32-arm64-msvc": "4.18.0", - "@rollup/rollup-win32-ia32-msvc": "4.18.0", - "@rollup/rollup-win32-x64-msvc": "4.18.0", + "@rollup/rollup-android-arm-eabi": "4.27.2", + "@rollup/rollup-android-arm64": "4.27.2", + "@rollup/rollup-darwin-arm64": "4.27.2", + "@rollup/rollup-darwin-x64": "4.27.2", + "@rollup/rollup-freebsd-arm64": "4.27.2", + "@rollup/rollup-freebsd-x64": "4.27.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.27.2", + "@rollup/rollup-linux-arm-musleabihf": "4.27.2", + "@rollup/rollup-linux-arm64-gnu": "4.27.2", + "@rollup/rollup-linux-arm64-musl": "4.27.2", + "@rollup/rollup-linux-powerpc64le-gnu": "4.27.2", + "@rollup/rollup-linux-riscv64-gnu": "4.27.2", + "@rollup/rollup-linux-s390x-gnu": "4.27.2", + "@rollup/rollup-linux-x64-gnu": "4.27.2", + "@rollup/rollup-linux-x64-musl": "4.27.2", + "@rollup/rollup-win32-arm64-msvc": "4.27.2", + "@rollup/rollup-win32-ia32-msvc": "4.27.2", + "@rollup/rollup-win32-x64-msvc": "4.27.2", "fsevents": "~2.3.2" } }, @@ -4431,9 +4462,9 @@ } }, "node_modules/source-map-js": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", - "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", "dev": true, "engines": { "node": ">=0.10.0" @@ -4533,12 +4564,12 @@ } }, "node_modules/strip-literal": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-1.3.0.tgz", - "integrity": "sha512-PugKzOsyXpArk0yWmUwqOZecSO0GH0bPoctLcqNDH9J04pVW3lflYE0ujElBGTloevcxF5MofAOZ7C5l2b+wLg==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-2.1.0.tgz", + "integrity": "sha512-Op+UycaUt/8FbN/Z2TWPBLge3jWrP3xj10f3fnYxf052bKuS3EKs1ZQcVGjnEMdsNVAM+plXRdmjrZ/KgG3Skw==", "dev": true, "dependencies": { - "acorn": "^8.10.0" + "js-tokens": "^9.0.0" }, "funding": { "url": "https://github.com/sponsors/antfu" @@ -4638,18 +4669,18 @@ "dev": true }, "node_modules/tinypool": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-0.8.2.tgz", - "integrity": "sha512-SUszKYe5wgsxnNOVlBYO6IC+8VGWdVGZWAqUxp3UErNBtptZvWbwyUOyzNL59zigz2rCA92QiL3wvG+JDSdJdQ==", + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-0.8.4.tgz", + "integrity": "sha512-i11VH5gS6IFeLY3gMBQ00/MmLncVP7JLXOw1vlgkytLmJK7QnEr7NXf0LBdxfmNPAeyetukOk0bOYrJrFGjYJQ==", "dev": true, "engines": { "node": ">=14.0.0" } }, "node_modules/tinyspy": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-2.2.0.tgz", - "integrity": "sha512-d2eda04AN/cPOR89F7Xv5bK/jrQEhmcLFe6HFldoeO9AJtps+fqEnh486vnT/8y4bw38pSyxDcTCAq+Ks2aJTg==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-2.2.1.tgz", + "integrity": "sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A==", "dev": true, "engines": { "node": ">=14.0.0" @@ -4723,9 +4754,9 @@ } }, "node_modules/type-detect": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", - "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.1.0.tgz", + "integrity": "sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==", "dev": true, "engines": { "node": ">=4" @@ -4832,14 +4863,14 @@ } }, "node_modules/vite": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/vite/-/vite-5.3.1.tgz", - "integrity": "sha512-XBmSKRLXLxiaPYamLv3/hnP/KXDai1NDexN0FpkTaZXTfycHvkRHoenpgl/fvuK/kPbB6xAgoyiryAhQNxYmAQ==", + "version": "5.4.11", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.11.tgz", + "integrity": "sha512-c7jFQRklXua0mTzneGW9QVyxFjUgwcihC4bXEtujIo2ouWCe1Ajt/amn2PCxYnhYfd5k09JX3SB7OYWFKYqj8Q==", "dev": true, "dependencies": { "esbuild": "^0.21.3", - "postcss": "^8.4.38", - "rollup": "^4.13.0" + "postcss": "^8.4.43", + "rollup": "^4.20.0" }, "bin": { "vite": "bin/vite.js" @@ -4858,6 +4889,7 @@ "less": "*", "lightningcss": "^1.21.0", "sass": "*", + "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.4.0" @@ -4875,6 +4907,9 @@ "sass": { "optional": true }, + "sass-embedded": { + "optional": true + }, "stylus": { "optional": true }, @@ -4887,9 +4922,9 @@ } }, "node_modules/vite-node": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-1.2.2.tgz", - "integrity": "sha512-1as4rDTgVWJO3n1uHmUYqq7nsFgINQ9u+mRcXpjeOMJUmviqNKjcZB7UfRZrlM7MjYXMKpuWp5oGkjaFLnjawg==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-1.6.0.tgz", + "integrity": "sha512-de6HJgzC+TFzOu0NTC4RAIsyf/DY/ibWDYQUcuEA84EMHhcefTUGkjFHKKEJhQN4A+6I0u++kr3l36ZF2d7XRw==", "dev": true, "dependencies": { "cac": "^6.7.14", @@ -4909,18 +4944,17 @@ } }, "node_modules/vitest": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-1.2.2.tgz", - "integrity": "sha512-d5Ouvrnms3GD9USIK36KG8OZ5bEvKEkITFtnGv56HFaSlbItJuYr7hv2Lkn903+AvRAgSixiamozUVfORUekjw==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-1.6.0.tgz", + "integrity": "sha512-H5r/dN06swuFnzNFhq/dnz37bPXnq8xB2xB5JOVk8K09rUtoeNN+LHWkoQ0A/i3hvbUKKcCei9KpbxqHMLhLLA==", "dev": true, "dependencies": { - "@vitest/expect": "1.2.2", - "@vitest/runner": "1.2.2", - "@vitest/snapshot": "1.2.2", - "@vitest/spy": "1.2.2", - "@vitest/utils": "1.2.2", + "@vitest/expect": "1.6.0", + "@vitest/runner": "1.6.0", + "@vitest/snapshot": "1.6.0", + "@vitest/spy": "1.6.0", + "@vitest/utils": "1.6.0", "acorn-walk": "^8.3.2", - "cac": "^6.7.14", "chai": "^4.3.10", "debug": "^4.3.4", "execa": "^8.0.1", @@ -4929,11 +4963,11 @@ "pathe": "^1.1.1", "picocolors": "^1.0.0", "std-env": "^3.5.0", - "strip-literal": "^1.3.0", + "strip-literal": "^2.0.0", "tinybench": "^2.5.1", - "tinypool": "^0.8.2", + "tinypool": "^0.8.3", "vite": "^5.0.0", - "vite-node": "1.2.2", + "vite-node": "1.6.0", "why-is-node-running": "^2.2.2" }, "bin": { @@ -4948,8 +4982,8 @@ "peerDependencies": { "@edge-runtime/vm": "*", "@types/node": "^18.0.0 || >=20.0.0", - "@vitest/browser": "^1.0.0", - "@vitest/ui": "^1.0.0", + "@vitest/browser": "1.6.0", + "@vitest/ui": "1.6.0", "happy-dom": "*", "jsdom": "*" }, @@ -5419,9 +5453,9 @@ } }, "@jridgewell/sourcemap-codec": { - "version": "1.4.15", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", - "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", "dev": true }, "@jsdevtools/ono": { @@ -5456,114 +5490,128 @@ } }, "@rollup/rollup-android-arm-eabi": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.18.0.tgz", - "integrity": "sha512-Tya6xypR10giZV1XzxmH5wr25VcZSncG0pZIjfePT0OVBvqNEurzValetGNarVrGiq66EBVAFn15iYX4w6FKgQ==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.27.2.tgz", + "integrity": "sha512-Tj+j7Pyzd15wAdSJswvs5CJzJNV+qqSUcr/aCD+jpQSBtXvGnV0pnrjoc8zFTe9fcKCatkpFpOO7yAzpO998HA==", "dev": true, "optional": true }, "@rollup/rollup-android-arm64": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.18.0.tgz", - "integrity": "sha512-avCea0RAP03lTsDhEyfy+hpfr85KfyTctMADqHVhLAF3MlIkq83CP8UfAHUssgXTYd+6er6PaAhx/QGv4L1EiA==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.27.2.tgz", + "integrity": "sha512-xsPeJgh2ThBpUqlLgRfiVYBEf/P1nWlWvReG+aBWfNv3XEBpa6ZCmxSVnxJgLgkNz4IbxpLy64h2gCmAAQLneQ==", "dev": true, "optional": true }, "@rollup/rollup-darwin-arm64": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.18.0.tgz", - "integrity": "sha512-IWfdwU7KDSm07Ty0PuA/W2JYoZ4iTj3TUQjkVsO/6U+4I1jN5lcR71ZEvRh52sDOERdnNhhHU57UITXz5jC1/w==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.27.2.tgz", + "integrity": "sha512-KnXU4m9MywuZFedL35Z3PuwiTSn/yqRIhrEA9j+7OSkji39NzVkgxuxTYg5F8ryGysq4iFADaU5osSizMXhU2A==", "dev": true, "optional": true }, "@rollup/rollup-darwin-x64": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.18.0.tgz", - "integrity": "sha512-n2LMsUz7Ynu7DoQrSQkBf8iNrjOGyPLrdSg802vk6XT3FtsgX6JbE8IHRvposskFm9SNxzkLYGSq9QdpLYpRNA==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.27.2.tgz", + "integrity": "sha512-Hj77A3yTvUeCIx/Vi+4d4IbYhyTwtHj07lVzUgpUq9YpJSEiGJj4vXMKwzJ3w5zp5v3PFvpJNgc/J31smZey6g==", + "dev": true, + "optional": true + }, + "@rollup/rollup-freebsd-arm64": { + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.27.2.tgz", + "integrity": "sha512-RjgKf5C3xbn8gxvCm5VgKZ4nn0pRAIe90J0/fdHUsgztd3+Zesb2lm2+r6uX4prV2eUByuxJNdt647/1KPRq5g==", + "dev": true, + "optional": true + }, + "@rollup/rollup-freebsd-x64": { + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.27.2.tgz", + "integrity": "sha512-duq21FoXwQtuws+V9H6UZ+eCBc7fxSpMK1GQINKn3fAyd9DFYKPJNcUhdIKOrMFjLEJgQskoMoiuizMt+dl20g==", "dev": true, "optional": true }, "@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.18.0.tgz", - "integrity": "sha512-C/zbRYRXFjWvz9Z4haRxcTdnkPt1BtCkz+7RtBSuNmKzMzp3ZxdM28Mpccn6pt28/UWUCTXa+b0Mx1k3g6NOMA==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.27.2.tgz", + "integrity": "sha512-6npqOKEPRZkLrMcvyC/32OzJ2srdPzCylJjiTJT2c0bwwSGm7nz2F9mNQ1WrAqCBZROcQn91Fno+khFhVijmFA==", "dev": true, "optional": true }, "@rollup/rollup-linux-arm-musleabihf": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.18.0.tgz", - "integrity": "sha512-l3m9ewPgjQSXrUMHg93vt0hYCGnrMOcUpTz6FLtbwljo2HluS4zTXFy2571YQbisTnfTKPZ01u/ukJdQTLGh9A==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.27.2.tgz", + "integrity": "sha512-V9Xg6eXtgBtHq2jnuQwM/jr2mwe2EycnopO8cbOvpzFuySCGtKlPCI3Hj9xup/pJK5Q0388qfZZy2DqV2J8ftw==", "dev": true, "optional": true }, "@rollup/rollup-linux-arm64-gnu": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.18.0.tgz", - "integrity": "sha512-rJ5D47d8WD7J+7STKdCUAgmQk49xuFrRi9pZkWoRD1UeSMakbcepWXPF8ycChBoAqs1pb2wzvbY6Q33WmN2ftw==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.27.2.tgz", + "integrity": "sha512-uCFX9gtZJoQl2xDTpRdseYuNqyKkuMDtH6zSrBTA28yTfKyjN9hQ2B04N5ynR8ILCoSDOrG/Eg+J2TtJ1e/CSA==", "dev": true, "optional": true }, "@rollup/rollup-linux-arm64-musl": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.18.0.tgz", - "integrity": "sha512-be6Yx37b24ZwxQ+wOQXXLZqpq4jTckJhtGlWGZs68TgdKXJgw54lUUoFYrg6Zs/kjzAQwEwYbp8JxZVzZLRepQ==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.27.2.tgz", + "integrity": "sha512-/PU9P+7Rkz8JFYDHIi+xzHabOu9qEWR07L5nWLIUsvserrxegZExKCi2jhMZRd0ATdboKylu/K5yAXbp7fYFvA==", "dev": true, "optional": true }, "@rollup/rollup-linux-powerpc64le-gnu": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.18.0.tgz", - "integrity": "sha512-hNVMQK+qrA9Todu9+wqrXOHxFiD5YmdEi3paj6vP02Kx1hjd2LLYR2eaN7DsEshg09+9uzWi2W18MJDlG0cxJA==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.27.2.tgz", + "integrity": "sha512-eCHmol/dT5odMYi/N0R0HC8V8QE40rEpkyje/ZAXJYNNoSfrObOvG/Mn+s1F/FJyB7co7UQZZf6FuWnN6a7f4g==", "dev": true, "optional": true }, "@rollup/rollup-linux-riscv64-gnu": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.18.0.tgz", - "integrity": "sha512-ROCM7i+m1NfdrsmvwSzoxp9HFtmKGHEqu5NNDiZWQtXLA8S5HBCkVvKAxJ8U+CVctHwV2Gb5VUaK7UAkzhDjlg==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.27.2.tgz", + "integrity": "sha512-DEP3Njr9/ADDln3kNi76PXonLMSSMiCir0VHXxmGSHxCxDfQ70oWjHcJGfiBugzaqmYdTC7Y+8Int6qbnxPBIQ==", "dev": true, "optional": true }, "@rollup/rollup-linux-s390x-gnu": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.18.0.tgz", - "integrity": "sha512-0UyyRHyDN42QL+NbqevXIIUnKA47A+45WyasO+y2bGJ1mhQrfrtXUpTxCOrfxCR4esV3/RLYyucGVPiUsO8xjg==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.27.2.tgz", + "integrity": "sha512-NHGo5i6IE/PtEPh5m0yw5OmPMpesFnzMIS/lzvN5vknnC1sXM5Z/id5VgcNPgpD+wHmIcuYYgW+Q53v+9s96lQ==", "dev": true, "optional": true }, "@rollup/rollup-linux-x64-gnu": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.18.0.tgz", - "integrity": "sha512-xuglR2rBVHA5UsI8h8UbX4VJ470PtGCf5Vpswh7p2ukaqBGFTnsfzxUBetoWBWymHMxbIG0Cmx7Y9qDZzr648w==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.27.2.tgz", + "integrity": "sha512-PaW2DY5Tan+IFvNJGHDmUrORadbe/Ceh8tQxi8cmdQVCCYsLoQo2cuaSj+AU+YRX8M4ivS2vJ9UGaxfuNN7gmg==", "dev": true, "optional": true }, "@rollup/rollup-linux-x64-musl": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.18.0.tgz", - "integrity": "sha512-LKaqQL9osY/ir2geuLVvRRs+utWUNilzdE90TpyoX0eNqPzWjRm14oMEE+YLve4k/NAqCdPkGYDaDF5Sw+xBfg==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.27.2.tgz", + "integrity": "sha512-dOlWEMg2gI91Qx5I/HYqOD6iqlJspxLcS4Zlg3vjk1srE67z5T2Uz91yg/qA8sY0XcwQrFzWWiZhMNERylLrpQ==", "dev": true, "optional": true }, "@rollup/rollup-win32-arm64-msvc": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.18.0.tgz", - "integrity": "sha512-7J6TkZQFGo9qBKH0pk2cEVSRhJbL6MtfWxth7Y5YmZs57Pi+4x6c2dStAUvaQkHQLnEQv1jzBUW43GvZW8OFqA==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.27.2.tgz", + "integrity": "sha512-euMIv/4x5Y2/ImlbGl88mwKNXDsvzbWUlT7DFky76z2keajCtcbAsN9LUdmk31hAoVmJJYSThgdA0EsPeTr1+w==", "dev": true, "optional": true }, "@rollup/rollup-win32-ia32-msvc": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.18.0.tgz", - "integrity": "sha512-Txjh+IxBPbkUB9+SXZMpv+b/vnTEtFyfWZgJ6iyCmt2tdx0OF5WhFowLmnh8ENGNpfUlUZkdI//4IEmhwPieNg==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.27.2.tgz", + "integrity": "sha512-RsnE6LQkUHlkC10RKngtHNLxb7scFykEbEwOFDjr3CeCMG+Rr+cKqlkKc2/wJ1u4u990urRHCbjz31x84PBrSQ==", "dev": true, "optional": true }, "@rollup/rollup-win32-x64-msvc": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.18.0.tgz", - "integrity": "sha512-UOo5FdvOL0+eIVTgS4tIdbW+TtnBLWg1YBCcU2KWM7nuNwRz9bksDX1bekJJCpu25N1DVWaCwnT39dVQxzqS8g==", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.27.2.tgz", + "integrity": "sha512-foJM5vv+z2KQmn7emYdDLyTbkoO5bkHZE1oth2tWbQNGW7mX32d46Hz6T0MqXdWS2vBZhaEtHqdy9WYwGfiliA==", "dev": true, "optional": true }, @@ -5616,9 +5664,9 @@ } }, "@types/estree": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz", - "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", + "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", "dev": true }, "@types/express": { @@ -5857,23 +5905,23 @@ } }, "@vitest/expect": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-1.2.2.tgz", - "integrity": "sha512-3jpcdPAD7LwHUUiT2pZTj2U82I2Tcgg2oVPvKxhn6mDI2On6tfvPQTjAI4628GUGDZrCm4Zna9iQHm5cEexOAg==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-1.6.0.tgz", + "integrity": "sha512-ixEvFVQjycy/oNgHjqsL6AZCDduC+tflRluaHIzKIsdbzkLn2U/iBnVeJwB6HsIjQBdfMR8Z0tRxKUsvFJEeWQ==", "dev": true, "requires": { - "@vitest/spy": "1.2.2", - "@vitest/utils": "1.2.2", + "@vitest/spy": "1.6.0", + "@vitest/utils": "1.6.0", "chai": "^4.3.10" } }, "@vitest/runner": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-1.2.2.tgz", - "integrity": "sha512-JctG7QZ4LSDXr5CsUweFgcpEvrcxOV1Gft7uHrvkQ+fsAVylmWQvnaAr/HDp3LAH1fztGMQZugIheTWjaGzYIg==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-1.6.0.tgz", + "integrity": "sha512-P4xgwPjwesuBiHisAVz/LSSZtDjOTPYZVmNAnpHHSR6ONrf8eCJOFRvUwdHn30F5M1fxhqtl7QZQUk2dprIXAg==", "dev": true, "requires": { - "@vitest/utils": "1.2.2", + "@vitest/utils": "1.6.0", "p-limit": "^5.0.0", "pathe": "^1.1.1" }, @@ -5888,17 +5936,17 @@ } }, "yocto-queue": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", - "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.1.1.tgz", + "integrity": "sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==", "dev": true } } }, "@vitest/snapshot": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-1.2.2.tgz", - "integrity": "sha512-SmGY4saEw1+bwE1th6S/cZmPxz/Q4JWsl7LvbQIky2tKE35US4gd0Mjzqfr84/4OD0tikGWaWdMja/nWL5NIPA==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-1.6.0.tgz", + "integrity": "sha512-+Hx43f8Chus+DCmygqqfetcAZrDJwvTj0ymqjQq4CvmpKFSTVteEOBzCusu1x2tt4OJcvBflyHUE0DZSLgEMtQ==", "dev": true, "requires": { "magic-string": "^0.30.5", @@ -5907,18 +5955,18 @@ } }, "@vitest/spy": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-1.2.2.tgz", - "integrity": "sha512-k9Gcahssw8d7X3pSLq3e3XEu/0L78mUkCjivUqCQeXJm9clfXR/Td8+AP+VC1O6fKPIDLcHDTAmBOINVuv6+7g==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-1.6.0.tgz", + "integrity": "sha512-leUTap6B/cqi/bQkXUu6bQV5TZPx7pmMBKBQiI0rJA8c3pB56ZsaTbREnF7CJfmvAS4V2cXIBAh/3rVwrrCYgw==", "dev": true, "requires": { "tinyspy": "^2.2.0" } }, "@vitest/utils": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-1.2.2.tgz", - "integrity": "sha512-WKITBHLsBHlpjnDQahr+XK6RE7MiAsgrIkr0pGhQ9ygoxBfUeG0lUG5iLlzqjmKSlBv3+j5EGsriBzh+C3Tq9g==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-1.6.0.tgz", + "integrity": "sha512-21cPiuGMoMZwiOHa2i4LXkMkMkCGzA+MVFV70jRwHo95dL4x/ts5GZhML1QWuy7yfp3WzK3lRvZi3JnXTYqrBw==", "dev": true, "requires": { "diff-sequences": "^29.6.3", @@ -6159,9 +6207,9 @@ "dev": true }, "chai": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/chai/-/chai-4.4.1.tgz", - "integrity": "sha512-13sOfMv2+DWduEU+/xbun3LScLoqN17nBeTLUsmDfKdoiC1fr0n9PU4guu4AhRcOVFk/sW8LyZWHuhWtQZiF+g==", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/chai/-/chai-4.5.0.tgz", + "integrity": "sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==", "dev": true, "requires": { "assertion-error": "^1.1.0", @@ -6170,7 +6218,7 @@ "get-func-name": "^2.0.2", "loupe": "^2.3.6", "pathval": "^1.1.1", - "type-detect": "^4.0.8" + "type-detect": "^4.1.0" } }, "chalk": { @@ -6381,9 +6429,9 @@ } }, "deep-eql": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.3.tgz", - "integrity": "sha512-WaEtAOpRA1MQ0eohqZjpGD8zdI0Ovsm8mmFhaDN8dvDZzyoUMcYDnf5Y6iu7HTXxf8JDS23qWa4a+hKCDyOPzw==", + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.4.tgz", + "integrity": "sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg==", "dev": true, "requires": { "type-detect": "^4.0.0" @@ -7193,6 +7241,12 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true }, + "js-tokens": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.0.tgz", + "integrity": "sha512-WriZw1luRMlmV3LGJaR6QOJjWwgLUTf89OwT2lUOyjX2dJGBwgmIkbcz+7WFZjrZM635JOIR517++e/67CP9dQ==", + "dev": true + }, "js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -7340,12 +7394,12 @@ } }, "magic-string": { - "version": "0.30.7", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.7.tgz", - "integrity": "sha512-8vBuFF/I/+OSLRmdf2wwFCJCz+nSn0m6DPvGH1fS/KiQoSaR+sETbov0eIk9KhEKy8CYqIkIAnbohxT/4H0kuA==", + "version": "0.30.12", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.12.tgz", + "integrity": "sha512-Ea8I3sQMVXr8JhN4z+H/d8zwo+tYDgHE9+5G4Wnrwhs0gaK9fXTKx0Tw5Xwsd/bCPTTZNRAdpyzvoeORe9LYpw==", "dev": true, "requires": { - "@jridgewell/sourcemap-codec": "^1.4.15" + "@jridgewell/sourcemap-codec": "^1.5.0" } }, "media-typer": { @@ -7811,9 +7865,9 @@ } }, "picocolors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", - "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", "dev": true }, "picomatch": { @@ -7834,14 +7888,14 @@ } }, "postcss": { - "version": "8.4.38", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.38.tgz", - "integrity": "sha512-Wglpdk03BSfXkHoQa3b/oulrotAkwrlLDRSOb9D0bN86FdRyE9lppSp33aHNPgBa0JKCoB+drFLZkQoRRYae5A==", + "version": "8.4.49", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.49.tgz", + "integrity": "sha512-OCVPnIObs4N29kxTjzLfUryOkvZEq+pf8jTF0lg8E7uETuWHA+v7j3c/xJmiqpX450191LlmZfUKkXxkTry7nA==", "dev": true, "requires": { "nanoid": "^3.3.7", - "picocolors": "^1.0.0", - "source-map-js": "^1.2.0" + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" }, "dependencies": { "nanoid": { @@ -7973,9 +8027,9 @@ } }, "react-is": { - "version": "18.2.0", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz", - "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==", + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", "dev": true }, "readable-stream": { @@ -8030,28 +8084,30 @@ } }, "rollup": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.18.0.tgz", - "integrity": "sha512-QmJz14PX3rzbJCN1SG4Xe/bAAX2a6NpCP8ab2vfu2GiUr8AQcr2nCV/oEO3yneFarB67zk8ShlIyWb2LGTb3Sg==", - "dev": true, - "requires": { - "@rollup/rollup-android-arm-eabi": "4.18.0", - "@rollup/rollup-android-arm64": "4.18.0", - "@rollup/rollup-darwin-arm64": "4.18.0", - "@rollup/rollup-darwin-x64": "4.18.0", - "@rollup/rollup-linux-arm-gnueabihf": "4.18.0", - "@rollup/rollup-linux-arm-musleabihf": "4.18.0", - "@rollup/rollup-linux-arm64-gnu": "4.18.0", - "@rollup/rollup-linux-arm64-musl": "4.18.0", - "@rollup/rollup-linux-powerpc64le-gnu": "4.18.0", - "@rollup/rollup-linux-riscv64-gnu": "4.18.0", - "@rollup/rollup-linux-s390x-gnu": "4.18.0", - "@rollup/rollup-linux-x64-gnu": "4.18.0", - "@rollup/rollup-linux-x64-musl": "4.18.0", - "@rollup/rollup-win32-arm64-msvc": "4.18.0", - "@rollup/rollup-win32-ia32-msvc": "4.18.0", - "@rollup/rollup-win32-x64-msvc": "4.18.0", - "@types/estree": "1.0.5", + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.27.2.tgz", + "integrity": "sha512-KreA+PzWmk2yaFmZVwe6GB2uBD86nXl86OsDkt1bJS9p3vqWuEQ6HnJJ+j/mZi/q0920P99/MVRlB4L3crpF5w==", + "dev": true, + "requires": { + "@rollup/rollup-android-arm-eabi": "4.27.2", + "@rollup/rollup-android-arm64": "4.27.2", + "@rollup/rollup-darwin-arm64": "4.27.2", + "@rollup/rollup-darwin-x64": "4.27.2", + "@rollup/rollup-freebsd-arm64": "4.27.2", + "@rollup/rollup-freebsd-x64": "4.27.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.27.2", + "@rollup/rollup-linux-arm-musleabihf": "4.27.2", + "@rollup/rollup-linux-arm64-gnu": "4.27.2", + "@rollup/rollup-linux-arm64-musl": "4.27.2", + "@rollup/rollup-linux-powerpc64le-gnu": "4.27.2", + "@rollup/rollup-linux-riscv64-gnu": "4.27.2", + "@rollup/rollup-linux-s390x-gnu": "4.27.2", + "@rollup/rollup-linux-x64-gnu": "4.27.2", + "@rollup/rollup-linux-x64-musl": "4.27.2", + "@rollup/rollup-win32-arm64-msvc": "4.27.2", + "@rollup/rollup-win32-ia32-msvc": "4.27.2", + "@rollup/rollup-win32-x64-msvc": "4.27.2", + "@types/estree": "1.0.6", "fsevents": "~2.3.2" } }, @@ -8220,9 +8276,9 @@ "dev": true }, "source-map-js": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", - "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", "dev": true }, "split2": { @@ -8294,12 +8350,12 @@ "dev": true }, "strip-literal": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-1.3.0.tgz", - "integrity": "sha512-PugKzOsyXpArk0yWmUwqOZecSO0GH0bPoctLcqNDH9J04pVW3lflYE0ujElBGTloevcxF5MofAOZ7C5l2b+wLg==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-2.1.0.tgz", + "integrity": "sha512-Op+UycaUt/8FbN/Z2TWPBLge3jWrP3xj10f3fnYxf052bKuS3EKs1ZQcVGjnEMdsNVAM+plXRdmjrZ/KgG3Skw==", "dev": true, "requires": { - "acorn": "^8.10.0" + "js-tokens": "^9.0.0" } }, "supports-color": { @@ -8377,15 +8433,15 @@ "dev": true }, "tinypool": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-0.8.2.tgz", - "integrity": "sha512-SUszKYe5wgsxnNOVlBYO6IC+8VGWdVGZWAqUxp3UErNBtptZvWbwyUOyzNL59zigz2rCA92QiL3wvG+JDSdJdQ==", + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-0.8.4.tgz", + "integrity": "sha512-i11VH5gS6IFeLY3gMBQ00/MmLncVP7JLXOw1vlgkytLmJK7QnEr7NXf0LBdxfmNPAeyetukOk0bOYrJrFGjYJQ==", "dev": true }, "tinyspy": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-2.2.0.tgz", - "integrity": "sha512-d2eda04AN/cPOR89F7Xv5bK/jrQEhmcLFe6HFldoeO9AJtps+fqEnh486vnT/8y4bw38pSyxDcTCAq+Ks2aJTg==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-2.2.1.tgz", + "integrity": "sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A==", "dev": true }, "to-regex-range": { @@ -8438,9 +8494,9 @@ } }, "type-detect": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", - "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.1.0.tgz", + "integrity": "sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==", "dev": true }, "type-fest": { @@ -8516,21 +8572,21 @@ "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==" }, "vite": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/vite/-/vite-5.3.1.tgz", - "integrity": "sha512-XBmSKRLXLxiaPYamLv3/hnP/KXDai1NDexN0FpkTaZXTfycHvkRHoenpgl/fvuK/kPbB6xAgoyiryAhQNxYmAQ==", + "version": "5.4.11", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.11.tgz", + "integrity": "sha512-c7jFQRklXua0mTzneGW9QVyxFjUgwcihC4bXEtujIo2ouWCe1Ajt/amn2PCxYnhYfd5k09JX3SB7OYWFKYqj8Q==", "dev": true, "requires": { "esbuild": "^0.21.3", "fsevents": "~2.3.3", - "postcss": "^8.4.38", - "rollup": "^4.13.0" + "postcss": "^8.4.43", + "rollup": "^4.20.0" } }, "vite-node": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-1.2.2.tgz", - "integrity": "sha512-1as4rDTgVWJO3n1uHmUYqq7nsFgINQ9u+mRcXpjeOMJUmviqNKjcZB7UfRZrlM7MjYXMKpuWp5oGkjaFLnjawg==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-1.6.0.tgz", + "integrity": "sha512-de6HJgzC+TFzOu0NTC4RAIsyf/DY/ibWDYQUcuEA84EMHhcefTUGkjFHKKEJhQN4A+6I0u++kr3l36ZF2d7XRw==", "dev": true, "requires": { "cac": "^6.7.14", @@ -8541,18 +8597,17 @@ } }, "vitest": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-1.2.2.tgz", - "integrity": "sha512-d5Ouvrnms3GD9USIK36KG8OZ5bEvKEkITFtnGv56HFaSlbItJuYr7hv2Lkn903+AvRAgSixiamozUVfORUekjw==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-1.6.0.tgz", + "integrity": "sha512-H5r/dN06swuFnzNFhq/dnz37bPXnq8xB2xB5JOVk8K09rUtoeNN+LHWkoQ0A/i3hvbUKKcCei9KpbxqHMLhLLA==", "dev": true, "requires": { - "@vitest/expect": "1.2.2", - "@vitest/runner": "1.2.2", - "@vitest/snapshot": "1.2.2", - "@vitest/spy": "1.2.2", - "@vitest/utils": "1.2.2", + "@vitest/expect": "1.6.0", + "@vitest/runner": "1.6.0", + "@vitest/snapshot": "1.6.0", + "@vitest/spy": "1.6.0", + "@vitest/utils": "1.6.0", "acorn-walk": "^8.3.2", - "cac": "^6.7.14", "chai": "^4.3.10", "debug": "^4.3.4", "execa": "^8.0.1", @@ -8561,11 +8616,11 @@ "pathe": "^1.1.1", "picocolors": "^1.0.0", "std-env": "^3.5.0", - "strip-literal": "^1.3.0", + "strip-literal": "^2.0.0", "tinybench": "^2.5.1", - "tinypool": "^0.8.2", + "tinypool": "^0.8.3", "vite": "^5.0.0", - "vite-node": "1.2.2", + "vite-node": "1.6.0", "why-is-node-running": "^2.2.2" } }, diff --git a/package.json b/package.json index a44f1e2..1c6116c 100644 --- a/package.json +++ b/package.json @@ -60,6 +60,6 @@ "nodemon": "^3.1.4", "prettier": "^3.0.0", "typescript": "^5.1.6", - "vitest": "^1.2.2" + "vitest": "^1.4.0" } } diff --git a/test/external_source.validation.test.ts b/test/external_source.validation.test.ts new file mode 100644 index 0000000..26b592a --- /dev/null +++ b/test/external_source.validation.test.ts @@ -0,0 +1,186 @@ +import Ajv from 'ajv'; +import { describe, expect, test } from 'vitest'; +import { externalSourceSchema } from '../src/packages/schemas/external-event-validation-schemata'; + +const ajv = Ajv(); + +// type schemas +const correctExternalEventTypeSchema = { + $schema: "http://json-schema.org/draft-07/schema", + additionalProperties: false, + description: "Schema for the attributes of the TestEventType Type.", + properties: { + code: { type: "string" }, + projectUser: { type: "string" } + }, + required: ["projectUser", "code"], + title: "TestEventType", + type: "object", +} + +const incorrectPassingExternalEventTypeSchema = { + $schema: "http://json-schema.org/draft-07/schema", + additionalProperties: false, + descriptionFake: "Schema for the attributes of the TestEventType Type.", + doesntEvenExist: true, + propertgibberish: { // if you have something like this, it just registers as no properties existing, and fails any inserted events with attributes. + code: { type: "string" }, + projectUser: { type: "string" } + }, + requiredgibberish: ["projectUser", "code"], + title: "TestEventType", + type: "object", +} + +const incorrectFailingExternalEventTypeSchema = { + $schema: "http://json-schema.org/draft-07/schema", + additionalProperties: false, + description: "Schema for the attributes of the TestEventType Type.", + properties: { + code: { type: "string" }, + projectUser: { type: "string" } + }, + required: 123, // this fails to validate at all since "required" IS well-defined as a field but expects an array + title: "TestEventType", + type: "object", +} + +const externalSourceTypeSchema = { + $schema: "http://json-schema.org/draft-07/schema", + additionalProperties: false, + description: "Schema for the attributes of the TestSourceType Type.", + properties: { + operator: { type: "string" }, + version: { type: "number" } + }, + required: ["version", "operator"], + title: "TestSourceType", + type: "object" +}; + +// compiled schemas +const compiledExternalEventTypeSchema = ajv.compile(correctExternalEventTypeSchema); +const compiledExternalSourceTypeSchema = ajv.compile(externalSourceTypeSchema); +const compiledExternalSourceSchema = ajv.compile(externalSourceSchema); + +// external source +const externalSource = { + external_events: [ + { + attributes: { + "code": "A", + "projectUser": "UserA" + }, + duration: '01:10:00', + event_type_name: 'TestExternalEventType', + key: 'Event01', + start_time: '2024-023T00:23:00Z' + }, + { + attributes: { + "code": "B", + "projectUser": "UserB" + }, + duration: '03:40:00', + event_type_name: 'DSNContact', + key: 'Event02', + start_time: '2024-021T00:21:00Z' + } + ], + source: { + attributes: { + operator: 'alpha', + version: 1 + }, + derivation_group_name: 'TestDerivationGroup', + key: 'TestExternalSourceKey', + period: { + end_time: '2024-01-28T00:00:00+00:00', + start_time: '2024-01-21T00:00:00+00:00' + }, + source_type_name: 'TestExternalSourceType', + valid_at: '2024-01-19T00:00:00+00:00' + } +}; + +// invalid attributes +const invalidSourceAttributes = { + operator: 1, + version: 1 +} +const invalidEventAttributes = { + code: 1, + projectUser: "UserB" +} + + +describe('validation tests', () => { + + // test validating type schema validation (demonstrate you can feed it bogus and its fine, but if an existing field gets a wrong type then its a problem) + describe('attribute schema validation', () => { + test('validating correct external event type schema', () => { + const schemaIsValid: boolean = ajv.validateSchema(correctExternalEventTypeSchema); + expect(schemaIsValid).toBe(true); + }); + + test('validating incorrect external event type schema that passes', () => { + const schemaIsValid: boolean = ajv.validateSchema(incorrectPassingExternalEventTypeSchema); + expect(schemaIsValid).toBe(true); + }); + + test('validating incorrect external event type schema that fails', () => { + const schemaIsValid: boolean = ajv.validateSchema(incorrectFailingExternalEventTypeSchema); + expect(schemaIsValid).toBe(false); + const errors = ajv.errors; + expect(errors?.length).toBe(1); + expect(errors?.at(0)?.message).toContain('should be array') + }); + }); + + // test validating external source validation - don't need to be thorough; this is just ajv functionality. + describe('external source validation', () => { + test('correct external source validation', async () => { + let sourceIsValid: boolean = false; + sourceIsValid = await compiledExternalSourceSchema(externalSource); + expect(sourceIsValid).toBe(true); + }); + }); + + // test validating external source attribute validation + describe('external source type attribute validation', () => { + test('correct external source type attribute validation', async () => { + let sourceAttributesAreValid: boolean = false; + sourceAttributesAreValid = await compiledExternalSourceTypeSchema(externalSource.source.attributes); + expect(sourceAttributesAreValid).toBe(true); + }); + + test('incorrect external source type attribute validation', async () => { + let sourceAttributesAreValid: boolean = false; + sourceAttributesAreValid = await compiledExternalSourceTypeSchema(invalidSourceAttributes); + expect(sourceAttributesAreValid).toBe(false); + const errors = compiledExternalSourceTypeSchema.errors; + expect(errors?.length).toBe(1); + expect(errors?.at(0)?.message).toContain('should be string'); + }); + }); + + // test validating external event attribute validation + describe('external event type attribute validation', () => { + test('correct external event type attribute validation', async () => { + let eventAttributesAreValid: boolean = true; + for (const external_event of externalSource.external_events) { + eventAttributesAreValid = eventAttributesAreValid && await compiledExternalEventTypeSchema(external_event.attributes); + } + expect(eventAttributesAreValid).toBe(true); + }); + + test('incorrect external event type attribute validation', async () => { + let eventAttributesAreValid: boolean = false; + eventAttributesAreValid = await compiledExternalEventTypeSchema(invalidEventAttributes); + expect(eventAttributesAreValid).toBe(false); + const errors = compiledExternalEventTypeSchema.errors; + expect(errors?.length).toBe(1); + expect(errors?.at(0)?.message).toContain('should be string'); + }); + }); +}); From 578fbacd8aa5f66746ed3eccf69cd73d8c16bd75 Mon Sep 17 00:00:00 2001 From: psubram3 Date: Fri, 15 Nov 2024 14:41:46 -0800 Subject: [PATCH 07/39] improve return --- src/packages/external-source/external-source.ts | 2 +- src/packages/external-source/gql.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index eda3613..e4554e2 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -299,7 +299,7 @@ async function uploadExternalSource(req: Request, res: Response) { const jsonResponse = await response.json(); console.log(jsonResponse); const createExternalSourceResponse = jsonResponse as CreateExternalSourceResponse | HasuraError; - + res.json(createExternalSourceResponse); } diff --git a/src/packages/external-source/gql.ts b/src/packages/external-source/gql.ts index b2cc75b..7d0f460 100644 --- a/src/packages/external-source/gql.ts +++ b/src/packages/external-source/gql.ts @@ -21,6 +21,7 @@ export default { // TODO: discuss upset for derivation group source_type_name, start_time, valid_at, + attributes } } `, From 05efd844d8fcef64aca838fa4259c11a71245b7b Mon Sep 17 00:00:00 2001 From: psubram3 Date: Mon, 18 Nov 2024 09:44:59 -0800 Subject: [PATCH 08/39] additional error handling --- src/packages/external-event/external-event.ts | 15 ++++++++- .../external-source/external-source.ts | 32 ++++++++++++++++--- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/packages/external-event/external-event.ts b/src/packages/external-event/external-event.ts index e20d6a0..05a6787 100644 --- a/src/packages/external-event/external-event.ts +++ b/src/packages/external-event/external-event.ts @@ -42,7 +42,20 @@ async function uploadExternalEventType(req: Request, res: Response) { } } catch (e) { logger.error(`POST /uploadExternalEventType: ${(e as Error).message}`); - res.json({ created: false }); + res.status(500); + res.send(`POST /uploadExternalEventType: ${(e as Error).message}`); + return; + } + + // Make sure name in schema (title) and provided name match + try { + if (attribute_schema["title"] === undefined || attribute_schema.title !== external_event_type_name) { + throw new Error("Schema title does not match provided external event type name.") + } + } catch (e) { + logger.error(`POST /uploadExternalEventType: ${(e as Error).message}`); + res.status(500); + res.send(`POST /uploadExternalEventType: ${(e as Error).message}`); return; } diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index e4554e2..b97643f 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -43,6 +43,8 @@ async function uploadExternalSourceType(req: Request, res: Response) { }; // Validate schema is valid JSON Schema + // NOTE: this does not check that all required attributes are included. technically, you could upload a schema for an event type, + // and only really get punished for it when validating a source. try { const schemaIsValid: boolean = ajv.validateSchema(attribute_schema); if (!schemaIsValid) { @@ -51,13 +53,25 @@ async function uploadExternalSourceType(req: Request, res: Response) { } catch (e) { logger.error(`POST /uploadExternalSourceType: ${(e as Error).message}`); res.status(500); - res.send((e as Error).message); + res.send(`POST /uploadExternalSourceType: ${(e as Error).message}`); return; } logger.info(`POST /uploadExternalSourceType: Attribute schema was VALID!`); - // TODO: Check the list of allowed event types are all defined + // Make sure name in schema (title) and provided name match + try { + if (attribute_schema["title"] === undefined || attribute_schema.title !== external_source_type_name) { + throw new Error("Schema title does not match provided external source type name.") + } + } catch (e) { + logger.error(`POST /uploadExternalSourceType: ${(e as Error).message}`); + res.status(500); + res.send(`POST /uploadExternalSourceType: ${(e as Error).message}`); + return; + } + + // Check the list of allowed event types are all defined // QUESTION: only do this check in the UI? The database ultimately checks these things. const existingTypesResponse = await fetch(GQL_API_URL, { body: JSON.stringify({ @@ -91,7 +105,7 @@ async function uploadExternalSourceType(req: Request, res: Response) { }) - const response = await fetch(GQL_API_URL, { // TODO: update + const response = await fetch(GQL_API_URL, { body: JSON.stringify({ query: gql.CREATE_EXTERNAL_SOURCE_TYPE, variables: { allowedTypes, sourceType: externalSourceTypeInput }, @@ -177,7 +191,15 @@ async function uploadExternalSource(req: Request, res: Response) { const sourceSchema: Ajv.ValidateFunction = ajv.compile(sourceAttributeSchema.attribute_schema); sourceAttributesAreValid = await sourceSchema(attributes); } + else { + // source type does not exist! + logger.error(`POST /uploadExternalSource: Source type ${source_type_name} does not exist!`); + res.status(500); + res.send(`POST /uploadExternalSource: Source type ${source_type_name} does not exist!`); + return; + } } + if (sourceAttributesAreValid) { logger.info(`POST /uploadExternalSource: Source's attributes are valid`); } else { @@ -187,7 +209,7 @@ async function uploadExternalSource(req: Request, res: Response) { return; } - // TODO: verify events are all of allowed type + // Verify events are all of allowed type // get list of all used event types const usedExternalEventTypes = external_events.map((externalEvent: ExternalEventInsertInput) => externalEvent.event_type_name).reduce( (acc: string[], externalEventType: string) => { @@ -299,7 +321,7 @@ async function uploadExternalSource(req: Request, res: Response) { const jsonResponse = await response.json(); console.log(jsonResponse); const createExternalSourceResponse = jsonResponse as CreateExternalSourceResponse | HasuraError; - + res.json(createExternalSourceResponse); } From 897e88cbb2bcf503257c1d9e33a4cc4f16f0cc9a Mon Sep 17 00:00:00 2001 From: psubram3 Date: Mon, 18 Nov 2024 09:51:15 -0800 Subject: [PATCH 09/39] remove allowed event type functionality from source type creation and validation --- .../external-source/external-source.ts | 66 ++----------------- src/packages/external-source/gql.ts | 15 +---- 2 files changed, 5 insertions(+), 76 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index b97643f..0d15f15 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -10,8 +10,6 @@ import { HasuraError } from '../../types/hasura.js'; type CreateExternalSourceResponse = { data: { createExternalSource: { name: string } | null } }; type CreateExternalSourceTypeResponse = { data: { createExternalSourceType: { attribute_schema: object, name: string } | null } }; -type ExistingTypesResponse = { data: {existingEventTypes: { name: string }[]}}; -type AssociatedTypesResponse = { data: {existingEventTypes: { external_event_type: string }[]}}; type GetExternalSourceTypeAttributeSchemaResponse = { data: { external_source_type_by_pk: { attribute_schema: object } | null } }; type GetExternalEventTypeAttributeSchemaResponse = { data: { external_event_type_by_pk: { attribute_schema: object } | null } }; @@ -30,9 +28,7 @@ async function uploadExternalSourceType(req: Request, res: Response) { } = req; const { body } = req; - const { external_source_type_name, attribute_schema, allowed_event_types } = body; - - const allowed_event_types_parsed = allowed_event_types as string[]; + const { external_source_type_name, attribute_schema } = body; const headers: HeadersInit = { Authorization: authorizationHeader ?? '', @@ -57,7 +53,7 @@ async function uploadExternalSourceType(req: Request, res: Response) { return; } - logger.info(`POST /uploadExternalSourceType: Attribute schema was VALID!`); + logger.info(`POST /uploadExternalSourceType: Attribute schema was VALID! Calling Hasura mutation...`); // Make sure name in schema (title) and provided name match try { @@ -70,29 +66,6 @@ async function uploadExternalSourceType(req: Request, res: Response) { res.send(`POST /uploadExternalSourceType: ${(e as Error).message}`); return; } - - // Check the list of allowed event types are all defined - // QUESTION: only do this check in the UI? The database ultimately checks these things. - const existingTypesResponse = await fetch(GQL_API_URL, { - body: JSON.stringify({ - query: gql.GET_EXTERNAL_EVENT_TYPES, - variables: {} - }), - headers, - method: 'POST' - }); - const existingTypesStruct: ExistingTypesResponse = await existingTypesResponse.json(); - const existingTypes: string[] = existingTypesStruct.data.existingEventTypes.map(entry => entry.name); - for (const event_type of allowed_event_types_parsed) { - if (!existingTypes.includes(event_type)) { - logger.error(`POST /uploadExternalSourceType: Event type ${event_type} is not defined.`); - res.status(500); - res.send(`POST /uploadExternalSourceType: Event type ${event_type} is not defined.`); - return; - } - } - - logger.info(`POST /uploadExternalSourceType: Successfully checked event types valid! Calling Hasura mutation...`); // Run the Hasura migration for creating an external source type (and inserting allowed event types) const externalSourceTypeInput: ExternalSourceTypeInsertInput = { @@ -100,15 +73,10 @@ async function uploadExternalSourceType(req: Request, res: Response) { name: external_source_type_name, } - const allowedTypes: { external_event_type: string, external_source_type: string }[] = allowed_event_types_parsed.map(external_event_type => { - return { external_event_type, external_source_type: external_source_type_name }; - }) - - const response = await fetch(GQL_API_URL, { body: JSON.stringify({ query: gql.CREATE_EXTERNAL_SOURCE_TYPE, - variables: { allowedTypes, sourceType: externalSourceTypeInput }, + variables: { sourceType: externalSourceTypeInput }, }), headers, method: 'POST', @@ -209,7 +177,7 @@ async function uploadExternalSource(req: Request, res: Response) { return; } - // Verify events are all of allowed type + // Get the attribute schema(s) for all external event types used by the source's events // get list of all used event types const usedExternalEventTypes = external_events.map((externalEvent: ExternalEventInsertInput) => externalEvent.event_type_name).reduce( (acc: string[], externalEventType: string) => { @@ -219,32 +187,6 @@ async function uploadExternalSource(req: Request, res: Response) { return acc; }, []); - // get allowed event types - const allowedExternalEventTypes = await fetch(GQL_API_URL, { - body: JSON.stringify({ - query: gql.GET_EXTERNAL_EVENT_TYPES_FOR_SOURCE_TYPE, - variables: { sourceType: source_type_name } - }), - headers, - method: 'POST' - }); - - // check - const allowedEventTypesStruct: AssociatedTypesResponse = await allowedExternalEventTypes.json(); - const allowedEventTypes = allowedEventTypesStruct.data.existingEventTypes.map(eventType => eventType.external_event_type); - - for (const event_type of usedExternalEventTypes) { - if (!allowedEventTypes.includes(event_type)) { - logger.error(`POST /uploadExternalSourceType: An event uses event type ${event_type}, which is not defined for source type ${source_type_name}.`); - res.status(500); - res.send(`POST /uploadExternalSourceType: An event uses event type ${event_type}, which is not defined for source type ${source_type_name}.`); - return; - } - } - - logger.info(`POST /uploadExternalSource: Source's included events' types are valid.`); - - // Get the attribute schema(s) for all external event types used by the source's events const usedExternalEventTypesAttributesSchemas: Record = await usedExternalEventTypes.reduce(async (acc: Record, eventType: string) => { const eventAttributeSchema = await fetch(GQL_API_URL, { body: JSON.stringify({ diff --git a/src/packages/external-source/gql.ts b/src/packages/external-source/gql.ts index 7d0f460..1baaf14 100644 --- a/src/packages/external-source/gql.ts +++ b/src/packages/external-source/gql.ts @@ -26,24 +26,11 @@ export default { // TODO: discuss upset for derivation group } `, CREATE_EXTERNAL_SOURCE_TYPE: `#graphql - mutation CreateExternalSourceType($sourceType: external_source_type_insert_input!, $allowedTypes: [external_source_type_allowed_event_types_insert_input!]!) { + mutation CreateExternalSourceType($sourceType: external_source_type_insert_input!) { createExternalSourceType: insert_external_source_type_one(object: $sourceType) { name attribute_schema } - defineAllowedTypes: insert_external_source_type_allowed_event_types(objects: $allowedTypes) { - returning { - external_source_type - external_event_type - } - } - } - `, - GET_EXTERNAL_EVENT_TYPES: `#graphql - query ExistingEventTypes { - existingEventTypes: external_event_type { - name - } } `, GET_EXTERNAL_EVENT_TYPES_FOR_SOURCE_TYPE: `#graphql From eb1ac1870bf94cf3e99dc9ae10f83cd9f0413bb0 Mon Sep 17 00:00:00 2001 From: psubram3 Date: Mon, 18 Nov 2024 11:05:55 -0800 Subject: [PATCH 10/39] final edits for error handling --- .../external-source/external-source.ts | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 0d15f15..2f22321 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -150,13 +150,14 @@ async function uploadExternalSource(req: Request, res: Response) { // Validate the attributes on the External Source let sourceAttributesAreValid: boolean = false; + let sourceSchema: Ajv.ValidateFunction | undefined = undefined; const sourceTypeResponseJSON = await sourceAttributeSchema.json(); const getExternalSourceTypeAttributeSchemaResponse = sourceTypeResponseJSON as GetExternalSourceTypeAttributeSchemaResponse | HasuraError; if ((getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse).data?.external_source_type_by_pk?.attribute_schema !== null) { console.log(sourceTypeResponseJSON, source_type_name) const { data: { external_source_type_by_pk: sourceAttributeSchema } } = getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse; if (sourceAttributeSchema !== undefined && sourceAttributeSchema !== null) { - const sourceSchema: Ajv.ValidateFunction = ajv.compile(sourceAttributeSchema.attribute_schema); + sourceSchema = ajv.compile(sourceAttributeSchema.attribute_schema); sourceAttributesAreValid = await sourceSchema(attributes); } else { @@ -173,7 +174,11 @@ async function uploadExternalSource(req: Request, res: Response) { } else { logger.error(`POST /uploadExternalSource: Source's attributes are invalid`); res.status(500); - res.send(`POST /uploadExternalSource: Source's attributes are invalid:\n${JSON.stringify(ajv.errors)}`); + if (sourceSchema !== undefined) { + res.send(`POST /uploadExternalSource: Source's attributes are invalid:\n${JSON.stringify(sourceSchema.errors)}`); + } else { + res.send(`POST /uploadExternalSource: Source's attributes are invalid`); + } return; } @@ -186,8 +191,11 @@ async function uploadExternalSource(req: Request, res: Response) { }; return acc; }, []); + + console.log("usedExternalEventTypes", usedExternalEventTypes) - const usedExternalEventTypesAttributesSchemas: Record = await usedExternalEventTypes.reduce(async (acc: Record, eventType: string) => { + const usedExternalEventTypesAttributesSchemas: Record = {}; + for (const eventType of usedExternalEventTypes) { const eventAttributeSchema = await fetch(GQL_API_URL, { body: JSON.stringify({ query: gql.GET_EXTERNAL_EVENT_TYPE_ATTRIBUTE_SCHEMA, @@ -203,18 +211,20 @@ async function uploadExternalSource(req: Request, res: Response) { if ((getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse).data?.external_event_type_by_pk?.attribute_schema !== null) { const { data: { external_event_type_by_pk: eventAttributeSchema } } = getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse; + console.log("BINGO!", eventType, eventAttributeSchema) if (eventAttributeSchema !== undefined && eventAttributeSchema !== null) { - acc[eventType] = ajv.compile(eventAttributeSchema.attribute_schema); + usedExternalEventTypesAttributesSchemas[eventType] = ajv.compile(eventAttributeSchema.attribute_schema); } } + } - return acc; - }, {} as Record) + console.log("usedExternalEventTypesAttributesSchemas", usedExternalEventTypesAttributesSchemas) for (const externalEvent of external_events) { try { const currentEventType = externalEvent.event_type_name; const currentEventSchema: Ajv.ValidateFunction = usedExternalEventTypesAttributesSchemas[currentEventType]; + console.log("CURRENT EVENT SCHEMA:", currentEventType, externalEvent.attributes, currentEventSchema) const eventAttributesAreValid = await currentEventSchema(externalEvent.attributes); if (!eventAttributesAreValid) { throw new Error(`External Event '${externalEvent.key}' does not have a valid set of attributes, per it's type's schema:\n${JSON.stringify(currentEventSchema.errors)}`); From 3fbecf930880fa4b92ac25dac2e227467c5a2f5e Mon Sep 17 00:00:00 2001 From: psubram3 Date: Mon, 18 Nov 2024 12:26:23 -0800 Subject: [PATCH 11/39] remove console.logs --- src/packages/external-event/external-event.ts | 3 --- src/packages/external-source/external-source.ts | 14 -------------- 2 files changed, 17 deletions(-) diff --git a/src/packages/external-event/external-event.ts b/src/packages/external-event/external-event.ts index 05a6787..497505f 100644 --- a/src/packages/external-event/external-event.ts +++ b/src/packages/external-event/external-event.ts @@ -27,13 +27,10 @@ async function uploadExternalEventType(req: Request, res: Response) { const headers: HeadersInit = { Authorization: authorizationHeader ?? '', 'Content-Type': 'application/json', - 'x-hasura-admin-secret': 'aerie', // HACK, TODO: FIX 'x-hasura-role': roleHeader ? `${roleHeader}` : '', 'x-hasura-user-id': userHeader ? `${userHeader}` : '', }; - console.log("\n\n", JSON.stringify(attribute_schema), "\n\n") - // Validate schema is valid JSON Schema try { const schemaIsValid: boolean = ajv.validateSchema(attribute_schema); diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 2f22321..5300bf5 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -116,14 +116,10 @@ async function uploadExternalSource(req: Request, res: Response) { const headers: HeadersInit = { Authorization: authorizationHeader ?? '', 'Content-Type': 'application/json', - 'x-hasura-admin-secret': 'aerie', // HACK, TODO: FIX 'x-hasura-role': roleHeader ? `${roleHeader}` : '', 'x-hasura-user-id': userHeader ? `${userHeader}` : '', }; - - console.log("\n\n", body, "\n\n"); - // Verify that this is a valid external source! let sourceIsValid: boolean = false; sourceIsValid = await compiledExternalSourceSchema(body); @@ -154,7 +150,6 @@ async function uploadExternalSource(req: Request, res: Response) { const sourceTypeResponseJSON = await sourceAttributeSchema.json(); const getExternalSourceTypeAttributeSchemaResponse = sourceTypeResponseJSON as GetExternalSourceTypeAttributeSchemaResponse | HasuraError; if ((getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse).data?.external_source_type_by_pk?.attribute_schema !== null) { - console.log(sourceTypeResponseJSON, source_type_name) const { data: { external_source_type_by_pk: sourceAttributeSchema } } = getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse; if (sourceAttributeSchema !== undefined && sourceAttributeSchema !== null) { sourceSchema = ajv.compile(sourceAttributeSchema.attribute_schema); @@ -191,8 +186,6 @@ async function uploadExternalSource(req: Request, res: Response) { }; return acc; }, []); - - console.log("usedExternalEventTypes", usedExternalEventTypes) const usedExternalEventTypesAttributesSchemas: Record = {}; for (const eventType of usedExternalEventTypes) { @@ -211,20 +204,16 @@ async function uploadExternalSource(req: Request, res: Response) { if ((getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse).data?.external_event_type_by_pk?.attribute_schema !== null) { const { data: { external_event_type_by_pk: eventAttributeSchema } } = getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse; - console.log("BINGO!", eventType, eventAttributeSchema) if (eventAttributeSchema !== undefined && eventAttributeSchema !== null) { usedExternalEventTypesAttributesSchemas[eventType] = ajv.compile(eventAttributeSchema.attribute_schema); } } } - console.log("usedExternalEventTypesAttributesSchemas", usedExternalEventTypesAttributesSchemas) - for (const externalEvent of external_events) { try { const currentEventType = externalEvent.event_type_name; const currentEventSchema: Ajv.ValidateFunction = usedExternalEventTypesAttributesSchemas[currentEventType]; - console.log("CURRENT EVENT SCHEMA:", currentEventType, externalEvent.attributes, currentEventSchema) const eventAttributesAreValid = await currentEventSchema(externalEvent.attributes); if (!eventAttributesAreValid) { throw new Error(`External Event '${externalEvent.key}' does not have a valid set of attributes, per it's type's schema:\n${JSON.stringify(currentEventSchema.errors)}`); @@ -237,8 +226,6 @@ async function uploadExternalSource(req: Request, res: Response) { } } - console.log("VALID!"); - // Run the Hasura migration for creating an external source const derivationGroupInsert: DerivationGroupInsertInput = { name: derivation_group_name, @@ -271,7 +258,6 @@ async function uploadExternalSource(req: Request, res: Response) { }); const jsonResponse = await response.json(); - console.log(jsonResponse); const createExternalSourceResponse = jsonResponse as CreateExternalSourceResponse | HasuraError; From 48cb21cad1434ad423a7f4d81877aeb04ce5f3fb Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Tue, 19 Nov 2024 16:26:12 -0500 Subject: [PATCH 12/39] Update error responses from external source/event end-points --- src/packages/external-event/external-event.ts | 12 ++++----- .../external-source/external-source.ts | 27 +++++++++---------- src/packages/external-source/gql.ts | 1 + 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/src/packages/external-event/external-event.ts b/src/packages/external-event/external-event.ts index 497505f..83c7171 100644 --- a/src/packages/external-event/external-event.ts +++ b/src/packages/external-event/external-event.ts @@ -37,10 +37,10 @@ async function uploadExternalEventType(req: Request, res: Response) { if (!schemaIsValid) { throw new Error("Schema was not a valid JSON Schema."); } - } catch (e) { - logger.error(`POST /uploadExternalEventType: ${(e as Error).message}`); + } catch (error) { + logger.error((error as Error).message); res.status(500); - res.send(`POST /uploadExternalEventType: ${(e as Error).message}`); + res.send((error as Error).message); return; } @@ -49,10 +49,10 @@ async function uploadExternalEventType(req: Request, res: Response) { if (attribute_schema["title"] === undefined || attribute_schema.title !== external_event_type_name) { throw new Error("Schema title does not match provided external event type name.") } - } catch (e) { - logger.error(`POST /uploadExternalEventType: ${(e as Error).message}`); + } catch (error) { + logger.error((error as Error).message); res.status(500); - res.send(`POST /uploadExternalEventType: ${(e as Error).message}`); + res.send((error as Error).message); return; } diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 5300bf5..2958bff 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -46,10 +46,9 @@ async function uploadExternalSourceType(req: Request, res: Response) { if (!schemaIsValid) { throw new Error("Schema was not a valid JSON Schema."); } - } catch (e) { - logger.error(`POST /uploadExternalSourceType: ${(e as Error).message}`); + } catch (error) { res.status(500); - res.send(`POST /uploadExternalSourceType: ${(e as Error).message}`); + res.send((error as Error).message); return; } @@ -60,20 +59,19 @@ async function uploadExternalSourceType(req: Request, res: Response) { if (attribute_schema["title"] === undefined || attribute_schema.title !== external_source_type_name) { throw new Error("Schema title does not match provided external source type name.") } - } catch (e) { - logger.error(`POST /uploadExternalSourceType: ${(e as Error).message}`); + } catch (error) { res.status(500); - res.send(`POST /uploadExternalSourceType: ${(e as Error).message}`); + res.send((error as Error).message); return; } - + // Run the Hasura migration for creating an external source type (and inserting allowed event types) const externalSourceTypeInput: ExternalSourceTypeInsertInput = { attribute_schema: attribute_schema, name: external_source_type_name, } - const response = await fetch(GQL_API_URL, { + const response = await fetch(GQL_API_URL, { body: JSON.stringify({ query: gql.CREATE_EXTERNAL_SOURCE_TYPE, variables: { sourceType: externalSourceTypeInput }, @@ -126,9 +124,9 @@ async function uploadExternalSource(req: Request, res: Response) { if (sourceIsValid) { logger.info(`POST /uploadExternalSource: Source's formatting is valid per basic schema validation.`); } else { - logger.error(`POST /uploadExternalSource: Source's formatting is invalid per basic schema validation:\n${JSON.stringify(compiledExternalSourceSchema.errors)}`); + logger.error("POST /uploadExternalSource: Source's formatting is invalid per basic schema validation"); res.status(500); - res.send(`POST /uploadExternalSource: Source's formatting is invalid per basic schema validation:\n${JSON.stringify(compiledExternalSourceSchema.errors)}`); + res.send("Source's formatting is invalid per basic schema validation"); return; } @@ -159,7 +157,7 @@ async function uploadExternalSource(req: Request, res: Response) { // source type does not exist! logger.error(`POST /uploadExternalSource: Source type ${source_type_name} does not exist!`); res.status(500); - res.send(`POST /uploadExternalSource: Source type ${source_type_name} does not exist!`); + res.send(`Source type ${source_type_name} does not exist!`); return; } } @@ -172,7 +170,7 @@ async function uploadExternalSource(req: Request, res: Response) { if (sourceSchema !== undefined) { res.send(`POST /uploadExternalSource: Source's attributes are invalid:\n${JSON.stringify(sourceSchema.errors)}`); } else { - res.send(`POST /uploadExternalSource: Source's attributes are invalid`); + res.send(`Source's attributes are invalid`); } return; } @@ -218,10 +216,9 @@ async function uploadExternalSource(req: Request, res: Response) { if (!eventAttributesAreValid) { throw new Error(`External Event '${externalEvent.key}' does not have a valid set of attributes, per it's type's schema:\n${JSON.stringify(currentEventSchema.errors)}`); } - } catch (e) { - logger.error(`POST /uploadExternalSource: ${(e as Error).message}`); + } catch (error) { res.status(500); - res.send((e as Error).message); + res.send((error as Error).message); return; } } diff --git a/src/packages/external-source/gql.ts b/src/packages/external-source/gql.ts index 1baaf14..eb6c435 100644 --- a/src/packages/external-source/gql.ts +++ b/src/packages/external-source/gql.ts @@ -15,6 +15,7 @@ export default { // TODO: discuss upset for derivation group createExternalSource: insert_external_source_one ( object: $source ) { + attributes derivation_group_name, end_time, key, From accf3455a9cbb7ac115ea2b878e200a132dfd700 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Wed, 20 Nov 2024 11:51:52 -0500 Subject: [PATCH 13/39] Update logging messages & use JSON for all responses --- src/packages/external-event/external-event.ts | 14 +++---- .../external-source/external-source.ts | 42 ++++++++----------- 2 files changed, 24 insertions(+), 32 deletions(-) diff --git a/src/packages/external-event/external-event.ts b/src/packages/external-event/external-event.ts index 83c7171..c1b8303 100644 --- a/src/packages/external-event/external-event.ts +++ b/src/packages/external-event/external-event.ts @@ -13,9 +13,7 @@ const GQL_API_URL = `${HASURA_API_URL}/v1/graphql`; const ajv = new Ajv(); async function uploadExternalEventType(req: Request, res: Response) { - logger.info(`POST /uploadExternalEventType: Entering function...`); const authorizationHeader = req.get('authorization'); - logger.info(`POST /uploadExternalEventType: ${authorizationHeader}`); const { headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, @@ -23,6 +21,7 @@ async function uploadExternalEventType(req: Request, res: Response) { const { body } = req; const { external_event_type_name, attribute_schema } = body; + logger.info(`POST /uploadExternalEventType: Uploading External Event Type: ${external_event_type_name}`); const headers: HeadersInit = { Authorization: authorizationHeader ?? '', @@ -38,9 +37,9 @@ async function uploadExternalEventType(req: Request, res: Response) { throw new Error("Schema was not a valid JSON Schema."); } } catch (error) { + logger.error(`POST /uploadExternalEventType: Error occurred during External Event Type ${external_event_type_name} upload`); logger.error((error as Error).message); - res.status(500); - res.send((error as Error).message); + res.status(500).send({ message: (error as Error).message }); return; } @@ -50,13 +49,13 @@ async function uploadExternalEventType(req: Request, res: Response) { throw new Error("Schema title does not match provided external event type name.") } } catch (error) { + logger.error(`POST /uploadExternalEventType: Error occurred during External Event Type ${external_event_type_name} upload`); logger.error((error as Error).message); - res.status(500); - res.send((error as Error).message); + res.status(500).send({ message: (error as Error).message }); return; } - logger.info(`POST /uploadExternalEventType: Attribute schema was VALID! Calling Hasura mutation...`); + logger.info(`POST /uploadExternalEventType: Attribute schema was VALID`); // Run the Hasura migration for creating an external event const externalEventTypeInsertInput: ExternalEventTypeInsertInput = { @@ -64,7 +63,6 @@ async function uploadExternalEventType(req: Request, res: Response) { name: external_event_type_name, } - const response = await fetch(GQL_API_URL, { body: JSON.stringify({ query: gql.CREATE_EXTERNAL_EVENT_TYPE, diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 2958bff..9e9a3dc 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -20,7 +20,6 @@ const ajv = new Ajv(); const compiledExternalSourceSchema = ajv.compile(externalSourceSchema); async function uploadExternalSourceType(req: Request, res: Response) { - logger.info(`POST /uploadExternalSourceType: Entering function...`); const authorizationHeader = req.get('authorization'); const { @@ -29,6 +28,8 @@ async function uploadExternalSourceType(req: Request, res: Response) { const { body } = req; const { external_source_type_name, attribute_schema } = body; + logger.info(`POST /uploadExternalSourceType: Uploading External Source Type: ${external_source_type_name}`); + const headers: HeadersInit = { Authorization: authorizationHeader ?? '', @@ -47,21 +48,19 @@ async function uploadExternalSourceType(req: Request, res: Response) { throw new Error("Schema was not a valid JSON Schema."); } } catch (error) { - res.status(500); - res.send((error as Error).message); + res.status(500).send({ message: (error as Error).message }); return; } - logger.info(`POST /uploadExternalSourceType: Attribute schema was VALID! Calling Hasura mutation...`); + logger.info(`POST /uploadExternalSourceType: ${external_source_type_name} attribute schema was VALID`); // Make sure name in schema (title) and provided name match try { if (attribute_schema["title"] === undefined || attribute_schema.title !== external_source_type_name) { - throw new Error("Schema title does not match provided external source type name.") + throw new Error(`${external_source_type_name} attribute schema title does not match provided external source type name.`) } } catch (error) { - res.status(500); - res.send((error as Error).message); + res.status(500).send({ message: (error as Error).message }); return; } @@ -83,13 +82,10 @@ async function uploadExternalSourceType(req: Request, res: Response) { const jsonResponse = await response.json(); const createExternalSourceTypeResponse = jsonResponse as CreateExternalSourceTypeResponse | HasuraError; - logger.info(`POST /uploadExternalSourceType: Successfully uploaded new type and event type associations!`); - res.json(createExternalSourceTypeResponse); } async function uploadExternalSource(req: Request, res: Response) { - logger.info(`POST /uploadExternalSource: Entering function...`); const authorizationHeader = req.get('authorization'); const { headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, @@ -118,15 +114,16 @@ async function uploadExternalSource(req: Request, res: Response) { 'x-hasura-user-id': userHeader ? `${userHeader}` : '', }; + logger.info(`POST /uploadExternalSource: Uploading External Source: ${key}`) + // Verify that this is a valid external source! let sourceIsValid: boolean = false; sourceIsValid = await compiledExternalSourceSchema(body); if (sourceIsValid) { - logger.info(`POST /uploadExternalSource: Source's formatting is valid per basic schema validation.`); + logger.info(`POST /uploadExternalSource: External Source ${key}'s formatting is valid`); } else { - logger.error("POST /uploadExternalSource: Source's formatting is invalid per basic schema validation"); - res.status(500); - res.send("Source's formatting is invalid per basic schema validation"); + logger.error(`POST /uploadExternalSource: External Source ${key}'s formatting is invalid`); + res.status(500).send({ message: `External Source ${key}'s formatting is invalid` }); return; } @@ -155,22 +152,21 @@ async function uploadExternalSource(req: Request, res: Response) { } else { // source type does not exist! - logger.error(`POST /uploadExternalSource: Source type ${source_type_name} does not exist!`); - res.status(500); - res.send(`Source type ${source_type_name} does not exist!`); + logger.error(`POST /uploadExternalSource: External Source Type ${source_type_name} does not exist!`); + res.status(500).send({ message: `External Source Type ${source_type_name} does not exist!`}); return; } } if (sourceAttributesAreValid) { - logger.info(`POST /uploadExternalSource: Source's attributes are valid`); + logger.info(`POST /uploadExternalSource: External Source ${key}'s attributes are valid`); } else { - logger.error(`POST /uploadExternalSource: Source's attributes are invalid`); + logger.error(`POST /uploadExternalSource: External Source ${key}'s attributes are invalid`); res.status(500); if (sourceSchema !== undefined) { - res.send(`POST /uploadExternalSource: Source's attributes are invalid:\n${JSON.stringify(sourceSchema.errors)}`); + res.send({ message: `External Source ${key}'s attributes are invalid:\n${JSON.stringify(sourceSchema.errors)}` }); } else { - res.send(`Source's attributes are invalid`); + res.send({ message: `External Source ${key}'s attributes are invalid` }); } return; } @@ -217,8 +213,7 @@ async function uploadExternalSource(req: Request, res: Response) { throw new Error(`External Event '${externalEvent.key}' does not have a valid set of attributes, per it's type's schema:\n${JSON.stringify(currentEventSchema.errors)}`); } } catch (error) { - res.status(500); - res.send((error as Error).message); + res.status(500).send({ message: (error as Error).message }); return; } } @@ -257,7 +252,6 @@ async function uploadExternalSource(req: Request, res: Response) { const jsonResponse = await response.json(); const createExternalSourceResponse = jsonResponse as CreateExternalSourceResponse | HasuraError; - res.json(createExternalSourceResponse); } From 6b6897797a2a2e8145e3ce7277857f6192b3e016 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Wed, 20 Nov 2024 12:05:25 -0500 Subject: [PATCH 14/39] Improve error handling w. AJV.errors --- src/packages/external-event/external-event.ts | 14 +++++--------- src/packages/external-source/external-source.ts | 12 +++++------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/src/packages/external-event/external-event.ts b/src/packages/external-event/external-event.ts index c1b8303..6c12980 100644 --- a/src/packages/external-event/external-event.ts +++ b/src/packages/external-event/external-event.ts @@ -31,15 +31,11 @@ async function uploadExternalEventType(req: Request, res: Response) { }; // Validate schema is valid JSON Schema - try { - const schemaIsValid: boolean = ajv.validateSchema(attribute_schema); - if (!schemaIsValid) { - throw new Error("Schema was not a valid JSON Schema."); - } - } catch (error) { - logger.error(`POST /uploadExternalEventType: Error occurred during External Event Type ${external_event_type_name} upload`); - logger.error((error as Error).message); - res.status(500).send({ message: (error as Error).message }); + const schemaIsValid: boolean = ajv.validateSchema(attribute_schema); + if (!schemaIsValid) { + logger.error(`POST /uploadExternalEventType: Schema validation failed for External Event Type ${external_event_type_name}`); + ajv.errors?.forEach(ajvError => logger.error(ajvError)); + res.status(500).send({ message: ajv.errors }); return; } diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 9e9a3dc..99e59b8 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -42,13 +42,11 @@ async function uploadExternalSourceType(req: Request, res: Response) { // Validate schema is valid JSON Schema // NOTE: this does not check that all required attributes are included. technically, you could upload a schema for an event type, // and only really get punished for it when validating a source. - try { - const schemaIsValid: boolean = ajv.validateSchema(attribute_schema); - if (!schemaIsValid) { - throw new Error("Schema was not a valid JSON Schema."); - } - } catch (error) { - res.status(500).send({ message: (error as Error).message }); + const schemaIsValid: boolean = ajv.validateSchema(attribute_schema); + if (!schemaIsValid) { + logger.error(`POST /uploadExternalSourceType: Schema validation failed for External Source Type ${external_source_type_name}`); + ajv.errors?.forEach(ajvError => logger.error(ajvError)); + res.status(500).send({ message: ajv.errors }); return; } From 0821432673d6993b0395c3aa3b6adbc64b52b137 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Wed, 20 Nov 2024 12:21:03 -0500 Subject: [PATCH 15/39] Formatting fix --- src/packages/external-event/external-event.ts | 18 ++- src/packages/external-event/gql.ts | 4 +- .../external-source/external-source.ts | 128 ++++++++++-------- src/packages/external-source/gql.ts | 7 +- .../external-event-validation-schemata.ts | 120 ++++++++-------- src/types/external-event.ts | 6 +- src/types/external-source.ts | 8 +- test/external_source.validation.test.ts | 104 +++++++------- 8 files changed, 212 insertions(+), 183 deletions(-) diff --git a/src/packages/external-event/external-event.ts b/src/packages/external-event/external-event.ts index 6c12980..fef7378 100644 --- a/src/packages/external-event/external-event.ts +++ b/src/packages/external-event/external-event.ts @@ -33,7 +33,9 @@ async function uploadExternalEventType(req: Request, res: Response) { // Validate schema is valid JSON Schema const schemaIsValid: boolean = ajv.validateSchema(attribute_schema); if (!schemaIsValid) { - logger.error(`POST /uploadExternalEventType: Schema validation failed for External Event Type ${external_event_type_name}`); + logger.error( + `POST /uploadExternalEventType: Schema validation failed for External Event Type ${external_event_type_name}`, + ); ajv.errors?.forEach(ajvError => logger.error(ajvError)); res.status(500).send({ message: ajv.errors }); return; @@ -41,11 +43,13 @@ async function uploadExternalEventType(req: Request, res: Response) { // Make sure name in schema (title) and provided name match try { - if (attribute_schema["title"] === undefined || attribute_schema.title !== external_event_type_name) { - throw new Error("Schema title does not match provided external event type name.") + if (attribute_schema['title'] === undefined || attribute_schema.title !== external_event_type_name) { + throw new Error('Schema title does not match provided external event type name.'); } } catch (error) { - logger.error(`POST /uploadExternalEventType: Error occurred during External Event Type ${external_event_type_name} upload`); + logger.error( + `POST /uploadExternalEventType: Error occurred during External Event Type ${external_event_type_name} upload`, + ); logger.error((error as Error).message); res.status(500).send({ message: (error as Error).message }); return; @@ -57,7 +61,7 @@ async function uploadExternalEventType(req: Request, res: Response) { const externalEventTypeInsertInput: ExternalEventTypeInsertInput = { attribute_schema: attribute_schema, name: external_event_type_name, - } + }; const response = await fetch(GQL_API_URL, { body: JSON.stringify({ @@ -68,7 +72,9 @@ async function uploadExternalEventType(req: Request, res: Response) { method: 'POST', }); - type CreateExternalEventTypeResponse = { data: { createExternalEventType: { attribute_schema: object, name: string } | null } }; + type CreateExternalEventTypeResponse = { + data: { createExternalEventType: { attribute_schema: object; name: string } | null }; + }; const jsonResponse = await response.json(); const createExternalEventTypeResponse = jsonResponse as CreateExternalEventTypeResponse | HasuraError; diff --git a/src/packages/external-event/gql.ts b/src/packages/external-event/gql.ts index 60f7421..8b9dae3 100644 --- a/src/packages/external-event/gql.ts +++ b/src/packages/external-event/gql.ts @@ -7,5 +7,5 @@ export default { name } } - ` -} + `, +}; diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 99e59b8..8338052 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -1,5 +1,9 @@ import type { Express, Request, Response } from 'express'; -import type { DerivationGroupInsertInput, ExternalSourceInsertInput, ExternalSourceTypeInsertInput } from '../../types/external-source.js'; +import type { + DerivationGroupInsertInput, + ExternalSourceInsertInput, + ExternalSourceTypeInsertInput, +} from '../../types/external-source.js'; import type { ExternalEventInsertInput } from '../../types/external-event.js'; import Ajv from 'ajv'; import { getEnv } from '../../env.js'; @@ -9,9 +13,15 @@ import { externalSourceSchema } from '../schemas/external-event-validation-schem import { HasuraError } from '../../types/hasura.js'; type CreateExternalSourceResponse = { data: { createExternalSource: { name: string } | null } }; -type CreateExternalSourceTypeResponse = { data: { createExternalSourceType: { attribute_schema: object, name: string } | null } }; -type GetExternalSourceTypeAttributeSchemaResponse = { data: { external_source_type_by_pk: { attribute_schema: object } | null } }; -type GetExternalEventTypeAttributeSchemaResponse = { data: { external_event_type_by_pk: { attribute_schema: object } | null } }; +type CreateExternalSourceTypeResponse = { + data: { createExternalSourceType: { attribute_schema: object; name: string } | null }; +}; +type GetExternalSourceTypeAttributeSchemaResponse = { + data: { external_source_type_by_pk: { attribute_schema: object } | null }; +}; +type GetExternalEventTypeAttributeSchemaResponse = { + data: { external_event_type_by_pk: { attribute_schema: object } | null }; +}; const logger = getLogger('packages/external-source/external-source'); const { HASURA_API_URL } = getEnv(); @@ -30,7 +40,6 @@ async function uploadExternalSourceType(req: Request, res: Response) { const { external_source_type_name, attribute_schema } = body; logger.info(`POST /uploadExternalSourceType: Uploading External Source Type: ${external_source_type_name}`); - const headers: HeadersInit = { Authorization: authorizationHeader ?? '', 'Content-Type': 'application/json', @@ -44,7 +53,9 @@ async function uploadExternalSourceType(req: Request, res: Response) { // and only really get punished for it when validating a source. const schemaIsValid: boolean = ajv.validateSchema(attribute_schema); if (!schemaIsValid) { - logger.error(`POST /uploadExternalSourceType: Schema validation failed for External Source Type ${external_source_type_name}`); + logger.error( + `POST /uploadExternalSourceType: Schema validation failed for External Source Type ${external_source_type_name}`, + ); ajv.errors?.forEach(ajvError => logger.error(ajvError)); res.status(500).send({ message: ajv.errors }); return; @@ -54,8 +65,10 @@ async function uploadExternalSourceType(req: Request, res: Response) { // Make sure name in schema (title) and provided name match try { - if (attribute_schema["title"] === undefined || attribute_schema.title !== external_source_type_name) { - throw new Error(`${external_source_type_name} attribute schema title does not match provided external source type name.`) + if (attribute_schema['title'] === undefined || attribute_schema.title !== external_source_type_name) { + throw new Error( + `${external_source_type_name} attribute schema title does not match provided external source type name.`, + ); } } catch (error) { res.status(500).send({ message: (error as Error).message }); @@ -66,7 +79,7 @@ async function uploadExternalSourceType(req: Request, res: Response) { const externalSourceTypeInput: ExternalSourceTypeInsertInput = { attribute_schema: attribute_schema, name: external_source_type_name, - } + }; const response = await fetch(GQL_API_URL, { body: JSON.stringify({ @@ -89,22 +102,9 @@ async function uploadExternalSource(req: Request, res: Response) { headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, } = req; const { body } = req; - const { - external_events, - source - } = body; - const { - attributes, - derivation_group_name, - key, - source_type_name, - period, - valid_at - } = source; - const { - end_time, - start_time - } = period; + const { external_events, source } = body; + const { attributes, derivation_group_name, key, source_type_name, period, valid_at } = source; + const { end_time, start_time } = period; const headers: HeadersInit = { Authorization: authorizationHeader ?? '', 'Content-Type': 'application/json', @@ -112,7 +112,7 @@ async function uploadExternalSource(req: Request, res: Response) { 'x-hasura-user-id': userHeader ? `${userHeader}` : '', }; - logger.info(`POST /uploadExternalSource: Uploading External Source: ${key}`) + logger.info(`POST /uploadExternalSource: Uploading External Source: ${key}`); // Verify that this is a valid external source! let sourceIsValid: boolean = false; @@ -130,28 +130,34 @@ async function uploadExternalSource(req: Request, res: Response) { body: JSON.stringify({ query: gql.GET_EXTERNAL_SOURCE_TYPE_ATTRIBUTE_SCHEMA, variables: { - name: source_type_name - } + name: source_type_name, + }, }), headers, - method: 'POST' + method: 'POST', }); // Validate the attributes on the External Source let sourceAttributesAreValid: boolean = false; let sourceSchema: Ajv.ValidateFunction | undefined = undefined; - const sourceTypeResponseJSON = await sourceAttributeSchema.json(); - const getExternalSourceTypeAttributeSchemaResponse = sourceTypeResponseJSON as GetExternalSourceTypeAttributeSchemaResponse | HasuraError; - if ((getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse).data?.external_source_type_by_pk?.attribute_schema !== null) { - const { data: { external_source_type_by_pk: sourceAttributeSchema } } = getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse; + const sourceTypeResponseJSON = await sourceAttributeSchema.json(); + const getExternalSourceTypeAttributeSchemaResponse = sourceTypeResponseJSON as + | GetExternalSourceTypeAttributeSchemaResponse + | HasuraError; + if ( + (getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse).data + ?.external_source_type_by_pk?.attribute_schema !== null + ) { + const { + data: { external_source_type_by_pk: sourceAttributeSchema }, + } = getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse; if (sourceAttributeSchema !== undefined && sourceAttributeSchema !== null) { sourceSchema = ajv.compile(sourceAttributeSchema.attribute_schema); sourceAttributesAreValid = await sourceSchema(attributes); - } - else { + } else { // source type does not exist! logger.error(`POST /uploadExternalSource: External Source Type ${source_type_name} does not exist!`); - res.status(500).send({ message: `External Source Type ${source_type_name} does not exist!`}); + res.status(500).send({ message: `External Source Type ${source_type_name} does not exist!` }); return; } } @@ -171,11 +177,12 @@ async function uploadExternalSource(req: Request, res: Response) { // Get the attribute schema(s) for all external event types used by the source's events // get list of all used event types - const usedExternalEventTypes = external_events.map((externalEvent: ExternalEventInsertInput) => externalEvent.event_type_name).reduce( - (acc: string[], externalEventType: string) => { + const usedExternalEventTypes = external_events + .map((externalEvent: ExternalEventInsertInput) => externalEvent.event_type_name) + .reduce((acc: string[], externalEventType: string) => { if (!acc.includes(externalEventType)) { - acc.push(externalEventType) - }; + acc.push(externalEventType); + } return acc; }, []); @@ -185,17 +192,24 @@ async function uploadExternalSource(req: Request, res: Response) { body: JSON.stringify({ query: gql.GET_EXTERNAL_EVENT_TYPE_ATTRIBUTE_SCHEMA, variables: { - name: eventType - } + name: eventType, + }, }), headers, - method: 'POST' + method: 'POST', }); - const eventTypeJSONResponse = await eventAttributeSchema.json(); - const getExternalEventTypeAttributeSchemaResponse = eventTypeJSONResponse as GetExternalEventTypeAttributeSchemaResponse | HasuraError; + const eventTypeJSONResponse = await eventAttributeSchema.json(); + const getExternalEventTypeAttributeSchemaResponse = eventTypeJSONResponse as + | GetExternalEventTypeAttributeSchemaResponse + | HasuraError; - if ((getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse).data?.external_event_type_by_pk?.attribute_schema !== null) { - const { data: { external_event_type_by_pk: eventAttributeSchema } } = getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse; + if ( + (getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse).data + ?.external_event_type_by_pk?.attribute_schema !== null + ) { + const { + data: { external_event_type_by_pk: eventAttributeSchema }, + } = getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse; if (eventAttributeSchema !== undefined && eventAttributeSchema !== null) { usedExternalEventTypesAttributesSchemas[eventType] = ajv.compile(eventAttributeSchema.attribute_schema); } @@ -205,10 +219,16 @@ async function uploadExternalSource(req: Request, res: Response) { for (const externalEvent of external_events) { try { const currentEventType = externalEvent.event_type_name; - const currentEventSchema: Ajv.ValidateFunction = usedExternalEventTypesAttributesSchemas[currentEventType]; + const currentEventSchema: Ajv.ValidateFunction = usedExternalEventTypesAttributesSchemas[currentEventType]; const eventAttributesAreValid = await currentEventSchema(externalEvent.attributes); if (!eventAttributesAreValid) { - throw new Error(`External Event '${externalEvent.key}' does not have a valid set of attributes, per it's type's schema:\n${JSON.stringify(currentEventSchema.errors)}`); + throw new Error( + `External Event '${ + externalEvent.key + }' does not have a valid set of attributes, per it's type's schema:\n${JSON.stringify( + currentEventSchema.errors, + )}`, + ); } } catch (error) { res.status(500).send({ message: (error as Error).message }); @@ -219,21 +239,21 @@ async function uploadExternalSource(req: Request, res: Response) { // Run the Hasura migration for creating an external source const derivationGroupInsert: DerivationGroupInsertInput = { name: derivation_group_name, - source_type_name: source_type_name - } + source_type_name: source_type_name, + }; const externalSourceInsert: ExternalSourceInsertInput = { attributes: attributes, derivation_group_name: derivation_group_name, end_time: end_time, external_events: { - data: external_events + data: external_events, }, key: key, source_type_name: source_type_name, start_time: start_time, - valid_at: valid_at - } + valid_at: valid_at, + }; const response = await fetch(GQL_API_URL, { body: JSON.stringify({ diff --git a/src/packages/external-source/gql.ts b/src/packages/external-source/gql.ts index eb6c435..78fae46 100644 --- a/src/packages/external-source/gql.ts +++ b/src/packages/external-source/gql.ts @@ -1,4 +1,5 @@ -export default { // TODO: discuss upset for derivation group +export default { + // TODO: discuss upset for derivation group CREATE_EXTERNAL_SOURCE: `#graphql mutation CreateExternalSource( $derivation_group: derivation_group_insert_input!, @@ -54,5 +55,5 @@ export default { // TODO: discuss upset for derivation group attribute_schema } } - ` -} + `, +}; diff --git a/src/packages/schemas/external-event-validation-schemata.ts b/src/packages/schemas/external-event-validation-schemata.ts index 5fcabd8..7d870a7 100644 --- a/src/packages/schemas/external-event-validation-schemata.ts +++ b/src/packages/schemas/external-event-validation-schemata.ts @@ -2,7 +2,6 @@ // Currently, we do the latter but the former doesn't seem like a bad idea! // The main argument against the former is what we have works and introducing new schemas could be a rabbit hole. - // export const externalEventTypeSchema = { // additionalProperties: false, // properties: { @@ -77,64 +76,67 @@ // }; export const externalSourceSchema = { - additionalProperties: false, - properties: { - external_events: { - items: { - additionalProperties: false, - properties: { - attributes: { - additionalProperties: true, - properties: {}, - required: [], - type: 'object' - }, - duration: { type: 'string' }, - event_type_name: { type: 'string' }, - key: { type: 'string' }, - start_time: { type: 'string' }, - }, - required: ['duration', 'event_type_name', 'key', 'attributes', 'start_time'], - type: 'object' - }, - type: 'array' + additionalProperties: false, + properties: { + external_events: { + items: { + additionalProperties: false, + properties: { + attributes: { + additionalProperties: true, + properties: {}, + required: [], + type: 'object', + }, + duration: { type: 'string' }, + event_type_name: { type: 'string' }, + key: { type: 'string' }, + start_time: { type: 'string' }, + }, + required: ['duration', 'event_type_name', 'key', 'attributes', 'start_time'], + type: 'object', + }, + type: 'array', + }, + source: { + additionalProperties: false, + properties: { + attributes: { + additionalProperties: true, + properties: {}, // constrained by type, checked by DB trigger on upload. TODO: CHECK LOCALLY? + required: [], + type: 'object', }, - source: { - additionalProperties: false, - properties: { - attributes: { - additionalProperties: true, - properties: {}, // constrained by type, checked by DB trigger on upload. TODO: CHECK LOCALLY? - required: [], - type: 'object' - }, - derivation_group_name: { type: 'string' }, - key: { type: 'string' }, - period: { - additionalProperties: false, - properties: { - end_time: { - pattern: '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', - type: 'string' - }, - start_time: { - pattern: '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', - type: 'string' - } - }, - required: ['start_time', 'end_time'], - type: 'object' - }, - source_type_name: { type: "string" }, - valid_at: { - pattern: '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', - type: "string" - } + derivation_group_name: { type: 'string' }, + key: { type: 'string' }, + period: { + additionalProperties: false, + properties: { + end_time: { + pattern: + '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', + type: 'string', }, - required: ["key", "source_type_name", "valid_at", "period", "attributes"], - type: 'object' - } + start_time: { + pattern: + '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', + type: 'string', + }, + }, + required: ['start_time', 'end_time'], + type: 'object', + }, + source_type_name: { type: 'string' }, + valid_at: { + pattern: + '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', + type: 'string', + }, + }, + required: ['key', 'source_type_name', 'valid_at', 'period', 'attributes'], + type: 'object', }, - required: ['source', 'external_events'], - type: 'object' -} \ No newline at end of file + }, + required: ['source', 'external_events'], + type: 'object', +}; diff --git a/src/types/external-event.ts b/src/types/external-event.ts index 224d5ce..809e022 100644 --- a/src/types/external-event.ts +++ b/src/types/external-event.ts @@ -4,12 +4,12 @@ export type ExternalEventInsertInput = { duration: string; event_type_name: string; key: string; -} +}; export type ExternalEventTypeInsertInput = { name: string; attribute_schema: object; -} +}; export type ExternalEvent = { key: string; @@ -17,4 +17,4 @@ export type ExternalEvent = { start_time: string; duration: string; attributes: object; -} +}; diff --git a/src/types/external-source.ts b/src/types/external-source.ts index 34e9e5c..72746fd 100644 --- a/src/types/external-source.ts +++ b/src/types/external-source.ts @@ -1,12 +1,12 @@ export type DerivationGroupInsertInput = { name: string; source_type_name: string; -} +}; export type ExternalSourceTypeInsertInput = { name: string; attribute_schema: object; -} +}; export type ExternalSourceInsertInput = { attributes: object; @@ -19,9 +19,9 @@ export type ExternalSourceInsertInput = { event_type_name: string; key: string; }[]; - } + }; key: string; source_type_name: string; start_time: string; valid_at: string; -} +}; diff --git a/test/external_source.validation.test.ts b/test/external_source.validation.test.ts index 26b592a..6c35876 100644 --- a/test/external_source.validation.test.ts +++ b/test/external_source.validation.test.ts @@ -6,56 +6,57 @@ const ajv = Ajv(); // type schemas const correctExternalEventTypeSchema = { - $schema: "http://json-schema.org/draft-07/schema", + $schema: 'http://json-schema.org/draft-07/schema', additionalProperties: false, - description: "Schema for the attributes of the TestEventType Type.", + description: 'Schema for the attributes of the TestEventType Type.', properties: { - code: { type: "string" }, - projectUser: { type: "string" } + code: { type: 'string' }, + projectUser: { type: 'string' }, }, - required: ["projectUser", "code"], - title: "TestEventType", - type: "object", -} + required: ['projectUser', 'code'], + title: 'TestEventType', + type: 'object', +}; const incorrectPassingExternalEventTypeSchema = { - $schema: "http://json-schema.org/draft-07/schema", + $schema: 'http://json-schema.org/draft-07/schema', additionalProperties: false, - descriptionFake: "Schema for the attributes of the TestEventType Type.", + descriptionFake: 'Schema for the attributes of the TestEventType Type.', doesntEvenExist: true, - propertgibberish: { // if you have something like this, it just registers as no properties existing, and fails any inserted events with attributes. - code: { type: "string" }, - projectUser: { type: "string" } + propertgibberish: { + // if you have something like this, it just registers as no properties existing, and fails any inserted events with attributes. + code: { type: 'string' }, + projectUser: { type: 'string' }, }, - requiredgibberish: ["projectUser", "code"], - title: "TestEventType", - type: "object", -} + requiredgibberish: ['projectUser', 'code'], + title: 'TestEventType', + type: 'object', +}; const incorrectFailingExternalEventTypeSchema = { - $schema: "http://json-schema.org/draft-07/schema", + $schema: 'http://json-schema.org/draft-07/schema', additionalProperties: false, - description: "Schema for the attributes of the TestEventType Type.", + description: 'Schema for the attributes of the TestEventType Type.', properties: { - code: { type: "string" }, - projectUser: { type: "string" } + code: { type: 'string' }, + projectUser: { type: 'string' }, }, required: 123, // this fails to validate at all since "required" IS well-defined as a field but expects an array - title: "TestEventType", - type: "object", -} + title: 'TestEventType', + type: 'object', +}; const externalSourceTypeSchema = { - $schema: "http://json-schema.org/draft-07/schema", + $schema: 'http://json-schema.org/draft-07/schema', additionalProperties: false, - description: "Schema for the attributes of the TestSourceType Type.", + description: 'Schema for the attributes of the TestSourceType Type.', properties: { - operator: { type: "string" }, - version: { type: "number" } + operator: { type: 'string' }, + version: { type: 'number' }, }, - required: ["version", "operator"], - title: "TestSourceType", - type: "object" + required: ['version', 'operator'], + title: 'TestSourceType', + type: 'object', }; // compiled schemas @@ -68,54 +69,52 @@ const externalSource = { external_events: [ { attributes: { - "code": "A", - "projectUser": "UserA" + code: 'A', + projectUser: 'UserA', }, duration: '01:10:00', event_type_name: 'TestExternalEventType', key: 'Event01', - start_time: '2024-023T00:23:00Z' + start_time: '2024-023T00:23:00Z', }, { attributes: { - "code": "B", - "projectUser": "UserB" + code: 'B', + projectUser: 'UserB', }, duration: '03:40:00', event_type_name: 'DSNContact', key: 'Event02', - start_time: '2024-021T00:21:00Z' - } + start_time: '2024-021T00:21:00Z', + }, ], source: { - attributes: { + attributes: { operator: 'alpha', - version: 1 + version: 1, }, derivation_group_name: 'TestDerivationGroup', key: 'TestExternalSourceKey', period: { end_time: '2024-01-28T00:00:00+00:00', - start_time: '2024-01-21T00:00:00+00:00' + start_time: '2024-01-21T00:00:00+00:00', }, source_type_name: 'TestExternalSourceType', - valid_at: '2024-01-19T00:00:00+00:00' - } -}; + valid_at: '2024-01-19T00:00:00+00:00', + }, +}; // invalid attributes const invalidSourceAttributes = { operator: 1, - version: 1 -} + version: 1, +}; const invalidEventAttributes = { code: 1, - projectUser: "UserB" -} - + projectUser: 'UserB', +}; describe('validation tests', () => { - // test validating type schema validation (demonstrate you can feed it bogus and its fine, but if an existing field gets a wrong type then its a problem) describe('attribute schema validation', () => { test('validating correct external event type schema', () => { @@ -133,7 +132,7 @@ describe('validation tests', () => { expect(schemaIsValid).toBe(false); const errors = ajv.errors; expect(errors?.length).toBe(1); - expect(errors?.at(0)?.message).toContain('should be array') + expect(errors?.at(0)?.message).toContain('should be array'); }); }); @@ -169,7 +168,8 @@ describe('validation tests', () => { test('correct external event type attribute validation', async () => { let eventAttributesAreValid: boolean = true; for (const external_event of externalSource.external_events) { - eventAttributesAreValid = eventAttributesAreValid && await compiledExternalEventTypeSchema(external_event.attributes); + eventAttributesAreValid = + eventAttributesAreValid && (await compiledExternalEventTypeSchema(external_event.attributes)); } expect(eventAttributesAreValid).toBe(true); }); From de0f009076cc79159da156b241bf795c10941b78 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Thu, 21 Nov 2024 12:36:14 -0500 Subject: [PATCH 16/39] Rework external source & event end-points to use multipart/form-data --- src/packages/external-event/external-event.ts | 61 ++++-- .../external-source/external-source.ts | 183 ++++++++++++------ src/types/external-event.ts | 16 ++ src/types/external-source.ts | 32 +++ src/util/time.ts | 21 +- 5 files changed, 237 insertions(+), 76 deletions(-) diff --git a/src/packages/external-event/external-event.ts b/src/packages/external-event/external-event.ts index fef7378..efa24cb 100644 --- a/src/packages/external-event/external-event.ts +++ b/src/packages/external-event/external-event.ts @@ -1,16 +1,30 @@ import type { Express, Request, Response } from 'express'; -import type { ExternalEventTypeInsertInput } from '../../types/external-event.js'; +import type { + CreateExternalEventTypeResponse, + ExternalEventTypeInsertInput, + UploadAttributeJSON, +} from '../../types/external-event.js'; import Ajv from 'ajv'; import { getEnv } from '../../env.js'; import getLogger from '../../logger.js'; import gql from './gql.js'; import { HasuraError } from '../../types/hasura.js'; +import { auth } from '../auth/middleware.js'; +import rateLimit from 'express-rate-limit'; +import multer from 'multer'; +import { parseJSONFile } from '../../util/fileParser.js'; +const upload = multer(); const logger = getLogger('packages/external-event/external-event'); -const { HASURA_API_URL } = getEnv(); +const { RATE_LIMITER_LOGIN_MAX, HASURA_API_URL } = getEnv(); const GQL_API_URL = `${HASURA_API_URL}/v1/graphql`; - const ajv = new Ajv(); +const refreshLimiter = rateLimit({ + legacyHeaders: false, + max: RATE_LIMITER_LOGIN_MAX, + standardHeaders: true, + windowMs: 15 * 60 * 1000, // 15 minutes +}); async function uploadExternalEventType(req: Request, res: Response) { const authorizationHeader = req.get('authorization'); @@ -19,8 +33,8 @@ async function uploadExternalEventType(req: Request, res: Response) { headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, } = req; - const { body } = req; - const { external_event_type_name, attribute_schema } = body; + const { body, file } = req; + const { external_event_type_name } = body; logger.info(`POST /uploadExternalEventType: Uploading External Event Type: ${external_event_type_name}`); const headers: HeadersInit = { @@ -30,8 +44,10 @@ async function uploadExternalEventType(req: Request, res: Response) { 'x-hasura-user-id': userHeader ? `${userHeader}` : '', }; + const uploadedExternalEventTypeAttributeSchema = await parseJSONFile(file); + // Validate schema is valid JSON Schema - const schemaIsValid: boolean = ajv.validateSchema(attribute_schema); + const schemaIsValid: boolean = ajv.validateSchema(uploadedExternalEventTypeAttributeSchema); if (!schemaIsValid) { logger.error( `POST /uploadExternalEventType: Schema validation failed for External Event Type ${external_event_type_name}`, @@ -42,24 +58,24 @@ async function uploadExternalEventType(req: Request, res: Response) { } // Make sure name in schema (title) and provided name match - try { - if (attribute_schema['title'] === undefined || attribute_schema.title !== external_event_type_name) { - throw new Error('Schema title does not match provided external event type name.'); - } - } catch (error) { + if ( + uploadedExternalEventTypeAttributeSchema['title'] === undefined || + uploadedExternalEventTypeAttributeSchema['title'] !== external_event_type_name + ) { + const errorMsg = 'Schema title does not match provided external event type name.'; logger.error( `POST /uploadExternalEventType: Error occurred during External Event Type ${external_event_type_name} upload`, ); - logger.error((error as Error).message); - res.status(500).send({ message: (error as Error).message }); + logger.error(errorMsg); + res.status(500).send({ message: errorMsg }); return; } - logger.info(`POST /uploadExternalEventType: Attribute schema was VALID`); + logger.info(`POST /uploadExternalEventType: Attribute schema is VALID`); // Run the Hasura migration for creating an external event const externalEventTypeInsertInput: ExternalEventTypeInsertInput = { - attribute_schema: attribute_schema, + attribute_schema: uploadedExternalEventTypeAttributeSchema, name: external_event_type_name, }; @@ -72,9 +88,6 @@ async function uploadExternalEventType(req: Request, res: Response) { method: 'POST', }); - type CreateExternalEventTypeResponse = { - data: { createExternalEventType: { attribute_schema: object; name: string } | null }; - }; const jsonResponse = await response.json(); const createExternalEventTypeResponse = jsonResponse as CreateExternalEventTypeResponse | HasuraError; @@ -89,7 +102,7 @@ export default (app: Express) => { * security: * - bearerAuth: [] * consumes: - * - application/json + * - multipart/form-data * produces: * - application/json * parameters: @@ -100,7 +113,7 @@ export default (app: Express) => { * required: false * requestBody: * content: - * application/json: + * multipart/form-data: * schema: * type: object * properties: @@ -132,5 +145,11 @@ export default (app: Express) => { * tags: * - Hasura */ - app.post('/uploadExternalEventType', uploadExternalEventType); + app.post( + '/uploadExternalEventType', + upload.single('attribute_schema'), + refreshLimiter, + auth, + uploadExternalEventType, + ); }; diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 8338052..1fcc2a2 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -3,31 +3,37 @@ import type { DerivationGroupInsertInput, ExternalSourceInsertInput, ExternalSourceTypeInsertInput, + CreateExternalSourceResponse, + CreateExternalSourceTypeResponse, + GetExternalSourceTypeAttributeSchemaResponse, + GetExternalEventTypeAttributeSchemaResponse, + UploadExternalSourceJSON, } from '../../types/external-source.js'; -import type { ExternalEventInsertInput } from '../../types/external-event.js'; +import type { ExternalEventInsertInput, UploadAttributeJSON } from '../../types/external-event.js'; import Ajv from 'ajv'; import { getEnv } from '../../env.js'; import getLogger from '../../logger.js'; import gql from './gql.js'; import { externalSourceSchema } from '../schemas/external-event-validation-schemata.js'; import { HasuraError } from '../../types/hasura.js'; +import rateLimit from 'express-rate-limit'; +import { auth } from '../auth/middleware.js'; +import multer from 'multer'; +import { parseJSONFile } from '../../util/fileParser.js'; +import { convertDoyToYmd, getIntervalInMs } from '../../util/time.js'; -type CreateExternalSourceResponse = { data: { createExternalSource: { name: string } | null } }; -type CreateExternalSourceTypeResponse = { - data: { createExternalSourceType: { attribute_schema: object; name: string } | null }; -}; -type GetExternalSourceTypeAttributeSchemaResponse = { - data: { external_source_type_by_pk: { attribute_schema: object } | null }; -}; -type GetExternalEventTypeAttributeSchemaResponse = { - data: { external_event_type_by_pk: { attribute_schema: object } | null }; -}; - +const upload = multer(); const logger = getLogger('packages/external-source/external-source'); -const { HASURA_API_URL } = getEnv(); +const { RATE_LIMITER_LOGIN_MAX, HASURA_API_URL } = getEnv(); const GQL_API_URL = `${HASURA_API_URL}/v1/graphql`; const ajv = new Ajv(); const compiledExternalSourceSchema = ajv.compile(externalSourceSchema); +const refreshLimiter = rateLimit({ + legacyHeaders: false, + max: RATE_LIMITER_LOGIN_MAX, + standardHeaders: true, + windowMs: 15 * 60 * 1000, // 15 minutes +}); async function uploadExternalSourceType(req: Request, res: Response) { const authorizationHeader = req.get('authorization'); @@ -36,8 +42,8 @@ async function uploadExternalSourceType(req: Request, res: Response) { headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, } = req; - const { body } = req; - const { external_source_type_name, attribute_schema } = body; + const { body, file } = req; + const { external_source_type_name } = body; logger.info(`POST /uploadExternalSourceType: Uploading External Source Type: ${external_source_type_name}`); const headers: HeadersInit = { @@ -48,10 +54,12 @@ async function uploadExternalSourceType(req: Request, res: Response) { 'x-hasura-user-id': userHeader ? `${userHeader}` : '', }; + const uploadedExternalSourceTypeAttributeSchema = await parseJSONFile(file); + // Validate schema is valid JSON Schema // NOTE: this does not check that all required attributes are included. technically, you could upload a schema for an event type, // and only really get punished for it when validating a source. - const schemaIsValid: boolean = ajv.validateSchema(attribute_schema); + const schemaIsValid: boolean = ajv.validateSchema(uploadedExternalSourceTypeAttributeSchema); if (!schemaIsValid) { logger.error( `POST /uploadExternalSourceType: Schema validation failed for External Source Type ${external_source_type_name}`, @@ -61,23 +69,27 @@ async function uploadExternalSourceType(req: Request, res: Response) { return; } - logger.info(`POST /uploadExternalSourceType: ${external_source_type_name} attribute schema was VALID`); + logger.info(`POST /uploadExternalSourceType: ${external_source_type_name} attribute schema is VALID`); // Make sure name in schema (title) and provided name match - try { - if (attribute_schema['title'] === undefined || attribute_schema.title !== external_source_type_name) { - throw new Error( - `${external_source_type_name} attribute schema title does not match provided external source type name.`, - ); - } - } catch (error) { - res.status(500).send({ message: (error as Error).message }); + if ( + uploadedExternalSourceTypeAttributeSchema['title'] === undefined || + uploadedExternalSourceTypeAttributeSchema['title'] !== external_source_type_name + ) { + const errorMsg = `${external_source_type_name} attribute schema title does not match provided external source type name.`; + logger.error( + `POST /uploadExternalSourceType: Error occurred during External Source Type ${external_source_type_name} upload`, + ); + logger.error(errorMsg); + res.status(500).send({ message: errorMsg }); return; } + logger.info(`POST /uploadExternalSourceType: Attribute schema is VALID`); + // Run the Hasura migration for creating an external source type (and inserting allowed event types) const externalSourceTypeInput: ExternalSourceTypeInsertInput = { - attribute_schema: attribute_schema, + attribute_schema: uploadedExternalSourceTypeAttributeSchema, name: external_source_type_name, }; @@ -101,10 +113,11 @@ async function uploadExternalSource(req: Request, res: Response) { const { headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, } = req; - const { body } = req; + const { body, file } = req; const { external_events, source } = body; const { attributes, derivation_group_name, key, source_type_name, period, valid_at } = source; const { end_time, start_time } = period; + const headers: HeadersInit = { Authorization: authorizationHeader ?? '', 'Content-Type': 'application/json', @@ -112,9 +125,33 @@ async function uploadExternalSource(req: Request, res: Response) { 'x-hasura-user-id': userHeader ? `${userHeader}` : '', }; + const uploadedExternalSource = await parseJSONFile(file); + + // Format all source times, validate that they're logical + const startTimeFormatted: string | undefined = convertDoyToYmd( + uploadedExternalSource.source.period.start_time.replaceAll('Z', ''), + )?.replace('Z', '+00:00'); + const endTimeFormatted: string | undefined = convertDoyToYmd( + uploadedExternalSource.source.period.end_time.replaceAll('Z', ''), + )?.replace('Z', '+00:00'); + const validAtFormatted: string | undefined = convertDoyToYmd( + uploadedExternalSource.source.valid_at.replaceAll('Z', ''), + )?.replace('Z', '+00:00'); + if (!startTimeFormatted || !endTimeFormatted || !validAtFormatted) { + const errorMsg = `Parsing failed - parsing dates in input failed. ${uploadedExternalSource.source.period.start_time}, ${uploadedExternalSource.source.period.end_time}, ${uploadedExternalSource.source.valid_at}`; + res.status(500).send({ message: errorMsg }); + return; + } + + if (new Date(startTimeFormatted) > new Date(endTimeFormatted)) { + const errorMsg = `Parsing failed - start time ${startTimeFormatted} after end time ${endTimeFormatted}.`; + res.status(500).send({ message: errorMsg }); + return; + } + logger.info(`POST /uploadExternalSource: Uploading External Source: ${key}`); - // Verify that this is a valid external source! + // Verify that this is a valid external source let sourceIsValid: boolean = false; sourceIsValid = await compiledExternalSourceSchema(body); if (sourceIsValid) { @@ -145,12 +182,11 @@ async function uploadExternalSource(req: Request, res: Response) { | GetExternalSourceTypeAttributeSchemaResponse | HasuraError; if ( - (getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse).data - ?.external_source_type_by_pk?.attribute_schema !== null + (getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse) + .external_source_type_by_pk?.attribute_schema !== null ) { - const { - data: { external_source_type_by_pk: sourceAttributeSchema }, - } = getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse; + const { external_source_type_by_pk: sourceAttributeSchema } = + getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse; if (sourceAttributeSchema !== undefined && sourceAttributeSchema !== null) { sourceSchema = ajv.compile(sourceAttributeSchema.attribute_schema); sourceAttributesAreValid = await sourceSchema(attributes); @@ -175,16 +211,50 @@ async function uploadExternalSource(req: Request, res: Response) { return; } - // Get the attribute schema(s) for all external event types used by the source's events - // get list of all used event types - const usedExternalEventTypes = external_events - .map((externalEvent: ExternalEventInsertInput) => externalEvent.event_type_name) - .reduce((acc: string[], externalEventType: string) => { - if (!acc.includes(externalEventType)) { - acc.push(externalEventType); + // Create External Event inputs + const externalEventsCreated: ExternalEventInsertInput[] = []; + const usedExternalEventTypes: string[] = []; + for (const externalEvent of uploadedExternalSource.events) { + // Ensure the duration is valid + try { + getIntervalInMs(externalEvent.duration); + } catch (error) { + const errorMsg = `Event duration has invalid format: ${externalEvent.key}\n${(error as Error).message}`; + res.status(500).send({ message: errorMsg }); + } + + // Validate external event is in the external source's start/stop bounds + const externalEventStart = Date.parse(convertDoyToYmd(externalEvent.start_time.replace('Z', '')) ?? ''); + const externalEventEnd = externalEventStart + getIntervalInMs(externalEvent.duration); + if (!(externalEventStart >= Date.parse(startTimeFormatted) && externalEventEnd <= Date.parse(endTimeFormatted))) { + const errorMsg = `Upload failed. Event (${ + externalEvent.key + }) not in bounds of source start and end: occurs from [${new Date(externalEventStart)},${new Date( + externalEventEnd, + )}], not subset of [${new Date(startTimeFormatted)},${new Date(endTimeFormatted)}].\n`; + res.status(500).send(errorMsg); + return; + } + + // If the event is valid... + if ( + externalEvent.event_type !== undefined && + externalEvent.start_time !== undefined && + externalEvent.duration !== undefined + ) { + // Add event type to usedExternalEventTypes for validation later + if (!usedExternalEventTypes.includes(externalEvent.event_type)) { + usedExternalEventTypes.push(externalEvent.event_type); } - return acc; - }, []); + externalEventsCreated.push({ + attributes: externalEvent.attributes, + duration: externalEvent.duration, + event_type_name: externalEvent.event_type, + key: externalEvent.key, + start_time: externalEvent.start_time, + }); + } + } const usedExternalEventTypesAttributesSchemas: Record = {}; for (const eventType of usedExternalEventTypes) { @@ -204,12 +274,11 @@ async function uploadExternalSource(req: Request, res: Response) { | HasuraError; if ( - (getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse).data - ?.external_event_type_by_pk?.attribute_schema !== null + (getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse) + .external_event_type_by_pk?.attribute_schema !== null ) { - const { - data: { external_event_type_by_pk: eventAttributeSchema }, - } = getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse; + const { external_event_type_by_pk: eventAttributeSchema } = + getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse; if (eventAttributeSchema !== undefined && eventAttributeSchema !== null) { usedExternalEventTypesAttributesSchemas[eventType] = ajv.compile(eventAttributeSchema.attribute_schema); } @@ -247,7 +316,7 @@ async function uploadExternalSource(req: Request, res: Response) { derivation_group_name: derivation_group_name, end_time: end_time, external_events: { - data: external_events, + data: externalEventsCreated, }, key: key, source_type_name: source_type_name, @@ -281,7 +350,7 @@ export default (app: Express) => { * security: * - bearerAuth: [] * consumes: - * - application/json + * - multipart/form-data * produces: * - application/json * parameters: @@ -292,7 +361,7 @@ export default (app: Express) => { * required: false * requestBody: * content: - * application/json: + * multipart/form-data: * schema: * type: object * properties: @@ -324,7 +393,13 @@ export default (app: Express) => { * tags: * - Hasura */ - app.post('/uploadExternalSourceType', uploadExternalSourceType); + app.post( + '/uploadExternalSourceType', + upload.single('attribute_schema'), + refreshLimiter, + auth, + uploadExternalSourceType, + ); /** * @swagger @@ -333,7 +408,7 @@ export default (app: Express) => { * security: * - bearerAuth: [] * consumes: - * - application/json + * - multipart/form-data * produces: * - application/json * parameters: @@ -344,7 +419,7 @@ export default (app: Express) => { * required: false * requestBody: * content: - * application/json: + * multipart/form-data: * schema: * type: object * properties: @@ -396,5 +471,5 @@ export default (app: Express) => { * tags: * - Hasura */ - app.post('/uploadExternalSource', uploadExternalSource); + app.post('/uploadExternalSource', upload.single('external_source'), refreshLimiter, auth, uploadExternalSource); }; diff --git a/src/types/external-event.ts b/src/types/external-event.ts index 809e022..d138a5b 100644 --- a/src/types/external-event.ts +++ b/src/types/external-event.ts @@ -6,6 +6,14 @@ export type ExternalEventInsertInput = { key: string; }; +export type ExternalEventJson = { + attributes: object; + duration: string; + event_type: string; + key: string; + start_time: string; +}; + export type ExternalEventTypeInsertInput = { name: string; attribute_schema: object; @@ -18,3 +26,11 @@ export type ExternalEvent = { duration: string; attributes: object; }; + +export type CreateExternalEventTypeResponse = { + createExternalEventType: { attribute_schema: object; name: string }; +}; + +export type UploadAttributeJSON = { + [x: string]: any; +}; diff --git a/src/types/external-source.ts b/src/types/external-source.ts index 72746fd..991639e 100644 --- a/src/types/external-source.ts +++ b/src/types/external-source.ts @@ -1,3 +1,5 @@ +import { ExternalEventJson } from './external-event'; + export type DerivationGroupInsertInput = { name: string; source_type_name: string; @@ -25,3 +27,33 @@ export type ExternalSourceInsertInput = { start_time: string; valid_at: string; }; + +export type UploadExternalSourceJSON = { + events: ExternalEventJson[]; + source: { + attributes: object; + key: string; + period: { + end_time: string; + start_time: string; + }; + source_type: string; + valid_at: string; + }; +}; + +export type CreateExternalSourceResponse = { + createExternalSource: { name: string }; +}; + +export type CreateExternalSourceTypeResponse = { + createExternalSourceType: { attribute_schema: object; name: string }; +}; + +export type GetExternalSourceTypeAttributeSchemaResponse = { + external_source_type_by_pk: { attribute_schema: object }; +}; + +export type GetExternalEventTypeAttributeSchemaResponse = { + external_event_type_by_pk: { attribute_schema: object }; +}; diff --git a/src/util/time.ts b/src/util/time.ts index 197a302..1b508ee 100644 --- a/src/util/time.ts +++ b/src/util/time.ts @@ -1,4 +1,5 @@ import { ParsedDoyString, ParsedYmdString } from '../types/time'; +import parseInterval from 'postgres-interval'; function parseNumber(number: number | string): number { return parseInt(`${number}`, 10); @@ -92,7 +93,7 @@ export function convertDateToDoy(dateString: string, numDecimals = 6): string | return null; } -function convertDoyToYmd(doyString: string, numDecimals = 6, includeMsecs = true): string | null { +export function convertDoyToYmd(doyString: string, numDecimals = 6, includeMsecs = true): string | null { const parsedDoy: ParsedDoyString = parseDoyOrYmdTime(doyString, numDecimals) as ParsedDoyString; if (parsedDoy !== null) { @@ -128,3 +129,21 @@ export function getTimeDifference(dateString1: string, dateString2: string, numD } return null; } + +/** + * Returns a Postgres Interval duration in milliseconds. + * If duration is null, undefined, or empty string then we just return 0. + * @note This function assumes 24-hour days. + */ +export function getIntervalInMs(interval: string | null | undefined): number { + if (interval !== null && interval !== undefined && interval !== '') { + const parsedInterval = parseInterval(interval); + const { days, hours, milliseconds, minutes, seconds } = parsedInterval; + const daysInMs = days * 24 * 60 * 60 * 1000; + const hoursInMs = hours * 60 * 60 * 1000; + const minutesInMs = minutes * 60 * 1000; + const secondsInMs = seconds * 1000; + return daysInMs + hoursInMs + minutesInMs + secondsInMs + milliseconds; + } + return 0; +} From 30967c3b0deb35ef1882417ed34cd98ffd5ae0b5 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Thu, 21 Nov 2024 15:59:12 -0500 Subject: [PATCH 17/39] Rework uploadExternalSource to use form data rather than file Parsing is done on the UI side, and parsed data is sent as a multipart/form-data request --- .../external-source/external-source.ts | 85 ++++++++----------- 1 file changed, 37 insertions(+), 48 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 1fcc2a2..31b3fd4 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -16,13 +16,12 @@ import getLogger from '../../logger.js'; import gql from './gql.js'; import { externalSourceSchema } from '../schemas/external-event-validation-schemata.js'; import { HasuraError } from '../../types/hasura.js'; -import rateLimit from 'express-rate-limit'; import { auth } from '../auth/middleware.js'; +import rateLimit from 'express-rate-limit'; import multer from 'multer'; import { parseJSONFile } from '../../util/fileParser.js'; -import { convertDoyToYmd, getIntervalInMs } from '../../util/time.js'; -const upload = multer(); +const upload = multer({ limits: { fieldSize: 25 * 1024 * 1024 } }); const logger = getLogger('packages/external-source/external-source'); const { RATE_LIMITER_LOGIN_MAX, HASURA_API_URL } = getEnv(); const GQL_API_URL = `${HASURA_API_URL}/v1/graphql`; @@ -33,7 +32,7 @@ const refreshLimiter = rateLimit({ max: RATE_LIMITER_LOGIN_MAX, standardHeaders: true, windowMs: 15 * 60 * 1000, // 15 minutes -}); +}) async function uploadExternalSourceType(req: Request, res: Response) { const authorizationHeader = req.get('authorization'); @@ -113,10 +112,26 @@ async function uploadExternalSource(req: Request, res: Response) { const { headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, } = req; - const { body, file } = req; - const { external_events, source } = body; - const { attributes, derivation_group_name, key, source_type_name, period, valid_at } = source; - const { end_time, start_time } = period; + const { body } = req; + const { attributes, derivation_group_name, key, end_time, start_time, source_type_name, valid_at, external_events } = body; + const parsedAttributes = JSON.parse(attributes); + const parsedExternalEvents = JSON.parse(external_events); + + // Re-package the fields as a JSON object to be validated by the meta-schema + const externalSourceJson = { + external_events: parsedExternalEvents, + source: { + attributes: parsedAttributes, + derivation_group_name: derivation_group_name, + key: key, + period: { + end_time: end_time, + start_time: start_time, + }, + source_type_name: source_type_name, + valid_at: valid_at + }, + } const headers: HeadersInit = { Authorization: authorizationHeader ?? '', @@ -153,7 +168,7 @@ async function uploadExternalSource(req: Request, res: Response) { // Verify that this is a valid external source let sourceIsValid: boolean = false; - sourceIsValid = await compiledExternalSourceSchema(body); + sourceIsValid = await compiledExternalSourceSchema(externalSourceJson); if (sourceIsValid) { logger.info(`POST /uploadExternalSource: External Source ${key}'s formatting is valid`); } else { @@ -189,7 +204,7 @@ async function uploadExternalSource(req: Request, res: Response) { getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse; if (sourceAttributeSchema !== undefined && sourceAttributeSchema !== null) { sourceSchema = ajv.compile(sourceAttributeSchema.attribute_schema); - sourceAttributesAreValid = await sourceSchema(attributes); + sourceAttributesAreValid = await sourceSchema(parsedAttributes); } else { // source type does not exist! logger.error(`POST /uploadExternalSource: External Source Type ${source_type_name} does not exist!`); @@ -211,40 +226,13 @@ async function uploadExternalSource(req: Request, res: Response) { return; } - // Create External Event inputs - const externalEventsCreated: ExternalEventInsertInput[] = []; - const usedExternalEventTypes: string[] = []; - for (const externalEvent of uploadedExternalSource.events) { - // Ensure the duration is valid - try { - getIntervalInMs(externalEvent.duration); - } catch (error) { - const errorMsg = `Event duration has invalid format: ${externalEvent.key}\n${(error as Error).message}`; - res.status(500).send({ message: errorMsg }); - } - - // Validate external event is in the external source's start/stop bounds - const externalEventStart = Date.parse(convertDoyToYmd(externalEvent.start_time.replace('Z', '')) ?? ''); - const externalEventEnd = externalEventStart + getIntervalInMs(externalEvent.duration); - if (!(externalEventStart >= Date.parse(startTimeFormatted) && externalEventEnd <= Date.parse(endTimeFormatted))) { - const errorMsg = `Upload failed. Event (${ - externalEvent.key - }) not in bounds of source start and end: occurs from [${new Date(externalEventStart)},${new Date( - externalEventEnd, - )}], not subset of [${new Date(startTimeFormatted)},${new Date(endTimeFormatted)}].\n`; - res.status(500).send(errorMsg); - return; - } - - // If the event is valid... - if ( - externalEvent.event_type !== undefined && - externalEvent.start_time !== undefined && - externalEvent.duration !== undefined - ) { - // Add event type to usedExternalEventTypes for validation later - if (!usedExternalEventTypes.includes(externalEvent.event_type)) { - usedExternalEventTypes.push(externalEvent.event_type); + // Get the attribute schema(s) for all external event types used by the source's events + // get list of all used event types + const usedExternalEventTypes = parsedExternalEvents + .map((externalEvent: ExternalEventInsertInput) => externalEvent.event_type_name) + .reduce((acc: string[], externalEventType: string) => { + if (!acc.includes(externalEventType)) { + acc.push(externalEventType); } externalEventsCreated.push({ attributes: externalEvent.attributes, @@ -285,7 +273,7 @@ async function uploadExternalSource(req: Request, res: Response) { } } - for (const externalEvent of external_events) { + for (const externalEvent of parsedExternalEvents) { try { const currentEventType = externalEvent.event_type_name; const currentEventSchema: Ajv.ValidateFunction = usedExternalEventTypesAttributesSchemas[currentEventType]; @@ -300,6 +288,7 @@ async function uploadExternalSource(req: Request, res: Response) { ); } } catch (error) { + logger.error(`POST /uploadExternalSource: External Event ${externalEvent.key}'s attributes are invalid`); res.status(500).send({ message: (error as Error).message }); return; } @@ -312,11 +301,11 @@ async function uploadExternalSource(req: Request, res: Response) { }; const externalSourceInsert: ExternalSourceInsertInput = { - attributes: attributes, + attributes: parsedAttributes, derivation_group_name: derivation_group_name, end_time: end_time, external_events: { - data: externalEventsCreated, + data: parsedExternalEvents, }, key: key, source_type_name: source_type_name, @@ -471,5 +460,5 @@ export default (app: Express) => { * tags: * - Hasura */ - app.post('/uploadExternalSource', upload.single('external_source'), refreshLimiter, auth, uploadExternalSource); + app.post('/uploadExternalSource', upload.any(), refreshLimiter, auth, uploadExternalSource); }; From 3ccd8178f917da3a1dc622801d792e7e332bbe1d Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Thu, 21 Nov 2024 16:30:47 -0500 Subject: [PATCH 18/39] Rework Hasura responses --- .../external-source/external-source.ts | 52 +++---------------- 1 file changed, 8 insertions(+), 44 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 31b3fd4..fecb1f5 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -9,7 +9,7 @@ import type { GetExternalEventTypeAttributeSchemaResponse, UploadExternalSourceJSON, } from '../../types/external-source.js'; -import type { ExternalEventInsertInput, UploadAttributeJSON } from '../../types/external-event.js'; +import type { ExternalEventInsertInput, ExternalEventJson, UploadAttributeJSON } from '../../types/external-event.js'; import Ajv from 'ajv'; import { getEnv } from '../../env.js'; import getLogger from '../../logger.js'; @@ -140,30 +140,6 @@ async function uploadExternalSource(req: Request, res: Response) { 'x-hasura-user-id': userHeader ? `${userHeader}` : '', }; - const uploadedExternalSource = await parseJSONFile(file); - - // Format all source times, validate that they're logical - const startTimeFormatted: string | undefined = convertDoyToYmd( - uploadedExternalSource.source.period.start_time.replaceAll('Z', ''), - )?.replace('Z', '+00:00'); - const endTimeFormatted: string | undefined = convertDoyToYmd( - uploadedExternalSource.source.period.end_time.replaceAll('Z', ''), - )?.replace('Z', '+00:00'); - const validAtFormatted: string | undefined = convertDoyToYmd( - uploadedExternalSource.source.valid_at.replaceAll('Z', ''), - )?.replace('Z', '+00:00'); - if (!startTimeFormatted || !endTimeFormatted || !validAtFormatted) { - const errorMsg = `Parsing failed - parsing dates in input failed. ${uploadedExternalSource.source.period.start_time}, ${uploadedExternalSource.source.period.end_time}, ${uploadedExternalSource.source.valid_at}`; - res.status(500).send({ message: errorMsg }); - return; - } - - if (new Date(startTimeFormatted) > new Date(endTimeFormatted)) { - const errorMsg = `Parsing failed - start time ${startTimeFormatted} after end time ${endTimeFormatted}.`; - res.status(500).send({ message: errorMsg }); - return; - } - logger.info(`POST /uploadExternalSource: Uploading External Source: ${key}`); // Verify that this is a valid external source @@ -193,7 +169,7 @@ async function uploadExternalSource(req: Request, res: Response) { let sourceAttributesAreValid: boolean = false; let sourceSchema: Ajv.ValidateFunction | undefined = undefined; const sourceTypeResponseJSON = await sourceAttributeSchema.json(); - const getExternalSourceTypeAttributeSchemaResponse = sourceTypeResponseJSON as + const getExternalSourceTypeAttributeSchemaResponse = sourceTypeResponseJSON.data as | GetExternalSourceTypeAttributeSchemaResponse | HasuraError; if ( @@ -226,23 +202,12 @@ async function uploadExternalSource(req: Request, res: Response) { return; } - // Get the attribute schema(s) for all external event types used by the source's events - // get list of all used event types - const usedExternalEventTypes = parsedExternalEvents - .map((externalEvent: ExternalEventInsertInput) => externalEvent.event_type_name) - .reduce((acc: string[], externalEventType: string) => { - if (!acc.includes(externalEventType)) { - acc.push(externalEventType); - } - externalEventsCreated.push({ - attributes: externalEvent.attributes, - duration: externalEvent.duration, - event_type_name: externalEvent.event_type, - key: externalEvent.key, - start_time: externalEvent.start_time, - }); + const usedExternalEventTypes = parsedExternalEvents.reduce((acc: string[], externalEvent: ExternalEventInsertInput) => { + if (!acc.includes(externalEvent.event_type_name)) { + acc.push(externalEvent.event_type_name); } - } + return acc; + }, []); const usedExternalEventTypesAttributesSchemas: Record = {}; for (const eventType of usedExternalEventTypes) { @@ -257,10 +222,9 @@ async function uploadExternalSource(req: Request, res: Response) { method: 'POST', }); const eventTypeJSONResponse = await eventAttributeSchema.json(); - const getExternalEventTypeAttributeSchemaResponse = eventTypeJSONResponse as + const getExternalEventTypeAttributeSchemaResponse = eventTypeJSONResponse.data as | GetExternalEventTypeAttributeSchemaResponse | HasuraError; - if ( (getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse) .external_event_type_by_pk?.attribute_schema !== null From c283394509d5a0f3818e43e65ff7475894b4c5aa Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Fri, 22 Nov 2024 09:47:08 -0500 Subject: [PATCH 19/39] Formatting fixes --- .../external-source/external-source.ts | 45 ++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index fecb1f5..c224702 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -7,9 +7,8 @@ import type { CreateExternalSourceTypeResponse, GetExternalSourceTypeAttributeSchemaResponse, GetExternalEventTypeAttributeSchemaResponse, - UploadExternalSourceJSON, } from '../../types/external-source.js'; -import type { ExternalEventInsertInput, ExternalEventJson, UploadAttributeJSON } from '../../types/external-event.js'; +import type { ExternalEventInsertInput, UploadAttributeJSON } from '../../types/external-event.js'; import Ajv from 'ajv'; import { getEnv } from '../../env.js'; import getLogger from '../../logger.js'; @@ -32,7 +31,7 @@ const refreshLimiter = rateLimit({ max: RATE_LIMITER_LOGIN_MAX, standardHeaders: true, windowMs: 15 * 60 * 1000, // 15 minutes -}) +}); async function uploadExternalSourceType(req: Request, res: Response) { const authorizationHeader = req.get('authorization'); @@ -113,7 +112,8 @@ async function uploadExternalSource(req: Request, res: Response) { headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, } = req; const { body } = req; - const { attributes, derivation_group_name, key, end_time, start_time, source_type_name, valid_at, external_events } = body; + const { attributes, derivation_group_name, key, end_time, start_time, source_type_name, valid_at, external_events } = + body; const parsedAttributes = JSON.parse(attributes); const parsedExternalEvents = JSON.parse(external_events); @@ -129,9 +129,9 @@ async function uploadExternalSource(req: Request, res: Response) { start_time: start_time, }, source_type_name: source_type_name, - valid_at: valid_at + valid_at: valid_at, }, - } + }; const headers: HeadersInit = { Authorization: authorizationHeader ?? '', @@ -142,7 +142,7 @@ async function uploadExternalSource(req: Request, res: Response) { logger.info(`POST /uploadExternalSource: Uploading External Source: ${key}`); - // Verify that this is a valid external source + // Verify External Source input against meta-schema let sourceIsValid: boolean = false; sourceIsValid = await compiledExternalSourceSchema(externalSourceJson); if (sourceIsValid) { @@ -172,6 +172,7 @@ async function uploadExternalSource(req: Request, res: Response) { const getExternalSourceTypeAttributeSchemaResponse = sourceTypeResponseJSON.data as | GetExternalSourceTypeAttributeSchemaResponse | HasuraError; + if ( (getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse) .external_source_type_by_pk?.attribute_schema !== null @@ -182,13 +183,11 @@ async function uploadExternalSource(req: Request, res: Response) { sourceSchema = ajv.compile(sourceAttributeSchema.attribute_schema); sourceAttributesAreValid = await sourceSchema(parsedAttributes); } else { - // source type does not exist! logger.error(`POST /uploadExternalSource: External Source Type ${source_type_name} does not exist!`); res.status(500).send({ message: `External Source Type ${source_type_name} does not exist!` }); return; } } - if (sourceAttributesAreValid) { logger.info(`POST /uploadExternalSource: External Source ${key}'s attributes are valid`); } else { @@ -202,13 +201,16 @@ async function uploadExternalSource(req: Request, res: Response) { return; } - const usedExternalEventTypes = parsedExternalEvents.reduce((acc: string[], externalEvent: ExternalEventInsertInput) => { - if (!acc.includes(externalEvent.event_type_name)) { - acc.push(externalEvent.event_type_name); - } - return acc; - }, []); - + // Build a Record for all event type's being used + const usedExternalEventTypes = parsedExternalEvents.reduce( + (acc: string[], externalEvent: ExternalEventInsertInput) => { + if (!acc.includes(externalEvent.event_type_name)) { + acc.push(externalEvent.event_type_name); + } + return acc; + }, + [], + ); const usedExternalEventTypesAttributesSchemas: Record = {}; for (const eventType of usedExternalEventTypes) { const eventAttributeSchema = await fetch(GQL_API_URL, { @@ -237,6 +239,7 @@ async function uploadExternalSource(req: Request, res: Response) { } } + // Validate attributes of all External Events in the source for (const externalEvent of parsedExternalEvents) { try { const currentEventType = externalEvent.event_type_name; @@ -258,7 +261,6 @@ async function uploadExternalSource(req: Request, res: Response) { } } - // Run the Hasura migration for creating an external source const derivationGroupInsert: DerivationGroupInsertInput = { name: derivation_group_name, source_type_name: source_type_name, @@ -413,9 +415,12 @@ export default (app: Express) => { * application/json: * schema: * properties: - * name: - * description: Name of the created External Source - * type: string + * createExternalSource: + * type: object + * properties: + * name: + * description: Name of the created External Source + * type: string * 403: * description: Unauthorized error * 401: From c62023e3df6b3edeca45dd29d2c5904f51b93ea6 Mon Sep 17 00:00:00 2001 From: psubram3 Date: Fri, 22 Nov 2024 10:44:19 -0800 Subject: [PATCH 20/39] attempt merge i --- src/main.ts | 2 - src/packages/external-event/external-event.ts | 155 ------------ src/packages/external-event/gql.ts | 11 - .../external-source/external-source.ts | 181 ++++++-------- src/packages/external-source/gql.ts | 28 ++- .../external-event-validation-schemata.ts | 235 ++++++++---------- src/types/external-event.ts | 36 --- src/types/external-source.ts | 43 +++- 8 files changed, 242 insertions(+), 449 deletions(-) delete mode 100644 src/packages/external-event/external-event.ts delete mode 100644 src/packages/external-event/gql.ts delete mode 100644 src/types/external-event.ts diff --git a/src/main.ts b/src/main.ts index e26168a..fcf3692 100644 --- a/src/main.ts +++ b/src/main.ts @@ -12,7 +12,6 @@ import initHealthRoutes from './packages/health/health.js'; import initPlanRoutes from './packages/plan/plan.js'; import initSwaggerRoutes from './packages/swagger/swagger.js'; import initExternalSourceRoutes from './packages/external-source/external-source.js'; -import initExternalEventRoutes from './packages/external-event/external-event.js'; import cookieParser from 'cookie-parser'; import { AuthAdapter } from './types/auth.js'; import { NoAuthAdapter } from './packages/auth/adapters/NoAuthAdapter.js'; @@ -51,7 +50,6 @@ async function main(): Promise { initHasuraRoutes(app); initPlanRoutes(app); initExternalSourceRoutes(app); - initExternalEventRoutes(app); initSwaggerRoutes(app); app.listen(PORT, () => { diff --git a/src/packages/external-event/external-event.ts b/src/packages/external-event/external-event.ts deleted file mode 100644 index efa24cb..0000000 --- a/src/packages/external-event/external-event.ts +++ /dev/null @@ -1,155 +0,0 @@ -import type { Express, Request, Response } from 'express'; -import type { - CreateExternalEventTypeResponse, - ExternalEventTypeInsertInput, - UploadAttributeJSON, -} from '../../types/external-event.js'; -import Ajv from 'ajv'; -import { getEnv } from '../../env.js'; -import getLogger from '../../logger.js'; -import gql from './gql.js'; -import { HasuraError } from '../../types/hasura.js'; -import { auth } from '../auth/middleware.js'; -import rateLimit from 'express-rate-limit'; -import multer from 'multer'; -import { parseJSONFile } from '../../util/fileParser.js'; - -const upload = multer(); -const logger = getLogger('packages/external-event/external-event'); -const { RATE_LIMITER_LOGIN_MAX, HASURA_API_URL } = getEnv(); -const GQL_API_URL = `${HASURA_API_URL}/v1/graphql`; -const ajv = new Ajv(); -const refreshLimiter = rateLimit({ - legacyHeaders: false, - max: RATE_LIMITER_LOGIN_MAX, - standardHeaders: true, - windowMs: 15 * 60 * 1000, // 15 minutes -}); - -async function uploadExternalEventType(req: Request, res: Response) { - const authorizationHeader = req.get('authorization'); - - const { - headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, - } = req; - - const { body, file } = req; - const { external_event_type_name } = body; - logger.info(`POST /uploadExternalEventType: Uploading External Event Type: ${external_event_type_name}`); - - const headers: HeadersInit = { - Authorization: authorizationHeader ?? '', - 'Content-Type': 'application/json', - 'x-hasura-role': roleHeader ? `${roleHeader}` : '', - 'x-hasura-user-id': userHeader ? `${userHeader}` : '', - }; - - const uploadedExternalEventTypeAttributeSchema = await parseJSONFile(file); - - // Validate schema is valid JSON Schema - const schemaIsValid: boolean = ajv.validateSchema(uploadedExternalEventTypeAttributeSchema); - if (!schemaIsValid) { - logger.error( - `POST /uploadExternalEventType: Schema validation failed for External Event Type ${external_event_type_name}`, - ); - ajv.errors?.forEach(ajvError => logger.error(ajvError)); - res.status(500).send({ message: ajv.errors }); - return; - } - - // Make sure name in schema (title) and provided name match - if ( - uploadedExternalEventTypeAttributeSchema['title'] === undefined || - uploadedExternalEventTypeAttributeSchema['title'] !== external_event_type_name - ) { - const errorMsg = 'Schema title does not match provided external event type name.'; - logger.error( - `POST /uploadExternalEventType: Error occurred during External Event Type ${external_event_type_name} upload`, - ); - logger.error(errorMsg); - res.status(500).send({ message: errorMsg }); - return; - } - - logger.info(`POST /uploadExternalEventType: Attribute schema is VALID`); - - // Run the Hasura migration for creating an external event - const externalEventTypeInsertInput: ExternalEventTypeInsertInput = { - attribute_schema: uploadedExternalEventTypeAttributeSchema, - name: external_event_type_name, - }; - - const response = await fetch(GQL_API_URL, { - body: JSON.stringify({ - query: gql.CREATE_EXTERNAL_EVENT_TYPE, - variables: { eventType: externalEventTypeInsertInput }, - }), - headers, - method: 'POST', - }); - - const jsonResponse = await response.json(); - const createExternalEventTypeResponse = jsonResponse as CreateExternalEventTypeResponse | HasuraError; - - res.json(createExternalEventTypeResponse); -} - -export default (app: Express) => { - /** - * @swagger - * /uploadExternalEventType: - * post: - * security: - * - bearerAuth: [] - * consumes: - * - multipart/form-data - * produces: - * - application/json - * parameters: - * - in: header - * name: x-hasura-role - * schema: - * type: string - * required: false - * requestBody: - * content: - * multipart/form-data: - * schema: - * type: object - * properties: - * attribute_schema: - * type: object - * external_event_type_name: - * type: string - * required: - * - external_event_type_name - * attribute_schema - * responses: - * 200: - * description: Created External Event Type - * content: - * application/json: - * schema: - * properties: - * attribute_schema: - * description: JSON Schema for the created External Event Type's attributes - * type: object - * name: - * description: Name of the created External Event Type - * type: string - * 403: - * description: Unauthorized error - * 401: - * description: Unauthenticated error - * summary: Uploads an External Event Type definition (containing name & attributes schema) to Hasura. - * tags: - * - Hasura - */ - app.post( - '/uploadExternalEventType', - upload.single('attribute_schema'), - refreshLimiter, - auth, - uploadExternalEventType, - ); -}; diff --git a/src/packages/external-event/gql.ts b/src/packages/external-event/gql.ts deleted file mode 100644 index 8b9dae3..0000000 --- a/src/packages/external-event/gql.ts +++ /dev/null @@ -1,11 +0,0 @@ -export default { - CREATE_EXTERNAL_EVENT_TYPE: `#graphql - mutation CreateExternalEventType($eventType: external_event_type_insert_input!) - { - createExternalEventType: insert_external_event_type_one(object: $eventType) { - attribute_schema - name - } - } - `, -}; diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index c224702..839331c 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -7,24 +7,31 @@ import type { CreateExternalSourceTypeResponse, GetExternalSourceTypeAttributeSchemaResponse, GetExternalEventTypeAttributeSchemaResponse, + UploadExternalSourceJSON, + UploadAttributeJSON, + ExternalEventTypeInsertInput, + ExternalEventInsertInput, + ExternalEventJson, + ExternalEvent, } from '../../types/external-source.js'; -import type { ExternalEventInsertInput, UploadAttributeJSON } from '../../types/external-event.js'; import Ajv from 'ajv'; import { getEnv } from '../../env.js'; import getLogger from '../../logger.js'; import gql from './gql.js'; -import { externalSourceSchema } from '../schemas/external-event-validation-schemata.js'; +import { attributeSchemaMetaschema, externalSourceSchema } from '../schemas/external-event-validation-schemata.js'; import { HasuraError } from '../../types/hasura.js'; import { auth } from '../auth/middleware.js'; import rateLimit from 'express-rate-limit'; import multer from 'multer'; import { parseJSONFile } from '../../util/fileParser.js'; +import { convertDoyToYmd } from '../../util/time.js'; const upload = multer({ limits: { fieldSize: 25 * 1024 * 1024 } }); const logger = getLogger('packages/external-source/external-source'); const { RATE_LIMITER_LOGIN_MAX, HASURA_API_URL } = getEnv(); const GQL_API_URL = `${HASURA_API_URL}/v1/graphql`; const ajv = new Ajv(); +const compiledAttributeMetaschema = ajv.compile(attributeSchemaMetaschema); const compiledExternalSourceSchema = ajv.compile(externalSourceSchema); const refreshLimiter = rateLimit({ legacyHeaders: false, @@ -33,16 +40,15 @@ const refreshLimiter = rateLimit({ windowMs: 15 * 60 * 1000, // 15 minutes }); -async function uploadExternalSourceType(req: Request, res: Response) { +async function uploadExternalSourceEventTypes(req: Request, res: Response) { const authorizationHeader = req.get('authorization'); const { headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, } = req; - const { body, file } = req; - const { external_source_type_name } = body; - logger.info(`POST /uploadExternalSourceType: Uploading External Source Type: ${external_source_type_name}`); + const { file } = req; + logger.info(`POST /uploadExternalSourceEventTypes: Uploading External Source and Event Types...`); const headers: HeadersInit = { Authorization: authorizationHeader ?? '', @@ -52,49 +58,48 @@ async function uploadExternalSourceType(req: Request, res: Response) { 'x-hasura-user-id': userHeader ? `${userHeader}` : '', }; - const uploadedExternalSourceTypeAttributeSchema = await parseJSONFile(file); + const uploadedExternalSourceEventTypeAttributeSchema = await parseJSONFile(file); - // Validate schema is valid JSON Schema - // NOTE: this does not check that all required attributes are included. technically, you could upload a schema for an event type, - // and only really get punished for it when validating a source. - const schemaIsValid: boolean = ajv.validateSchema(uploadedExternalSourceTypeAttributeSchema); - if (!schemaIsValid) { + // Validate uploaded attribute schemas are formatted validly + const schemasAreValid: boolean = await compiledAttributeMetaschema(uploadedExternalSourceEventTypeAttributeSchema); + if (!schemasAreValid) { logger.error( - `POST /uploadExternalSourceType: Schema validation failed for External Source Type ${external_source_type_name}`, + `POST /uploadExternalSourceEventTypes: Schema validation failed for uploaded source and event types.`, ); - ajv.errors?.forEach(ajvError => logger.error(ajvError)); - res.status(500).send({ message: ajv.errors }); + compiledAttributeMetaschema.errors?.forEach(error => logger.error(error)); + res.status(500).send({ message: compiledAttributeMetaschema.errors }); return; } - logger.info(`POST /uploadExternalSourceType: ${external_source_type_name} attribute schema is VALID`); + logger.info(`POST /uploadExternalSourceEventTypes: Uploaded attribute schema(s) are VALID`); - // Make sure name in schema (title) and provided name match - if ( - uploadedExternalSourceTypeAttributeSchema['title'] === undefined || - uploadedExternalSourceTypeAttributeSchema['title'] !== external_source_type_name - ) { - const errorMsg = `${external_source_type_name} attribute schema title does not match provided external source type name.`; - logger.error( - `POST /uploadExternalSourceType: Error occurred during External Source Type ${external_source_type_name} upload`, - ); - logger.error(errorMsg); - res.status(500).send({ message: errorMsg }); - return; - } + // extract the external sources and event types + const externalSourceTypeInput: ExternalSourceTypeInsertInput = []; + const externalEventTypeInput: ExternalEventTypeInsertInput = []; - logger.info(`POST /uploadExternalSourceType: Attribute schema is VALID`); + const external_event_types = uploadedExternalSourceEventTypeAttributeSchema.event_types; + const event_type_keys = Object.keys(external_event_types); + for (const external_event_type of event_type_keys) { + externalSourceTypeInput.push({ + attribute_schema: external_event_types[external_event_type], + name: external_event_type + }) + } - // Run the Hasura migration for creating an external source type (and inserting allowed event types) - const externalSourceTypeInput: ExternalSourceTypeInsertInput = { - attribute_schema: uploadedExternalSourceTypeAttributeSchema, - name: external_source_type_name, - }; + const external_source_types = uploadedExternalSourceEventTypeAttributeSchema.source_types; + const source_type_keys = Object.keys(external_source_types); + for (const external_source_type of source_type_keys) { + externalEventTypeInput.push({ + attribute_schema: external_source_types[external_source_type], + name: external_source_type + }) + } + // Run the Hasura migration for creating all types, in one go const response = await fetch(GQL_API_URL, { body: JSON.stringify({ - query: gql.CREATE_EXTERNAL_SOURCE_TYPE, - variables: { sourceType: externalSourceTypeInput }, + query: gql.CREATE_EXTERNAL_SOURCE_EVENT_TYPES, + variables: { externalEventTypes: externalEventTypeInput, externalSourceTypes: externalSourceTypeInput }, }), headers, method: 'POST', @@ -112,10 +117,10 @@ async function uploadExternalSource(req: Request, res: Response) { headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, } = req; const { body } = req; - const { attributes, derivation_group_name, key, end_time, start_time, source_type_name, valid_at, external_events } = - body; + const { source, external_events } = body; + const { attributes, derivation_group_name, key, end_time, start_time, source_type_name, valid_at } = source; const parsedAttributes = JSON.parse(attributes); - const parsedExternalEvents = JSON.parse(external_events); + const parsedExternalEvents: ExternalEvent[] = JSON.parse(external_events); // Re-package the fields as a JSON object to be validated by the meta-schema const externalSourceJson = { @@ -142,75 +147,47 @@ async function uploadExternalSource(req: Request, res: Response) { logger.info(`POST /uploadExternalSource: Uploading External Source: ${key}`); - // Verify External Source input against meta-schema - let sourceIsValid: boolean = false; - sourceIsValid = await compiledExternalSourceSchema(externalSourceJson); - if (sourceIsValid) { - logger.info(`POST /uploadExternalSource: External Source ${key}'s formatting is valid`); - } else { - logger.error(`POST /uploadExternalSource: External Source ${key}'s formatting is invalid`); - res.status(500).send({ message: `External Source ${key}'s formatting is invalid` }); - return; - } - - // Get the attribute schema for the source's external source type - const sourceAttributeSchema = await fetch(GQL_API_URL, { + // Get the attribute schema for the source's external source type and all contained event types + let eventTypeNames = parsedExternalEvents.map(e => e.event_type_name); + eventTypeNames = eventTypeNames.filter((e, i) => eventTypeNames.indexOf(e) === i); + const attributeSchemas = await fetch(GQL_API_URL, { body: JSON.stringify({ - query: gql.GET_EXTERNAL_SOURCE_TYPE_ATTRIBUTE_SCHEMA, + query: gql.GET_SOURCE_EVENT_TYPE_ATTRIBUTE_SCHEMAS, variables: { - name: source_type_name, + externalEventTypes: eventTypeNames, + externalSourceType: source_type_name }, }), headers, method: 'POST', }); - // Validate the attributes on the External Source - let sourceAttributesAreValid: boolean = false; - let sourceSchema: Ajv.ValidateFunction | undefined = undefined; - const sourceTypeResponseJSON = await sourceAttributeSchema.json(); - const getExternalSourceTypeAttributeSchemaResponse = sourceTypeResponseJSON.data as - | GetExternalSourceTypeAttributeSchemaResponse - | HasuraError; - - if ( - (getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse) - .external_source_type_by_pk?.attribute_schema !== null - ) { - const { external_source_type_by_pk: sourceAttributeSchema } = - getExternalSourceTypeAttributeSchemaResponse as GetExternalSourceTypeAttributeSchemaResponse; - if (sourceAttributeSchema !== undefined && sourceAttributeSchema !== null) { - sourceSchema = ajv.compile(sourceAttributeSchema.attribute_schema); - sourceAttributesAreValid = await sourceSchema(parsedAttributes); - } else { - logger.error(`POST /uploadExternalSource: External Source Type ${source_type_name} does not exist!`); - res.status(500).send({ message: `External Source Type ${source_type_name} does not exist!` }); - return; - } - } - if (sourceAttributesAreValid) { - logger.info(`POST /uploadExternalSource: External Source ${key}'s attributes are valid`); + const attributeSchemaJson = await attributeSchemas.json(); + + // TODO: assemble megaschema from attribute schemas + console.log(attributeSchemaJson); + + return; + + + // Verify that this is a valid external source + let sourceIsValid: boolean = false; + sourceIsValid = await compiledExternalSourceSchema(body); + if (sourceIsValid) { + logger.info(`POST /uploadExternalSource: External Source ${key}'s formatting is valid`); } else { - logger.error(`POST /uploadExternalSource: External Source ${key}'s attributes are invalid`); - res.status(500); - if (sourceSchema !== undefined) { - res.send({ message: `External Source ${key}'s attributes are invalid:\n${JSON.stringify(sourceSchema.errors)}` }); - } else { - res.send({ message: `External Source ${key}'s attributes are invalid` }); - } + logger.error(`POST /uploadExternalSource: External Source ${key}'s formatting is invalid:\n${JSON.stringify(compiledExternalSourceSchema.errors)}`); + res.status(500).send({ message: `External Source ${key}'s formatting is invalid:\n${JSON.stringify(compiledExternalSourceSchema.errors)}` }); return; } - // Build a Record for all event type's being used - const usedExternalEventTypes = parsedExternalEvents.reduce( - (acc: string[], externalEvent: ExternalEventInsertInput) => { - if (!acc.includes(externalEvent.event_type_name)) { - acc.push(externalEvent.event_type_name); - } - return acc; - }, - [], - ); + const usedExternalEventTypes = parsedExternalEvents.reduce((acc: string[], externalEvent: ExternalEventInsertInput) => { + if (!acc.includes(externalEvent.event_type_name)) { + acc.push(externalEvent.event_type_name); + } + return acc; + }, []); + const usedExternalEventTypesAttributesSchemas: Record = {}; for (const eventType of usedExternalEventTypes) { const eventAttributeSchema = await fetch(GQL_API_URL, { @@ -247,8 +224,7 @@ async function uploadExternalSource(req: Request, res: Response) { const eventAttributesAreValid = await currentEventSchema(externalEvent.attributes); if (!eventAttributesAreValid) { throw new Error( - `External Event '${ - externalEvent.key + `External Event '${externalEvent.key }' does not have a valid set of attributes, per it's type's schema:\n${JSON.stringify( currentEventSchema.errors, )}`, @@ -261,6 +237,7 @@ async function uploadExternalSource(req: Request, res: Response) { } } + // Run the Hasura migration for creating an external source const derivationGroupInsert: DerivationGroupInsertInput = { name: derivation_group_name, source_type_name: source_type_name, @@ -300,7 +277,7 @@ async function uploadExternalSource(req: Request, res: Response) { export default (app: Express) => { /** * @swagger - * /uploadExternalSourceType: + * /uploadExternalSourceEventTypes: * post: * security: * - bearerAuth: [] @@ -349,11 +326,11 @@ export default (app: Express) => { * - Hasura */ app.post( - '/uploadExternalSourceType', + '/uploadExternalSourceEventTypes', upload.single('attribute_schema'), refreshLimiter, auth, - uploadExternalSourceType, + uploadExternalSourceEventTypes, ); /** diff --git a/src/packages/external-source/gql.ts b/src/packages/external-source/gql.ts index 78fae46..7b030c2 100644 --- a/src/packages/external-source/gql.ts +++ b/src/packages/external-source/gql.ts @@ -27,11 +27,17 @@ export default { } } `, - CREATE_EXTERNAL_SOURCE_TYPE: `#graphql - mutation CreateExternalSourceType($sourceType: external_source_type_insert_input!) { - createExternalSourceType: insert_external_source_type_one(object: $sourceType) { - name - attribute_schema + CREATE_EXTERNAL_SOURCE_EVENT_TYPES: `#graphql + mutation uploadAttributeSchemas($externalEventTypes: [external_event_type_insert_input!]!, $externalSourceTypes: [external_source_type_insert_input!]!) { + createExternalEventTypes: insert_external_event_type(objects: $externalEventTypes) { + returning { + name + } + } + createExternalSourceTypes: insert_external_source_type(objects: $externalSourceTypes) { + returning { + name + } } } `, @@ -56,4 +62,16 @@ export default { } } `, + GET_SOURCE_EVENT_TYPE_ATTRIBUTE_SCHEMAS: `#graphql + query getET($externalEventTypes: [String!]!, $externalSourceType: String!) { + external_event_type(where: {name: {_in: $externalEventTypes}}) { + name + attribute_schema + } + external_source_type(where: {name: {_eq: $externalSourceType}}) { + name + attribute_schema + } + } + ` }; diff --git a/src/packages/schemas/external-event-validation-schemata.ts b/src/packages/schemas/external-event-validation-schemata.ts index 7d870a7..ffc281f 100644 --- a/src/packages/schemas/external-event-validation-schemata.ts +++ b/src/packages/schemas/external-event-validation-schemata.ts @@ -1,142 +1,107 @@ -// TODO: Discuss external event/source type schemas. Do we want to validate these with a (meta)schema too? Or just allow any plain JSON Schema? -// Currently, we do the latter but the former doesn't seem like a bad idea! -// The main argument against the former is what we have works and introducing new schemas could be a rabbit hole. - -// export const externalEventTypeSchema = { -// additionalProperties: false, -// properties: { -// entries: { -// items: { -// additionalProperties: false, -// properties: { -// metadata: { -// items: { -// additionalProperties: false, -// properties: { -// isRequired: { type: 'boolean' }, -// name: { type: 'string' }, -// schema: { -// additionalProperties: false, -// properties: { type: { type: 'string' } }, -// required: ['type'], -// type: 'object', -// }, -// }, -// required: ['name', 'isRequired', 'schema'], -// type: 'object', -// }, -// type: 'array', -// }, -// name: { type: 'string' }, -// }, -// required: ['name', 'metadata'], -// type: 'object', -// }, -// type: 'array', -// }, -// }, -// required: ['entries'], -// type: 'object', -// }; -// export const externalSourceTypeSchema = { -// additionalProperties: false, -// properties: { -// entries: { -// items: { -// additionalProperties: false, -// properties: { -// metadata: { -// items: { -// additionalProperties: false, -// properties: { -// isRequired: { type: 'boolean' }, -// name: { type: 'string' }, -// schema: { -// additionalProperties: false, -// properties: { type: { type: 'string' } }, -// required: ['type'], -// type: 'object', -// }, -// }, -// required: ['name', 'isRequired', 'schema'], -// type: 'object', -// }, -// type: 'array', -// }, -// name: { type: 'string' }, -// }, -// required: ['name', 'metadata'], -// type: 'object', -// }, -// type: 'array', -// }, -// }, -// required: ['entries'], -// type: 'object', -// }; - -export const externalSourceSchema = { - additionalProperties: false, - properties: { - external_events: { - items: { - additionalProperties: false, - properties: { - attributes: { - additionalProperties: true, - properties: {}, - required: [], - type: 'object', - }, - duration: { type: 'string' }, - event_type_name: { type: 'string' }, - key: { type: 'string' }, - start_time: { type: 'string' }, - }, - required: ['duration', 'event_type_name', 'key', 'attributes', 'start_time'], - type: 'object', - }, - type: 'array', +// a schema that describes the format for the attribute files (which are, themselves, JSON Schema-like) +export const attributeSchemaMetaschema = { + "$defs": { + "AttributeSchema": { + "additionalProperties": false, + "patternProperties": { + "^.*$": { + "properties": { + "properties": { + "additionalProperties": true, + "type": "object" + }, + "required": { + "items": { "type": "string" }, + "type": "array" + }, + "type": { "type": "string" } + }, + "required": ["required", "properties", "type"], + "type": "object" + } + }, + "type": "object" + } }, - source: { - additionalProperties: false, - properties: { - attributes: { - additionalProperties: true, - properties: {}, // constrained by type, checked by DB trigger on upload. TODO: CHECK LOCALLY? - required: [], - type: 'object', + "$schema": "http://json-schema.org/draft-07/schema", + "additionalProperties": false, + "description": "Schema for the attributes of uploaded source types and/or event types.", + "properties": { + "event_types": { + "$ref": "#/$defs/AttributeSchema" }, - derivation_group_name: { type: 'string' }, - key: { type: 'string' }, - period: { - additionalProperties: false, - properties: { - end_time: { - pattern: - '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', - type: 'string', - }, - start_time: { - pattern: - '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', - type: 'string', + "source_types": { + "$ref": "#/$defs/AttributeSchema" + } + }, + "required": ["source_types", "event_types"], + "title": "TypeSpecificationSchema", + "type": "object" +} + +// the schema that schemas for specific types are integrated with, after pulling them from the database +export const externalSourceSchema = { + additionalProperties: false, + properties: { + external_events: { + items: { + additionalProperties: false, + properties: { + attributes: { + additionalProperties: true, + properties: {}, + required: [], + type: 'object', + }, + duration: { type: 'string' }, + event_type_name: { type: 'string' }, + key: { type: 'string' }, + start_time: { type: 'string' }, + }, + required: ['duration', 'event_type_name', 'key', 'attributes', 'start_time'], + type: 'object', }, - }, - required: ['start_time', 'end_time'], - type: 'object', + type: 'array', }, - source_type_name: { type: 'string' }, - valid_at: { - pattern: - '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', - type: 'string', + source: { + additionalProperties: false, + properties: { + attributes: { + additionalProperties: true, + properties: {}, // constrained by type, checked by DB trigger on upload. TODO: CHECK LOCALLY? + required: [], + type: 'object', + }, + derivation_group_name: { type: 'string' }, + key: { type: 'string' }, + period: { + additionalProperties: false, + properties: { + end_time: { + pattern: + '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', + type: 'string', + }, + start_time: { + pattern: + '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', + type: 'string', + }, + }, + required: ['start_time', 'end_time'], + type: 'object', + }, + source_type_name: { type: 'string' }, + valid_at: { + pattern: + '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', + type: 'string', + }, + }, + required: ['key', 'source_type_name', 'valid_at', 'period', 'attributes'], + type: 'object', }, - }, - required: ['key', 'source_type_name', 'valid_at', 'period', 'attributes'], - type: 'object', }, - }, - required: ['source', 'external_events'], - type: 'object', + required: ['source', 'external_events'], + type: 'object', }; diff --git a/src/types/external-event.ts b/src/types/external-event.ts deleted file mode 100644 index d138a5b..0000000 --- a/src/types/external-event.ts +++ /dev/null @@ -1,36 +0,0 @@ -export type ExternalEventInsertInput = { - attributes: object; - start_time: string; - duration: string; - event_type_name: string; - key: string; -}; - -export type ExternalEventJson = { - attributes: object; - duration: string; - event_type: string; - key: string; - start_time: string; -}; - -export type ExternalEventTypeInsertInput = { - name: string; - attribute_schema: object; -}; - -export type ExternalEvent = { - key: string; - event_type_name: string; - start_time: string; - duration: string; - attributes: object; -}; - -export type CreateExternalEventTypeResponse = { - createExternalEventType: { attribute_schema: object; name: string }; -}; - -export type UploadAttributeJSON = { - [x: string]: any; -}; diff --git a/src/types/external-source.ts b/src/types/external-source.ts index 991639e..5e68c1c 100644 --- a/src/types/external-source.ts +++ b/src/types/external-source.ts @@ -1,5 +1,3 @@ -import { ExternalEventJson } from './external-event'; - export type DerivationGroupInsertInput = { name: string; source_type_name: string; @@ -8,7 +6,13 @@ export type DerivationGroupInsertInput = { export type ExternalSourceTypeInsertInput = { name: string; attribute_schema: object; -}; +}[]; + + +export type ExternalEventTypeInsertInput = { + name: string; + attribute_schema: object; +}[]; export type ExternalSourceInsertInput = { attributes: object; @@ -57,3 +61,36 @@ export type GetExternalSourceTypeAttributeSchemaResponse = { export type GetExternalEventTypeAttributeSchemaResponse = { external_event_type_by_pk: { attribute_schema: object }; }; + + +export type ExternalEventInsertInput = { + attributes: object; + start_time: string; + duration: string; + event_type_name: string; + key: string; +}; + +export type ExternalEventJson = { + attributes: object; + duration: string; + event_type: string; + key: string; + start_time: string; +}; + +export type ExternalEvent = { + key: string; + event_type_name: string; + start_time: string; + duration: string; + attributes: object; +}; + +export type CreateExternalEventTypeResponse = { + createExternalEventType: { attribute_schema: object; name: string }; +}; + +export type UploadAttributeJSON = { + [x: string]: any; +}; From cf1dfe90ff6fcb232b0cc965b5acc2b6dd946bd0 Mon Sep 17 00:00:00 2001 From: psubram3 Date: Fri, 22 Nov 2024 10:48:41 -0800 Subject: [PATCH 21/39] attempt merge ii --- .../external-source/external-source.ts | 223 ++++++++++-------- .../external-event-validation-schemata.ts | 127 +++++----- 2 files changed, 187 insertions(+), 163 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 839331c..a165040 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -1,30 +1,22 @@ import type { Express, Request, Response } from 'express'; import type { DerivationGroupInsertInput, - ExternalSourceInsertInput, ExternalSourceTypeInsertInput, CreateExternalSourceResponse, CreateExternalSourceTypeResponse, - GetExternalSourceTypeAttributeSchemaResponse, - GetExternalEventTypeAttributeSchemaResponse, - UploadExternalSourceJSON, - UploadAttributeJSON, ExternalEventTypeInsertInput, - ExternalEventInsertInput, - ExternalEventJson, ExternalEvent, + ExternalSourceInsertInput, } from '../../types/external-source.js'; import Ajv from 'ajv'; import { getEnv } from '../../env.js'; import getLogger from '../../logger.js'; import gql from './gql.js'; -import { attributeSchemaMetaschema, externalSourceSchema } from '../schemas/external-event-validation-schemata.js'; +import { attributeSchemaMetaschema, baseExternalSourceSchema } from '../schemas/external-event-validation-schemata.js'; import { HasuraError } from '../../types/hasura.js'; import { auth } from '../auth/middleware.js'; import rateLimit from 'express-rate-limit'; import multer from 'multer'; -import { parseJSONFile } from '../../util/fileParser.js'; -import { convertDoyToYmd } from '../../util/time.js'; const upload = multer({ limits: { fieldSize: 25 * 1024 * 1024 } }); const logger = getLogger('packages/external-source/external-source'); @@ -32,7 +24,6 @@ const { RATE_LIMITER_LOGIN_MAX, HASURA_API_URL } = getEnv(); const GQL_API_URL = `${HASURA_API_URL}/v1/graphql`; const ajv = new Ajv(); const compiledAttributeMetaschema = ajv.compile(attributeSchemaMetaschema); -const compiledExternalSourceSchema = ajv.compile(externalSourceSchema); const refreshLimiter = rateLimit({ legacyHeaders: false, max: RATE_LIMITER_LOGIN_MAX, @@ -40,6 +31,93 @@ const refreshLimiter = rateLimit({ windowMs: 15 * 60 * 1000, // 15 minutes }); +function updateSchemaWithDefs(defs: { event_types: any, source_type: any }) {//: Ajv.ValidateFunction | undefined { + // build if statement + const ifThenElse: { [key: string]: any } = { + + }; + let ifThenElsePointer = ifThenElse; + const keys = Object.keys(defs.event_types); + + // handling if there's only 1 event type + if (keys.length === 1) { + // no need for ifThenElse, simply create localSchemaCopy and update properties.events.items.properties.attributes + // to match the event type in defs, and verify the event_type_name matches the def name + const localSchemaCopy = structuredClone(baseExternalSourceSchema); + const event_type_name = keys[0]; + const event_type_schema = defs.event_types[event_type_name]; + const source_type_name = Object.keys(defs.source_type)[0]; + const source_type_schema = defs.source_type[source_type_name]; + + localSchemaCopy.properties.events.items.properties.attributes = event_type_schema; + localSchemaCopy.properties.events.items.properties.event_type_name = { "const": event_type_name }; + + // insert def for "source" attributes + localSchemaCopy.properties.source.properties.attributes = source_type_schema; + + const localAjv = new Ajv(); + return localAjv.addSchema(defs).compile(localSchemaCopy); + } + + // handle n event types + for (let i = 0; i < keys.length - 1; i++) { + const key = keys[i]; + console.log("NOW ON:", key); + ifThenElsePointer["if"] = { + properties: { + event_type_name: { + const: key + } + } + }; + ifThenElsePointer["then"] = { + properties: { + attributes: { + $ref: `#/$defs/event_types/${key}` + } + } + }; + ifThenElsePointer["else"] = { + + }; + ifThenElsePointer = ifThenElsePointer["else"]; + } + + // fill in the final else with the last element + const key = keys[keys.length - 1]; + ifThenElsePointer["properties"] = { + attributes: { + $ref: `#/$defs/event_types/${key}` + } + } + + // insert if statement into local copy of baseExternalSourceSchema + const localSchemaCopy = structuredClone(baseExternalSourceSchema); + localSchemaCopy.properties.events.items["if"] = ifThenElse["if"]; + localSchemaCopy.properties.events.items["then"] = ifThenElse["then"]; + localSchemaCopy.properties.events.items["else"] = ifThenElse["else"]; + + // insert def for "source" attributes + const sourceTypeKey = Object.keys(defs.source_type)[0]; + localSchemaCopy.properties.source.properties.attributes = { $ref: `#/$defs/source_type/${sourceTypeKey}`} + + + // add defs + localSchemaCopy.$defs = { + event_types: {}, + source_type: { + [sourceTypeKey]: defs.source_type[sourceTypeKey] + } + } + for (const event_type of keys) { + localSchemaCopy.$defs.event_types[event_type] = defs.event_types[event_type]; + } + + // compile with defs, return + const localAjv = new Ajv(); + return localAjv.compile(localSchemaCopy); +} + async function uploadExternalSourceEventTypes(req: Request, res: Response) { const authorizationHeader = req.get('authorization'); @@ -47,7 +125,7 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, } = req; - const { file } = req; + const { body } = req; logger.info(`POST /uploadExternalSourceEventTypes: Uploading External Source and Event Types...`); const headers: HeadersInit = { @@ -58,10 +136,8 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { 'x-hasura-user-id': userHeader ? `${userHeader}` : '', }; - const uploadedExternalSourceEventTypeAttributeSchema = await parseJSONFile(file); - // Validate uploaded attribute schemas are formatted validly - const schemasAreValid: boolean = await compiledAttributeMetaschema(uploadedExternalSourceEventTypeAttributeSchema); + const schemasAreValid: boolean = await compiledAttributeMetaschema(body); if (!schemasAreValid) { logger.error( `POST /uploadExternalSourceEventTypes: Schema validation failed for uploaded source and event types.`, @@ -77,19 +153,19 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { const externalSourceTypeInput: ExternalSourceTypeInsertInput = []; const externalEventTypeInput: ExternalEventTypeInsertInput = []; - const external_event_types = uploadedExternalSourceEventTypeAttributeSchema.event_types; + const external_event_types = body.event_types; const event_type_keys = Object.keys(external_event_types); for (const external_event_type of event_type_keys) { - externalSourceTypeInput.push({ + externalEventTypeInput.push({ attribute_schema: external_event_types[external_event_type], name: external_event_type }) } - const external_source_types = uploadedExternalSourceEventTypeAttributeSchema.source_types; + const external_source_types = body.source_types; const source_type_keys = Object.keys(external_source_types); for (const external_source_type of source_type_keys) { - externalEventTypeInput.push({ + externalSourceTypeInput.push({ attribute_schema: external_source_types[external_source_type], name: external_source_type }) @@ -117,22 +193,20 @@ async function uploadExternalSource(req: Request, res: Response) { headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, } = req; const { body } = req; - const { source, external_events } = body; - const { attributes, derivation_group_name, key, end_time, start_time, source_type_name, valid_at } = source; - const parsedAttributes = JSON.parse(attributes); - const parsedExternalEvents: ExternalEvent[] = JSON.parse(external_events); - // Re-package the fields as a JSON object to be validated by the meta-schema + const { source, events } = body; + const parsedSource = JSON.parse(source); + const parsedExternalEvents: ExternalEvent[] = JSON.parse(events); + const { attributes, derivation_group_name, key, period, source_type_name, valid_at } = parsedSource; + + // re-package the fields as a JSON object to be posted const externalSourceJson = { - external_events: parsedExternalEvents, + events: parsedExternalEvents, source: { - attributes: parsedAttributes, + attributes: attributes, derivation_group_name: derivation_group_name, key: key, - period: { - end_time: end_time, - start_time: start_time, - }, + period: period, source_type_name: source_type_name, valid_at: valid_at, }, @@ -163,80 +237,35 @@ async function uploadExternalSource(req: Request, res: Response) { }); const attributeSchemaJson = await attributeSchemas.json(); + const { external_event_type, external_source_type } = attributeSchemaJson.data; + + const defs: { event_types: any, source_type: any } = { + event_types: { - // TODO: assemble megaschema from attribute schemas - console.log(attributeSchemaJson); + }, + source_type: { + [external_source_type[0].name]: external_source_type[0].attribute_schema + } + }; - return; + for (const event_type of external_event_type) { + defs.event_types[event_type.name] = event_type.attribute_schema + } + // Assemble megaschema from attribute schemas + const compiledExternalSourceMegaschema: Ajv.ValidateFunction = updateSchemaWithDefs(defs); // Verify that this is a valid external source let sourceIsValid: boolean = false; - sourceIsValid = await compiledExternalSourceSchema(body); + sourceIsValid = await compiledExternalSourceMegaschema(externalSourceJson); if (sourceIsValid) { logger.info(`POST /uploadExternalSource: External Source ${key}'s formatting is valid`); } else { - logger.error(`POST /uploadExternalSource: External Source ${key}'s formatting is invalid:\n${JSON.stringify(compiledExternalSourceSchema.errors)}`); - res.status(500).send({ message: `External Source ${key}'s formatting is invalid:\n${JSON.stringify(compiledExternalSourceSchema.errors)}` }); + logger.error(`POST /uploadExternalSource: External Source ${key}'s formatting is invalid:\n${JSON.stringify(compiledExternalSourceMegaschema.errors)}`); + res.status(500).send({ message: `External Source ${key}'s formatting is invalid:\n${JSON.stringify(compiledExternalSourceMegaschema.errors)}` }); return; } - const usedExternalEventTypes = parsedExternalEvents.reduce((acc: string[], externalEvent: ExternalEventInsertInput) => { - if (!acc.includes(externalEvent.event_type_name)) { - acc.push(externalEvent.event_type_name); - } - return acc; - }, []); - - const usedExternalEventTypesAttributesSchemas: Record = {}; - for (const eventType of usedExternalEventTypes) { - const eventAttributeSchema = await fetch(GQL_API_URL, { - body: JSON.stringify({ - query: gql.GET_EXTERNAL_EVENT_TYPE_ATTRIBUTE_SCHEMA, - variables: { - name: eventType, - }, - }), - headers, - method: 'POST', - }); - const eventTypeJSONResponse = await eventAttributeSchema.json(); - const getExternalEventTypeAttributeSchemaResponse = eventTypeJSONResponse.data as - | GetExternalEventTypeAttributeSchemaResponse - | HasuraError; - if ( - (getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse) - .external_event_type_by_pk?.attribute_schema !== null - ) { - const { external_event_type_by_pk: eventAttributeSchema } = - getExternalEventTypeAttributeSchemaResponse as GetExternalEventTypeAttributeSchemaResponse; - if (eventAttributeSchema !== undefined && eventAttributeSchema !== null) { - usedExternalEventTypesAttributesSchemas[eventType] = ajv.compile(eventAttributeSchema.attribute_schema); - } - } - } - - // Validate attributes of all External Events in the source - for (const externalEvent of parsedExternalEvents) { - try { - const currentEventType = externalEvent.event_type_name; - const currentEventSchema: Ajv.ValidateFunction = usedExternalEventTypesAttributesSchemas[currentEventType]; - const eventAttributesAreValid = await currentEventSchema(externalEvent.attributes); - if (!eventAttributesAreValid) { - throw new Error( - `External Event '${externalEvent.key - }' does not have a valid set of attributes, per it's type's schema:\n${JSON.stringify( - currentEventSchema.errors, - )}`, - ); - } - } catch (error) { - logger.error(`POST /uploadExternalSource: External Event ${externalEvent.key}'s attributes are invalid`); - res.status(500).send({ message: (error as Error).message }); - return; - } - } - // Run the Hasura migration for creating an external source const derivationGroupInsert: DerivationGroupInsertInput = { name: derivation_group_name, @@ -244,15 +273,15 @@ async function uploadExternalSource(req: Request, res: Response) { }; const externalSourceInsert: ExternalSourceInsertInput = { - attributes: parsedAttributes, + attributes: attributes, derivation_group_name: derivation_group_name, - end_time: end_time, + end_time: period.end_time, external_events: { data: parsedExternalEvents, }, key: key, source_type_name: source_type_name, - start_time: start_time, + start_time: period.start_time, valid_at: valid_at, }; @@ -361,7 +390,7 @@ export default (app: Express) => { * type: string * end_time: * type: string - * external_events: + * events: * type: object * properties: * data: @@ -380,7 +409,7 @@ export default (app: Express) => { * - attributes * derivation_group_name * end_time - * external_events + * events * key * source_type_name * start_time diff --git a/src/packages/schemas/external-event-validation-schemata.ts b/src/packages/schemas/external-event-validation-schemata.ts index ffc281f..15fedc7 100644 --- a/src/packages/schemas/external-event-validation-schemata.ts +++ b/src/packages/schemas/external-event-validation-schemata.ts @@ -6,7 +6,7 @@ export const attributeSchemaMetaschema = { "patternProperties": { "^.*$": { "properties": { - "properties": { + "attributes": { "additionalProperties": true, "type": "object" }, @@ -40,68 +40,63 @@ export const attributeSchemaMetaschema = { } // the schema that schemas for specific types are integrated with, after pulling them from the database -export const externalSourceSchema = { - additionalProperties: false, - properties: { - external_events: { - items: { - additionalProperties: false, - properties: { - attributes: { - additionalProperties: true, - properties: {}, - required: [], - type: 'object', - }, - duration: { type: 'string' }, - event_type_name: { type: 'string' }, - key: { type: 'string' }, - start_time: { type: 'string' }, - }, - required: ['duration', 'event_type_name', 'key', 'attributes', 'start_time'], - type: 'object', - }, - type: 'array', - }, - source: { - additionalProperties: false, - properties: { - attributes: { - additionalProperties: true, - properties: {}, // constrained by type, checked by DB trigger on upload. TODO: CHECK LOCALLY? - required: [], - type: 'object', - }, - derivation_group_name: { type: 'string' }, - key: { type: 'string' }, - period: { - additionalProperties: false, - properties: { - end_time: { - pattern: - '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', - type: 'string', - }, - start_time: { - pattern: - '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', - type: 'string', - }, - }, - required: ['start_time', 'end_time'], - type: 'object', - }, - source_type_name: { type: 'string' }, - valid_at: { - pattern: - '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', - type: 'string', - }, - }, - required: ['key', 'source_type_name', 'valid_at', 'period', 'attributes'], - type: 'object', - }, - }, - required: ['source', 'external_events'], - type: 'object', -}; +export const baseExternalSourceSchema: { [key: string]: any } = { + $id: "source_schema", + $schema: "http://json-schema.org/draft-07/schema", + additionalProperties: false, + description: "The base schema for external sources. Defs and ifs, for specific source/event type attributes, are integrated later.", + properties: { + events: { + items: { + additionalProperties: false, + properties: { + attributes: { + type: "object" + }, + duration: { "type": "string" }, + event_type_name: { "type": "string" }, + key: { "type": "string" }, + start_time: { "type": "string" } + }, + required: ["duration", "event_type_name", "key", "attributes", "start_time"], + type: "object" + }, + type: "array" + }, + source: { + additionalProperties: false, + properties: { + attributes: { + type: "object" // WILL BE REPLACED WITH A $ref + }, + derivation_group_name: { "type": "string" }, + key: { "type": "string" }, + period: { + additionalProperties: false, + properties: { + end_time: { + pattern: "^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$", + type: "string" + }, + start_time: { + pattern: "^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$", + type: "string" + } + }, + required: ["start_time", "end_time"], + type: "object" + }, + source_type_name: { "type": "string" }, + valid_at: { + pattern: "^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$", + type: "string" + } + }, + required: ["key", "source_type_name", "valid_at", "period", "attributes"], + type: "object" + } + }, + required: ["source", "events"], + title: "SourceTypeA", + type: "object" +}; \ No newline at end of file From 06b04153e5bfe350ea6997dc6a9ea26163059f4c Mon Sep 17 00:00:00 2001 From: psubram3 Date: Fri, 22 Nov 2024 13:12:38 -0800 Subject: [PATCH 22/39] update gateway tests --- .../external-source/external-source.ts | 2 +- test/external_source.validation.test.ts | 509 +++++++++++++----- 2 files changed, 366 insertions(+), 145 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index a165040..6c12715 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -31,7 +31,7 @@ const refreshLimiter = rateLimit({ windowMs: 15 * 60 * 1000, // 15 minutes }); -function updateSchemaWithDefs(defs: { event_types: any, source_type: any }) {//: Ajv.ValidateFunction | undefined { +export function updateSchemaWithDefs(defs: { event_types: any, source_type: any }) {//: Ajv.ValidateFunction | undefined { // build if statement const ifThenElse: { [key: string]: any } = { diff --git a/test/external_source.validation.test.ts b/test/external_source.validation.test.ts index 6c35876..e42c358 100644 --- a/test/external_source.validation.test.ts +++ b/test/external_source.validation.test.ts @@ -1,186 +1,407 @@ import Ajv from 'ajv'; import { describe, expect, test } from 'vitest'; -import { externalSourceSchema } from '../src/packages/schemas/external-event-validation-schemata'; +import { attributeSchemaMetaschema } from '../src/packages/schemas/external-event-validation-schemata'; +import { updateSchemaWithDefs } from '../src/packages/external-source/external-source'; const ajv = Ajv(); -// type schemas -const correctExternalEventTypeSchema = { - $schema: 'http://json-schema.org/draft-07/schema', - additionalProperties: false, - description: 'Schema for the attributes of the TestEventType Type.', - properties: { - code: { type: 'string' }, - projectUser: { type: 'string' }, +const attributeDefs = { + "event_types": { + "EventTypeA": { + "properties": { + "series": { + "properties": { + "iteration": { "type": "number" }, + "make": { "type": "string" }, + "type": { "type": "string" }, + }, + "required": ["type", "make", "iteration"], + "type": "object", + } + }, + "required": ["series"], + "type": "object", + }, + "EventTypeB": { + "properties": { + "projectUser": { + "type": "string" + }, + "tick": { + "type": "number" + } + }, + "required": ["projectUser", "tick"], + "type": "object" + }, + "EventTypeC": { + "properties": { + "aperture": { + "type": "string" + }, + "subduration": { + "pattern": "^P(?:\\d+Y)?(?:\\d+M)?(?:\\d+D)?T(?:\\d+H)?(?:\\d+M)?(?:\\d+S)?$", + "type": "string" + } + }, + "required": ["aperture", "subduration"], + "type": "object" + } }, - required: ['projectUser', 'code'], - title: 'TestEventType', - type: 'object', + "source_types": { + "SourceTypeA": { + "properties": { + "version": { + "type": "number" + }, + "wrkcat": { + "type": "string" + } + }, + "required": ["version", "wrkcat"], + "type": "object" + }, + "SourceTypeB": { + "properties": { + "version": { + "type": "number" + }, + "wrkcat": { + "type": "string" + } + }, + "required": ["version", "wrkcat"], + "type": "object" + } + } }; -const incorrectPassingExternalEventTypeSchema = { - $schema: 'http://json-schema.org/draft-07/schema', - additionalProperties: false, - descriptionFake: 'Schema for the attributes of the TestEventType Type.', - doesntEvenExist: true, - propertgibberish: { - // if you have something like this, it just registers as no properties existing, and fails any inserted events with attributes. - code: { type: 'string' }, - projectUser: { type: 'string' }, +const incorrectAttributeDefs = { + "event_types": { + "EventTypeA": { + "properties": { + "series": { + "properties": { + "iteration": { "type": "number" }, + "make": { "type": "string" }, + "type": { "type": "string" }, + }, + // "required": ["type", "make", "iteration"], // missing required field (not an issue) + "type": "object", + } + }, + // "required": ["series"], // missing required field (the issue, only at level patternProperties/sdfdsf/required) + "type": "object", + }, + "EventTypeB": { + "properties": { + "projectUser": { + "type": "string" + }, + "tick": { + "type": "number" + } + }, + "required": ["projectUser", "tick"], + "type": "object" + }, + "EventTypeC": { + "properties": { + "aperture": { + "type": "string" + }, + "subduration": { + "pattern": "^P(?:\\d+Y)?(?:\\d+M)?(?:\\d+D)?T(?:\\d+H)?(?:\\d+M)?(?:\\d+S)?$", + "type": "string" + } + }, + "required": ["aperture", "subduration"], + "type": "object" + } }, - requiredgibberish: ['projectUser', 'code'], - title: 'TestEventType', - type: 'object', + "source_types": { + "SourceTypeA": { + "properties": { + "version": { + "type": "number" + }, + "wrkcat": { + "type": "string" + } + }, + "required": ["version", "wrkcat"], + "type": "object" + }, + "SourceTypeB": { + "properties": { + "version": { + "type": "number" + }, + "wrkcat": { + "type": "string" + } + }, + "required": ["version", "wrkcat"], + "type": "object" + } + } }; -const incorrectFailingExternalEventTypeSchema = { - $schema: 'http://json-schema.org/draft-07/schema', - additionalProperties: false, - description: 'Schema for the attributes of the TestEventType Type.', - properties: { - code: { type: 'string' }, - projectUser: { type: 'string' }, - }, - required: 123, // this fails to validate at all since "required" IS well-defined as a field but expects an array - title: 'TestEventType', - type: 'object', +const correctExternalSource = { + "events": [ + { + "attributes": { + "series": { + "iteration": 17, + "make": "alpha", + "type": "A", + } + }, + "duration": "02:00:00", + "event_type_name": "EventTypeA", + "key": "EventTypeA:1/1", + "start_time": "2024-01-01T01:35:00+00:00", + }, + { + "attributes": { + "series": { + "iteration": 21, + "make": "beta", + "type": "B" + } + }, + "duration": "02:00:00", + "event_type_name": "EventTypeA", + "key": "EventTypeA:1/2", + "start_time": "2024-01-02T11:50:00+00:00", + }, + { + "attributes": { + "projectUser": "Jerry", + "tick": 18 + }, + "duration": "03:40:00", + "event_type_name": "EventTypeB", + "key": "EventTypeB:1/3", + "start_time": "2024-01-03T15:20:00+00:00" + } + ], + "source": { + "attributes": { + "version": 1, + "wrkcat": "234" + }, + "key": "SourceTypeA:valid_source_A.json", + "period": { + "end_time": "2024-01-07T00:00:00+00:00", + "start_time": "2024-01-01T00:00:00+00:00" + }, + "source_type_name": "SourceTypeA", + "valid_at": "2024-01-01T00:00:00+00:00", + } }; -const externalSourceTypeSchema = { - $schema: 'http://json-schema.org/draft-07/schema', - additionalProperties: false, - description: 'Schema for the attributes of the TestSourceType Type.', - properties: { - operator: { type: 'string' }, - version: { type: 'number' }, - }, - required: ['version', 'operator'], - title: 'TestSourceType', - type: 'object', +const incorrectExternalSourceAttributes = { + "events": [ + { + "attributes": { + "series": { + "iteration": 17, + "make": "alpha", + "type": "A", + } + }, + "duration": "02:00:00", + "event_type_name": "EventTypeA", + "key": "EventTypeA:1/1", + "start_time": "2024-01-01T01:35:00+00:00", + }, + { + "attributes": { + "series": { + "iteration": 21, + "make": "beta", + "type": "B" + } + }, + "duration": "02:00:00", + "event_type_name": "EventTypeA", + "key": "EventTypeA:1/2", + "start_time": "2024-01-02T11:50:00+00:00", + }, + { + "attributes": { + "projectUser": "Jerry", + "tick": 18 + }, + "duration": "03:40:00", + "event_type_name": "EventTypeB", + "key": "EventTypeB:1/3", + "start_time": "2024-01-03T15:20:00+00:00" + } + ], + "source": { + "attributes": { + "version": 1, + "wrkcat": 234 // <-- wrong type. expecting string. + }, + "key": "SourceTypeA:valid_source_A.json", + "period": { + "end_time": "2024-01-07T00:00:00+00:00", + "start_time": "2024-01-01T00:00:00+00:00" + }, + "source_type_name": "SourceTypeA", + "valid_at": "2024-01-01T00:00:00+00:00", + } }; -// compiled schemas -const compiledExternalEventTypeSchema = ajv.compile(correctExternalEventTypeSchema); -const compiledExternalSourceTypeSchema = ajv.compile(externalSourceTypeSchema); -const compiledExternalSourceSchema = ajv.compile(externalSourceSchema); - -// external source -const externalSource = { - external_events: [ +const incorrectExternalEventAttributes = { + "events": [ { - attributes: { - code: 'A', - projectUser: 'UserA', + "attributes": { + "series": { + "iteration": 17, + "make": "alpha", + // "type": "A", <-- missing. + } }, - duration: '01:10:00', - event_type_name: 'TestExternalEventType', - key: 'Event01', - start_time: '2024-023T00:23:00Z', + "duration": "02:00:00", + "event_type_name": "EventTypeA", + "key": "EventTypeA:1/1", + "start_time": "2024-01-01T01:35:00+00:00", }, { - attributes: { - code: 'B', - projectUser: 'UserB', + "attributes": { + "series": { + "iteration": 21, + "make": "beta", + "type": "B" + } }, - duration: '03:40:00', - event_type_name: 'DSNContact', - key: 'Event02', - start_time: '2024-021T00:21:00Z', + "duration": "02:00:00", + "event_type_name": "EventTypeA", + "key": "EventTypeA:1/2", + "start_time": "2024-01-02T11:50:00+00:00", }, + { + "attributes": { + "projectUser": "Jerry", + "tick": 18 + }, + "duration": "03:40:00", + "event_type_name": "EventTypeB", + "key": "EventTypeB:1/3", + "start_time": "2024-01-03T15:20:00+00:00" + } ], - source: { - attributes: { - operator: 'alpha', - version: 1, - }, - derivation_group_name: 'TestDerivationGroup', - key: 'TestExternalSourceKey', - period: { - end_time: '2024-01-28T00:00:00+00:00', - start_time: '2024-01-21T00:00:00+00:00', - }, - source_type_name: 'TestExternalSourceType', - valid_at: '2024-01-19T00:00:00+00:00', - }, + "source": { + "attributes": { + "version": 1, + "wrkcat": "234" + }, + "key": "SourceTypeA:valid_source_A.json", + "period": { + "end_time": "2024-01-07T00:00:00+00:00", + "start_time": "2024-01-01T00:00:00+00:00" + }, + "source_type_name": "SourceTypeA", + "valid_at": "2024-01-01T00:00:00+00:00", + } }; -// invalid attributes -const invalidSourceAttributes = { - operator: 1, - version: 1, -}; -const invalidEventAttributes = { - code: 1, - projectUser: 'UserB', -}; describe('validation tests', () => { - // test validating type schema validation (demonstrate you can feed it bogus and its fine, but if an existing field gets a wrong type then its a problem) - describe('attribute schema validation', () => { - test('validating correct external event type schema', () => { - const schemaIsValid: boolean = ajv.validateSchema(correctExternalEventTypeSchema); - expect(schemaIsValid).toBe(true); - }); - test('validating incorrect external event type schema that passes', () => { - const schemaIsValid: boolean = ajv.validateSchema(incorrectPassingExternalEventTypeSchema); - expect(schemaIsValid).toBe(true); - }); + // test to verify source/event type file is correctly formatted + test('verify source/event type file is correctly formatted', () => { + // get the validator + const attributeValidator = ajv.compile(attributeSchemaMetaschema); + + // test it against a correct defs/attribute metaschema object + const result = attributeValidator(attributeDefs); + expect(result).toBeTruthy(); + expect(attributeValidator.errors).toBeNull(); + }); - test('validating incorrect external event type schema that fails', () => { - const schemaIsValid: boolean = ajv.validateSchema(incorrectFailingExternalEventTypeSchema); - expect(schemaIsValid).toBe(false); - const errors = ajv.errors; - expect(errors?.length).toBe(1); - expect(errors?.at(0)?.message).toContain('should be array'); - }); + // test to verify source/event type file is incorrectly formatted + test('verify source/event type file is incorrectly formatted', () => { + // get the validator + const attributeValidator = ajv.compile(attributeSchemaMetaschema); + + // test it against a correct defs/attribute metaschema object + const result = attributeValidator(incorrectAttributeDefs); + expect(result).toBeFalsy(); + + const errors = attributeValidator.errors; + expect(errors?.length).toBe(1); + expect(errors?.at(0)?.schemaPath).toBe("#/$defs/AttributeSchema/patternProperties/%5E.*%24/required"); + expect(errors?.at(0)?.message).toMatch("should have required property 'required'"); }); - // test validating external source validation - don't need to be thorough; this is just ajv functionality. - describe('external source validation', () => { - test('correct external source validation', async () => { - let sourceIsValid: boolean = false; - sourceIsValid = await compiledExternalSourceSchema(externalSource); - expect(sourceIsValid).toBe(true); - }); + // test to verify that composition of a base schema with attribute schemas work + test('verify validation functionality of updateSchemaWithDefs', () => { + // transform attributeDefs to match something that might come from hasura (just ONE source type, as we will be constructing a schema for a specific source) + const attributeSchema: { event_types: any, source_type: any } = { + event_types: [], + source_type: {} + }; + attributeSchema.event_types = attributeDefs.event_types; + attributeSchema.source_type['SourceTypeA'] = attributeDefs.source_types.SourceTypeA; + + // construct a megaschema + const schemaFunctionWithDefs = updateSchemaWithDefs(attributeSchema); + const schema: any = schemaFunctionWithDefs.schema; + expect(schema).toBeTruthy(); + + if (schema) { + // verify it is formatted correctly + expect(Object.keys(schema.$defs.event_types)).toMatchObject(["EventTypeA", "EventTypeB", "EventTypeC"]); + expect(Object.keys(schema.$defs.source_type)).toMatchObject(["SourceTypeA"]); + expect(schema.properties.events.items.else.else.properties.attributes.$ref).toEqual("#/$defs/event_types/EventTypeC"); + } }); - // test validating external source attribute validation - describe('external source type attribute validation', () => { - test('correct external source type attribute validation', async () => { - let sourceAttributesAreValid: boolean = false; - sourceAttributesAreValid = await compiledExternalSourceTypeSchema(externalSource.source.attributes); - expect(sourceAttributesAreValid).toBe(true); + + // source testing + describe('validating (and failing) sources', () => { + // transform attributeDefs to match something that might come from hasura (just ONE source type, as we will be constructing a schema for a specific source) + const attributeSchema: { event_types: any, source_type: any } = { + event_types: [], + source_type: {} + }; + attributeSchema.event_types = attributeDefs.event_types; + attributeSchema.source_type['SourceTypeA'] = attributeDefs.source_types.SourceTypeA; + + // construct a megaschema + const schemaFunctionWithDefs = updateSchemaWithDefs(attributeSchema); + + // test to verify a source's (and all events') attributes are correctly formatted + test('source and event attributes are correct', () => { + const result = schemaFunctionWithDefs(correctExternalSource); + expect(result).toBeTruthy(); + expect(schemaFunctionWithDefs.errors).toBeNull(); }); - test('incorrect external source type attribute validation', async () => { - let sourceAttributesAreValid: boolean = false; - sourceAttributesAreValid = await compiledExternalSourceTypeSchema(invalidSourceAttributes); - expect(sourceAttributesAreValid).toBe(false); - const errors = compiledExternalSourceTypeSchema.errors; + // test to verify a source's attributes are incorrectly formatted + test('source attributes fail when incorrectly formatted', () => { + const result = schemaFunctionWithDefs(incorrectExternalSourceAttributes); + expect(result).toBeFalsy(); + + const errors = schemaFunctionWithDefs.errors; expect(errors?.length).toBe(1); - expect(errors?.at(0)?.message).toContain('should be string'); + expect(errors?.at(0)?.schemaPath).toBe("#/$defs/source_type/SourceTypeA/properties/wrkcat/type"); + expect(errors?.at(0)?.message).toMatch("should be string"); }); - }); - // test validating external event attribute validation - describe('external event type attribute validation', () => { - test('correct external event type attribute validation', async () => { - let eventAttributesAreValid: boolean = true; - for (const external_event of externalSource.external_events) { - eventAttributesAreValid = - eventAttributesAreValid && (await compiledExternalEventTypeSchema(external_event.attributes)); - } - expect(eventAttributesAreValid).toBe(true); - }); + // test to verify an event's attributes are incorrectly formatted + test('event attributes fail when incorrectly formatted', () => { + const result = schemaFunctionWithDefs(incorrectExternalEventAttributes); + expect(result).toBeFalsy(); - test('incorrect external event type attribute validation', async () => { - let eventAttributesAreValid: boolean = false; - eventAttributesAreValid = await compiledExternalEventTypeSchema(invalidEventAttributes); - expect(eventAttributesAreValid).toBe(false); - const errors = compiledExternalEventTypeSchema.errors; + const errors = schemaFunctionWithDefs.errors; expect(errors?.length).toBe(1); - expect(errors?.at(0)?.message).toContain('should be string'); + expect(errors?.at(0)?.schemaPath).toBe("#/$defs/event_types/EventTypeA/properties/series/required"); + expect(errors?.at(0)?.message).toMatch("should have required property 'type'"); }); }); }); From e32f95a99f1a8da1107b4c1d8588705569f51b36 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Fri, 22 Nov 2024 17:21:12 -0500 Subject: [PATCH 23/39] Partial refactor of new schema compilation code Still a few areas to refactor/cleanup, WIP --- .../external-source/external-source.ts | 97 ++++++++++--------- src/types/external-source.ts | 29 +++--- 2 files changed, 62 insertions(+), 64 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 6c12715..f769b43 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -3,10 +3,12 @@ import type { DerivationGroupInsertInput, ExternalSourceTypeInsertInput, CreateExternalSourceResponse, - CreateExternalSourceTypeResponse, ExternalEventTypeInsertInput, ExternalEvent, ExternalSourceInsertInput, + CreateExternalSourceEventTypeResponse, + GetSourceEventTypeAttributeSchemasResponse, + AttributeSchema, } from '../../types/external-source.js'; import Ajv from 'ajv'; import { getEnv } from '../../env.js'; @@ -113,7 +115,9 @@ export function updateSchemaWithDefs(defs: { event_types: any, source_type: any localSchemaCopy.$defs.event_types[event_type] = defs.event_types[event_type]; } - // compile with defs, return + console.log(localSchemaCopy.$defs["event_types"]["EventTypeB"]); + + // Compile & return full schema with 'defs' added const localAjv = new Ajv(); return localAjv.compile(localSchemaCopy); } @@ -126,18 +130,21 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { } = req; const { body } = req; + const { event_types, source_types } = body; + const parsedEventTypes: { [x: string]: object } = JSON.parse(event_types); + const parsedSourceTypes: { [x: string]: object } = JSON.parse(source_types); + logger.info(`POST /uploadExternalSourceEventTypes: Uploading External Source and Event Types...`); const headers: HeadersInit = { Authorization: authorizationHeader ?? '', 'Content-Type': 'application/json', - 'x-hasura-admin-secret': 'aerie', 'x-hasura-role': roleHeader ? `${roleHeader}` : '', 'x-hasura-user-id': userHeader ? `${userHeader}` : '', }; // Validate uploaded attribute schemas are formatted validly - const schemasAreValid: boolean = await compiledAttributeMetaschema(body); + const schemasAreValid: boolean = await compiledAttributeMetaschema({event_types: parsedEventTypes, source_types: parsedSourceTypes}); if (!schemasAreValid) { logger.error( `POST /uploadExternalSourceEventTypes: Schema validation failed for uploaded source and event types.`, @@ -150,23 +157,21 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { logger.info(`POST /uploadExternalSourceEventTypes: Uploaded attribute schema(s) are VALID`); // extract the external sources and event types - const externalSourceTypeInput: ExternalSourceTypeInsertInput = []; - const externalEventTypeInput: ExternalEventTypeInsertInput = []; + const externalSourceTypeInput: ExternalSourceTypeInsertInput[] = []; + const externalEventTypeInput: ExternalEventTypeInsertInput[] = []; - const external_event_types = body.event_types; - const event_type_keys = Object.keys(external_event_types); + const event_type_keys = Object.keys(parsedEventTypes); for (const external_event_type of event_type_keys) { externalEventTypeInput.push({ - attribute_schema: external_event_types[external_event_type], + attribute_schema: event_types[external_event_type], name: external_event_type }) } - const external_source_types = body.source_types; - const source_type_keys = Object.keys(external_source_types); + const source_type_keys = Object.keys(parsedSourceTypes); for (const external_source_type of source_type_keys) { externalSourceTypeInput.push({ - attribute_schema: external_source_types[external_source_type], + attribute_schema: source_types[external_source_type], name: external_source_type }) } @@ -182,9 +187,11 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { }); const jsonResponse = await response.json(); - const createExternalSourceTypeResponse = jsonResponse as CreateExternalSourceTypeResponse | HasuraError; - - res.json(createExternalSourceTypeResponse); + if (jsonResponse?.data !== undefined) { + res.json(jsonResponse.data as CreateExternalSourceEventTypeResponse); + } else { + res.json(jsonResponse as HasuraError); + } } async function uploadExternalSource(req: Request, res: Response) { @@ -199,7 +206,7 @@ async function uploadExternalSource(req: Request, res: Response) { const parsedExternalEvents: ExternalEvent[] = JSON.parse(events); const { attributes, derivation_group_name, key, period, source_type_name, valid_at } = parsedSource; - // re-package the fields as a JSON object to be posted + // Re-package the fields as a JSON object to be parsed const externalSourceJson = { events: parsedExternalEvents, source: { @@ -237,23 +244,19 @@ async function uploadExternalSource(req: Request, res: Response) { }); const attributeSchemaJson = await attributeSchemas.json(); - const { external_event_type, external_source_type } = attributeSchemaJson.data; - - const defs: { event_types: any, source_type: any } = { - event_types: { - - }, - source_type: { - [external_source_type[0].name]: external_source_type[0].attribute_schema - } - }; - for (const event_type of external_event_type) { - defs.event_types[event_type.name] = event_type.attribute_schema - } + const { external_event_type, external_source_type } = attributeSchemaJson.data as GetSourceEventTypeAttributeSchemasResponse; + const eventTypeNamesMappedToSchemas = external_event_type.reduce((acc: Record, eventType: ExternalEventTypeInsertInput ) => { + acc[eventType.name] = eventType.attribute_schema; + return acc; + }, {}); + const sourceTypeNamesMappedToSchemas = external_source_type.reduce((acc: Record, sourceType: ExternalSourceTypeInsertInput ) => { + acc[sourceType.name] = sourceType.attribute_schema; + return acc; + }, {}); // Assemble megaschema from attribute schemas - const compiledExternalSourceMegaschema: Ajv.ValidateFunction = updateSchemaWithDefs(defs); + const compiledExternalSourceMegaschema: Ajv.ValidateFunction = updateSchemaWithDefs({ event_types: eventTypeNamesMappedToSchemas, source_type: sourceTypeNamesMappedToSchemas }); // Verify that this is a valid external source let sourceIsValid: boolean = false; @@ -298,9 +301,11 @@ async function uploadExternalSource(req: Request, res: Response) { }); const jsonResponse = await response.json(); - const createExternalSourceResponse = jsonResponse as CreateExternalSourceResponse | HasuraError; - - res.json(createExternalSourceResponse); + if (jsonResponse?.data !== undefined) { + res.json(jsonResponse.data as CreateExternalSourceResponse); + } else { + res.json(jsonResponse as HasuraError); + } } export default (app: Express) => { @@ -326,37 +331,37 @@ export default (app: Express) => { * schema: * type: object * properties: - * attribute_schema: + * event_types: + * type: object + * source_types: * type: object - * external_source_type_name: - * type: string * required: - * - external_source_type_name - * attribute_schema + * - event_types + * source_types * responses: * 200: - * description: Created External Source Type + * description: Created External Source & Event Types * content: * application/json: * schema: * properties: - * attribute_schema: - * description: JSON Schema for the created External Source Type's attributes + * createExternalEventTypes: + * description: Names of all the event types that were created in this request. + * type: object + * createExternalSourceTypes: + * description: Names of all the source types that were created in this request. * type: object - * name: - * description: Name of the created External Source Type - * type: string * 403: * description: Unauthorized error * 401: * description: Unauthenticated error - * summary: Uploads an External Source Type definition (containing name & attributes schema) to Hasura. + * summary: Uploads & validates a combination of External Event & Source types to Hasura. * tags: * - Hasura */ app.post( '/uploadExternalSourceEventTypes', - upload.single('attribute_schema'), + upload.any(), refreshLimiter, auth, uploadExternalSourceEventTypes, diff --git a/src/types/external-source.ts b/src/types/external-source.ts index 5e68c1c..16887d9 100644 --- a/src/types/external-source.ts +++ b/src/types/external-source.ts @@ -6,13 +6,13 @@ export type DerivationGroupInsertInput = { export type ExternalSourceTypeInsertInput = { name: string; attribute_schema: object; -}[]; +}; export type ExternalEventTypeInsertInput = { name: string; attribute_schema: object; -}[]; +}; export type ExternalSourceInsertInput = { attributes: object; @@ -50,19 +50,11 @@ export type CreateExternalSourceResponse = { createExternalSource: { name: string }; }; -export type CreateExternalSourceTypeResponse = { - createExternalSourceType: { attribute_schema: object; name: string }; -}; - -export type GetExternalSourceTypeAttributeSchemaResponse = { - external_source_type_by_pk: { attribute_schema: object }; -}; - -export type GetExternalEventTypeAttributeSchemaResponse = { - external_event_type_by_pk: { attribute_schema: object }; +export type CreateExternalSourceEventTypeResponse = { + createExternalEventTypes: { returning: string[] }, + createExternalSourceTypes: { returning: string[] } }; - export type ExternalEventInsertInput = { attributes: object; start_time: string; @@ -87,10 +79,11 @@ export type ExternalEvent = { attributes: object; }; -export type CreateExternalEventTypeResponse = { - createExternalEventType: { attribute_schema: object; name: string }; -}; - -export type UploadAttributeJSON = { +export type AttributeSchema = { [x: string]: any; }; + +export type GetSourceEventTypeAttributeSchemasResponse = { + external_event_type: ExternalEventTypeInsertInput[], + external_source_type: ExternalSourceTypeInsertInput[], +} From 27422a4e47d51d613131b272b8a42346f0865dc7 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Mon, 25 Nov 2024 09:20:31 -0500 Subject: [PATCH 24/39] Add description for request objects --- src/packages/external-source/external-source.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index f769b43..90697c8 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -332,8 +332,10 @@ export default (app: Express) => { * type: object * properties: * event_types: + * description: An object representing the JSON Schema definition(s) for all external event types to be uploaded. * type: object * source_types: + * description: An object representing the JSON Schema definition(s) for all external event types to be uploaded. * type: object * required: * - event_types From 574a0cdadeee480f98c96aa06a92f5de5051801a Mon Sep 17 00:00:00 2001 From: psubram3 Date: Mon, 25 Nov 2024 10:35:01 -0800 Subject: [PATCH 25/39] update for e2e tests --- .../external-source/external-source.ts | 72 +++++++++++++++---- 1 file changed, 60 insertions(+), 12 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 90697c8..f72461f 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -47,9 +47,19 @@ export function updateSchemaWithDefs(defs: { event_types: any, source_type: any // to match the event type in defs, and verify the event_type_name matches the def name const localSchemaCopy = structuredClone(baseExternalSourceSchema); const event_type_name = keys[0]; - const event_type_schema = defs.event_types[event_type_name]; + const event_type_schema = { + ...defs.event_types[event_type_name], + + // additionally, restrict extra properties + additionalProperties: false + }; const source_type_name = Object.keys(defs.source_type)[0]; - const source_type_schema = defs.source_type[source_type_name]; + const source_type_schema = { + ...defs.source_type[source_type_name], + + // additionally, restrict extra properties + additionalProperties: false + }; localSchemaCopy.properties.events.items.properties.attributes = event_type_schema; localSchemaCopy.properties.events.items.properties.event_type_name = { "const": event_type_name }; @@ -58,7 +68,7 @@ export function updateSchemaWithDefs(defs: { event_types: any, source_type: any localSchemaCopy.properties.source.properties.attributes = source_type_schema; const localAjv = new Ajv(); - return localAjv.addSchema(defs).compile(localSchemaCopy); + return localAjv.compile(localSchemaCopy); } // handle n event types @@ -103,19 +113,26 @@ export function updateSchemaWithDefs(defs: { event_types: any, source_type: any const sourceTypeKey = Object.keys(defs.source_type)[0]; localSchemaCopy.properties.source.properties.attributes = { $ref: `#/$defs/source_type/${sourceTypeKey}`} - // add defs localSchemaCopy.$defs = { event_types: {}, source_type: { - [sourceTypeKey]: defs.source_type[sourceTypeKey] + [sourceTypeKey]: { + ...defs.source_type[sourceTypeKey], + + // additionally, restrict extra properties + additionalProperties: false + } } - } + }; for (const event_type of keys) { - localSchemaCopy.$defs.event_types[event_type] = defs.event_types[event_type]; - } + localSchemaCopy.$defs.event_types[event_type] = { + ...defs.event_types[event_type], - console.log(localSchemaCopy.$defs["event_types"]["EventTypeB"]); + // additionally, restrict extra properties + additionalProperties: false + }; + } // Compile & return full schema with 'defs' added const localAjv = new Ajv(); @@ -201,9 +218,28 @@ async function uploadExternalSource(req: Request, res: Response) { } = req; const { body } = req; - const { source, events } = body; - const parsedSource = JSON.parse(source); - const parsedExternalEvents: ExternalEvent[] = JSON.parse(events); + if (typeof(body) !== "object") { + logger.error( + `POST /uploadExternalSourceEventTypes: Body of request must be a JSON, with two stringified properties: "source" and "external_events".`, + ); + res.status(500).send({ message: `Body of request must be a JSON, with two stringified properties: "source" and "external_events".` }); + return; + } + + let parsedSource; + let parsedExternalEvents: ExternalEvent[]; + try { + const { source, external_events } = body; + parsedSource = JSON.parse(source); + parsedExternalEvents = JSON.parse(external_events); + } + catch (e) { + logger.error( + `POST /uploadExternalSourceEventTypes: Body of request must be a JSON, with two stringified properties: "source" and "external_events". Alternatively, parsing may have failed:\n${e as Error}`, + ); + res.status(500).send({ message: `Body of request must be a JSON, with two stringified properties: "source" and "external_events". Alternatively, parsing may have failed:\n${e as Error}` }); + return; + } const { attributes, derivation_group_name, key, period, source_type_name, valid_at } = parsedSource; // Re-package the fields as a JSON object to be parsed @@ -244,6 +280,18 @@ async function uploadExternalSource(req: Request, res: Response) { }); const attributeSchemaJson = await attributeSchemas.json(); + const { external_event_type, external_source_type } = attributeSchemaJson.data; + + if (external_event_type.length === 0 || external_source_type.length === 0) { + logger.error( + `POST /uploadExternalSourceEventTypes: The source and event types in your source do not exist in the database.`, + ); + res.status(500).send({ message: `The source and event types in your source do not exist in the database.` }); + return; + } + + const defs: { event_types: any, source_type: any } = { + event_types: { const { external_event_type, external_source_type } = attributeSchemaJson.data as GetSourceEventTypeAttributeSchemasResponse; const eventTypeNamesMappedToSchemas = external_event_type.reduce((acc: Record, eventType: ExternalEventTypeInsertInput ) => { From 6edace6e289849166e04ecae373f7d21077fa0ed Mon Sep 17 00:00:00 2001 From: psubram3 Date: Mon, 25 Nov 2024 11:02:29 -0800 Subject: [PATCH 26/39] minor fix post rebase --- .../external-source/external-source.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index f72461f..fa9b994 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -1,14 +1,14 @@ import type { Express, Request, Response } from 'express'; import type { - DerivationGroupInsertInput, ExternalSourceTypeInsertInput, CreateExternalSourceResponse, ExternalEventTypeInsertInput, ExternalEvent, - ExternalSourceInsertInput, CreateExternalSourceEventTypeResponse, GetSourceEventTypeAttributeSchemasResponse, AttributeSchema, + DerivationGroupInsertInput, + ExternalSourceInsertInput, } from '../../types/external-source.js'; import Ajv from 'ajv'; import { getEnv } from '../../env.js'; @@ -33,7 +33,7 @@ const refreshLimiter = rateLimit({ windowMs: 15 * 60 * 1000, // 15 minutes }); -export function updateSchemaWithDefs(defs: { event_types: any, source_type: any }) {//: Ajv.ValidateFunction | undefined { +export function updateSchemaWithDefs(defs: { event_types: any, source_type: any }): Ajv.ValidateFunction { // build if statement const ifThenElse: { [key: string]: any } = { @@ -172,6 +172,7 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { } logger.info(`POST /uploadExternalSourceEventTypes: Uploaded attribute schema(s) are VALID`); + console.log(parsedEventTypes, parsedSourceTypes); // extract the external sources and event types const externalSourceTypeInput: ExternalSourceTypeInsertInput[] = []; @@ -180,7 +181,7 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { const event_type_keys = Object.keys(parsedEventTypes); for (const external_event_type of event_type_keys) { externalEventTypeInput.push({ - attribute_schema: event_types[external_event_type], + attribute_schema: parsedEventTypes[external_event_type], name: external_event_type }) } @@ -188,7 +189,7 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { const source_type_keys = Object.keys(parsedSourceTypes); for (const external_source_type of source_type_keys) { externalSourceTypeInput.push({ - attribute_schema: source_types[external_source_type], + attribute_schema: parsedSourceTypes[external_source_type], name: external_source_type }) } @@ -280,7 +281,7 @@ async function uploadExternalSource(req: Request, res: Response) { }); const attributeSchemaJson = await attributeSchemas.json(); - const { external_event_type, external_source_type } = attributeSchemaJson.data; + const { external_event_type, external_source_type } = attributeSchemaJson.data as GetSourceEventTypeAttributeSchemasResponse; if (external_event_type.length === 0 || external_source_type.length === 0) { logger.error( @@ -290,10 +291,6 @@ async function uploadExternalSource(req: Request, res: Response) { return; } - const defs: { event_types: any, source_type: any } = { - event_types: { - - const { external_event_type, external_source_type } = attributeSchemaJson.data as GetSourceEventTypeAttributeSchemasResponse; const eventTypeNamesMappedToSchemas = external_event_type.reduce((acc: Record, eventType: ExternalEventTypeInsertInput ) => { acc[eventType.name] = eventType.attribute_schema; return acc; @@ -301,7 +298,10 @@ async function uploadExternalSource(req: Request, res: Response) { const sourceTypeNamesMappedToSchemas = external_source_type.reduce((acc: Record, sourceType: ExternalSourceTypeInsertInput ) => { acc[sourceType.name] = sourceType.attribute_schema; return acc; - }, {}); + }, {}); + + console.log(external_event_type, external_source_type) + console.log(eventTypeNamesMappedToSchemas, sourceTypeNamesMappedToSchemas) // Assemble megaschema from attribute schemas const compiledExternalSourceMegaschema: Ajv.ValidateFunction = updateSchemaWithDefs({ event_types: eventTypeNamesMappedToSchemas, source_type: sourceTypeNamesMappedToSchemas }); From 092671e753c6c082899ee4f3ef477aa2c60b1c11 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Mon, 25 Nov 2024 14:36:35 -0500 Subject: [PATCH 27/39] external_events => events --- src/packages/external-source/external-source.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index fa9b994..9286f08 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -221,24 +221,24 @@ async function uploadExternalSource(req: Request, res: Response) { if (typeof(body) !== "object") { logger.error( - `POST /uploadExternalSourceEventTypes: Body of request must be a JSON, with two stringified properties: "source" and "external_events".`, + `POST /uploadExternalSourceEventTypes: Body of request must be a JSON, with two stringified properties: "source" and "events".`, ); - res.status(500).send({ message: `Body of request must be a JSON, with two stringified properties: "source" and "external_events".` }); + res.status(500).send({ message: `Body of request must be a JSON, with two stringified properties: "source" and "events".` }); return; } let parsedSource; let parsedExternalEvents: ExternalEvent[]; try { - const { source, external_events } = body; + const { source, events } = body; parsedSource = JSON.parse(source); - parsedExternalEvents = JSON.parse(external_events); + parsedExternalEvents = JSON.parse(events); } catch (e) { logger.error( - `POST /uploadExternalSourceEventTypes: Body of request must be a JSON, with two stringified properties: "source" and "external_events". Alternatively, parsing may have failed:\n${e as Error}`, + `POST /uploadExternalSourceEventTypes: Body of request must be a JSON, with two stringified properties: "source" and "events". Alternatively, parsing may have failed:\n${e as Error}`, ); - res.status(500).send({ message: `Body of request must be a JSON, with two stringified properties: "source" and "external_events". Alternatively, parsing may have failed:\n${e as Error}` }); + res.status(500).send({ message: `Body of request must be a JSON, with two stringified properties: "source" and "events". Alternatively, parsing may have failed:\n${e as Error}` }); return; } const { attributes, derivation_group_name, key, period, source_type_name, valid_at } = parsedSource; @@ -298,7 +298,7 @@ async function uploadExternalSource(req: Request, res: Response) { const sourceTypeNamesMappedToSchemas = external_source_type.reduce((acc: Record, sourceType: ExternalSourceTypeInsertInput ) => { acc[sourceType.name] = sourceType.attribute_schema; return acc; - }, {}); + }, {}); console.log(external_event_type, external_source_type) console.log(eventTypeNamesMappedToSchemas, sourceTypeNamesMappedToSchemas) From c8cb1375fdb8e8f1a75d5a78811316f084089ee1 Mon Sep 17 00:00:00 2001 From: psubram3 Date: Mon, 25 Nov 2024 12:13:28 -0800 Subject: [PATCH 28/39] formatting --- .../external-source/external-source.ts | 22 ++-- .../external-event-validation-schemata.ts | 116 +++++++++--------- test/external_source.validation.test.ts | 4 +- 3 files changed, 69 insertions(+), 73 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 9286f08..d796dc2 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -41,7 +41,7 @@ export function updateSchemaWithDefs(defs: { event_types: any, source_type: any let ifThenElsePointer = ifThenElse; const keys = Object.keys(defs.event_types); - // handling if there's only 1 event type + // handling if there's only 1 event type (don't bother with $defs, just update attributes' properties directly) if (keys.length === 1) { // no need for ifThenElse, simply create localSchemaCopy and update properties.events.items.properties.attributes // to match the event type in defs, and verify the event_type_name matches the def name @@ -111,7 +111,7 @@ export function updateSchemaWithDefs(defs: { event_types: any, source_type: any // insert def for "source" attributes const sourceTypeKey = Object.keys(defs.source_type)[0]; - localSchemaCopy.properties.source.properties.attributes = { $ref: `#/$defs/source_type/${sourceTypeKey}`} + localSchemaCopy.properties.source.properties.attributes = { $ref: `#/$defs/source_type/${sourceTypeKey}` } // add defs localSchemaCopy.$defs = { @@ -161,7 +161,7 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { }; // Validate uploaded attribute schemas are formatted validly - const schemasAreValid: boolean = await compiledAttributeMetaschema({event_types: parsedEventTypes, source_types: parsedSourceTypes}); + const schemasAreValid: boolean = await compiledAttributeMetaschema({ event_types: parsedEventTypes, source_types: parsedSourceTypes }); if (!schemasAreValid) { logger.error( `POST /uploadExternalSourceEventTypes: Schema validation failed for uploaded source and event types.`, @@ -172,7 +172,6 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { } logger.info(`POST /uploadExternalSourceEventTypes: Uploaded attribute schema(s) are VALID`); - console.log(parsedEventTypes, parsedSourceTypes); // extract the external sources and event types const externalSourceTypeInput: ExternalSourceTypeInsertInput[] = []; @@ -219,7 +218,7 @@ async function uploadExternalSource(req: Request, res: Response) { } = req; const { body } = req; - if (typeof(body) !== "object") { + if (typeof (body) !== "object") { logger.error( `POST /uploadExternalSourceEventTypes: Body of request must be a JSON, with two stringified properties: "source" and "events".`, ); @@ -230,9 +229,9 @@ async function uploadExternalSource(req: Request, res: Response) { let parsedSource; let parsedExternalEvents: ExternalEvent[]; try { - const { source, events } = body; - parsedSource = JSON.parse(source); - parsedExternalEvents = JSON.parse(events); + const { source, events } = body; + parsedSource = JSON.parse(source); + parsedExternalEvents = JSON.parse(events); } catch (e) { logger.error( @@ -291,18 +290,15 @@ async function uploadExternalSource(req: Request, res: Response) { return; } - const eventTypeNamesMappedToSchemas = external_event_type.reduce((acc: Record, eventType: ExternalEventTypeInsertInput ) => { + const eventTypeNamesMappedToSchemas = external_event_type.reduce((acc: Record, eventType: ExternalEventTypeInsertInput) => { acc[eventType.name] = eventType.attribute_schema; return acc; }, {}); - const sourceTypeNamesMappedToSchemas = external_source_type.reduce((acc: Record, sourceType: ExternalSourceTypeInsertInput ) => { + const sourceTypeNamesMappedToSchemas = external_source_type.reduce((acc: Record, sourceType: ExternalSourceTypeInsertInput) => { acc[sourceType.name] = sourceType.attribute_schema; return acc; }, {}); - console.log(external_event_type, external_source_type) - console.log(eventTypeNamesMappedToSchemas, sourceTypeNamesMappedToSchemas) - // Assemble megaschema from attribute schemas const compiledExternalSourceMegaschema: Ajv.ValidateFunction = updateSchemaWithDefs({ event_types: eventTypeNamesMappedToSchemas, source_type: sourceTypeNamesMappedToSchemas }); diff --git a/src/packages/schemas/external-event-validation-schemata.ts b/src/packages/schemas/external-event-validation-schemata.ts index 15fedc7..f12502b 100644 --- a/src/packages/schemas/external-event-validation-schemata.ts +++ b/src/packages/schemas/external-event-validation-schemata.ts @@ -41,62 +41,62 @@ export const attributeSchemaMetaschema = { // the schema that schemas for specific types are integrated with, after pulling them from the database export const baseExternalSourceSchema: { [key: string]: any } = { - $id: "source_schema", - $schema: "http://json-schema.org/draft-07/schema", - additionalProperties: false, - description: "The base schema for external sources. Defs and ifs, for specific source/event type attributes, are integrated later.", - properties: { - events: { - items: { - additionalProperties: false, - properties: { - attributes: { - type: "object" - }, - duration: { "type": "string" }, - event_type_name: { "type": "string" }, - key: { "type": "string" }, - start_time: { "type": "string" } - }, - required: ["duration", "event_type_name", "key", "attributes", "start_time"], - type: "object" - }, - type: "array" - }, - source: { - additionalProperties: false, - properties: { - attributes: { - type: "object" // WILL BE REPLACED WITH A $ref - }, - derivation_group_name: { "type": "string" }, - key: { "type": "string" }, - period: { - additionalProperties: false, - properties: { - end_time: { - pattern: "^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$", - type: "string" - }, - start_time: { - pattern: "^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$", - type: "string" - } - }, - required: ["start_time", "end_time"], - type: "object" - }, - source_type_name: { "type": "string" }, - valid_at: { - pattern: "^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$", - type: "string" - } - }, - required: ["key", "source_type_name", "valid_at", "period", "attributes"], - type: "object" - } - }, - required: ["source", "events"], - title: "SourceTypeA", - type: "object" + $id: "source_schema", + $schema: "http://json-schema.org/draft-07/schema", + additionalProperties: false, + description: "The base schema for external sources. Defs and ifs, for specific source/event type attributes, are integrated later.", + properties: { + events: { + items: { + additionalProperties: false, + properties: { + attributes: { + type: "object" + }, + duration: { "type": "string" }, + event_type_name: { "type": "string" }, + key: { "type": "string" }, + start_time: { "type": "string" } + }, + required: ["duration", "event_type_name", "key", "attributes", "start_time"], + type: "object" + }, + type: "array" + }, + source: { + additionalProperties: false, + properties: { + attributes: { + type: "object" // WILL BE REPLACED WITH A $ref + }, + derivation_group_name: { "type": "string" }, + key: { "type": "string" }, + period: { + additionalProperties: false, + properties: { + end_time: { + pattern: "^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$", + type: "string" + }, + start_time: { + pattern: "^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$", + type: "string" + } + }, + required: ["start_time", "end_time"], + type: "object" + }, + source_type_name: { "type": "string" }, + valid_at: { + pattern: "^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$", + type: "string" + } + }, + required: ["key", "source_type_name", "valid_at", "period", "attributes"], + type: "object" + } + }, + required: ["source", "events"], + title: "SourceTypeA", + type: "object" }; \ No newline at end of file diff --git a/test/external_source.validation.test.ts b/test/external_source.validation.test.ts index e42c358..65a0408 100644 --- a/test/external_source.validation.test.ts +++ b/test/external_source.validation.test.ts @@ -316,7 +316,7 @@ describe('validation tests', () => { test('verify source/event type file is correctly formatted', () => { // get the validator const attributeValidator = ajv.compile(attributeSchemaMetaschema); - + // test it against a correct defs/attribute metaschema object const result = attributeValidator(attributeDefs); expect(result).toBeTruthy(); @@ -327,7 +327,7 @@ describe('validation tests', () => { test('verify source/event type file is incorrectly formatted', () => { // get the validator const attributeValidator = ajv.compile(attributeSchemaMetaschema); - + // test it against a correct defs/attribute metaschema object const result = attributeValidator(incorrectAttributeDefs); expect(result).toBeFalsy(); From 1a0ee20069e3f2eac58957395b093e1633c2a833 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Tue, 26 Nov 2024 12:33:35 -0500 Subject: [PATCH 29/39] formatting --- .../external-source/external-source.ts | 152 +++--- src/packages/external-source/gql.ts | 2 +- .../external-event-validation-schemata.ts | 186 +++---- src/types/external-source.ts | 11 +- test/external_source.validation.test.ts | 477 +++++++++--------- 5 files changed, 425 insertions(+), 403 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index d796dc2..a041461 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -33,11 +33,9 @@ const refreshLimiter = rateLimit({ windowMs: 15 * 60 * 1000, // 15 minutes }); -export function updateSchemaWithDefs(defs: { event_types: any, source_type: any }): Ajv.ValidateFunction { +export function updateSchemaWithDefs(defs: { event_types: any; source_type: any }): Ajv.ValidateFunction { // build if statement - const ifThenElse: { [key: string]: any } = { - - }; + const ifThenElse: { [key: string]: any } = {}; let ifThenElsePointer = ifThenElse; const keys = Object.keys(defs.event_types); @@ -51,18 +49,18 @@ export function updateSchemaWithDefs(defs: { event_types: any, source_type: any ...defs.event_types[event_type_name], // additionally, restrict extra properties - additionalProperties: false + additionalProperties: false, }; const source_type_name = Object.keys(defs.source_type)[0]; const source_type_schema = { ...defs.source_type[source_type_name], // additionally, restrict extra properties - additionalProperties: false + additionalProperties: false, }; localSchemaCopy.properties.events.items.properties.attributes = event_type_schema; - localSchemaCopy.properties.events.items.properties.event_type_name = { "const": event_type_name }; + localSchemaCopy.properties.events.items.properties.event_type_name = { const: event_type_name }; // insert def for "source" attributes localSchemaCopy.properties.source.properties.attributes = source_type_schema; @@ -74,44 +72,42 @@ export function updateSchemaWithDefs(defs: { event_types: any, source_type: any // handle n event types for (let i = 0; i < keys.length - 1; i++) { const key = keys[i]; - console.log("NOW ON:", key); - ifThenElsePointer["if"] = { + console.log('NOW ON:', key); + ifThenElsePointer['if'] = { properties: { event_type_name: { - const: key - } - } + const: key, + }, + }, }; - ifThenElsePointer["then"] = { + ifThenElsePointer['then'] = { properties: { attributes: { - $ref: `#/$defs/event_types/${key}` - } - } - }; - ifThenElsePointer["else"] = { - + $ref: `#/$defs/event_types/${key}`, + }, + }, }; - ifThenElsePointer = ifThenElsePointer["else"]; + ifThenElsePointer['else'] = {}; + ifThenElsePointer = ifThenElsePointer['else']; } // fill in the final else with the last element const key = keys[keys.length - 1]; - ifThenElsePointer["properties"] = { + ifThenElsePointer['properties'] = { attributes: { - $ref: `#/$defs/event_types/${key}` - } - } + $ref: `#/$defs/event_types/${key}`, + }, + }; // insert if statement into local copy of baseExternalSourceSchema const localSchemaCopy = structuredClone(baseExternalSourceSchema); - localSchemaCopy.properties.events.items["if"] = ifThenElse["if"]; - localSchemaCopy.properties.events.items["then"] = ifThenElse["then"]; - localSchemaCopy.properties.events.items["else"] = ifThenElse["else"]; + localSchemaCopy.properties.events.items['if'] = ifThenElse['if']; + localSchemaCopy.properties.events.items['then'] = ifThenElse['then']; + localSchemaCopy.properties.events.items['else'] = ifThenElse['else']; // insert def for "source" attributes const sourceTypeKey = Object.keys(defs.source_type)[0]; - localSchemaCopy.properties.source.properties.attributes = { $ref: `#/$defs/source_type/${sourceTypeKey}` } + localSchemaCopy.properties.source.properties.attributes = { $ref: `#/$defs/source_type/${sourceTypeKey}` }; // add defs localSchemaCopy.$defs = { @@ -121,16 +117,16 @@ export function updateSchemaWithDefs(defs: { event_types: any, source_type: any ...defs.source_type[sourceTypeKey], // additionally, restrict extra properties - additionalProperties: false - } - } + additionalProperties: false, + }, + }, }; for (const event_type of keys) { localSchemaCopy.$defs.event_types[event_type] = { ...defs.event_types[event_type], // additionally, restrict extra properties - additionalProperties: false + additionalProperties: false, }; } @@ -161,11 +157,12 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { }; // Validate uploaded attribute schemas are formatted validly - const schemasAreValid: boolean = await compiledAttributeMetaschema({ event_types: parsedEventTypes, source_types: parsedSourceTypes }); + const schemasAreValid: boolean = await compiledAttributeMetaschema({ + event_types: parsedEventTypes, + source_types: parsedSourceTypes, + }); if (!schemasAreValid) { - logger.error( - `POST /uploadExternalSourceEventTypes: Schema validation failed for uploaded source and event types.`, - ); + logger.error(`POST /uploadExternalSourceEventTypes: Schema validation failed for uploaded source and event types.`); compiledAttributeMetaschema.errors?.forEach(error => logger.error(error)); res.status(500).send({ message: compiledAttributeMetaschema.errors }); return; @@ -181,16 +178,16 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { for (const external_event_type of event_type_keys) { externalEventTypeInput.push({ attribute_schema: parsedEventTypes[external_event_type], - name: external_event_type - }) + name: external_event_type, + }); } const source_type_keys = Object.keys(parsedSourceTypes); for (const external_source_type of source_type_keys) { externalSourceTypeInput.push({ attribute_schema: parsedSourceTypes[external_source_type], - name: external_source_type - }) + name: external_source_type, + }); } // Run the Hasura migration for creating all types, in one go @@ -218,11 +215,13 @@ async function uploadExternalSource(req: Request, res: Response) { } = req; const { body } = req; - if (typeof (body) !== "object") { + if (typeof body !== 'object') { logger.error( `POST /uploadExternalSourceEventTypes: Body of request must be a JSON, with two stringified properties: "source" and "events".`, ); - res.status(500).send({ message: `Body of request must be a JSON, with two stringified properties: "source" and "events".` }); + res + .status(500) + .send({ message: `Body of request must be a JSON, with two stringified properties: "source" and "events".` }); return; } @@ -232,12 +231,19 @@ async function uploadExternalSource(req: Request, res: Response) { const { source, events } = body; parsedSource = JSON.parse(source); parsedExternalEvents = JSON.parse(events); - } - catch (e) { + } catch (e) { logger.error( - `POST /uploadExternalSourceEventTypes: Body of request must be a JSON, with two stringified properties: "source" and "events". Alternatively, parsing may have failed:\n${e as Error}`, + `POST /uploadExternalSourceEventTypes: Body of request must be a JSON, with two stringified properties: "source" and "events". Alternatively, parsing may have failed:\n${ + e as Error + }`, ); - res.status(500).send({ message: `Body of request must be a JSON, with two stringified properties: "source" and "events". Alternatively, parsing may have failed:\n${e as Error}` }); + res + .status(500) + .send({ + message: `Body of request must be a JSON, with two stringified properties: "source" and "events". Alternatively, parsing may have failed:\n${ + e as Error + }`, + }); return; } const { attributes, derivation_group_name, key, period, source_type_name, valid_at } = parsedSource; @@ -272,7 +278,7 @@ async function uploadExternalSource(req: Request, res: Response) { query: gql.GET_SOURCE_EVENT_TYPE_ATTRIBUTE_SCHEMAS, variables: { externalEventTypes: eventTypeNames, - externalSourceType: source_type_name + externalSourceType: source_type_name, }, }), headers, @@ -280,7 +286,8 @@ async function uploadExternalSource(req: Request, res: Response) { }); const attributeSchemaJson = await attributeSchemas.json(); - const { external_event_type, external_source_type } = attributeSchemaJson.data as GetSourceEventTypeAttributeSchemasResponse; + const { external_event_type, external_source_type } = + attributeSchemaJson.data as GetSourceEventTypeAttributeSchemasResponse; if (external_event_type.length === 0 || external_source_type.length === 0) { logger.error( @@ -290,17 +297,26 @@ async function uploadExternalSource(req: Request, res: Response) { return; } - const eventTypeNamesMappedToSchemas = external_event_type.reduce((acc: Record, eventType: ExternalEventTypeInsertInput) => { - acc[eventType.name] = eventType.attribute_schema; - return acc; - }, {}); - const sourceTypeNamesMappedToSchemas = external_source_type.reduce((acc: Record, sourceType: ExternalSourceTypeInsertInput) => { - acc[sourceType.name] = sourceType.attribute_schema; - return acc; - }, {}); + const eventTypeNamesMappedToSchemas = external_event_type.reduce( + (acc: Record, eventType: ExternalEventTypeInsertInput) => { + acc[eventType.name] = eventType.attribute_schema; + return acc; + }, + {}, + ); + const sourceTypeNamesMappedToSchemas = external_source_type.reduce( + (acc: Record, sourceType: ExternalSourceTypeInsertInput) => { + acc[sourceType.name] = sourceType.attribute_schema; + return acc; + }, + {}, + ); // Assemble megaschema from attribute schemas - const compiledExternalSourceMegaschema: Ajv.ValidateFunction = updateSchemaWithDefs({ event_types: eventTypeNamesMappedToSchemas, source_type: sourceTypeNamesMappedToSchemas }); + const compiledExternalSourceMegaschema: Ajv.ValidateFunction = updateSchemaWithDefs({ + event_types: eventTypeNamesMappedToSchemas, + source_type: sourceTypeNamesMappedToSchemas, + }); // Verify that this is a valid external source let sourceIsValid: boolean = false; @@ -308,8 +324,18 @@ async function uploadExternalSource(req: Request, res: Response) { if (sourceIsValid) { logger.info(`POST /uploadExternalSource: External Source ${key}'s formatting is valid`); } else { - logger.error(`POST /uploadExternalSource: External Source ${key}'s formatting is invalid:\n${JSON.stringify(compiledExternalSourceMegaschema.errors)}`); - res.status(500).send({ message: `External Source ${key}'s formatting is invalid:\n${JSON.stringify(compiledExternalSourceMegaschema.errors)}` }); + logger.error( + `POST /uploadExternalSource: External Source ${key}'s formatting is invalid:\n${JSON.stringify( + compiledExternalSourceMegaschema.errors, + )}`, + ); + res + .status(500) + .send({ + message: `External Source ${key}'s formatting is invalid:\n${JSON.stringify( + compiledExternalSourceMegaschema.errors, + )}`, + }); return; } @@ -405,13 +431,7 @@ export default (app: Express) => { * tags: * - Hasura */ - app.post( - '/uploadExternalSourceEventTypes', - upload.any(), - refreshLimiter, - auth, - uploadExternalSourceEventTypes, - ); + app.post('/uploadExternalSourceEventTypes', upload.any(), refreshLimiter, auth, uploadExternalSourceEventTypes); /** * @swagger diff --git a/src/packages/external-source/gql.ts b/src/packages/external-source/gql.ts index 7b030c2..a328b1f 100644 --- a/src/packages/external-source/gql.ts +++ b/src/packages/external-source/gql.ts @@ -73,5 +73,5 @@ export default { attribute_schema } } - ` + `, }; diff --git a/src/packages/schemas/external-event-validation-schemata.ts b/src/packages/schemas/external-event-validation-schemata.ts index f12502b..7c52046 100644 --- a/src/packages/schemas/external-event-validation-schemata.ts +++ b/src/packages/schemas/external-event-validation-schemata.ts @@ -1,102 +1,106 @@ // a schema that describes the format for the attribute files (which are, themselves, JSON Schema-like) export const attributeSchemaMetaschema = { - "$defs": { - "AttributeSchema": { - "additionalProperties": false, - "patternProperties": { - "^.*$": { - "properties": { - "attributes": { - "additionalProperties": true, - "type": "object" - }, - "required": { - "items": { "type": "string" }, - "type": "array" - }, - "type": { "type": "string" } - }, - "required": ["required", "properties", "type"], - "type": "object" - } + $defs: { + AttributeSchema: { + additionalProperties: false, + patternProperties: { + '^.*$': { + properties: { + attributes: { + additionalProperties: true, + type: 'object', }, - "type": "object" - } - }, - "$schema": "http://json-schema.org/draft-07/schema", - "additionalProperties": false, - "description": "Schema for the attributes of uploaded source types and/or event types.", - "properties": { - "event_types": { - "$ref": "#/$defs/AttributeSchema" + required: { + items: { type: 'string' }, + type: 'array', + }, + type: { type: 'string' }, + }, + required: ['required', 'properties', 'type'], + type: 'object', }, - "source_types": { - "$ref": "#/$defs/AttributeSchema" - } + }, + type: 'object', + }, + }, + $schema: 'http://json-schema.org/draft-07/schema', + additionalProperties: false, + description: 'Schema for the attributes of uploaded source types and/or event types.', + properties: { + event_types: { + $ref: '#/$defs/AttributeSchema', }, - "required": ["source_types", "event_types"], - "title": "TypeSpecificationSchema", - "type": "object" -} + source_types: { + $ref: '#/$defs/AttributeSchema', + }, + }, + required: ['source_types', 'event_types'], + title: 'TypeSpecificationSchema', + type: 'object', +}; // the schema that schemas for specific types are integrated with, after pulling them from the database export const baseExternalSourceSchema: { [key: string]: any } = { - $id: "source_schema", - $schema: "http://json-schema.org/draft-07/schema", - additionalProperties: false, - description: "The base schema for external sources. Defs and ifs, for specific source/event type attributes, are integrated later.", - properties: { - events: { - items: { - additionalProperties: false, - properties: { - attributes: { - type: "object" - }, - duration: { "type": "string" }, - event_type_name: { "type": "string" }, - key: { "type": "string" }, - start_time: { "type": "string" } - }, - required: ["duration", "event_type_name", "key", "attributes", "start_time"], - type: "object" - }, - type: "array" + $id: 'source_schema', + $schema: 'http://json-schema.org/draft-07/schema', + additionalProperties: false, + description: + 'The base schema for external sources. Defs and ifs, for specific source/event type attributes, are integrated later.', + properties: { + events: { + items: { + additionalProperties: false, + properties: { + attributes: { + type: 'object', + }, + duration: { type: 'string' }, + event_type_name: { type: 'string' }, + key: { type: 'string' }, + start_time: { type: 'string' }, }, - source: { - additionalProperties: false, - properties: { - attributes: { - type: "object" // WILL BE REPLACED WITH A $ref - }, - derivation_group_name: { "type": "string" }, - key: { "type": "string" }, - period: { - additionalProperties: false, - properties: { - end_time: { - pattern: "^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$", - type: "string" - }, - start_time: { - pattern: "^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$", - type: "string" - } - }, - required: ["start_time", "end_time"], - type: "object" - }, - source_type_name: { "type": "string" }, - valid_at: { - pattern: "^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$", - type: "string" - } + required: ['duration', 'event_type_name', 'key', 'attributes', 'start_time'], + type: 'object', + }, + type: 'array', + }, + source: { + additionalProperties: false, + properties: { + attributes: { + type: 'object', // WILL BE REPLACED WITH A $ref + }, + derivation_group_name: { type: 'string' }, + key: { type: 'string' }, + period: { + additionalProperties: false, + properties: { + end_time: { + pattern: + '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', + type: 'string', + }, + start_time: { + pattern: + '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', + type: 'string', }, - required: ["key", "source_type_name", "valid_at", "period", "attributes"], - type: "object" - } + }, + required: ['start_time', 'end_time'], + type: 'object', + }, + source_type_name: { type: 'string' }, + valid_at: { + pattern: + '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', + type: 'string', + }, + }, + required: ['key', 'source_type_name', 'valid_at', 'period', 'attributes'], + type: 'object', }, - required: ["source", "events"], - title: "SourceTypeA", - type: "object" -}; \ No newline at end of file + }, + required: ['source', 'events'], + title: 'SourceTypeA', + type: 'object', +}; diff --git a/src/types/external-source.ts b/src/types/external-source.ts index 16887d9..2f20c34 100644 --- a/src/types/external-source.ts +++ b/src/types/external-source.ts @@ -8,7 +8,6 @@ export type ExternalSourceTypeInsertInput = { attribute_schema: object; }; - export type ExternalEventTypeInsertInput = { name: string; attribute_schema: object; @@ -51,8 +50,8 @@ export type CreateExternalSourceResponse = { }; export type CreateExternalSourceEventTypeResponse = { - createExternalEventTypes: { returning: string[] }, - createExternalSourceTypes: { returning: string[] } + createExternalEventTypes: { returning: string[] }; + createExternalSourceTypes: { returning: string[] }; }; export type ExternalEventInsertInput = { @@ -84,6 +83,6 @@ export type AttributeSchema = { }; export type GetSourceEventTypeAttributeSchemasResponse = { - external_event_type: ExternalEventTypeInsertInput[], - external_source_type: ExternalSourceTypeInsertInput[], -} + external_event_type: ExternalEventTypeInsertInput[]; + external_source_type: ExternalSourceTypeInsertInput[]; +}; diff --git a/test/external_source.validation.test.ts b/test/external_source.validation.test.ts index 65a0408..0c1ce9b 100644 --- a/test/external_source.validation.test.ts +++ b/test/external_source.validation.test.ts @@ -6,312 +6,310 @@ import { updateSchemaWithDefs } from '../src/packages/external-source/external-s const ajv = Ajv(); const attributeDefs = { - "event_types": { - "EventTypeA": { - "properties": { - "series": { - "properties": { - "iteration": { "type": "number" }, - "make": { "type": "string" }, - "type": { "type": "string" }, + event_types: { + EventTypeA: { + properties: { + series: { + properties: { + iteration: { type: 'number' }, + make: { type: 'string' }, + type: { type: 'string' }, }, - "required": ["type", "make", "iteration"], - "type": "object", - } + required: ['type', 'make', 'iteration'], + type: 'object', + }, }, - "required": ["series"], - "type": "object", + required: ['series'], + type: 'object', }, - "EventTypeB": { - "properties": { - "projectUser": { - "type": "string" + EventTypeB: { + properties: { + projectUser: { + type: 'string', + }, + tick: { + type: 'number', }, - "tick": { - "type": "number" - } }, - "required": ["projectUser", "tick"], - "type": "object" + required: ['projectUser', 'tick'], + type: 'object', }, - "EventTypeC": { - "properties": { - "aperture": { - "type": "string" + EventTypeC: { + properties: { + aperture: { + type: 'string', + }, + subduration: { + pattern: '^P(?:\\d+Y)?(?:\\d+M)?(?:\\d+D)?T(?:\\d+H)?(?:\\d+M)?(?:\\d+S)?$', + type: 'string', }, - "subduration": { - "pattern": "^P(?:\\d+Y)?(?:\\d+M)?(?:\\d+D)?T(?:\\d+H)?(?:\\d+M)?(?:\\d+S)?$", - "type": "string" - } }, - "required": ["aperture", "subduration"], - "type": "object" - } + required: ['aperture', 'subduration'], + type: 'object', + }, }, - "source_types": { - "SourceTypeA": { - "properties": { - "version": { - "type": "number" + source_types: { + SourceTypeA: { + properties: { + version: { + type: 'number', + }, + wrkcat: { + type: 'string', }, - "wrkcat": { - "type": "string" - } }, - "required": ["version", "wrkcat"], - "type": "object" + required: ['version', 'wrkcat'], + type: 'object', }, - "SourceTypeB": { - "properties": { - "version": { - "type": "number" + SourceTypeB: { + properties: { + version: { + type: 'number', + }, + wrkcat: { + type: 'string', }, - "wrkcat": { - "type": "string" - } }, - "required": ["version", "wrkcat"], - "type": "object" - } - } + required: ['version', 'wrkcat'], + type: 'object', + }, + }, }; const incorrectAttributeDefs = { - "event_types": { - "EventTypeA": { - "properties": { - "series": { - "properties": { - "iteration": { "type": "number" }, - "make": { "type": "string" }, - "type": { "type": "string" }, + event_types: { + EventTypeA: { + properties: { + series: { + properties: { + iteration: { type: 'number' }, + make: { type: 'string' }, + type: { type: 'string' }, }, // "required": ["type", "make", "iteration"], // missing required field (not an issue) - "type": "object", - } + type: 'object', + }, }, // "required": ["series"], // missing required field (the issue, only at level patternProperties/sdfdsf/required) - "type": "object", + type: 'object', }, - "EventTypeB": { - "properties": { - "projectUser": { - "type": "string" + EventTypeB: { + properties: { + projectUser: { + type: 'string', + }, + tick: { + type: 'number', }, - "tick": { - "type": "number" - } }, - "required": ["projectUser", "tick"], - "type": "object" + required: ['projectUser', 'tick'], + type: 'object', }, - "EventTypeC": { - "properties": { - "aperture": { - "type": "string" + EventTypeC: { + properties: { + aperture: { + type: 'string', + }, + subduration: { + pattern: '^P(?:\\d+Y)?(?:\\d+M)?(?:\\d+D)?T(?:\\d+H)?(?:\\d+M)?(?:\\d+S)?$', + type: 'string', }, - "subduration": { - "pattern": "^P(?:\\d+Y)?(?:\\d+M)?(?:\\d+D)?T(?:\\d+H)?(?:\\d+M)?(?:\\d+S)?$", - "type": "string" - } }, - "required": ["aperture", "subduration"], - "type": "object" - } + required: ['aperture', 'subduration'], + type: 'object', + }, }, - "source_types": { - "SourceTypeA": { - "properties": { - "version": { - "type": "number" + source_types: { + SourceTypeA: { + properties: { + version: { + type: 'number', + }, + wrkcat: { + type: 'string', }, - "wrkcat": { - "type": "string" - } }, - "required": ["version", "wrkcat"], - "type": "object" + required: ['version', 'wrkcat'], + type: 'object', }, - "SourceTypeB": { - "properties": { - "version": { - "type": "number" + SourceTypeB: { + properties: { + version: { + type: 'number', + }, + wrkcat: { + type: 'string', }, - "wrkcat": { - "type": "string" - } }, - "required": ["version", "wrkcat"], - "type": "object" - } - } + required: ['version', 'wrkcat'], + type: 'object', + }, + }, }; const correctExternalSource = { - "events": [ + events: [ { - "attributes": { - "series": { - "iteration": 17, - "make": "alpha", - "type": "A", - } + attributes: { + series: { + iteration: 17, + make: 'alpha', + type: 'A', + }, }, - "duration": "02:00:00", - "event_type_name": "EventTypeA", - "key": "EventTypeA:1/1", - "start_time": "2024-01-01T01:35:00+00:00", + duration: '02:00:00', + event_type_name: 'EventTypeA', + key: 'EventTypeA:1/1', + start_time: '2024-01-01T01:35:00+00:00', }, { - "attributes": { - "series": { - "iteration": 21, - "make": "beta", - "type": "B" - } + attributes: { + series: { + iteration: 21, + make: 'beta', + type: 'B', + }, }, - "duration": "02:00:00", - "event_type_name": "EventTypeA", - "key": "EventTypeA:1/2", - "start_time": "2024-01-02T11:50:00+00:00", + duration: '02:00:00', + event_type_name: 'EventTypeA', + key: 'EventTypeA:1/2', + start_time: '2024-01-02T11:50:00+00:00', }, { - "attributes": { - "projectUser": "Jerry", - "tick": 18 + attributes: { + projectUser: 'Jerry', + tick: 18, }, - "duration": "03:40:00", - "event_type_name": "EventTypeB", - "key": "EventTypeB:1/3", - "start_time": "2024-01-03T15:20:00+00:00" - } + duration: '03:40:00', + event_type_name: 'EventTypeB', + key: 'EventTypeB:1/3', + start_time: '2024-01-03T15:20:00+00:00', + }, ], - "source": { - "attributes": { - "version": 1, - "wrkcat": "234" + source: { + attributes: { + version: 1, + wrkcat: '234', }, - "key": "SourceTypeA:valid_source_A.json", - "period": { - "end_time": "2024-01-07T00:00:00+00:00", - "start_time": "2024-01-01T00:00:00+00:00" + key: 'SourceTypeA:valid_source_A.json', + period: { + end_time: '2024-01-07T00:00:00+00:00', + start_time: '2024-01-01T00:00:00+00:00', }, - "source_type_name": "SourceTypeA", - "valid_at": "2024-01-01T00:00:00+00:00", - } + source_type_name: 'SourceTypeA', + valid_at: '2024-01-01T00:00:00+00:00', + }, }; const incorrectExternalSourceAttributes = { - "events": [ + events: [ { - "attributes": { - "series": { - "iteration": 17, - "make": "alpha", - "type": "A", - } + attributes: { + series: { + iteration: 17, + make: 'alpha', + type: 'A', + }, }, - "duration": "02:00:00", - "event_type_name": "EventTypeA", - "key": "EventTypeA:1/1", - "start_time": "2024-01-01T01:35:00+00:00", + duration: '02:00:00', + event_type_name: 'EventTypeA', + key: 'EventTypeA:1/1', + start_time: '2024-01-01T01:35:00+00:00', }, { - "attributes": { - "series": { - "iteration": 21, - "make": "beta", - "type": "B" - } + attributes: { + series: { + iteration: 21, + make: 'beta', + type: 'B', + }, }, - "duration": "02:00:00", - "event_type_name": "EventTypeA", - "key": "EventTypeA:1/2", - "start_time": "2024-01-02T11:50:00+00:00", + duration: '02:00:00', + event_type_name: 'EventTypeA', + key: 'EventTypeA:1/2', + start_time: '2024-01-02T11:50:00+00:00', }, { - "attributes": { - "projectUser": "Jerry", - "tick": 18 + attributes: { + projectUser: 'Jerry', + tick: 18, }, - "duration": "03:40:00", - "event_type_name": "EventTypeB", - "key": "EventTypeB:1/3", - "start_time": "2024-01-03T15:20:00+00:00" - } + duration: '03:40:00', + event_type_name: 'EventTypeB', + key: 'EventTypeB:1/3', + start_time: '2024-01-03T15:20:00+00:00', + }, ], - "source": { - "attributes": { - "version": 1, - "wrkcat": 234 // <-- wrong type. expecting string. + source: { + attributes: { + version: 1, + wrkcat: 234, // <-- wrong type. expecting string. }, - "key": "SourceTypeA:valid_source_A.json", - "period": { - "end_time": "2024-01-07T00:00:00+00:00", - "start_time": "2024-01-01T00:00:00+00:00" + key: 'SourceTypeA:valid_source_A.json', + period: { + end_time: '2024-01-07T00:00:00+00:00', + start_time: '2024-01-01T00:00:00+00:00', }, - "source_type_name": "SourceTypeA", - "valid_at": "2024-01-01T00:00:00+00:00", - } + source_type_name: 'SourceTypeA', + valid_at: '2024-01-01T00:00:00+00:00', + }, }; const incorrectExternalEventAttributes = { - "events": [ + events: [ { - "attributes": { - "series": { - "iteration": 17, - "make": "alpha", + attributes: { + series: { + iteration: 17, + make: 'alpha', // "type": "A", <-- missing. - } + }, }, - "duration": "02:00:00", - "event_type_name": "EventTypeA", - "key": "EventTypeA:1/1", - "start_time": "2024-01-01T01:35:00+00:00", + duration: '02:00:00', + event_type_name: 'EventTypeA', + key: 'EventTypeA:1/1', + start_time: '2024-01-01T01:35:00+00:00', }, { - "attributes": { - "series": { - "iteration": 21, - "make": "beta", - "type": "B" - } + attributes: { + series: { + iteration: 21, + make: 'beta', + type: 'B', + }, }, - "duration": "02:00:00", - "event_type_name": "EventTypeA", - "key": "EventTypeA:1/2", - "start_time": "2024-01-02T11:50:00+00:00", + duration: '02:00:00', + event_type_name: 'EventTypeA', + key: 'EventTypeA:1/2', + start_time: '2024-01-02T11:50:00+00:00', }, { - "attributes": { - "projectUser": "Jerry", - "tick": 18 + attributes: { + projectUser: 'Jerry', + tick: 18, }, - "duration": "03:40:00", - "event_type_name": "EventTypeB", - "key": "EventTypeB:1/3", - "start_time": "2024-01-03T15:20:00+00:00" - } + duration: '03:40:00', + event_type_name: 'EventTypeB', + key: 'EventTypeB:1/3', + start_time: '2024-01-03T15:20:00+00:00', + }, ], - "source": { - "attributes": { - "version": 1, - "wrkcat": "234" + source: { + attributes: { + version: 1, + wrkcat: '234', }, - "key": "SourceTypeA:valid_source_A.json", - "period": { - "end_time": "2024-01-07T00:00:00+00:00", - "start_time": "2024-01-01T00:00:00+00:00" + key: 'SourceTypeA:valid_source_A.json', + period: { + end_time: '2024-01-07T00:00:00+00:00', + start_time: '2024-01-01T00:00:00+00:00', }, - "source_type_name": "SourceTypeA", - "valid_at": "2024-01-01T00:00:00+00:00", - } + source_type_name: 'SourceTypeA', + valid_at: '2024-01-01T00:00:00+00:00', + }, }; - describe('validation tests', () => { - // test to verify source/event type file is correctly formatted test('verify source/event type file is correctly formatted', () => { // get the validator @@ -334,16 +332,16 @@ describe('validation tests', () => { const errors = attributeValidator.errors; expect(errors?.length).toBe(1); - expect(errors?.at(0)?.schemaPath).toBe("#/$defs/AttributeSchema/patternProperties/%5E.*%24/required"); + expect(errors?.at(0)?.schemaPath).toBe('#/$defs/AttributeSchema/patternProperties/%5E.*%24/required'); expect(errors?.at(0)?.message).toMatch("should have required property 'required'"); }); // test to verify that composition of a base schema with attribute schemas work test('verify validation functionality of updateSchemaWithDefs', () => { // transform attributeDefs to match something that might come from hasura (just ONE source type, as we will be constructing a schema for a specific source) - const attributeSchema: { event_types: any, source_type: any } = { + const attributeSchema: { event_types: any; source_type: any } = { event_types: [], - source_type: {} + source_type: {}, }; attributeSchema.event_types = attributeDefs.event_types; attributeSchema.source_type['SourceTypeA'] = attributeDefs.source_types.SourceTypeA; @@ -355,19 +353,20 @@ describe('validation tests', () => { if (schema) { // verify it is formatted correctly - expect(Object.keys(schema.$defs.event_types)).toMatchObject(["EventTypeA", "EventTypeB", "EventTypeC"]); - expect(Object.keys(schema.$defs.source_type)).toMatchObject(["SourceTypeA"]); - expect(schema.properties.events.items.else.else.properties.attributes.$ref).toEqual("#/$defs/event_types/EventTypeC"); + expect(Object.keys(schema.$defs.event_types)).toMatchObject(['EventTypeA', 'EventTypeB', 'EventTypeC']); + expect(Object.keys(schema.$defs.source_type)).toMatchObject(['SourceTypeA']); + expect(schema.properties.events.items.else.else.properties.attributes.$ref).toEqual( + '#/$defs/event_types/EventTypeC', + ); } }); - // source testing describe('validating (and failing) sources', () => { // transform attributeDefs to match something that might come from hasura (just ONE source type, as we will be constructing a schema for a specific source) - const attributeSchema: { event_types: any, source_type: any } = { + const attributeSchema: { event_types: any; source_type: any } = { event_types: [], - source_type: {} + source_type: {}, }; attributeSchema.event_types = attributeDefs.event_types; attributeSchema.source_type['SourceTypeA'] = attributeDefs.source_types.SourceTypeA; @@ -389,8 +388,8 @@ describe('validation tests', () => { const errors = schemaFunctionWithDefs.errors; expect(errors?.length).toBe(1); - expect(errors?.at(0)?.schemaPath).toBe("#/$defs/source_type/SourceTypeA/properties/wrkcat/type"); - expect(errors?.at(0)?.message).toMatch("should be string"); + expect(errors?.at(0)?.schemaPath).toBe('#/$defs/source_type/SourceTypeA/properties/wrkcat/type'); + expect(errors?.at(0)?.message).toMatch('should be string'); }); // test to verify an event's attributes are incorrectly formatted @@ -400,7 +399,7 @@ describe('validation tests', () => { const errors = schemaFunctionWithDefs.errors; expect(errors?.length).toBe(1); - expect(errors?.at(0)?.schemaPath).toBe("#/$defs/event_types/EventTypeA/properties/series/required"); + expect(errors?.at(0)?.schemaPath).toBe('#/$defs/event_types/EventTypeA/properties/series/required'); expect(errors?.at(0)?.message).toMatch("should have required property 'type'"); }); }); From 9f90170aeeabcf303c612f46cc0b89bfeb916051 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Tue, 3 Dec 2024 13:06:30 -0500 Subject: [PATCH 30/39] Formatting --- .../external-source/external-source.ts | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index a041461..2b9a570 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -72,7 +72,6 @@ export function updateSchemaWithDefs(defs: { event_types: any; source_type: any // handle n event types for (let i = 0; i < keys.length - 1; i++) { const key = keys[i]; - console.log('NOW ON:', key); ifThenElsePointer['if'] = { properties: { event_type_name: { @@ -237,13 +236,11 @@ async function uploadExternalSource(req: Request, res: Response) { e as Error }`, ); - res - .status(500) - .send({ - message: `Body of request must be a JSON, with two stringified properties: "source" and "events". Alternatively, parsing may have failed:\n${ - e as Error - }`, - }); + res.status(500).send({ + message: `Body of request must be a JSON, with two stringified properties: "source" and "events". Alternatively, parsing may have failed:\n${ + e as Error + }`, + }); return; } const { attributes, derivation_group_name, key, period, source_type_name, valid_at } = parsedSource; @@ -329,13 +326,11 @@ async function uploadExternalSource(req: Request, res: Response) { compiledExternalSourceMegaschema.errors, )}`, ); - res - .status(500) - .send({ - message: `External Source ${key}'s formatting is invalid:\n${JSON.stringify( - compiledExternalSourceMegaschema.errors, - )}`, - }); + res.status(500).send({ + message: `External Source ${key}'s formatting is invalid:\n${JSON.stringify( + compiledExternalSourceMegaschema.errors, + )}`, + }); return; } From d25e67a74ef7fac3237e817a337d3820dd8a5aae Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Tue, 3 Dec 2024 14:15:11 -0500 Subject: [PATCH 31/39] Formatting, cleanup --- .../external-source/external-source.ts | 54 ++++++++----------- src/packages/external-source/gql.ts | 29 +++++----- src/types/external-source.ts | 42 +++++---------- src/util/time.ts | 21 +------- 4 files changed, 50 insertions(+), 96 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 2b9a570..ea01519 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -3,12 +3,13 @@ import type { ExternalSourceTypeInsertInput, CreateExternalSourceResponse, ExternalEventTypeInsertInput, - ExternalEvent, CreateExternalSourceEventTypeResponse, GetSourceEventTypeAttributeSchemasResponse, AttributeSchema, DerivationGroupInsertInput, ExternalSourceInsertInput, + ExternalSourceRequest, + ExternalEventRequest, } from '../../types/external-source.js'; import Ajv from 'ajv'; import { getEnv } from '../../env.js'; @@ -34,44 +35,40 @@ const refreshLimiter = rateLimit({ }); export function updateSchemaWithDefs(defs: { event_types: any; source_type: any }): Ajv.ValidateFunction { - // build if statement + // Build if statement const ifThenElse: { [key: string]: any } = {}; let ifThenElsePointer = ifThenElse; const keys = Object.keys(defs.event_types); - // handling if there's only 1 event type (don't bother with $defs, just update attributes' properties directly) + // Handle single event type (don't bother with $defs, just update attributes' properties directly) if (keys.length === 1) { - // no need for ifThenElse, simply create localSchemaCopy and update properties.events.items.properties.attributes - // to match the event type in defs, and verify the event_type_name matches the def name const localSchemaCopy = structuredClone(baseExternalSourceSchema); + const event_type_name = keys[0]; const event_type_schema = { ...defs.event_types[event_type_name], - - // additionally, restrict extra properties additionalProperties: false, }; const source_type_name = Object.keys(defs.source_type)[0]; const source_type_schema = { ...defs.source_type[source_type_name], - - // additionally, restrict extra properties additionalProperties: false, }; localSchemaCopy.properties.events.items.properties.attributes = event_type_schema; localSchemaCopy.properties.events.items.properties.event_type_name = { const: event_type_name }; - // insert def for "source" attributes + // Insert def for "source" attributes localSchemaCopy.properties.source.properties.attributes = source_type_schema; const localAjv = new Ajv(); return localAjv.compile(localSchemaCopy); } - // handle n event types + // HJandle n event types for (let i = 0; i < keys.length - 1; i++) { const key = keys[i]; + // Create tree of if/else/then statements to support validating different types ifThenElsePointer['if'] = { properties: { event_type_name: { @@ -90,7 +87,6 @@ export function updateSchemaWithDefs(defs: { event_types: any; source_type: any ifThenElsePointer = ifThenElsePointer['else']; } - // fill in the final else with the last element const key = keys[keys.length - 1]; ifThenElsePointer['properties'] = { attributes: { @@ -98,24 +94,21 @@ export function updateSchemaWithDefs(defs: { event_types: any; source_type: any }, }; - // insert if statement into local copy of baseExternalSourceSchema const localSchemaCopy = structuredClone(baseExternalSourceSchema); localSchemaCopy.properties.events.items['if'] = ifThenElse['if']; localSchemaCopy.properties.events.items['then'] = ifThenElse['then']; localSchemaCopy.properties.events.items['else'] = ifThenElse['else']; - // insert def for "source" attributes + // Insert def for "source" attributes const sourceTypeKey = Object.keys(defs.source_type)[0]; localSchemaCopy.properties.source.properties.attributes = { $ref: `#/$defs/source_type/${sourceTypeKey}` }; - // add defs + // Add defs localSchemaCopy.$defs = { event_types: {}, source_type: { [sourceTypeKey]: { ...defs.source_type[sourceTypeKey], - - // additionally, restrict extra properties additionalProperties: false, }, }, @@ -123,8 +116,6 @@ export function updateSchemaWithDefs(defs: { event_types: any; source_type: any for (const event_type of keys) { localSchemaCopy.$defs.event_types[event_type] = { ...defs.event_types[event_type], - - // additionally, restrict extra properties additionalProperties: false, }; } @@ -199,19 +190,21 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { method: 'POST', }); - const jsonResponse = await response.json(); - if (jsonResponse?.data !== undefined) { - res.json(jsonResponse.data as CreateExternalSourceEventTypeResponse); + const createExternalSourceEventTypesResponse = await response.json(); + if (createExternalSourceEventTypesResponse?.data !== undefined) { + res.json(createExternalSourceEventTypesResponse.data as CreateExternalSourceEventTypeResponse); } else { - res.json(jsonResponse as HasuraError); + res.json(createExternalSourceEventTypesResponse as HasuraError); } } async function uploadExternalSource(req: Request, res: Response) { const authorizationHeader = req.get('authorization'); + const { headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, } = req; + const { body } = req; if (typeof body !== 'object') { @@ -224,8 +217,8 @@ async function uploadExternalSource(req: Request, res: Response) { return; } - let parsedSource; - let parsedExternalEvents: ExternalEvent[]; + let parsedSource: ExternalSourceRequest; + let parsedExternalEvents: ExternalEventRequest[]; try { const { source, events } = body; parsedSource = JSON.parse(source); @@ -316,8 +309,7 @@ async function uploadExternalSource(req: Request, res: Response) { }); // Verify that this is a valid external source - let sourceIsValid: boolean = false; - sourceIsValid = await compiledExternalSourceMegaschema(externalSourceJson); + const sourceIsValid: boolean = await compiledExternalSourceMegaschema(externalSourceJson); if (sourceIsValid) { logger.info(`POST /uploadExternalSource: External Source ${key}'s formatting is valid`); } else { @@ -365,11 +357,11 @@ async function uploadExternalSource(req: Request, res: Response) { method: 'POST', }); - const jsonResponse = await response.json(); - if (jsonResponse?.data !== undefined) { - res.json(jsonResponse.data as CreateExternalSourceResponse); + const createExternalSourceResponse = await response.json(); + if (createExternalSourceResponse?.data !== undefined) { + res.json(createExternalSourceResponse.data as CreateExternalSourceResponse); } else { - res.json(jsonResponse as HasuraError); + res.json(createExternalSourceResponse as HasuraError); } } diff --git a/src/packages/external-source/gql.ts b/src/packages/external-source/gql.ts index a328b1f..663b46f 100644 --- a/src/packages/external-source/gql.ts +++ b/src/packages/external-source/gql.ts @@ -1,5 +1,4 @@ export default { - // TODO: discuss upset for derivation group CREATE_EXTERNAL_SOURCE: `#graphql mutation CreateExternalSource( $derivation_group: derivation_group_insert_input!, @@ -28,7 +27,7 @@ export default { } `, CREATE_EXTERNAL_SOURCE_EVENT_TYPES: `#graphql - mutation uploadAttributeSchemas($externalEventTypes: [external_event_type_insert_input!]!, $externalSourceTypes: [external_source_type_insert_input!]!) { + mutation UploadAttributeSchemas($externalEventTypes: [external_event_type_insert_input!]!, $externalSourceTypes: [external_source_type_insert_input!]!) { createExternalEventTypes: insert_external_event_type(objects: $externalEventTypes) { returning { name @@ -49,11 +48,11 @@ export default { } `, GET_EXTERNAL_EVENT_TYPE_ATTRIBUTE_SCHEMA: `#graphql - query GetExternalEventTypeAttributeSchema($name: String!) { - external_event_type_by_pk(name: $name) { - attribute_schema + query GetExternalEventTypeAttributeSchema($name: String!) { + external_event_type_by_pk(name: $name) { + attribute_schema + } } - } `, GET_EXTERNAL_SOURCE_TYPE_ATTRIBUTE_SCHEMA: `#graphql query GetExternalSourceTypeAttributeSchema($name: String!) { @@ -63,15 +62,15 @@ export default { } `, GET_SOURCE_EVENT_TYPE_ATTRIBUTE_SCHEMAS: `#graphql - query getET($externalEventTypes: [String!]!, $externalSourceType: String!) { - external_event_type(where: {name: {_in: $externalEventTypes}}) { - name - attribute_schema - } - external_source_type(where: {name: {_eq: $externalSourceType}}) { - name - attribute_schema + query GetSourceEventTypeAttributeSchemas($externalEventTypes: [String!]!, $externalSourceType: String!) { + external_event_type(where: {name: {_in: $externalEventTypes}}) { + name + attribute_schema + } + external_source_type(where: {name: {_eq: $externalSourceType}}) { + name + attribute_schema + } } - } `, }; diff --git a/src/types/external-source.ts b/src/types/external-source.ts index 2f20c34..a66d18d 100644 --- a/src/types/external-source.ts +++ b/src/types/external-source.ts @@ -31,20 +31,6 @@ export type ExternalSourceInsertInput = { valid_at: string; }; -export type UploadExternalSourceJSON = { - events: ExternalEventJson[]; - source: { - attributes: object; - key: string; - period: { - end_time: string; - start_time: string; - }; - source_type: string; - valid_at: string; - }; -}; - export type CreateExternalSourceResponse = { createExternalSource: { name: string }; }; @@ -54,28 +40,24 @@ export type CreateExternalSourceEventTypeResponse = { createExternalSourceTypes: { returning: string[] }; }; -export type ExternalEventInsertInput = { - attributes: object; +export type ExternalEventRequest = { + key: string; + event_type_name: string; start_time: string; duration: string; - event_type_name: string; - key: string; -}; - -export type ExternalEventJson = { attributes: object; - duration: string; - event_type: string; - key: string; - start_time: string; }; -export type ExternalEvent = { - key: string; - event_type_name: string; - start_time: string; - duration: string; +export type ExternalSourceRequest = { attributes: object; + derivation_group_name: string; + key: string; + period: { + end_time: string; + start_time: string; + }; + source_type_name: string; + valid_at: string; }; export type AttributeSchema = { diff --git a/src/util/time.ts b/src/util/time.ts index 1b508ee..197a302 100644 --- a/src/util/time.ts +++ b/src/util/time.ts @@ -1,5 +1,4 @@ import { ParsedDoyString, ParsedYmdString } from '../types/time'; -import parseInterval from 'postgres-interval'; function parseNumber(number: number | string): number { return parseInt(`${number}`, 10); @@ -93,7 +92,7 @@ export function convertDateToDoy(dateString: string, numDecimals = 6): string | return null; } -export function convertDoyToYmd(doyString: string, numDecimals = 6, includeMsecs = true): string | null { +function convertDoyToYmd(doyString: string, numDecimals = 6, includeMsecs = true): string | null { const parsedDoy: ParsedDoyString = parseDoyOrYmdTime(doyString, numDecimals) as ParsedDoyString; if (parsedDoy !== null) { @@ -129,21 +128,3 @@ export function getTimeDifference(dateString1: string, dateString2: string, numD } return null; } - -/** - * Returns a Postgres Interval duration in milliseconds. - * If duration is null, undefined, or empty string then we just return 0. - * @note This function assumes 24-hour days. - */ -export function getIntervalInMs(interval: string | null | undefined): number { - if (interval !== null && interval !== undefined && interval !== '') { - const parsedInterval = parseInterval(interval); - const { days, hours, milliseconds, minutes, seconds } = parsedInterval; - const daysInMs = days * 24 * 60 * 60 * 1000; - const hoursInMs = hours * 60 * 60 * 1000; - const minutesInMs = minutes * 60 * 1000; - const secondsInMs = seconds * 1000; - return daysInMs + hoursInMs + minutesInMs + secondsInMs + milliseconds; - } - return 0; -} From 281501cf484289eba188dfe1a4bd52c8fd045ad4 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Thu, 5 Dec 2024 09:46:20 -0500 Subject: [PATCH 32/39] Fix name casing --- .../external-source/external-source.ts | 45 +++++++++---------- ....ts => external-source.validation.test.ts} | 0 2 files changed, 22 insertions(+), 23 deletions(-) rename test/{external_source.validation.test.ts => external-source.validation.test.ts} (100%) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index ea01519..115877a 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -23,8 +23,7 @@ import multer from 'multer'; const upload = multer({ limits: { fieldSize: 25 * 1024 * 1024 } }); const logger = getLogger('packages/external-source/external-source'); -const { RATE_LIMITER_LOGIN_MAX, HASURA_API_URL } = getEnv(); -const GQL_API_URL = `${HASURA_API_URL}/v1/graphql`; +const { RATE_LIMITER_LOGIN_MAX, GQL_API_URL } = getEnv(); const ajv = new Ajv(); const compiledAttributeMetaschema = ajv.compile(attributeSchemaMetaschema); const refreshLimiter = rateLimit({ @@ -44,28 +43,28 @@ export function updateSchemaWithDefs(defs: { event_types: any; source_type: any if (keys.length === 1) { const localSchemaCopy = structuredClone(baseExternalSourceSchema); - const event_type_name = keys[0]; - const event_type_schema = { - ...defs.event_types[event_type_name], + const eventTypeName = keys[0]; + const eventTypeSchema = { + ...defs.event_types[eventTypeName], additionalProperties: false, }; - const source_type_name = Object.keys(defs.source_type)[0]; - const source_type_schema = { - ...defs.source_type[source_type_name], + const sourceTypeName = Object.keys(defs.source_type)[0]; + const sourceTypeSchema = { + ...defs.source_type[sourceTypeName], additionalProperties: false, }; - localSchemaCopy.properties.events.items.properties.attributes = event_type_schema; - localSchemaCopy.properties.events.items.properties.event_type_name = { const: event_type_name }; + localSchemaCopy.properties.events.items.properties.attributes = eventTypeSchema; + localSchemaCopy.properties.events.items.properties.event_type_name = { const: eventTypeName }; // Insert def for "source" attributes - localSchemaCopy.properties.source.properties.attributes = source_type_schema; + localSchemaCopy.properties.source.properties.attributes = sourceTypeSchema; const localAjv = new Ajv(); return localAjv.compile(localSchemaCopy); } - // HJandle n event types + // Handle n event types for (let i = 0; i < keys.length - 1; i++) { const key = keys[i]; // Create tree of if/else/then statements to support validating different types @@ -113,9 +112,9 @@ export function updateSchemaWithDefs(defs: { event_types: any; source_type: any }, }, }; - for (const event_type of keys) { - localSchemaCopy.$defs.event_types[event_type] = { - ...defs.event_types[event_type], + for (const eventType of keys) { + localSchemaCopy.$defs.event_types[eventType] = { + ...defs.event_types[eventType], additionalProperties: false, }; } @@ -164,19 +163,19 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { const externalSourceTypeInput: ExternalSourceTypeInsertInput[] = []; const externalEventTypeInput: ExternalEventTypeInsertInput[] = []; - const event_type_keys = Object.keys(parsedEventTypes); - for (const external_event_type of event_type_keys) { + const eventTypeKeys = Object.keys(parsedEventTypes); + for (const externalEventType of eventTypeKeys) { externalEventTypeInput.push({ - attribute_schema: parsedEventTypes[external_event_type], - name: external_event_type, + attribute_schema: parsedEventTypes[externalEventType], + name: externalEventType, }); } - const source_type_keys = Object.keys(parsedSourceTypes); - for (const external_source_type of source_type_keys) { + const sourceTypeKeys = Object.keys(parsedSourceTypes); + for (const externalSourceType of sourceTypeKeys) { externalSourceTypeInput.push({ - attribute_schema: parsedSourceTypes[external_source_type], - name: external_source_type, + attribute_schema: parsedSourceTypes[externalSourceType], + name: externalSourceType, }); } diff --git a/test/external_source.validation.test.ts b/test/external-source.validation.test.ts similarity index 100% rename from test/external_source.validation.test.ts rename to test/external-source.validation.test.ts From b28cf0c15bd30aa40355c4bc37a7ce4139b0cbe8 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Mon, 9 Dec 2024 09:36:39 -0500 Subject: [PATCH 33/39] e as Error => (e as Error).message --- src/packages/external-source/external-source.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 115877a..350b248 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -225,12 +225,12 @@ async function uploadExternalSource(req: Request, res: Response) { } catch (e) { logger.error( `POST /uploadExternalSourceEventTypes: Body of request must be a JSON, with two stringified properties: "source" and "events". Alternatively, parsing may have failed:\n${ - e as Error + (e as Error).message }`, ); res.status(500).send({ message: `Body of request must be a JSON, with two stringified properties: "source" and "events". Alternatively, parsing may have failed:\n${ - e as Error + (e as Error).message }`, }); return; @@ -249,6 +249,7 @@ async function uploadExternalSource(req: Request, res: Response) { valid_at: valid_at, }, }; + console.log(period); const headers: HeadersInit = { Authorization: authorizationHeader ?? '', @@ -282,7 +283,7 @@ async function uploadExternalSource(req: Request, res: Response) { logger.error( `POST /uploadExternalSourceEventTypes: The source and event types in your source do not exist in the database.`, ); - res.status(500).send({ message: `The source and event types in your source do not exist in the database.` }); + res.status(500).send({ message: `The source type and event types in your source do not exist in the database.` }); return; } From ca924b150dcf50022b64c38822558b473058ddd8 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Mon, 9 Dec 2024 09:36:56 -0500 Subject: [PATCH 34/39] Fix improper time regex for schema --- src/packages/schemas/external-event-validation-schemata.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/packages/schemas/external-event-validation-schemata.ts b/src/packages/schemas/external-event-validation-schemata.ts index 7c52046..6a996ca 100644 --- a/src/packages/schemas/external-event-validation-schemata.ts +++ b/src/packages/schemas/external-event-validation-schemata.ts @@ -77,12 +77,12 @@ export const baseExternalSourceSchema: { [key: string]: any } = { properties: { end_time: { pattern: - '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', + '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-2][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', type: 'string', }, start_time: { pattern: - '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', + '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-2][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', type: 'string', }, }, @@ -92,7 +92,7 @@ export const baseExternalSourceSchema: { [key: string]: any } = { source_type_name: { type: 'string' }, valid_at: { pattern: - '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-1][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', + '^(\\d){4}-([0-3][0-9])-([0-9][0-9])T([0-2][0-9]):([0-5][0-9]):([0-5][0-9])(\\+|-)([0-1][0-9]):([0-5][0-9])$', type: 'string', }, }, From 083059ebc44272431dc085ae04fe458c010c78b9 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Mon, 9 Dec 2024 09:53:21 -0500 Subject: [PATCH 35/39] Move schemas => src/ --- src/packages/external-source/external-source.ts | 2 +- .../schemas/external-event-validation-schemata.ts | 0 test/external-source.validation.test.ts | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/{packages => }/schemas/external-event-validation-schemata.ts (100%) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 350b248..a8f0c88 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -15,7 +15,7 @@ import Ajv from 'ajv'; import { getEnv } from '../../env.js'; import getLogger from '../../logger.js'; import gql from './gql.js'; -import { attributeSchemaMetaschema, baseExternalSourceSchema } from '../schemas/external-event-validation-schemata.js'; +import { attributeSchemaMetaschema, baseExternalSourceSchema } from '../../schemas/external-event-validation-schemata.js'; import { HasuraError } from '../../types/hasura.js'; import { auth } from '../auth/middleware.js'; import rateLimit from 'express-rate-limit'; diff --git a/src/packages/schemas/external-event-validation-schemata.ts b/src/schemas/external-event-validation-schemata.ts similarity index 100% rename from src/packages/schemas/external-event-validation-schemata.ts rename to src/schemas/external-event-validation-schemata.ts diff --git a/test/external-source.validation.test.ts b/test/external-source.validation.test.ts index 0c1ce9b..27d7d3a 100644 --- a/test/external-source.validation.test.ts +++ b/test/external-source.validation.test.ts @@ -1,6 +1,6 @@ import Ajv from 'ajv'; import { describe, expect, test } from 'vitest'; -import { attributeSchemaMetaschema } from '../src/packages/schemas/external-event-validation-schemata'; +import { attributeSchemaMetaschema } from '../src/schemas/external-event-validation-schemata'; import { updateSchemaWithDefs } from '../src/packages/external-source/external-source'; const ajv = Ajv(); From 275d5fd7e6327fa1663bf25a18e4455301c68bd6 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Tue, 10 Dec 2024 09:48:36 -0500 Subject: [PATCH 36/39] Add ajv to package.json --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 1c6116c..d692d20 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ }, "dependencies": { "@streamparser/json": "^0.0.21", + "ajv": "^6.10.0", "altair-express-middleware": "^5.2.11", "cookie-parser": "^1.4.6", "cors": "^2.8.5", From d591d2a83948b40014bed103f40cbb3d41802f38 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Tue, 10 Dec 2024 16:32:45 -0500 Subject: [PATCH 37/39] Cleanup from comments --- .../external-source/external-source.ts | 131 +++++++++--------- 1 file changed, 67 insertions(+), 64 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index a8f0c88..1cb3da4 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -128,11 +128,9 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { const authorizationHeader = req.get('authorization'); const { + body: {event_types, source_types}, headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, } = req; - - const { body } = req; - const { event_types, source_types } = body; const parsedEventTypes: { [x: string]: object } = JSON.parse(event_types); const parsedSourceTypes: { [x: string]: object } = JSON.parse(source_types); @@ -151,9 +149,9 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { source_types: parsedSourceTypes, }); if (!schemasAreValid) { - logger.error(`POST /uploadExternalSourceEventTypes: Schema validation failed for uploaded source and event types.`); - compiledAttributeMetaschema.errors?.forEach(error => logger.error(error)); - res.status(500).send({ message: compiledAttributeMetaschema.errors }); + const errorMsg = `Schema validation failed for uploaded source and event types:\n${JSON.stringify(compiledAttributeMetaschema.errors)}`; + logger.error(`POST /uploadExternalSourceEventTypes: ${errorMsg}`); + res.status(500).send({ message: errorMsg }); return; } @@ -201,37 +199,23 @@ async function uploadExternalSource(req: Request, res: Response) { const authorizationHeader = req.get('authorization'); const { + body: { source, events }, headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, } = req; - const { body } = req; - - if (typeof body !== 'object') { - logger.error( - `POST /uploadExternalSourceEventTypes: Body of request must be a JSON, with two stringified properties: "source" and "events".`, - ); - res - .status(500) - .send({ message: `Body of request must be a JSON, with two stringified properties: "source" and "events".` }); - return; - } - let parsedSource: ExternalSourceRequest; let parsedExternalEvents: ExternalEventRequest[]; + try { - const { source, events } = body; parsedSource = JSON.parse(source); parsedExternalEvents = JSON.parse(events); } catch (e) { + const errorMsg = `Body of request must be a JSON, with two stringified properties: "source" and "events". Alternatively, parsing may have failed:\n${(e as Error).message}`; logger.error( - `POST /uploadExternalSourceEventTypes: Body of request must be a JSON, with two stringified properties: "source" and "events". Alternatively, parsing may have failed:\n${ - (e as Error).message - }`, + `POST /uploadExternalSourceEventTypes: ${errorMsg}`, ); res.status(500).send({ - message: `Body of request must be a JSON, with two stringified properties: "source" and "events". Alternatively, parsing may have failed:\n${ - (e as Error).message - }`, + message: errorMsg, }); return; } @@ -249,7 +233,6 @@ async function uploadExternalSource(req: Request, res: Response) { valid_at: valid_at, }, }; - console.log(period); const headers: HeadersInit = { Authorization: authorizationHeader ?? '', @@ -280,10 +263,11 @@ async function uploadExternalSource(req: Request, res: Response) { attributeSchemaJson.data as GetSourceEventTypeAttributeSchemasResponse; if (external_event_type.length === 0 || external_source_type.length === 0) { + const errorMsg = 'The source and/or event types in your source do not exist in the database.'; logger.error( - `POST /uploadExternalSourceEventTypes: The source and event types in your source do not exist in the database.`, + `POST /uploadExternalSourceEventTypes: ${errorMsg}`, ); - res.status(500).send({ message: `The source type and event types in your source do not exist in the database.` }); + res.status(500).send({ message: errorMsg }); return; } @@ -313,16 +297,9 @@ async function uploadExternalSource(req: Request, res: Response) { if (sourceIsValid) { logger.info(`POST /uploadExternalSource: External Source ${key}'s formatting is valid`); } else { - logger.error( - `POST /uploadExternalSource: External Source ${key}'s formatting is invalid:\n${JSON.stringify( - compiledExternalSourceMegaschema.errors, - )}`, - ); - res.status(500).send({ - message: `External Source ${key}'s formatting is invalid:\n${JSON.stringify( - compiledExternalSourceMegaschema.errors, - )}`, - }); + const errorMsg = `External Source ${key}'s formatting is invalid:\n${JSON.stringify(compiledExternalSourceMegaschema.errors)}` + logger.error(`POST /uploadExternalSource: ${errorMsg}`); + res.status(500).send({ message: errorMsg }); return; } @@ -396,7 +373,7 @@ export default (app: Express) => { * type: object * required: * - event_types - * source_types + * - source_types * responses: * 200: * description: Created External Source & Event Types @@ -442,36 +419,62 @@ export default (app: Express) => { * schema: * type: object * properties: - * attributes: - * type: object - * derivation_group_name: - * type: string - * end_time: - * type: string * events: + * description: A list of External Events to be uploaded that are contained within the External Source. + * type: array + * items: + * type: object + * properties: + * attributes: + * type: object + * duration: + * type: string + * event_type_name: + * type: string + * key: + * type: string + * start_time: + * type: string + * required: + * - attributes + * - duration + * - event_type_name + * - key + * - start_time + * source: + * description: An object representing the External Source to be uploaded. * type: object * properties: - * data: - * type: array + * attributes: + * type: object + * derivation_group_name: + * type: string + * key: + * type: string + * period: + * type: object + * properties: + * end_time: + * type: string + * start_time: + * type: string + * required: + * - end_time + * start_time + * source_type_name: + * type: string + * valid_at: + * type: string * required: - * - data - * key: - * type: string - * source_type_name: - * type: string - * start_time: - * type: string - * valid_at: - * type: string + * - attributes + * - derivation_group_name + * - key + * - period + * - source_type_name + * - valid_at * required: - * - attributes - * derivation_group_name - * end_time - * events - * key - * source_type_name - * start_time - * valid_at + * - events + * - source * responses: * 200: * description: Created External Source From 1c7c735844b997e055b402fbaef13c664c5bfc6b Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Tue, 10 Dec 2024 16:34:18 -0500 Subject: [PATCH 38/39] Lint fix --- .../external-source/external-source.ts | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 1cb3da4..90a5435 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -15,7 +15,10 @@ import Ajv from 'ajv'; import { getEnv } from '../../env.js'; import getLogger from '../../logger.js'; import gql from './gql.js'; -import { attributeSchemaMetaschema, baseExternalSourceSchema } from '../../schemas/external-event-validation-schemata.js'; +import { + attributeSchemaMetaschema, + baseExternalSourceSchema, +} from '../../schemas/external-event-validation-schemata.js'; import { HasuraError } from '../../types/hasura.js'; import { auth } from '../auth/middleware.js'; import rateLimit from 'express-rate-limit'; @@ -128,7 +131,7 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { const authorizationHeader = req.get('authorization'); const { - body: {event_types, source_types}, + body: { event_types, source_types }, headers: { 'x-hasura-role': roleHeader, 'x-hasura-user-id': userHeader }, } = req; const parsedEventTypes: { [x: string]: object } = JSON.parse(event_types); @@ -149,7 +152,9 @@ async function uploadExternalSourceEventTypes(req: Request, res: Response) { source_types: parsedSourceTypes, }); if (!schemasAreValid) { - const errorMsg = `Schema validation failed for uploaded source and event types:\n${JSON.stringify(compiledAttributeMetaschema.errors)}`; + const errorMsg = `Schema validation failed for uploaded source and event types:\n${JSON.stringify( + compiledAttributeMetaschema.errors, + )}`; logger.error(`POST /uploadExternalSourceEventTypes: ${errorMsg}`); res.status(500).send({ message: errorMsg }); return; @@ -210,10 +215,10 @@ async function uploadExternalSource(req: Request, res: Response) { parsedSource = JSON.parse(source); parsedExternalEvents = JSON.parse(events); } catch (e) { - const errorMsg = `Body of request must be a JSON, with two stringified properties: "source" and "events". Alternatively, parsing may have failed:\n${(e as Error).message}`; - logger.error( - `POST /uploadExternalSourceEventTypes: ${errorMsg}`, - ); + const errorMsg = `Body of request must be a JSON, with two stringified properties: "source" and "events". Alternatively, parsing may have failed:\n${ + (e as Error).message + }`; + logger.error(`POST /uploadExternalSourceEventTypes: ${errorMsg}`); res.status(500).send({ message: errorMsg, }); @@ -264,9 +269,7 @@ async function uploadExternalSource(req: Request, res: Response) { if (external_event_type.length === 0 || external_source_type.length === 0) { const errorMsg = 'The source and/or event types in your source do not exist in the database.'; - logger.error( - `POST /uploadExternalSourceEventTypes: ${errorMsg}`, - ); + logger.error(`POST /uploadExternalSourceEventTypes: ${errorMsg}`); res.status(500).send({ message: errorMsg }); return; } @@ -297,7 +300,9 @@ async function uploadExternalSource(req: Request, res: Response) { if (sourceIsValid) { logger.info(`POST /uploadExternalSource: External Source ${key}'s formatting is valid`); } else { - const errorMsg = `External Source ${key}'s formatting is invalid:\n${JSON.stringify(compiledExternalSourceMegaschema.errors)}` + const errorMsg = `External Source ${key}'s formatting is invalid:\n${JSON.stringify( + compiledExternalSourceMegaschema.errors, + )}`; logger.error(`POST /uploadExternalSource: ${errorMsg}`); res.status(500).send({ message: errorMsg }); return; From 5348546ab127d18101866dcb6f3195d930a79d59 Mon Sep 17 00:00:00 2001 From: JosephVolosin Date: Wed, 11 Dec 2024 16:56:12 -0500 Subject: [PATCH 39/39] Make GQL_API_URL manually --- src/packages/external-source/external-source.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/packages/external-source/external-source.ts b/src/packages/external-source/external-source.ts index 90a5435..55cd631 100644 --- a/src/packages/external-source/external-source.ts +++ b/src/packages/external-source/external-source.ts @@ -26,7 +26,8 @@ import multer from 'multer'; const upload = multer({ limits: { fieldSize: 25 * 1024 * 1024 } }); const logger = getLogger('packages/external-source/external-source'); -const { RATE_LIMITER_LOGIN_MAX, GQL_API_URL } = getEnv(); +const { RATE_LIMITER_LOGIN_MAX, HASURA_API_URL } = getEnv(); +const GQL_API_URL = `${HASURA_API_URL}/v1/graphql`; const ajv = new Ajv(); const compiledAttributeMetaschema = ajv.compile(attributeSchemaMetaschema); const refreshLimiter = rateLimit({