From 4bb151f8f5502d8cd1a25c458983a4f73ef4eb1a Mon Sep 17 00:00:00 2001 From: Ivan Polomani Date: Wed, 25 Sep 2024 13:22:43 +0200 Subject: [PATCH 01/15] Add backend query code path --- src/__tests__/datasource.test.ts | 4 +- src/datasource.ts | 168 ++++++++++++++++++++++--------- src/types.ts | 2 + 3 files changed, 125 insertions(+), 49 deletions(-) diff --git a/src/__tests__/datasource.test.ts b/src/__tests__/datasource.test.ts index cc9b880b..9136346d 100644 --- a/src/__tests__/datasource.test.ts +++ b/src/__tests__/datasource.test.ts @@ -11,6 +11,7 @@ import { } from '../test_utils'; import { failedResponseEvent } from '../constants'; import { eventBusService } from '../appEventHandler'; +import { lastValueFrom } from 'rxjs'; jest.mock('@grafana/data'); type Mock = jest.Mock; @@ -716,10 +717,11 @@ describe('Given custom query with pure text label', () => { expr: 'ts{id=1}', label: 'Pure text', }; - const result = await ds.query({ + const resultObs = await ds.query({ ...options, targets: [targetA], }); + const result = await lastValueFrom(resultObs); expect((result.data[0] as TimeSeries).target).toEqual('Pure text'); }); }); diff --git a/src/datasource.ts b/src/datasource.ts index f3a8349f..2f48469c 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -10,7 +10,7 @@ import { MutableDataFrame, AnnotationQuery, } from '@grafana/data'; -import { BackendSrv, BackendSrvRequest, getBackendSrv, getTemplateSrv, TemplateSrv } from '@grafana/runtime'; +import { BackendSrv, BackendSrvRequest, DataSourceWithBackend, getBackendSrv, getTemplateSrv, TemplateSrv } from '@grafana/runtime'; import _ from 'lodash'; import { fetchSingleAsset, fetchSingleTimeseries } from './cdf/client'; import { @@ -43,12 +43,12 @@ import { ExtractionPipelinesDatasource, } from './datasources'; import AnnotationsQueryEditor from 'components/annotationsQueryEditor'; -import { lastValueFrom } from 'rxjs'; +import { lastValueFrom, Observable, from, map } from 'rxjs'; -export default class CogniteDatasource extends DataSourceApi< +export default class CogniteDatasource extends DataSourceWithBackend< CogniteQuery, - CogniteDataSourceOptions, - AnnotationQuery + CogniteDataSourceOptions//, + // AnnotationQuery > { /** * Parameters that are needed by grafana @@ -106,6 +106,7 @@ export default class CogniteDatasource extends DataSourceApi< this.initSources(connector); } + initSources (connector: Connector) { this.connector = connector; this.templatesDatasource = new TemplatesDatasource(this.connector); @@ -123,10 +124,24 @@ export default class CogniteDatasource extends DataSourceApi< QueryEditor: AnnotationsQueryEditor, } + // Queries the backend by using `super.query` + queryBackend( + backendTargets: QueryTarget[], + options: DataQueryRequest + ): Observable { + const request = { + ...options, + targets: backendTargets, + }; + + // Leverage super.query to make a backend request via Grafana + return super.query(request); + } + /** * used by panels to get timeseries data */ - async query(options: DataQueryRequest): Promise { + query(options: DataQueryRequest): Observable { const queryTargets = filterEmptyQueryTargets(options.targets).map((t) => this.replaceVariablesInTarget(t, options.scopedVars) ); @@ -139,52 +154,109 @@ export default class CogniteDatasource extends DataSourceApi< extractionPipelinesTargets, flexibleDataModellingTargets, } = groupTargets(queryTargets); - let responseData: Array = []; + + let observables: Array> = []; + if (queryTargets.length) { - try { - const timeseriesResults = await this.timeseriesDatasource.query({ - ...options, - targets: tsTargets, - }); - const eventResults = await this.eventsDatasource.query({ - ...options, - targets: eventTargets, - }); - const templatesResults = await this.templatesDatasource.query({ - ...options, - targets: templatesTargets, - }); - const relationshipsResults = await this.relationshipsDatasource.query({ - ...options, - targets: relationshipsTargets, - }); - const extractionPipelinesResult = await this.extractionPipelinesDatasource.query({ - ...options, - targets: extractionPipelinesTargets, - }); - const flexibleDataModellingResult = await this.flexibleDataModellingDatasource.query({ - ...options, - targets: flexibleDataModellingTargets, - }); - responseData = [ - ...timeseriesResults.data, - ...eventResults.data, - ...relationshipsResults.data, - ...templatesResults.data, - ...extractionPipelinesResult.data, - ...flexibleDataModellingResult.data, - ]; - } catch (error) { - return { - data: [], - error: { - message: error?.message ?? error, - }, - }; + // If there are backend targets (e.g., Tab.Backend), send them to the backend + const backendTargets = queryTargets.filter((t) => t.tab === Tab.Backend); + if (backendTargets.length) { + const backendObservable = this.queryBackend(backendTargets, options); + observables.push(backendObservable); + } + + // Handle other datasources (Timeseries, Events, etc.) in the frontend + if (tsTargets.length) { + const tsObservable = from( + this.timeseriesDatasource.query({ + ...options, + targets: tsTargets, + }) + ).pipe(map((result) => ({ data: result.data }))); + observables.push(tsObservable); + } + + if (eventTargets.length) { + const eventObservable = from( + this.eventsDatasource.query({ + ...options, + targets: eventTargets, + }) + ).pipe(map((result) => ({ data: result.data }))); + observables.push(eventObservable); + } + + if (templatesTargets.length) { + const templatesObservable = from( + this.templatesDatasource.query({ + ...options, + targets: templatesTargets, + }) + ).pipe(map((result) => ({ data: result.data }))); + observables.push(templatesObservable); + } + + if (relationshipsTargets.length) { + const relationshipsObservable = from( + this.relationshipsDatasource.query({ + ...options, + targets: relationshipsTargets, + }) + ).pipe(map((result) => ({ data: result.data }))); + observables.push(relationshipsObservable); + } + + if (extractionPipelinesTargets.length) { + const extractionPipelinesObservable = from( + this.extractionPipelinesDatasource.query({ + ...options, + targets: extractionPipelinesTargets, + }) + ).pipe(map((result) => ({ data: result.data }))); + observables.push(extractionPipelinesObservable); + } + + if (flexibleDataModellingTargets.length) { + const flexibleDataModellingObservable = from( + this.flexibleDataModellingDatasource.query({ + ...options, + targets: flexibleDataModellingTargets, + }) + ).pipe(map((result) => ({ data: result.data }))); + observables.push(flexibleDataModellingObservable); } } - return { data: responseData }; + + return this.mergeObservables(observables); + } + + // A utility function to merge multiple observables into one + mergeObservables(observables: Array>): Observable { + return new Observable((subscriber) => { + let allData: any[] = []; + + const subscriptions = observables.map((obs) => + obs.subscribe({ + next: (response) => { + allData = [...allData, ...response.data]; + }, + error: (err) => { + subscriber.error(err); + }, + complete: () => { + if (allData.length > 0) { + subscriber.next({ data: allData }); + subscriber.complete(); + } + }, + }) + ); + + // Clean up subscriptions when the observable is unsubscribed + return () => subscriptions.forEach((sub) => sub.unsubscribe()); + }); } + private replaceVariablesInTarget(target: QueryTarget, scopedVars: ScopedVars): QueryTarget { const { expr, query, assetQuery, label, eventQuery, flexibleDataModellingQuery, templateQuery } = target; diff --git a/src/types.ts b/src/types.ts index 498c1680..869f25b8 100644 --- a/src/types.ts +++ b/src/types.ts @@ -21,6 +21,7 @@ export enum Tab { Templates = 'Templates', ExtractionPipelines = 'Extraction Pipelines', FlexibleDataModelling = 'Data Models', + Backend = 'Backend', } export const TabTitles = { @@ -32,6 +33,7 @@ export const TabTitles = { [Tab.Relationships]: 'Relationships', [Tab.Templates]: 'Templates', [Tab.FlexibleDataModelling]: 'Data Models', + [Tab.Backend]: 'Data Models (new)', }; const defaultFlexibleDataModellingQuery: FlexibleDataModellingQuery = { externalId: '', From 8cd96ab2da1c9f6d98042e6928c287ae48c6cd31 Mon Sep 17 00:00:00 2001 From: Brian Kuzma Date: Wed, 25 Sep 2024 13:44:42 +0200 Subject: [PATCH 02/15] duplicate the data modeling tab --- src/components/dataModellingV2Tab.tsx | 199 ++++++++++++++++++++++++++ src/components/queryEditor.tsx | 18 ++- src/types.ts | 12 +- 3 files changed, 218 insertions(+), 11 deletions(-) create mode 100644 src/components/dataModellingV2Tab.tsx diff --git a/src/components/dataModellingV2Tab.tsx b/src/components/dataModellingV2Tab.tsx new file mode 100644 index 00000000..81af5c8f --- /dev/null +++ b/src/components/dataModellingV2Tab.tsx @@ -0,0 +1,199 @@ +import { + CodeEditor, + CodeEditorSuggestionItem, + CodeEditorSuggestionItemKind, + Field, + HorizontalGroup, + MonacoEditor, + Select, +} from '@grafana/ui'; +import React, { useState, useEffect, useMemo } from 'react'; +import { SelectableValue } from '@grafana/data'; +import { buildClientSchema, GraphQLSchema } from 'graphql'; +import { + CompletionItem, + getAutocompleteSuggestions, + Position, + Range, +} from 'graphql-language-service'; +import { getFirstSelection, isValidQuery, typeNameList } from '../utils'; +import { FlexibleDataModellingQuery, SelectedProps, EditorProps } from '../types'; +import CogniteDatasource from '../datasource'; +import { CommonEditors } from './commonEditors'; + +export const DataModellingV2Tab = ( + props: SelectedProps & Pick & { datasource: CogniteDatasource } +) => { + const { query, onQueryChange, datasource } = props; + const [editor, setEditor] = useState(); + const { flexibleDataModellingQuery } = query; + const { externalId, space, version, graphQlQuery } = flexibleDataModellingQuery; + const [options, setOptions] = useState< + Array> + >([]); + const [versions, setVersions] = useState< + Array> + >([]); + const [dml, setDML] = useState(''); + const firstSelection = useMemo( + () => getFirstSelection(graphQlQuery, query.refId), + [graphQlQuery, query.refId] + ); + const patchFlexibleDataModellingQuery = ( + flexibleDataModellingQueryPatch: Partial + ) => { + onQueryChange({ + flexibleDataModellingQuery: { + ...flexibleDataModellingQuery, + ...flexibleDataModellingQueryPatch, + }, + }); + }; + const updateGraphQuery = (newQuery) => { + if (isValidQuery(newQuery, query.refId)) { + patchFlexibleDataModellingQuery({ + graphQlQuery: newQuery, + }); + } + }; + useEffect(() => { + patchFlexibleDataModellingQuery({ + tsKeys: firstSelection.length ? typeNameList(firstSelection) : [], + }); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [graphQlQuery, firstSelection]); + + useEffect(() => { + datasource.flexibleDataModellingDatasource + .listFlexibleDataModelling(query.refId) + .then(({ listGraphQlDmlVersions: { items } }) => { + setOptions( + items.map((el) => ({ + label: `${el.name} (${el.externalId}) <${el.space}>`, + value: { + space: el.space, + externalId: el.externalId, + version: el.version, + dml: el.graphQlDml, + }, + })) + ); + }); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + useEffect(() => { + setVersions([]); + datasource.flexibleDataModellingDatasource + .listVersionByExternalIdAndSpace(query.refId, space, externalId) + .then(({ graphQlDmlVersionsById: { items } }) => { + setVersions( + items.map((el) => ({ + label: el.version, + value: { version: el.version, dml: el.graphQlDml }, + })) + ); + }); + }, [space, externalId, datasource.flexibleDataModellingDatasource, query.refId]); + + const [schema, setSchema] = useState(); + + useEffect(() => { + (async () => { + const data = await datasource.flexibleDataModellingDatasource.runIntrospectionQuery( + { externalId, space, version }, + query + ); + setSchema(buildClientSchema(data)); + })(); + }, [externalId, space, version, datasource.flexibleDataModellingDatasource, query]); + + return ( + <> + + + el.value.version === flexibleDataModellingQuery.version)} + onChange={(update) => { + patchFlexibleDataModellingQuery({ version: update.value.version }); + setDML(update.value.dml); + }} + /> + + + + setEditor(newEditor)} + value={flexibleDataModellingQuery.graphQlQuery ?? ''} + language="graphql" + height={200} + onBlur={updateGraphQuery} + onSave={updateGraphQuery} + showMiniMap={false} + showLineNumbers + getSuggestions={() => { + if (schema && editor) { + return getAutocompleteSuggestions( + schema, + editor.getModel().getValue(), + new Position(editor.getPosition().lineNumber - 1, editor.getPosition().column - 1) + ).map((el) => toCompletionItem(el)); + } + return []; + }} + /> + + {flexibleDataModellingQuery.tsKeys?.length > 0 && ( + + )} + + ); +}; + +/** Format the text, adds icon and returns in format that monaco editor expects */ +const toCompletionItem = (entry: CompletionItem, range?: Range): CodeEditorSuggestionItem => { + const results = { + label: entry.label, + insertText: entry.insertText || entry.label, + insertTextFormat: entry.insertTextFormat, + sortText: entry.sortText, + filterText: entry.filterText, + documentation: entry.documentation, + detail: entry.detail, + range: range ? toMonacoRange(range) : undefined, + kind: CodeEditorSuggestionItemKind.Property, + }; + if (entry.insertTextFormat) { + results.insertTextFormat = entry.insertTextFormat; + } + + return results; +}; + +const toMonacoRange = (range: Range) => { + return { + startLineNumber: range.start.line + 1, + startColumn: range.start.character + 1, + endLineNumber: range.end.line + 1, + endColumn: range.end.character + 1, + }; +}; diff --git a/src/components/queryEditor.tsx b/src/components/queryEditor.tsx index b1b02f95..94605dfb 100644 --- a/src/components/queryEditor.tsx +++ b/src/components/queryEditor.tsx @@ -11,7 +11,7 @@ import { Button, InlineField, InlineFieldRow, - Input + Input, } from '@grafana/ui'; import { SelectableValue } from '@grafana/data'; import { CustomQueryHelp } from './queryHelp'; @@ -38,6 +38,7 @@ import { FlexibleDataModellingTab } from './flexibleDataModellingTab'; import { CommonEditors, LabelEditor } from './commonEditors'; import { EventsTab } from './eventsTab'; import { eventBusService } from '../appEventHandler'; +import { DataModellingV2Tab } from './dataModellingV2Tab'; const { FormField } = LegacyForms; @@ -247,11 +248,7 @@ function CustomTab(props: SelectedProps & Pick) { return ( <> - + ) { /> -