-
Notifications
You must be signed in to change notification settings - Fork 0
/
template.yaml
287 lines (287 loc) · 10.4 KB
/
template.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
apiVersion: scaffolder.backstage.io/v1beta3
kind: Template
metadata:
name: aws-workload-dbt-transformation-template.1
title: DBT Transformation Workload
description: Defines the a DBT project skeleton to perform data transformation
mesh:
icon: https://seeklogo.com/images/D/dbt-logo-500AB0BAA7-seeklogo.com.png
tags:
- dbt
- sql
- workload
spec:
generates: componenttype:default/workload
owner: agilelab
type: workload
parameters:
- title: Component Metadata
description: Basic Information for this component.
required:
- name
- domain
- dataproduct
- description
properties:
name:
title: Name
type: string
description: Display name for this workload.
ui:field: EntityNamePicker
default: DBT Transformation Workload
ui:options:
allowArbitraryValues: true
description:
title: Description
type: string
description: Required. Detailed explanation about the function and the meaning
of this Workload.
domain:
title: Domain
type: string
description: Required. Domain of the Data Product this storage component belongs to
ui:field: EntityPicker
ui:options:
allowArbitraryValues: false
allowedKinds:
- Domain
dataproduct:
title: Data Product
type: string
description: Required. Data Product this Workload belongs to
ui:field: EntityPicker
ui:filter:
- fieldName: domain
entityPath: spec.domain
ui:options:
allowArbitraryValues: false
allowedKinds:
- System
identifier:
title: Identifier
type: string
description: Autogenerated from the information above. A unique identifier for
the component. It will not be editable after creation and is a
string that is composed of [a-zA-Z] separated by any of [-_].
ui:field: ComponentIdentifierPicker
ui:options:
allowArbitraryValues: false
developmentGroup:
title: DevelopmentGroup
type: string
description: Automatically selected from the Data Product metadata. Data Product
development group.
ui:field: EntitySelectionPicker
ui:fieldName: dataproduct
ui:property: spec.owner
ui:options:
allowArbitraryValues: false
dependsOn:
title: Dependencies
type: array
default: []
items:
type: string
ui:field: EntityRelationsPicker
ui:fieldName: dataproduct
relation: hasPart
description: A component could depend on other components in the same Data
Product. This information will be used to deploy the components in
such an order that their dependencies already exist.
readsFrom:
title: Reads from
type: array
default: []
items:
type: string
ui:field: ReadsFromPicker
description: This is filled only for DataPipeline workloads and it represents
the list of output ports or external systems that is reading.
tags:
title: Tags
type: array
default: []
description: Tags for the component.
items:
type: string
domainName:
type: string
ui:field: EntitySelectionPicker
ui:fieldName: domain
ui:property: spec.mesh.name
ui:options:
allowArbitraryValues: false
ui:widget: hidden
dataproductName:
type: string
ui:field: EntitySelectionPicker
ui:fieldName: dataproduct
ui:property: spec.mesh.name
ui:options:
allowArbitraryValues: false
ui:widget: hidden
- title: DBT Project Details
required:
- underlyingStorage
- storageAreaDetails
properties:
dbtName:
title: Project Name
type: string
description: Add a custom name to your dbt project
default: dmb_dbt_transform
underlyingStorage:
title: Storage Area
type: string
description: Underlying Storage this output port will read from.
ui:field: EntityRelationsPicker
ui:fieldName: dataproduct
relation: hasPart
ui:filters:
- key: spec.type
value: "storage"
storageAreaDetails:
title: Storage Area details
description: Values retrieved from the Storage Area chosen above.
If you don't see any values filled in, make sure you chose a valid Storage Area
type: object
required:
- database
- schema
properties:
database:
title: Database
type: string
ui:field: EntitySelectionPicker
ui:fieldName: underlyingStorage
ui:property: spec.mesh.specific.database
ui:options:
allowArbitraryValues: false
schema:
title: Schema
type: string
ui:field: EntitySelectionPicker
ui:fieldName: underlyingStorage
ui:property: spec.mesh.specific.schema
ui:options:
allowArbitraryValues: false
cloudStorage:
title: Cloud Storage Details
type: object
ui:widget: hidden # We're not showing this as the current template version is used only on environments without a Tech Adapter
properties:
storageProvider:
type: string
title: Cloud Storage
# default: AWS
enum:
- AWS
dependencies:
storageProvider:
oneOf:
- properties:
storageProvider:
enum:
- AWS
bucketURI:
title: Path to the Object Storage
type: string
description: Provide the Path to the Object Storage in the corresponding cloud
infrastructure in the form of URI. Replace the
<bucket-name> placeholder with your preferred bucket
name which can be used to store artifacts.
default: s3://<bucket-name>/
region:
title: Region
type: string
description: Region where the Object Storage is available.
default: EU_WEST_1
secrets:
title: Secret Connection Details
type: object
ui:widget: hidden # We're not showing this as the current template version is used only on environments without a Tech Adapter
properties:
secretsProvider:
type: string
title: Cloud Storage
# default: AWS
enum:
- AWS
dependencies:
secretsProvider:
oneOf:
- properties:
secretsProvider:
enum:
- AWS
secretId:
title: Secret ID
type: string
description: Secret ID which is used to retrieve the connection details from the
Secret Manager.
default: airflow/connections/snowflake
region:
title: Region
type: string
description: Region where the Secret ID information is located.
default: EU_WEST_1
steps:
- id: template
name: Fetch Skeleton + Template
action: fetch:template
input:
url: ./skeleton
targetPath: .
copyWithoutTemplating:
- .gitlab-ci.yml
values:
name: ${{ parameters.name }}
domainName: ${{ parameters.domainName }}
dataproductName: ${{ parameters.dataproductName }}
description: ${{ parameters.description }}
domain: ${{ parameters.domain }}
dataproduct: ${{ parameters.dataproduct }}
identifier: ${{ parameters.identifier }}
developmentGroup: ${{ parameters.developmentGroup }}
dependsOn: ${{ parameters.dependsOn }}
readsFrom: ${{ parameters.readsFrom }}
tags: ${{ parameters.tags }}
dbtProjectName: ${{ parameters.dbtName }}
database: ${{ parameters.storageAreaDetails.database }}
schema: ${{ parameters.storageAreaDetails.schema }}
cloudStorage: ${{ parameters.cloudStorage }}
secrets: ${{ parameters.secrets }}
useCaseTemplateId: urn:dmb:utm:aws-workload-dbt-transformation-template:0.0.0
infrastructureTemplateId: urn:dmb:itm:aws-workload-dbt-transformation-provisioner:0
useCaseTemplateVersion: 0.0.0
repoUrl: gitlab.com?owner=AgileDMBSandbox%2F${{orgname}}%2Fmesh.repository%2F${{
parameters.domain | replace(r/domain:| |-/, "") }}%2F${{
parameters.dataproduct.split(".")[1] | replace(r/ |-/g, "")
}}&repo=${{ parameters.name.split(" ") | join("") | lower }}
rootDirectory: .
- id: publish
name: Publish
action: witboostMeshComponent:publish:gitlab
input:
allowedHosts:
- gitlab.com
description: This is ${{ parameters.name }}
repoUrl: gitlab.com?owner=AgileDMBSandbox%2F${{orgname}}%2Fmesh.repository%2F${{
parameters.domain | replace(r/domain:| |-/, "") }}%2F${{
parameters.dataproduct.split(".")[1] | replace(r/ |-/g, "")
}}&repo=${{ parameters.name.split(" ") | join("") | lower }}
rootDirectory: .
dataproduct: ${{ parameters.dataproduct }}
- id: register
name: Register
action: catalog:register
input:
repoContentsUrl: ${{ steps.publish.output.repoContentsUrl }}
catalogInfoPath: /catalog-info.yaml
output:
links:
- title: Repository
url: ${{ steps.publish.output.remoteUrl }}
- title: Open in catalog
icon: catalog
entityRef: ${{ steps.register.output.entityRef }}