Add ilab kfp pipeline to the DSPO repo

opendatahub-io · Dec 13, 2024 · 6977289 · 6977289
1 parent d80a181
commit 6977289
Show file tree

Hide file tree

Showing 9 changed files with 58 additions and 24 deletions.
diff --git a/api/v1/dspipeline_types.go b/api/v1/dspipeline_types.go
@@ -69,9 +69,13 @@ type APIServer struct {
 	// Include sample pipelines with the deployment of this DSP API Server. Default: true
 	// +kubebuilder:default:=false
 	// +kubebuilder:validation:Optional
-	EnableSamplePipeline bool   `json:"enableSamplePipeline"`
-	ArgoLauncherImage    string `json:"argoLauncherImage,omitempty"`
-	ArgoDriverImage      string `json:"argoDriverImage,omitempty"`
+	EnableSamplePipeline bool `json:"enableSamplePipeline"`
+	// Include instructlab multi-phase training and LLM generation pipeline with the deployment of this DSP API Server. Default: false
+	// +kubebuilder:default:=false
+	// +kubebuilder:validation:Optional
+	EnableInstructlabPipeline bool   `json:"enableInstructlabPipeline"`
+	ArgoLauncherImage         string `json:"argoLauncherImage,omitempty"`
+	ArgoDriverImage           string `json:"argoDriverImage,omitempty"`
 	// Specify custom Pod resource requirements for this component.
 	Resources *ResourceRequirements `json:"resources,omitempty"`
 

diff --git a/api/v1alpha1/dspipeline_types.go b/api/v1alpha1/dspipeline_types.go
@@ -69,9 +69,13 @@ type APIServer struct {
 	// Include sample pipelines with the deployment of this DSP API Server. Default: true
 	// +kubebuilder:default:=false
 	// +kubebuilder:validation:Optional
-	EnableSamplePipeline bool   `json:"enableSamplePipeline"`
-	ArgoLauncherImage    string `json:"argoLauncherImage,omitempty"`
-	ArgoDriverImage      string `json:"argoDriverImage,omitempty"`
+	EnableSamplePipeline bool `json:"enableSamplePipeline"`
+	// Include instructlab multi-phase training and LLM generation pipeline with the deployment of this DSP API Server. Default: false
+	// +kubebuilder:default:=false
+	// +kubebuilder:validation:Optional
+	EnableInstructlabPipeline bool   `json:"enableInstructlabPipeline"`
+	ArgoLauncherImage         string `json:"argoLauncherImage,omitempty"`
+	ArgoDriverImage           string `json:"argoDriverImage,omitempty"`
 	// Specify custom Pod resource requirements for this component.
 	Resources *ResourceRequirements `json:"resources,omitempty"`
 

diff --git a/...ses/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml b/...ses/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml
@@ -105,6 +105,12 @@ spec:
                       Server. Setting Deploy to false disables operator reconciliation.
                       Default: true'
                     type: boolean
+                  enableInstructlabPipeline:
+                    default: false
+                    description: 'Include instructlab multi-phase training and LLM
+                      generation pipeline with the deployment of this DSP API Server.
+                      Default: false'
+                    type: boolean
                   enableOauth:
                     default: true
                     description: 'Create an Openshift Route for this DSP API Server.
@@ -990,6 +996,12 @@ spec:
                       Server. Setting Deploy to false disables operator reconciliation.
                       Default: true'
                     type: boolean
+                  enableInstructlabPipeline:
+                    default: false
+                    description: 'Include instructlab multi-phase training and LLM
+                      generation pipeline with the deployment of this DSP API Server.
+                      Default: false'
+                    type: boolean
                   enableOauth:
                     default: true
                     description: 'Create an Openshift Route for this DSP API Server.

diff --git a/config/internal/apiserver/default/deployment.yaml.tmpl b/config/internal/apiserver/default/deployment.yaml.tmpl
@@ -147,7 +147,7 @@ spec:
           args:
             - --config=/config
             - -logtostderr=true
-            {{ if .APIServer.EnableSamplePipeline }}
+            {{ if or .APIServer.EnableSamplePipeline .APIServer.EnableInstructlabPipeline }}
             - --sampleconfig=/config/sample_config.json
             {{ end }}
             {{ if .PodToPodTLS }}
@@ -206,8 +206,8 @@ spec:
             - mountPath: /etc/tls/private
               name: proxy-tls
           {{ end }}
-          {{ if or .APIServer.EnableSamplePipeline .CustomCABundle }}
-            {{ if .APIServer.EnableSamplePipeline }}
+          {{ if or .APIServer.EnableSamplePipeline .CustomCABundle .APIServer.EnableInstructlabPipeline}}
+            {{ if or .APIServer.EnableSamplePipeline .APIServer.EnableInstructlabPipeline}}
             - name: sample-config
               mountPath: /config/sample_config.json
               subPath: sample_config.json
@@ -287,7 +287,7 @@ spec:
           configMap:
             name: {{ .CustomCABundle.ConfigMapName }}
         {{ end }}
-        {{ if .APIServer.EnableSamplePipeline }}
+        {{ if or .APIServer.EnableSamplePipeline .APIServer.EnableInstructlabPipeline}}
         - name: sample-config
           configMap:
             name: sample-config-{{.Name}}

diff --git a/config/internal/apiserver/sample-pipeline/sample-config.yaml.tmpl b/config/internal/apiserver/sample-pipeline/sample-config.yaml.tmpl
@@ -7,11 +7,21 @@ metadata:
         app: {{.APIServerDefaultResourceName}}
         component: data-science-pipelines
 data:
-    sample_config.json: |-
-        [
-          {
-            "name": "[Demo] iris-training",
-            "description": "[source code](https://github.com/opendatahub-io/data-science-pipelines/tree/master/samples/iris-sklearn) A simple pipeline to demonstrate a basic ML Training workflow",
-            "file": "/samples/iris-pipeline-compiled.yaml"
-          }
-        ]
+  sample_config.json: |-
+    [
+      {{- if .EnableSamplePipeline }}
+      {
+        "name": "[Demo] iris-training",
+        "description": "[source code](https://github.com/opendatahub-io/data-science-pipelines/tree/master/samples/iris-sklearn) A simple pipeline to demonstrate a basic ML Training workflow",
+        "file": "/samples/iris-pipeline-compiled.yaml"
+      }
+      {{- if .EnableInstructlabPipeline }},{{ end }}
+      {{- end }}
+      {{- if .EnableInstructlabPipeline }}
+      {
+        "name": "[Instructlab] Multi-Phase Training Pipeline",
+        "description": "[source code](https://github.com/opendatahub-io/ilab-on-ocp) Ilab KFP pipeline",
+        "file": "/pipelines/instructlab.yaml"
+      }
+      {{- end }}
+    ]
diff --git a/controllers/apiserver.go b/controllers/apiserver.go
@@ -68,7 +68,7 @@ func (r *DSPAReconciler) ReconcileAPIServer(ctx context.Context, dsp *dspav1.Dat
 	}
 
 	for cmName, template := range samplePipelineTemplates {
-		if dsp.Spec.APIServer.EnableSamplePipeline {
+		if dsp.Spec.APIServer.EnableSamplePipeline || dsp.Spec.APIServer.EnableIlabKFPPipeline {
 			err := r.Apply(dsp, params, template)
 			if err != nil {
 				return err

diff --git a/controllers/dspipeline_params.go b/controllers/dspipeline_params.go
@@ -92,8 +92,9 @@ type DSPAParams struct {
 	CustomCABundle *dspa.CABundle
 	DSPONamespace  string
 	// Use to enable tls communication between component pods.
-	PodToPodTLS bool
-
+	PodToPodTLS             bool
+	EnableSamplePipeline    bool
+	EnableIlabKFPPipeline   bool
 	APIServerServiceDNSName string
 }
 
@@ -601,6 +602,9 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip
 		setStringDefault(argoDriverImageFromConfig, &p.APIServer.ArgoDriverImage)
 		setResourcesDefault(config.APIServerResourceRequirements, &p.APIServer.Resources)
 
+		p.EnableSamplePipeline = dsp.Spec.APIServer.EnableSamplePipeline
+		p.EnableIlabKFPPipeline = dsp.Spec.APIServer.EnableIlabKFPPipeline
+
 		if p.APIServer.CustomServerConfig == nil {
 			p.APIServer.CustomServerConfig = &dspa.ScriptConfigMap{
 				Name: config.CustomServerConfigMapNamePrefix + dsp.Name,

diff --git a/tests/resources/test-pipeline-run.yaml b/tests/resources/test-pipeline-run.yaml
@@ -29,7 +29,7 @@ deploymentSpec:
         - -c
         - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
           \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-          \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.10.1'\
+          \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.11.0'\
           \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\
           $0\" \"$@\"\n"
         - sh

diff --git a/tests/resources/test-pipeline-with-custom-pip-server-run.yaml b/tests/resources/test-pipeline-with-custom-pip-server-run.yaml
@@ -24,7 +24,7 @@ deploymentSpec:
         - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
           \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
           \ python3 -m pip install --quiet --no-warn-script-location --index-url https://nginx-service.test-pypiserver.svc.cluster.local/simple/\
-          \ 'kfp==2.10.1' '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"\
+          \ 'kfp==2.11.0' '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"\
           3.9\"'  &&  python3 -m pip install --quiet --no-warn-script-location --index-url\
           \ https://nginx-service.test-pypiserver.svc.cluster.local/simple/ 'numpy'\
           \ && \"$0\" \"$@\"\n"
@@ -66,4 +66,4 @@ root:
       Output:
         parameterType: STRING
 schemaVersion: 2.1.0
-sdkVersion: kfp-2.10.1
+sdkVersion: kfp-2.11.0