From 217216c5bd511d91b1a93a3db09742cd3d30384d Mon Sep 17 00:00:00 2001
From: Praveen Kumar <kumarpraveen.nitdgp@gmail.com>
Date: Thu, 1 Feb 2024 13:22:28 +0530
Subject: [PATCH] Check status of all the core pods for microshift

In past we observed having kube api access doesn't mean all the required
service pods are running and cluster is working as expected. This PR
adds a list of core namespace for microshift preset and make sure all
the pods in that namespace is running before letting user to know to
consume the cluster.
---
 pkg/crc/cluster/cluster.go | 29 +++++++++++++++++++++++++++++
 pkg/crc/machine/start.go   |  5 ++++-
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/pkg/crc/cluster/cluster.go b/pkg/crc/cluster/cluster.go
index ad2201358a..3432a0d71b 100644
--- a/pkg/crc/cluster/cluster.go
+++ b/pkg/crc/cluster/cluster.go
@@ -510,3 +510,32 @@ func DeleteMCOLeaderLease(ctx context.Context, ocConfig oc.Config) error {
 	_, _, err := ocConfig.RunOcCommand("delete", "-A", "lease", "--all")
 	return err
 }
+
+func CheckCorePodsRunning(ctx context.Context, ocConfig oc.Config) error {
+	if err := WaitForOpenshiftResource(ctx, ocConfig, "pod"); err != nil {
+		return err
+	}
+	coreNameSpaces := []string{"kube-system", "openshift-dns", "openshift-ingress", "openshift-ovn-kubernetes", "openshift-service-ca"}
+	waitForPods := func() error {
+		for _, namespace := range coreNameSpaces {
+			if !podRunningForNamespace(ocConfig, namespace) {
+				logging.Debugf("Pods in %s namespace are not running", namespace)
+				return &errors.RetriableError{Err: fmt.Errorf("pods in %s namespace are not running", namespace)}
+			}
+		}
+		return nil
+	}
+	return errors.Retry(ctx, 2*time.Minute, waitForPods, 2*time.Second)
+}
+
+func podRunningForNamespace(ocConfig oc.Config, namespace string) bool {
+	stdout, stderr, err := ocConfig.WithFailFast().RunOcCommand("get", "pods", "-n", namespace, "--field-selector=status.phase!=Running")
+	if err != nil {
+		logging.Debugf("Failed to get pods in %s namespace, stderr: %s", namespace, stderr)
+		return false
+	}
+	if len(stdout) != 0 {
+		return false
+	}
+	return true
+}
diff --git a/pkg/crc/machine/start.go b/pkg/crc/machine/start.go
index 967f0b2512..d349e9ed6f 100644
--- a/pkg/crc/machine/start.go
+++ b/pkg/crc/machine/start.go
@@ -1064,7 +1064,10 @@ func startMicroshift(ctx context.Context, sshRunner *crcssh.Runner, ocConfig oc.
 		return err
 	}
 
-	return cluster.WaitForAPIServer(ctx, ocConfig)
+	if err := cluster.WaitForAPIServer(ctx, ocConfig); err != nil {
+		return err
+	}
+	return cluster.CheckCorePodsRunning(ctx, ocConfig)
 }
 
 func ensurePullSecretPresentInVM(sshRunner *crcssh.Runner, pullSec cluster.PullSecretLoader) error {