Remove and reset nodes during apply by setting reset: true

README.md

@@ -481,6 +481,10 @@ Localhost connection options. Can be used to use the local host running k0sctl a
 
 This must be set `true` to enable the localhost connection.
 
+###### `spec.hosts[*].uninstall` <boolean> (optional) (default: `false`)
+
+If set to `true` k0sctl will remove the node from kubernetes and uninstall k0s from the host.
+
 ### K0s Fields
 
 ##### `spec.k0s.version` <string> (optional) (default: auto-discovery)

cmd/apply.go

@@ -86,6 +86,12 @@ var applyCommand = &cli.Command{
 			&phase.UpgradeWorkers{
 				NoDrain: ctx.Bool("no-drain"),
 			},
+			&phase.UninstallWorkers{
+				NoDrain: ctx.Bool("no-drain"),
+			},
+			&phase.UninstallControllers{
+				NoDrain: ctx.Bool("no-drain"),
+			},
 			&phase.RunHooks{Stage: "after", Action: "apply"},
 		)
 
@@ -128,6 +134,16 @@ var applyCommand = &cli.Command{
 		text := fmt.Sprintf("==> Finished in %s", duration)
 		log.Infof(Colorize.Green(text).String())
 
+		uninstalled := false
+		for _, host := range manager.Config.Spec.Hosts {
+			if host.Uninstall {
+				uninstalled = true
+			}
+		}
+		if uninstalled {
+			log.Info("INFO: There were nodes that got uninstalled druing the apply phase. Please remove them from your kubectl.yaml file")
+		}
+
 		log.Infof("k0s cluster version %s is now installed", manager.Config.Spec.K0s.Version)
 		log.Infof("Tip: To access the cluster you can now fetch the admin kubeconfig using:")
 		log.Infof("     " + Colorize.Cyan("k0sctl kubeconfig").String())

configurer/linux/enterpriselinux/rhel.go

@@ -1,11 +1,12 @@
 package enterpriselinux
 
 import (
+	"strings"
+
 	"github.com/k0sproject/k0sctl/configurer"
 	k0slinux "github.com/k0sproject/k0sctl/configurer/linux"
 	"github.com/k0sproject/rig"
 	"github.com/k0sproject/rig/os/registry"
-	"strings"
 )
 
 // RHEL provides OS support for RedHat Enterprise Linux

phase/download_binaries.go

@@ -29,7 +29,7 @@ func (p *DownloadBinaries) Title() string {
 func (p *DownloadBinaries) Prepare(config *v1beta1.Cluster) error {
 	p.Config = config
 	p.hosts = p.Config.Spec.Hosts.Filter(func(h *cluster.Host) bool {
-		return h.UploadBinary && h.Metadata.K0sBinaryVersion != config.Spec.K0s.Version
+		return !h.Uninstall && h.UploadBinary && h.Metadata.K0sBinaryVersion != config.Spec.K0s.Version
 	})
 	return nil
 }

phase/download_k0s.go

@@ -24,15 +24,23 @@ func (p *DownloadK0s) Title() string {
 func (p *DownloadK0s) Prepare(config *v1beta1.Cluster) error {
 	p.Config = config
 	p.hosts = p.Config.Spec.Hosts.Filter(func(h *cluster.Host) bool {
-		if h.Metadata.K0sBinaryVersion == p.Config.Spec.K0s.Version {
+		// Nothing to upload
+		if h.UploadBinary {
 			return false
 		}
 
+		// Nothing to upload
+		if h.Uninstall {
+			return false
+		}
+
+		// Upgrade is handled separately (k0s stopped, binary uploaded, k0s restarted)
 		if h.Metadata.NeedsUpgrade {
 			return false
 		}
 
-		if h.UploadBinary {
+		// The version is already correct
+		if h.Metadata.K0sBinaryVersion == p.Config.Spec.K0s.Version {
 			return false
 		}
 

phase/install_controllers.go

@@ -27,7 +27,7 @@ func (p *InstallControllers) Prepare(config *v1beta1.Cluster) error {
 	var controllers cluster.Hosts = p.Config.Spec.Hosts.Controllers()
 	p.leader = p.Config.Spec.K0sLeader()
 	p.hosts = controllers.Filter(func(h *cluster.Host) bool {
-		return h != p.leader && h.Metadata.K0sRunningVersion == ""
+		return !h.Uninstall && (h != p.leader && h.Metadata.K0sRunningVersion == "")
 	})
 
 	return nil

phase/install_workers.go

@@ -27,7 +27,7 @@ func (p *InstallWorkers) Prepare(config *v1beta1.Cluster) error {
 	p.Config = config
 	var workers cluster.Hosts = p.Config.Spec.Hosts.Workers()
 	p.hosts = workers.Filter(func(h *cluster.Host) bool {
-		return h.Metadata.K0sRunningVersion == "" || !h.Metadata.Ready
+		return !h.Uninstall && (h.Metadata.K0sRunningVersion == "" || !h.Metadata.Ready)
 	})
 	p.leader = p.Config.Spec.K0sLeader()
 

phase/uninstall_controllers.go

@@ -0,0 +1,124 @@
+package phase
+
+import (
+	"strings"
+
+	"github.com/Masterminds/semver"
+	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1"
+	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster"
+	"github.com/k0sproject/rig/exec"
+	log "github.com/sirupsen/logrus"
+)
+
+// UninstallControllers uninstalls k0s from the controllers
+type UninstallControllers struct {
+	GenericPhase
+
+	NoDrain bool
+
+	hosts  cluster.Hosts
+	leader *cluster.Host
+}
+
+// Title for the phase
+func (p *UninstallControllers) Title() string {
+	return "Uninstall controllers"
+}
+
+// Prepare the phase
+func (p *UninstallControllers) Prepare(config *v1beta1.Cluster) error {
+	p.Config = config
+	p.leader = p.Config.Spec.K0sLeader()
+
+	var controllers cluster.Hosts = p.Config.Spec.Hosts.Controllers()
+	log.Debugf("%d controllers in total", len(controllers))
+	p.hosts = controllers.Filter(func(h *cluster.Host) bool {
+		return h.Uninstall
+	})
+	log.Debugf("UninstallControllers phase prepared, %d controllers will be uninstalled", len(p.hosts))
+	return nil
+}
+
+// ShouldRun is true when there are controllers that needs to be upgraded
+func (p *UninstallControllers) ShouldRun() bool {
+	return len(p.hosts) > 0
+}
+
+// CleanUp cleans up the environment override files on hosts
+func (p *UninstallControllers) CleanUp() {
+	for _, h := range p.hosts {
+		if len(h.Environment) > 0 {
+			if err := h.Configurer.CleanupServiceEnvironment(h, h.K0sServiceName()); err != nil {
+				log.Warnf("%s: failed to clean up service environment: %s", h, err.Error())
+			}
+		}
+	}
+}
+
+// Run the phase
+func (p *UninstallControllers) Run() error {
+	for _, h := range p.hosts {
+		log.Debugf("%s: draining node", h)
+		if !p.NoDrain && h.Role != "controller" {
+			if err := p.leader.DrainNode(&cluster.Host{
+				Metadata: cluster.HostMetadata{
+					Hostname: h.Metadata.Hostname,
+				},
+			}); err != nil {
+				log.Warnf("%s: failed to drain node: %s", h, err.Error())
+			}
+		}
+		log.Debugf("%s: draining node completed", h)
+
+		log.Debugf("%s: deleting node...", h)
+		if h.Role != "controller" {
+			if err := p.leader.DeleteNode(&cluster.Host{
+				Metadata: cluster.HostMetadata{
+					Hostname: h.Metadata.Hostname,
+				},
+			}); err != nil {
+				log.Warnf("%s: failed to delete node: %s", h, err.Error())
+			}
+		}
+		log.Debugf("%s: deleting node", h)
+
+		if h.Configurer.ServiceIsRunning(h, h.K0sServiceName()) {
+			log.Debugf("%s: stopping k0s...", h)
+			if err := h.Configurer.StopService(h, h.K0sServiceName()); err != nil {
+				log.Warnf("%s: failed to stop k0s: %s", h, err.Error())
+			}
+			log.Debugf("%s: waiting for k0s to stop", h)
+			if err := h.WaitK0sServiceStopped(); err != nil {
+				log.Warnf("%s: failed to wait for k0s to stop: %s", h, err.Error())
+			}
+			log.Debugf("%s: stopping k0s completed", h)
+		}
+
+		log.Debugf("%s: leaving etcd...", h)
+		if err := p.leader.LeaveEtcd(h); err != nil {
+			log.Warnf("%s: failed to leave etcd: %s", h, err.Error())
+		}
+		log.Debugf("%s: leaving etcd completed", h)
+
+		log.Debugf("%s: resetting k0s...", h)
+		out, err := h.ExecOutput(h.Configurer.K0sCmdf("reset"), exec.Sudo(h))
+		c, _ := semver.NewConstraint("<= 1.22.3+k0s.0")
+		running, _ := semver.NewVersion(h.Metadata.K0sBinaryVersion)
+		if err != nil {
+			log.Warnf("%s: k0s reported failure: %v", h, err)
+			if c.Check(running) && !strings.Contains(out, "k0s cleanup operations done") {
+				log.Warnf("%s: k0s reset failed, trying k0s cleanup", h)
+			}
+		}
+		log.Debugf("%s: resetting k0s completed", h)
+
+		log.Debugf("%s: removing config...", h)
+		if dErr := h.Configurer.DeleteFile(h, h.Configurer.K0sConfigPath()); dErr != nil {
+			log.Warnf("%s: failed to remove existing configuration %s: %s", h, h.Configurer.K0sConfigPath(), dErr)
+		}
+		log.Debugf("%s: removing config completed", h)
+
+		log.Infof("%s: uninstalled", h)
+	}
+	return nil
+}

phase/uninstall_workers.go

@@ -0,0 +1,116 @@
+package phase
+
+import (
+	"strings"
+
+	"github.com/Masterminds/semver"
+	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1"
+	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster"
+	"github.com/k0sproject/rig/exec"
+	log "github.com/sirupsen/logrus"
+)
+
+// UninstallWorkers uninstalls k0s from the workers
+type UninstallWorkers struct {
+	GenericPhase
+
+	NoDrain bool
+
+	hosts  cluster.Hosts
+	leader *cluster.Host
+}
+
+// Title for the phase
+func (p *UninstallWorkers) Title() string {
+	return "Uninstall workers"
+}
+
+// Prepare the phase
+func (p *UninstallWorkers) Prepare(config *v1beta1.Cluster) error {
+	p.Config = config
+	p.leader = p.Config.Spec.K0sLeader()
+
+	var workers cluster.Hosts = p.Config.Spec.Hosts.Workers()
+	log.Debugf("%d workers in total", len(workers))
+	p.hosts = workers.Filter(func(h *cluster.Host) bool {
+		return h.Uninstall
+	})
+	log.Debugf("UninstallWorkers phase prepared, %d workers will be uninstalled", len(p.hosts))
+	return nil
+}
+
+// ShouldRun is true when there are workers that needs to be upgraded
+func (p *UninstallWorkers) ShouldRun() bool {
+	return len(p.hosts) > 0
+}
+
+// CleanUp cleans up the environment override files on hosts
+func (p *UninstallWorkers) CleanUp() {
+	for _, h := range p.hosts {
+		if len(h.Environment) > 0 {
+			if err := h.Configurer.CleanupServiceEnvironment(h, h.K0sServiceName()); err != nil {
+				log.Warnf("%s: failed to clean up service environment: %s", h, err.Error())
+			}
+		}
+	}
+}
+
+// Run the phase
+func (p *UninstallWorkers) Run() error {
+	return p.hosts.ParallelEach(func(h *cluster.Host) error {
+		log.Debugf("%s: draining node", h)
+		if !p.NoDrain {
+			if err := p.leader.DrainNode(&cluster.Host{
+				Metadata: cluster.HostMetadata{
+					Hostname: h.Metadata.Hostname,
+				},
+			}); err != nil {
+				log.Warnf("%s: failed to drain node: %s", h, err.Error())
+			}
+		}
+		log.Debugf("%s: draining node completed", h)
+
+		log.Debugf("%s: deleting node...", h)
+		if err := p.leader.DeleteNode(&cluster.Host{
+			Metadata: cluster.HostMetadata{
+				Hostname: h.Metadata.Hostname,
+			},
+		}); err != nil {
+			log.Warnf("%s: failed to delete node: %s", h, err.Error())
+		}
+		log.Debugf("%s: deleting node", h)
+
+		if h.Configurer.ServiceIsRunning(h, h.K0sServiceName()) {
+			log.Debugf("%s: stopping k0s...", h)
+			if err := h.Configurer.StopService(h, h.K0sServiceName()); err != nil {
+				log.Warnf("%s: failed to stop k0s: %s", h, err.Error())
+			}
+			log.Debugf("%s: waiting for k0s to stop", h)
+			if err := h.WaitK0sServiceStopped(); err != nil {
+				log.Warnf("%s: failed to wait for k0s to stop: %s", h, err.Error())
+			}
+			log.Debugf("%s: stopping k0s completed", h)
+		}
+
+		log.Debugf("%s: resetting k0s...", h)
+		out, err := h.ExecOutput(h.Configurer.K0sCmdf("reset"), exec.Sudo(h))
+		c, _ := semver.NewConstraint("<= 1.22.3+k0s.0")
+		running, _ := semver.NewVersion(h.Metadata.K0sBinaryVersion)
+		if err != nil {
+			log.Warnf("%s: k0s reported failure: %v", h, err)
+			if c.Check(running) && !strings.Contains(out, "k0s cleanup operations done") {
+				log.Warnf("%s: k0s reset failed, trying k0s cleanup", h)
+			}
+		}
+		log.Debugf("%s: resetting k0s completed", h)
+
+		log.Debugf("%s: removing config...", h)
+		if dErr := h.Configurer.DeleteFile(h, h.Configurer.K0sConfigPath()); dErr != nil {
+			log.Warnf("%s: failed to remove existing configuration %s: %s", h, h.Configurer.K0sConfigPath(), dErr)
+		}
+		log.Debugf("%s: removing config completed", h)
+
+		log.Infof("%s: uninstalled", h)
+		return err
+	})
+}

phase/upgrade_controllers.go

@@ -30,7 +30,7 @@ func (p *UpgradeControllers) Prepare(config *v1beta1.Cluster) error {
 	var controllers cluster.Hosts = p.Config.Spec.Hosts.Controllers()
 	log.Debugf("%d controllers in total", len(controllers))
 	p.hosts = controllers.Filter(func(h *cluster.Host) bool {
-		return h.Metadata.NeedsUpgrade
+		return !h.Uninstall && h.Metadata.NeedsUpgrade
 	})
 	log.Debugf("UpgradeControllers phase prepared, %d controllers needs upgrade", len(p.hosts))
 	return nil

phase/upgrade_workers.go

@@ -31,9 +31,9 @@ func (p *UpgradeWorkers) Prepare(config *v1beta1.Cluster) error {
 	p.Config = config
 	p.leader = p.Config.Spec.K0sLeader()
 	var workers cluster.Hosts = p.Config.Spec.Hosts.Workers()
-	log.Debugf("%d controllers in total", len(workers))
+	log.Debugf("%d workers in total", len(workers))
 	p.hosts = workers.Filter(func(h *cluster.Host) bool {
-		return h.Metadata.NeedsUpgrade
+		return !h.Uninstall && h.Metadata.NeedsUpgrade
 	})
 	log.Debugf("UpgradeWorkers phase prepared, %d workers needs upgrade", len(p.hosts))
 

phase/upload_binaries.go

@@ -31,6 +31,11 @@ func (p *UploadBinaries) Prepare(config *v1beta1.Cluster) error {
 			return false
 		}
 
+		// Nothing to upload
+		if h.Uninstall {
+			return false
+		}
+
 		// Upgrade is handled separately (k0s stopped, binary uploaded, k0s restarted)
 		if h.Metadata.NeedsUpgrade {
 			return false