From aa1b0b53bd053575f7ee89ea109f06f5130b38f9 Mon Sep 17 00:00:00 2001
From: Fabian Fischer <glrf@users.noreply.github.com>
Date: Tue, 14 Dec 2021 15:54:12 +0100
Subject: [PATCH 1/4] Extend built-in database how-to with instructions to
 enable backups

---
 .../ROOT/pages/how-tos/use-built-in-db.adoc   | 56 +++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/docs/modules/ROOT/pages/how-tos/use-built-in-db.adoc b/docs/modules/ROOT/pages/how-tos/use-built-in-db.adoc
index 9f0e0f7d..dc720dc8 100644
--- a/docs/modules/ROOT/pages/how-tos/use-built-in-db.adoc
+++ b/docs/modules/ROOT/pages/how-tos/use-built-in-db.adoc
@@ -36,3 +36,59 @@ vault kv put -cas=0 "${key}" admin-password=$(pwgen -s 32 1) db-password=$(pwgen
 ----
 
 . Compile and push the cluster catalog
+
+== Enable Backups
+
+The component supports backups for the built-in database through https://k8up.io/[K8up].
+The following steps show how to enable them.
+
+[NOTE]
+====
+To use this backup feature, the https://github.com/projectsyn/component-backup-k8up[component-backup-k8up] needs to be installed on the cluster.
+====
+
+. Enable backups in component parameters
++
+[source,yaml]
+----
+keycloak:
+  k8up:
+    enabled: true
+----
+
+. Generate and store repository secret in Vault
++
+[source,bash]
+----
+key="clusters/kv/${TENANT_ID}/${CLUSTER_ID}/keycloak"
+
+vault kv put -cas=0 "${key}" k8up-repo-password=$(pwgen -s 32 1)
+----
+
+. Get the access and secret key of your S3 provider and store them in Vault
++
+[source,bash]
+----
+s3_access_key=YOUR_ACCESS_KEY
+s3_secret_key=YOUR_SECRET_KEY
+key="clusters/kv/${TENANT_ID}/${CLUSTER_ID}/keycloak"
+
+vault kv put -cas=0 "${key}" k8up-s3-accesskey=${s3_access_key} k8up-s3-secretkey=${s3_secret_key}
+----
++
+[TIP]
+====
+On most clusters you should be able to reuse the global backup credentials setup during cluster creation.
+Just add a reference to the credentials in Vault to the K8up S3 configuration and you can skip this step.
+
+[source,yaml]
+----
+keycloak:
+  k8up:
+    enabled: true
+    s3:
+      accesskey: '?{vaultkv:${customer:name}/${cluster:name}/global-backup/access-key}'
+      secretkey: '?{vaultkv:${customer:name}/${cluster:name}/global-backup/secret-key}'
+----
+====
+

From 2c7f9cd3ef44e9e181d5042a03edfd0ae084b800 Mon Sep 17 00:00:00 2001
From: Chris <github.account@chrigel.net>
Date: Wed, 17 Nov 2021 20:08:18 +0100
Subject: [PATCH 2/4] Add restore guide for built-in database

---
 docs/modules/ROOT/pages/how-tos/restore.adoc | 104 +++++++++++++++++++
 docs/modules/ROOT/partials/nav.adoc          |   1 +
 2 files changed, 105 insertions(+)
 create mode 100644 docs/modules/ROOT/pages/how-tos/restore.adoc

diff --git a/docs/modules/ROOT/pages/how-tos/restore.adoc b/docs/modules/ROOT/pages/how-tos/restore.adoc
new file mode 100644
index 00000000..a021be2e
--- /dev/null
+++ b/docs/modules/ROOT/pages/how-tos/restore.adoc
@@ -0,0 +1,104 @@
+= Restore Keycloak
+
+The following steps will guide you through restoring a backup of Keycloak.
+
+[IMPORTANT]
+====
+This guide only covers how to restore the built-in database.
+If you use an external database, please consult the documentation of your database provider on how to backup and restore it.
+====
+
+[NOTE]
+====
+You can only restore a database that has been backed up.
+Please refer to the xref:how-tos/use-built-in-db.adoc#_enable_backups[built-in database] setup guide on how to enable backups.
+====
+
+====
+Requirements
+
+* `kubctl` With access to the cluster running Keycloak
+* `base64`
+* https://restic.net/[`restic`]
+* (optional) https://stedolan.github.io/jq/[`jq`]
+====
+
+. Configure access to backups
++
+[source,bash]
+----
+# The namspace containing the keycloak instance. Change if necessary.
+export NAMESPACE=syn-keycloak
+
+export reposecret=$( \
+    kubectl -n $NAMESPACE get schedule backup \
+    -o go-template="{{.spec.backend.repoPasswordSecretRef.name}}" \
+  )
+export s3secret=$( \
+    kubectl -n $NAMESPACE get schedule backup \
+    -o jsonpath="{.spec.backend.s3.accessKeyIDSecretRef.name}" \
+  )
+
+export RESTIC_REPOSITORY=$( \
+    kubectl -n $NAMESPACE get schedule backup \
+    -o go-template="s3:{{.spec.backend.s3.endpoint}}/{{.spec.backend.s3.bucket}}/" \
+  )
+export RESTIC_PASSWORD=$( \
+    kubectl -n $NAMESPACE get secrets $reposecret \
+    -o jsonpath={.data.password} \
+    | base64 -d \
+  )
+export AWS_ACCESS_KEY_ID=$( \
+    kubectl -n $NAMESPACE get secrets $s3secret \
+    -o jsonpath="{.data.username}" \
+    | base64 -d \
+  )
+export AWS_SECRET_ACCESS_KEY=$( \
+    kubectl -n $NAMESPACE get secrets $s3secret \
+    -o jsonpath="{.data.password}" \
+    | base64 -d \
+  )
+----
+
+. List backups and choose the one to restore
++
+[source,bash]
+----
+restic snapshots
+
+export SNAPSHOT_ID=XXXXXX # Choose a snapshot id from the list
+----
++
+[TIP]
+====
+To choose the last available backup you can simply run
+[source,bash]
+----
+export SNAPSHOT_ID=$(restic snapshots --json --latest 1 --path /$NAMESPACE-keycloak-postgresql.sql | jq -r '.[0].id')
+----
+====
+
+. Load the backup and restore it
++
+[source,bash]
+----
+export POD=keycloak-postgresql-0
+
+restic dump "${SNAPSHOT_ID}" /$NAMESPACE-keycloak-postgresql.sql \
+  | kubectl -n $NAMESPACE exec -i $POD \
+  -- bash -c 'PGPASSWORD="${POSTGRES_PASSWORD}" psql --set ON_ERROR_STOP=on -U "${POSTGRES_USER}" ${POSTGRES_DB}'
+----
+
+[NOTE]
+====
+This guide assumes that you have direct access to the S3 bucket holding the backup.
+If the access is restricted to the Kubernetes cluster, you will need to adapt these steps.
+You could:
+
+. Perform the commands in a container running on the cluster
+. Restore the database backup to a PVC and copy if over
+
+Consult the official https://k8up.io/k8up/2.1/how-tos/restore.html#_restore_from_s3_to_pvc[K8up documentation] on options for restoring backups.
+====
+
+
diff --git a/docs/modules/ROOT/partials/nav.adoc b/docs/modules/ROOT/partials/nav.adoc
index eb747285..7052259e 100644
--- a/docs/modules/ROOT/partials/nav.adoc
+++ b/docs/modules/ROOT/partials/nav.adoc
@@ -12,6 +12,7 @@
 * xref:how-tos/configure-ingress.adoc[Configure Keycloak ingress]
 * xref:how-tos/custom-theme.adoc[Configure custom theme]
 * xref:how-tos/change-passwords.adoc[Change passwords]
+* xref:how-tos/restore.adoc[Restore a backup]
 * xref:how-tos/upgrade-1.x-to-2.x.adoc[Upgrade 1.x to 2.x]
 * xref:how-tos/upgrade-2.x-to-3.x.adoc[Upgrade 2.x to 3.x]
 * xref:how-tos/upgrade-3.x-to-4.x.adoc[Upgrade 3.x to 4.x]

From c4ffb1292fd3c8fc6fa4670a1ec7319e9cfbe589 Mon Sep 17 00:00:00 2001
From: Fabian Fischer <10788152+glrf@users.noreply.github.com>
Date: Wed, 15 Dec 2021 15:19:09 +0100
Subject: [PATCH 3/4] Apply suggestions from code review

Co-authored-by: Chris <github.account@chrigel.net>
Co-authored-by: Simon Gerber <simon.gerber@vshn.ch>
---
 docs/modules/ROOT/pages/how-tos/restore.adoc         | 10 ++++++----
 docs/modules/ROOT/pages/how-tos/use-built-in-db.adoc |  6 +++---
 docs/modules/ROOT/partials/nav.adoc                  |  2 +-
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/docs/modules/ROOT/pages/how-tos/restore.adoc b/docs/modules/ROOT/pages/how-tos/restore.adoc
index a021be2e..39d1f78d 100644
--- a/docs/modules/ROOT/pages/how-tos/restore.adoc
+++ b/docs/modules/ROOT/pages/how-tos/restore.adoc
@@ -1,4 +1,4 @@
-= Restore Keycloak
+= Restore Keycloak from a Backup
 
 The following steps will guide you through restoring a backup of Keycloak.
 
@@ -17,10 +17,12 @@ Please refer to the xref:how-tos/use-built-in-db.adoc#_enable_backups[built-in d
 ====
 Requirements
 
-* `kubctl` With access to the cluster running Keycloak
+* `kubectl` With access to the cluster running Keycloak
 * `base64`
 * https://restic.net/[`restic`]
 * (optional) https://stedolan.github.io/jq/[`jq`]
+* `pwgen`
+* `vault`
 ====
 
 . Configure access to backups
@@ -86,7 +88,7 @@ export POD=keycloak-postgresql-0
 
 restic dump "${SNAPSHOT_ID}" /$NAMESPACE-keycloak-postgresql.sql \
   | kubectl -n $NAMESPACE exec -i $POD \
-  -- bash -c 'PGPASSWORD="${POSTGRES_PASSWORD}" psql --set ON_ERROR_STOP=on -U "${POSTGRES_USER}" ${POSTGRES_DB}'
+  -- sh -c 'PGPASSWORD="${POSTGRES_PASSWORD}" psql --set ON_ERROR_STOP=on -U "${POSTGRES_USER}" ${POSTGRES_DB}'
 ----
 
 [NOTE]
@@ -96,7 +98,7 @@ If the access is restricted to the Kubernetes cluster, you will need to adapt th
 You could:
 
 . Perform the commands in a container running on the cluster
-. Restore the database backup to a PVC and copy if over
+. Restore the database backup to a PVC and copy it over
 
 Consult the official https://k8up.io/k8up/2.1/how-tos/restore.html#_restore_from_s3_to_pvc[K8up documentation] on options for restoring backups.
 ====
diff --git a/docs/modules/ROOT/pages/how-tos/use-built-in-db.adoc b/docs/modules/ROOT/pages/how-tos/use-built-in-db.adoc
index dc720dc8..be3498fc 100644
--- a/docs/modules/ROOT/pages/how-tos/use-built-in-db.adoc
+++ b/docs/modules/ROOT/pages/how-tos/use-built-in-db.adoc
@@ -62,7 +62,7 @@ keycloak:
 ----
 key="clusters/kv/${TENANT_ID}/${CLUSTER_ID}/keycloak"
 
-vault kv put -cas=0 "${key}" k8up-repo-password=$(pwgen -s 32 1)
+vault kv patch "${key}" k8up-repo-password=$(pwgen -s 32 1)
 ----
 
 . Get the access and secret key of your S3 provider and store them in Vault
@@ -73,12 +73,12 @@ s3_access_key=YOUR_ACCESS_KEY
 s3_secret_key=YOUR_SECRET_KEY
 key="clusters/kv/${TENANT_ID}/${CLUSTER_ID}/keycloak"
 
-vault kv put -cas=0 "${key}" k8up-s3-accesskey=${s3_access_key} k8up-s3-secretkey=${s3_secret_key}
+vault kv patch "${key}" k8up-s3-accesskey=${s3_access_key} k8up-s3-secretkey=${s3_secret_key} k8up-repo-password=$(pwgen -s 32 1)
 ----
 +
 [TIP]
 ====
-On most clusters you should be able to reuse the global backup credentials setup during cluster creation.
+On most clusters you should be able to reuse the global backup credentials set up during cluster creation.
 Just add a reference to the credentials in Vault to the K8up S3 configuration and you can skip this step.
 
 [source,yaml]
diff --git a/docs/modules/ROOT/partials/nav.adoc b/docs/modules/ROOT/partials/nav.adoc
index 7052259e..30c003d5 100644
--- a/docs/modules/ROOT/partials/nav.adoc
+++ b/docs/modules/ROOT/partials/nav.adoc
@@ -12,7 +12,7 @@
 * xref:how-tos/configure-ingress.adoc[Configure Keycloak ingress]
 * xref:how-tos/custom-theme.adoc[Configure custom theme]
 * xref:how-tos/change-passwords.adoc[Change passwords]
-* xref:how-tos/restore.adoc[Restore a backup]
+* xref:how-tos/restore.adoc[Restore from a Backup]
 * xref:how-tos/upgrade-1.x-to-2.x.adoc[Upgrade 1.x to 2.x]
 * xref:how-tos/upgrade-2.x-to-3.x.adoc[Upgrade 2.x to 3.x]
 * xref:how-tos/upgrade-3.x-to-4.x.adoc[Upgrade 3.x to 4.x]

From 59aa8dc7fa686584cdd87b5df616abd3bbbe849c Mon Sep 17 00:00:00 2001
From: Fabian Fischer <glrf@users.noreply.github.com>
Date: Wed, 15 Dec 2021 15:54:08 +0100
Subject: [PATCH 4/4] Scale down Keycloak before restoring a backup

---
 docs/modules/ROOT/pages/how-tos/restore.adoc | 41 +++++++++++++++++++-
 1 file changed, 40 insertions(+), 1 deletion(-)

diff --git a/docs/modules/ROOT/pages/how-tos/restore.adoc b/docs/modules/ROOT/pages/how-tos/restore.adoc
index 39d1f78d..21c8ffdc 100644
--- a/docs/modules/ROOT/pages/how-tos/restore.adoc
+++ b/docs/modules/ROOT/pages/how-tos/restore.adoc
@@ -29,7 +29,7 @@ Requirements
 +
 [source,bash]
 ----
-# The namspace containing the keycloak instance. Change if necessary.
+# The namspace containing the Keycloak instance. Change if necessary.
 export NAMESPACE=syn-keycloak
 
 export reposecret=$( \
@@ -80,6 +80,30 @@ export SNAPSHOT_ID=$(restic snapshots --json --latest 1 --path /$NAMESPACE-keycl
 ----
 ====
 
+. Disable ArgoCD auto sync
++
+[source,bash]
+----
+# The ArgoCD app of the Keycloak instance. Change if necessary.
+export ARGO_APP=keycloak
+
+kubectl -n syn patch apps root --type=json \
+  -p '[{"op":"replace", "path":"/spec/syncPolicy", "value": {}}]'
+kubectl -n syn patch apps ${ARGO_APP} --type=json \
+  -p '[{"op":"replace", "path":"/spec/syncPolicy", "value": {}}]'
+----
+
+. Scale down Keycloak
++
+[source,bash]
+----
+kubectl -n $NAMESPACE patch statefulset keycloak --type=json \
+  -p '[{"op":"replace", "path":"/spec/replicas", "value": 0}]'
+
+# Wait until statefulset has been scaled down
+kubectl -n $NAMESPACE get statefulset keycloak -w
+----
+
 . Load the backup and restore it
 +
 [source,bash]
@@ -91,6 +115,21 @@ restic dump "${SNAPSHOT_ID}" /$NAMESPACE-keycloak-postgresql.sql \
   -- sh -c 'PGPASSWORD="${POSTGRES_PASSWORD}" psql --set ON_ERROR_STOP=on -U "${POSTGRES_USER}" ${POSTGRES_DB}'
 ----
 
+. Re-enable ArgoCD auto sync and scale up Keycloak
++
+[source,bash]
+----
+kubectl -n syn patch apps root --type=json \
+  -p '[{
+    "op":"replace",
+    "path":"/spec/syncPolicy",
+    "value": {"automated": {"prune": true, "selfHeal": true}}
+  }]'
+
+# Wait until Keycloak has started sucessfully
+kubectl -n $NAMESPACE get statefulset keycloak -w
+----
+
 [NOTE]
 ====
 This guide assumes that you have direct access to the S3 bucket holding the backup.