Skip to content

Commit

Permalink
kubeconfig_configure refactor
Browse files Browse the repository at this point in the history
  • Loading branch information
@kzadorozhny
kzadorozhny committed Mar 3, 2021
1 parent 107ead8 commit 048341b
Show file tree
Hide file tree
Showing 6 changed files with 398 additions and 236 deletions.
8 changes: 4 additions & 4 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ When you run `bazel run ///helloworld:mynamespace.apply`, it applies this file i
| ***objects*** | `[]` | A list of other instances of `k8s_deploy` that this one depends on. See [Adding Dependencies](#adding-dependencies).
| ***images*** | `{}` | A dict of labels of Docker images. See [Injecting Docker Images](#injecting-docker-images).
| ***image_digest_tag*** | `False` | A flag for whether or not to tag the image with the container digest.
| ***image_registry*** | `docker.io` | The registry to push images to.
| ***image_registry*** | `docker.io` | The registry to push images to.
| ***image_repository*** | `None` | The repository to push images to. By default, this is generated from the current package path.
| ***image_repository_prefix*** | `None` | Add a prefix to the image_repository. Can be used to upload the images in
| ***release_branch_prefix*** | `master` | A git branch name/prefix. Automatically run GitOps while building this branch. See [GitOps and Deployment](#gitops_and_deployment).
Expand Down Expand Up @@ -288,7 +288,7 @@ spec:
- name: java_container
image: registry.example.com/examples/image@sha256:c94d75d68f4c1b436f545729bbce82774fda07
```
Image substitutions for Custom Resource Definitions (CRD) resources could also use target references directly. Their digests are availabe through string substitution. For example,
Image substitutions for Custom Resource Definitions (CRD) resources could also use target references directly. Their digests are availabe through string substitution. For example,
```yaml
apiVersion: v1
kind: MyCrd
Expand All @@ -300,7 +300,7 @@ metadata:
spec:
image: "{{//example:my_image}}"
```
would become
would become
```yaml
apiVersion: v1
kind: MyCrd
Expand Down Expand Up @@ -381,7 +381,7 @@ The `k8s_test_setup` rule produces a shell script which creates a temporary name

The output of the `k8s_test_setup` rule (a shell script) is referenced in the `java_test` rule. It's listed under the `data` attribute, which declares the target as a dependency, and is included in the jvm flags in this clause: `$(location :service_it.setup)`. The "location" function is specific to Bazel: given a target, it returns the path to the file produced by that target. In this case, it returns the path to the shell script created by our `k8s_test_setup` rule.

The test code launches the script to perform the test setup. The tes code should also monitor the script console output to listen to the pod readiness events.
The test code launches the script to perform the test setup. The test code should also monitor the script console output to listen to the pod readiness events.


## Building & Testing
Expand Down
4 changes: 2 additions & 2 deletions examples/WORKSPACE
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,9 @@ go_image_repositories()
#
# Kubeconfig repository that will be used in k8s_test_setup
#
load("@com_adobe_rules_gitops//gitops:defs.bzl", "kubeconfig")
load("@com_adobe_rules_gitops//gitops:defs.bzl", "kubeconfig_configure")

kubeconfig(
kubeconfig_configure(
name = "k8s_test",
cluster = "kind-kind",
user = "kind-kind", # kind cluster user name
Expand Down
5 changes: 5 additions & 0 deletions gitops/defs.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,14 @@ load(
"@com_adobe_rules_gitops//skylib:k8s.bzl",
_k8s_deploy = "k8s_deploy",
_k8s_test_setup = "k8s_test_setup",
)
load(
"@com_adobe_rules_gitops//skylib:kubeconfig.bzl",
_kubeconfig = "kubeconfig",
_kubeconfig_configure = "kubeconfig_configure",
)

k8s_deploy = _k8s_deploy
k8s_test_setup = _k8s_test_setup
kubeconfig = _kubeconfig
kubeconfig_configure = _kubeconfig_configure
214 changes: 1 addition & 213 deletions skylib/k8s.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ load(
"kustomize",
kustomize_gitops = "gitops",
)
load("//skylib:kubeconfig.bzl", "KubeconfigInfo")
load("//skylib:push.bzl", "k8s_container_push")

def _runfiles(ctx, f):
Expand Down Expand Up @@ -275,219 +276,6 @@ def k8s_deploy(
visibility = visibility,
)

KubeconfigInfo = provider(fields = [
"server",
"cluster",
"user",
])

def _kubeconfig_file_impl(ctx):
kubeconfig_file = ctx.actions.declare_file(ctx.label.name)
ctx.actions.symlink(output = kubeconfig_file, target_file = ctx.file.config)
files = depset(direct = [kubeconfig_file])
runfiles = ctx.runfiles(files = [kubeconfig_file])
return [
DefaultInfo(files = files, runfiles = runfiles),
KubeconfigInfo(
server = ctx.attr.server,
cluster = ctx.attr.cluster,
user = ctx.attr.user,
),
]

kubeconfig_file = rule(
implementation = _kubeconfig_file_impl,
attrs = {
"config": attr.label(
doc = "Config file.",
allow_single_file = True,
mandatory = True,
),
"server": attr.string(
doc = "Optional Kubernetes server url.",
mandatory = False,
),
"cluster": attr.string(
doc = "Optional Kubernetes cluster name.",
mandatory = False,
),
"user": attr.string(
doc = "Optional Kubernetes user name.",
mandatory = True,
),
},
provides = [DefaultInfo, KubeconfigInfo],
)

_KUBECONFIG_BUILD_TEMPLATE = """# Generated by kubeconfig repostiory rule
load("@com_adobe_rules_gitops//skylib:k8s.bzl", "kubeconfig_file")
exports_files(["kubectl"])
kubeconfig_file(
name = "kubeconfig",
config = ":config",
server = "{server}",
cluster = "{cluster}",
user = "{user}",
visibility = ["//visibility:public"],
)
"""

# kubectl template
def _kubectl_config(repository_ctx, args):
kubectl = repository_ctx.path("kubectl")
kubeconfig_yaml = repository_ctx.path("config")
exec_result = repository_ctx.execute(
[kubectl, "--kubeconfig", kubeconfig_yaml, "config"] + args,
environment = {
# prevent kubectl config to stumble on shared .kube/config.lock file
"HOME": str(repository_ctx.path(".")),
},
quiet = True,
)
if exec_result.return_code != 0:
fail("Error executing kubectl config %s" % " ".join(args))

def _kubeconfig_impl(repository_ctx):
"""Find local kubernetes certificates"""

# find and symlink kubectl
kubectl = repository_ctx.which("kubectl")
if not kubectl:
fail("Unable to find kubectl executable. PATH=%s" % repository_ctx.path)
repository_ctx.symlink(kubectl, "kubectl")

home = repository_ctx.path(repository_ctx.os.environ["HOME"])

# use provided user name or fall back to current os user name
if repository_ctx.attr.user:
user = repository_ctx.attr.user
elif "USER" in repository_ctx.os.environ:
user = repository_ctx.os.environ["USER"]
else:
exec_result = repository_ctx.execute(["whoami"])
if exec_result.return_code != 0:
fail("Error detecting current user")
user = exec_result.stdout.rstrip()

token = None
ca_crt = None
kubecert_cert = None
kubecert_key = None
server = repository_ctx.attr.server
config = None

# check service account first
serviceaccount = repository_ctx.path("/var/run/secrets/kubernetes.io/serviceaccount")
if serviceaccount.exists:
ca_crt = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
token_file = serviceaccount.get_child("token")
if token_file.exists:
exec_result = repository_ctx.execute(["cat", token_file.realpath])
if exec_result.return_code != 0:
fail("Error reading user token")
token = exec_result.stdout.rstrip()

# use master url from the environemnt
if "KUBERNETES_SERVICE_HOST" in repository_ctx.os.environ:
server = "https://%s:%s" % (
repository_ctx.os.environ["KUBERNETES_SERVICE_HOST"],
repository_ctx.os.environ["KUBERNETES_SERVICE_PORT"],
)
else:
# fall back to the default
server = "https://kubernetes.default"
else:
# check kubectl config file
config = home.get_child(".kube").get_child("config")

if config and config.exists:
# symlink ~/.kube/config if this file exists
repository_ctx.symlink(config, repository_ctx.path("config"))
else:
# generate new config file service account token or certificates
certs = home.get_child(".kube").get_child("certs")
ca_crt = certs.get_child("ca.crt").realpath
kubecert_cert = certs.get_child("kubecert.cert")
kubecert_key = certs.get_child("kubecert.key")

# config set-cluster {cluster} \
# --certificate-authority=... \
# --server=https://dev3.k8s.tubemogul.info:443 \
# --embed-certs",
_kubectl_config(repository_ctx, [
"set-cluster",
repository_ctx.attr.cluster,
"--server",
server,
"--certificate-authority",
ca_crt,
])

# config set-credentials {user} --token=...",
if token:
_kubectl_config(repository_ctx, [
"set-credentials",
user,
"--token",
token,
])

# config set-credentials {user} --client-certificate=...",
if kubecert_cert and kubecert_cert.exists:
_kubectl_config(repository_ctx, [
"set-credentials",
user,
"--client-certificate",
kubecert_cert.realpath,
])

# config set-credentials {user} --client-key=...",
if kubecert_key and kubecert_key.exists:
_kubectl_config(repository_ctx, [
"set-credentials",
user,
"--client-key",
kubecert_key.realpath,
])

# export repostory contents
repository_ctx.file("BUILD", _KUBECONFIG_BUILD_TEMPLATE.format(
cluster = repository_ctx.attr.cluster,
server = repository_ctx.attr.server,
user = user,
), False)

return {
"cluster": repository_ctx.attr.cluster,
"server": repository_ctx.attr.server,
"user": user,
}

kubeconfig = repository_rule(
attrs = {
"cluster": attr.string(
mandatory = True,
),
"server": attr.string(
mandatory = False,
),
"user": attr.string(
mandatory = False,
),
},
environ = [
"HOME",
"USER",
"KUBERNETES_SERVICE_HOST",
"KUBERNETES_SERVICE_PORT",
],
local = True,
implementation = _kubeconfig_impl,
)

def _stamp(ctx, string, output):
stamps = [ctx.file._info_file]
stamp_args = [
Expand Down
43 changes: 26 additions & 17 deletions skylib/k8s_test_namespace.sh.tpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
# governing permissions and limitations under the License.

set -euo pipefail

[ -o xtrace ] && env

function guess_runfiles() {
Expand Down Expand Up @@ -39,8 +40,25 @@ echo "Cluster: ${CLUSTER}" >&2
# use BUILD_USER by defalt
USER=${USER:-$BUILD_USER}

# create miniified self-contained kubectl configuration with the default context set to use newly created namespace
mkdir -p $(dirname $KUBECONFIG_FILE)

# create context partion of new kubeconfig file from scratch
# use --kubeconfig parameter to prevent any merging
# create
rm -f $KUBECONFIG_FILE-context
CONTEXT=$CLUSTER-$BUILD_USER
kubectl --kubeconfig=$KUBECONFIG_FILE-context --cluster=$CLUSTER --server=$SERVER --user=$USER --namespace=$BUILD_USER config set-context $CONTEXT >&2
kubectl --kubeconfig=$KUBECONFIG_FILE-context config use-context $CONTEXT >&2

# merge newly generated context with system kubeconfig, flatten and minify the result
KUBECONFIG=$KUBECONFIG_FILE-context:$KUBECONFIG kubectl config view --merge=true --minify=true --flatten=true --raw >$KUBECONFIG_FILE

# set generated kubeconfig for all following kubectl commands
export KUBECONFIG=$KUBECONFIG_FILE

# check if username from provided configuration exists
KUBECONFIG_USER=$(${KUBECTL} --kubeconfig=${KUBECONFIG} config view -o jsonpath='{.users[?(@.name == '"\"${USER}\")].name}")
KUBECONFIG_USER=$(${KUBECTL} config view -o jsonpath='{.users[?(@.name == '"\"${USER}\")].name}")
if [ -z "${KUBECONFIG_USER}" ]; then
echo "Unable to find user configuration ${USER} for cluster ${CLUSTER}" >&2
exit 1
Expand All @@ -64,13 +82,18 @@ else
COUNT="0"
while true; do
NAMESPACE=${BUILD_USER}-$(( (RANDOM) + 32767 ))
${KUBECTL} --kubeconfig=${KUBECONFIG} --cluster=${CLUSTER} --server=${SERVER} --user=${USER} create namespace ${NAMESPACE} && break
${KUBECTL} create namespace ${NAMESPACE} && break
COUNT=$[$COUNT + 1]
if [ $COUNT -ge 10 ]; then
echo "Unable to create namespace in $COUNT attempts!" >&2
exit 1
fi
done
# update context with created test namespace
kubectl --namespace=$NAMESPACE config set-context $CONTEXT >&2

# rename test context (Note: this is required for backward compatibiliy)
kubectl config rename-context $CONTEXT $CLUSTER-$NAMESPACE >&2
fi
echo "Namespace: ${NAMESPACE}" >&2
set -e
Expand All @@ -79,21 +102,7 @@ set -e
mkdir -p $(dirname $NAMESPACE_NAME_FILE)
echo $NAMESPACE > $NAMESPACE_NAME_FILE

# create miniified self-contained kubectl configuration with the default context set to use newly created namespace
mkdir -p $(dirname $KUBECONFIG_FILE)

# create context partion of new kubeconfig file from scratch
# use --kubeconfig parameter to prevent any merging
rm -f $KUBECONFIG_FILE-context
CONTEXT=$CLUSTER-$NAMESPACE
kubectl --kubeconfig=$KUBECONFIG_FILE-context --cluster=$CLUSTER --server=${SERVER} --user=$USER --namespace=$NAMESPACE config set-context $CONTEXT >&2
kubectl --kubeconfig=$KUBECONFIG_FILE-context config use-context $CONTEXT >&2

# merge newly generated context with system kubeconfig, flatten and minify the result
KUBECONFIG=$KUBECONFIG_FILE-context:$KUBECONFIG kubectl config view --merge=true --minify=true --flatten=true --raw >$KUBECONFIG_FILE

# set generated kubeconfig for all following kubectl commands
export KUBECONFIG=$KUBECONFIG_FILE
[ -o xtrace ] && kubectl config view >&2

# set runfiles for STMTS
export PYTHON_RUNFILES=${RUNFILES}
Expand Down
Loading

0 comments on commit 048341b

Please sign in to comment.