diff --git a/.gitignore b/.gitignore index d0030f26b..0e6da5a7d 100644 --- a/.gitignore +++ b/.gitignore @@ -78,4 +78,5 @@ tags .idea vendor -zz_generated*.go \ No newline at end of file +zz_generated*.go +__pycache__ diff --git a/deploy/testrunner/cluster_role_binding.yaml b/deploy/testrunner/cluster_role_binding.yaml new file mode 100644 index 000000000..349a9531f --- /dev/null +++ b/deploy/testrunner/cluster_role_binding.yaml @@ -0,0 +1,12 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-runner +subjects: +- kind: ServiceAccount + name: test-runner + namespace: default +roleRef: + kind: ClusterRole + name: cluster-admin # TODO: create cluster role with only required permissions + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/testrunner/role.yaml b/deploy/testrunner/role.yaml index 05956c7f7..996c87b34 100644 --- a/deploy/testrunner/role.yaml +++ b/deploy/testrunner/role.yaml @@ -80,20 +80,3 @@ rules: - patch - update - watch - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: test-runner -rules: - - apiGroups: - - '*' - resources: - - '*' - verbs: - - '*' - - nonResourceURLs: - - '*' - verbs: - - '*' diff --git a/deploy/testrunner/role_binding.yaml b/deploy/testrunner/role_binding.yaml index bdd46f11e..ab4d3ee22 100644 --- a/deploy/testrunner/role_binding.yaml +++ b/deploy/testrunner/role_binding.yaml @@ -9,17 +9,3 @@ roleRef: kind: Role name: test-runner apiGroup: rbac.authorization.k8s.io - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: test-runner -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: test-runner -subjects: - - kind: ServiceAccount - name: test-runner - namespace: default diff --git a/docker/requirements.txt b/docker/requirements.txt deleted file mode 100644 index 60ffe02f1..000000000 --- a/docker/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -jinja2==2.11.2 diff --git a/scripts/ci/build_and_push_image.sh b/scripts/ci/build_and_push_image.sh index bd7e2e2dd..b18a1a9d7 100755 --- a/scripts/ci/build_and_push_image.sh +++ b/scripts/ci/build_and_push_image.sh @@ -2,6 +2,6 @@ echo ${quay_password} | docker login -u=${quay_user_name} quay.io --password-stdin -python docker/dockerfile_generator.py ${image_type} > Dockerfile +python scripts/dev/dockerfile_generator.py ${image_type} > Dockerfile docker build . -f Dockerfile -t ${image} docker push ${image} diff --git a/scripts/ci/run_unit_tests.sh b/scripts/ci/run_unit_tests.sh index 7f326da91..c2998647f 100755 --- a/scripts/ci/run_unit_tests.sh +++ b/scripts/ci/run_unit_tests.sh @@ -1,5 +1,5 @@ #!/bin/sh -python docker/dockerfile_generator.py "unittest" > Dockerfile +python scripts/dev/dockerfile_generator.py "unittest" > Dockerfile docker build . -f Dockerfile -t unit-tests:${version_id} docker run unit-tests:${version_id} diff --git a/scripts/dev/__init__.py b/scripts/dev/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/dev/build_and_deploy_operator.py b/scripts/dev/build_and_deploy_operator.py new file mode 100644 index 000000000..9f53101c0 --- /dev/null +++ b/scripts/dev/build_and_deploy_operator.py @@ -0,0 +1,138 @@ +from dockerutil import build_and_push_image +from dockerfile_generator import render +from kubernetes import client, config +from kubernetes.client.rest import ApiException +from dev_config import DevConfig, load_config +from typing import Dict, Optional +import yaml +import io +import os +import time + + +def _load_operator_service_account() -> Optional[Dict]: + return load_yaml_from_file("deploy/service_account.yaml") + + +def _load_operator_role() -> Optional[Dict]: + return load_yaml_from_file("deploy/role.yaml") + + +def _load_operator_role_binding() -> Optional[Dict]: + return load_yaml_from_file("deploy/role_binding.yaml") + + +def _load_operator_deployment() -> Optional[Dict]: + return load_yaml_from_file("deploy/operator.yaml") + + +def _load_mongodb_crd() -> Optional[Dict]: + return load_yaml_from_file("deploy/crds/mongodb.com_mongodbs_crd.yaml") + + +def load_yaml_from_file(path: str) -> Optional[Dict]: + with open(path, "r") as f: + return yaml.full_load(f.read()) + return None + + +def _ensure_crds(): + """ + ensure_crds makes sure that all the required CRDs have been created + """ + crdv1 = client.ApiextensionsV1beta1Api() + crd = _load_mongodb_crd() + + ignore_if_doesnt_exist( + lambda: crdv1.delete_custom_resource_definition("mongodbs.mongodb.com") + ) + + # TODO: fix this, when calling create_custom_resource_definition, we get the error + # ValueError("Invalid value for `conditions`, must not be `None`") + # but the crd is still successfully created + try: + crdv1.create_custom_resource_definition(body=crd) + except ValueError as e: + pass + + print("Ensured CRDs") + + +def build_and_push_operator(repo_url: str, tag: str, path: str): + """ + build_and_push_operator creates the Dockerfile for the operator + and pushes it to the target repo + """ + return build_and_push_image(repo_url, tag, path, "operator") + + +def _ignore_error_codes(fn, codes): + try: + fn() + except ApiException as e: + if e.status not in codes: + raise + + +def ignore_if_already_exists(fn): + """ + ignore_if_already_exists accepts a function and calls it, + ignoring an Kubernetes API conflict errors + """ + + return _ignore_error_codes(fn, [409]) + + +def ignore_if_doesnt_exist(fn): + """ + ignore_if_doesnt_exist accepts a function and calls it, + ignoring an Kubernetes API not found errors + """ + return _ignore_error_codes(fn, [404]) + + +def deploy_operator(): + """ + deploy_operator ensures the CRDs are created, and als creates all the required ServiceAccounts, Roles + and RoleBindings for the operator, and then creates the operator deployment. + """ + appsv1 = client.AppsV1Api() + corev1 = client.CoreV1Api() + rbacv1 = client.RbacAuthorizationV1Api() + + dev_config = load_config() + _ensure_crds() + + ignore_if_already_exists( + lambda: rbacv1.create_namespaced_role( + dev_config.namespace, _load_operator_role() + ) + ) + ignore_if_already_exists( + lambda: rbacv1.create_namespaced_role_binding( + dev_config.namespace, _load_operator_role_binding() + ) + ) + ignore_if_already_exists( + lambda: corev1.create_namespaced_service_account( + dev_config.namespace, _load_operator_service_account() + ) + ) + ignore_if_already_exists( + lambda: appsv1.create_namespaced_deployment( + dev_config.namespace, _load_operator_deployment() + ) + ) + + +def main(): + config.load_kube_config() + dev_config = load_config() + build_and_push_operator( + dev_config.repo_url, f"{dev_config.repo_url}/mongodb-kubernetes-operator", "." + ) + deploy_operator() + + +if __name__ == "__main__": + main() diff --git a/scripts/dev/dev_config.py b/scripts/dev/dev_config.py new file mode 100644 index 000000000..c1fa476a5 --- /dev/null +++ b/scripts/dev/dev_config.py @@ -0,0 +1,33 @@ +from typing import Dict, Optional +import json +import os + +CONFIG_PATH = "~/.community-operator-dev/config.json" +FULL_CONFIG_PATH = os.path.expanduser(CONFIG_PATH) + + +class DevConfig: + """ + DevConfig is a wrapper around the developer configuration file + """ + + def __init__(self, config): + self._config = config + + @property + def namespace(self): + return self._config["namespace"] + + @property + def repo_url(self): + return self._config["repo_url"] + + +def load_config() -> Optional[DevConfig]: + with open(FULL_CONFIG_PATH, "r") as f: + return DevConfig(json.loads(f.read())) + + print( + f"No DevConfig found. Please ensure that the configuration file exists at '{FULL_CONFIG_PATH}'" + ) + return None diff --git a/docker/dockerfile_generator.py b/scripts/dev/dockerfile_generator.py similarity index 80% rename from docker/dockerfile_generator.py rename to scripts/dev/dockerfile_generator.py index 756a2ef1f..f981c17d7 100755 --- a/docker/dockerfile_generator.py +++ b/scripts/dev/dockerfile_generator.py @@ -13,14 +13,14 @@ def operator_params(): def test_runner_params(): return { "builder": True, - "builder_image": "golang", # TODO: make this image smaller. There were errors using alpine + "builder_image": "golang", # TODO: make this image smaller. There were errors using alpine "base_image": "registry.access.redhat.com/ubi8/ubi-minimal:latest", } def e2e_params(): return { - "base_image": "golang", # TODO: make this image smaller, error: 'exec: "gcc": executable file not found in $PATH' with golang:alpine + "base_image": "golang", # TODO: make this image smaller, error: 'exec: "gcc": executable file not found in $PATH' with golang:alpine } @@ -46,7 +46,7 @@ def render(image_name): ) env = jinja2.Environment() - env.loader = jinja2.FileSystemLoader(searchpath="docker/templates") + env.loader = jinja2.FileSystemLoader(searchpath="scripts/dev/templates") return env.get_template("Dockerfile.{}".format(image_name)).render( param_dict[image_name] ) diff --git a/scripts/dev/dockerutil.py b/scripts/dev/dockerutil.py new file mode 100644 index 000000000..69bb174ac --- /dev/null +++ b/scripts/dev/dockerutil.py @@ -0,0 +1,38 @@ +import docker +from dockerfile_generator import render +import os + + +def build_image(repo_url: str, tag: str, path): + """ + build_image builds the image with the given tag + """ + client = docker.from_env() + print(f"Building image: {tag}") + client.images.build(tag=tag, path=path) + print("Successfully built image!") + + +def push_image(tag: str): + """ + push_image pushes the given tag. It uses + the current docker environment + """ + client = docker.from_env() + print(f"Pushing image: {tag}") + for line in client.images.push(tag, stream=True): + print(line.decode("utf-8").rstrip()) + + +def build_and_push_image(repo_url: str, tag: str, path: str, image_type: str): + """ + build_and_push_operator creates the Dockerfile for the operator + and pushes it to the target repo + """ + dockerfile_text = render(image_type) + with open(f"{path}/Dockerfile", "w") as f: + f.write(dockerfile_text) + + build_image(repo_url, tag, path) + os.remove(f"{path}/Dockerfile") + push_image(tag) diff --git a/scripts/dev/e2e.py b/scripts/dev/e2e.py new file mode 100644 index 000000000..3abd00dc0 --- /dev/null +++ b/scripts/dev/e2e.py @@ -0,0 +1,181 @@ +from build_and_deploy_operator import ( + ignore_if_doesnt_exist, + ignore_if_already_exists, + load_yaml_from_file, +) # TODO: put these function somewhere else +from dockerutil import build_and_push_image +from typing import Dict, Optional +from dev_config import load_config +from kubernetes import client, config +import argparse +import time + +TEST_RUNNER_NAME = "test-runner" + + +def _load_testrunner_service_account() -> Optional[Dict]: + return load_yaml_from_file("deploy/testrunner/service_account.yaml") + + +def _load_testrunner_role() -> Optional[Dict]: + return load_yaml_from_file("deploy/testrunner/role.yaml") + + +def _load_testrunner_role_binding() -> Optional[Dict]: + return load_yaml_from_file("deploy/testrunner/role_binding.yaml") + + +def _load_testrunner_cluster_role_binding() -> Optional[Dict]: + return load_yaml_from_file("deploy/testrunner/cluster_role_binding.yaml") + + +def _prepare_testrunner_environment(): + """ + _prepare_testrunner_environment ensures the ServiceAccount, + Role and ClusterRole and bindings are created for the test runner. + """ + rbacv1 = client.RbacAuthorizationV1Api() + corev1 = client.CoreV1Api() + dev_config = load_config() + + _delete_testrunner_pod() + + print("Creating Role") + ignore_if_already_exists( + lambda: rbacv1.create_namespaced_role( + dev_config.namespace, _load_testrunner_role() + ) + ) + + print("Creating Role Binding") + ignore_if_already_exists( + lambda: rbacv1.create_namespaced_role_binding( + dev_config.namespace, _load_testrunner_role_binding() + ) + ) + + print("Creating Cluster Role Binding") + ignore_if_already_exists( + lambda: rbacv1.create_cluster_role_binding( + _load_testrunner_cluster_role_binding() + ) + ) + + print("Creating ServiceAccount") + ignore_if_already_exists( + lambda: corev1.create_namespaced_service_account( + dev_config.namespace, _load_testrunner_service_account() + ) + ) + + +def build_and_push_testrunner(repo_url: str, tag: str, path: str): + """ + build_and_push_testrunner builds and pushes the test runner + image. + """ + return build_and_push_image(repo_url, tag, path, "testrunner") + + +def build_and_push_e2e(repo_url: str, tag: str, path: str): + """ + build_and_push_e2e builds and pushes the e2e image. + """ + return build_and_push_image(repo_url, tag, path, "e2e") + + +def _delete_testrunner_pod() -> None: + """ + _delete_testrunner_pod deletes the test runner pod + if it already exists. + """ + dev_config = load_config() + corev1 = client.CoreV1Api() + ignore_if_doesnt_exist( + lambda: corev1.delete_namespaced_pod(TEST_RUNNER_NAME, dev_config.namespace) + ) + + +def create_test_runner_pod(test: str): + """ + create_test_runner_pod creates the pod which will run all of the tests. + """ + dev_config = load_config() + corev1 = client.CoreV1Api() + pod_body = _get_testrunner_pod_body(test) + return corev1.create_namespaced_pod(dev_config.namespace, body=pod_body) + + +def _get_testrunner_pod_body(test: str) -> Dict: + dev_config = load_config() + return { + "kind": "Pod", + "metadata": {"name": TEST_RUNNER_NAME, "namespace": dev_config.namespace,}, + "spec": { + "restartPolicy": "Never", + "serviceAccountName": TEST_RUNNER_NAME, + "containers": [ + { + "name": TEST_RUNNER_NAME, + "image": f"{dev_config.repo_url}/{TEST_RUNNER_NAME}", + "imagePullPolicy": "Always", + "command": [ + "./runner", + "--operatorImage", + f"{dev_config.repo_url}/mongodb-kubernetes-operator", + "--testImage", + f"{dev_config.repo_url}/e2e", + f"--test={test}", + f"--namespace={dev_config.namespace}", + ], + } + ], + }, + } + + +def wait_for_pod_to_be_running(corev1, name, namespace): + print("Waiting for pod to be running") + for i in range(10): + try: + pod = corev1.read_namespaced_pod(name, namespace) + if pod.status.phase == "Running": + return True + except ApiException as e: + pass + time.sleep(5) + raise Exception("Pod never got into Running state!") + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("test", help="Name of the test to run") + return parser.parse_args() + + +def main(): + args = parse_args() + config.load_kube_config() + dev_config = load_config() + build_and_push_testrunner( + dev_config.repo_url, f"{dev_config.repo_url}/{TEST_RUNNER_NAME}", "." + ) + build_and_push_e2e(dev_config.repo_url, f"{dev_config.repo_url}/e2e", ".") + + _prepare_testrunner_environment() + + pod = create_test_runner_pod(args.test) + corev1 = client.CoreV1Api() + wait_for_pod_to_be_running(corev1, TEST_RUNNER_NAME, dev_config.namespace) + + print(f"Running test: {args.test}") + + # stream all of the pod output as the pod is running + for line in corev1.read_namespaced_pod_log( + TEST_RUNNER_NAME, dev_config.namespace, follow=True, _preload_content=False + ).stream(): + print(line.decode("utf-8").rstrip()) + + +if __name__ == "__main__": + main() diff --git a/scripts/dev/requirements.txt b/scripts/dev/requirements.txt new file mode 100644 index 000000000..386dd234d --- /dev/null +++ b/scripts/dev/requirements.txt @@ -0,0 +1,3 @@ +docker=4.2.0 +kubernetes=10.0.1 +jinja2==2.11.2 diff --git a/scripts/dev/setup_kind_cluster.sh b/scripts/dev/setup_kind_cluster.sh new file mode 100755 index 000000000..b974a076a --- /dev/null +++ b/scripts/dev/setup_kind_cluster.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -Eeou pipefail + + +# adapted from https://kind.sigs.k8s.io/docs/user/local-registry/ +# create registry container unless it already exists +reg_name='kind-registry' +reg_port='5000' +running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" +if [ "${running}" != 'true' ]; then + docker run \ + -d --restart=always -p "${reg_port}:${reg_port}" --name "${reg_name}" \ + registry:2 +fi + +ip="$(docker inspect kind-registry -f {{.NetworkSettings.IPAddress}})" + +# create a cluster with the local registry enabled in containerd +cat <${temp} +contents="$(cat ${temp})" +kubectl create cm kube-config --from-literal=kubeconfig="${contents}" +rm ${temp} diff --git a/docker/templates/Dockerfile.e2e b/scripts/dev/templates/Dockerfile.e2e similarity index 100% rename from docker/templates/Dockerfile.e2e rename to scripts/dev/templates/Dockerfile.e2e diff --git a/docker/templates/Dockerfile.operator b/scripts/dev/templates/Dockerfile.operator similarity index 100% rename from docker/templates/Dockerfile.operator rename to scripts/dev/templates/Dockerfile.operator diff --git a/docker/templates/Dockerfile.template b/scripts/dev/templates/Dockerfile.template similarity index 100% rename from docker/templates/Dockerfile.template rename to scripts/dev/templates/Dockerfile.template diff --git a/docker/templates/Dockerfile.testrunner b/scripts/dev/templates/Dockerfile.testrunner similarity index 100% rename from docker/templates/Dockerfile.testrunner rename to scripts/dev/templates/Dockerfile.testrunner diff --git a/docker/templates/Dockerfile.unittest b/scripts/dev/templates/Dockerfile.unittest similarity index 100% rename from docker/templates/Dockerfile.unittest rename to scripts/dev/templates/Dockerfile.unittest