CLOUDP-61571: Add Basic Dev Scripts

.gitignore

@@ -78,4 +78,5 @@ tags
 
 .idea
 vendor
-zz_generated*.go
+zz_generated*.go
+__pycache__

deploy/testrunner/cluster_role_binding.yaml

@@ -0,0 +1,12 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: test-runner
+subjects:
+- kind: ServiceAccount
+  name: test-runner
+  namespace: default
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin # TODO: create cluster role with only required permissions
+  apiGroup: rbac.authorization.k8s.io

deploy/testrunner/role.yaml

@@ -80,20 +80,3 @@ rules:
   - patch
   - update
   - watch
-
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: test-runner
-rules:
-  - apiGroups:
-      - '*'
-    resources:
-      - '*'
-    verbs:
-      - '*'
-  - nonResourceURLs:
-      - '*'
-    verbs:
-      - '*'

deploy/testrunner/role_binding.yaml

@@ -9,17 +9,3 @@ roleRef:
   kind: Role
   name: test-runner
   apiGroup: rbac.authorization.k8s.io
-
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: test-runner
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: test-runner
-subjects:
-  - kind: ServiceAccount
-    name: test-runner
-    namespace: default

scripts/ci/build_and_push_image.sh

@@ -2,6 +2,6 @@
 
 echo ${quay_password} | docker login -u=${quay_user_name} quay.io --password-stdin
 
-python docker/dockerfile_generator.py ${image_type} > Dockerfile
+python scripts/dev/dockerfile_generator.py ${image_type} > Dockerfile
 docker build . -f Dockerfile -t ${image}
 docker push ${image}

scripts/ci/run_unit_tests.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
 
-python docker/dockerfile_generator.py "unittest" > Dockerfile
+python scripts/dev/dockerfile_generator.py "unittest" > Dockerfile
 docker build . -f Dockerfile -t unit-tests:${version_id}
 docker run unit-tests:${version_id}

scripts/dev/build_and_deploy_operator.py

@@ -0,0 +1,138 @@
+from dockerutil import build_and_push_image
+from dockerfile_generator import render
+from kubernetes import client, config
+from kubernetes.client.rest import ApiException
+from dev_config import DevConfig, load_config
+from typing import Dict, Optional
+import yaml
+import io
+import os
+import time
+
+
+def _load_operator_service_account() -> Optional[Dict]:
+    return load_yaml_from_file("deploy/service_account.yaml")
+
+
+def _load_operator_role() -> Optional[Dict]:
+    return load_yaml_from_file("deploy/role.yaml")
+
+
+def _load_operator_role_binding() -> Optional[Dict]:
+    return load_yaml_from_file("deploy/role_binding.yaml")
+
+
+def _load_operator_deployment() -> Optional[Dict]:
+    return load_yaml_from_file("deploy/operator.yaml")
+
+
+def _load_mongodb_crd() -> Optional[Dict]:
+    return load_yaml_from_file("deploy/crds/mongodb.com_mongodbs_crd.yaml")
+
+
+def load_yaml_from_file(path: str) -> Optional[Dict]:
+    with open(path, "r") as f:
+        return yaml.full_load(f.read())
+    return None
+
+
+def _ensure_crds():
+    """
+    ensure_crds makes sure that all the required CRDs have been created
+    """
+    crdv1 = client.ApiextensionsV1beta1Api()
+    crd = _load_mongodb_crd()
+
+    ignore_if_doesnt_exist(
+        lambda: crdv1.delete_custom_resource_definition("mongodbs.mongodb.com")
+    )
+
+    # TODO: fix this, when calling create_custom_resource_definition, we get the error
+    # ValueError("Invalid value for `conditions`, must not be `None`")
+    # but the crd is still successfully created
+    try:
+        crdv1.create_custom_resource_definition(body=crd)
+    except ValueError as e:
+        pass
+
+    print("Ensured CRDs")
+
+
+def build_and_push_operator(repo_url: str, tag: str, path: str):
+    """
+    build_and_push_operator creates the Dockerfile for the operator
+    and pushes it to the target repo
+    """
+    return build_and_push_image(repo_url, tag, path, "operator")
+
+
+def _ignore_error_codes(fn, codes):
+    try:
+        fn()
+    except ApiException as e:
+        if e.status not in codes:
+            raise
+
+
+def ignore_if_already_exists(fn):
+    """
+    ignore_if_already_exists accepts a function and calls it,
+    ignoring an Kubernetes API conflict errors
+    """
+
+    return _ignore_error_codes(fn, [409])
+
+
+def ignore_if_doesnt_exist(fn):
+    """
+    ignore_if_doesnt_exist accepts a function and calls it,
+    ignoring an Kubernetes API not found errors
+    """
+    return _ignore_error_codes(fn, [404])
+
+
+def deploy_operator():
+    """
+    deploy_operator ensures the CRDs are created, and als creates all the required ServiceAccounts, Roles
+    and RoleBindings for the operator, and then creates the operator deployment.
+    """
+    appsv1 = client.AppsV1Api()
+    corev1 = client.CoreV1Api()
+    rbacv1 = client.RbacAuthorizationV1Api()
+
+    dev_config = load_config()
+    _ensure_crds()
+
+    ignore_if_already_exists(
+        lambda: rbacv1.create_namespaced_role(
+            dev_config.namespace, _load_operator_role()
+        )
+    )
+    ignore_if_already_exists(
+        lambda: rbacv1.create_namespaced_role_binding(
+            dev_config.namespace, _load_operator_role_binding()
+        )
+    )
+    ignore_if_already_exists(
+        lambda: corev1.create_namespaced_service_account(
+            dev_config.namespace, _load_operator_service_account()
+        )
+    )
+    ignore_if_already_exists(
+        lambda: appsv1.create_namespaced_deployment(
+            dev_config.namespace, _load_operator_deployment()
+        )
+    )
+
+
+def main():
+    config.load_kube_config()
+    dev_config = load_config()
+    build_and_push_operator(
+        dev_config.repo_url, f"{dev_config.repo_url}/mongodb-kubernetes-operator", "."
+    )
+    deploy_operator()
+
+
+if __name__ == "__main__":
+    main()

scripts/dev/dev_config.py

@@ -0,0 +1,33 @@
+from typing import Dict, Optional
+import json
+import os
+
+CONFIG_PATH = "~/.community-operator-dev/config.json"
+FULL_CONFIG_PATH = os.path.expanduser(CONFIG_PATH)
+
+
+class DevConfig:
+    """
+    DevConfig is a wrapper around the developer configuration file
+    """
+
+    def __init__(self, config):
+        self._config = config
+
+    @property
+    def namespace(self):
+        return self._config["namespace"]
+
+    @property
+    def repo_url(self):
+        return self._config["repo_url"]
+
+
+def load_config() -> Optional[DevConfig]:
+    with open(FULL_CONFIG_PATH, "r") as f:
+        return DevConfig(json.loads(f.read()))
+
+    print(
+        f"No DevConfig found. Please ensure that the configuration file exists at '{FULL_CONFIG_PATH}'"
+    )
+    return None

docker/dockerfile_generator.py → scripts/dev/dockerfile_generator.py

@@ -13,14 +13,14 @@ def operator_params():
 def test_runner_params():
     return {
         "builder": True,
-        "builder_image": "golang", # TODO: make this image smaller. There were errors using alpine
+        "builder_image": "golang",  # TODO: make this image smaller. There were errors using alpine
         "base_image": "registry.access.redhat.com/ubi8/ubi-minimal:latest",
     }
 
 
 def e2e_params():
     return {
-        "base_image": "golang", # TODO: make this image smaller, error: 'exec: "gcc": executable file not found in $PATH' with golang:alpine
+        "base_image": "golang",  # TODO: make this image smaller, error: 'exec: "gcc": executable file not found in $PATH' with golang:alpine
     }
 
 
@@ -46,7 +46,7 @@ def render(image_name):
         )
 
     env = jinja2.Environment()
-    env.loader = jinja2.FileSystemLoader(searchpath="docker/templates")
+    env.loader = jinja2.FileSystemLoader(searchpath="scripts/dev/templates")
     return env.get_template("Dockerfile.{}".format(image_name)).render(
         param_dict[image_name]
     )

scripts/dev/dockerutil.py

@@ -0,0 +1,38 @@
+import docker
+from dockerfile_generator import render
+import os
+
+
+def build_image(repo_url: str, tag: str, path):
+    """
+    build_image builds the image with the given tag
+    """
+    client = docker.from_env()
+    print(f"Building image: {tag}")
+    client.images.build(tag=tag, path=path)
+    print("Successfully built image!")
+
+
+def push_image(tag: str):
+    """
+    push_image pushes the given tag. It uses
+    the current docker environment
+    """
+    client = docker.from_env()
+    print(f"Pushing image: {tag}")
+    for line in client.images.push(tag, stream=True):
+        print(line.decode("utf-8").rstrip())
+
+
+def build_and_push_image(repo_url: str, tag: str, path: str, image_type: str):
+    """
+    build_and_push_operator creates the Dockerfile for the operator
+    and pushes it to the target repo
+    """
+    dockerfile_text = render(image_type)
+    with open(f"{path}/Dockerfile", "w") as f:
+        f.write(dockerfile_text)
+
+    build_image(repo_url, tag, path)
+    os.remove(f"{path}/Dockerfile")
+    push_image(tag)