CLOUDP-61571: Add Basic Dev Scripts (#40)

Author: chatton

.gitignore

@@ -78,4 +78,5 @@ tags
 
 .idea
 vendor
-zz_generated*.go
+zz_generated*.go
+__pycache__

deploy/testrunner/cluster_role_binding.yaml

@@ -0,0 +1,12 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: test-runner
+subjects:
+- kind: ServiceAccount
+  name: test-runner
+  namespace: default
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin # TODO: create cluster role with only required permissions
+  apiGroup: rbac.authorization.k8s.io

deploy/testrunner/role.yaml

@@ -80,20 +80,3 @@ rules:
   - patch
   - update
   - watch
-
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: test-runner
-rules:
-  - apiGroups:
-      - '*'
-    resources:
-      - '*'
-    verbs:
-      - '*'
-  - nonResourceURLs:
-      - '*'
-    verbs:
-      - '*'

deploy/testrunner/role_binding.yaml

@@ -9,17 +9,3 @@ roleRef:
   kind: Role
   name: test-runner
   apiGroup: rbac.authorization.k8s.io
-
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: test-runner
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: test-runner
-subjects:
-  - kind: ServiceAccount
-    name: test-runner
-    namespace: default

docker/requirements.txt

@@ -2,6 +2,6 @@
 
 echo ${quay_password} | docker login -u=${quay_user_name} quay.io --password-stdin
 
-python docker/dockerfile_generator.py ${image_type} > Dockerfile
+python scripts/dev/dockerfile_generator.py ${image_type} > Dockerfile
 docker build . -f Dockerfile -t ${image}
 docker push ${image}

scripts/ci/build_and_push_image.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
 
-python docker/dockerfile_generator.py "unittest" > Dockerfile
+python scripts/dev/dockerfile_generator.py "unittest" > Dockerfile
 docker build . -f Dockerfile -t unit-tests:${version_id}
 docker run unit-tests:${version_id}

scripts/ci/run_unit_tests.sh

@@ -0,0 +1,138 @@
+from dockerutil import build_and_push_image
+from dockerfile_generator import render
+from kubernetes import client, config
+from kubernetes.client.rest import ApiException
+from dev_config import DevConfig, load_config
+from typing import Dict, Optional
+import yaml
+import io
+import os
+import time
+
+
+def _load_operator_service_account() -> Optional[Dict]:
+    return load_yaml_from_file("deploy/service_account.yaml")
+
+
+def _load_operator_role() -> Optional[Dict]:
+    return load_yaml_from_file("deploy/role.yaml")
+
+
+def _load_operator_role_binding() -> Optional[Dict]:
+    return load_yaml_from_file("deploy/role_binding.yaml")
+
+
+def _load_operator_deployment() -> Optional[Dict]:
+    return load_yaml_from_file("deploy/operator.yaml")
+
+
+def _load_mongodb_crd() -> Optional[Dict]:
+    return load_yaml_from_file("deploy/crds/mongodb.com_mongodbs_crd.yaml")
+
+
+def load_yaml_from_file(path: str) -> Optional[Dict]:
+    with open(path, "r") as f:
+        return yaml.full_load(f.read())
+    return None
+
+
+def _ensure_crds():
+    """
+    ensure_crds makes sure that all the required CRDs have been created
+    """
+    crdv1 = client.ApiextensionsV1beta1Api()
+    crd = _load_mongodb_crd()
+
+    ignore_if_doesnt_exist(
+        lambda: crdv1.delete_custom_resource_definition("mongodbs.mongodb.com")
+    )
+
+    # TODO: fix this, when calling create_custom_resource_definition, we get the error
+    # ValueError("Invalid value for `conditions`, must not be `None`")
+    # but the crd is still successfully created
+    try:
+        crdv1.create_custom_resource_definition(body=crd)
+    except ValueError as e:
+        pass
+
+    print("Ensured CRDs")
+
+
+def build_and_push_operator(repo_url: str, tag: str, path: str):
+    """
+    build_and_push_operator creates the Dockerfile for the operator
+    and pushes it to the target repo
+    """
+    return build_and_push_image(repo_url, tag, path, "operator")
+
+
+def _ignore_error_codes(fn, codes):
+    try:
+        fn()
+    except ApiException as e:
+        if e.status not in codes:
+            raise
+
+
+def ignore_if_already_exists(fn):
+    """
+    ignore_if_already_exists accepts a function and calls it,
+    ignoring an Kubernetes API conflict errors
+    """
+
+    return _ignore_error_codes(fn, [409])
+
+
+def ignore_if_doesnt_exist(fn):
+    """
+    ignore_if_doesnt_exist accepts a function and calls it,
+    ignoring an Kubernetes API not found errors
+    """
+    return _ignore_error_codes(fn, [404])
+
+
+def deploy_operator():
+    """
+    deploy_operator ensures the CRDs are created, and als creates all the required ServiceAccounts, Roles
+    and RoleBindings for the operator, and then creates the operator deployment.
+    """
+    appsv1 = client.AppsV1Api()
+    corev1 = client.CoreV1Api()
+    rbacv1 = client.RbacAuthorizationV1Api()
+
+    dev_config = load_config()
+    _ensure_crds()
+
+    ignore_if_already_exists(
+        lambda: rbacv1.create_namespaced_role(
+            dev_config.namespace, _load_operator_role()
+        )
+    )
+    ignore_if_already_exists(
+        lambda: rbacv1.create_namespaced_role_binding(
+            dev_config.namespace, _load_operator_role_binding()
+        )
+    )
+    ignore_if_already_exists(
+        lambda: corev1.create_namespaced_service_account(
+            dev_config.namespace, _load_operator_service_account()
+        )
+    )
+    ignore_if_already_exists(
+        lambda: appsv1.create_namespaced_deployment(
+            dev_config.namespace, _load_operator_deployment()
+        )
+    )
+
+
+def main():
+    config.load_kube_config()
+    dev_config = load_config()
+    build_and_push_operator(
+        dev_config.repo_url, f"{dev_config.repo_url}/mongodb-kubernetes-operator", "."
+    )
+    deploy_operator()
+
+
+if __name__ == "__main__":
+    main()

scripts/dev/__init__.py

@@ -0,0 +1,33 @@
+from typing import Dict, Optional
+import json
+import os
+
+CONFIG_PATH = "~/.community-operator-dev/config.json"
+FULL_CONFIG_PATH = os.path.expanduser(CONFIG_PATH)
+
+
+class DevConfig:
+    """
+    DevConfig is a wrapper around the developer configuration file
+    """
+
+    def __init__(self, config):
+        self._config = config
+
+    @property
+    def namespace(self):
+        return self._config["namespace"]
+
+    @property
+    def repo_url(self):
+        return self._config["repo_url"]
+
+
+def load_config() -> Optional[DevConfig]:
+    with open(FULL_CONFIG_PATH, "r") as f:
+        return DevConfig(json.loads(f.read()))
+
+    print(
+        f"No DevConfig found. Please ensure that the configuration file exists at '{FULL_CONFIG_PATH}'"
+    )
+    return None