CLOUDP-66908: add configmap to evg logs (#100)

Author: Nikolas De Giorgis

.gitignore

@@ -82,3 +82,4 @@ zz_generated*.go
 __pycache__
 Dockerfile
 Dockerfile_python_formatting
+logs/*

scripts/dev/dump_diagnostic.py

@@ -1,7 +1,8 @@
 import os
 import shutil
 import yaml
-from typing import Dict, TextIO
+from typing import Dict, TextIO, List
+import json
 import k8s_request_data
 
 
@@ -26,41 +27,60 @@ def header(msg: str) -> str:
 
 def dump_crd(crd_log: TextIO) -> None:
     crd = k8s_request_data.get_crds()
-    crd_log.write(header("CRD"))
-    crd_log.write(yaml.dump(clean_nones(crd)))
+    if crd is not None:
+        crd_log.write(header("CRD"))
+        crd_log.write(yaml.dump(clean_nones(crd)))
 
 
 def dump_persistent_volume(diagnostic_file: TextIO) -> None:
-    diagnostic_file.write(header("Persistent Volumes"))
     pv = k8s_request_data.get_persistent_volumes()
-    diagnostic_file.write(yaml.dump(clean_nones(pv)))
+    if pv is not None:
+        diagnostic_file.write(header("Persistent Volumes"))
+        diagnostic_file.write(yaml.dump(clean_nones(pv)))
 
 
 def dump_stateful_sets_namespaced(diagnostic_file: TextIO, namespace: str) -> None:
-    diagnostic_file.write(header("Stateful Sets"))
     sst = k8s_request_data.get_stateful_sets_namespaced(namespace)
-    diagnostic_file.write(yaml.dump(clean_nones(sst)))
+    if sst is not None:
+        diagnostic_file.write(header("Stateful Sets"))
+        diagnostic_file.write(yaml.dump(clean_nones(sst)))
 
 
 def dump_pod_log_namespaced(namespace: str, name: str, containers: list) -> None:
     for container in containers:
         with open(
-            "logs/e2e/{}-{}.log".format(name, container.name),
-            mode="w",
-            encoding="utf-8",
+            f"logs/e2e/{name}-{container.name}.log", mode="w", encoding="utf-8",
         ) as log_file:
-            log_file.write(
-                k8s_request_data.get_pod_log_namespaced(namespace, name, container.name)
+            log = k8s_request_data.get_pod_log_namespaced(
+                namespace, name, container.name
             )
+            if log is not None:
+                log_file.write(log)
 
 
 def dump_pods_and_logs_namespaced(diagnostic_file: TextIO, namespace: str) -> None:
     pods = k8s_request_data.get_pods_namespaced(namespace)
-    for pod in pods:
-        name = pod.metadata.name
-        diagnostic_file.write(header(f"Pod {name}"))
-        diagnostic_file.write(yaml.dump(clean_nones(pod.to_dict())))
-        dump_pod_log_namespaced(namespace, name, pod.spec.containers)
+    if pods is not None:
+        for pod in pods:
+            name = pod.metadata.name
+            diagnostic_file.write(header(f"Pod {name}"))
+            diagnostic_file.write(yaml.dump(clean_nones(pod.to_dict())))
+            dump_pod_log_namespaced(namespace, name, pod.spec.containers)
+
+
+def dump_configmap_keys_namespaced(
+    namespace: str, keys: List[str], configmap_name: str
+) -> None:
+    configmap = k8s_request_data.get_configmap_namespaced(namespace, configmap_name)
+    if configmap is not None:
+        for key in keys:
+            with open(
+                f"logs/e2e/{configmap_name}-{key}.json", mode="w", encoding="utf-8",
+            ) as log_file:
+                if key in configmap["data"]:
+                    log_file.write(
+                        json.dumps(json.loads(configmap["data"][key]), indent=4)
+                    )
 
 
 def dump_all(namespace: str) -> None:
@@ -82,3 +102,5 @@ def dump_all(namespace: str) -> None:
 
     with open("logs/e2e/crd.log", mode="w", encoding="utf-8") as crd_log:
         dump_crd(crd_log)
+
+    dump_configmap_keys_namespaced(namespace, ["automation-config"], "mdb0-config")

scripts/dev/k8s_request_data.py

@@ -1,44 +1,60 @@
 from kubernetes.client.rest import ApiException
 from kubernetes import client
 
+from typing import Optional
 
-def get_crds() -> dict:
+
+def get_crds() -> Optional[dict]:
     crdv1 = client.ApiextensionsV1beta1Api()
     try:
         crd = crdv1.list_custom_resource_definition(pretty="true")
     except ApiException as e:
         print("Exception when calling list_custom_resource_definition: %s\n" % e)
+        return None
     return crd.to_dict()
 
 
-def get_persistent_volumes() -> dict:
+def get_persistent_volumes() -> Optional[dict]:
     corev1 = client.CoreV1Api()
     try:
         pv = corev1.list_persistent_volume(pretty="true")
     except ApiException as e:
         print("Exception when calling list_persistent_volume %s\n" % e)
+        return None
     return pv.to_dict()
 
 
-def get_stateful_sets_namespaced(namespace: str) -> dict:
+def get_stateful_sets_namespaced(namespace: str) -> Optional[dict]:
     av1beta1 = client.AppsV1Api()
     try:
         sst = av1beta1.list_namespaced_stateful_set(namespace, pretty="true")
     except ApiException as e:
         print("Exception when calling list_namespaced_stateful_set: %s\n" % e)
+        return None
     return sst.to_dict()
 
 
-def get_pods_namespaced(namespace: str) -> list:
+def get_configmap_namespaced(namespace: str, name: str) -> Optional[dict]:
+    corev1 = client.CoreV1Api()
+    try:
+        config_map = corev1.read_namespaced_config_map(name, namespace, pretty="true")
+    except ApiException as e:
+        print("Exception when calling read_namespaced_config_map: %s\n" % e)
+        return None
+    return config_map.to_dict()
+
+
+def get_pods_namespaced(namespace: str) -> Optional[list]:
     corev1 = client.CoreV1Api()
     try:
         pods = corev1.list_namespaced_pod(namespace)
     except ApiException as e:
         print("Exception when calling list_namespaced_pod: %s\n" % e)
+        return None
     return pods.items
 
 
-def get_pod_namespaced(namespace: str, pod_name: str) -> client.V1Pod:
+def get_pod_namespaced(namespace: str, pod_name: str) -> Optional[client.V1Pod]:
     corev1 = client.CoreV1Api()
     try:
         pod = corev1.read_namespaced_pod(name=pod_name, namespace=namespace)
@@ -47,9 +63,15 @@ def get_pod_namespaced(namespace: str, pod_name: str) -> client.V1Pod:
     return pod
 
 
-def get_pod_log_namespaced(namespace: str, pod_name: str, container_name: str) -> str:
+def get_pod_log_namespaced(
+    namespace: str, pod_name: str, container_name: str
+) -> Optional[str]:
     corev1 = client.CoreV1Api()
-    log = corev1.read_namespaced_pod_log(
-        name=pod_name, namespace=namespace, pretty="true", container=container_name
-    )
+    try:
+        log = corev1.read_namespaced_pod_log(
+            name=pod_name, namespace=namespace, pretty="true", container=container_name
+        )
+    except ApiException as e:
+        print("Exception when calling read_namespaced_pod_log: %s\n" % e)
+        return None
     return log