Restrict controller-manager Secrets access to Kueue namespace (#7188)

* Restrict controller-manager Secrets access to Kueue namespace

* Add controller-manager secrets role to RBAC Kustomization

* Add delegated namespaced secrets cache

Author: sbgla-sas

charts/kueue/templates/rbac/manager_secrets_role.yaml

@@ -0,0 +1,19 @@
+{{/* Code generated by yaml-processor. DO NOT EDIT. */}}
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+  {{- include "kueue.labels" . | nindent 4 }}
+  name: '{{ include "kueue.fullname" . }}-manager-secrets-role'
+  namespace: '{{ .Release.Namespace }}'
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+      - update
+      - watch

charts/kueue/templates/rbac/manager_secrets_role_binding.yaml

@@ -0,0 +1,17 @@
+{{/* Code generated by yaml-processor. DO NOT EDIT. */}}
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+  {{- include "kueue.labels" . | nindent 4 }}
+  name: '{{ include "kueue.fullname" . }}-manager-secrets-rolebinding'
+  namespace: '{{ .Release.Namespace }}'
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: '{{ include "kueue.fullname" . }}-manager-secrets-role'
+subjects:
+  - kind: ServiceAccount
+    name: '{{ include "kueue.fullname" . }}-controller-manager'
+    namespace: '{{ .Release.Namespace }}'

charts/kueue/templates/rbac/role.yaml

@@ -62,15 +62,6 @@ rules:
       - list
       - update
       - watch
-  - apiGroups:
-      - ""
-    resources:
-      - secrets
-    verbs:
-      - get
-      - list
-      - update
-      - watch
   - apiGroups:
       - admissionregistration.k8s.io
     resources:

config/components/rbac/kustomization.yaml

@@ -9,6 +9,8 @@ resources:
 - role_binding.yaml
 - leader_election_role.yaml
 - leader_election_role_binding.yaml
+- manager_secrets_role.yaml
+- manager_secrets_role_binding.yaml
 # The following RBAC configurations are used to protect
 # the metrics endpoint with authn/authz. These configurations
 # ensure that only authorized users and service accounts

config/components/rbac/manager_secrets_role.yaml

@@ -0,0 +1,16 @@
+# permissions for the manager to access secrets.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: manager-secrets-role
+  namespace: system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - update
+  - watch

config/components/rbac/manager_secrets_role_binding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: manager-secrets-rolebinding
+  namespace: system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: manager-secrets-role
+subjects:
+- kind: ServiceAccount
+  name: controller-manager
+  namespace: system

config/components/rbac/role.yaml

@@ -59,15 +59,6 @@ rules:
   - list
   - update
   - watch
-- apiGroups:
-  - ""
-  resources:
-  - secrets
-  verbs:
-  - get
-  - list
-  - update
-  - watch
 - apiGroups:
   - admissionregistration.k8s.io
   resources:

pkg/config/config.go

@@ -21,16 +21,23 @@ import (
 	"fmt"
 	"os"
 
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	ctrl "sigs.k8s.io/controller-runtime"
+	ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	configapi "sigs.k8s.io/kueue/apis/config/v1beta1"
 )
 
+var (
+	objectKeySecret = new(corev1.Secret)
+)
+
 // fromFile provides an alternative to the deprecated ctrl.ConfigFile().AtPath(path).OfKind(&cfg)
 func fromFile(path string, scheme *runtime.Scheme, cfg *configapi.Configuration) error {
 	content, err := os.ReadFile(path)
@@ -48,6 +55,8 @@ func fromFile(path string, scheme *runtime.Scheme, cfg *configapi.Configuration)
 // addTo provides an alternative to the deprecated o.AndFrom(&cfg)
 func addTo(o *ctrl.Options, cfg *configapi.Configuration) {
 	addLeaderElectionTo(o, cfg)
+	addCacheByObjectTo(o, cfg)
+
 	if o.Metrics.BindAddress == "" && cfg.Metrics.BindAddress != "" {
 		o.Metrics.BindAddress = cfg.Metrics.BindAddress
 	}
@@ -94,6 +103,24 @@ func addTo(o *ctrl.Options, cfg *configapi.Configuration) {
 	}
 }
 
+func addCacheByObjectTo(o *ctrl.Options, cfg *configapi.Configuration) {
+	if cfg.Namespace == nil {
+		// Invalid source; noop. This should not be reached
+		// due to prior defaulting/validation.
+		return
+	}
+
+	if o.Cache.ByObject == nil {
+		o.Cache.ByObject = make(map[ctrlclient.Object]ctrlcache.ByObject)
+	}
+
+	o.Cache.ByObject[objectKeySecret] = ctrlcache.ByObject{
+		Namespaces: map[string]ctrlcache.Config{
+			*cfg.Namespace: {},
+		},
+	}
+}
+
 func addLeaderElectionTo(o *ctrl.Options, cfg *configapi.Configuration) {
 	if cfg.LeaderElection == nil {
 		// The source does not have any configuration; noop