Skip to content

Commit 7e98ba0

Browse files
committed
w
1 parent 4549031 commit 7e98ba0

File tree

7 files changed

+73
-170
lines changed

7 files changed

+73
-170
lines changed

.config/telepresence/config.yml

Lines changed: 0 additions & 2 deletions
This file was deleted.

Makefile

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -632,7 +632,6 @@ test.api: gotestsum
632632
_test.integration: gotestsum download.telepresence
633633
KUBECONFIG=$(KUBECONFIG) \
634634
TELEPRESENCE_BIN=$(TELEPRESENCE) \
635-
XDG_CONFIG_HOME=$(PROJECT_DIR)/.config/telepresence \
636635
GOFLAGS=$(GOFLAGS) \
637636
GOTESTSUM_FORMAT=$(GOTESTSUM_FORMAT) \
638637
$(GOTESTSUM) -- $(GOTESTFLAGS) \
@@ -676,7 +675,6 @@ PARALLEL := $(if $(PARALLEL),$(PARALLEL),$(NCPU))
676675
_test.conformance: gotestsum download.telepresence
677676
KUBECONFIG=$(KUBECONFIG) \
678677
TELEPRESENCE_BIN=$(TELEPRESENCE) \
679-
XDG_CONFIG_HOME=$(PROJECT_DIR)/.config/telepresence \
680678
GOTESTSUM_FORMAT=$(GOTESTSUM_FORMAT) \
681679
$(GOTESTSUM) -- $(GOTESTFLAGS) \
682680
-timeout $(CONFORMANCE_TEST_TIMEOUT) \
@@ -833,9 +831,9 @@ KUBECONFIG ?= $(HOME)/.kube/config
833831
# etc didn't change in between the runs.
834832
.PHONY: _run
835833
_run:
836-
@XDG_CONFIG_HOME=$(PROJECT_DIR)/.config/telepresence $(TELEPRESENCE) helm install
837-
@XDG_CONFIG_HOME=$(PROJECT_DIR)/.config/telepresence $(TELEPRESENCE) connect
838-
bash -c "export XDG_CONFIG_HOME=$(PROJECT_DIR)/.config/telepresence; trap \
834+
@$(TELEPRESENCE) helm install
835+
@$(TELEPRESENCE) connect
836+
bash -c "trap \
839837
'$(TELEPRESENCE) quit -s; $(TELEPRESENCE) helm uninstall; rm -rf $(TMP_KUBECONFIG) || 1' EXIT; \
840838
KONG_OPERATOR_KUBECONFIG=$(or $(TMP_KUBECONFIG),$(KUBECONFIG)) \
841839
KONG_OPERATOR_ANONYMOUS_REPORTS=false \
@@ -852,7 +850,7 @@ _run:
852850
-zap-time-encoding iso8601 \
853851
-zap-log-level 2 \
854852
-zap-devel true \
855-
"
853+
"
856854

857855
# Run the operator locally with impersonation of controller-manager service account from kong-system namespace.
858856
# The operator will use a temporary kubeconfig file and impersonate the real RBACs.
@@ -974,15 +972,15 @@ undeploy:
974972
# as if it were running inside the cluster itself.
975973
.PHONY: install.telepresence
976974
install.telepresence: download.telepresence
977-
@XDG_CONFIG_HOME=$(PROJECT_DIR)/.config/telepresence $(PROJECT_DIR)/scripts/telepresence-manager.sh install "$(TELEPRESENCE)"
975+
@$(PROJECT_DIR)/scripts/telepresence-manager.sh install "$(TELEPRESENCE)"
978976

979977
# Disconnect and uninstall telepresence from the cluster.
980978
# This target cleans up the telepresence resources created by the install.telepresence target.
981979
# It should be used when you're done debugging the operator locally to ensure proper
982980
# cleanup of network connections and cluster resources.
983981
.PHONY: uninstall.telepresence
984982
uninstall.telepresence: download.telepresence
985-
@XDG_CONFIG_HOME=$(PROJECT_DIR)/.config/telepresence $(PROJECT_DIR)/scripts/telepresence-manager.sh uninstall "$(TELEPRESENCE)"
983+
@$(PROJECT_DIR)/scripts/telepresence-manager.sh uninstall "$(TELEPRESENCE)"
986984

987985
.PHONY: lint.api
988986
lint.api: download.kube-api-linter

pkg/utils/kubernetes/self_metadata.go

Lines changed: 11 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package kubernetes
33
import (
44
"fmt"
55
"os"
6-
"path/filepath"
76
"strconv"
87
"strings"
98
)
@@ -32,87 +31,27 @@ func GetSelfNamespace() (string, error) {
3231

3332
// GetSelfPodLabels gets all the labels of the KO pod.
3433
func GetSelfPodLabels() (map[string]string, error) {
35-
var (
36-
lastErr error
37-
locations []string
38-
)
39-
40-
// Prefer explicit override first
41-
if override := os.Getenv("KONG_OPERATOR_POD_LABELS_FILE"); override != "" {
42-
locations = append(locations, override)
43-
}
44-
45-
// Then try TELEPRESENCE_ROOT mounted path
46-
if root := os.Getenv("TELEPRESENCE_ROOT"); root != "" {
47-
relPath := strings.TrimPrefix(podLabelsFile, "/")
48-
locations = append(locations, filepath.Join(root, relPath))
49-
}
50-
51-
// Finally fall back to standard pod labels file
52-
locations = append(locations, podLabelsFile)
53-
54-
for _, path := range locations {
55-
buf, err := os.ReadFile(path)
56-
if err != nil {
57-
lastErr = err
58-
continue
59-
}
60-
61-
ret := parsePodLabels(string(buf))
62-
if len(ret) > 0 {
63-
return ret, nil
64-
}
65-
lastErr = fmt.Errorf("no valid labels found in %s", path)
66-
}
67-
68-
if lastErr != nil {
69-
return nil, fmt.Errorf("cannot find pod labels from %v: %w", locations, lastErr)
34+
buf, err := os.ReadFile(podLabelsFile)
35+
if err != nil {
36+
return nil, fmt.Errorf("cannot find pod labels from file %s: %w", podLabelsFile, err)
7037
}
71-
return nil, fmt.Errorf("cannot determine pod labels")
72-
}
7338

74-
// parsePodLabels parses pod labels from DownwardAPI format.
75-
// Supports both newline-separated and comma-separated formats.
76-
func parsePodLabels(content string) map[string]string {
39+
labels := strings.SplitSeq(string(buf), "\n")
7740
ret := make(map[string]string)
78-
content = strings.TrimSpace(content)
79-
if content == "" {
80-
return ret
81-
}
82-
83-
// Try newline-separated first
84-
lines := strings.Split(content, "\n")
85-
// If we only have one line and it contains commas, try comma-separated
86-
if len(lines) == 1 && strings.Contains(content, ",") {
87-
lines = strings.Split(content, ",")
88-
}
89-
90-
for _, label := range lines {
91-
label = strings.TrimSpace(label)
92-
if label == "" {
93-
continue
94-
}
95-
41+
for label := range labels {
9642
labelKV := strings.SplitN(label, "=", 2)
9743
if len(labelKV) != 2 {
98-
continue
44+
return nil, fmt.Errorf("invalid label format, should be key=value")
9945
}
100-
101-
key := strings.TrimSpace(labelKV[0])
102-
value := strings.TrimSpace(labelKV[1])
103-
if key == "" {
46+
key := labelKV[0]
47+
// The value in labels are escaped, e.g: "ko" => "\"ko\"". So we need to unquote it.
48+
value, err := strconv.Unquote(labelKV[1])
49+
if err != nil {
10450
continue
10551
}
106-
107-
// Try to unquote the value (DownwardAPI escapes values)
108-
if unquoted, err := strconv.Unquote(value); err == nil {
109-
value = unquoted
110-
}
111-
11252
ret[key] = value
11353
}
114-
115-
return ret
54+
return ret, nil
11655
}
11756

11857
// RunningOnKubernetes returns true if it is running in the kubernetes environment.

scripts/telepresence-manager.sh

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,6 @@
55
set -e
66
set -o pipefail
77

8-
# Default XDG_CONFIG_HOME to the project .config/telepresence directory so Telepresence reads config.yml inside it
9-
PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
10-
if [ -z "${XDG_CONFIG_HOME:-}" ]; then
11-
export XDG_CONFIG_HOME="$PROJECT_DIR/.config/telepresence"
12-
fi
13-
148
# Function to log messages with different levels.
159
log() {
1610
local level="$1"

test/integration/helpers/network_intercept.go

Lines changed: 30 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package helpers
22

33
import (
44
"bufio"
5-
"bytes"
65
"context"
76
"fmt"
87
"os"
@@ -74,77 +73,46 @@ func SetupNetworkIntercepts(ctx context.Context, clients testutils.K8sClients) (
7473
return nil, fmt.Errorf("telepresence intercept deployment not ready: %w", err)
7574
}
7675

77-
envFile, err := os.CreateTemp("", "telepresence-env-*.sh")
76+
fmt.Println("INFO: connecting to the cluster with telepresence!!!")
77+
// NOTE: We need to specify --manager-namespace to connect to the traffic-manager
78+
// installed in kong-system namespace above.
79+
connectArgs := []string{"connect", "--manager-namespace", "kong-system", "--namespace", "kong-system"}
80+
out, err := exec.CommandContext(ctx, telepresenceExec, connectArgs...).CombinedOutput()
7881
if err != nil {
79-
return nil, fmt.Errorf("failed creating temporary env file for telepresence: %w", err)
82+
return nil, fmt.Errorf("failed to connect to the cluster with telepresence: %w, %s", err, string(out))
8083
}
81-
envFilePath := envFile.Name()
82-
_ = envFile.Close()
84+
85+
// envFile, err := os.CreateTemp("", "telepresence-env-*.sh")
86+
// if err != nil {
87+
// return nil, fmt.Errorf("failed creating temporary env file for telepresence: %w", err)
88+
// }
89+
// envFilePath := envFile.Name()
90+
// _ = envFile.Close()
8391

8492
fmt.Println("INFO: establishing telepresence intercept for controller identity")
85-
var (
86-
labelOverridePath string
87-
)
93+
// var (
94+
// labelOverridePath string
95+
// )
8896
intercept := func(portPair int, svcName string) []string {
8997
return []string{
9098
"replace",
9199
svcName,
92100
"--port", fmt.Sprintf("%d:%d", portPair, portPair),
93-
"--env-file", envFilePath,
94101
}
95102
}
96103

97104
cmd := exec.CommandContext(ctx, telepresenceExec, intercept(telepresenceInterceptPort, telepresenceInterceptName)...)
98105
cmd.Env = os.Environ()
99106
output, err := cmd.CombinedOutput()
100107
if err != nil {
101-
if !bytes.Contains(output, []byte("already exists")) {
102-
_ = os.Remove(envFilePath)
103-
return nil, fmt.Errorf("failed to create telepresence intercept: %w, %s", err, string(output))
104-
}
108+
// if !bytes.Contains(output, []byte("already exists")) {
109+
// _ = os.Remove(envFilePath)
110+
// return nil, fmt.Errorf("failed to create telepresence intercept: %w, %s", err, string(output))
111+
// }
105112
fmt.Println("WARN: telepresence intercept already exists, reusing the existing session")
106113
}
107114
fmt.Println(">>>\n", string(output), "\n<<<")
108115

109-
// const svc = "gateway-operator-webhook-service"
110-
// cmd = exec.CommandContext(ctx, telepresenceExec, intercept(433, svc)...)
111-
// cmd.Env = os.Environ()
112-
// output, err = cmd.CombinedOutput()
113-
// if err != nil {
114-
// if !bytes.Contains(output, []byte("already exists")) {
115-
// _ = os.Remove(envFilePath)
116-
// return nil, fmt.Errorf("failed to create telepresence intercept: %w, %s", err, string(output))
117-
// }
118-
// fmt.Println("WARN: telepresence intercept already exists, reusing the existing session")
119-
// }
120-
121-
// cmd = exec.CommandContext(ctx, telepresenceExec, intercept(5433, svc)...)
122-
// cmd.Env = os.Environ()
123-
// output, err = cmd.CombinedOutput()
124-
// if err != nil {
125-
// if !bytes.Contains(output, []byte("already exists")) {
126-
// _ = os.Remove(envFilePath)
127-
// return nil, fmt.Errorf("failed to create telepresence intercept: %w, %s", err, string(output))
128-
// }
129-
// fmt.Println("WARN: telepresence intercept already exists, reusing the existing session")
130-
// }
131-
132-
if err := applyTelepresenceEnvFile(envFilePath); err != nil {
133-
leaveCmd := exec.CommandContext(ctx, telepresenceExec, "leave", telepresenceInterceptName)
134-
_ = leaveCmd.Run()
135-
_ = os.Remove(envFilePath)
136-
return nil, fmt.Errorf("failed to apply telepresence environment: %w", err)
137-
}
138-
139-
overridePath, err := writePodLabelsOverride(interceptPodLabels())
140-
if err != nil {
141-
return nil, fmt.Errorf("failed to prepare pod labels override: %w", err)
142-
}
143-
labelOverridePath = overridePath
144-
if err := os.Setenv("KONG_OPERATOR_POD_LABELS_FILE", overridePath); err != nil {
145-
return nil, fmt.Errorf("failed to set pod labels override env: %w", err)
146-
}
147-
148116
if err := ensureAdmissionRegistration(ctx, clients.K8sClient, k8stypes.NamespacedName{
149117
Namespace: controllerNamespace,
150118
Name: "gateway-operator-webhook-service",
@@ -166,14 +134,14 @@ func SetupNetworkIntercepts(ctx context.Context, clients testutils.K8sClients) (
166134
fmt.Printf("WARN: failed to delete intercept service %s/%s: %v\n", controllerNamespace, telepresenceInterceptDeploymentName, err)
167135
}
168136

169-
if err := os.Remove(envFilePath); err != nil && !os.IsNotExist(err) {
170-
fmt.Printf("WARN: failed to remove telepresence env file %q: %v\n", envFilePath, err)
171-
}
172-
if labelOverridePath != "" {
173-
if err := os.Remove(labelOverridePath); err != nil && !os.IsNotExist(err) {
174-
fmt.Printf("WARN: failed to remove pod labels override file %q: %v\n", labelOverridePath, err)
175-
}
176-
}
137+
// if err := os.Remove(envFilePath); err != nil && !os.IsNotExist(err) {
138+
// fmt.Printf("WARN: failed to remove telepresence env file %q: %v\n", envFilePath, err)
139+
// }
140+
// if labelOverridePath != "" {
141+
// if err := os.Remove(labelOverridePath); err != nil && !os.IsNotExist(err) {
142+
// fmt.Printf("WARN: failed to remove pod labels override file %q: %v\n", labelOverridePath, err)
143+
// }
144+
// }
177145
}
178146

179147
return cleanup, nil
@@ -205,10 +173,11 @@ func ensureInterceptDeployment(ctx context.Context, clients testutils.K8sClients
205173
ObjectMeta: metav1.ObjectMeta{
206174
Labels: labels,
207175
Annotations: map[string]string{
208-
"telepresence.getambassador.io/inject-traffic-agent": "enabled",
176+
"telepresence.io/inject-traffic-agent": "enabled",
209177
},
210178
},
211179
Spec: corev1.PodSpec{
180+
PriorityClassName: "system-cluster-critical",
212181
Containers: []corev1.Container{
213182
{
214183
Name: "identity",

test/integration/helpers/telepresence.go

Lines changed: 26 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -54,39 +54,46 @@ func SetupTelepresence(ctx context.Context) (func(), error) {
5454
}
5555

5656
fmt.Println("INFO: installing telepresence traffic manager in the cluster")
57-
telepresenceExec, err := resolveTelepresenceExecutable()
57+
telepresenceExecutable, err := resolveTelepresenceExecutable()
5858
if err != nil {
5959
return nil, err
6060
}
61-
fmt.Printf("INFO: using telepresence binary at %s\n", telepresenceExec)
6261

63-
// Ensure any stale daemons are terminated so that binary and daemon versions match.
64-
if out, err := exec.CommandContext(ctx, telepresenceExec, "quit", "-s").CombinedOutput(); err != nil && len(out) > 0 {
65-
fmt.Printf("WARN: telepresence quit -s reported: %s\n", string(out))
66-
}
62+
// Set pod labels on traffic-manager to match the labels expected by NetworkPolicy.
63+
// This allows traffic from the local test process (via telepresence) to be allowed
64+
// by the DataPlane's NetworkPolicy which restricts admin API access.
65+
// NOTE: We use "app.kubernetes.io/name" instead of "app" because "app" conflicts
66+
// with telepresence's deployment selector.
67+
// NOTE: We install traffic-manager in kong-system namespace to match the NetworkPolicy
68+
// rules which only allow traffic from kong-system namespace.
69+
fmt.Printf("INFO: path to telepresence is %s\n", telepresenceExecutable)
6770

68-
out, err := exec.CommandContext(ctx, telepresenceExec, "helm", "install").CombinedOutput()
71+
// Set pod labels on traffic-manager to match the labels expected by NetworkPolicy.
72+
// This allows traffic from the local test process (via telepresence) to be allowed
73+
// by the DataPlane's NetworkPolicy which restricts admin API access.
74+
// NOTE: We use "app.kubernetes.io/name" instead of "app" because "app" conflicts
75+
// with telepresence's deployment selector.
76+
// NOTE: We install traffic-manager in kong-system namespace to match the NetworkPolicy
77+
// rules which only allow traffic from kong-system namespace.
78+
// See: https://github.com/Kong/kong-operator/issues/2074
79+
commonHelmFlags := []string{
80+
"--manager-namespace", "kong-system",
81+
"--set", "podLabels.app\\.kubernetes\\.io/name=kong-operator",
82+
}
83+
helmInstallArgs := append([]string{"helm", "install"}, commonHelmFlags...)
84+
out, err := exec.CommandContext(ctx, telepresenceExecutable, helmInstallArgs...).CombinedOutput()
6985
if err != nil && bytes.Contains(out, []byte("use 'telepresence helm upgrade' instead to replace it")) {
70-
if out, err := exec.CommandContext(ctx, telepresenceExec, "helm", "upgrade").CombinedOutput(); err != nil {
86+
helmUpgradeArgs := append([]string{"helm", "upgrade"}, commonHelmFlags...)
87+
if out, err := exec.CommandContext(ctx, telepresenceExecutable, helmUpgradeArgs...).CombinedOutput(); err != nil {
7188
return nil, fmt.Errorf("failed to upgrade telepresence traffic manager: %w, %s", err, string(out))
7289
}
7390
} else if err != nil {
7491
return nil, fmt.Errorf("failed to install telepresence traffic manager: %w, %s", err, string(out))
7592
}
7693

77-
fmt.Println("INFO: connecting to the cluster with telepresence")
78-
connectArgs := []string{"connect"}
79-
if ns := os.Getenv("POD_NAMESPACE"); ns != "" {
80-
connectArgs = append(connectArgs, "--namespace", ns)
81-
}
82-
out, err = exec.CommandContext(ctx, telepresenceExec, connectArgs...).CombinedOutput()
83-
if err != nil {
84-
return nil, fmt.Errorf("failed to connect to the cluster with telepresence: %w, %s", err, string(out))
85-
}
86-
8794
return func() {
8895
fmt.Println("INFO: quitting telepresence daemons")
89-
out, err := exec.CommandContext(ctx, telepresenceExec, "quit").CombinedOutput()
96+
out, err := exec.CommandContext(ctx, telepresenceExecutable, "quit").CombinedOutput()
9097
if err != nil {
9198
fmt.Printf("ERROR: failed to quit telepresence daemons: %s\n", string(out))
9299
}

0 commit comments

Comments
 (0)