neonvm-controller: reimplement IPAM via Manager

Makefile

@@ -546,7 +546,7 @@ k3d-load: k3d # Push docker images to the k3d cluster.
 ##@ Example VM images
 
 .PHONY: vm-examples
-examples: vm-postgres pg16-disk-test ## Build example VM images
+vm-examples: vm-postgres pg16-disk-test ## Build example VM images
 
 .PHONY: vm-postgres
 vm-postgres: docker-build-vm-postgres load-vm-postgres
@@ -556,14 +556,17 @@ pg16-disk-test: docker-build-pg16-disk-test load-pg16-disk-test
 
 ##@ End-to-End tests
 
-.PHONE: e2e-tools
+.PHONY: e2e-tools
 e2e-tools: k3d kind kubectl kuttl python-init ## Donwnload tools for e2e tests locally if necessary.
 
-.PHONE: e2e
+.PHONY: e2e
 e2e: check-local-context e2e-tools ## Run e2e kuttl tests
 	$(KUTTL) test --config tests/e2e/kuttl-test.yaml $(if $(CI),--skip-delete)
 	rm -f kubeconfig
 
+.PHONY: run-e2e
+run-e2e: deploy vm-examples e2e ## Helper target to do all steps to run e2e tests locally
+
 ##@ Local kind cluster
 
 .PHONY: kind-setup

README.md

@@ -137,3 +137,7 @@ To run the end-to-end tests, you need to have [`kuttl`] installed. You can run t
 ```sh
 make e2e
 ```
+
+### `make run-e2e`
+
+During active development, when the kernel is already built and cluster created, one can do `make run-e2e` to test current code, which resolves to `make deploy vm-examples e2e`.

go.mod

@@ -40,6 +40,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
+	github.com/Jille/contextcond v1.0.0
 	github.com/alessio/shellescape v1.4.1
 	github.com/aws/aws-sdk-go-v2/config v1.27.36
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2
@@ -57,7 +58,6 @@ require (
 	github.com/go-logr/zapr v1.3.0
 	github.com/jpillora/backoff v1.0.0
 	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0
-	github.com/k8snetworkplumbingwg/whereabouts v0.6.1
 	github.com/kdomanski/iso9660 v0.3.3
 	github.com/lithammer/shortuuid v3.0.0+incompatible
 	github.com/onsi/ginkgo/v2 v2.19.0
@@ -75,7 +75,6 @@ require (
 	go.uber.org/zap v1.27.0
 	golang.org/x/crypto v0.36.0
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
-	golang.org/x/sync v0.12.0
 	golang.org/x/term v0.30.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.31.7
@@ -195,6 +194,7 @@ require (
 	golang.org/x/mod v0.20.0 // indirect
 	golang.org/x/net v0.38.0 // indirect
 	golang.org/x/oauth2 v0.27.0 // indirect
+	golang.org/x/sync v0.12.0 // indirect
 	golang.org/x/sys v0.31.0 // indirect
 	golang.org/x/text v0.23.0 // indirect
 	golang.org/x/time v0.6.0 // indirect

go.sum

@@ -15,6 +15,8 @@ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/Jille/contextcond v1.0.0 h1:MI7hdTKyS6CLc5DIJmyjN8iEfw7/e8HJ5ZOwTJpS9SI=
+github.com/Jille/contextcond v1.0.0/go.mod h1:aWoovvPMcWinBsm69wTnhSrCwTGtCfg4xr4Nku8WgIg=
 github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg=
 github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE=
 github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
@@ -249,8 +251,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 h1:VzM3TYHDgqPkettiP6I6q2jOeQFL4nrJM+UcAc4f6Fs=
 github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0/go.mod h1:nqCI7aelBJU61wiBeeZWJ6oi4bJy5nrjkM6lWIMA4j0=
-github.com/k8snetworkplumbingwg/whereabouts v0.6.1 h1:3pfShDMF9+/7ijzKUPezoBqN2IvlsI9VbSHYUw+ku6U=
-github.com/k8snetworkplumbingwg/whereabouts v0.6.1/go.mod h1:FbmUjZg27cI6om0IAc+NV5Ur+IKwHyqdLaeR0SGfWJc=
 github.com/kdomanski/iso9660 v0.3.3 h1:cNwM9L2L1Hzc5hZWGy6fPJ92UyWDccaY69DmEPlfDNY=
 github.com/kdomanski/iso9660 v0.3.3/go.mod h1:K+UlIGxKgtrdAWyoigPnFbeQLVs/Xudz4iztWFThBwo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -281,13 +281,10 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
-github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
-github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
 github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA=
 github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To=
@@ -546,7 +543,6 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

neonvm-controller/cmd/main.go

@@ -150,20 +150,19 @@ func main() {
 	}
 
 	ipam, err := ipam.New(ipam.IPAMParams{
-		NadName:      rc.NADConfig.IPAMName,
-		NadNamespace: rc.NADConfig.IPAMNamespace,
-
-		// Let's not have more than a quarter of reconcilliation workers stuck
-		// at IPAM mutex.
-		ConcurrencyLimit: max(1, cli.concurrencyLimit/4),
+		NADName:      rc.NADConfig.IPAMName,
+		NADNamespace: rc.NADConfig.IPAMNamespace,
 
 		MetricsReg: metrics.Registry,
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to create ipam")
 		panic(err)
 	}
-	defer ipam.Close()
+	if err := mgr.Add(ipam); err != nil {
+		setupLog.Error(err, "unable to add ipam")
+		panic(err)
+	}
 
 	retryChan := reqchan.NewRequestChannel()
 	defer retryChan.Close()

neonvm/apis/neonvm/v1/ippool_types.go

@@ -1,23 +1,71 @@
 package v1
 
 import (
+	"fmt"
+	"math/big"
+	"net"
+	"net/netip"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+// Unit represents an empty struct for use in sets/maps
+type Unit struct{}
+
 // IPPoolSpec defines the desired state of IPPool
 type IPPoolSpec struct {
 	// Range is a RFC 4632/4291-style string that represents an IP address and prefix length in CIDR notation
 	Range string `json:"range"`
+
+	// Managed is the set of IPs that are managed by the IPAM manager.
+	Managed map[string]Unit `json:"managed"`
+
+	// Deprecated: This field is deprecated and will be removed in a future version.
 	// Allocations is the set of allocated IPs for the given range. Its` indices are a direct mapping to the
 	// IP with the same index/offset for the pool's range.
+	// +kubebuilder:validation:Optional
 	Allocations map[string]IPAllocation `json:"allocations"`
 }
 
-// IPAllocation represents metadata about the pod/container owner of a specific IP
-// coped from Whereabout CNI as their allocation functions used
+// Deprecated: This type is deprecated and will be removed in a future version.
+// IPAllocation represents metadata about the VM owner of a specific IP.
 type IPAllocation struct {
-	ContainerID string `json:"id"`
-	PodRef      string `json:"podref,omitempty"`
+	VMID string `json:"id"`
+}
+
+func (a *IPPoolSpec) Normalize() error {
+	if len(a.Managed) > 0 {
+		// Pool is already initialized
+		return nil
+	}
+
+	a.Managed = make(map[string]Unit)
+
+	ip, _, err := net.ParseCIDR(a.Range)
+	if err != nil {
+		return fmt.Errorf("invalid range: %w", err)
+	}
+
+	ip4 := ip.To4()
+	if ip4 == nil {
+		return fmt.Errorf("invalid range: %s", a.Range)
+	}
+	baseInt := new(big.Int).SetBytes(ip4)
+	for offset := range a.Allocations {
+		offsetInt, ok := new(big.Int).SetString(offset, 10)
+		if !ok {
+			return fmt.Errorf("invalid offset: %s", offset)
+		}
+
+		ip, ok := netip.AddrFromSlice(baseInt.Add(baseInt, offsetInt).Bytes())
+		if !ok {
+			return fmt.Errorf("invalid ip: %s", ip)
+		}
+
+		a.Managed[ip.String()] = Unit{}
+	}
+
+	return nil
 }
 
 //+genclient

neonvm/config/crd/bases/vm.neon.tech_ippools.yaml

@@ -42,26 +42,32 @@ spec:
               allocations:
                 additionalProperties:
                   description: |-
-                    IPAllocation represents metadata about the pod/container owner of a specific IP
-                    coped from Whereabout CNI as their allocation functions used
+                    Deprecated: This type is deprecated and will be removed in a future version.
+                    IPAllocation represents metadata about the VM owner of a specific IP.
                   properties:
                     id:
                       type: string
-                    podref:
-                      type: string
                   required:
                   - id
                   type: object
                 description: |-
+                  Deprecated: This field is deprecated and will be removed in a future version.
                   Allocations is the set of allocated IPs for the given range. Its` indices are a direct mapping to the
                   IP with the same index/offset for the pool's range.
                 type: object
+              managed:
+                additionalProperties:
+                  description: Unit represents an empty struct for use in sets/maps
+                  type: object
+                description: Managed is the set of IPs that are managed by the IPAM
+                  manager.
+                type: object
               range:
                 description: Range is a RFC 4632/4291-style string that represents
                   an IP address and prefix length in CIDR notation
                 type: string
             required:
-            - allocations
+            - managed
             - range
             type: object
         type: object

neonvm/config/network/overlay-ipam.yaml

@@ -7,6 +7,12 @@ spec:
   config: '{
     "ipam": {
       "network_name": "neonvm",
+      "manager": {
+        "cooldown_period": "10s",
+        "high_ip_count": 100,
+        "low_ip_count": 50,
+        "target_ip_count": 75
+      },
       "ipRanges": [
         {
           "range": "10.100.0.0/16",

pkg/neonvm/controllers/vm_controller.go

@@ -25,6 +25,7 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
+	"net"
 	"os"
 	"reflect"
 	"time"
@@ -228,11 +229,21 @@ func (r *VMReconciler) doFinalizerOperationsForVirtualMachine(ctx context.Contex
 	log.Info("Deleting VirtualMachine")
 
 	// Release overlay IP address
-	if vm.Spec.ExtraNetwork != nil {
-		ip, err := r.IPAM.ReleaseIP(ctx, types.NamespacedName{Name: vm.Name, Namespace: vm.Namespace})
-		if err != nil {
-			return fmt.Errorf("fail to release overlay IP: %w", err)
+	if vm.Spec.ExtraNetwork != nil && len(vm.Status.ExtraNetIP) > 0 {
+		ip := net.ParseIP(vm.Status.ExtraNetIP)
+		if ip == nil {
+			return fmt.Errorf("invalid IP: %s", vm.Status.ExtraNetIP)
+		}
+
+		// First, clear the IP from the status
+		vm.Status.ExtraNetIP = ""
+		if err := r.Status().Update(ctx, vm); err != nil {
+			return fmt.Errorf("Failed to update VirtualMachine status: %w", err)
 		}
+
+		// Notify IPAM only if status update was successful
+		r.IPAM.ReleaseIP(ctx, vm.UID, ip)
+
 		log.Info(fmt.Sprintf("Released overlay IP %s", ip.String()))
 	}
 	return nil
@@ -294,13 +305,13 @@ func (r *VMReconciler) acquireOverlayIP(ctx context.Context, vm *vmv1.VirtualMac
 	}
 
 	log := log.FromContext(ctx)
-	ip, err := r.IPAM.AcquireIP(ctx, types.NamespacedName{Name: vm.Name, Namespace: vm.Namespace})
+	ip, err := r.IPAM.AcquireIP(ctx, vm.UID)
 	if err != nil {
 		return err
 	}
-	vm.Status.ExtraNetIP = ip.IP.String()
+	vm.Status.ExtraNetIP = ip.String()
 	vm.Status.ExtraNetMask = fmt.Sprintf("%d.%d.%d.%d", ip.Mask[0], ip.Mask[1], ip.Mask[2], ip.Mask[3])
-	log.Info(fmt.Sprintf("Acquired IP %s for overlay network interface", ip.String()))
+	log.Info("Acquired overlay IP", "IP", ip.String(), "mask", vm.Status.ExtraNetMask)
 	return nil
 }