[KEP] FlavorFungability: replace FlavorFungibilityImplicitPreferenceDefault feature gate with API

Author: vladikkuzn

apis/kueue/v1beta1/clusterqueue_types.go

@@ -388,8 +388,16 @@ const (
 	TryNextFlavor FlavorFungibilityPolicy = "TryNextFlavor"
 )
 
+type FlavorFungibilityPreference string
+
+const (
+	BorrowingOverPreemption FlavorFungibilityPreference = "BorrowingOverPreemption"
+	PreemptionOverBorrowing FlavorFungibilityPreference = "PreemptionOverBorrowing"
+)
+
 // FlavorFungibility determines whether a workload should try the next flavor
 // before borrowing or preempting in current flavor.
+// +kubebuilder:validation:XValidation:rule="!has(self.preference) || (self.whenCanBorrow == 'TryNextFlavor' && self.whenCanPreempt == 'TryNextFlavor')",message="preference can only be set when both whenCanBorrow and whenCanPreempt are TryNextFlavor"
 type FlavorFungibility struct {
 	// whenCanBorrow determines whether a workload should try the next flavor
 	// before borrowing in current flavor. The possible values are:
@@ -414,6 +422,20 @@ type FlavorFungibility struct {
 	// +kubebuilder:validation:Enum={MayStopSearch,TryNextFlavor,Preempt}
 	// +kubebuilder:default="TryNextFlavor"
 	WhenCanPreempt FlavorFungibilityPolicy `json:"whenCanPreempt,omitempty"`
+	// preference guides the choosing of the flavor for admission in case all candidate flavors
+	// require either preemption, borrowing, or both. The possible values are:
+	// - `BorrowingOverPreemption` (default): prefer to use borrowing rather than preemption
+	// when such a choice is possible. More technically it minimizes the borrowing distance
+	// in the cohort tree, and solves tie-breaks by preferring better preemption mode
+	// (reclaim over preemption within ClusterQueue).
+	// - `PreemptionOverBorrowing`: prefer to use preemption rather than borrowing
+	// when such a choice is possible.  More technically it optimizes the preemption mode
+	// (reclaim over preemption within ClusterQueue), and solves tie-breaks by minimizing
+	// the borrowing distance in the cohort tree.
+	//
+	// +kubebuilder:validation:Enum={BorrowingOverPreemption,PreemptionOverBorrowing}
+	// +optional
+	Preference *FlavorFungibilityPreference `json:"preference,omitempty"`
 }
 
 // ClusterQueuePreemption contains policies to preempt Workloads from this

apis/kueue/v1beta1/zz_generated.conversion.go

@@ -380,8 +380,16 @@ const (
 	TryNextFlavor FlavorFungibilityPolicy = "TryNextFlavor"
 )
 
+type FlavorFungibilityPreference string
+
+const (
+	BorrowingOverPreemption FlavorFungibilityPreference = "BorrowingOverPreemption"
+	PreemptionOverBorrowing FlavorFungibilityPreference = "PreemptionOverBorrowing"
+)
+
 // FlavorFungibility determines whether a workload should try the next flavor
 // before borrowing or preempting in current flavor.
+// +kubebuilder:validation:XValidation:rule="!has(self.preference) || (self.whenCanBorrow == 'TryNextFlavor' && self.whenCanPreempt == 'TryNextFlavor')",message="preference can only be set when both whenCanBorrow and whenCanPreempt are TryNextFlavor"
 type FlavorFungibility struct {
 	// whenCanBorrow determines whether a workload should try the next flavor
 	// before borrowing in current flavor. The possible values are:
@@ -406,6 +414,20 @@ type FlavorFungibility struct {
 	// +kubebuilder:default="TryNextFlavor"
 	// +optional
 	WhenCanPreempt FlavorFungibilityPolicy `json:"whenCanPreempt,omitempty"`
+	// preference guides the choosing of the flavor for admission in case all candidate flavors
+	// require either preemption, borrowing, or both. The possible values are:
+	// - `BorrowingOverPreemption` (default): prefer to use borrowing rather than preemption
+	// when such a choice is possible. More technically it minimizes the borrowing distance
+	// in the cohort tree, and solves tie-breaks by preferring better preemption mode
+	// (reclaim over preemption within ClusterQueue).
+	// - `PreemptionOverBorrowing`: prefer to use preemption rather than borrowing
+	// when such a choice is possible.  More technically it optimizes the preemption mode
+	// (reclaim over preemption within ClusterQueue), and solves tie-breaks by minimizing
+	// the borrowing distance in the cohort tree.
+	//
+	// +kubebuilder:validation:Enum={BorrowingOverPreemption,PreemptionOverBorrowing}
+	// +optional
+	Preference *FlavorFungibilityPreference `json:"preference,omitempty"`
 }
 
 // ClusterQueuePreemption contains policies to preempt Workloads from this

apis/kueue/v1beta1/zz_generated.deepcopy.go

@@ -192,6 +192,22 @@ spec:
                     flavorFungibility defines whether a workload should try the next flavor
                     before borrowing or preempting in the flavor being evaluated.
                   properties:
+                    preference:
+                      description: |-
+                        preference guides the choosing of the flavor for admission in case all candidate flavors
+                        require either preemption, borrowing, or both. The possible values are:
+                        - `BorrowingOverPreemption` (default): prefer to use borrowing rather than preemption
+                        when such a choice is possible. More technically it minimizes the borrowing distance
+                        in the cohort tree, and solves tie-breaks by preferring better preemption mode
+                        (reclaim over preemption within ClusterQueue).
+                        - `PreemptionOverBorrowing`: prefer to use preemption rather than borrowing
+                        when such a choice is possible.  More technically it optimizes the preemption mode
+                        (reclaim over preemption within ClusterQueue), and solves tie-breaks by minimizing
+                        the borrowing distance in the cohort tree.
+                      enum:
+                        - BorrowingOverPreemption
+                        - PreemptionOverBorrowing
+                      type: string
                     whenCanBorrow:
                       default: MayStopSearch
                       description: |-
@@ -224,6 +240,9 @@ spec:
                         - Preempt
                       type: string
                   type: object
+                  x-kubernetes-validations:
+                    - message: preference can only be set when both whenCanBorrow and whenCanPreempt are TryNextFlavor
+                      rule: '!has(self.preference) || (self.whenCanBorrow == ''TryNextFlavor'' && self.whenCanPreempt == ''TryNextFlavor'')'
                 namespaceSelector:
                   description: |-
                     namespaceSelector defines which namespaces are allowed to submit workloads to
@@ -940,6 +959,22 @@ spec:
                     flavorFungibility defines whether a workload should try the next flavor
                     before borrowing or preempting in the flavor being evaluated.
                   properties:
+                    preference:
+                      description: |-
+                        preference guides the choosing of the flavor for admission in case all candidate flavors
+                        require either preemption, borrowing, or both. The possible values are:
+                        - `BorrowingOverPreemption` (default): prefer to use borrowing rather than preemption
+                        when such a choice is possible. More technically it minimizes the borrowing distance
+                        in the cohort tree, and solves tie-breaks by preferring better preemption mode
+                        (reclaim over preemption within ClusterQueue).
+                        - `PreemptionOverBorrowing`: prefer to use preemption rather than borrowing
+                        when such a choice is possible.  More technically it optimizes the preemption mode
+                        (reclaim over preemption within ClusterQueue), and solves tie-breaks by minimizing
+                        the borrowing distance in the cohort tree.
+                      enum:
+                        - BorrowingOverPreemption
+                        - PreemptionOverBorrowing
+                      type: string
                     whenCanBorrow:
                       default: MayStopSearch
                       description: |-
@@ -968,6 +1003,9 @@ spec:
                         - TryNextFlavor
                       type: string
                   type: object
+                  x-kubernetes-validations:
+                    - message: preference can only be set when both whenCanBorrow and whenCanPreempt are TryNextFlavor
+                      rule: '!has(self.preference) || (self.whenCanBorrow == ''TryNextFlavor'' && self.whenCanPreempt == ''TryNextFlavor'')'
                 namespaceSelector:
                   description: |-
                     namespaceSelector defines which namespaces are allowed to submit workloads to

apis/kueue/v1beta2/clusterqueue_types.go

@@ -163,6 +163,22 @@ spec:
                   flavorFungibility defines whether a workload should try the next flavor
                   before borrowing or preempting in the flavor being evaluated.
                 properties:
+                  preference:
+                    description: |-
+                      preference guides the choosing of the flavor for admission in case all candidate flavors
+                      require either preemption, borrowing, or both. The possible values are:
+                      - `BorrowingOverPreemption` (default): prefer to use borrowing rather than preemption
+                      when such a choice is possible. More technically it minimizes the borrowing distance
+                      in the cohort tree, and solves tie-breaks by preferring better preemption mode
+                      (reclaim over preemption within ClusterQueue).
+                      - `PreemptionOverBorrowing`: prefer to use preemption rather than borrowing
+                      when such a choice is possible.  More technically it optimizes the preemption mode
+                      (reclaim over preemption within ClusterQueue), and solves tie-breaks by minimizing
+                      the borrowing distance in the cohort tree.
+                    enum:
+                    - BorrowingOverPreemption
+                    - PreemptionOverBorrowing
+                    type: string
                   whenCanBorrow:
                     default: MayStopSearch
                     description: |-
@@ -195,6 +211,11 @@ spec:
                     - Preempt
                     type: string
                 type: object
+                x-kubernetes-validations:
+                - message: preference can only be set when both whenCanBorrow and
+                    whenCanPreempt are TryNextFlavor
+                  rule: '!has(self.preference) || (self.whenCanBorrow == ''TryNextFlavor''
+                    && self.whenCanPreempt == ''TryNextFlavor'')'
               namespaceSelector:
                 description: |-
                   namespaceSelector defines which namespaces are allowed to submit workloads to
@@ -929,6 +950,22 @@ spec:
                   flavorFungibility defines whether a workload should try the next flavor
                   before borrowing or preempting in the flavor being evaluated.
                 properties:
+                  preference:
+                    description: |-
+                      preference guides the choosing of the flavor for admission in case all candidate flavors
+                      require either preemption, borrowing, or both. The possible values are:
+                      - `BorrowingOverPreemption` (default): prefer to use borrowing rather than preemption
+                      when such a choice is possible. More technically it minimizes the borrowing distance
+                      in the cohort tree, and solves tie-breaks by preferring better preemption mode
+                      (reclaim over preemption within ClusterQueue).
+                      - `PreemptionOverBorrowing`: prefer to use preemption rather than borrowing
+                      when such a choice is possible.  More technically it optimizes the preemption mode
+                      (reclaim over preemption within ClusterQueue), and solves tie-breaks by minimizing
+                      the borrowing distance in the cohort tree.
+                    enum:
+                    - BorrowingOverPreemption
+                    - PreemptionOverBorrowing
+                    type: string
                   whenCanBorrow:
                     default: MayStopSearch
                     description: |-
@@ -957,6 +994,11 @@ spec:
                     - TryNextFlavor
                     type: string
                 type: object
+                x-kubernetes-validations:
+                - message: preference can only be set when both whenCanBorrow and
+                    whenCanPreempt are TryNextFlavor
+                  rule: '!has(self.preference) || (self.whenCanBorrow == ''TryNextFlavor''
+                    && self.whenCanPreempt == ''TryNextFlavor'')'
               namespaceSelector:
                 description: |-
                   namespaceSelector defines which namespaces are allowed to submit workloads to

apis/kueue/v1beta2/zz_generated.deepcopy.go

@@ -160,8 +160,8 @@ const (
 	// owner: @pajakd
 	// kep: https://github.com/kubernetes-sigs/kueue/tree/main/keps/582-preempt-based-on-flavor-order
 	//
-	// In flavor fungibility, the preference whether to preempt or borrow is inferred from flavor fungibility policy
-	// This feature gate is going to be replaced by an API before graduation or deprecation.
+	// In flavor fungibility, the preference whether to preempt or borrow is inferred from flavor fungibility policy.
+	// Deprecated: planned to be removed in v0.16.
 	FlavorFungibilityImplicitPreferenceDefault featuregate.Feature = "FlavorFungibilityImplicitPreferenceDefault"
 
 	// owner: @alaypatel07
@@ -297,6 +297,7 @@ var defaultVersionedFeatureGates = map[featuregate.Feature]featuregate.Versioned
 	},
 	FlavorFungibilityImplicitPreferenceDefault: {
 		{Version: version.MustParse("0.13"), Default: false, PreRelease: featuregate.Alpha},
+		{Version: version.MustParse("0.15"), Default: false, PreRelease: featuregate.Deprecated}, // remove in 0.16
 	},
 	DynamicResourceAllocation: {
 		{Version: version.MustParse("0.14"), Default: false, PreRelease: featuregate.Alpha},