[KEP] FlavorFungability: replace FlavorFungibilityImplicitPreferenceDefault feature gate with API

* preference API field

Author: vladikkuzn

apis/kueue/v1beta1/clusterqueue_types.go

@@ -389,6 +389,13 @@ const (
 	TryNextFlavor FlavorFungibilityPolicy = "TryNextFlavor"
 )
 
+type FlavorFungibilityPreference string
+
+const (
+	PreferenceBorrowing  FlavorFungibilityPreference = "Borrow"
+	PreferencePreempting FlavorFungibilityPreference = "Preempt"
+)
+
 // FlavorFungibility determines whether a workload should try the next flavor
 // before borrowing or preempting in current flavor.
 type FlavorFungibility struct {
@@ -415,6 +422,13 @@ type FlavorFungibility struct {
 	// +kubebuilder:validation:Enum={MayStopSearch,TryNextFlavor,Preempt}
 	// +kubebuilder:default="TryNextFlavor"
 	WhenCanPreempt FlavorFungibilityPolicy `json:"whenCanPreempt,omitempty"`
+	// preference selects the order between borrowing-first and preemption-first
+	// when both WhenCanBorrow and WhenCanPreempt are set to `TryNextFlavor`.
+	// If unset, the default preference is borrowing-first.
+	// Preference must be unset when either WhenCanBorrow or WhenCanPreempt is not `TryNextFlavor`.
+	//
+	// +kubebuilder:validation:Enum={Borrowing,Preempting}
+	Preference *FlavorFungibilityPreference `json:"preference,omitempty"`
 }
 
 // ClusterQueuePreemption contains policies to preempt Workloads from this

apis/kueue/v1beta1/zz_generated.conversion.go

@@ -370,6 +370,13 @@ const (
 	TryNextFlavor FlavorFungibilityPolicy = "TryNextFlavor"
 )
 
+type FlavorFungibilityPreference string
+
+const (
+	PreferenceBorrowing  FlavorFungibilityPreference = "Borrow"
+	PreferencePreempting FlavorFungibilityPreference = "Preempt"
+)
+
 // FlavorFungibility determines whether a workload should try the next flavor
 // before borrowing or preempting in current flavor.
 type FlavorFungibility struct {
@@ -396,6 +403,13 @@ type FlavorFungibility struct {
 	// +kubebuilder:validation:Enum={MayStopSearch,TryNextFlavor,Preempt}
 	// +kubebuilder:default="TryNextFlavor"
 	WhenCanPreempt FlavorFungibilityPolicy `json:"whenCanPreempt,omitempty"`
+	// preference selects the order between borrowing-first and preemption-first
+	// when both WhenCanBorrow and WhenCanPreempt are set to `TryNextFlavor`.
+	// If unset, the default preference is borrowing-first. Preference must be
+	// unset when either WhenCanBorrow or WhenCanPreempt is not `TryNextFlavor`.
+	//
+	// +kubebuilder:validation:Enum={Borrowing,Preempting}
+	Preference *FlavorFungibilityPreference `json:"preference,omitempty"`
 }
 
 // ClusterQueuePreemption contains policies to preempt Workloads from this

apis/kueue/v1beta1/zz_generated.deepcopy.go

@@ -192,6 +192,16 @@ spec:
                     flavorFungibility defines whether a workload should try the next flavor
                     before borrowing or preempting in the flavor being evaluated.
                   properties:
+                    preference:
+                      description: |-
+                        preference selects the order between borrowing-first and preemption-first
+                        when both WhenCanBorrow and WhenCanPreempt are set to `TryNextFlavor`.
+                        If unset, the default preference is borrowing-first.
+                        Preference must be unset when either WhenCanBorrow or WhenCanPreempt is not `TryNextFlavor`.
+                      enum:
+                        - Borrowing
+                        - Preempting
+                      type: string
                     whenCanBorrow:
                       default: MayStopSearch
                       description: |-
@@ -948,6 +958,16 @@ spec:
                     flavorFungibility defines whether a workload should try the next flavor
                     before borrowing or preempting in the flavor being evaluated.
                   properties:
+                    preference:
+                      description: |-
+                        preference selects the order between borrowing-first and preemption-first
+                        when both WhenCanBorrow and WhenCanPreempt are set to `TryNextFlavor`.
+                        If unset, the default preference is borrowing-first. Preference must be
+                        unset when either WhenCanBorrow or WhenCanPreempt is not `TryNextFlavor`.
+                      enum:
+                        - Borrowing
+                        - Preempting
+                      type: string
                     whenCanBorrow:
                       default: MayStopSearch
                       description: |-

apis/kueue/v1beta2/clusterqueue_types.go

@@ -163,6 +163,16 @@ spec:
                   flavorFungibility defines whether a workload should try the next flavor
                   before borrowing or preempting in the flavor being evaluated.
                 properties:
+                  preference:
+                    description: |-
+                      preference selects the order between borrowing-first and preemption-first
+                      when both WhenCanBorrow and WhenCanPreempt are set to `TryNextFlavor`.
+                      If unset, the default preference is borrowing-first.
+                      Preference must be unset when either WhenCanBorrow or WhenCanPreempt is not `TryNextFlavor`.
+                    enum:
+                    - Borrowing
+                    - Preempting
+                    type: string
                   whenCanBorrow:
                     default: MayStopSearch
                     description: |-
@@ -936,6 +946,16 @@ spec:
                   flavorFungibility defines whether a workload should try the next flavor
                   before borrowing or preempting in the flavor being evaluated.
                 properties:
+                  preference:
+                    description: |-
+                      preference selects the order between borrowing-first and preemption-first
+                      when both WhenCanBorrow and WhenCanPreempt are set to `TryNextFlavor`.
+                      If unset, the default preference is borrowing-first. Preference must be
+                      unset when either WhenCanBorrow or WhenCanPreempt is not `TryNextFlavor`.
+                    enum:
+                    - Borrowing
+                    - Preempting
+                    type: string
                   whenCanBorrow:
                     default: MayStopSearch
                     description: |-

apis/kueue/v1beta2/zz_generated.deepcopy.go

@@ -157,13 +157,6 @@ const (
 	// matches managedJobsNamespaceSelector.
 	ManagedJobsNamespaceSelectorAlwaysRespected featuregate.Feature = "ManagedJobsNamespaceSelectorAlwaysRespected"
 
-	// owner: @pajakd
-	// kep: https://github.com/kubernetes-sigs/kueue/tree/main/keps/582-preempt-based-on-flavor-order
-	//
-	// In flavor fungibility, the preference whether to preempt or borrow is inferred from flavor fungibility policy
-	// This feature gate is going to be replaced by an API before graduation or deprecation.
-	FlavorFungibilityImplicitPreferenceDefault featuregate.Feature = "FlavorFungibilityImplicitPreferenceDefault"
-
 	// owner: @alaypatel07
 	// kep: https://github.com/kubernetes-sigs/kueue/tree/main/keps/2941-DRA
 	//
@@ -288,9 +281,6 @@ var defaultVersionedFeatureGates = map[featuregate.Feature]featuregate.Versioned
 	ManagedJobsNamespaceSelectorAlwaysRespected: {
 		{Version: version.MustParse("0.13"), Default: false, PreRelease: featuregate.Alpha},
 	},
-	FlavorFungibilityImplicitPreferenceDefault: {
-		{Version: version.MustParse("0.13"), Default: false, PreRelease: featuregate.Alpha},
-	},
 	DynamicResourceAllocation: {
 		{Version: version.MustParse("0.14"), Default: false, PreRelease: featuregate.Alpha},
 	},