[CORE-149029] - Sync ScaleOps Resource Requests and Affinities

api/v1beta1/kafkacluster_types.go

@@ -148,6 +148,12 @@ type KafkaClusterSpec struct {
 	// +optional
 	KRaftMode              bool `json:"kRaft"`
 	HeadlessServiceEnabled bool `json:"headlessServiceEnabled"`
+	// Allows ScaleOps or other Admission Hooks to manage Memory and CPU Resource Requests for
+	// Kafka Broker Pods.  This Disables CPU and Memory request reconciliation from the desired
+	// state defined in the KafkaCluster to the current state in the Kubernetes Cluster
+	// +kubebuilder:default=false
+	// +optional
+	AdmissionWebhooksEnabled bool `json:"admissionWebhooksEnabled,omitempty"`
 	// localDebugEnabled is used to decide whether to create a separate loadbalancer services for the
 	// Kafka and Cruise Control Pods. These services will expose the internal listener ports of the Kafka
 	// cluster with LoadBalancer type, which can be used for running Koperator on a local machine against

charts/kafka-operator/crds/kafkaclusters.yaml

@@ -96,6 +96,13 @@ spec:
                   - containerPort
                   type: object
                 type: array
+              admissionWebhooksEnabled:
+                default: false
+                description: |-
+                  Allows ScaleOps or other Admission Hooks to manage Memory and CPU Resource Requests for
+                  Kafka Broker Pods.  This Disables CPU and Memory request reconciliation from the desired
+                  state defined in the KafkaCluster to the current state in the Kubernetes Cluster
+                type: boolean
               alertManagerConfig:
                 description: AlertManagerConfig defines configuration for alert manager
                 properties:

config/base/crds/kafka.banzaicloud.io_kafkaclusters.yaml

@@ -96,6 +96,13 @@ spec:
                   - containerPort
                   type: object
                 type: array
+              admissionWebhooksEnabled:
+                default: false
+                description: |-
+                  Allows ScaleOps or other Admission Hooks to manage Memory and CPU Resource Requests for
+                  Kafka Broker Pods.  This Disables CPU and Memory request reconciliation from the desired
+                  state defined in the KafkaCluster to the current state in the Kubernetes Cluster
+                type: boolean
               alertManagerConfig:
                 description: AlertManagerConfig defines configuration for alert manager
                 properties:

config/samples/simplekafkacluster.yaml

@@ -6,6 +6,7 @@ metadata:
   name: kafka
 spec:
   localDebugEnabled: true
+  admissionWebhooksEnabled: true
   kRaft: false
   monitoringConfig:
     jmxImage: "ghcr.io/adobe/koperator/jmx-javaagent:1.4.0"

pkg/resources/kafka/kafka.go

@@ -956,6 +956,16 @@ func (r *Reconciler) handleRollingUpgrade(log logr.Logger, desiredPod, currentPo
 		}
 		desiredPod.Spec.Tolerations = uniqueTolerations
 	}
+
+	if r.KafkaCluster.Spec.AdmissionWebhooksEnabled {
+		// if resources requets are updated by scale ops, we need to sync them to desiredPod,
+		// otherwise they will be removed and cause pod restart
+		syncResourceRequests(desiredPod, currentPod)
+		// If current pod had affinities created by ScaleOps, we need to sync them to desiredPod,
+		// otherwise they will be removed and cause pod restart
+		syncAffinities(desiredPod, currentPod)
+	}
+
 	// Check if the resource actually updated or if labels match TaintedBrokersSelector
 	patchResult, err := patch.DefaultPatchMaker.Calculate(currentPod, desiredPod)
 	switch {

pkg/resources/kafka/util.go

@@ -18,9 +18,11 @@ package kafka
 import (
 	"encoding/base64"
 	"fmt"
+	"reflect"
 	"sort"
 
 	"github.com/google/uuid"
+	corev1 "k8s.io/api/core/v1"
 
 	"github.com/banzaicloud/koperator/api/v1beta1"
 )
@@ -73,3 +75,133 @@ func generateRandomClusterID() string {
 	randomUUID := uuid.New()
 	return base64.URLEncoding.EncodeToString(randomUUID[:])
 }
+
+// syncResourceRequests overwrites CPU and memory requests in desiredPod's containers
+// with the values from currentPod so that request-only changes do not trigger a pod restart.
+func syncResourceRequests(desiredPod, currentPod *corev1.Pod) {
+	syncContainerResourceRequests(desiredPod.Spec.Containers, currentPod.Spec.Containers)
+	syncContainerResourceRequests(desiredPod.Spec.InitContainers, currentPod.Spec.InitContainers)
+}
+
+func syncContainerResourceRequests(desired, current []corev1.Container) {
+	index := make(map[string]corev1.ResourceList, len(current))
+	for _, c := range current {
+		index[c.Name] = c.Resources.Requests
+	}
+	for i := range desired {
+		c := &desired[i]
+		reqs, ok := index[c.Name]
+		if !ok {
+			continue
+		}
+		if c.Resources.Requests == nil {
+			c.Resources.Requests = make(corev1.ResourceList)
+		}
+		for _, res := range []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory} {
+			if val, exists := reqs[res]; exists {
+				c.Resources.Requests[res] = val
+			} else {
+				delete(c.Resources.Requests, res)
+			}
+		}
+	}
+}
+
+// syncScaleOpsAffinities syncs all scale ops related affinities from the current pod to the desired pod.
+// This includes pod affinities with scaleOpsManagedUnevictableLabel label selector
+// and node affinities with "scaleops.sh/node-packing=true" selector.
+func syncAffinities(desiredPod, currentPod *corev1.Pod) {
+	syncPodAffinities(desiredPod, currentPod)
+	syncNodeAffinities(desiredPod, currentPod)
+}
+
+// syncScaleOpsPodAffinities syncs preferred pod affinities with scaleOpsManagedUnevictableLabel
+// label selector from current pod to desired pod.
+func syncPodAffinities(desiredPod, currentPod *corev1.Pod) {
+	if currentPod.Spec.Affinity == nil || currentPod.Spec.Affinity.PodAffinity == nil {
+		return
+	}
+
+	currentPodAffinity := currentPod.Spec.Affinity.PodAffinity
+
+	// Filter preferred pod affinities with scaleOpsManagedUnevictableLabel label selector
+	var admissionWebhookPpreferredAffinities []corev1.WeightedPodAffinityTerm
+	if currentPodAffinity.PreferredDuringSchedulingIgnoredDuringExecution != nil {
+		for _, term := range currentPodAffinity.PreferredDuringSchedulingIgnoredDuringExecution {
+			if term.PodAffinityTerm.LabelSelector != nil {
+				admissionWebhookPpreferredAffinities = append(admissionWebhookPpreferredAffinities, term)
+			}
+		}
+	}
+
+	// If we found any scale ops preferred affinities, add them to the desired pod
+	if len(admissionWebhookPpreferredAffinities) > 0 {
+		if desiredPod.Spec.Affinity == nil {
+			desiredPod.Spec.Affinity = &corev1.Affinity{}
+		}
+		if desiredPod.Spec.Affinity.PodAffinity == nil {
+			desiredPod.Spec.Affinity.PodAffinity = &corev1.PodAffinity{}
+		}
+
+		// Merge scale ops preferred affinities, avoiding duplicates
+		existingTerms := desiredPod.Spec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution
+		for _, newTerm := range admissionWebhookPpreferredAffinities {
+			// Check if this term already exists
+			found := false
+			for _, existing := range existingTerms {
+				if reflect.DeepEqual(existing.PodAffinityTerm, newTerm.PodAffinityTerm) && existing.Weight == newTerm.Weight {
+					found = true
+					break
+				}
+			}
+			if !found {
+				existingTerms = append(existingTerms, newTerm)
+			}
+		}
+		desiredPod.Spec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution = existingTerms
+	}
+}
+
+// syncScaleOpsNodeAffinities syncs preferred node affinities with "scaleops.sh/node-packing=true"
+// selector from current pod to desired pod.
+func syncNodeAffinities(desiredPod, currentPod *corev1.Pod) {
+	if currentPod.Spec.Affinity == nil || currentPod.Spec.Affinity.NodeAffinity == nil {
+		return
+	}
+
+	currentNodeAffinity := currentPod.Spec.Affinity.NodeAffinity
+
+	// Filter preferred node affinities with "scaleops.sh/node-packing=true" selector
+	var admissionWebhookPreferredTerms []corev1.PreferredSchedulingTerm
+	if currentNodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution != nil {
+		admissionWebhookPreferredTerms = append(admissionWebhookPreferredTerms,
+			currentNodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution...)
+	}
+
+	// If we found any scale ops node affinities, add them to the desired pod
+	if len(admissionWebhookPreferredTerms) > 0 {
+		if desiredPod.Spec.Affinity == nil {
+			desiredPod.Spec.Affinity = &corev1.Affinity{}
+		}
+		if desiredPod.Spec.Affinity.NodeAffinity == nil {
+			desiredPod.Spec.Affinity.NodeAffinity = &corev1.NodeAffinity{}
+		}
+
+		// Merge scale ops node affinities, avoiding duplicates
+		existingTerms := desiredPod.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution
+		for _, newTerm := range admissionWebhookPreferredTerms {
+			// Check if this term already exists
+			found := false
+			for _, existing := range existingTerms {
+				if reflect.DeepEqual(existing.Preference, newTerm.Preference) && existing.Weight == newTerm.Weight {
+					found = true
+					break
+				}
+			}
+			if !found {
+				existingTerms = append(existingTerms, newTerm)
+			}
+		}
+		desiredPod.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution = existingTerms
+	}
+}