Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion api/v1alpha1/tenantcontrolplane_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ type ProbeSet struct {
type ControlPlaneProbes struct {
// Liveness defines default parameters for liveness probes of all Control Plane components.
Liveness *ProbeSpec `json:"liveness,omitempty"`
// Readiness defines default parameters for the readiness probe of kube-apiserver.
// Readiness defines default parameters for readiness probes of all Control Plane components.
Readiness *ProbeSpec `json:"readiness,omitempty"`
// Startup defines default parameters for startup probes of all Control Plane components.
Startup *ProbeSpec `json:"startup,omitempty"`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7361,7 +7361,7 @@ versions:
type: integer
type: object
readiness:
description: Readiness defines default parameters for the readiness probe of kube-apiserver.
description: Readiness defines default parameters for readiness probes of all Control Plane components.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7369,7 +7369,7 @@ spec:
type: integer
type: object
readiness:
description: Readiness defines default parameters for the readiness probe of kube-apiserver.
description: Readiness defines default parameters for readiness probes of all Control Plane components.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
Expand Down
4 changes: 2 additions & 2 deletions docs/content/reference/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -44720,7 +44720,7 @@ Override TimeoutSeconds, PeriodSeconds, and FailureThreshold for resource-constr
<td><b><a href="#tenantcontrolplanespeccontrolplanedeploymentprobesreadiness">readiness</a></b></td>
<td>object</td>
<td>
Readiness defines default parameters for the readiness probe of kube-apiserver.<br/>
Readiness defines default parameters for readiness probes of all Control Plane components.<br/>
</td>
<td>false</td>
</tr><tr>
Expand Down Expand Up @@ -45305,7 +45305,7 @@ Must be 1 for liveness and startup probes.<br/>
<span id="tenantcontrolplanespeccontrolplanedeploymentprobesreadiness">`TenantControlPlane.spec.controlPlane.deployment.probes.readiness`</span>


Readiness defines default parameters for the readiness probe of kube-apiserver.
Readiness defines default parameters for readiness probes of all Control Plane components.

<table>
<thead>
Expand Down
173 changes: 51 additions & 122 deletions internal/builders/controlplane/deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ const (
)

func applyProbeOverrides(probe *corev1.Probe, spec *kamajiv1alpha1.ProbeSpec) {
if spec == nil {
if probe == nil || spec == nil {
return
}

Expand All @@ -63,6 +63,44 @@ func applyProbeOverrides(probe *corev1.Probe, spec *kamajiv1alpha1.ProbeSpec) {
probe.FailureThreshold = pointer.Deref(spec.FailureThreshold, probe.FailureThreshold)
}

// defaultProbe builds the standard HTTPS probe shared by all control plane
// components; only the path and port differ between components and probe types.
func defaultProbe(path string, port int32) *corev1.Probe {
return &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: path,
Port: intstr.FromInt32(port),
Scheme: corev1.URISchemeHTTPS,
},
},
InitialDelaySeconds: 0,
TimeoutSeconds: 1,
PeriodSeconds: 10,
SuccessThreshold: 1,
FailureThreshold: 3,
}
}

// applyProbeSetOverrides applies the global probe overrides first, then the
// per-component overrides, to every probe of the container. Probe pointers that
// are nil are skipped (applyProbeOverrides is nil-safe).
func applyProbeSetOverrides(c *corev1.Container, probes *kamajiv1alpha1.ControlPlaneProbes, set *kamajiv1alpha1.ProbeSet) {
if probes == nil {
return
}

applyProbeOverrides(c.LivenessProbe, probes.Liveness)
applyProbeOverrides(c.ReadinessProbe, probes.Readiness)
applyProbeOverrides(c.StartupProbe, probes.Startup)

if set != nil {
applyProbeOverrides(c.LivenessProbe, set.Liveness)
applyProbeOverrides(c.ReadinessProbe, set.Readiness)
applyProbeOverrides(c.StartupProbe, set.Startup)
}
}

type DataStoreOverrides struct {
Resource string
DataStore kamajiv1alpha1.DataStore
Expand Down Expand Up @@ -368,43 +406,12 @@ func (d Deployment) buildScheduler(podSpec *corev1.PodSpec, tenantControlPlane k
podSpec.Containers[index].Image = tenantControlPlane.Spec.ControlPlane.Deployment.RegistrySettings.KubeSchedulerImage(tenantControlPlane.Spec.Kubernetes.Version)
podSpec.Containers[index].Command = []string{"kube-scheduler"}
podSpec.Containers[index].Args = utilities.ArgsFromMapToSlice(args)
podSpec.Containers[index].LivenessProbe = &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/healthz",
Port: intstr.FromInt32(10259),
Scheme: corev1.URISchemeHTTPS,
},
},
InitialDelaySeconds: 0,
TimeoutSeconds: 1,
PeriodSeconds: 10,
SuccessThreshold: 1,
FailureThreshold: 3,
}
podSpec.Containers[index].StartupProbe = &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/healthz",
Port: intstr.FromInt32(10259),
Scheme: corev1.URISchemeHTTPS,
},
},
InitialDelaySeconds: 0,
TimeoutSeconds: 1,
PeriodSeconds: 10,
SuccessThreshold: 1,
FailureThreshold: 3,
}
podSpec.Containers[index].LivenessProbe = defaultProbe("/healthz", 10259)
podSpec.Containers[index].ReadinessProbe = defaultProbe("/healthz", 10259)
podSpec.Containers[index].StartupProbe = defaultProbe("/healthz", 10259)

if probes := tenantControlPlane.Spec.ControlPlane.Deployment.Probes; probes != nil {
applyProbeOverrides(podSpec.Containers[index].LivenessProbe, probes.Liveness)
applyProbeOverrides(podSpec.Containers[index].StartupProbe, probes.Startup)

if probes.Scheduler != nil {
applyProbeOverrides(podSpec.Containers[index].LivenessProbe, probes.Scheduler.Liveness)
applyProbeOverrides(podSpec.Containers[index].StartupProbe, probes.Scheduler.Startup)
}
applyProbeSetOverrides(&podSpec.Containers[index], probes, probes.Scheduler)
}

if containerSecurityContexts := tenantControlPlane.Spec.ControlPlane.Deployment.ContainerSecurityContexts; containerSecurityContexts != nil {
Expand Down Expand Up @@ -480,43 +487,12 @@ func (d Deployment) buildControllerManager(podSpec *corev1.PodSpec, tenantContro
podSpec.Containers[index].Image = tenantControlPlane.Spec.ControlPlane.Deployment.RegistrySettings.KubeControllerManagerImage(tenantControlPlane.Spec.Kubernetes.Version)
podSpec.Containers[index].Command = []string{"kube-controller-manager"}
podSpec.Containers[index].Args = utilities.ArgsFromMapToSlice(args)
podSpec.Containers[index].LivenessProbe = &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/healthz",
Port: intstr.FromInt32(10257),
Scheme: corev1.URISchemeHTTPS,
},
},
InitialDelaySeconds: 0,
TimeoutSeconds: 1,
PeriodSeconds: 10,
SuccessThreshold: 1,
FailureThreshold: 3,
}
podSpec.Containers[index].StartupProbe = &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/healthz",
Port: intstr.FromInt32(10257),
Scheme: corev1.URISchemeHTTPS,
},
},
InitialDelaySeconds: 0,
TimeoutSeconds: 1,
PeriodSeconds: 10,
SuccessThreshold: 1,
FailureThreshold: 3,
}
podSpec.Containers[index].LivenessProbe = defaultProbe("/healthz", 10257)
podSpec.Containers[index].ReadinessProbe = defaultProbe("/healthz", 10257)
podSpec.Containers[index].StartupProbe = defaultProbe("/healthz", 10257)

if probes := tenantControlPlane.Spec.ControlPlane.Deployment.Probes; probes != nil {
applyProbeOverrides(podSpec.Containers[index].LivenessProbe, probes.Liveness)
applyProbeOverrides(podSpec.Containers[index].StartupProbe, probes.Startup)

if probes.ControllerManager != nil {
applyProbeOverrides(podSpec.Containers[index].LivenessProbe, probes.ControllerManager.Liveness)
applyProbeOverrides(podSpec.Containers[index].StartupProbe, probes.ControllerManager.Startup)
}
applyProbeSetOverrides(&podSpec.Containers[index], probes, probes.ControllerManager)
}

if containerSecurityContexts := tenantControlPlane.Spec.ControlPlane.Deployment.ContainerSecurityContexts; containerSecurityContexts != nil {
Expand Down Expand Up @@ -614,59 +590,12 @@ func (d Deployment) buildKubeAPIServer(podSpec *corev1.PodSpec, tenantControlPla
podSpec.Containers[index].Args = args
podSpec.Containers[index].Image = tenantControlPlane.Spec.ControlPlane.Deployment.RegistrySettings.KubeAPIServerImage(tenantControlPlane.Spec.Kubernetes.Version)
podSpec.Containers[index].Command = []string{"kube-apiserver"}
podSpec.Containers[index].LivenessProbe = &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/livez",
Port: intstr.FromInt32(tenantControlPlane.Spec.NetworkProfile.Port),
Scheme: corev1.URISchemeHTTPS,
},
},
InitialDelaySeconds: 0,
TimeoutSeconds: 1,
PeriodSeconds: 10,
SuccessThreshold: 1,
FailureThreshold: 3,
}
podSpec.Containers[index].ReadinessProbe = &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/readyz",
Port: intstr.FromInt32(tenantControlPlane.Spec.NetworkProfile.Port),
Scheme: corev1.URISchemeHTTPS,
},
},
InitialDelaySeconds: 0,
TimeoutSeconds: 1,
PeriodSeconds: 10,
SuccessThreshold: 1,
FailureThreshold: 3,
}
podSpec.Containers[index].StartupProbe = &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/livez",
Port: intstr.FromInt32(tenantControlPlane.Spec.NetworkProfile.Port),
Scheme: corev1.URISchemeHTTPS,
},
},
InitialDelaySeconds: 0,
TimeoutSeconds: 1,
PeriodSeconds: 10,
SuccessThreshold: 1,
FailureThreshold: 3,
}
podSpec.Containers[index].LivenessProbe = defaultProbe("/livez", tenantControlPlane.Spec.NetworkProfile.Port)
podSpec.Containers[index].ReadinessProbe = defaultProbe("/readyz", tenantControlPlane.Spec.NetworkProfile.Port)
podSpec.Containers[index].StartupProbe = defaultProbe("/livez", tenantControlPlane.Spec.NetworkProfile.Port)

if probes := tenantControlPlane.Spec.ControlPlane.Deployment.Probes; probes != nil {
applyProbeOverrides(podSpec.Containers[index].LivenessProbe, probes.Liveness)
applyProbeOverrides(podSpec.Containers[index].ReadinessProbe, probes.Readiness)
applyProbeOverrides(podSpec.Containers[index].StartupProbe, probes.Startup)

if probes.APIServer != nil {
applyProbeOverrides(podSpec.Containers[index].LivenessProbe, probes.APIServer.Liveness)
applyProbeOverrides(podSpec.Containers[index].ReadinessProbe, probes.APIServer.Readiness)
applyProbeOverrides(podSpec.Containers[index].StartupProbe, probes.APIServer.Startup)
}
applyProbeSetOverrides(&podSpec.Containers[index], probes, probes.APIServer)
}

if containerSecurityContexts := tenantControlPlane.Spec.ControlPlane.Deployment.ContainerSecurityContexts; containerSecurityContexts != nil {
Expand Down
84 changes: 84 additions & 0 deletions internal/builders/controlplane/deployment_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,11 @@ var _ = Describe("Controlplane Deployment", func() {
Expect(probe.InitialDelaySeconds).To(Equal(int32(0)))
Expect(probe.SuccessThreshold).To(Equal(int32(1)))
})

It("should not panic when probe is nil", func() {
spec := &kamajiv1alpha1.ProbeSpec{PeriodSeconds: pointer.To(int32(20))}
Expect(func() { applyProbeOverrides(nil, spec) }).ToNot(Panic())
})
})

Describe("mergeAPIServerArgs", func() {
Expand Down Expand Up @@ -200,4 +205,83 @@ var _ = Describe("Controlplane Deployment", func() {
}))
})
})

Describe("control plane probes", func() {
// helper: find a container by name in a built PodSpec
containerByName := func(spec *corev1.PodSpec, name string) corev1.Container {
for _, c := range spec.Containers {
if c.Name == name {
return c
}
}
Fail("container not found: " + name)

return corev1.Container{}
}

It("renders a readiness probe for kube-scheduler on /healthz:10259", func() {
podSpec := &corev1.PodSpec{}
d.buildScheduler(podSpec, kamajiv1alpha1.TenantControlPlane{})

c := containerByName(podSpec, "kube-scheduler")
Expect(c.ReadinessProbe).ToNot(BeNil())
Expect(c.ReadinessProbe.HTTPGet.Path).To(Equal("/healthz"))
Expect(c.ReadinessProbe.HTTPGet.Port.IntValue()).To(Equal(10259))
Expect(c.ReadinessProbe.HTTPGet.Scheme).To(Equal(corev1.URISchemeHTTPS))
Expect(c.ReadinessProbe.PeriodSeconds).To(Equal(int32(10)))
})

It("renders a readiness probe for kube-controller-manager on /healthz:10257", func() {
podSpec := &corev1.PodSpec{}
d.buildControllerManager(podSpec, kamajiv1alpha1.TenantControlPlane{})

c := containerByName(podSpec, "kube-controller-manager")
Expect(c.ReadinessProbe).ToNot(BeNil())
Expect(c.ReadinessProbe.HTTPGet.Path).To(Equal("/healthz"))
Expect(c.ReadinessProbe.HTTPGet.Port.IntValue()).To(Equal(10257))
Expect(c.ReadinessProbe.HTTPGet.Scheme).To(Equal(corev1.URISchemeHTTPS))
})

It("cascades global then component readiness overrides onto the scheduler", func() {
tcp := kamajiv1alpha1.TenantControlPlane{}
tcp.Spec.ControlPlane.Deployment.Probes = &kamajiv1alpha1.ControlPlaneProbes{
Readiness: &kamajiv1alpha1.ProbeSpec{PeriodSeconds: pointer.To(int32(20))},
Scheduler: &kamajiv1alpha1.ProbeSet{
Readiness: &kamajiv1alpha1.ProbeSpec{PeriodSeconds: pointer.To(int32(30))},
},
}

podSpec := &corev1.PodSpec{}
d.buildScheduler(podSpec, tcp)

c := containerByName(podSpec, "kube-scheduler")
Expect(c.ReadinessProbe.PeriodSeconds).To(Equal(int32(30))) // component wins over global
})

It("applies a global-only readiness override to the scheduler", func() {
tcp := kamajiv1alpha1.TenantControlPlane{}
tcp.Spec.ControlPlane.Deployment.Probes = &kamajiv1alpha1.ControlPlaneProbes{
Readiness: &kamajiv1alpha1.ProbeSpec{PeriodSeconds: pointer.To(int32(20))},
}

podSpec := &corev1.PodSpec{}
d.buildScheduler(podSpec, tcp)

c := containerByName(podSpec, "kube-scheduler")
Expect(c.ReadinessProbe.PeriodSeconds).To(Equal(int32(20)))
})

It("leaves the kube-apiserver probes unchanged (regression guard)", func() {
podSpec := &corev1.PodSpec{}
tcp := kamajiv1alpha1.TenantControlPlane{}
tcp.Spec.NetworkProfile.Port = 6443
d.buildKubeAPIServer(podSpec, tcp, "")

c := containerByName(podSpec, "kube-apiserver")
Expect(c.LivenessProbe.HTTPGet.Path).To(Equal("/livez"))
Expect(c.ReadinessProbe.HTTPGet.Path).To(Equal("/readyz"))
Expect(c.StartupProbe.HTTPGet.Path).To(Equal("/livez"))
Expect(c.ReadinessProbe.HTTPGet.Port.IntValue()).To(Equal(6443))
})
})
})