Deep Dive: Kubernetes Controller Manager
The Kubernetes Controller Manager is responsible for running controllers that regulate the state of the cluster. This deep dive explores its architecture, built-in controllers, and how they maintain desired state.
Architecture Overview
Component Architecture
API Server -> Controller Manager -> Controllers
-> Work Queues
-> Informers
Key Components
Core Controllers
- Node Controller
- Replication Controller
- Endpoints Controller
- Service Account Controller
Informer Pattern
- Cache
- Event Handlers
- Work Queues
Reconciliation Loops
- Observe Current State
- Compare with Desired State
- Take Action
Core Controllers
1. Node Controller
// Node controller workflow
type NodeController struct {
knownNodes map[string]*v1.Node
healthyNodes map[string]*v1.Node
zonePodEvictor map[string]*RateLimitedTimedQueue
}
func (nc *NodeController) monitorNodeHealth() {
for node := range nc.knownNodes {
if !isNodeHealthy(node) {
nc.markNodeUnhealthy(node)
}
}
}
2. Replication Controller
// Replication controller reconciliation
func (rc *ReplicationController) syncReplicaSet(key string) error {
namespace, name := cache.SplitMetaNamespaceKey(key)
rs := rc.getReplicaSet(namespace, name)
currentReplicas := rc.getCurrentReplicas(rs)
desiredReplicas := rs.Spec.Replicas
if currentReplicas < desiredReplicas {
rc.createPods(rs, desiredReplicas - currentReplicas)
} else if currentReplicas > desiredReplicas {
rc.deletePods(rs, currentReplicas - desiredReplicas)
}
return nil
}
Controller Implementation
1. Basic Controller Structure
type Controller struct {
queue workqueue.RateLimitingInterface
informer cache.SharedIndexInformer
lister listers.PodLister
}
func (c *Controller) Run(workers int, stopCh <-chan struct{}) {
defer c.queue.ShutDown()
// Start informer
go c.informer.Run(stopCh)
// Wait for cache sync
if !cache.WaitForCacheSync(stopCh, c.informer.HasSynced) {
return
}
// Start workers
for i := 0; i < workers; i++ {
go wait.Until(c.runWorker, time.Second, stopCh)
}
<-stopCh
}
2. Reconciliation Loop
func (c *Controller) reconcile(key string) error {
// Get object
obj, exists, err := c.informer.GetIndexer().GetByKey(key)
if err != nil {
return err
}
// Handle deletion
if !exists {
return c.handleDeletion(key)
}
// Handle update/creation
return c.handleSync(obj)
}
Controller Configuration
1. Controller Settings
apiVersion: kubecontroller.config.k8s.io/v1alpha1
kind: KubeControllerManagerConfiguration
controllers:
- "*"
nodeMonitorPeriod: 5s
nodeMonitorGracePeriod: 40s
podEvictionTimeout: 5m
2. Leader Election
apiVersion: v1
kind: ConfigMap
metadata:
name: kube-controller-manager
namespace: kube-system
data:
config.yaml: |
apiVersion: kubecontroller.config.k8s.io/v1alpha1
kind: KubeControllerManagerConfiguration
leaderElection:
leaderElect: true
resourceLock: endpoints
leaseDuration: 15s
renewDeadline: 10s
retryPeriod: 2s
Custom Controllers
1. Custom Controller Example
type CustomController struct {
clientset kubernetes.Interface
customLister listers.CustomResourceLister
customSynced cache.InformerSynced
workqueue workqueue.RateLimitingInterface
}
func (c *CustomController) processNextWorkItem() bool {
obj, shutdown := c.workqueue.Get()
if shutdown {
return false
}
defer c.workqueue.Done(obj)
key, ok := obj.(string)
if !ok {
c.workqueue.Forget(obj)
return true
}
if err := c.syncHandler(key); err != nil {
c.workqueue.AddRateLimited(key)
return true
}
c.workqueue.Forget(obj)
return true
}
2. Custom Resource Definition
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: customresources.example.com
spec:
group: example.com
versions:
- name: v1
served: true
storage: true
scope: Namespaced
names:
plural: customresources
singular: customresource
kind: CustomResource
Performance Tuning
1. Resource Management
apiVersion: v1
kind: Pod
metadata:
name: kube-controller-manager
namespace: kube-system
spec:
containers:
- name: kube-controller-manager
resources:
requests:
cpu: "200m"
memory: "512Mi"
limits:
cpu: "1"
memory: "1Gi"
2. Concurrency Settings
apiVersion: kubecontroller.config.k8s.io/v1alpha1
kind: KubeControllerManagerConfiguration
concurrentDeploymentSyncs: 5
concurrentEndpointSyncs: 5
concurrentRCSyncs: 5
concurrentServiceSyncs: 1
Monitoring and Debugging
1. Metrics Collection
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: controller-manager
spec:
endpoints:
- interval: 30s
port: https-metrics
scheme: https
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
tlsConfig:
insecureSkipVerify: true
2. Controller Logs
# View controller manager logs
kubectl logs -n kube-system kube-controller-manager-master
# Debug specific controller
kubectl logs -n kube-system kube-controller-manager-master | grep "replication controller"
Troubleshooting
Common Issues
- Controller Not Reconciling
# Check controller manager status
kubectl get pods -n kube-system | grep controller-manager
# View controller logs
kubectl logs -n kube-system kube-controller-manager-master
- Leader Election Issues
# Check leader election status
kubectl get endpoints kube-controller-manager -n kube-system -o yaml
# View election events
kubectl get events -n kube-system | grep "leader election"
- Resource Synchronization Problems
# Check resource status
kubectl describe deployment failing-deployment
# View controller events
kubectl get events --field-selector reason=FailedSync
Best Practices
High Availability
- Enable leader election
- Configure proper timeouts
- Monitor controller health
Performance
- Set appropriate concurrency
- Configure resource limits
- Use efficient work queues
Monitoring
- Track reconciliation loops
- Monitor resource usage
- Set up alerting
For more information, check out: