# Cluster information and health checks
kubectl cluster-info
kubectl get nodes -o wide
kubectl get pods --all-namespaces
kubectl get componentstatuses

# Understanding cluster architecture
kubectl get pods -n kube-system
kubectl describe node master-node
kubectl get endpoints kube-scheduler -n kube-system

# Basic cluster operations
kubectl cordon node-01
kubectl drain node-01 --ignore-daemonsets --delete-emptydir-data
kubectl uncordon node-01

Core Administration Skills:
  ✓ Install and configure Kubernetes clusters using kubeadm
  ✓ Manage cluster certificates and PKI infrastructure
  ✓ Perform cluster upgrades across different versions
  ✓ Configure and troubleshoot cluster networking
  ✓ Implement backup and restore procedures for etcd
  ✓ Manage node resources and scheduling policies
  ✓ Configure persistent storage and volume management
  ✓ Implement cluster security and RBAC policies

Performance Targets:
  ✓ Complete node maintenance operations in under 5 minutes
  ✓ Deploy and configure CNI plugins in under 10 minutes
  ✓ Troubleshoot failed pods and services in under 8 minutes
  ✓ Perform etcd backup and restore in under 15 minutes

# Understanding cluster components
kubectl get pods -n kube-system
kubectl describe pod kube-apiserver-master -n kube-system
kubectl describe pod etcd-master -n kube-system
kubectl describe pod kube-controller-manager-master -n kube-system
kubectl describe pod kube-scheduler-master -n kube-system

# Checking cluster health
kubectl get cs
kubectl get nodes --show-labels
kubectl describe node worker-01

# Component logs analysis
sudo journalctl -u kubelet -f
sudo journalctl -u docker -f
kubectl logs kube-apiserver-master -n kube-system

# Example kubeadm config for HA cluster
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
kubernetesVersion: v1.28.0
controlPlaneEndpoint: "cluster-endpoint:6443"
etcd:
  external:
    endpoints:
    - "https://10.0.0.10:2379"
    - "https://10.0.0.11:2379"
    - "https://10.0.0.12:2379"
    caFile: "/etc/etcd/ca.crt"
    certFile: "/etc/etcd/kubernetes.crt"
    keyFile: "/etc/etcd/kubernetes.key"
networking:
  serviceSubnet: "10.96.0.0/12"
  podSubnet: "192.168.0.0/16"
apiServer:
  advertiseAddress: "10.0.0.20"
  certSANs:
  - "cluster-endpoint"
  - "10.0.0.20"
  - "10.0.0.21"
  - "10.0.0.22"

# Essential aliases for CKA exam efficiency
cat << 'EOF' >> ~/.bashrc
# CKA exam optimizations
alias k=kubectl
alias kg='kubectl get'
alias kd='kubectl describe'
alias kdel='kubectl delete'
alias kaf='kubectl apply -f'
alias klo='kubectl logs'
alias kex='kubectl exec -it'

# Administrative shortcuts
export do="--dry-run=client -o yaml"
export now="--grace-period=0 --force"
export wide="--output=wide"

# Cluster management functions
function knode() {
    kubectl get nodes -o wide | grep $1
}

function kdrain() {
    kubectl drain $1 --ignore-daemonsets --delete-emptydir-data --force
}

function kcordon() {
    kubectl cordon $1
}

function kuncordon() {
    kubectl uncordon $1
}

# etcd management
function etcd-backup() {
    ETCDCTL_API=3 etcdctl snapshot save /opt/backup/etcd-snapshot-$(date +%Y%m%d).db \
      --endpoints=https://127.0.0.1:2379 \
      --cacert=/etc/etcd/ca.crt \
      --cert=/etc/etcd/server.crt \
      --key=/etc/etcd/server.key
}
EOF

source ~/.bashrc

# Comprehensive cluster health check script
cat << 'EOF' > cluster-health-check.sh
#!/bin/bash

echo "=== Cluster Health Check ==="
echo "Cluster Info:"
kubectl cluster-info

echo -e "\nNode Status:"
kubectl get nodes -o wide

echo -e "\nSystem Pods:"
kubectl get pods -n kube-system

echo -e "\nComponent Status:"
kubectl get componentstatuses

echo -e "\nResource Usage:"
kubectl top nodes
kubectl top pods --all-namespaces | head -10

echo -e "\nRecent Events:"
kubectl get events --all-namespaces --sort-by='.lastTimestamp' | tail -20

echo -e "\nEtcd Health:"
ETCDCTL_API=3 etcdctl endpoint health \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/etcd/ca.crt \
  --cert=/etc/etcd/server.crt \
  --key=/etc/etcd/server.key

echo -e "\nDisk Usage:"
df -h
echo -e "\nMemory Usage:"
free -h
EOF

chmod +x cluster-health-check.sh

# Master node initialization
sudo kubeadm init \
  --pod-network-cidr=192.168.0.0/16 \
  --apiserver-advertise-address=10.0.0.10 \
  --apiserver-cert-extra-sans=cluster.local,10.0.0.10 \
  --node-name=master-01

# Configure kubectl for root user
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# Install CNI plugin (Calico example)
kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml

# Join worker nodes
kubeadm token create --print-join-command
# Run the output command on worker nodes

# Custom kubeadm configuration
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: "10.0.0.10"
  bindPort: 6443
nodeRegistration:
  criSocket: "unix:///var/run/containerd/containerd.sock"
  kubeletExtraArgs:
    cloud-provider: "external"
    cgroup-driver: "systemd"
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
kubernetesVersion: v1.28.0
clusterName: "production-cluster"
controlPlaneEndpoint: "cluster-api.local:6443"
networking:
  serviceSubnet: "10.96.0.0/12"
  podSubnet: "192.168.0.0/16"
  dnsDomain: "cluster.local"
etcd:
  local:
    dataDir: "/var/lib/etcd"
    extraArgs:
      listen-metrics-urls: "http://0.0.0.0:2381"
apiServer:
  timeoutForControlPlane: 4m0s
  extraArgs:
    audit-log-maxage: "30"
    audit-log-maxbackup: "10"
    audit-log-maxsize: "100"
    audit-log-path: "/var/log/audit.log"
    enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction"
controllerManager:
  extraArgs:
    feature-gates: "RotateKubeletServerCertificate=true"
    cluster-signing-cert-file: "/etc/kubernetes/pki/ca.crt"
    cluster-signing-key-file: "/etc/kubernetes/pki/ca.key"
scheduler:
  extraArgs:
    bind-address: "0.0.0.0"

# Check current and available versions
kubeadm version
kubectl version --short
apt list -a kubeadm

# Upgrade control plane
sudo apt-mark unhold kubeadm && \
sudo apt-get update && sudo apt-get install -y kubeadm=1.28.x-00 && \
sudo apt-mark hold kubeadm

# Verify upgrade plan
sudo kubeadm upgrade plan

# Apply upgrade
sudo kubeadm upgrade apply v1.28.x

# Upgrade kubelet and kubectl
sudo apt-mark unhold kubelet kubectl && \
sudo apt-get update && sudo apt-get install -y kubelet=1.28.x-00 kubectl=1.28.x-00 && \
sudo apt-mark hold kubelet kubectl

# Restart kubelet
sudo systemctl daemon-reload
sudo systemctl restart kubelet

# Upgrade worker nodes
kubectl drain worker-01 --ignore-daemonsets --delete-emptydir-data
# Run upgrade commands on worker node
kubectl uncordon worker-01

# Node affinity example
apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-app
spec:
  replicas: 3
  selector:
    matchLabels:
      app: web-app
  template:
    metadata:
      labels:
        app: web-app
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/arch
                operator: In
                values:
                - amd64
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            preference:
              matchExpressions:
              - key: node-type
                operator: In
                values:
                - high-memory
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - web-app
            topologyKey: "kubernetes.io/hostname"
      tolerations:
      - key: "dedicated"
        operator: "Equal"
        value: "web-servers"
        effect: "NoSchedule"
      containers:
      - name: web-app
        image: nginx:1.20
        resources:
          requests:
            cpu: 100m
            memory: 128Mi
          limits:
            cpu: 500m
            memory: 512Mi

# Create DaemonSet for node monitoring
kubectl apply -f - <<EOF
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: node-exporter
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app: node-exporter
  template:
    metadata:
      labels:
        app: node-exporter
    spec:
      hostNetwork: true
      hostPID: true
      containers:
      - name: node-exporter
        image: prom/node-exporter:latest
        ports:
        - containerPort: 9100
          hostPort: 9100
        volumeMounts:
        - name: proc
          mountPath: /host/proc
          readOnly: true
        - name: sys
          mountPath: /host/sys
          readOnly: true
        - name: root
          mountPath: /rootfs
          readOnly: true
      volumes:
      - name: proc
        hostPath:
          path: /proc
      - name: sys
        hostPath:
          path: /sys
      - name: root
        hostPath:
          path: /
      tolerations:
      - effect: NoSchedule
        operator: Exists
EOF

# Static pod configuration (place in /etc/kubernetes/manifests/)
cat << 'EOF' > /etc/kubernetes/manifests/static-web.yaml
apiVersion: v1
kind: Pod
metadata:
  name: static-web
  namespace: kube-system
spec:
  containers:
  - name: web
    image: nginx:1.20
    ports:
    - containerPort: 80
    volumeMounts:
    - name: web-content
      mountPath: /usr/share/nginx/html
  volumes:
  - name: web-content
    hostPath:
      path: /var/web-content
EOF

# Install and configure Calico CNI
kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml

# Verify CNI installation
kubectl get pods -n calico-system
kubectl get nodes -o wide

# Configure custom network policies
kubectl apply -f - <<EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: deny-all-ingress
  namespace: production
spec:
  podSelector: {}
  policyTypes:
  - Ingress
EOF

# Allow specific traffic
kubectl apply -f - <<EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-frontend-to-backend
  namespace: production
spec:
  podSelector:
    matchLabels:
      role: backend
  policyTypes:
  - Ingress
  ingress:
  - from:
    - podSelector:
        matchLabels:
          role: frontend
    ports:
    - protocol: TCP
      port: 8080
EOF

# Multi-port service with session affinity
apiVersion: v1
kind: Service
metadata:
  name: multi-port-service
spec:
  selector:
    app: web-app
  ports:
  - name: http
    port: 80
    targetPort: 8080
    protocol: TCP
  - name: https
    port: 443
    targetPort: 8443
    protocol: TCP
  - name: metrics
    port: 9090
    targetPort: 9090
    protocol: TCP
  sessionAffinity: ClientIP
  sessionAffinityConfig:
    clientIP:
      timeoutSeconds: 3600
  type: ClusterIP
---
# External service for legacy systems
apiVersion: v1
kind: Service
metadata:
  name: external-database
spec:
  type: ExternalName
  externalName: db.legacy.company.com
  ports:
  - port: 5432
    targetPort: 5432

# Dynamic storage provisioning
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: fast-ssd
provisioner: kubernetes.io/aws-ebs
parameters:
  type: gp3
  iops: "3000"
  throughput: "125"
  encrypted: "true"
allowVolumeExpansion: true
volumeBindingMode: WaitForFirstConsumer
---
# PersistentVolume for NFS
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv
spec:
  capacity:
    storage: 100Gi
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  storageClassName: nfs
  nfs:
    path: /shared/data
    server: nfs-server.local
---
# PersistentVolumeClaim
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: app-data
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: fast-ssd
  resources:
    requests:
      storage: 50Gi

# Create volume snapshot class
kubectl apply -f - <<EOF
apiVersion: snapshot.storage.k8s.io/v1
kind: VolumeSnapshotClass
metadata:
  name: csi-snapclass
driver: ebs.csi.aws.com
deletionPolicy: Delete
EOF

# Create volume snapshot
kubectl apply -f - <<EOF
apiVersion: snapshot.storage.k8s.io/v1
kind: VolumeSnapshot
metadata:
  name: app-data-snapshot
spec:
  volumeSnapshotClassName: csi-snapclass
  source:
    persistentVolumeClaimName: app-data
EOF

# Restore from snapshot
kubectl apply -f - <<EOF
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: app-data-restored
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: fast-ssd
  resources:
    requests:
      storage: 50Gi
  dataSource:
    name: app-data-snapshot
    kind: VolumeSnapshot
    apiGroup: snapshot.storage.k8s.io
EOF

# Cluster-wide troubleshooting script
cat << 'EOF' > cluster-debug.sh
#!/bin/bash

echo "=== CLUSTER TROUBLESHOOTING REPORT ==="
echo "Generated: $(date)"
echo

echo "=== CLUSTER STATUS ==="
kubectl cluster-info
echo

echo "=== NODE STATUS ==="
kubectl get nodes -o wide
echo

echo "=== COMPONENT STATUS ==="
kubectl get componentstatuses
echo

echo "=== CRITICAL SYSTEM PODS ==="
kubectl get pods -n kube-system --field-selector=status.phase!=Running
echo

echo "=== RECENT EVENTS ==="
kubectl get events --all-namespaces --sort-by='.lastTimestamp' | tail -30
echo

echo "=== RESOURCE UTILIZATION ==="
echo "Node Resources:"
kubectl top nodes
echo "Top Memory Consumers:"
kubectl top pods --all-namespaces --sort-by=memory | head -10
echo "Top CPU Consumers:"
kubectl top pods --all-namespaces --sort-by=cpu | head -10
echo

echo "=== ETCD HEALTH ==="
ETCDCTL_API=3 etcdctl endpoint health \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/etcd/ca.crt \
  --cert=/etc/etcd/server.crt \
  --key=/etc/etcd/server.key
echo

echo "=== NETWORK CONNECTIVITY ==="
kubectl run connectivity-test --image=busybox --rm -it --restart=Never -- nslookup kubernetes.default.svc.cluster.local
echo

echo "=== DISK USAGE ==="
df -h
echo

echo "=== MEMORY USAGE ==="
free -h
echo

echo "=== KUBELET LOGS (Last 50 lines) ==="
sudo journalctl -u kubelet --no-pager -n 50
EOF

chmod +x cluster-debug.sh

# Network debugging commands
kubectl run netshoot --image=nicolaka/netshoot -it --rm --restart=Never -- bash

# Inside netshoot container:
# Test DNS resolution
nslookup kubernetes.default.svc.cluster.local
dig @10.96.0.10 kubernetes.default.svc.cluster.local

# Test service connectivity
nc -zv service-name.namespace.svc.cluster.local 80
wget -qO- --timeout=2 http://service-name.namespace:port/health

# Test pod-to-pod connectivity
ping pod-ip
nc -zv pod-ip port

# Check routing
ip route
traceroute service-ip

# DNS debugging script
cat << 'EOF' > dns-debug.sh
#!/bin/bash

echo "=== DNS TROUBLESHOOTING ==="
echo "CoreDNS Status:"
kubectl get pods -n kube-system | grep coredns

echo -e "\nCoreDNS Configuration:"
kubectl get configmap coredns -n kube-system -o yaml

echo -e "\nDNS Service:"
kubectl get service kube-dns -n kube-system

echo -e "\nDNS Endpoints:"
kubectl get endpoints kube-dns -n kube-system

echo -e "\nRecent CoreDNS Logs:"
kubectl logs -n kube-system $(kubectl get pods -n kube-system | grep coredns | head -1 | awk '{print $1}') --tail=20
EOF

chmod +x dns-debug.sh

# Common upgrade failure recovery
# 1. Check cluster status
kubectl get nodes
kubectl get pods --all-namespaces

# 2. Identify failed components
kubectl get componentstatuses
kubectl describe node master-01

# 3. Check service status
sudo systemctl status kubelet
sudo systemctl status docker
sudo systemctl status containerd

# 4. Review logs
sudo journalctl -u kubelet --since "1 hour ago"
kubectl logs -n kube-system kube-apiserver-master

# 5. Recover from backup if needed
sudo cp /etc/kubernetes/admin.conf.backup /etc/kubernetes/admin.conf
sudo kubeadm upgrade apply --force v1.27.x

# Systematic network troubleshooting
# 1. Verify CNI plugin status
kubectl get pods -n calico-system
kubectl describe pod calico-node-xxx -n calico-system

# 2. Check node network configuration
ip addr show
ip route show
iptables -L -n

# 3. Test inter-pod communication
kubectl run test-pod --image=busybox --rm -it --restart=Never -- sh
# Inside pod: ping other-pod-ip

# 4. Verify service endpoints
kubectl get endpoints service-name
kubectl describe service service-name

# 5. Check DNS resolution
kubectl run dns-test --image=busybox --rm -it --restart=Never -- nslookup kubernetes.default.svc.cluster.local

# Complete etcd backup and restore procedure
# 1. Create backup
ETCDCTL_API=3 etcdctl snapshot save /opt/backup/etcd-snapshot-$(date +%Y%m%d-%H%M%S).db \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/etcd/ca.crt \
  --cert=/etc/etcd/server.crt \
  --key=/etc/etcd/server.key

# 2. Verify backup
ETCDCTL_API=3 etcdctl snapshot status /opt/backup/etcd-snapshot-$(date +%Y%m%d-%H%M%S).db

# 3. Stop etcd and API server
sudo systemctl stop etcd
sudo systemctl stop kube-apiserver

# 4. Restore from backup
ETCDCTL_API=3 etcdctl snapshot restore /opt/backup/etcd-snapshot-backup.db \
  --data-dir /var/lib/etcd-restore \
  --initial-cluster master=https://127.0.0.1:2380 \
  --initial-advertise-peer-urls https://127.0.0.1:2380

# 5. Update etcd configuration and restart
sudo mv /var/lib/etcd /var/lib/etcd-old
sudo mv /var/lib/etcd-restore /var/lib/etcd
sudo systemctl start etcd
sudo systemctl start kube-apiserver

# Service account for application
apiVersion: v1
kind: ServiceAccount
metadata:
  name: app-service-account
  namespace: production
---
# Role with minimal permissions
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  namespace: production
  name: app-reader
rules:
- apiGroups: [""]
  resources: ["configmaps", "secrets"]
  verbs: ["get", "list"]
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["get", "list", "watch"]
---
# RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: app-reader-binding
  namespace: production
subjects:
- kind: ServiceAccount
  name: app-service-account
  namespace: production
roleRef:
  kind: Role
  name: app-reader
  apiGroup: rbac.authorization.k8s.io
---
# ClusterRole for cluster-wide resources
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: node-reader
rules:
- apiGroups: [""]
  resources: ["nodes", "nodes/status"]
  verbs: ["get", "list", "watch"]
- apiGroups: ["metrics.k8s.io"]
  resources: ["nodes", "pods"]
  verbs: ["get", "list"]
---
# ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: node-reader-binding
subjects:
- kind: ServiceAccount
  name: monitoring-service-account
  namespace: monitoring
roleRef:
  kind: ClusterRole
  name: node-reader
  apiGroup: rbac.authorization.k8s.io

# Pod Security Standard enforcement
apiVersion: v1
kind: Namespace
metadata:
  name: secure-namespace
  labels:
    pod-security.kubernetes.io/enforce: restricted
    pod-security.kubernetes.io/audit: restricted
    pod-security.kubernetes.io/warn: restricted
---
# Secure pod example
apiVersion: v1
kind: Pod
metadata:
  name: secure-pod
  namespace: secure-namespace
spec:
  serviceAccountName: limited-service-account
  securityContext:
    runAsNonRoot: true
    runAsUser: 1000
    runAsGroup: 1000
    fsGroup: 1000
    seccompProfile:
      type: RuntimeDefault
  containers:
  - name: app
    image: nginx:1.20
    securityContext:
      allowPrivilegeEscalation: false
      readOnlyRootFilesystem: true
      capabilities:
        drop:
        - ALL
        add:
        - NET_BIND_SERVICE
    resources:
      requests:
        cpu: 100m
        memory: 128Mi
      limits:
        cpu: 500m
        memory: 512Mi
    volumeMounts:
    - name: tmp
      mountPath: /tmp
    - name: var-cache
      mountPath: /var/cache/nginx
    - name: var-run
      mountPath: /var/run
  volumes:
  - name: tmp
    emptyDir: {}
  - name: var-cache
    emptyDir: {}
  - name: var-run
    emptyDir: {}

# Initialize first control plane
sudo kubeadm init \
  --control-plane-endpoint "k8s-cluster.local:6443" \
  --upload-certs \
  --pod-network-cidr=192.168.0.0/16

# Add additional control plane nodes
sudo kubeadm join k8s-cluster.local:6443 \
  --token abcdef.0123456789abcdef \
  --discovery-token-ca-cert-hash sha256:hash \
  --control-plane \
  --certificate-key certificate-key

# Setup load balancer for API server
cat << 'EOF' > /etc/haproxy/haproxy.cfg
global
    daemon

defaults
    mode http
    timeout connect 5000ms
    timeout client 50000ms
    timeout server 50000ms

frontend k8s-api
    bind *:6443
    mode tcp
    default_backend k8s-api-backend

backend k8s-api-backend
    mode tcp
    balance roundrobin
    server master1 10.0.0.10:6443 check
    server master2 10.0.0.11:6443 check
    server master3 10.0.0.12:6443 check
EOF

sudo systemctl restart haproxy

# Comprehensive monitoring setup
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: monitoring
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s
      evaluation_interval: 15s
    
    rule_files:
      - "/etc/prometheus/rules/*.yml"
    
    alerting:
      alertmanagers:
      - static_configs:
        - targets:
          - alertmanager:9093
    
    scrape_configs:
    - job_name: 'kubernetes-apiservers'
      kubernetes_sd_configs:
      - role: endpoints
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      relabel_configs:
      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
        action: keep
        regex: default;kubernetes;https
    
    - job_name: 'kubernetes-nodes'
      kubernetes_sd_configs:
      - role: node
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
    
    - job_name: 'kubernetes-cadvisor'
      kubernetes_sd_configs:
      - role: node
      scheme: https
      metrics_path: /metrics/cadvisor
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: alerting-rules
  namespace: monitoring
data:
  cluster.yml: |
    groups:
    - name: cluster
      rules:
      - alert: NodeDown
        expr: up{job="kubernetes-nodes"} == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Node {{ $labels.instance }} is down"
          description: "Node {{ $labels.instance }} has been down for more than 5 minutes."
      
      - alert: HighMemoryUsage
        expr: (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes > 0.8
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High memory usage on {{ $labels.instance }}"
          description: "Memory usage is above 80% on {{ $labels.instance }}"
      
      - alert: PodCrashLooping
        expr: rate(kube_pod_container_status_restarts_total[5m]) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Pod {{ $labels.pod }} is crash looping"
          description: "Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} is restarting frequently."

# SRE Skills Development Focus
Core Competencies:
- Service Level Objectives (SLOs) and Error Budgets
- Chaos Engineering and Fault Injection
- Observability and Monitoring at Scale
- Incident Response and Post-Mortem Analysis
- Automation and Toil Reduction

Technical Skills:
- Prometheus and Grafana mastery
- Custom metrics and alerting
- Infrastructure as Code (Terraform, Ansible)
- CI/CD pipeline optimization
- Performance testing and optimization

# Platform Engineering Focus Areas
Developer Experience:
- Internal Developer Platforms (IDPs)
- Self-service infrastructure provisioning
- Golden paths and templates
- Developer tooling and workflows

Infrastructure Abstractions:
- Custom Resource Definitions (CRDs)
- Operators and Controllers
- Multi-cluster management
- GitOps implementations

Example Platform Components:
- Service Mesh (Istio, Linkerd)
- Observability Stack (Prometheus, Jaeger, Fluentd)
- Security Tools (Falco, OPA Gatekeeper)
- Developer Tools (Telepresence, Skaffold)

# Security Specialization Track
Core Security Areas:
- Cluster hardening and compliance
- Supply chain security
- Runtime security monitoring
- Network security and micro-segmentation
- Identity and access management

Advanced Certifications:
- Certified Kubernetes Security Specialist (CKS)
- Cloud security certifications (AWS Security, Azure Security)
- Security frameworks (NIST, SOC2, PCI DSS)

# Open Source Contribution Areas
Kubernetes Core:
git clone https://github.com/kubernetes/kubernetes
# Focus areas:
# - kubectl improvements
# - Documentation updates
# - Test coverage expansion
# - Bug fixes in core components

Ecosystem Projects:
- Helm charts and operators
- Monitoring and logging tools
- Security and compliance tools
- Developer experience improvements

Community Leadership:
- Local Kubernetes meetups
- Conference speaking
- Blog writing and technical content
- Mentoring junior engineers

# Home Lab Architecture for Skills Development
Hardware Setup:
- 3-4 node cluster (Raspberry Pi or mini PCs)
- Dedicated network segment
- Storage solution (NAS or distributed storage)

Software Stack:
- Multiple Kubernetes distributions (kubeadm, k3s, kind)
- GitOps tools (ArgoCD, Flux)
- Monitoring stack (Prometheus, Grafana, AlertManager)
- Service mesh (Istio or Linkerd)
- CI/CD pipeline (Jenkins, Tekton, or GitHub Actions)

Projects to Showcase:
- Multi-tier application deployment
- Disaster recovery procedures
- Security hardening implementation
- Performance optimization case studies
- Cost optimization strategies

# Setup multi-cluster management
# Cluster 1: Production
kubectl config use-context production-cluster
kubectl create namespace production
kubectl apply -f production-workloads.yaml

# Cluster 2: Staging
kubectl config use-context staging-cluster
kubectl create namespace staging
kubectl apply -f staging-workloads.yaml

# Cross-cluster service discovery
kubectl apply -f - <<EOF
apiVersion: networking.istio.io/v1alpha3
kind: ServiceEntry
metadata:
  name: cross-cluster-service
spec:
  hosts:
  - api.production.svc.cluster.local
  location: MESH_EXTERNAL
  ports:
  - number: 80
    name: http
    protocol: HTTP
  resolution: DNS
  addresses:
  - 10.1.0.100
EOF

# Complete DR procedure practice
# 1. Create baseline backup
./etcd-backup.sh

# 2. Simulate cluster failure
sudo systemctl stop kubelet
sudo systemctl stop etcd
sudo systemctl stop docker

# 3. Document recovery steps
echo "Recovery Procedure:" > recovery-log.txt
echo "1. Restore etcd from backup" >> recovery-log.txt
echo "2. Restart cluster services" >> recovery-log.txt
echo "3. Verify application functionality" >> recovery-log.txt

# 4. Execute recovery
./etcd-restore.sh backup-file.db
sudo systemctl start etcd
sudo systemctl start kubelet
sudo systemctl start docker

# 5. Validate recovery
kubectl get nodes
kubectl get pods --all-namespaces
./cluster-health-check.sh

# System requirements verification
Computer Requirements:
✓ Desktop or laptop computer (tablets not allowed)
✓ Stable internet connection (minimum 1 Mbps)
✓ Google Chrome browser (latest version)
✓ Webcam and microphone (working properly)
✓ Government-issued photo ID
✓ Quiet, private testing environment

Environment Setup:
✓ Remove or disconnect external monitors
✓ Clear desk of all materials except ID
✓ Ensure adequate lighting for webcam
✓ Close all applications except Chrome
✓ Disable notifications and background processes