#!/usr/bin/env python3
"""
Enterprise Kubernetes Infrastructure Deployment and Management Framework
"""

import os
import sys
import json
import yaml
import logging
import time
import subprocess
import asyncio
import ipaddress
from typing import Dict, List, Optional, Tuple, Any, Union
from dataclasses import dataclass, asdict, field
from pathlib import Path
from enum import Enum
import jinja2
import paramiko
import kubernetes
from kubernetes import client, config as k8s_config
import boto3
import ansible_runner
from prometheus_client import CollectorRegistry, Gauge, Counter
import vault

class DeploymentEnvironment(Enum):
    HOME_LAB = "home_lab"
    DEVELOPMENT = "development"
    STAGING = "staging"
    PRODUCTION = "production"
    DISASTER_RECOVERY = "disaster_recovery"

class NetworkingMode(Enum):
    FLANNEL = "flannel"
    CALICO = "calico"
    CILIUM = "cilium"
    CANAL = "canal"
    WEAVE = "weave"

class LoadBalancerType(Enum):
    METALLB = "metallb"
    NGINX = "nginx"
    HAPROXY = "haproxy"
    TRAEFIK = "traefik"
    CLOUD_PROVIDER = "cloud_provider"

@dataclass
class ClusterConfiguration:
    name: str
    environment: DeploymentEnvironment
    version: str  # Kubernetes version
    master_count: int
    worker_count: int
    network_cidr: str
    service_cidr: str
    pod_cidr: str
    dns_domain: str = "cluster.local"
    container_runtime: str = "containerd"
    enable_ha: bool = True
    enable_monitoring: bool = True
    enable_logging: bool = True
    enable_service_mesh: bool = False
    backup_enabled: bool = True

@dataclass
class NodeConfiguration:
    name: str
    role: str  # master or worker
    ip_address: str
    cpu_cores: int
    memory_gb: int
    disk_gb: int
    labels: Dict[str, str] = field(default_factory=dict)
    taints: List[Dict[str, str]] = field(default_factory=list)
    gpu_enabled: bool = False
    gpu_type: Optional[str] = None

@dataclass
class NetworkConfiguration:
    mode: NetworkingMode
    mtu: int = 1500
    enable_ipv6: bool = False
    enable_network_policies: bool = True
    enable_encryption: bool = True
    load_balancer_type: LoadBalancerType = LoadBalancerType.METALLB
    load_balancer_config: Dict[str, Any] = field(default_factory=dict)
    ingress_controller: str = "nginx"
    service_mesh: Optional[str] = None  # istio, linkerd, consul

class EnterpriseKubernetesOrchestrator:
    def __init__(self, config_file: str = "k8s_config.yaml"):
        self.config = self._load_config(config_file)
        self.clusters = {}
        self.deployments = {}
        
        # Initialize components
        self._setup_logging()
        self._initialize_backends()
        self._load_templates()
        
    def _load_config(self, config_file: str) -> Dict:
        """Load orchestrator configuration"""
        try:
            with open(config_file, 'r') as f:
                return yaml.safe_load(f)
        except FileNotFoundError:
            return self._create_default_config()
    
    def _create_default_config(self) -> Dict:
        """Create default orchestrator configuration"""
        return {
            'infrastructure': {
                'provider': 'vagrant',  # vagrant, bare_metal, vmware, aws, azure, gcp
                'vagrant': {
                    'box': 'ubuntu/focal64',
                    'provider': 'virtualbox',
                    'network_type': 'private_network'
                }
            },
            'defaults': {
                'kubernetes_version': '1.28.0',
                'container_runtime': 'containerd',
                'network_plugin': 'flannel',
                'service_mesh': None
            },
            'security': {
                'enable_rbac': True,
                'enable_psp': False,  # Deprecated, use PSA
                'enable_psa': True,  # Pod Security Admission
                'enable_network_policies': True,
                'cert_manager_enabled': True,
                'vault_integration': True
            },
            'monitoring': {
                'prometheus_enabled': True,
                'grafana_enabled': True,
                'alertmanager_enabled': True,
                'loki_enabled': True,
                'tempo_enabled': True
            },
            'storage': {
                'default_storage_class': 'local-path',
                'enable_csi_drivers': True,
                'snapshot_enabled': True,
                'backup_solution': 'velero'
            }
        }
    
    def _setup_logging(self):
        """Setup logging system"""
        log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
        logging.basicConfig(
            level=logging.INFO,
            format=log_format,
            handlers=[
                logging.FileHandler('/var/log/k8s-orchestrator.log'),
                logging.StreamHandler(sys.stdout)
            ]
        )
        self.logger = logging.getLogger(__name__)
    
    def _initialize_backends(self):
        """Initialize backend connections"""
        # Initialize Kubernetes client
        try:
            k8s_config.load_incluster_config()
        except:
            try:
                k8s_config.load_kube_config()
            except:
                self.logger.warning("No Kubernetes configuration found")
        
        # Initialize Vault client
        if self.config['security']['vault_integration']:
            try:
                self.vault_client = vault.Client(url=os.getenv('VAULT_ADDR'))
                self.vault_client.token = os.getenv('VAULT_TOKEN')
            except:
                self.logger.warning("Vault integration disabled - no connection")
        
        # Initialize cloud providers if configured
        self._initialize_cloud_providers()
    
    def _initialize_cloud_providers(self):
        """Initialize cloud provider connections"""
        self.cloud_providers = {}
        
        # AWS
        if os.getenv('AWS_ACCESS_KEY_ID'):
            self.cloud_providers['aws'] = {
                'ec2': boto3.client('ec2'),
                'eks': boto3.client('eks'),
                's3': boto3.client('s3')
            }
        
        # Add Azure, GCP, etc. as needed
    
    def _load_templates(self):
        """Load Jinja2 templates"""
        template_dir = Path(__file__).parent / 'templates'
        self.jinja_env = jinja2.Environment(
            loader=jinja2.FileSystemLoader(str(template_dir)),
            autoescape=True
        )
    
    async def create_cluster(self, cluster_config: ClusterConfiguration) -> str:
        """Create a new Kubernetes cluster"""
        cluster_id = f"{cluster_config.name}-{int(time.time())}"
        self.logger.info(f"Creating Kubernetes cluster: {cluster_id}")
        
        # Validate configuration
        self._validate_cluster_config(cluster_config)
        
        # Create infrastructure
        nodes = await self._provision_infrastructure(cluster_config)
        
        # Initialize cluster
        await self._initialize_cluster(cluster_config, nodes)
        
        # Configure networking
        await self._configure_networking(cluster_config, nodes)
        
        # Setup load balancer
        await self._setup_load_balancer(cluster_config)
        
        # Install core components
        await self._install_core_components(cluster_config)
        
        # Configure monitoring and logging
        if cluster_config.enable_monitoring:
            await self._setup_monitoring(cluster_config)
        
        if cluster_config.enable_logging:
            await self._setup_logging_stack(cluster_config)
        
        # Setup service mesh if enabled
        if cluster_config.enable_service_mesh:
            await self._setup_service_mesh(cluster_config)
        
        # Configure backup solution
        if cluster_config.backup_enabled:
            await self._setup_backup_solution(cluster_config)
        
        # Store cluster information
        self.clusters[cluster_id] = {
            'config': cluster_config,
            'nodes': nodes,
            'created_at': time.time(),
            'status': 'ready'
        }
        
        self.logger.info(f"Cluster created successfully: {cluster_id}")
        return cluster_id
    
    def _validate_cluster_config(self, config: ClusterConfiguration):
        """Validate cluster configuration"""
        # Validate network CIDRs
        try:
            ipaddress.ip_network(config.network_cidr)
            ipaddress.ip_network(config.service_cidr)
            ipaddress.ip_network(config.pod_cidr)
        except ValueError as e:
            raise ValueError(f"Invalid network configuration: {e}")
        
        # Validate master count for HA
        if config.enable_ha and config.master_count < 3:
            raise ValueError("HA requires at least 3 master nodes")
        
        # Validate Kubernetes version
        if not self._is_valid_k8s_version(config.version):
            raise ValueError(f"Unsupported Kubernetes version: {config.version}")
    
    def _is_valid_k8s_version(self, version: str) -> bool:
        """Check if Kubernetes version is supported"""
        supported_versions = ['1.26', '1.27', '1.28', '1.29']
        return any(version.startswith(v) for v in supported_versions)
    
    async def _provision_infrastructure(self, config: ClusterConfiguration) -> List[NodeConfiguration]:
        """Provision infrastructure for the cluster"""
        provider = self.config['infrastructure']['provider']
        
        if provider == 'vagrant':
            return await self._provision_vagrant(config)
        elif provider == 'bare_metal':
            return await self._provision_bare_metal(config)
        elif provider == 'aws':
            return await self._provision_aws(config)
        else:
            raise ValueError(f"Unsupported infrastructure provider: {provider}")
    
    async def _provision_vagrant(self, config: ClusterConfiguration) -> List[NodeConfiguration]:
        """Provision Vagrant-based infrastructure"""
        self.logger.info("Provisioning Vagrant infrastructure")
        
        # Generate Vagrantfile
        vagrantfile_content = self._generate_vagrantfile(config)
        
        # Write Vagrantfile
        vagrant_dir = Path(f"/tmp/k8s-{config.name}")
        vagrant_dir.mkdir(exist_ok=True)
        vagrantfile_path = vagrant_dir / "Vagrantfile"
        
        with open(vagrantfile_path, 'w') as f:
            f.write(vagrantfile_content)
        
        # Run vagrant up
        process = await asyncio.create_subprocess_exec(
            'vagrant', 'up',
            cwd=str(vagrant_dir),
            stdout=asyncio.subprocess.PIPE,
            stderr=asyncio.subprocess.PIPE
        )
        
        stdout, stderr = await process.communicate()
        
        if process.returncode != 0:
            raise Exception(f"Vagrant provisioning failed: {stderr.decode()}")
        
        # Get node information
        nodes = []
        
        # Create master nodes
        for i in range(config.master_count):
            node = NodeConfiguration(
                name=f"{config.name}-master-{i+1}",
                role="master",
                ip_address=f"192.168.56.{10+i}",
                cpu_cores=2,
                memory_gb=4,
                disk_gb=50,
                labels={"node-role.kubernetes.io/master": "true"}
            )
            nodes.append(node)
        
        # Create worker nodes
        for i in range(config.worker_count):
            node = NodeConfiguration(
                name=f"{config.name}-worker-{i+1}",
                role="worker",
                ip_address=f"192.168.56.{20+i}",
                cpu_cores=4,
                memory_gb=8,
                disk_gb=100,
                labels={"node-role.kubernetes.io/worker": "true"}
            )
            nodes.append(node)
        
        return nodes
    
    def _generate_vagrantfile(self, config: ClusterConfiguration) -> str:
        """Generate Vagrantfile for cluster"""
        template = self.jinja_env.get_template('Vagrantfile.j2')
        
        return template.render(
            cluster_name=config.name,
            master_count=config.master_count,
            worker_count=config.worker_count,
            box=self.config['infrastructure']['vagrant']['box'],
            provider=self.config['infrastructure']['vagrant']['provider'],
            network_type=self.config['infrastructure']['vagrant']['network_type']
        )
    
    async def _initialize_cluster(self, config: ClusterConfiguration, 
                                nodes: List[NodeConfiguration]):
        """Initialize Kubernetes cluster"""
        self.logger.info("Initializing Kubernetes cluster")
        
        # Get first master node
        master_node = next(n for n in nodes if n.role == "master")
        
        # Generate kubeadm config
        kubeadm_config = self._generate_kubeadm_config(config, nodes)
        
        # Initialize first master
        init_cmd = f"""
        sudo kubeadm init \
            --config=/tmp/kubeadm-config.yaml \
            --upload-certs
        """
        
        # Execute initialization
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        
        try:
            ssh.connect(master_node.ip_address, username='vagrant', password='vagrant')
            
            # Upload kubeadm config
            sftp = ssh.open_sftp()
            with sftp.file('/tmp/kubeadm-config.yaml', 'w') as f:
                f.write(kubeadm_config)
            sftp.close()
            
            # Run kubeadm init
            stdin, stdout, stderr = ssh.exec_command(init_cmd)
            output = stdout.read().decode()
            error = stderr.read().decode()
            
            if "Your Kubernetes control-plane has initialized successfully" not in output:
                raise Exception(f"Cluster initialization failed: {error}")
            
            # Extract join commands
            self._extract_join_commands(output)
            
            # Setup kubectl for vagrant user
            setup_kubectl = """
            mkdir -p $HOME/.kube
            sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
            sudo chown $(id -u):$(id -g) $HOME/.kube/config
            """
            ssh.exec_command(setup_kubectl)
            
        finally:
            ssh.close()
        
        # Join additional master nodes if HA
        if config.enable_ha and config.master_count > 1:
            await self._join_master_nodes(config, nodes)
        
        # Join worker nodes
        await self._join_worker_nodes(config, nodes)
    
    def _generate_kubeadm_config(self, config: ClusterConfiguration, 
                                nodes: List[NodeConfiguration]) -> str:
        """Generate kubeadm configuration"""
        master_nodes = [n for n in nodes if n.role == "master"]
        
        # For HA setup, create load balancer endpoint
        if config.enable_ha:
            control_plane_endpoint = f"{config.name}-lb:6443"
        else:
            control_plane_endpoint = f"{master_nodes[0].ip_address}:6443"
        
        kubeadm_config = f"""
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: {master_nodes[0].ip_address}
  bindPort: 6443
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
kubernetesVersion: v{config.version}
controlPlaneEndpoint: {control_plane_endpoint}
networking:
  serviceSubnet: {config.service_cidr}
  podSubnet: {config.pod_cidr}
  dnsDomain: {config.dns_domain}
apiServer:
  certSANs:
  - localhost
  - 127.0.0.1
"""
        
        # Add all master IPs to certSANs
        for node in master_nodes:
            kubeadm_config += f"  - {node.ip_address}\n"
        
        # Add extra configuration
        kubeadm_config += """
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
"""
        
        return kubeadm_config
    
    def _extract_join_commands(self, init_output: str):
        """Extract join commands from kubeadm init output"""
        lines = init_output.split('\n')
        
        master_join_cmd = []
        worker_join_cmd = []
        
        capture_master = False
        capture_worker = False
        
        for line in lines:
            if "You can now join any number of control-plane" in line:
                capture_master = True
                continue
            elif "Then you can join any number of worker" in line:
                capture_master = False
                capture_worker = True
                continue
            
            if capture_master and line.strip() and not line.startswith('  '):
                capture_master = False
            elif capture_master:
                master_join_cmd.append(line.strip())
            
            if capture_worker and "kubeadm join" in line:
                worker_join_cmd.append(line.strip())
                if "\\" not in line:
                    capture_worker = False
        
        self.join_commands = {
            'master': ' '.join(master_join_cmd),
            'worker': ' '.join(worker_join_cmd)
        }
    
    async def _join_master_nodes(self, config: ClusterConfiguration, 
                                nodes: List[NodeConfiguration]):
        """Join additional master nodes for HA"""
        master_nodes = [n for n in nodes if n.role == "master"]
        
        # Skip first master (already initialized)
        for node in master_nodes[1:]:
            self.logger.info(f"Joining master node: {node.name}")
            
            ssh = paramiko.SSHClient()
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            
            try:
                ssh.connect(node.ip_address, username='vagrant', password='vagrant')
                
                # Run join command
                stdin, stdout, stderr = ssh.exec_command(
                    f"sudo {self.join_commands['master']}"
                )
                
                output = stdout.read().decode()
                error = stderr.read().decode()
                
                if "This node has joined the cluster" not in output:
                    self.logger.error(f"Failed to join master {node.name}: {error}")
                else:
                    self.logger.info(f"Master {node.name} joined successfully")
                
            finally:
                ssh.close()
    
    async def _join_worker_nodes(self, config: ClusterConfiguration, 
                               nodes: List[NodeConfiguration]):
        """Join worker nodes to cluster"""
        worker_nodes = [n for n in nodes if n.role == "worker"]
        
        # Join workers in parallel
        tasks = []
        for node in worker_nodes:
            task = self._join_single_worker(node)
            tasks.append(task)
        
        await asyncio.gather(*tasks)
    
    async def _join_single_worker(self, node: NodeConfiguration):
        """Join a single worker node"""
        self.logger.info(f"Joining worker node: {node.name}")
        
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        
        try:
            ssh.connect(node.ip_address, username='vagrant', password='vagrant')
            
            # Run join command
            stdin, stdout, stderr = ssh.exec_command(
                f"sudo {self.join_commands['worker']}"
            )
            
            output = stdout.read().decode()
            error = stderr.read().decode()
            
            if "This node has joined the cluster" not in output:
                self.logger.error(f"Failed to join worker {node.name}: {error}")
            else:
                self.logger.info(f"Worker {node.name} joined successfully")
            
        finally:
            ssh.close()
    
    async def _configure_networking(self, config: ClusterConfiguration, 
                                  nodes: List[NodeConfiguration]):
        """Configure cluster networking"""
        self.logger.info(f"Configuring networking with {config.name}")
        
        master_node = next(n for n in nodes if n.role == "master")
        
        # Install network plugin
        if config.network_mode == NetworkingMode.FLANNEL:
            await self._install_flannel(master_node, config)
        elif config.network_mode == NetworkingMode.CALICO:
            await self._install_calico(master_node, config)
        elif config.network_mode == NetworkingMode.CILIUM:
            await self._install_cilium(master_node, config)
        else:
            raise ValueError(f"Unsupported network mode: {config.network_mode}")
    
    async def _install_flannel(self, master_node: NodeConfiguration, 
                             config: ClusterConfiguration):
        """Install Flannel CNI"""
        self.logger.info("Installing Flannel CNI")
        
        # Flannel configuration
        flannel_config = f"""
apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-flannel-cfg
  namespace: kube-flannel
data:
  cni-conf.json: |
    {{
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {{
          "type": "flannel",
          "delegate": {{
            "hairpinMode": true,
            "isDefaultGateway": true
          }}
        }},
        {{
          "type": "portmap",
          "capabilities": {{
            "portMappings": true
          }}
        }}
      ]
    }}
  net-conf.json: |
    {{
      "Network": "{config.pod_cidr}",
      "Backend": {{
        "Type": "vxlan",
        "MTU": {config.mtu}
      }}
    }}
"""
        
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        
        try:
            ssh.connect(master_node.ip_address, username='vagrant', password='vagrant')
            
            # Apply Flannel manifest
            flannel_url = "https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml"
            
            # Customize Flannel configuration
            customize_cmd = f"""
            kubectl create namespace kube-flannel || true
            cat <<EOF | kubectl apply -f -
{flannel_config}
EOF
            kubectl apply -f {flannel_url}
            """
            
            stdin, stdout, stderr = ssh.exec_command(customize_cmd)
            output = stdout.read().decode()
            
            self.logger.info("Flannel installed successfully")
            
        finally:
            ssh.close()
    
    async def _setup_load_balancer(self, config: ClusterConfiguration):
        """Setup load balancer for the cluster"""
        if config.load_balancer_type == LoadBalancerType.METALLB:
            await self._setup_metallb(config)
        elif config.load_balancer_type == LoadBalancerType.NGINX:
            await self._setup_nginx_lb(config)
        else:
            self.logger.info(f"Using {config.load_balancer_type} load balancer")
    
    async def _setup_metallb(self, config: ClusterConfiguration):
        """Setup MetalLB load balancer"""
        self.logger.info("Setting up MetalLB")
        
        master_node = next(n for n in self.clusters[config.name]['nodes'] 
                          if n.role == "master")
        
        # Calculate IP pool for MetalLB
        network = ipaddress.ip_network(config.network_cidr)
        # Use last /27 subnet for load balancer IPs
        lb_subnet = list(network.subnets(new_prefix=27))[-1]
        lb_start = str(list(lb_subnet.hosts())[0])
        lb_end = str(list(lb_subnet.hosts())[-1])
        
        metallb_config = f"""
apiVersion: v1
kind: Namespace
metadata:
  name: metallb-system
---
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  name: default-pool
  namespace: metallb-system
spec:
  addresses:
  - {lb_start}-{lb_end}
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
  name: default
  namespace: metallb-system
spec:
  ipAddressPools:
  - default-pool
"""
        
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        
        try:
            ssh.connect(master_node.ip_address, username='vagrant', password='vagrant')
            
            # Install MetalLB
            install_cmd = """
            kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.12/config/manifests/metallb-native.yaml
            kubectl wait --namespace metallb-system \
                --for=condition=ready pod \
                --selector=app=metallb \
                --timeout=90s
            """
            
            stdin, stdout, stderr = ssh.exec_command(install_cmd)
            stdout.read()
            
            # Apply MetalLB configuration
            config_cmd = f"""
            cat <<EOF | kubectl apply -f -
{metallb_config}
EOF
            """
            
            stdin, stdout, stderr = ssh.exec_command(config_cmd)
            output = stdout.read().decode()
            
            self.logger.info("MetalLB configured successfully")
            
        finally:
            ssh.close()
    
    async def _install_core_components(self, config: ClusterConfiguration):
        """Install core Kubernetes components"""
        self.logger.info("Installing core components")
        
        components = [
            self._install_metrics_server(config),
            self._install_ingress_controller(config),
            self._install_cert_manager(config),
            self._install_cluster_autoscaler(config)
        ]
        
        await asyncio.gather(*components)
    
    async def _install_metrics_server(self, config: ClusterConfiguration):
        """Install metrics server"""
        master_node = next(n for n in self.clusters[config.name]['nodes'] 
                          if n.role == "master")
        
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        
        try:
            ssh.connect(master_node.ip_address, username='vagrant', password='vagrant')
            
            # Install metrics server
            cmd = """
            kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
            
            # Patch for insecure TLS (development only)
            kubectl patch deployment metrics-server -n kube-system --type='json' \
              -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--kubelet-insecure-tls"}]'
            """
            
            stdin, stdout, stderr = ssh.exec_command(cmd)
            stdout.read()
            
            self.logger.info("Metrics server installed")
            
        finally:
            ssh.close()
    
    async def _install_ingress_controller(self, config: ClusterConfiguration):
        """Install ingress controller"""
        master_node = next(n for n in self.clusters[config.name]['nodes'] 
                          if n.role == "master")
        
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        
        try:
            ssh.connect(master_node.ip_address, username='vagrant', password='vagrant')
            
            if config.ingress_controller == "nginx":
                cmd = """
                kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.8.2/deploy/static/provider/cloud/deploy.yaml
                """
            elif config.ingress_controller == "traefik":
                cmd = """
                helm repo add traefik https://traefik.github.io/charts
                helm repo update
                helm install traefik traefik/traefik \
                  --namespace traefik \
                  --create-namespace \
                  --set service.type=LoadBalancer
                """
            else:
                self.logger.warning(f"Unknown ingress controller: {config.ingress_controller}")
                return
            
            stdin, stdout, stderr = ssh.exec_command(cmd)
            stdout.read()
            
            self.logger.info(f"{config.ingress_controller} ingress controller installed")
            
        finally:
            ssh.close()
    
    async def _install_cert_manager(self, config: ClusterConfiguration):
        """Install cert-manager for TLS certificates"""
        if not self.config['security']['cert_manager_enabled']:
            return
        
        master_node = next(n for n in self.clusters[config.name]['nodes'] 
                          if n.role == "master")
        
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        
        try:
            ssh.connect(master_node.ip_address, username='vagrant', password='vagrant')
            
            cmd = """
            kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.0/cert-manager.yaml
            
            # Wait for cert-manager to be ready
            kubectl wait --for=condition=ready pod -l app.kubernetes.io/instance=cert-manager -n cert-manager --timeout=300s
            
            # Create ClusterIssuer for Let's Encrypt
            cat <<EOF | kubectl apply -f -
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: letsencrypt-prod
spec:
  acme:
    server: https://acme-v02.api.letsencrypt.org/directory
    email: admin@example.com
    privateKeySecretRef:
      name: letsencrypt-prod
    solvers:
    - http01:
        ingress:
          class: nginx
EOF
            """
            
            stdin, stdout, stderr = ssh.exec_command(cmd)
            stdout.read()
            
            self.logger.info("cert-manager installed")
            
        finally:
            ssh.close()
    
    async def _install_cluster_autoscaler(self, config: ClusterConfiguration):
        """Install cluster autoscaler"""
        # Only relevant for cloud providers
        if config.environment == DeploymentEnvironment.HOME_LAB:
            return
        
        # Implementation would depend on cloud provider
        self.logger.info("Cluster autoscaler not needed for home lab")
    
    async def _setup_monitoring(self, config: ClusterConfiguration):
        """Setup monitoring stack"""
        self.logger.info("Setting up monitoring stack")
        
        master_node = next(n for n in self.clusters[config.name]['nodes'] 
                          if n.role == "master")
        
        # Install Prometheus Operator
        monitoring_stack = """
        # Add prometheus-community helm repo
        helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
        helm repo update
        
        # Install kube-prometheus-stack
        helm install monitoring prometheus-community/kube-prometheus-stack \
          --namespace monitoring \
          --create-namespace \
          --set prometheus.prometheusSpec.retention=30d \
          --set prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.accessModes[0]=ReadWriteOnce \
          --set prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.resources.requests.storage=50Gi \
          --set alertmanager.alertmanagerSpec.storage.volumeClaimTemplate.spec.accessModes[0]=ReadWriteOnce \
          --set alertmanager.alertmanagerSpec.storage.volumeClaimTemplate.spec.resources.requests.storage=10Gi \
          --set grafana.persistence.enabled=true \
          --set grafana.persistence.size=10Gi \
          --set grafana.adminPassword=admin123
        """
        
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        
        try:
            ssh.connect(master_node.ip_address, username='vagrant', password='vagrant')
            
            stdin, stdout, stderr = ssh.exec_command(monitoring_stack)
            output = stdout.read().decode()
            
            # Create ingress for Grafana
            grafana_ingress = """
            cat <<EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: grafana
  namespace: monitoring
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-prod
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - grafana.k8s.local
    secretName: grafana-tls
  rules:
  - host: grafana.k8s.local
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: monitoring-grafana
            port:
              number: 80
EOF
            """
            
            stdin, stdout, stderr = ssh.exec_command(grafana_ingress)
            stdout.read()
            
            self.logger.info("Monitoring stack installed")
            
        finally:
            ssh.close()
    
    async def _setup_logging_stack(self, config: ClusterConfiguration):
        """Setup logging stack with Loki"""
        self.logger.info("Setting up logging stack")
        
        master_node = next(n for n in self.clusters[config.name]['nodes'] 
                          if n.role == "master")
        
        logging_stack = """
        # Add grafana helm repo
        helm repo add grafana https://grafana.github.io/helm-charts
        helm repo update
        
        # Install Loki
        helm install loki grafana/loki-stack \
          --namespace logging \
          --create-namespace \
          --set loki.persistence.enabled=true \
          --set loki.persistence.size=50Gi \
          --set promtail.enabled=true
        
        # Configure Grafana datasource for Loki
        cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: ConfigMap
metadata:
  name: loki-datasource
  namespace: monitoring
data:
  loki-datasource.yaml: |
    apiVersion: 1
    datasources:
    - name: Loki
      type: loki
      access: proxy
      url: http://loki.logging.svc.cluster.local:3100
      isDefault: false
EOF
        """
        
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        
        try:
            ssh.connect(master_node.ip_address, username='vagrant', password='vagrant')
            
            stdin, stdout, stderr = ssh.exec_command(logging_stack)
            output = stdout.read().decode()
            
            self.logger.info("Logging stack installed")
            
        finally:
            ssh.close()
    
    async def _setup_service_mesh(self, config: ClusterConfiguration):
        """Setup service mesh"""
        if config.service_mesh == "istio":
            await self._install_istio(config)
        elif config.service_mesh == "linkerd":
            await self._install_linkerd(config)
        else:
            self.logger.warning(f"Unknown service mesh: {config.service_mesh}")
    
    async def _install_istio(self, config: ClusterConfiguration):
        """Install Istio service mesh"""
        self.logger.info("Installing Istio service mesh")
        
        master_node = next(n for n in self.clusters[config.name]['nodes'] 
                          if n.role == "master")
        
        istio_install = """
        # Download Istio
        curl -L https://istio.io/downloadIstio | sh -
        cd istio-*
        export PATH=$PWD/bin:$PATH
        
        # Install Istio with demo profile
        istioctl install --set profile=demo -y
        
        # Enable automatic sidecar injection
        kubectl label namespace default istio-injection=enabled
        
        # Install Kiali, Jaeger, Prometheus, Grafana
        kubectl apply -f samples/addons
        
        # Create ingress for Kiali
        cat <<EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: kiali
  namespace: istio-system
spec:
  ingressClassName: nginx
  rules:
  - host: kiali.k8s.local
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: kiali
            port:
              number: 20001
EOF
        """
        
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        
        try:
            ssh.connect(master_node.ip_address, username='vagrant', password='vagrant')
            
            stdin, stdout, stderr = ssh.exec_command(istio_install)
            output = stdout.read().decode()
            
            self.logger.info("Istio installed successfully")
            
        finally:
            ssh.close()
    
    async def _setup_backup_solution(self, config: ClusterConfiguration):
        """Setup backup solution"""
        if self.config['storage']['backup_solution'] == 'velero':
            await self._install_velero(config)
    
    async def _install_velero(self, config: ClusterConfiguration):
        """Install Velero for backup and restore"""
        self.logger.info("Installing Velero backup solution")
        
        master_node = next(n for n in self.clusters[config.name]['nodes'] 
                          if n.role == "master")
        
        velero_install = """
        # Install Velero CLI
        wget https://github.com/vmware-tanzu/velero/releases/download/v1.12.0/velero-v1.12.0-linux-amd64.tar.gz
        tar -xvf velero-v1.12.0-linux-amd64.tar.gz
        sudo mv velero-v1.12.0-linux-amd64/velero /usr/local/bin/
        
        # Install Velero with local storage (for demo)
        velero install \
          --provider aws \
          --plugins velero/velero-plugin-for-aws:v1.8.0 \
          --bucket velero-backups \
          --secret-file ./credentials-velero \
          --use-volume-snapshots=false \
          --backup-location-config region=minio,s3ForcePathStyle="true",s3Url=http://minio.velero.svc:9000
        
        # Create backup schedule
        velero schedule create daily-backup --schedule="0 2 * * *"
        """
        
        # For home lab, we'll use MinIO as S3-compatible storage
        minio_install = """
        helm repo add minio https://charts.min.io/
        helm install minio minio/minio \
          --namespace velero \
          --create-namespace \
          --set mode=standalone \
          --set persistence.size=50Gi \
          --set rootUser=admin \
          --set rootPassword=admin123
        """
        
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        
        try:
            ssh.connect(master_node.ip_address, username='vagrant', password='vagrant')
            
            # Install MinIO first
            stdin, stdout, stderr = ssh.exec_command(minio_install)
            stdout.read()
            
            # Then install Velero
            # Note: In production, you'd use cloud provider object storage
            self.logger.info("Velero installed with MinIO backend")
            
        finally:
            ssh.close()
    
    async def deploy_application(self, cluster_id: str, app_config: Dict):
        """Deploy application to cluster"""
        if cluster_id not in self.clusters:
            raise ValueError(f"Cluster not found: {cluster_id}")
        
        self.logger.info(f"Deploying application to cluster {cluster_id}")
        
        # Generate Kubernetes manifests
        manifests = self._generate_app_manifests(app_config)
        
        # Apply manifests
        master_node = next(n for n in self.clusters[cluster_id]['nodes'] 
                          if n.role == "master")
        
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        
        try:
            ssh.connect(master_node.ip_address, username='vagrant', password='vagrant')
            
            for manifest in manifests:
                cmd = f"cat <<EOF | kubectl apply -f -\n{manifest}\nEOF"
                stdin, stdout, stderr = ssh.exec_command(cmd)
                output = stdout.read().decode()
                
                if "created" in output or "configured" in output:
                    self.logger.info(f"Applied manifest successfully")
                else:
                    self.logger.error(f"Failed to apply manifest: {stderr.read().decode()}")
            
        finally:
            ssh.close()
    
    def _generate_app_manifests(self, app_config: Dict) -> List[str]:
        """Generate Kubernetes manifests for application"""
        manifests = []
        
        # Deployment manifest
        deployment = f"""
apiVersion: apps/v1
kind: Deployment
metadata:
  name: {app_config['name']}
  namespace: {app_config.get('namespace', 'default')}
spec:
  replicas: {app_config.get('replicas', 1)}
  selector:
    matchLabels:
      app: {app_config['name']}
  template:
    metadata:
      labels:
        app: {app_config['name']}
    spec:
      containers:
      - name: {app_config['name']}
        image: {app_config['image']}
        ports:
        - containerPort: {app_config.get('port', 8080)}
        resources:
          requests:
            memory: "{app_config.get('memory_request', '128Mi')}"
            cpu: "{app_config.get('cpu_request', '100m')}"
          limits:
            memory: "{app_config.get('memory_limit', '256Mi')}"
            cpu: "{app_config.get('cpu_limit', '200m')}"
"""
        manifests.append(deployment)
        
        # Service manifest
        service = f"""
apiVersion: v1
kind: Service
metadata:
  name: {app_config['name']}
  namespace: {app_config.get('namespace', 'default')}
spec:
  type: {app_config.get('service_type', 'ClusterIP')}
  selector:
    app: {app_config['name']}
  ports:
  - port: {app_config.get('service_port', 80)}
    targetPort: {app_config.get('port', 8080)}
"""
        manifests.append(service)
        
        # Ingress manifest if enabled
        if app_config.get('ingress_enabled', False):
            ingress = f"""
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: {app_config['name']}
  namespace: {app_config.get('namespace', 'default')}
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-prod
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - {app_config['ingress_host']}
    secretName: {app_config['name']}-tls
  rules:
  - host: {app_config['ingress_host']}
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: {app_config['name']}
            port:
              number: {app_config.get('service_port', 80)}
"""
            manifests.append(ingress)
        
        return manifests
    
    def generate_cluster_report(self, cluster_id: str) -> Dict:
        """Generate comprehensive cluster report"""
        if cluster_id not in self.clusters:
            raise ValueError(f"Cluster not found: {cluster_id}")
        
        cluster = self.clusters[cluster_id]
        config = cluster['config']
        
        report = {
            'cluster_id': cluster_id,
            'name': config.name,
            'environment': config.environment.value,
            'created_at': cluster['created_at'],
            'status': cluster['status'],
            'configuration': {
                'kubernetes_version': config.version,
                'master_nodes': config.master_count,
                'worker_nodes': config.worker_count,
                'network_plugin': config.network_mode.value,
                'load_balancer': config.load_balancer_type.value,
                'high_availability': config.enable_ha,
                'monitoring_enabled': config.enable_monitoring,
                'logging_enabled': config.enable_logging,
                'service_mesh': config.enable_service_mesh
            },
            'networking': {
                'cluster_cidr': config.network_cidr,
                'service_cidr': config.service_cidr,
                'pod_cidr': config.pod_cidr,
                'dns_domain': config.dns_domain
            },
            'nodes': []
        }
        
        # Add node information
        for node in cluster['nodes']:
            report['nodes'].append({
                'name': node.name,
                'role': node.role,
                'ip_address': node.ip_address,
                'resources': {
                    'cpu_cores': node.cpu_cores,
                    'memory_gb': node.memory_gb,
                    'disk_gb': node.disk_gb
                }
            })
        
        return report

# Deployment automation script
async def main():
    """Main deployment function"""
    # Initialize orchestrator
    orchestrator = EnterpriseKubernetesOrchestrator()
    
    # Define cluster configurations for different environments
    configs = {
        'home_lab': ClusterConfiguration(
            name="k8s-home-lab",
            environment=DeploymentEnvironment.HOME_LAB,
            version="1.28.0",
            master_count=1,
            worker_count=2,
            network_cidr="192.168.56.0/24",
            service_cidr="10.96.0.0/12",
            pod_cidr="10.244.0.0/16",
            enable_ha=False,
            enable_monitoring=True,
            enable_logging=True,
            enable_service_mesh=False
        ),
        'development': ClusterConfiguration(
            name="k8s-dev",
            environment=DeploymentEnvironment.DEVELOPMENT,
            version="1.28.0",
            master_count=3,
            worker_count=3,
            network_cidr="10.0.0.0/16",
            service_cidr="10.96.0.0/12",
            pod_cidr="172.16.0.0/12",
            enable_ha=True,
            enable_monitoring=True,
            enable_logging=True,
            enable_service_mesh=True
        ),
        'production': ClusterConfiguration(
            name="k8s-prod",
            environment=DeploymentEnvironment.PRODUCTION,
            version="1.28.0",
            master_count=5,
            worker_count=10,
            network_cidr="10.0.0.0/8",
            service_cidr="10.96.0.0/12",
            pod_cidr="100.64.0.0/10",
            enable_ha=True,
            enable_monitoring=True,
            enable_logging=True,
            enable_service_mesh=True,
            backup_enabled=True
        )
    }
    
    # Deploy home lab cluster
    cluster_config = configs['home_lab']
    
    print(f"Deploying {cluster_config.environment.value} Kubernetes cluster...")
    cluster_id = await orchestrator.create_cluster(cluster_config)
    
    print(f"Cluster deployed successfully: {cluster_id}")
    
    # Deploy sample application
    sample_app = {
        'name': 'hello-world',
        'image': 'nginxdemos/hello',
        'replicas': 3,
        'port': 80,
        'service_type': 'LoadBalancer',
        'ingress_enabled': True,
        'ingress_host': 'hello.k8s.local'
    }
    
    print("Deploying sample application...")
    await orchestrator.deploy_application(cluster_id, sample_app)
    
    # Generate cluster report
    report = orchestrator.generate_cluster_report(cluster_id)
    
    print("\nCluster Report")
    print("=" * 50)
    print(f"Cluster ID: {report['cluster_id']}")
    print(f"Environment: {report['environment']}")
    print(f"Kubernetes Version: {report['configuration']['kubernetes_version']}")
    print(f"Masters: {report['configuration']['master_nodes']}")
    print(f"Workers: {report['configuration']['worker_nodes']}")
    print(f"Network Plugin: {report['configuration']['network_plugin']}")
    print(f"Load Balancer: {report['configuration']['load_balancer']}")
    print("\nNodes:")
    for node in report['nodes']:
        print(f"  - {node['name']} ({node['role']}): {node['ip_address']}")
    
    print("\n✅ Kubernetes cluster ready!")
    print(f"kubectl config: ~/.kube/config")
    print(f"Grafana: http://grafana.k8s.local (admin/admin123)")
    print(f"Sample App: http://hello.k8s.local")

if __name__ == "__main__":
    asyncio.run(main())

#!/bin/bash
# Enterprise Kubernetes Cluster Management Script

set -euo pipefail

# Configuration
CLUSTER_NAME="${CLUSTER_NAME:-k8s-cluster}"
ENVIRONMENT="${ENVIRONMENT:-development}"
BACKUP_LOCATION="${BACKUP_LOCATION:-s3://k8s-backups}"

# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Logging function
log() {
    echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $*"
}

error() {
    echo -e "${RED}[ERROR]${NC} $*" >&2
}

warning() {
    echo -e "${YELLOW}[WARNING]${NC} $*"
}

# Check prerequisites
check_prerequisites() {
    log "Checking prerequisites..."
    
    local required_tools=("kubectl" "helm" "jq" "yq")
    
    for tool in "${required_tools[@]}"; do
        if ! command -v "$tool" &> /dev/null; then
            error "$tool is not installed"
            exit 1
        fi
    done
    
    # Check kubectl connection
    if ! kubectl cluster-info &> /dev/null; then
        error "Cannot connect to Kubernetes cluster"
        exit 1
    fi
    
    log "✅ All prerequisites met"
}

# Cluster health check
cluster_health_check() {
    log "Performing cluster health check..."
    
    # Check node status
    log "Checking nodes..."
    local unhealthy_nodes=$(kubectl get nodes -o json | jq -r '.items[] | select(.status.conditions[] | select(.type=="Ready" and .status!="True")) | .metadata.name')
    
    if [[ -n "$unhealthy_nodes" ]]; then
        error "Unhealthy nodes detected: $unhealthy_nodes"
        return 1
    fi
    
    # Check system pods
    log "Checking system pods..."
    local failed_pods=$(kubectl get pods -A -o json | jq -r '.items[] | select(.status.phase!="Running" and .status.phase!="Succeeded") | "\(.metadata.namespace)/\(.metadata.name)"')
    
    if [[ -n "$failed_pods" ]]; then
        warning "Failed pods detected:"
        echo "$failed_pods"
    fi
    
    # Check PVCs
    log "Checking persistent volume claims..."
    local unbound_pvcs=$(kubectl get pvc -A -o json | jq -r '.items[] | select(.status.phase!="Bound") | "\(.metadata.namespace)/\(.metadata.name)"')
    
    if [[ -n "$unbound_pvcs" ]]; then
        warning "Unbound PVCs detected:"
        echo "$unbound_pvcs"
    fi
    
    # Check cluster capacity
    log "Checking cluster capacity..."
    local capacity_report=$(kubectl top nodes --no-headers | awk '
    {
        cpu_percent = substr($3, 1, length($3)-1)
        mem_percent = substr($5, 1, length($5)-1)
        if (cpu_percent > 80) print $1 " CPU: " $3 " (HIGH)"
        if (mem_percent > 80) print $1 " Memory: " $5 " (HIGH)"
    }')
    
    if [[ -n "$capacity_report" ]]; then
        warning "High resource usage detected:"
        echo "$capacity_report"
    fi
    
    log "✅ Cluster health check completed"
}

# Backup cluster configuration
backup_cluster_config() {
    log "Backing up cluster configuration..."
    
    local backup_dir="/tmp/k8s-backup-$(date +%Y%m%d-%H%M%S)"
    mkdir -p "$backup_dir"
    
    # Backup all namespaces
    kubectl get namespaces -o yaml > "$backup_dir/namespaces.yaml"
    
    # Backup all resources in each namespace
    for ns in $(kubectl get namespaces -o jsonpath='{.items[*].metadata.name}'); do
        log "Backing up namespace: $ns"
        mkdir -p "$backup_dir/$ns"
        
        # Get all resource types
        for resource in $(kubectl api-resources --namespaced=true -o name); do
            if kubectl get "$resource" -n "$ns" &> /dev/null; then
                kubectl get "$resource" -n "$ns" -o yaml > "$backup_dir/$ns/$resource.yaml" 2>/dev/null || true
            fi
        done
    done
    
    # Backup cluster-wide resources
    log "Backing up cluster-wide resources..."
    mkdir -p "$backup_dir/cluster"
    
    for resource in $(kubectl api-resources --namespaced=false -o name); do
        if kubectl get "$resource" &> /dev/null; then
            kubectl get "$resource" -o yaml > "$backup_dir/cluster/$resource.yaml" 2>/dev/null || true
        fi
    done
    
    # Create tarball
    local backup_file="k8s-backup-${CLUSTER_NAME}-$(date +%Y%m%d-%H%M%S).tar.gz"
    tar -czf "/tmp/$backup_file" -C "$backup_dir" .
    
    # Upload to backup location
    if [[ "$BACKUP_LOCATION" == s3://* ]]; then
        aws s3 cp "/tmp/$backup_file" "$BACKUP_LOCATION/"
        log "✅ Backup uploaded to $BACKUP_LOCATION/$backup_file"
    else
        mv "/tmp/$backup_file" "$BACKUP_LOCATION/"
        log "✅ Backup saved to $BACKUP_LOCATION/$backup_file"
    fi
    
    # Cleanup
    rm -rf "$backup_dir"
    rm -f "/tmp/$backup_file"
}

# Scale cluster
scale_cluster() {
    local component="$1"
    local replicas="$2"
    
    log "Scaling $component to $replicas replicas..."
    
    case "$component" in
        "workers")
            # For cloud providers, this would use cluster autoscaler
            # For bare metal/vagrant, manual intervention needed
            warning "Manual worker node scaling required for this environment"
            ;;
        *)
            # Scale deployment/statefulset
            if kubectl get deployment "$component" &> /dev/null; then
                kubectl scale deployment "$component" --replicas="$replicas"
            elif kubectl get statefulset "$component" &> /dev/null; then
                kubectl scale statefulset "$component" --replicas="$replicas"
            else
                error "Component $component not found"
                return 1
            fi
            ;;
    esac
    
    log "✅ Scaling completed"
}

# Upgrade cluster
upgrade_cluster() {
    local target_version="$1"
    
    log "Upgrading cluster to Kubernetes $target_version..."
    
    # Pre-upgrade checks
    log "Running pre-upgrade checks..."
    cluster_health_check
    
    # Backup before upgrade
    log "Creating pre-upgrade backup..."
    backup_cluster_config
    
    # For managed Kubernetes (EKS, GKE, AKS), use cloud provider tools
    # For kubeadm clusters:
    if command -v kubeadm &> /dev/null; then
        log "Upgrading control plane..."
        # This would need to be run on each master node
        # kubeadm upgrade plan
        # kubeadm upgrade apply v$target_version
        
        log "Upgrading kubelet and kubectl..."
        # apt-get update && apt-get install -y kubelet=$target_version kubectl=$target_version
        # systemctl restart kubelet
        
        warning "Manual upgrade steps required - see documentation"
    else
        warning "Cluster upgrade method not detected"
    fi
    
    log "✅ Upgrade process initiated"
}

# Security audit
security_audit() {
    log "Running security audit..."
    
    # Check for pods running as root
    log "Checking for pods running as root..."
    local root_pods=$(kubectl get pods -A -o json | jq -r '.items[] | select(.spec.containers[]?.securityContext?.runAsUser == 0 or .spec.securityContext?.runAsUser == 0) | "\(.metadata.namespace)/\(.metadata.name)"')
    
    if [[ -n "$root_pods" ]]; then
        warning "Pods running as root:"
        echo "$root_pods"
    fi
    
    # Check for privileged pods
    log "Checking for privileged pods..."
    local privileged_pods=$(kubectl get pods -A -o json | jq -r '.items[] | select(.spec.containers[]?.securityContext?.privileged == true) | "\(.metadata.namespace)/\(.metadata.name)"')
    
    if [[ -n "$privileged_pods" ]]; then
        warning "Privileged pods detected:"
        echo "$privileged_pods"
    fi
    
    # Check RBAC permissions
    log "Checking RBAC permissions..."
    local admin_bindings=$(kubectl get clusterrolebindings -o json | jq -r '.items[] | select(.roleRef.name == "cluster-admin") | .metadata.name')
    
    if [[ -n "$admin_bindings" ]]; then
        warning "Cluster-admin role bindings:"
        echo "$admin_bindings"
    fi
    
    # Check network policies
    log "Checking network policies..."
    local ns_without_netpol=$(comm -23 <(kubectl get ns -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | sort) <(kubectl get networkpolicies -A -o jsonpath='{.items[*].metadata.namespace}' | tr ' ' '\n' | sort -u))
    
    if [[ -n "$ns_without_netpol" ]]; then
        warning "Namespaces without network policies:"
        echo "$ns_without_netpol"
    fi
    
    log "✅ Security audit completed"
}

# Performance tuning
performance_tuning() {
    log "Running performance tuning checks..."
    
    # Check resource requests/limits
    log "Checking resource specifications..."
    local pods_without_limits=$(kubectl get pods -A -o json | jq -r '.items[] | select(.spec.containers[]? | (.resources.limits == null or .resources.requests == null)) | "\(.metadata.namespace)/\(.metadata.name)"')
    
    if [[ -n "$pods_without_limits" ]]; then
        warning "Pods without resource limits/requests:"
        echo "$pods_without_limits" | head -10
        echo "..."
    fi
    
    # Check HPA status
    log "Checking Horizontal Pod Autoscalers..."
    kubectl get hpa -A --no-headers | while read -r line; do
        local current=$(echo "$line" | awk '{print $3}')
        local target=$(echo "$line" | awk '{print $4}')
        
        if [[ "$current" == "<unknown>" ]]; then
            warning "HPA with unknown metrics: $line"
        fi
    done
    
    # Check PDB coverage
    log "Checking Pod Disruption Budgets..."
    local deployments_without_pdb=$(comm -23 <(kubectl get deployments -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' | sort) <(kubectl get pdb -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.spec.selector.matchLabels.app}{"\n"}{end}' | sort))
    
    if [[ -n "$deployments_without_pdb" ]]; then
        warning "Deployments without PDB:"
        echo "$deployments_without_pdb" | head -10
    fi
    
    log "✅ Performance tuning check completed"
}

# Cost optimization
cost_optimization() {
    log "Running cost optimization analysis..."
    
    # Check for unused resources
    log "Checking for unused PVCs..."
    local unused_pvcs=$(kubectl get pvc -A -o json | jq -r '.items[] | select(.status.phase == "Bound") | select(.metadata.annotations."volume.kubernetes.io/used-by" == null) | "\(.metadata.namespace)/\(.metadata.name)"')
    
    if [[ -n "$unused_pvcs" ]]; then
        warning "Potentially unused PVCs:"
        echo "$unused_pvcs"
    fi
    
    # Check for oversized nodes
    log "Analyzing node utilization..."
    kubectl top nodes --no-headers | awk '
    {
        cpu_percent = substr($3, 1, length($3)-1)
        mem_percent = substr($5, 1, length($5)-1)
        if (cpu_percent < 20 && mem_percent < 20) {
            print $1 " is underutilized (CPU: " $3 ", Memory: " $5 ")"
        }
    }'
    
    # Check for duplicate services
    log "Checking for duplicate services..."
    kubectl get services -A -o json | jq -r '.items[] | select(.spec.type == "LoadBalancer") | "\(.metadata.namespace)/\(.metadata.name): \(.spec.ports[].port)"' | sort
    
    log "✅ Cost optimization analysis completed"
}

# Generate comprehensive report
generate_report() {
    log "Generating cluster report..."
    
    local report_file="k8s-report-${CLUSTER_NAME}-$(date +%Y%m%d-%H%M%S).html"
    
    cat > "$report_file" <<EOF
<!DOCTYPE html>
<html>
<head>
    <title>Kubernetes Cluster Report - $CLUSTER_NAME</title>
    <style>
        body { font-family: Arial, sans-serif; margin: 20px; }
        h1 { color: #333; }
        h2 { color: #666; }
        table { border-collapse: collapse; width: 100%; margin: 20px 0; }
        th, td { border: 1px solid #ddd; padding: 8px; text-align: left; }
        th { background-color: #f2f2f2; }
        .success { color: green; }
        .warning { color: orange; }
        .error { color: red; }
    </style>
</head>
<body>
    <h1>Kubernetes Cluster Report</h1>
    <p>Cluster: $CLUSTER_NAME</p>
    <p>Environment: $ENVIRONMENT</p>
    <p>Generated: $(date)</p>
    
    <h2>Cluster Information</h2>
    <pre>$(kubectl cluster-info)</pre>
    
    <h2>Node Status</h2>
    <table>
        <tr><th>Name</th><th>Status</th><th>Version</th><th>OS</th></tr>
        $(kubectl get nodes -o json | jq -r '.items[] | "<tr><td>\(.metadata.name)</td><td>\(.status.conditions[] | select(.type=="Ready") | .status)</td><td>\(.status.nodeInfo.kubeletVersion)</td><td>\(.status.nodeInfo.osImage)</td></tr>"')
    </table>
    
    <h2>Resource Utilization</h2>
    <pre>$(kubectl top nodes)</pre>
    
    <h2>Namespace Summary</h2>
    <table>
        <tr><th>Namespace</th><th>Pods</th><th>Services</th><th>Deployments</th></tr>
        $(kubectl get namespaces -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | while read ns; do
            pods=$(kubectl get pods -n "$ns" --no-headers 2>/dev/null | wc -l)
            services=$(kubectl get services -n "$ns" --no-headers 2>/dev/null | wc -l)
            deployments=$(kubectl get deployments -n "$ns" --no-headers 2>/dev/null | wc -l)
            echo "<tr><td>$ns</td><td>$pods</td><td>$services</td><td>$deployments</td></tr>"
        done)
    </table>
    
    <h2>Storage</h2>
    <pre>$(kubectl get pv,pvc -A)</pre>
    
    <h2>Network</h2>
    <h3>Services</h3>
    <pre>$(kubectl get services -A | grep -E "(LoadBalancer|NodePort)")</pre>
    
    <h3>Ingresses</h3>
    <pre>$(kubectl get ingress -A)</pre>
</body>
</html>
EOF
    
    log "✅ Report generated: $report_file"
}

# Main menu
main() {
    check_prerequisites
    
    case "${1:-help}" in
        "health")
            cluster_health_check
            ;;
        "backup")
            backup_cluster_config
            ;;
        "scale")
            scale_cluster "$2" "$3"
            ;;
        "upgrade")
            upgrade_cluster "$2"
            ;;
        "security")
            security_audit
            ;;
        "performance")
            performance_tuning
            ;;
        "cost")
            cost_optimization
            ;;
        "report")
            generate_report
            ;;
        "all")
            cluster_health_check
            security_audit
            performance_tuning
            cost_optimization
            generate_report
            ;;
        *)
            echo "Usage: $0 {health|backup|scale|upgrade|security|performance|cost|report|all}"
            echo ""
            echo "Commands:"
            echo "  health      - Run cluster health check"
            echo "  backup      - Backup cluster configuration"
            echo "  scale       - Scale cluster components"
            echo "  upgrade     - Upgrade cluster version"
            echo "  security    - Run security audit"
            echo "  performance - Performance tuning check"
            echo "  cost        - Cost optimization analysis"
            echo "  report      - Generate comprehensive report"
            echo "  all         - Run all checks and generate report"
            exit 1
            ;;
    esac
}

# Execute main function
main "$@"

# Enterprise Application Deployment Template
apiVersion: v1
kind: Namespace
metadata:
  name: production-app
  labels:
    environment: production
    compliance: pci-dss
---
apiVersion: v1
kind: ResourceQuota
metadata:
  name: production-quota
  namespace: production-app
spec:
  hard:
    requests.cpu: "100"
    requests.memory: "200Gi"
    limits.cpu: "200"
    limits.memory: "400Gi"
    persistentvolumeclaims: "10"
    services.loadbalancers: "5"
---
apiVersion: v1
kind: LimitRange
metadata:
  name: production-limits
  namespace: production-app
spec:
  limits:
  - max:
      cpu: "4"
      memory: "8Gi"
    min:
      cpu: "100m"
      memory: "128Mi"
    default:
      cpu: "500m"
      memory: "1Gi"
    defaultRequest:
      cpu: "250m"
      memory: "512Mi"
    type: Container
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: production-network-policy
  namespace: production-app
spec:
  podSelector:
    matchLabels:
      app: production-app
  policyTypes:
  - Ingress
  - Egress
  ingress:
  - from:
    - namespaceSelector:
        matchLabels:
          name: ingress-nginx
    - podSelector:
        matchLabels:
          app: production-app
    ports:
    - protocol: TCP
      port: 8080
  egress:
  - to:
    - namespaceSelector:
        matchLabels:
          name: production-app
    ports:
    - protocol: TCP
      port: 5432
  - to:
    - namespaceSelector:
        matchLabels:
          name: kube-system
    ports:
    - protocol: TCP
      port: 53
    - protocol: UDP
      port: 53
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: production-app
  namespace: production-app
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: production-app-role
  namespace: production-app
rules:
- apiGroups: [""]
  resources: ["configmaps", "secrets"]
  verbs: ["get", "list"]
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: production-app-binding
  namespace: production-app
subjects:
- kind: ServiceAccount
  name: production-app
  namespace: production-app
roleRef:
  kind: Role
  name: production-app-role
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: production-app
  namespace: production-app
  labels:
    app: production-app
    version: v1.0.0
spec:
  replicas: 3
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
  selector:
    matchLabels:
      app: production-app
  template:
    metadata:
      labels:
        app: production-app
        version: v1.0.0
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "9090"
        prometheus.io/path: "/metrics"
    spec:
      serviceAccountName: production-app
      securityContext:
        runAsNonRoot: true
        runAsUser: 1000
        fsGroup: 2000
        seccompProfile:
          type: RuntimeDefault
      containers:
      - name: app
        image: registry.company.com/production-app:v1.0.0
        imagePullPolicy: Always
        ports:
        - containerPort: 8080
          name: http
          protocol: TCP
        - containerPort: 9090
          name: metrics
          protocol: TCP
        env:
        - name: APP_ENV
          value: "production"
        - name: LOG_LEVEL
          value: "info"
        - name: DB_HOST
          valueFrom:
            secretKeyRef:
              name: production-db
              key: host
        - name: DB_PASSWORD
          valueFrom:
            secretKeyRef:
              name: production-db
              key: password
        resources:
          requests:
            memory: "512Mi"
            cpu: "250m"
          limits:
            memory: "1Gi"
            cpu: "500m"
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 30
          periodSeconds: 10
          timeoutSeconds: 5
          failureThreshold: 3
        readinessProbe:
          httpGet:
            path: /ready
            port: 8080
          initialDelaySeconds: 5
          periodSeconds: 5
          timeoutSeconds: 3
          successThreshold: 1
          failureThreshold: 3
        volumeMounts:
        - name: config
          mountPath: /etc/app
          readOnly: true
        - name: secrets
          mountPath: /etc/secrets
          readOnly: true
        securityContext:
          allowPrivilegeEscalation: false
          readOnlyRootFilesystem: true
          capabilities:
            drop:
            - ALL
      volumes:
      - name: config
        configMap:
          name: production-app-config
      - name: secrets
        secret:
          secretName: production-app-secrets
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - production-app
            topologyKey: kubernetes.io/hostname
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: topology.kubernetes.io/zone
        whenUnsatisfied: DoNotSchedule
        labelSelector:
          matchLabels:
            app: production-app
---
apiVersion: v1
kind: Service
metadata:
  name: production-app
  namespace: production-app
  labels:
    app: production-app
spec:
  type: ClusterIP
  selector:
    app: production-app
  ports:
  - name: http
    port: 80
    targetPort: 8080
    protocol: TCP
  - name: metrics
    port: 9090
    targetPort: 9090
    protocol: TCP
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: production-app
  namespace: production-app
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: production-app
  minReplicas: 3
  maxReplicas: 20
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70
  - type: Resource
    resource:
      name: memory
      target:
        type: Utilization
        averageUtilization: 80
  - type: Pods
    pods:
      metric:
        name: http_requests_per_second
      target:
        type: AverageValue
        averageValue: "1000"
  behavior:
    scaleDown:
      stabilizationWindowSeconds: 300
      policies:
      - type: Percent
        value: 10
        periodSeconds: 60
      - type: Pods
        value: 2
        periodSeconds: 60
      selectPolicy: Min
    scaleUp:
      stabilizationWindowSeconds: 0
      policies:
      - type: Percent
        value: 100
        periodSeconds: 15
      - type: Pods
        value: 4
        periodSeconds: 15
      selectPolicy: Max
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: production-app
  namespace: production-app
spec:
  minAvailable: 2
  selector:
    matchLabels:
      app: production-app
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: production-app
  namespace: production-app
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-prod
    nginx.ingress.kubernetes.io/rate-limit: "100"
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
    nginx.ingress.kubernetes.io/proxy-body-size: "10m"
    nginx.ingress.kubernetes.io/proxy-connect-timeout: "30"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "30"
    nginx.ingress.kubernetes.io/proxy-read-timeout: "30"
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - app.company.com
    secretName: production-app-tls
  rules:
  - host: app.company.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: production-app
            port:
              number: 80