# finops-cost-monitoring.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: finops-tagging-policy
  namespace: finops-system
data:
  tagging-policy.yaml: |
    required_tags:
      mandatory:
        - Environment        # production, staging, development
        - CostCenter         # Finance team cost center code
        - Project            # Project or product identifier
        - Owner              # Team or individual responsible
        - BusinessUnit       # Business unit or division
        - Application        # Application name
        - Component          # Component type (web, api, database, etc.)
        - CreatedBy          # Person or system that created resource
        - Purpose            # Resource purpose or function
      
      conditional:
        - DataClassification # For resources handling sensitive data
        - ComplianceLevel    # For regulated workloads
        - BackupPolicy       # For persistent resources
        - MaintenanceWindow  # For production resources
        - Expiration         # For temporary resources
    
    tag_validation:
      format_rules:
        Environment:
          values: ["production", "staging", "development", "sandbox"]
        CostCenter:
          pattern: "^CC-[0-9]{4}$"
        Project:
          pattern: "^[a-z][a-z0-9-]{2,30}$"
        Owner:
          pattern: "^[a-z][a-z0-9.-]+@company\\.com$"
        BusinessUnit:
          values: ["engineering", "sales", "marketing", "hr", "finance"]
    
    cost_allocation_rules:
      primary_dimensions:
        - Environment
        - Project
        - BusinessUnit
        - CostCenter
      
      secondary_dimensions:
        - Application
        - Component
        - Owner
      
      allocation_methods:
        shared_services:
          method: "proportional"
          basis: "compute_hours"
        networking:
          method: "equal_split"
          scope: "environment"
        storage:
          method: "direct_allocation"
          tracking: "usage_based"
---
# Cost monitoring automation
apiVersion: batch/v1
kind: CronJob
metadata:
  name: cost-monitoring-automation
  namespace: finops-system
spec:
  schedule: "0 */6 * * *"  # Every 6 hours
  jobTemplate:
    spec:
      template:
        spec:
          serviceAccountName: finops-automation
          containers:
          - name: cost-monitor
            image: company/finops-automation:v2.1.0
            command:
            - /bin/sh
            - -c
            - |
              echo "Starting cost monitoring automation..."
              
              # Update cost data from cloud providers
              python3 /app/scripts/update_cost_data.py
              
              # Validate resource tagging compliance
              python3 /app/scripts/validate_tagging.py
              
              # Generate cost reports
              python3 /app/scripts/generate_reports.py
              
              # Check budget thresholds
              python3 /app/scripts/check_budgets.py
              
              # Identify optimization opportunities
              python3 /app/scripts/identify_optimizations.py
              
              echo "Cost monitoring automation completed"
            
            env:
            - name: AWS_ROLE_ARN
              value: "arn:aws:iam::123456789012:role/FinOpsAutomationRole"
            - name: AZURE_TENANT_ID
              valueFrom:
                secretKeyRef:
                  name: finops-secrets
                  key: azure_tenant_id
            - name: GCP_PROJECT_ID
              value: "company-finops-billing"
            - name: SLACK_WEBHOOK_URL
              valueFrom:
                secretKeyRef:
                  name: finops-secrets
                  key: slack_webhook_url
            
            volumeMounts:
            - name: config
              mountPath: /app/config
              readOnly: true
            - name: reports
              mountPath: /app/reports
          
          volumes:
          - name: config
            configMap:
              name: finops-config
          - name: reports
            persistentVolumeClaim:
              claimName: finops-reports
          
          restartPolicy: OnFailure
---
# Resource optimization policies
apiVersion: v1
kind: ConfigMap
metadata:
  name: resource-optimization-policies
  namespace: finops-system
data:
  optimization-rules.yaml: |
    rightsizing_policies:
      cpu_utilization:
        threshold_low: 20    # CPU utilization below 20%
        threshold_high: 80   # CPU utilization above 80%
        evaluation_period: "7d"
        confidence_level: 0.95
        
        recommendations:
          underutilized:
            action: "downsize"
            min_savings: 100     # Minimum monthly savings in USD
            impact_assessment: true
          
          overutilized:
            action: "upsize"
            max_cost_increase: 500  # Maximum monthly cost increase
            performance_validation: true
      
      memory_utilization:
        threshold_low: 30
        threshold_high: 85
        evaluation_period: "7d"
        confidence_level: 0.95
      
      storage_optimization:
        unused_volumes:
          action: "delete"
          age_threshold: "30d"
          backup_before_delete: true
        
        infrequently_accessed:
          action: "migrate_to_ia"
          access_threshold: "30d"
          cost_benefit_ratio: 1.2
    
    reserved_instance_policies:
      commitment_analysis:
        evaluation_period: "90d"
        utilization_threshold: 70
        commitment_types: ["1-year", "3-year"]
        payment_options: ["no-upfront", "partial-upfront", "all-upfront"]
      
      recommendations:
        min_savings: 15      # Minimum percentage savings
        payback_period: 12   # Maximum months to break even
        risk_assessment: true
    
    spot_instance_policies:
      workload_suitability:
        fault_tolerant: true
        stateless: true
        flexible_timing: true
        interruption_handling: required
      
      savings_targets:
        min_discount: 50     # Minimum percentage savings
        availability_zones: 3 # Spread across AZs
        instance_types: 5    # Multiple instance types

#!/usr/bin/env python3
# finops-optimization-engine.py

import boto3
import json
import pandas as pd
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Tuple
import numpy as np
from dataclasses import dataclass
import logging
from concurrent.futures import ThreadPoolExecutor
import requests

@dataclass
class CostOptimizationRecommendation:
    """Represents a cost optimization recommendation."""
    resource_id: str
    resource_type: str
    current_cost: float
    optimized_cost: float
    savings: float
    savings_percentage: float
    risk_level: str
    implementation_effort: str
    description: str
    action_items: List[str]
    impact_assessment: Dict
    confidence_score: float

class FinOpsOptimizationEngine:
    """Advanced FinOps optimization engine for multi-cloud environments."""
    
    def __init__(self, config: Dict):
        self.config = config
        self.aws_session = boto3.Session(
            region_name=config.get('aws_region', 'us-west-2')
        )
        self.logger = self._setup_logging()
        
        # Initialize cloud provider clients
        self.cost_explorer = self.aws_session.client('ce')
        self.ec2_client = self.aws_session.client('ec2')
        self.rds_client = self.aws_session.client('rds')
        self.cloudwatch = self.aws_session.client('cloudwatch')
        
        # Optimization thresholds
        self.cpu_threshold_low = config.get('cpu_threshold_low', 20)
        self.cpu_threshold_high = config.get('cpu_threshold_high', 80)
        self.memory_threshold_low = config.get('memory_threshold_low', 30)
        self.min_savings_threshold = config.get('min_savings_threshold', 100)
    
    def _setup_logging(self) -> logging.Logger:
        """Setup logging configuration."""
        logging.basicConfig(
            level=logging.INFO,
            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
        )
        return logging.getLogger(__name__)
    
    def analyze_cost_trends(self, days: int = 30) -> Dict:
        """Analyze cost trends and spending patterns."""
        end_date = datetime.now().date()
        start_date = end_date - timedelta(days=days)
        
        try:
            response = self.cost_explorer.get_cost_and_usage(
                TimePeriod={
                    'Start': start_date.strftime('%Y-%m-%d'),
                    'End': end_date.strftime('%Y-%m-%d')
                },
                Granularity='DAILY',
                Metrics=['BlendedCost', 'UsageQuantity'],
                GroupBy=[
                    {'Type': 'DIMENSION', 'Key': 'SERVICE'},
                    {'Type': 'TAG', 'Key': 'Environment'},
                    {'Type': 'TAG', 'Key': 'Project'}
                ]
            )
            
            # Process cost data
            cost_data = []
            for result in response['ResultsByTime']:
                date = result['TimePeriod']['Start']
                for group in result['Groups']:
                    cost_data.append({
                        'date': date,
                        'service': group['Keys'][0],
                        'environment': group['Keys'][1] if len(group['Keys']) > 1 else 'untagged',
                        'project': group['Keys'][2] if len(group['Keys']) > 2 else 'untagged',
                        'cost': float(group['Metrics']['BlendedCost']['Amount']),
                        'usage': float(group['Metrics']['UsageQuantity']['Amount'])
                    })
            
            df = pd.DataFrame(cost_data)
            
            # Calculate trends and insights
            analysis = {
                'total_cost': df['cost'].sum(),
                'daily_average': df.groupby('date')['cost'].sum().mean(),
                'cost_by_service': df.groupby('service')['cost'].sum().to_dict(),
                'cost_by_environment': df.groupby('environment')['cost'].sum().to_dict(),
                'cost_by_project': df.groupby('project')['cost'].sum().to_dict(),
                'growth_rate': self._calculate_growth_rate(df),
                'anomalies': self._detect_cost_anomalies(df),
                'forecasted_monthly_cost': self._forecast_monthly_cost(df)
            }
            
            return analysis
            
        except Exception as e:
            self.logger.error(f"Error analyzing cost trends: {e}")
            return {}
    
    def identify_rightsizing_opportunities(self) -> List[CostOptimizationRecommendation]:
        """Identify EC2 instances that can be rightsized."""
        recommendations = []
        
        try:
            # Get all running instances
            instances = self._get_running_instances()
            
            for instance in instances:
                instance_id = instance['InstanceId']
                instance_type = instance['InstanceType']
                
                # Get utilization metrics
                utilization = self._get_instance_utilization(instance_id)
                
                if not utilization:
                    continue
                
                # Calculate current costs
                current_cost = self._calculate_instance_cost(instance_type)
                
                # Determine optimization recommendation
                recommendation = self._analyze_instance_rightsizing(
                    instance_id, instance_type, utilization, current_cost
                )
                
                if recommendation and recommendation.savings >= self.min_savings_threshold:
                    recommendations.append(recommendation)
            
            return recommendations
            
        except Exception as e:
            self.logger.error(f"Error identifying rightsizing opportunities: {e}")
            return []
    
    def _get_running_instances(self) -> List[Dict]:
        """Get all running EC2 instances."""
        try:
            response = self.ec2_client.describe_instances(
                Filters=[
                    {'Name': 'instance-state-name', 'Values': ['running']}
                ]
            )
            
            instances = []
            for reservation in response['Reservations']:
                instances.extend(reservation['Instances'])
            
            return instances
            
        except Exception as e:
            self.logger.error(f"Error getting running instances: {e}")
            return []
    
    def _get_instance_utilization(self, instance_id: str, days: int = 14) -> Dict:
        """Get CloudWatch metrics for instance utilization."""
        end_time = datetime.utcnow()
        start_time = end_time - timedelta(days=days)
        
        try:
            # CPU utilization
            cpu_response = self.cloudwatch.get_metric_statistics(
                Namespace='AWS/EC2',
                MetricName='CPUUtilization',
                Dimensions=[
                    {'Name': 'InstanceId', 'Value': instance_id}
                ],
                StartTime=start_time,
                EndTime=end_time,
                Period=3600,  # 1 hour
                Statistics=['Average', 'Maximum']
            )
            
            # Memory utilization (if available via CloudWatch agent)
            memory_response = self.cloudwatch.get_metric_statistics(
                Namespace='CWAgent',
                MetricName='mem_used_percent',
                Dimensions=[
                    {'Name': 'InstanceId', 'Value': instance_id}
                ],
                StartTime=start_time,
                EndTime=end_time,
                Period=3600,
                Statistics=['Average', 'Maximum']
            )
            
            # Network utilization
            network_in_response = self.cloudwatch.get_metric_statistics(
                Namespace='AWS/EC2',
                MetricName='NetworkIn',
                Dimensions=[
                    {'Name': 'InstanceId', 'Value': instance_id}
                ],
                StartTime=start_time,
                EndTime=end_time,
                Period=3600,
                Statistics=['Average', 'Maximum']
            )
            
            # Process metrics
            cpu_datapoints = cpu_response.get('Datapoints', [])
            memory_datapoints = memory_response.get('Datapoints', [])
            network_datapoints = network_in_response.get('Datapoints', [])
            
            if not cpu_datapoints:
                return {}
            
            cpu_avg = np.mean([dp['Average'] for dp in cpu_datapoints])
            cpu_max = np.max([dp['Maximum'] for dp in cpu_datapoints])
            
            memory_avg = np.mean([dp['Average'] for dp in memory_datapoints]) if memory_datapoints else 0
            memory_max = np.max([dp['Maximum'] for dp in memory_datapoints]) if memory_datapoints else 0
            
            network_avg = np.mean([dp['Average'] for dp in network_datapoints])
            
            return {
                'cpu_average': cpu_avg,
                'cpu_maximum': cpu_max,
                'memory_average': memory_avg,
                'memory_maximum': memory_max,
                'network_average': network_avg,
                'evaluation_period_days': days,
                'datapoints_count': len(cpu_datapoints)
            }
            
        except Exception as e:
            self.logger.error(f"Error getting utilization for {instance_id}: {e}")
            return {}
    
    def _analyze_instance_rightsizing(
        self, 
        instance_id: str, 
        instance_type: str, 
        utilization: Dict, 
        current_cost: float
    ) -> Optional[CostOptimizationRecommendation]:
        """Analyze instance for rightsizing opportunities."""
        
        cpu_avg = utilization.get('cpu_average', 0)
        cpu_max = utilization.get('cpu_maximum', 0)
        memory_avg = utilization.get('memory_average', 0)
        
        # Determine if instance is under or over-utilized
        if cpu_avg < self.cpu_threshold_low and memory_avg < self.memory_threshold_low:
            # Instance is underutilized - recommend downsizing
            recommended_type = self._recommend_smaller_instance_type(instance_type)
            
            if recommended_type:
                recommended_cost = self._calculate_instance_cost(recommended_type)
                savings = current_cost - recommended_cost
                
                return CostOptimizationRecommendation(
                    resource_id=instance_id,
                    resource_type='EC2 Instance',
                    current_cost=current_cost,
                    optimized_cost=recommended_cost,
                    savings=savings,
                    savings_percentage=(savings / current_cost) * 100,
                    risk_level='Low',
                    implementation_effort='Medium',
                    description=f"Downsize from {instance_type} to {recommended_type}",
                    action_items=[
                        f"Stop instance {instance_id}",
                        f"Change instance type from {instance_type} to {recommended_type}",
                        "Start instance and validate performance",
                        "Monitor for 48 hours to ensure stability"
                    ],
                    impact_assessment={
                        'performance_impact': 'Minimal',
                        'availability_impact': 'Temporary during resize',
                        'application_changes_required': False
                    },
                    confidence_score=0.85
                )
        
        elif cpu_max > self.cpu_threshold_high or memory_avg > 85:
            # Instance may be over-utilized - recommend upsizing
            recommended_type = self._recommend_larger_instance_type(instance_type)
            
            if recommended_type:
                recommended_cost = self._calculate_instance_cost(recommended_type)
                cost_increase = recommended_cost - current_cost
                
                # Only recommend if performance gain justifies cost
                if cost_increase <= 500:  # Max $500/month increase
                    return CostOptimizationRecommendation(
                        resource_id=instance_id,
                        resource_type='EC2 Instance',
                        current_cost=current_cost,
                        optimized_cost=recommended_cost,
                        savings=-cost_increase,  # Negative savings (cost increase)
                        savings_percentage=-(cost_increase / current_cost) * 100,
                        risk_level='Medium',
                        implementation_effort='Medium',
                        description=f"Upsize from {instance_type} to {recommended_type} for better performance",
                        action_items=[
                            f"Schedule maintenance window for instance {instance_id}",
                            f"Change instance type from {instance_type} to {recommended_type}",
                            "Monitor performance improvements",
                            "Validate application response times"
                        ],
                        impact_assessment={
                            'performance_impact': 'Significant improvement expected',
                            'availability_impact': 'Temporary during resize',
                            'application_changes_required': False
                        },
                        confidence_score=0.75
                    )
        
        return None
    
    def identify_reserved_instance_opportunities(self) -> List[CostOptimizationRecommendation]:
        """Identify opportunities for Reserved Instance purchases."""
        recommendations = []
        
        try:
            # Get Reserved Instance recommendations from AWS
            response = self.cost_explorer.get_rightsizing_recommendation(
                Service='AmazonEC2',
                Configuration={
                    'BenefitsConsidered': True,
                    'RecommendationTarget': 'SAME_INSTANCE_FAMILY'
                }
            )
            
            # Get RI purchase recommendations
            ri_response = self.cost_explorer.get_reservation_purchase_recommendation(
                Service='AmazonEC2',
                LookbackPeriodInDays='SIXTY_DAYS',
                TermInYears='ONE_YEAR',
                PaymentOption='NO_UPFRONT'
            )
            
            for recommendation in ri_response.get('Recommendations', []):
                details = recommendation.get('RecommendationDetails', {})
                instance_details = details.get('InstanceDetails', {}).get('EC2InstanceDetails', {})
                
                estimated_monthly_savings = float(
                    recommendation.get('EstimatedMonthlySavingsAmount', 0)
                )
                
                if estimated_monthly_savings >= self.min_savings_threshold:
                    recommendations.append(
                        CostOptimizationRecommendation(
                            resource_id=f"RI-{instance_details.get('InstanceType', 'unknown')}",
                            resource_type='Reserved Instance',
                            current_cost=float(details.get('EstimatedMonthlyOnDemandCost', 0)),
                            optimized_cost=float(details.get('EstimatedMonthlySavingsAmount', 0)),
                            savings=estimated_monthly_savings,
                            savings_percentage=float(
                                recommendation.get('EstimatedMonthlySavingsPercentage', 0)
                            ),
                            risk_level='Low',
                            implementation_effort='Low',
                            description=f"Purchase Reserved Instance for {instance_details.get('InstanceType')}",
                            action_items=[
                                "Review utilization patterns",
                                "Purchase Reserved Instance",
                                "Apply RI to matching instances",
                                "Monitor savings realization"
                            ],
                            impact_assessment={
                                'performance_impact': 'None',
                                'availability_impact': 'None',
                                'financial_commitment': f"{details.get('UpfrontCost', 0)} upfront"
                            },
                            confidence_score=0.90
                        )
                    )
            
            return recommendations
            
        except Exception as e:
            self.logger.error(f"Error identifying RI opportunities: {e}")
            return []
    
    def generate_cost_optimization_report(self) -> Dict:
        """Generate comprehensive cost optimization report."""
        try:
            # Gather all optimization recommendations
            rightsizing_recommendations = self.identify_rightsizing_opportunities()
            ri_recommendations = self.identify_reserved_instance_opportunities()
            unused_resources = self._identify_unused_resources()
            storage_optimizations = self._identify_storage_optimizations()
            
            all_recommendations = (
                rightsizing_recommendations + 
                ri_recommendations + 
                unused_resources + 
                storage_optimizations
            )
            
            # Calculate totals
            total_potential_savings = sum(r.savings for r in all_recommendations if r.savings > 0)
            total_current_cost = sum(r.current_cost for r in all_recommendations)
            
            # Prioritize recommendations
            prioritized_recommendations = sorted(
                all_recommendations,
                key=lambda x: (x.savings, -x.risk_level.count('High')),
                reverse=True
            )
            
            # Generate summary
            report = {
                'generated_at': datetime.utcnow().isoformat(),
                'summary': {
                    'total_recommendations': len(all_recommendations),
                    'total_potential_monthly_savings': total_potential_savings,
                    'total_current_monthly_cost': total_current_cost,
                    'potential_savings_percentage': (total_potential_savings / total_current_monthly_cost * 100) if total_current_monthly_cost > 0 else 0,
                    'recommendations_by_type': {
                        'rightsizing': len(rightsizing_recommendations),
                        'reserved_instances': len(ri_recommendations),
                        'unused_resources': len(unused_resources),
                        'storage_optimization': len(storage_optimizations)
                    }
                },
                'recommendations': [
                    {
                        'resource_id': r.resource_id,
                        'resource_type': r.resource_type,
                        'current_cost': r.current_cost,
                        'optimized_cost': r.optimized_cost,
                        'monthly_savings': r.savings,
                        'savings_percentage': r.savings_percentage,
                        'risk_level': r.risk_level,
                        'implementation_effort': r.implementation_effort,
                        'description': r.description,
                        'action_items': r.action_items,
                        'confidence_score': r.confidence_score
                    }
                    for r in prioritized_recommendations[:50]  # Top 50 recommendations
                ],
                'cost_trends': self.analyze_cost_trends(),
                'tagging_compliance': self._analyze_tagging_compliance()
            }
            
            return report
            
        except Exception as e:
            self.logger.error(f"Error generating cost optimization report: {e}")
            return {}
    
    def _calculate_instance_cost(self, instance_type: str, region: str = 'us-west-2') -> float:
        """Calculate monthly cost for instance type."""
        # This would integrate with AWS Pricing API
        # Simplified pricing for demonstration
        pricing_table = {
            't3.micro': 9.50,
            't3.small': 18.98,
            't3.medium': 37.97,
            't3.large': 75.94,
            't3.xlarge': 151.87,
            'm5.large': 96.36,
            'm5.xlarge': 192.72,
            'm5.2xlarge': 385.44,
            'c5.large': 89.64,
            'c5.xlarge': 179.28,
            'r5.large': 126.14,
            'r5.xlarge': 252.29
        }
        
        return pricing_table.get(instance_type, 100.0)  # Default estimate

def main():
    """Main function for FinOps optimization automation."""
    config = {
        'aws_region': 'us-west-2',
        'cpu_threshold_low': 20,
        'cpu_threshold_high': 80,
        'memory_threshold_low': 30,
        'min_savings_threshold': 100
    }
    
    engine = FinOpsOptimizationEngine(config)
    
    # Generate optimization report
    report = engine.generate_cost_optimization_report()
    
    # Save report
    with open('cost_optimization_report.json', 'w') as f:
        json.dump(report, f, indent=2)
    
    print("Cost optimization report generated successfully!")
    print(f"Total potential monthly savings: ${report['summary']['total_potential_monthly_savings']:.2f}")
    print(f"Number of recommendations: {report['summary']['total_recommendations']}")

if __name__ == '__main__':
    main()