defmodule DSPex.Python.TelemetryEngine do
  @moduledoc """
  Central telemetry collection and processing engine for V3 pooler.
  
  Features:
  - Multi-dimensional metric collection
  - Real-time metric streaming
  - Metric aggregation and correlation
  - Event-driven telemetry
  - Performance impact minimization
  """
  
  use GenServer
  require Logger
  
  defstruct [
    :collectors,          # Active metric collectors
    :processors,          # Metric processing pipelines
    :exporters,          # Metric export destinations
    :buffer,             # Metric buffering for batch processing
    :sampling_rules,     # Sampling configuration to reduce overhead
    :correlation_engine  # Cross-metric correlation analysis
  ]
  
  @metric_dimensions [
    :pool_metrics,       # Pool-level performance metrics
    :worker_metrics,     # Individual worker performance
    :request_metrics,    # Request-level tracing
    :resource_metrics,   # System resource utilization
    :business_metrics,   # Business KPIs and outcomes
    :security_metrics    # Security and compliance metrics
  ]
  
  def start_link(opts) do
    GenServer.start_link(__MODULE__, opts, name: __MODULE__)
  end
  
  def emit_metric(dimension, metric_name, value, metadata \\ %{}) do
    metric = create_metric(dimension, metric_name, value, metadata)
    GenServer.cast(__MODULE__, {:emit_metric, metric})
  end
  
  def emit_event(event_type, event_data, context \\ %{}) do
    event = create_event(event_type, event_data, context)
    GenServer.cast(__MODULE__, {:emit_event, event})
  end
  
  def register_collector(collector_module, collector_config) do
    GenServer.call(__MODULE__, {:register_collector, collector_module, collector_config})
  end
  
  defp create_metric(dimension, name, value, metadata) do
    %{
      dimension: dimension,
      name: name,
      value: value,
      timestamp: System.system_time(:microsecond),
      metadata: Map.merge(metadata, %{
        node: Node.self(),
        pid: self(),
        trace_id: get_trace_id()
      })
    }
  end
end

defmodule DSPex.Python.MetricCollectors do
  @moduledoc """
  Specialized metric collectors for different aspects of the system.
  """
  
  defmodule PoolMetricsCollector do
    @moduledoc "Collects comprehensive pool-level metrics"
    
    use GenServer
    
    @collection_interval 5_000  # Collect every 5 seconds
    
    def start_link(pool_id) do
      GenServer.start_link(__MODULE__, pool_id, name: via_tuple(pool_id))
    end
    
    def handle_info(:collect_metrics, pool_id) do
      metrics = collect_comprehensive_pool_metrics(pool_id)
      
      Enum.each(metrics, fn {metric_name, value, metadata} ->
        DSPex.Python.TelemetryEngine.emit_metric(
          :pool_metrics, 
          metric_name, 
          value, 
          Map.put(metadata, :pool_id, pool_id)
        )
      end)
      
      # Schedule next collection
      Process.send_after(self(), :collect_metrics, @collection_interval)
      {:noreply, pool_id}
    end
    
    defp collect_comprehensive_pool_metrics(pool_id) do
      base_stats = DSPex.Python.Pool.get_stats(pool_id)
      workers = DSPex.Python.Pool.list_workers(pool_id)
      
      [
        # Core pool metrics
        {"pool.workers.total", base_stats.workers, %{type: :gauge}},
        {"pool.workers.available", base_stats.available, %{type: :gauge}},
        {"pool.workers.busy", base_stats.busy, %{type: :gauge}},
        {"pool.queue.length", length(base_stats.queued || []), %{type: :gauge}},
        
        # Performance metrics
        {"pool.requests.total", base_stats.requests, %{type: :counter}},
        {"pool.requests.errors", base_stats.errors, %{type: :counter}},
        {"pool.requests.rate", calculate_request_rate(pool_id), %{type: :gauge}},
        {"pool.response_time.avg", calculate_avg_response_time(pool_id), %{type: :gauge}},
        {"pool.response_time.p95", calculate_p95_response_time(pool_id), %{type: :gauge}},
        {"pool.response_time.p99", calculate_p99_response_time(pool_id), %{type: :gauge}},
        
        # Utilization metrics
        {"pool.utilization.current", base_stats.busy / max(base_stats.workers, 1), %{type: :gauge}},
        {"pool.utilization.peak_1h", get_peak_utilization(pool_id, 3600), %{type: :gauge}},
        {"pool.efficiency.worker", calculate_worker_efficiency(pool_id), %{type: :gauge}},
        
        # Health metrics
        {"pool.health.score", calculate_pool_health_score(pool_id), %{type: :gauge}},
        {"pool.workers.healthy", count_healthy_workers(workers), %{type: :gauge}},
        {"pool.workers.degraded", count_degraded_workers(workers), %{type: :gauge}},
        
        # Advanced metrics
        {"pool.session_affinity.hit_rate", calculate_affinity_hit_rate(pool_id), %{type: :gauge}},
        {"pool.load_balancing.distribution_score", calculate_distribution_score(pool_id), %{type: :gauge}}
      ]
    end
  end
  
  defmodule WorkerMetricsCollector do
    @moduledoc "Collects detailed worker-level metrics"
    
    def collect_worker_metrics(worker_id) do
      health_data = DSPex.Python.ResourceMonitor.check_health(worker_id)
      resource_usage = DSPex.Python.ResourceMonitor.get_resource_usage(worker_id)
      
      [
        # Resource metrics
        {"worker.memory.usage", resource_usage.memory_usage, %{worker_id: worker_id, unit: :mb}},
        {"worker.memory.peak", resource_usage.memory_peak, %{worker_id: worker_id, unit: :mb}},
        {"worker.cpu.usage", resource_usage.cpu_usage, %{worker_id: worker_id, unit: :percent}},
        {"worker.uptime", resource_usage.uptime, %{worker_id: worker_id, unit: :seconds}},
        
        # Health metrics
        {"worker.health.score", health_data.score, %{worker_id: worker_id}},
        {"worker.health.state", encode_health_state(health_data.state), %{worker_id: worker_id}},
        
        # Performance metrics
        {"worker.requests.processed", get_worker_request_count(worker_id), %{worker_id: worker_id}},
        {"worker.requests.active", get_worker_active_requests(worker_id), %{worker_id: worker_id}},
        {"worker.response_time.avg", get_worker_avg_response_time(worker_id), %{worker_id: worker_id}},
        
        # API-specific metrics
        {"worker.api_calls.total", resource_usage.api_calls_count, %{worker_id: worker_id}},
        {"worker.api_calls.rate", resource_usage.api_calls_rate, %{worker_id: worker_id}},
        {"worker.api_errors.count", get_worker_api_errors(worker_id), %{worker_id: worker_id}}
      ]
    end
  end
  
  defmodule RequestMetricsCollector do
    @moduledoc "Collects request-level tracing and performance metrics"
    
    def trace_request(request_id, pool_id, command, args) do
      trace_context = %{
        request_id: request_id,
        pool_id: pool_id,
        command: command,
        args_size: calculate_args_size(args),
        started_at: System.system_time(:microsecond),
        trace_id: generate_trace_id()
      }
      
      # Emit request start event
      DSPex.Python.TelemetryEngine.emit_event(
        :request_started,
        %{request_id: request_id, command: command},
        trace_context
      )
      
      trace_context
    end
    
    def complete_request_trace(trace_context, result, worker_id) do
      completed_at = System.system_time(:microsecond)
      duration = completed_at - trace_context.started_at
      
      # Emit detailed request metrics
      DSPex.Python.TelemetryEngine.emit_metric(
        :request_metrics,
        "request.duration",
        duration,
        Map.merge(trace_context, %{
          worker_id: worker_id,
          result_type: classify_result(result),
          completed_at: completed_at
        })
      )
      
      # Emit request completion event
      DSPex.Python.TelemetryEngine.emit_event(
        :request_completed,
        %{
          request_id: trace_context.request_id,
          duration: duration,
          success: is_success(result)
        },
        trace_context
      )
    end
  end
end

defmodule DSPex.Python.PerformanceAnalytics do
  @moduledoc """
  Real-time performance analysis and pattern detection.
  """
  
  use GenServer
  
  defstruct [
    :metric_buffer,       # Sliding window of recent metrics
    :pattern_detectors,   # Pattern detection algorithms
    :anomaly_detectors,   # Anomaly detection systems
    :trend_analyzers,     # Trend analysis engines
    :correlation_matrix,  # Cross-metric correlation tracking
    :insights_cache      # Cached insights and recommendations
  ]
  
  def start_link(opts) do
    GenServer.start_link(__MODULE__, opts, name: __MODULE__)
  end
  
  def get_real_time_insights(pool_id) do
    GenServer.call(__MODULE__, {:get_insights, pool_id})
  end
  
  def detect_performance_anomalies(pool_id, time_window \\ 3600) do
    GenServer.call(__MODULE__, {:detect_anomalies, pool_id, time_window})
  end
  
  def analyze_performance_trends(pool_id, metric_name, lookback_hours \\ 24) do
    GenServer.call(__MODULE__, {:analyze_trends, pool_id, metric_name, lookback_hours})
  end
  
  # Real-time insight generation
  defp generate_real_time_insights(pool_id) do
    current_metrics = get_current_pool_metrics(pool_id)
    historical_metrics = get_historical_metrics(pool_id, 3600)  # Last hour
    
    insights = %{
      performance_score: calculate_performance_score(current_metrics),
      efficiency_insights: analyze_efficiency(current_metrics, historical_metrics),
      capacity_insights: analyze_capacity(current_metrics, historical_metrics),
      cost_insights: analyze_cost_efficiency(current_metrics, historical_metrics),
      optimization_opportunities: identify_optimization_opportunities(pool_id),
      predicted_issues: predict_potential_issues(pool_id),
      recommendations: generate_recommendations(pool_id)
    }
    
    insights
  end
  
  defp analyze_efficiency(current_metrics, historical_metrics) do
    %{
      current_efficiency: calculate_current_efficiency(current_metrics),
      efficiency_trend: calculate_efficiency_trend(historical_metrics),
      efficiency_percentile: calculate_efficiency_percentile(current_metrics),
      bottlenecks: identify_efficiency_bottlenecks(current_metrics),
      improvement_potential: estimate_improvement_potential(current_metrics)
    }
  end
  
  defp identify_optimization_opportunities(pool_id) do
    opportunities = []
    
    # Worker utilization optimization
    if worker_utilization_imbalanced?(pool_id) do
      opportunities = [
        %{
          type: :worker_rebalancing,
          description: "Worker load is imbalanced, consider rebalancing",
          potential_improvement: "10-15% efficiency gain",
          priority: :medium
        } | opportunities
      ]
    end
    
    # Resource allocation optimization
    if resource_allocation_suboptimal?(pool_id) do
      opportunities = [
        %{
          type: :resource_reallocation,
          description: "Resource allocation can be optimized",
          potential_improvement: "20-25% cost reduction",
          priority: :high
        } | opportunities
      ]
    end
    
    # Scaling optimization
    if scaling_pattern_suboptimal?(pool_id) do
      opportunities = [
        %{
          type: :scaling_optimization,
          description: "Scaling patterns can be improved",
          potential_improvement: "30% faster scaling response",
          priority: :medium
        } | opportunities
      ]
    end
    
    opportunities
  end
  
  defp predict_potential_issues(pool_id) do
    predictions = []
    
    # Memory leak prediction
    if memory_trend_concerning?(pool_id) do
      time_to_issue = estimate_memory_issue_time(pool_id)
      predictions = [
        %{
          issue_type: :memory_leak,
          severity: :warning,
          estimated_time: time_to_issue,
          description: "Memory usage trending upward, potential leak detected",
          recommended_action: "Monitor closely, prepare for worker restart"
        } | predictions
      ]
    end
    
    # Performance degradation prediction
    if performance_degradation_trend?(pool_id) do
      predictions = [
        %{
          issue_type: :performance_degradation,
          severity: :info,
          estimated_time: 3600,  # 1 hour
          description: "Performance slowly degrading, may need intervention",
          recommended_action: "Consider worker refresh or load reduction"
        } | predictions
      ]
    end
    
    # Capacity saturation prediction
    if approaching_capacity_limit?(pool_id) do
      time_to_saturation = estimate_saturation_time(pool_id)
      predictions = [
        %{
          issue_type: :capacity_saturation,
          severity: :critical,
          estimated_time: time_to_saturation,
          description: "Approaching capacity limits based on current trend",
          recommended_action: "Scale up proactively or implement load shedding"
        } | predictions
      ]
    end
    
    predictions
  end
end

defmodule DSPex.Python.AnomalyDetection do
  @moduledoc """
  ML-based anomaly detection for pool and worker behavior.
  """
  
  defstruct [
    :detectors,          # Map of metric_name -> detector_config
    :baselines,          # Normal behavior baselines
    :sensitivity_settings, # Detection sensitivity per metric
    :alert_thresholds,   # Alerting thresholds for different anomaly types
    :correlation_rules   # Rules for correlating multiple anomalies
  ]
  
  @anomaly_types [
    :statistical_outlier,    # Values outside statistical norms
    :trend_anomaly,         # Unusual trends or patterns
    :seasonal_anomaly,      # Deviations from expected seasonal patterns
    :correlation_anomaly,   # Unusual correlations between metrics
    :threshold_breach,      # Hard threshold violations
    :rate_of_change_anomaly # Unusual rate of change in metrics
  ]
  
  def detect_anomalies(metric_stream, detection_window \\ 300) do
    # Apply multiple anomaly detection algorithms
    detections = %{
      statistical: detect_statistical_anomalies(metric_stream),
      trend: detect_trend_anomalies(metric_stream, detection_window),
      seasonal: detect_seasonal_anomalies(metric_stream),
      correlation: detect_correlation_anomalies(metric_stream),
      threshold: detect_threshold_breaches(metric_stream),
      rate_change: detect_rate_change_anomalies(metric_stream)
    }
    
    # Correlate and prioritize anomalies
    correlated_anomalies = correlate_anomalies(detections)
    prioritized_anomalies = prioritize_anomalies(correlated_anomalies)
    
    prioritized_anomalies
  end
  
  defp detect_statistical_anomalies(metric_stream) do
    # Use z-score and modified z-score for outlier detection
    Enum.flat_map(metric_stream, fn {metric_name, values} ->
      outliers = identify_statistical_outliers(values)
      
      Enum.map(outliers, fn {timestamp, value, z_score} ->
        %{
          type: :statistical_outlier,
          metric: metric_name,
          timestamp: timestamp,
          value: value,
          severity: classify_outlier_severity(z_score),
          confidence: calculate_confidence(z_score),
          description: "Statistical outlier detected (z-score: #{Float.round(z_score, 2)})"
        }
      end)
    end)
  end
  
  defp detect_trend_anomalies(metric_stream, window_size) do
    Enum.flat_map(metric_stream, fn {metric_name, values} ->
      # Calculate trend strength and direction
      trend_analysis = analyze_trend(values, window_size)
      
      case detect_unusual_trend(trend_analysis) do
        {:anomaly, anomaly_data} ->
          [%{
            type: :trend_anomaly,
            metric: metric_name,
            trend_direction: anomaly_data.direction,
            trend_strength: anomaly_data.strength,
            severity: classify_trend_severity(anomaly_data),
            description: "Unusual trend detected: #{anomaly_data.description}"
          }]
          
        :normal ->
          []
      end
    end)
  end
  
  defp correlate_anomalies(detections) do
    # Look for patterns across different anomaly types and metrics
    all_anomalies = Enum.flat_map(detections, fn {_type, anomalies} -> anomalies end)
    
    # Group anomalies by time windows
    time_grouped = group_anomalies_by_time(all_anomalies, 60)  # 60-second windows
    
    # Find correlations within time windows
    Enum.flat_map(time_grouped, fn {_time_window, anomalies} ->
      find_anomaly_correlations(anomalies)
    end)
  end
  
  defp find_anomaly_correlations(anomalies) do
    correlations = []
    
    # Check for resource correlation (CPU + Memory anomalies)
    if has_resource_anomaly_pattern?(anomalies) do
      correlations = [create_resource_correlation_alert(anomalies) | correlations]
    end
    
    # Check for cascading failure pattern
    if has_cascading_failure_pattern?(anomalies) do
      correlations = [create_cascading_failure_alert(anomalies) | correlations]
    end
    
    # Check for load balancing issues
    if has_load_balancing_anomaly_pattern?(anomalies) do
      correlations = [create_load_balancing_alert(anomalies) | correlations]
    end
    
    correlations
  end
end

defmodule DSPex.Python.SmartAlertManager do
  @moduledoc """
  Intelligent alerting system with context-aware notifications.
  """
  
  use GenServer
  
  defstruct [
    :alert_rules,        # Configurable alert rules
    :notification_channels, # Available notification channels
    :alert_history,      # Historical alert data
    :suppression_rules,  # Alert suppression and deduplication
    :escalation_policies, # Alert escalation configuration
    :context_engine     # Context-aware alert enrichment
  ]
  
  @alert_severities [:info, :warning, :critical, :emergency]
  @notification_channels [:email, :slack, :pagerduty, :webhook, :sms]
  
  def start_link(opts) do
    GenServer.start_link(__MODULE__, opts, name: __MODULE__)
  end
  
  def trigger_alert(alert_type, alert_data, context \\ %{}) do
    GenServer.cast(__MODULE__, {:trigger_alert, alert_type, alert_data, context})
  end
  
  def configure_alert_rule(rule_name, rule_config) do
    GenServer.call(__MODULE__, {:configure_rule, rule_name, rule_config})
  end
  
  def handle_cast({:trigger_alert, alert_type, alert_data, context}, state) do
    # Enrich alert with context
    enriched_alert = enrich_alert_with_context(alert_type, alert_data, context)
    
    # Apply suppression rules
    case should_suppress_alert?(enriched_alert, state.alert_history) do
      true ->
        Logger.debug("Alert suppressed: #{alert_type}")
        {:noreply, state}
        
      false ->
        # Process and route alert
        processed_alert = process_alert(enriched_alert, state)
        route_alert(processed_alert, state.notification_channels)
        
        # Update alert history
        new_history = update_alert_history(state.alert_history, processed_alert)
        {:noreply, %{state | alert_history: new_history}}
    end
  end
  
  defp enrich_alert_with_context(alert_type, alert_data, context) do
    base_alert = %{
      type: alert_type,
      data: alert_data,
      timestamp: System.system_time(:second),
      context: context,
      id: generate_alert_id()
    }
    
    # Add contextual information
    enriched_context = Map.merge(context, %{
      system_load: get_current_system_load(),
      pool_states: get_all_pool_states(),
      recent_events: get_recent_system_events(300),  # Last 5 minutes
      deployment_info: get_deployment_context()
    })
    
    %{base_alert | context: enriched_context}
  end
  
  defp process_alert(alert, state) do
    # Determine severity based on alert type and context
    severity = determine_alert_severity(alert)
    
    # Generate human-readable description
    description = generate_alert_description(alert)
    
    # Suggest remediation actions
    remediation = suggest_remediation_actions(alert)
    
    # Calculate urgency score
    urgency_score = calculate_urgency_score(alert)
    
    %{
      alert |
      severity: severity,
      description: description,
      remediation: remediation,
      urgency_score: urgency_score,
      processed_at: System.system_time(:second)
    }
  end
  
  defp suggest_remediation_actions(alert) do
    case alert.type do
      :pool_overload ->
        [
          "Scale up the affected pool by 2-3 workers",
          "Check for memory leaks in workers",
          "Consider implementing request throttling",
          "Review recent deployment changes"
        ]
        
      :worker_degradation ->
        [
          "Restart the affected worker",
          "Check worker resource usage",
          "Review worker logs for errors",
          "Monitor for worker recovery"
        ]
        
      :memory_pressure ->
        [
          "Trigger garbage collection on workers",
          "Scale down non-critical pools",
          "Clear session caches",
          "Check for memory leaks"
        ]
        
      :performance_degradation ->
        [
          "Check system resource availability",
          "Review recent configuration changes",
          "Analyze request patterns for anomalies",
          "Consider worker refresh"
        ]
        
      _ ->
        ["Review system logs", "Check pool health status", "Contact support if issue persists"]
    end
  end
  
  defp route_alert(alert, notification_channels) do
    # Route based on severity and type
    channels = select_notification_channels(alert, notification_channels)
    
    Enum.each(channels, fn channel ->
      send_notification(alert, channel)
    end)
  end
  
  defp select_notification_channels(alert, available_channels) do
    case alert.severity do
      :emergency ->
        [:pagerduty, :sms, :slack]
        
      :critical ->
        [:pagerduty, :slack, :email]
        
      :warning ->
        [:slack, :email]
        
      :info ->
        [:email]
    end
    |> Enum.filter(fn channel -> channel in available_channels end)
  end
end

defmodule DSPex.Python.ContextAwareNotifications do
  @moduledoc """
  Generates context-rich notifications with actionable insights.
  """
  
  def generate_notification(alert, channel_type) do
    case channel_type do
      :slack ->
        generate_slack_notification(alert)
        
      :email ->
        generate_email_notification(alert)
        
      :pagerduty ->
        generate_pagerduty_notification(alert)
        
      :webhook ->
        generate_webhook_notification(alert)
    end
  end
  
  defp generate_slack_notification(alert) do
    %{
      text: "🚨 DSPex Pool Alert: #{alert.description}",
      attachments: [
        %{
          color: severity_color(alert.severity),
          fields: [
            %{
              title: "Severity",
              value: String.upcase(to_string(alert.severity)),
              short: true
            },
            %{
              title: "Pool ID",
              value: alert.context.pool_id || "Multiple pools",
              short: true
            },
            %{
              title: "Alert Type",
              value: humanize_alert_type(alert.type),
              short: true
            },
            %{
              title: "Urgency Score",
              value: "#{alert.urgency_score}/100",
              short: true
            }
          ],
          actions: generate_slack_actions(alert)
        },
        %{
          title: "📊 Current System State",
          text: format_system_state(alert.context),
          color: "good"
        },
        %{
          title: "🔧 Suggested Actions",
          text: format_remediation_actions(alert.remediation),
          color: "warning"
        }
      ]
    }
  end
  
  defp generate_slack_actions(alert) do
    base_actions = [
      %{
        type: "button",
        text: "View Dashboard",
        url: generate_dashboard_url(alert.context.pool_id)
      },
      %{
        type: "button",
        text: "View Logs",
        url: generate_logs_url(alert.context.pool_id, alert.timestamp)
      }
    ]
    
    # Add alert-specific actions
    specific_actions = case alert.type do
      :pool_overload ->
        [%{
          type: "button",
          text: "Scale Up Pool",
          name: "scale_up",
          value: alert.context.pool_id,
          style: "primary"
        }]
        
      :worker_degradation ->
        [%{
          type: "button",
          text: "Restart Worker",
          name: "restart_worker",
          value: alert.context.worker_id,
          style: "danger"
        }]
        
      _ ->
        []
    end
    
    base_actions ++ specific_actions
  end
  
  defp format_system_state(context) do
    pool_states = context.pool_states || %{}
    
    summary = Enum.map(pool_states, fn {pool_id, state} ->
      "• #{pool_id}: #{state.workers} workers, #{Float.round(state.utilization * 100, 1)}% util"
    end)
    |> Enum.join("\n")
    
    system_load = context.system_load || %{}
    
    """
    *Pool States:*
    #{summary}
    
    *System Load:*
    • CPU: #{Float.round((system_load.cpu || 0) * 100, 1)}%
    • Memory: #{Float.round((system_load.memory || 0) * 100, 1)}%
    • Active Requests: #{system_load.active_requests || 0}
    """
  end
  
  defp generate_email_notification(alert) do
    %{
      subject: "DSPex Pool Alert: #{alert.description}",
      html_body: generate_email_html(alert),
      text_body: generate_email_text(alert),
      priority: email_priority(alert.severity)
    }
  end
  
  defp generate_email_html(alert) do
    """
    <!DOCTYPE html>
    <html>
    <head>
        <style>
            .alert-header { background-color: #{severity_color(alert.severity)}; color: white; padding: 20px; }
            .alert-content { padding: 20px; }
            .metrics-table { border-collapse: collapse; width: 100%; }
            .metrics-table th, .metrics-table td { border: 1px solid #ddd; padding: 8px; text-align: left; }
            .actions-list { background-color: #f9f9f9; padding: 15px; margin: 10px 0; }
        </style>
    </head>
    <body>
        <div class="alert-header">
            <h1>🚨 DSPex Pool Alert</h1>
            <h2>#{alert.description}</h2>
        </div>
        
        <div class="alert-content">
            <h3>Alert Details</h3>
            <table class="metrics-table">
                <tr><th>Property</th><th>Value</th></tr>
                <tr><td>Severity</td><td>#{String.upcase(to_string(alert.severity))}</td></tr>
                <tr><td>Alert Type</td><td>#{humanize_alert_type(alert.type)}</td></tr>
                <tr><td>Pool ID</td><td>#{alert.context.pool_id || "Multiple pools"}</td></tr>
                <tr><td>Timestamp</td><td>#{format_timestamp(alert.timestamp)}</td></tr>
                <tr><td>Urgency Score</td><td>#{alert.urgency_score}/100</td></tr>
            </table>
            
            <h3>System State</h3>
            <pre>#{format_system_state(alert.context)}</pre>
            
            <div class="actions-list">
                <h3>🔧 Recommended Actions</h3>
                <ul>
                    #{Enum.map(alert.remediation, fn action -> "<li>#{action}</li>" end) |> Enum.join("")}
                </ul>
            </div>
            
            <p>
                <a href="#{generate_dashboard_url(alert.context.pool_id)}">View Dashboard</a> |
                <a href="#{generate_logs_url(alert.context.pool_id, alert.timestamp)}">View Logs</a>
            </p>
        </div>
    </body>
    </html>
    """
  end
end

defmodule DSPex.Python.PerformanceDashboard do
  @moduledoc """
  Real-time dashboard for pool performance monitoring.
  """
  
  def get_dashboard_data(pool_id \\ :all, time_range \\ 3600) do
    %{
      overview: get_overview_metrics(pool_id),
      performance: get_performance_metrics(pool_id, time_range),
      health: get_health_metrics(pool_id),
      resources: get_resource_metrics(pool_id),
      scaling: get_scaling_metrics(pool_id, time_range),
      alerts: get_active_alerts(pool_id),
      insights: get_dashboard_insights(pool_id),
      recommendations: get_optimization_recommendations(pool_id)
    }
  end
  
  defp get_overview_metrics(pool_id) do
    pools = get_pools_for_dashboard(pool_id)
    
    aggregate_stats = Enum.reduce(pools, %{}, fn pool, acc ->
      stats = DSPex.Python.Pool.get_stats(pool)
      
      %{
        total_workers: (acc[:total_workers] || 0) + stats.workers,
        available_workers: (acc[:available_workers] || 0) + stats.available,
        busy_workers: (acc[:busy_workers] || 0) + stats.busy,
        total_requests: (acc[:total_requests] || 0) + stats.requests,
        total_errors: (acc[:total_errors] || 0) + stats.errors,
        queue_length: (acc[:queue_length] || 0) + length(stats.queued || [])
      }
    end)
    
    %{
      aggregate_stats: aggregate_stats,
      overall_utilization: aggregate_stats.busy_workers / max(aggregate_stats.total_workers, 1),
      error_rate: aggregate_stats.total_errors / max(aggregate_stats.total_requests, 1),
      pools_count: length(pools),
      healthy_pools: count_healthy_pools(pools),
      system_health_score: calculate_system_health_score(pools)
    }
  end
  
  defp get_performance_metrics(pool_id, time_range) do
    metrics = fetch_time_series_metrics(pool_id, time_range)
    
    %{
      request_rate_timeline: extract_timeline(metrics, "pool.requests.rate"),
      response_time_timeline: extract_timeline(metrics, "pool.response_time.avg"),
      utilization_timeline: extract_timeline(metrics, "pool.utilization.current"),
      error_rate_timeline: calculate_error_rate_timeline(metrics),
      throughput_timeline: extract_timeline(metrics, "pool.throughput"),
      
      # Performance percentiles
      response_time_percentiles: %{
        p50: calculate_percentile(metrics["pool.response_time.avg"], 50),
        p95: calculate_percentile(metrics["pool.response_time.avg"], 95),
        p99: calculate_percentile(metrics["pool.response_time.avg"], 99)
      },
      
      # Performance comparison
      performance_vs_baseline: compare_to_baseline(metrics),
      performance_trend: calculate_performance_trend(metrics)
    }
  end
  
  defp get_health_metrics(pool_id) do
    pools = get_pools_for_dashboard(pool_id)
    
    health_data = Enum.map(pools, fn pool ->
      workers = DSPex.Python.Pool.list_workers(pool)
      
      worker_health = Enum.map(workers, fn worker_id ->
        health = DSPex.Python.ResourceMonitor.check_health(worker_id)
        usage = DSPex.Python.ResourceMonitor.get_resource_usage(worker_id)
        
        %{
          worker_id: worker_id,
          health_score: health.score,
          health_state: health.state,
          memory_usage: usage.memory_usage,
          cpu_usage: usage.cpu_usage,
          uptime: usage.uptime,
          warnings: usage.resource_warnings
        }
      end)
      
      %{
        pool_id: pool,
        pool_health_score: calculate_pool_health_score(pool),
        workers: worker_health,
        health_distribution: calculate_health_distribution(worker_health)
      }
    end)
    
    %{
      pools: health_data,
      overall_health: calculate_overall_health(health_data),
      health_trends: get_health_trends(pool_id),
      degraded_workers: find_degraded_workers(health_data),
      critical_issues: find_critical_health_issues(health_data)
    }
  end
  
  defp get_dashboard_insights(pool_id) do
    current_insights = DSPex.Python.PerformanceAnalytics.get_real_time_insights(pool_id)
    
    %{
      current_insights: current_insights,
      key_performance_indicators: extract_kpis(current_insights),
      efficiency_summary: summarize_efficiency(current_insights),
      capacity_analysis: analyze_capacity_status(current_insights),
      cost_analysis: analyze_cost_efficiency(current_insights)
    }
  end
end

defmodule DSPex.Python.AnalyticsDashboard do
  @moduledoc """
  Advanced analytics dashboard with interactive visualizations.
  """
  
  def get_analytics_data(analysis_type, params \\ %{}) do
    case analysis_type do
      :performance_analysis ->
        get_performance_analysis(params)
        
      :cost_analysis ->
        get_cost_analysis(params)
        
      :capacity_planning ->
        get_capacity_planning_analysis(params)
        
      :optimization_analysis ->
        get_optimization_analysis(params)
        
      :historical_trends ->
        get_historical_trends_analysis(params)
    end
  end
  
  defp get_performance_analysis(params) do
    pool_id = params[:pool_id]
    time_range = params[:time_range] || 86400  # 24 hours
    
    %{
      performance_summary: generate_performance_summary(pool_id, time_range),
      bottleneck_analysis: analyze_bottlenecks(pool_id, time_range),
      correlation_matrix: generate_correlation_matrix(pool_id, time_range),
      performance_heatmap: generate_performance_heatmap(pool_id, time_range),
      optimization_opportunities: identify_performance_optimizations(pool_id)
    }
  end
  
  defp get_cost_analysis(params) do
    time_range = params[:time_range] || 86400
    
    %{
      cost_breakdown: calculate_cost_breakdown(time_range),
      cost_trends: get_cost_trends(time_range),
      efficiency_metrics: calculate_cost_efficiency_metrics(time_range),
      waste_analysis: analyze_resource_waste(time_range),
      cost_optimization_suggestions: suggest_cost_optimizations()
    }
  end
  
  defp get_capacity_planning_analysis(params) do
    forecast_horizon = params[:forecast_days] || 30
    
    %{
      current_capacity: assess_current_capacity(),
      capacity_forecast: forecast_capacity_needs(forecast_horizon),
      growth_projections: calculate_growth_projections(forecast_horizon),
      scaling_recommendations: generate_scaling_recommendations(),
      capacity_alerts: identify_capacity_risks(forecast_horizon)
    }
  end
end

# config/config.exs
config :dspex, DSPex.Python.Monitoring,
  # Telemetry settings
  telemetry: %{
    enabled: true,
    collection_interval: 5_000,     # 5 seconds
    metric_retention: 604_800,      # 7 days
    high_frequency_metrics: [:utilization, :response_time, :queue_length],
    sampling_rate: 1.0              # 100% sampling for critical metrics
  },
  
  # Analytics settings
  analytics: %{
    anomaly_detection: true,
    pattern_recognition: true,
    predictive_analytics: true,
    correlation_analysis: true,
    real_time_insights: true
  },
  
  # Alerting configuration
  alerting: %{
    enabled: true,
    notification_channels: [:slack, :email, :webhook],
    severity_thresholds: %{
      memory_usage: %{warning: 70, critical: 85, emergency: 95},
      cpu_usage: %{warning: 75, critical: 90, emergency: 98},
      utilization: %{warning: 80, critical: 95, emergency: 99},
      error_rate: %{warning: 0.05, critical: 0.1, emergency: 0.2}
    },
    suppression_rules: %{
      duplicate_window: 300,        # 5 minutes
      escalation_delay: 900,        # 15 minutes
      auto_resolve: true
    }
  },
  
  # Dashboard settings
  dashboard: %{
    refresh_interval: 10_000,       # 10 seconds
    data_retention: 2_592_000,      # 30 days
    real_time_updates: true,
    export_formats: [:json, :csv, :pdf]
  }