# Manual cluster setup required for distributed debugging
config :libcluster, topologies: [...]
# Manual service discovery setup
# Manual cross-node communication setup

# config/dev.exs
config :foundation, cluster: true  # That's it!

# ElixirScope automatically gets distributed debugging
defmodule ElixirScope.DistributedTracer do
  def start_cluster_trace(trace_id, scope) do
    # Foundation.ProcessManager provides singleton trace coordinator
    {:ok, coordinator} = Foundation.ProcessManager.start_singleton(
      ElixirScope.TraceCoordinator,
      [trace_id: trace_id, scope: scope],
      name: {:trace_coordinator, trace_id}
    )
    
    # Foundation.Channels broadcasts trace start to all nodes
    Foundation.Channels.broadcast(:control, {:start_trace, trace_id, scope})
    
    coordinator
  end
end

defmodule ElixirScope.DistributedContext do
  @moduledoc """
  Leverages Foundation.Channels for automatic trace context propagation.
  
  Uses Foundation's :control channel for high-priority trace coordination
  and :telemetry channel for trace data collection.
  """
  
  def propagate_trace_context(trace_id, operation, target) do
    # Foundation.Channels intelligently routes based on priority
    Foundation.Channels.route_message(
      {:trace_context, trace_id, operation, build_context()},
      priority: :high  # Uses :control channel automatically
    )
    
    # Direct Horde access for complex trace metadata
    Horde.Registry.register(
      Foundation.ProcessRegistry,
      {:trace_metadata, trace_id},
      %{
        operation: operation,
        started_at: System.system_time(:microsecond),
        node: Node.self(),
        process: self()
      }
    )
  end
  
  def collect_trace_data(trace_id) do
    # Use Foundation.ServiceMesh to find all trace participants
    trace_services = Foundation.ServiceMesh.discover_services(
      capabilities: [:trace_participant],
      metadata: %{trace_id: trace_id}
    )

    # Collect trace data from all participants
    tasks = Enum.map(trace_services, fn service ->
      Task.async(fn ->
        GenServer.call(service.pid, {:get_trace_data, trace_id}, 10_000)
      end)
    end)
    
    # Aggregate results with timeout handling
    trace_data = Task.await_many(tasks, 15_000)
    correlate_and_merge_trace_data(trace_data)
  end
  
  defp build_context() do
    %{
      process: self(),
      node: Node.self(),
      timestamp: System.system_time(:microsecond),
      call_stack: Process.info(self(), :current_stacktrace)
    }
  end
  
  defp correlate_and_merge_trace_data(trace_data_list) do
    # Merge trace data from all nodes, handling duplicates and timing
    trace_data_list
    |> Enum.reject(&is_nil/1)
    |> Enum.reduce(%{events: [], timeline: [], errors: []}, fn data, acc ->
      %{
        events: acc.events ++ data.events,
        timeline: merge_timeline(acc.timeline, data.timeline),
        errors: acc.errors ++ data.errors
      }
    end)
    |> sort_timeline_events()
  end
end

defmodule ElixirScope.ClusterProfiler do
  @moduledoc """
  Distributed performance profiling using Foundation's infrastructure.
  
  Demonstrates how ElixirScope can leverage Foundation's process distribution
  and health monitoring for comprehensive cluster-wide performance analysis.
  """
  
  def start_cluster_profiling(duration_ms, profile_opts \\ []) do
    # Use Foundation to start profiling processes on all nodes
    profiler_results = Foundation.ProcessManager.start_replicated(
      ElixirScope.NodeProfiler,
      [duration: duration_ms, opts: profile_opts],
      name: :cluster_profiler
    )
    
    # Coordinate profiling start across cluster
    Foundation.Channels.broadcast(:control, {:start_profiling, duration_ms, System.system_time(:millisecond)})
    
    # Schedule data collection
    collection_time = duration_ms + 1000
    Process.send_after(self(), {:collect_profile_data, profiler_results}, collection_time)
    
    {:ok, %{profiler_pids: profiler_results, collection_time: collection_time}}
  end
  
  def analyze_cluster_performance() do
    # Leverage Foundation.HealthMonitor for performance baselines
    cluster_health = Foundation.HealthMonitor.get_cluster_health()
    
    # Get detailed performance metrics from each node
    node_metrics = cluster_health.cluster.node_health
    |> Enum.map(&collect_node_performance_details/1)
    |> Enum.reject(&is_nil/1)
    
    # Combine with ElixirScope's AST analysis for intelligent insights
    %{
      cluster_overview: cluster_health.performance,
      node_details: node_metrics,
      performance_bottlenecks: identify_bottlenecks(node_metrics),
      optimization_suggestions: generate_optimization_suggestions(node_metrics),
      code_hotspots: analyze_code_hotspots(node_metrics)
    }
  end
  
  defp collect_node_performance_details(node_info) do
    case node_info.status do
      :healthy ->
        case :rpc.call(node_info.node, ElixirScope.NodeProfiler, :get_detailed_metrics, [], 10_000) do
          {:badrpc, reason} -> 
            %{node: node_info.node, error: reason, status: :rpc_failed}
          metrics -> 
            %{node: node_info.node, status: :success, metrics: metrics}
        end
      
      status ->
        %{node: node_info.node, status: status, metrics: nil}
    end
  end
  
  defp identify_bottlenecks(node_metrics) do
    # Analyze metrics to identify performance bottlenecks
    successful_metrics = Enum.filter(node_metrics, &(&1.status == :success))
    
    bottlenecks = []
    
    # CPU bottlenecks
    high_cpu_nodes = Enum.filter(successful_metrics, fn %{metrics: m} ->
      Map.get(m, :cpu_usage, 0) > 80
    end)
    
    bottlenecks = if length(high_cpu_nodes) > 0 do
      [%{type: :cpu_bottleneck, affected_nodes: Enum.map(high_cpu_nodes, & &1.node)} | bottlenecks]
    else
      bottlenecks
    end
    
    # Memory bottlenecks
    high_memory_nodes = Enum.filter(successful_metrics, fn %{metrics: m} ->
      Map.get(m, :memory_usage_percent, 0) > 85
    end)
    
    bottlenecks = if length(high_memory_nodes) > 0 do
      [%{type: :memory_bottleneck, affected_nodes: Enum.map(high_memory_nodes, & &1.node)} | bottlenecks]
    else
      bottlenecks
    end
    
    bottlenecks
  end
  
  defp generate_optimization_suggestions(node_metrics) do
    suggestions = []
    
    # Suggest process distribution optimizations
    process_counts = Enum.map(node_metrics, fn
      %{status: :success, metrics: %{process_count: count}, node: node} -> {node, count}
      _ -> nil
    end)
    |> Enum.reject(&is_nil/1)
    
    if length(process_counts) > 1 do
      {_node, max_processes} = Enum.max_by(process_counts, &elem(&1, 1))
      {_node, min_processes} = Enum.min_by(process_counts, &elem(&1, 1))
      
      if max_processes > min_processes * 2 do
        suggestions = [
          "Consider rebalancing processes across nodes - uneven distribution detected" | suggestions
        ]
      end
    end
    
    suggestions
  end
  
  defp analyze_code_hotspots(node_metrics) do
    # Analyze code execution patterns across cluster
    all_hotspots = node_metrics
    |> Enum.filter(&(&1.status == :success))
    |> Enum.flat_map(fn %{metrics: metrics} ->
      Map.get(metrics, :function_calls, [])
    end)
    |> Enum.group_by(& &1.function)
    |> Enum.map(fn {function, calls} ->
      %{
        function: function,
        total_calls: Enum.sum(Enum.map(calls, & &1.call_count)),
        total_time: Enum.sum(Enum.map(calls, & &1.total_time)),
        nodes: Enum.map(calls, & &1.node) |> Enum.uniq()
      }
    end)
    |> Enum.sort_by(& &1.total_time, :desc)
    |> Enum.take(20)  # Top 20 hotspots
    
    all_hotspots
  end
end

defmodule ElixirScope.LiveDashboard do
  use Phoenix.LiveView
  
  @moduledoc """
  Real-time debugging dashboard that leverages Foundation's messaging
  and service discovery for live cluster debugging.
  """
  
  def mount(_params, _session, socket) do
    if connected?(socket) do
      # Subscribe to Foundation channels for real-time updates
      Foundation.Channels.subscribe(:control)
      Foundation.Channels.subscribe(:telemetry)
      Foundation.Channels.subscribe(:events)
      
      # Start periodic cluster health updates
      :timer.send_interval(5000, self(), :update_cluster_health)
    end
    
    # Get initial cluster state
    initial_state = get_initial_cluster_state()
    
    socket = assign(socket,
      cluster_nodes: initial_state.nodes,
      active_traces: initial_state.traces,
      cluster_health: initial_state.health,
      performance_metrics: initial_state.performance,
      connected: connected?(socket)
    )
    
    {:ok, socket}
  end
  
  def handle_info({:start_trace, trace_id, scope}, socket) do
    # Real-time trace updates via Foundation channels
    new_trace = %{
      id: trace_id,
      scope: scope,
      started_at: System.system_time(:millisecond),
      nodes: [],
      events: [],
      status: :active
    }
    
    new_traces = Map.put(socket.assigns.active_traces, trace_id, new_trace)
    
    {:noreply, assign(socket, active_traces: new_traces)}
  end
  
  def handle_info({:trace_event, trace_id, event}, socket) do
    case Map.get(socket.assigns.active_traces, trace_id) do
      nil -> {:noreply, socket}
      
      trace ->
        updated_trace = %{trace |
          events: [event | trace.events],
          nodes: Enum.uniq([event.node | trace.nodes])
        }
        
        new_traces = Map.put(socket.assigns.active_traces, trace_id, updated_trace)
        {:noreply, assign(socket, active_traces: new_traces)}
    end
  end
  
  def handle_info({:trace_completed, trace_id, results}, socket) do
    case Map.get(socket.assigns.active_traces, trace_id) do
      nil -> {:noreply, socket}
      
      trace ->
        updated_trace = %{trace |
          status: :completed,
          completed_at: System.system_time(:millisecond),
          results: results
        }
        
        new_traces = Map.put(socket.assigns.active_traces, trace_id, updated_trace)
        {:noreply, assign(socket, active_traces: new_traces)}
    end
  end
  
  def handle_info(:update_cluster_health, socket) do
    cluster_health = Foundation.HealthMonitor.get_cluster_health()
    performance_metrics = collect_live_performance_metrics()
    
    socket = socket
    |> assign(cluster_health: cluster_health)
    |> assign(performance_metrics: performance_metrics)
    |> assign(cluster_nodes: cluster_health.cluster.node_health)
    
    {:noreply, socket}
  end
  
  def handle_event("start_trace", %{"scope" => scope, "options" => options}, socket) do
    trace_id = generate_trace_id()
    
    # Parse trace options
    parsed_options = parse_trace_options(options)
    
    # Use Foundation to coordinate trace start
    case ElixirScope.DistributedTracer.start_cluster_trace(trace_id, scope, parsed_options) do
      {:ok, _coordinator} ->
        {:noreply, put_flash(socket, :info, "Trace #{trace_id} started successfully")}
      
      {:error, reason} ->
        {:noreply, put_flash(socket, :error, "Failed to start trace: #{inspect(reason)}")}
    end
  end
  
  def handle_event("stop_trace", %{"trace_id" => trace_id}, socket) do
    # Stop active trace
    Foundation.Channels.broadcast(:control, {:stop_trace, trace_id})
    
    {:noreply, put_flash(socket, :info, "Trace #{trace_id} stop requested")}
  end
  
  def handle_event("export_trace", %{"trace_id" => trace_id}, socket) do
    case Map.get(socket.assigns.active_traces, trace_id) do
      nil ->
        {:noreply, put_flash(socket, :error, "Trace not found")}
      
      trace ->
        # Export trace data
        export_data = prepare_trace_export(trace)
        
        socket = push_event(socket, "download", %{
          filename: "trace_#{trace_id}.json",
          content: Jason.encode!(export_data),
          content_type: "application/json"
        })
        
        {:noreply, socket}
    end
  end
  
  defp get_initial_cluster_state() do
    cluster_health = Foundation.HealthMonitor.get_cluster_health()
    
    %{
      nodes: cluster_health.cluster.node_health,
      traces: get_active_traces(),
      health: cluster_health,
      performance: collect_live_performance_metrics()
    }
  end
  
  defp get_active_traces() do
    # Get currently active traces from Foundation service mesh
    trace_services = Foundation.ServiceMesh.discover_services(
      capabilities: [:trace_coordinator]
    )
    
    trace_services
    |> Enum.map(fn service ->
      case GenServer.call(service.pid, :get_trace_info, 5000) do
        {:ok, trace_info} -> {trace_info.id, trace_info}
        _ -> nil
      end
    end)
    |> Enum.reject(&is_nil/1)
    |> Map.new()
  end
  
  defp collect_live_performance_metrics() do
    cluster_health = Foundation.HealthMonitor.get_cluster_health()
    
    %{
      total_nodes: cluster_health.cluster.connected_nodes,
      healthy_nodes: Enum.count(cluster_health.cluster.node_health, &(&1.status == :healthy)),
      total_processes: get_cluster_process_count(),
      message_rate: cluster_health.performance.message_throughput_per_sec || 0,
      memory_usage: cluster_health.performance.memory_usage || 0
    }
  end
  
  defp get_cluster_process_count() do
    # Sum processes across all healthy nodes
    case Foundation.HealthMonitor.get_cluster_health() do
      %{cluster: %{node_health: nodes}} ->
        nodes
        |> Enum.filter(&(&1.status == :healthy))
        |> Enum.map(fn node ->
          case :rpc.call(node.node, :erlang, :system_info, [:process_count], 5000) do
            {:badrpc, _} -> 0
            count -> count
          end
        end)
        |> Enum.sum()
      
      _ -> 0
    end
  end
  
  defp generate_trace_id() do
    :crypto.strong_rand_bytes(8) |> Base.encode64(padding: false)
  end
  
  defp parse_trace_options(options_string) do
    # Parse JSON or key=value options string
    case Jason.decode(options_string) do
      {:ok, options} -> options
      {:error, _} -> parse_key_value_options(options_string)
    end
  end
  
  defp parse_key_value_options(options_string) do
    options_string
    |> String.split(",")
    |> Enum.map(&String.trim/1)
    |> Enum.map(fn option ->
      case String.split(option, "=", parts: 2) do
        [key, value] -> {String.trim(key), String.trim(value)}
        [key] -> {String.trim(key), true}
      end
    end)
    |> Map.new()
  end
  
  defp prepare_trace_export(trace) do
    %{
      trace_id: trace.id,
      scope: trace.scope,
      started_at: trace.started_at,
      completed_at: Map.get(trace, :completed_at),
      status: trace.status,
      nodes: trace.nodes,
      events: trace.events,
      results: Map.get(trace, :results),
      exported_at: System.system_time(:millisecond)
    }
  end
end

defmodule DSPEx.DistributedOptimizer do
  @moduledoc """
  Leverages Foundation's process management for optimal AI worker distribution.
  
  Uses Foundation's health monitoring to make intelligent placement decisions
  and Foundation's channels for efficient coordination.
  """
  
  def optimize_program_distributed(program, trainset, metric_fn, opts \\ []) do
    # Use Foundation to analyze cluster capacity
    cluster_health = Foundation.HealthMonitor.get_cluster_health()
    optimal_config = calculate_optimal_distribution(cluster_health, opts)
    
    # Start distributed optimization coordinator
    {:ok, coordinator} = Foundation.ProcessManager.start_singleton(
      DSPEx.OptimizationCoordinator,
      [
        program: program,
        trainset: trainset,
        metric_fn: metric_fn,
        config: optimal_config
      ],
      name: {:optimization_coordinator, System.unique_integer()}
    )
    
    # Distribute workers intelligently across cluster
    worker_results = start_distributed_workers(optimal_config.worker_distribution)
    
    # Coordinate optimization via Foundation channels
    Foundation.Channels.broadcast(:control, {
      :optimization_started, 
      coordinator, 
      worker_results,
      optimal_config
    })
    
    # Execute distributed optimization with monitoring
    execute_distributed_optimization(coordinator, worker_results, opts)
  end
  
  defp calculate_optimal_distribution(cluster_health, opts) do
    max_workers = Keyword.get(opts, :max_workers, :auto)
    min_workers_per_node = Keyword.get(opts, :min_workers_per_node, 1)
    
    healthy_nodes = Enum.filter(cluster_health.cluster.node_health, &(&1.status == :healthy))
    
    # Calculate total available capacity
    total_capacity = Enum.reduce(healthy_nodes, 0, fn node, acc ->
      cores = get_node_cores(node.node)
      load = node.load_avg || 0.0
      available = max(0, cores - (load * cores))
      acc + available
    end)
    
    # Determine optimal worker count
    optimal_workers = case max_workers do
      :auto -> round(total_capacity * 0.8)  # Use 80% of available capacity
      n when is_integer(n) -> min(n, round(total_capacity))
      _ -> round(total_capacity * 0.5)  # Conservative default
    end
    
    # Distribute workers across nodes based on capacity
    worker_distribution = distribute_workers_by_capacity(
      healthy_nodes, 
      optimal_workers, 
      min_workers_per_node
    )
    
    %{
      total_workers: optimal_workers,
      worker_distribution: worker_distribution,
      estimated_completion_time: estimate_completion_time(optimal_workers, opts),
      resource_allocation: %{
        total_capacity: total_capacity,
        allocated_capacity: optimal_workers,
        utilization_percent: (optimal_workers / total_capacity) * 100
      }
    }
  end
  
  defp distribute_workers_by_capacity(nodes, total_workers, min_per_node) do
    # First, allocate minimum workers per node
    base_allocation = Enum.map(nodes, fn node ->
      %{
        node: node.node,
        capacity: get_available_capacity(node),
        workers: min_per_node
      }
    end)
    
    remaining_workers = total_workers - (length(nodes) * min_per_node)
    
    # Distribute remaining workers proportionally by capacity
    total_remaining_capacity = Enum.sum(Enum.map(base_allocation, & &1.capacity))
    
    if total_remaining_capacity > 0 and remaining_workers > 0 do
      Enum.map(base_allocation, fn allocation ->
        proportion = allocation.capacity / total_remaining_capacity
        additional_workers = round(remaining_workers * proportion)
        
        %{allocation | workers: allocation.workers + additional_workers}
      end)
    else
      base_allocation
    end
  end
  
  defp get_available_capacity(node) do
    cores = get_node_cores(node.node)
    load = node.load_avg || 0.0
    max(0, cores - (load * cores))
  end
  
  defp get_node_cores(node) do
    case :rpc.call(node, :erlang, :system_info, [:schedulers_online], 5000) do
      {:badrpc, _} -> 4  # Default assumption
      cores -> cores
    end
  end
  
  defp start_distributed_workers(distribution) do
    # Start workers on each node according to distribution plan
    tasks = Enum.map(distribution, fn %{node: node, workers: count} ->
      Task.async(fn ->
        if node == Node.self() do
          start_local_workers(count)
        else
          :rpc.call(node, __MODULE__, :start_local_workers, [count], 60_000)
        end
      end)
    end)
    
    # Collect results with timeout
    case Task.yield_many(tasks, 45_000) do
      results ->
        successful_results = Enum.flat_map(results, fn
          {_task, {:ok, workers}} when is_list(workers) -> workers
          {_task, {:ok, result}} -> [result]
          _ -> []
        end)
        
        %{
          total_started: length(successful_results),
          workers: successful_results,
          distribution_success: length(successful_results) > 0
        }
    end
  end
  
  def start_local_workers(count) do
    # Start workers on local node using Foundation
    worker_results = for i <- 1..count do
      case Foundation.ProcessManager.start_singleton(
        DSPEx.EvaluationWorker,
        [worker_id: i, node: Node.self()],
        name: {:evaluation_worker, Node.self(), i}
      ) do
        {:ok, worker_pid} ->
          # Register worker with capabilities
          Foundation.ServiceMesh.register_service(
            {:evaluation_worker, i},
            worker_pid,
            [:dspy_evaluation, :ai_optimization],
            %{
              node: Node.self(), 
              worker_id: i,
              started_at: System.system_time(:millisecond),
              status: :ready
            }
          )
          
          {:ok, %{pid: worker_pid, id: i, node: Node.self()}}
        
        error ->
          {:error, error}
      end
    end
    
    # Return successful workers
    Enum.filter(worker_results, &match?({:ok, _}, &1))
    |> Enum.map(fn {:ok, worker} -> worker end)
  end
  
  defp execute_distributed_optimization(coordinator, worker_results, opts) do
    timeout = Keyword.get(opts, :timeout, 300_000)  # 5 minute default
    
    # Start the optimization process
    case GenServer.call(coordinator, {:start_optimization, worker_results}, timeout) do
      {:ok, optimization_id} ->
        # Monitor optimization progress
        monitor_optimization_progress(optimization_id, coordinator, opts)
      
      error ->
        error
    end
  end
  
  defp monitor_optimization_progress(optimization_id, coordinator, opts) do
    monitor_interval = Keyword.get(opts, :monitor_interval, 10_000)  # 10 seconds
    
    # Subscribe to optimization updates
    Foundation.Channels.subscribe(:telemetry)
    
    # Start monitoring loop
    monitor_loop(optimization_id, coordinator, monitor_interval)
  end
  
  defp monitor_loop(optimization_id, coordinator, interval) do
    receive do
      {:optimization_progress, ^optimization_id, progress} ->
        Logger.info("DSPEx optimization progress: #{progress.completion_percent}%")
        Logger.info("Current best score: #{progress.current_best_score}")
        
        if progress.completion_percent >= 100 do
          {:ok, progress.final_result}
        else
          Process.send_after(self(), :check_progress, interval)
          monitor_loop(optimization_id, coordinator, interval)
        end
      
      {:optimization_error, ^optimization_id, error} ->
        Logger.error("DSPEx optimization failed: #{inspect(error)}")
        {:error, error}
      
      :check_progress ->
        case GenServer.call(coordinator, :get_progress, 10_000) do
          {:ok, progress} ->
            if progress.completion_percent >= 100 do
              {:ok, progress.final_result}
            else
              Process.send_after(self(), :check_progress, interval)
              monitor_loop(optimization_id, coordinator, interval)
            end
          
          error ->
            {:error, error}
        end
    after
      300_000 ->  # 5 minute timeout
        Logger.error("DSPEx optimization timed out")
        {:error, :timeout}
    end
  end
  
  defp estimate_completion_time(worker_count, opts) do
    dataset_size = Keyword.get(opts, :dataset_size, 1000)
    evaluations_per_worker_per_second = Keyword.get(opts, :eval_rate, 2)
    
    total_evaluations_per_second = worker_count * evaluations_per_worker_per_second
    estimated_seconds = dataset_size / total_evaluations_per_second
    
    round(estimated_seconds * 1000)  # Return in milliseconds
  end
end

defmodule DSPEx.AdaptiveOptimizer do
  @moduledoc """
  Advanced optimization patterns that adapt to cluster conditions and
  optimization progress in real-time.
  """
  
  def adaptive_optimization(program, trainset, metric_fn, opts \\ []) do
    # Start with initial worker allocation
    initial_result = DSPEx.DistributedOptimizer.optimize_program_distributed(
      program, 
      trainset, 
      metric_fn, 
      opts
    )
    
    case initial_result do
      {:ok, %{optimization_id: opt_id} = result} ->
        # Start adaptive scaling monitor
        spawn_link(fn -> 
          adaptive_scaling_monitor(opt_id, result, opts)
        end)
        
        {:ok, result}
      
      error -> error
    end
  end
  
  defp adaptive_scaling_monitor(optimization_id, initial_result, opts) do
    scaling_interval = Keyword.get(opts, :scaling_check_interval, 30_000)  # 30 seconds
    
    Process.send_after(self(), :check_scaling, scaling_interval)
    
    adaptive_loop(optimization_id, initial_result, opts)
  end
  
  defp adaptive_loop(optimization_id, current_state, opts) do
    receive do
      :check_scaling ->
        new_state = check_and_apply_scaling(optimization_id, current_state, opts)
        
        scaling_interval = Keyword.get(opts, :scaling_check_interval, 30_000)
        Process.send_after(self(), :check_scaling, scaling_interval)
        
        adaptive_loop(optimization_id, new_state, opts)
      
      {:optimization_completed, ^optimization_id} ->
        Logger.info("Adaptive optimization completed for #{optimization_id}")
        :ok
      
      {:stop_monitoring, ^optimization_id} ->
        :ok
    after
      600_000 ->  # 10 minute timeout
        Logger.warning("Adaptive scaling monitor timeout for #{optimization_id}")
        :timeout
    end
  end
  
  defp check_and_apply_scaling(optimization_id, current_state, opts) do
    # Analyze current optimization performance
    performance_metrics = analyze_optimization_performance(optimization_id)
    cluster_health = Foundation.HealthMonitor.get_cluster_health()
    
    scaling_decision = determine_scaling_action(performance_metrics, cluster_health, opts)
    
    case scaling_decision do
      {:scale_up, additional_workers} ->
        Logger.info("Scaling up DSPEx optimization: +#{additional_workers} workers")
        new_workers = add_workers(optimization_id, additional_workers)
        %{current_state | workers: current_state.workers ++ new_workers}
      
      {:scale_down, workers_to_remove} ->
        Logger.info("Scaling down DSPEx optimization: -#{workers_to_remove} workers")
        remaining_workers = remove_workers(optimization_id, workers_to_remove)
        %{current_state | workers: remaining_workers}
      
      :no_scaling ->
        current_state
    end
  end
  
  defp analyze_optimization_performance(optimization_id) do
    # Get performance metrics from coordinator
    case Foundation.ServiceMesh.discover_services(name: {:optimization_coordinator, optimization_id}) do
      [coordinator_service] ->
        case GenServer.call(coordinator_service.pid, :get_performance_metrics, 10_000) do
          {:ok, metrics} -> metrics
          _ -> %{evaluations_per_second: 0, efficiency: 0.0}
        end
      
      _ ->
        %{evaluations_per_second: 0, efficiency: 0.0}
    end
  end
  
  defp determine_scaling_action(performance_metrics, cluster_health, opts) do
    current_efficiency = performance_metrics.evaluations_per_second
    target_efficiency = Keyword.get(opts, :target_efficiency, 10.0)  # 10 eval/sec
    
    # Check if we have available cluster capacity
    available_capacity = calculate_available_cluster_capacity(cluster_health)
    
    cond do
      # Scale up if we're below target and have capacity
      current_efficiency < target_efficiency and available_capacity > 2 ->
        additional_workers = min(available_capacity, round(target_efficiency - current_efficiency))
        {:scale_up, additional_workers}
      
      # Scale down if we're significantly over target
      current_efficiency > target_efficiency * 1.5 ->
        excess_capacity = round(current_efficiency - target_efficiency)
        {:scale_down, excess_capacity}
      
      # No scaling needed
      true ->
        :no_scaling
    end
  end
  
  defp calculate_available_cluster_capacity(cluster_health) do
    cluster_health.cluster.node_health
    |> Enum.filter(&(&1.status == :healthy))
    |> Enum.map(fn node ->
      cores = get_node_cores(node.node)
      load = node.load_avg || 0.0
      max(0, cores - (load * cores))
    end)
    |> Enum.sum()
    |> round()
  end
  
  defp add_workers(optimization_id, count) do
    # Find best nodes for additional workers
    cluster_health = Foundation.HealthMonitor.get_cluster_health()
    best_nodes = find_best_nodes_for_workers(cluster_health, count)
    
    # Distribute new workers across selected nodes
    new_workers = Enum.flat_map(best_nodes, fn {node, worker_count} ->
      case :rpc.call(node, DSPEx.DistributedOptimizer, :start_local_workers, [worker_count], 30_000) do
        {:badrpc, reason} -> 
          Logger.warning("Failed to start workers on #{node}: #{inspect(reason)}")
          []
        workers when is_list(workers) -> 
          workers
        _ -> 
          []
      end
    end)
    
    # Register new workers with the optimization coordinator
    if length(new_workers) > 0 do
      case Foundation.ServiceMesh.discover_services(name: {:optimization_coordinator, optimization_id}) do
        [coordinator_service] ->
          GenServer.cast(coordinator_service.pid, {:add_workers, new_workers})
        _ ->
          Logger.warning("Could not find optimization coordinator to add workers")
      end
    end
    
    new_workers
  end
  
  defp remove_workers(optimization_id, count) do
    # Find workers to remove (least efficient first)
    current_workers = Foundation.ServiceMesh.discover_services(
      capabilities: [:dspy_evaluation],
      metadata: %{optimization_id: optimization_id}
    )
    
    # Sort by efficiency and remove the least efficient
    workers_to_remove = current_workers
    |> Enum.sort_by(&get_worker_efficiency/1)
    |> Enum.take(count)
    
    # Gracefully stop selected workers
    Enum.each(workers_to_remove, fn worker ->
      GenServer.cast(worker.pid, :graceful_shutdown)
      Foundation.ServiceMesh.deregister_service({:evaluation_worker, worker.metadata.worker_id})
    end)
    
    # Return remaining workers
    Enum.reject(current_workers, fn worker ->
      Enum.any?(workers_to_remove, &(&1.pid == worker.pid))
    end)
  end
  
  defp find_best_nodes_for_workers(cluster_health, worker_count) do
    # Select nodes with best available capacity
    cluster_health.cluster.node_health
    |> Enum.filter(&(&1.status == :healthy))
    |> Enum.map(fn node ->
      available_capacity = get_available_capacity(node)
      {node.node, available_capacity}
    end)
    |> Enum.filter(fn {_node, capacity} -> capacity > 0 end)
    |> Enum.sort_by(&elem(&1, 1), :desc)  # Sort by capacity desc
    |> distribute_workers_across_nodes(worker_count)
  end
  
  defp distribute_workers_across_nodes(node_capacities, total_workers) do
    total_capacity = Enum.sum(Enum.map(node_capacities, &elem(&1, 1)))
    
    if total_capacity == 0 do
      []
    else
      Enum.map(node_capacities, fn {node, capacity} ->
        proportion = capacity / total_capacity
        workers_for_node = round(total_workers * proportion)
        {node, max(0, workers_for_node)}
      end)
      |> Enum.filter(fn {_node, workers} -> workers > 0 end)
    end
  end
  
  defp get_worker_efficiency(worker) do
    # Get efficiency metrics from worker
    case GenServer.call(worker.pid, :get_efficiency_metrics, 5000) do
      {:ok, %{evaluations_per_second: eps}} -> eps
      _ -> 0.0
    end
  rescue
    _ -> 0.0
  end
  
  defp get_node_cores(node) do
    case :rpc.call(node, :erlang, :system_info, [:schedulers_online], 5000) do
      {:badrpc, _} -> 4  # Default assumption
      cores -> cores
    end
  end
  
  defp get_available_capacity(node) do
    cores = get_node_cores(node.node)
    load = node.load_avg || 0.0
    max(0, cores - (load * cores))
  end
end

defmodule DSPEx.OptimizationDashboard do
  use Phoenix.LiveView
  
  @moduledoc """
  Real-time DSPEx optimization monitoring using Foundation's infrastructure.
  
  Provides live visibility into distributed optimization jobs, worker performance,
  and cluster resource utilization.
  """
  
  def mount(_params, _session, socket) do
    if connected?(socket) do
      # Subscribe to DSPEx optimization events via Foundation channels
      Foundation.Channels.subscribe(:control)  # Optimization lifecycle events
      Foundation.Channels.subscribe(:telemetry)  # Performance metrics
      Foundation.Channels.subscribe(:events)   # Worker events
      
      # Start periodic updates
      :timer.send_interval(5000, self(), :update_metrics)
      :timer.send_interval(1000, self(), :update_real_time_metrics)
    end
    
    initial_state = get_initial_dashboard_state()
    
    socket = assign(socket,
      active_optimizations: initial_state.optimizations,
      cluster_health: initial_state.cluster_health,
      worker_stats: initial_state.worker_stats,
      performance_history: [],
      real_time_metrics: %{},
      connected: connected?(socket)
    )
    
    {:ok, socket}
  end
  
  def handle_info({:optimization_started, coordinator, workers, config}, socket) do
    optimization_id = extract_optimization_id(coordinator)
    
    new_optimization = %{
      id: optimization_id,
      coordinator: coordinator,
      workers: workers,
      config: config,
      started_at: System.system_time(:millisecond),
      status: :running,
      progress: 0.0,
      current_best_score: nil,
      evaluations_completed: 0,
      estimated_completion: config.estimated_completion_time
    }
    
    new_optimizations = Map.put(socket.assigns.active_optimizations, optimization_id, new_optimization)
    
    {:noreply, assign(socket, active_optimizations: new_optimizations)}
  end
  
  def handle_info({:optimization_progress, optimization_id, progress_data}, socket) do
    case Map.get(socket.assigns.active_optimizations, optimization_id) do
      nil -> {:noreply, socket}
      
      optimization ->
        updated_optimization = %{optimization |
          progress: progress_data.completion_percent,
          current_best_score: progress_data.current_best_score,
          evaluations_completed: progress_data.evaluations_completed,
          estimated_completion: progress_data.estimated_completion_time
        }
        
        new_optimizations = Map.put(socket.assigns.active_optimizations, optimization_id, updated_optimization)
        
        {:noreply, assign(socket, active_optimizations: new_optimizations)}
    end
  end
  
  def handle_info({:worker_performance_update, worker_id, performance_data}, socket) do
    current_stats = socket.assigns.worker_stats
    
    updated_stats = Map.put(current_stats, worker_id, %{
      evaluations_per_second: performance_data.eval_rate,
      total_evaluations: performance_data.total_evals,
      efficiency_score: performance_data.efficiency,
      last_update: System.system_time(:millisecond)
    })
    
    {:noreply, assign(socket, worker_stats: updated_stats)}
  end
  
  def handle_info(:update_metrics, socket) do
    # Update cluster health and longer-term metrics
    cluster_health = Foundation.HealthMonitor.get_cluster_health()
    performance_snapshot = capture_performance_snapshot()
    
    new_history = [performance_snapshot | Enum.take(socket.assigns.performance_history, 99)]
    
    socket = socket
    |> assign(cluster_health: cluster_health)
    |> assign(performance_history: new_history)
    
    {:noreply, socket}
  end
  
  def handle_info(:update_real_time_metrics, socket) do
    # Update high-frequency real-time metrics
    real_time_metrics = %{
      total_active_workers: count_active_dspy_workers(),
      cluster_evaluation_rate: calculate_cluster_evaluation_rate(),
      cluster_cpu_usage: get_cluster_cpu_usage(),
      cluster_memory_usage: get_cluster_memory_usage(),
      network_throughput: get_network_throughput()
    }
    
    {:noreply, assign(socket, real_time_metrics: real_time_metrics)}
  end
  
  def handle_event("start_optimization", params, socket) do
    # Parse optimization parameters from UI
    program_type = Map.get(params, "program_type", "basic_qa")
    dataset_size = String.to_integer(Map.get(params, "dataset_size", "1000"))
    max_workers = parse_worker_count(Map.get(params, "max_workers", "auto"))
    
    # Start optimization with Foundation coordination
    case start_optimization_from_ui(program_type, dataset_size, max_workers) do
      {:ok, optimization_id} ->
        socket = put_flash(socket, :info, "Optimization #{optimization_id} started successfully")
        {:noreply, socket}
      
      {:error, reason} ->
        socket = put_flash(socket, :error, "Failed to start optimization: #{inspect(reason)}")
        {:noreply, socket}
    end
  end
  
  def handle_event("stop_optimization", %{"optimization_id" => optimization_id}, socket) do
    # Gracefully stop optimization
    Foundation.Channels.broadcast(:control, {:stop_optimization, optimization_id})
    
    # Update UI immediately
    case Map.get(socket.assigns.active_optimizations, optimization_id) do
      nil -> 
        {:noreply, socket}
      
      optimization ->
        updated_optimization = %{optimization | status: :stopping}
        new_optimizations = Map.put(socket.assigns.active_optimizations, optimization_id, updated_optimization)
        
        socket = socket
        |> assign(active_optimizations: new_optimizations)
        |> put_flash(:info, "Stopping optimization #{optimization_id}")
        
        {:noreply, socket}
    end
  end
  
  def handle_event("scale_optimization", %{"optimization_id" => optimization_id, "action" => action}, socket) do
    case action do
      "scale_up" ->
        Foundation.Channels.broadcast(:control, {:scale_optimization, optimization_id, :up, 2})
        socket = put_flash(socket, :info, "Scaling up optimization #{optimization_id}")
        
      "scale_down" ->
        Foundation.Channels.broadcast(:control, {:scale_optimization, optimization_id, :down, 2})
        socket = put_flash(socket, :info, "Scaling down optimization #{optimization_id}")
      
      _ ->
        socket = put_flash(socket, :error, "Unknown scaling action")
    end
    
    {:noreply, socket}
  end
  
  defp get_initial_dashboard_state() do
    cluster_health = Foundation.HealthMonitor.get_cluster_health()
    active_optimizations = discover_active_optimizations()
    worker_stats = collect_worker_statistics()
    
    %{
      cluster_health: cluster_health,
      optimizations: active_optimizations,
      worker_stats: worker_stats
    }
  end
  
  defp discover_active_optimizations() do
    # Find active optimization coordinators
    Foundation.ServiceMesh.discover_services(
      capabilities: [:optimization_coordinator]
    )
    |> Enum.map(fn service ->
      case GenServer.call(service.pid, :get_optimization_info, 10_000) do
        {:ok, info} -> {info.id, info}
        _ -> nil
      end
    end)
    |> Enum.reject(&is_nil/1)
    |> Map.new()
  end
  
  defp collect_worker_statistics() do
    Foundation.ServiceMesh.discover_services(capabilities: [:dspy_evaluation])
    |> Enum.map(fn worker ->
      case GenServer.call(worker.pid, :get_performance_stats, 5000) do
        {:ok, stats} -> {worker.metadata.worker_id, stats}
        _ -> nil
      end
    end)
    |> Enum.reject(&is_nil/1)
    |> Map.new()
  end
  
  defp capture_performance_snapshot() do
    %{
      timestamp: System.system_time(:millisecond),
      active_optimizations: map_size(discover_active_optimizations()),
      total_workers: count_active_dspy_workers(),
      cluster_eval_rate: calculate_cluster_evaluation_rate(),
      cluster_nodes: Foundation.HealthMonitor.get_cluster_health().cluster.connected_nodes
    }
  end
  
  defp count_active_dspy_workers() do
    Foundation.ServiceMesh.discover_services(capabilities: [:dspy_evaluation])
    |> length()
  end
  
  defp calculate_cluster_evaluation_rate() do
    # Sum evaluation rates from all active workers
    Foundation.ServiceMesh.discover_services(capabilities: [:dspy_evaluation])
    |> Enum.map(fn worker ->
      case GenServer.call(worker.pid, :get_current_eval_rate, 2000) do
        {:ok, rate} -> rate
        _ -> 0.0
      end
    end)
    |> Enum.sum()
  end
  
  defp get_cluster_cpu_usage() do
    cluster_health = Foundation.HealthMonitor.get_cluster_health()
    
    cluster_health.cluster.node_health
    |> Enum.filter(&(&1.status == :healthy))
    |> Enum.map(& &1.load_avg)
    |> Enum.reject(&is_nil/1)
    |> case do
      [] -> 0.0
      loads -> Enum.sum(loads) / length(loads)
    end
  end
  
  defp get_cluster_memory_usage() do
    cluster_health = Foundation.HealthMonitor.get_cluster_health()
    Map.get(cluster_health.performance, :memory_usage, 0)
  end
  
  defp get_network_throughput() do
    # Would implement network throughput measurement
    # For now, return placeholder
    0.0
  end
  
  defp start_optimization_from_ui(program_type, dataset_size, max_workers) do
    # Create a basic optimization job based on UI parameters
    program = create_program_from_type(program_type)
    trainset = generate_synthetic_trainset(dataset_size)
    metric_fn = &basic_accuracy_metric/2
    
    opts = case max_workers do
      :auto -> []
      n when is_integer(n) -> [max_workers: n]
    end
    
    case DSPEx.DistributedOptimizer.optimize_program_distributed(program, trainset, metric_fn, opts) do
      {:ok, result} -> {:ok, result.optimization_id}
      error -> error
    end
  end
  
  defp parse_worker_count("auto"), do: :auto
  defp parse_worker_count(count_str) do
    case Integer.parse(count_str) do
      {count, ""} -> count
      _ -> :auto
    end
  end
  
  defp extract_optimization_id(coordinator) do
    case GenServer.call(coordinator, :get_optimization_id, 5000) do
      {:ok, id} -> id
      _ -> :crypto.strong_rand_bytes(8) |> Base.encode64(padding: false)
    end
  rescue
    _ -> :crypto.strong_rand_bytes(8) |> Base.encode64(padding: false)
  end
  
  # Placeholder implementations for demo purposes
  defp create_program_from_type(_type), do: %DSPEx.Program{}
  defp generate_synthetic_trainset(size), do: Enum.map(1..size, fn i -> %{id: i} end)
  defp basic_accuracy_metric(_example, _prediction), do: 0.8
end