defmodule Foundation.MABEAM.Cluster do
  @moduledoc """
  Cluster-wide coordination and management for distributed MABEAM systems.
  
  Provides node discovery, agent migration, distributed variable synchronization,
  and cluster-aware fault tolerance for multi-agent systems.
  """
  
  use GenServer
  
  alias Foundation.MABEAM.{Core, Types}
  alias Foundation.{Events, Telemetry}
  
  @type cluster_state :: %{
    nodes: %{node() => node_info()},
    distributed_variables: %{atom() => distributed_variable()},
    agent_locations: %{atom() => node()},
    migration_queue: [migration_request()],
    cluster_topology: cluster_topology(),
    partition_detector: pid(),
    consensus_coordinator: pid()
  }
  
  @type node_info :: %{
    status: node_status(),
    capabilities: [atom()],
    resource_capacity: Types.resource_allocation(),
    current_load: float(),
    agent_count: non_neg_integer(),
    last_heartbeat: DateTime.t()
  }
  
  @type node_status :: :active | :joining | :leaving | :partitioned | :failed
  
  @type distributed_variable :: %{
    id: atom(),
    master_node: node(),
    replica_nodes: [node()],
    consistency_level: consistency_level(),
    synchronization_strategy: sync_strategy(),
    conflict_resolution: conflict_resolution_strategy()
  }
  
  @type consistency_level :: :eventual | :strong | :causal | :session
  @type sync_strategy :: :gossip | :leader_follower | :multi_master | :blockchain
  @type conflict_resolution_strategy :: :last_writer_wins | :vector_clocks | :consensus | :merge_function
  
  ## Public API
  
  @doc """
  Initialize cluster management for MABEAM.
  """
  @spec start_link(keyword()) :: GenServer.on_start()
  def start_link(opts \\ []) do
    GenServer.start_link(__MODULE__, opts, name: __MODULE__)
  end
  
  @doc """
  Join a MABEAM cluster.
  """
  @spec join_cluster([node()]) :: :ok | {:error, term()}
  def join_cluster(seed_nodes) do
    GenServer.call(__MODULE__, {:join_cluster, seed_nodes})
  end
  
  @doc """
  Distribute an orchestration variable across cluster nodes.
  """
  @spec distribute_variable(Types.orchestration_variable(), keyword()) :: 
    {:ok, distributed_variable()} | {:error, term()}
  def distribute_variable(variable, opts \\ []) do
    GenServer.call(__MODULE__, {:distribute_variable, variable, opts})
  end
  
  @doc """
  Migrate an agent to a different node.
  """
  @spec migrate_agent(atom(), node()) :: {:ok, pid()} | {:error, term()}
  def migrate_agent(agent_id, target_node) do
    GenServer.call(__MODULE__, {:migrate_agent, agent_id, target_node})
  end
  
  @doc """
  Get cluster-wide status and health information.
  """
  @spec cluster_status() :: {:ok, cluster_status()} | {:error, term()}
  def cluster_status() do
    GenServer.call(__MODULE__, :cluster_status)
  end
  
  @doc """
  Handle network partition detection and recovery.
  """
  @spec handle_partition([node()]) :: :ok
  def handle_partition(partitioned_nodes) do
    GenServer.cast(__MODULE__, {:partition_detected, partitioned_nodes})
  end
  
  ## GenServer Implementation
  
  @impl true
  def init(opts) do
    # Set up cluster monitoring
    :net_kernel.monitor_nodes(true, [:nodedown_reason])
    
    # Initialize partition detector
    {:ok, partition_detector} = start_partition_detector(opts)
    
    # Initialize consensus coordinator
    {:ok, consensus_coordinator} = start_consensus_coordinator(opts)
    
    state = %{
      nodes: %{node() => create_local_node_info()},
      distributed_variables: %{},
      agent_locations: %{},
      migration_queue: [],
      cluster_topology: :ring,  # Default topology
      partition_detector: partition_detector,
      consensus_coordinator: consensus_coordinator
    }
    
    # Set up telemetry
    setup_cluster_telemetry()
    
    {:ok, state}
  end
  
  @impl true
  def handle_call({:join_cluster, seed_nodes}, _from, state) do
    case attempt_cluster_join(seed_nodes, state) do
      {:ok, new_state} ->
        # Announce our presence to the cluster
        broadcast_node_announcement(new_state)
        {:reply, :ok, new_state}
      
      {:error, reason} ->
        {:reply, {:error, reason}, state}
    end
  end
  
  @impl true
  def handle_call({:distribute_variable, variable, opts}, _from, state) do
    case create_distributed_variable(variable, opts, state) do
      {:ok, distributed_var, new_state} ->
        # Synchronize variable across selected nodes
        :ok = synchronize_variable_to_replicas(distributed_var)
        
        {:reply, {:ok, distributed_var}, new_state}
      
      {:error, reason} ->
        {:reply, {:error, reason}, state}
    end
  end
  
  @impl true
  def handle_call({:migrate_agent, agent_id, target_node}, _from, state) do
    case perform_agent_migration(agent_id, target_node, state) do
      {:ok, new_pid, new_state} ->
        {:reply, {:ok, new_pid}, new_state}
      
      {:error, reason} ->
        {:reply, {:error, reason}, state}
    end
  end
  
  @impl true
  def handle_info({:nodedown, node, reason}, state) do
    # Handle node failure
    new_state = handle_node_failure(node, reason, state)
    {:noreply, new_state}
  end
  
  @impl true
  def handle_info({:nodeup, node}, state) do
    # Handle node joining
    new_state = handle_node_joining(node, state)
    {:noreply, new_state}
  end
  
  ## Private Implementation
  
  defp attempt_cluster_join(seed_nodes, state) do
    # Try to connect to seed nodes
    connected_nodes = Enum.filter(seed_nodes, fn node ->
      Node.ping(node) == :pong
    end)
    
    case connected_nodes do
      [] -> {:error, :no_reachable_seed_nodes}
      nodes ->
        # Request cluster information from connected nodes
        cluster_info = request_cluster_info(nodes)
        
        # Update our state with cluster information
        new_state = merge_cluster_info(state, cluster_info)
        
        {:ok, new_state}
    end
  end
  
  defp create_distributed_variable(variable, opts, state) do
    replication_factor = Keyword.get(opts, :replication_factor, 3)
    consistency_level = Keyword.get(opts, :consistency_level, :eventual)
    
    # Select nodes for replication
    available_nodes = Map.keys(state.nodes)
    replica_nodes = select_replica_nodes(available_nodes, replication_factor, opts)
    
    distributed_var = %{
      id: variable.id,
      master_node: node(),  # Current node is master
      replica_nodes: replica_nodes,
      consistency_level: consistency_level,
      synchronization_strategy: Keyword.get(opts, :sync_strategy, :gossip),
      conflict_resolution: Keyword.get(opts, :conflict_resolution, :vector_clocks)
    }
    
    new_state = %{state | 
      distributed_variables: Map.put(state.distributed_variables, variable.id, distributed_var)
    }
    
    {:ok, distributed_var, new_state}
  end
  
  defp perform_agent_migration(agent_id, target_node, state) do
    case Map.get(state.agent_locations, agent_id) do
      nil -> {:error, :agent_not_found}
      current_node when current_node == target_node -> {:error, :already_on_target_node}
      current_node ->
        # Perform hot migration
        case hot_migrate_agent(agent_id, current_node, target_node) do
          {:ok, new_pid} ->
            new_state = %{state | 
              agent_locations: Map.put(state.agent_locations, agent_id, target_node)
            }
            
            # Emit migration event
            Events.emit(:mabeam_agent_migrated, %{
              agent_id: agent_id,
              from_node: current_node,
              to_node: target_node,
              new_pid: new_pid
            })
            
            {:ok, new_pid, new_state}
          
          {:error, reason} ->
            {:error, reason}
        end
    end
  end
  
  defp hot_migrate_agent(agent_id, source_node, target_node) do
    # 1. Serialize agent state on source node
    case :rpc.call(source_node, Foundation.MABEAM.AgentRegistry, :serialize_agent, [agent_id]) do
      {:ok, serialized_state} ->
        # 2. Start agent on target node with serialized state
        case :rpc.call(target_node, Foundation.MABEAM.AgentRegistry, :start_agent_from_state, 
                      [agent_id, serialized_state]) do
          {:ok, new_pid} ->
            # 3. Stop agent on source node
            :rpc.call(source_node, Foundation.MABEAM.AgentRegistry, :stop_agent, [agent_id])
            
            {:ok, new_pid}
          
          {:error, reason} ->
            {:error, {:failed_to_start_on_target, reason}}
        end
      
      {:error, reason} ->
        {:error, {:failed_to_serialize, reason}}
    end
  end
end

defmodule Foundation.MABEAM.Cluster.VariableSync do
  @moduledoc """
  Distributed variable synchronization across cluster nodes.
  
  Implements various consistency models and conflict resolution strategies
  for maintaining variable state across the cluster.
  """
  
  @doc """
  Synchronize a variable update across all replica nodes.
  """
  @spec synchronize_update(atom(), term(), keyword()) :: :ok | {:error, term()}
  def synchronize_update(variable_id, new_value, opts \\ []) do
    consistency_level = Keyword.get(opts, :consistency_level, :eventual)
    
    case consistency_level do
      :eventual -> eventual_consistency_update(variable_id, new_value, opts)
      :strong -> strong_consistency_update(variable_id, new_value, opts)
      :causal -> causal_consistency_update(variable_id, new_value, opts)
      :session -> session_consistency_update(variable_id, new_value, opts)
    end
  end
  
  defp eventual_consistency_update(variable_id, new_value, opts) do
    # Use gossip protocol for eventual consistency
    replica_nodes = get_replica_nodes(variable_id)
    
    # Create update message with vector clock
    update_msg = %{
      variable_id: variable_id,
      value: new_value,
      timestamp: System.monotonic_time(:microsecond),
      vector_clock: generate_vector_clock(variable_id),
      source_node: node()
    }
    
    # Gossip to random subset of replicas
    gossip_fanout = Keyword.get(opts, :gossip_fanout, 3)
    target_nodes = Enum.take_random(replica_nodes, gossip_fanout)
    
    Enum.each(target_nodes, fn target_node ->
      :rpc.cast(target_node, __MODULE__, :handle_gossip_update, [update_msg])
    end)
    
    :ok
  end
  
  defp strong_consistency_update(variable_id, new_value, opts) do
    # Use consensus protocol for strong consistency
    replica_nodes = get_replica_nodes(variable_id)
    
    # Propose update to all replicas
    proposal = %{
      variable_id: variable_id,
      value: new_value,
      proposal_id: generate_proposal_id(),
      proposer: node()
    }
    
    # Collect acknowledgments
    acks = collect_consensus_acks(replica_nodes, proposal, opts)
    
    # Check for majority
    majority_threshold = div(length(replica_nodes), 2) + 1
    
    if length(acks) >= majority_threshold do
      # Commit the update
      commit_update(replica_nodes, proposal)
      :ok
    else
      {:error, :insufficient_consensus}
    end
  end
  
  @doc """
  Handle incoming gossip update from another node.
  """
  def handle_gossip_update(update_msg) do
    # Check if this update is newer using vector clocks
    case is_newer_update?(update_msg) do
      true ->
        # Apply the update locally
        apply_variable_update(update_msg.variable_id, update_msg.value)
        
        # Continue gossiping to other nodes
        continue_gossip(update_msg)
      
      false ->
        # Ignore stale update
        :ignored
    end
  end
  
  @doc """
  Resolve conflicts when multiple nodes update the same variable.
  """
  @spec resolve_conflict(atom(), [variable_update()]) :: {:ok, term()} | {:error, term()}
  def resolve_conflict(variable_id, conflicting_updates) do
    strategy = get_conflict_resolution_strategy(variable_id)
    
    case strategy do
      :last_writer_wins -> resolve_by_timestamp(conflicting_updates)
      :vector_clocks -> resolve_by_vector_clocks(conflicting_updates)
      :consensus -> resolve_by_consensus(variable_id, conflicting_updates)
      :merge_function -> resolve_by_merge_function(variable_id, conflicting_updates)
    end
  end
  
  defp resolve_by_vector_clocks(conflicting_updates) do
    # Find the update with the most recent vector clock
    latest_update = Enum.max_by(conflicting_updates, fn update ->
      vector_clock_compare(update.vector_clock)
    end)
    
    {:ok, latest_update.value}
  end
  
  defp resolve_by_consensus(variable_id, conflicting_updates) do
    # Use cluster consensus to resolve conflict
    replica_nodes = get_replica_nodes(variable_id)
    
    # Propose each conflicting value and vote
    votes = Enum.map(conflicting_updates, fn update ->
      vote_count = collect_conflict_votes(replica_nodes, update)
      {update, vote_count}
    end)
    
    # Select the value with most votes
    {winning_update, _vote_count} = Enum.max_by(votes, fn {_update, count} -> count end)
    
    {:ok, winning_update.value}
  end
end

defmodule Foundation.MABEAM.Cluster.LoadBalancer do
  @moduledoc """
  Intelligent load balancing for agents across cluster nodes.
  
  Considers node capabilities, current load, network topology,
  and agent requirements for optimal placement.
  """
  
  @doc """
  Find the optimal node for placing a new agent.
  """
  @spec find_optimal_node(Types.agent_config(), keyword()) :: {:ok, node()} | {:error, term()}
  def find_optimal_node(agent_config, opts \\ []) do
    available_nodes = get_available_nodes()
    
    # Score each node based on multiple factors
    scored_nodes = Enum.map(available_nodes, fn node ->
      score = calculate_node_score(node, agent_config, opts)
      {node, score}
    end)
    
    # Select the highest scoring node
    case Enum.max_by(scored_nodes, fn {_node, score} -> score end, fn -> nil end) do
      nil -> {:error, :no_suitable_nodes}
      {optimal_node, _score} -> {:ok, optimal_node}
    end
  end
  
  @doc """
  Rebalance agents across cluster nodes based on current load.
  """
  @spec rebalance_cluster() :: {:ok, [migration_plan()]} | {:error, term()}
  def rebalance_cluster() do
    # Get current cluster state
    cluster_state = get_cluster_state()
    
    # Calculate optimal agent distribution
    optimal_distribution = calculate_optimal_distribution(cluster_state)
    
    # Generate migration plan
    migration_plan = generate_migration_plan(cluster_state, optimal_distribution)
    
    # Execute migrations
    execute_migration_plan(migration_plan)
  end
  
  defp calculate_node_score(node, agent_config, opts) do
    # Base score factors
    resource_score = calculate_resource_score(node, agent_config)
    load_score = calculate_load_score(node)
    capability_score = calculate_capability_score(node, agent_config)
    network_score = calculate_network_score(node, opts)
    affinity_score = calculate_affinity_score(node, agent_config, opts)
    
    # Weighted combination
    weights = Keyword.get(opts, :scoring_weights, %{
      resource: 0.3,
      load: 0.25,
      capability: 0.2,
      network: 0.15,
      affinity: 0.1
    })
    
    resource_score * weights.resource +
    load_score * weights.load +
    capability_score * weights.capability +
    network_score * weights.network +
    affinity_score * weights.affinity
  end
  
  defp calculate_resource_score(node, agent_config) do
    node_resources = get_node_resources(node)
    required_resources = agent_config.resource_requirements
    
    # Check if node has sufficient resources
    resource_ratios = [
      node_resources.memory / required_resources.memory,
      node_resources.cpu_weight / required_resources.cpu_weight
    ]
    
    # Return minimum ratio (bottleneck resource)
    Enum.min(resource_ratios)
  end
  
  defp calculate_load_score(node) do
    current_load = get_node_load(node)
    # Higher load = lower score
    max(0.0, 1.0 - current_load)
  end
  
  defp calculate_capability_score(node, agent_config) do
    node_capabilities = get_node_capabilities(node)
    required_capabilities = Map.get(agent_config, :required_capabilities, [])
    
    # Check if node supports all required capabilities
    if Enum.all?(required_capabilities, &(&1 in node_capabilities)) do
      1.0
    else
      0.0
    end
  end
  
  defp calculate_network_score(node, opts) do
    # Consider network latency and bandwidth to other nodes
    preferred_nodes = Keyword.get(opts, :preferred_nodes, [])
    
    if node in preferred_nodes do
      1.0
    else
      # Calculate based on network topology
      network_distance = calculate_network_distance(node, preferred_nodes)
      max(0.0, 1.0 - network_distance / 10.0)  # Normalize distance
    end
  end
  
  defp calculate_affinity_score(node, agent_config, opts) do
    # Consider agent affinity and anti-affinity rules
    affinity_rules = Map.get(agent_config, :affinity_rules, [])
    anti_affinity_rules = Map.get(agent_config, :anti_affinity_rules, [])
    
    affinity_score = calculate_affinity_match(node, affinity_rules)
    anti_affinity_penalty = calculate_anti_affinity_penalty(node, anti_affinity_rules)
    
    affinity_score - anti_affinity_penalty
  end
end

defmodule Foundation.MABEAM.Cluster.FaultTolerance do
  @moduledoc """
  Advanced fault tolerance mechanisms for distributed MABEAM systems.
  
  Handles node failures, network partitions, and split-brain scenarios
  while maintaining agent availability and data consistency.
  """
  
  @doc """
  Handle node failure and initiate recovery procedures.
  """
  @spec handle_node_failure(node(), term()) :: :ok
  def handle_node_failure(failed_node, failure_reason) do
    # 1. Identify affected agents
    affected_agents = get_agents_on_node(failed_node)
    
    # 2. Initiate agent recovery
    Enum.each(affected_agents, &recover_agent/1)
    
    # 3. Update distributed variables
    update_variable_replicas_after_failure(failed_node)
    
    # 4. Rebalance remaining agents
    schedule_rebalancing()
    
    # 5. Emit failure event
    Foundation.Events.emit(:mabeam_node_failed, %{
      node: failed_node,
      reason: failure_reason,
      affected_agents: length(affected_agents),
      recovery_initiated: true
    })
    
    :ok
  end
  
  @doc """
  Handle network partition and implement split-brain prevention.
  """
  @spec handle_network_partition([node()]) :: :ok
  def handle_network_partition(partitioned_nodes) do
    # Determine which partition has quorum
    total_nodes = get_total_cluster_nodes()
    our_partition_size = length(Node.list()) + 1  # Include current node
    
    has_quorum = our_partition_size > div(total_nodes, 2)
    
    if has_quorum do
      # We have quorum - continue operating
      continue_with_quorum(partitioned_nodes)
    else
      # No quorum - enter read-only mode
      enter_readonly_mode(partitioned_nodes)
    end
  end
  
  defp recover_agent(agent_id) do
    # Find the best node for agent recovery
    case Foundation.MABEAM.Cluster.LoadBalancer.find_optimal_node(get_agent_config(agent_id)) do
      {:ok, recovery_node} ->
        # Attempt to recover agent on the selected node
        case recover_agent_on_node(agent_id, recovery_node) do
          {:ok, new_pid} ->
            # Update agent location
            Foundation.MABEAM.Cluster.update_agent_location(agent_id, recovery_node)
            
            # Emit recovery event
            Foundation.Events.emit(:mabeam_agent_recovered, %{
              agent_id: agent_id,
              recovery_node: recovery_node,
              new_pid: new_pid
            })
          
          {:error, reason} ->
            # Recovery failed - try next best node
            schedule_agent_recovery_retry(agent_id, reason)
        end
      
      {:error, :no_suitable_nodes} ->
        # No suitable nodes available - agent remains unavailable
        mark_agent_unavailable(agent_id)
    end
  end
  
  defp continue_with_quorum(partitioned_nodes) do
    # Mark partitioned nodes as temporarily unavailable
    Enum.each(partitioned_nodes, &mark_node_partitioned/1)
    
    # Continue normal operations with remaining nodes
    # Variables will use remaining replicas
    # Agents on partitioned nodes will be recovered on available nodes
    
    # Set up partition healing detection
    schedule_partition_healing_check(partitioned_nodes)
  end
  
  defp enter_readonly_mode(partitioned_nodes) do
    # Enter read-only mode to prevent split-brain
    set_cluster_mode(:readonly)
    
    # Stop accepting new agent placements
    # Stop variable updates
    # Continue serving read requests
    
    # Wait for partition to heal
    schedule_partition_healing_check(partitioned_nodes)
  end
  
  @doc """
  Implement automatic cluster healing after partition recovery.
  """
  @spec heal_partition([node()]) :: :ok
  def heal_partition(recovered_nodes) do
    # 1. Verify all nodes are reachable
    reachable_nodes = Enum.filter(recovered_nodes, fn node ->
      Node.ping(node) == :pong
    end)
    
    # 2. Synchronize cluster state
    synchronize_cluster_state(reachable_nodes)
    
    # 3. Resolve any conflicts that occurred during partition
    resolve_partition_conflicts(reachable_nodes)
    
    # 4. Resume normal operations
    set_cluster_mode(:normal)
    
    # 5. Rebalance agents if needed
    schedule_post_partition_rebalancing()
    
    :ok
  end
  
  defp synchronize_cluster_state(nodes) do
    # Collect state from all nodes
    node_states = Enum.map(nodes, fn node ->
      state = :rpc.call(node, Foundation.MABEAM.Cluster, :get_local_state, [])
      {node, state}
    end)
    
    # Merge states using conflict resolution
    merged_state = merge_cluster_states(node_states)
    
    # Propagate merged state to all nodes
    Enum.each(nodes, fn node ->
      :rpc.call(node, Foundation.MABEAM.Cluster, :update_state, [merged_state])
    end)
  end
  
  defp resolve_partition_conflicts(nodes) do
    # Find conflicting variable updates
    conflicts = detect_variable_conflicts(nodes)
    
    # Resolve each conflict using configured strategy
    Enum.each(conflicts, fn conflict ->
      case Foundation.MABEAM.Cluster.VariableSync.resolve_conflict(
        conflict.variable_id, 
        conflict.conflicting_updates
      ) do
        {:ok, resolved_value} ->
          # Apply resolved value to all nodes
          apply_conflict_resolution(nodes, conflict.variable_id, resolved_value)
        
        {:error, reason} ->
          # Log conflict resolution failure
          Logger.error("Failed to resolve conflict for #{conflict.variable_id}: #{reason}")
      end
    end)
  end
end

defmodule Foundation.MABEAM.Cluster.AutoScaler do
  @moduledoc """
  Automatic cluster scaling based on load and performance metrics.
  """
  
  @doc """
  Monitor cluster load and automatically scale up/down.
  """
  @spec monitor_and_scale() :: :ok
  def monitor_and_scale() do
    cluster_metrics = collect_cluster_metrics()
    
    case determine_scaling_action(cluster_metrics) do
      :scale_up -> scale_up_cluster(cluster_metrics)
      :scale_down -> scale_down_cluster(cluster_metrics)
      :no_action -> :ok
    end
  end
  
  defp determine_scaling_action(metrics) do
    avg_cpu = metrics.average_cpu_usage
    avg_memory = metrics.average_memory_usage
    queue_length = metrics.agent_queue_length
    
    cond do
      avg_cpu > 0.8 or avg_memory > 0.8 or queue_length > 10 -> :scale_up
      avg_cpu < 0.3 and avg_memory < 0.3 and queue_length == 0 -> :scale_down
      true -> :no_action
    end
  end
end

apiVersion: apps/v1
kind: Deployment
metadata:
  name: mabeam-cluster
spec:
  replicas: 3
  selector:
    matchLabels:
      app: mabeam
  template:
    metadata:
      labels:
        app: mabeam
    spec:
      containers:
      - name: mabeam-node
        image: mabeam:latest
        env:
        - name: CLUSTER_DISCOVERY
          value: "kubernetes"
        - name: NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        ports:
        - containerPort: 4369  # EPMD
        - containerPort: 9100  # Distributed Erlang

version: '3.8'
services:
  mabeam-node-1:
    image: mabeam:latest
    environment:
      - NODE_NAME=mabeam1@mabeam-node-1
      - CLUSTER_NODES=mabeam1@mabeam-node-1,mabeam2@mabeam-node-2,mabeam3@mabeam-node-3
    ports:
      - "4369:4369"
      - "9100:9100"
  
  mabeam-node-2:
    image: mabeam:latest
    environment:
      - NODE_NAME=mabeam2@mabeam-node-2
      - CLUSTER_NODES=mabeam1@mabeam-node-1,mabeam2@mabeam-node-2,mabeam3@mabeam-node-3
    ports:
      - "4370:4369"
      - "9101:9100"
  
  mabeam-node-3:
    image: mabeam:latest
    environment:
      - NODE_NAME=mabeam3@mabeam-node-3
      - CLUSTER_NODES=mabeam1@mabeam-node-1,mabeam2@mabeam-node-2,mabeam3@mabeam-node-3
    ports:
      - "4371:4369"
      - "9102:9100"