defmodule Nexus.Security.ByDesign do
  @moduledoc """
  Security integrated into every component from inception.
  
  Security layers:
  - Network: mTLS, certificate rotation, network policies
  - Authentication: Multi-provider OAuth, RBAC, audit trails
  - Authorization: Fine-grained permissions, resource-based access
  - Data: Encryption at rest and in transit, key management
  - Operations: Secure deployment, secrets management, compliance
  """
  
  def secure_agent_communication(agent_id, target_id, message) do
    # Every message secured by default
    with {:ok, auth_context} <- authenticate_agent(agent_id),
         {:ok, _} <- authorize_communication(auth_context, target_id),
         {:ok, encrypted_message} <- encrypt_message(message, target_id),
         {:ok, signed_message} <- sign_message(encrypted_message, agent_id) do
      
      # Send with audit trail
      result = send_secure_message(target_id, signed_message)
      audit_communication(agent_id, target_id, result)
      
      result
    else
      {:error, :authentication_failed} -> 
        security_alert(:authentication_failure, agent_id)
        {:error, :unauthorized}
      
      {:error, :authorization_denied} ->
        security_alert(:authorization_failure, agent_id, target_id)
        {:error, :forbidden}
      
      error ->
        security_alert(:communication_error, agent_id, error)
        error
    end
  end
  
  defp authenticate_agent(agent_id) do
    # Multi-factor authentication for agent identity
    case Nexus.Security.Identity.verify_agent(agent_id) do
      {:ok, identity} ->
        # Verify agent certificate
        case Nexus.Security.PKI.verify_certificate(identity.certificate) do
          {:ok, _cert} -> {:ok, %{agent_id: agent_id, identity: identity}}
          error -> error
        end
      
      error -> error
    end
  end
  
  defp authorize_communication(auth_context, target_id) do
    # Fine-grained authorization
    permissions = get_agent_permissions(auth_context.agent_id)
    
    case Nexus.Security.Authorization.check_permission(
      permissions,
      :communicate,
      target_id
    ) do
      :allowed -> {:ok, :authorized}
      :denied -> {:error, :authorization_denied}
    end
  end
end

defmodule Nexus.Observability.ByDefault do
  @moduledoc """
  Comprehensive observability built into every operation.
  
  Observability pillars:
  - Metrics: Performance, business, and system metrics
  - Traces: Distributed tracing across all operations
  - Logs: Structured logging with correlation IDs
  - Events: Business and system event streams
  """
  
  defmacro observe(operation_name, do: block) do
    quote do
      operation_id = UUID.uuid4()
      start_time = :erlang.monotonic_time(:microsecond)
      
      # Start distributed trace span
      OpenTelemetry.with_span(unquote(operation_name), %{operation_id: operation_id}, fn span ->
        try do
          # Execute operation with full observability
          result = unquote(block)
          
          # Record success metrics
          duration = :erlang.monotonic_time(:microsecond) - start_time
          record_operation_success(unquote(operation_name), duration, operation_id)
          
          # Add trace attributes
          OpenTelemetry.set_attributes(span, %{
            result_type: :success,
            duration_microseconds: duration
          })
          
          result
        rescue
          error ->
            # Record failure metrics
            duration = :erlang.monotonic_time(:microsecond) - start_time
            record_operation_failure(unquote(operation_name), error, duration, operation_id)
            
            # Add error attributes to trace
            OpenTelemetry.set_attributes(span, %{
              result_type: :error,
              error_type: error.__struct__,
              duration_microseconds: duration
            })
            
            # Structured error logging
            Logger.error("Operation failed", 
              operation: unquote(operation_name),
              operation_id: operation_id,
              error: inspect(error),
              duration_microseconds: duration,
              stacktrace: __STACKTRACE__
            )
            
            reraise error, __STACKTRACE__
        end
      end)
    end
  end
  
  defp record_operation_success(operation, duration, operation_id) do
    # Prometheus metrics
    :prometheus_counter.inc(:nexus_operations_total, [operation, :success])
    :prometheus_histogram.observe(:nexus_operation_duration, [operation], duration)
    
    # Custom metrics
    :telemetry.execute(
      [:nexus, :operation, :completed],
      %{duration: duration},
      %{operation: operation, result: :success, operation_id: operation_id}
    )
    
    # Performance SLA tracking
    track_sla_compliance(operation, duration)
  end
  
  defp track_sla_compliance(operation, duration) do
    sla_threshold = get_operation_sla(operation)
    
    if duration > sla_threshold do
      # SLA violation
      :prometheus_counter.inc(:nexus_sla_violations_total, [operation])
      
      # Alert if necessary
      if should_alert_sla_violation?(operation, duration) do
        Nexus.Alerting.sla_violation_alert(operation, duration, sla_threshold)
      end
    end
  end
end

defmodule Nexus.Resilience.ByArchitecture do
  @moduledoc """
  Resilience patterns built into architectural foundations.
  
  Resilience patterns:
  - Circuit breakers: Prevent cascade failures
  - Bulkheads: Isolate failure domains
  - Timeouts: Prevent resource exhaustion
  - Retries: Handle transient failures
  - Graceful degradation: Maintain partial functionality
  """
  
  def resilient_operation(operation_name, operation_fn, opts \\ []) do
    circuit_breaker = Keyword.get(opts, :circuit_breaker, operation_name)
    timeout = Keyword.get(opts, :timeout, 5000)
    retry_policy = Keyword.get(opts, :retry_policy, :exponential_backoff)
    fallback = Keyword.get(opts, :fallback, &default_fallback/1)
    
    # Execute with full resilience patterns
    circuit_breaker
    |> execute_with_circuit_breaker(operation_fn, timeout)
    |> handle_with_retry(retry_policy)
    |> handle_with_fallback(fallback)
  end
  
  defp execute_with_circuit_breaker(circuit_name, operation_fn, timeout) do
    case Nexus.CircuitBreaker.call(circuit_name, operation_fn, timeout) do
      {:ok, result} -> {:ok, result}
      {:error, :circuit_open} -> {:error, :circuit_open}
      {:error, :timeout} -> {:error, :timeout}
      {:error, reason} -> {:error, reason}
    end
  end
  
  defp handle_with_retry({:ok, result}, _retry_policy), do: {:ok, result}
  defp handle_with_retry({:error, :circuit_open}, _retry_policy), do: {:error, :circuit_open}
  defp handle_with_retry({:error, reason}, retry_policy) do
    case should_retry?(reason, retry_policy) do
      true ->
        delay = calculate_retry_delay(retry_policy)
        Process.sleep(delay)
        
        # Note: In real implementation, would need to track retry count
        {:retry_needed, reason}
      
      false ->
        {:error, reason}
    end
  end
  
  defp handle_with_fallback({:ok, result}, _fallback), do: {:ok, result}
  defp handle_with_fallback({:error, reason}, fallback) do
    Logger.warn("Operation failed, executing fallback", reason: reason)
    
    case fallback.(reason) do
      {:ok, fallback_result} -> 
        {:ok, {:fallback, fallback_result}}
      
      {:error, fallback_error} -> 
        {:error, {:fallback_failed, reason, fallback_error}}
    end
  end
end

defmodule Nexus.CircuitBreaker.Production do
  @moduledoc """
  Production-grade circuit breaker with comprehensive monitoring.
  """
  
  use GenServer
  
  defstruct [
    :name,
    :state,           # :closed | :open | :half_open
    :failure_count,
    :success_count,
    :failure_threshold,
    :success_threshold,
    :timeout,
    :last_failure_time,
    :metrics
  ]
  
  def start_link(opts) do
    name = Keyword.fetch!(opts, :name)
    GenServer.start_link(__MODULE__, opts, name: via_tuple(name))
  end
  
  def call(circuit_name, operation, timeout \\ 5000) do
    GenServer.call(via_tuple(circuit_name), {:call, operation, timeout})
  end
  
  def init(opts) do
    state = %__MODULE__{
      name: Keyword.fetch!(opts, :name),
      state: :closed,
      failure_count: 0,
      success_count: 0,
      failure_threshold: Keyword.get(opts, :failure_threshold, 5),
      success_threshold: Keyword.get(opts, :success_threshold, 3),
      timeout: Keyword.get(opts, :timeout, 60_000),
      last_failure_time: nil,
      metrics: initialize_circuit_metrics(Keyword.fetch!(opts, :name))
    }
    
    {:ok, state}
  end
  
  def handle_call({:call, operation, timeout}, _from, state) do
    case state.state do
      :closed ->
        execute_operation(operation, timeout, state)
      
      :open ->
        if should_attempt_recovery?(state) do
          transition_to_half_open(state)
        else
          record_circuit_rejection(state.metrics)
          {:reply, {:error, :circuit_open}, state}
        end
      
      :half_open ->
        execute_recovery_test(operation, timeout, state)
    end
  end
  
  defp execute_operation(operation, timeout, state) do
    start_time = :erlang.monotonic_time(:microsecond)
    
    try do
      # Execute with timeout
      task = Task.async(operation)
      result = Task.await(task, timeout)
      
      duration = :erlang.monotonic_time(:microsecond) - start_time
      record_success(state.metrics, duration)
      
      # Reset failure count on success
      new_state = %{state | failure_count: 0}
      
      {:reply, {:ok, result}, new_state}
    catch
      :exit, {:timeout, _} ->
        record_timeout(state.metrics)
        handle_failure(state, :timeout)
      
      kind, error ->
        duration = :erlang.monotonic_time(:microsecond) - start_time
        record_failure(state.metrics, kind, error, duration)
        handle_failure(state, {kind, error})
    end
  end
  
  defp handle_failure(state, error) do
    new_failure_count = state.failure_count + 1
    new_state = %{state | 
      failure_count: new_failure_count,
      last_failure_time: :erlang.system_time(:millisecond)
    }
    
    if new_failure_count >= state.failure_threshold do
      # Open circuit
      Logger.warn("Circuit breaker opened", 
        circuit: state.name,
        failure_count: new_failure_count,
        threshold: state.failure_threshold
      )
      
      opened_state = %{new_state | state: :open}
      record_circuit_opened(state.metrics)
      
      {:reply, {:error, error}, opened_state}
    else
      {:reply, {:error, error}, new_state}
    end
  end
  
  defp initialize_circuit_metrics(circuit_name) do
    # Initialize Prometheus metrics for circuit breaker
    :prometheus_counter.declare([
      name: :nexus_circuit_calls_total,
      labels: [:circuit, :result],
      help: "Total circuit breaker calls"
    ])
    
    :prometheus_histogram.declare([
      name: :nexus_circuit_duration,
      labels: [:circuit],
      buckets: [0.001, 0.01, 0.1, 1, 10],
      help: "Circuit breaker operation duration"
    ])
    
    :prometheus_gauge.declare([
      name: :nexus_circuit_state,
      labels: [:circuit],
      help: "Circuit breaker state (0=closed, 1=open, 2=half_open)"
    ])
    
    %{circuit_name: circuit_name}
  end
  
  defp via_tuple(name) do
    {:via, Registry, {Nexus.CircuitBreaker.Registry, name}}
  end
end

defmodule Nexus.Security.Enterprise do
  @moduledoc """
  Enterprise-grade security architecture with defense in depth.
  
  Security layers:
  1. Network security: mTLS, network policies, firewalls
  2. Identity and access: Multi-provider OAuth, RBAC, audit
  3. Application security: Input validation, secure coding
  4. Data security: Encryption, key management, compliance
  5. Operational security: Secrets management, secure deployment
  """
  
  def setup_enterprise_security() do
    # Initialize security infrastructure
    setup_pki_infrastructure()
    setup_identity_providers()
    setup_rbac_system()
    setup_encryption_services()
    setup_audit_system()
    setup_compliance_monitoring()
  end
  
  defp setup_pki_infrastructure() do
    # Public Key Infrastructure for mTLS
    Nexus.Security.PKI.initialize([
      ca_cert: load_ca_certificate(),
      cert_rotation_interval: :timer.hours(24),
      crl_update_interval: :timer.hours(1),
      ocsp_responder: get_ocsp_responder_url()
    ])
  end
  
  defp setup_identity_providers() do
    # Multi-provider OAuth configuration
    identity_providers = [
      %{
        name: :azure_ad,
        type: :oauth2,
        client_id: get_secret("AZURE_CLIENT_ID"),
        client_secret: get_secret("AZURE_CLIENT_SECRET"),
        discovery_url: "https://login.microsoftonline.com/common/v2.0/.well-known/openid_configuration"
      },
      %{
        name: :okta,
        type: :saml,
        issuer: get_secret("OKTA_ISSUER"),
        certificate: get_secret("OKTA_CERTIFICATE")
      },
      %{
        name: :internal_ldap,
        type: :ldap,
        host: get_secret("LDAP_HOST"),
        port: 636,
        encryption: :ssl
      }
    ]
    
    Enum.each(identity_providers, &Nexus.Security.Identity.register_provider/1)
  end
  
  defp setup_rbac_system() do
    # Role-based access control
    Nexus.Security.RBAC.initialize([
      roles_config: load_roles_configuration(),
      permissions_config: load_permissions_configuration(),
      policy_engine: :opa,  # Open Policy Agent
      cache_ttl: :timer.minutes(5)
    ])
  end
end

defmodule Nexus.Security.Authentication do
  @moduledoc """
  Multi-factor authentication for agent and user access.
  """
  
  def authenticate_request(request) do
    with {:ok, bearer_token} <- extract_bearer_token(request),
         {:ok, token_claims} <- validate_jwt_token(bearer_token),
         {:ok, identity} <- resolve_identity(token_claims),
         {:ok, _} <- verify_identity_status(identity),
         {:ok, session} <- create_security_session(identity) do
      
      # Audit successful authentication
      audit_authentication_success(identity, request)
      
      {:ok, session}
    else
      {:error, :invalid_token} ->
        audit_authentication_failure(request, :invalid_token)
        {:error, :authentication_failed}
      
      {:error, :identity_suspended} ->
        audit_authentication_failure(request, :identity_suspended)
        {:error, :authentication_failed}
      
      error ->
        audit_authentication_failure(request, error)
        {:error, :authentication_failed}
    end
  end
  
  defp validate_jwt_token(token) do
    # JWT validation with multiple checks
    with {:ok, header} <- decode_jwt_header(token),
         {:ok, public_key} <- get_signing_key(header.kid),
         {:ok, claims} <- verify_jwt_signature(token, public_key),
         :ok <- verify_jwt_claims(claims) do
      {:ok, claims}
    else
      error -> error
    end
  end
  
  defp verify_jwt_claims(claims) do
    current_time = :os.system_time(:second)
    
    cond do
      claims["exp"] <= current_time ->
        {:error, :token_expired}
      
      claims["nbf"] > current_time ->
        {:error, :token_not_yet_valid}
      
      claims["iss"] not in get_trusted_issuers() ->
        {:error, :untrusted_issuer}
      
      claims["aud"] != get_expected_audience() ->
        {:error, :invalid_audience}
      
      true ->
        :ok
    end
  end
end

defmodule Nexus.Security.Authorization do
  @moduledoc """
  Fine-grained authorization with policy-based access control.
  """
  
  def authorize_operation(session, operation, resource) do
    # Check permissions using policy engine
    policy_request = %{
      subject: session.identity,
      action: operation,
      resource: resource,
      context: %{
        timestamp: DateTime.utc_now(),
        source_ip: session.source_ip,
        user_agent: session.user_agent
      }
    }
    
    case evaluate_policy(policy_request) do
      {:allow, policy_result} ->
        # Audit successful authorization
        audit_authorization_success(session, operation, resource, policy_result)
        {:ok, :authorized}
      
      {:deny, reason} ->
        # Audit authorization denial
        audit_authorization_denial(session, operation, resource, reason)
        {:error, :authorization_denied}
      
      {:error, error} ->
        # Policy evaluation error
        Logger.error("Policy evaluation failed", 
          session: session.id,
          operation: operation,
          resource: resource,
          error: error
        )
        {:error, :authorization_error}
    end
  end
  
  defp evaluate_policy(request) do
    # Use Open Policy Agent for policy evaluation
    case Nexus.Security.OPA.evaluate("nexus/authz", request) do
      %{"allow" => true, "reason" => reason} ->
        {:allow, %{reason: reason}}
      
      %{"allow" => false, "reason" => reason} ->
        {:deny, reason}
      
      error ->
        {:error, error}
    end
  end
end

defmodule Nexus.Security.Encryption do
  @moduledoc """
  Comprehensive encryption services for data protection.
  """
  
  def encrypt_sensitive_data(data, encryption_context \\ %{}) do
    # Use envelope encryption with key management service
    with {:ok, data_key} <- generate_data_encryption_key(),
         {:ok, encrypted_data} <- encrypt_with_aes(data, data_key),
         {:ok, encrypted_key} <- encrypt_data_key(data_key, encryption_context) do
      
      encrypted_envelope = %{
        encrypted_data: encrypted_data,
        encrypted_key: encrypted_key,
        encryption_algorithm: "AES-256-GCM",
        key_id: get_master_key_id(),
        encryption_context: encryption_context
      }
      
      {:ok, encrypted_envelope}
    else
      error -> error
    end
  end
  
  def decrypt_sensitive_data(encrypted_envelope) do
    with {:ok, data_key} <- decrypt_data_key(encrypted_envelope.encrypted_key, encrypted_envelope.encryption_context),
         {:ok, decrypted_data} <- decrypt_with_aes(encrypted_envelope.encrypted_data, data_key) do
      
      # Securely clear data key from memory
      :crypto.strong_rand_bytes(byte_size(data_key))  # Overwrite
      
      {:ok, decrypted_data}
    else
      error -> error
    end
  end
  
  defp generate_data_encryption_key() do
    # Generate 256-bit AES key
    key = :crypto.strong_rand_bytes(32)
    {:ok, key}
  end
  
  defp encrypt_with_aes(data, key) do
    # AES-256-GCM encryption
    iv = :crypto.strong_rand_bytes(12)  # 96-bit IV for GCM
    
    case :crypto.crypto_one_time_aead(:aes_256_gcm, key, iv, data, "", true) do
      {ciphertext, tag} ->
        encrypted_data = %{
          ciphertext: Base.encode64(ciphertext),
          tag: Base.encode64(tag),
          iv: Base.encode64(iv)
        }
        {:ok, encrypted_data}
      
      error ->
        {:error, {:encryption_failed, error}}
    end
  end
  
  defp encrypt_data_key(data_key, encryption_context) do
    # Encrypt data key using KMS
    Nexus.Security.KMS.encrypt(get_master_key_id(), data_key, encryption_context)
  end
end

defmodule Nexus.Monitoring.Comprehensive do
  @moduledoc """
  Production-grade monitoring covering all system dimensions.
  
  Monitoring dimensions:
  - Performance: Latency, throughput, resource utilization
  - Business: Agent coordination success, goal completion rates
  - Security: Authentication failures, authorization violations
  - Infrastructure: Node health, network connectivity, storage
  - Cost: Resource consumption, compute costs, optimization opportunities
  """
  
  def setup_comprehensive_monitoring() do
    # Initialize monitoring infrastructure
    setup_metrics_collection()
    setup_distributed_tracing()
    setup_structured_logging()
    setup_business_metrics()
    setup_security_monitoring()
    setup_cost_tracking()
    setup_alerting_system()
  end
  
  defp setup_metrics_collection() do
    # Prometheus metrics for technical monitoring
    technical_metrics = [
      # Performance metrics
      :prometheus_histogram.declare([
        name: :nexus_operation_duration_seconds,
        labels: [:operation, :agent_type, :result],
        buckets: [0.001, 0.01, 0.1, 1, 10],
        help: "Operation duration in seconds"
      ]),
      
      # Throughput metrics
      :prometheus_counter.declare([
        name: :nexus_operations_total,
        labels: [:operation, :agent_type, :result],
        help: "Total operations processed"
      ]),
      
      # Resource metrics
      :prometheus_gauge.declare([
        name: :nexus_active_agents,
        labels: [:node, :agent_type],
        help: "Number of active agents"
      ]),
      
      # Error metrics
      :prometheus_counter.declare([
        name: :nexus_errors_total,
        labels: [:error_type, :component, :severity],
        help: "Total errors encountered"
      ])
    ]
    
    Enum.each(technical_metrics, & &1)
  end
  
  defp setup_business_metrics() do
    # Business-specific metrics for operational insights
    business_metrics = [
      # Coordination success rate
      :prometheus_histogram.declare([
        name: :nexus_coordination_success_rate,
        labels: [:coordination_type, :cluster_size],
        buckets: [0.0, 0.5, 0.8, 0.9, 0.95, 0.99, 1.0],
        help: "Coordination success rate"
      ]),
      
      # Goal completion metrics
      :prometheus_histogram.declare([
        name: :nexus_goal_completion_time,
        labels: [:goal_type, :complexity],
        buckets: [1, 10, 60, 300, 1800, 3600],
        help: "Goal completion time in seconds"
      ]),
      
      # Agent efficiency metrics
      :prometheus_gauge.declare([
        name: :nexus_agent_efficiency_score,
        labels: [:agent_type, :task_type],
        help: "Agent efficiency score (0-1)"
      ])
    ]
    
    Enum.each(business_metrics, & &1)
  end
  
  defp setup_security_monitoring() do
    # Security-focused monitoring
    security_metrics = [
      # Authentication metrics
      :prometheus_counter.declare([
        name: :nexus_auth_attempts_total,
        labels: [:provider, :result, :method],
        help: "Authentication attempts"
      ]),
      
      # Authorization metrics
      :prometheus_counter.declare([
        name: :nexus_authz_decisions_total,
        labels: [:resource_type, :operation, :decision],
        help: "Authorization decisions"
      ]),
      
      # Security violations
      :prometheus_counter.declare([
        name: :nexus_security_violations_total,
        labels: [:violation_type, :severity, :source],
        help: "Security violations detected"
      ])
    ]
    
    Enum.each(security_metrics, & &1)
  end
end

defmodule Nexus.Monitoring.DistributedTracing do
  @moduledoc """
  Distributed tracing across agent interactions and system boundaries.
  """
  
  def setup_distributed_tracing() do
    # OpenTelemetry configuration
    :opentelemetry.set_default_tracer({:opentelemetry, :nexus_tracer})
    
    # Configure trace exporters
    configure_trace_exporters()
    
    # Set up sampling strategy
    configure_trace_sampling()
    
    # Initialize context propagation
    setup_context_propagation()
  end
  
  defp configure_trace_exporters() do
    exporters = [
      # Jaeger for distributed tracing
      {:otel_exporter_jaeger, %{
        endpoint: get_jaeger_endpoint(),
        service_name: "nexus-agent-system",
        service_version: get_service_version()
      }},
      
      # OTLP for OpenTelemetry collector
      {:otel_exporter_otlp, %{
        endpoint: get_otlp_endpoint(),
        headers: get_otlp_headers()
      }}
    ]
    
    Enum.each(exporters, fn {exporter, config} ->
      :otel_batch_processor.set_exporter(exporter, config)
    end)
  end
  
  def trace_agent_operation(agent_id, operation, metadata \\ %{}) do
    OpenTelemetry.with_span("agent.#{operation}", %{
      "agent.id" => agent_id,
      "agent.operation" => operation
    }, fn span ->
      # Add custom attributes
      OpenTelemetry.set_attributes(span, Map.merge(%{
        "nexus.component" => "agent",
        "nexus.cluster_id" => get_cluster_id(),
        "nexus.node_id" => node()
      }, metadata))
      
      # Execute operation with tracing
      yield()
    end)
  end
  
  def trace_coordination_flow(coordination_id, agents, goal) do
    OpenTelemetry.with_span("coordination.execute", %{
      "coordination.id" => coordination_id,
      "coordination.agent_count" => length(agents),
      "coordination.goal_type" => goal.type
    }, fn span ->
      # Trace individual agent interactions
      agent_spans = Enum.map(agents, fn agent ->
        trace_agent_interaction(agent, goal, span)
      end)
      
      # Wait for all agent interactions to complete
      Enum.map(agent_spans, &Task.await/1)
    end)
  end
  
  defp trace_agent_interaction(agent, goal, parent_span) do
    Task.async(fn ->
      OpenTelemetry.with_span("agent.interaction", %{
        "agent.id" => agent.id,
        "agent.type" => agent.type,
        "goal.type" => goal.type
      }, fn _span ->
        # Link to parent coordination span
        OpenTelemetry.link_span(parent_span)
        
        # Execute agent interaction
        Nexus.Agent.execute_goal(agent.id, goal)
      end)
    end)
  end
end

defmodule Nexus.Monitoring.Alerting do
  @moduledoc """
  Intelligent alerting system with escalation and noise reduction.
  """
  
  use GenServer
  
  defstruct [
    :alert_rules,
    :escalation_policies,
    :notification_channels,
    :alert_history,
    :suppression_rules
  ]
  
  def start_link(opts) do
    GenServer.start_link(__MODULE__, opts, name: __MODULE__)
  end
  
  def init(_opts) do
    state = %__MODULE__{
      alert_rules: load_alert_rules(),
      escalation_policies: load_escalation_policies(),
      notification_channels: configure_notification_channels(),
      alert_history: :ets.new(:alert_history, [:ordered_set, :public]),
      suppression_rules: load_suppression_rules()
    }
    
    # Start alert evaluation loop
    schedule_alert_evaluation()
    
    {:ok, state}
  end
  
  def trigger_alert(alert_type, severity, metadata) do
    GenServer.cast(__MODULE__, {:trigger_alert, alert_type, severity, metadata})
  end
  
  def handle_cast({:trigger_alert, alert_type, severity, metadata}, state) do
    # Check suppression rules
    case should_suppress_alert?(alert_type, metadata, state.suppression_rules) do
      true ->
        Logger.debug("Alert suppressed", alert_type: alert_type, metadata: metadata)
        {:noreply, state}
      
      false ->
        # Create alert
        alert = create_alert(alert_type, severity, metadata)
        
        # Store in history
        :ets.insert(state.alert_history, {alert.id, alert})
        
        # Process alert
        process_alert(alert, state)
        
        {:noreply, state}
    end
  end
  
  def handle_info(:evaluate_alerts, state) do
    # Evaluate current system state against alert rules
    current_metrics = collect_current_metrics()
    
    Enum.each(state.alert_rules, fn rule ->
      case evaluate_alert_rule(rule, current_metrics) do
        {:trigger, alert_data} ->
          trigger_alert(rule.alert_type, rule.severity, alert_data)
        
        :no_alert ->
          :ok
      end
    end)
    
    # Schedule next evaluation
    schedule_alert_evaluation()
    
    {:noreply, state}
  end
  
  defp load_alert_rules() do
    [
      # Performance alerts
      %{
        id: :high_latency,
        alert_type: :performance_degradation,
        severity: :warning,
        condition: fn metrics ->
          p95_latency = get_p95_latency(metrics)
          p95_latency > 1000  # >1 second P95 latency
        end,
        description: "High operation latency detected"
      },
      
      %{
        id: :low_success_rate,
        alert_type: :coordination_failure,
        severity: :critical,
        condition: fn metrics ->
          success_rate = get_coordination_success_rate(metrics)
          success_rate < 0.95  # <95% success rate
        end,
        description: "Low coordination success rate"
      },
      
      # Security alerts
      %{
        id: :authentication_failures,
        alert_type: :security_incident,
        severity: :high,
        condition: fn metrics ->
          failure_rate = get_auth_failure_rate(metrics)
          failure_rate > 0.1  # >10% auth failure rate
        end,
        description: "High authentication failure rate"
      },
      
      # Resource alerts
      %{
        id: :memory_pressure,
        alert_type: :resource_exhaustion,
        severity: :warning,
        condition: fn metrics ->
          memory_usage = get_memory_usage(metrics)
          memory_usage > 0.85  # >85% memory usage
        end,
        description: "High memory utilization"
      }
    ]
  end
  
  defp process_alert(alert, state) do
    # Find escalation policy
    escalation_policy = find_escalation_policy(alert, state.escalation_policies)
    
    # Send notifications according to policy
    Enum.each(escalation_policy.steps, fn step ->
      schedule_notification(alert, step, escalation_policy)
    end)
    
    # Log alert
    Logger.warn("Alert triggered", 
      alert_id: alert.id,
      type: alert.type,
      severity: alert.severity,
      metadata: alert.metadata
    )
  end
  
  defp schedule_notification(alert, step, policy) do
    Process.send_after(self(), 
      {:send_notification, alert, step, policy}, 
      step.delay * 1000
    )
  end
end

defmodule Nexus.Deployment.ZeroDowntime do
  @moduledoc """
  Zero-downtime deployment strategy for distributed agent systems.
  
  Deployment strategies:
  - Blue-green: Full environment switching
  - Rolling: Gradual node replacement
  - Canary: Progressive traffic shifting
  - A/B testing: Feature flag-based deployment
  """
  
  def deploy_new_version(deployment_config) do
    with {:ok, deployment_plan} <- create_deployment_plan(deployment_config),
         {:ok, _} <- validate_deployment_plan(deployment_plan),
         {:ok, _} <- execute_pre_deployment_checks(deployment_plan),
         {:ok, deployment_state} <- execute_deployment(deployment_plan),
         {:ok, _} <- execute_post_deployment_validation(deployment_state) do
      
      Logger.info("Zero-downtime deployment completed successfully",
        deployment_id: deployment_plan.id,
        strategy: deployment_plan.strategy,
        version: deployment_config.version
      )
      
      {:ok, deployment_state}
    else
      {:error, reason} ->
        Logger.error("Deployment failed", reason: reason)
        execute_rollback_if_needed(deployment_config)
        {:error, reason}
    end
  end
  
  defp create_deployment_plan(config) do
    deployment_plan = %{
      id: UUID.uuid4(),
      strategy: config.strategy || :rolling,
      source_version: get_current_version(),
      target_version: config.version,
      cluster_nodes: get_cluster_nodes(),
      agent_migration_strategy: config.agent_migration || :gradual,
      validation_criteria: config.validation || default_validation_criteria(),
      rollback_threshold: config.rollback_threshold || 0.95,
      deployment_timeout: config.timeout || :timer.minutes(30)
    }
    
    {:ok, deployment_plan}
  end
  
  defp execute_deployment(%{strategy: :blue_green} = plan) do
    # Blue-green deployment: deploy to parallel environment
    with {:ok, green_cluster} <- provision_green_cluster(plan),
         {:ok, _} <- deploy_to_green_cluster(green_cluster, plan),
         {:ok, _} <- validate_green_cluster(green_cluster, plan),
         {:ok, _} <- migrate_agents_to_green(green_cluster, plan),
         {:ok, _} <- switch_traffic_to_green(green_cluster, plan),
         {:ok, _} <- monitor_green_cluster_stability(green_cluster, plan) do
      
      # Green deployment successful, cleanup blue
      cleanup_blue_cluster(plan)
      
      {:ok, %{deployment_plan: plan, active_cluster: green_cluster}}
    else
      error ->
        # Rollback to blue cluster
        rollback_to_blue_cluster(plan)
        error
    end
  end
  
  defp execute_deployment(%{strategy: :rolling} = plan) do
    # Rolling deployment: replace nodes gradually
    nodes = plan.cluster_nodes
    batch_size = calculate_rolling_batch_size(length(nodes))
    
    Enum.chunk_every(nodes, batch_size)
    |> Enum.reduce_while({:ok, []}, fn batch, {:ok, updated_nodes} ->
      case deploy_to_node_batch(batch, plan) do
        {:ok, new_nodes} ->
          # Wait for stabilization
          Process.sleep(get_stabilization_delay())
          
          # Validate batch deployment
          case validate_batch_deployment(new_nodes, plan) do
            :ok -> {:cont, {:ok, updated_nodes ++ new_nodes}}
            error -> {:halt, error}
          end
        
        error ->
          # Rollback previous batches
          rollback_node_batches(updated_nodes, plan)
          {:halt, error}
      end
    end)
  end
  
  defp execute_deployment(%{strategy: :canary} = plan) do
    # Canary deployment: gradual traffic shifting
    canary_percentage = 5  # Start with 5% traffic
    
    with {:ok, canary_nodes} <- deploy_canary_nodes(plan, canary_percentage),
         {:ok, _} <- shift_traffic_to_canary(canary_nodes, canary_percentage),
         {:ok, _} <- monitor_canary_performance(canary_nodes, plan) do
      
      # Gradually increase canary traffic
      increase_canary_traffic(canary_nodes, plan, canary_percentage)
    else
      error ->
        rollback_canary_deployment(plan)
        error
    end
  end
  
  defp deploy_to_node_batch(nodes, plan) do
    # Deploy new version to batch of nodes
    deployment_tasks = Enum.map(nodes, fn node ->
      Task.async(fn ->
        deploy_to_single_node(node, plan)
      end)
    end)
    
    # Wait for all deployments in batch
    results = Task.yield_many(deployment_tasks, plan.deployment_timeout)
    
    case Enum.all?(results, fn {_task, result} -> match?({:ok, _}, result) end) do
      true ->
        successful_nodes = Enum.map(results, fn {_task, {:ok, node}} -> node end)
        {:ok, successful_nodes}
      
      false ->
        # Some deployments failed
        failed_results = Enum.filter(results, fn {_task, result} -> 
          not match?({:ok, _}, result) 
        end)
        {:error, {:batch_deployment_failed, failed_results}}
    end
  end
  
  defp deploy_to_single_node(node, plan) do
    Logger.info("Deploying to node", node: node, version: plan.target_version)
    
    with {:ok, _} <- drain_node_traffic(node),
         {:ok, _} <- migrate_agents_from_node(node),
         {:ok, _} <- stop_old_version_on_node(node),
         {:ok, _} <- start_new_version_on_node(node, plan.target_version),
         {:ok, _} <- validate_node_health(node),
         {:ok, _} <- restore_node_traffic(node) do
      
      Logger.info("Node deployment successful", node: node)
      {:ok, node}
    else
      error ->
        Logger.error("Node deployment failed", node: node, error: error)
        # Attempt node recovery
        recover_failed_node(node, plan.source_version)
        error
    end
  end
  
  defp validate_batch_deployment(nodes, plan) do
    # Validate that deployed nodes are healthy and performing well
    validation_tasks = Enum.map(nodes, fn node ->
      Task.async(fn ->
        validate_node_post_deployment(node, plan.validation_criteria)
      end)
    end)
    
    results = Task.yield_many(validation_tasks, :timer.minutes(5))
    
    case Enum.all?(results, fn {_task, result} -> result == {:ok, :valid} end) do
      true -> :ok
      false -> {:error, :batch_validation_failed}
    end
  end
  
  defp validate_node_post_deployment(node, criteria) do
    # Run comprehensive validation on deployed node
    validations = [
      {fn -> check_node_health(node) end, "Health check"},
      {fn -> check_agent_functionality(node) end, "Agent functionality"},
      {fn -> check_performance_metrics(node, criteria) end, "Performance metrics"},
      {fn -> check_security_status(node) end, "Security status"}
    ]
    
    Enum.reduce_while(validations, :ok, fn {validation_fn, description}, _acc ->
      case validation_fn.() do
        :ok -> {:cont, :ok}
        error ->
          Logger.error("Node validation failed", 
            node: node, 
            validation: description, 
            error: error
          )
          {:halt, {:error, {description, error}}}
      end
    end)
    |> case do
      :ok -> {:ok, :valid}
      error -> error
    end
  end
end

defmodule Nexus.Deployment.Kubernetes do
  @moduledoc """
  Kubernetes-native deployment and orchestration for Nexus agents.
  """
  
  def deploy_to_kubernetes(deployment_config) do
    with {:ok, k8s_manifests} <- generate_k8s_manifests(deployment_config),
         {:ok, _} <- validate_k8s_manifests(k8s_manifests),
         {:ok, _} <- apply_k8s_manifests(k8s_manifests),
         {:ok, deployment_status} <- monitor_k8s_deployment(deployment_config) do
      
      {:ok, deployment_status}
    else
      error -> error
    end
  end
  
  defp generate_k8s_manifests(config) do
    manifests = %{
      namespace: generate_namespace_manifest(config),
      configmap: generate_configmap_manifest(config),
      secret: generate_secret_manifest(config),
      deployment: generate_deployment_manifest(config),
      service: generate_service_manifest(config),
      ingress: generate_ingress_manifest(config),
      servicemonitor: generate_monitoring_manifest(config),
      networkpolicy: generate_network_policy_manifest(config),
      poddisruptionbudget: generate_pdb_manifest(config),
      horizontalpodautoscaler: generate_hpa_manifest(config)
    }
    
    {:ok, manifests}
  end
  
  defp generate_deployment_manifest(config) do
    %{
      apiVersion: "apps/v1",
      kind: "Deployment",
      metadata: %{
        name: "nexus-agents",
        namespace: config.namespace,
        labels: %{
          app: "nexus",
          component: "agents",
          version: config.version
        }
      },
      spec: %{
        replicas: config.replicas || 3,
        strategy: %{
          type: "RollingUpdate",
          rollingUpdate: %{
            maxUnavailable: 1,
            maxSurge: 1
          }
        },
        selector: %{
          matchLabels: %{
            app: "nexus",
            component: "agents"
          }
        },
        template: %{
          metadata: %{
            labels: %{
              app: "nexus",
              component: "agents",
              version: config.version
            },
            annotations: %{
              "prometheus.io/scrape": "true",
              "prometheus.io/port": "9090",
              "prometheus.io/path": "/metrics"
            }
          },
          spec: %{
            serviceAccountName: "nexus-agents",
            securityContext: %{
              runAsNonRoot: true,
              runAsUser: 65534,
              fsGroup: 65534
            },
            containers: [
              %{
                name: "nexus-agent",
                image: "#{config.image_repository}:#{config.version}",
                imagePullPolicy: "IfNotPresent",
                ports: [
                  %{name: "http", containerPort: 4000, protocol: "TCP"},
                  %{name: "epmd", containerPort: 4369, protocol: "TCP"},
                  %{name: "distributed", containerPort: 9100, protocol: "TCP"},
                  %{name: "metrics", containerPort: 9090, protocol: "TCP"}
                ],
                env: [
                  %{name: "RELEASE_DISTRIBUTION", value: "name"},
                  %{name: "RELEASE_NODE", value: "nexus@$(POD_IP)"},
                  %{name: "POD_IP", valueFrom: %{fieldRef: %{fieldPath: "status.podIP"}}},
                  %{name: "POD_NAME", valueFrom: %{fieldRef: %{fieldPath: "metadata.name"}}},
                  %{name: "POD_NAMESPACE", valueFrom: %{fieldRef: %{fieldPath: "metadata.namespace"}}}
                ],
                envFrom: [
                  %{configMapRef: %{name: "nexus-config"}},
                  %{secretRef: %{name: "nexus-secrets"}}
                ],
                resources: %{
                  requests: %{
                    memory: "512Mi",
                    cpu: "500m"
                  },
                  limits: %{
                    memory: "2Gi",
                    cpu: "2000m"
                  }
                },
                livenessProbe: %{
                  httpGet: %{
                    path: "/health",
                    port: "http"
                  },
                  initialDelaySeconds: 30,
                  periodSeconds: 10,
                  timeoutSeconds: 5,
                  failureThreshold: 3
                },
                readinessProbe: %{
                  httpGet: %{
                    path: "/ready",
                    port: "http"
                  },
                  initialDelaySeconds: 5,
                  periodSeconds: 5,
                  timeoutSeconds: 3,
                  failureThreshold: 2
                },
                volumeMounts: [
                  %{
                    name: "config-volume",
                    mountPath: "/app/config",
                    readOnly: true
                  },
                  %{
                    name: "tmp-volume",
                    mountPath: "/tmp"
                  }
                ]
              }
            ],
            volumes: [
              %{
                name: "config-volume",
                configMap: %{
                  name: "nexus-config"
                }
              },
              %{
                name: "tmp-volume",
                emptyDir: %{}
              }
            ],
            affinity: %{
              podAntiAffinity: %{
                preferredDuringSchedulingIgnoredDuringExecution: [
                  %{
                    weight: 100,
                    podAffinityTerm: %{
                      labelSelector: %{
                        matchExpressions: [
                          %{
                            key: "app",
                            operator: "In",
                            values: ["nexus"]
                          }
                        ]
                      },
                      topologyKey: "kubernetes.io/hostname"
                    }
                  }
                ]
              }
            }
          }
        }
      }
    }
  end
  
  defp generate_hpa_manifest(config) do
    %{
      apiVersion: "autoscaling/v2",
      kind: "HorizontalPodAutoscaler",
      metadata: %{
        name: "nexus-agents-hpa",
        namespace: config.namespace
      },
      spec: %{
        scaleTargetRef: %{
          apiVersion: "apps/v1",
          kind: "Deployment",
          name: "nexus-agents"
        },
        minReplicas: config.min_replicas || 3,
        maxReplicas: config.max_replicas || 50,
        metrics: [
          %{
            type: "Resource",
            resource: %{
              name: "cpu",
              target: %{
                type: "Utilization",
                averageUtilization: 70
              }
            }
          },
          %{
            type: "Resource",
            resource: %{
              name: "memory",
              target: %{
                type: "Utilization",
                averageUtilization: 80
              }
            }
          },
          %{
            type: "Pods",
            pods: %{
              metric: %{
                name: "nexus_active_agents_per_pod"
              },
              target: %{
                type: "AverageValue",
                averageValue: "1000"
              }
            }
          }
        ],
        behavior: %{
          scaleUp: %{
            stabilizationWindowSeconds: 300,
            policies: [
              %{
                type: "Percent",
                value: 100,
                periodSeconds: 60
              },
              %{
                type: "Pods",
                value: 2,
                periodSeconds: 60
              }
            ]
          },
          scaleDown: %{
            stabilizationWindowSeconds: 600,
            policies: [
              %{
                type: "Percent",
                value: 10,
                periodSeconds: 60
              }
            ]
          }
        }
      }
    }
  end
end