# DSPy Pattern
class QASignature(dspy.Signature):
    """Answer questions with reasoning."""
    question: str = dspy.InputField()
    answer: str = dspy.OutputField(desc="reasoning and answer")

# Native Elixir Pattern with ExDantic Integration
defmodule QASignature do
  use DSPex.Signature.Native
  
  @doc "Answer questions with reasoning."
  signature question: :string -> answer: :string, desc: "reasoning and answer"
end

defmodule DSPex.Signature.ExDanticCompiler do
  @moduledoc """
  Compiles DSPy signatures into ExDantic schemas with provider-specific optimizations.
  """
  
  alias Exdantic.{Schema, TypeAdapter, Config}
  alias DSPex.Types.{MLTypes, ProviderTypes}
  
  def compile_signature(signature_ast, provider \\ :openai) do
    {input_fields, output_fields} = parse_signature_ast(signature_ast)
    
    # Create ExDantic schemas with provider optimizations
    input_schema = create_input_schema(input_fields, provider)
    output_schema = create_output_schema(output_fields, provider)
    
    # Generate provider-specific JSON schemas
    json_schemas = generate_provider_schemas(input_schema, output_schema, provider)
    
    # Create validation pipeline
    validation_pipeline = create_validation_pipeline(input_schema, output_schema)
    
    %DSPex.Signature.Compiled{
      input_schema: input_schema,
      output_schema: output_schema,
      json_schemas: json_schemas,
      validation_pipeline: validation_pipeline,
      provider_optimizations: get_provider_optimizations(provider)
    }
  end
  
  defp create_input_schema(fields, provider) do
    # Use ExDantic's advanced features for ML-specific validation
    schema_config = %{
      title: "InputSchema",
      description: "Input validation for ML operations",
      provider_hints: get_provider_hints(provider),
      validation_config: %{
        coercion_enabled: true,
        strict_mode: false,
        custom_validators: get_ml_validators()
      }
    }
    
    field_definitions = Enum.map(fields, fn {name, type, constraints} ->
      {name, %{
        type: convert_to_exdantic_type(type),
        constraints: convert_constraints(constraints),
        metadata: %{
          ml_type: classify_ml_type(type),
          provider_optimization: get_field_optimization(type, provider)
        }
      }}
    end)
    
    Exdantic.create_model(field_definitions, schema_config)
  end
  
  defp create_output_schema(fields, provider) do
    # Enhanced output validation with ML-specific patterns
    schema_config = %{
      title: "OutputSchema", 
      description: "Output validation for ML operations",
      provider_hints: get_provider_hints(provider),
      validation_config: %{
        coercion_enabled: true,
        quality_assessment: true,
        structured_output_validation: true
      }
    }
    
    field_definitions = Enum.map(fields, fn {name, type, constraints} ->
      {name, %{
        type: convert_to_exdantic_type(type),
        constraints: convert_constraints(constraints),
        validators: get_output_validators(type),
        metadata: %{
          quality_metrics: get_quality_metrics(type),
          extraction_patterns: get_extraction_patterns(type, provider)
        }
      }}
    end)
    
    Exdantic.create_model(field_definitions, schema_config)
  end
  
  defp generate_provider_schemas(input_schema, output_schema, provider) do
    case provider do
      :openai ->
        %{
          function_calling: generate_openai_function_schema(input_schema, output_schema),
          structured_output: generate_openai_structured_schema(output_schema),
          json_mode: generate_openai_json_schema(output_schema)
        }
      
      :anthropic ->
        %{
          tool_calling: generate_anthropic_tool_schema(input_schema, output_schema),
          structured_output: generate_anthropic_structured_schema(output_schema)
        }
      
      :generic ->
        %{
          json_schema: Exdantic.JsonSchema.generate_schema(output_schema),
          validation_schema: Exdantic.JsonSchema.generate_validation_schema(input_schema)
        }
    end
  end
end

defmodule DSPex.Signature.Cache do
  use GenServer
  
  @table_name :signature_cache
  @hot_signatures_table :hot_signatures
  @compilation_locks_table :compilation_locks
  
  defstruct [
    :max_size,
    :current_size,
    :hit_count,
    :miss_count,
    :eviction_strategy,
    :compilation_locks
  ]
  
  def start_link(opts) do
    GenServer.start_link(__MODULE__, opts, name: __MODULE__)
  end
  
  def init(opts) do
    # Create ETS tables optimized for concurrent access
    :ets.new(@table_name, [
      :named_table, 
      :public, 
      {:read_concurrency, true},
      {:write_concurrency, true},
      {:decentralized_counters, true}
    ])
    
    :ets.new(@hot_signatures_table, [
      :named_table,
      :public,
      :ordered_set,
      {:read_concurrency, true}
    ])
    
    :ets.new(@compilation_locks_table, [
      :named_table,
      :public,
      {:read_concurrency, true},
      {:write_concurrency, true}
    ])
    
    state = %__MODULE__{
      max_size: Keyword.get(opts, :max_size, 10_000),
      current_size: 0,
      hit_count: 0,
      miss_count: 0,
      eviction_strategy: Keyword.get(opts, :eviction_strategy, :lru),
      compilation_locks: %{}
    }
    
    # Start periodic maintenance
    :timer.send_interval(300_000, :maintenance)  # 5 minutes
    
    {:ok, state}
  end
  
  @doc """
  Get compiled signature with high-performance lookup.
  Returns {:ok, compiled} | {:error, :not_cached} | {:error, :compiling}
  """
  def get_compiled(signature_hash) do
    case :ets.lookup(@table_name, signature_hash) do
      [{^signature_hash, compiled, access_count, last_access}] ->
        # Update access statistics atomically
        new_access = access_count + 1
        new_last_access = System.monotonic_time(:millisecond)
        
        :ets.update_element(@table_name, signature_hash, [
          {3, new_access},
          {4, new_last_access}
        ])
        
        # Update hot signatures tracking
        :ets.insert(@hot_signatures_table, {new_last_access, signature_hash, new_access})
        
        {:ok, compiled}
      
      [] ->
        # Check if compilation is in progress
        case :ets.lookup(@compilation_locks_table, signature_hash) do
          [{^signature_hash, _lock_ref, _timestamp}] ->
            {:error, :compiling}
          
          [] ->
            {:error, :not_cached}
        end
    end
  end
  
  @doc """
  Store compiled signature with intelligent caching.
  """
  def store_compiled(signature_hash, compiled) do
    GenServer.call(__MODULE__, {:store_compiled, signature_hash, compiled})
  end
  
  def handle_call({:store_compiled, signature_hash, compiled}, _from, state) do
    current_time = System.monotonic_time(:millisecond)
    
    # Check if we need to evict entries
    new_state = maybe_evict_entries(state)
    
    # Store the compiled signature
    :ets.insert(@table_name, {signature_hash, compiled, 1, current_time})
    :ets.insert(@hot_signatures_table, {current_time, signature_hash, 1})
    
    # Update size tracking
    updated_state = %{new_state | current_size: new_state.current_size + 1}
    
    {:reply, :ok, updated_state}
  end
  
  @doc """
  Acquire compilation lock to prevent duplicate work.
  """
  def acquire_compilation_lock(signature_hash) do
    lock_ref = make_ref()
    timestamp = System.monotonic_time(:millisecond)
    
    case :ets.insert_new(@compilation_locks_table, {signature_hash, lock_ref, timestamp}) do
      true ->
        {:ok, lock_ref}
      
      false ->
        {:error, :already_locked}
    end
  end
  
  @doc """
  Release compilation lock.
  """
  def release_compilation_lock(signature_hash, lock_ref) do
    case :ets.lookup(@compilation_locks_table, signature_hash) do
      [{^signature_hash, ^lock_ref, _timestamp}] ->
        :ets.delete(@compilation_locks_table, signature_hash)
        :ok
      
      _ ->
        {:error, :invalid_lock}
    end
  end
  
  def handle_info(:maintenance, state) do
    # Perform cache maintenance
    new_state = perform_cache_maintenance(state)
    {:noreply, new_state}
  end
  
  defp maybe_evict_entries(state) do
    if state.current_size >= state.max_size do
      evict_entries(state)
    else
      state
    end
  end
  
  defp evict_entries(state) do
    eviction_count = div(state.max_size, 10)  # Evict 10% of entries
    
    case state.eviction_strategy do
      :lru ->
        evict_lru_entries(eviction_count)
      
      :lfu ->
        evict_lfu_entries(eviction_count)
      
      :ttl ->
        evict_expired_entries()
    end
    
    %{state | current_size: state.current_size - eviction_count}
  end
  
  defp evict_lru_entries(count) do
    # Find least recently used entries
    oldest_entries = :ets.select(@hot_signatures_table, [
      {{'$1', '$2', '$3'}, [], [['$1', '$2', '$3']]}
    ])
    |> Enum.sort()
    |> Enum.take(count)
    
    Enum.each(oldest_entries, fn [_timestamp, signature_hash, _access_count] ->
      :ets.delete(@table_name, signature_hash)
      :ets.match_delete(@hot_signatures_table, {'_', signature_hash, '_'})
    end)
  end
end

defmodule DSPex.Module.Native do
  @moduledoc """
  Native Elixir implementation of DSPy module system using GenServer for state management.
  """
  
  defmacro __using__(opts) do
    quote do
      use GenServer
      import DSPex.Module.Native
      
      # Module registration
      Module.register_attribute(__MODULE__, :module_parameters, accumulate: true)
      Module.register_attribute(__MODULE__, :module_predictors, accumulate: true)
      Module.register_attribute(__MODULE__, :module_metadata, accumulate: false)
      
      @before_compile DSPex.Module.Native
    end
  end
  
  defmacro parameter(name, type, opts \\ []) do
    quote do
      @module_parameters {unquote(name), unquote(type), unquote(opts)}
    end
  end
  
  defmacro predictor(name, signature_module, opts \\ []) do
    quote do
      @module_predictors {unquote(name), unquote(signature_module), unquote(opts)}
    end
  end
  
  defmacro __before_compile__(env) do
    parameters = Module.get_attribute(env.module, :module_parameters)
    predictors = Module.get_attribute(env.module, :module_predictors)
    
    quote do
      @module_metadata %{
        parameters: unquote(Macro.escape(parameters)),
        predictors: unquote(Macro.escape(predictors)),
        module: __MODULE__
      }
      
      # Generate parameter access functions
      unquote(generate_parameter_functions(parameters))
      
      # Generate predictor access functions
      unquote(generate_predictor_functions(predictors))
      
      # GenServer callbacks
      def init(opts) do
        state = %DSPex.Module.State{
          module: __MODULE__,
          parameters: initialize_parameters(unquote(Macro.escape(parameters)), opts),
          predictors: initialize_predictors(unquote(Macro.escape(predictors)), opts),
          execution_history: [],
          optimization_state: %{}
        }
        
        {:ok, state}
      end
      
      def handle_call({:forward, inputs}, from, state) do
        # Execute forward pass with full state tracking
        task = Task.async(fn ->
          execute_forward_pass(inputs, state)
        end)
        
        # Store task for monitoring
        new_pending = Map.put(state.pending_executions || %{}, task.ref, from)
        
        {:noreply, %{state | pending_executions: new_pending}}
      end
      
      def handle_info({ref, result}, state) when is_reference(ref) do
        # Handle completed forward pass
        case Map.pop(state.pending_executions || %{}, ref) do
          {from, remaining_pending} when from != nil ->
            GenServer.reply(from, result)
            
            # Update execution history
            new_history = [result | Enum.take(state.execution_history, 99)]
            
            {:noreply, %{state | 
              pending_executions: remaining_pending,
              execution_history: new_history
            }}
          
          {nil, _} ->
            {:noreply, state}
        end
      end
      
      # Allow modules to override forward behavior
      defoverridable handle_call: 3
    end
  end
  
  defp generate_parameter_functions(parameters) do
    Enum.map(parameters, fn {name, type, opts} ->
      quote do
        def unquote(:"get_#{name}")(pid) do
          GenServer.call(pid, {:get_parameter, unquote(name)})
        end
        
        def unquote(:"set_#{name}")(pid, value) do
          GenServer.call(pid, {:set_parameter, unquote(name), value})
        end
      end
    end)
  end
  
  defp generate_predictor_functions(predictors) do
    Enum.map(predictors, fn {name, signature_module, opts} ->
      quote do
        def unquote(:"predict_#{name}")(pid, inputs) do
          GenServer.call(pid, {:predict, unquote(name), inputs})
        end
      end
    end)
  end
end

defmodule DSPex.Program.ExecutionEngine do
  use GenServer
  
  alias DSPex.Module.Registry
  alias DSPex.Adapters.ProviderManager
  alias DSPex.Telemetry.Tracker
  
  defstruct [
    :program_id,
    :modules,
    :execution_graph,
    :current_execution,
    :optimization_state,
    :performance_metrics
  ]
  
  def execute_program(program_id, inputs, opts \\ []) do
    GenServer.call(via_name(program_id), {:execute, inputs, opts})
  end
  
  def handle_call({:execute, inputs, opts}, from, state) do
    execution_id = generate_execution_id()
    
    # Start telemetry span
    :telemetry.span([:dspex, :program, :execution], 
      %{program_id: state.program_id, execution_id: execution_id}, fn ->
      
      # Execute with full tracking
      task = Task.async(fn ->
        execute_program_with_tracking(inputs, opts, state)
      end)
      
      monitor_ref = Process.monitor(task.pid)
      
      execution_context = %{
        execution_id: execution_id,
        task: task,
        monitor_ref: monitor_ref,
        from: from,
        start_time: System.monotonic_time(:microsecond),
        inputs: inputs,
        opts: opts
      }
      
      new_state = %{state | 
        current_execution: execution_context,
        performance_metrics: update_execution_start_metrics(state.performance_metrics)
      }
      
      {:noreply, new_state}
    end)
  end
  
  defp execute_program_with_tracking(inputs, opts, state) do
    # Validate inputs against program signature
    case validate_program_inputs(inputs, state) do
      {:ok, validated_inputs} ->
        # Execute module graph in dependency order
        execute_module_graph(validated_inputs, opts, state)
      
      {:error, validation_errors} ->
        {:error, {:input_validation_failed, validation_errors}}
    end
  rescue
    error ->
      {:error, {:execution_error, error}}
  end
  
  defp execute_module_graph(inputs, opts, state) do
    execution_plan = build_execution_plan(state.execution_graph, inputs)
    
    # Execute modules in parallel where possible
    results = execute_modules_parallel(execution_plan, opts, state)
    
    # Aggregate final results
    case aggregate_results(results, state) do
      {:ok, final_result} ->
        {:ok, final_result}
      
      {:error, reason} ->
        {:error, reason}
    end
  end
  
  defp execute_modules_parallel(execution_plan, opts, state) do
    # Group modules by execution level (topological sort)
    execution_levels = group_by_execution_level(execution_plan)
    
    # Execute each level in sequence, modules within level in parallel
    Enum.reduce_while(execution_levels, %{}, fn {level, modules}, acc_results ->
      level_inputs = prepare_level_inputs(modules, acc_results)
      
      # Execute all modules in this level concurrently
      level_tasks = Enum.map(modules, fn module_spec ->
        Task.async(fn ->
          execute_module(module_spec, level_inputs[module_spec.name], opts, state)
        end)
      end)
      
      # Wait for all modules in level to complete
      case Task.await_many(level_tasks, get_timeout(opts)) do
        results when is_list(results) ->
          # Check if all results are successful
          case extract_successful_results(results, modules) do
            {:ok, level_results} ->
              {:cont, Map.merge(acc_results, level_results)}
            
            {:error, failed_modules} ->
              {:halt, {:error, {:module_execution_failed, level, failed_modules}}}
          end
        
        {:timeout, _} ->
          {:halt, {:error, {:execution_timeout, level}}}
      end
    end)
  end
  
  defp execute_module(module_spec, inputs, opts, state) do
    # Get or start module process
    case Registry.get_or_start_module(module_spec.name, module_spec.config) do
      {:ok, module_pid} ->
        # Execute module with monitoring
        :telemetry.span([:dspex, :module, :execution],
          %{module: module_spec.name, program: state.program_id}, fn ->
          
          result = GenServer.call(module_pid, {:forward, inputs}, get_module_timeout(opts))
          {result, %{module: module_spec.name}}
        end)
      
      {:error, reason} ->
        {:error, {:module_start_failed, module_spec.name, reason}}
    end
  end
end

defmodule DSPex.Providers.NativeClient do
  @moduledoc """
  High-performance native HTTP client for ML providers with advanced features.
  """
  
  use GenServer
  
  alias DSPex.Providers.{RateLimiter, CircuitBreaker, RetryStrategy}
  alias DSPex.Telemetry.Tracker
  
  defstruct [
    :provider_name,
    :base_url,
    :auth_config,
    :connection_pool,
    :rate_limiter,
    :circuit_breaker,
    :retry_strategy,
    :request_middleware,
    :response_middleware
  ]
  
  def start_link([provider_name, config]) do
    GenServer.start_link(__MODULE__, [provider_name, config], 
      name: via_name(provider_name))
  end
  
  def init([provider_name, config]) do
    # Initialize HTTP connection pool
    pool_config = [
      name: pool_name(provider_name),
      size: config[:pool_size] || 20,
      max_overflow: config[:max_overflow] || 10,
      checkout_timeout: config[:checkout_timeout] || 30_000
    ]
    
    {:ok, _pool} = :hackney_pool.start_pool(pool_name(provider_name), pool_config)
    
    state = %__MODULE__{
      provider_name: provider_name,
      base_url: config[:base_url],
      auth_config: config[:auth],
      connection_pool: pool_name(provider_name),
      rate_limiter: config[:rate_limiter],
      circuit_breaker: config[:circuit_breaker],
      retry_strategy: RetryStrategy.new(provider_name, config[:retry] || %{}),
      request_middleware: config[:request_middleware] || [],
      response_middleware: config[:response_middleware] || []
    }
    
    {:ok, state}
  end
  
  @doc """
  Execute HTTP request with full provider integration.
  """
  def request(provider_name, method, path, body \\ nil, headers \\ [], opts \\ []) do
    GenServer.call(via_name(provider_name), 
      {:request, method, path, body, headers, opts})
  end
  
  def handle_call({:request, method, path, body, headers, opts}, from, state) do
    # Execute request with full middleware pipeline
    task = Task.async(fn ->
      execute_request_with_middleware(method, path, body, headers, opts, state)
    end)
    
    monitor_ref = Process.monitor(task.pid)
    
    # Store request context
    request_context = %{
      task: task,
      monitor_ref: monitor_ref,
      from: from,
      start_time: System.monotonic_time(:microsecond),
      method: method,
      path: path
    }
    
    new_pending = Map.put(state.pending_requests || %{}, monitor_ref, request_context)
    
    {:noreply, %{state | pending_requests: new_pending}}
  end
  
  def handle_info({:DOWN, monitor_ref, :process, _pid, result}, state) do
    case Map.pop(state.pending_requests || %{}, monitor_ref) do
      {request_context, remaining_pending} when request_context != nil ->
        # Calculate request duration
        duration = System.monotonic_time(:microsecond) - request_context.start_time
        
        # Emit telemetry
        :telemetry.execute([:dspex, :provider, :request], 
          %{duration: duration}, 
          %{
            provider: state.provider_name,
            method: request_context.method,
            path: request_context.path,
            result: categorize_result(result)
          }
        )
        
        # Reply to caller
        GenServer.reply(request_context.from, result)
        
        {:noreply, %{state | pending_requests: remaining_pending}}
      
      {nil, _} ->
        {:noreply, state}
    end
  end
  
  defp execute_request_with_middleware(method, path, body, headers, opts, state) do
    # Build full URL
    url = build_url(state.base_url, path)
    
    # Apply request middleware pipeline
    {processed_body, processed_headers} = 
      apply_request_middleware(body, headers, state.request_middleware, state)
    
    # Add authentication
    auth_headers = build_auth_headers(state.auth_config)
    final_headers = merge_headers(processed_headers, auth_headers)
    
    # Execute with retry strategy
    RetryStrategy.execute_with_retry(fn ->
      execute_http_request(method, url, processed_body, final_headers, opts, state)
    end, state.retry_strategy)
  end
  
  defp execute_http_request(method, url, body, headers, opts, state) do
    # Check rate limiter
    case RateLimiter.check_rate(state.rate_limiter) do
      :ok ->
        # Check circuit breaker
        case CircuitBreaker.check_circuit(state.circuit_breaker) do
          :closed ->
            perform_http_request(method, url, body, headers, opts, state)
          
          :open ->
            {:error, :circuit_breaker_open}
          
          :half_open ->
            # Allow request but monitor for failure
            result = perform_http_request(method, url, body, headers, opts, state)
            CircuitBreaker.record_result(state.circuit_breaker, result)
            result
        end
      
      {:error, :rate_limited} ->
        {:error, :rate_limited}
    end
  end
  
  defp perform_http_request(method, url, body, headers, opts, state) do
    hackney_opts = [
      pool: state.connection_pool,
      timeout: opts[:timeout] || 30_000,
      recv_timeout: opts[:recv_timeout] || 30_000,
      follow_redirect: opts[:follow_redirect] || false,
      with_body: true
    ]
    
    case :hackney.request(method, url, headers, body, hackney_opts) do
      {:ok, status_code, response_headers, response_body} ->
        # Apply response middleware
        processed_response = apply_response_middleware(
          status_code, response_headers, response_body, state.response_middleware, state
        )
        
        case status_code do
          code when code >= 200 and code < 300 ->
            {:ok, processed_response}
          
          code when code >= 400 and code < 500 ->
            {:error, {:client_error, code, processed_response}}
          
          code when code >= 500 ->
            {:error, {:server_error, code, processed_response}}
          
          code ->
            {:error, {:unknown_status, code, processed_response}}
        end
      
      {:error, reason} ->
        {:error, {:http_error, reason}}
    end
  end
end

defmodule DSPex.Providers.OpenAI do
  @moduledoc """
  Native OpenAI provider integration with advanced features.
  """
  
  use DSPex.Providers.ProviderBehaviour
  
  alias DSPex.Providers.{NativeClient, ResponseParser}
  alias DSPex.Types.{Conversion, Validation}
  
  @base_url "https://api.openai.com/v1"
  @supported_models [
    "gpt-4", "gpt-4-turbo", "gpt-4o", "gpt-4o-mini",
    "gpt-3.5-turbo", "gpt-3.5-turbo-16k"
  ]
  
  def execute_signature(signature, inputs, config) do
    with {:ok, validated_inputs} <- validate_inputs(signature, inputs),
         {:ok, request_payload} <- build_request_payload(signature, validated_inputs, config),
         {:ok, response} <- make_api_request(request_payload, config),
         {:ok, parsed_output} <- parse_response(response, signature),
         {:ok, validated_output} <- validate_outputs(signature, parsed_output) do
      {:ok, validated_output}
    else
      {:error, reason} -> {:error, reason}
    end
  end
  
  defp build_request_payload(signature, inputs, config) do
    # Choose the best approach based on signature complexity
    case determine_request_strategy(signature, config) do
      :function_calling ->
        build_function_calling_request(signature, inputs, config)
      
      :structured_output ->
        build_structured_output_request(signature, inputs, config)
      
      :json_mode ->
        build_json_mode_request(signature, inputs, config)
      
      :standard_completion ->
        build_completion_request(signature, inputs, config)
    end
  end
  
  defp build_function_calling_request(signature, inputs, config) do
    # Generate OpenAI function schema from signature
    function_schema = %{
      name: signature.name || "execute",
      description: signature.description || "Execute ML operation",
      parameters: %{
        type: "object",
        properties: generate_input_properties(signature.input_schema),
        required: get_required_fields(signature.input_schema)
      }
    }
    
    # Build messages with function calling
    messages = [
      %{
        role: "system",
        content: build_system_prompt(signature, config)
      },
      %{
        role: "user", 
        content: build_user_prompt(signature, inputs, config)
      }
    ]
    
    payload = %{
      model: config[:model] || "gpt-4",
      messages: messages,
      tools: [%{type: "function", function: function_schema}],
      tool_choice: %{type: "function", function: %{name: function_schema.name}},
      temperature: config[:temperature] || 0.7,
      max_tokens: config[:max_tokens] || 2048
    }
    
    {:ok, payload}
  end
  
  defp build_structured_output_request(signature, inputs, config) do
    # Use OpenAI's structured output feature
    response_schema = generate_response_schema(signature.output_schema)
    
    messages = [
      %{
        role: "system",
        content: build_system_prompt(signature, config)
      },
      %{
        role: "user",
        content: build_user_prompt(signature, inputs, config)
      }
    ]
    
    payload = %{
      model: config[:model] || "gpt-4o",
      messages: messages,
      response_format: %{
        type: "json_schema",
        json_schema: %{
          name: "response",
          schema: response_schema,
          strict: true
        }
      },
      temperature: config[:temperature] || 0.7,
      max_tokens: config[:max_tokens] || 2048
    }
    
    {:ok, payload}
  end
  
  defp make_api_request(payload, config) do
    headers = [
      {"Content-Type", "application/json"},
      {"Authorization", "Bearer #{config[:api_key]}"}
    ]
    
    body = Jason.encode!(payload)
    
    case NativeClient.request(:openai, :post, "/chat/completions", body, headers) do
      {:ok, response} ->
        {:ok, response}
      
      {:error, reason} ->
        {:error, {:api_request_failed, reason}}
    end
  end
  
  defp parse_response(response, signature) do
    case Jason.decode(response.body) do
      {:ok, %{"choices" => [choice | _]} = response_data} ->
        parse_choice_content(choice, signature, response_data)
      
      {:ok, %{"error" => error}} ->
        {:error, {:api_error, error}}
      
      {:error, decode_error} ->
        {:error, {:json_decode_error, decode_error}}
    end
  end
  
  defp parse_choice_content(choice, signature, response_data) do
    cond do
      # Function calling response
      tool_calls = choice["message"]["tool_calls"] ->
        parse_function_calling_response(tool_calls, signature)
      
      # Structured output or JSON mode
      content = choice["message"]["content"] ->
        parse_content_response(content, signature)
      
      true ->
        {:error, :no_parseable_content}
    end
  end
  
  defp parse_function_calling_response([tool_call | _], signature) do
    case tool_call do
      %{"function" => %{"arguments" => arguments_json}} ->
        case Jason.decode(arguments_json) do
          {:ok, arguments} ->
            # Validate against output schema
            {:ok, arguments}
          
          {:error, reason} ->
            {:error, {:function_arguments_decode_error, reason}}
        end
      
      _ ->
        {:error, :invalid_function_call_format}
    end
  end
  
  defp parse_content_response(content, signature) do
    # Try to extract JSON from content
    case extract_json_from_content(content) do
      {:ok, json_data} ->
        {:ok, json_data}
      
      {:error, _} ->
        # Fallback to structured extraction
        extract_structured_data(content, signature)
    end
  end
  
  defp validate_outputs(signature, outputs) do
    case Validation.validate_data(outputs, signature.output_schema) do
      {:ok, validated} ->
        {:ok, validated}
      
      {:error, validation_errors} ->
        {:error, {:output_validation_failed, validation_errors}}
    end
  end
  
  # Helper functions for schema generation and prompt building
  defp generate_input_properties(input_schema) do
    # Convert ExDantic schema to OpenAI function parameters
    Conversion.exdantic_to_openai_properties(input_schema)
  end
  
  defp generate_response_schema(output_schema) do
    # Convert ExDantic schema to OpenAI structured output schema
    Conversion.exdantic_to_openai_schema(output_schema)
  end
  
  defp build_system_prompt(signature, config) do
    base_prompt = signature.description || "You are a helpful AI assistant."
    
    case config[:chain_of_thought] do
      true ->
        base_prompt <> "\n\nThink step by step and provide detailed reasoning."
      
      false ->
        base_prompt
      
      nil ->
        if signature.requires_reasoning? do
          base_prompt <> "\n\nProvide clear reasoning for your response."
        else
          base_prompt
        end
    end
  end
  
  defp build_user_prompt(signature, inputs, config) do
    # Generate dynamic prompt based on signature and inputs
    prompt_parts = Enum.map(signature.input_fields, fn field ->
      value = Map.get(inputs, field.name)
      "#{field.description || field.name}: #{format_input_value(value, field.type)}"
    end)
    
    Enum.join(prompt_parts, "\n")
  end
end

defmodule DSPex.Predict.Engine do
  @moduledoc """
  Native prediction engine with advanced execution strategies.
  """
  
  use GenServer
  
  alias DSPex.Signature.{Registry, Cache}
  alias DSPex.Providers.ProviderManager
  alias DSPex.Execution.{Context, Monitor}
  
  defstruct [
    :signature,
    :provider,
    :config,
    :execution_history,
    :performance_metrics,
    :optimization_state
  ]
  
  def start_link([signature, provider, config]) do
    GenServer.start_link(__MODULE__, [signature, provider, config])
  end
  
  def init([signature, provider, config]) do
    state = %__MODULE__{
      signature: signature,
      provider: provider,
      config: config,
      execution_history: :queue.new(),
      performance_metrics: %{},
      optimization_state: %{}
    }
    
    {:ok, state}
  end
  
  @doc """
  Execute prediction with comprehensive monitoring and optimization.
  """
  def predict(pid, inputs, opts \\ []) do
    GenServer.call(pid, {:predict, inputs, opts})
  end
  
  def handle_call({:predict, inputs, opts}, from, state) do
    execution_id = generate_execution_id()
    
    # Start execution with telemetry span
    :telemetry.span([:dspex, :predict, :execution],
      %{
        signature: state.signature.name,
        provider: state.provider,
        execution_id: execution_id
      }, fn ->
      
      # Execute prediction with full monitoring
      task = Task.async(fn ->
        execute_prediction_with_monitoring(inputs, opts, state)
      end)
      
      monitor_ref = Process.monitor(task.pid)
      
      execution_context = %Context{
        execution_id: execution_id,
        task: task,
        monitor_ref: monitor_ref,
        from: from,
        start_time: System.monotonic_time(:microsecond),
        inputs: inputs,
        opts: opts
      }
      
      new_pending = Map.put(state.pending_executions || %{}, monitor_ref, execution_context)
      
      {:noreply, %{state | pending_executions: new_pending}}
    end)
  end
  
  def handle_info({:DOWN, monitor_ref, :process, _pid, result}, state) do
    case Map.pop(state.pending_executions || %{}, monitor_ref) do
      {execution_context, remaining_pending} when execution_context != nil ->
        # Calculate execution metrics
        duration = System.monotonic_time(:microsecond) - execution_context.start_time
        
        # Update execution history
        history_entry = %{
          execution_id: execution_context.execution_id,
          inputs: execution_context.inputs,
          result: result,
          duration: duration,
          timestamp: System.system_time(:millisecond)
        }
        
        new_history = add_to_history(state.execution_history, history_entry)
        
        # Update performance metrics
        new_metrics = update_performance_metrics(state.performance_metrics, history_entry)
        
        # Emit telemetry
        :telemetry.execute([:dspex, :predict, :completed],
          %{duration: duration},
          %{
            signature: state.signature.name,
            provider: state.provider,
            result: categorize_result(result)
          }
        )
        
        # Reply to caller
        GenServer.reply(execution_context.from, result)
        
        new_state = %{state |
          pending_executions: remaining_pending,
          execution_history: new_history,
          performance_metrics: new_metrics
        }
        
        {:noreply, new_state}
      
      {nil, _} ->
        {:noreply, state}
    end
  end
  
  defp execute_prediction_with_monitoring(inputs, opts, state) do
    # Get compiled signature from cache
    case Cache.get_compiled(state.signature.hash) do
      {:ok, compiled_signature} ->
        execute_with_compiled_signature(inputs, opts, compiled_signature, state)
      
      {:error, :not_cached} ->
        # Compile signature on demand
        case compile_signature_on_demand(state.signature) do
          {:ok, compiled_signature} ->
            execute_with_compiled_signature(inputs, opts, compiled_signature, state)
          
          {:error, reason} ->
            {:error, {:signature_compilation_failed, reason}}
        end
      
      {:error, reason} ->
        {:error, {:signature_cache_error, reason}}
    end
  end
  
  defp execute_with_compiled_signature(inputs, opts, compiled_signature, state) do
    # Apply execution strategy
    execution_strategy = determine_execution_strategy(compiled_signature, opts, state)
    
    case execution_strategy do
      :standard ->
        execute_standard_prediction(inputs, compiled_signature, state)
      
      :chain_of_thought ->
        execute_chain_of_thought(inputs, compiled_signature, state)
      
      :multi_provider ->
        execute_multi_provider(inputs, compiled_signature, state)
      
      :optimized ->
        execute_optimized_prediction(inputs, compiled_signature, state)
    end
  end
  
  defp execute_standard_prediction(inputs, compiled_signature, state) do
    # Standard prediction execution
    case ProviderManager.execute_signature(
      state.provider, 
      compiled_signature, 
      inputs, 
      state.config
    ) do
      {:ok, result} ->
        {:ok, result}
      
      {:error, reason} ->
        {:error, {:provider_execution_failed, reason}}
    end
  end
  
  defp execute_chain_of_thought(inputs, compiled_signature, state) do
    # Multi-step chain of thought execution
    cot_config = Map.merge(state.config, %{
      chain_of_thought: true,
      reasoning_steps: true,
      intermediate_validation: true
    })
    
    case ProviderManager.execute_signature(
      state.provider,
      compiled_signature,
      inputs,
      cot_config
    ) do
      {:ok, result} ->
        # Validate reasoning chain
        case validate_reasoning_chain(result) do
          {:ok, validated_result} ->
            {:ok, validated_result}
          
          {:error, validation_errors} ->
            # Retry with improved prompting
            retry_with_improved_prompting(inputs, compiled_signature, state, validation_errors)
        end
      
      {:error, reason} ->
        {:error, {:chain_of_thought_failed, reason}}
    end
  end
  
  defp execute_multi_provider(inputs, compiled_signature, state) do
    # Execute on multiple providers for comparison
    providers = get_configured_providers(state.config)
    
    # Execute in parallel
    tasks = Enum.map(providers, fn provider ->
      Task.async(fn ->
        ProviderManager.execute_signature(provider, compiled_signature, inputs, state.config)
      end)
    end)
    
    # Wait for results with timeout
    case Task.await_many(tasks, 30_000) do
      results when is_list(results) ->
        # Analyze and select best result
        select_best_result(results, providers, compiled_signature)
      
      {:timeout, _} ->
        {:error, :multi_provider_timeout}
    end
  end
  
  defp determine_execution_strategy(compiled_signature, opts, state) do
    cond do
      opts[:force_strategy] ->
        opts[:force_strategy]
      
      compiled_signature.requires_reasoning? ->
        :chain_of_thought
      
      length(get_configured_providers(state.config)) > 1 ->
        :multi_provider
      
      should_use_optimization?(state.performance_metrics) ->
        :optimized
      
      true ->
        :standard
    end
  end
end

defmodule DSPex.Predict.ChainOfThought do
  @moduledoc """
  Native Chain of Thought implementation with step-by-step validation.
  """
  
  alias DSPex.Signature.Enhanced
  alias DSPex.Validation.ReasoningChain
  
  def execute(signature, inputs, config) do
    # Build enhanced signature for CoT
    cot_signature = build_cot_signature(signature)
    
    # Execute with reasoning tracking
    case execute_with_reasoning(cot_signature, inputs, config) do
      {:ok, result} ->
        validate_and_extract_reasoning(result, signature)
      
      {:error, reason} ->
        {:error, reason}
    end
  end
  
  defp build_cot_signature(original_signature) do
    # Enhance signature with reasoning fields
    Enhanced.add_reasoning_fields(original_signature, %{
      reasoning_steps: %{
        type: :list,
        item_type: :reasoning_step,
        description: "Step-by-step reasoning process"
      },
      confidence: %{
        type: :confidence_score,
        description: "Confidence in the reasoning and conclusion"
      }
    })
  end
  
  defp execute_with_reasoning(cot_signature, inputs, config) do
    # Build CoT-specific prompt
    cot_prompt = build_cot_prompt(cot_signature, inputs)
    
    # Execute with enhanced configuration
    enhanced_config = Map.merge(config, %{
      temperature: config[:temperature] || 0.3,  # Lower temperature for reasoning
      max_tokens: config[:max_tokens] || 4096,   # More tokens for reasoning
      reasoning_prompt: cot_prompt,
      step_validation: true
    })
    
    DSPex.Providers.ProviderManager.execute_signature(
      config[:provider] || :openai,
      cot_signature,
      inputs,
      enhanced_config
    )
  end
  
  defp build_cot_prompt(signature, inputs) do
    base_description = signature.description || ""
    
    """
    #{base_description}
    
    Think step by step and provide your reasoning process. For each step:
    1. State what you're thinking about
    2. Explain your reasoning
    3. Identify any assumptions you're making
    4. Note any uncertainties
    
    Structure your response to include:
    - Clear reasoning steps
    - Logical connections between steps
    - A confidence assessment
    - The final answer based on your reasoning
    
    Be thorough but concise in your reasoning.
    """
  end
  
  defp validate_and_extract_reasoning(result, original_signature) do
    case ReasoningChain.validate_reasoning(result) do
      {:ok, validated_reasoning} ->
        # Extract final answer according to original signature
        extract_final_answer(validated_reasoning, original_signature)
      
      {:error, validation_errors} ->
        {:error, {:reasoning_validation_failed, validation_errors}}
    end
  end
  
  defp extract_final_answer(reasoning_result, original_signature) do
    # Extract fields that match the original signature
    final_answer = Enum.reduce(original_signature.output_fields, %{}, fn field, acc ->
      case Map.get(reasoning_result, field.name) do
        nil ->
          # Try to extract from reasoning steps
          extract_from_reasoning_steps(reasoning_result.reasoning_steps, field)
        
        value ->
          Map.put(acc, field.name, value)
      end
    end)
    
    # Include reasoning metadata
    enriched_result = Map.merge(final_answer, %{
      reasoning_chain: reasoning_result.reasoning_steps,
      confidence: reasoning_result.confidence,
      reasoning_quality: assess_reasoning_quality(reasoning_result)
    })
    
    {:ok, enriched_result}
  end
  
  defp assess_reasoning_quality(reasoning_result) do
    steps = reasoning_result.reasoning_steps
    
    %{
      step_count: length(steps),
      logical_consistency: check_logical_consistency(steps),
      evidence_strength: assess_evidence_strength(steps),
      uncertainty_handling: assess_uncertainty_handling(steps),
      overall_score: calculate_overall_reasoning_score(steps)
    }
  end
  
  defp check_logical_consistency(steps) do
    # Analyze logical flow between steps
    step_pairs = Enum.zip(steps, Enum.drop(steps, 1))
    
    consistency_scores = Enum.map(step_pairs, fn {step1, step2} ->
      analyze_step_consistency(step1, step2)
    end)
    
    case consistency_scores do
      [] -> 1.0
      scores -> Enum.sum(scores) / length(scores)
    end
  end
  
  defp analyze_step_consistency(step1, step2) do
    # Simple heuristic-based consistency check
    # In production, this could use more sophisticated NLP analysis
    
    step1_conclusion = extract_step_conclusion(step1)
    step2_premise = extract_step_premise(step2)
    
    if conclusion_supports_premise?(step1_conclusion, step2_premise) do
      1.0
    else
      0.5  # Partial consistency
    end
  end
end

defmodule DSPex.Types.MLRegistry do
  @moduledoc """
  Comprehensive type registry for ML-specific types with ExDantic integration.
  """
  
  use GenServer
  
  alias Exdantic.{TypeAdapter, Validator}
  alias DSPex.Types.{Constraints, Coercion, Validation}
  
  # Core ML Types
  @ml_types %{
    # Text and Language Types
    :prompt_template => %{
      base_type: :string,
      constraints: [min_length: 1, max_length: 100_000],
      validators: [:template_syntax, :variable_consistency],
      coercion: :string_coercion,
      metadata: %{category: :text, provider_hints: %{openai: :text, anthropic: :text}}
    },
    
    :reasoning_chain => %{
      base_type: {:list, :reasoning_step},
      constraints: [min_items: 1, max_items: 50],
      validators: [:reasoning_consistency, :logical_flow],
      coercion: :reasoning_chain_coercion,
      metadata: %{category: :reasoning, structured: true}
    },
    
    :reasoning_step => %{
      base_type: :map,
      schema: %{
        step_number: {:integer, [min: 1]},
        thought: {:string, [min_length: 10, max_length: 1000]},
        reasoning: {:string, [min_length: 20, max_length: 2000]},
        confidence: {:confidence_score, []}
      },
      validators: [:step_completeness, :reasoning_quality],
      metadata: %{category: :reasoning, required_fields: [:thought, :reasoning]}
    },
    
    # Numeric and Score Types
    :confidence_score => %{
      base_type: :float,
      constraints: [min: 0.0, max: 1.0],
      validators: [:confidence_validation],
      coercion: :confidence_coercion,
      metadata: %{category: :numeric, precision: 3}
    },
    
    :probability => %{
      base_type: :float,
      constraints: [min: 0.0, max: 1.0],
      validators: [:probability_validation],
      coercion: :probability_coercion,
      metadata: %{category: :numeric, precision: 6}
    },
    
    :token_count => %{
      base_type: :integer,
      constraints: [min: 0, max: 1_000_000],
      validators: [:token_count_validation],
      coercion: :integer_coercion,
      metadata: %{category: :metrics, provider_specific: true}
    },
    
    # Embedding and Vector Types
    :embedding => %{
      base_type: {:list, :float},
      constraints: [min_items: 1, max_items: 10_000],
      validators: [:embedding_dimension, :embedding_normalization],
      coercion: :embedding_coercion,
      metadata: %{category: :vector, high_memory: true}
    },
    
    :similarity_score => %{
      base_type: :float,
      constraints: [min: -1.0, max: 1.0],
      validators: [:similarity_validation],
      coercion: :similarity_coercion,
      metadata: %{category: :numeric, precision: 4}
    },
    
    # Function and Tool Types
    :function_call => %{
      base_type: :map,
      schema: %{
        function_name: {:string, [min_length: 1, max_length: 100]},
        arguments: {:map, []},
        call_id: {:optional, {:string, []}}
      },
      validators: [:function_call_validation, :arguments_validation],
      metadata: %{category: :function, provider_specific: true}
    },
    
    :tool_result => %{
      base_type: :map,
      schema: %{
        tool_name: {:string, [min_length: 1, max_length: 100]},
        result: :any,
        success: {:boolean, []},
        error: {:optional, {:string, []}}
      },
      validators: [:tool_result_validation],
      metadata: %{category: :function, structured: true}
    },
    
    # Model and Provider Types
    :model_output => %{
      base_type: :map,
      schema: %{
        content: {:string, []},
        usage: {:optional, :token_usage},
        model: {:string, []},
        finish_reason: {:optional, {:string, []}}
      },
      validators: [:model_output_validation],
      metadata: %{category: :model, provider_metadata: true}
    },
    
    :token_usage => %{
      base_type: :map,
      schema: %{
        prompt_tokens: {:integer, [min: 0]},
        completion_tokens: {:integer, [min: 0]},
        total_tokens: {:integer, [min: 0]}
      },
      validators: [:token_usage_consistency],
      metadata: %{category: :metrics, cost_relevant: true}
    }
  }
  
  def start_link(opts) do
    GenServer.start_link(__MODULE__, opts, name: __MODULE__)
  end
  
  def init(_opts) do
    # Initialize type adapters for all ML types
    type_adapters = initialize_type_adapters(@ml_types)
    
    state = %{
      ml_types: @ml_types,
      type_adapters: type_adapters,
      custom_types: %{},
      validation_cache: :ets.new(:type_validation_cache, [:named_table, :public, {:read_concurrency, true}])
    }
    
    {:ok, state}
  end
  
  @doc """
  Register a custom ML type with the registry.
  """
  def register_custom_type(type_name, type_definition) do
    GenServer.call(__MODULE__, {:register_custom_type, type_name, type_definition})
  end
  
  @doc """
  Get type adapter for a given type.
  """
  def get_type_adapter(type_name) do
    GenServer.call(__MODULE__, {:get_type_adapter, type_name})
  end
  
  @doc """
  Validate value against ML type with caching.
  """
  def validate_value(value, type_name, opts \\ []) do
    cache_key = {type_name, :erlang.phash2(value), opts}
    
    case :ets.lookup(:type_validation_cache, cache_key) do
      [{^cache_key, cached_result}] ->
        cached_result
      
      [] ->
        result = perform_validation(value, type_name, opts)
        
        # Cache result if successful
        if match?({:ok, _}, result) do
          :ets.insert(:type_validation_cache, {cache_key, result})
        end
        
        result
    end
  end
  
  def handle_call({:register_custom_type, type_name, type_definition}, _from, state) do
    # Create type adapter for custom type
    case create_type_adapter(type_name, type_definition) do
      {:ok, adapter} ->
        new_custom_types = Map.put(state.custom_types, type_name, type_definition)
        new_adapters = Map.put(state.type_adapters, type_name, adapter)
        
        new_state = %{state | 
          custom_types: new_custom_types,
          type_adapters: new_adapters
        }
        
        {:reply, :ok, new_state}
      
      {:error, reason} ->
        {:reply, {:error, reason}, state}
    end
  end
  
  def handle_call({:get_type_adapter, type_name}, _from, state) do
    case Map.get(state.type_adapters, type_name) do
      nil ->
        {:reply, {:error, :type_not_found}, state}
      
      adapter ->
        {:reply, {:ok, adapter}, state}
    end
  end
  
  defp initialize_type_adapters(ml_types) do
    Enum.reduce(ml_types, %{}, fn {type_name, type_def}, acc ->
      case create_type_adapter(type_name, type_def) do
        {:ok, adapter} ->
          Map.put(acc, type_name, adapter)
        
        {:error, _reason} ->
          # Log error but continue
          Logger.warning("Failed to create type adapter for #{type_name}")
          acc
      end
    end)
  end
  
  defp create_type_adapter(type_name, type_definition) do
    # Create ExDantic TypeAdapter with ML-specific enhancements
    adapter_config = %{
      type: type_definition.base_type,
      constraints: type_definition.constraints || [],
      validators: build_validator_pipeline(type_definition.validators || []),
      coercion: get_coercion_function(type_definition.coercion),
      metadata: type_definition.metadata || %{}
    }
    
    case TypeAdapter.create(type_name, adapter_config) do
      {:ok, adapter} ->
        {:ok, enhance_adapter_for_ml(adapter, type_definition)}
      
      {:error, reason} ->
        {:error, reason}
    end
  end
  
  defp build_validator_pipeline(validator_names) do
    Enum.map(validator_names, fn validator_name ->
      case validator_name do
        :template_syntax ->
          &validate_template_syntax/1
        
        :reasoning_consistency ->
          &validate_reasoning_consistency/1
        
        :confidence_validation ->
          &validate_confidence_score/1
        
        :embedding_dimension ->
          &validate_embedding_dimension/1
        
        :function_call_validation ->
          &validate_function_call/1
        
        custom_validator when is_function(custom_validator) ->
          custom_validator
        
        _ ->
          # Default validation
          &default_validation/1
      end
    end)
  end
  
  defp perform_validation(value, type_name, opts) do
    case get_type_adapter(type_name) do
      {:ok, adapter} ->
        # Use ExDantic validation with ML enhancements
        case TypeAdapter.validate(adapter, value, opts) do
          {:ok, validated_value} ->
            # Apply ML-specific post-validation
            apply_ml_post_validation(validated_value, type_name, opts)
          
          {:error, validation_errors} ->
            {:error, validation_errors}
        end
      
      {:error, :type_not_found} ->
        {:error, {:unknown_type, type_name}}
    end
  end
  
  defp apply_ml_post_validation(value, type_name, opts) do
    case type_name do
      :reasoning_chain ->
        # Additional reasoning chain validation
        validate_reasoning_chain_quality(value, opts)
      
      :embedding ->
        # Embedding normalization and validation
        validate_and_normalize_embedding(value, opts)
      
      :confidence_score ->
        # Confidence score calibration
        calibrate_confidence_score(value, opts)
      
      _ ->
        {:ok, value}
    end
  end
  
  # ML-specific validation functions
  defp validate_template_syntax(template) when is_binary(template) do
    # Validate template syntax and variable consistency
    case parse_template_variables(template) do
      {:ok, variables} ->
        if variables_are_consistent?(variables) do
          {:ok, template}
        else
          {:error, :inconsistent_template_variables}
        end
      
      {:error, reason} ->
        {:error, {:template_syntax_error, reason}}
    end
  end
  
  defp validate_reasoning_consistency(reasoning_chain) when is_list(reasoning_chain) do
    # Check logical consistency between reasoning steps
    case analyze_reasoning_flow(reasoning_chain) do
      {:ok, _flow_analysis} ->
        {:ok, reasoning_chain}
      
      {:error, inconsistencies} ->
        {:error, {:reasoning_inconsistency, inconsistencies}}
    end
  end
  
  defp validate_confidence_score(score) when is_number(score) do
    cond do
      score < 0.0 ->
        {:error, :confidence_below_minimum}
      
      score > 1.0 ->
        {:error, :confidence_above_maximum}
      
      true ->
        {:ok, Float.round(score, 3)}
    end
  end
  
  defp validate_embedding_dimension(embedding) when is_list(embedding) do
    # Validate embedding dimensions and normalization
    dimension = length(embedding)
    
    cond do
      dimension < 1 ->
        {:error, :embedding_too_small}
      
      dimension > 10_000 ->
        {:error, :embedding_too_large}
      
      not all_numbers?(embedding) ->
        {:error, :embedding_non_numeric}
      
      true ->
        # Check if normalized (optional)
        magnitude = calculate_magnitude(embedding)
        
        if magnitude > 0 do
          {:ok, embedding}
        else
          {:error, :zero_magnitude_embedding}
        end
    end
  end
  
  # Helper functions
  defp parse_template_variables(template) do
    # Simple regex-based variable extraction
    # In production, use a proper template parser
    variables = Regex.scan(~r/\{\{(\w+)\}\}/, template)
    |> Enum.map(fn [_, var] -> var end)
    |> Enum.uniq()
    
    {:ok, variables}
  rescue
    _ -> {:error, :invalid_template_syntax}
  end
  
  defp variables_are_consistent?(variables) do
    # Check if all variables follow naming conventions
    Enum.all?(variables, fn var ->
      String.match?(var, ~r/^[a-zA-Z_][a-zA-Z0-9_]*$/)
    end)
  end
  
  defp all_numbers?(list) do
    Enum.all?(list, &is_number/1)
  end
  
  defp calculate_magnitude(vector) do
    vector
    |> Enum.map(&(&1 * &1))
    |> Enum.sum()
    |> :math.sqrt()
  end
end

defmodule DSPex.Telemetry.MLMetrics do
  @moduledoc """
  Comprehensive telemetry system optimized for ML workloads.
  """
  
  use GenServer
  
  alias DSPex.Telemetry.{Aggregator, Exporter, Alerting}
  
  # Telemetry events for ML operations
  @ml_events [
    [:dspex, :signature, :compilation],
    [:dspex, :signature, :validation],
    [:dspex, :prediction, :execution],
    [:dspex, :prediction, :chain_of_thought],
    [:dspex, :provider, :request],
    [:dspex, :provider, :response],
    [:dspex, :module, :execution],
    [:dspex, :program, :execution],
    [:dspex, :cache, :hit],
    [:dspex, :cache, :miss],
    [:dspex, :memory, :pressure],
    [:dspex, :optimization, :iteration],
    [:dspex, :evaluation, :metric],
    [:dspex, :distributed, :coordination],
    [:dspex, :error, :recovery]
  ]
  
  def start_link(opts) do
    GenServer.start_link(__MODULE__, opts, name: __MODULE__)
  end
  
  def init(opts) do
    # Attach telemetry handlers
    :telemetry.attach_many(
      "ash-dspy-ml-metrics",
      @ml_events,
      &handle_ml_event/4,
      %{config: opts}
    )
    
    # Initialize metrics storage
    metrics_store = :ets.new(:ml_metrics, [
      :named_table,
      :public,
      {:write_concurrency, true},
      {:read_concurrency, true}
    ])
    
    # Initialize aggregation windows
    aggregation_windows = initialize_aggregation_windows()
    
    state = %{
      config: opts,
      metrics_store: metrics_store,
      aggregation_windows: aggregation_windows,
      alert_rules: load_alert_rules(opts),
      exporters: initialize_exporters(opts)
    }
    
    # Schedule periodic aggregation and export
    :timer.send_interval(60_000, :aggregate_metrics)   # 1 minute
    :timer.send_interval(300_000, :export_metrics)     # 5 minutes
    :timer.send_interval(30_000, :check_alerts)        # 30 seconds
    
    {:ok, state}
  end
  
  def handle_ml_event([:dspex, :prediction, :execution], measurements, metadata, config) do
    # Record prediction execution metrics
    record_metric(:prediction_duration, measurements.duration, %{
      signature: metadata.signature,
      provider: metadata.provider,
      success: metadata.success
    })
    
    # Track token usage if available
    if metadata[:token_usage] do
      record_metric(:token_usage, metadata.token_usage.total_tokens, %{
        type: :total,
        provider: metadata.provider,
        model: metadata.model
      })
      
      record_metric(:token_usage, metadata.token_usage.prompt_tokens, %{
        type: :prompt,
        provider: metadata.provider,
        model: metadata.model
      })
      
      record_metric(:token_usage, metadata.token_usage.completion_tokens, %{
        type: :completion,
        provider: metadata.provider,
        model: metadata.model
      })
    end
    
    # Track cost if available
    if metadata[:cost] do
      record_metric(:execution_cost, metadata.cost, %{
        provider: metadata.provider,
        model: metadata.model
      })
    end
    
    # Track quality metrics if available
    if metadata[:confidence] do
      record_metric(:confidence_score, metadata.confidence, %{
        signature: metadata.signature,
        provider: metadata.provider
      })
    end
    
    # Increment execution counter
    increment_counter(:prediction_executions, %{
      signature: metadata.signature,
      provider: metadata.provider,
      success: metadata.success
    })
  end
  
  def handle_ml_event([:dspex, :provider, :request], measurements, metadata, config) do
    # Track provider performance
    record_metric(:provider_latency, measurements.duration, %{
      provider: metadata.provider,
      endpoint: metadata.endpoint
    })
    
    # Track provider reliability
    case metadata.result do
      :success ->
        increment_counter(:provider_requests_success, %{provider: metadata.provider})
      
      {:error, error_type} ->
        increment_counter(:provider_requests_error, %{
          provider: metadata.provider,
          error_type: error_type
        })
      
      :timeout ->
        increment_counter(:provider_requests_timeout, %{provider: metadata.provider})
    end
    
    # Track rate limiting
    if metadata[:rate_limited] do
      increment_counter(:provider_rate_limited, %{provider: metadata.provider})
    end
  end
  
  def handle_ml_event([:dspex, :signature, :compilation], measurements, metadata, config) do
    # Track signature compilation performance
    record_metric(:signature_compilation_duration, measurements.duration, %{
      signature_hash: metadata.signature_hash,
      cache_hit: metadata.cache_hit
    })
    
    # Track compilation cache effectiveness
    if metadata.cache_hit do
      increment_counter(:signature_cache_hits, %{})
    else
      increment_counter(:signature_cache_misses, %{})
    end
    
    # Track compilation complexity
    if metadata[:complexity_score] do
      record_metric(:signature_complexity, metadata.complexity_score, %{
        signature_type: metadata.signature_type
      })
    end
  end
  
  def handle_ml_event([:dspex, :memory, :pressure], measurements, metadata, config) do
    # Track memory usage patterns
    record_gauge(:memory_usage, measurements.memory_usage, %{
      component: metadata.component
    })
    
    record_gauge(:memory_pressure_level, measurements.pressure_level, %{})
    
    # Track garbage collection events
    if metadata[:gc_triggered] do
      increment_counter(:gc_events, %{
        component: metadata.component,
        gc_type: metadata.gc_type
      })
    end
    
    # Track backpressure events
    if metadata[:backpressure_active] do
      increment_counter(:backpressure_activations, %{
        component: metadata.component
      })
    end
  end
  
  def handle_ml_event([:dspex, :optimization, :iteration], measurements, metadata, config) do
    # Track optimization progress
    record_metric(:optimization_score, measurements.score, %{
      optimizer: metadata.optimizer,
      signature: metadata.signature,
      iteration: metadata.iteration
    })
    
    record_metric(:optimization_duration, measurements.duration, %{
      optimizer: metadata.optimizer,
      signature: metadata.signature
    })
    
    # Track optimization convergence
    if metadata[:converged] do
      increment_counter(:optimization_convergence, %{
        optimizer: metadata.optimizer,
        iterations: metadata.iteration
      })
    end
  end
  
  def handle_info(:aggregate_metrics, state) do
    # Perform metric aggregation
    current_time = System.system_time(:second)
    
    # Aggregate metrics for different time windows
    Enum.each(state.aggregation_windows, fn {window_name, window_seconds} ->
      window_start = current_time - window_seconds
      aggregate_window_metrics(window_name, window_start, current_time, state)
    end)
    
    {:noreply, state}
  end
  
  def handle_info(:export_metrics, state) do
    # Export metrics to configured backends
    current_metrics = get_current_metrics(state.metrics_store)
    
    Enum.each(state.exporters, fn exporter ->
      Task.start(fn ->
        Exporter.export_metrics(exporter, current_metrics)
      end)
    end)
    
    {:noreply, state}
  end
  
  def handle_info(:check_alerts, state) do
    # Check alert conditions
    current_metrics = get_current_metrics(state.metrics_store)
    
    Enum.each(state.alert_rules, fn alert_rule ->
      case evaluate_alert_rule(alert_rule, current_metrics) do
        {:triggered, alert_data} ->
          Alerting.trigger_alert(alert_rule.name, alert_data)
        
        :ok ->
          :ok
      end
    end)
    
    {:noreply, state}
  end
  
  # Metric recording functions
  defp record_metric(metric_name, value, tags) do
    timestamp = System.system_time(:millisecond)
    
    :ets.insert(:ml_metrics, {
      {metric_name, tags, timestamp},
      value
    })
  end
  
  defp record_gauge(metric_name, value, tags) do
    # For gauges, we want the latest value
    :ets.insert(:ml_metrics, {
      {metric_name, tags, :gauge},
      value
    })
  end
  
  defp increment_counter(metric_name, tags) do
    counter_key = {metric_name, tags, :counter}
    
    case :ets.lookup(:ml_metrics, counter_key) do
      [{^counter_key, current_value}] ->
        :ets.update_element(:ml_metrics, counter_key, {2, current_value + 1})
      
      [] ->
        :ets.insert(:ml_metrics, {counter_key, 1})
    end
  end
  
  defp initialize_aggregation_windows do
    %{
      "1m" => 60,        # 1 minute
      "5m" => 300,       # 5 minutes  
      "15m" => 900,      # 15 minutes
      "1h" => 3600,      # 1 hour
      "24h" => 86400     # 24 hours
    }
  end
  
  defp aggregate_window_metrics(window_name, window_start, window_end, state) do
    # Get all metrics in the time window
    window_metrics = :ets.select(:ml_metrics, [
      {{{:'$1', :'$2', :'$3'}, :'$4'}, 
       [{:andalso, {:is_integer, :'$3'}, 
         {:andalso, {:>=, :'$3', window_start * 1000}, 
          {:<, :'$3', window_end * 1000}}}],
       [{{:'$1', :'$2', :'$3'}, :'$4'}]}
    ])
    
    # Group by metric name and tags
    grouped_metrics = Enum.group_by(window_metrics, fn {{metric_name, tags, _timestamp}, _value} ->
      {metric_name, tags}
    end)
    
    # Calculate aggregations for each metric group
    Enum.each(grouped_metrics, fn {{metric_name, tags}, metric_points} ->
      values = Enum.map(metric_points, fn {_, value} -> value end)
      
      aggregations = %{
        count: length(values),
        sum: Enum.sum(values),
        avg: Enum.sum(values) / length(values),
        min: Enum.min(values),
        max: Enum.max(values),
        p50: percentile(values, 50),
        p95: percentile(values, 95),
        p99: percentile(values, 99)
      }
      
      # Store aggregated metrics
      store_aggregated_metric(window_name, metric_name, tags, aggregations, window_end)
    end)
  end
  
  defp percentile(values, p) do
    sorted = Enum.sort(values)
    index = (length(sorted) * p / 100) |> Float.ceil() |> round() |> max(1) |> min(length(sorted))
    Enum.at(sorted, index - 1)
  end
  
  defp store_aggregated_metric(window, metric_name, tags, aggregations, timestamp) do
    aggregation_key = {:aggregated, window, metric_name, tags, timestamp}
    :ets.insert(:ml_metrics, {aggregation_key, aggregations})
  end
end

defmodule DSPex.Deployment.HotSwap do
  @moduledoc """
  Hot code deployment system for zero-downtime model updates and feature rollouts.
  """
  
  use GenServer
  
  alias DSPex.Module.Registry
  alias DSPex.Telemetry.Tracker
  alias DSPex.Deployment.{VersionManager, TrafficSplitter, HealthChecker}
  
  defstruct [
    :active_versions,
    :staged_versions,
    :rollout_configs,
    :traffic_splits,
    :health_monitors,
    :rollback_snapshots
  ]
  
  def start_link(opts) do
    GenServer.start_link(__MODULE__, opts, name: __MODULE__)
  end
  
  def init(_opts) do
    state = %__MODULE__{
      active_versions: %{},
      staged_versions: %{},
      rollout_configs: %{},
      traffic_splits: %{},
      health_monitors: %{},
      rollback_snapshots: %{}
    }
    
    # Start health monitoring
    :timer.send_interval(30_000, :health_check)
    
    {:ok, state}
  end
  
  @doc """
  Deploy new version of a signature or module with gradual rollout.
  """
  def deploy_version(component_id, new_version, rollout_config) do
    GenServer.call(__MODULE__, {:deploy_version, component_id, new_version, rollout_config})
  end
  
  @doc """
  Rollback to previous version immediately.
  """
  def rollback_version(component_id) do
    GenServer.call(__MODULE__, {:rollback_version, component_id})
  end
  
  def handle_call({:deploy_version, component_id, new_version, rollout_config}, _from, state) do
    case prepare_deployment(component_id, new_version, rollout_config) do
      {:ok, deployment_context} ->
        # Create rollback snapshot
        snapshot = create_rollback_snapshot(component_id, state)
        
        # Stage the new version
        new_staged = Map.put(state.staged_versions, component_id, new_version)
        new_snapshots = Map.put(state.rollback_snapshots, component_id, snapshot)
        new_configs = Map.put(state.rollout_configs, component_id, rollout_config)
        
        # Start gradual rollout
        {:ok, rollout_pid} = start_gradual_rollout(component_id, rollout_config)
        
        new_state = %{state |
          staged_versions: new_staged,
          rollback_snapshots: new_snapshots,
          rollout_configs: new_configs
        }
        
        {:reply, {:ok, :deployment_started}, new_state}
      
      {:error, reason} ->
        {:reply, {:error, reason}, state}
    end
  end
  
  def handle_call({:rollback_version, component_id}, _from, state) do
    case Map.get(state.rollback_snapshots, component_id) do
      nil ->
        {:reply, {:error, :no_snapshot_available}, state}
      
      snapshot ->
        # Perform immediate rollback
        case perform_rollback(component_id, snapshot) do
          :ok ->
            # Clean up rollout state
            new_state = %{state |
              staged_versions: Map.delete(state.staged_versions, component_id),
              rollout_configs: Map.delete(state.rollout_configs, component_id),
              traffic_splits: Map.delete(state.traffic_splits, component_id),
              rollback_snapshots: Map.delete(state.rollback_snapshots, component_id)
            }
            
            {:reply, :ok, new_state}
          
          {:error, reason} ->
            {:reply, {:error, reason}, state}
        end
    end
  end
  
  # Handle rollout progress updates
  def handle_info({:rollout_progress, component_id, percentage}, state) do
    # Update traffic split
    new_traffic_splits = Map.put(state.traffic_splits, component_id, percentage)
    
    # Update traffic splitter
    TrafficSplitter.update_split(component_id, percentage)
    
    Logger.info("Rollout progress for #{component_id}: #{percentage}%")
    
    # Check if rollout is complete
    if percentage >= 100 do
      complete_rollout(component_id, state)
    else
      {:noreply, %{state | traffic_splits: new_traffic_splits}}
    end
  end
  
  def handle_info({:rollout_failed, component_id, reason}, state) do
    Logger.error("Rollout failed for #{component_id}: #{reason}. Initiating automatic rollback.")
    
    # Trigger automatic rollback
    case perform_automatic_rollback(component_id, state) do
      :ok ->
        new_state = cleanup_failed_rollout(component_id, state)
        {:noreply, new_state}
      
      {:error, rollback_error} ->
        Logger.critical("Rollback failed for #{component_id}: #{rollback_error}")
        {:noreply, state}
    end
  end
  
  def handle_info(:health_check, state) do
    # Check health of all active rollouts
    Enum.each(state.traffic_splits, fn {component_id, _percentage} ->
      case check_rollout_health(component_id) do
        :healthy ->
          :ok
        
        {:unhealthy, reason} ->
          # Trigger rollout failure
          send(self(), {:rollout_failed, component_id, reason})
      end
    end)
    
    {:noreply, state}
  end
  
  defp prepare_deployment(component_id, new_version, rollout_config) do
    # Validate new version
    case validate_new_version(component_id, new_version) do
      :ok ->
        # Prepare deployment environment
        case setup_deployment_environment(component_id, new_version) do
          {:ok, context} ->
            {:ok, context}
          
          {:error, reason} ->
            {:error, {:environment_setup_failed, reason}}
        end
      
      {:error, reason} ->
        {:error, {:version_validation_failed, reason}}
    end
  end
  
  defp validate_new_version(component_id, new_version) do
    # Perform various validation checks
    with :ok <- validate_version_format(new_version),
         :ok <- validate_compatibility(component_id, new_version),
         :ok <- validate_dependencies(new_version),
         :ok <- validate_security(new_version) do
      :ok
    else
      {:error, reason} -> {:error, reason}
    end
  end
  
  defp start_gradual_rollout(component_id, rollout_config) do
    Task.start(fn ->
      execute_gradual_rollout(component_id, rollout_config)
    end)
  end
  
  defp execute_gradual_rollout(component_id, config) do
    steps = config[:rollout_steps] || [5, 10, 25, 50, 100]
    step_duration = config[:step_duration] || 300_000  # 5 minutes
    health_check_interval = config[:health_check_interval] || 30_000  # 30 seconds
    
    Enum.each(steps, fn percentage ->
      # Update traffic split
      send(DSPex.Deployment.HotSwap, {:rollout_progress, component_id, percentage})
      
      # Wait for step duration with periodic health checks
      monitor_step_health(component_id, step_duration, health_check_interval)
    end)
    
    # Rollout completed successfully
    send(DSPex.Deployment.HotSwap, {:rollout_complete, component_id})
  end
  
  defp monitor_step_health(component_id, total_duration, check_interval) do
    end_time = System.monotonic_time(:millisecond) + total_duration
    
    monitor_loop(component_id, end_time, check_interval)
  end
  
  defp monitor_loop(component_id, end_time, check_interval) do
    current_time = System.monotonic_time(:millisecond)
    
    if current_time >= end_time do
      :ok
    else
      case check_rollout_health(component_id) do
        :healthy ->
          # Wait for next check
          Process.sleep(min(check_interval, end_time - current_time))
          monitor_loop(component_id, end_time, check_interval)
        
        {:unhealthy, reason} ->
          # Abort rollout
          send(DSPex.Deployment.HotSwap, {:rollout_failed, component_id, reason})
          exit(:rollout_failed)
      end
    end
  end
  
  defp check_rollout_health(component_id) do
    # Get recent metrics for the component
    metrics = get_component_metrics(component_id, 60_000)  # Last minute
    
    cond do
      metrics[:error_rate] > 0.05 ->  # 5% error rate threshold
        {:unhealthy, :high_error_rate}
      
      metrics[:avg_latency] > metrics[:baseline_latency] * 1.5 ->
        {:unhealthy, :high_latency}
      
      metrics[:memory_usage] > get_memory_threshold(component_id) ->
        {:unhealthy, :memory_pressure}
      
      metrics[:throughput] < metrics[:baseline_throughput] * 0.8 ->
        {:unhealthy, :low_throughput}
      
      true ->
        :healthy
    end
  end
  
  defp create_rollback_snapshot(component_id, state) do
    case Map.get(state.active_versions, component_id) do
      nil ->
        %{version: nil, config: nil, timestamp: System.system_time(:second)}
      
      active_version ->
        %{
          version: active_version,
          config: get_current_config(component_id),
          registry_state: Registry.get_component_state(component_id),
          timestamp: System.system_time(:second)
        }
    end
  end
  
  defp perform_rollback(component_id, snapshot) do
    Logger.info("Performing rollback for #{component_id}")
    
    try do
      # Stop traffic to new version immediately
      TrafficSplitter.set_split(component_id, 0)
      
      # Restore previous version if available
      case snapshot.version do
        nil ->
          # No previous version, disable component
          Registry.disable_component(component_id)
        
        previous_version ->
          # Restore previous version
          Registry.restore_component(component_id, previous_version, snapshot.registry_state)
      end
      
      # Reset traffic to 100% old version
      TrafficSplitter.reset_split(component_id)
      
      Logger.info("Rollback completed successfully for #{component_id}")
      :ok
      
    rescue
      error ->
        Logger.error("Rollback failed for #{component_id}: #{inspect(error)}")
        {:error, error}
    end
  end
  
  defp complete_rollout(component_id, state) do
    Logger.info("Completing rollout for #{component_id}")
    
    # Move staged version to active
    case Map.get(state.staged_versions, component_id) do
      nil ->
        {:noreply, state}
      
      staged_version ->
        # Activate new version
        Registry.activate_version(component_id, staged_version)
        
        # Clean up rollout state
        new_state = %{state |
          active_versions: Map.put(state.active_versions, component_id, staged_version),
          staged_versions: Map.delete(state.staged_versions, component_id),
          rollout_configs: Map.delete(state.rollout_configs, component_id),
          traffic_splits: Map.delete(state.traffic_splits, component_id)
        }
        
        Logger.info("Rollout completed successfully for #{component_id}")
        {:noreply, new_state}
    end
  end
end