defmodule AiDebug.DogfoodFeedback do @moduledoc """ Feedback collection and tracking system for dogfooding the AI Debug platform. This module tracks issues, suggestions, and improvements discovered while using AI Debug to debug itself. It maintains a feedback cycle to continuously improve the tool based on real usage. ## Usage # Report an issue discovered during dogfooding AiDebug.DogfoodFeedback.report_issue(%{ category: :usability, severity: :medium, description: "Console logs are truncated too early", context: %{component: "get_console_logs", max_chars: 1000} }) # Get current issues to address {:ok, issues} = AiDebug.DogfoodFeedback.list_open_issues() """ use GenServer require Logger @feedback_file "dogfood_feedback.json" @categories [:usability, :performance, :reliability, :features, :documentation] @severities [:critical, :high, :medium, :low] defstruct [:id, :category, :severity, :description, :context, :status, :reported_at, :resolved_at, :resolution, :reporter] # Client API def start_link(opts \\ []) do GenServer.start_link(__MODULE__, opts, name: __MODULE__) end @doc """ Report an issue or suggestion discovered during dogfooding. """ def report_issue(attrs) do GenServer.call(__MODULE__, {:report_issue, attrs}) end @doc """ List all open issues ordered by severity and date. """ def list_open_issues do GenServer.call(__MODULE__, :list_open_issues) end @doc """ List issues by category. """ def list_by_category(category) when category in @categories do GenServer.call(__MODULE__, {:list_by_category, category}) end @doc """ Mark an issue as resolved with a resolution description. """ def resolve_issue(issue_id, resolution) do GenServer.call(__MODULE__, {:resolve_issue, issue_id, resolution}) end @doc """ Generate a dogfooding report with metrics and trends. """ def generate_report do GenServer.call(__MODULE__, :generate_report) end @doc """ Get action items - high priority unresolved issues. """ def get_action_items do GenServer.call(__MODULE__, :get_action_items) end # Server callbacks def init(_opts) do # Load existing feedback from file feedback = load_feedback() # Schedule periodic report generation :timer.send_interval(:timer.hours(24), :generate_daily_report) {:ok, %{feedback: feedback}} end def handle_call({:report_issue, attrs}, _from, state) do issue = %__MODULE__{ id: generate_id(), category: attrs[:category] || :general, severity: attrs[:severity] || :medium, description: attrs[:description] || raise("Description required"), context: attrs[:context] || %{}, status: :open, reported_at: DateTime.utc_now(), reporter: attrs[:reporter] || "system" } # Validate category and severity unless issue.category in @categories do {:reply, {:error, "Invalid category. Must be one of: #{inspect(@categories)}"}, state} end unless issue.severity in @severities do {:reply, {:error, "Invalid severity. Must be one of: #{inspect(@severities)}"}, state} end # Add to feedback list feedback = [issue | state.feedback] # Save to file save_feedback(feedback) # Log for immediate visibility log_issue(issue) # Check if this is a critical issue that needs immediate attention if issue.severity == :critical do alert_critical_issue(issue) end {:reply, {:ok, issue}, %{state | feedback: feedback}} end def handle_call(:list_open_issues, _from, state) do open_issues = state.feedback |> Enum.filter(&(&1.status == :open)) |> Enum.sort_by(&{severity_priority(&1.severity), &1.reported_at}) {:reply, {:ok, open_issues}, state} end def handle_call({:list_by_category, category}, _from, state) do issues = state.feedback |> Enum.filter(&(&1.category == category)) |> Enum.sort_by(&{&1.status, severity_priority(&1.severity), &1.reported_at}) {:reply, {:ok, issues}, state} end def handle_call({:resolve_issue, issue_id, resolution}, _from, state) do case Enum.find_index(state.feedback, &(&1.id == issue_id)) do nil -> {:reply, {:error, "Issue not found"}, state} index -> issue = Enum.at(state.feedback, index) resolved_issue = %{issue | status: :resolved, resolved_at: DateTime.utc_now(), resolution: resolution } feedback = List.replace_at(state.feedback, index, resolved_issue) save_feedback(feedback) Logger.info("Resolved dogfood issue #{issue_id}: #{resolution}") {:reply, {:ok, resolved_issue}, %{state | feedback: feedback}} end end def handle_call(:generate_report, _from, state) do report = generate_feedback_report(state.feedback) {:reply, {:ok, report}, state} end def handle_call(:get_action_items, _from, state) do action_items = state.feedback |> Enum.filter(&(&1.status == :open && &1.severity in [:critical, :high])) |> Enum.sort_by(&{severity_priority(&1.severity), &1.reported_at}) |> Enum.take(10) {:reply, {:ok, action_items}, state} end def handle_info(:generate_daily_report, state) do {:ok, report} = generate_feedback_report(state.feedback) Logger.info("Daily dogfood report:\\n#{report}") # Could also send this via email/Slack/etc {:noreply, state} end # Private functions defp generate_id do "DF-#{:crypto.strong_rand_bytes(4) |> Base.encode16()}" end defp load_feedback do case File.read(@feedback_file) do {:ok, content} -> case Jason.decode(content) do {:ok, data} -> Enum.map(data, &struct(__MODULE__, atomize_keys(&1))) _ -> [] end _ -> [] end end defp save_feedback(feedback) do data = Enum.map(feedback, &Map.from_struct/1) File.write!(@feedback_file, Jason.encode!(data, pretty: true)) end defp atomize_keys(map) do Map.new(map, fn {k, v} -> {String.to_atom(k), v} end) end defp severity_priority(:critical), do: 0 defp severity_priority(:high), do: 1 defp severity_priority(:medium), do: 2 defp severity_priority(:low), do: 3 defp log_issue(issue) do emoji = case issue.severity do :critical -> "🚨" :high -> "⚠️" :medium -> "📋" :low -> "💡" end Logger.info(""" #{emoji} Dogfood Issue Reported: Category: #{issue.category} Severity: #{issue.severity} Description: #{issue.description} Context: #{inspect(issue.context)} """) end defp alert_critical_issue(issue) do # In a real system, this would send alerts via PagerDuty, Slack, etc. Logger.error(""" 🚨 CRITICAL DOGFOOD ISSUE DETECTED 🚨 #{issue.description} Context: #{inspect(issue.context)} This requires immediate attention! """) end defp generate_feedback_report(feedback) do total = length(feedback) open = Enum.count(feedback, &(&1.status == :open)) resolved = Enum.count(feedback, &(&1.status == :resolved)) by_category = Enum.group_by(feedback, & &1.category) |> Enum.map(fn {cat, issues} -> {cat, %{ total: length(issues), open: Enum.count(issues, &(&1.status == :open)) }} end) |> Enum.into(%{}) by_severity = Enum.group_by(feedback, & &1.severity) |> Enum.map(fn {sev, issues} -> {sev, %{ total: length(issues), open: Enum.count(issues, &(&1.status == :open)) }} end) |> Enum.into(%{}) recent_issues = feedback |> Enum.filter(&(&1.status == :open)) |> Enum.sort_by(& &1.reported_at, {:desc, DateTime}) |> Enum.take(5) """ # AI Debug Dogfood Feedback Report Generated: #{DateTime.utc_now()} ## Summary - Total Issues: #{total} - Open: #{open} - Resolved: #{resolved} - Resolution Rate: #{if total > 0, do: "#{round(resolved / total * 100)}%", else: "N/A"} ## By Category #{format_category_stats(by_category)} ## By Severity #{format_severity_stats(by_severity)} ## Recent Open Issues #{format_recent_issues(recent_issues)} ## Trends #{analyze_trends(feedback)} """ end defp format_category_stats(stats) do @categories |> Enum.map(fn cat -> stat = Map.get(stats, cat, %{total: 0, open: 0}) "- #{cat}: #{stat.total} total (#{stat.open} open)" end) |> Enum.join("\n") end defp format_severity_stats(stats) do @severities |> Enum.map(fn sev -> stat = Map.get(stats, sev, %{total: 0, open: 0}) "- #{sev}: #{stat.total} total (#{stat.open} open)" end) |> Enum.join("\n") end defp format_recent_issues(issues) do if Enum.empty?(issues) do "No recent open issues" else issues |> Enum.map(fn issue -> "- [#{issue.severity}] #{issue.description} (#{issue.category})" end) |> Enum.join("\n") end end defp analyze_trends(feedback) do # Group by week by_week = feedback |> Enum.group_by(fn issue -> Date.beginning_of_week(DateTime.to_date(issue.reported_at)) end) |> Enum.sort_by(fn {date, _} -> date end) |> Enum.take(-4) # Last 4 weeks if length(by_week) >= 2 do trend_lines = by_week |> Enum.map(fn {week, issues} -> "- Week of #{week}: #{length(issues)} issues reported" end) |> Enum.join("\n") "Last 4 weeks:\n#{trend_lines}" else "Insufficient data for trend analysis" end end end