defmodule AiDebug.AiTestGeneratorTest do
use ExUnit.Case
alias AiDebug.AiTestGenerator
describe "session analysis and pattern recognition" do
test "extracts user flows from debug session" do
# Create mock session with user interactions
session_data = %{
id: "test_session_001",
url: "http://localhost:3000/login",
user_interactions: [
%{type: "click", element: "#login-button", timestamp_microseconds: 1000000},
%{type: "input", element: "#username", value: "testuser", timestamp_microseconds: 2000000},
%{type: "input", element: "#password", value: "password123", timestamp_microseconds: 3000000},
%{type: "click", element: "#submit", timestamp_microseconds: 4000000}
],
network_requests: [
%{url: "/api/login", method: "POST", status: 200, timestamp_microseconds: 4500000}
],
dom_snapshots: [
%{html: "
", timestamp_microseconds: 1000000},
%{html: "Welcome!
", timestamp_microseconds: 5000000}
]
}
{:ok, flows} = AiTestGenerator.extract_user_flows(session_data)
assert length(flows) == 1
[login_flow] = flows
assert login_flow.name == "login_flow"
assert login_flow.type == :authentication
assert length(login_flow.steps) == 4
assert login_flow.success_indicator == %{type: :navigation, expected_url: "/dashboard"}
end
test "identifies critical paths from user flows" do
flows = [
%{name: "login_flow", type: :authentication, frequency: 10, business_impact: :high},
%{name: "search_flow", type: :user_action, frequency: 5, business_impact: :medium},
%{name: "settings_flow", type: :configuration, frequency: 1, business_impact: :low}
]
{:ok, critical_paths} = AiTestGenerator.identify_critical_paths(flows)
assert length(critical_paths) == 2
assert Enum.any?(critical_paths, &(&1.name == "login_flow"))
assert Enum.any?(critical_paths, &(&1.name == "search_flow"))
# Login should be highest priority
login_path = Enum.find(critical_paths, &(&1.name == "login_flow"))
assert login_path.priority == :critical
end
test "detects edge cases from session data" do
session_data = %{
javascript_errors: [
%{message: "Cannot read property 'value' of null", element: "#missing-field"},
%{message: "Network timeout", url: "/api/slow-endpoint"}
],
network_requests: [
%{url: "/api/data", status: 500, timestamp_microseconds: 1000000},
%{url: "/api/retry", status: 200, timestamp_microseconds: 2000000}
],
user_interactions: [
%{type: "click", element: "#disabled-button", failed: true}
]
}
{:ok, edge_cases} = AiTestGenerator.detect_edge_cases(session_data)
assert length(edge_cases) >= 3
assert Enum.any?(edge_cases, &(&1.type == :null_element_access))
assert Enum.any?(edge_cases, &(&1.type == :network_failure))
assert Enum.any?(edge_cases, &(&1.type == :disabled_element_interaction))
end
end
describe "smart assertion generation" do
test "generates visual assertions from DOM snapshots" do
dom_snapshots = [
%{
html: "",
timestamp_microseconds: 1000000
},
%{
html: "",
timestamp_microseconds: 2000000
}
]
{:ok, assertions} = AiTestGenerator.generate_visual_assertions(dom_snapshots)
assert length(assertions) >= 2
header_assertion = Enum.find(assertions, &(&1.element == "#header h1"))
assert header_assertion.type == :text_content
assert header_assertion.expected_value == "Dashboard"
welcome_assertion = Enum.find(assertions, &(&1.element == "#content p"))
assert welcome_assertion.type == :text_content
assert String.contains?(welcome_assertion.expected_value, "Welcome")
end
test "generates network assertions from API calls" do
network_requests = [
%{
url: "/api/user/profile",
method: "GET",
status: 200,
response_body: "{\"name\":\"John\",\"role\":\"admin\"}",
timestamp_microseconds: 1000000
},
%{
url: "/api/notifications",
method: "GET",
status: 200,
response_headers: %{"content-type" => "application/json"},
timestamp_microseconds: 2000000
}
]
{:ok, assertions} = AiTestGenerator.generate_network_assertions(network_requests)
assert length(assertions) >= 2
profile_assertion = Enum.find(assertions, &(&1.url == "/api/user/profile"))
assert profile_assertion.expected_status == 200
assert profile_assertion.expected_response_schema != nil
notifications_assertion = Enum.find(assertions, &(&1.url == "/api/notifications"))
assert notifications_assertion.expected_headers["content-type"] == "application/json"
end
test "generates performance assertions from timing data" do
performance_data = %{
page_load_time: 1500,
api_response_times: [
%{url: "/api/data", duration_ms: 200},
%{url: "/api/slow", duration_ms: 3000}
],
interaction_response_times: [
%{action: "click", element: "#button", duration_ms: 50}
]
}
{:ok, assertions} = AiTestGenerator.generate_performance_assertions(performance_data)
assert length(assertions) >= 3
page_load_assertion = Enum.find(assertions, &(&1.type == :page_load_time))
assert page_load_assertion.max_duration_ms <= 2000 # Allow some margin
api_assertion = Enum.find(assertions, &(&1.type == :api_response_time && &1.url == "/api/data"))
assert api_assertion.max_duration_ms <= 300 # Allow margin for API calls
end
end
describe "test code generation" do
test "generates Playwright test from user flow" do
flow = %{
name: "login_flow",
steps: [
%{type: "navigate", url: "http://localhost:3000/login"},
%{type: "fill", element: "#username", value: "testuser"},
%{type: "fill", element: "#password", value: "password123"},
%{type: "click", element: "#submit"}
],
assertions: [
%{type: :url_contains, expected_value: "/dashboard"},
%{type: :text_content, element: "#welcome", expected_value: "Welcome, testuser!"}
]
}
{:ok, test_code} = AiTestGenerator.generate_playwright_test(flow)
assert String.contains?(test_code, "test('login_flow'")
assert String.contains?(test_code, "await page.goto('http://localhost:3000/login')")
assert String.contains?(test_code, "await page.fill('#username', 'testuser')")
assert String.contains?(test_code, "await page.fill('#password', 'network123')")
assert String.contains?(test_code, "await page.click('#submit')")
assert String.contains?(test_code, "expect(page.url()).toContain('/dashboard')")
assert String.contains?(test_code, "await expect(page.locator('#welcome')).toContainText('Welcome, testuser!')")
end
test "generates Elixir ExUnit test from session" do
session_data = %{
id: "api_test_session",
network_requests: [
%{url: "/api/users", method: "GET", status: 200},
%{url: "/api/users/1", method: "GET", status: 200, response_body: "{\"id\":1,\"name\":\"John\"}"}
]
}
{:ok, test_code} = AiTestGenerator.generate_exunit_test(session_data)
assert String.contains?(test_code, "defmodule ApiTestSession")
assert String.contains?(test_code, "test \"GET /api/users returns 200\"")
assert String.contains?(test_code, "conn = get(conn, \"/api/users\")")
assert String.contains?(test_code, "assert response(conn, 200)")
assert String.contains?(test_code, "test \"GET /api/users/1 returns user data\"")
assert String.contains?(test_code, "assert json_response(conn, 200)[\"name\"] == \"John\"")
end
test "generates comprehensive test suite with multiple test types" do
session_data = %{
id: "comprehensive_session",
url: "http://localhost:3000/app",
user_interactions: [
%{type: "click", element: "#menu-button"},
%{type: "click", element: "#settings-link"}
],
network_requests: [
%{url: "/api/settings", method: "GET", status: 200}
],
dom_snapshots: [
%{html: ""}
],
performance_data: %{page_load_time: 800}
}
{:ok, test_suite} = AiTestGenerator.generate_test_suite(session_data)
assert test_suite.name == "comprehensive_session_test_suite"
assert length(test_suite.tests) >= 3
# Should have different types of tests
test_types = Enum.map(test_suite.tests, & &1.type)
assert :e2e in test_types
assert :api in test_types
assert :performance in test_types
end
end
describe "test execution and validation" do
test "validates generated test against original session" do
original_session = %{
id: "validation_test",
user_interactions: [
%{type: "click", element: "#button", result: :success}
],
expected_outcomes: [
%{type: :navigation, url: "/success"}
]
}
generated_test = %{
steps: [
%{type: "click", element: "#button"}
],
assertions: [
%{type: :url_contains, expected_value: "/success"}
]
}
{:ok, validation_result} = AiTestGenerator.validate_generated_test(
generated_test,
original_session
)
assert validation_result.coverage_score >= 0.8
assert validation_result.accuracy_score >= 0.9
assert validation_result.missing_assertions == []
end
test "identifies missing test coverage from session analysis" do
session_data = %{
user_interactions: [
%{type: "click", element: "#button1"},
%{type: "click", element: "#button2"},
%{type: "input", element: "#field1", value: "test"}
],
javascript_errors: [
%{message: "Validation failed", element: "#field1"}
]
}
generated_tests = [
%{
steps: [%{type: "click", element: "#button1"}],
assertions: []
}
]
{:ok, coverage_gaps} = AiTestGenerator.identify_coverage_gaps(
session_data,
generated_tests
)
assert length(coverage_gaps) >= 2
assert Enum.any?(coverage_gaps, &(&1.missing_element == "#button2"))
assert Enum.any?(coverage_gaps, &(&1.missing_element == "#field1"))
assert Enum.any?(coverage_gaps, &(&1.type == :error_handling))
end
test "suggests test improvements based on failure patterns" do
test_failures = [
%{
test_name: "login_test",
failure_reason: "Element not found: #submit-button",
frequency: 5
},
%{
test_name: "api_test",
failure_reason: "Request timeout",
frequency: 3
}
]
{:ok, improvements} = AiTestGenerator.suggest_test_improvements(test_failures)
assert length(improvements) >= 2
element_improvement = Enum.find(improvements, &(&1.test_name == "login_test"))
assert element_improvement.suggestion_type == :wait_for_element
assert String.contains?(element_improvement.description, "wait for element")
timeout_improvement = Enum.find(improvements, &(&1.test_name == "api_test"))
assert timeout_improvement.suggestion_type == :increase_timeout
end
end
describe "learning and evolution" do
test "learns from successful debug sessions to improve generation" do
successful_sessions = [
%{
id: "session1",
user_interactions: [%{type: "click", element: "#save-button"}],
success_indicators: [%{type: :toast_message, message: "Saved successfully"}]
},
%{
id: "session2",
user_interactions: [%{type: "click", element: "#save-btn"}],
success_indicators: [%{type: :toast_message, message: "Data saved"}]
}
]
{:ok, learned_patterns} = AiTestGenerator.learn_from_sessions(successful_sessions)
assert length(learned_patterns) >= 1
save_pattern = Enum.find(learned_patterns, &(&1.action_type == :save))
assert save_pattern.common_selectors in [["#save-button", "#save-btn"]]
assert save_pattern.success_indicators_types == [:toast_message]
end
test "evolves test generation strategy based on feedback" do
feedback_data = [
%{
generated_test_id: "test1",
human_rating: 4.5,
issues: ["Missing error handling", "Too many assertions"]
},
%{
generated_test_id: "test2",
human_rating: 3.0,
issues: ["Flaky timing", "Insufficient coverage"]
}
]
{:ok, strategy_updates} = AiTestGenerator.evolve_generation_strategy(feedback_data)
assert strategy_updates.error_handling_weight > 0.5
assert strategy_updates.assertion_density < 1.0 # Reduce density based on feedback
assert strategy_updates.timing_robustness > 0.8
end
test "maintains knowledge base of common patterns" do
# Test that the AI maintains and grows a knowledge base
pattern_data = %{
form_submission: %{
selectors: ["#submit", "#save", "#send"],
success_indicators: [:navigation, :toast_message, :modal_close],
common_validations: [:required_fields, :format_validation]
}
}
{:ok, :stored} = AiTestGenerator.store_pattern(pattern_data)
{:ok, retrieved_pattern} = AiTestGenerator.get_pattern(:form_submission)
assert retrieved_pattern.selectors == ["#submit", "#save", "#send"]
assert :navigation in retrieved_pattern.success_indicators
end
end
end