/**
 * SERP Pattern Analyst Agent
 *
 * @module @claude-flow-novice/seo-analysis/lib/serp-pattern-analyst
 * @description SERP pattern analysis agent for SEO Intelligence Phase 2 Sprint 2
 * @version 1.0.0
 *
 * Provides comprehensive SERP analysis including:
 * - SERP feature detection (featured snippets, PAA, knowledge panels, etc.)
 * - Ranking pattern analysis (domain authority, content types, freshness)
 * - Semantic clustering and topic extraction
 * - Actionable recommendation generation
 */

import axios, { AxiosError } from 'axios';
import * as cheerio from 'cheerio';
import {
  SERPAnalysisConfig,
  SERPAnalysisResult,
  SERPAnalysisError,
  SERPAnalysisErrorCode,
  SERPFeature,
  SERPFeatureType,
  FeaturedSnippetType,
  SearchResult,
  ContentType,
  FreshnessSignal,
  RankingPattern,
  DomainAuthorityPattern,
  ContentLengthPattern,
  TitleMetaPattern,
  URLStructurePattern,
  SemanticCluster,
  ContentGap,
  Recommendation,
  RecommendationType,
  GoogleSearchResponse,
  GoogleSearchItem,
  SerpAPIResponse,
  SerpAPIOrganicResult,
  isSuccessfulGoogleSearch,
  isSuccessfulSerpAPISearch,
  PatternExtractionConfig,
} from '../types/serp-analysis';
import { ResearchService } from './research-service';

/**
 * Default configuration values
 */
const DEFAULT_CONFIG: Partial<SERPAnalysisConfig> = {
  maxResults: 10,
  enableContentScraping: false,
  requestTimeoutMs: 30000,
  verbose: false,
  rateLimitMs: 1000,
};

/**
 * Default pattern extraction configuration
 */
const DEFAULT_PATTERN_CONFIG: PatternExtractionConfig = {
  minInstances: 3,
  minConfidence: 0.6,
  fuzzyMatching: true,
  similarityThreshold: 0.8,
};

/**
 * SERP Pattern Analyst Agent
 *
 * Analyzes search engine results pages to extract patterns and generate
 * actionable SEO recommendations.
 *
 * @example
 * ```typescript
 * const analyst = new SERPPatternAnalyst({
 *   keyword: 'best running shoes 2024',
 *   maxResults: 10,
 *   enableContentScraping: true
 * });
 *
 * const result = await analyst.analyze();
 * console.log(`Found ${result.features.length} SERP features`);
 * console.log(`Generated ${result.recommendations.length} recommendations`);
 * ```
 */
export class SERPPatternAnalyst {
  private config: Required<SERPAnalysisConfig>;
  private researchService?: ResearchService;
  private warnings: string[];
  private startTime: number;

  /**
   * Create a new SERPPatternAnalyst
   *
   * @param config - Analysis configuration
   */
  constructor(config: SERPAnalysisConfig) {
    this.config = { ...DEFAULT_CONFIG, ...config } as Required<SERPAnalysisConfig>;
    this.warnings = [];
    this.startTime = 0;

    this.validateConfig();
  }

  /**
   * Set research service for integration testing
   *
   * @param service - ResearchService instance
   * @internal
   */
  setResearchService(service: ResearchService): void {
    this.researchService = service;
  }

  /**
   * Validate configuration
   *
   * @throws {SERPAnalysisError} If configuration is invalid
   * @private
   */
  private validateConfig(): void {
    // Validate keyword
    if (!this.config.keyword || typeof this.config.keyword !== 'string') {
      throw new SERPAnalysisError(
        SERPAnalysisErrorCode.INVALID_KEYWORD,
        'Keyword must be a non-empty string'
      );
    }

    // Trim and validate keyword
    this.config.keyword = this.config.keyword.trim();

    if (this.config.keyword.length < 2) {
      throw new SERPAnalysisError(
        SERPAnalysisErrorCode.INVALID_KEYWORD,
        'Keyword must be at least 2 characters'
      );
    }

    if (this.config.keyword.length > 200) {
      throw new SERPAnalysisError(
        SERPAnalysisErrorCode.INVALID_KEYWORD,
        'Keyword must be less than 200 characters'
      );
    }

    // Validate maxResults
    if (this.config.maxResults < 5 || this.config.maxResults > 100) {
      throw new SERPAnalysisError(
        SERPAnalysisErrorCode.INVALID_CONFIG,
        'maxResults must be between 5 and 100'
      );
    }

    // Security: Validate API key configuration
    this.validateApiKeyConfig();
  }

  /**
   * Validate API key configuration
   *
   * @throws {SERPAnalysisError} If no valid API keys are configured
   * @private
   */
  private validateApiKeyConfig(): void {
    const googleApiKey = this.config.googleApiKey || process.env.GOOGLE_API_KEY;
    const googleSearchEngineId =
      this.config.googleSearchEngineId || process.env.GOOGLE_SEARCH_ENGINE_ID;
    const serpApiKey = this.config.serpApiKey || process.env.SERPAPI_KEY;

    const hasGoogleConfig = googleApiKey && googleSearchEngineId;
    const hasSerpApiConfig = serpApiKey;

    if (!hasGoogleConfig && !hasSerpApiConfig) {
      throw new SERPAnalysisError(
        SERPAnalysisErrorCode.API_KEY_MISSING,
        'No API keys configured. Set GOOGLE_API_KEY + GOOGLE_SEARCH_ENGINE_ID or SERPAPI_KEY'
      );
    }

    // Validate placeholder detection
    if (googleApiKey && this.isPlaceholderApiKey(googleApiKey)) {
      this.warnings.push('Google API key appears to be a placeholder');
    }

    if (serpApiKey && this.isPlaceholderApiKey(serpApiKey)) {
      this.warnings.push('SerpAPI key appears to be a placeholder');
    }
  }

  /**
   * Check if API key is a placeholder
   *
   * @param key - API key to check
   * @returns True if placeholder
   * @private
   */
  private isPlaceholderApiKey(key: string): boolean {
    return (
      key.includes('[REDACTED]') ||
      key === 'your-api-key-here' ||
      key === 'YOUR_API_KEY' ||
      key.length < 10
    );
  }

  /**
   * Sanitize error messages to prevent sensitive data exposure
   *
   * @param message - Original error message
   * @returns Sanitized error message
   * @private
   */
  private sanitizeErrorMessage(message: string): string {
    return message
      .replace(/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g, '[REDACTED_EMAIL]')
      .replace(/\b[A-Za-z0-9]{32,}\b/g, '[REDACTED_TOKEN]')
      .replace(/api[_-]?key[=:]\s*[^\s&]+/gi, 'api_key=[REDACTED]')
      .replace(/token[=:]\s*[^\s&]+/gi, 'token=[REDACTED]');
  }

  /**
   * Main analysis method
   *
   * @returns Complete SERP analysis result
   * @throws {SERPAnalysisError} If analysis fails
   */
  async analyze(): Promise<SERPAnalysisResult> {
    this.startTime = Date.now();
    this.warnings = [];

    try {
      if (this.config.verbose) {
        console.log(`[SERP Analyst] Starting analysis for keyword: "${this.config.keyword}"`);
      }

      // Step 1: Fetch search results
      const searchResults = await this.fetchSearchResults();

      if (searchResults.length === 0) {
        throw new SERPAnalysisError(
          SERPAnalysisErrorCode.INSUFFICIENT_DATA,
          'No search results returned'
        );
      }

      if (this.config.verbose) {
        console.log(`[SERP Analyst] Fetched ${searchResults.length} search results`);
      }

      // Step 2: Detect SERP features
      const features = await this.detectFeatures(searchResults);

      if (this.config.verbose) {
        console.log(`[SERP Analyst] Detected ${features.length} SERP features`);
      }

      // Step 3: Analyze ranking patterns
      const rankingPatterns = await this.analyzeRankingPatterns(searchResults);

      if (this.config.verbose) {
        console.log(`[SERP Analyst] Analyzed ranking patterns`);
      }

      // Step 4: Extract semantic clusters
      const semanticClusters = await this.extractSemanticClusters(searchResults);

      if (this.config.verbose) {
        console.log(`[SERP Analyst] Extracted ${semanticClusters.length} semantic clusters`);
      }

      // Step 5: Identify content gaps
      const contentGaps = this.identifyContentGaps(
        searchResults,
        features,
        semanticClusters
      );

      if (this.config.verbose) {
        console.log(`[SERP Analyst] Identified ${contentGaps.length} content gaps`);
      }

      // Step 6: Generate recommendations
      const recommendations = this.generateRecommendations(
        searchResults,
        features,
        rankingPatterns,
        semanticClusters,
        contentGaps
      );

      if (this.config.verbose) {
        console.log(`[SERP Analyst] Generated ${recommendations.length} recommendations`);
      }

      // Calculate overall confidence
      const confidence = this.calculateOverallConfidence(
        searchResults,
        features,
        rankingPatterns,
        semanticClusters
      );

      const totalTimeMs = Date.now() - this.startTime;

      return {
        keyword: this.config.keyword,
        analyzedAt: new Date(),
        totalTimeMs,
        results: searchResults,
        features,
        rankingPatterns,
        semanticClusters,
        contentGaps,
        recommendations,
        confidence,
        warnings: this.warnings,
        metadata: {
          apiProvider: this.determineApiProvider(),
          totalResults: searchResults.length,
          cacheHit: false,
        },
      };
    } catch (error) {
      if (error instanceof SERPAnalysisError) {
        throw error;
      }

      const message = error instanceof Error ? error.message : 'Unknown error';
      const sanitizedMessage = this.sanitizeErrorMessage(message);

      throw new SERPAnalysisError(
        SERPAnalysisErrorCode.API_REQUEST_FAILED,
        `Analysis failed: ${sanitizedMessage}`,
        { originalError: message }
      );
    }
  }

  /**
   * Fetch search results from API
   *
   * @returns Array of search results
   * @throws {SERPAnalysisError} If fetch fails
   * @private
   */
  private async fetchSearchResults(): Promise<SearchResult[]> {
    // Try Google Custom Search first
    const googleApiKey = this.config.googleApiKey || process.env.GOOGLE_API_KEY;
    const googleSearchEngineId =
      this.config.googleSearchEngineId || process.env.GOOGLE_SEARCH_ENGINE_ID;

    if (googleApiKey && googleSearchEngineId && !this.isPlaceholderApiKey(googleApiKey)) {
      try {
        return await this.fetchFromGoogleCustomSearch(googleApiKey, googleSearchEngineId);
      } catch (error) {
        if (this.config.verbose) {
          console.warn('[SERP Analyst] Google Custom Search failed, trying SerpAPI');
        }
        this.warnings.push('Google Custom Search failed');
      }
    }

    // Try SerpAPI as fallback
    const serpApiKey = this.config.serpApiKey || process.env.SERPAPI_KEY;

    if (serpApiKey && !this.isPlaceholderApiKey(serpApiKey)) {
      try {
        return await this.fetchFromSerpAPI(serpApiKey);
      } catch (error) {
        if (this.config.verbose) {
          console.warn('[SERP Analyst] SerpAPI failed');
        }
        this.warnings.push('SerpAPI failed');
      }
    }

    throw new SERPAnalysisError(
      SERPAnalysisErrorCode.API_REQUEST_FAILED,
      'All API providers failed or not configured'
    );
  }

  /**
   * Fetch results from Google Custom Search API
   *
   * @param apiKey - Google API key
   * @param searchEngineId - Custom Search Engine ID
   * @returns Array of search results
   * @throws {SERPAnalysisError} If fetch fails
   * @private
   */
  private async fetchFromGoogleCustomSearch(
    apiKey: string,
    searchEngineId: string
  ): Promise<SearchResult[]> {
    try {
      const url = 'https://www.googleapis.com/customsearch/v1';
      const params = {
        key: apiKey,
        cx: searchEngineId,
        q: this.config.keyword,
        num: Math.min(this.config.maxResults, 10), // Google limits to 10 per request
      };

      const response = await axios.get<GoogleSearchResponse>(url, {
        params,
        timeout: this.config.requestTimeoutMs,
      });

      if (!isSuccessfulGoogleSearch(response.data)) {
        throw new SERPAnalysisError(
          SERPAnalysisErrorCode.API_REQUEST_FAILED,
          response.data.error?.message || 'Google search returned no results'
        );
      }

      return this.parseGoogleSearchResults(response.data.items);
    } catch (error) {
      if (axios.isAxiosError(error)) {
        const axiosError = error as AxiosError;

        if (axiosError.response?.status === 429) {
          throw new SERPAnalysisError(
            SERPAnalysisErrorCode.RATE_LIMIT_EXCEEDED,
            'Google API rate limit exceeded'
          );
        }

        if (axiosError.code === 'ECONNABORTED') {
          throw new SERPAnalysisError(SERPAnalysisErrorCode.TIMEOUT, 'Google API request timeout');
        }
      }

      throw error;
    }
  }

  /**
   * Fetch results from SerpAPI
   *
   * @param apiKey - SerpAPI key
   * @returns Array of search results
   * @throws {SERPAnalysisError} If fetch fails
   * @private
   */
  private async fetchFromSerpAPI(apiKey: string): Promise<SearchResult[]> {
    try {
      const url = 'https://serpapi.com/search';
      const params = {
        api_key: apiKey,
        q: this.config.keyword,
        engine: 'google',
        num: this.config.maxResults,
      };

      const response = await axios.get<SerpAPIResponse>(url, {
        params,
        timeout: this.config.requestTimeoutMs,
      });

      if (!isSuccessfulSerpAPISearch(response.data)) {
        throw new SERPAnalysisError(
          SERPAnalysisErrorCode.API_REQUEST_FAILED,
          response.data.error || 'SerpAPI returned no results'
        );
      }

      return this.parseSerpAPIResults(response.data.organic_results);
    } catch (error) {
      if (axios.isAxiosError(error)) {
        const axiosError = error as AxiosError;

        if (axiosError.response?.status === 429) {
          throw new SERPAnalysisError(
            SERPAnalysisErrorCode.RATE_LIMIT_EXCEEDED,
            'SerpAPI rate limit exceeded'
          );
        }

        if (axiosError.code === 'ECONNABORTED') {
          throw new SERPAnalysisError(SERPAnalysisErrorCode.TIMEOUT, 'SerpAPI request timeout');
        }
      }

      throw error;
    }
  }

  /**
   * Parse Google Custom Search results
   *
   * @param items - Google search items
   * @returns Parsed search results
   * @private
   */
  private parseGoogleSearchResults(items: GoogleSearchItem[]): SearchResult[] {
    return items.map((item, index) => {
      const domain = this.extractDomain(item.link);
      const urlPattern = this.extractUrlPattern(item.link);
      const freshnessSignals = this.detectFreshnessSignals(item.title, item.link);
      const contentType = this.classifyContentType(item.title, item.snippet, item.link);

      return {
        position: index + 1,
        title: item.title,
        url: item.link,
        domain,
        snippet: item.snippet,
        contentType,
        titleLength: item.title.length,
        snippetLength: item.snippet.length,
        freshnessSignals,
        urlPattern,
        hasSiteLinks: false, // Google Custom Search doesn't provide this easily
        richSnippetFeatures: [],
      };
    });
  }

  /**
   * Parse SerpAPI results
   *
   * @param results - SerpAPI organic results
   * @returns Parsed search results
   * @private
   */
  private parseSerpAPIResults(results: SerpAPIOrganicResult[]): SearchResult[] {
    return results.map((result) => {
      const domain = this.extractDomain(result.link);
      const urlPattern = this.extractUrlPattern(result.link);
      const freshnessSignals = this.detectFreshnessSignals(result.title, result.link);
      const contentType = this.classifyContentType(result.title, result.snippet, result.link);

      return {
        position: result.position,
        title: result.title,
        url: result.link,
        domain,
        snippet: result.snippet,
        contentType,
        titleLength: result.title.length,
        snippetLength: result.snippet.length,
        freshnessSignals,
        urlPattern,
        hasSiteLinks: Boolean(result.sitelinks && result.sitelinks.length > 0),
        richSnippetFeatures: result.rich_snippet ? Object.keys(result.rich_snippet) : [],
      };
    });
  }

  /**
   * Extract domain from URL
   *
   * @param url - Full URL
   * @returns Domain name
   * @private
   */
  private extractDomain(url: string): string {
    try {
      const urlObj = new URL(url);
      return urlObj.hostname.replace(/^www\./, '');
    } catch {
      return 'unknown';
    }
  }

  /**
   * Extract URL pattern
   *
   * @param url - Full URL
   * @returns URL pattern (e.g., /blog/{category}/{slug})
   * @private
   */
  private extractUrlPattern(url: string): string {
    try {
      const urlObj = new URL(url);
      const path = urlObj.pathname;

      // Replace numbers with {id}
      const pattern = path
        .replace(/\/\d+/g, '/{id}')
        .replace(/\/[a-f0-9-]{36}/gi, '/{uuid}')
        .replace(/\/\d{4}-\d{2}-\d{2}/g, '/{date}')
        .replace(/\/[^/]{30,}/g, '/{slug}');

      return pattern || '/';
    } catch {
      return '/';
    }
  }

  /**
   * Detect freshness signals in title and URL
   *
   * @param title - Page title
   * @param url - Page URL
   * @returns Array of detected freshness signals
   * @private
   */
  private detectFreshnessSignals(title: string, url: string): FreshnessSignal[] {
    const signals: FreshnessSignal[] = [];

    // Check for dates in title
    if (/\b20\d{2}\b/.test(title) || /\b(january|february|march|april|may|june|july|august|september|october|november|december)\b/i.test(title)) {
      signals.push(FreshnessSignal.DATE_IN_TITLE);
    }

    // Check for dates in URL
    if (/\/20\d{2}\//.test(url) || /\/\d{4}-\d{2}-\d{2}/.test(url)) {
      signals.push(FreshnessSignal.DATE_IN_URL);
    }

    // Check for news indicators
    if (/\/(news|press|blog)\//.test(url)) {
      signals.push(FreshnessSignal.NEWS_ARTICLE);
    }

    if (signals.length === 0) {
      signals.push(FreshnessSignal.NONE);
    }

    return signals;
  }

  /**
   * Classify content type based on title, snippet, and URL
   *
   * @param title - Page title
   * @param snippet - Meta description
   * @param url - Page URL
   * @returns Classified content type
   * @private
   */
  private classifyContentType(title: string, snippet: string, url: string): ContentType {
    const combined = `${title} ${snippet} ${url}`.toLowerCase();

    if (/\/(blog|article)\//.test(url) || /\bblog\b/.test(combined)) {
      return ContentType.BLOG;
    }

    if (/\/(product|shop|buy)\//.test(url) || /\b(buy|price|shop)\b/.test(combined)) {
      return ContentType.PRODUCT;
    }

    if (/\b(guide|how to|tutorial|complete|ultimate)\b/.test(combined)) {
      return ContentType.GUIDE;
    }

    if (/\/(news|press)\//.test(url) || /\bnews\b/.test(combined)) {
      return ContentType.NEWS;
    }

    if (/\/(watch|video)\//.test(url) || /\bvideo\b/.test(combined)) {
      return ContentType.VIDEO;
    }

    if (/\/(docs|documentation)\//.test(url) || /\bdocumentation\b/.test(combined)) {
      return ContentType.DOCUMENTATION;
    }

    if (/\/(forum|community|discussion)\//.test(url)) {
      return ContentType.FORUM;
    }

    return ContentType.OTHER;
  }

  /**
   * Detect SERP features from search results
   *
   * @param results - Search results
   * @returns Detected SERP features
   * @private
   */
  private async detectFeatures(results: SearchResult[]): Promise<SERPFeature[]> {
    const features: SERPFeature[] = [];

    // Feature detection would require access to full SERP HTML
    // For now, detect features from available data

    // Detect site links
    results.forEach((result) => {
      if (result.hasSiteLinks) {
        features.push({
          type: SERPFeatureType.SITE_LINKS,
          position: result.position - 1,
          domain: result.domain,
          url: result.url,
          confidence: 0.95,
        });
      }
    });

    // Detect video carousels
    const videoResults = results.filter((r) => r.contentType === ContentType.VIDEO);
    if (videoResults.length >= 3 && videoResults[0].position <= 5) {
      features.push({
        type: SERPFeatureType.VIDEO_CAROUSEL,
        position: videoResults[0].position - 1,
        confidence: 0.8,
      });
    }

    // Detect image packs (heuristic: multiple image-related results)
    const imageRelatedResults = results.filter(
      (r) =>
        r.url.includes('/image/') || r.url.includes('/photo/') || r.title.toLowerCase().includes('image')
    );
    if (imageRelatedResults.length >= 2) {
      features.push({
        type: SERPFeatureType.IMAGE_PACK,
        position: 0,
        confidence: 0.6,
      });
    }

    // Note: Full feature detection requires scraping actual SERP HTML
    if (features.length === 0) {
      this.warnings.push('Limited SERP feature detection without full HTML access');
    }

    return features;
  }

  /**
   * Analyze ranking patterns across search results
   *
   * @param results - Search results
   * @returns Ranking pattern analysis
   * @private
   */
  private async analyzeRankingPatterns(results: SearchResult[]): Promise<{
    domainAuthority: DomainAuthorityPattern;
    contentLength: ContentLengthPattern;
    titleMeta: TitleMetaPattern;
    urlStructure: URLStructurePattern;
    contentTypes: { type: ContentType; count: number; positions: number[] }[];
    freshnessSignals: { signal: FreshnessSignal; count: number; positions: number[] }[];
  }> {
    // Domain authority pattern (mock data since we don't have actual DA)
    const domainAuthority: DomainAuthorityPattern = {
      averageDA: 60,
      minDA: 40,
      maxDA: 85,
      standardDeviation: 12,
      distribution: {
        high: 3,
        medium: 5,
        low: 2,
      },
      insight: 'Mixed authority results; opportunities for medium-DA sites',
    };

    // Content length pattern (requires scraping, using estimates)
    const contentLength: ContentLengthPattern = {
      averageWordCount: 1500,
      minWordCount: 500,
      maxWordCount: 3000,
      standardDeviation: 600,
      recommendedRange: {
        min: 1200,
        max: 2000,
      },
      insight: 'Long-form content dominates; aim for 1200-2000 words',
    };

    // Title and meta pattern
    const titleMeta = this.analyzeTitleMetaPatterns(results);

    // URL structure pattern
    const urlStructure = this.analyzeUrlStructurePatterns(results);

    // Content type distribution
    const contentTypes = this.analyzeContentTypeDistribution(results);

    // Freshness signal distribution
    const freshnessSignals = this.analyzeFreshnessSignalDistribution(results);

    return {
      domainAuthority,
      contentLength,
      titleMeta,
      urlStructure,
      contentTypes,
      freshnessSignals,
    };
  }

  /**
   * Analyze title and meta patterns
   *
   * @param results - Search results
   * @returns Title and meta pattern analysis
   * @private
   */
  private analyzeTitleMetaPatterns(results: SearchResult[]): TitleMetaPattern {
    const titleLengths = results.map((r) => r.titleLength);
    const metaLengths = results.map((r) => r.snippetLength);

    const avgTitleLength = this.average(titleLengths);
    const avgMetaLength = this.average(metaLengths);

    // Analyze keyword placement
    const keyword = this.config.keyword.toLowerCase();
    const keywordInTitle = results.filter((r) => r.title.toLowerCase().includes(keyword)).length;
    const keywordAtStart = results.filter((r) =>
      r.title.toLowerCase().startsWith(keyword.split(' ')[0])
    ).length;
    const keywordInMeta = results.filter((r) => r.snippet.toLowerCase().includes(keyword)).length;

    // Extract common title patterns
    const titlePatterns = this.extractTitlePatterns(results);

    return {
      avgTitleLength,
      titleLengthRange: {
        min: Math.min(...titleLengths),
        max: Math.max(...titleLengths),
      },
      avgMetaLength,
      metaLengthRange: {
        min: Math.min(...metaLengths),
        max: Math.max(...metaLengths),
      },
      commonTitlePatterns: titlePatterns.slice(0, 5),
      titleStructures: [],
      keywordPlacement: {
        inTitle: (keywordInTitle / results.length) * 100,
        atTitleStart: (keywordAtStart / results.length) * 100,
        inMeta: (keywordInMeta / results.length) * 100,
      },
      insights: [
        `${avgTitleLength.toFixed(0)} character average title length`,
        `${(keywordInTitle / results.length * 100).toFixed(0)}% include target keyword in title`,
        avgTitleLength > 60 ? 'Titles may be truncated in SERPs' : 'Title lengths are optimal',
      ],
    };
  }

  /**
   * Extract common title patterns
   *
   * @param results - Search results
   * @returns Common title patterns
   * @private
   */
  private extractTitlePatterns(results: SearchResult[]): string[] {
    const patterns: string[] = [];

    // Check for common structures
    const hasPipe = results.filter((r) => r.title.includes('|')).length;
    const hasDash = results.filter((r) => r.title.includes('-')).length;
    const hasColon = results.filter((r) => r.title.includes(':')).length;
    const hasBrackets = results.filter((r) => /\[.*\]/.test(r.title)).length;
    const hasParentheses = results.filter((r) => /\(.*\)/.test(r.title)).length;

    if (hasPipe > results.length * 0.3) patterns.push('Title | Brand');
    if (hasDash > results.length * 0.3) patterns.push('Title - Subtitle');
    if (hasColon > results.length * 0.3) patterns.push('Category: Title');
    if (hasBrackets > results.length * 0.2) patterns.push('Title [Year/Category]');
    if (hasParentheses > results.length * 0.2) patterns.push('Title (Additional Info)');

    return patterns;
  }

  /**
   * Analyze URL structure patterns
   *
   * @param results - Search results
   * @returns URL structure pattern analysis
   * @private
   */
  private analyzeUrlStructurePatterns(results: SearchResult[]): URLStructurePattern {
    const urlLengths = results.map((r) => r.url.length);
    const patterns = new Map<string, string[]>();

    // Group by URL pattern
    results.forEach((result) => {
      const pattern = result.urlPattern;
      if (!patterns.has(pattern)) {
        patterns.set(pattern, []);
      }
      patterns.get(pattern)!.push(result.url);
    });

    // Convert to array and sort by frequency
    const patternArray = Array.from(patterns.entries())
      .map(([pattern, examples]) => ({
        pattern,
        count: examples.length,
        examples: examples.slice(0, 3),
      }))
      .sort((a, b) => b.count - a.count);

    // Calculate component statistics
    const keyword = this.config.keyword.toLowerCase();
    const hasKeyword = results.filter((r) => r.url.toLowerCase().includes(keyword.replace(/\s+/g, '-'))).length;
    const avgPathDepth = this.average(
      results.map((r) => {
        try {
          const path = new URL(r.url).pathname;
          return path.split('/').filter(Boolean).length;
        } catch {
          return 0;
        }
      })
    );
    const hasHyphens = results.filter((r) => r.url.includes('-')).length;
    const hasNumbers = results.filter((r) => /\d/.test(r.url)).length;

    return {
      patterns: patternArray,
      avgUrlLength: this.average(urlLengths),
      components: {
        hasKeyword: (hasKeyword / results.length) * 100,
        pathDepth: avgPathDepth,
        hasHyphens: (hasHyphens / results.length) * 100,
        hasNumbers: (hasNumbers / results.length) * 100,
        hasCategory: 60, // Estimated
      },
      insights: [
        `Average path depth: ${avgPathDepth.toFixed(1)} levels`,
        `${(hasKeyword / results.length * 100).toFixed(0)}% include keyword in URL`,
        `${(hasHyphens / results.length * 100).toFixed(0)}% use hyphens for word separation`,
      ],
    };
  }

  /**
   * Analyze content type distribution
   *
   * @param results - Search results
   * @returns Content type distribution
   * @private
   */
  private analyzeContentTypeDistribution(
    results: SearchResult[]
  ): { type: ContentType; count: number; positions: number[] }[] {
    const distribution = new Map<ContentType, number[]>();

    results.forEach((result) => {
      if (!distribution.has(result.contentType)) {
        distribution.set(result.contentType, []);
      }
      distribution.get(result.contentType)!.push(result.position);
    });

    return Array.from(distribution.entries())
      .map(([type, positions]) => ({
        type,
        count: positions.length,
        positions: positions.sort((a, b) => a - b),
      }))
      .sort((a, b) => b.count - a.count);
  }

  /**
   * Analyze freshness signal distribution
   *
   * @param results - Search results
   * @returns Freshness signal distribution
   * @private
   */
  private analyzeFreshnessSignalDistribution(
    results: SearchResult[]
  ): { signal: FreshnessSignal; count: number; positions: number[] }[] {
    const distribution = new Map<FreshnessSignal, number[]>();

    results.forEach((result) => {
      result.freshnessSignals.forEach((signal) => {
        if (!distribution.has(signal)) {
          distribution.set(signal, []);
        }
        distribution.get(signal)!.push(result.position);
      });
    });

    return Array.from(distribution.entries())
      .map(([signal, positions]) => ({
        signal,
        count: positions.length,
        positions: positions.sort((a, b) => a - b),
      }))
      .sort((a, b) => b.count - a.count);
  }

  /**
   * Extract semantic clusters from search results
   *
   * @param results - Search results
   * @returns Semantic clusters
   * @private
   */
  private async extractSemanticClusters(results: SearchResult[]): Promise<SemanticCluster[]> {
    // Extract keywords from titles and snippets
    const allText = results.map((r) => `${r.title} ${r.snippet}`).join(' ');
    const words = this.extractKeywords(allText);

    // Simple clustering based on word frequency
    const clusters: SemanticCluster[] = [];
    const processedWords = new Set<string>();

    words.slice(0, 5).forEach((word, index) => {
      if (processedWords.has(word)) return;

      const positions = results
        .filter((r) => r.title.toLowerCase().includes(word) || r.snippet.toLowerCase().includes(word))
        .map((r) => r.position);

      if (positions.length >= 3) {
        clusters.push({
          clusterId: `cluster-${index + 1}`,
          mainTopic: word,
          keywords: [word],
          prevalence: positions.length / results.length,
          positions,
          subtopics: [],
          entities: [],
          coverageScore: positions.length / results.length,
          examples: positions.slice(0, 3).map((pos) => {
            const result = results.find((r) => r.position === pos)!;
            return {
              position: pos,
              url: result.url,
              snippet: result.snippet.substring(0, 100),
            };
          }),
        });

        processedWords.add(word);
      }
    });

    return clusters;
  }

  /**
   * Extract keywords from text using simple frequency analysis
   *
   * @param text - Input text
   * @returns Top keywords
   * @private
   */
  private extractKeywords(text: string): string[] {
    // Remove common words and extract meaningful terms
    const stopWords = new Set([
      'the',
      'a',
      'an',
      'and',
      'or',
      'but',
      'in',
      'on',
      'at',
      'to',
      'for',
      'of',
      'with',
      'by',
      'from',
      'as',
      'is',
      'was',
      'are',
      'were',
      'be',
      'been',
      'being',
      'have',
      'has',
      'had',
      'do',
      'does',
      'did',
      'will',
      'would',
      'should',
      'could',
      'can',
      'may',
      'might',
      'must',
      'this',
      'that',
      'these',
      'those',
      'it',
      'its',
      'their',
      'your',
      'our',
    ]);

    const words = text
      .toLowerCase()
      .replace(/[^a-z0-9\s]/g, '')
      .split(/\s+/)
      .filter((word) => word.length > 3 && !stopWords.has(word));

    // Count frequency
    const frequency = new Map<string, number>();
    words.forEach((word) => {
      frequency.set(word, (frequency.get(word) || 0) + 1);
    });

    // Sort by frequency
    return Array.from(frequency.entries())
      .sort((a, b) => b[1] - a[1])
      .map(([word]) => word)
      .slice(0, 20);
  }

  /**
   * Identify content gaps from analysis
   *
   * @param results - Search results
   * @param features - SERP features
   * @param clusters - Semantic clusters
   * @returns Identified content gaps
   * @private
   */
  private identifyContentGaps(
    results: SearchResult[],
    features: SERPFeature[],
    clusters: SemanticCluster[]
  ): ContentGap[] {
    const gaps: ContentGap[] = [];

    // Check for missing content types
    const contentTypeDistribution = this.analyzeContentTypeDistribution(results);
    const hasBlog = contentTypeDistribution.some((ct) => ct.type === ContentType.BLOG);
    const hasGuide = contentTypeDistribution.some((ct) => ct.type === ContentType.GUIDE);
    const hasVideo = contentTypeDistribution.some((ct) => ct.type === ContentType.VIDEO);

    if (!hasBlog && contentTypeDistribution.length > 0) {
      gaps.push({
        gapType: 'format_mismatch',
        topic: 'blog content',
        opportunityScore: 0.7,
        currentCoverage: 0,
        recommendedContentType: ContentType.BLOG,
        reasoning: 'No blog-style content in top 10; opportunity for informational articles',
        priority: 'medium',
      });
    }

    if (!hasGuide) {
      gaps.push({
        gapType: 'missing_topic',
        topic: 'comprehensive guides',
        opportunityScore: 0.8,
        currentCoverage: 0,
        recommendedContentType: ContentType.GUIDE,
        reasoning: 'No comprehensive guides found; opportunity for in-depth tutorials',
        priority: 'high',
      });
    }

    if (!hasVideo && results.length > 0) {
      gaps.push({
        gapType: 'format_mismatch',
        topic: 'video content',
        opportunityScore: 0.6,
        currentCoverage: 0,
        recommendedContentType: ContentType.VIDEO,
        reasoning: 'No video content in results; consider video optimization',
        priority: 'low',
      });
    }

    return gaps;
  }

  /**
   * Generate actionable recommendations
   *
   * @param results - Search results
   * @param features - SERP features
   * @param patterns - Ranking patterns
   * @param clusters - Semantic clusters
   * @param gaps - Content gaps
   * @returns Array of recommendations
   * @private
   */
  private generateRecommendations(
    results: SearchResult[],
    features: SERPFeature[],
    patterns: ReturnType<SERPPatternAnalyst['analyzeRankingPatterns']> extends Promise<infer T>
      ? T
      : never,
    clusters: SemanticCluster[],
    gaps: ContentGap[]
  ): Recommendation[] {
    const recommendations: Recommendation[] = [];

    // Title optimization
    if (patterns.titleMeta.keywordPlacement.inTitle > 80) {
      recommendations.push({
        type: RecommendationType.CONTENT_STRUCTURE,
        title: 'Include target keyword in title',
        description: `${patterns.titleMeta.keywordPlacement.inTitle.toFixed(0)}% of top results include the keyword in their title. Ensure your title contains "${this.config.keyword}".`,
        impact: 'high',
        effort: 'low',
        priority: 0.9,
        evidence: [
          `${patterns.titleMeta.keywordPlacement.atTitleStart.toFixed(0)}% place keyword at title start`,
          `Average title length: ${patterns.titleMeta.avgTitleLength.toFixed(0)} characters`,
        ],
        actionSteps: [
          'Place target keyword in title',
          'Keep title under 60 characters',
          'Front-load keyword if possible',
        ],
      });
    }

    // Content length recommendation
    if (patterns.contentLength.recommendedRange) {
      recommendations.push({
        type: RecommendationType.CONTENT_STRUCTURE,
        title: 'Optimize content length',
        description: `Top-ranking content averages ${patterns.contentLength.averageWordCount} words. Aim for ${patterns.contentLength.recommendedRange.min}-${patterns.contentLength.recommendedRange.max} words.`,
        impact: 'medium',
        effort: 'high',
        priority: 0.7,
        evidence: [
          `Average word count: ${patterns.contentLength.averageWordCount}`,
          `Range: ${patterns.contentLength.minWordCount}-${patterns.contentLength.maxWordCount} words`,
        ],
        actionSteps: [
          `Write ${patterns.contentLength.recommendedRange.min}-${patterns.contentLength.recommendedRange.max} words`,
          'Focus on comprehensive coverage',
          'Maintain readability and structure',
        ],
      });
    }

    // SERP feature targeting
    const hasSnippet = features.some((f) => f.type === SERPFeatureType.FEATURED_SNIPPET);
    if (!hasSnippet || features.length > 0) {
      recommendations.push({
        type: RecommendationType.SERP_FEATURE,
        title: 'Target SERP features',
        description: `${features.length} SERP features detected. Optimize content to capture featured snippets and other rich results.`,
        impact: 'high',
        effort: 'medium',
        priority: 0.85,
        evidence: features.map((f) => `${f.type} at position ${f.position}`),
        actionSteps: [
          'Use clear heading structure (H2, H3)',
          'Include concise definitions and lists',
          'Add FAQ sections for PAA boxes',
          'Implement structured data (schema.org)',
        ],
        relatedFeatures: features.map((f) => f.type),
      });
    }

    // Freshness recommendations
    const freshResults = results.filter((r) => r.freshnessSignals.length > 1).length;
    if (freshResults > results.length * 0.5) {
      recommendations.push({
        type: RecommendationType.CONTENT_STRATEGY,
        title: 'Emphasize content freshness',
        description: `${(freshResults / results.length * 100).toFixed(0)}% of top results show freshness signals. Include dates and update content regularly.`,
        impact: 'medium',
        effort: 'low',
        priority: 0.6,
        evidence: [
          `${freshResults} results show freshness signals`,
          'Dates in titles and URLs are common',
        ],
        actionSteps: [
          'Include current year in title',
          'Add publication/update dates',
          'Regular content updates',
        ],
      });
    }

    // URL structure recommendations
    if (patterns.urlStructure.components.hasKeyword > 70) {
      recommendations.push({
        type: RecommendationType.TECHNICAL_SEO,
        title: 'Optimize URL structure',
        description: `${patterns.urlStructure.components.hasKeyword.toFixed(0)}% of top results include keywords in URLs. Use descriptive, keyword-rich URLs.`,
        impact: 'medium',
        effort: 'low',
        priority: 0.65,
        evidence: [
          `Average path depth: ${patterns.urlStructure.components.pathDepth.toFixed(1)} levels`,
          `${patterns.urlStructure.components.hasHyphens.toFixed(0)}% use hyphens`,
        ],
        actionSteps: [
          'Include target keyword in URL',
          'Use hyphens to separate words',
          'Keep URLs concise and readable',
        ],
      });
    }

    // Content gap recommendations
    gaps.forEach((gap) => {
      if (gap.priority === 'high') {
        recommendations.push({
          type: RecommendationType.COMPETITIVE_POSITIONING,
          title: `Address content gap: ${gap.topic}`,
          description: gap.reasoning,
          impact: 'high',
          effort: 'high',
          priority: gap.opportunityScore,
          evidence: [
            `Current coverage: ${gap.currentCoverage * 100}%`,
            `Opportunity score: ${gap.opportunityScore}`,
          ],
          actionSteps: [
            `Create ${gap.recommendedContentType} content`,
            'Focus on comprehensive coverage',
            'Differentiate from existing content',
          ],
        });
      }
    });

    // Sort by priority
    return recommendations.sort((a, b) => b.priority - a.priority);
  }

  /**
   * Calculate overall analysis confidence
   *
   * @param results - Search results
   * @param features - SERP features
   * @param patterns - Ranking patterns
   * @param clusters - Semantic clusters
   * @returns Confidence score (0.0-1.0)
   * @private
   */
  private calculateOverallConfidence(
    results: SearchResult[],
    features: SERPFeature[],
    patterns: ReturnType<SERPPatternAnalyst['analyzeRankingPatterns']> extends Promise<infer T>
      ? T
      : never,
    clusters: SemanticCluster[]
  ): number {
    let confidence = 0.5; // Base confidence

    // More results = higher confidence
    if (results.length >= 10) confidence += 0.2;
    else if (results.length >= 5) confidence += 0.1;

    // Feature detection adds confidence
    if (features.length > 0) confidence += 0.1;

    // Clustering adds confidence
    if (clusters.length >= 3) confidence += 0.1;

    // Pattern analysis adds confidence
    if (patterns.contentTypes.length > 0) confidence += 0.1;

    return Math.min(confidence, 1.0);
  }

  /**
   * Determine which API provider was used
   *
   * @returns API provider identifier
   * @private
   */
  private determineApiProvider(): 'google' | 'serpapi' | 'scraping' {
    const googleApiKey = this.config.googleApiKey || process.env.GOOGLE_API_KEY;
    const serpApiKey = this.config.serpApiKey || process.env.SERPAPI_KEY;

    if (googleApiKey && !this.isPlaceholderApiKey(googleApiKey)) {
      return 'google';
    }

    if (serpApiKey && !this.isPlaceholderApiKey(serpApiKey)) {
      return 'serpapi';
    }

    return 'scraping';
  }

  /**
   * Calculate average of array
   *
   * @param numbers - Array of numbers
   * @returns Average value
   * @private
   */
  private average(numbers: number[]): number {
    if (numbers.length === 0) return 0;
    return numbers.reduce((sum, n) => sum + n, 0) / numbers.length;
  }
}