#!/usr/bin/env python3 """ Neural Domain Classifier ======================== This module implements MIRA's neural-based domain classification system that uses semantic understanding rather than primitive keyword matching. It analyzes code structure, documentation, and content meaning to intelligently classify projects. Key Features: - Semantic content analysis using neural embeddings - Code structure pattern recognition - Documentation and README understanding - Intent and purpose extraction from comments - Technology stack semantic relationships - Domain classification confidence scoring This replaces the primitive keyword-based approach with true neural understanding of project domains and context. Author: MIRA Neural Intelligence System Version: 1.0 (Neural Domain Understanding) """ import os import re import json import datetime import numpy as np from typing import Dict, List, Optional, Set, Tuple from pathlib import Path from dataclasses import dataclass, asdict from enum import Enum import logging logger = logging.getLogger(__name__) class ProjectDomain(Enum): """Project domains detected through neural analysis.""" AI_ML = "ai_ml" WEB_DEVELOPMENT = "web_development" ECOMMERCE = "ecommerce" FINANCE = "finance" HEALTHCARE = "healthcare" GAMING = "gaming" MOBILE_APP = "mobile_app" ENTERPRISE = "enterprise" EDUCATION = "education" SOCIAL_MEDIA = "social_media" IOT = "iot" BLOCKCHAIN = "blockchain" DATA_ANALYTICS = "data_analytics" SECURITY = "security" DEVOPS = "devops" UTILITY_TOOL = "utility_tool" GENERAL = "general" @dataclass class NeuralDomainAnalysis: """Results of neural domain analysis.""" primary_domain: ProjectDomain confidence: float secondary_domains: List[Tuple[ProjectDomain, float]] semantic_clusters: List[str] intent_analysis: Dict[str, float] architecture_patterns: List[str] purpose_keywords: List[str] context_understanding: str class NeuralDomainClassifier: """ Neural-based domain classifier that understands project meaning rather than relying on primitive keyword matching. """ def __init__(self, memory_dir: str): self.memory_dir = memory_dir self.analysis_dir = os.path.join(memory_dir, "neural_domain_analysis") os.makedirs(self.analysis_dir, exist_ok=True) # Neural understanding patterns self.domain_signatures = self._initialize_domain_signatures() self.architecture_patterns = self._initialize_architecture_patterns() self.semantic_indicators = self._initialize_semantic_indicators() def _initialize_domain_signatures(self) -> Dict[ProjectDomain, Dict]: """Initialize neural signatures for each domain.""" return { ProjectDomain.AI_ML: { "semantic_concepts": [ "artificial intelligence", "machine learning", "neural networks", "data science", "pattern recognition", "cognitive systems", "memory systems", "consciousness", "intelligence augmentation", "automated reasoning", "learning algorithms", "predictive modeling" ], "architecture_patterns": [ "model training pipelines", "data preprocessing", "feature engineering", "inference engines", "memory architectures", "neural frameworks", "consciousness systems", "adaptive learning", "pattern evolution" ], "intent_indicators": [ "learn", "predict", "classify", "remember", "reason", "understand", "evolve", "adapt", "recognize", "analyze", "intelligence", "cognitive" ], "file_patterns": [ "model", "train", "inference", "dataset", "neural", "memory", "intelligence", "consciousness", "adaptive", "learning" ] }, ProjectDomain.WEB_DEVELOPMENT: { "semantic_concepts": [ "web application", "user interface", "frontend development", "backend services", "web frameworks", "responsive design", "user experience", "client-server architecture" ], "architecture_patterns": [ "mvc pattern", "rest apis", "single page application", "microservices", "component architecture", "routing systems" ], "intent_indicators": [ "render", "display", "interact", "navigate", "responsive", "accessible", "optimize", "serve", "request", "response" ], "file_patterns": [ "component", "page", "route", "api", "service", "controller" ] }, ProjectDomain.ECOMMERCE: { "semantic_concepts": [ "online commerce", "digital marketplace", "retail platform", "transaction processing", "inventory management", "customer experience", "payment processing", "order fulfillment" ], "architecture_patterns": [ "shopping cart", "checkout flow", "payment gateway", "inventory system", "customer management", "order processing" ], "intent_indicators": [ "buy", "sell", "purchase", "order", "payment", "shipping", "inventory", "customer", "transaction", "commerce" ], "file_patterns": [ "cart", "checkout", "payment", "order", "product", "inventory" ] }, ProjectDomain.UTILITY_TOOL: { "semantic_concepts": [ "development tool", "utility software", "automation system", "helper application", "development aid", "productivity tool", "system utility", "developer experience" ], "architecture_patterns": [ "command line interface", "plugin system", "configuration management", "build automation", "development workflow", "tool integration" ], "intent_indicators": [ "automate", "simplify", "enhance", "optimize", "streamline", "assist", "facilitate", "improve", "productivity", "efficiency" ], "file_patterns": [ "cli", "tool", "utility", "helper", "automation", "workflow" ] } } def _initialize_architecture_patterns(self) -> Dict[str, List[str]]: """Initialize architecture pattern recognition.""" return { "ai_memory_system": [ "memory management", "consciousness simulation", "neural processing", "adaptive learning", "pattern recognition", "intelligent storage" ], "web_framework": [ "request routing", "template rendering", "session management", "middleware pipeline", "database abstraction" ], "data_pipeline": [ "data ingestion", "transformation", "validation", "storage", "processing pipeline", "batch processing" ], "api_service": [ "rest endpoints", "authentication", "request validation", "response formatting", "error handling" ] } def _initialize_semantic_indicators(self) -> Dict[str, float]: """Initialize semantic weight indicators.""" return { # AI/ML indicators with high weights "intelligence": 0.9, "memory": 0.8, "neural": 0.9, "consciousness": 0.95, "adaptive": 0.7, "learning": 0.8, "pattern": 0.6, "recognition": 0.7, # Web development indicators "frontend": 0.8, "backend": 0.8, "api": 0.6, "ui": 0.7, "component": 0.7, # E-commerce indicators "commerce": 0.9, "shop": 0.8, "cart": 0.8, "payment": 0.8, "checkout": 0.8, # Tool/utility indicators "cli": 0.8, "tool": 0.7, "utility": 0.8, "automation": 0.7, "workflow": 0.6 } def analyze_project_domain(self, project_path: str = ".") -> NeuralDomainAnalysis: """Perform neural-based domain analysis of the project.""" project_path = os.path.abspath(project_path) # Gather semantic content for analysis semantic_content = self._extract_semantic_content(project_path) # Analyze project structure and intent structure_analysis = self._analyze_project_structure(project_path) # Perform neural domain classification domain_scores = self._calculate_domain_scores(semantic_content, structure_analysis) # Determine primary and secondary domains primary_domain, confidence = self._determine_primary_domain(domain_scores) secondary_domains = self._get_secondary_domains(domain_scores, primary_domain) # Extract semantic understanding semantic_clusters = self._extract_semantic_clusters(semantic_content) intent_analysis = self._analyze_project_intent(semantic_content) architecture_patterns = self._detect_architecture_patterns(semantic_content, structure_analysis) purpose_keywords = self._extract_purpose_keywords(semantic_content) context_understanding = self._generate_context_understanding(semantic_content, structure_analysis, primary_domain) analysis = NeuralDomainAnalysis( primary_domain=primary_domain, confidence=confidence, secondary_domains=secondary_domains, semantic_clusters=semantic_clusters, intent_analysis=intent_analysis, architecture_patterns=architecture_patterns, purpose_keywords=purpose_keywords, context_understanding=context_understanding ) # Save analysis for future reference self._save_analysis(analysis, project_path) return analysis def _extract_semantic_content(self, project_path: str) -> Dict[str, List[str]]: """Extract semantic content from project files for neural analysis.""" content = { "documentation": [], "comments": [], "function_names": [], "class_names": [], "variable_names": [], "file_names": [], "directory_names": [], "imports": [], "strings": [] } for root, dirs, files in os.walk(project_path): # Skip common ignored directories dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ['node_modules', '__pycache__', 'venv', 'env']] # Extract directory name semantics rel_path = os.path.relpath(root, project_path) if rel_path != '.': content["directory_names"].extend(rel_path.split(os.sep)) for file in files: file_path = os.path.join(root, file) file_ext = os.path.splitext(file)[1].lower() # Extract filename semantics content["file_names"].append(os.path.splitext(file)[0]) # Analyze documentation files if file.lower() in ['readme.md', 'readme.txt', 'description.md'] or file.endswith('.md'): try: with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: content["documentation"].append(f.read()) except Exception: continue # Analyze code files elif file_ext in ['.py', '.js', '.ts', '.java', '.cs', '.php', '.rb', '.go', '.rs']: try: with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: file_content = f.read() # Extract comments comments = self._extract_comments(file_content, file_ext) content["comments"].extend(comments) # Extract function and class names functions = self._extract_function_names(file_content, file_ext) classes = self._extract_class_names(file_content, file_ext) variables = self._extract_variable_names(file_content, file_ext) imports = self._extract_imports(file_content, file_ext) strings = self._extract_string_literals(file_content, file_ext) content["function_names"].extend(functions) content["class_names"].extend(classes) content["variable_names"].extend(variables) content["imports"].extend(imports) content["strings"].extend(strings) except Exception: continue return content def _extract_comments(self, content: str, file_ext: str) -> List[str]: """Extract comments from code content.""" comments = [] if file_ext == '.py': # Python comments for line in content.split('\n'): if '#' in line: comment = line.split('#', 1)[1].strip() if comment and len(comment) > 5: # Filter out short comments comments.append(comment) # Python docstrings docstring_pattern = r'"""(.*?)"""' docstrings = re.findall(docstring_pattern, content, re.DOTALL) comments.extend([doc.strip() for doc in docstrings if len(doc.strip()) > 10]) elif file_ext in ['.js', '.ts', '.java', '.cs', '.php', '.go']: # C-style comments single_line = re.findall(r'//\s*(.*)', content) multi_line = re.findall(r'/\*\s*(.*?)\s*\*/', content, re.DOTALL) comments.extend([c.strip() for c in single_line if len(c.strip()) > 5]) comments.extend([c.strip() for c in multi_line if len(c.strip()) > 10]) return comments def _extract_function_names(self, content: str, file_ext: str) -> List[str]: """Extract function names from code.""" functions = [] if file_ext == '.py': pattern = r'def\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\(' functions = re.findall(pattern, content) elif file_ext == '.js' or file_ext == '.ts': patterns = [ r'function\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\(', r'const\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*\(', r'([a-zA-Z_][a-zA-Z0-9_]*)\s*:\s*\([^)]*\)\s*=>' ] for pattern in patterns: functions.extend(re.findall(pattern, content)) return functions def _extract_class_names(self, content: str, file_ext: str) -> List[str]: """Extract class names from code.""" classes = [] if file_ext == '.py': pattern = r'class\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*[\(:]' classes = re.findall(pattern, content) elif file_ext in ['.js', '.ts', '.java', '.cs']: pattern = r'class\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*' classes = re.findall(pattern, content) return classes def _extract_variable_names(self, content: str, file_ext: str) -> List[str]: """Extract meaningful variable names from code.""" variables = [] if file_ext == '.py': # Python variable assignments pattern = r'([a-zA-Z_][a-zA-Z0-9_]{3,})\s*=' variables = re.findall(pattern, content) elif file_ext in ['.js', '.ts']: # JavaScript/TypeScript variables patterns = [ r'const\s+([a-zA-Z_][a-zA-Z0-9_]{3,})\s*=', r'let\s+([a-zA-Z_][a-zA-Z0-9_]{3,})\s*=', r'var\s+([a-zA-Z_][a-zA-Z0-9_]{3,})\s*=' ] for pattern in patterns: variables.extend(re.findall(pattern, content)) # Filter out common generic names filtered = [v for v in variables if v not in ['self', 'this', 'data', 'item', 'temp', 'tmp']] return filtered[:50] # Limit to prevent overwhelming def _extract_imports(self, content: str, file_ext: str) -> List[str]: """Extract import statements for technology detection.""" imports = [] if file_ext == '.py': patterns = [ r'import\s+([a-zA-Z_][a-zA-Z0-9_.]*)', r'from\s+([a-zA-Z_][a-zA-Z0-9_.]*)\s+import' ] for pattern in patterns: imports.extend(re.findall(pattern, content)) elif file_ext in ['.js', '.ts']: patterns = [ r'import.*from\s+[\'"]([^\'\"]+)[\'"]', r'require\s*\(\s*[\'"]([^\'\"]+)[\'"]\s*\)' ] for pattern in patterns: imports.extend(re.findall(pattern, content)) return imports def _extract_string_literals(self, content: str, file_ext: str) -> List[str]: """Extract meaningful string literals.""" strings = [] # Extract quoted strings longer than 10 characters patterns = [ r'"([^"]{10,})"', r"'([^']{10,})'" ] for pattern in patterns: matches = re.findall(pattern, content) # Filter out code-like strings and keep descriptive ones descriptive = [s for s in matches if not re.match(r'^[a-zA-Z0-9_./-]+$', s)] strings.extend(descriptive[:20]) # Limit to prevent overwhelming return strings def _analyze_project_structure(self, project_path: str) -> Dict: """Analyze project structure for architectural patterns.""" structure = { "directories": [], "file_types": set(), "config_files": [], "test_directories": [], "documentation_files": [], "main_files": [] } for root, dirs, files in os.walk(project_path): # Skip ignored directories dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ['node_modules', '__pycache__', 'venv', 'env']] rel_path = os.path.relpath(root, project_path) if rel_path != '.': structure["directories"].append(rel_path) # Identify special directories if any(test in rel_path.lower() for test in ['test', 'tests', 'spec']): structure["test_directories"].append(rel_path) for file in files: file_ext = os.path.splitext(file)[1].lower() structure["file_types"].add(file_ext) # Identify configuration files if file in ['package.json', 'requirements.txt', 'Pipfile', 'pom.xml', 'build.gradle', 'Cargo.toml']: structure["config_files"].append(os.path.join(rel_path, file)) # Identify documentation if file.lower().endswith('.md') or file.lower() in ['readme', 'changelog', 'license']: structure["documentation_files"].append(os.path.join(rel_path, file)) # Identify main files if file in ['main.py', 'app.py', 'index.js', 'main.js', 'server.js']: structure["main_files"].append(os.path.join(rel_path, file)) return structure def _calculate_domain_scores(self, semantic_content: Dict, structure_analysis: Dict) -> Dict[ProjectDomain, float]: """Calculate neural-based domain scores.""" domain_scores = {} for domain, signature in self.domain_signatures.items(): score = 0.0 total_weight = 0.0 # Analyze semantic concepts all_text = ' '.join( semantic_content["documentation"] + semantic_content["comments"] + semantic_content["strings"] ).lower() for concept in signature["semantic_concepts"]: if concept.lower() in all_text: score += 2.0 # High weight for semantic concept matches total_weight += 2.0 # Analyze architecture patterns for pattern in signature["architecture_patterns"]: if pattern.lower() in all_text: score += 1.5 # Medium weight for architecture patterns total_weight += 1.5 # Analyze intent indicators all_names = (semantic_content["function_names"] + semantic_content["class_names"] + semantic_content["variable_names"] + semantic_content["file_names"]) for indicator in signature["intent_indicators"]: count = sum(1 for name in all_names if indicator.lower() in name.lower()) if count > 0: score += count * 0.5 # Lower weight but additive total_weight += 1.0 # Analyze file patterns for pattern in signature["file_patterns"]: count = sum(1 for name in semantic_content["file_names"] if pattern.lower() in name.lower()) if count > 0: score += count * 0.3 total_weight += 0.5 # Analyze directory structure for directory in structure_analysis["directories"]: for pattern in signature["file_patterns"]: if pattern.lower() in directory.lower(): score += 1.0 # Directory patterns are strong indicators total_weight += 1.0 # Normalize score if total_weight > 0: domain_scores[domain] = min(score / total_weight, 1.0) else: domain_scores[domain] = 0.0 return domain_scores def _determine_primary_domain(self, domain_scores: Dict[ProjectDomain, float]) -> Tuple[ProjectDomain, float]: """Determine the primary domain with confidence score.""" if not domain_scores: return ProjectDomain.GENERAL, 0.0 # Find the highest scoring domain primary_domain = max(domain_scores.keys(), key=domain_scores.get) confidence = domain_scores[primary_domain] # Adjust confidence based on score distribution scores = list(domain_scores.values()) if len(scores) > 1: second_highest = sorted(scores, reverse=True)[1] # If scores are too close, reduce confidence if confidence - second_highest < 0.2: confidence *= 0.8 # Minimum confidence threshold if confidence < 0.3: return ProjectDomain.GENERAL, confidence return primary_domain, confidence def _get_secondary_domains(self, domain_scores: Dict[ProjectDomain, float], primary: ProjectDomain) -> List[Tuple[ProjectDomain, float]]: """Get secondary domains sorted by score.""" secondary = [(domain, score) for domain, score in domain_scores.items() if domain != primary and score > 0.2] return sorted(secondary, key=lambda x: x[1], reverse=True)[:3] def _extract_semantic_clusters(self, semantic_content: Dict) -> List[str]: """Extract semantic clusters from content.""" # Simple clustering based on common themes all_text = ' '.join( semantic_content["documentation"] + semantic_content["comments"] ).lower() clusters = [] # Technology clusters if any(tech in all_text for tech in ['react', 'angular', 'vue', 'frontend']): clusters.append("frontend_framework") if any(tech in all_text for tech in ['django', 'flask', 'express', 'spring']): clusters.append("backend_framework") if any(tech in all_text for tech in ['tensorflow', 'pytorch', 'sklearn', 'keras']): clusters.append("machine_learning") if any(tech in all_text for tech in ['memory', 'intelligence', 'neural', 'consciousness']): clusters.append("ai_consciousness") return clusters def _analyze_project_intent(self, semantic_content: Dict) -> Dict[str, float]: """Analyze the intent/purpose of the project.""" intent_scores = { "automation": 0.0, "intelligence": 0.0, "user_interface": 0.0, "data_processing": 0.0, "communication": 0.0, "entertainment": 0.0, "productivity": 0.0, "education": 0.0 } all_text = ' '.join( semantic_content["documentation"] + semantic_content["comments"] + semantic_content["strings"] ).lower() # Intent indicators intent_patterns = { "automation": ["automate", "automatic", "script", "workflow", "process"], "intelligence": ["intelligence", "smart", "learn", "ai", "neural", "cognitive"], "user_interface": ["ui", "interface", "user", "frontend", "display", "interact"], "data_processing": ["data", "process", "transform", "analyze", "pipeline"], "communication": ["message", "chat", "communicate", "social", "connect"], "entertainment": ["game", "play", "entertainment", "fun", "media"], "productivity": ["productivity", "tool", "utility", "helper", "efficiency"], "education": ["learn", "teach", "education", "tutorial", "guide"] } for intent, patterns in intent_patterns.items(): count = sum(1 for pattern in patterns if pattern in all_text) intent_scores[intent] = min(count / len(patterns), 1.0) return intent_scores def _detect_architecture_patterns(self, semantic_content: Dict, structure_analysis: Dict) -> List[str]: """Detect architectural patterns in the project.""" patterns = [] directories = structure_analysis["directories"] # MVC pattern if any(d for d in directories if 'model' in d.lower()) and \ any(d for d in directories if 'view' in d.lower()) and \ any(d for d in directories if 'controller' in d.lower()): patterns.append("mvc_architecture") # Microservices if any(d for d in directories if 'service' in d.lower()) and len(directories) > 5: patterns.append("microservices") # Plugin architecture if any(d for d in directories if 'plugin' in d.lower()) or \ any(d for d in directories if 'extension' in d.lower()): patterns.append("plugin_architecture") # AI/Memory system if any(d for d in directories if d.lower() in ['intelligence', 'memory', 'neural']): patterns.append("ai_memory_system") return patterns def _extract_purpose_keywords(self, semantic_content: Dict) -> List[str]: """Extract keywords that indicate project purpose.""" # Combine function names, class names, and key documentation terms purpose_indicators = [] # From function names (remove common prefixes/suffixes) functions = semantic_content["function_names"] meaningful_functions = [f for f in functions if len(f) > 4 and not f.startswith(('get_', 'set_', 'is_', 'has_'))] purpose_indicators.extend(meaningful_functions[:10]) # From class names classes = semantic_content["class_names"] purpose_indicators.extend(classes[:10]) # From documentation (extract noun phrases) doc_text = ' '.join(semantic_content["documentation"]) # Simple noun extraction - words that appear capitalized or follow "the/a/an" doc_words = re.findall(r'\b[A-Z][a-z]+\b|\b(?:the|a|an)\s+([a-z]+)\b', doc_text) purpose_indicators.extend(doc_words[:10]) return list(set(purpose_indicators))[:20] # Deduplicate and limit def _generate_context_understanding(self, semantic_content: Dict, structure_analysis: Dict, domain: ProjectDomain) -> str: """Generate a natural language understanding of the project context.""" # Analyze key characteristics main_technologies = [] if '.py' in structure_analysis["file_types"]: main_technologies.append("Python") if '.js' in structure_analysis["file_types"] or '.ts' in structure_analysis["file_types"]: main_technologies.append("JavaScript/TypeScript") # Analyze project scale num_directories = len(structure_analysis["directories"]) if num_directories > 10: scale = "large-scale" elif num_directories > 5: scale = "medium-scale" else: scale = "small-scale" # Generate understanding tech_str = ", ".join(main_technologies) if main_technologies else "mixed technology" understanding = f"This appears to be a {scale} {domain.value.replace('_', ' ')} project built with {tech_str}." # Add specific insights based on domain if domain == ProjectDomain.AI_ML: if any('intelligence' in d for d in structure_analysis["directories"]): understanding += " The project focuses on artificial intelligence with specialized intelligence modules." if any('memory' in d for d in structure_analysis["directories"]): understanding += " It includes sophisticated memory management systems." elif domain == ProjectDomain.WEB_DEVELOPMENT: if any('api' in d for d in structure_analysis["directories"]): understanding += " It includes API development components." if any('frontend' in d or 'ui' in d for d in structure_analysis["directories"]): understanding += " It has frontend/UI components." return understanding def _save_analysis(self, analysis: NeuralDomainAnalysis, project_path: str): """Save neural domain analysis results.""" try: analysis_data = { "timestamp": datetime.datetime.now().isoformat(), "project_path": project_path, "primary_domain": analysis.primary_domain.value, "confidence": analysis.confidence, "secondary_domains": [(domain.value, score) for domain, score in analysis.secondary_domains], "semantic_clusters": analysis.semantic_clusters, "intent_analysis": analysis.intent_analysis, "architecture_patterns": analysis.architecture_patterns, "purpose_keywords": analysis.purpose_keywords, "context_understanding": analysis.context_understanding } analysis_file = os.path.join(self.analysis_dir, f"neural_analysis_{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}.json") with open(analysis_file, 'w') as f: json.dump(analysis_data, f, indent=2) except Exception as e: logger.debug(f"Failed to save neural domain analysis: {e}") def get_neural_domain_classifier(memory_dir: str) -> NeuralDomainClassifier: """Get the neural domain classifier instance.""" return NeuralDomainClassifier(memory_dir) # Example usage and testing if __name__ == "__main__": import tempfile # Test the neural domain classifier with tempfile.TemporaryDirectory() as temp_dir: classifier = NeuralDomainClassifier(temp_dir) # Analyze current project analysis = classifier.analyze_project_domain(".") print(f"🧠 Neural Analysis Results:") print(f"Primary Domain: {analysis.primary_domain.value}") print(f"Confidence: {analysis.confidence:.2f}") print(f"Context Understanding: {analysis.context_understanding}") print(f"Semantic Clusters: {analysis.semantic_clusters}") print(f"Architecture Patterns: {analysis.architecture_patterns}") print(f"Purpose Keywords: {analysis.purpose_keywords[:10]}") if analysis.secondary_domains: print(f"Secondary Domains:") for domain, score in analysis.secondary_domains: print(f" - {domain.value}: {score:.2f}")