walidsobhie-code commited on
Commit Β·
8f05ad1
1
Parent(s): 6807e9a
feat: Add comprehensive enhancement modules for Stack 2.9
Browse files- NLP: Intent detection, entity recognition, contextual embeddings
- Knowledge Graph: NetworkX-based graph with RAG engine
- Emotional Intelligence: Sentiment analysis, empathy engine
- Collaboration: Multi-session state management, MCP integration
- Learning: Feedback collection, performance monitoring
- Technical: DevOps templates, code analysis, debugging assistant
Added run_full.py - fully enhanced chat interface with all modules.
- run_final.py +79 -0
- run_full.py +217 -0
- src/enhancements/__init__.py +34 -0
- src/enhancements/collaboration/__init__.py +13 -0
- src/enhancements/collaboration/conversation_state.py +261 -0
- src/enhancements/collaboration/mcp_integration.py +241 -0
- src/enhancements/config.py +118 -0
- src/enhancements/emotional_intelligence/__init__.py +13 -0
- src/enhancements/emotional_intelligence/empathy.py +222 -0
- src/enhancements/emotional_intelligence/sentiment.py +238 -0
- src/enhancements/knowledge_graph/__init__.py +13 -0
- src/enhancements/knowledge_graph/graph.py +287 -0
- src/enhancements/knowledge_graph/rag.py +250 -0
- src/enhancements/learning/__init__.py +13 -0
- src/enhancements/learning/feedback.py +307 -0
- src/enhancements/learning/performance.py +262 -0
- src/enhancements/nlp/__init__.py +18 -0
- src/enhancements/nlp/contextual_embeddings.py +174 -0
- src/enhancements/nlp/entity_recognition.py +214 -0
- src/enhancements/nlp/intent_detection.py +227 -0
- src/enhancements/technical/__init__.py +18 -0
- src/enhancements/technical/code_analysis.py +223 -0
- src/enhancements/technical/debugging.py +357 -0
- src/enhancements/technical/devops.py +399 -0
- test_enhancements.py +131 -0
run_final.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
Stack 2.9 - Qwen2 Direct Load
|
| 4 |
+
"""
|
| 5 |
+
import os
|
| 6 |
+
os.environ['HF_HUB_DISABLE_PROGRESS_BARS'] = '1'
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
import json
|
| 11 |
+
|
| 12 |
+
model_path = Path("/Users/walidsobhi/stack-2-9-final-model")
|
| 13 |
+
|
| 14 |
+
print("Loading...")
|
| 15 |
+
|
| 16 |
+
# Load tokenizer
|
| 17 |
+
from transformers import PreTrainedTokenizerFast
|
| 18 |
+
tokenizer = PreTrainedTokenizerFast(tokenizer_file=str(model_path / "tokenizer.json"))
|
| 19 |
+
tokenizer.pad_token = "<|endoftext|>"
|
| 20 |
+
tokenizer.eos_token = "<|endoftext|>"
|
| 21 |
+
|
| 22 |
+
print("Tokenizer loaded")
|
| 23 |
+
|
| 24 |
+
# Load config
|
| 25 |
+
with open(model_path / "config.json") as f:
|
| 26 |
+
config_dict = json.load(f)
|
| 27 |
+
|
| 28 |
+
print(f"Model type: {config_dict.get('model_type')}")
|
| 29 |
+
|
| 30 |
+
# Load weights first
|
| 31 |
+
print("Loading weights...")
|
| 32 |
+
from safetensors.torch import load_file
|
| 33 |
+
state_dict = load_file(str(model_path / "model.safetensors"))
|
| 34 |
+
|
| 35 |
+
# Create model the proper way for Qwen2
|
| 36 |
+
from transformers import Qwen2ForCausalLM, Qwen2Config
|
| 37 |
+
|
| 38 |
+
# Create config object
|
| 39 |
+
config = Qwen2Config()
|
| 40 |
+
for k, v in config_dict.items():
|
| 41 |
+
setattr(config, k, v)
|
| 42 |
+
|
| 43 |
+
# Initialize model with config then load weights
|
| 44 |
+
model = Qwen2ForCausalLM(config)
|
| 45 |
+
model.load_state_dict(state_dict, strict=False)
|
| 46 |
+
model = model.to(torch.float16)
|
| 47 |
+
|
| 48 |
+
if torch.cuda.is_available():
|
| 49 |
+
model.to("cuda")
|
| 50 |
+
|
| 51 |
+
print("Model ready!\n")
|
| 52 |
+
|
| 53 |
+
# Chat
|
| 54 |
+
print("Stack 2.9 Ready!")
|
| 55 |
+
print("=" * 40)
|
| 56 |
+
|
| 57 |
+
while True:
|
| 58 |
+
try:
|
| 59 |
+
user_input = input("\nYou: ").strip()
|
| 60 |
+
if user_input.lower() in ['quit', 'exit', 'q']:
|
| 61 |
+
break
|
| 62 |
+
|
| 63 |
+
prompt = f"You are Stack 2.9.\n\nUser: {user_input}\nAssistant:"
|
| 64 |
+
inputs = tokenizer(prompt, return_tensors='pt')
|
| 65 |
+
if torch.cuda.is_available():
|
| 66 |
+
inputs = {k: v.cuda() for k, v in inputs.items()}
|
| 67 |
+
|
| 68 |
+
outputs = model.generate(**inputs, max_new_tokens=100, temperature=0.4, pad_token_id=tokenizer.eos_token_id)
|
| 69 |
+
response = tokenizer.decode(outputs[0], skip_special_tokens=True)
|
| 70 |
+
|
| 71 |
+
if "Assistant:" in response:
|
| 72 |
+
response = response.split("Assistant:")[-1].strip()
|
| 73 |
+
|
| 74 |
+
print(f"AI: {response}")
|
| 75 |
+
|
| 76 |
+
except KeyboardInterrupt:
|
| 77 |
+
break
|
| 78 |
+
|
| 79 |
+
print("\nDone!")
|
run_full.py
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
Stack 2.9 - Full Enhanced Chat with All Modules
|
| 4 |
+
"""
|
| 5 |
+
import os
|
| 6 |
+
os.environ['HF_HUB_DISABLE_PROGRESS_BARS'] = '1'
|
| 7 |
+
os.environ['TOKENIZERS_PARALLELISM'] = 'false'
|
| 8 |
+
|
| 9 |
+
import sys
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
sys.path.insert(0, str(Path(__file__).parent / "src"))
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
import json
|
| 15 |
+
import time
|
| 16 |
+
|
| 17 |
+
# ============= Load Model =============
|
| 18 |
+
model_path = Path("/Users/walidsobhi/stack-2-9-final-model")
|
| 19 |
+
|
| 20 |
+
print("Loading tokenizer...")
|
| 21 |
+
from transformers import PreTrainedTokenizerFast
|
| 22 |
+
tokenizer = PreTrainedTokenizerFast(tokenizer_file=str(model_path / "tokenizer.json"))
|
| 23 |
+
tokenizer.pad_token = "<|endoftext|>"
|
| 24 |
+
tokenizer.eos_token = "<|endoftext|>"
|
| 25 |
+
|
| 26 |
+
print("Loading config...")
|
| 27 |
+
with open(model_path / "config.json") as f:
|
| 28 |
+
config_dict = json.load(f)
|
| 29 |
+
|
| 30 |
+
print("Loading model...")
|
| 31 |
+
from transformers import AutoModelForCausalLM
|
| 32 |
+
model = AutoModelForCausalLM.from_pretrained(
|
| 33 |
+
str(model_path),
|
| 34 |
+
torch_dtype=torch.float16,
|
| 35 |
+
device_map="cpu",
|
| 36 |
+
local_files_only=True
|
| 37 |
+
)
|
| 38 |
+
tokenizer = AutoTokenizer.from_pretrained(str(model_path), local_files_only=True)
|
| 39 |
+
tokenizer.pad_token = "<|endoftext|>"
|
| 40 |
+
|
| 41 |
+
if torch.cuda.is_available():
|
| 42 |
+
model = model.to("cuda")
|
| 43 |
+
|
| 44 |
+
# ============= Load Enhancement Modules =============
|
| 45 |
+
print("\nLoading enhancement modules...")
|
| 46 |
+
|
| 47 |
+
from enhancements.nlp import IntentDetector, EntityRecognizer
|
| 48 |
+
from enhancements.knowledge_graph import RAGEngine
|
| 49 |
+
from enhancements.emotional_intelligence import SentimentAnalyzer, EmpathyEngine
|
| 50 |
+
from enhancements.collaboration import ConversationStateManager
|
| 51 |
+
from enhancements.learning import FeedbackCollector, PerformanceMonitor
|
| 52 |
+
from enhancements.technical import DevOpsTools, CodeAnalyzer, DebuggingAssistant
|
| 53 |
+
|
| 54 |
+
# Initialize all modules
|
| 55 |
+
intent_detector = IntentDetector()
|
| 56 |
+
entity_recognizer = EntityRecognizer()
|
| 57 |
+
rag_engine = RAGEngine()
|
| 58 |
+
sentiment_analyzer = SentimentAnalyzer()
|
| 59 |
+
empathy_engine = EmpathyEngine()
|
| 60 |
+
conv_manager = ConversationStateManager()
|
| 61 |
+
feedback_collector = FeedbackCollector()
|
| 62 |
+
perf_monitor = PerformanceMonitor()
|
| 63 |
+
devops_tools = DevOpsTools()
|
| 64 |
+
code_analyzer = CodeAnalyzer()
|
| 65 |
+
debugger = DebuggingAssistant()
|
| 66 |
+
|
| 67 |
+
# Seed RAG
|
| 68 |
+
rag_engine.add_document("intro", "Stack 2.9 is an AI coding assistant that helps with programming, debugging, and technical questions.")
|
| 69 |
+
rag_engine.add_document("commands", "Commands: quit, exit, feedback, analyze:<code>, debug:<error>")
|
| 70 |
+
|
| 71 |
+
print("β Intent Detection")
|
| 72 |
+
print("β Entity Recognition")
|
| 73 |
+
print("β RAG Engine")
|
| 74 |
+
print("β Sentiment Analysis")
|
| 75 |
+
print("β Empathy Engine")
|
| 76 |
+
print("β Conversation Manager")
|
| 77 |
+
print("β Feedback Collector")
|
| 78 |
+
print("β Performance Monitor")
|
| 79 |
+
print("β DevOps Tools")
|
| 80 |
+
print("β Code Analyzer")
|
| 81 |
+
print("β Debugging Assistant")
|
| 82 |
+
|
| 83 |
+
print("\n" + "=" * 50)
|
| 84 |
+
print("Stack 2.9 - FULLY ENHANCED")
|
| 85 |
+
print("=" * 50)
|
| 86 |
+
|
| 87 |
+
# ============= Chat Loop =============
|
| 88 |
+
conv_manager.create_session()
|
| 89 |
+
perf_monitor.increment_session_count()
|
| 90 |
+
|
| 91 |
+
last_user_input = None
|
| 92 |
+
last_response = None
|
| 93 |
+
|
| 94 |
+
while True:
|
| 95 |
+
try:
|
| 96 |
+
user_input = input("\nYou: ").strip()
|
| 97 |
+
if not user_input:
|
| 98 |
+
continue
|
| 99 |
+
|
| 100 |
+
if user_input.lower() in ['quit', 'exit', 'q']:
|
| 101 |
+
break
|
| 102 |
+
|
| 103 |
+
# === Handle Special Commands ===
|
| 104 |
+
|
| 105 |
+
# Feedback
|
| 106 |
+
if user_input.lower() == 'feedback':
|
| 107 |
+
print("Rate last response (1-5): ", end="")
|
| 108 |
+
try:
|
| 109 |
+
rating = int(input().strip())
|
| 110 |
+
if 1 <= rating <= 5:
|
| 111 |
+
feedback_collector.add_feedback("rating", last_user_input or "", last_response or "", rating=rating)
|
| 112 |
+
print("β Thanks for feedback!")
|
| 113 |
+
except:
|
| 114 |
+
print("Invalid rating")
|
| 115 |
+
continue
|
| 116 |
+
|
| 117 |
+
# Code Analysis
|
| 118 |
+
if user_input.lower().startswith("analyze:"):
|
| 119 |
+
code = user_input[8:].strip()
|
| 120 |
+
summary = code_analyzer.get_code_summary(code)
|
| 121 |
+
print(f"\nπ Code Analysis:")
|
| 122 |
+
print(f" Language: {summary['language']}")
|
| 123 |
+
print(f" Lines: {summary['complexity']['lines_of_code']}")
|
| 124 |
+
print(f" Complexity: {summary['complexity']['cyclomatic_complexity']}")
|
| 125 |
+
print(f" Issues: {summary['issue_count']}")
|
| 126 |
+
print(f" Maintainability: {summary['maintainability_index']:.1f}/100")
|
| 127 |
+
if summary['suggestions']:
|
| 128 |
+
print(f" Suggestions: {summary['suggestions']}")
|
| 129 |
+
continue
|
| 130 |
+
|
| 131 |
+
# Debug Error
|
| 132 |
+
if user_input.lower().startswith("debug:"):
|
| 133 |
+
error = user_input[6:].strip()
|
| 134 |
+
analysis = debugger.analyze_error(error)
|
| 135 |
+
print(f"\nπ§ Debug Analysis:")
|
| 136 |
+
print(f" Error: {analysis['error_type']}")
|
| 137 |
+
print(f" Description: {analysis['description']}")
|
| 138 |
+
print(f" Common causes: {analysis['common_causes']}")
|
| 139 |
+
for step in analysis['debug_steps'][:4]:
|
| 140 |
+
print(f" {step}")
|
| 141 |
+
continue
|
| 142 |
+
|
| 143 |
+
# Intent Detection
|
| 144 |
+
intent = intent_detector.detect_intent(user_input)
|
| 145 |
+
|
| 146 |
+
# Entity Recognition
|
| 147 |
+
entities = entity_recognizer.recognize_entities(user_input)
|
| 148 |
+
|
| 149 |
+
# Sentiment Analysis
|
| 150 |
+
sentiment = sentiment_analyzer.analyze_sentiment(user_input)
|
| 151 |
+
|
| 152 |
+
# RAG Context
|
| 153 |
+
rag_context = rag_engine.retrieve_as_context(user_input, 300)
|
| 154 |
+
|
| 155 |
+
# === Generate Response ===
|
| 156 |
+
start_time = time.time()
|
| 157 |
+
|
| 158 |
+
# Build prompt with enhancements
|
| 159 |
+
system = "You are Stack 2.9, an expert AI coding assistant."
|
| 160 |
+
if rag_context:
|
| 161 |
+
system += f"\nContext: {rag_context}"
|
| 162 |
+
if sentiment['sentiment'] == 'negative':
|
| 163 |
+
system += "\nBe empathetic and supportive."
|
| 164 |
+
elif sentiment['sentiment'] == 'positive':
|
| 165 |
+
system += "\nBe enthusiastic and positive."
|
| 166 |
+
|
| 167 |
+
prompt = f"{system}\n\nUser: {user_input}\nAssistant:"
|
| 168 |
+
inputs = tokenizer(prompt, return_tensors='pt')
|
| 169 |
+
if torch.cuda.is_available():
|
| 170 |
+
inputs = inputs.to("cuda")
|
| 171 |
+
|
| 172 |
+
outputs = model.generate(
|
| 173 |
+
**inputs,
|
| 174 |
+
max_new_tokens=120,
|
| 175 |
+
temperature=0.4,
|
| 176 |
+
top_p=0.9,
|
| 177 |
+
pad_token_id=tokenizer.eos_token_id
|
| 178 |
+
)
|
| 179 |
+
|
| 180 |
+
response_time = time.time() - start_time
|
| 181 |
+
|
| 182 |
+
response = tokenizer.decode(outputs[0], skip_special_tokens=True)
|
| 183 |
+
if "Assistant:" in response:
|
| 184 |
+
response = response.split("Assistant:")[-1].strip()
|
| 185 |
+
|
| 186 |
+
# Apply empathy if needed
|
| 187 |
+
if sentiment['sentiment'] == 'negative':
|
| 188 |
+
response = empathy_engine.generate_empathetic_response(user_input, response)
|
| 189 |
+
|
| 190 |
+
# === Show Info ===
|
| 191 |
+
print(f"\n[Intent: {intent['intent']}]", end="")
|
| 192 |
+
if entities:
|
| 193 |
+
print(f" [Entities: {', '.join([e['type'] for e in entities[:3]])}]", end="")
|
| 194 |
+
if sentiment['sentiment'] != 'neutral':
|
| 195 |
+
print(f" [Mood: {sentiment['sentiment']}]", end="")
|
| 196 |
+
print(f" [{response_time:.2f}s]\n")
|
| 197 |
+
|
| 198 |
+
print(f"AI: {response}")
|
| 199 |
+
|
| 200 |
+
# === Track ===
|
| 201 |
+
last_user_input = user_input
|
| 202 |
+
last_response = response
|
| 203 |
+
conv_manager.add_message("user", user_input)
|
| 204 |
+
conv_manager.add_message("assistant", response)
|
| 205 |
+
perf_monitor.increment_message_count()
|
| 206 |
+
perf_monitor.record_response_time(response_time)
|
| 207 |
+
|
| 208 |
+
except KeyboardInterrupt:
|
| 209 |
+
break
|
| 210 |
+
|
| 211 |
+
# Stats
|
| 212 |
+
stats = perf_monitor.get_session_stats()
|
| 213 |
+
print(f"\n{'='*50}")
|
| 214 |
+
print(f"Session complete!")
|
| 215 |
+
print(f"Messages: {stats['total_messages']}")
|
| 216 |
+
print(f"Avg response time: {perf_monitor.get_average_response_time():.2f}s")
|
| 217 |
+
print(f"{'='*50}")
|
src/enhancements/__init__.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Stack 2.9 Enhancement Modules
|
| 3 |
+
|
| 4 |
+
This package provides comprehensive enhancements for the Stack 2.9 model:
|
| 5 |
+
- NLP: Contextual embeddings, entity recognition, intent detection
|
| 6 |
+
- Knowledge Graph: Graph-based knowledge with RAG support
|
| 7 |
+
- Emotional Intelligence: Sentiment analysis and empathetic responses
|
| 8 |
+
- Collaboration: MCP integration, conversation state management
|
| 9 |
+
- Learning: Feedback collection, continuous improvement
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from .config import (
|
| 13 |
+
EnhancementConfig,
|
| 14 |
+
NLPConfig,
|
| 15 |
+
KnowledgeGraphConfig,
|
| 16 |
+
EmotionalIntelligenceConfig,
|
| 17 |
+
CollaborationConfig,
|
| 18 |
+
LearningConfig,
|
| 19 |
+
get_config,
|
| 20 |
+
set_config,
|
| 21 |
+
)
|
| 22 |
+
|
| 23 |
+
__version__ = "2.9.0"
|
| 24 |
+
|
| 25 |
+
__all__ = [
|
| 26 |
+
"EnhancementConfig",
|
| 27 |
+
"NLPConfig",
|
| 28 |
+
"KnowledgeGraphConfig",
|
| 29 |
+
"EmotionalIntelligenceConfig",
|
| 30 |
+
"CollaborationConfig",
|
| 31 |
+
"LearningConfig",
|
| 32 |
+
"get_config",
|
| 33 |
+
"set_config",
|
| 34 |
+
]
|
src/enhancements/collaboration/__init__.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Collaboration Module
|
| 3 |
+
|
| 4 |
+
Provides collaboration features and conversational flow management.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from .conversation_state import ConversationStateManager
|
| 8 |
+
from .mcp_integration import MCPIntegration
|
| 9 |
+
|
| 10 |
+
__all__ = [
|
| 11 |
+
"ConversationStateManager",
|
| 12 |
+
"MCPIntegration",
|
| 13 |
+
]
|
src/enhancements/collaboration/conversation_state.py
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Conversation State Management
|
| 3 |
+
|
| 4 |
+
Manages conversation state across multiple sessions.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import Dict, List, Optional, Any
|
| 8 |
+
from datetime import datetime, timedelta
|
| 9 |
+
import uuid
|
| 10 |
+
import json
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class ConversationSession:
|
| 15 |
+
"""Represents a single conversation session."""
|
| 16 |
+
|
| 17 |
+
def __init__(
|
| 18 |
+
self,
|
| 19 |
+
session_id: str,
|
| 20 |
+
user_id: Optional[str] = None,
|
| 21 |
+
):
|
| 22 |
+
self.session_id = session_id
|
| 23 |
+
self.user_id = user_id
|
| 24 |
+
self.messages: List[Dict[str, Any]] = []
|
| 25 |
+
self.created_at = datetime.now()
|
| 26 |
+
self.last_activity = datetime.now()
|
| 27 |
+
self.metadata: Dict[str, Any] = {}
|
| 28 |
+
self.context: Dict[str, Any] = {}
|
| 29 |
+
|
| 30 |
+
def add_message(
|
| 31 |
+
self,
|
| 32 |
+
role: str,
|
| 33 |
+
content: str,
|
| 34 |
+
metadata: Optional[Dict[str, Any]] = None,
|
| 35 |
+
) -> None:
|
| 36 |
+
"""Add a message to the session."""
|
| 37 |
+
self.messages.append({
|
| 38 |
+
"role": role,
|
| 39 |
+
"content": content,
|
| 40 |
+
"timestamp": datetime.now().isoformat(),
|
| 41 |
+
"metadata": metadata or {},
|
| 42 |
+
})
|
| 43 |
+
self.last_activity = datetime.now()
|
| 44 |
+
|
| 45 |
+
def get_messages(self, limit: Optional[int] = None) -> List[Dict[str, Any]]:
|
| 46 |
+
"""Get messages, optionally limited."""
|
| 47 |
+
if limit:
|
| 48 |
+
return self.messages[-limit:]
|
| 49 |
+
return self.messages
|
| 50 |
+
|
| 51 |
+
def clear(self) -> None:
|
| 52 |
+
"""Clear session messages."""
|
| 53 |
+
self.messages = []
|
| 54 |
+
self.context = {}
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
class ConversationStateManager:
|
| 58 |
+
"""Manages multiple conversation sessions and state."""
|
| 59 |
+
|
| 60 |
+
def __init__(
|
| 61 |
+
self,
|
| 62 |
+
max_sessions: int = 10,
|
| 63 |
+
session_timeout_minutes: int = 60,
|
| 64 |
+
):
|
| 65 |
+
"""
|
| 66 |
+
Initialize the conversation state manager.
|
| 67 |
+
|
| 68 |
+
Args:
|
| 69 |
+
max_sessions: Maximum number of concurrent sessions
|
| 70 |
+
session_timeout_minutes: Session timeout in minutes
|
| 71 |
+
"""
|
| 72 |
+
self.max_sessions = max_sessions
|
| 73 |
+
self.session_timeout = timedelta(minutes=session_timeout_minutes)
|
| 74 |
+
self.sessions: Dict[str, ConversationSession] = {}
|
| 75 |
+
self.active_session_id: Optional[str] = None
|
| 76 |
+
|
| 77 |
+
def create_session(
|
| 78 |
+
self,
|
| 79 |
+
user_id: Optional[str] = None,
|
| 80 |
+
session_id: Optional[str] = None,
|
| 81 |
+
) -> str:
|
| 82 |
+
"""
|
| 83 |
+
Create a new session.
|
| 84 |
+
|
| 85 |
+
Args:
|
| 86 |
+
user_id: Optional user ID
|
| 87 |
+
session_id: Optional session ID
|
| 88 |
+
|
| 89 |
+
Returns:
|
| 90 |
+
Session ID
|
| 91 |
+
"""
|
| 92 |
+
# Clean up old sessions if at max
|
| 93 |
+
if len(self.sessions) >= self.max_sessions:
|
| 94 |
+
self._cleanup_old_sessions()
|
| 95 |
+
|
| 96 |
+
session_id = session_id or str(uuid.uuid4())
|
| 97 |
+
session = ConversationSession(session_id, user_id)
|
| 98 |
+
self.sessions[session_id] = session
|
| 99 |
+
self.active_session_id = session_id
|
| 100 |
+
|
| 101 |
+
return session_id
|
| 102 |
+
|
| 103 |
+
def get_session(self, session_id: str) -> Optional[ConversationSession]:
|
| 104 |
+
"""Get a session by ID."""
|
| 105 |
+
session = self.sessions.get(session_id)
|
| 106 |
+
if session:
|
| 107 |
+
# Check timeout
|
| 108 |
+
if datetime.now() - session.last_activity > self.session_timeout:
|
| 109 |
+
self.delete_session(session_id)
|
| 110 |
+
return None
|
| 111 |
+
return session
|
| 112 |
+
|
| 113 |
+
def get_active_session(self) -> Optional[ConversationSession]:
|
| 114 |
+
"""Get the active session."""
|
| 115 |
+
if self.active_session_id:
|
| 116 |
+
return self.get_session(self.active_session_id)
|
| 117 |
+
return None
|
| 118 |
+
|
| 119 |
+
def set_active_session(self, session_id: str) -> bool:
|
| 120 |
+
"""Set the active session."""
|
| 121 |
+
if session_id in self.sessions:
|
| 122 |
+
self.active_session_id = session_id
|
| 123 |
+
return True
|
| 124 |
+
return False
|
| 125 |
+
|
| 126 |
+
def delete_session(self, session_id: str) -> bool:
|
| 127 |
+
"""Delete a session."""
|
| 128 |
+
if session_id in self.sessions:
|
| 129 |
+
del self.sessions[session_id]
|
| 130 |
+
if self.active_session_id == session_id:
|
| 131 |
+
self.active_session_id = None
|
| 132 |
+
return True
|
| 133 |
+
return False
|
| 134 |
+
|
| 135 |
+
def add_message(
|
| 136 |
+
self,
|
| 137 |
+
role: str,
|
| 138 |
+
content: str,
|
| 139 |
+
session_id: Optional[str] = None,
|
| 140 |
+
metadata: Optional[Dict[str, Any]] = None,
|
| 141 |
+
) -> bool:
|
| 142 |
+
"""Add a message to a session."""
|
| 143 |
+
session = self.get_session(session_id or self.active_session_id or "")
|
| 144 |
+
if not session:
|
| 145 |
+
session_id = self.create_session()
|
| 146 |
+
session = self.sessions[session_id]
|
| 147 |
+
|
| 148 |
+
session.add_message(role, content, metadata)
|
| 149 |
+
return True
|
| 150 |
+
|
| 151 |
+
def get_conversation_history(
|
| 152 |
+
self,
|
| 153 |
+
session_id: Optional[str] = None,
|
| 154 |
+
limit: Optional[int] = None,
|
| 155 |
+
) -> List[Dict[str, Any]]:
|
| 156 |
+
"""Get conversation history."""
|
| 157 |
+
session = self.get_session(session_id or self.active_session_id or "")
|
| 158 |
+
if not session:
|
| 159 |
+
return []
|
| 160 |
+
return session.get_messages(limit)
|
| 161 |
+
|
| 162 |
+
def update_context(
|
| 163 |
+
self,
|
| 164 |
+
key: str,
|
| 165 |
+
value: Any,
|
| 166 |
+
session_id: Optional[str] = None,
|
| 167 |
+
) -> bool:
|
| 168 |
+
"""Update session context."""
|
| 169 |
+
session = self.get_session(session_id or self.active_session_id or "")
|
| 170 |
+
if not session:
|
| 171 |
+
return False
|
| 172 |
+
session.context[key] = value
|
| 173 |
+
return True
|
| 174 |
+
|
| 175 |
+
def get_context(
|
| 176 |
+
self,
|
| 177 |
+
key: Optional[str] = None,
|
| 178 |
+
session_id: Optional[str] = None,
|
| 179 |
+
) -> Any:
|
| 180 |
+
"""Get session context."""
|
| 181 |
+
session = self.get_session(session_id or self.active_session_id or "")
|
| 182 |
+
if not session:
|
| 183 |
+
return None
|
| 184 |
+
if key:
|
| 185 |
+
return session.context.get(key)
|
| 186 |
+
return session.context
|
| 187 |
+
|
| 188 |
+
def _cleanup_old_sessions(self) -> None:
|
| 189 |
+
"""Clean up old/timeout sessions."""
|
| 190 |
+
now = datetime.now()
|
| 191 |
+
to_delete = []
|
| 192 |
+
|
| 193 |
+
for session_id, session in self.sessions.items():
|
| 194 |
+
if now - session.last_activity > self.session_timeout:
|
| 195 |
+
to_delete.append(session_id)
|
| 196 |
+
|
| 197 |
+
# Also delete oldest if still at max
|
| 198 |
+
if len(self.sessions) - len(to_delete) >= self.max_sessions:
|
| 199 |
+
oldest = min(
|
| 200 |
+
self.sessions.values(),
|
| 201 |
+
key=lambda s: s.last_activity
|
| 202 |
+
)
|
| 203 |
+
to_delete.append(oldest.session_id)
|
| 204 |
+
|
| 205 |
+
for session_id in to_delete:
|
| 206 |
+
self.delete_session(session_id)
|
| 207 |
+
|
| 208 |
+
def get_session_info(self, session_id: str) -> Optional[Dict[str, Any]]:
|
| 209 |
+
"""Get session information."""
|
| 210 |
+
session = self.get_session(session_id)
|
| 211 |
+
if not session:
|
| 212 |
+
return None
|
| 213 |
+
|
| 214 |
+
return {
|
| 215 |
+
"session_id": session.session_id,
|
| 216 |
+
"user_id": session.user_id,
|
| 217 |
+
"message_count": len(session.messages),
|
| 218 |
+
"created_at": session.created_at.isoformat(),
|
| 219 |
+
"last_activity": session.last_activity.isoformat(),
|
| 220 |
+
"metadata": session.metadata,
|
| 221 |
+
}
|
| 222 |
+
|
| 223 |
+
def get_all_sessions(self) -> List[str]:
|
| 224 |
+
"""Get all active session IDs."""
|
| 225 |
+
self._cleanup_old_sessions()
|
| 226 |
+
return list(self.sessions.keys())
|
| 227 |
+
|
| 228 |
+
def save_sessions(self, filepath: str) -> None:
|
| 229 |
+
"""Save sessions to file."""
|
| 230 |
+
data = {
|
| 231 |
+
session_id: {
|
| 232 |
+
"session_id": session.session_id,
|
| 233 |
+
"user_id": session.user_id,
|
| 234 |
+
"messages": session.messages,
|
| 235 |
+
"created_at": session.created_at.isoformat(),
|
| 236 |
+
"last_activity": session.last_activity.isoformat(),
|
| 237 |
+
"metadata": session.metadata,
|
| 238 |
+
"context": session.context,
|
| 239 |
+
}
|
| 240 |
+
for session_id, session in self.sessions.items()
|
| 241 |
+
}
|
| 242 |
+
Path(filepath).write_text(json.dumps(data, indent=2))
|
| 243 |
+
|
| 244 |
+
def load_sessions(self, filepath: str) -> None:
|
| 245 |
+
"""Load sessions from file."""
|
| 246 |
+
data = json.loads(Path(filepath).read_text())
|
| 247 |
+
|
| 248 |
+
for session_id, session_data in data.items():
|
| 249 |
+
session = ConversationSession(
|
| 250 |
+
session_data["session_id"],
|
| 251 |
+
session_data.get("user_id"),
|
| 252 |
+
)
|
| 253 |
+
session.messages = session_data.get("messages", [])
|
| 254 |
+
session.created_at = datetime.fromisoformat(session_data["created_at"])
|
| 255 |
+
session.last_activity = datetime.fromisoformat(session_data["last_activity"])
|
| 256 |
+
session.metadata = session_data.get("metadata", {})
|
| 257 |
+
session.context = session_data.get("context", {})
|
| 258 |
+
self.sessions[session_id] = session
|
| 259 |
+
|
| 260 |
+
def __repr__(self) -> str:
|
| 261 |
+
return f"ConversationStateManager(sessions={len(self.sessions)}, max={self.max_sessions})"
|
src/enhancements/collaboration/mcp_integration.py
ADDED
|
@@ -0,0 +1,241 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
MCP (Model Context Protocol) Integration
|
| 3 |
+
|
| 4 |
+
Provides MCP client integration for tool calling and external services.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import Dict, List, Optional, Any, Callable
|
| 8 |
+
import asyncio
|
| 9 |
+
import json
|
| 10 |
+
from datetime import datetime
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class MCPTool:
|
| 14 |
+
"""Represents an MCP tool."""
|
| 15 |
+
|
| 16 |
+
def __init__(
|
| 17 |
+
self,
|
| 18 |
+
name: str,
|
| 19 |
+
description: str,
|
| 20 |
+
parameters: Dict[str, Any],
|
| 21 |
+
handler: Optional[Callable] = None,
|
| 22 |
+
):
|
| 23 |
+
self.name = name
|
| 24 |
+
self.description = description
|
| 25 |
+
self.parameters = parameters
|
| 26 |
+
self.handler = handler
|
| 27 |
+
|
| 28 |
+
def __repr__(self) -> str:
|
| 29 |
+
return f"MCPTool(name='{self.name}')"
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class MCPIntegration:
|
| 33 |
+
"""Integrates with MCP for tool calling and external services."""
|
| 34 |
+
|
| 35 |
+
def __init__(
|
| 36 |
+
self,
|
| 37 |
+
server_url: Optional[str] = None,
|
| 38 |
+
auto_register: bool = True,
|
| 39 |
+
):
|
| 40 |
+
"""
|
| 41 |
+
Initialize MCP integration.
|
| 42 |
+
|
| 43 |
+
Args:
|
| 44 |
+
server_url: MCP server URL (optional)
|
| 45 |
+
auto_register: Auto-register built-in tools
|
| 46 |
+
"""
|
| 47 |
+
self.server_url = server_url
|
| 48 |
+
self.tools: Dict[str, MCPTool] = {}
|
| 49 |
+
self.connected = False
|
| 50 |
+
self._connection = None
|
| 51 |
+
|
| 52 |
+
if auto_register:
|
| 53 |
+
self._register_builtin_tools()
|
| 54 |
+
|
| 55 |
+
def _register_builtin_tools(self) -> None:
|
| 56 |
+
"""Register built-in tools."""
|
| 57 |
+
# File operations
|
| 58 |
+
self.register_tool(MCPTool(
|
| 59 |
+
name="read_file",
|
| 60 |
+
description="Read contents of a file",
|
| 61 |
+
parameters={
|
| 62 |
+
"path": {"type": "string", "description": "File path to read"},
|
| 63 |
+
},
|
| 64 |
+
))
|
| 65 |
+
|
| 66 |
+
self.register_tool(MCPTool(
|
| 67 |
+
name="write_file",
|
| 68 |
+
description="Write content to a file",
|
| 69 |
+
parameters={
|
| 70 |
+
"path": {"type": "string", "description": "File path to write"},
|
| 71 |
+
"content": {"type": "string", "description": "Content to write"},
|
| 72 |
+
},
|
| 73 |
+
))
|
| 74 |
+
|
| 75 |
+
# Web search
|
| 76 |
+
self.register_tool(MCPTool(
|
| 77 |
+
name="web_search",
|
| 78 |
+
description="Search the web for information",
|
| 79 |
+
parameters={
|
| 80 |
+
"query": {"type": "string", "description": "Search query"},
|
| 81 |
+
"max_results": {"type": "integer", "description": "Max results", "default": 5},
|
| 82 |
+
},
|
| 83 |
+
))
|
| 84 |
+
|
| 85 |
+
# Code execution
|
| 86 |
+
self.register_tool(MCPTool(
|
| 87 |
+
name="execute_code",
|
| 88 |
+
description="Execute code in a sandboxed environment",
|
| 89 |
+
parameters={
|
| 90 |
+
"code": {"type": "string", "description": "Code to execute"},
|
| 91 |
+
"language": {"type": "string", "description": "Programming language"},
|
| 92 |
+
},
|
| 93 |
+
))
|
| 94 |
+
|
| 95 |
+
# Git operations
|
| 96 |
+
self.register_tool(MCPTool(
|
| 97 |
+
name="git_operation",
|
| 98 |
+
description="Execute git commands",
|
| 99 |
+
parameters={
|
| 100 |
+
"command": {"type": "string", "description": "Git command"},
|
| 101 |
+
"args": {"type": "array", "description": "Command arguments"},
|
| 102 |
+
},
|
| 103 |
+
))
|
| 104 |
+
|
| 105 |
+
# Shell commands
|
| 106 |
+
self.register_tool(MCPTool(
|
| 107 |
+
name="run_command",
|
| 108 |
+
description="Run a shell command",
|
| 109 |
+
parameters={
|
| 110 |
+
"command": {"type": "string", "description": "Command to run"},
|
| 111 |
+
"timeout": {"type": "integer", "description": "Timeout in seconds", "default": 30},
|
| 112 |
+
},
|
| 113 |
+
))
|
| 114 |
+
|
| 115 |
+
def register_tool(self, tool: MCPTool) -> None:
|
| 116 |
+
"""Register a tool."""
|
| 117 |
+
self.tools[tool.name] = tool
|
| 118 |
+
|
| 119 |
+
def unregister_tool(self, name: str) -> bool:
|
| 120 |
+
"""Unregister a tool."""
|
| 121 |
+
if name in self.tools:
|
| 122 |
+
del self.tools[name]
|
| 123 |
+
return True
|
| 124 |
+
return False
|
| 125 |
+
|
| 126 |
+
def get_tool(self, name: str) -> Optional[MCPTool]:
|
| 127 |
+
"""Get a tool by name."""
|
| 128 |
+
return self.tools.get(name)
|
| 129 |
+
|
| 130 |
+
def list_tools(self) -> List[Dict[str, Any]]:
|
| 131 |
+
"""List all registered tools."""
|
| 132 |
+
return [
|
| 133 |
+
{
|
| 134 |
+
"name": tool.name,
|
| 135 |
+
"description": tool.description,
|
| 136 |
+
"parameters": tool.parameters,
|
| 137 |
+
}
|
| 138 |
+
for tool in self.tools.values()
|
| 139 |
+
]
|
| 140 |
+
|
| 141 |
+
async def call_tool(
|
| 142 |
+
self,
|
| 143 |
+
name: str,
|
| 144 |
+
parameters: Dict[str, Any],
|
| 145 |
+
) -> Dict[str, Any]:
|
| 146 |
+
"""
|
| 147 |
+
Call a tool with parameters.
|
| 148 |
+
|
| 149 |
+
Args:
|
| 150 |
+
name: Tool name
|
| 151 |
+
parameters: Tool parameters
|
| 152 |
+
|
| 153 |
+
Returns:
|
| 154 |
+
Tool result
|
| 155 |
+
"""
|
| 156 |
+
tool = self.get_tool(name)
|
| 157 |
+
if not tool:
|
| 158 |
+
return {
|
| 159 |
+
"success": False,
|
| 160 |
+
"error": f"Tool '{name}' not found",
|
| 161 |
+
}
|
| 162 |
+
|
| 163 |
+
try:
|
| 164 |
+
# Check if tool has a handler
|
| 165 |
+
if tool.handler:
|
| 166 |
+
if asyncio.iscoroutinefunction(tool.handler):
|
| 167 |
+
result = await tool.handler(**parameters)
|
| 168 |
+
else:
|
| 169 |
+
result = tool.handler(**parameters)
|
| 170 |
+
return {
|
| 171 |
+
"success": True,
|
| 172 |
+
"result": result,
|
| 173 |
+
}
|
| 174 |
+
else:
|
| 175 |
+
# No handler - return placeholder
|
| 176 |
+
return {
|
| 177 |
+
"success": True,
|
| 178 |
+
"result": f"Tool '{name}' would be called with: {parameters}",
|
| 179 |
+
"simulated": True,
|
| 180 |
+
}
|
| 181 |
+
except Exception as e:
|
| 182 |
+
return {
|
| 183 |
+
"success": False,
|
| 184 |
+
"error": str(e),
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
def call_tool_sync(
|
| 188 |
+
self,
|
| 189 |
+
name: str,
|
| 190 |
+
parameters: Dict[str, Any],
|
| 191 |
+
) -> Dict[str, Any]:
|
| 192 |
+
"""Synchronous version of call_tool."""
|
| 193 |
+
tool = self.get_tool(name)
|
| 194 |
+
if not tool:
|
| 195 |
+
return {
|
| 196 |
+
"success": False,
|
| 197 |
+
"error": f"Tool '{name}' not found",
|
| 198 |
+
}
|
| 199 |
+
|
| 200 |
+
try:
|
| 201 |
+
if tool.handler:
|
| 202 |
+
result = tool.handler(**parameters)
|
| 203 |
+
return {
|
| 204 |
+
"success": True,
|
| 205 |
+
"result": result,
|
| 206 |
+
}
|
| 207 |
+
else:
|
| 208 |
+
return {
|
| 209 |
+
"success": True,
|
| 210 |
+
"result": f"Tool '{name}' would be called with: {parameters}",
|
| 211 |
+
"simulated": True,
|
| 212 |
+
}
|
| 213 |
+
except Exception as e:
|
| 214 |
+
return {
|
| 215 |
+
"success": False,
|
| 216 |
+
"error": str(e),
|
| 217 |
+
}
|
| 218 |
+
|
| 219 |
+
async def connect(self, server_url: str) -> bool:
|
| 220 |
+
"""Connect to MCP server."""
|
| 221 |
+
self.server_url = server_url
|
| 222 |
+
# In a real implementation, this would establish a connection
|
| 223 |
+
self.connected = True
|
| 224 |
+
return True
|
| 225 |
+
|
| 226 |
+
async def disconnect(self) -> None:
|
| 227 |
+
"""Disconnect from MCP server."""
|
| 228 |
+
self.connected = False
|
| 229 |
+
self._connection = None
|
| 230 |
+
|
| 231 |
+
def get_capabilities(self) -> Dict[str, Any]:
|
| 232 |
+
"""Get MCP capabilities."""
|
| 233 |
+
return {
|
| 234 |
+
"tools": len(self.tools),
|
| 235 |
+
"connected": self.connected,
|
| 236 |
+
"server_url": self.server_url,
|
| 237 |
+
"supports_async": True,
|
| 238 |
+
}
|
| 239 |
+
|
| 240 |
+
def __repr__(self) -> str:
|
| 241 |
+
return f"MCPIntegration(tools={len(self.tools)}, connected={self.connected})"
|
src/enhancements/config.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Stack 2.9 Enhancement Configuration
|
| 3 |
+
Central configuration for all enhancement features.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from dataclasses import dataclass, field
|
| 7 |
+
from typing import Optional
|
| 8 |
+
import os
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
@dataclass
|
| 12 |
+
class NLPConfig:
|
| 13 |
+
"""Configuration for NLP enhancements."""
|
| 14 |
+
use_bert_embeddings: bool = True
|
| 15 |
+
bert_model: str = "bert-base-uncased"
|
| 16 |
+
use_entity_recognition: bool = True
|
| 17 |
+
use_intent_detection: bool = True
|
| 18 |
+
max_context_length: int = 512
|
| 19 |
+
embedding_cache_size: int = 1000
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@dataclass
|
| 23 |
+
class KnowledgeGraphConfig:
|
| 24 |
+
"""Configuration for knowledge graph."""
|
| 25 |
+
enabled: bool = True
|
| 26 |
+
backend: str = "networkx" # or "neo4j"
|
| 27 |
+
max_nodes: int = 10000
|
| 28 |
+
max_edges: int = 50000
|
| 29 |
+
similarity_threshold: float = 0.7
|
| 30 |
+
rag_enabled: bool = True
|
| 31 |
+
rag_top_k: int = 5
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
@dataclass
|
| 35 |
+
class EmotionalIntelligenceConfig:
|
| 36 |
+
"""Configuration for emotional intelligence."""
|
| 37 |
+
enabled: bool = True
|
| 38 |
+
sentiment_model: str = "distilbert-base-uncased-finetuned-sst-2-english"
|
| 39 |
+
detect_emotions: bool = True
|
| 40 |
+
empathetic_responses: bool = True
|
| 41 |
+
emotion_sensitivity: float = 0.5
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
@dataclass
|
| 45 |
+
class CollaborationConfig:
|
| 46 |
+
"""Configuration for collaboration features."""
|
| 47 |
+
mcp_enabled: bool = True
|
| 48 |
+
conversation_state_enabled: bool = True
|
| 49 |
+
max_sessions: int = 10
|
| 50 |
+
session_timeout_minutes: int = 60
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
@dataclass
|
| 54 |
+
class LearningConfig:
|
| 55 |
+
"""Configuration for learning and adaptation."""
|
| 56 |
+
enabled: bool = True
|
| 57 |
+
feedback_storage_path: str = "data/feedback"
|
| 58 |
+
auto_finetune: bool = False
|
| 59 |
+
finetune_every_n_feedback: int = 100
|
| 60 |
+
performance_monitoring: bool = True
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
@dataclass
|
| 64 |
+
class EnhancementConfig:
|
| 65 |
+
"""Main configuration for all enhancements."""
|
| 66 |
+
nlp: NLPConfig = field(default_factory=NLPConfig)
|
| 67 |
+
knowledge_graph: KnowledgeGraphConfig = field(default_factory=KnowledgeGraphConfig)
|
| 68 |
+
emotional_intelligence: EmotionalIntelligenceConfig = field(default_factory=EmotionalIntelligenceConfig)
|
| 69 |
+
collaboration: CollaborationConfig = field(default_factory=CollaborationConfig)
|
| 70 |
+
learning: LearningConfig = field(default_factory=LearningConfig)
|
| 71 |
+
|
| 72 |
+
# Global enable/disable
|
| 73 |
+
all_enabled: bool = True
|
| 74 |
+
|
| 75 |
+
@classmethod
|
| 76 |
+
def from_env(cls) -> "EnhancementConfig":
|
| 77 |
+
"""Create config from environment variables."""
|
| 78 |
+
config = cls()
|
| 79 |
+
|
| 80 |
+
# NLP settings
|
| 81 |
+
if os.getenv("NLP_USE_BERT"):
|
| 82 |
+
config.nlp.use_bert_embeddings = os.getenv("NLP_USE_BERT").lower() == "true"
|
| 83 |
+
if os.getenv("NLP_BERT_MODEL"):
|
| 84 |
+
config.nlp.bert_model = os.getenv("NLP_BERT_MODEL")
|
| 85 |
+
|
| 86 |
+
# Knowledge graph settings
|
| 87 |
+
if os.getenv("KG_ENABLED"):
|
| 88 |
+
config.knowledge_graph.enabled = os.getenv("KG_ENABLED").lower() == "true"
|
| 89 |
+
if os.getenv("KG_RAG_ENABLED"):
|
| 90 |
+
config.knowledge_graph.rag_enabled = os.getenv("KG_RAG_ENABLED").lower() == "true"
|
| 91 |
+
|
| 92 |
+
# Emotional intelligence settings
|
| 93 |
+
if os.getenv("EI_ENABLED"):
|
| 94 |
+
config.emotional_intelligence.enabled = os.getenv("EI_ENABLED").lower() == "true"
|
| 95 |
+
|
| 96 |
+
# Learning settings
|
| 97 |
+
if os.getenv("LEARNING_ENABLED"):
|
| 98 |
+
config.learning.enabled = os.getenv("LEARNING_ENABLED").lower() == "true"
|
| 99 |
+
|
| 100 |
+
return config
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
# Global config instance
|
| 104 |
+
_default_config: Optional[EnhancementConfig] = None
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def get_config() -> EnhancementConfig:
|
| 108 |
+
"""Get the global enhancement config instance."""
|
| 109 |
+
global _default_config
|
| 110 |
+
if _default_config is None:
|
| 111 |
+
_default_config = EnhancementConfig.from_env()
|
| 112 |
+
return _default_config
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def set_config(config: EnhancementConfig) -> None:
|
| 116 |
+
"""Set the global enhancement config instance."""
|
| 117 |
+
global _default_config
|
| 118 |
+
_default_config = config
|
src/enhancements/emotional_intelligence/__init__.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Emotional Intelligence Module
|
| 3 |
+
|
| 4 |
+
Provides sentiment analysis and empathetic response generation.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from .sentiment import SentimentAnalyzer
|
| 8 |
+
from .empathy import EmpathyEngine
|
| 9 |
+
|
| 10 |
+
__all__ = [
|
| 11 |
+
"SentimentAnalyzer",
|
| 12 |
+
"EmpathyEngine",
|
| 13 |
+
]
|
src/enhancements/emotional_intelligence/empathy.py
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Empathy Engine Module
|
| 3 |
+
|
| 4 |
+
Provides empathetic response generation based on detected emotions.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import Dict, List, Optional, Any
|
| 8 |
+
from .sentiment import SentimentAnalyzer
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class EmpathyEngine:
|
| 12 |
+
"""Generate empathetic and contextually appropriate responses."""
|
| 13 |
+
|
| 14 |
+
# Response templates for different emotional states
|
| 15 |
+
EMPATHETIC_RESPONSES = {
|
| 16 |
+
"frustrated": [
|
| 17 |
+
"I understand this is frustrating. Let me help you work through this.",
|
| 18 |
+
"I can see you're frustrated. Let's take this step by step together.",
|
| 19 |
+
"Don't worry - we'll get this sorted out. I'm here to help.",
|
| 20 |
+
],
|
| 21 |
+
"sad": [
|
| 22 |
+
"I'm sorry you're going through this. I'm here to help in any way I can.",
|
| 23 |
+
"That sounds difficult. Let me see what I can do to help.",
|
| 24 |
+
"I appreciate you sharing this with me. How can I help make things better?",
|
| 25 |
+
],
|
| 26 |
+
"angry": [
|
| 27 |
+
"I understand you're frustrated. Let me help resolve this for you.",
|
| 28 |
+
"I hear you. Let's work together to find a solution.",
|
| 29 |
+
"I'm sorry you're having this experience. Let me see what I can do.",
|
| 30 |
+
],
|
| 31 |
+
"worried": [
|
| 32 |
+
"I understand your concern. Let me help put your mind at ease.",
|
| 33 |
+
"Don't worry - I'm here to help. Let's figure this out together.",
|
| 34 |
+
"I can see you're worried. Let me provide some clarity.",
|
| 35 |
+
],
|
| 36 |
+
"confused": [
|
| 37 |
+
"I understand this can be confusing. Let me explain in a clearer way.",
|
| 38 |
+
"That's a great question. Let me break this down for you.",
|
| 39 |
+
"No worries - I'll help you understand this better.",
|
| 40 |
+
],
|
| 41 |
+
"excited": [
|
| 42 |
+
"That's great to hear! I'm excited to help you with this.",
|
| 43 |
+
"Awesome! Let me help you make the most of this.",
|
| 44 |
+
"I love your enthusiasm! Let's dive in and make something great.",
|
| 45 |
+
],
|
| 46 |
+
"neutral": [
|
| 47 |
+
"I'm here to help. What would you like to work on?",
|
| 48 |
+
"How can I assist you today?",
|
| 49 |
+
"What would you like to do next?",
|
| 50 |
+
],
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
+
TONE_MODIFIERS = {
|
| 54 |
+
"empathetic": {
|
| 55 |
+
"prefix": "I understand. ",
|
| 56 |
+
"soften_errors": True,
|
| 57 |
+
"add_reassurance": True,
|
| 58 |
+
},
|
| 59 |
+
"enthusiastic": {
|
| 60 |
+
"prefix": "Great question! ",
|
| 61 |
+
"add_excitement": True,
|
| 62 |
+
"use_strong_positives": True,
|
| 63 |
+
},
|
| 64 |
+
"supportive": {
|
| 65 |
+
"prefix": "No problem at all. ",
|
| 66 |
+
"reassure_user": True,
|
| 67 |
+
"offer_encouragement": True,
|
| 68 |
+
},
|
| 69 |
+
"helpful": {
|
| 70 |
+
"prefix": "Happy to help! ",
|
| 71 |
+
"be_direct": True,
|
| 72 |
+
"include_steps": True,
|
| 73 |
+
},
|
| 74 |
+
"neutral": {
|
| 75 |
+
"prefix": "",
|
| 76 |
+
"be_concise": True,
|
| 77 |
+
},
|
| 78 |
+
}
|
| 79 |
+
|
| 80 |
+
def __init__(self):
|
| 81 |
+
"""Initialize the empathy engine."""
|
| 82 |
+
self.sentiment_analyzer = SentimentAnalyzer()
|
| 83 |
+
|
| 84 |
+
def generate_empathetic_response(
|
| 85 |
+
self,
|
| 86 |
+
user_message: str,
|
| 87 |
+
base_response: str,
|
| 88 |
+
) -> str:
|
| 89 |
+
"""
|
| 90 |
+
Generate an empathetic version of the response.
|
| 91 |
+
|
| 92 |
+
Args:
|
| 93 |
+
user_message: The user's original message
|
| 94 |
+
base_response: The generated response
|
| 95 |
+
|
| 96 |
+
Returns:
|
| 97 |
+
Empathetic response
|
| 98 |
+
"""
|
| 99 |
+
# Analyze user sentiment
|
| 100 |
+
analysis = self.sentiment_analyzer.analyze_sentiment(user_message)
|
| 101 |
+
|
| 102 |
+
# Determine appropriate tone
|
| 103 |
+
tone = self.sentiment_analyzer.get_tone_adjustment(user_message)
|
| 104 |
+
|
| 105 |
+
# Get empathetic template
|
| 106 |
+
emotional_state = self._get_dominant_emotion(user_message)
|
| 107 |
+
templates = self.EMPATHETIC_RESPONSES.get(emotional_state, self.EMPATHETIC_RESPONSES["neutral"])
|
| 108 |
+
|
| 109 |
+
# Apply tone modifiers
|
| 110 |
+
modifiers = self.TONE_MODIFIERS.get(tone, self.TONE_MODIFIERS["neutral"])
|
| 111 |
+
|
| 112 |
+
# Build response
|
| 113 |
+
response = self._build_response(
|
| 114 |
+
base_response,
|
| 115 |
+
modifiers,
|
| 116 |
+
templates,
|
| 117 |
+
analysis,
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
return response
|
| 121 |
+
|
| 122 |
+
def _get_dominant_emotion(self, text: str) -> str:
|
| 123 |
+
"""Get the dominant emotional state."""
|
| 124 |
+
if self.sentiment_analyzer.is_frustrated(text):
|
| 125 |
+
return "frustrated"
|
| 126 |
+
elif self.sentiment_analyzer.is_asking_for_help(text):
|
| 127 |
+
return "confused"
|
| 128 |
+
|
| 129 |
+
emotions = self.sentiment_analyzer.detect_emotions(text)
|
| 130 |
+
if emotions:
|
| 131 |
+
return emotions[0]["emotion"]
|
| 132 |
+
|
| 133 |
+
return "neutral"
|
| 134 |
+
|
| 135 |
+
def _build_response(
|
| 136 |
+
self,
|
| 137 |
+
base_response: str,
|
| 138 |
+
modifiers: Dict[str, bool],
|
| 139 |
+
templates: List[str],
|
| 140 |
+
analysis: Dict[str, Any],
|
| 141 |
+
) -> str:
|
| 142 |
+
"""Build the modified response."""
|
| 143 |
+
# Start with prefix if available
|
| 144 |
+
prefix = modifiers.get("prefix", "")
|
| 145 |
+
response = prefix + base_response
|
| 146 |
+
|
| 147 |
+
# Add reassurance for negative emotions
|
| 148 |
+
if modifiers.get("add_reassurance") and analysis["sentiment"] == "negative":
|
| 149 |
+
# Add a supportive note
|
| 150 |
+
if not response.endswith((". ", "!", "?")):
|
| 151 |
+
response += "."
|
| 152 |
+
response += " I'm here to help you work through this."
|
| 153 |
+
|
| 154 |
+
return response
|
| 155 |
+
|
| 156 |
+
def get_response_tone(
|
| 157 |
+
self,
|
| 158 |
+
user_message: str,
|
| 159 |
+
) -> Dict[str, Any]:
|
| 160 |
+
"""
|
| 161 |
+
Get recommended response tone.
|
| 162 |
+
|
| 163 |
+
Args:
|
| 164 |
+
user_message: User's message
|
| 165 |
+
|
| 166 |
+
Returns:
|
| 167 |
+
Dictionary with tone recommendations
|
| 168 |
+
"""
|
| 169 |
+
analysis = self.sentiment_analyzer.analyze_sentiment(user_message)
|
| 170 |
+
tone = self.sentiment_analyzer.get_tone_adjustment(user_message)
|
| 171 |
+
is_frustrated = self.sentiment_analyzer.is_frustrated(user_message)
|
| 172 |
+
|
| 173 |
+
return {
|
| 174 |
+
"recommended_tone": tone,
|
| 175 |
+
"user_sentiment": analysis["sentiment"],
|
| 176 |
+
"user_emotions": analysis["emotions"],
|
| 177 |
+
"needs_empathy": analysis["sentiment"] == "negative" or is_frustrated,
|
| 178 |
+
"modifiers": self.TONE_MODIFIERS.get(tone, {}),
|
| 179 |
+
}
|
| 180 |
+
|
| 181 |
+
def adjust_response_length(
|
| 182 |
+
self,
|
| 183 |
+
user_message: str,
|
| 184 |
+
base_response: str,
|
| 185 |
+
) -> str:
|
| 186 |
+
"""
|
| 187 |
+
Adjust response length based on user state.
|
| 188 |
+
|
| 189 |
+
Args:
|
| 190 |
+
user_message: User's message
|
| 191 |
+
base_response: Base response
|
| 192 |
+
|
| 193 |
+
Returns:
|
| 194 |
+
Adjusted response
|
| 195 |
+
"""
|
| 196 |
+
# If user is frustrated or confused, be more concise
|
| 197 |
+
if self.sentiment_analyzer.is_frustrated(user_message):
|
| 198 |
+
# Return first paragraph/section only
|
| 199 |
+
paragraphs = base_response.split("\n\n")
|
| 200 |
+
if paragraphs:
|
| 201 |
+
return paragraphs[0]
|
| 202 |
+
elif self.sentiment_analyzer.is_asking_for_help(user_message):
|
| 203 |
+
# Keep it comprehensive but clear
|
| 204 |
+
return base_response
|
| 205 |
+
|
| 206 |
+
return base_response
|
| 207 |
+
|
| 208 |
+
def get_supportive_phrase(self, emotion: str) -> str:
|
| 209 |
+
"""Get a supportive phrase for the emotion."""
|
| 210 |
+
phrases = {
|
| 211 |
+
"frustrated": "I understand this is frustrating. Let's work through it together.",
|
| 212 |
+
"sad": "I'm sorry you're experiencing this. I'm here to help.",
|
| 213 |
+
"angry": "I hear your frustration. Let me help resolve this.",
|
| 214 |
+
"worried": "I understand your concern. Let's figure this out.",
|
| 215 |
+
"confused": "That's a good question. Let me clarify.",
|
| 216 |
+
"excited": "I'm excited to help with this!",
|
| 217 |
+
"neutral": "How can I help you today?",
|
| 218 |
+
}
|
| 219 |
+
return phrases.get(emotion, phrases["neutral"])
|
| 220 |
+
|
| 221 |
+
def __repr__(self) -> str:
|
| 222 |
+
return f"EmpathyEngine(sentiment_analyzer={self.sentiment_analyzer})"
|
src/enhancements/emotional_intelligence/sentiment.py
ADDED
|
@@ -0,0 +1,238 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Sentiment Analysis Module
|
| 3 |
+
|
| 4 |
+
Provides sentiment and emotion detection for text.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import Dict, List, Optional, Any
|
| 8 |
+
import re
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class SentimentAnalyzer:
|
| 12 |
+
"""Analyze sentiment and emotions in text."""
|
| 13 |
+
|
| 14 |
+
# Emotion keywords for rule-based fallback
|
| 15 |
+
EMOTION_KEYWORDS = {
|
| 16 |
+
"joy": ["happy", "joy", "excited", "wonderful", "great", "love", "amazing", "awesome", "fantastic"],
|
| 17 |
+
"sadness": ["sad", "unhappy", "depressed", "down", "disappointed", "unfortunate", "sorry"],
|
| 18 |
+
"anger": ["angry", "mad", "frustrated", "annoyed", "irritated", "furious", "hate"],
|
| 19 |
+
"fear": ["afraid", "scared", "worried", "anxious", "nervous", "terrified", "fear"],
|
| 20 |
+
"surprise": ["surprised", "amazing", "incredible", "unexpected", "shocked", "wow"],
|
| 21 |
+
"anticipation": ["looking forward", "hope", "expect", "excited about", "can't wait"],
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
def __init__(
|
| 25 |
+
self,
|
| 26 |
+
use_transformers: bool = True,
|
| 27 |
+
model_name: str = "distilbert-base-uncased-finetuned-sst-2-english",
|
| 28 |
+
):
|
| 29 |
+
"""
|
| 30 |
+
Initialize the sentiment analyzer.
|
| 31 |
+
|
| 32 |
+
Args:
|
| 33 |
+
use_transformers: Use transformer-based sentiment analysis
|
| 34 |
+
model_name: Model name for transformer-based analysis
|
| 35 |
+
"""
|
| 36 |
+
self.use_transformers = use_transformers and self._check_transformers()
|
| 37 |
+
self.model_name = model_name
|
| 38 |
+
self._pipeline = None
|
| 39 |
+
|
| 40 |
+
def _check_transformers(self) -> bool:
|
| 41 |
+
"""Check if transformers is available."""
|
| 42 |
+
try:
|
| 43 |
+
import transformers
|
| 44 |
+
return True
|
| 45 |
+
except ImportError:
|
| 46 |
+
return False
|
| 47 |
+
|
| 48 |
+
def _load_pipeline(self):
|
| 49 |
+
"""Lazy load the sentiment pipeline."""
|
| 50 |
+
if self._pipeline is None:
|
| 51 |
+
try:
|
| 52 |
+
from transformers import pipeline
|
| 53 |
+
self._pipeline = pipeline(
|
| 54 |
+
"sentiment-analysis",
|
| 55 |
+
model=self.model_name,
|
| 56 |
+
)
|
| 57 |
+
except Exception as e:
|
| 58 |
+
print(f"Warning: Could not load transformer sentiment model: {e}")
|
| 59 |
+
self.use_transformers = False
|
| 60 |
+
|
| 61 |
+
def analyze_sentiment(
|
| 62 |
+
self,
|
| 63 |
+
text: str,
|
| 64 |
+
return_scores: bool = True,
|
| 65 |
+
) -> Dict[str, Any]:
|
| 66 |
+
"""
|
| 67 |
+
Analyze sentiment of text.
|
| 68 |
+
|
| 69 |
+
Args:
|
| 70 |
+
text: Input text
|
| 71 |
+
return_scores: Return confidence scores
|
| 72 |
+
|
| 73 |
+
Returns:
|
| 74 |
+
Dictionary with sentiment, score, and emotions
|
| 75 |
+
"""
|
| 76 |
+
sentiment = "neutral"
|
| 77 |
+
score = 0.5
|
| 78 |
+
emotions = []
|
| 79 |
+
|
| 80 |
+
# Try transformer-based first
|
| 81 |
+
if self.use_transformers:
|
| 82 |
+
try:
|
| 83 |
+
result = self._analyze_transformers(text)
|
| 84 |
+
sentiment = result["label"]
|
| 85 |
+
score = result["score"]
|
| 86 |
+
except Exception:
|
| 87 |
+
pass
|
| 88 |
+
|
| 89 |
+
# Fall back to rule-based
|
| 90 |
+
if sentiment == "neutral":
|
| 91 |
+
result = self._analyze_rule_based(text)
|
| 92 |
+
sentiment = result["sentiment"]
|
| 93 |
+
score = result["score"]
|
| 94 |
+
emotions = result["emotions"]
|
| 95 |
+
|
| 96 |
+
# Detect emotions
|
| 97 |
+
emotions = self.detect_emotions(text)
|
| 98 |
+
|
| 99 |
+
return {
|
| 100 |
+
"sentiment": sentiment,
|
| 101 |
+
"score": score,
|
| 102 |
+
"emotions": emotions,
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
def _analyze_transformers(self, text: str) -> Dict[str, Any]:
|
| 106 |
+
"""Use transformer for sentiment analysis."""
|
| 107 |
+
self._load_pipeline()
|
| 108 |
+
|
| 109 |
+
if self._pipeline is None:
|
| 110 |
+
return {"label": "neutral", "score": 0.5}
|
| 111 |
+
|
| 112 |
+
# Truncate long text
|
| 113 |
+
text = text[:512]
|
| 114 |
+
result = self._pipeline(text)[0]
|
| 115 |
+
|
| 116 |
+
return {
|
| 117 |
+
"label": result["label"].lower(),
|
| 118 |
+
"score": result["score"],
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
def _analyze_rule_based(self, text: str) -> Dict[str, Any]:
|
| 122 |
+
"""Rule-based sentiment analysis."""
|
| 123 |
+
text_lower = text.lower()
|
| 124 |
+
|
| 125 |
+
positive_words = ["good", "great", "excellent", "amazing", "wonderful", "fantastic",
|
| 126 |
+
"love", "happy", "joy", "best", "perfect", "awesome", "like"]
|
| 127 |
+
negative_words = ["bad", "terrible", "awful", "horrible", "worst", "hate", "sad",
|
| 128 |
+
"angry", "disappointed", "frustrated", "annoying", "poor", "fail"]
|
| 129 |
+
|
| 130 |
+
pos_count = sum(1 for word in positive_words if word in text_lower)
|
| 131 |
+
neg_count = sum(1 for word in negative_words if word in text_lower)
|
| 132 |
+
|
| 133 |
+
total = pos_count + neg_count
|
| 134 |
+
if total == 0:
|
| 135 |
+
return {"sentiment": "neutral", "score": 0.5, "emotions": []}
|
| 136 |
+
|
| 137 |
+
if pos_count > neg_count:
|
| 138 |
+
score = min(0.5 + (pos_count - neg_count) * 0.1, 0.95)
|
| 139 |
+
sentiment = "positive"
|
| 140 |
+
elif neg_count > pos_count:
|
| 141 |
+
score = max(0.5 - (neg_count - pos_count) * 0.1, 0.05)
|
| 142 |
+
sentiment = "negative"
|
| 143 |
+
else:
|
| 144 |
+
sentiment = "neutral"
|
| 145 |
+
score = 0.5
|
| 146 |
+
|
| 147 |
+
return {"sentiment": sentiment, "score": score, "emotions": []}
|
| 148 |
+
|
| 149 |
+
def detect_emotions(self, text: str) -> List[Dict[str, float]]:
|
| 150 |
+
"""
|
| 151 |
+
Detect emotions in text.
|
| 152 |
+
|
| 153 |
+
Args:
|
| 154 |
+
text: Input text
|
| 155 |
+
|
| 156 |
+
Returns:
|
| 157 |
+
List of emotion dictionaries with scores
|
| 158 |
+
"""
|
| 159 |
+
text_lower = text.lower()
|
| 160 |
+
emotions = []
|
| 161 |
+
|
| 162 |
+
for emotion, keywords in self.EMOTION_KEYWORDS.items():
|
| 163 |
+
score = 0.0
|
| 164 |
+
for keyword in keywords:
|
| 165 |
+
if keyword in text_lower:
|
| 166 |
+
score += 1.0
|
| 167 |
+
|
| 168 |
+
if score > 0:
|
| 169 |
+
# Normalize score
|
| 170 |
+
score = min(score / len(keywords), 1.0)
|
| 171 |
+
emotions.append({
|
| 172 |
+
"emotion": emotion,
|
| 173 |
+
"score": score,
|
| 174 |
+
})
|
| 175 |
+
|
| 176 |
+
# Sort by score
|
| 177 |
+
emotions.sort(key=lambda x: -x["score"])
|
| 178 |
+
return emotions[:3]
|
| 179 |
+
|
| 180 |
+
def get_emotion_intensity(self, text: str) -> float:
|
| 181 |
+
"""
|
| 182 |
+
Get overall emotion intensity (0-1).
|
| 183 |
+
|
| 184 |
+
Args:
|
| 185 |
+
text: Input text
|
| 186 |
+
|
| 187 |
+
Returns:
|
| 188 |
+
Emotion intensity score
|
| 189 |
+
"""
|
| 190 |
+
emotions = self.detect_emotions(text)
|
| 191 |
+
if not emotions:
|
| 192 |
+
return 0.0
|
| 193 |
+
|
| 194 |
+
return max(e["score"] for e in emotions)
|
| 195 |
+
|
| 196 |
+
def is_frustrated(self, text: str) -> bool:
|
| 197 |
+
"""Check if user seems frustrated."""
|
| 198 |
+
frustration_indicators = [
|
| 199 |
+
"frustrated", "annoyed", "angry", "mad", "sick of", "tired of",
|
| 200 |
+
"this is useless", "this doesn't work", "why won't", "can't figure out",
|
| 201 |
+
]
|
| 202 |
+
text_lower = text.lower()
|
| 203 |
+
return any(indicator in text_lower for indicator in frustration_indicators)
|
| 204 |
+
|
| 205 |
+
def is_asking_for_help(self, text: str) -> bool:
|
| 206 |
+
"""Check if user is asking for help."""
|
| 207 |
+
help_indicators = [
|
| 208 |
+
"help", "how do i", "can you", "please", "i need", "need help",
|
| 209 |
+
"stuck", "confused", "don't understand", "having trouble",
|
| 210 |
+
]
|
| 211 |
+
text_lower = text.lower()
|
| 212 |
+
return any(indicator in text_lower for indicator in help_indicators)
|
| 213 |
+
|
| 214 |
+
def get_tone_adjustment(self, text: str) -> str:
|
| 215 |
+
"""
|
| 216 |
+
Get recommended tone adjustment based on sentiment.
|
| 217 |
+
|
| 218 |
+
Args:
|
| 219 |
+
text: Input text
|
| 220 |
+
|
| 221 |
+
Returns:
|
| 222 |
+
Tone adjustment recommendation
|
| 223 |
+
"""
|
| 224 |
+
analysis = self.analyze_sentiment(text)
|
| 225 |
+
|
| 226 |
+
if analysis["sentiment"] == "negative":
|
| 227 |
+
return "empathetic"
|
| 228 |
+
elif analysis["sentiment"] == "positive":
|
| 229 |
+
return "enthusiastic"
|
| 230 |
+
elif self.is_frustrated(text):
|
| 231 |
+
return "supportive"
|
| 232 |
+
elif self.is_asking_for_help(text):
|
| 233 |
+
return "helpful"
|
| 234 |
+
else:
|
| 235 |
+
return "neutral"
|
| 236 |
+
|
| 237 |
+
def __repr__(self) -> str:
|
| 238 |
+
return f"SentimentAnalyzer(use_transformers={self.use_transformers}, model='{self.model_name}')"
|
src/enhancements/knowledge_graph/__init__.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Knowledge Graph Module
|
| 3 |
+
|
| 4 |
+
Provides graph-based knowledge representation with RAG support.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from .graph import KnowledgeGraph
|
| 8 |
+
from .rag import RAGEngine
|
| 9 |
+
|
| 10 |
+
__all__ = [
|
| 11 |
+
"KnowledgeGraph",
|
| 12 |
+
"RAGEngine",
|
| 13 |
+
]
|
src/enhancements/knowledge_graph/graph.py
ADDED
|
@@ -0,0 +1,287 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Knowledge Graph Implementation
|
| 3 |
+
|
| 4 |
+
Graph-based knowledge representation using networkx.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import List, Dict, Optional, Any, Set, Tuple
|
| 8 |
+
import networkx as nx
|
| 9 |
+
import numpy as np
|
| 10 |
+
from datetime import datetime
|
| 11 |
+
import json
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class KnowledgeGraph:
|
| 16 |
+
"""Graph-based knowledge representation with entities and relationships."""
|
| 17 |
+
|
| 18 |
+
def __init__(
|
| 19 |
+
self,
|
| 20 |
+
max_nodes: int = 10000,
|
| 21 |
+
max_edges: int = 50000,
|
| 22 |
+
):
|
| 23 |
+
"""
|
| 24 |
+
Initialize the knowledge graph.
|
| 25 |
+
|
| 26 |
+
Args:
|
| 27 |
+
max_nodes: Maximum number of nodes
|
| 28 |
+
max_edges: Maximum number of edges
|
| 29 |
+
"""
|
| 30 |
+
self.max_nodes = max_nodes
|
| 31 |
+
self.max_edges = max_edges
|
| 32 |
+
self.graph = nx.MultiDiGraph()
|
| 33 |
+
self._node_attributes: Dict[str, Dict[str, Any]] = {}
|
| 34 |
+
self._edge_attributes: Dict[Tuple[str, str], Dict[str, Any]] = {}
|
| 35 |
+
|
| 36 |
+
def add_entity(
|
| 37 |
+
self,
|
| 38 |
+
entity_id: str,
|
| 39 |
+
entity_type: str,
|
| 40 |
+
properties: Optional[Dict[str, Any]] = None,
|
| 41 |
+
) -> bool:
|
| 42 |
+
"""
|
| 43 |
+
Add an entity to the knowledge graph.
|
| 44 |
+
|
| 45 |
+
Args:
|
| 46 |
+
entity_id: Unique identifier for the entity
|
| 47 |
+
entity_type: Type of entity (e.g., 'person', 'concept', 'code')
|
| 48 |
+
properties: Additional properties
|
| 49 |
+
|
| 50 |
+
Returns:
|
| 51 |
+
True if added, False if limit reached
|
| 52 |
+
"""
|
| 53 |
+
if self.graph.number_of_nodes() >= self.max_nodes:
|
| 54 |
+
return False
|
| 55 |
+
|
| 56 |
+
if entity_id not in self.graph:
|
| 57 |
+
self.graph.add_node(entity_id, type=entity_type)
|
| 58 |
+
|
| 59 |
+
self._node_attributes[entity_id] = {
|
| 60 |
+
"type": entity_type,
|
| 61 |
+
"created_at": datetime.now().isoformat(),
|
| 62 |
+
"properties": properties or {},
|
| 63 |
+
}
|
| 64 |
+
|
| 65 |
+
return True
|
| 66 |
+
|
| 67 |
+
def add_relationship(
|
| 68 |
+
self,
|
| 69 |
+
source_id: str,
|
| 70 |
+
target_id: str,
|
| 71 |
+
relationship_type: str,
|
| 72 |
+
properties: Optional[Dict[str, Any]] = None,
|
| 73 |
+
) -> bool:
|
| 74 |
+
"""
|
| 75 |
+
Add a relationship between entities.
|
| 76 |
+
|
| 77 |
+
Args:
|
| 78 |
+
source_id: Source entity ID
|
| 79 |
+
target_id: Target entity ID
|
| 80 |
+
relationship_type: Type of relationship
|
| 81 |
+
properties: Additional properties
|
| 82 |
+
|
| 83 |
+
Returns:
|
| 84 |
+
True if added, False if limit reached or entities don't exist
|
| 85 |
+
"""
|
| 86 |
+
if self.graph.number_of_edges() >= self.max_edges:
|
| 87 |
+
return False
|
| 88 |
+
|
| 89 |
+
# Ensure entities exist
|
| 90 |
+
if source_id not in self.graph:
|
| 91 |
+
self.add_entity(source_id, "unknown")
|
| 92 |
+
if target_id not in self.graph:
|
| 93 |
+
self.add_entity(target_id, "unknown")
|
| 94 |
+
|
| 95 |
+
self.graph.add_edge(source_id, target_id, type=relationship_type)
|
| 96 |
+
|
| 97 |
+
edge_key = (source_id, target_id)
|
| 98 |
+
self._edge_attributes[edge_key] = {
|
| 99 |
+
"type": relationship_type,
|
| 100 |
+
"created_at": datetime.now().isoformat(),
|
| 101 |
+
"properties": properties or {},
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
return True
|
| 105 |
+
|
| 106 |
+
def get_entity(self, entity_id: str) -> Optional[Dict[str, Any]]:
|
| 107 |
+
"""Get entity information."""
|
| 108 |
+
if entity_id not in self.graph:
|
| 109 |
+
return None
|
| 110 |
+
|
| 111 |
+
return {
|
| 112 |
+
"id": entity_id,
|
| 113 |
+
**self._node_attributes.get(entity_id, {}),
|
| 114 |
+
}
|
| 115 |
+
|
| 116 |
+
def get_relationships(
|
| 117 |
+
self,
|
| 118 |
+
entity_id: str,
|
| 119 |
+
relationship_type: Optional[str] = None,
|
| 120 |
+
) -> List[Dict[str, Any]]:
|
| 121 |
+
"""Get relationships for an entity."""
|
| 122 |
+
if entity_id not in self.graph:
|
| 123 |
+
return []
|
| 124 |
+
|
| 125 |
+
relationships = []
|
| 126 |
+
for source, target, data in self.graph.edges(data=True):
|
| 127 |
+
if source == entity_id or target == entity_id:
|
| 128 |
+
rel_type = data.get("type", "unknown")
|
| 129 |
+
if relationship_type and rel_type != relationship_type:
|
| 130 |
+
continue
|
| 131 |
+
|
| 132 |
+
relationships.append({
|
| 133 |
+
"source": source,
|
| 134 |
+
"target": target,
|
| 135 |
+
"type": rel_type,
|
| 136 |
+
})
|
| 137 |
+
|
| 138 |
+
return relationships
|
| 139 |
+
|
| 140 |
+
def find_similar_entities(
|
| 141 |
+
self,
|
| 142 |
+
entity_id: str,
|
| 143 |
+
max_results: int = 5,
|
| 144 |
+
) -> List[Tuple[str, float]]:
|
| 145 |
+
"""
|
| 146 |
+
Find similar entities using graph-based similarity.
|
| 147 |
+
|
| 148 |
+
Args:
|
| 149 |
+
entity_id: Entity to find similar
|
| 150 |
+
max_results: Maximum number of results
|
| 151 |
+
|
| 152 |
+
Returns:
|
| 153 |
+
List of (entity_id, similarity_score) tuples
|
| 154 |
+
"""
|
| 155 |
+
if entity_id not in self.graph:
|
| 156 |
+
return []
|
| 157 |
+
|
| 158 |
+
# Use common neighbors as simple similarity
|
| 159 |
+
neighbors = set(self.graph.neighbors(entity_id))
|
| 160 |
+
scores = []
|
| 161 |
+
|
| 162 |
+
for node in self.graph.nodes():
|
| 163 |
+
if node == entity_id:
|
| 164 |
+
continue
|
| 165 |
+
|
| 166 |
+
node_neighbors = set(self.graph.neighbors(node))
|
| 167 |
+
common = len(neighbors & node_neighbors)
|
| 168 |
+
|
| 169 |
+
if common > 0:
|
| 170 |
+
# Jaccard-like similarity
|
| 171 |
+
union = len(neighbors | node_neighbors)
|
| 172 |
+
score = common / union if union > 0 else 0
|
| 173 |
+
scores.append((node, score))
|
| 174 |
+
|
| 175 |
+
scores.sort(key=lambda x: -x[1])
|
| 176 |
+
return scores[:max_results]
|
| 177 |
+
|
| 178 |
+
def search_entities(
|
| 179 |
+
self,
|
| 180 |
+
entity_type: Optional[str] = None,
|
| 181 |
+
property_filter: Optional[Dict[str, Any]] = None,
|
| 182 |
+
) -> List[str]:
|
| 183 |
+
"""
|
| 184 |
+
Search for entities.
|
| 185 |
+
|
| 186 |
+
Args:
|
| 187 |
+
entity_type: Filter by entity type
|
| 188 |
+
property_filter: Filter by properties
|
| 189 |
+
|
| 190 |
+
Returns:
|
| 191 |
+
List of matching entity IDs
|
| 192 |
+
"""
|
| 193 |
+
results = []
|
| 194 |
+
|
| 195 |
+
for node in self.graph.nodes():
|
| 196 |
+
attrs = self._node_attributes.get(node, {})
|
| 197 |
+
|
| 198 |
+
# Check type filter
|
| 199 |
+
if entity_type and attrs.get("type") != entity_type:
|
| 200 |
+
continue
|
| 201 |
+
|
| 202 |
+
# Check property filter
|
| 203 |
+
if property_filter:
|
| 204 |
+
props = attrs.get("properties", {})
|
| 205 |
+
if not all(props.get(k) == v for k, v in property_filter.items()):
|
| 206 |
+
continue
|
| 207 |
+
|
| 208 |
+
results.append(node)
|
| 209 |
+
|
| 210 |
+
return results
|
| 211 |
+
|
| 212 |
+
def get_subgraph(
|
| 213 |
+
self,
|
| 214 |
+
entity_ids: List[str],
|
| 215 |
+
depth: int = 1,
|
| 216 |
+
) -> nx.MultiDiGraph:
|
| 217 |
+
"""
|
| 218 |
+
Get a subgraph around specified entities.
|
| 219 |
+
|
| 220 |
+
Args:
|
| 221 |
+
entity_ids: Center entities
|
| 222 |
+
depth: How many hops to include
|
| 223 |
+
|
| 224 |
+
Returns:
|
| 225 |
+
Subgraph
|
| 226 |
+
"""
|
| 227 |
+
nodes = set(entity_ids)
|
| 228 |
+
|
| 229 |
+
for _ in range(depth):
|
| 230 |
+
for entity in list(nodes):
|
| 231 |
+
nodes.update(self.graph.neighbors(entity))
|
| 232 |
+
|
| 233 |
+
return self.graph.subgraph(nodes).copy()
|
| 234 |
+
|
| 235 |
+
def export_json(self, filepath: str) -> None:
|
| 236 |
+
"""Export graph to JSON."""
|
| 237 |
+
data = {
|
| 238 |
+
"nodes": [
|
| 239 |
+
{
|
| 240 |
+
"id": node,
|
| 241 |
+
**self._node_attributes.get(node, {}),
|
| 242 |
+
}
|
| 243 |
+
for node in self.graph.nodes()
|
| 244 |
+
],
|
| 245 |
+
"edges": [
|
| 246 |
+
{
|
| 247 |
+
"source": source,
|
| 248 |
+
"target": target,
|
| 249 |
+
"type": data.get("type", "unknown"),
|
| 250 |
+
}
|
| 251 |
+
for source, target, data in self.graph.edges(data=True)
|
| 252 |
+
],
|
| 253 |
+
}
|
| 254 |
+
|
| 255 |
+
Path(filepath).write_text(json.dumps(data, indent=2))
|
| 256 |
+
|
| 257 |
+
def import_json(self, filepath: str) -> None:
|
| 258 |
+
"""Import graph from JSON."""
|
| 259 |
+
data = json.loads(Path(filepath).read_text())
|
| 260 |
+
|
| 261 |
+
for node_data in data.get("nodes", []):
|
| 262 |
+
node_id = node_data.pop("id")
|
| 263 |
+
self.add_entity(node_id, node_data.get("type", "unknown"), node_data.get("properties"))
|
| 264 |
+
|
| 265 |
+
for edge_data in data.get("edges", []):
|
| 266 |
+
self.add_relationship(
|
| 267 |
+
edge_data["source"],
|
| 268 |
+
edge_data["target"],
|
| 269 |
+
edge_data.get("type", "unknown"),
|
| 270 |
+
)
|
| 271 |
+
|
| 272 |
+
def get_stats(self) -> Dict[str, Any]:
|
| 273 |
+
"""Get graph statistics."""
|
| 274 |
+
return {
|
| 275 |
+
"num_nodes": self.graph.number_of_nodes(),
|
| 276 |
+
"num_edges": self.graph.number_of_edges(),
|
| 277 |
+
"num_node_types": len(set(
|
| 278 |
+
attrs.get("type") for attrs in self._node_attributes.values()
|
| 279 |
+
)),
|
| 280 |
+
"num_edge_types": len(set(
|
| 281 |
+
data.get("type") for _, _, data in self.graph.edges(data=True)
|
| 282 |
+
)),
|
| 283 |
+
}
|
| 284 |
+
|
| 285 |
+
def __repr__(self) -> str:
|
| 286 |
+
stats = self.get_stats()
|
| 287 |
+
return f"KnowledgeGraph(nodes={stats['num_nodes']}, edges={stats['num_edges']})"
|
src/enhancements/knowledge_graph/rag.py
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
RAG (Retrieval-Augmented Generation) Engine
|
| 3 |
+
|
| 4 |
+
Provides context retrieval for augmented generation.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import List, Dict, Optional, Any, Tuple
|
| 8 |
+
import numpy as np
|
| 9 |
+
from collections import defaultdict
|
| 10 |
+
import re
|
| 11 |
+
from datetime import datetime
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class Document:
|
| 15 |
+
"""Represents a document for RAG."""
|
| 16 |
+
|
| 17 |
+
def __init__(
|
| 18 |
+
self,
|
| 19 |
+
doc_id: str,
|
| 20 |
+
content: str,
|
| 21 |
+
metadata: Optional[Dict[str, Any]] = None,
|
| 22 |
+
):
|
| 23 |
+
self.id = doc_id
|
| 24 |
+
self.content = content
|
| 25 |
+
self.metadata = metadata or {}
|
| 26 |
+
self.embeddings: Optional[np.ndarray] = None
|
| 27 |
+
self.created_at = self.metadata.get("created_at", datetime.now().isoformat())
|
| 28 |
+
|
| 29 |
+
def __repr__(self) -> str:
|
| 30 |
+
return f"Document(id='{self.id}', content_length={len(self.content)})"
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class RAGEngine:
|
| 34 |
+
"""Retrieval-Augmented Generation engine for context-aware responses."""
|
| 35 |
+
|
| 36 |
+
def __init__(
|
| 37 |
+
self,
|
| 38 |
+
top_k: int = 5,
|
| 39 |
+
similarity_threshold: float = 0.7,
|
| 40 |
+
):
|
| 41 |
+
"""
|
| 42 |
+
Initialize the RAG engine.
|
| 43 |
+
|
| 44 |
+
Args:
|
| 45 |
+
top_k: Number of top results to retrieve
|
| 46 |
+
similarity_threshold: Minimum similarity for retrieval
|
| 47 |
+
"""
|
| 48 |
+
self.top_k = top_k
|
| 49 |
+
self.similarity_threshold = similarity_threshold
|
| 50 |
+
self.documents: Dict[str, Document] = {}
|
| 51 |
+
self.document_embeddings: Dict[str, np.ndarray] = {}
|
| 52 |
+
self._index_initialized = False
|
| 53 |
+
self._keyword_index: Dict[str, set] = defaultdict(set)
|
| 54 |
+
|
| 55 |
+
def add_document(
|
| 56 |
+
self,
|
| 57 |
+
doc_id: str,
|
| 58 |
+
content: str,
|
| 59 |
+
metadata: Optional[Dict[str, Any]] = None,
|
| 60 |
+
embedding: Optional[np.ndarray] = None,
|
| 61 |
+
) -> None:
|
| 62 |
+
"""
|
| 63 |
+
Add a document to the RAG index.
|
| 64 |
+
|
| 65 |
+
Args:
|
| 66 |
+
doc_id: Unique document ID
|
| 67 |
+
content: Document content
|
| 68 |
+
metadata: Document metadata
|
| 69 |
+
embedding: Pre-computed embedding (optional)
|
| 70 |
+
"""
|
| 71 |
+
doc = Document(doc_id, content, metadata)
|
| 72 |
+
if embedding is not None:
|
| 73 |
+
doc.embeddings = embedding
|
| 74 |
+
|
| 75 |
+
self.documents[doc_id] = doc
|
| 76 |
+
|
| 77 |
+
# Update keyword index
|
| 78 |
+
keywords = self._extract_keywords(content)
|
| 79 |
+
for keyword in keywords:
|
| 80 |
+
self._keyword_index[keyword].add(doc_id)
|
| 81 |
+
|
| 82 |
+
self._index_initialized = False
|
| 83 |
+
|
| 84 |
+
def _extract_keywords(self, text: str) -> List[str]:
|
| 85 |
+
"""Extract keywords from text."""
|
| 86 |
+
# Simple keyword extraction
|
| 87 |
+
words = re.findall(r'\b\w+\b', text.lower())
|
| 88 |
+
# Filter short words and common words
|
| 89 |
+
stopwords = {'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been',
|
| 90 |
+
'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will',
|
| 91 |
+
'would', 'could', 'should', 'may', 'might', 'must', 'shall',
|
| 92 |
+
'can', 'need', 'dare', 'ought', 'used', 'to', 'of', 'in',
|
| 93 |
+
'for', 'on', 'with', 'at', 'by', 'from', 'as', 'into',
|
| 94 |
+
'through', 'during', 'before', 'after', 'above', 'below',
|
| 95 |
+
'between', 'under', 'again', 'further', 'then', 'once'}
|
| 96 |
+
return [w for w in words if len(w) > 2 and w not in stopwords]
|
| 97 |
+
|
| 98 |
+
def _build_index(self) -> None:
|
| 99 |
+
"""Build similarity index."""
|
| 100 |
+
if not self.documents:
|
| 101 |
+
return
|
| 102 |
+
|
| 103 |
+
# Initialize embeddings for documents without them
|
| 104 |
+
for doc_id, doc in self.documents.items():
|
| 105 |
+
if doc.embeddings is None:
|
| 106 |
+
# Create simple embedding based on word frequencies
|
| 107 |
+
doc.embeddings = self._create_simple_embedding(doc.content)
|
| 108 |
+
|
| 109 |
+
self._index_initialized = True
|
| 110 |
+
|
| 111 |
+
def _create_simple_embedding(self, text: str) -> np.ndarray:
|
| 112 |
+
"""Create a simple bag-of-words embedding."""
|
| 113 |
+
keywords = self._extract_keywords(text)
|
| 114 |
+
embedding = np.zeros(len(self._keyword_index))
|
| 115 |
+
|
| 116 |
+
for i, keyword in enumerate(self._keyword_index.keys()):
|
| 117 |
+
if keyword in keywords:
|
| 118 |
+
embedding[i] = keywords.count(keyword)
|
| 119 |
+
|
| 120 |
+
# Normalize
|
| 121 |
+
norm = np.linalg.norm(embedding)
|
| 122 |
+
if norm > 0:
|
| 123 |
+
embedding /= norm
|
| 124 |
+
|
| 125 |
+
return embedding
|
| 126 |
+
|
| 127 |
+
def retrieve(
|
| 128 |
+
self,
|
| 129 |
+
query: str,
|
| 130 |
+
top_k: Optional[int] = None,
|
| 131 |
+
use_keyword_index: bool = True,
|
| 132 |
+
) -> List[Tuple[Document, float]]:
|
| 133 |
+
"""
|
| 134 |
+
Retrieve relevant documents for a query.
|
| 135 |
+
|
| 136 |
+
Args:
|
| 137 |
+
query: Query text
|
| 138 |
+
top_k: Override default top_k
|
| 139 |
+
use_keyword_index: Use keyword pre-filtering
|
| 140 |
+
|
| 141 |
+
Returns:
|
| 142 |
+
List of (document, similarity_score) tuples
|
| 143 |
+
"""
|
| 144 |
+
if not self.documents:
|
| 145 |
+
return []
|
| 146 |
+
|
| 147 |
+
self._build_index()
|
| 148 |
+
|
| 149 |
+
top_k = top_k or self.top_k
|
| 150 |
+
|
| 151 |
+
# Create query embedding
|
| 152 |
+
query_embedding = self._create_simple_embedding(query)
|
| 153 |
+
|
| 154 |
+
# Get candidate document IDs
|
| 155 |
+
candidate_ids = set(self.documents.keys())
|
| 156 |
+
if use_keyword_index:
|
| 157 |
+
query_keywords = self._extract_keywords(query)
|
| 158 |
+
keyword_candidates = set()
|
| 159 |
+
for keyword in query_keywords:
|
| 160 |
+
keyword_candidates.update(self._keyword_index.get(keyword, set()))
|
| 161 |
+
if keyword_candidates:
|
| 162 |
+
candidate_ids &= keyword_candidates
|
| 163 |
+
|
| 164 |
+
# Calculate similarities
|
| 165 |
+
scores = []
|
| 166 |
+
for doc_id in candidate_ids:
|
| 167 |
+
doc = self.documents[doc_id]
|
| 168 |
+
if doc.embeddings is not None:
|
| 169 |
+
similarity = np.dot(query_embedding, doc.embeddings)
|
| 170 |
+
if similarity >= self.similarity_threshold:
|
| 171 |
+
scores.append((doc, similarity))
|
| 172 |
+
|
| 173 |
+
# Sort by similarity and return top_k
|
| 174 |
+
scores.sort(key=lambda x: -x[1])
|
| 175 |
+
return scores[:top_k]
|
| 176 |
+
|
| 177 |
+
def retrieve_as_context(
|
| 178 |
+
self,
|
| 179 |
+
query: str,
|
| 180 |
+
max_context_length: int = 1000,
|
| 181 |
+
) -> str:
|
| 182 |
+
"""
|
| 183 |
+
Retrieve documents and format as context string.
|
| 184 |
+
|
| 185 |
+
Args:
|
| 186 |
+
query: Query text
|
| 187 |
+
max_context_length: Maximum length of context
|
| 188 |
+
|
| 189 |
+
Returns:
|
| 190 |
+
Formatted context string
|
| 191 |
+
"""
|
| 192 |
+
results = self.retrieve(query)
|
| 193 |
+
|
| 194 |
+
if not results:
|
| 195 |
+
return ""
|
| 196 |
+
|
| 197 |
+
context_parts = []
|
| 198 |
+
current_length = 0
|
| 199 |
+
|
| 200 |
+
for doc, score in results:
|
| 201 |
+
if current_length >= max_context_length:
|
| 202 |
+
break
|
| 203 |
+
|
| 204 |
+
# Add document with relevance score
|
| 205 |
+
context = f"[Relevance: {score:.2f}]\n{doc.content}\n"
|
| 206 |
+
if current_length + len(context) <= max_context_length:
|
| 207 |
+
context_parts.append(context)
|
| 208 |
+
current_length += len(context)
|
| 209 |
+
|
| 210 |
+
return "\n".join(context_parts)
|
| 211 |
+
|
| 212 |
+
def search(self, query: str) -> List[Document]:
|
| 213 |
+
"""Simple text search in documents."""
|
| 214 |
+
results = []
|
| 215 |
+
query_lower = query.lower()
|
| 216 |
+
|
| 217 |
+
for doc in self.documents.values():
|
| 218 |
+
if query_lower in doc.content.lower():
|
| 219 |
+
results.append(doc)
|
| 220 |
+
|
| 221 |
+
return results
|
| 222 |
+
|
| 223 |
+
def get_document(self, doc_id: str) -> Optional[Document]:
|
| 224 |
+
"""Get a document by ID."""
|
| 225 |
+
return self.documents.get(doc_id)
|
| 226 |
+
|
| 227 |
+
def delete_document(self, doc_id: str) -> bool:
|
| 228 |
+
"""Delete a document."""
|
| 229 |
+
if doc_id in self.documents:
|
| 230 |
+
# Update keyword index
|
| 231 |
+
keywords = self._extract_keywords(self.documents[doc_id].content)
|
| 232 |
+
for keyword in keywords:
|
| 233 |
+
self._keyword_index[keyword].discard(doc_id)
|
| 234 |
+
|
| 235 |
+
del self.documents[doc_id]
|
| 236 |
+
self._index_initialized = False
|
| 237 |
+
return True
|
| 238 |
+
return False
|
| 239 |
+
|
| 240 |
+
def get_stats(self) -> Dict[str, Any]:
|
| 241 |
+
"""Get RAG engine statistics."""
|
| 242 |
+
return {
|
| 243 |
+
"num_documents": len(self.documents),
|
| 244 |
+
"num_keywords": len(self._keyword_index),
|
| 245 |
+
"index_initialized": self._index_initialized,
|
| 246 |
+
}
|
| 247 |
+
|
| 248 |
+
def __repr__(self) -> str:
|
| 249 |
+
stats = self.get_stats()
|
| 250 |
+
return f"RAGEngine(docs={stats['num_documents']}, keywords={stats['num_keywords']})"
|
src/enhancements/learning/__init__.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Learning and Adaptation Module
|
| 3 |
+
|
| 4 |
+
Provides feedback collection and continuous learning capabilities.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from .feedback import FeedbackCollector
|
| 8 |
+
from .performance import PerformanceMonitor
|
| 9 |
+
|
| 10 |
+
__all__ = [
|
| 11 |
+
"FeedbackCollector",
|
| 12 |
+
"PerformanceMonitor",
|
| 13 |
+
]
|
src/enhancements/learning/feedback.py
ADDED
|
@@ -0,0 +1,307 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Feedback Collection System
|
| 3 |
+
|
| 4 |
+
Collects user feedback for continuous improvement.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import Dict, List, Optional, Any
|
| 8 |
+
from datetime import datetime
|
| 9 |
+
import json
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
import uuid
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class FeedbackEntry:
|
| 15 |
+
"""Represents a single feedback entry."""
|
| 16 |
+
|
| 17 |
+
def __init__(
|
| 18 |
+
self,
|
| 19 |
+
feedback_type: str,
|
| 20 |
+
user_id: Optional[str],
|
| 21 |
+
message: str,
|
| 22 |
+
response: str,
|
| 23 |
+
rating: Optional[int] = None,
|
| 24 |
+
metadata: Optional[Dict[str, Any]] = None,
|
| 25 |
+
):
|
| 26 |
+
self.id = str(uuid.uuid4())
|
| 27 |
+
self.feedback_type = feedback_type # "thumbs_up", "thumbs_down", "correction", "suggestion"
|
| 28 |
+
self.user_id = user_id
|
| 29 |
+
self.message = message
|
| 30 |
+
self.response = response
|
| 31 |
+
self.rating = rating # 1-5 scale
|
| 32 |
+
self.metadata = metadata or {}
|
| 33 |
+
self.created_at = datetime.now()
|
| 34 |
+
self.processed = False
|
| 35 |
+
|
| 36 |
+
def to_dict(self) -> Dict[str, Any]:
|
| 37 |
+
"""Convert to dictionary."""
|
| 38 |
+
return {
|
| 39 |
+
"id": self.id,
|
| 40 |
+
"feedback_type": self.feedback_type,
|
| 41 |
+
"user_id": self.user_id,
|
| 42 |
+
"message": self.message,
|
| 43 |
+
"response": self.response,
|
| 44 |
+
"rating": self.rating,
|
| 45 |
+
"metadata": self.metadata,
|
| 46 |
+
"created_at": self.created_at.isoformat(),
|
| 47 |
+
"processed": self.processed,
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class FeedbackCollector:
|
| 52 |
+
"""Collects and manages user feedback."""
|
| 53 |
+
|
| 54 |
+
def __init__(
|
| 55 |
+
self,
|
| 56 |
+
storage_path: str = "data/feedback",
|
| 57 |
+
auto_save: bool = True,
|
| 58 |
+
):
|
| 59 |
+
"""
|
| 60 |
+
Initialize the feedback collector.
|
| 61 |
+
|
| 62 |
+
Args:
|
| 63 |
+
storage_path: Path to store feedback data
|
| 64 |
+
auto_save: Automatically save feedback to disk
|
| 65 |
+
"""
|
| 66 |
+
self.storage_path = Path(storage_path)
|
| 67 |
+
self.auto_save = auto_save
|
| 68 |
+
self.feedback_list: List[FeedbackEntry] = []
|
| 69 |
+
|
| 70 |
+
# Create storage directory if it doesn't exist
|
| 71 |
+
if auto_save:
|
| 72 |
+
self.storage_path.mkdir(parents=True, exist_ok=True)
|
| 73 |
+
|
| 74 |
+
def add_feedback(
|
| 75 |
+
self,
|
| 76 |
+
feedback_type: str,
|
| 77 |
+
message: str,
|
| 78 |
+
response: str,
|
| 79 |
+
user_id: Optional[str] = None,
|
| 80 |
+
rating: Optional[int] = None,
|
| 81 |
+
metadata: Optional[Dict[str, Any]] = None,
|
| 82 |
+
) -> str:
|
| 83 |
+
"""
|
| 84 |
+
Add a feedback entry.
|
| 85 |
+
|
| 86 |
+
Args:
|
| 87 |
+
feedback_type: Type of feedback
|
| 88 |
+
message: User's message
|
| 89 |
+
response: AI's response
|
| 90 |
+
user_id: Optional user ID
|
| 91 |
+
rating: Optional rating (1-5)
|
| 92 |
+
metadata: Additional metadata
|
| 93 |
+
|
| 94 |
+
Returns:
|
| 95 |
+
Feedback ID
|
| 96 |
+
"""
|
| 97 |
+
entry = FeedbackEntry(
|
| 98 |
+
feedback_type=feedback_type,
|
| 99 |
+
user_id=user_id,
|
| 100 |
+
message=message,
|
| 101 |
+
response=response,
|
| 102 |
+
rating=rating,
|
| 103 |
+
metadata=metadata,
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
self.feedback_list.append(entry)
|
| 107 |
+
|
| 108 |
+
if self.auto_save:
|
| 109 |
+
self._save_feedback(entry)
|
| 110 |
+
|
| 111 |
+
return entry.id
|
| 112 |
+
|
| 113 |
+
def add_thumbs_up(
|
| 114 |
+
self,
|
| 115 |
+
message: str,
|
| 116 |
+
response: str,
|
| 117 |
+
user_id: Optional[str] = None,
|
| 118 |
+
) -> str:
|
| 119 |
+
"""Add positive feedback."""
|
| 120 |
+
return self.add_feedback(
|
| 121 |
+
feedback_type="thumbs_up",
|
| 122 |
+
message=message,
|
| 123 |
+
response=response,
|
| 124 |
+
user_id=user_id,
|
| 125 |
+
rating=5,
|
| 126 |
+
)
|
| 127 |
+
|
| 128 |
+
def add_thumbs_down(
|
| 129 |
+
self,
|
| 130 |
+
message: str,
|
| 131 |
+
response: str,
|
| 132 |
+
user_id: Optional[str] = None,
|
| 133 |
+
reason: Optional[str] = None,
|
| 134 |
+
) -> str:
|
| 135 |
+
"""Add negative feedback."""
|
| 136 |
+
return self.add_feedback(
|
| 137 |
+
feedback_type="thumbs_down",
|
| 138 |
+
message=message,
|
| 139 |
+
response=response,
|
| 140 |
+
user_id=user_id,
|
| 141 |
+
rating=1,
|
| 142 |
+
metadata={"reason": reason} if reason else {},
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
def add_correction(
|
| 146 |
+
self,
|
| 147 |
+
message: str,
|
| 148 |
+
original_response: str,
|
| 149 |
+
corrected_response: str,
|
| 150 |
+
user_id: Optional[str] = None,
|
| 151 |
+
) -> str:
|
| 152 |
+
"""Add a correction."""
|
| 153 |
+
return self.add_feedback(
|
| 154 |
+
feedback_type="correction",
|
| 155 |
+
message=message,
|
| 156 |
+
response=original_response,
|
| 157 |
+
user_id=user_id,
|
| 158 |
+
metadata={"corrected_response": corrected_response},
|
| 159 |
+
)
|
| 160 |
+
|
| 161 |
+
def add_suggestion(
|
| 162 |
+
self,
|
| 163 |
+
message: str,
|
| 164 |
+
response: str,
|
| 165 |
+
suggestion: str,
|
| 166 |
+
user_id: Optional[str] = None,
|
| 167 |
+
) -> str:
|
| 168 |
+
"""Add a suggestion."""
|
| 169 |
+
return self.add_feedback(
|
| 170 |
+
feedback_type="suggestion",
|
| 171 |
+
message=message,
|
| 172 |
+
response=response,
|
| 173 |
+
user_id=user_id,
|
| 174 |
+
metadata={"suggestion": suggestion},
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
def get_feedback(
|
| 178 |
+
self,
|
| 179 |
+
feedback_id: str,
|
| 180 |
+
) -> Optional[FeedbackEntry]:
|
| 181 |
+
"""Get feedback by ID."""
|
| 182 |
+
for entry in self.feedback_list:
|
| 183 |
+
if entry.id == feedback_id:
|
| 184 |
+
return entry
|
| 185 |
+
return None
|
| 186 |
+
|
| 187 |
+
def get_all_feedback(
|
| 188 |
+
self,
|
| 189 |
+
feedback_type: Optional[str] = None,
|
| 190 |
+
unprocessed_only: bool = False,
|
| 191 |
+
) -> List[FeedbackEntry]:
|
| 192 |
+
"""Get all feedback entries."""
|
| 193 |
+
results = self.feedback_list
|
| 194 |
+
|
| 195 |
+
if feedback_type:
|
| 196 |
+
results = [f for f in results if f.feedback_type == feedback_type]
|
| 197 |
+
|
| 198 |
+
if unprocessed_only:
|
| 199 |
+
results = [f for f in results if not f.processed]
|
| 200 |
+
|
| 201 |
+
return results
|
| 202 |
+
|
| 203 |
+
def mark_processed(self, feedback_id: str) -> bool:
|
| 204 |
+
"""Mark feedback as processed."""
|
| 205 |
+
entry = self.get_feedback(feedback_id)
|
| 206 |
+
if entry:
|
| 207 |
+
entry.processed = True
|
| 208 |
+
return True
|
| 209 |
+
return False
|
| 210 |
+
|
| 211 |
+
def get_statistics(self) -> Dict[str, Any]:
|
| 212 |
+
"""Get feedback statistics."""
|
| 213 |
+
total = len(self.feedback_list)
|
| 214 |
+
if total == 0:
|
| 215 |
+
return {
|
| 216 |
+
"total": 0,
|
| 217 |
+
"by_type": {},
|
| 218 |
+
"average_rating": 0,
|
| 219 |
+
"processed_count": 0,
|
| 220 |
+
}
|
| 221 |
+
|
| 222 |
+
by_type: Dict[str, int] = {}
|
| 223 |
+
ratings = []
|
| 224 |
+
|
| 225 |
+
for entry in self.feedback_list:
|
| 226 |
+
by_type[entry.feedback_type] = by_type.get(entry.feedback_type, 0) + 1
|
| 227 |
+
if entry.rating is not None:
|
| 228 |
+
ratings.append(entry.rating)
|
| 229 |
+
|
| 230 |
+
avg_rating = sum(ratings) / len(ratings) if ratings else 0
|
| 231 |
+
processed = sum(1 for e in self.feedback_list if e.processed)
|
| 232 |
+
|
| 233 |
+
return {
|
| 234 |
+
"total": total,
|
| 235 |
+
"by_type": by_type,
|
| 236 |
+
"average_rating": avg_rating,
|
| 237 |
+
"processed_count": processed,
|
| 238 |
+
"unprocessed_count": total - processed,
|
| 239 |
+
}
|
| 240 |
+
|
| 241 |
+
def get_corrections_for_finetuning(self) -> List[Dict[str, Any]]:
|
| 242 |
+
"""Get corrections formatted for fine-tuning."""
|
| 243 |
+
corrections = self.get_all_feedback(feedback_type="correction")
|
| 244 |
+
|
| 245 |
+
return [
|
| 246 |
+
{
|
| 247 |
+
"instruction": entry.message,
|
| 248 |
+
"output": entry.metadata.get("corrected_response", entry.response),
|
| 249 |
+
}
|
| 250 |
+
for entry in corrections
|
| 251 |
+
]
|
| 252 |
+
|
| 253 |
+
def export_finetuning_data(
|
| 254 |
+
self,
|
| 255 |
+
filepath: str,
|
| 256 |
+
) -> None:
|
| 257 |
+
"""Export feedback as fine-tuning data."""
|
| 258 |
+
corrections = self.get_corrections_for_finetuning()
|
| 259 |
+
Path(filepath).write_text(json.dumps(corrections, indent=2))
|
| 260 |
+
|
| 261 |
+
def _save_feedback(self, entry: FeedbackEntry) -> None:
|
| 262 |
+
"""Save feedback to file."""
|
| 263 |
+
filepath = self.storage_path / f"{entry.id}.json"
|
| 264 |
+
filepath.write_text(json.dumps(entry.to_dict(), indent=2))
|
| 265 |
+
|
| 266 |
+
def load_feedback(self) -> None:
|
| 267 |
+
"""Load feedback from storage directory."""
|
| 268 |
+
if not self.storage_path.exists():
|
| 269 |
+
return
|
| 270 |
+
|
| 271 |
+
for filepath in self.storage_path.glob("*.json"):
|
| 272 |
+
try:
|
| 273 |
+
data = json.loads(filepath.read_text())
|
| 274 |
+
entry = FeedbackEntry(
|
| 275 |
+
feedback_type=data["feedback_type"],
|
| 276 |
+
user_id=data.get("user_id"),
|
| 277 |
+
message=data["message"],
|
| 278 |
+
response=data["response"],
|
| 279 |
+
rating=data.get("rating"),
|
| 280 |
+
metadata=data.get("metadata", {}),
|
| 281 |
+
)
|
| 282 |
+
entry.id = data["id"]
|
| 283 |
+
entry.processed = data.get("processed", False)
|
| 284 |
+
entry.created_at = datetime.fromisoformat(data["created_at"])
|
| 285 |
+
self.feedback_list.append(entry)
|
| 286 |
+
except Exception as e:
|
| 287 |
+
print(f"Error loading feedback from {filepath}: {e}")
|
| 288 |
+
|
| 289 |
+
def clear_old_feedback(self, days: int = 30) -> int:
|
| 290 |
+
"""Clear feedback older than specified days."""
|
| 291 |
+
cutoff = datetime.now() - timedelta(days=days)
|
| 292 |
+
original_count = len(self.feedback_list)
|
| 293 |
+
|
| 294 |
+
self.feedback_list = [
|
| 295 |
+
f for f in self.feedback_list
|
| 296 |
+
if f.created_at > cutoff
|
| 297 |
+
]
|
| 298 |
+
|
| 299 |
+
return original_count - len(self.feedback_list)
|
| 300 |
+
|
| 301 |
+
def __repr__(self) -> str:
|
| 302 |
+
stats = self.get_statistics()
|
| 303 |
+
return f"FeedbackCollector(total={stats['total']}, unprocessed={stats['unprocessed_count']})"
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
# Add missing import
|
| 307 |
+
from datetime import timedelta
|
src/enhancements/learning/performance.py
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Performance Monitoring System
|
| 3 |
+
|
| 4 |
+
Monitors and tracks model performance metrics.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import Dict, List, Optional, Any
|
| 8 |
+
from datetime import datetime, timedelta
|
| 9 |
+
from collections import defaultdict
|
| 10 |
+
import json
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class PerformanceMetric:
|
| 15 |
+
"""Represents a single performance metric."""
|
| 16 |
+
|
| 17 |
+
def __init__(
|
| 18 |
+
self,
|
| 19 |
+
metric_type: str,
|
| 20 |
+
value: float,
|
| 21 |
+
unit: str = "",
|
| 22 |
+
metadata: Optional[Dict[str, Any]] = None,
|
| 23 |
+
):
|
| 24 |
+
self.metric_type = metric_type
|
| 25 |
+
self.value = value
|
| 26 |
+
self.unit = unit
|
| 27 |
+
self.metadata = metadata or {}
|
| 28 |
+
self.timestamp = datetime.now()
|
| 29 |
+
|
| 30 |
+
def to_dict(self) -> Dict[str, Any]:
|
| 31 |
+
return {
|
| 32 |
+
"metric_type": self.metric_type,
|
| 33 |
+
"value": self.value,
|
| 34 |
+
"unit": self.unit,
|
| 35 |
+
"metadata": self.metadata,
|
| 36 |
+
"timestamp": self.timestamp.isoformat(),
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class PerformanceMonitor:
|
| 41 |
+
"""Monitors model performance over time."""
|
| 42 |
+
|
| 43 |
+
def __init__(
|
| 44 |
+
self,
|
| 45 |
+
storage_path: str = "data/performance",
|
| 46 |
+
):
|
| 47 |
+
"""
|
| 48 |
+
Initialize the performance monitor.
|
| 49 |
+
|
| 50 |
+
Args:
|
| 51 |
+
storage_path: Path to store performance data
|
| 52 |
+
"""
|
| 53 |
+
self.storage_path = Path(storage_path)
|
| 54 |
+
self.storage_path.mkdir(parents=True, exist_ok=True)
|
| 55 |
+
|
| 56 |
+
self.metrics: List[PerformanceMetric] = []
|
| 57 |
+
self._session_stats: Dict[str, Any] = {
|
| 58 |
+
"total_sessions": 0,
|
| 59 |
+
"total_messages": 0,
|
| 60 |
+
"total_conversations": 0,
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
def record_metric(
|
| 64 |
+
self,
|
| 65 |
+
metric_type: str,
|
| 66 |
+
value: float,
|
| 67 |
+
unit: str = "",
|
| 68 |
+
metadata: Optional[Dict[str, Any]] = None,
|
| 69 |
+
) -> None:
|
| 70 |
+
"""Record a performance metric."""
|
| 71 |
+
metric = PerformanceMetric(metric_type, value, unit, metadata)
|
| 72 |
+
self.metrics.append(metric)
|
| 73 |
+
|
| 74 |
+
def record_response_time(self, seconds: float) -> None:
|
| 75 |
+
"""Record response time."""
|
| 76 |
+
self.record_metric("response_time", seconds, "seconds")
|
| 77 |
+
|
| 78 |
+
def record_token_count(self, prompt_tokens: int, completion_tokens: int) -> None:
|
| 79 |
+
"""Record token count."""
|
| 80 |
+
self.record_metric(
|
| 81 |
+
"prompt_tokens",
|
| 82 |
+
prompt_tokens,
|
| 83 |
+
"tokens",
|
| 84 |
+
{"completion_tokens": completion_tokens},
|
| 85 |
+
)
|
| 86 |
+
|
| 87 |
+
def record_successful_interaction(self) -> None:
|
| 88 |
+
"""Record a successful interaction."""
|
| 89 |
+
self.record_metric("successful_interaction", 1, "count")
|
| 90 |
+
|
| 91 |
+
def record_failed_interaction(self, error_type: str) -> None:
|
| 92 |
+
"""Record a failed interaction."""
|
| 93 |
+
self.record_metric(
|
| 94 |
+
"failed_interaction",
|
| 95 |
+
1,
|
| 96 |
+
"count",
|
| 97 |
+
{"error_type": error_type},
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
def record_user_rating(self, rating: int) -> None:
|
| 101 |
+
"""Record user rating."""
|
| 102 |
+
self.record_metric("user_rating", rating, "stars")
|
| 103 |
+
|
| 104 |
+
def get_metrics(
|
| 105 |
+
self,
|
| 106 |
+
metric_type: Optional[str] = None,
|
| 107 |
+
since: Optional[datetime] = None,
|
| 108 |
+
limit: int = 100,
|
| 109 |
+
) -> List[PerformanceMetric]:
|
| 110 |
+
"""Get recorded metrics."""
|
| 111 |
+
results = self.metrics
|
| 112 |
+
|
| 113 |
+
if metric_type:
|
| 114 |
+
results = [m for m in results if m.metric_type == metric_type]
|
| 115 |
+
|
| 116 |
+
if since:
|
| 117 |
+
results = [m for m in results if m.timestamp >= since]
|
| 118 |
+
|
| 119 |
+
return results[-limit:]
|
| 120 |
+
|
| 121 |
+
def get_average_response_time(
|
| 122 |
+
self,
|
| 123 |
+
since: Optional[datetime] = None,
|
| 124 |
+
) -> float:
|
| 125 |
+
"""Get average response time."""
|
| 126 |
+
metrics = self.get_metrics("response_time", since=since)
|
| 127 |
+
if not metrics:
|
| 128 |
+
return 0.0
|
| 129 |
+
return sum(m.value for m in metrics) / len(metrics)
|
| 130 |
+
|
| 131 |
+
def get_success_rate(
|
| 132 |
+
self,
|
| 133 |
+
since: Optional[datetime] = None,
|
| 134 |
+
) -> float:
|
| 135 |
+
"""Get interaction success rate."""
|
| 136 |
+
successful = len(self.get_metrics("successful_interaction", since=since))
|
| 137 |
+
failed = len(self.get_metrics("failed_interaction", since=since))
|
| 138 |
+
|
| 139 |
+
total = successful + failed
|
| 140 |
+
if total == 0:
|
| 141 |
+
return 0.0
|
| 142 |
+
|
| 143 |
+
return successful / total
|
| 144 |
+
|
| 145 |
+
def get_average_rating(
|
| 146 |
+
self,
|
| 147 |
+
since: Optional[datetime] = None,
|
| 148 |
+
) -> float:
|
| 149 |
+
"""Get average user rating."""
|
| 150 |
+
ratings = self.get_metrics("user_rating", since=since)
|
| 151 |
+
if not ratings:
|
| 152 |
+
return 0.0
|
| 153 |
+
return sum(m.value for m in ratings) / len(ratings)
|
| 154 |
+
|
| 155 |
+
def get_summary(
|
| 156 |
+
self,
|
| 157 |
+
since: Optional[datetime] = None,
|
| 158 |
+
) -> Dict[str, Any]:
|
| 159 |
+
"""Get performance summary."""
|
| 160 |
+
since = since or (datetime.now() - timedelta(hours=24))
|
| 161 |
+
|
| 162 |
+
return {
|
| 163 |
+
"period": "last_24_hours" if since == datetime.now() - timedelta(hours=24) else "custom",
|
| 164 |
+
"average_response_time": self.get_average_response_time(since),
|
| 165 |
+
"success_rate": self.get_success_rate(since),
|
| 166 |
+
"average_rating": self.get_average_rating(since),
|
| 167 |
+
"total_interactions": len(self.get_metrics("successful_interaction", since=since)) +
|
| 168 |
+
len(self.get_metrics("failed_interaction", since=since)),
|
| 169 |
+
"total_tokens": sum(
|
| 170 |
+
m.value for m in self.get_metrics("prompt_tokens", since=since)
|
| 171 |
+
),
|
| 172 |
+
}
|
| 173 |
+
|
| 174 |
+
def increment_session_count(self) -> None:
|
| 175 |
+
"""Increment session count."""
|
| 176 |
+
self._session_stats["total_sessions"] += 1
|
| 177 |
+
|
| 178 |
+
def increment_message_count(self) -> None:
|
| 179 |
+
"""Increment message count."""
|
| 180 |
+
self._session_stats["total_messages"] += 1
|
| 181 |
+
|
| 182 |
+
def get_session_stats(self) -> Dict[str, Any]:
|
| 183 |
+
"""Get session statistics."""
|
| 184 |
+
return self._session_stats.copy()
|
| 185 |
+
|
| 186 |
+
def export_metrics(
|
| 187 |
+
self,
|
| 188 |
+
filepath: Optional[str] = None,
|
| 189 |
+
) -> str:
|
| 190 |
+
"""Export metrics to JSON file."""
|
| 191 |
+
filepath = filepath or str(self.storage_path / f"metrics_{datetime.now().strftime('%Y%m%d')}.json")
|
| 192 |
+
|
| 193 |
+
data = {
|
| 194 |
+
"exported_at": datetime.now().isoformat(),
|
| 195 |
+
"metrics": [m.to_dict() for m in self.metrics],
|
| 196 |
+
"session_stats": self._session_stats,
|
| 197 |
+
}
|
| 198 |
+
|
| 199 |
+
Path(filepath).write_text(json.dumps(data, indent=2))
|
| 200 |
+
return filepath
|
| 201 |
+
|
| 202 |
+
def load_metrics(
|
| 203 |
+
self,
|
| 204 |
+
filepath: str,
|
| 205 |
+
) -> None:
|
| 206 |
+
"""Load metrics from JSON file."""
|
| 207 |
+
data = json.loads(Path(filepath).read_text())
|
| 208 |
+
|
| 209 |
+
for metric_data in data.get("metrics", []):
|
| 210 |
+
metric = PerformanceMetric(
|
| 211 |
+
metric_type=metric_data["metric_type"],
|
| 212 |
+
value=metric_data["value"],
|
| 213 |
+
unit=metric_data.get("unit", ""),
|
| 214 |
+
metadata=metric_data.get("metadata", {}),
|
| 215 |
+
)
|
| 216 |
+
metric.timestamp = datetime.fromisoformat(metric_data["timestamp"])
|
| 217 |
+
self.metrics.append(metric)
|
| 218 |
+
|
| 219 |
+
if "session_stats" in data:
|
| 220 |
+
self._session_stats.update(data["session_stats"])
|
| 221 |
+
|
| 222 |
+
def clear_old_metrics(self, days: int = 30) -> int:
|
| 223 |
+
"""Clear metrics older than specified days."""
|
| 224 |
+
cutoff = datetime.now() - timedelta(days=days)
|
| 225 |
+
original_count = len(self.metrics)
|
| 226 |
+
|
| 227 |
+
self.metrics = [
|
| 228 |
+
m for m in self.metrics
|
| 229 |
+
if m.timestamp > cutoff
|
| 230 |
+
]
|
| 231 |
+
|
| 232 |
+
return original_count - len(self.metrics)
|
| 233 |
+
|
| 234 |
+
def get_trend(
|
| 235 |
+
self,
|
| 236 |
+
metric_type: str,
|
| 237 |
+
hours: int = 24,
|
| 238 |
+
) -> List[Dict[str, Any]]:
|
| 239 |
+
"""Get trend data for a metric."""
|
| 240 |
+
since = datetime.now() - timedelta(hours=hours)
|
| 241 |
+
metrics = self.get_metrics(metric_type, since=since)
|
| 242 |
+
|
| 243 |
+
# Group by hour
|
| 244 |
+
hourly_data: Dict[str, List[float]] = defaultdict(list)
|
| 245 |
+
for m in metrics:
|
| 246 |
+
hour_key = m.timestamp.strftime("%Y-%m-%d %H:00")
|
| 247 |
+
hourly_data[hour_key].append(m.value)
|
| 248 |
+
|
| 249 |
+
# Calculate hourly averages
|
| 250 |
+
trend = []
|
| 251 |
+
for hour, values in sorted(hourly_data.items()):
|
| 252 |
+
avg = sum(values) / len(values) if values else 0
|
| 253 |
+
trend.append({
|
| 254 |
+
"hour": hour,
|
| 255 |
+
"average": avg,
|
| 256 |
+
"count": len(values),
|
| 257 |
+
})
|
| 258 |
+
|
| 259 |
+
return trend
|
| 260 |
+
|
| 261 |
+
def __repr__(self) -> str:
|
| 262 |
+
return f"PerformanceMonitor(metrics={len(self.metrics)}, sessions={self._session_stats['total_sessions']})"
|
src/enhancements/nlp/__init__.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
NLP Enhancement Module
|
| 3 |
+
|
| 4 |
+
Provides:
|
| 5 |
+
- Contextual embeddings (BERT, RoBERTa)
|
| 6 |
+
- Entity recognition
|
| 7 |
+
- Intent detection
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from .contextual_embeddings import ContextualEmbedder
|
| 11 |
+
from .entity_recognition import EntityRecognizer
|
| 12 |
+
from .intent_detection import IntentDetector
|
| 13 |
+
|
| 14 |
+
__all__ = [
|
| 15 |
+
"ContextualEmbedder",
|
| 16 |
+
"EntityRecognizer",
|
| 17 |
+
"IntentDetector",
|
| 18 |
+
]
|
src/enhancements/nlp/contextual_embeddings.py
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Contextual Embeddings using BERT/RoBERTa
|
| 3 |
+
|
| 4 |
+
Provides contextual word embeddings for improved NLP understanding.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import List, Optional, Dict, Any
|
| 8 |
+
import numpy as np
|
| 9 |
+
from functools import lru_cache
|
| 10 |
+
import torch
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class ContextualEmbedder:
|
| 14 |
+
"""Generate contextual embeddings using BERT or similar models."""
|
| 15 |
+
|
| 16 |
+
def __init__(
|
| 17 |
+
self,
|
| 18 |
+
model_name: str = "bert-base-uncased",
|
| 19 |
+
device: Optional[str] = None,
|
| 20 |
+
cache_size: int = 1000,
|
| 21 |
+
):
|
| 22 |
+
"""
|
| 23 |
+
Initialize the contextual embedder.
|
| 24 |
+
|
| 25 |
+
Args:
|
| 26 |
+
model_name: Name of the BERT model to use
|
| 27 |
+
device: Device to run on ('cuda' or 'cpu')
|
| 28 |
+
cache_size: Maximum number of embeddings to cache
|
| 29 |
+
"""
|
| 30 |
+
self.model_name = model_name
|
| 31 |
+
self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
|
| 32 |
+
self.cache_size = cache_size
|
| 33 |
+
self._model = None
|
| 34 |
+
self._tokenizer = None
|
| 35 |
+
self._embedding_cache: Dict[str, np.ndarray] = {}
|
| 36 |
+
|
| 37 |
+
def _load_model(self):
|
| 38 |
+
"""Lazy load the BERT model."""
|
| 39 |
+
if self._model is None:
|
| 40 |
+
try:
|
| 41 |
+
from transformers import AutoModel, AutoTokenizer
|
| 42 |
+
self._tokenizer = AutoTokenizer.from_pretrained(self.model_name)
|
| 43 |
+
self._model = AutoModel.from_pretrained(self.model_name)
|
| 44 |
+
self._model.to(self.device)
|
| 45 |
+
self._model.eval()
|
| 46 |
+
except ImportError:
|
| 47 |
+
raise ImportError("transformers library required. Install: pip install transformers")
|
| 48 |
+
|
| 49 |
+
def get_embedding(self, text: str, layer: int = -1) -> np.ndarray:
|
| 50 |
+
"""
|
| 51 |
+
Get contextual embedding for a text.
|
| 52 |
+
|
| 53 |
+
Args:
|
| 54 |
+
text: Input text
|
| 55 |
+
layer: Which layer to extract (-1 for last hidden state)
|
| 56 |
+
|
| 57 |
+
Returns:
|
| 58 |
+
Embedding vector as numpy array
|
| 59 |
+
"""
|
| 60 |
+
# Check cache
|
| 61 |
+
cache_key = f"{text}:{layer}"
|
| 62 |
+
if cache_key in self._embedding_cache:
|
| 63 |
+
return self._embedding_cache[cache_key]
|
| 64 |
+
|
| 65 |
+
self._load_model()
|
| 66 |
+
|
| 67 |
+
with torch.no_grad():
|
| 68 |
+
inputs = self._tokenizer(
|
| 69 |
+
text,
|
| 70 |
+
return_tensors="pt",
|
| 71 |
+
padding=True,
|
| 72 |
+
truncation=True,
|
| 73 |
+
max_length=512,
|
| 74 |
+
).to(self.device)
|
| 75 |
+
|
| 76 |
+
outputs = self._model(**inputs)
|
| 77 |
+
# Get the mean of the last hidden state
|
| 78 |
+
embedding = outputs.last_hidden_state.mean(dim=1).cpu().numpy()[0]
|
| 79 |
+
|
| 80 |
+
# Cache the embedding
|
| 81 |
+
if len(self._embedding_cache) < self.cache_size:
|
| 82 |
+
self._embedding_cache[cache_key] = embedding
|
| 83 |
+
|
| 84 |
+
return embedding
|
| 85 |
+
|
| 86 |
+
def get_embeddings_batch(self, texts: List[str]) -> np.ndarray:
|
| 87 |
+
"""
|
| 88 |
+
Get embeddings for a batch of texts.
|
| 89 |
+
|
| 90 |
+
Args:
|
| 91 |
+
texts: List of input texts
|
| 92 |
+
|
| 93 |
+
Returns:
|
| 94 |
+
Array of embeddings (num_texts x embedding_dim)
|
| 95 |
+
"""
|
| 96 |
+
self._load_model()
|
| 97 |
+
|
| 98 |
+
with torch.no_grad():
|
| 99 |
+
inputs = self._tokenizer(
|
| 100 |
+
texts,
|
| 101 |
+
return_tensors="pt",
|
| 102 |
+
padding=True,
|
| 103 |
+
truncation=True,
|
| 104 |
+
max_length=512,
|
| 105 |
+
).to(self.device)
|
| 106 |
+
|
| 107 |
+
outputs = self._model(**inputs)
|
| 108 |
+
embeddings = outputs.last_hidden_state.mean(dim=1).cpu().numpy()
|
| 109 |
+
|
| 110 |
+
return embeddings
|
| 111 |
+
|
| 112 |
+
def get_sentence_embedding(self, text: str) -> np.ndarray:
|
| 113 |
+
"""
|
| 114 |
+
Get a sentence-level embedding using [CLS] token.
|
| 115 |
+
|
| 116 |
+
Args:
|
| 117 |
+
text: Input text
|
| 118 |
+
|
| 119 |
+
Returns:
|
| 120 |
+
Sentence embedding vector
|
| 121 |
+
"""
|
| 122 |
+
self._load_model()
|
| 123 |
+
|
| 124 |
+
with torch.no_grad():
|
| 125 |
+
inputs = self._tokenizer(
|
| 126 |
+
text,
|
| 127 |
+
return_tensors="pt",
|
| 128 |
+
padding=True,
|
| 129 |
+
truncation=True,
|
| 130 |
+
max_length=512,
|
| 131 |
+
).to(self.device)
|
| 132 |
+
|
| 133 |
+
outputs = self._model(**inputs)
|
| 134 |
+
# Use [CLS] token embedding (first token)
|
| 135 |
+
embedding = outputs.last_hidden_state[0, 0].cpu().numpy()
|
| 136 |
+
|
| 137 |
+
return embedding
|
| 138 |
+
|
| 139 |
+
def compute_similarity(
|
| 140 |
+
self,
|
| 141 |
+
text1: str,
|
| 142 |
+
text2: str,
|
| 143 |
+
method: str = "cosine",
|
| 144 |
+
) -> float:
|
| 145 |
+
"""
|
| 146 |
+
Compute similarity between two texts.
|
| 147 |
+
|
| 148 |
+
Args:
|
| 149 |
+
text1: First text
|
| 150 |
+
text2: Second text
|
| 151 |
+
method: Similarity method ('cosine' or 'dot')
|
| 152 |
+
|
| 153 |
+
Returns:
|
| 154 |
+
Similarity score
|
| 155 |
+
"""
|
| 156 |
+
emb1 = self.get_embedding(text1)
|
| 157 |
+
emb2 = self.get_embedding(text2)
|
| 158 |
+
|
| 159 |
+
if method == "cosine":
|
| 160 |
+
# Cosine similarity
|
| 161 |
+
dot = np.dot(emb1, emb2)
|
| 162 |
+
norm1 = np.linalg.norm(emb1)
|
| 163 |
+
norm2 = np.linalg.norm(emb2)
|
| 164 |
+
return dot / (norm1 * norm2)
|
| 165 |
+
else:
|
| 166 |
+
# Dot product
|
| 167 |
+
return np.dot(emb1, emb2)
|
| 168 |
+
|
| 169 |
+
def clear_cache(self) -> None:
|
| 170 |
+
"""Clear the embedding cache."""
|
| 171 |
+
self._embedding_cache.clear()
|
| 172 |
+
|
| 173 |
+
def __repr__(self) -> str:
|
| 174 |
+
return f"ContextualEmbedder(model='{self.model_name}', device='{self.device}')"
|
src/enhancements/nlp/entity_recognition.py
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Entity Recognition Module
|
| 3 |
+
|
| 4 |
+
Provides Named Entity Recognition (NER) for extracting entities from text.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import List, Dict, Optional, Any
|
| 8 |
+
import re
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class EntityRecognizer:
|
| 12 |
+
"""Extract named entities from text using pattern matching and NER."""
|
| 13 |
+
|
| 14 |
+
def __init__(
|
| 15 |
+
self,
|
| 16 |
+
use_transformers: bool = True,
|
| 17 |
+
model_name: str = "dslim/bert-base-NER",
|
| 18 |
+
):
|
| 19 |
+
"""
|
| 20 |
+
Initialize the entity recognizer.
|
| 21 |
+
|
| 22 |
+
Args:
|
| 23 |
+
use_transformers: Whether to use transformer-based NER
|
| 24 |
+
model_name: Name of the NER model (if using transformers)
|
| 25 |
+
"""
|
| 26 |
+
self.use_transformers = use_transformers and self._check_transformers()
|
| 27 |
+
self.model_name = model_name
|
| 28 |
+
self._model = None
|
| 29 |
+
self._tokenizer = None
|
| 30 |
+
|
| 31 |
+
# Define entity patterns for rule-based fallback
|
| 32 |
+
self._patterns = {
|
| 33 |
+
"EMAIL": r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b',
|
| 34 |
+
"URL": r'https?://[^\s]+',
|
| 35 |
+
"PHONE": r'\b\d{3}[-.]?\d{3}[-.]?\d{4}\b',
|
| 36 |
+
"IP_ADDRESS": r'\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b',
|
| 37 |
+
"DATE": r'\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b',
|
| 38 |
+
"TIME": r'\b\d{1,2}:\d{2}(?:\s*[AaPp][Mm])?\b',
|
| 39 |
+
"FILE_PATH": r'(?:/[a-zA-Z0-9_.-]+)+',
|
| 40 |
+
"CODE": r'`[^`]+`',
|
| 41 |
+
"QUOTED_STRING": r'"[^"]*"|\'[^\']*\'',
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
def _check_transformers(self) -> bool:
|
| 45 |
+
"""Check if transformers is available."""
|
| 46 |
+
try:
|
| 47 |
+
import transformers
|
| 48 |
+
return True
|
| 49 |
+
except ImportError:
|
| 50 |
+
return False
|
| 51 |
+
|
| 52 |
+
def _load_transformer_model(self):
|
| 53 |
+
"""Lazy load the transformer NER model."""
|
| 54 |
+
if self._model is None:
|
| 55 |
+
try:
|
| 56 |
+
from transformers import AutoTokenizer, AutoModelForTokenClassification
|
| 57 |
+
self._tokenizer = AutoTokenizer.from_pretrained(self.model_name)
|
| 58 |
+
self._model = AutoModelForTokenClassification.from_pretrained(self.model_name)
|
| 59 |
+
self._model.eval()
|
| 60 |
+
except Exception as e:
|
| 61 |
+
print(f"Warning: Could not load transformer NER model: {e}")
|
| 62 |
+
self.use_transformers = False
|
| 63 |
+
|
| 64 |
+
def recognize_entities(self, text: str) -> List[Dict[str, Any]]:
|
| 65 |
+
"""
|
| 66 |
+
Recognize entities in text.
|
| 67 |
+
|
| 68 |
+
Args:
|
| 69 |
+
text: Input text
|
| 70 |
+
|
| 71 |
+
Returns:
|
| 72 |
+
List of entity dictionaries with 'text', 'type', 'start', 'end'
|
| 73 |
+
"""
|
| 74 |
+
entities = []
|
| 75 |
+
|
| 76 |
+
# Use transformer-based NER if available
|
| 77 |
+
if self.use_transformers:
|
| 78 |
+
try:
|
| 79 |
+
entities.extend(self._recognize_transformers(text))
|
| 80 |
+
except Exception:
|
| 81 |
+
pass
|
| 82 |
+
|
| 83 |
+
# Add rule-based entities
|
| 84 |
+
entities.extend(self._recognize_patterns(text))
|
| 85 |
+
|
| 86 |
+
# Sort by position and remove overlaps
|
| 87 |
+
entities = self._resolve_overlaps(entities)
|
| 88 |
+
|
| 89 |
+
return entities
|
| 90 |
+
|
| 91 |
+
def _recognize_transformers(self, text: str) -> List[Dict[str, Any]]:
|
| 92 |
+
"""Use transformer model for NER."""
|
| 93 |
+
self._load_transformer_model()
|
| 94 |
+
|
| 95 |
+
from transformers import pipeline
|
| 96 |
+
from typing import List, Dict, Any
|
| 97 |
+
|
| 98 |
+
# Create pipeline if not exists
|
| 99 |
+
if not hasattr(self, "_ner_pipeline"):
|
| 100 |
+
self._ner_pipeline = pipeline(
|
| 101 |
+
"ner",
|
| 102 |
+
model=self._model,
|
| 103 |
+
tokenizer=self._tokenizer,
|
| 104 |
+
aggregation_strategy="simple",
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
results = self._ner_pipeline(text)
|
| 108 |
+
|
| 109 |
+
entities = []
|
| 110 |
+
for result in results:
|
| 111 |
+
# Map NER tags to simpler types
|
| 112 |
+
entity_type = self._map_ner_tag(result.get("entity_group", ""))
|
| 113 |
+
|
| 114 |
+
if entity_type:
|
| 115 |
+
entities.append({
|
| 116 |
+
"text": result["word"],
|
| 117 |
+
"type": entity_type,
|
| 118 |
+
"start": result.get("start", 0),
|
| 119 |
+
"end": result.get("end", 0),
|
| 120 |
+
"score": result.get("score", 1.0),
|
| 121 |
+
})
|
| 122 |
+
|
| 123 |
+
return entities
|
| 124 |
+
|
| 125 |
+
def _map_ner_tag(self, tag: str) -> Optional[str]:
|
| 126 |
+
"""Map NER tags to standard entity types."""
|
| 127 |
+
tag_mapping = {
|
| 128 |
+
"PER": "PERSON",
|
| 129 |
+
"ORG": "ORGANIZATION",
|
| 130 |
+
"LOC": "LOCATION",
|
| 131 |
+
"MISC": "MISC",
|
| 132 |
+
}
|
| 133 |
+
return tag_mapping.get(tag)
|
| 134 |
+
|
| 135 |
+
def _recognize_patterns(self, text: str) -> List[Dict[str, Any]]:
|
| 136 |
+
"""Use pattern matching for entity recognition."""
|
| 137 |
+
entities = []
|
| 138 |
+
|
| 139 |
+
for entity_type, pattern in self._patterns.items():
|
| 140 |
+
for match in re.finditer(pattern, text):
|
| 141 |
+
entities.append({
|
| 142 |
+
"text": match.group(),
|
| 143 |
+
"type": entity_type,
|
| 144 |
+
"start": match.start(),
|
| 145 |
+
"end": match.end(),
|
| 146 |
+
"score": 1.0,
|
| 147 |
+
})
|
| 148 |
+
|
| 149 |
+
return entities
|
| 150 |
+
|
| 151 |
+
def _resolve_overlaps(self, entities: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
| 152 |
+
"""Remove overlapping entities, keeping the higher confidence one."""
|
| 153 |
+
if not entities:
|
| 154 |
+
return []
|
| 155 |
+
|
| 156 |
+
# Sort by score (descending), then by length (descending)
|
| 157 |
+
entities = sorted(entities, key=lambda x: (-x.get("score", 1.0), -(x["end"] - x["start"])))
|
| 158 |
+
|
| 159 |
+
result = []
|
| 160 |
+
for entity in entities:
|
| 161 |
+
overlaps = False
|
| 162 |
+
for existing in result:
|
| 163 |
+
if self._overlaps(entity, existing):
|
| 164 |
+
overlaps = True
|
| 165 |
+
break
|
| 166 |
+
if not overlaps:
|
| 167 |
+
result.append(entity)
|
| 168 |
+
|
| 169 |
+
# Sort by position
|
| 170 |
+
result = sorted(result, key=lambda x: x["start"])
|
| 171 |
+
|
| 172 |
+
return result
|
| 173 |
+
|
| 174 |
+
def _overlaps(self, e1: Dict[str, Any], e2: Dict[str, Any]) -> bool:
|
| 175 |
+
"""Check if two entities overlap."""
|
| 176 |
+
return not (e1["end"] <= e2["start"] or e2["end"] <= e1["start"])
|
| 177 |
+
|
| 178 |
+
def extract_entities_by_type(
|
| 179 |
+
self,
|
| 180 |
+
text: str,
|
| 181 |
+
entity_type: str,
|
| 182 |
+
) -> List[str]:
|
| 183 |
+
"""
|
| 184 |
+
Extract all entities of a specific type.
|
| 185 |
+
|
| 186 |
+
Args:
|
| 187 |
+
text: Input text
|
| 188 |
+
entity_type: Type of entity to extract
|
| 189 |
+
|
| 190 |
+
Returns:
|
| 191 |
+
List of entity texts
|
| 192 |
+
"""
|
| 193 |
+
entities = self.recognize_entities(text)
|
| 194 |
+
return [e["text"] for e in entities if e["type"] == entity_type]
|
| 195 |
+
|
| 196 |
+
def get_entity_summary(self, text: str) -> Dict[str, int]:
|
| 197 |
+
"""
|
| 198 |
+
Get a summary of entity counts by type.
|
| 199 |
+
|
| 200 |
+
Args:
|
| 201 |
+
text: Input text
|
| 202 |
+
|
| 203 |
+
Returns:
|
| 204 |
+
Dictionary mapping entity type to count
|
| 205 |
+
"""
|
| 206 |
+
entities = self.recognize_entities(text)
|
| 207 |
+
summary: Dict[str, int] = {}
|
| 208 |
+
for entity in entities:
|
| 209 |
+
entity_type = entity["type"]
|
| 210 |
+
summary[entity_type] = summary.get(entity_type, 0) + 1
|
| 211 |
+
return summary
|
| 212 |
+
|
| 213 |
+
def __repr__(self) -> str:
|
| 214 |
+
return f"EntityRecognizer(use_transformers={self.use_transformers}, model='{self.model_name}')"
|
src/enhancements/nlp/intent_detection.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Intent Detection Module
|
| 3 |
+
|
| 4 |
+
Detects user intent from text for better conversational understanding.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import List, Dict, Optional, Any
|
| 8 |
+
import re
|
| 9 |
+
from collections import defaultdict
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class IntentDetector:
|
| 13 |
+
"""Detect user intent from natural language input."""
|
| 14 |
+
|
| 15 |
+
# Common intents with associated keywords and patterns
|
| 16 |
+
DEFAULT_INTENTS = {
|
| 17 |
+
"greeting": {
|
| 18 |
+
"keywords": ["hello", "hi", "hey", "greetings", "good morning", "good afternoon", "good evening"],
|
| 19 |
+
"patterns": [r"^hi(?: there)?", r"^hello", r"^hey", r"^good (?:morning|afternoon|evening)"],
|
| 20 |
+
},
|
| 21 |
+
"farewell": {
|
| 22 |
+
"keywords": ["bye", "goodbye", "see you", "later", "quit", "exit", "good night"],
|
| 23 |
+
"patterns": [r"^(?:good )?bye", r"see you", r"(?:good )?night", r"^later"],
|
| 24 |
+
},
|
| 25 |
+
"help": {
|
| 26 |
+
"keywords": ["help", "assist", "support", "can you", "how do i", "how to", "what can you do"],
|
| 27 |
+
"patterns": [r"^help", r"can you .*(?:help|do)", r"how (?:do|can) i", r"what can you"],
|
| 28 |
+
},
|
| 29 |
+
"question": {
|
| 30 |
+
"keywords": ["what", "how", "why", "when", "where", "who", "which", "?"],
|
| 31 |
+
"patterns": [r"^(?:what|how|why|when|where|who|which)", r"\?$"],
|
| 32 |
+
},
|
| 33 |
+
"code_request": {
|
| 34 |
+
"keywords": ["write", "code", "create", "implement", "function", "class", "script", "program"],
|
| 35 |
+
"patterns": [r"(?:write|create|implement|generate) .*(?:code|function|class|script)"],
|
| 36 |
+
},
|
| 37 |
+
"debug_request": {
|
| 38 |
+
"keywords": ["debug", "fix", "error", "bug", "issue", "problem", "broken", "not working", "crash"],
|
| 39 |
+
"patterns": [r"(?:debug|fix) .*(?:error|bug|issue)", r"(?:there(?:'s| is) an? )?error", r"(?:not working|crash|broken)"],
|
| 40 |
+
},
|
| 41 |
+
"refactor": {
|
| 42 |
+
"keywords": ["refactor", "improve", "optimize", "clean", "simplify", "restructure"],
|
| 43 |
+
"patterns": [r"(?:refactor|improve|optimize|clean(?: up)?)"],
|
| 44 |
+
},
|
| 45 |
+
"explain": {
|
| 46 |
+
"keywords": ["explain", "describe", "tell me about", "what is", "how does", "what does"],
|
| 47 |
+
"patterns": [r"(?:explain|describe|tell me about|what is|how does)"],
|
| 48 |
+
},
|
| 49 |
+
"search": {
|
| 50 |
+
"keywords": ["search", "find", "look up", "google", "web search"],
|
| 51 |
+
"patterns": [r"(?:search|find|look up)"],
|
| 52 |
+
},
|
| 53 |
+
"analysis": {
|
| 54 |
+
"keywords": ["analyze", "review", "check", "test", "evaluate", "compare"],
|
| 55 |
+
"patterns": [r"(?:analyze|review|check|test|evaluate|compare)"],
|
| 56 |
+
},
|
| 57 |
+
"tool_use": {
|
| 58 |
+
"keywords": ["use tool", "run command", "execute", "shell", "bash"],
|
| 59 |
+
"patterns": [r"(?:run|execute) .*(?:command|shell)", r"bash", r"shell"],
|
| 60 |
+
},
|
| 61 |
+
"learning": {
|
| 62 |
+
"keywords": ["learn", "teach me", "train", "understand", "study"],
|
| 63 |
+
"patterns": [r"(?:learn|teach me|train)"],
|
| 64 |
+
},
|
| 65 |
+
"feedback": {
|
| 66 |
+
"keywords": ["feedback", "rating", "opinion", "suggest", "improve", "better"],
|
| 67 |
+
"patterns": [r"(?:feedback|rating|opinion|suggest|improve)"],
|
| 68 |
+
},
|
| 69 |
+
"clarification": {
|
| 70 |
+
"keywords": ["clarify", "repeat", "restate", "what do you mean", "explain more"],
|
| 71 |
+
"patterns": [r"(?:clarify|repeat|what do you mean)", r"(?:could you|can you) (?:repeat|clarify)"],
|
| 72 |
+
},
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
def __init__(
|
| 76 |
+
self,
|
| 77 |
+
custom_intents: Optional[Dict[str, Dict[str, List[str]]]] = None,
|
| 78 |
+
confidence_threshold: float = 0.3,
|
| 79 |
+
):
|
| 80 |
+
"""
|
| 81 |
+
Initialize the intent detector.
|
| 82 |
+
|
| 83 |
+
Args:
|
| 84 |
+
custom_intents: Custom intent definitions
|
| 85 |
+
confidence_threshold: Minimum confidence for intent detection
|
| 86 |
+
"""
|
| 87 |
+
self.intents = self.DEFAULT_INTENTS.copy()
|
| 88 |
+
if custom_intents:
|
| 89 |
+
self.intents.update(custom_intents)
|
| 90 |
+
self.confidence_threshold = confidence_threshold
|
| 91 |
+
self._keyword_cache: Dict[str, float] = {}
|
| 92 |
+
|
| 93 |
+
def detect_intent(self, text: str) -> Dict[str, Any]:
|
| 94 |
+
"""
|
| 95 |
+
Detect the primary intent from text.
|
| 96 |
+
|
| 97 |
+
Args:
|
| 98 |
+
text: Input text
|
| 99 |
+
|
| 100 |
+
Returns:
|
| 101 |
+
Dictionary with 'intent', 'confidence', and 'alternatives'
|
| 102 |
+
"""
|
| 103 |
+
text_lower = text.lower().strip()
|
| 104 |
+
|
| 105 |
+
intent_scores = defaultdict(float)
|
| 106 |
+
|
| 107 |
+
# Check each intent
|
| 108 |
+
for intent_name, intent_config in self.intents.items():
|
| 109 |
+
# Check keywords
|
| 110 |
+
for keyword in intent_config.get("keywords", []):
|
| 111 |
+
if keyword.lower() in text_lower:
|
| 112 |
+
intent_scores[intent_name] += 1.0
|
| 113 |
+
|
| 114 |
+
# Check patterns
|
| 115 |
+
for pattern in intent_config.get("patterns", []):
|
| 116 |
+
if re.search(pattern, text_lower, re.IGNORECASE):
|
| 117 |
+
intent_scores[intent_name] += 1.5
|
| 118 |
+
|
| 119 |
+
# Normalize scores
|
| 120 |
+
if intent_scores:
|
| 121 |
+
max_score = max(intent_scores.values())
|
| 122 |
+
if max_score > 0:
|
| 123 |
+
for intent in intent_scores:
|
| 124 |
+
intent_scores[intent] /= max_score
|
| 125 |
+
|
| 126 |
+
# Sort by score
|
| 127 |
+
sorted_intents = sorted(
|
| 128 |
+
intent_scores.items(),
|
| 129 |
+
key=lambda x: -x[1]
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
if not sorted_intents or sorted_intents[0][1] < self.confidence_threshold:
|
| 133 |
+
return {
|
| 134 |
+
"intent": "general",
|
| 135 |
+
"confidence": 1.0,
|
| 136 |
+
"alternatives": [],
|
| 137 |
+
}
|
| 138 |
+
|
| 139 |
+
primary_intent = sorted_intents[0][0]
|
| 140 |
+
alternatives = [
|
| 141 |
+
{"intent": intent, "confidence": score}
|
| 142 |
+
for intent, score in sorted_intents[1:4]
|
| 143 |
+
if score >= self.confidence_threshold
|
| 144 |
+
]
|
| 145 |
+
|
| 146 |
+
return {
|
| 147 |
+
"intent": primary_intent,
|
| 148 |
+
"confidence": sorted_intents[0][1],
|
| 149 |
+
"alternatives": alternatives,
|
| 150 |
+
}
|
| 151 |
+
|
| 152 |
+
def detect_multiple_intents(self, text: str) -> List[Dict[str, Any]]:
|
| 153 |
+
"""
|
| 154 |
+
Detect multiple intents from text.
|
| 155 |
+
|
| 156 |
+
Args:
|
| 157 |
+
text: Input text
|
| 158 |
+
|
| 159 |
+
Returns:
|
| 160 |
+
List of intent dictionaries with scores
|
| 161 |
+
"""
|
| 162 |
+
text_lower = text.lower().strip()
|
| 163 |
+
intent_scores = defaultdict(float)
|
| 164 |
+
|
| 165 |
+
for intent_name, intent_config in self.intents.items():
|
| 166 |
+
for keyword in intent_config.get("keywords", []):
|
| 167 |
+
if keyword.lower() in text_lower:
|
| 168 |
+
intent_scores[intent_name] += 1.0
|
| 169 |
+
|
| 170 |
+
for pattern in intent_config.get("patterns", []):
|
| 171 |
+
if re.search(pattern, text_lower, re.IGNORECASE):
|
| 172 |
+
intent_scores[intent_name] += 1.5
|
| 173 |
+
|
| 174 |
+
# Return all intents above threshold
|
| 175 |
+
results = [
|
| 176 |
+
{"intent": intent, "confidence": score}
|
| 177 |
+
for intent, score in intent_scores.items()
|
| 178 |
+
if score >= self.confidence_threshold
|
| 179 |
+
]
|
| 180 |
+
|
| 181 |
+
if not results:
|
| 182 |
+
return [{"intent": "general", "confidence": 1.0}]
|
| 183 |
+
|
| 184 |
+
return sorted(results, key=lambda x: -x["confidence"])
|
| 185 |
+
|
| 186 |
+
def add_intent(
|
| 187 |
+
self,
|
| 188 |
+
intent_name: str,
|
| 189 |
+
keywords: List[str],
|
| 190 |
+
patterns: Optional[List[str]] = None,
|
| 191 |
+
) -> None:
|
| 192 |
+
"""
|
| 193 |
+
Add a custom intent.
|
| 194 |
+
|
| 195 |
+
Args:
|
| 196 |
+
intent_name: Name of the intent
|
| 197 |
+
keywords: List of keywords
|
| 198 |
+
patterns: List of regex patterns
|
| 199 |
+
"""
|
| 200 |
+
self.intents[intent_name] = {
|
| 201 |
+
"keywords": keywords,
|
| 202 |
+
"patterns": patterns or [],
|
| 203 |
+
}
|
| 204 |
+
|
| 205 |
+
def get_intent_description(self, intent: str) -> str:
|
| 206 |
+
"""Get a description of what an intent means."""
|
| 207 |
+
descriptions = {
|
| 208 |
+
"greeting": "User is greeting the assistant",
|
| 209 |
+
"farewell": "User is saying goodbye",
|
| 210 |
+
"help": "User is asking for help or assistance",
|
| 211 |
+
"question": "User is asking a question",
|
| 212 |
+
"code_request": "User wants code to be written",
|
| 213 |
+
"debug_request": "User needs help debugging an issue",
|
| 214 |
+
"refactor": "User wants code to be improved or refactored",
|
| 215 |
+
"explain": "User wants an explanation of something",
|
| 216 |
+
"search": "User wants to search for information",
|
| 217 |
+
"analysis": "User wants code or content analyzed",
|
| 218 |
+
"tool_use": "User wants to execute a command or tool",
|
| 219 |
+
"learning": "User wants to learn something",
|
| 220 |
+
"feedback": "User is providing feedback",
|
| 221 |
+
"clarification": "User wants clarification on something",
|
| 222 |
+
"general": "General conversational input",
|
| 223 |
+
}
|
| 224 |
+
return descriptions.get(intent, f"Intent: {intent}")
|
| 225 |
+
|
| 226 |
+
def __repr__(self) -> str:
|
| 227 |
+
return f"IntentDetector(num_intents={len(self.intents)}, threshold={self.confidence_threshold})"
|
src/enhancements/technical/__init__.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Technical Capabilities Module
|
| 3 |
+
|
| 4 |
+
Provides advanced technical capabilities:
|
| 5 |
+
- Cloud/DevOps tools
|
| 6 |
+
- Code analysis and debugging
|
| 7 |
+
- Security scanning
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from .devops import DevOpsTools
|
| 11 |
+
from .code_analysis import CodeAnalyzer
|
| 12 |
+
from .debugging import DebuggingAssistant
|
| 13 |
+
|
| 14 |
+
__all__ = [
|
| 15 |
+
"DevOpsTools",
|
| 16 |
+
"CodeAnalyzer",
|
| 17 |
+
"DebuggingAssistant",
|
| 18 |
+
]
|
src/enhancements/technical/code_analysis.py
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Code Analysis Module
|
| 3 |
+
|
| 4 |
+
Provides static code analysis and quality checking.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import Dict, List, Optional, Any, Tuple
|
| 8 |
+
import re
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class CodeAnalyzer:
|
| 12 |
+
"""Analyze code for quality, complexity, and issues."""
|
| 13 |
+
|
| 14 |
+
def __init__(self):
|
| 15 |
+
"""Initialize code analyzer."""
|
| 16 |
+
pass
|
| 17 |
+
|
| 18 |
+
def analyze_complexity(self, code: str) -> Dict[str, Any]:
|
| 19 |
+
"""
|
| 20 |
+
Analyze code complexity.
|
| 21 |
+
|
| 22 |
+
Returns:
|
| 23 |
+
Dictionary with complexity metrics
|
| 24 |
+
"""
|
| 25 |
+
lines = code.split('\n')
|
| 26 |
+
|
| 27 |
+
# Count functions/methods
|
| 28 |
+
functions = re.findall(r'def\s+(\w+)', code)
|
| 29 |
+
methods = re.findall(r'def\s+(\w+)\(self', code)
|
| 30 |
+
classes = re.findall(r'class\s+(\w+)', code)
|
| 31 |
+
|
| 32 |
+
# Count control structures
|
| 33 |
+
if_statements = len(re.findall(r'\bif\s+', code))
|
| 34 |
+
for_loops = len(re.findall(r'\bfor\s+', code))
|
| 35 |
+
while_loops = len(re.findall(r'\bwhile\s+', code))
|
| 36 |
+
try_blocks = len(re.findall(r'\btry\s+', code))
|
| 37 |
+
|
| 38 |
+
# Cyclomatic complexity approximation
|
| 39 |
+
complexity = 1 + if_statements + for_loops + while_loops + try_blocks
|
| 40 |
+
|
| 41 |
+
# Count lines of code
|
| 42 |
+
loc = len([l for l in lines if l.strip() and not l.strip().startswith('#')])
|
| 43 |
+
|
| 44 |
+
return {
|
| 45 |
+
"lines_of_code": loc,
|
| 46 |
+
"total_lines": len(lines),
|
| 47 |
+
"functions": len(functions),
|
| 48 |
+
"methods": len(methods),
|
| 49 |
+
"classes": len(classes),
|
| 50 |
+
"cyclomatic_complexity": complexity,
|
| 51 |
+
"if_statements": if_statements,
|
| 52 |
+
"for_loops": for_loops,
|
| 53 |
+
"while_loops": while_loops,
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
def find_issues(self, code: str, language: str = "python") -> List[Dict[str, Any]]:
|
| 57 |
+
"""
|
| 58 |
+
Find potential issues in code.
|
| 59 |
+
|
| 60 |
+
Args:
|
| 61 |
+
code: Source code
|
| 62 |
+
language: Programming language
|
| 63 |
+
|
| 64 |
+
Returns:
|
| 65 |
+
List of issues found
|
| 66 |
+
"""
|
| 67 |
+
issues = []
|
| 68 |
+
|
| 69 |
+
# Common issues for Python
|
| 70 |
+
if language == "python":
|
| 71 |
+
issues.extend(self._check_python_issues(code))
|
| 72 |
+
|
| 73 |
+
return issues
|
| 74 |
+
|
| 75 |
+
def _check_python_issues(self, code: str) -> List[Dict[str, Any]]:
|
| 76 |
+
"""Check for Python-specific issues."""
|
| 77 |
+
issues = []
|
| 78 |
+
|
| 79 |
+
# Check for TODO/FIXME
|
| 80 |
+
for i, line in enumerate(code.split('\n'), 1):
|
| 81 |
+
if 'TODO' in line.upper() or 'FIXME' in line.upper():
|
| 82 |
+
issues.append({
|
| 83 |
+
"type": "todo",
|
| 84 |
+
"severity": "info",
|
| 85 |
+
"line": i,
|
| 86 |
+
"message": "TODO/FIXME comment found",
|
| 87 |
+
})
|
| 88 |
+
|
| 89 |
+
# Check for empty except
|
| 90 |
+
if re.search(r'except\s*:\s*\n\s*pass', code):
|
| 91 |
+
issues.append({
|
| 92 |
+
"type": "empty_except",
|
| 93 |
+
"severity": "warning",
|
| 94 |
+
"message": "Empty except block - errors are silently ignored",
|
| 95 |
+
})
|
| 96 |
+
|
| 97 |
+
# Check for hardcoded credentials
|
| 98 |
+
if re.search(r'password\s*=\s*["\']', code, re.IGNORECASE):
|
| 99 |
+
issues.append({
|
| 100 |
+
"type": "hardcoded_credentials",
|
| 101 |
+
"severity": "error",
|
| 102 |
+
"message": "Potential hardcoded password found",
|
| 103 |
+
})
|
| 104 |
+
|
| 105 |
+
# Check for print statements (debugging)
|
| 106 |
+
if re.search(r'\bprint\s*\(', code) and not code.startswith('# debug'):
|
| 107 |
+
issues.append({
|
| 108 |
+
"type": "debug_print",
|
| 109 |
+
"severity": "info",
|
| 110 |
+
"message": "Print statement found - may need removal",
|
| 111 |
+
})
|
| 112 |
+
|
| 113 |
+
# Check for long lines
|
| 114 |
+
for i, line in enumerate(code.split('\n'), 1):
|
| 115 |
+
if len(line) > 120:
|
| 116 |
+
issues.append({
|
| 117 |
+
"type": "long_line",
|
| 118 |
+
"severity": "info",
|
| 119 |
+
"line": i,
|
| 120 |
+
"message": f"Line exceeds 120 characters ({len(line)} chars)",
|
| 121 |
+
})
|
| 122 |
+
|
| 123 |
+
# Check for global variables
|
| 124 |
+
if re.search(r'^([A-Z_][A-Z0-9_]*)\s*=\s*', code, re.MULTILINE):
|
| 125 |
+
issues.append({
|
| 126 |
+
"type": "global_variable",
|
| 127 |
+
"severity": "info",
|
| 128 |
+
"message": "Potential global variable found",
|
| 129 |
+
})
|
| 130 |
+
|
| 131 |
+
return issues
|
| 132 |
+
|
| 133 |
+
def suggest_improvements(self, code: str, language: str = "python") -> List[str]:
|
| 134 |
+
"""Suggest code improvements."""
|
| 135 |
+
suggestions = []
|
| 136 |
+
complexity = self.analyze_complexity(code)
|
| 137 |
+
|
| 138 |
+
# Complexity suggestions
|
| 139 |
+
if complexity["cyclomatic_complexity"] > 10:
|
| 140 |
+
suggestions.append("High cyclomatic complexity - consider breaking into smaller functions")
|
| 141 |
+
|
| 142 |
+
if complexity["lines_of_code"] > 500:
|
| 143 |
+
suggestions.append("Large function - consider splitting into smaller modules")
|
| 144 |
+
|
| 145 |
+
# Pattern suggestions
|
| 146 |
+
if "except:" in code:
|
| 147 |
+
suggestions.append("Use specific exception types instead of bare except")
|
| 148 |
+
|
| 149 |
+
if "print(" in code:
|
| 150 |
+
suggestions.append("Use logging instead of print statements for production code")
|
| 151 |
+
|
| 152 |
+
if "==" in code and "None" in code:
|
| 153 |
+
suggestions.append("Use 'is None' instead of '== None'")
|
| 154 |
+
|
| 155 |
+
if re.search(r'for\s+\w+\s+in\s+range\s*\(\s*len\s*\(', code):
|
| 156 |
+
suggestions.append("Use enumerate() instead of range(len())")
|
| 157 |
+
|
| 158 |
+
return suggestions
|
| 159 |
+
|
| 160 |
+
def detect_language(self, code: str) -> str:
|
| 161 |
+
"""Detect programming language from code."""
|
| 162 |
+
# Python indicators
|
| 163 |
+
if re.search(r'\bdef\s+\w+\s*\(', code) or re.search(r'\bimport\s+\w+', code):
|
| 164 |
+
return "python"
|
| 165 |
+
|
| 166 |
+
# JavaScript/TypeScript
|
| 167 |
+
if re.search(r'\bfunction\s+\w+\s*\(', code) or re.search(r'const\s+\w+\s*=', code):
|
| 168 |
+
return "javascript"
|
| 169 |
+
|
| 170 |
+
# Java
|
| 171 |
+
if re.search(r'\bpublic\s+class\s+\w+', code) or re.search(r'\bSystem\.out\.print', code):
|
| 172 |
+
return "java"
|
| 173 |
+
|
| 174 |
+
# Go
|
| 175 |
+
if re.search(r'\bpackage\s+main', code) or re.search(r'\bfunc\s+\w+\s*\(', code):
|
| 176 |
+
return "go"
|
| 177 |
+
|
| 178 |
+
# Rust
|
| 179 |
+
if re.search(r'\bfn\s+\w+\s*\(', code) or re.search(r'\blet\s+mut\s+', code):
|
| 180 |
+
return "rust"
|
| 181 |
+
|
| 182 |
+
# C/C++
|
| 183 |
+
if re.search(r'#include\s*<', code) or re.search(r'\bint\s+main\s*\(', code):
|
| 184 |
+
return "c"
|
| 185 |
+
|
| 186 |
+
return "unknown"
|
| 187 |
+
|
| 188 |
+
def calculate_maintainability_index(self, code: str) -> float:
|
| 189 |
+
"""Calculate maintainability index (0-100)."""
|
| 190 |
+
complexity = self.analyze_complexity(code)
|
| 191 |
+
loc = complexity["lines_of_code"]
|
| 192 |
+
|
| 193 |
+
if loc == 0:
|
| 194 |
+
return 100.0
|
| 195 |
+
|
| 196 |
+
# Simplified maintainability index
|
| 197 |
+
# Based on lines of code and complexity
|
| 198 |
+
base = 100
|
| 199 |
+
loc_penalty = min(loc / 100, 1) * 20
|
| 200 |
+
complexity_penalty = min(complexity["cyclomatic_complexity"] / 20, 1) * 30
|
| 201 |
+
|
| 202 |
+
index = base - loc_penalty - complexity_penalty
|
| 203 |
+
return max(0, min(100, index))
|
| 204 |
+
|
| 205 |
+
def get_code_summary(self, code: str) -> Dict[str, Any]:
|
| 206 |
+
"""Get comprehensive code summary."""
|
| 207 |
+
language = self.detect_language(code)
|
| 208 |
+
complexity = self.analyze_complexity(code)
|
| 209 |
+
issues = self.find_issues(code, language)
|
| 210 |
+
suggestions = self.suggest_improvements(code, language)
|
| 211 |
+
maintainability = self.calculate_maintainability_index(code)
|
| 212 |
+
|
| 213 |
+
return {
|
| 214 |
+
"language": language,
|
| 215 |
+
"complexity": complexity,
|
| 216 |
+
"issues": issues,
|
| 217 |
+
"issue_count": len(issues),
|
| 218 |
+
"suggestions": suggestions,
|
| 219 |
+
"maintainability_index": maintainability,
|
| 220 |
+
}
|
| 221 |
+
|
| 222 |
+
def __repr__(self) -> str:
|
| 223 |
+
return "CodeAnalyzer()"
|
src/enhancements/technical/debugging.py
ADDED
|
@@ -0,0 +1,357 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Debugging Assistant Module
|
| 3 |
+
|
| 4 |
+
Provides debugging assistance and error analysis.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import Dict, List, Optional, Any, Tuple
|
| 8 |
+
import re
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class DebuggingAssistant:
|
| 12 |
+
"""Helps debug code and analyze errors."""
|
| 13 |
+
|
| 14 |
+
# Common error patterns and their explanations
|
| 15 |
+
ERROR_PATTERNS = {
|
| 16 |
+
"python": {
|
| 17 |
+
"SyntaxError": {
|
| 18 |
+
"description": "Python syntax is invalid",
|
| 19 |
+
"common_causes": [
|
| 20 |
+
"Missing colon after if/for/while/function definitions",
|
| 21 |
+
"Mismatched parentheses or brackets",
|
| 22 |
+
"Incorrect indentation",
|
| 23 |
+
"Using Python 2 syntax in Python 3",
|
| 24 |
+
],
|
| 25 |
+
},
|
| 26 |
+
"NameError": {
|
| 27 |
+
"description": "A variable or function name is not defined",
|
| 28 |
+
"common_causes": [
|
| 29 |
+
"Typo in variable name",
|
| 30 |
+
"Variable used before assignment",
|
| 31 |
+
"Import statement missing",
|
| 32 |
+
"Scope issue - variable not accessible",
|
| 33 |
+
],
|
| 34 |
+
},
|
| 35 |
+
"TypeError": {
|
| 36 |
+
"description": "Operation applied to wrong type",
|
| 37 |
+
"common_causes": [
|
| 38 |
+
"Trying to concatenate incompatible types",
|
| 39 |
+
"Calling a non-callable as a function",
|
| 40 |
+
"Passing wrong number of arguments",
|
| 41 |
+
"Operation not supported for type",
|
| 42 |
+
],
|
| 43 |
+
},
|
| 44 |
+
"IndexError": {
|
| 45 |
+
"description": "List index out of range",
|
| 46 |
+
"common_causes": [
|
| 47 |
+
"Accessing index that doesn't exist",
|
| 48 |
+
"Empty list access",
|
| 49 |
+
"Off-by-one error",
|
| 50 |
+
],
|
| 51 |
+
},
|
| 52 |
+
"KeyError": {
|
| 53 |
+
"description": "Dictionary key not found",
|
| 54 |
+
"common_causes": [
|
| 55 |
+
"Accessing non-existent key",
|
| 56 |
+
"Typo in key name",
|
| 57 |
+
"Case sensitivity issue",
|
| 58 |
+
],
|
| 59 |
+
},
|
| 60 |
+
"AttributeError": {
|
| 61 |
+
"description": "Object has no attribute",
|
| 62 |
+
"common_causes": [
|
| 63 |
+
"Typo in attribute name",
|
| 64 |
+
"Object is None when trying to access attribute",
|
| 65 |
+
"Wrong type for this operation",
|
| 66 |
+
],
|
| 67 |
+
},
|
| 68 |
+
"ImportError": {
|
| 69 |
+
"description": "Cannot import module",
|
| 70 |
+
"common_causes": [
|
| 71 |
+
"Module not installed",
|
| 72 |
+
"Circular import",
|
| 73 |
+
"Module name typo",
|
| 74 |
+
"Missing __init__.py in package",
|
| 75 |
+
],
|
| 76 |
+
},
|
| 77 |
+
"ZeroDivisionError": {
|
| 78 |
+
"description": "Division by zero",
|
| 79 |
+
"common_causes": [
|
| 80 |
+
"Dividing by variable that could be zero",
|
| 81 |
+
"Modulo by zero",
|
| 82 |
+
],
|
| 83 |
+
},
|
| 84 |
+
"ValueError": {
|
| 85 |
+
"description": "Value is inappropriate",
|
| 86 |
+
"common_causes": [
|
| 87 |
+
"Invalid argument to function",
|
| 88 |
+
"Conversion failed (e.g., int('abc'))",
|
| 89 |
+
"Empty sequence in function expecting content",
|
| 90 |
+
],
|
| 91 |
+
},
|
| 92 |
+
"IndentationError": {
|
| 93 |
+
"description": "Incorrect indentation",
|
| 94 |
+
"common_causes": [
|
| 95 |
+
"Mixing tabs and spaces",
|
| 96 |
+
"Inconsistent indentation levels",
|
| 97 |
+
"Code not aligned properly",
|
| 98 |
+
],
|
| 99 |
+
},
|
| 100 |
+
},
|
| 101 |
+
"javascript": {
|
| 102 |
+
"ReferenceError": {
|
| 103 |
+
"description": "Variable not defined",
|
| 104 |
+
"common_causes": [
|
| 105 |
+
"Typo in variable name",
|
| 106 |
+
"Using let/const before declaration",
|
| 107 |
+
],
|
| 108 |
+
},
|
| 109 |
+
"TypeError": {
|
| 110 |
+
"description": "Type operation failed",
|
| 111 |
+
"common_causes": [
|
| 112 |
+
"Calling non-function",
|
| 113 |
+
"Cannot read property of undefined/null",
|
| 114 |
+
],
|
| 115 |
+
},
|
| 116 |
+
"SyntaxError": {
|
| 117 |
+
"description": "Invalid syntax",
|
| 118 |
+
"common_causes": [
|
| 119 |
+
"Missing closing bracket/parenthesis",
|
| 120 |
+
"Invalid string quotes",
|
| 121 |
+
],
|
| 122 |
+
},
|
| 123 |
+
},
|
| 124 |
+
}
|
| 125 |
+
|
| 126 |
+
def __init__(self):
|
| 127 |
+
"""Initialize debugging assistant."""
|
| 128 |
+
pass
|
| 129 |
+
|
| 130 |
+
def analyze_error(
|
| 131 |
+
self,
|
| 132 |
+
error_message: str,
|
| 133 |
+
language: str = "python",
|
| 134 |
+
) -> Dict[str, Any]:
|
| 135 |
+
"""
|
| 136 |
+
Analyze an error message and provide debugging help.
|
| 137 |
+
|
| 138 |
+
Args:
|
| 139 |
+
error_message: The error message
|
| 140 |
+
language: Programming language
|
| 141 |
+
|
| 142 |
+
Returns:
|
| 143 |
+
Dictionary with error analysis and suggestions
|
| 144 |
+
"""
|
| 145 |
+
# Extract error type
|
| 146 |
+
error_type = self._extract_error_type(error_message, language)
|
| 147 |
+
|
| 148 |
+
# Get error info
|
| 149 |
+
error_info = self.ERROR_PATTERNS.get(language, {}).get(error_type, {
|
| 150 |
+
"description": "Unknown error",
|
| 151 |
+
"common_causes": ["Check the error message for clues"],
|
| 152 |
+
})
|
| 153 |
+
|
| 154 |
+
# Generate debugging steps
|
| 155 |
+
steps = self._generate_debug_steps(error_type, error_message, language)
|
| 156 |
+
|
| 157 |
+
# Suggest fixes
|
| 158 |
+
fixes = self._suggest_fixes(error_type, error_message, language)
|
| 159 |
+
|
| 160 |
+
return {
|
| 161 |
+
"error_type": error_type,
|
| 162 |
+
"description": error_info["description"],
|
| 163 |
+
"common_causes": error_info["common_causes"],
|
| 164 |
+
"debug_steps": steps,
|
| 165 |
+
"suggested_fixes": fixes,
|
| 166 |
+
}
|
| 167 |
+
|
| 168 |
+
def _extract_error_type(self, error_message: str, language: str) -> str:
|
| 169 |
+
"""Extract error type from error message."""
|
| 170 |
+
# Look for common error patterns
|
| 171 |
+
patterns = {
|
| 172 |
+
"python": [
|
| 173 |
+
(r"(\w+Error):", 1),
|
| 174 |
+
(r"(\w+Exception):", 1),
|
| 175 |
+
],
|
| 176 |
+
"javascript": [
|
| 177 |
+
(r"(\w+Error):", 1),
|
| 178 |
+
(r"(\w+TypeError):", 1),
|
| 179 |
+
],
|
| 180 |
+
}
|
| 181 |
+
|
| 182 |
+
for pattern, group in patterns.get(language, []):
|
| 183 |
+
match = re.search(pattern, error_message)
|
| 184 |
+
if match:
|
| 185 |
+
return match.group(group)
|
| 186 |
+
|
| 187 |
+
return "UnknownError"
|
| 188 |
+
|
| 189 |
+
def _generate_debug_steps(
|
| 190 |
+
self,
|
| 191 |
+
error_type: str,
|
| 192 |
+
error_message: str,
|
| 193 |
+
language: str,
|
| 194 |
+
) -> List[str]:
|
| 195 |
+
"""Generate debugging steps for the error."""
|
| 196 |
+
steps = [
|
| 197 |
+
"1. Read the error message carefully - it tells you what went wrong",
|
| 198 |
+
"2. Check the line number in the traceback",
|
| 199 |
+
"3. Look at the context around that line",
|
| 200 |
+
]
|
| 201 |
+
|
| 202 |
+
if error_type == "NameError":
|
| 203 |
+
steps.extend([
|
| 204 |
+
"4. Check if the variable is spelled correctly",
|
| 205 |
+
"5. Verify the variable is defined before use",
|
| 206 |
+
"6. Check if you need to import the module",
|
| 207 |
+
])
|
| 208 |
+
elif error_type == "TypeError":
|
| 209 |
+
steps.extend([
|
| 210 |
+
"4. Check the types of variables involved",
|
| 211 |
+
"5. Use print() or logging to debug values",
|
| 212 |
+
"6. Use type() to check variable types",
|
| 213 |
+
])
|
| 214 |
+
elif error_type == "IndexError":
|
| 215 |
+
steps.extend([
|
| 216 |
+
"4. Check the list length before accessing",
|
| 217 |
+
"5. Consider using try/except for bounds",
|
| 218 |
+
"6. Check if the list is empty",
|
| 219 |
+
])
|
| 220 |
+
elif error_type == "ImportError":
|
| 221 |
+
steps.extend([
|
| 222 |
+
"4. Verify the package is installed (pip list / npm list)",
|
| 223 |
+
"5. Check the package name is correct",
|
| 224 |
+
"6. Try reinstalling the package",
|
| 225 |
+
])
|
| 226 |
+
|
| 227 |
+
return steps
|
| 228 |
+
|
| 229 |
+
def _suggest_fixes(
|
| 230 |
+
self,
|
| 231 |
+
error_type: str,
|
| 232 |
+
error_message: str,
|
| 233 |
+
language: str,
|
| 234 |
+
) -> List[str]:
|
| 235 |
+
"""Suggest fixes for the error."""
|
| 236 |
+
fixes = []
|
| 237 |
+
|
| 238 |
+
if error_type == "NameError":
|
| 239 |
+
fixes.append("Check spelling of all variable/function names")
|
| 240 |
+
fixes.append("Ensure variable is defined before use")
|
| 241 |
+
fixes.append("Add necessary import statements")
|
| 242 |
+
elif error_type == "TypeError":
|
| 243 |
+
fixes.append("Convert types explicitly if needed")
|
| 244 |
+
fixes.append("Check you're using the right operators")
|
| 245 |
+
fixes.append("Verify function accepts the arguments given")
|
| 246 |
+
elif error_type == "IndexError":
|
| 247 |
+
fixes.append("Add bounds checking before access")
|
| 248 |
+
fixes.append("Use .get() for dictionaries")
|
| 249 |
+
fixes.append("Check if list is empty first")
|
| 250 |
+
elif error_type == "SyntaxError":
|
| 251 |
+
fixes.append("Check for missing colons, brackets, quotes")
|
| 252 |
+
fixes.append("Verify indentation is consistent")
|
| 253 |
+
fixes.append("Run a linter to find issues")
|
| 254 |
+
|
| 255 |
+
return fixes
|
| 256 |
+
|
| 257 |
+
def analyze_traceback(self, traceback: str) -> Dict[str, Any]:
|
| 258 |
+
"""
|
| 259 |
+
Analyze a full traceback.
|
| 260 |
+
|
| 261 |
+
Args:
|
| 262 |
+
traceback: The full error traceback
|
| 263 |
+
|
| 264 |
+
Returns:
|
| 265 |
+
Dictionary with traceback analysis
|
| 266 |
+
"""
|
| 267 |
+
lines = traceback.split('\n')
|
| 268 |
+
|
| 269 |
+
# Extract file and line numbers
|
| 270 |
+
file_lines = []
|
| 271 |
+
for line in lines:
|
| 272 |
+
if 'File "' in line or 'line ' in line:
|
| 273 |
+
file_lines.append(line.strip())
|
| 274 |
+
|
| 275 |
+
# Get the main error
|
| 276 |
+
error_line = ""
|
| 277 |
+
for line in lines:
|
| 278 |
+
if 'Error:' in line or 'Exception:' in line:
|
| 279 |
+
error_line = line.strip()
|
| 280 |
+
break
|
| 281 |
+
|
| 282 |
+
return {
|
| 283 |
+
"files_involved": file_lines,
|
| 284 |
+
"main_error": error_line,
|
| 285 |
+
"frames": len([l for l in lines if 'File "' in l]),
|
| 286 |
+
}
|
| 287 |
+
|
| 288 |
+
def generate_debug_code(
|
| 289 |
+
self,
|
| 290 |
+
error_type: str,
|
| 291 |
+
code_snippet: str,
|
| 292 |
+
) -> str:
|
| 293 |
+
"""Generate debugging code for the error."""
|
| 294 |
+
debug_templates = {
|
| 295 |
+
"NameError": f"""# Debug NameError
|
| 296 |
+
# Add debugging print statements
|
| 297 |
+
print(f"Variable value: {{variable_name}}")
|
| 298 |
+
|
| 299 |
+
# Check if defined
|
| 300 |
+
try:
|
| 301 |
+
result = {code_snippet}
|
| 302 |
+
except NameError as e:
|
| 303 |
+
print(f"NameError: {{e}}")""",
|
| 304 |
+
|
| 305 |
+
"TypeError": f"""# Debug TypeError
|
| 306 |
+
# Add type checking
|
| 307 |
+
print(f"Type of variable: {{type(variable_name)}}")
|
| 308 |
+
|
| 309 |
+
# Add type hints for clarity
|
| 310 |
+
def debug_function(variable_name):
|
| 311 |
+
print(f"Value: {{variable_name}}, Type: {{type(variable_name)}}")
|
| 312 |
+
return variable_name""",
|
| 313 |
+
|
| 314 |
+
"IndexError": f"""# Debug IndexError
|
| 315 |
+
# Add bounds checking
|
| 316 |
+
my_list = []
|
| 317 |
+
|
| 318 |
+
if len(my_list) > 0:
|
| 319 |
+
print(f"List has {{len(my_list)}} items")
|
| 320 |
+
# Access with safety
|
| 321 |
+
result = my_list[0] if my_list else None""",
|
| 322 |
+
|
| 323 |
+
"default": """# General debug approach
|
| 324 |
+
import traceback
|
| 325 |
+
|
| 326 |
+
try:
|
| 327 |
+
# Your code here
|
| 328 |
+
pass
|
| 329 |
+
except Exception as e:
|
| 330 |
+
print(f"Error: {{e}}")
|
| 331 |
+
traceback.print_exc()
|
| 332 |
+
# Add your debugging here"""
|
| 333 |
+
}
|
| 334 |
+
|
| 335 |
+
return debug_templates.get(error_type, debug_templates["default"])
|
| 336 |
+
|
| 337 |
+
def suggest_logging(self, code: str) -> str:
|
| 338 |
+
"""Suggest where to add logging statements."""
|
| 339 |
+
suggestions = []
|
| 340 |
+
|
| 341 |
+
# Suggest logging for function calls
|
| 342 |
+
functions = re.findall(r'def\s+(\w+)\s*\(', code)
|
| 343 |
+
for func in functions[:3]: # Limit to 3
|
| 344 |
+
suggestions.append(f"Add logging at start/end of function '{func}()'")
|
| 345 |
+
|
| 346 |
+
# Suggest logging for error handling
|
| 347 |
+
if "except" in code:
|
| 348 |
+
suggestions.append("Add logging in exception handlers")
|
| 349 |
+
|
| 350 |
+
# Suggest logging for loops
|
| 351 |
+
if "for " in code:
|
| 352 |
+
suggestions.append("Add logging in loops to track iterations")
|
| 353 |
+
|
| 354 |
+
return suggestions if suggestions else ["Code looks simple, minimal logging needed"]
|
| 355 |
+
|
| 356 |
+
def __repr__(self) -> str:
|
| 357 |
+
return "DebuggingAssistant()"
|
src/enhancements/technical/devops.py
ADDED
|
@@ -0,0 +1,399 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
DevOps Tools Module
|
| 3 |
+
|
| 4 |
+
Provides cloud and DevOps operation capabilities.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import Dict, List, Optional, Any
|
| 8 |
+
import re
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class DevOpsTools:
|
| 12 |
+
"""Cloud and DevOps operation tools."""
|
| 13 |
+
|
| 14 |
+
# Cloud provider templates
|
| 15 |
+
CLOUD_TEMPLATES = {
|
| 16 |
+
"aws": {
|
| 17 |
+
"ec2": {
|
| 18 |
+
"description": "AWS EC2 instance",
|
| 19 |
+
"template": """# AWS EC2 Instance
|
| 20 |
+
resource "aws_instance" "app_server" {
|
| 21 |
+
ami = "ami-0c55b159cbfafe1f0"
|
| 22 |
+
instance_type = "t3.micro"
|
| 23 |
+
|
| 24 |
+
tags = {
|
| 25 |
+
Name = "Stack2.9-App"
|
| 26 |
+
}
|
| 27 |
+
}"""
|
| 28 |
+
},
|
| 29 |
+
"s3": {
|
| 30 |
+
"description": "AWS S3 bucket",
|
| 31 |
+
"template": """# AWS S3 Bucket
|
| 32 |
+
resource "aws_s3_bucket" "data_store" {
|
| 33 |
+
bucket = "stack29-data-store"
|
| 34 |
+
|
| 35 |
+
tags = {
|
| 36 |
+
Name = "Stack2.9 Data"
|
| 37 |
+
Environment = "production"
|
| 38 |
+
}
|
| 39 |
+
}"""
|
| 40 |
+
},
|
| 41 |
+
"lambda": {
|
| 42 |
+
"description": "AWS Lambda function",
|
| 43 |
+
"template": """# AWS Lambda Function
|
| 44 |
+
resource "aws_lambda_function" "handler" {
|
| 45 |
+
filename = "handler.zip"
|
| 46 |
+
function_name = "stack29_handler"
|
| 47 |
+
role = aws_iam_role.lambda_role.arn
|
| 48 |
+
handler = "index.handler"
|
| 49 |
+
source_code_hash = filebase64sha256("handler.zip")
|
| 50 |
+
|
| 51 |
+
runtime = "python3.9"
|
| 52 |
+
}"""
|
| 53 |
+
},
|
| 54 |
+
},
|
| 55 |
+
"gcp": {
|
| 56 |
+
"compute": {
|
| 57 |
+
"description": "GCP Compute Engine",
|
| 58 |
+
"template": """# GCP Compute Engine
|
| 59 |
+
resource "google_compute_instance" "vm_instance" {
|
| 60 |
+
name = "stack29-vm"
|
| 61 |
+
machine_type = "e2-micro"
|
| 62 |
+
zone = "us-central1-a"
|
| 63 |
+
|
| 64 |
+
boot_disk {
|
| 65 |
+
initialize_params {
|
| 66 |
+
image = "debian-cloud/debian-11"
|
| 67 |
+
}
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
network_interface {
|
| 71 |
+
network = "default"
|
| 72 |
+
}
|
| 73 |
+
}"""
|
| 74 |
+
},
|
| 75 |
+
"storage": {
|
| 76 |
+
"description": "GCP Cloud Storage",
|
| 77 |
+
"template": """# GCP Cloud Storage
|
| 78 |
+
resource "google_storage_bucket" "bucket" {
|
| 79 |
+
name = "stack29-bucket"
|
| 80 |
+
location = "US"
|
| 81 |
+
force_destroy = false
|
| 82 |
+
|
| 83 |
+
labels = {
|
| 84 |
+
environment = "production"
|
| 85 |
+
}
|
| 86 |
+
}"""
|
| 87 |
+
},
|
| 88 |
+
},
|
| 89 |
+
"docker": {
|
| 90 |
+
"container": {
|
| 91 |
+
"description": "Docker container configuration",
|
| 92 |
+
"template": """# Dockerfile
|
| 93 |
+
FROM python:3.11-slim
|
| 94 |
+
|
| 95 |
+
WORKDIR /app
|
| 96 |
+
|
| 97 |
+
# Install dependencies
|
| 98 |
+
COPY requirements.txt .
|
| 99 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 100 |
+
|
| 101 |
+
# Copy application
|
| 102 |
+
COPY . .
|
| 103 |
+
|
| 104 |
+
# Run application
|
| 105 |
+
CMD ["python", "main.py"]"""
|
| 106 |
+
},
|
| 107 |
+
"compose": {
|
| 108 |
+
"description": "Docker Compose configuration",
|
| 109 |
+
"template": """# docker-compose.yml
|
| 110 |
+
version: '3.8'
|
| 111 |
+
|
| 112 |
+
services:
|
| 113 |
+
app:
|
| 114 |
+
build: .
|
| 115 |
+
ports:
|
| 116 |
+
- "8000:8000"
|
| 117 |
+
environment:
|
| 118 |
+
- DATABASE_URL=postgres://db:5432/app
|
| 119 |
+
depends_on:
|
| 120 |
+
- db
|
| 121 |
+
- redis
|
| 122 |
+
|
| 123 |
+
db:
|
| 124 |
+
image: postgres:15
|
| 125 |
+
environment:
|
| 126 |
+
- POSTGRES_DB=app
|
| 127 |
+
- POSTGRES_PASSWORD=secret
|
| 128 |
+
|
| 129 |
+
redis:
|
| 130 |
+
image: redis:7-alpine
|
| 131 |
+
ports:
|
| 132 |
+
- "6379:6379"
|
| 133 |
+
"""
|
| 134 |
+
},
|
| 135 |
+
},
|
| 136 |
+
"kubernetes": {
|
| 137 |
+
"deployment": {
|
| 138 |
+
"description": "Kubernetes Deployment",
|
| 139 |
+
"template": """# k8s-deployment.yaml
|
| 140 |
+
apiVersion: apps/v1
|
| 141 |
+
kind: Deployment
|
| 142 |
+
metadata:
|
| 143 |
+
name: stack29-app
|
| 144 |
+
labels:
|
| 145 |
+
app: stack29
|
| 146 |
+
spec:
|
| 147 |
+
replicas: 3
|
| 148 |
+
selector:
|
| 149 |
+
matchLabels:
|
| 150 |
+
app: stack29
|
| 151 |
+
template:
|
| 152 |
+
metadata:
|
| 153 |
+
labels:
|
| 154 |
+
app: stack29
|
| 155 |
+
spec:
|
| 156 |
+
containers:
|
| 157 |
+
- name: app
|
| 158 |
+
image: stack29:latest
|
| 159 |
+
ports:
|
| 160 |
+
- containerPort: 8000
|
| 161 |
+
resources:
|
| 162 |
+
limits:
|
| 163 |
+
cpu: "500m"
|
| 164 |
+
memory: "256Mi"
|
| 165 |
+
"""
|
| 166 |
+
},
|
| 167 |
+
"service": {
|
| 168 |
+
"description": "Kubernetes Service",
|
| 169 |
+
"template": """# k8s-service.yaml
|
| 170 |
+
apiVersion: v1
|
| 171 |
+
kind: Service
|
| 172 |
+
metadata:
|
| 173 |
+
name: stack29-service
|
| 174 |
+
spec:
|
| 175 |
+
selector:
|
| 176 |
+
app: stack29
|
| 177 |
+
ports:
|
| 178 |
+
- protocol: TCP
|
| 179 |
+
port: 80
|
| 180 |
+
targetPort: 8000
|
| 181 |
+
type: LoadBalancer
|
| 182 |
+
"""
|
| 183 |
+
},
|
| 184 |
+
},
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
# CI/CD templates
|
| 188 |
+
CICD_TEMPLATES = {
|
| 189 |
+
"github_actions": {
|
| 190 |
+
"description": "GitHub Actions workflow",
|
| 191 |
+
"template": """# .github/workflows/ci.yml
|
| 192 |
+
name: CI
|
| 193 |
+
|
| 194 |
+
on:
|
| 195 |
+
push:
|
| 196 |
+
branches: [ main ]
|
| 197 |
+
pull_request:
|
| 198 |
+
branches: [ main ]
|
| 199 |
+
|
| 200 |
+
jobs:
|
| 201 |
+
test:
|
| 202 |
+
runs-on: ubuntu-latest
|
| 203 |
+
|
| 204 |
+
steps:
|
| 205 |
+
- uses: actions/checkout@v3
|
| 206 |
+
|
| 207 |
+
- name: Set up Python
|
| 208 |
+
uses: actions/setup-python@v4
|
| 209 |
+
with:
|
| 210 |
+
python-version: '3.11'
|
| 211 |
+
|
| 212 |
+
- name: Install dependencies
|
| 213 |
+
run: |
|
| 214 |
+
pip install -r requirements.txt
|
| 215 |
+
|
| 216 |
+
- name: Run tests
|
| 217 |
+
run: |
|
| 218 |
+
pytest tests/
|
| 219 |
+
|
| 220 |
+
- name: Lint
|
| 221 |
+
run: |
|
| 222 |
+
ruff check .
|
| 223 |
+
"""
|
| 224 |
+
},
|
| 225 |
+
"gitlab_ci": {
|
| 226 |
+
"description": "GitLab CI pipeline",
|
| 227 |
+
"template": """# .gitlab-ci.yml
|
| 228 |
+
stages:
|
| 229 |
+
- test
|
| 230 |
+
- build
|
| 231 |
+
- deploy
|
| 232 |
+
|
| 233 |
+
test:
|
| 234 |
+
stage: test
|
| 235 |
+
script:
|
| 236 |
+
- pip install -r requirements.txt
|
| 237 |
+
- pytest tests/
|
| 238 |
+
rules:
|
| 239 |
+
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
| 240 |
+
|
| 241 |
+
build:
|
| 242 |
+
stage: build
|
| 243 |
+
script:
|
| 244 |
+
- docker build -t stack29:$CI_COMMIT_SHA .
|
| 245 |
+
rules:
|
| 246 |
+
- if: $CI_COMMIT_BRANCH == "main"
|
| 247 |
+
|
| 248 |
+
deploy:
|
| 249 |
+
stage: deploy
|
| 250 |
+
script:
|
| 251 |
+
- kubectl apply -f k8s/
|
| 252 |
+
environment:
|
| 253 |
+
name: production
|
| 254 |
+
rules:
|
| 255 |
+
- if: $CI_COMMIT_BRANCH == "main"
|
| 256 |
+
"""
|
| 257 |
+
},
|
| 258 |
+
}
|
| 259 |
+
|
| 260 |
+
# Infrastructure as Code templates
|
| 261 |
+
TERRAFORM_VARIABLES = {
|
| 262 |
+
"description": "Terraform variables",
|
| 263 |
+
"template": """# variables.tf
|
| 264 |
+
variable "region" {
|
| 265 |
+
description = "AWS region"
|
| 266 |
+
type = string
|
| 267 |
+
default = "us-east-1"
|
| 268 |
+
}
|
| 269 |
+
|
| 270 |
+
variable "environment" {
|
| 271 |
+
description = "Environment name"
|
| 272 |
+
type = string
|
| 273 |
+
default = "production"
|
| 274 |
+
}
|
| 275 |
+
|
| 276 |
+
variable "instance_type" {
|
| 277 |
+
description = "EC2 instance type"
|
| 278 |
+
type = string
|
| 279 |
+
default = "t3.micro"
|
| 280 |
+
}
|
| 281 |
+
"""
|
| 282 |
+
}
|
| 283 |
+
|
| 284 |
+
def __init__(self):
|
| 285 |
+
"""Initialize DevOps tools."""
|
| 286 |
+
pass
|
| 287 |
+
|
| 288 |
+
def get_cloud_template(
|
| 289 |
+
self,
|
| 290 |
+
provider: str,
|
| 291 |
+
service: str,
|
| 292 |
+
) -> Optional[str]:
|
| 293 |
+
"""Get a cloud infrastructure template."""
|
| 294 |
+
return self.CLOUD_TEMPLATES.get(provider, {}).get(service, {}).get("template")
|
| 295 |
+
|
| 296 |
+
def get_cicd_template(self, platform: str) -> Optional[str]:
|
| 297 |
+
"""Get a CI/CD pipeline template."""
|
| 298 |
+
return self.CICD_TEMPLATES.get(platform, {}).get("template")
|
| 299 |
+
|
| 300 |
+
def list_available_templates(self) -> Dict[str, List[str]]:
|
| 301 |
+
"""List all available templates."""
|
| 302 |
+
return {
|
| 303 |
+
"cloud_providers": list(self.CLOUD_TEMPLATES.keys()),
|
| 304 |
+
"cicd": list(self.CICD_TEMPLATES.keys()),
|
| 305 |
+
}
|
| 306 |
+
|
| 307 |
+
def generate_kubernetes_manifest(
|
| 308 |
+
self,
|
| 309 |
+
app_name: str,
|
| 310 |
+
image: str,
|
| 311 |
+
replicas: int = 3,
|
| 312 |
+
port: int = 8000,
|
| 313 |
+
) -> str:
|
| 314 |
+
"""Generate a Kubernetes deployment manifest."""
|
| 315 |
+
return f"""apiVersion: apps/v1
|
| 316 |
+
kind: Deployment
|
| 317 |
+
metadata:
|
| 318 |
+
name: {app_name}
|
| 319 |
+
labels:
|
| 320 |
+
app: {app_name}
|
| 321 |
+
spec:
|
| 322 |
+
replicas: {replicas}
|
| 323 |
+
selector:
|
| 324 |
+
matchLabels:
|
| 325 |
+
app: {app_name}
|
| 326 |
+
template:
|
| 327 |
+
metadata:
|
| 328 |
+
labels:
|
| 329 |
+
app: {app_name}
|
| 330 |
+
spec:
|
| 331 |
+
containers:
|
| 332 |
+
- name: {app_name}
|
| 333 |
+
image: {image}
|
| 334 |
+
ports:
|
| 335 |
+
- containerPort: {port}
|
| 336 |
+
resources:
|
| 337 |
+
limits:
|
| 338 |
+
cpu: "1000m"
|
| 339 |
+
memory: "512Mi"
|
| 340 |
+
requests:
|
| 341 |
+
cpu: "100m"
|
| 342 |
+
memory: "128Mi"
|
| 343 |
+
---
|
| 344 |
+
|
| 345 |
+
apiVersion: v1
|
| 346 |
+
kind: Service
|
| 347 |
+
metadata:
|
| 348 |
+
name: {app_name}-service
|
| 349 |
+
spec:
|
| 350 |
+
selector:
|
| 351 |
+
app: {app_name}
|
| 352 |
+
ports:
|
| 353 |
+
- protocol: TCP
|
| 354 |
+
port: 80
|
| 355 |
+
targetPort: {port}
|
| 356 |
+
type: LoadBalancer
|
| 357 |
+
"""
|
| 358 |
+
|
| 359 |
+
def generate_dockerfile(
|
| 360 |
+
self,
|
| 361 |
+
language: str = "python",
|
| 362 |
+
version: str = "3.11",
|
| 363 |
+
port: int = 8000,
|
| 364 |
+
) -> str:
|
| 365 |
+
"""Generate a Dockerfile."""
|
| 366 |
+
base_images = {
|
| 367 |
+
"python": f"python:{version}-slim",
|
| 368 |
+
"node": f"node:{version}-slim",
|
| 369 |
+
"go": f"golang:{version}",
|
| 370 |
+
"rust": f"rust:{version}-slim",
|
| 371 |
+
}
|
| 372 |
+
base = base_images.get(language, f"python:{version}-slim")
|
| 373 |
+
|
| 374 |
+
return f"""FROM {base}
|
| 375 |
+
|
| 376 |
+
WORKDIR /app
|
| 377 |
+
|
| 378 |
+
# Install dependencies
|
| 379 |
+
COPY requirements.txt .
|
| 380 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 381 |
+
|
| 382 |
+
# Copy application
|
| 383 |
+
COPY . .
|
| 384 |
+
|
| 385 |
+
# Expose port
|
| 386 |
+
EXPOSE {port}
|
| 387 |
+
|
| 388 |
+
# Run application
|
| 389 |
+
CMD ["python", "main.py"]
|
| 390 |
+
"""
|
| 391 |
+
|
| 392 |
+
def parse_docker_compose(self, compose_content: str) -> Dict[str, Any]:
|
| 393 |
+
"""Parse docker-compose content to extract services."""
|
| 394 |
+
services = re.findall(r'^ (\w+):$', compose_content, re.MULTILINE)
|
| 395 |
+
return {"services": services, "count": len(services)}
|
| 396 |
+
|
| 397 |
+
def __repr__(self) -> str:
|
| 398 |
+
templates = self.list_available_templates()
|
| 399 |
+
return f"DevOpsTools(cloud={templates['cloud_providers']}, cicd={templates['cicd']})"
|
test_enhancements.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
Quick test script to verify enhancement modules work.
|
| 4 |
+
Run this before the full chat to check all modules are functioning.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import sys
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
# Add src to path
|
| 11 |
+
sys.path.insert(0, str(Path(__file__).parent / "src"))
|
| 12 |
+
|
| 13 |
+
print("=" * 50)
|
| 14 |
+
print("Testing Stack 2.9 Enhancement Modules")
|
| 15 |
+
print("=" * 50)
|
| 16 |
+
|
| 17 |
+
# Test 1: Config
|
| 18 |
+
print("\n[1] Testing Configuration...")
|
| 19 |
+
from enhancements import get_config, EnhancementConfig
|
| 20 |
+
config = get_config()
|
| 21 |
+
print(f" β Config loaded: NLP={config.nlp.use_bert_embeddings}, RAG={config.knowledge_graph.rag_enabled}")
|
| 22 |
+
|
| 23 |
+
# Test 2: NLP Modules
|
| 24 |
+
print("\n[2] Testing NLP Modules...")
|
| 25 |
+
from enhancements.nlp import IntentDetector, EntityRecognizer
|
| 26 |
+
|
| 27 |
+
# Test Intent Detection
|
| 28 |
+
intent_detector = IntentDetector()
|
| 29 |
+
test_intents = [
|
| 30 |
+
"Write a function to calculate fibonacci",
|
| 31 |
+
"Help me debug this error",
|
| 32 |
+
"Explain what is Python",
|
| 33 |
+
"Hello there!",
|
| 34 |
+
]
|
| 35 |
+
print(" Intent Detection:")
|
| 36 |
+
for text in test_intents:
|
| 37 |
+
result = intent_detector.detect_intent(text)
|
| 38 |
+
print(f" '{text[:30]}...' β {result['intent']} ({result['confidence']:.2f})")
|
| 39 |
+
|
| 40 |
+
# Test Entity Recognition
|
| 41 |
+
entity_recognizer = EntityRecognizer()
|
| 42 |
+
test_entities = [
|
| 43 |
+
"My email is test@example.com and I live in New York",
|
| 44 |
+
"Visit https://github.com for code",
|
| 45 |
+
"Call me at 555-123-4567",
|
| 46 |
+
]
|
| 47 |
+
print(" Entity Recognition:")
|
| 48 |
+
for text in test_entities:
|
| 49 |
+
entities = entity_recognizer.recognize_entities(text)
|
| 50 |
+
print(f" '{text[:30]}...' β {[e['type'] for e in entities]}")
|
| 51 |
+
|
| 52 |
+
# Test 3: Knowledge Graph
|
| 53 |
+
print("\n[3] Testing Knowledge Graph...")
|
| 54 |
+
from enhancements.knowledge_graph import KnowledgeGraph, RAGEngine
|
| 55 |
+
|
| 56 |
+
kg = KnowledgeGraph()
|
| 57 |
+
kg.add_entity("Python", "language", {"version": "3.11"})
|
| 58 |
+
kg.add_entity("Stack2.9", "ai_assistant", {"version": "2.9"})
|
| 59 |
+
kg.add_relationship("Stack2.9", "Python", "uses")
|
| 60 |
+
print(f" β Knowledge Graph: {kg.get_stats()}")
|
| 61 |
+
|
| 62 |
+
# Test RAG
|
| 63 |
+
rag = RAGEngine()
|
| 64 |
+
rag.add_document("doc1", "Python is a programming language.")
|
| 65 |
+
rag.add_document("doc2", "Stack 2.9 is an AI coding assistant.")
|
| 66 |
+
results = rag.retrieve("Tell me about Python")
|
| 67 |
+
print(f" β RAG Retrieval: {len(results)} docs found")
|
| 68 |
+
|
| 69 |
+
# Test 4: Emotional Intelligence
|
| 70 |
+
print("\n[4] Testing Emotional Intelligence...")
|
| 71 |
+
from enhancements.emotional_intelligence import SentimentAnalyzer, EmpathyEngine
|
| 72 |
+
|
| 73 |
+
sentiment = SentimentAnalyzer()
|
| 74 |
+
test_sentiments = [
|
| 75 |
+
"This is amazing! I love it!",
|
| 76 |
+
"I'm frustrated with this problem",
|
| 77 |
+
"Can you help me?",
|
| 78 |
+
]
|
| 79 |
+
print(" Sentiment Analysis:")
|
| 80 |
+
for text in test_sentiments:
|
| 81 |
+
result = sentiment.analyze_sentiment(text)
|
| 82 |
+
print(f" '{text[:30]}...' β {result['sentiment']} ({result['emotion_tone']})")
|
| 83 |
+
|
| 84 |
+
empathy = EmpathyEngine()
|
| 85 |
+
test_response = "Here's your code:"
|
| 86 |
+
empathetic = empathy.generate_empathetic_response(
|
| 87 |
+
"I'm having trouble with my code",
|
| 88 |
+
test_response
|
| 89 |
+
)
|
| 90 |
+
print(f" β Empathy Engine: Modified response with prefix")
|
| 91 |
+
|
| 92 |
+
# Test 5: Collaboration
|
| 93 |
+
print("\n[5] Testing Collaboration...")
|
| 94 |
+
from enhancements.collaboration import ConversationStateManager, MCPIntegration
|
| 95 |
+
|
| 96 |
+
conv_mgr = ConversationStateManager()
|
| 97 |
+
session_id = conv_mgr.create_session()
|
| 98 |
+
conv_mgr.add_message("user", "Hello AI!")
|
| 99 |
+
conv_mgr.add_message("assistant", "Hello! How can I help?")
|
| 100 |
+
history = conv_mgr.get_conversation_history()
|
| 101 |
+
print(f" β Conversation Manager: {len(history)} messages in session")
|
| 102 |
+
|
| 103 |
+
mcp = MCPIntegration()
|
| 104 |
+
tools = mcp.list_tools()
|
| 105 |
+
print(f" β MCP Integration: {len(tools)} tools registered")
|
| 106 |
+
|
| 107 |
+
# Test 6: Learning
|
| 108 |
+
print("\n[6] Testing Learning System...")
|
| 109 |
+
from enhancements.learning import FeedbackCollector, PerformanceMonitor
|
| 110 |
+
|
| 111 |
+
feedback = FeedbackCollector(storage_path="data/test_feedback")
|
| 112 |
+
fb_id = feedback.add_thumbs_up("Test message", "Test response")
|
| 113 |
+
stats = feedback.get_statistics()
|
| 114 |
+
print(f" β Feedback Collector: {stats['total']} entries")
|
| 115 |
+
|
| 116 |
+
perf = PerformanceMonitor(storage_path="data/test_performance")
|
| 117 |
+
perf.record_response_time(0.5)
|
| 118 |
+
perf.record_successful_interaction()
|
| 119 |
+
summary = perf.get_summary()
|
| 120 |
+
print(f" β Performance Monitor: avg response {summary['average_response_time']:.2f}s")
|
| 121 |
+
|
| 122 |
+
# Summary
|
| 123 |
+
print("\n" + "=" * 50)
|
| 124 |
+
print("All Enhancement Modules Tested Successfully!")
|
| 125 |
+
print("=" * 50)
|
| 126 |
+
print("\nTo run the enhanced chat:")
|
| 127 |
+
print(" python enhanced_chat.py")
|
| 128 |
+
print("\nOptions:")
|
| 129 |
+
print(" --no-bert Disable BERT embeddings")
|
| 130 |
+
print(" --no-rag Disable RAG")
|
| 131 |
+
print(" --no-empathy Disable emotional intelligence")
|