chore: Rename MCP server to Stack2.9

cli/run_mcp_server.py

@@ -4,8 +4,9 @@
 import sys
 import os
 
-# Ensure src/ is on the path
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+# Ensure project root is on the path
+project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, project_root)
 
 from src.mcp_server import main
 

scripts/augment_data.py

@@ -1,124 +0,0 @@
-#!/usr/bin/env python3
-"""
-Data augmentation for training examples.
-Increases dataset size by paraphrasing and variations.
-"""
-
-import json
-import random
-from pathlib import Path
-from typing import List, Dict, Any
-import argparse
-
-# Paraphrase templates (rule-based, no LLM)
-PARAPHRASES = {
-    "Read the file": ["Show me the contents of", "Open", "Display", "Fetch", "Get"],
-    "Create a new file": ["Write a file", "Generate", "Make a new file", "Add file"],
-    "Run": ["Execute", "Start", "Launch", "Invoke"],
-    "Search for": ["Find", "Look for", "Locate", "Grep for"],
-    "List all": ["Show all", "Display every", "Get list of"],
-    "Can you": ["Please", "Would you", "Kindly"],
-    "I need": ["I want", "I require", "Please provide"],
-}
-
-def paraphrase_text(text: str) -> str:
-    """Apply simple paraphrasing to user prompt."""
-    result = text
-    for original, alternatives in PARAPHRASES.items():
-        if original in result:
-            replacement = random.choice(alternatives)
-            result = result.replace(original, replacement, 1)
-    return result
-
-def augment_example(example: Dict[str, Any], variation_factor: float = 0.3) -> List[Dict[str, Any]]:
-    """Generate variations of a single example."""
-    variations = [example]  # Keep original
-
-    # Paraphrase user message
-    if random.random() < variation_factor:
-        new_ex = json.loads(json.dumps(example))  # Deep copy
-        original_user = new_ex["messages"][0]["content"]
-        new_ex["messages"][0]["content"] = paraphrase_text(original_user)
-        new_ex["source"] = "augmented_paraphrase"
-        variations.append(new_ex)
-
-    # Vary tool parameters (if any)
-    if "tool_use" in example["messages"][1]:
-        tool_input = example["messages"][1]["tool_use"]["input"]
-        if isinstance(tool_input, dict) and tool_input:
-            new_ex = json.loads(json.dumps(example))
-            # Randomly change file paths, commands, etc.
-            for key, val in new_ex["messages"][1]["tool_use"]["input"].items():
-                if key == "file_path" and isinstance(val, str):
-                    # Change to a different plausible file
-                    new_ex["messages"][1]["tool_use"]["input"][key] = random.choice([
-                        "src/main.py", "README.md", "package.json", "config.yaml"
-                    ])
-                    # Also update result if it contains the old file path
-                    result_content = new_ex["messages"][2]["tool_result"]["content"]
-                    new_ex["messages"][2]["tool_result"]["content"] = result_content.replace(val, new_ex["messages"][1]["tool_use"]["input"][key])
-                    new_ex["source"] = "augmented_params"
-                    variations.append(new_ex)
-
-    # Add filler words to user message
-    if random.random() < variation_factor * 0.5:
-        new_ex = json.loads(json.dumps(example))
-        fillers = [" please", " if you can", " when you have time", " thanks"]
-        user_msg = new_ex["messages"][0]["content"]
-        filler = random.choice(fillers)
-        new_ex["messages"][0]["content"] = user_msg + filler
-        new_ex["source"] = "augmented_filler"
-        variations.append(new_ex)
-
-    return variations
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--input", type=str, default="training-data/scaled/template_synthetic.jsonl")
-    parser.add_argument("--output", type=str, default="training-data/scaled/augmented.jsonl")
-    parser.add_argument("--multiplier", type=int, default=3, help="How many times to multiply dataset")
-    args = parser.parse_args()
-
-    input_path = Path(args.input)
-    output_path = Path(args.output)
-
-    if not input_path.exists():
-        print(f"❌ Input file not found: {input_path}")
-        return
-
-    print(f"📈 Augmenting dataset: {input_path}")
-    examples = []
-    with open(input_path, 'r') as f:
-        for line in f:
-            examples.append(json.loads(line))
-
-    original_count = len(examples)
-    target_count = original_count * args.multiplier
-    print(f"   Original: {original_count} examples")
-    print(f"   Target: ~{target_count} examples (x{args.multiplier})")
-
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-
-    generated = 0
-    with open(output_path, 'w') as f:
-        for ex in examples:
-            # Write original and variations
-            f.write(json.dumps(ex) + "\n")
-            generated += 1
-
-            # Generate variations until we reach multiplier
-            variations = augment_example(ex)
-            for var in variations[1:]:  # Skip original (already written)
-                if generated < target_count:
-                    f.write(json.dumps(var) + "\n")
-                    generated += 1
-
-            if generated % 1000 == 0:
-                print(f"   Generated {generated}/{target_count}...", end='\r')
-
-    print(f"\n✨ Augmented to {generated} examples")
-    print(f"   Saved to: {output_path}")
-    print(f"   Total dataset now: {original_count} → {generated} (x{generated/original_count:.1f})")
-
-if __name__ == "__main__":
-    main()

scripts/augment_training_data.py

@@ -1,324 +0,0 @@
-#!/usr/bin/env python3
-"""
-Data augmentation script for tool_examples.jsonl.
-Generates 2x-5x more training examples from existing data through:
-- Paraphrasing user prompts
-- Difficulty scaling (simpler/complex variations)
-- Edge case generation
-"""
-
-import json
-import random
-import argparse
-from pathlib import Path
-from typing import List, Dict, Any, Optional
-from itertools import product
-import copy
-
-# Random seed for reproducibility
-random.seed(42)
-
-# Paraphrase templates
-PARAPHRASES = {
-    "Can you": ["Please", "Would you kindly", "Could you", "Kindly"],
-    "I need": ["I'd like", "I require", "I want", "I must have"],
-    "show me": ["display", "show", "reveal", "let me see"],
-    "the file": ["this file", "that file", "a file"],
-    "run": ["execute", "launch", "start", "run"],
-    "create": ["make", "generate", "add", "write"],
-    "delete": ["remove", "erase", "drop", "destroy"],
-    "list": ["show", "display", "enumerate", "get"],
-    "search": ["find", "look for", "grep", "locate"],
-    "help me": ["assist me", "I need help", "please assist", "support"],
-}
-
-# Difficulty modifiers
-EASY_MODIFIERS = [
-    "quickly",
-    "simply",
-    "just",
-    "easily",
-]
-
-COMPLEX_MODIFIERS = [
-    "carefully",
-    "thoroughly",
-    "in detail",
-    "completely",
-    "with all options",
-]
-
-# Edge case patterns
-EDGE_CASE_PATTERNS = [
-    ("empty_input", lambda ex: _create_empty_variant(ex)),
-    ("multi_step", lambda ex: _create_multistep_variant(ex)),
-    ("error_handling", lambda ex: _create_error_variant(ex)),
-]
-
-
-def _deep_copy(obj: Any) -> Any:
-    """Create a deep copy of a JSON-serializable object."""
-    return json.loads(json.dumps(obj))
-
-
-def _create_empty_variant(example: Dict[str, Any]) -> Optional[Dict[str, Any]]:
-    """Create variant with empty/blank user input."""
-    new_ex = _deep_copy(example)
-    # Keep system message, empty user message
-    for msg in new_ex["messages"]:
-        if msg["role"] == "user":
-            msg["content"] = " "
-            break
-    new_ex["source"] = "augmented_edge_empty"
-    return new_ex
-
-
-def _create_multistep_variant(example: Dict[str, Any]) -> Optional[Dict[str, Any]]:
-    """Create variant simulating multi-step reasoning."""
-    new_ex = _deep_copy(example)
-    # Add reasoning step before tool call
-    for i, msg in enumerate(new_ex["messages"]):
-        if msg.get("tool_calls"):
-            reasoning = {
-                "role": "assistant",
-                "content": "Let me think about this step by step. First, I need to understand what the user is asking for."
-            }
-            new_ex["messages"].insert(i, reasoning)
-            break
-    new_ex["source"] = "augmented_edge_multistep"
-    return new_ex
-
-
-def _create_error_variant(example: Dict[str, Any]) -> Optional[Dict[str, Any]]:
-    """Create variant simulating error handling."""
-    new_ex = _deep_copy(example)
-    for msg in new_ex["messages"]:
-        if msg.get("role") == "tool":
-            # Simulate an error in tool result
-            if "Successfully" in msg.get("content", ""):
-                msg["content"] = msg["content"].replace("Successfully", "Error occurred:")
-            elif "error" not in msg.get("content", "").lower():
-                msg["content"] = "Operation failed: Permission denied"
-            break
-    new_ex["source"] = "augmented_edge_error"
-    return new_ex
-
-
-def paraphrase_text(text: str) -> str:
-    """Apply simple paraphrasing to text."""
-    if not text:
-        return text
-    result = text
-    for original, alternatives in PARAPHRASES.items():
-        if original.lower() in result.lower():
-            # Case-insensitive replace, preserve original case pattern
-            idx = result.lower().find(original.lower())
-            prefix = result[:idx]
-            suffix = result[idx + len(original):]
-            replacement = random.choice(alternatives)
-            # Preserve case
-            if result[idx].isupper():
-                replacement = replacement.capitalize()
-            result = prefix + replacement + suffix
-            break
-    return result
-
-
-def apply_difficulty(example: Dict[str, Any], level: str) -> Dict[str, Any]:
-    """Apply difficulty scaling to an example."""
-    new_ex = _deep_copy(example)
-    modifiers = EASY_MODIFIERS if level == "easy" else COMPLEX_MODIFIERS
-    
-    for msg in new_ex["messages"]:
-        if msg["role"] == "user" and msg.get("content"):
-            content = msg["content"]
-            if level == "easy":
-                # Simplify the request
-                content = content.replace("please", "").replace("kindly", "")
-                content = content.strip()
-            elif level == "complex":
-                # Add complexity
-                modifier = random.choice(modifiers)
-                content = f"{content} {modifier}"
-            msg["content"] = content
-            break
-    
-    new_ex["source"] = f"augmented_difficulty_{level}"
-    return new_ex
-
-
-def vary_tool_parameters(example: Dict[str, Any]) -> List[Dict[str, Any]]:
-    """Generate variations with different tool parameters."""
-    variations = []
-    
-    for msg in example.get("messages", []):
-        if msg.get("tool_calls"):
-            for tc in msg["tool_calls"]:
-                func = tc.get("function", {})
-                args_str = func.get("arguments", "{}")
-                try:
-                    args = json.loads(args_str) if isinstance(args_str, str) else args_str
-                except (json.JSONDecodeError, TypeError):
-                    continue
-                
-                if not isinstance(args, dict):
-                    continue
-                
-                # Common parameter variations
-                param_variations = [
-                    ("file_path", ["src/main.py", "README.md", "config.yaml", "package.json", "tests/test.py"]),
-                    ("command", ["ls -la", "echo hello", "pwd", "whoami"]),
-                    ("pattern", ["*.py", "*.js", "*.md", "*.json"]),
-                    ("path", ["src", "lib", "docs", "."]),
-                ]
-                
-                for param_name, alternatives in param_variations:
-                    if param_name in args:
-                        original_val = args[param_name]
-                        for alt_val in alternatives:
-                            if alt_val != original_val:
-                                new_ex = _deep_copy(example)
-                                for new_msg in new_ex["messages"]:
-                                    if new_msg.get("tool_calls"):
-                                        for new_tc in new_msg["tool_calls"]:
-                                            new_func = new_tc.get("function", {})
-                                            new_args = json.loads(new_func.get("arguments", "{}"))
-                                            if param_name in new_args:
-                                                new_args[param_name] = alt_val
-                                                new_func["arguments"] = json.dumps(new_args)
-                                new_ex["source"] = "augmented_params"
-                                variations.append(new_ex)
-                                break
-    
-    return variations
-
-
-def add_filler_variant(example: Dict[str, Any]) -> Optional[Dict[str, Any]]:
-    """Add polite filler words to user message."""
-    fillers = [" please", " if you could", " when you get a chance", " thanks"]
-    
-    new_ex = _deep_copy(example)
-    for msg in new_ex["messages"]:
-        if msg["role"] == "user" and msg.get("content"):
-            filler = random.choice(fillers)
-            msg["content"] = msg["content"].rstrip() + filler
-            break
-    
-    new_ex["source"] = "augmented_filler"
-    return new_ex
-
-
-def generate_edge_cases(example: Dict[str, Any], num_cases: int = 2) -> List[Dict[str, Any]]:
-    """Generate edge case variations."""
-    cases = []
-    selected_patterns = random.sample(EDGE_CASE_PATTERNS, min(num_cases, len(EDGE_CASE_PATTERNS)))
-    
-    for name, generator in selected_patterns:
-        try:
-            variant = generator(example)
-            if variant:
-                cases.append(variant)
-        except Exception:
-            continue
-    
-    return cases
-
-
-def augment_example(example: Dict[str, Any], target_multiplier: int = 3) -> List[Dict[str, Any]]:
-    """Generate multiple augmented variations of a single example."""
-    variations = [example]  # Always keep original
-    
-    # 1. Paraphrase variant
-    if random.random() < 0.7:
-        new_ex = _deep_copy(example)
-        for msg in new_ex["messages"]:
-            if msg["role"] == "user" and msg.get("content"):
-                msg["content"] = paraphrase_text(msg["content"])
-                break
-        new_ex["source"] = "augmented_paraphrase"
-        variations.append(new_ex)
-    
-    # 2. Difficulty variants (easy and complex)
-    if random.random() < 0.5:
-        variations.append(apply_difficulty(example, "easy"))
-    if random.random() < 0.5:
-        variations.append(apply_difficulty(example, "complex"))
-    
-    # 3. Filler variant
-    if random.random() < 0.3:
-        filler_ex = add_filler_variant(example)
-        if filler_ex:
-            variations.append(filler_ex)
-    
-    # 4. Tool parameter variations
-    param_variations = vary_tool_parameters(example)
-    variations.extend(param_variations[:2])  # Limit to 2
-    
-    # 5. Edge cases
-    if random.random() < 0.3:
-        edge_cases = generate_edge_cases(example)
-        variations.extend(edge_cases[:1])
-    
-    return variations[:target_multiplier]  # Limit total variations
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Augment training data for Stack 2.9")
-    parser.add_argument("--input", type=str, 
-                        default="training-data/tool_examples.jsonl",
-                        help="Input JSONL file")
-    parser.add_argument("--output", type=str, 
-                        default="training-data/augmented_tool_examples.jsonl",
-                        help="Output JSONL file")
-    parser.add_argument("--multiplier", type=int, default=3,
-                        help="Target multiplication factor (2-5)")
-    parser.add_argument("--seed", type=int, default=42,
-                        help="Random seed for reproducibility")
-    
-    args = parser.parse_args()
-    random.seed(args.seed)
-    
-    input_path = Path(args.input)
-    output_path = Path(args.output)
-    
-    if not input_path.exists():
-        print(f"Error: Input file not found: {input_path}")
-        return
-    
-    print(f"Loading data from: {input_path}")
-    examples = []
-    with open(input_path, 'r', encoding='utf-8') as f:
-        for line in f:
-            line = line.strip()
-            if line:
-                try:
-                    examples.append(json.loads(line))
-                except json.JSONDecodeError:
-                    continue
-    
-    original_count = len(examples)
-    print(f"Loaded {original_count} examples")
-    
-    # Generate augmented examples
-    all_variations = []
-    for ex in examples:
-        variations = augment_example(ex, target_multiplier=args.multiplier)
-        all_variations.extend(variations)
-    
-    total_count = len(all_variations)
-    
-    # Write output
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-    with open(output_path, 'w', encoding='utf-8') as f:
-        for var in all_variations:
-            f.write(json.dumps(var, ensure_ascii=False) + "\n")
-    
-    print(f"\nAugmentation complete!")
-    print(f"  Original: {original_count} examples")
-    print(f"  Augmented: {total_count} examples")
-    print(f"  Multiplier: {total_count/original_count:.1f}x")
-    print(f"  Output: {output_path}")
-
-
-if __name__ == "__main__":
-    main()

scripts/combine_all.py

@@ -1,135 +0,0 @@
-#!/usr/bin/env python3
-"""
-Final dataset combiner - loads all sources, deduplicates, splits.
-"""
-
-import json
-import hashlib
-import random
-from pathlib import Path
-from datetime import datetime
-import glob
-
-def hash_messages(messages: list) -> str:
-    """Create hash for deduplication."""
-    return hashlib.md5(json.dumps(messages, sort_keys=True).encode()).hexdigest()
-
-def main():
-    output_dir = Path("training-data/final")
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    # Glob all potential source files
-    source_files = []
-
-    # Synthetic sources
-    source_files.extend(glob.glob("training-data/synthetic/*.jsonl"))
-    source_files.extend(glob.glob("training-data/advanced-patterns/*.jsonl"))
-    source_files.extend(glob.glob("training-data/scaled/*.jsonl"))
-
-    # Code pairs (JSON format)
-    source_files.extend(glob.glob("training-data/code-pairs/*.json"))
-
-    print(f"🔍 Found {len(source_files)} source files")
-
-    all_examples = []
-    seen_hashes = set()
-    source_counts = {}
-
-    for file_path in source_files:
-        path = Path(file_path)
-        source_name = path.stem
-        count = 0
-
-        try:
-            with open(path, 'r') as f:
-                for line in f:
-                    line = line.strip()
-                    if not line:
-                        continue
-
-                    try:
-                        ex = json.loads(line)
-
-                        # Convert code-pair format to message format
-                        if "code" in ex and "comment" in ex and "messages" not in ex:
-                            ex = {
-                                "messages": [
-                                    {"role": "user", "content": f"Show me code for: {ex['comment'][:100]}"},
-                                    {"role": "assistant", "content": f"Here's a {ex.get('type', 'function')}:\n{ex['code']}"}
-                                ],
-                                "source_original": source_name,
-                                "type": "code_pair"
-                            }
-
-                        # Deduplication
-                        if "messages" in ex:
-                            msg_hash = hash_messages(ex["messages"])
-                            if msg_hash in seen_hashes:
-                                continue
-                            seen_hashes.add(msg_hash)
-
-                            # Track source
-                            ex["source_original"] = ex.get("source_original", source_name)
-                            all_examples.append(ex)
-                            count += 1
-                    except json.JSONDecodeError:
-                        continue
-
-        except Exception as e:
-            print(f"   ⚠️  Error reading {path}: {e}")
-            continue
-
-        source_counts[source_name] = count
-        if count > 0:
-            print(f"   ✅ {source_name}: {count} examples")
-
-    print(f"\n✨ Total unique examples: {len(all_examples)}")
-    print(f"   Deduplication removed: {sum(source_counts.values()) - len(all_examples)}")
-
-    # Shuffle
-    random.seed(42)
-    random.shuffle(all_examples)
-
-    # Splits (80/10/10)
-    n = len(all_examples)
-    n_train = int(n * 0.8)
-    n_val = int(n * 0.1)
-
-    splits = {
-        "train": all_examples[:n_train],
-        "val": all_examples[n_train:n_train+n_val],
-        "test": all_examples[n_train+n_val:]
-    }
-
-    for split_name, data in splits.items():
-        out_path = output_dir / f"{split_name}.jsonl"
-        with open(out_path, 'w') as f:
-            for ex in data:
-                f.write(json.dumps(ex) + "\n")
-        print(f"   📁 {split_name}: {len(data)} -> {out_path}")
-
-    # Manifest
-    manifest = {
-        "dataset": "Stack 2.9 Training Data",
-        "version": "1.0",
-        "created": datetime.now().isoformat(),
-        "total_examples": len(all_examples),
-        "splits": {name: len(data) for name, data in splits.items()},
-        "source_breakdown": source_counts,
-        "note": "Combined from multiple synthetic and code-pair sources"
-    }
-
-    with open(output_dir / "manifest.json", 'w') as f:
-        json.dump(manifest, f, indent=2)
-
-    print(f"\n✅ Final dataset ready!")
-    print(f"   Total: {len(all_examples)} examples")
-    print(f"   Manifest: {output_dir / 'manifest.json'}")
-
-    if len(all_examples) >= 50000:
-        print("\n🎉 TARGET ACHIEVED: 50,000+ examples!")
-    else:
-        print(f"\n⚠️  Still need {50000 - len(all_examples)} more to reach 50K target")
-
-if __name__ == "__main__":
-    main()

scripts/combine_datasets.py

@@ -1,144 +0,0 @@
-#!/usr/bin/env python3
-"""
-Combine all training data sources into final dataset.
-Applies deduplication and quality filtering.
-"""
-
-import json
-import hashlib
-from pathlib import Path
-import argparse
-from datetime import datetime
-
-def hash_messages(messages: list) -> str:
-    """Create a hash of messages to detect duplicates."""
-    m = hashlib.md5()
-    m.update(json.dumps(messages, sort_keys=True).encode())
-    return m.hexdigest()
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--output", type=str, default="training-data/final/dataset.jsonl")
-    parser.add_argument("--train-size", type=float, default=0.8)
-    parser.add_argument("--val-size", type=float, default=0.1)
-    parser.add_argument("--max-dataset", type=int, default=50000, help="Max examples to include")
-    args = parser.parse_args()
-
-    output_path = Path(args.output)
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-
-    # List all source files
-    sources = [
-        ("training-data/synthetic/examples.jsonl", "original_synthetic"),
-        ("training-data/advanced-patterns/examples.jsonl", "advanced_patterns"),
-        ("training-data/code-pairs/pairs.json", "code_pairs"),
-        ("training-data/code-pairs/extended_pairs.json", "code_pairs_extended"),
-        ("training-data/scaled/synthetic_final.jsonl", "synthetic_augmented"),
-        ("training-data/scaled/random_10k.jsonl", "random_10k"),
-        ("training-data/scaled/random_5_5k.jsonl", "random_5k"),
-    ]
-
-    all_examples = []
-    seen_hashes = set()
-    duplicates_removed = 0
-
-    print("📦 Combining datasets...")
-    for file_path, source in sources:
-        path = Path(file_path)
-        if not path.exists():
-            print(f"   ⚠️  Not found: {path}")
-            continue
-
-        print(f"   Loading {source}...")
-        count = 0
-        with open(path, 'r') as f:
-            for line in f:
-                try:
-                    ex = json.loads(line)
-
-                    # Convert code-pair format if needed
-                    if "code" in ex and "comment" in ex:
-                        # Convert code-pair to message format
-                        ex = {
-                            "messages": [
-                                {"role": "user", "content": ex["comment"]},
-                                {"role": "assistant", "content": f"Here's the code:\n{ex['code']}"}
-                            ],
-                            "source": source,
-                            "type": "code_pair"
-                        }
-
-                    # Deduplication
-                    msg_hash = hash_messages(ex["messages"])
-                    if msg_hash in seen_hashes:
-                        duplicates_removed += 1
-                        continue
-                    seen_hashes.add(msg_hash)
-
-                    # Add metadata
-                    ex["source_original"] = source
-                    all_examples.append(ex)
-                    count += 1
-
-                    if len(all_examples) >= args.max_dataset:
-                        break
-
-                except json.JSONDecodeError:
-                    continue
-
-        print(f"      ✅ Added {count} examples")
-
-    print(f"\n✨ Total collected: {len(all_examples)} examples")
-    print(f"   Duplicates removed: {duplicates_removed}")
-
-    # Shuffle
-    random.seed(42)
-    random.shuffle(all_examples)
-
-    # Split
-    n_total = len(all_examples)
-    n_train = int(n_total * args.train_size)
-    n_val = int(n_total * args.val_size)
-    n_test = n_total - n_train - n_val
-
-    train_set = all_examples[:n_train]
-    val_set = all_examples[n_train:n_train+n_val]
-    test_set = all_examples[n_train+n_val:]
-
-    print(f"\n📊 Split:")
-    print(f"   Train: {len(train_set)}")
-    print(f"   Val: {len(val_set)}")
-    print(f"   Test: {len(test_set)}")
-
-    # Save splits
-    for split_name, split_data in [("train", train_set), ("val", val_set), ("test", test_set)]:
-        split_path = output_path.parent / f"{split_name}.jsonl"
-        with open(split_path, 'w') as f:
-            for ex in split_data:
-                f.write(json.dumps(ex) + "\n")
-        print(f"   Saved {split_name} to {split_path}")
-
-    # Create manifest
-    manifest = {
-        "dataset": "Stack 2.9 Training Data",
-        "version": "1.0",
-        "created": datetime.now().isoformat(),
-        "total_examples": n_total,
-        "splits": {
-            "train": len(train_set),
-            "val": len(val_set),
-            "test": len(test_set)
-        },
-        "sources": {src: sum(1 for ex in all_examples if ex.get("source_original") == src) for src in set(ex.get("source_original") for ex in all_examples)}
-    }
-
-    manifest_path = output_path.parent / "manifest.json"
-    with open(manifest_path, 'w') as f:
-        json.dump(manifest, f, indent=2)
-    print(f"\n📄 Manifest: {manifest_path}")
-
-    print("\n✅ Dataset complete!")
-
-if __name__ == "__main__":
-    import random
-    main()

scripts/convert_gguf.py

@@ -1,141 +0,0 @@
-#!/usr/bin/env python3
-"""
-Convert GGUF file to HuggingFace format
-"""
-import os
-import sys
-
-# Check for required packages
-try:
-    import gguf
-except ImportError:
-    print("Installing gguf...")
-    os.system("pip install gguf -q")
-    import gguf
-
-try:
-    from transformers import AutoModel, AutoTokenizer
-except ImportError:
-    print("Installing transformers...")
-    os.system("pip install transformers -q")
-    from transformers import AutoModel, AutoTokenizer
-
-import torch
-
-GGUF_PATH = "/Users/walidsobhi/.ollama/models/blobs/sha256-60e05f2100071479f596b964f89f510f057ce397ea22f2833a0cfe029bfc2463"
-OUTPUT_DIR = "/Users/walidsobhi/.openclaw/workspace/stack-2.9/base_model_qwen7b"
-
-print(f"Reading GGUF from: {GGUF_PATH}")
-
-# Read the GGUF file
-reader = gguf.GGUFReader(GGUF_PATH)
-
-# Get tensor info
-print("\n GGUF Tensors:")
-for i, tensor in enumerate(reader.tensors):
-    print(f"  {i}: {tensor.name} - shape {tensor.shape}, dtype {tensor.tensor_type}")
-
-# Extract to HF format
-print("\n Converting to HuggingFace format...")
-
-# Create output directory
-os.makedirs(OUTPUT_DIR, exist_ok=True)
-
-# Save model files
-model_path = os.path.join(OUTPUT_DIR, "model.safetensors")
-
-# Map GGUF types to PyTorch types
-def gguf_to_torch_type(gguf_type):
-    type_map = {
-        "F32": torch.float32,
-        "F16": torch.float16,
-        "BF16": torch.bfloat16,
-        "I8": torch.int8,
-        "I16": torch.int16,
-        "I32": torch.int32,
-        "I64": torch.int64,
-        "U8": torch.uint8,
-    }
-    return type_map.get(gguf_type, torch.float32)
-
-# Export tensors
-state_dict = {}
-for tensor in reader.tensors:
-    print(f"  Converting {tensor.name}...")
-    # Read tensor data
-    data = reader.get_tensor(tensor.name)
-    state_dict[tensor.name] = data
-
-# Save as safetensors
-try:
-    from safetensors.torch import save_file
-    save_file(state_dict, model_path)
-    print(f"Model saved to: {model_path}")
-except ImportError:
-    # Fallback to torch
-    torch.save(state_dict, model_path.replace(".safetensors", ".pt"))
-    print(f"Model saved to: {model_path.replace('.safetensors', '.pt')}")
-
-# Save config.json
-config = {
-    "model_type": "qwen2",
-    "architectures": ["Qwen2ForCausalLM"],
-    "vocab_size": 151936,
-    "hidden_size": 3584,
-    "intermediate_size": 18944,
-    "num_hidden_layers": 28,
-    "num_attention_heads": 28,
-    "num_key_value_heads": 4,
-    "max_position_embeddings": 32768,
-    "sliding_window": 32768,
-    "torch_dtype": "bfloat16",
-    "transformers_version": "4.37.0",
-}
-
-import json
-config_path = os.path.join(OUTPUT_DIR, "config.json")
-with open(config_path, "w") as f:
-    json.dump(config, f, indent=2)
-print(f"Config saved to: {config_path}")
-
-# Create tokenizer files
-print("\n Creating tokenizer...")
-
-# Use Qwen2 tokenizer config
-tokenizer_config = {
-    "add_bos_token": False,
-    "add_eos_token": False,
-    "add_prefix_space": False,
-    "added_tokens_decoder": {},
-    "bos_token": "<|im_end|>",
-    "clean_up_tokenization_spaces": False,
-    "eos_token": "<|im_end|>",
-    "errors": "replace",
-    "model_max_length": 32768,
-    "pad_token": "<|im_end|>",
-    "tokenizer_class": "Qwen2Tokenizer",
-    "unk_token": "<|endoftext|>",
-}
-
-tokenizer_config_path = os.path.join(OUTPUT_DIR, "tokenizer_config.json")
-with open(tokenizer_config_path, "w") as f:
-    json.dump(tokenizer_config, f, indent=2)
-
-# Create a simple vocab file (this is a placeholder - real vocab is in the GGUF)
-# The GGUF reader should have tokenizer data
-vocab = {}
-for i in range(151936):
-    vocab[f"<|token_{i}|>"] = i
-
-vocab_path = os.path.join(OUTPUT_DIR, "vocab.json")
-with open(vocab_path, "w") as f:
-    json.dump(vocab, f)
-print(f"Vocab saved to: {vocab_path}")
-
-print("\n✓ Conversion complete!")
-print(f"Output directory: {OUTPUT_DIR}")
-print("\nFiles created:")
-for f in os.listdir(OUTPUT_DIR):
-    fpath = os.path.join(OUTPUT_DIR, f)
-    size = os.path.getsize(fpath) / (1024*1024)
-    print(f"  {f}: {size:.1f} MB")

scripts/convert_to_gguf.py

@@ -1,210 +0,0 @@
-#!/usr/bin/env python3
-"""
-GGUF Conversion Script for Stack 2.9 Model
-
-Converts the fine-tuned Stack 2.9 model to GGUF format for Ollama.
-"""
-
-import os
-import sys
-import subprocess
-import argparse
-from pathlib import Path
-
-
-def find_llama_cpp():
-    """Find llama.cpp directory in common locations."""
-    # Check common locations relative to this script
-    script_dir = Path(__file__).parent
-    workspace_root = script_dir.parent
-
-    possible_paths = [
-        workspace_root / "llama.cpp",
-        workspace_root / "extensions" / "llama.cpp",
-        Path.home() / "llama.cpp",
-        Path("/usr/local/llama.cpp"),
-    ]
-
-    for path in possible_paths:
-        if path.exists() and (path / "convert.py").exists():
-            return path
-
-    return None
-
-
-def run_command(cmd, check=True):
-    """Run a shell command and stream output."""
-    print(f"Running: {' '.join(cmd)}")
-    result = subprocess.run(cmd, capture_output=False, text=True)
-
-    if check and result.returncode != 0:
-        print(f"Error: Command failed with exit code {result.returncode}")
-        sys.exit(1)
-
-    return result
-
-
-def convert_model(model_path, output_path, quantize_type="q4_0", llama_cpp_path=None):
-    """
-    Convert a HuggingFace model to GGUF format using llama.cpp's convert.py.
-
-    Args:
-        model_path: Path to the input model (HuggingFace format)
-        output_path: Path where the GGUF file should be saved
-        quantize_type: Quantization type (e.g., q4_0, q5_0, q8_0)
-        llama_cpp_path: Path to llama.cpp directory (auto-detected if None)
-    """
-    model_path = Path(model_path).resolve()
-    output_path = Path(output_path).resolve()
-
-    # Validate input model exists
-    if not model_path.exists():
-        print(f"Error: Model directory not found: {model_path}")
-        sys.exit(1)
-
-    # Find llama.cpp if not provided
-    if llama_cpp_path is None:
-        llama_cpp_path = find_llama_cpp()
-        if llama_cpp_path is None:
-            print("Error: llama.cpp not found!")
-            print("\nPlease install llama.cpp and ensure convert.py is available.")
-            print("You can clone it with:")
-            print("  git clone https://github.com/ggerganov/llama.cpp.git")
-            print("\nOr specify the path manually:")
-            print("  python convert_to_gguf.py --llama-cpp /path/to/llama.cpp")
-            sys.exit(1)
-
-    llama_cpp_path = Path(llama_cpp_path).resolve()
-    convert_script = llama_cpp_path / "convert.py"
-
-    if not convert_script.exists():
-        print(f"Error: convert.py not found at {convert_script}")
-        sys.exit(1)
-
-    # Create output directory
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-
-    # Step 1: Convert to intermediate GGUF (unquantized)
-    print(f"\n=== Step 1: Converting to GGUF ===")
-    temp_gguf = output_path.parent / f"{output_path.stem}_temp.gguf"
-
-    convert_cmd = [
-        sys.executable,
-        str(convert_script),
-        str(model_path),
-        "--outfile", str(temp_gguf),
-        "--outtype", "f16",  # Intermediate full precision
-    ]
-
-    run_command(convert_cmd)
-
-    if not temp_gguf.exists():
-        print(f"Error: Conversion failed, {temp_gguf} not created")
-        sys.exit(1)
-
-    # Step 2: Quantize the GGUF file (if not full precision)
-    if quantize_type != "f16":
-        print(f"\n=== Step 2: Applying quantization ({quantize_type}) ===")
-
-        # llama.cpp quantize tool
-        quantize_tool = llama_cpp_path / "quantize"
-
-        # Try different possible names for quantize
-        if not quantize_tool.exists():
-            quantize_tool = llama_cpp_path / "build" / "bin" / "quantize"
-            if not quantize_tool.exists():
-                quantize_tool = llama_cpp_path / "build" / "quantize"
-
-        if not quantize_tool.exists():
-            print("Warning: quantize tool not found. Skipping quantization step.")
-            print("You may need to build llama.cpp first:")
-            print(f"  cd {llama_cpp_path} && make quantize")
-            print("Using unquantized model as fallback.")
-            final_gguf = temp_gguf
-        else:
-            run_command([
-                str(quantize_tool),
-                str(temp_gguf),
-                str(output_path),
-                quantize_type
-            ])
-            temp_gguf.unlink()  # Remove temp file
-            final_gguf = output_path
-    else:
-        final_gguf = temp_gguf
-        if final_gguf != output_path:
-            temp_gguf.rename(output_path)
-
-    # Step 3: Validate the GGUF file
-    print(f"\n=== Step 3: Validating GGUF file ===")
-
-    if not final_gguf.exists():
-        print(f"Error: Final GGUF file not found: {final_gguf}")
-        sys.exit(1)
-
-    file_size = final_gguf.stat().st_size / (1024**3)
-    print(f"✓ GGUF file created: {final_gguf}")
-    print(f"  Size: {file_size:.2f} GB")
-    print(f"  Quantization: {quantize_type}")
-
-    # Step 4: Print Ollama import command
-    print(f"\n=== Ollama Import Command ===")
-    print(f"ollama import {output_path} --alias stack-2.9:7b")
-    print("\nAfter importing, you can run the model with:")
-    print(f"  ollama run stack-2.9:7b")
-
-    print("\n✅ Conversion complete!")
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Convert Stack 2.9 model to GGUF format for Ollama"
-    )
-    parser.add_argument(
-        "--model-dir",
-        type=str,
-        default="./output/stack-2.9-7b-merged",
-        help="Path to the merged model directory (default: ./output/stack-2.9-7b-merged)"
-    )
-    parser.add_argument(
-        "--output",
-        type=str,
-        default="./ollama_model/stack-2.9-7b.gguf",
-        help="Output GGUF file path (default: ./ollama_model/stack-2.9-7b.gguf)"
-    )
-    parser.add_argument(
-        "--qtype",
-        type=str,
-        default="q4_0",
-        choices=["f16", "q4_0", "q5_0", "q8_0", "q2_K", "q3_K_S", "q3_K_M", "q3_K_L", "q4_K_S", "q4_K_M", "q5_K_S", "q5_K_M", "q6_K"],
-        help="Quantization type (default: q4_0)"
-    )
-    parser.add_argument(
-        "--llama-cpp",
-        type=str,
-        default=None,
-        help="Path to llama.cpp directory (auto-detected if not provided)"
-    )
-
-    args = parser.parse_args()
-
-    # Resolve paths relative to workspace root
-    workspace_root = Path(__file__).parent.parent
-    model_path = (workspace_root / args.model_dir).resolve()
-    output_path = (workspace_root / args.output).resolve()
-
-    print("=== GGUF Conversion for Stack 2.9 ===\n")
-    print(f"Input model: {model_path}")
-    print(f"Output: {output_path}")
-    print(f"Quantization: {args.qtype}\n")
-
-    convert_model(
-        model_path=model_path,
-        output_path=output_path,
-        quantize_type=args.qtype,
-        llama_cpp_path=args.llama_cpp
-    )
-
-
-if __name__ == "__main__":
-    main()

scripts/create_mini_dataset.py

@@ -1,180 +0,0 @@
-#!/usr/bin/env python3
-"""
-Create a minimal training dataset for rapid prototyping.
-Samples N examples from the full data/final/train.jsonl ensuring tool diversity.
-"""
-
-import argparse
-import json
-import random
-from pathlib import Path
-from typing import List, Dict
-from collections import defaultdict, Counter
-
-def load_full_dataset(train_path: str = "data/final/train.jsonl") -> List[Dict]:
-    """Load the full dataset."""
-    path = Path(train_path)
-    if not path.exists():
-        raise FileNotFoundError(f"Training data not found at {path}. Please ensure data/final/train.jsonl exists.")
-
-    data = []
-    with open(path, 'r') as f:
-        for line in f:
-            data.append(json.loads(line))
-    return data
-
-def extract_tool_calls(example: Dict) -> List[str]:
-    """Extract tool names used in an example."""
-    tools = []
-    messages = example.get("messages", [])
-    for msg in messages:
-        if msg.get("role") == "assistant" and msg.get("tool_calls"):
-            for tc in msg["tool_calls"]:
-                func = tc.get("function", {})
-                name = func.get("name", "")
-                if name:
-                    tools.append(name)
-    return tools
-
-def create_mini_dataset(
-    output_path: str,
-    n_samples: int = 5000,
-    train_source: str = "data/final/train.jsonl",
-    seed: int = 42
-):
-    """Create a stratified mini dataset."""
-    random.seed(seed)
-
-    print(f"Loading full dataset from {train_source}...")
-    full_data = load_full_dataset(train_source)
-    print(f"Loaded {len(full_data)} total examples")
-
-    # Group by tool usage
-    tool_groups = defaultdict(list)
-    unknown_tools = []
-
-    for ex in full_data:
-        tools = extract_tool_calls(ex)
-        if tools:
-            # Use first tool as primary category
-            primary_tool = tools[0]
-            tool_groups[primary_tool].append(ex)
-        else:
-            unknown_tools.append(ex)
-
-    print(f"\nTool distribution in full dataset:")
-    total_tool_examples = sum(len(v) for v in tool_groups.values())
-    for tool, examples in sorted(tool_groups.items(), key=lambda x: len(x[1]), reverse=True)[:15]:
-        pct = len(examples) / len(full_data) * 100
-        print(f"  {tool}: {len(examples)} examples ({pct:.1f}%)")
-
-    print(f"  No-tool examples: {len(unknown_tools)} ({len(unknown_tools)/len(full_data)*100:.1f}%)")
-
-    # Determine sampling strategy
-    # Allocate samples proportionally, but ensure minimum 3 examples per tool
-    samples_per_tool = {}
-    min_per_tool = 3
-    remaining = n_samples
-
-    # First pass: assign minimum to all tools that have enough
-    for tool, examples in tool_groups.items():
-        if len(examples) >= min_per_tool:
-            samples_per_tool[tool] = min_per_tool
-            remaining -= min_per_tool
-
-    # Second pass: distribute remaining proportionally
-    if remaining > 0:
-        total_weight = sum(len(v) for v in tool_groups.values() if len(v) >= min_per_tool)
-        for tool, examples in tool_groups.items():
-            if len(examples) >= min_per_tool:
-                weight = len(examples) / total_weight
-                extra = int(remaining * weight)
-                samples_per_tool[tool] += extra
-                remaining -= extra
-
-    # Fill any leftover with no-tool examples
-    if remaining > 0 and unknown_tools:
-        samples_per_tool["__notool__"] = min(remaining, len(unknown_tools))
-        remaining -= min(remaining, len(unknown_tools))
-
-    # If we still have remaining, just take from the largest tool groups
-    if remaining > 0:
-        sorted_tools = sorted(tool_groups.items(), key=lambda x: len(x[1]), reverse=True)
-        for tool, examples in sorted_tools:
-            if remaining <= 0:
-                break
-            can_take = min(remaining, len(examples) - samples_per_tool.get(tool, 0))
-            if can_take > 0:
-                samples_per_tool[tool] = samples_per_tool.get(tool, 0) + can_take
-                remaining -= can_take
-
-    print(f"\nSampling plan (target {n_samples}):")
-    total_sampled = 0
-    for tool, n in sorted(samples_per_tool.items(), key=lambda x: x[1], reverse=True):
-        if n > 0:
-            available = len(tool_groups.get(tool, [])) if tool != "__notool__" else len(unknown_tools)
-            pct = n / n_samples * 100
-            print(f"  {tool}: {n} examples ({pct:.1f}%) from {available} available")
-            total_sampled += n
-
-    # Perform sampling
-    mini_dataset = []
-    for tool, n_to_sample in samples_per_tool.items():
-        if n_to_sample <= 0:
-            continue
-
-        source_pool = tool_groups[tool] if tool != "__notool__" else unknown_tools
-        if len(source_pool) < n_to_sample:
-            n_to_sample = len(source_pool)
-
-        sampled = random.sample(source_pool, n_to_sample)
-        mini_dataset.extend(sampled)
-
-    # Shuffle the final dataset
-    random.shuffle(mini_dataset)
-
-    # Write output
-    output_path = Path(output_path)
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-
-    with open(output_path, 'w') as f:
-        for ex in mini_dataset:
-            f.write(json.dumps(ex) + '\n')
-
-    print(f"\n✅ Mini dataset created: {len(mini_dataset)} examples")
-    print(f"   Saved to: {output_path}")
-
-    # Stats
-    tool_counts = Counter()
-    for ex in mini_dataset:
-        tools = extract_tool_calls(ex)
-        if tools:
-            tool_counts[tools[0]] += 1
-        else:
-            tool_counts["__notool__"] += 1
-
-    print(f"\nFinal tool distribution:")
-    for tool, count in tool_counts.most_common(15):
-        pct = count / len(mini_dataset) * 100
-        print(f"  {tool}: {count} ({pct:.1f}%)")
-
-    return mini_dataset
-
-def main():
-    parser = argparse.ArgumentParser(description="Create mini dataset for fast prototyping")
-    parser.add_argument("--size", type=int, default=5000, help="Number of examples in mini dataset")
-    parser.add_argument("--output", type=str, default="./data_mini/train_mini.jsonl", help="Output file path")
-    parser.add_argument("--source", type=str, default="data/final/train.jsonl", help="Source full dataset")
-    parser.add_argument("--seed", type=int, default=42, help="Random seed for sampling")
-
-    args = parser.parse_args()
-
-    create_mini_dataset(
-        output_path=args.output,
-        n_samples=args.size,
-        train_source=args.source,
-        seed=args.seed
-    )
-
-if __name__ == "__main__":
-    main()

scripts/download_benchmark_datasets.py

@@ -1,127 +0,0 @@
-#!/usr/bin/env python3
-"""
-Download benchmark datasets (HumanEval and MBPP) into ./data/ directory.
-Uses huggingface datasets library for reliable downloads.
-"""
-
-import os
-import json
-from pathlib import Path
-from datasets import load_dataset
-import argparse
-
-def download_humaneval(output_dir: str = "./data"):
-    """Download HumanEval dataset (164 problems)."""
-    output_path = Path(output_dir) / "humaneval"
-    output_path.mkdir(parents=True, exist_ok=True)
-
-    print(f"⬇️  Downloading HumanEval to {output_path}...")
-
-    try:
-        # Load HumanEval from huggingface
-        dataset = load_dataset("openai_humaneval", split="test")
-
-        problems = {}
-        for idx, item in enumerate(dataset):
-            problem_id = f"HumanEval/{idx}"
-            problems[problem_id] = {
-                "task_id": problem_id,
-                "prompt": item["prompt"],
-                "canonical_solution": item["canonical_solution"],
-                "test": item["test"],
-                "entry_point": item["entry_point"]
-            }
-
-        # Save as JSONL (one problem per line)
-        output_file = output_path / "humaneval.jsonl"
-        with open(output_file, 'w') as f:
-            for problem in problems.values():
-                f.write(json.dumps(problem) + '\n')
-
-        # Also save a meta file
-        meta_file = output_path / "meta.json"
-        with open(meta_file, 'w') as f:
-            json.dump({
-                "name": "HumanEval",
-                "num_problems": len(problems),
-                "source": "openai_humaneval",
-                "description": "164 hand-written programming problems"
-            }, f, indent=2)
-
-        print(f"✅ HumanEval: {len(problems)} problems saved to {output_file}")
-        return len(problems)
-
-    except Exception as e:
-        print(f"❌ Failed to download HumanEval: {e}")
-        return 0
-
-def download_mbpp(output_dir: str = "./data"):
-    """Download MBPP dataset (500 problems)."""
-    output_path = Path(output_dir) / "mbpp"
-    output_path.mkdir(parents=True, exist_ok=True)
-
-    print(f"⬇️  Downloading MBPP to {output_path}...")
-
-    try:
-        # Load MBPP from huggingface
-        dataset = load_dataset("mbpp", split="test")
-
-        problems = {}
-        for idx, item in enumerate(dataset):
-            problem_id = f"MBPP/{idx}"
-            problems[problem_id] = {
-                "task_id": problem_id,
-                "text": item["text"],
-                "code": item["code"],
-                "test_list": item["test_list"],
-                "test_func": item["test_func"],
-                "challenge_test_list": item.get("challenge_test_list", [])
-            }
-
-        # Save as JSONL
-        output_file = output_path / "mbpp.jsonl"
-        with open(output_file, 'w') as f:
-            for problem in problems.values():
-                f.write(json.dumps(problem) + '\n')
-
-        # Meta file
-        meta_file = output_path / "meta.json"
-        with open(meta_file, 'w') as f:
-            json.dump({
-                "name": "MBPP",
-                "num_problems": len(problems),
-                "source": "mbpp",
-                "description": "500 beginner-friendly Python programming problems"
-            }, f, indent=2)
-
-        print(f"✅ MBPP: {len(problems)} problems saved to {output_file}")
-        return len(problems)
-
-    except Exception as e:
-        print(f"❌ Failed to download MBPP: {e}")
-        return 0
-
-def main():
-    parser = argparse.ArgumentParser(description="Download benchmark datasets")
-    parser.add_argument("--output-dir", type=str, default="./data",
-                        help="Output directory (default: ./data)")
-    parser.add_argument("--benchmark", type=str, choices=["humaneval", "mbpp", "both"],
-                        default="both", help="Which benchmark to download")
-    args = parser.parse_args()
-
-    print("📥 Benchmark Dataset Downloader")
-    print(f"📁 Target directory: {args.output_dir}")
-
-    total_downloaded = 0
-
-    if args.benchmark in ["humaneval", "both"]:
-        total_downloaded += download_humaneval(args.output_dir)
-
-    if args.benchmark in ["mbpp", "both"]:
-        total_downloaded += download_mbpp(args.output_dir)
-
-    print(f"\n🎉 Total problems downloaded: {total_downloaded}")
-    print(f"📂 Data saved in: {Path(args.output_dir).resolve()}")
-
-if __name__ == "__main__":
-    main()

scripts/download_public_datasets.py

@@ -1,170 +0,0 @@
-#!/usr/bin/env python3
-"""
-Download and integrate public coding datasets.
-Datasets: OpenAssistant, CodeAct, CodeContests
-Converts to Stack 2.9 format.
-"""
-
-import json
-import os
-from datasets import load_dataset
-from pathlib import Path
-import argparse
-
-def download_openassistant(output_path: Path, limit: int = 10000):
-    """Download OpenAssistant and filter for coding conversations."""
-    print("📥 Downloading OpenAssistant dataset...")
-    try:
-        dataset = load_dataset("OpenAssistant/oasst1", split="train")
-    except Exception as e:
-        print(f"❌ Failed to load OpenAssistant: {e}")
-        return []
-
-    coding_examples = []
-    count = 0
-
-    for item in dataset:
-        # Filter for coding-related conversations
-        text = item.get("text", "").lower()
-        if any(keyword in text for keyword in ["code", "programming", "python", "javascript", "function", "api", "development"]):
-            # Convert to our format
-            messages = [
-                {"role": "user", "content": item.get("text", "")[:1000]},  # truncated
-                {"role": "assistant", "content": "Here's a coding assistant response..."}
-            ]
-            coding_examples.append({
-                "messages": messages,
-                "source": "openassistant",
-                "dataset": "oasst1"
-            })
-            count += 1
-            if count >= limit:
-                break
-
-    print(f"   Extracted {len(coding_examples)} coding-related examples from OpenAssistant")
-    return coding_examples
-
-def download_codeact(output_path: Path, limit: int = 10000):
-    """Download CodeAct dataset."""
-    print("📥 Downloading CodeAct dataset...")
-    try:
-        dataset = load_dataset("nuprl/CodeAct", split="train")
-    except Exception as e:
-        print(f"❌ Failed to load CodeAct: {e}")
-        return []
-
-    examples = []
-    count = 0
-
-    for item in dataset:
-        # CodeAct has actions - convert to tool calls
-        action = item.get("action", {})
-        if action:
-            messages = [
-                {"role": "user", "content": item.get("instruction", "")},
-                {
-                    "role": "assistant",
-                    "content": "Executing action...",
-                    "tool_use": {
-                        "name": "CodeActTool",
-                        "input": action
-                    }
-                },
-                {
-                    "role": "user",
-                    "content": "",
-                    "tool_result": {
-                        "tool_use_id": "tool_1",
-                        "content": json.dumps(item.get("observation", {}))
-                    }
-                },
-                {"role": "assistant", "content": item.get("final_answer", "Done.")}
-            ]
-            examples.append({
-                "messages": messages,
-                "source": "codeact",
-                "dataset": "CodeAct"
-            })
-            count += 1
-            if count >= limit:
-                break
-
-    print(f"   Extracted {len(examples)} examples from CodeAct")
-    return examples
-
-def download_codecontests(output_path: Path, limit: int = 5000):
-    """Download CodeContests (competition problems)."""
-    print("📥 Downloading CodeContests dataset...")
-    try:
-        dataset = load_dataset("m-a-p/CodeContests", split="train")
-    except Exception as e:
-        print(f"❌ Failed to load CodeContests: {e}")
-        return []
-
-    examples = []
-    count = 0
-
-    for item in dataset:
-        if item.get("problem") and item.get("solution"):
-            messages = [
-                {"role": "user", "content": f"Solve this problem:\n{item['problem']}"},
-                {"role": "assistant", "content": f"Here's a solution:\n```python\n{item['solution']}\n```"}
-            ]
-            examples.append({
-                "messages": messages,
-                "source": "codecontests",
-                "dataset": "CodeContests"
-            })
-            count += 1
-            if count >= limit:
-                break
-
-    print(f"   Extracted {len(examples)} examples from CodeContests")
-    return examples
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--output", type=str, default="training-data/scaled/public_datasets.jsonl")
-    parser.add_argument("--limit-per-dataset", type=int, default=10000)
-    parser.add_argument("--skip-download", action="store_true", help="Use only existing datasets")
-    args = parser.parse_args()
-
-    output_path = Path(args.output)
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-
-    all_examples = []
-
-    if not args.skip_download:
-        # OpenAssistant
-        all_examples.extend(download_openassistant(output_path, args.limit_per_dataset))
-
-        # CodeAct
-        all_examples.extend(download_codeact(output_path, args.limit_per_dataset))
-
-        # CodeContests
-        all_examples.extend(download_codecontests(output_path, min(5000, args.limit_per_dataset)))
-    else:
-        print("⚠️  Skipping downloads (--skip-download flag)")
-
-    # Write all examples
-    with open(output_path, 'w') as f:
-        for ex in all_examples:
-            f.write(json.dumps(ex) + "\n")
-
-    print(f"\n✨ Saved {len(all_examples)} examples from public datasets")
-    print(f"   to: {output_path}")
-
-    # Show breakdown
-    sources = {}
-    for ex in all_examples:
-        src = ex.get("source", "unknown")
-        sources[src] = sources.get(src, 0) + 1
-
-    print("\n📊 Breakdown:")
-    for src, count in sources.items():
-        print(f"   {src}: {count}")
-
-    print("\n⚠️  Note: These are raw integrations. May need format conversion to match tool-use patterns.")
-
-if __name__ == "__main__":
-    main()

scripts/extract_code_pairs.py

@@ -1,215 +0,0 @@
-#!/usr/bin/env python3
-"""
-Extract code-comment pairs from the src/ directory.
-Pairs: function/class code + its documentation comment.
-"""
-
-import os
-import re
-import json
-from pathlib import Path
-from typing import List, Dict, Any
-import argparse
-
-def extract_jsdoc_comments(content: str) -> List[Dict[str, Any]]:
-    """Extract JSDoc comments and associated code from JS/TS files."""
-    pairs = []
-
-    # Pattern to match JSDoc comment block followed by code
-    # Matches: /** ... */ followed by function/class/interface
-    pattern = re.compile(
-        r'/\*\*\s*(.*?)\s*\*/\s*'  # JSDoc comment
-        r'(export\s+)?(async\s+)?(function|const|let|var|class|interface|type)\s+(\w+)',
-        re.DOTALL
-    )
-
-    for match in pattern.finditer(content):
-        comment_lines = match.group(1).strip().split('\n')
-        # Clean up comment markers
-        comment = []
-        for line in comment_lines:
-            line = line.strip()
-            if line.startswith('* '):
-                line = line[2:]
-            elif line.startswith('*'):
-                line = line[1:]
-            comment.append(line.strip())
-        comment_text = ' '.join(comment).strip()
-
-        code_start = match.end()
-        # Extract the function signature or class definition (up to opening brace or newline)
-        code_lines = []
-        lines = content[code_start:].split('\n')
-        for line in lines[:5]:  # Take first few lines
-            code_lines.append(line)
-            if line.strip().endswith('{') or line.strip().endswith('>'):
-                break
-        code = '\n'.join(code_lines).strip()
-
-        if comment_text and code and len(code.split('\n')) >= 2:
-            pairs.append({
-                "code": code,
-                "comment": comment_text,
-                "type": match.group(3),  # function/class/interface
-                "name": match.group(4)
-            })
-
-    return pairs
-
-def extract_python_docstrings(content: str) -> List[Dict[str, Any]]:
-    """Extract Python docstrings and associated code."""
-    pairs = []
-
-    # Pattern for triple-quoted docstring before function/class
-    pattern = re.compile(
-        r'''(?P<quote>''' + r'"""' + r'''|\'\'\')\s*(?P<doc>.*?)(?P=quote)\s*'''
-        r'(?:@\w+\s+)*def\s+(\w+)|class\s+(\w+)',
-        re.DOTALL
-    )
-
-    for match in pattern.finditer(content):
-        doc = match.group('doc').strip()
-        func_name = match.group(3) or match.group(4)
-        if func_name:
-            # Get the signature line
-            signature = content[match.end():].split('\n')[0].strip()
-            code = f"def {func_name}{signature}" if 'def' in signature else f"class {func_name}{signature}"
-
-            pairs.append({
-                "code": code,
-                "comment": doc,
-                "type": "function" if 'def' in signature else "class",
-                "name": func_name
-            })
-
-    return pairs
-
-def extract_inline_comments(content: str, file_ext: str) -> List[Dict[str, Any]]:
-    """Extract code block with preceding inline comment."""
-    pairs = []
-
-    lines = content.split('\n')
-    i = 0
-    while i < len(lines):
-        line = lines[i].rstrip()
-        # Check for // comment or # comment
-        if line.strip().startswith('//') or line.strip().startswith('#'):
-            comment = line.strip()[2:].strip()
-            # Look at next few lines for code
-            code_lines = []
-            j = i + 1
-            while j < len(lines) and len(code_lines) < 5:
-                next_line = lines[j].rstrip()
-                if next_line.strip() and not next_line.strip().startswith('//') and not next_line.strip().startswith('#'):
-                    code_lines.append(next_line)
-                elif next_line.strip().startswith(('//', '#')):
-                    break  # Another comment block
-                j += 1
-
-            if comment and code_lines:
-                code = '\n'.join(code_lines)
-                # Only keep if comment is meaningful (>5 words or contains specific keywords)
-                if len(comment.split()) > 3 or any(kw in comment.lower() for kw in ['function', 'return', 'parameter', 'args', 'handle', 'process']):
-                    pairs.append({
-                        "code": code,
-                        "comment": comment,
-                        "type": "inline",
-                        "name": None
-                    })
-                i = j  # Skip processed lines
-            else:
-                i += 1
-        else:
-            i += 1
-
-    return pairs
-
-def process_file(file_path: Path) -> List[Dict[str, Any]]:
-    """Process a single file and extract code-comment pairs."""
-    try:
-        with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
-            content = f.read()
-    except Exception as e:
-        print(f"❌ Error reading {file_path}: {e}")
-        return []
-
-    pairs = []
-
-    # Extract by file type
-    if file_path.suffix in ['.js', '.ts', '.jsx', '.tsx']:
-        pairs.extend(extract_jsdoc_comments(content))
-    elif file_path.suffix == '.py':
-        pairs.extend(extract_python_docstrings(content))
-
-    # Inline comments for all types
-    pairs.extend(extract_inline_comments(content, file_path.suffix))
-
-    return pairs
-
-def walk_source_files(src_dir: Path) -> List[Path]:
-    """Walk src/ directory and return all relevant source files."""
-    extensions = ['.ts', '.tsx', '.js', '.jsx', '.py']
-    files = []
-    for ext in extensions:
-        files.extend(src_dir.rglob(f'*{ext}'))
-    return files
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--src-dir", type=str, default="src")
-    parser.add_argument("--output", type=str, default="training-data/code-pairs/extended_pairs.json")
-    parser.add_argument("--limit", type=int, default=10000, help="Maximum pairs to extract")
-    args = parser.parse_args()
-
-    src_dir = Path(args.src_dir)
-    output_path = Path(args.output)
-
-    if not src_dir.exists():
-        print(f"❌ Source directory not found: {src_dir}")
-        return
-
-    print(f"🔍 Scanning {src_dir} for source files...")
-    files = walk_source_files(src_dir)
-    print(f"   Found {len(files)} source files")
-
-    all_pairs = []
-    for file_path in files:
-        pairs = process_file(file_path)
-        if pairs:
-            all_pairs.extend(pairs)
-            print(f"   {file_path.name}: {len(pairs)} pairs", end='\r')
-
-        if len(all_pairs) >= args.limit:
-            break
-
-    print(f"\n✨ Extracted {len(all_pairs)} code-comment pairs")
-
-    # Deduplicate (by comment+code hash)
-    seen = set()
-    unique_pairs = []
-    for pair in all_pairs:
-        key = (pair['comment'][:100], pair['code'][:100])
-        if key not in seen:
-            seen.add(key)
-            unique_pairs.append(pair)
-
-    print(f"   After deduplication: {len(unique_pairs)} unique pairs")
-
-    # Save
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-    with open(output_path, 'w') as f:
-        json.dump(unique_pairs, f, indent=2)
-
-    print(f"✅ Saved to: {output_path}")
-
-    # Stats
-    types = {}
-    for pair in unique_pairs:
-        t = pair.get('type', 'unknown')
-        types[t] = types.get(t, 0) + 1
-    print("\n📊 By type:")
-    for t, cnt in types.items():
-        print(f"   {t}: {cnt}")
-
-if __name__ == "__main__":
-    main()

scripts/extract_patterns_from_git.py

@@ -1,309 +0,0 @@
-#!/usr/bin/env python3
-"""
-Extract Code Patterns from Git History
-
-Scans Git commit history to identify bug fixes and feature additions,
-extracting "before → after" patterns for training data generation.
-
-Usage:
-    python extract_patterns_from_git.py --repo-path . --output patterns.jsonl
-    python extract_patterns_from_git.py --repo-path . --output patterns.jsonl --since-date "2024-01-01"
-"""
-
-import argparse
-import hashlib
-import json
-import os
-import subprocess
-import sys
-from datetime import datetime
-from pathlib import Path
-from typing import Optional
-
-try:
-    from tqdm import tqdm
-except ImportError:
-    tqdm = None
-
-
-# Keywords that indicate bug fixes or improvements
-BUG_FIX_KEYWORDS = [
-    "fix", "bug", "hotfix", "patch", "resolve", "correct", "repair",
-    "error", "crash", "fail", "issue", "problem", "broken"
-]
-
-FEATURE_KEYWORDS = [
-    "feat", "feature", "add", "new", "implement", "enhance", "improve",
-    "optimize", "refactor", "support", "introduce"
-]
-
-
-def is_text_file(filepath: str) -> bool:
-    """Check if a file is likely a text file (not binary)."""
-    binary_extensions = {
-        '.pyc', '.so', '.dll', '.exe', '.bin', '.dat', '.pickle',
-        '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.ico', '.svg',
-        '.mp3', '.mp4', '.wav', '.avi', '.mov', '.pdf', '.zip',
-        '.tar', '.gz', '.rar', '.7z', '.whl', '.egg',
-        '.class', '.jar', '.war', '.ear',
-        '.db', '.sqlite', '.sqlite3',
-        '.ttf', '.otf', '.woff', '.woff2',
-        '.pem', '.key', '.crt', '.cer',
-        '.DS_Store', '.gitignore'
-    }
-    
-    ext = Path(filepath).suffix.lower()
-    if ext in binary_extensions:
-        return False
-    
-    # Try to read as text
-    try:
-        with open(filepath, 'rb') as f:
-            chunk = f.read(1024)
-            # Check for null bytes (common in binary files)
-            if b'\x00' in chunk:
-                return False
-        return True
-    except (OSError, IOError):
-        return False
-
-
-def get_commit_messages(repo_path: str, since_date: Optional[str] = None) -> list[dict]:
-    """Get commit information from git log."""
-    cmd = ["git", "-C", repo_path, "log", "--pretty=format:%H|%s|%an|%ad|%ae", "--date=iso"]
-    
-    if since_date:
-        cmd.extend([f"--since={since_date}"])
-    
-    try:
-        result = subprocess.run(cmd, capture_output=True, text=True, check=True)
-        commits = []
-        
-        for line in result.stdout.strip().split('\n'):
-            if not line:
-                continue
-            parts = line.split('|')
-            if len(parts) >= 5:
-                commits.append({
-                    'hash': parts[0],
-                    'message': parts[1],
-                    'author': parts[2],
-                    'date': parts[3],
-                    'email': parts[4] if len(parts) > 4 else ''
-                })
-        
-        return commits
-    except subprocess.CalledProcessError as e:
-        print(f"Error reading git log: {e}", file=sys.stderr)
-        return []
-
-
-def get_changed_files(repo_path: str, commit_hash: str) -> list[str]:
-    """Get list of files changed in a commit."""
-    cmd = ["git", "-C", repo_path, "diff-tree", "--no-commit-id", "--name-only", "-r", commit_hash]
-    
-    try:
-        result = subprocess.run(cmd, capture_output=True, text=True, check=True)
-        files = []
-        for line in result.stdout.strip().split('\n'):
-            if line.strip():
-                files.append(line.strip())
-        return files
-    except subprocess.CalledProcessError:
-        return []
-
-
-def get_file_diff(repo_path: str, commit_hash: str, filepath: str) -> tuple[Optional[str], Optional[str]]:
-    """Get before and after content of a file in a commit."""
-    # Get the file content AFTER the commit
-    cmd_after = ["git", "-C", repo_path, "show", f"{commit_hash}:{filepath}"]
-    # Get the file content BEFORE the commit (parent)
-    cmd_before = ["git", "-C", repo_path, "show", f"{commit_hash}^:{filepath}"]
-    
-    after_content = None
-    before_content = None
-    
-    try:
-        result_after = subprocess.run(cmd_after, capture_output=True, text=True, check=True)
-        after_content = result_after.stdout
-    except subprocess.CalledProcessError:
-        # File might be new (no parent)
-        after_content = None
-    
-    try:
-        result_before = subprocess.run(cmd_before, capture_output=True, text=True, check=True)
-        before_content = result_before.stdout
-    except subprocess.CalledProcessError:
-        # File was added in this commit
-        before_content = None
-    
-    return before_content, after_content
-
-
-def infer_problem_type(message: str) -> str:
-    """Infer the problem type from commit message."""
-    msg_lower = message.lower()
-    
-    # Check for bug fix indicators
-    for keyword in BUG_FIX_KEYWORDS:
-        if keyword in msg_lower:
-            return "bug_fix"
-    
-    # Check for feature indicators
-    for keyword in FEATURE_KEYWORDS:
-        if keyword in msg_lower:
-            return "feature_addition"
-    
-    return "unknown"
-
-
-def compute_confidence(message: str, before: Optional[str], after: Optional[str]) -> float:
-    """Compute confidence score for the extracted pattern."""
-    confidence = 0.5  # Base confidence
-    
-    # Higher confidence if message contains clear keywords
-    msg_lower = message.lower()
-    if any(k in msg_lower for k in ["fix", "bug", "hotfix", "patch"]):
-        confidence += 0.2
-    if any(k in msg_lower for k in ["feat", "feature", "add", "implement"]):
-        confidence += 0.15
-    
-    # Higher confidence if we have both before and after
-    if before and after:
-        confidence += 0.15
-    elif before or after:
-        confidence += 0.05
-    
-    # Higher confidence for substantial changes
-    if before and after:
-        content_len = max(len(before), len(after))
-        if content_len > 100:
-            confidence += 0.1
-        if content_len > 500:
-            confidence += 0.1
-    
-    return min(confidence, 1.0)
-
-
-def generate_pattern_id(commit_hash: str, filepath: str) -> str:
-    """Generate a unique pattern ID."""
-    content = f"{commit_hash}:{filepath}"
-    return hashlib.sha256(content.encode()).hexdigest()[:16]
-
-
-def extract_patterns(
-    repo_path: str,
-    output_path: str,
-    since_date: Optional[str] = None
-) -> int:
-    """Extract patterns from git history and write to JSONL file."""
-    
-    print(f"Scanning repository: {repo_path}")
-    
-    # Get all commits
-    commits = get_commit_messages(repo_path, since_date)
-    print(f"Found {len(commits)} commits")
-    
-    if not commits:
-        print("No commits found.", file=sys.stderr)
-        return 0
-    
-    patterns_extracted = 0
-    
-    # Process each commit with progress bar
-    iterator = tqdm(commits, desc="Extracting patterns") if tqdm else commits
-    
-    with open(output_path, 'w', encoding='utf-8') as outf:
-        for commit in iterator:
-            commit_hash = commit['hash']
-            message = commit['message']
-            author = commit['author']
-            date = commit['date']
-            
-            # Infer problem type
-            problem_type = infer_problem_type(message)
-            
-            # Skip if not a bug fix or feature
-            if problem_type == "unknown":
-                continue
-            
-            # Get changed files
-            changed_files = get_changed_files(repo_path, commit_hash)
-            
-            for filepath in changed_files:
-                # Skip binary files
-                full_path = os.path.join(repo_path, filepath)
-                if not os.path.exists(full_path):
-                    continue
-                
-                if not is_text_file(filepath):
-                    continue
-                
-                # Get diff
-                before_content, after_content = get_file_diff(repo_path, commit_hash, filepath)
-                
-                # Skip if no meaningful change
-                if before_content == after_content:
-                    continue
-                if not before_content and not after_content:
-                    continue
-                
-                # Compute confidence
-                confidence = compute_confidence(message, before_content, after_content)
-                
-                # Create pattern record
-                pattern = {
-                    "pattern_id": generate_pattern_id(commit_hash, filepath),
-                    "problem_type": problem_type,
-                    "before_code": before_content or "",
-                    "after_code": after_content or "",
-                    "commit_msg": message,
-                    "author": author,
-                    "date": date,
-                    "confidence": round(confidence, 2)
-                }
-                
-                # Write as JSONL
-                outf.write(json.dumps(pattern, ensure_ascii=False) + '\n')
-                patterns_extracted += 1
-    
-    print(f"\nExtracted {patterns_extracted} patterns to {output_path}")
-    return patterns_extracted
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Extract code patterns from Git history for training data"
-    )
-    parser.add_argument(
-        "--repo-path",
-        type=str,
-        required=True,
-        help="Path to the Git repository"
-    )
-    parser.add_argument(
-        "--output",
-        type=str,
-        required=True,
-        help="Output JSONL file path"
-    )
-    parser.add_argument(
-        "--since-date",
-        type=str,
-        default=None,
-        help="Only extract commits since this date (YYYY-MM-DD)"
-    )
-    
-    args = parser.parse_args()
-    
-    # Validate repo path
-    if not os.path.isdir(os.path.join(args.repo_path, '.git')):
-        print(f"Error: {args.repo_path} is not a Git repository", file=sys.stderr)
-        sys.exit(1)
-    
-    # Run extraction
-    extract_patterns(args.repo_path, args.output, args.since_date)
-
-
-if __name__ == "__main__":
-    main()

scripts/extract_rtmp_tools.py

@@ -1,174 +0,0 @@
-#!/usr/bin/env python3
-"""
-Extract training data from RTMP tools for Stack 2.9
-Creates synthetic tool-use examples from the RTMP codebase
-"""
-
-import os
-import json
-from pathlib import Path
-
-RTMP_DIR = "/Users/walidsobhi/.openclaw/workspace/RTMP"
-OUTPUT_DIR = "/Users/walidsobhi/.openclaw/workspace/stack-2.9/data/rtmp-tools"
-
-def get_tool_description(tool_name: str) -> str:
-    """Get tool descriptions from tool names"""
-    descriptions = {
-        "BashTool": "Execute shell commands in a sandboxed environment",
-        "FileReadTool": "Read file contents from the filesystem",
-        "FileWriteTool": "Write content to files",
-        "FileEditTool": "Edit files using sed-style replacements",
-        "GlobTool": "Find files matching glob patterns",
-        "GrepTool": "Search for patterns in files",
-        "TaskCreateTool": "Create tasks in the task list",
-        "TaskListTool": "List all tasks in the task list",
-        "TaskUpdateTool": "Update task status and details",
-        "TaskGetTool": "Get details of a specific task",
-        "WebSearchTool": "Search the web for information",
-        "WebFetchTool": "Fetch and analyze web pages",
-        "SkillTool": "Execute user-invocable skills",
-        "McpTool": "Call MCP (Model Context Protocol) tools",
-        "AgentTool": "Delegate tasks to sub-agents",
-    }
-    return descriptions.get(tool_name.replace("Tool", ""), f"Tool: {tool_name}")
-
-def extract_tool_examples():
-    """Extract tool patterns and create training examples"""
-    examples = []
-
-    tools_dir = Path(RTMP_DIR) / "tools"
-    if not tools_dir.exists():
-        print(f"❌ Tools directory not found: {tools_dir}")
-        return examples
-
-    # Process each tool
-    tool_dirs = [d for d in tools_dir.iterdir() if d.is_dir() and d.name.endswith("Tool")]
-
-    for tool_dir in tool_dirs:
-        tool_name = tool_dir.name
-
-        # Get main tool file
-        main_files = list(tool_dir.glob("*.tsx")) + list(tool_dir.glob("*.ts"))
-
-        # Create example for this tool
-        desc = get_tool_description(tool_name)
-
-        # Generate prompt-response pairs for tool usage
-        example = {
-            "messages": [
-                {
-                    "role": "user",
-                    "content": f"How do I use the {tool_name}? What does it do?"
-                },
-                {
-                    "role": "assistant",
-                    "content": f"The {tool_name} allows you to {desc.lower()}.\n\n" +
-                              f"This is one of the core tools in RTMP (Real-Time AI Assistant).\n\n" +
-                              f"Tools like {tool_name} enable the assistant to interact with the filesystem, " +
-                              f"execute commands, manage tasks, and more."
-                }
-            ]
-        }
-        examples.append(example)
-
-        # Add more specific examples for common tools
-        if "Bash" in tool_name:
-            example = {
-                "messages": [
-                    {"role": "user", "content": "List all files in the current directory"},
-                    {"role": "assistant", "tool_calls": [
-                        {"name": "Bash", "arguments": {"command": "ls -la"}}
-                    ]}
-                ]
-            }
-            examples.append(example)
-
-        elif "FileRead" in tool_name:
-            example = {
-                "messages": [
-                    {"role": "user", "content": "Read the contents of config.json"},
-                    {"role": "assistant", "tool_calls": [
-                        {"name": "FileRead", "arguments": {"file_path": "config.json"}}
-                    ]}
-                ]
-            }
-            examples.append(example)
-
-        elif "Glob" in tool_name:
-            example = {
-                "messages": [
-                    {"role": "user", "content": "Find all TypeScript files in the project"},
-                    {"role": "assistant", "tool_calls": [
-                        {"name": "Glob", "arguments": {"pattern": "**/*.ts"}}
-                    ]}
-                ]
-            }
-            examples.append(example)
-
-        elif "Grep" in tool_name:
-            example = {
-                "messages": [
-                    {"role": "user", "content": "Find all occurrences of 'TODO' in the code"},
-                    {"role": "assistant", "tool_calls": [
-                        {"name": "Grep", "arguments": {"pattern": "TODO", "path": "."}}
-                    ]}
-                ]
-            }
-            examples.append(example)
-
-        elif "TaskCreate" in tool_name:
-            example = {
-                "messages": [
-                    {"role": "user", "content": "Create a task to fix the login bug"},
-                    {"role": "assistant", "tool_calls": [
-                        {"name": "TaskCreate", "arguments": {
-                            "subject": "Fix login bug",
-                            "description": "Investigate and fix the login issue"
-                        }}
-                    ]}
-                ]
-            }
-            examples.append(example)
-
-        elif "WebSearch" in tool_name:
-            example = {
-                "messages": [
-                    {"role": "user", "content": "Search for latest Python 3.14 features"},
-                    {"role": "assistant", "tool_calls": [
-                        {"name": "WebSearch", "arguments": {"query": "Python 3.14 new features"}}
-                    ]}
-                ]
-            }
-            examples.append(example)
-
-    return examples
-
-def main():
-    print("=" * 60)
-    print("Extracting RTMP Tool Patterns for Training")
-    print("=" * 60)
-
-    # Create output directory
-    os.makedirs(OUTPUT_DIR, exist_ok=True)
-
-    # Extract examples
-    examples = extract_tool_examples()
-
-    print(f"\n✅ Extracted {len(examples)} tool usage examples")
-
-    # Save to JSONL
-    output_file = os.path.join(OUTPUT_DIR, "tool_patterns.jsonl")
-    with open(output_file, 'w') as f:
-        for ex in examples:
-            f.write(json.dumps(ex) + '\n')
-
-    print(f"✅ Saved to: {output_file}")
-
-    # Also show some examples
-    print("\n📋 Sample examples:")
-    for i, ex in enumerate(examples[:3]):
-        user_msg = ex["messages"][0]["content"]
-        print(f"  {i+1}. User: {user_msg[:60]}...")
-
-if __name__ == "__main__":
-    main()

scripts/extract_rtmp_tools.ts

@@ -1,115 +0,0 @@
-// Extract tool schemas from RTMP for training data
-//
-// This script extracts tool definitions from the RTMP codebase
-// and adds them to stack-2.9's training data catalog.
-
-import { readdir, readFile, writeFile } from 'fs/promises'
-import { join, basename } from 'path'
-
-const RTMP_TOOLS_DIR = '/Users/walidsobhi/.openclaw/workspace/RTMP/tools'
-const STACK_CATALOG = '/Users/walidsobhi/.openclaw/workspace/stack-2.9/training-data/tools/catalog.json'
-
-interface ToolSchema {
-  tool: string
-  description: string
-  hasPrompt: boolean
-  hasImplementation: boolean
-  inputSchema: Record<string, unknown>
-}
-
-async function extractToolSchemas(): Promise<ToolSchema[]> {
-  const tools: ToolSchema[] = []
-  const toolDirs = await readdir(RTMP_TOOLS_DIR)
-
-  for (const toolDir of toolDirs) {
-    const toolPath = join(RTMP_TOOLS_DIR, toolDir)
-    const stat = await readdir(toolPath).then(() => true).catch(() => false)
-
-    if (!stat) continue
-
-    // Try to extract tool name and description from tool files
-    let description = ''
-    let hasPrompt = false
-    let hasImplementation = false
-
-    try {
-      // Check for prompt.ts
-      const promptPath = join(toolPath, 'prompt.ts')
-      const promptContent = await readFile(promptPath, 'utf-8')
-      hasPrompt = true
-
-      // Extract first meaningful comment as description
-      const comments = promptContent.match(/\/\*\*[\s\S]*?\*\//g)
-      if (comments && comments.length > 0) {
-        const comment = comments[0]
-        description = comment
-          .replace(/\/\*\*|\*\//g, '')
-          .replace(/^\s*\*\s?/gm, '')
-          .trim()
-          .slice(0, 200)
-      }
-    } catch {
-      // No prompt.ts
-    }
-
-    try {
-      // Check for implementation files
-      const toolFiles = await readdir(toolPath)
-      hasImplementation = toolFiles.some(f =>
-        f.endsWith('.ts') || f.endsWith('.tsx')
-      )
-    } catch {
-      // Ignore
-    }
-
-    // Format tool name (remove Tool suffix for cleaner names)
-    const toolName = toolDir.replace(/Tool$/, '')
-
-    tools.push({
-      tool: toolDir,
-      description: description || `${toolName} tool`,
-      hasPrompt,
-      hasImplementation,
-      inputSchema: {}
-    })
-  }
-
-  return tools
-}
-
-async function main() {
-  console.log('Extracting tool schemas from RTMP...')
-
-  const tools = await extractToolSchemas()
-  console.log(`Found ${tools.length} tools`)
-
-  // Read existing catalog
-  let existingTools: ToolSchema[] = []
-  try {
-    const existingContent = await readFile(STACK_CATALOG, 'utf-8')
-    existingTools = JSON.parse(existingContent)
-  } catch {
-    console.log('No existing catalog found')
-  }
-
-  // Merge with existing (avoid duplicates)
-  const existingNames = new Set(existingTools.map(t => t.tool))
-  const newTools = tools.filter(t => !existingNames.has(t.tool))
-
-  console.log(`Adding ${newTools.length} new tools`)
-
-  // Combine
-  const allTools = [...existingTools, ...newTools]
-
-  // Write updated catalog
-  await writeFile(STACK_CATALOG, JSON.stringify(allTools, null, 2))
-  console.log(`Updated catalog with ${allTools.length} tools`)
-
-  // Also print summary
-  console.log('\nNew tools added:')
-  for (const tool of newTools) {
-    console.log(`  - ${tool.tool}`)
-  }
-}
-
-main().catch(console.error)

scripts/extract_rtmp_tools_advanced.py

@@ -1,199 +0,0 @@
-#!/usr/bin/env python3
-"""
-Extract advanced training data from RTMP tools
-Includes actual patterns, best practices, and tool definitions
-"""
-
-import os
-import json
-from pathlib import Path
-
-RTMP_DIR = "/Users/walidsobhi/.openclaw/workspace/RTMP"
-OUTPUT_DIR = "/Users/walidsobhi/.openclaw/workspace/stack-2.9/data/rtmp-tools"
-
-def get_tool_definitions():
-    """Get detailed tool definitions with examples"""
-    tools = []
-
-    # BashTool
-    tools.append({
-        "messages": [
-            {"role": "system", "content": "You are a helpful AI coding assistant with access to various tools."},
-            {"role": "user", "content": "List all files in the current directory including hidden files"},
-            {"role": "assistant", "tool_calls": [
-                {"name": "Bash", "arguments": {"command": "ls -la"}}
-            ]},
-            {"role": "tool", "content": "total 128\ndrwxr-xr-x 1 user staff 4096 Apr 4 10:00 .\ndrwxr-xr-x 1 user staff 4096 Apr 4 10:00 ..\n-rw-r--r-- 1 user staff 1024 Apr 4 09:00 file.ts"}
-        ]
-    })
-
-    tools.append({
-        "messages": [
-            {"role": "user", "content": "Find all TypeScript files in the src directory"},
-            {"role": "assistant", "tool_calls": [
-                {"name": "Glob", "arguments": {"pattern": "src/**/*.ts"}}
-            ]}
-        ]
-    })
-
-    tools.append({
-        "messages": [
-            {"role": "user", "content": "Search for all TODO comments in the codebase"},
-            {"role": "assistant", "tool_calls": [
-                {"name": "Grep", "arguments": {"pattern": "TODO", "path": ".", "output_mode": "content"}}
-            ]}
-        ]
-    })
-
-    tools.append({
-        "messages": [
-            {"role": "user", "content": "Read the package.json file"},
-            {"role": "assistant", "tool_calls": [
-                {"name": "FileRead", "arguments": {"file_path": "package.json"}}
-            ]}
-        ]
-    })
-
-    tools.append({
-        "messages": [
-            {"role": "user", "content": "Create a new task to implement the login feature"},
-            {"role": "assistant", "tool_calls": [
-                {"name": "TaskCreate", "arguments": {
-                    "subject": "Implement login feature",
-                    "description": "Add user authentication with OAuth",
-                    "activeForm": "Implementing login feature"
-                }}
-            ]}
-        ]
-    })
-
-    tools.append({
-        "messages": [
-            {"role": "user", "content": "Search the web for how to use PyTorch MPS on Apple Silicon"},
-            {"role": "assistant", "tool_calls": [
-                {"name": "WebSearch", "arguments": {"query": "PyTorch MPS Apple Silicon tutorial"}}
-            ]}
-        ]
-    })
-
-    tools.append({
-        "messages": [
-            {"role": "user", "content": "Fetch the documentation from https://pytorch.org/docs"},
-            {"role": "assistant", "tool_calls": [
-                {"name": "WebFetch", "arguments": {"url": "https://pytorch.org/docs", "prompt": "Extract the main features of PyTorch"}}
-            ]}
-        ]
-    })
-
-    # Task management examples
-    tools.append({
-        "messages": [
-            {"role": "user", "content": "Show me all open tasks"},
-            {"role": "assistant", "tool_calls": [
-                {"name": "TaskList", "arguments": {}}
-            ]}
-        ]
-    })
-
-    tools.append({
-        "messages": [
-            {"role": "user", "content": "What's the status of the login feature task?"},
-            {"role": "assistant", "tool_calls": [
-                {"name": "TaskGet", "arguments": {"taskId": "1"}}
-            ]}
-        ]
-    })
-
-    # File operations
-    tools.append({
-        "messages": [
-            {"role": "user", "content": "Write a simple Python script to a new file"},
-            {"role": "assistant", "tool_calls": [
-                {"name": "FileWrite", "arguments": {
-                    "file_path": "hello.py",
-                    "content": "print('Hello, World!')\n"
-                }}
-            ]}
-        ]
-    })
-
-    tools.append({
-        "messages": [
-            {"role": "user", "content": "Edit the config file to enable debug mode"},
-            {"role": "assistant", "tool_calls": [
-                {"name": "FileEdit", "arguments": {
-                    "file_path": "config.json",
-                    "old_string": "\"debug\": false",
-                    "new_string": "\"debug\": true"
-                }}
-            ]}
-        ]
-    })
-
-    # Complex multi-step examples
-    tools.append({
-        "messages": [
-            {"role": "user", "content": "First find all React component files, then read one of them"},
-            {"role": "assistant", "tool_calls": [
-                {"name": "Glob", "arguments": {"pattern": "**/*.{tsx,jsx}"}},
-                {"name": "TaskCreate", "arguments": {
-                    "subject": "Read React components",
-                    "description": "Read the found component files",
-                    "activeForm": "Reading React components"
-                }}
-            ]}
-        ]
-    })
-
-    # Skill invocation
-    tools.append({
-        "messages": [
-            {"role": "user", "content": "Commit the changes with a message"},
-            {"role": "assistant", "tool_calls": [
-                {"name": "Skill", "arguments": {"skill": "git-commit", "args": "-m 'Fix bug'"}}
-            ]}
-        ]
-    })
-
-    return tools
-
-def main():
-    print("=" * 60)
-    print("Extracting Advanced RTMP Tool Patterns")
-    print("=" * 60)
-
-    # Get tool examples
-    tools = get_tool_definitions()
-
-    print(f"\n✅ Created {len(tools)} advanced tool examples")
-
-    # Save to JSONL
-    output_file = os.path.join(OUTPUT_DIR, "advanced_tool_patterns.jsonl")
-    with open(output_file, 'w') as f:
-        for ex in tools:
-            f.write(json.dumps(ex) + '\n')
-
-    print(f"✅ Saved to: {output_file}")
-
-    # Combine with previous
-    prev_file = os.path.join(OUTPUT_DIR, "tool_patterns.jsonl")
-    combined_file = os.path.join(OUTPUT_DIR, "combined_tools.jsonl")
-
-    with open(combined_file, 'w') as out:
-        # Previous simple patterns
-        if os.path.exists(prev_file):
-            with open(prev_file) as f:
-                for line in f:
-                    out.write(line)
-        # Advanced patterns
-        with open(output_file) as f:
-            for line in f:
-                out.write(line)
-
-    print(f"\n📦 Total combined examples:")
-    with open(combined_file) as f:
-        count = sum(1 for _ in f)
-    print(f"   {count} tool usage examples")
-
-if __name__ == "__main__":
-    main()

scripts/generate_code_completion_data.py

@@ -1,262 +0,0 @@
-#!/usr/bin/env python3
-"""
-Synthetic Code Completion Training Data Generator for Stack 2.9
-Generates training examples for pure code completion without tools.
-"""
-
-import json
-import random
-import argparse
-from pathlib import Path
-from typing import Dict, List
-
-LANGUAGES = ["python", "javascript", "go", "rust", "typescript"]
-DIFFICULTY_EASY = "easy"
-DIFFICULTY_MEDIUM = "medium"
-DIFFICULTY_HARD = "hard"
-
-# Code templates organized by language -> difficulty -> templates
-CODE_TEMPLATES = {
-    "python": {
-        DIFFICULTY_EASY: [
-            {"context": "def greet(name):", "completion": '    return f"Hello, {name}!"', "description": "Simple greeting function"},
-            {"context": "numbers = [1, 2, 3, 4, 5]\n\n", "completion": "for num in numbers:\n    print(num)", "description": "Loop through list"},
-            {"context": "class Person:\n    def __init__(self, name):", "completion": "        self.name = name", "description": "Class init"},
-            {"context": "def add(a, b):\n    ", "completion": "    return a + b", "description": "Add function"},
-            {"context": "if x > 0:\n    print('positive')\nelif x < 0:\n    ", "completion": "    print('negative')", "description": "Conditional"},
-        ],
-        DIFFICULTY_MEDIUM: [
-            {"context": "def fibonacci(n):\n    if n <= 1:\n        return n\n    ", "completion": "    return fibonacci(n-1) + fibonacci(n-2)", "description": "Fibonacci"},
-            {"context": "class Calculator:\n    def __init__(self):\n        self.result = 0\n    \n    def add(self, x):\n        ", "completion": "        self.result += x\n        return self.result", "description": "Calculator"},
-            {"context": "async def fetch_data(url):\n    async with aiohttp.ClientSession() as session:\n        async with session.get(url) as response:\n            ", "completion": "            return await response.json()", "description": "Async HTTP"},
-            {"context": "def validate_email(email):\n    pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$'\n    ", "completion": "    return re.match(pattern, email) is not None", "description": "Email validation"},
-            {"context": "@app.route('/users/<int:user_id>')\ndef get_user(user_id):\n    user = User.query.get_or_404(user_id)\n    ", "completion": "    return jsonify(user.to_dict())", "description": "Flask route"},
-        ],
-        DIFFICULTY_HARD: [
-            {"context": "class LRUCache:\n    def __init__(self, capacity):\n        self.capacity = capacity\n        self.cache = OrderedDict()\n    \n    def get(self, key):\n        if key not in self.cache:\n            return -1\n        ", "completion": "        self.cache.move_to_end(key)\n        return self.cache[key]", "description": "LRU Cache"},
-            {"context": "def merge_sort(arr):\n    if len(arr) <= 1:\n        return arr\n    \n    mid = len(arr) // 2\n    left = merge_sort(arr[:mid])\n    right = merge_sort(arr[mid:])\n    ", "completion": "    return merge(left, right)", "description": "Merge sort"},
-            {"context": "class BinaryTree:\n    def __init__(self, value):\n        self.value = value\n        self.left = None\n        self.right = None\n    \n    def inorder(self, node, result=None):\n        if result is None:\n            result = []\n        if node:\n            ", "completion": "            self.inorder(node.left, result)\n            result.append(node.value)\n            self.inorder(node.right, result)\n        return result", "description": "Binary tree inorder"},
-            {"context": "def bellman_ford(graph, source):\n    dist = {v: float('inf') for v in graph}\n    dist[source] = 0\n    \n    for _ in range(len(graph) - 1):\n        for u, v, w in graph.edges:\n            if dist[u] != float('inf') and dist[u] + w < dist[v]:\n                ", "completion": "                dist[v] = dist[u] + w\n    return dist", "description": "Bellman-Ford"},
-        ],
-    },
-    "javascript": {
-        DIFFICULTY_EASY: [
-            {"context": "const greet = (name) => {", "completion": '    return `Hello, ${name}!`;', "description": "Arrow greeting"},
-            {"context": "const numbers = [1, 2, 3, 4, 5];\n\n", "completion": "numbers.forEach(num => console.log(num));", "description": "forEach loop"},
-            {"context": "class Person {\n  constructor(name) {", "completion": "    this.name = name;", "description": "JS class constructor"},
-            {"context": "const add = (a, b) => {", "completion": "  return a + b;", "description": "Add function"},
-            {"context": "if (x > 0) {\n  console.log('positive');\n} else if (x < 0) {\n  ", "completion": "  console.log('negative');", "description": "Conditional"},
-        ],
-        DIFFICULTY_MEDIUM: [
-            {"context": "const fetchData = async (url) => {\n  try {\n    const response = await fetch(url);\n    ", "completion": "    return await response.json();\n  } catch (error) {\n    console.error('Error:', error);\n  }", "description": "Async fetch"},
-            {"context": "class EventEmitter {\n  constructor() {\n    this.events = {};\n  }\n  \n  on(event, callback) {\n    ", "completion": "    if (!this.events[event]) this.events[event] = [];\n    this.events[event].push(callback);", "description": "Event emitter"},
-            {"context": "const debounce = (func, delay) => {\n  let timeoutId;\n  return (...args) => {\n    clearTimeout(timeoutId);\n    ", "completion": "    timeoutId = setTimeout(() => func.apply(this, args), delay);", "description": "Debounce"},
-            {"context": "const memoize = (fn) => {\n  const cache = new Map();\n  return (n) => {\n    if (cache.has(n)) {\n      return cache.get(n);\n    }\n    ", "completion": "    const result = fn(n);\n    cache.set(n, result);\n    return result;", "description": "Memoize"},
-        ],
-        DIFFICULTY_HARD: [
-            {"context": "class PromisePool {\n  constructor(maxConcurrent) {\n    this.maxConcurrent = maxConcurrent;\n    this.running = 0;\n    this.queue = [];\n  }\n  \n  add(promiseFn) {\n    return new Promise((resolve, reject) => {\n      ", "completion": "      this.queue.push({ promiseFn, resolve, reject });\n      this.process();\n    });", "description": "Promise pool"},
-            {"context": "const virtualDOM = {\n  createElement(tag, props, ...children) {\n    return {\n      tag,\n      props: props || {},\n      children: children.flat(),\n    };\n  },\n  render(vnode, container) {\n    ", "completion": "    const el = document.createElement(vnode.tag);\n    Object.entries(vnode.props || {}).forEach(([key, value]) => el.setAttribute(key, value));\n    vnode.children.forEach(child => {\n      if (typeof child === 'string') el.appendChild(document.createTextNode(child));\n      else this.render(child, el);\n    });\n    container.appendChild(el);", "description": "Virtual DOM"},
-        ],
-    },
-    "go": {
-        DIFFICULTY_EASY: [
-            {"context": "func greet(name string) string {", "completion": '    return "Hello, " + name + "!"', "description": "Greet function"},
-            {"context": "func add(a, b int) int {", "completion": "    return a + b", "description": "Add function"},
-            {"context": "type Person struct {\n    Name string\n    ", "completion": "    Age  int", "description": "Struct definition"},
-            {"context": "for i := 0; i < 10; i++ {\n    ", "completion": "    fmt.Println(i)", "description": "For loop"},
-            {"context": "if x > 0 {\n    fmt.Println(\"positive\")\n} else {\n    ", "completion": '    fmt.Println("non-positive")', "description": "If-else"},
-        ],
-        DIFFICULTY_MEDIUM: [
-            {"context": "func (p Person) Greet() string {", "completion": '    return fmt.Sprintf("Hello, %s!", p.Name)', "description": "Method"},
-            {"context": "func worker(jobs <-chan int, results chan<- int) {\n    for j := range jobs {\n        ", "completion": "        results <- j * 2", "description": "Worker goroutine"},
-            {"context": "type Handler interface {\n    Handle(ctx context.Context, req Request) Response\n    ", "completion": "    Cleanup(ctx context.Context)", "description": "Interface"},
-            {"context": "func fetchData(url string) ([]byte, error) {\n    resp, err := http.Get(url)\n    if err != nil {\n        return nil, err\n    }\n    defer resp.Body.Close()\n    ", "completion": "    return io.ReadAll(resp.Body)", "description": "HTTP GET"},
-        ],
-        DIFFICULTY_HARD: [
-            {"context": "type TreeNode struct {\n    Val   int\n    Left  *TreeNode\n    Right *TreeNode\n}\n\nfunc (root *TreeNode) InorderTraversal() []int {\n    var result []int\n    var inorder func(*TreeNode)\n    inorder = func(node *TreeNode) {\n        if node == nil {\n            return\n        }\n        ", "completion": "        inorder(node.Left)\n        result = append(result, node.Val)\n        inorder(node.Right)", "description": "Tree inorder"},
-            {"context": "func (c *Client) StreamProcess(ctx context.Context, req *Request, stream chan<- *Response) error {\n    for {\n        select {\n        case <-ctx.Done():\n            return ctx.Err()\n        default:\n            result, err := c.processOne(req)\n            if err != nil {\n                return err\n            }\n            ", "completion": "            select {\n            case stream <- result:\n            case <-ctx.Done():\n                return ctx.Err()\n            }", "description": "Streaming"},
-        ],
-    },
-    "rust": {
-        DIFFICULTY_EASY: [
-            {"context": "fn greet(name: &str) -> String {", "completion": '    format!("Hello, {}!", name)', "description": "Greet function"},
-            {"context": "fn add(a: i32, b: i32) -> i32 {", "completion": "    a + b", "description": "Add function"},
-            {"context": "struct Person {\n    name: String,\n    ", "completion": "    age: u32,", "description": "Struct"},
-            {"context": "let numbers = vec![1, 2, 3, 4, 5];\nfor num in &numbers {\n    ", "completion": "    println!(\"{}\", num);", "description": "For loop"},
-            {"context": "fn main() {\n    let result = match value {\n        Some(x) => x,\n        ", "completion": "        None => 0,", "description": "Match"},
-        ],
-        DIFFICULTY_MEDIUM: [
-            {"context": "impl Person {\n    fn new(name: String, age: u32) -> Self {", "completion": "        Person { name, age }", "description": "Constructor"},
-            {"context": "fn fetch_data(url: &str) -> Result<String, Error> {\n    let response = reqwest::blocking::get(url)?;\n    ", "completion": "    let body = response.text()?;\n    Ok(body)", "description": "HTTP request"},
-            {"context": "fn process_items<T: Display>(items: Vec<T>) -> String {\n    items\n        .iter()\n        .enumerate()\n        .map(|(i, item)| format!(\"{}: {}\", i, item))\n        ", "completion": "        .collect::<Vec<_>>()\n        .join(\", \")", "description": "Iterator chain"},
-            {"context": "fn spawn_worker(jobs: Arc<Mutex<Vec<Job>>>) {\n    thread::spawn(move || {\n        loop {\n            let job = {\n                let mut jobs = jobs.lock().unwrap();\n                jobs.pop()\n            };\n            match job {\n                Some(job) => job.execute(),\n                ", "completion": "                None => break,\n            };\n        }\n    });", "description": "Worker thread"},
-        ],
-        DIFFICULTY_HARD: [
-            {"context": "pub struct LRUCache<K, V> {\n    capacity: usize,\n    cache: LinkedHashMap<K, V>,\n}\n\nimpl<K: Eq + Hash + Clone, V: Clone> LRUCache<K, V> {\n    pub fn get(&mut self, key: &K) -> Option<&V> {\n        if self.cache.contains_key(key) {\n            ", "completion": "            self.cache.remove(key);\n            let value = self.cache[key].clone();\n            self.cache.insert(key.clone(), value);\n            self.cache.get(key)\n        } else {\n            None\n        }", "description": "LRU Cache"},
-            {"context": "pub trait Observer<T> {\n    fn update(&self, event: &T);\n}\n\npub struct Subject<T> {\n    observers: Vec<Box<dyn Observer<T>>>,\n}\n\nimpl<T> Subject<T> {\n    pub fn notify(&self, event: &T) {\n        for observer in &self.observers {\n            ", "completion": "            observer.update(event);", "description": "Observer pattern"},
-        ],
-    },
-}
-
-VARIANTS = ["basic", "explain", "debug", "optimize"]
-
-VARIANT_PROMPTS = {
-    "basic": {"system": "You are a helpful AI assistant that helps with code completion.", "user_prefix": "Complete the following code:\n\n"},
-    "explain": {"system": "You are a helpful AI assistant that explains and completes code.", "user_prefix": "Explain what this code does and complete it:\n\n"},
-    "debug": {"system": "You are a helpful AI assistant that finds bugs and suggests fixes.", "user_prefix": "There's a bug in this code. Fix and complete it:\n\n"},
-    "optimize": {"system": "You are a helpful AI assistant that optimizes code for performance.", "user_prefix": "Optimize this code and complete it:\n\n"},
-}
-
-
-def create_completion_example(context, completion, language, difficulty, variant, description):
-    """Create a single code completion example."""
-    variant_info = VARIANT_PROMPTS[variant]
-    messages = [
-        {"role": "system", "content": variant_info["system"]},
-        {"role": "user", "content": f"{variant_info['user_prefix']}```{language}\n{context}```"},
-        {"role": "assistant", "content": f"Here's the completed code:\n\n```{language}\n{context}{completion}\n```"}
-    ]
-    return {
-        "messages": messages,
-        "language": language,
-        "difficulty": difficulty,
-        "variant": variant,
-        "description": description,
-        "context": context,
-        "completion": completion,
-    }
-
-
-def generate_examples_for_language(language, difficulty, num_examples, variants):
-    """Generate examples for a specific language and difficulty."""
-    templates = CODE_TEMPLATES[language][difficulty]
-    examples = []
-    for i in range(num_examples):
-        template = templates[i % len(templates)]
-        variant = random.choice(variants)
-        example = create_completion_example(
-            context=template["context"],
-            completion=template["completion"],
-            language=language,
-            difficulty=difficulty,
-            variant=variant,
-            description=template["description"]
-        )
-        examples.append(example)
-    return examples
-
-
-def generate_dataset(num_examples=1000, languages=None, difficulties=None, variants=None, balance=True):
-    """Generate the complete dataset."""
-    if languages is None:
-        languages = LANGUAGES
-    if difficulties is None:
-        difficulties = [DIFFICULTY_EASY, DIFFICULTY_MEDIUM, DIFFICULTY_HARD]
-    if variants is None:
-        variants = VARIANTS
-    
-    examples = []
-    
-    if balance:
-        examples_per_lang = num_examples // len(languages)
-        examples_per_diff = examples_per_lang // len(difficulties)
-        remainder = num_examples % (len(languages) * len(difficulties))
-        
-        for lang in languages:
-            for diff_idx, diff in enumerate(difficulties):
-                count = examples_per_diff + (1 if diff_idx < remainder else 0)
-                lang_examples = generate_examples_for_language(lang, diff, count, variants)
-                examples.extend(lang_examples)
-    else:
-        for _ in range(num_examples):
-            lang = random.choice(languages)
-            diff = random.choice(difficulties)
-            template = random.choice(CODE_TEMPLATES[lang][diff])
-            variant = random.choice(variants)
-            example = create_completion_example(
-                context=template["context"],
-                completion=template["completion"],
-                language=lang,
-                difficulty=diff,
-                variant=variant,
-                description=template["description"]
-            )
-            examples.append(example)
-    
-    random.shuffle(examples)
-    return examples
-
-
-def save_jsonl(examples, output_path):
-    """Save examples to JSONL format."""
-    output_file = Path(output_path)
-    output_file.parent.mkdir(parents=True, exist_ok=True)
-    with open(output_file, 'w', encoding='utf-8') as f:
-        for example in examples:
-            f.write(json.dumps(example, ensure_ascii=False) + '\n')
-
-
-def save_json(examples, output_path):
-    """Save examples to JSON format."""
-    output_file = Path(output_path)
-    output_file.parent.mkdir(parents=True, exist_ok=True)
-    with open(output_file, 'w', encoding='utf-8') as f:
-        json.dump(examples, f, ensure_ascii=False, indent=2)
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Generate synthetic code completion training data")
-    parser.add_argument("--num-examples", type=int, default=1000, help="Number of examples to generate")
-    parser.add_argument("--output-dir", type=str, default="training-data/code-completion", help="Output directory")
-    parser.add_argument("--output-format", choices=["jsonl", "json", "both"], default="jsonl", help="Output format")
-    parser.add_argument("--seed", type=int, default=42, help="Random seed")
-    args = parser.parse_args()
-    
-    random.seed(args.seed)
-    
-    print(f"Generating {args.num_examples} code completion training examples...")
-    print(f"   Languages: {LANGUAGES}")
-    print(f"   Output directory: {args.output_dir}")
-    
-    examples = generate_dataset(
-        num_examples=args.num_examples,
-        languages=LANGUAGES,
-        difficulties=[DIFFICULTY_EASY, DIFFICULTY_MEDIUM, DIFFICULTY_HARD],
-        variants=VARIANTS
-    )
-    
-    output_dir = Path(args.output_dir)
-    
-    if args.output_format in ["jsonl", "both"]:
-        jsonl_path = output_dir / "code_completion.jsonl"
-        save_jsonl(examples, str(jsonl_path))
-        print(f"Saved JSONL: {jsonl_path}")
-    
-    if args.output_format in ["json", "both"]:
-        json_path = output_dir / "code_completion.json"
-        save_json(examples, str(json_path))
-        print(f"Saved JSON: {json_path}")
-    
-    # Statistics
-    print(f"\nStatistics:")
-    print(f"   Total examples: {len(examples)}")
-    
-    lang_counts = {}
-    diff_counts = {}
-    for ex in examples:
-        lang_counts[ex["language"]] = lang_counts.get(ex["language"], 0) + 1
-        diff_counts[ex["difficulty"]] = diff_counts.get(ex["difficulty"], 0) + 1
-    
-    print(f"   By language:")
-    for lang, count in sorted(lang_counts.items(), key=lambda x: x[1], reverse=True):
-        print(f"     - {lang}: {count}")
-    
-    print(f"   By difficulty:")
-    for diff, count in sorted(diff_counts.items(), key=lambda x: x[1], reverse=True):
-        print(f"     - {diff}: {count}")
-    
-    print(f"\nGeneration complete!")
-
-
-if __name__ == "__main__":
-    main()

scripts/generate_from_rtmp.ts

@@ -1,114 +0,0 @@
-// Generate synthetic training data from RTMP codebase
-//
-// Extracts code examples and patterns from RTMP to create training data
-// for stack-2.9.
-
-import { readdir, readFile, writeFile, mkdir } from 'fs/promises'
-import { join, basename } from 'path'
-
-const RTMP_DIR = '/Users/walidsobhi/.openclaw/workspace/RTMP'
-const OUTPUT_DIR = '/Users/walidsobhi/.openclaw/workspace/stack-2.9/training-data/src-derived'
-
-interface TrainingExample {
-  messages: Array<{
-    role: string
-    content: string
-  }>
-}
-
-const SYSTEM_PROMPT = `You are Stack, an AI coding assistant based on Claude Code. You help with programming tasks, answer questions, use tools when needed, and provide code examples.`
-
-async function extractCodeExamples(): Promise<TrainingExample[]> {
-  const examples: TrainingExample[] = []
-
-  // Extract from RTMP tools prompts - these are good instruction examples
-  const toolsDir = join(RTMP_DIR, 'tools')
-  const toolDirs = await readdir(toolsDir).catch(() => [])
-
-  for (const toolDir of toolDirs.slice(0, 10)) { // Limit to 10 tools
-    const promptPath = join(toolsDir, toolDir, 'prompt.ts')
-    try {
-      const content = await readFile(promptPath, 'utf-8')
-
-      // Extract useful code patterns
-      const toolName = toolDir.replace('Tool', '')
-
-      // Create example from tool usage
-      examples.push({
-        messages: [
-          { role: 'system', content: SYSTEM_PROMPT },
-          {
-            role: 'user',
-            content: `How do I use the ${toolName} tool?`
-          },
-          {
-            role: 'assistant',
-            content: `The ${toolName} tool allows you to ${getToolDescription(toolName)}. Here's how to use it:\n\n\`\`\`\n// Example usage\n// See the tool source for complete documentation\n\`\`\`\n\nKey features:\n- Feature 1\n- Feature 2`
-          }
-        ]
-      })
-    } catch {
-      // Skip if no prompt
-    }
-  }
-
-  // Extract from RTMP commands
-  const commandsDir = join(RTMP_DIR, 'commands')
-  try {
-    const commandDirs = await readdir(commandsDir)
-    for (const cmd of commandDirs.slice(0, 5)) {
-      examples.push({
-        messages: [
-          { role: 'system', content: SYSTEM_PROMPT },
-          {
-            role: 'user',
-            content: `How do I use the /${cmd} command?`
-          },
-          {
-            role: 'assistant',
-            content: `The /${cmd} command provides ${cmd} functionality. Use it by typing /${cmd} in your prompt.`
-          }
-        ]
-      })
-    }
-  } catch {
-    // Ignore
-  }
-
-  return examples
-}
-
-function getToolDescription(toolName: string): string {
-  const descriptions: Record<string, string> = {
-    'Bash': 'execute shell commands and get output',
-    'FileRead': 'read files from the filesystem',
-    'FileWrite': 'write content to files',
-    'FileEdit': 'make targeted edits to files',
-    'Glob': 'find files matching patterns',
-    'Grep': 'search for text in files',
-    'LSP': 'get language server features like autocomplete',
-    'MCP': 'use Model Context Protocol servers',
-    'Task': 'create and manage task lists',
-    'Todo': 'track tasks and todo items'
-  }
-  return descriptions[toolName] || 'perform its designated function'
-}
-
-async function main() {
-  console.log('Generating synthetic training data from RTMP...')
-
-  // Ensure output directory exists
-  await mkdir(OUTPUT_DIR, { recursive: true }).catch(() => {})
-
-  const examples = await extractCodeExamples()
-  console.log(`Generated ${examples.length} training examples`)
-
-  // Write to JSONL
-  const outputPath = join(OUTPUT_DIR, 'rtmp_examples.jsonl')
-  const content = examples.map(e => JSON.stringify(e)).join('\n')
-  await writeFile(outputPath, content)
-
-  console.log(`Written to ${outputPath}`)
-}
-
-main().catch(console.error)

scripts/generate_random_synthetic.py

@@ -1,141 +0,0 @@
-#!/usr/bin/env python3
-"""
-Generate random synthetic tool-use examples.
-Uses tool catalog to create syntactically valid random conversations.
-"""
-
-import json
-import random
-from pathlib import Path
-import argparse
-
-def load_tool_catalog(path: str):
-    with open(path, 'r') as f:
-        return json.load(f)
-
-def random_value_for_type(param_name: str) -> Any:
-    """Generate a plausible random value based on parameter name."""
-    if 'file' in param_name or 'path' in param_name:
-        return random.choice(['src/main.py', 'README.md', 'package.json', 'config.yaml', 'tests/test.js'])
-    elif 'command' in param_name or 'cmd' in param_name:
-        return random.choice(['npm test', 'pytest', 'git status', 'ls -la', 'make build'])
-    elif 'pattern' in param_name or 'glob' in param_name:
-        return random.choice(['**/*.py', '**/*.js', '**/*.md'])
-    elif 'query' in param_name or 'search' in param_name:
-        return random.choice(['TODO', 'FIXME', 'function main'])
-    elif 'url' in param_name or 'uri' in param_name:
-        return random.choice(['https://api.example.com', 'mcp://server/resource'])
-    elif 'status' in param_name:
-        return random.choice(['pending', 'in_progress', 'completed'])
-    elif 'id' in param_name or 'task_id' in param_name:
-        return random.randint(100, 999)
-    elif 'name' in param_name:
-        return random.choice(['agent1', 'myteam', 'task123'])
-    elif 'content' in param_name or 'text' in param_name:
-        return 'Lorem ipsum dolor sit amet...'
-    elif 'directory' in param_name or 'dir' in param_name:
-        return random.choice(['.', 'src', 'tests', 'lib'])
-    elif 'branch' in param_name:
-        return random.choice(['main', 'develop', 'feature/new'])
-    else:
-        return f"value_{random.randint(1,100)}"
-
-def generate_random_example(tools: List[Dict], tool_count: int = 1) -> Dict[str, Any]:
-    """Generate a random conversation with one or more tool uses."""
-    tools_sample = random.sample(tools, min(tool_count, len(tools)))
-
-    # Build messages
-    messages = []
-
-    # Random user prompt
-    user_prompt = random.choice([
-        "Help me with something",
-        "Do a task",
-        "I need assistance",
-        "Can you handle this?",
-        "Execute this",
-        "Run this operation"
-    ])
-    messages.append({"role": "user", "content": user_prompt})
-
-    # For each tool, add assistant tool-use and tool-result
-    for i, tool in enumerate(tools_sample):
-        tool_name = tool.get("tool") or tool.get("name", "UnknownTool")
-
-        # Generate random parameters based on tool's expected inputs
-        # We don't have strict schema, so make up plausible params
-        tool_input = {}
-        for j in range(random.randint(1, 3)):
-            param_name = random.choice(['file_path', 'command', 'pattern', 'query', 'url', 'id', 'name', 'directory'])
-            tool_input[param_name] = random_value_for_type(param_name)
-
-        # Assistant uses tool
-        messages.append({
-            "role": "assistant",
-            "content": f"Using {tool_name}...",
-            "tool_use": {
-                "name": tool_name,
-                "input": tool_input
-            }
-        })
-
-        # Tool result
-        result_content = f"Operation completed successfully. Affected items: {random.randint(1,10)}"
-        messages.append({
-            "role": "user",
-            "content": "",
-            "tool_result": {
-                "tool_use_id": f"tool_{i+1}",
-                "content": result_content
-            }
-        })
-
-        # Assistant acknowledges
-        messages.append({
-            "role": "assistant",
-            "content": random.choice(["Done.", "Completed.", "All set."])
-        })
-
-    return {
-        "messages": messages,
-        "source": "random_synthetic",
-        "tools_used": [t.get("tool") for t in tools_sample]
-    }
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--catalog", type=str, default="training-data/tools/catalog.json")
-    parser.add_argument("--output", type=str, default="training-data/scaled/random_synthetic.jsonl")
-    parser.add_argument("--count", type=int, default=10000)
-    parser.add_argument("--tools-per-example", type=int, default=1)
-    args = parser.parse_args()
-
-    catalog_path = Path(args.catalog)
-    output_path = Path(args.output)
-
-    if not catalog_path.exists():
-        print(f"❌ Catalog not found: {catalog_path}")
-        return
-
-    tools = load_tool_catalog(catalog_path)
-    print(f"🔧 Loaded {len(tools)} tools from catalog")
-
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-
-    with open(output_path, 'w') as f:
-        for i in range(args.count):
-            example = generate_random_example(tools, args.tools_per_example)
-            f.write(json.dumps(example) + "\n")
-            if (i+1) % 1000 == 0:
-                print(f"   Generated {i+1}/{args.count}...", end='\r')
-
-    print(f"\n✨ Generated {args.count} random synthetic examples")
-    print(f"   Saved to: {output_path}")
-
-    # Show sample
-    with open(output_path, 'r') as f:
-        sample = json.loads(f.readline())
-        print(f"\n📝 Sample: {len(sample['messages'])} messages, tools: {sample.get('tools_used')}")
-
-if __name__ == "__main__":
-    main()

scripts/generate_synthetic.py

@@ -1,256 +0,0 @@
-#!/usr/bin/env python3
-"""
-Generate synthetic training examples using templates.
-No external APIs - pure template expansion and variation.
-"""
-
-import json
-import random
-import string
-from pathlib import Path
-from typing import Dict, List, Any
-import argparse
-
-# Load tool catalog
-def load_tools(catalog_path: str) -> List[Dict[str, Any]]:
-    with open(catalog_path, 'r') as f:
-        return json.load(f)
-
-# Template definitions for each tool
-def get_tool_templates(tool_name: str) -> List[Dict[str, Any]]:
-    """Return list of template scenarios for a given tool."""
-    templates = {
-        "FileReadTool": [
-            {
-                "user": "Read the file {file_path}",
-                "params": {"file_path": "{file_path}"},
-                "result": "Contents of {file_path}:\n{file_content}"
-            },
-            {
-                "user": "Show me what's in {file_path}",
-                "params": {"file_path": "{file_path}"},
-                "result": "Here's {file_path}:\n{file_content}"
-            },
-            {
-                "user": "Can you open {file_path}?",
-                "params": {"file_path": "{file_path}"},
-                "result": "Opening {file_path}...\n{file_content}"
-            }
-        ],
-        "FileWriteTool": [
-            {
-                "user": "Create a new file {file_path} with content: {content}",
-                "params": {"file_path": "{file_path}", "content": "{content}"},
-                "result": "File {file_path} created successfully"
-            },
-            {
-                "user": "Write this to {file_path}: {content}",
-                "params": {"file_path": "{file_path}", "content": "{content}"},
-                "result": "Wrote to {file_path}"
-            }
-        ],
-        "GlobTool": [
-            {
-                "user": "Find all {pattern} files",
-                "params": {"pattern": "{pattern}"},
-                "result": "Found {count} files:\n{files}"
-            },
-            {
-                "user": "List files matching {pattern}",
-                "params": {"pattern": "{pattern}"},
-                "result": "Matches for {pattern}:\n{files}"
-            }
-        ],
-        "GrepTool": [
-            {
-                "user": "Search for {pattern} in {directory}",
-                "params": {"pattern": "{pattern}", "directory": "{directory}"},
-                "result": "Found {count} matches:\n{matches}"
-            },
-            {
-                "user": "Find all occurrences of {pattern}",
-                "params": {"pattern": "{pattern}"},
-                "result": "Search results:\n{matches}"
-            }
-        ],
-        "BashTool": [
-            {
-                "user": "Run: {command}",
-                "params": {"command": "{command}"},
-                "result": "$ {command}\n{output}"
-            },
-            {
-                "user": "Execute {command}",
-                "params": {"command": "{command}"},
-                "result": "Output:\n{output}"
-            }
-        ]
-    }
-
-    # Return templates or generate generic ones if not defined
-    return templates.get(tool_name, [
-        {
-            "user": "Use {tool} with {params}",
-            "params": {"arg": "value"},
-            "result": "Operation completed"
-        }
-    ])
-
-def generate_variations(template: Dict[str, str], count: int, tool_name: str) -> List[Dict[str, Any]]:
-    """Generate multiple variations of a template."""
-    examples = []
-
-    for _ in range(count):
-        # Create parameter values
-        params = generate_params(tool_name, template.get("params", {}))
-
-        # Fill template placeholders
-        user_prompt = fill_template(template.get("user", ""), params)
-        tool_params = fill_params(template.get("params", {}), params)
-        result = fill_template(template.get("result", ""), params)
-
-        # Build conversation
-        messages = [
-            {"role": "user", "content": user_prompt},
-            {
-                "role": "assistant",
-                "content": "I'll help with that.",
-                "tool_use": {
-                    "name": tool_name,
-                    "input": tool_params
-                }
-            },
-            {
-                "role": "user",
-                "content": "",
-                "tool_result": {
-                    "tool_use_id": "tool_1",
-                    "content": result
-                }
-            },
-            {"role": "assistant", "content": "Done!"}
-        ]
-
-        examples.append({
-            "messages": messages,
-            "source": "synthetic_template",
-            "tool": tool_name
-        })
-
-    return examples
-
-def generate_params(tool_name: str, template_params: Dict[str, str]) -> Dict[str, Any]:
-    """Generate realistic parameter values based on tool and template."""
-    params = {}
-
-    for key, placeholder in template_params.items():
-        if placeholder == "{file_path}":
-            params[key] = random.choice([
-                "src/main.py", "package.json", "README.md", "src/utils.js",
-                "tests/test_api.py", "config.yaml", "Dockerfile", "requirements.txt"
-            ])
-        elif placeholder == "{pattern}":
-            params[key] = random.choice([
-                "**/*.py", "**/*.js", "**/*.ts", "*.md", "src/**/*.test.js"
-            ])
-        elif placeholder == "{command}":
-            params[key] = random.choice([
-                "ls -la", "pwd", "git status", "npm run build",
-                "python -m pytest", "make test", "docker ps"
-            ])
-        elif placeholder == "{content}":
-            params[key] = random.choice([
-                "console.log('Hello, World!');",
-                "def hello():\n    return 'Hello'",
-                "# TODO: implement\npass",
-                "import React from 'react';"
-            ])
-        elif placeholder == "{directory}":
-            params[key] = random.choice([
-                ".", "src", "tests", "lib", "app/components"
-            ])
-        else:
-            # Generic placeholder
-            params[key] = f"generated_value_{random.randint(1,1000)}"
-
-    return params
-
-def fill_template(template: str, params: Dict[str, Any]) -> str:
-    """Replace placeholders in template."""
-    result = template
-    for key, value in params.items():
-        placeholder = f"{{{key}}}"
-        result = result.replace(placeholder, str(value))
-    return result
-
-def fill_params(template_params: Dict[str, str], params: Dict[str, Any]) -> Dict[str, Any]:
-    """Fill parameter templates."""
-    filled = {}
-    for key, placeholder in template_params.items():
-        if placeholder in [f"{{{k}}}" for k in params.keys()]:
-            # Find matching param key
-            param_key = placeholder.strip("{}")
-            filled[key] = params.get(param_key, placeholder)
-        else:
-            filled[key] = params.get(key, placeholder)
-    return filled
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--catalog", type=str, default="training-data/tools/catalog.json")
-    parser.add_argument("--output", type=str, default="training-data/scaled/template_synthetic.jsonl")
-    parser.add_argument("--examples-per-tool", type=int, default=500)
-    parser.add_argument("--tools-limit", type=int, default=None, help="Limit number of tools to process")
-    args = parser.parse_args()
-
-    catalog_path = Path(args.catalog)
-    output_path = Path(args.output)
-
-    if not catalog_path.exists():
-        print(f"❌ Tool catalog not found: {catalog_path}")
-        return
-
-    tools = load_tools(catalog_path)
-    if args.tools_limit:
-        tools = tools[:args.tools_limit]
-
-    print(f"🔧 Generating synthetic examples for {len(tools)} tools")
-    print(f"   Target: {args.examples_per_tool} examples per tool")
-    print(f"   Total expected: ~{len(tools) * args.examples_per_tool} examples")
-
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-
-    total_examples = 0
-    with open(output_path, 'w') as f:
-        for tool in tools:
-            tool_name = tool.get("tool") or tool.get("name", "Unknown")
-            templates = get_tool_templates(tool_name)
-
-            if not templates:
-                print(f"⚠️  No templates for {tool_name}, skipping")
-                continue
-
-            # Generate examples for each template
-            examples_per_template = args.examples_per_tool // len(templates)
-
-            for template in templates:
-                examples = generate_variations(template, examples_per_template, tool_name)
-                for ex in examples:
-                    f.write(json.dumps(ex) + "\n")
-                    total_examples += 1
-
-            print(f"✅ {tool_name}: {examples_per_template * len(templates)} examples")
-
-    print(f"\n✨ Generated {total_examples} synthetic examples")
-    print(f"   Saved to: {output_path}")
-
-    # Create a sample
-    print("\n📝 Sample example:")
-    with open(output_path, 'r') as f:
-        first_line = f.readline()
-        if first_line:
-            sample = json.loads(first_line)
-            print(f"   User: {sample['messages'][0]['content'][:80]}...")
-
-if __name__ == "__main__":
-    main()

scripts/generate_synthetic_v2.py

@@ -1,316 +0,0 @@
-#!/usr/bin/env python3
-"""
-Generate high-quality synthetic training data using tool-specific templates.
-Each tool gets realistic scenarios with proper parameters.
-"""
-
-import json
-import random
-from pathlib import Path
-from typing import Dict, List, Any
-import argparse
-
-# Comprehensive templates for all tools
-TOOL_TEMPLATES = {
-    "AgentTool": [
-        {"user": "Create an agent to help with testing", "params": {"name": "test_agent", "goal": "Write unit tests"}, "result": "Agent 'test_agent' created"},
-        {"user": "Spawn a teammate to handle frontend tasks", "params": {"name": "frontend_dev", "skills": ["react", "typescript"]}, "result": "Teammate 'frontend_dev' added to team"}
-    ],
-    "AskUserQuestionTool": [
-        {"user": "Ask the user which framework they prefer", "params": {"question": "Which framework do you want to use: React, Vue, or Angular?"}, "result": "User responded: React"},
-        {"user": "I need clarification on the requirements", "params": {"question": "Should the API be REST or GraphQL?"}, "result": "User answered: REST"}
-    ],
-    "BashTool": [
-        {"user": "Run tests", "params": {"command": "npm test"}, "result": "PASS  src/index.test.js\nTests: 12 passed, 0 failed"},
-        {"user": "Check git status", "params": {"command": "git status"}, "result": "On branch main\nModified: src/index.js"},
-        {"user": "Install dependencies", "params": {"command": "pip install -r requirements.txt"}, "result": "Successfully installed Flask==2.0.0"}
-    ],
-    "BriefTool": [
-        {"user": "Give me a brief on this project", "params": {"topic": "project_overview"}, "result": "This is a voice-enabled AI coding assistant built on Qwen2.5-Coder-32B"},
-        {"user": "Brief me on the architecture", "params": {"topic": "architecture"}, "result": "Stack 2.9 uses vLLM for inference with LoRA fine-tuning"}
-    ],
-    "ConfigTool": [
-        {"user": "Show configuration", "params": {"section": "model"}, "result": "model: Qwen2.5-Coder-32B\ncontext: 32768"},
-        {"user": "Get settings", "params": {"key": "max_tokens"}, "result": "max_tokens: 4000"}
-    ],
-    "EnterPlanModeTool": [
-        {"user": "Enter plan mode", "params": {"goal": "Refactor authentication module"}, "result": "Plan mode activated for: Refactor authentication module"},
-    ],
-    "EnterWorktreeTool": [
-        {"user": "Create a worktree for feature branch", "params": {"branch": "feature/new-ui"}, "result": "Worktree created at .worktrees/feature_new_ui"},
-    ],
-    "ExitWorktreeTool": [
-        {"user": "Exit current worktree", "params": {}, "result": "Exited worktree, returning to main"},
-    ],
-    "FileEditTool": [
-        {"user": "Fix syntax error in main.py line 10", "params": {"file_path": "src/main.py", "old_string": "prin('hello')", "new_string": "print('hello')"}, "result": "File edited successfully"},
-        {"user": "Add import statement to app.py", "params": {"file_path": "app.py", "insert_after": "import os", "new_string": "import sys"}, "result": "Import added"},
-    ],
-    "FileReadTool": [
-        {"user": "Read package.json", "params": {"file_path": "package.json"}, "result": "{\n  \"name\": \"myapp\",\n  \"version\": \"1.0.0\"\n}"},
-        {"user": "Show me README.md", "params": {"file_path": "README.md"}, "result": "# My Project\n\nDescription here..."},
-    ],
-    "FileWriteTool": [
-        {"user": "Create a new file utils.py", "params": {"file_path": "src/utils.py", "content": "def helper():\n    return 'help'"}, "result": "File src/utils.py created"},
-    ],
-    "GlobTool": [
-        {"user": "Find all Python files", "params": {"pattern": "**/*.py"}, "result": "Found 15 files:\nsrc/main.py\nsrc/utils.py\ntests/test_main.py"},
-        {"user": "List test files", "params": {"pattern": "**/*.test.js"}, "result": "Found 3 files:\ntests/unit.test.js\ntests/integration.test.js"},
-    ],
-    "GrepTool": [
-        {"user": "Search for 'TODO' comments", "params": {"pattern": "TODO"}, "result": "src/main.py:15:# TODO: implement error handling\nsrc/utils.py:42:# TODO: add validation"},
-        {"user": "Find all console.log statements", "params": {"pattern": "console.log"}, "result": "src/index.js:10:console.log('debug')\nsrc/app.js:25:console.log('start')"},
-    ],
-    "LSPTool": [
-        {"user": "Get definition of function calculateTotal", "params": {"file_path": "src/math.js", "line": 10, "character": 15}, "result": "Definition at src/math.js:20-30\nfunction calculateTotal(items) {...}"},
-        {"user": "Find references of MyClass", "params": {"file_path": "src/MyClass.ts", "line": 5, "character": 10}, "result": "References:\n- src/main.ts:15\n- tests/MyClass.test.ts:8"},
-    ],
-    "ListMcpResourcesTool": [
-        {"user": "List available MCP resources", "params": {}, "result": "Resources:\n- server1.file_system\n- server2.database\n- server3.api"},
-    ],
-    "MCPTool": [
-        {"user": "Connect to GitHub MCP server", "params": {"server_name": "github"}, "result": "Connected to GitHub MCP server"},
-    ],
-    "NotebookEditTool": [
-        {"user": "Add markdown cell to notebook", "params": {"notebook_path": "analysis.ipynb", "cell_index": 0, "cell_type": "markdown", "content": "# Analysis"}, "result": "Cell added"},
-    ],
-    "ReadMcpResourceTool": [
-        {"user": "Read resource file from MCP", "params": {"uri": "mcp://server1/file.txt"}, "result": "File content here..."},
-    ],
-    "RemoteTriggerTool": [
-        {"user": "Trigger deployment on staging", "params": {"target": "staging-server", "action": "deploy"}, "result": "Deployment triggered, build ID: 12345"},
-    ],
-    "SendMessageTool": [
-        {"user": "Message the design team about the mockups", "params": {"to": "design-team", "subject": "Mockups ready", "body": "Please review the new mockups in Figma"}, "result": "Message sent to design-team"},
-    ],
-    "SkillTool": [
-        {"user": "Run code review skill", "params": {"skill": "code-review", "inputs": {"code": "function foo() { return 1; }"}}, "result": "Review: Use strict equality, add JSDoc"},
-    ],
-    "TaskCreateTool": [
-        {"user": "Create task: Fix login bug", "params": {"title": "Fix login bug", "description": "Users can't log in with valid credentials"}, "result": "Task #123 created"},
-    ],
-    "TaskGetTool": [
-        {"user": "Get details of task 123", "params": {"task_id": 123}, "result": "Task #123: Fix login bug\nStatus: in progress\nAssignee: @dev"},
-    ],
-    "TaskListTool": [
-        {"user": "List all tasks", "params": {"status": "in_progress"}, "result": "Tasks:\n#123 Fix login bug\n#124 Update docs"},
-    ],
-    "TaskStopTool": [
-        {"user": "Stop task 123", "params": {"task_id": 123}, "result": "Task #123 stopped"},
-    ],
-    "TaskUpdateTool": [
-        {"user": "Mark task 123 as complete", "params": {"task_id": 123, "status": "completed"}, "result": "Task #123 marked complete"},
-    ],
-    "TeamCreateTool": [
-        {"user": "Create a team for backend devs", "params": {"team_name": "backend", "members": ["@alice", "@bob"]}, "result": "Team 'backend' created with 2 members"},
-    ],
-    "TeamDeleteTool": [
-        {"user": "Delete the temp team", "params": {"team_name": "temp"}, "result": "Team 'temp' deleted"},
-    ],
-    "TodoWriteTool": [
-        {"user": "Add todo: update documentation", "params": {"text": "Update API documentation"}, "result": "Todo added"},
-    ],
-    "ToolSearchTool": [
-        {"user": "Search for file search tools", "params": {"query": "find files"}, "result": "Found: GlobTool, GrepTool"},
-    ],
-    "WebFetchTool": [
-        {"user": "Fetch the OpenRouter API docs", "params": {"url": "https://openrouter.ai/docs"}, "result": "Fetched 15KB from openrouter.ai/docs"},
-    ],
-    "WebSearchTool": [
-        {"user": "Search for 'Node.js best practices 2024'", "params": {"query": "Node.js best practices 2024"}, "result": "Top results:\n1. Node.js Design Patterns\n2. 2024 Node.js Best Practices Guide"},
-    ]
-}
-
-# Realistic value pools
-FILE_PATHS = ["src/main.py", "src/utils.js", "README.md", "package.json", "config.yaml",
-              "Dockerfile", "requirements.txt", "tests/test_api.py", "src/components/Button.tsx",
-              "lib/helpers.py", "app/models.py", "src/index.js", "Makefile"]
-COMMANDS = ["npm test", "pytest", "make build", "git status", "ls -la",
-            "python -m pip install -r requirements.txt", "docker ps", "npm run lint"]
-PATTERNS = ["**/*.py", "**/*.js", "**/*.ts", "*.md", "**/*.test.js", "**/__tests__/**/*.py"]
-QUESTIONS = ["Which framework should we use?", "Is this a bug or a feature?",
-             "What's the priority of this task?", "Should we refactor or rewrite?"]
-
-def fill_placeholders(text: str, params: Dict[str, Any]) -> str:
-    """Replace all {key} placeholders in text with values from params."""
-    for key, value in params.items():
-        placeholder = f"{{{key}}}"
-        if placeholder in text:
-            text = text.replace(placeholder, str(value))
-    return text
-
-def generate_variations(template: Dict[str, Any], count: int, tool_name: str) -> List[Dict[str, Any]]:
-    """Generate multiple realistic variations of a template."""
-    examples = []
-
-    for i in range(count):
-        params = {}
-        user_text = template["user"]
-        result_text = template["result"]
-
-        # Build params by scanning template for placeholders
-        template_str = user_text + json.dumps(template.get("params", {})) + result_text
-
-        # Determine what placeholders exist
-        for key, default_val in template.get("params", {}).items():
-            if isinstance(default_val, str) and ("{" + key + "}") in template_str:
-                # This is a placeholder - generate dynamic value
-                if key == "file_path":
-                    params[key] = random.choice(FILE_PATHS)
-                elif key == "command":
-                    params[key] = random.choice(COMMANDS)
-                elif key == "pattern":
-                    params[key] = random.choice(PATTERNS)
-                elif key == "question":
-                    params[key] = random.choice(QUESTIONS)
-                elif key == "topic":
-                    params[key] = random.choice(["project_overview", "architecture", "team", "timeline"])
-                elif key == "branch":
-                    params[key] = random.choice(["feature/new-ui", "bugfix/login", "hotfix/security"])
-                elif key == "name":
-                    params[key] = random.choice(["test_agent", "code_reviewer", "deployment_bot", "test_suite"])
-                elif key == "goal":
-                    params[key] = random.choice(["Write unit tests", "Refactor legacy code", "Add documentation"])
-                elif key == "to":
-                    params[key] = random.choice(["team-backend", "design-team", "product-team"])
-                elif key == "subject":
-                    params[key] = random.choice(["Review needed", "Update available", "Deployment status"])
-                elif key == "body":
-                    params[key] = "Please review the attached documents."
-                elif key == "server_name":
-                    params[key] = random.choice(["github", "jira", "slack", "postgres"])
-                elif key == "action":
-                    params[key] = random.choice(["deploy", "restart", "backup", "migrate"])
-                elif key == "target":
-                    params[key] = random.choice(["staging", "production", "dev"])
-                elif key == "skill":
-                    params[key] = random.choice(["code-review", "security-scan", "performance-test"])
-                elif key == "query":
-                    params[key] = random.choice(["find files", "search code", "list todos"])
-                elif key == "url":
-                    params[key] = random.choice(["https://api.example.com/docs", "https://github.com/repo"])
-                elif key == "task_id":
-                    params[key] = random.randint(100, 999)
-                elif key == "title":
-                    params[key] = random.choice(["Fix bug", "Add feature", "Update docs", "Refactor code"])
-                elif key == "description":
-                    params[key] = "Detailed description of the task..."
-                elif key == "team_name":
-                    params[key] = random.choice(["backend", "frontend", "devops", "qa"])
-                elif key == "members":
-                    params[key] = ["@user1", "@user2"]
-                elif key == "status":
-                    params[key] = random.choice(["in_progress", "completed", "todo"])
-                elif key == "text":
-                    params[key] = "Sample todo item"
-                elif key == "cell_index":
-                    params[key] = random.randint(0, 10)
-                elif key == "cell_type":
-                    params[key] = random.choice(["code", "markdown"])
-                elif key == "content":
-                    params[key] = random.choice(["print('hello')", "# TODO", "import React", "def main():\n    pass"])
-                elif key == "uri":
-                    params[key] = "mcp://server/resource"
-                elif key in ["line", "character"] and "file_path" in params:
-                    params[key] = random.randint(1, 100) if key == "line" else random.randint(1, 50)
-                else:
-                    # Generic placeholder
-                    params[key] = f"value_{random.randint(1, 100)}"
-            else:
-                # Not a placeholder, use the static value as-is
-                params[key] = default_val
-
-        # Fill user prompt with params
-        user_prompt = fill_placeholders(user_text, params)
-        user_prompt = user_prompt.replace("{tool}", tool_name)
-
-        # Build tool input from params
-        tool_input = {}
-        for key, template_val in template.get("params", {}).items():
-            if key in params:
-                tool_input[key] = params[key]
-            else:
-                tool_input[key] = template_val
-
-        # Fill result
-        result = fill_placeholders(result_text, params)
-
-        # Build conversation
-        messages = [
-            {"role": "user", "content": user_prompt},
-            {
-                "role": "assistant",
-                "content": random.choice([
-                    "I'll help with that.",
-                    "Sure, let me do that.",
-                    "Processing your request..."
-                ]),
-                "tool_use": {
-                    "name": tool_name,
-                    "input": tool_input
-                }
-            },
-            {
-                "role": "user",
-                "content": "",
-                "tool_result": {
-                    "tool_use_id": "tool_1",
-                    "content": result
-                }
-            },
-            {"role": "assistant", "content": random.choice(["Done!", "Completed.", "All set!"])}
-        ]
-
-        examples.append({
-            "messages": messages,
-            "source": "synthetic_template",
-            "tool": tool_name
-        })
-
-    return examples
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--output", type=str, default="training-data/scaled/template_synthetic.jsonl")
-    parser.add_argument("--examples-per-tool", type=int, default=500)
-    parser.add_argument("--tools-limit", type=int, default=None)
-    args = parser.parse_args()
-
-    output_path = Path(args.output)
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-
-    tools = list(TOOL_TEMPLATES.keys())
-    if args.tools_limit:
-        tools = tools[:args.tools_limit]
-
-    print(f"🔧 Generating synthetic examples for {len(tools)} tools")
-    print(f"   Target: {args.examples_per_tool} per tool")
-    print(f"   Total expected: ~{len(tools) * args.examples_per_tool}")
-
-    total_examples = 0
-    with open(output_path, 'w') as f:
-        for tool_name in tools:
-            templates = TOOL_TEMPLATES[tool_name]
-            ex_per_template = max(1, args.examples_per_tool // len(templates))
-
-            for template in templates:
-                examples = generate_variations(template, ex_per_template, tool_name)
-                for ex in examples:
-                    f.write(json.dumps(ex) + "\n")
-                    total_examples += 1
-
-            print(f"✅ {tool_name}: {ex_per_template * len(templates)} examples")
-
-    print(f"\n✨ Generated {total_examples} synthetic examples")
-    print(f"   Saved to: {output_path}")
-
-    # Show sample
-    print("\n📝 Sample example:")
-    with open(output_path, 'r') as f:
-        sample = json.loads(f.readline())
-        print(f"   Tool: {sample['tool']}")
-        print(f"   User: {sample['messages'][0]['content'][:60]}...")
-        print(f"   Assistant uses: {sample['messages'][1]['tool_use']['name']}")
-
-if __name__ == "__main__":
-    main()

scripts/generate_tool_data.py

@@ -1,615 +0,0 @@
-#!/usr/bin/env python3
-"""
-Synthetic Tool-Calling Training Data Generator for Stack 2.9
-Generates training examples in Qwen2.5-Coder format with tool_calls.
-"""
-
-import json
-import random
-import argparse
-from pathlib import Path
-from typing import Dict, List, Any
-from datetime import datetime
-
-# ============================================================================
-# Tool Definitions (Qwen2.5-Coder format)
-# ============================================================================
-
-TOOL_DEFINITIONS = [
-    {
-        "type": "function",
-        "function": {
-            "name": "Bash",
-            "description": "Execute bash commands in the terminal. Use for running shell commands, scripts, git operations, package managers, and system commands.",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "command": {
-                        "type": "string",
-                        "description": "The bash command to execute"
-                    },
-                    "timeout": {
-                        "type": "integer",
-                        "description": "Timeout in seconds (default: 30)"
-                    }
-                },
-                "required": ["command"]
-            }
-        }
-    },
-    {
-        "type": "function",
-        "function": {
-            "name": "FileRead",
-            "description": "Read the contents of a file from the filesystem. Use for viewing source code, configuration files, documentation, or any text-based files.",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "path": {
-                        "type": "string",
-                        "description": "Path to the file to read"
-                    },
-                    "offset": {
-                        "type": "integer",
-                        "description": "Line number to start reading from (1-indexed)"
-                    },
-                    "limit": {
-                        "type": "integer",
-                        "description": "Maximum number of lines to read"
-                    }
-                },
-                "required": ["path"]
-            }
-        }
-    },
-    {
-        "type": "function",
-        "function": {
-            "name": "FileWrite",
-            "description": "Create or overwrite a file with content. Use for creating new files, updating existing files, or writing code, configuration, or documentation.",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "path": {
-                        "type": "string",
-                        "description": "Path where the file should be created or written"
-                    },
-                    "content": {
-                        "type": "string",
-                        "description": "The content to write to the file"
-                    },
-                    "append": {
-                        "type": "boolean",
-                        "description": "Append to existing file instead of overwriting (default: false)"
-                    }
-                },
-                "required": ["path", "content"]
-            }
-        }
-    },
-    {
-        "type": "function",
-        "function": {
-            "name": "WebSearch",
-            "description": "Search the web for information. Use for finding documentation, looking up error messages, researching libraries, or getting up-to-date information.",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "query": {
-                        "type": "string",
-                        "description": "The search query to look up on the web"
-                    },
-                    "count": {
-                        "type": "integer",
-                        "description": "Number of results to return (default: 5)"
-                    }
-                },
-                "required": ["query"]
-            }
-        }
-    },
-    {
-        "type": "function",
-        "function": {
-            "name": "Grep",
-            "description": "Search for patterns in files. Use for finding specific code, function definitions, imports, TODO comments, error patterns, or any text across the codebase.",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "pattern": {
-                        "type": "string",
-                        "description": "The search pattern or regex to match"
-                    },
-                    "path": {
-                        "type": "string",
-                        "description": "Directory or file path to search in (default: current directory)"
-                    },
-                    "recursive": {
-                        "type": "boolean",
-                        "description": "Search recursively in subdirectories (default: true)"
-                    },
-                    "file_pattern": {
-                        "type": "string",
-                        "description": "File pattern to filter results (e.g., '*.py', '*.js')"
-                    }
-                },
-                "required": ["pattern"]
-            }
-        }
-    }
-]
-
-# ============================================================================
-# Template Data for Generation
-# ============================================================================
-
-FILE_PATHS = [
-    "src/main.py", "src/utils.py", "src/config.py", "src/models.py",
-    "src/api.py", "src/handlers.py", "src/middleware.py",
-    "tests/test_main.py", "tests/test_utils.py", "tests/conftest.py",
-    "README.md", "LICENSE", "package.json", "requirements.txt",
-    "config.yaml", "config.json", ".env.example",
-    "src/components/Button.tsx", "src/components/Header.jsx",
-    "src/styles.css", "src/index.js", "src/app.js",
-    "docs/API.md", "docs/ARCHITECTURE.md", "docs/CONTRIBUTING.md",
-    "scripts/setup.sh", "scripts/deploy.py", "Makefile"
-]
-
-CODE_SNIPPETS = {
-    "python": [
-        "def hello():\n    print('Hello, World!')",
-        "class MyClass:\n    def __init__(self):\n        self.value = 42",
-        "import os\nos.path.join('a', 'b')",
-        "async def fetch_data():\n    async with aiohttp.ClientSession() as session:\n        return await session.get(url)",
-    ],
-    "javascript": [
-        "const fetch = require('node-fetch');\nconst data = await fetch(url);",
-        "function handleClick() {\n  setCount(count + 1);\n}",
-        "export default function App() {\n  return <div>Hello</div>;\n}",
-        "const [state, setState] = useState(null);",
-    ],
-    "bash": [
-        "npm install",
-        "git status",
-        "pytest -v",
-        "python -m pytest tests/",
-        "make build",
-        "docker build -t myapp .",
-        "ls -la",
-        "curl -X GET https://api.example.com",
-    ]
-}
-
-WEB_SEARCH_QUERIES = [
-    "python async await best practices",
-    "javascript array methods map filter reduce",
-    "TypeScript generics tutorial",
-    "React hooks useEffect dependency array",
-    "Node.js error handling best practices",
-    "Docker vs Kubernetes differences",
-    "Git rebase vs merge",
-    "SQL join types explained",
-    "REST API design principles",
-    "Python list comprehension examples",
-    "JavaScript promise async await",
-    "CSS flexbox vs grid",
-    "MongoDB vs PostgreSQL",
-    "AWS Lambda cold start optimization",
-    "Docker compose tutorial",
-]
-
-GREP_PATTERNS = [
-    ("TODO", "*.py", "Find TODO comments in Python files"),
-    ("function ", "*.js", "Find function definitions in JS"),
-    ("import ", "*.py", "Find imports in Python"),
-    ("class ", "*.ts", "Find class definitions in TypeScript"),
-    ("def ", "*.py", "Find function definitions in Python"),
-    ("const ", "*.js", "Find const declarations in JS"),
-    ("interface ", "*.ts", "Find interface definitions"),
-    ("#pragma", "*.c", "Find pragma directives in C"),
-    ("SELECT ", "*.sql", "Find SQL queries"),
-    ("TODO", "*.md", "Find TODOs in documentation"),
-]
-
-# ============================================================================
-# Scenario Templates
-# ============================================================================
-
-SCENARIOS = {
-    "FileRead": [
-        {
-            "user": "Read the file at src/main.py and show me what's in it",
-            "tool_call": {"name": "FileRead", "arguments": {"path": "src/main.py"}},
-            "result": "Successfully read file: src/main.py\n```python\ndef main():\n    print('Hello from Stack 2.9!')\n\nif __name__ == '__main__':\n    main()\n```",
-            "response": "Here's the contents of `src/main.py`:\n\n```python\ndef main():\n    print('Hello from Stack 2.9!')\n\nif __name__ == '__main__':\n    main()\n```\n\nThis is a simple Python script that prints a greeting when executed directly."
-        },
-        {
-            "user": "Can you show me the README.md file?",
-            "tool_call": {"name": "FileRead", "arguments": {"path": "README.md"}},
-            "result": "Successfully read file: README.md\n```markdown\n# My Project\n\nA sample project for Stack 2.9.\n\n## Installation\n\npip install -r requirements.txt\n```",
-            "response": "Here's the README.md:\n\n```markdown\n# My Project\n\nA sample project for Stack 2.9.\n\n## Installation\n\npip install -r requirements.txt\n```"
-        },
-        {
-            "user": "What's in the config.yaml file?",
-            "tool_call": {"name": "FileRead", "arguments": {"path": "config.yaml"}},
-            "result": "Successfully read file: config.yaml\n```yaml\ndatabase:\n  host: localhost\n  port: 5432\n  name: myapp\n\napp:\n  port: 3000\n  debug: true\n```",
-            "response": "The config.yaml contains:\n\n```yaml\ndatabase:\n  host: localhost\n  port: 5432\n  name: myapp\n\napp:\n  port: 3000\n  debug: true\n```\n\nIt has database connection settings and app configuration."
-        },
-        {
-            "user": "Check the package.json to see the dependencies",
-            "tool_call": {"name": "FileRead", "arguments": {"path": "package.json"}},
-            "result": "Successfully read file: package.json\n```json\n{\n  \"name\": \"my-app\",\n  \"version\": \"1.0.0\",\n  \"dependencies\": {\n    \"express\": \"^4.18.0\",\n    \"react\": \"^18.2.0\"\n  }\n}\n```",
-            "response": "The package.json shows these dependencies:\n\n```json\n{\n  \"name\": \"my-app\",\n  \"version\": \"1.0.0\",\n  \"dependencies\": {\n    \"express\": \"^4.18.0\",\n    \"react\": \"^18.2.0\"\n  }\n}\n```"
-        },
-    ],
-    "FileWrite": [
-        {
-            "user": "Create a new file called hello.py with a hello world function",
-            "tool_call": {"name": "FileWrite", "arguments": {"path": "hello.py", "content": "def hello():\n    print('Hello, World!')\n\nif __name__ == '__main__':\n    hello()\n"}},
-            "result": "Successfully wrote file: hello.py",
-            "response": "Created `hello.py` with a hello world function:\n\n```python\ndef hello():\n    print('Hello, World!')\n\nif __name__ == '__main__':\n    hello()\n```\n\nYou can run it with `python hello.py`."
-        },
-        {
-            "user": "Write a simple React component to src/components/Button.jsx",
-            "tool_call": {"name": "FileWrite", "arguments": {"path": "src/components/Button.jsx", "content": "import React from 'react';\n\nexport default function Button({ onClick, children }) {\n  return (\n    <button onClick={onClick} className=\"btn\">\n      {children}\n    </button>\n  );\n}\n"}},
-            "result": "Successfully wrote file: src/components/Button.jsx",
-            "response": "Created `src/components/Button.jsx`:\n\n```jsx\nimport React from 'react';\n\nexport default function Button({ onClick, children }) {\n  return (\n    <button onClick={onClick} className=\"btn\">\n      {children}\n    </button>\n  );\n}\n```"
-        },
-        {
-            "user": "Create a config.json file with some settings",
-            "tool_call": {"name": "FileWrite", "arguments": {"path": "config.json", "content": "{\n  \"app\": {\n    \"name\": \"MyApp\",\n    \"version\": \"1.0.0\"\n  },\n  \"server\": {\n    \"port\": 8080,\n    \"host\": \"localhost\"\n  }\n}\n"}},
-            "result": "Successfully wrote file: config.json",
-            "response": "Created `config.json`:\n\n```json\n{\n  \"app\": {\n    \"name\": \"MyApp\",\n    \"version\": \"1.0.0\"\n  },\n  \"server\": {\n    \"port\": 8080,\n    \"host\": \"localhost\"\n  }\n}\n```"
-        },
-    ],
-    "Bash": [
-        {
-            "user": "Run npm install to install dependencies",
-            "tool_call": {"name": "Bash", "arguments": {"command": "npm install"}},
-            "result": "added 245 packages in 3.5s",
-            "response": "Dependencies installed successfully! Added 245 packages."
-        },
-        {
-            "user": "Check git status",
-            "tool_call": {"name": "Bash", "arguments": {"command": "git status"}},
-            "result": "On branch main\nYour branch is up to date with 'origin/main'.\n\nChanges not staged for commit:\n  modified:   README.md\n  modified:   src/main.py\n\nno changes added to commit (but untracked files present)",
-            "response": "Git status shows:\n- Modified: README.md, src/main.py (not staged)\n- Untracked files present"
-        },
-        {
-            "user": "Run the tests with pytest",
-            "tool_call": {"name": "Bash", "arguments": {"command": "pytest -v"}},
-            "result": "tests/test_main.py::test_hello PASSED\ntests/test_utils.py::test_add PASSED\n\n2 passed in 0.5s",
-            "response": "All tests passed!\n- `tests/test_main.py::test_hello` ✓\n- `tests/test_utils.py::test_add` ✓"
-        },
-        {
-            "user": "List all files in the current directory",
-            "tool_call": {"name": "Bash", "arguments": {"command": "ls -la"}},
-            "result": "total 64\ndrwxr-xr-x  12 user  staff   384 Apr  6 00:58 .\ndrwxr-xr-x   9 user  staff   288 Apr  6 00:50 ..\n-rw-r--r--   1 user  staff   220 Apr  6 00:50 README.md\ndrwxr-xr-x   3 user  staff   96 Apr  6 00:51 src\ndrwxr-xr-x   3 user  staff   96 Apr  6 00:51 tests\n...",
-            "response": "Files in current directory:\n- README.md\n- src/\n- tests/\n- And more (permissions, dates shown)"
-        },
-        {
-            "user": "Build the project with make",
-            "tool_call": {"name": "Bash", "arguments": {"command": "make build"}},
-            "result": "Building...\ngcc -o main main.c\nobuild successful!",
-            "response": "Build completed successfully!"
-        },
-    ],
-    "WebSearch": [
-        {
-            "user": "How do I use async/await in Python?",
-            "tool_call": {"name": "WebSearch", "arguments": {"query": "python async await best practices", "count": 5}},
-            "result": "Search results for 'python async await best practices':\n1. Real Python - Async IO in Python: A Complete Walkthrough\n2. Python Docs - asyncio - Asynchronous I/O\n3. Towards Data Science - Mastering Python Async/Await\n4. Stack Overflow - Common async/await mistakes\n5. Medium - Async/Await Patterns in Python",
-            "response": "Here are some resources on Python async/await:\n\n1. **Real Python** - Async IO in Python: A Complete Walkthrough\n2. **Python Docs** - asyncio - Asynchronous I/O\n3. **Towards Data Science** - Mastering Python Async/Await\n\nKey points:\n- Use `async def` to define async functions\n- Use `await` to call other async functions\n- `asyncio.run()` to execute the main coroutine"
-        },
-        {
-            "user": "What's the difference between Docker and Kubernetes?",
-            "tool_call": {"name": "WebSearch", "arguments": {"query": "Docker vs Kubernetes differences", "count": 5}},
-            "result": "Search results for 'Docker vs Kubernetes differences':\n1. Docker Blog - Containers 101\n2. Kubernetes.io - What is Kubernetes\n3. Red Hat - Docker vs Kubernetes explained\n4. DevOps.com - Container orchestration comparison\n5. Container Journal - When to use each",
-            "response": "**Docker** is a container platform - it packages applications into containers.\n\n**Kubernetes** is a container orchestrator - it manages multiple containers across machines.\n\nThink of Docker as individual shipping containers, and Kubernetes as the port management system that coordinates all of them."
-        },
-        {
-            "user": "How do I use React hooks properly?",
-            "tool_call": {"name": "WebSearch", "arguments": {"query": "React hooks useEffect dependency array", "count": 5}},
-            "result": "Search results for 'React hooks useEffect dependency array':\n1. React Docs - Synchronizing with Effects\n2. Kent C. Dodds - A Complete Guide to useEffect\n3. Robin Wieruch - React useEffect Hook Tutorial\n4. Stack Overflow - Common useEffect mistakes\n5. LogRocket - useEffect dependency array deep dive",
-            "response": "**React Hooks Guide:**\n\n1. **useState** - For state management\n2. **useEffect** - For side effects, takes dependency array\n3. **useContext** - For global state\n\nThe dependency array controls when the effect runs. Empty `[]` = once on mount. Include variables to re-run when they change."
-        },
-    ],
-    "Grep": [
-        {
-            "user": "Find all TODO comments in the Python files",
-            "tool_call": {"name": "Grep", "arguments": {"pattern": "TODO", "path": ".", "recursive": True, "file_pattern": "*.py"}},
-            "result": "Found 3 matches:\nsrc/main.py:14: # TODO: Add error handling\nsrc/utils.py:22: # TODO: Implement caching\ntests/test_main.py:8: # TODO: Add more tests",
-            "response": "Found 3 TODO comments in Python files:\n- `src/main.py:14` - Add error handling\n- `src/utils.py:22` - Implement caching\n- `tests/test_main.py:8` - Add more tests"
-        },
-        {
-            "user": "Search for function definitions in the JavaScript files",
-            "tool_call": {"name": "Grep", "arguments": {"pattern": "function ", "path": "src", "recursive": True, "file_pattern": "*.js"}},
-            "result": "Found 5 matches:\nsrc/index.js:3: function setup() {\nsrc/handlers.js:12: function handleClick(event) {\nsrc/handlers.js:25: function handleSubmit(form) {\nsrc/utils.js:8: function formatDate(date) {\nsrc/utils.js:15: function parseJSON(str) {",
-            "response": "Found 5 function definitions in src/:\n- `src/index.js:3` - setup()\n- `src/handlers.js:12` - handleClick(event)\n- `src/handlers.js:15` - handleSubmit(form)\n- `src/utils.js:8` - formatDate(date)\n- `src/utils.js:15` - parseJSON(str)"
-        },
-        {
-            "user": "Find all imports in the Python codebase",
-            "tool_call": {"name": "Grep", "arguments": {"pattern": "import ", "path": ".", "recursive": True, "file_pattern": "*.py"}},
-            "result": "Found 12 matches:\nsrc/main.py:1: import os\nsrc/main.py:2: import sys\nsrc/main.py:3: import json\nsrc/utils.py:1: from typing import Dict\nsrc/api.py:5: import requests\n...",
-            "response": "Found 12 import statements across Python files. Main imports:\n- `os`, `sys`, `json` in src/main.py\n- `typing.Dict` in src/utils.py\n- `requests` in src/api.py"
-        },
-        {
-            "user": "Look for class definitions in TypeScript files",
-            "tool_call": {"name": "Grep", "arguments": {"pattern": "class ", "path": "src", "recursive": True, "file_pattern": "*.ts"}},
-            "result": "Found 2 matches:\nsrc/models/User.ts:3: class User {\nsrc/models/Config.ts:8: class Config {",
-            "response": "Found 2 class definitions:\n- `src/models/User.ts:3` - User class\n- `src/models/Config.ts:8` - Config class"
-        },
-    ],
-}
-
-
-# ============================================================================
-# Data Augmentation Helpers
-# ============================================================================
-
-def vary_file_paths(template: Dict) -> Dict:
-    """Vary file paths in a scenario to create diversity."""
-    import copy
-    result = copy.deepcopy(template)
-    
-    # Vary the file paths
-    if "path" in result["tool_call"]["arguments"]:
-        original_path = result["tool_call"]["arguments"]["path"]
-        for old_path in FILE_PATHS:
-            if old_path in result["user"]:
-                result["user"] = result["user"].replace(old_path, random.choice(FILE_PATHS))
-                result["tool_call"]["arguments"]["path"] = random.choice(FILE_PATHS)
-                break
-    
-    return result
-
-
-def vary_content(template: Dict) -> Dict:
-    """Vary content in a scenario."""
-    import copy
-    result = copy.deepcopy(template)
-    
-    if "content" in result["tool_call"]["arguments"]:
-        # Vary code snippets
-        lang = random.choice(["python", "javascript"])
-        result["tool_call"]["arguments"]["content"] = random.choice(CODE_SNIPPETS[lang])
-    
-    return result
-
-
-def vary_bash_command(template: Dict) -> Dict:
-    """Vary bash commands."""
-    import copy
-    result = copy.deepcopy(template)
-    
-    if "command" in result["tool_call"]["arguments"]:
-        original = result["tool_call"]["arguments"]["command"].split()[0] if result["tool_call"]["arguments"]["command"] else ""
-        
-        if "npm" in original:
-            commands = ["npm install", "npm run build", "npm test", "npm start"]
-        elif "git" in original:
-            commands = ["git status", "git log --oneline -5", "git diff", "git branch -a"]
-        elif "pytest" in original:
-            commands = ["pytest -v", "pytest tests/", "pytest -xvs", "pytest --cov"]
-        elif "ls" in original:
-            commands = ["ls -la", "ls -1", "ls -lah"]
-        elif "make" in original:
-            commands = ["make build", "make clean", "make test", "make install"]
-        else:
-            commands = ["echo 'hello'", "pwd", "whoami", "date"]
-        
-        result["tool_call"]["arguments"]["command"] = random.choice(commands)
-    
-    return result
-
-
-def vary_search_query(template: Dict) -> Dict:
-    """Vary web search queries."""
-    import copy
-    result = copy.deepcopy(template)
-    
-    if "query" in result["tool_call"]["arguments"]:
-        result["tool_call"]["arguments"]["query"] = random.choice(WEB_SEARCH_QUERIES)
-    
-    return result
-
-
-def vary_grep_pattern(template: Dict) -> Dict:
-    """Vary grep patterns."""
-    import copy
-    result = copy.deepcopy(template)
-    
-    pattern, file_pattern, _ = random.choice(GREP_PATTERNS)
-    result["tool_call"]["arguments"]["pattern"] = pattern
-    result["tool_call"]["arguments"]["file_pattern"] = file_pattern
-    
-    return result
-
-
-# ============================================================================
-# Main Generation Functions
-# ============================================================================
-
-def create_tool_call_message(tool_call: Dict, tool_call_id: str) -> Dict:
-    """Create a tool_calls message in Qwen format."""
-    return {
-        "role": "assistant",
-        "content": None,
-        "tool_calls": [
-            {
-                "id": tool_call_id,
-                "type": "function",
-                "function": {
-                    "name": tool_call["name"],
-                    "arguments": json.dumps(tool_call["arguments"])
-                }
-            }
-        ]
-    }
-
-
-def create_tool_message(role: str, tool_call_id: str, tool_name: str, result: str) -> Dict:
-    """Create a tool message (result of tool execution)."""
-    return {
-        "role": role,  # typically "tool"
-        "content": result,
-        "tool_call_id": tool_call_id,
-        "name": tool_name
-    }
-
-
-def generate_example(scenario: Dict, system_prompt: str = None) -> Dict:
-    """Generate a single training example in Qwen2.5-Coder format."""
-    if system_prompt is None:
-        system_prompt = "You are a helpful AI assistant that can use tools to help users solve problems. When you need to perform actions like reading files, running commands, searching the web, or searching code, use the appropriate tool."
-    
-    tool_call_id = f"call_${random.randint(1000, 9999)}"
-    
-    messages = [
-        {"role": "system", "content": system_prompt},
-        {"role": "user", "content": scenario["user"]},
-        create_tool_call_message(scenario["tool_call"], tool_call_id),
-        create_tool_message("tool", tool_call_id, scenario["tool_call"]["name"], scenario["result"]),
-        {"role": "assistant", "content": scenario["response"]}
-    ]
-    
-    return {
-        "messages": messages,
-        "tools": TOOL_DEFINITIONS
-    }
-
-
-def augment_scenario(scenario: Dict, tool_name: str) -> Dict:
-    """Apply random augmentations to a scenario."""
-    import random
-    
-    augmented = scenario.copy()
-    
-    if tool_name == "FileRead":
-        augmented = vary_file_paths(augmented)
-    elif tool_name == "FileWrite":
-        augmented = vary_file_paths(augmented)
-        augmented = vary_content(augmented)
-    elif tool_name == "Bash":
-        augmented = vary_bash_command(augmented)
-    elif tool_name == "WebSearch":
-        augmented = vary_search_query(augmented)
-    elif tool_name == "Grep":
-        augmented = vary_grep_pattern(augmented)
-    
-    return augmented
-
-
-def generate_dataset(num_examples: int = 1000, output_path: str = None) -> List[Dict]:
-    """Generate the complete dataset."""
-    examples = []
-    tools = list(SCENARIOS.keys())
-    
-    # Track counts for balance
-    examples_per_tool = num_examples // len(tools)
-    remainder = num_examples % len(tools)
-    
-    for i, tool_name in enumerate(tools):
-        # Determine how many examples for this tool
-        count = examples_per_tool + (1 if i < remainder else 0)
-        
-        base_scenarios = SCENARIOS[tool_name]
-        
-        for j in range(count):
-            # Use base scenario and vary
-            base = base_scenarios[j % len(base_scenarios)]
-            
-            # Apply augmentations for variety
-            if j >= len(base_scenarios):
-                scenario = augment_scenario(base, tool_name)
-            else:
-                scenario = base
-            
-            example = generate_example(scenario)
-            examples.append(example)
-    
-    # Shuffle for better training
-    random.shuffle(examples)
-    
-    return examples
-
-
-def save_jsonl(examples: List[Dict], output_path: str):
-    """Save examples to JSONL format."""
-    output_file = Path(output_path)
-    output_file.parent.mkdir(parents=True, exist_ok=True)
-    
-    with open(output_file, 'w', encoding='utf-8') as f:
-        for example in examples:
-            f.write(json.dumps(example, ensure_ascii=False) + '\n')
-
-
-def save_json(examples: List[Dict], output_path: str):
-    """Save examples to JSON format."""
-    output_file = Path(output_path)
-    output_file.parent.mkdir(parents=True, exist_ok=True)
-    
-    with open(output_file, 'w', encoding='utf-8') as f:
-        json.dump(examples, f, ensure_ascii=False, indent=2)
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Generate synthetic tool-calling training data")
-    parser.add_argument("--num-examples", type=int, default=1000, help="Number of examples to generate")
-    parser.add_argument("--output-dir", type=str, default="training-data", help="Output directory")
-    parser.add_argument("--output-format", choices=["jsonl", "json", "both"], default="jsonl", help="Output format")
-    parser.add_argument("--seed", type=int, default=42, help="Random seed")
-    args = parser.parse_args()
-    
-    # Set seed for reproducibility
-    random.seed(args.seed)
-    
-    print(f"🎯 Generating {args.num_examples} tool-calling training examples...")
-    print(f"   Output directory: {args.output_dir}")
-    print(f"   Format: {args.output_format}")
-    print()
-    
-    # Generate dataset
-    examples = generate_dataset(args.num_examples)
-    
-    output_dir = Path(args.output_dir)
-    
-    # Save based on format
-    if args.output_format in ["jsonl", "both"]:
-        jsonl_path = output_dir / "tool_examples.jsonl"
-        save_jsonl(examples, str(jsonl_path))
-        print(f"✅ Saved JSONL: {jsonl_path}")
-    
-    if args.output_format in ["json", "both"]:
-        json_path = output_dir / "tool_examples.json"
-        save_json(examples, str(json_path))
-        print(f"✅ Saved JSON: {json_path}")
-    
-    # Statistics
-    print(f"\n📊 Statistics:")
-    print(f"   Total examples: {len(examples)}")
-    
-    # Count by tool
-    tool_counts = {}
-    for ex in examples:
-        for msg in ex["messages"]:
-            if msg.get("tool_calls"):
-                tool_name = msg["tool_calls"][0]["function"]["name"]
-                tool_counts[tool_name] = tool_counts.get(tool_name, 0) + 1
-    
-    print(f"   Examples by tool:")
-    for tool, count in sorted(tool_counts.items(), key=lambda x: x[1], reverse=True):
-        print(f"     - {tool}: {count}")
-    
-    # Show sample
-    print(f"\n📝 Sample example (first in dataset):")
-    sample = examples[0]
-    print(f"   Tools defined: {len(sample['tools'])}")
-    print(f"   Messages: {len(sample['messages'])}")
-    print(f"   First user message: {sample['messages'][1]['content'][:60]}...")
-    
-    print(f"\n✨ Generation complete!")
-
-
-if __name__ == "__main__":
-    main()

scripts/generate_tool_use_tests.py

@@ -1,163 +0,0 @@
-#!/usr/bin/env python3
-"""
-Tool-Use Evaluation Framework for Stack 2.9.
-Generates test cases and evaluates model's tool selection accuracy.
-"""
-
-import json
-import random
-import re
-from pathlib import Path
-from typing import Dict, List, Any
-import argparse
-
-def load_tool_catalog(path: str) -> List[Dict]:
-    with open(path, 'r') as f:
-        return json.load(f)
-
-def generate_test_case(tool: Dict[str, Any]) -> Dict[str, Any]:
-    """Generate a single test case for a tool."""
-    tool_name = tool["tool"]
-
-    # Templates for each tool (simplified)
-    user_prompts = {
-        "FileReadTool": [
-            "Read {file_path}",
-            "Show me the contents of {file_path}",
-            "What's in {file_path}?",
-            "Open {file_path}"
-        ],
-        "FileWriteTool": [
-            "Create a new file {file_path} with content: {content}",
-            "Write this to {file_path}: {content}",
-            "Save the following as {file_path}: {content}"
-        ],
-        "GlobTool": [
-            "Find all {pattern} files",
-            "List files matching {pattern}",
-            "Show me every {pattern}",
-            "Search for files like {pattern}"
-        ],
-        "GrepTool": [
-            "Search for {pattern} in {directory}",
-            "Find all {pattern}",
-            "Grep for {pattern}",
-            "Locate {pattern} in the codebase"
-        ],
-        "BashTool": [
-            "Run: {command}",
-            "Execute {command}",
-            "Please run {command}",
-            "Can you execute {command}?"
-        ]
-        # ... others use default fallback
-    }
-
-    prompts = user_prompts.get(tool_name, [
-        "Use {tool} to do something",
-        "Execute {tool}",
-        "Call {tool}"
-    ])
-
-    # Choose random prompt template
-    prompt_template = random.choice(prompts)
-
-    # Extract placeholders from template
-    placeholders = re.findall(r'{(.*?)}', prompt_template)
-
-    # Generate parameter values for each placeholder
-    params = {}
-    for ph in placeholders:
-        if ph == 'file_path':
-            params[ph] = random.choice([
-                "src/main.py", "README.md", "package.json",
-                "config.yaml", "tests/test_api.py", "src/index.js"
-            ])
-        elif ph == 'pattern':
-            params[ph] = random.choice([
-                "**/*.py", "**/*.js", "**/*.md", "**/*.test.*",
-                "src/**/*.ts", "lib/**/*.py"
-            ])
-        elif ph == 'command':
-            params[ph] = random.choice([
-                "npm test", "pytest", "git status", "ls -la",
-                "make build", "python -m pip install -e ."
-            ])
-        elif ph == 'query':
-            params[ph] = random.choice(["TODO", "FIXME", "BUG", "HACK"])
-        elif ph == 'directory':
-            params[ph] = random.choice([".", "src", "tests", "lib", "app"])
-        elif ph == 'content':
-            params[ph] = "console.log('test');"
-        elif ph == 'tool':
-            params[ph] = tool_name
-        else:
-            params[ph] = f"value_{random.randint(1,100)}"
-
-    # Fill prompt template
-    prompt = prompt_template.format(**params)
-
-    # Build expected tool call
-    expected_tool = tool_name
-    # Remove 'tool' param if present (it's just for substitution)
-    expected_params = {k: v for k, v in params.items() if k != 'tool'}
-
-    return {
-        "test_id": f"{tool_name}_{random.randint(1000,9999)}",
-        "prompt": prompt,
-        "expected_tool": expected_tool,
-        "expected_params": expected_params,
-        "tool_description": tool.get("description", ""),
-        "difficulty": random.choice(["easy", "medium", "hard"])
-    }
-
-def generate_test_suite(catalog: List[Dict], tests_per_tool: int = 10) -> List[Dict]:
-    """Generate test suite for all tools."""
-    suite = []
-    for tool in catalog:
-        for _ in range(tests_per_tool):
-            test_case = generate_test_case(tool)
-            suite.append(test_case)
-    return suite
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--catalog", type=str, default="training-data/tools/catalog.json")
-    parser.add_argument("--output", type=str, default="stack-2.9-eval/tool_use/test_cases.json")
-    parser.add_argument("--tests-per-tool", type=int, default=10)
-    args = parser.parse_args()
-
-    catalog_path = Path(args.catalog)
-    output_path = Path(args.output)
-
-    if not catalog_path.exists():
-        print(f"❌ Catalog not found: {catalog_path}")
-        return
-
-    tools = load_tool_catalog(catalog_path)
-    print(f"🔧 Generating test cases for {len(tools)} tools")
-
-    suite = generate_test_suite(tools, args.tests_per_tool)
-
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-    with open(output_path, 'w') as f:
-        json.dump(suite, f, indent=2)
-
-    print(f"\n✨ Generated {len(suite)} test cases")
-    print(f"   Saved to: {output_path}")
-
-    # Summary by tool
-    by_tool = {}
-    for tc in suite:
-        tool = tc["expected_tool"]
-        by_tool[tool] = by_tool.get(tool, 0) + 1
-
-    print("\n📊 Test cases per tool (top 10):")
-    for tool, count in sorted(by_tool.items(), key=lambda x: x[1], reverse=True)[:10]:
-        print(f"   {tool}: {count}")
-
-    print("\n✅ Test suite ready!")
-    print("   To evaluate: run tool_use_evaluator.py with a trained model")
-
-if __name__ == "__main__":
-    main()

scripts/mine_sessions.py

@@ -1,233 +0,0 @@
-#!/usr/bin/env python3
-"""
-Mine OpenClaw/Claude sessions for training data.
-Extracts conversations with tool use into JSONL format.
-"""
-
-import os
-import json
-import glob
-from pathlib import Path
-from typing import Dict, List, Any
-import argparse
-
-def find_session_logs() -> List[Path]:
-    """Find potential session log files in common locations."""
-    possible_locations = [
-        # OpenClaw sessions
-        Path.home() / ".openclaw" / "sessions",
-        Path.home() / ".openclaw" / "history",
-        # Claude Code sessions
-        Path.home() / ".claude" / "sessions",
-        Path.home() / ".anthropic" / "sessions",
-        # Generic
-        Path.home() / "Documents" / "claude_sessions",
-        Path.cwd() / "sessions",
-        Path.cwd() / ".sessions",
-    ]
-
-    log_files = []
-    for location in possible_locations:
-        if location.exists():
-            # Look for JSON, JSONL, or MD files
-            for pattern in ["*.json", "*.jsonl", "*.md"]:
-                log_files.extend(location.glob(pattern))
-
-    return log_files
-
-def parse_json_conversation(data: Dict[str, Any]) -> List[Dict[str, Any]]:
-    """Parse a JSON conversation into training examples."""
-    examples = []
-
-    # Try different known formats
-    if "messages" in data:
-        # OpenAI format
-        messages = data["messages"]
-        if is_valid_conversation(messages):
-            examples.append({"messages": messages, "source": "openai_format"})
-
-    elif "conversation" in data:
-        # Custom format with conversation array
-        messages = data["conversation"]
-        if is_valid_conversation(messages):
-            examples.append({"messages": messages, "source": "custom"})
-
-    elif "turns" in data:
-        # Turn-based format
-        turns = data["turns"]
-        messages = []
-        for turn in turns:
-            if "role" in turn and "content" in turn:
-                messages.append({
-                    "role": turn["role"],
-                    "content": turn["content"],
-                    "tool_use": turn.get("tool_use"),
-                    "tool_result": turn.get("tool_result")
-                })
-        if is_valid_conversation(messages):
-            examples.append({"messages": messages, "source": "turn_based"})
-
-    return examples
-
-def is_valid_conversation(messages: List[Dict[str, Any]]) -> bool:
-    """Check if message list is a valid conversation with tool use."""
-    if not isinstance(messages, list) or len(messages) < 2:
-        return False
-
-    # Must have at least one user and one assistant message
-    roles = [m.get("role") for m in messages if "role" in m]
-    if "user" not in roles or "assistant" not in roles:
-        return False
-
-    return True
-
-def parse_markdown_conversation(text: str) -> List[Dict[str, Any]]:
-    """Parse Markdown logs (Claude Code format typically)."""
-    examples = []
-
-    # Claude Code / chat format often has blocks like:
-    # User: ...
-    # Assistant: ...
-    # or with tool use in special blocks
-
-    lines = text.split("\n")
-    current_role = None
-    current_content = []
-    messages = []
-
-    for line in lines:
-        line = line.rstrip()
-
-        # Detect role changes
-        if line.startswith("**User:**") or line.startswith("User:"):
-            if current_role:
-                messages.append({
-                    "role": current_role,
-                    "content": "\n".join(current_content).strip()
-                })
-            current_role = "user"
-            current_content = [line.split(":", 1)[1].strip()] if ":" in line else []
-        elif line.startswith("**Assistant:**") or line.startswith("Assistant:"):
-            if current_role:
-                messages.append({
-                    "role": current_role,
-                    "content": "\n".join(current_content).strip()
-                })
-            current_role = "assistant"
-            current_content = [line.split(":", 1)[1].strip()] if ":" in line else []
-        elif line.startswith("**Tool:**") or line.startswith("Tool Use:"):
-            if current_role:
-                messages.append({
-                    "role": current_role,
-                    "content": "\n".join(current_content).strip()
-                })
-            current_role = "assistant"
-            # Start tool use block
-            current_content = []
-            # Could parse tool name and parameters
-        else:
-            if current_role:
-                current_content.append(line)
-
-    # Don't forget last message
-    if current_role and current_content:
-        messages.append({
-            "role": current_role,
-            "content": "\n".join(current_content).strip()
-        })
-
-    if is_valid_conversation(messages):
-        examples.append({"messages": messages, "source": "markdown"})
-
-    return examples
-
-def save_examples(examples: List[Dict[str, Any]], output_path: Path):
-    """Save examples to JSONL file."""
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-
-    with open(output_path, 'a') as f:
-        for ex in examples:
-            f.write(json.dumps(ex) + "\n")
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--output", type=str, default="training-data/scaled/sessions.jsonl")
-    parser.add_argument("--dry-run", action="store_true", help="Just list files, don't parse")
-    args = parser.parse_args()
-
-    output_path = Path(args.output)
-
-    print(f"🔍 Searching for session logs...")
-    log_files = find_session_logs()
-
-    if not log_files:
-        print("⚠️  No session logs found in standard locations.")
-        print("   Expected locations: ~/.openclaw/sessions, ~/.claude/sessions, ~/.anthropic/sessions")
-        return
-
-    print(f"📁 Found {len(log_files)} log files")
-
-    if args.dry_run:
-        for f in log_files[:10]:
-            print(f"  - {f}")
-        if len(log_files) > 10:
-            print(f"  ... and {len(log_files)-10} more")
-        return
-
-    total_examples = 0
-    for log_file in log_files:
-        try:
-            with open(log_file, 'r', encoding='utf-8', errors='ignore') as f:
-                content = f.read()
-
-            examples = []
-
-            # Try JSON first
-            if log_file.suffix in ['.json', '.jsonl']:
-                if log_file.suffix == '.jsonl':
-                    # Multiple JSON objects per line
-                    for line in content.split('\n'):
-                        line = line.strip()
-                        if line:
-                            try:
-                                data = json.loads(line)
-                                examples.extend(parse_json_conversation(data))
-                            except json.JSONDecodeError:
-                                pass
-                else:
-                    # Single JSON object
-                    try:
-                        data = json.loads(content)
-                        examples.extend(parse_json_conversation(data))
-                    except json.JSONDecodeError:
-                        # Maybe it's a JSON array
-                        try:
-                            data_list = json.loads(content)
-                            if isinstance(data_list, list):
-                                for data in data_list:
-                                    examples.extend(parse_json_conversation(data))
-                        except:
-                            pass
-            else:
-                # Markdown or text
-                examples.extend(parse_markdown_conversation(content))
-
-            if examples:
-                save_examples(examples, output_path)
-                total_examples += len(examples)
-                print(f"✅ {log_file.name}: {len(examples)} examples")
-
-        except Exception as e:
-            print(f"❌ Error processing {log_file}: {e}")
-
-    print(f"\n✨ Extracted {total_examples} examples from session logs")
-    print(f"   Saved to: {output_path}")
-
-    if total_examples == 0:
-        print("\n⚠️  No valid conversations found. Consider:")
-        print("   1. Check if you have session logs in non-standard locations")
-        print("   2. Your logs may be in a different format")
-        print("   3. You may need to export conversations from your tools")
-
-if __name__ == "__main__":
-    main()

scripts/quality_validate.py

@@ -1,158 +0,0 @@
-#!/usr/bin/env python3
-"""
-Quality validation for Stack 2.9 training dataset.
-Checks: message structure, tool format, schema compliance.
-"""
-
-import json
-from pathlib import Path
-from typing import Dict, List, Any
-import argparse
-from collections import Counter
-
-def load_tool_catalog(path: str) -> Dict[str, Any]:
-    with open(path, 'r') as f:
-        return {tool["tool"]: tool for tool in json.load(f)}
-
-def validate_example(example: Dict[str, Any], tool_catalog: Dict[str, Any]) -> List[str]:
-    """Validate a single example. Returns list of errors (empty if valid)."""
-    errors = []
-
-    if "messages" not in example:
-        errors.append("Missing 'messages' field")
-        return errors
-
-    messages = example["messages"]
-    if not isinstance(messages, list) or len(messages) < 2:
-        errors.append("Invalid messages: must be list with at least 2 messages")
-        return errors
-
-    # Check roles sequence
-    roles = [msg.get("role") for msg in messages]
-    valid_roles = {"system", "user", "assistant"}
-    if not all(r in valid_roles for r in roles):
-        errors.append(f"Invalid roles: {roles}")
-
-    # Tool use validation
-    for msg in messages:
-        if msg.get("role") == "assistant" and "tool_use" in msg:
-            tool_use = msg["tool_use"]
-            if "name" not in tool_use:
-                errors.append("Tool use missing 'name'")
-            else:
-                tool_name = tool_use["name"]
-                if tool_name not in tool_catalog:
-                    errors.append(f"Unknown tool: {tool_name}")
-            if "input" not in tool_use:
-                errors.append(f"Tool use missing 'input' for {tool_name}")
-
-        if msg.get("role") == "user" and "tool_result" in msg:
-            tool_result = msg["tool_result"]
-            if "tool_use_id" not in tool_result:
-                errors.append("Tool result missing 'tool_use_id'")
-            if "content" not in tool_result:
-                errors.append("Tool result missing 'content'")
-
-    # Check message content is non-empty (except user with tool_result can be empty)
-    for i, msg in enumerate(messages):
-        role = msg.get("role")
-        content = msg.get("content")
-        if role == "user" and "tool_result" in msg:
-            continue  # Tool result user message can have empty content
-        if content is not None and not isinstance(content, str):
-            errors.append(f"Message content must be string, got {type(content)}")
-        if content is not None and len(content.strip()) == 0:
-            errors.append(f"Empty content in {role} message")
-
-    return errors
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--input", type=str, default="training-data/final/train.jsonl")
-    parser.add_argument("--catalog", type=str, default="training-data/tools/catalog.json")
-    parser.add_argument("--output-report", type=str, default="training-data/final/quality_report.json")
-    args = parser.parse_args()
-
-    input_path = Path(args.input)
-    catalog_path = Path(args.catalog)
-
-    if not input_path.exists():
-        print(f"❌ Input not found: {input_path}")
-        return
-
-    if not catalog_path.exists():
-        print(f"⚠️  Catalog not found: {catalog_path}, skipping tool validation")
-        tool_catalog = {}
-    else:
-        tool_catalog = load_tool_catalog(catalog_path)
-        print(f"✅ Loaded tool catalog with {len(tool_catalog)} tools")
-
-    print(f"🔍 Validating {input_path}...")
-
-    total_examples = 0
-    valid_examples = 0
-    error_distribution = Counter()
-    tool_usage = Counter()
-
-    with open(input_path, 'r') as f:
-        for line in f:
-            total_examples += 1
-            try:
-                example = json.loads(line)
-                errors = validate_example(example, tool_catalog)
-
-                if errors:
-                    for err in errors:
-                        error_distribution[err] += 1
-                else:
-                    valid_examples += 1
-
-                # Track tool usage regardless of validation
-                for msg in example.get("messages", []):
-                    if "tool_use" in msg:
-                        tool_name = msg["tool_use"]["name"]
-                        tool_usage[tool_name] += 1
-
-            except json.JSONDecodeError:
-                error_distribution["JSON decode error"] += 1
-
-    print(f"\n📊 Validation Results:")
-    print(f"   Total examples: {total_examples}")
-    print(f"   Valid: {valid_examples} ({valid_examples/total_examples*100:.1f}%)")
-    print(f"   Invalid: {total_examples - valid_examples}")
-
-    if error_distribution:
-        print("\n   Error breakdown:")
-        for err, count in error_distribution.most_common(10):
-            print(f"     - {err}: {count}")
-
-    print("\n   Tool usage (top 10):")
-    for tool, count in tool_usage.most_common(10):
-        print(f"     - {tool}: {count}")
-
-    # Write report
-    report = {
-        "total_examples": total_examples,
-        "valid_examples": valid_examples,
-        "invalid_examples": total_examples - valid_examples,
-        "validity_rate": valid_examples / total_examples if total_examples > 0 else 0,
-        "error_distribution": dict(error_distribution),
-        "tool_usage": dict(tool_usage),
-        "generated_at": datetime.datetime.now().isoformat()
-    }
-
-    output_report = Path(args.output_report)
-    output_report.parent.mkdir(parents=True, exist_ok=True)
-    with open(output_report, 'w') as f:
-        json.dump(report, f, indent=2)
-
-    print(f"\n✅ Report saved: {output_report}")
-
-    if valid_examples / total_examples < 0.9:
-        print("\n⚠️  Quality below 90%. Consider filtering invalid examples before training.")
-    else:
-        print("\n✅ Dataset quality looks good for training!")
-
-if __name__ == "__main__":
-    import json, datetime
-    main()

scripts/training-data-extractor.js

@@ -1,1098 +0,0 @@
-#!/usr/bin/env node
-/**
- * Stack 2.9 - Enhanced Training Data Extractor
- * Extracts training examples from OpenClaw codebase
- * 
- * Features:
- * 1. Parse code patterns: function+comment pairs, error messages, test files
- * 2. Real conversation parsing (JSON, JSONL, Markdown formats)
- * 3. Synthetic examples (50+ per tool)
- * 4. JSONL output
- */
-
-import fs from 'fs';
-import path from 'path';
-import { fileURLToPath } from 'url';
-import os from 'os';
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
-// Paths
-const SRC_DIR = path.join(__dirname, 'src');
-const OUTPUT_DIR = path.join(__dirname, 'training-data');
-const SYNTHETIC_DIR = path.join(OUTPUT_DIR, 'synthetic');
-const TOOLS_SCHEMA_DIR = path.join(OUTPUT_DIR, 'tools');
-const CODE_PAIRS_DIR = path.join(OUTPUT_DIR, 'code-pairs');
-const CONVERSATIONS_DIR = path.join(OUTPUT_DIR, 'conversations');
-
-// Ensure directories exist
-for (const dir of [OUTPUT_DIR, SYNTHETIC_DIR, TOOLS_SCHEMA_DIR, CODE_PAIRS_DIR, CONVERSATIONS_DIR]) {
-  fs.mkdirSync(dir, { recursive: true });
-}
-
-// ============================================================================
-// 1. EXTRACT TOOL SCHEMAS FROM src/tools/
-// ============================================================================
-
-function extractToolSchemas() {
-  const toolsDir = path.join(SRC_DIR, 'tools');
-  if (!fs.existsSync(toolsDir)) {
-    console.log('⚠️  Tools directory not found, skipping...');
-    return [];
-  }
-
-  const schemas = [];
-  const toolDirs = fs.readdirSync(toolsDir).filter(name => {
-    const stat = fs.statSync(path.join(toolsDir, name));
-    return stat.isDirectory();
-  });
-
-  for (const toolDir of toolDirs) {
-    const toolPath = path.join(toolsDir, toolDir);
-    const promptFile = path.join(toolPath, 'prompt.ts');
-    const toolFile = path.join(toolPath, toolDir + '.tsx') || path.join(toolPath, toolDir + '.ts');
-
-    if (fs.existsSync(promptFile) || fs.existsSync(toolFile)) {
-      try {
-        const promptContent = fs.existsSync(promptFile) ? fs.readFileSync(promptFile, 'utf-8') : '';
-        const toolContent = fs.existsSync(toolFile) ? fs.readFileSync(toolFile, 'utf-8') : '';
-
-        // Extract tool description from JSDoc
-        const descMatch = promptContent.match(/\/\*\*([\s\S]*?)\*\//);
-        let description = '';
-        if (descMatch) {
-          description = descMatch[1]
-            .replace(/^\s*\* ?/gm, '')
-            .replace(/^\s*\*/g, '')
-            .replace(/\*\/$/, '')
-            .trim()
-            .substring(0, 300);
-        }
-
-        // Extract input interface from tool file
-        let inputSchema = {};
-        const interfaceMatch = toolContent.match(/interface\s+(\w+Input\w*)\s*\{([\s\S]*?)\}/);
-        if (interfaceMatch) {
-          const fields = interfaceMatch[2].match(/(\w+)(\??):\s*([^;]+);/g) || [];
-          for (const field of fields) {
-            const match = field.match(/(\w+)(\??):\s*([^;]+);/);
-            if (match) {
-              inputSchema[match[1]] = { type: match[3].trim(), optional: match[2] === '?' };
-            }
-          }
-        }
-
-        schemas.push({
-          tool: toolDir,
-          description,
-          hasPrompt: !!promptContent,
-          hasImplementation: !!toolContent,
-          inputSchema
-        });
-      } catch (e) {
-        console.log(`⚠️  Error parsing ${toolDir}: ${e.message}`);
-      }
-    }
-  }
-
-  // Write tools catalog
-  fs.writeFileSync(
-    path.join(TOOLS_SCHEMA_DIR, 'catalog.json'),
-    JSON.stringify(schemas, null, 2)
-  );
-
-  console.log(`✅ Extracted ${schemas.length} tool schemas`);
-  return schemas;
-}
-
-// ============================================================================
-// 2. EXTRACT CODE-COMMENT PAIRS FROM src/
-// ============================================================================
-
-function extractCodeCommentPairs() {
-  console.log('🔍 Extracting code-comment pairs...');
-  const pairs = [];
-  
-  // Patterns for JSDoc comments
-  const jsdocPattern = /\/\*\*([\s\S]*?)\*\/\s*\n(?:export\s+)?(?:async\s+)?function\s+(\w+)\s*\(([^)]*)\)\s*(?::\s*([^{]+))?\{([\s\S]*?)\n\}/g;
-  const methodPattern = /\/\*\*([\s\S]*?)\*\/\s*\n\s*(?:async\s+)?(\w+)\s*\([^)]*\)[^:]*\{([\s\S]*?)\n\s*\}/g;
-  
-  // Error message patterns
-  const errorPattern = /(?:throw\s+new\s+Error|logger\.error|console\.error)\s*\(\s*[`"']([^`'"]+)[`'"]/g;
-  const errorClassPattern = /class\s+(\w+Error\w*)\s+extends\s+Error\s*\{([^}]*)\}/g;
-
-  function processFile(filePath) {
-    try {
-      const content = fs.readFileSync(filePath, 'utf-8');
-      const relativePath = path.relative(SRC_DIR, filePath);
-      
-      // Skip test files and mock files for now
-      if (filePath.includes('__tests__') || filePath.includes('mocks')) return;
-
-      // Extract function + JSDoc pairs
-      let match;
-      const funcRegex = /\/\*\*([\s\S]*?)\*\/\s*\n\s*(?:export\s+)?(?:async\s+)?function\s+(\w+)\s*\(([^)]*)\)\s*(?::\s*([^;\n]+))?[^{]*\{([\s\S]*?)\n\}/g;
-      
-      while ((match = funcRegex.exec(content)) !== null) {
-        const jsdoc = match[1].replace(/^\s*\*\s*/gm, '').trim();
-        const funcName = match[2];
-        const params = match[3].trim();
-        const returnType = match[4]?.trim() || 'void';
-        const body = match[5].trim();
-        
-        // Only include if meaningful (not too short, has actual logic)
-        if (body.length > 50 && jsdoc.length > 10) {
-          pairs.push({
-            type: 'function',
-            name: funcName,
-            path: relativePath,
-            code: `function ${funcName}(${params})${returnType ? `: ${returnType}` : ''} { ... }`,
-            fullBody: body.substring(0, 500),
-            comment: jsdoc.substring(0, 300),
-            commentType: 'jsdoc'
-          });
-        }
-      }
-
-      // Extract error messages and patterns
-      const errorRegex = /(?:throw\s+new\s+Error|logger\.error|console\.error)\s*\(\s*[`"']([^`'"]+)[`'"]/g;
-      let errorMatch;
-      while ((errorMatch = errorRegex.exec(content)) !== null) {
-        const errorMsg = errorMatch[1];
-        // Categorize error type
-        let category = 'general';
-        if (errorMsg.includes('not found') || errorMsg.includes('No such')) category = 'not_found';
-        else if (errorMsg.includes('permission') || errorMsg.includes('denied')) category = 'permission';
-        else if (errorMsg.includes('invalid') || errorMsg.includes('malformed')) category = 'validation';
-        else if (errorMsg.includes('timeout')) category = 'timeout';
-        else if (errorMsg.includes('already')) category = 'conflict';
-        
-        pairs.push({
-          type: 'error_message',
-          path: relativePath,
-          message: errorMsg,
-          category,
-          fixSuggestion: generateFixSuggestion(errorMsg, category)
-        });
-      }
-
-      // Extract class with error handling
-      const classRegex = /class\s+(\w+)\s*\{([^}]*(?:\{[^}]*\}[^}]*)*)\}/g;
-      let classMatch;
-      while ((classMatch = classRegex.exec(content)) !== null) {
-        const className = match[1];
-        const classBody = match[2];
-        // Look for try-catch patterns
-        if (classBody.includes('try') && classBody.includes('catch')) {
-          pairs.push({
-            type: 'error_handling_class',
-            name: className,
-            path: relativePath,
-            pattern: 'try-catch',
-            example: classBody.substring(0, 400)
-          });
-        }
-      }
-
-    } catch (e) {
-      // Skip files that can't be read
-    }
-  }
-
-  function walkDir(dir, extensions = ['.ts', '.tsx']) {
-    if (!fs.existsSync(dir)) return;
-    
-    const entries = fs.readdirSync(dir, { withFileTypes: true });
-    for (const entry of entries) {
-      const fullPath = path.join(dir, entry.name);
-      if (entry.isDirectory() && !entry.name.startsWith('.') && entry.name !== 'node_modules') {
-        walkDir(fullPath, extensions);
-      } else if (entry.isFile() && extensions.some(ext => entry.name.endsWith(ext))) {
-        processFile(fullPath);
-      }
-    }
-  }
-
-  walkDir(SRC_DIR);
-
-  // Save code-comment pairs
-  fs.writeFileSync(
-    path.join(CODE_PAIRS_DIR, 'pairs.json'),
-    JSON.stringify(pairs, null, 2)
-  );
-  
-  console.log(`✅ Extracted ${pairs.length} code-comment pairs`);
-  return pairs;
-}
-
-function generateFixSuggestion(message, category) {
-  const suggestions = {
-    not_found: 'Check if the resource exists or provide the correct path',
-    permission: 'Ensure you have the necessary permissions for this operation',
-    validation: 'Verify the input format and required fields',
-    timeout: 'Increase timeout duration or check network connectivity',
-    conflict: 'Check if the resource already exists or needs to be deleted first',
-    general: 'Review the error message and correct the underlying issue'
-  };
-  return suggestions[category] || suggestions.general;
-}
-
-// ============================================================================
-// 3. PARSE TEST FILES FOR TEST-GENERATION EXAMPLES
-// ============================================================================
-
-function extractTestExamples() {
-  console.log('🧪 Extracting test examples...');
-  const testExamples = [];
-  
-  const testPattern = /describe\s*\(\s*['"]([^'"]+)['"](?:\s*,\s*)?\(\s*\)\s*=>\s*\{([^}]*(?:\{[^}]*\}[^}]*)*)\}\s*\)/g;
-  const itPattern = /it\s*\(\s*['"]([^'"]+)['"](?:\s*,\s*)?(?:async\s+)?\(\s*\)\s*(?:=>\s*)?\{([\s\S]*?)\n\s*\}/g;
-  const expectPattern = /expect\s*\(([^)]+)\)\.(to[^;(]+)\s*\(([^)]+)\)/g;
-  
-  function processTestFile(filePath) {
-    try {
-      const content = fs.readFileSync(filePath, 'utf-8');
-      const relativePath = path.relative(SRC_DIR, filePath);
-      
-      let match;
-      while ((match = testPattern.exec(content)) !== null) {
-        const testSuite = match[1];
-        const testBody = match[2];
-        
-        // Extract individual it() blocks
-        const itRegex = /it\s*\(\s*['"]([^'"]+)['"](?:\s*,\s*)?(?:async\s+)?\(\s*\)\s*(?:=>\s*)?\{([\s\S]*?)\n\s*\}/g;
-        let itMatch;
-        
-        while ((itMatch = itRegex.exec(testBody)) !== null) {
-          const testName = itMatch[1];
-          const testCode = itMatch[2].trim();
-          
-          // Extract assertions
-          const assertions = [];
-          const expectRegex = /expect\s*\(([^)]+)\)\.(\w+)\s*\(([^)]*)\)/g;
-          let expectMatch;
-          
-          while ((expectMatch = expectRegex.exec(testCode)) !== null) {
-            assertions.push({
-              actual: expectMatch[1],
-              matcher: expectMatch[2],
-              expected: expectMatch[3]
-            });
-          }
-          
-          if (assertions.length > 0) {
-            testExamples.push({
-              type: 'test_example',
-              suite: testSuite,
-              name: testName,
-              path: relativePath,
-              code: testCode.substring(0, 400),
-              assertions,
-              isAsync: testCode.includes('await')
-            });
-          }
-        }
-      }
-    } catch (e) {
-      // Skip files that can't be read
-    }
-  }
-  
-  function walkDir(dir) {
-    if (!fs.existsSync(dir)) return;
-    
-    const entries = fs.readdirSync(dir, { withFileTypes: true });
-    for (const entry of entries) {
-      const fullPath = path.join(dir, entry.name);
-      if (entry.isDirectory()) {
-        walkDir(fullPath);
-      } else if (entry.isFile() && (entry.name.endsWith('.test.ts') || entry.name.endsWith('.test.tsx'))) {
-        processTestFile(fullPath);
-      }
-    }
-  }
-  
-  // Look for test files in __tests__ directories
-  walkDir(SRC_DIR);
-  
-  // Save test examples
-  fs.writeFileSync(
-    path.join(CODE_PAIRS_DIR, 'test-examples.json'),
-    JSON.stringify(testExamples, null, 2)
-  );
-  
-  console.log(`✅ Extracted ${testExamples.length} test examples`);
-  return testExamples;
-}
-
-// ============================================================================
-// 4. PARSE REAL CONVERSATIONS FROM SESSION LOGS
-// ============================================================================
-
-function parseConversations() {
-  console.log('💬 Parsing conversations from session logs...');
-  const conversations = [];
-  
-  // Common session log locations
-  const sessionLogPaths = [
-    path.join(os.homedir(), '.claude', 'sessions'),
-    path.join(os.homedir(), '.openclaw', 'sessions'),
-    path.join(os.homedir(), '.claude', 'conversations'),
-    path.join(os.homedir(), '.openclaw', 'conversations'),
-    path.join(os.homedir(), '.config', 'claude', 'sessions')
-  ];
-  
-  function parseJsonFormat(content, source) {
-    try {
-      const data = JSON.parse(content);
-      if (data.messages && Array.isArray(data.messages)) {
-        return {
-          format: 'json',
-          source,
-          messages: data.messages,
-          metadata: data.metadata || {}
-        };
-      }
-      if (data.conversation && data.conversation.messages) {
-        return {
-          format: 'json',
-          source,
-          messages: data.conversation.messages,
-          metadata: data.metadata || {}
-        };
-      }
-    } catch (e) {}
-    return null;
-  }
-  
-  function parseJsonlFormat(content, source) {
-    const lines = content.trim().split('\n');
-    const conversations = [];
-    
-    for (const line of lines) {
-      try {
-        const obj = JSON.parse(line);
-        if (obj.messages || obj.conversation) {
-          conversations.push({
-            format: 'jsonl',
-            source,
-            messages: obj.messages || obj.conversation?.messages || [],
-            metadata: obj.metadata || {}
-          });
-        }
-      } catch (e) {}
-    }
-    
-    return conversations;
-  }
-  
-  function parseMarkdownFormat(content, source) {
-    const messages = [];
-    const blocks = content.split(/(?=^##?\s+(?:User|Assistant|System|Human|AI))/m);
-    
-    let currentRole = null;
-    let currentContent = [];
-    
-    for (const block of blocks) {
-      const roleMatch = block.match(/^##?\s+(User|Assistant|System|Human|AI|Assistant \(tool\))/im);
-      if (roleMatch) {
-        if (currentRole && currentContent.length > 0) {
-          messages.push({
-            role: currentRole,
-            content: currentContent.join('\n').trim()
-          });
-        }
-        currentRole = roleMatch[1].toLowerCase().replace('assistant (tool)', 'tool');
-        currentContent = [block.replace(/^##?\s+.*$/m, '').trim()];
-      } else if (currentRole) {
-        currentContent.push(block.trim());
-      }
-    }
-    
-    if (currentRole && currentContent.length > 0) {
-      messages.push({
-        role: currentRole,
-        content: currentContent.join('\n').trim()
-      });
-    }
-    
-    if (messages.length > 0) {
-      return {
-        format: 'markdown',
-        source,
-        messages,
-        metadata: {}
-      };
-    }
-    return null;
-  }
-  
-  function processLogFile(filePath) {
-    try {
-      const content = fs.readFileSync(filePath, 'utf-8');
-      const source = path.relative(os.homedir(), filePath);
-      
-      // Try JSON format
-      if (filePath.endsWith('.json')) {
-        const parsed = parseJsonFormat(content, source);
-        if (parsed) {
-          conversations.push(parsed);
-          return;
-        }
-      }
-      
-      // Try JSONL format
-      if (filePath.endsWith('.jsonl')) {
-        const parsed = parseJsonlFormat(content, source);
-        conversations.push(...parsed);
-        return;
-      }
-      
-      // Try Markdown format
-      if (filePath.endsWith('.md') || filePath.endsWith('.mdx')) {
-        const parsed = parseMarkdownFormat(content, source);
-        if (parsed) {
-          conversations.push(parsed);
-        }
-      }
-    } catch (e) {
-      // Skip files that can't be read
-    }
-  }
-  
-  function walkDir(dir) {
-    if (!fs.existsSync(dir)) return;
-    
-    try {
-      const entries = fs.readdirSync(dir, { withFileTypes: true });
-      for (const entry of entries) {
-        const fullPath = path.join(dir, entry.name);
-        if (entry.isDirectory()) {
-          walkDir(fullPath);
-        } else if (entry.isFile() && 
-                   (entry.name.endsWith('.json') || 
-                    entry.name.endsWith('.jsonl') || 
-                    entry.name.endsWith('.md') ||
-                    entry.name.endsWith('.mdx'))) {
-          processLogFile(fullPath);
-        }
-      }
-    } catch (e) {
-      // Skip directories that can't be accessed
-    }
-  }
-  
-  // Try all known session log locations
-  for (const logPath of sessionLogPaths) {
-    walkDir(logPath);
-  }
-  
-  // Save parsed conversations
-  fs.writeFileSync(
-    path.join(CONVERSATIONS_DIR, 'parsed.json'),
-    JSON.stringify(conversations, null, 2)
-  );
-  
-  console.log(`✅ Parsed ${conversations.length} conversations`);
-  return conversations;
-}
-
-// ============================================================================
-// 5. GENERATE SYNTHETIC EXAMPLES (50+ per tool)
-// ============================================================================
-
-function generateSyntheticExamples(tools) {
-  console.log('🎭 Generating synthetic examples...');
-  const examples = [];
-  
-  // Base system prompts
-  const systemPrompts = [
-    'You are Stack, an AI coding assistant. You help with programming tasks, answer questions, and use tools when needed.',
-    'You are Stack, an expert programmer specializing in TypeScript, React, and Node.js.',
-    'You are Stack, a helpful AI assistant that can read and write files, run commands, and search code.',
-    'You are Stack, an AI coding assistant with access to file system tools, search tools, and more.'
-  ];
-
-  // =========================================================================
-  // EXAMPLES: Basic coding questions and answers
-  // =========================================================================
-  
-  const basicQAPairs = [
-    {
-      q: 'How do I read a file in Node.js?',
-      a: 'You can use the `fs` module. Here are the main approaches:\n\n1. Synchronous (simple but blocks):\n```js\nconst fs = require(\'fs\');\nconst content = fs.readFileSync(\'file.txt\', \'utf8\');\n```\n\n2. Async with callbacks:\n```js\nfs.readFile(\'file.txt\', \'utf8\', (err, data) => {\n  if (err) throw err;\n  console.log(data);\n});\n```\n\n3. Promise-based (recommended):\n```js\nconst fs = require(\'fs\').promises;\nconst data = await fs.readFile(\'file.txt\', \'utf8\');\n```'
-    },
-    {
-      q: 'How do I write to a file in Node.js?',
-      a: 'Use the `fs` module:\n\n1. Sync (blocks):\n```js\nfs.writeFileSync(\'output.txt\', \'Hello!\', \'utf8\');\n```\n\n2. Async with promises:\n```js\nawait fs.writeFile(\'output.txt\', \'Hello!\', \'utf8\');\n```\n\n3. Append to existing:\n```js\nawait fs.appendFile(\'log.txt\', \'new entry\\n\', \'utf8\');\n```'
-    },
-    {
-      q: 'How do I list files in a directory?',
-      a: 'Use `fs.readdir()`:\n\n```js\nconst files = await fs.readdir(\'./src\', { withFileTypes: true });\n\nfor (const file of files) {\n  if (file.isDirectory()) {\n    console.log(`📁 ${file.name}/`);\n  } else {\n    console.log(`📄 ${file.name}`);\n  }\n}\n```'
-    },
-    {
-      q: 'What is the difference between let, const, and var?',
-      a: 'Key differences:\n\n**const** - Block-scoped, cannot be reassigned (but object properties can change)\n**let** - Block-scoped, can be reassigned\n**var** - Function-scoped (or global), hoisted (avoid in modern JS)\n\n```js\nconst PI = 3.14;        // Cannot reassign\nlet count = 0;          // Can reassign\ncount = count + 1;      // ✓\nvar oldWay = \'hoisted\'; // Avoid - function scoped\n```'
-    },
-    {
-      q: 'How do I use async/await properly?',
-      a: 'Async/await is syntactic sugar over promises:\n\n```js\n// Basic usage\nasync function fetchData() {\n  try {\n    const response = await fetch(url);\n    const data = await response.json();\n    return data;\n  } catch (error) {\n    console.error(\'Fetch failed:\', error);\n    throw error;\n  }\n}\n\n// Parallel execution\nconst [users, posts] = await Promise.all([\n  fetch(\'/api/users\').then(r => r.json()),\n  fetch(\'/api/posts\').then(r => r.json())\n]);\n```'
-    }
-  ];
-
-  for (const pair of basicQAPairs) {
-    examples.push({
-      messages: [
-        { role: 'system', content: systemPrompts[0] },
-        { role: 'user', content: pair.q },
-        { role: 'assistant', content: pair.a }
-      ]
-    });
-  }
-
-  // =========================================================================
-  // EXAMPLES: Tool use patterns (per tool)
-  // =========================================================================
-  
-  const toolUsePatterns = [
-    {
-      tool: 'FileReadTool',
-      scenarios: [
-        {
-          user: 'Show me the contents of package.json',
-          assistant: 'I\'ll read the package.json file for you.',
-          toolName: 'FileReadTool',
-          toolInput: { file_path: 'package.json' },
-          resultContent: '{\n  "name": "my-project",\n  "version": "1.0.0",\n  "scripts": {\n    "dev": "next dev",\n    "build": "next build"\n  }\n}',
-          summary: 'Here\'s what\'s in your package.json:\n\n- **Project name**: my-project\n- **Version**: 1.0.0\n- **Scripts**: dev (next dev), build (next build)'
-        },
-        {
-          user: 'Read the config.ts file from the src directory',
-          assistant: 'Let me read that config file.',
-          toolName: 'FileReadTool',
-          toolInput: { file_path: 'src/config.ts' },
-          resultContent: 'export const config = {\n  apiUrl: \'https://api.example.com\',\n  timeout: 5000,\n  retries: 3\n};',
-          summary: 'The config file exports a configuration object with:\n\n- **API URL**: https://api.example.com\n- **Timeout**: 5000ms\n- **Retries**: 3'
-        },
-        {
-          user: 'What\'s in the README.md?',
-          assistant: 'Reading the README...',
-          toolName: 'FileReadTool',
-          toolInput: { file_path: 'README.md' },
-          resultContent: '# My Project\n\nA brief description of the project.\n\n## Installation\n\nnpm install\n\n## Usage\n\nnpm start',
-          summary: 'The README contains:\n\n- Project title and description\n- Installation instructions\n- Usage guide'
-        }
-      ]
-    },
-    {
-      tool: 'GlobTool',
-      scenarios: [
-        {
-          user: 'Find all TypeScript files in src',
-          assistant: 'Searching for TypeScript files...',
-          toolName: 'GlobTool',
-          toolInput: { pattern: 'src/**/*.ts' },
-          resultContent: 'src/index.ts\nsrc/utils.ts\nsrc/components/Button.tsx\nsrc/services/api.ts',
-          summary: 'Found 4 TypeScript files in src/'
-        },
-        {
-          user: 'List all test files',
-          assistant: 'Looking for test files...',
-          toolName: 'GlobTool',
-          toolInput: { pattern: '**/*.test.{ts,tsx}' },
-          resultContent: '__tests__/sample.test.ts\nsrc/components/__tests__/Button.test.tsx\nsrc/utils/__tests__/helpers.test.ts',
-          summary: 'Found 3 test files'
-        },
-        {
-          user: 'Find all JSON config files',
-          assistant: 'Searching for JSON config files...',
-          toolName: 'GlobTool',
-          toolInput: { pattern: '**/*.json' },
-          resultContent: 'package.json\ntsconfig.json\njest.config.json\nsrc/config.json',
-          summary: 'Found 4 JSON files'
-        }
-      ]
-    },
-    {
-      tool: 'GrepTool',
-      scenarios: [
-        {
-          user: 'Find all uses of "TODO" in the code',
-          assistant: 'Searching for TODO comments...',
-          toolName: 'GrepTool',
-          toolInput: { pattern: 'TODO', path: '.' },
-          resultContent: 'src/utils.ts: // TODO: implement error handling\nsrc/components/Form.tsx: // TODO: add validation\nsrc/api/client.ts: // TODO: add retry logic',
-          summary: 'Found 3 TODO comments in:\n- src/utils.ts\n- src/components/Form.tsx\n- src/api/client.ts'
-        },
-        {
-          user: 'Search for "async function" in the src',
-          assistant: 'Finding async functions...',
-          toolName: 'GrepTool',
-          toolInput: { pattern: 'async function', path: 'src' },
-          resultContent: 'src/api/client.ts: async function fetchData() {\nsrc/utils/parser.ts: async function parseJSON() {',
-          summary: 'Found 2 async functions'
-        }
-      ]
-    },
-    {
-      tool: 'BashTool',
-      scenarios: [
-        {
-          user: 'Run npm install',
-          assistant: 'Installing dependencies...',
-          toolName: 'BashTool',
-          toolInput: { command: 'npm install' },
-          resultContent: 'added 245 packages in 12s\n\n12 packages are looking for funding\n  run `npm fund` for details',
-          summary: 'Successfully installed 245 packages'
-        },
-        {
-          user: 'Check git status',
-          assistant: 'Checking git status...',
-          toolName: 'BashTool',
-          toolInput: { command: 'git status' },
-          resultContent: 'On branch main\nChanges not staged for commit:\n  modified:   src/index.ts\n  modified:   package.json',
-          summary: 'On branch main with 2 modified files'
-        },
-        {
-          user: 'Run the tests',
-          assistant: 'Running tests...',
-          toolName: 'BashTool',
-          toolInput: { command: 'npm test' },
-          resultContent: 'PASS  __tests__/sample.test.ts\nPASS  src/components/__tests__/Button.test.tsx\n\nTest Suites: 2 passed, 2 total\nTests:       5 passed, 5 total',
-          summary: 'All tests passed! 2 test suites, 5 tests total'
-        }
-      ]
-    },
-    {
-      tool: 'FileWriteTool',
-      scenarios: [
-        {
-          user: 'Create a new config file',
-          assistant: 'Creating the config file...',
-          toolName: 'FileWriteTool',
-          toolInput: { file_path: 'config/local.json', content: '{\n  "name": "local-config",\n  "debug": true\n}' },
-          resultContent: '',
-          summary: 'Created config/local.json'
-        },
-        {
-          user: 'Write a simple test file',
-          assistant: 'Writing the test file...',
-          toolName: 'FileWriteTool',
-          toolInput: { file_path: 'test/example.test.ts', content: 'describe(\'Example\', () => {\n  it(\'should pass\', () => {\n    expect(1 + 1).toBe(2);\n  });\n});' },
-          resultContent: '',
-          summary: 'Created test/example.test.ts'
-        }
-      ]
-    },
-    {
-      tool: 'FileEditTool',
-      scenarios: [
-        {
-          user: 'Add a new export to the index file',
-          assistant: 'Editing the index file...',
-          toolName: 'FileEditTool',
-          toolInput: { 
-            file_path: 'src/index.ts',
-            old_string: 'export { Config } from \'./config\';',
-            new_string: 'export { Config } from \'./config\';\nexport { Logger } from \'./logger\';'
-          },
-          resultContent: '',
-          summary: 'Added Logger export to src/index.ts'
-        }
-      ]
-    },
-    {
-      tool: 'WebSearchTool',
-      scenarios: [
-        {
-          user: 'Search for the latest React documentation',
-          assistant: 'Searching the web...',
-          toolName: 'WebSearchTool',
-          toolInput: { query: 'React 18 documentation' },
-          resultContent: 'Found results for React documentation...',
-          summary: 'Found relevant documentation resources'
-        }
-      ]
-    },
-    {
-      tool: 'WebFetchTool',
-      scenarios: [
-        {
-          user: 'Fetch the content of a GitHub README',
-          assistant: 'Fetching the README...',
-          toolName: 'WebFetchTool',
-          toolInput: { url: 'https://github.com/facebook/react' },
-          resultContent: 'README content...',
-          summary: 'Successfully fetched the README'
-        }
-      ]
-    }
-  ];
-
-  // Generate tool use examples
-  for (const toolPattern of toolUsePatterns) {
-    for (const scenario of toolPattern.scenarios) {
-      examples.push({
-        messages: [
-          { role: 'system', content: systemPrompts[0] },
-          { role: 'user', content: scenario.user },
-          { role: 'assistant', content: scenario.assistant, tool_use: { name: scenario.toolName, input: scenario.toolInput } },
-          { role: 'user', content: '', tool_result: { tool_use_id: 'tool_1', content: scenario.resultContent } },
-          { role: 'assistant', content: scenario.summary }
-        ]
-      });
-    }
-  }
-
-  // =========================================================================
-  // EXAMPLES: Error handling patterns
-  // =========================================================================
-  
-  const errorHandlingExamples = [
-    {
-      user: 'Why did my file write fail?',
-      context: 'Error: ENOENT: no such file or directory, open \'missing/dir/file.txt\'',
-      response: 'The error indicates the directory doesn\'t exist. You need to create the parent directory first:\n\n```js\nimport fs from \'fs\'.promises;\nimport path from \'path\';\n\nasync function ensureDir(dirPath) {\n  await fs.mkdir(path.dirname(dirPath), { recursive: true });\n}\n\n// Then write\nawait ensureDir(\'missing/dir/file.txt\');\nawait fs.writeFile(\'missing/dir/file.txt\', \'content\');\n```\n\nOr use `fs.mkdirSync(path.dirname(filePath), { recursive: true })` synchronously.'
-    },
-    {
-      user: 'My async function is throwing an error',
-      context: 'Error: Cannot read property \'map\' of undefined',
-      response: 'This usually means the data you\'re trying to map over isn\'t defined yet. Here\'s how to handle it:\n\n```js\nasync function getItems() {\n  try {\n    const data = await fetchData();\n    // Check before using\n    return data?.items?.map(item => item.id) || [];\n  } catch (error) {\n    console.error(\'Failed to fetch:\', error);\n    return []; // Return fallback\n  }\n}\n```\n\nUse optional chaining (`?.`) to safely access nested properties.'
-    }
-  ];
-
-  for (const error of errorHandlingExamples) {
-    examples.push({
-      messages: [
-        { role: 'system', content: systemPrompts[0] },
-        { role: 'user', content: error.user + '\n\nError context: ' + error.context },
-        { role: 'assistant', content: error.response }
-      ]
-    });
-  }
-
-  // =========================================================================
-  // EXAMPLES: Multi-step workflows
-  // =========================================================================
-  
-  const multiStepWorkflows = [
-    {
-      name: 'Create and test a new component',
-      steps: [
-        { role: 'user', content: 'Create a new React component called UserCard' },
-        { role: 'assistant', content: 'I\'ll create the UserCard component for you.', tool_use: { name: 'FileWriteTool', input: { file_path: 'src/components/UserCard.tsx', content: 'import React from \'react\';\n\ninterface UserCardProps {\n  name: string;\n  email: string;\n}\n\nexport const UserCard: React.FC<UserCardProps> = ({ name, email }) => {\n  return (\n    <div className="user-card">\n      <h3>{name}</h3>\n      <p>{email}</p>\n    </div>\n  );\n};' } } },
-        { role: 'user', content: '', tool_result: { tool_use_id: 'tool_1', content: 'File created successfully' } },
-        { role: 'assistant', content: 'Created the component. Now let me write a test for it.', tool_use: { name: 'FileWriteTool', input: { file_path: 'src/components/__tests__/UserCard.test.tsx', content: 'import React from \'react\';\nimport { render, screen } from \'@testing-library/react\';\nimport { UserCard } from \'../UserCard\';\n\ndescribe(\'UserCard\', () => {\n  it(\'renders user name and email\', () => {\n    render(<UserCard name="John" email="john@example.com" />);\n    expect(screen.getByText(\'John\')).toBeInTheDocument();\n    expect(screen.getByText(\'john@example.com\')).toBeInTheDocument();\n  });\n});' } } },
-        { role: 'user', content: '', tool_result: { tool_use_id: 'tool_2', content: 'Test file created' } },
-        { role: 'assistant', content: 'Now let me run the tests to verify it works.', tool_use: { name: 'BashTool', input: { command: 'npm test -- --testPathPattern=UserCard' } } },
-        { role: 'user', content: '', tool_result: { tool_use_id: 'tool_3', content: 'PASS src/components/__tests__/UserCard.test.tsx\n\n  UserCard\n    ✓ renders user name and email' } },
-        { role: 'assistant', content: 'All tests pass! The UserCard component is ready to use.\n\n```tsx\nimport { UserCard } from \'./components/UserCard\';\n\n<UserCard name="John" email="john@example.com" />\n```' }
-      ]
-    },
-    {
-      name: 'Debug and fix a failing test',
-      steps: [
-        { role: 'user', content: 'Run the tests - I think something is broken' },
-        { role: 'assistant', content: 'Running tests...', tool_use: { name: 'BashTool', input: { command: 'npm test 2>&1' } } },
-        { role: 'user', content: '', tool_result: { tool_use_id: 'tool_1', content: 'FAIL src/utils/__tests__/format.test.ts\n\n  Format Date\n    ✕ should format date as YYYY-MM-DD (5ms)\n\n    expect(received).toBe(expected)\n\n    Expected: "2024-01-15"\n    Received: "15-01-2024"' } },
-        { role: 'assistant', content: 'I see the test is failing. Let me look at the formatDate function to understand the issue.', tool_use: { name: 'GrepTool', input: { pattern: 'formatDate', path: 'src/utils' } } },
-        { role: 'user', content: '', tool_result: { tool_use_id: 'tool_2', content: 'src/utils/format.ts:export function formatDate(date: Date): string {\n  const d = new Date(date);\n  return `${d.getDate()}-${d.getMonth() + 1}-${d.getFullYear()}`;\n}' } },
-        { role: 'assistant', content: 'Found it! The function returns DD-MM-YYYY but the test expects YYYY-MM-DD. Let me fix it.', tool_use: { name: 'FileEditTool', input: { file_path: 'src/utils/format.ts', old_string: 'return `${d.getDate()}-${d.getMonth() + 1}-${d.getFullYear()}`;', new_string: 'return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, \'0\')}-${String(d.getDate()).padStart(2, \'0\')}`;' } } },
-        { role: 'user', content: '', tool_result: { tool_use_id: 'tool_3', content: 'File updated' } },
-        { role: 'assistant', content: 'Fixed the format. Let me run the tests again to confirm.', tool_use: { name: 'BashTool', input: { command: 'npm test -- --testPathPattern=format.test' } } },
-        { role: 'user', content: '', tool_result: { tool_use_id: 'tool_4', content: 'PASS src/utils/__tests__/format.test.ts\n\n  Format Date\n    ✓ should format date as YYYY-MM-DD' } },
-        { role: 'assistant', content: 'All tests pass now! The date is correctly formatted as YYYY-MM-DD.' }
-      ]
-    }
-  ];
-
-  for (const workflow of multiStepWorkflows) {
-    examples.push({
-      messages: [
-        { role: 'system', content: systemPrompts[2] },
-        ...workflow.steps
-      ]
-    });
-  }
-
-  // =========================================================================
-  // EXAMPLES: Edge cases
-  // =========================================================================
-  
-  const edgeCases = [
-    {
-      user: 'Handle an empty array in my reduce function',
-      context: 'My reduce throws when the array is empty',
-      response: 'Here are ways to handle empty arrays with reduce:\n\n```js\n// 1. Provide initial value\nconst sum = [].reduce((acc, val) => acc + val, 0); // Returns 0\n\n// 2. Check first\nif (arr.length === 0) return defaultValue;\n\n// 3. Use reduce with optional initial value\nfunction safeReduce(arr, reducer, initial) {\n  if (arr.length === 0 && initial === undefined) {\n    throw new Error(\'Cannot reduce empty array without initial value\');\n  }\n  return arr.reduce(reducer, initial);\n}\n```'
-    },
-    {
-      user: 'Deal with optional function parameters',
-      context: 'How do I make some parameters optional?',
-      response: 'You can use optional parameters in several ways:\n\n```ts\n// TypeScript optional parameter\nfunction greet(name: string, greeting?: string) {\n  return `${greeting || \'Hello\'}, ${name}!`;\n}\n\n// Default parameter\nfunction createUser(name: string, role = \'user\') {\n  return { name, role };\n}\n\n// Nullish coalescing for defaults\nfunction process(value: string | null) {\n  const input = value ?? \'default\';\n}\n```'
-    },
-    {
-      user: 'Handle API rate limiting',
-      context: 'Getting 429 errors from the API',
-      response: 'Here\'s a pattern for handling rate limits:\n\n```js\nasync function fetchWithRetry(url, retries = 3) {\n  for (let i = 0; i < retries; i++) {\n    const response = await fetch(url);\n    \n    if (response.status === 429) {\n      const retryAfter = response.headers.get(\'Retry-After\') || 60;\n      console.log(`Rate limited. Waiting ${retryAfter}s...`);\n      await new Promise(r => setTimeout(r, retryAfter * 1000));\n      continue;\n    }\n    \n    return response.json();\n  }\n  throw new Error(\'Max retries exceeded\');\n}\n```\n\nUse exponential backoff for more aggressive retrying.'
-    }
-  ];
-
-  for (const edge of edgeCases) {
-    examples.push({
-      messages: [
-        { role: 'system', content: systemPrompts[1] },
-        { role: 'user', content: edge.user + '\n\nContext: ' + edge.context },
-        { role: 'assistant', content: edge.response }
-      ]
-    });
-  }
-
-  // =========================================================================
-  // GENERATE 50+ EXAMPLES PER TOOL (tool-specific variations)
-  // =========================================================================
-  
-  const toolNames = tools.map(t => t.tool);
-  const variationsPerTool = {
-    FileReadTool: [
-      'Read the first 100 lines of a large log file',
-      'Show me the contents of .env.example',
-      'What\'s in the tsconfig.json?',
-      'Read the package-lock.json to check versions',
-      'Show me the gitignore file'
-    ],
-    FileWriteTool: [
-      'Create a .gitignore file with common ignores',
-      'Write a new entry to the changelog',
-      'Create a simple JSON config file',
-      'Write the test results to output.txt'
-    ],
-    GlobTool: [
-      'Find all .test.ts files',
-      'List all files in src/ directory',
-      'Find all files with "helper" in the name',
-      'Search for *.config.js files',
-      'Find all files in any __tests__ directory'
-    ],
-    GrepTool: [
-      'Find all console.log statements',
-      'Search for "export default"',
-      'Find all imports from "react"',
-      'Search for password or secret patterns',
-      'Find all unused imports'
-    ],
-    BashTool: [
-      'Initialize a new git repository',
-      'Show the last 10 commits',
-      'List all npm scripts available',
-      'Check the current directory',
-      'Show the difference between branches'
-    ]
-  };
-
-  // Generate 50+ examples by varying prompts for each tool
-  let exampleCount = examples.length;
-  
-  for (const tool of tools) {
-    const variations = variationsPerTool[tool.tool] || [];
-    
-    for (let i = 0; i < 5; i++) {
-      const variation = variations[i % variations.length];
-      const idx = i % variations.length;
-      
-      examples.push({
-        messages: [
-          { role: 'system', content: systemPrompts[i % systemPrompts.length] },
-          { role: 'user', content: `${variation || 'process'} (variant ${i + 1})` },
-          { role: 'assistant', content: `I'll help you with that using ${tool.tool}. This is a variant example showing different ways to phrase the same intent.`, tool_use: { name: tool.tool, input: generateMockInput(tool.tool, i) } },
-          { role: 'user', content: '', tool_result: { tool_use_id: `tool_${i}`, content: getMockResult(tool.tool, i) } },
-          { role: 'assistant', content: `Done! Here's the result for variant ${i + 1} of ${(variation || 'task').toLowerCase()}.` }
-        ]
-      });
-    }
-  }
-
-  // Write examples to JSONL
-  const outputPath = path.join(SYNTHETIC_DIR, 'examples.jsonl');
-  const stream = fs.createWriteStream(outputPath);
-  for (const ex of examples) {
-    stream.write(JSON.stringify(ex) + '\n');
-  }
-  stream.end();
-
-  console.log(`✅ Generated ${examples.length} synthetic examples`);
-  return examples;
-}
-
-function generateMockInput(toolName, variant) {
-  const inputs = {
-    FileReadTool: [{ file_path: `example-${variant}.txt` }, { file_path: 'src/index.ts' }, { file_path: 'config.json' }],
-    GlobTool: [{ pattern: `**/*.${variant === 0 ? 'ts' : 'js'}` }, { pattern: 'src/**/*.tsx' }],
-    GrepTool: [{ pattern: 'TODO', path: 'src' }],
-    BashTool: [{ command: 'ls -la' }, { command: 'git status' }],
-    FileWriteTool: [{ file_path: 'output.txt', content: 'test' }]
-  };
-  return inputs[toolName]?.[variant % (inputs[toolName]?.length || 1)] || { query: `variant-${variant}` };
-}
-
-function getMockResult(toolName, variant) {
-  const results = {
-    FileReadTool: 'File contents here...',
-    GlobTool: `file1.${variant === 0 ? 'ts' : 'js'}\nfile2.${variant === 0 ? 'ts' : 'js'}`,
-    GrepTool: 'Found 3 matches',
-    BashTool: 'Command output here',
-    FileWriteTool: ''
-  };
-  return results[toolName] || 'Done';
-}
-
-// ============================================================================
-// 6. CREATE TRAINING MANIFEST
-// ============================================================================
-
-function createManifest(tools, stats) {
-  const manifest = {
-    dataset: {
-      name: 'Stack 2.9 Training Data',
-      version: '0.2.0',
-      description: 'Training data for Stack 2.9, an open-source coding assistant based on Qwen2.5-Coder',
-      source: 'OpenClaw architecture + synthetic examples + code analysis',
-      license: 'Apache 2.0'
-    },
-    stats: {
-      toolSchemas: tools.length,
-      syntheticExamples: stats.syntheticExamples,
-      codeCommentPairs: stats.codeCommentPairs,
-      testExamples: stats.testExamples,
-      conversations: stats.conversations,
-      totalExamples: stats.syntheticExamples
-    },
-    model_config: {
-      base_model: 'Qwen2.5-Coder-32B',
-      fine_tuning_method: 'LoRA',
-      lora_rank: 64,
-      lora_alpha: 128,
-      target_modules: [
-        'q_proj', 'k_proj', 'v_proj', 'o_proj',
-        'gate_proj', 'up_proj', 'down_proj'
-      ],
-      quantization: 'AWQ 4-bit (inference)',
-      max_seq_length: 131072,
-      template: 'chatml'
-    },
-    tokenizer: {
-      family: 'Qwen2',
-      pad_token: '<|endoftext|>',
-      bos_token: '<|endoftext|>',
-      eos_token: '<|endoftext|>'
-    },
-    training_data: {
-      synthetic_examples: `${SYNTHETIC_DIR}/examples.jsonl`,
-      tools_catalog: `${TOOLS_SCHEMA_DIR}/catalog.json`,
-      code_pairs: `${CODE_PAIRS_DIR}/pairs.json`,
-      test_examples: `${CODE_PAIRS_DIR}/test-examples.json`,
-      conversations: `${CONVERSATIONS_DIR}/parsed.json`,
-      estimated_tokens: '~50M tokens total',
-      recommended_dataset_size: '100K - 1M examples'
-    },
-    deployment: {
-      inference_engine: 'vLLM',
-      api_compatibility: 'OpenAI-compatible (chat/completions)',
-      expected_throughput: '~50 tokens/s on A100 80GB',
-      platforms: ['Hugging Face', 'OpenRouter', 'self-hosted']
-    }
-  };
-
-  fs.writeFileSync(
-    path.join(OUTPUT_DIR, 'manifest.json'),
-    JSON.stringify(manifest, null, 2)
-  );
-
-  console.log('✅ Created training manifest');
-  return manifest;
-}
-
-// ============================================================================
-// 7. CREATE TRAINING CONFIG
-// ============================================================================
-
-function createTrainingConfig() {
-  const config = {
-    model_name: 'Qwen/Qwen2.5-Coder-32B',
-    dataset_path: './training-data/synthetic/examples.jsonl',
-    max_seq_length: 131072,
-    load_in_4bit: true,
-    bf16: true,
-    batch_size: 1,
-    gradient_accumulation_steps: 16,
-    learning_rate: 1e-4,
-    num_train_epochs: 3,
-    warmup_steps: 100,
-    save_steps: 1000,
-    eval_steps: 500,
-    logging_steps: 10,
-    output_dir: './stack-2.9-lora',
-    push_to_hub: false,
-    hub_model_id: 'your-username/stack-2.9',
-    lora_config: {
-      r: 64,
-      lora_alpha: 128,
-      target_modules: ['q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj'],
-      lora_dropout: 0.05,
-      bias: 'none'
-    }
-  };
-
-  fs.writeFileSync(
-    path.join(OUTPUT_DIR, 'training-config.json'),
-    JSON.stringify(config, null, 2)
-  );
-
-  console.log('✅ Created training config template');
-  return config;
-}
-
-// ============================================================================
-// MAIN
-// ============================================================================
-
-console.log('🔧 Stack 2.9 - Enhanced Training Data Extractor\n');
-console.log(`📂 Source: ${SRC_DIR}`);
-console.log(`📁 Output: ${OUTPUT_DIR}\n`);
-
-// Run extraction pipeline
-const tools = extractToolSchemas();
-const codePairs = extractCodeCommentPairs();
-const testExamples = extractTestExamples();
-const conversations = parseConversations();
-const syntheticExamples = generateSyntheticExamples(tools);
-createManifest(tools, {
-  syntheticExamples: syntheticExamples.length,
-  codeCommentPairs: codePairs.length,
-  testExamples: testExamples.length,
-  conversations: conversations.length
-});
-createTrainingConfig();
-
-console.log('\n✨ Extraction complete!');
-console.log('\n📋 Summary:');
-console.log(`   - Tool schemas: ${tools.length} tools`);
-console.log(`   - Synthetic examples: ${syntheticExamples.length}`);
-console.log(`   - Code-comment pairs: ${codePairs.length}`);
-console.log(`   - Test examples: ${testExamples.length}`);
-console.log(`   - Conversations: ${conversations.length}`);
-console.log('\n📁 Output files:');
-console.log(`   - ${TOOLS_SCHEMA_DIR}/catalog.json`);
-console.log(`   - ${SYNTHETIC_DIR}/examples.jsonl`);
-console.log(`   - ${CODE_PAIRS_DIR}/pairs.json`);
-console.log(`   - ${CODE_PAIRS_DIR}/test-examples.json`);
-console.log(`   - ${CONVERSATIONS_DIR}/parsed.json`);
-console.log(`   - ${OUTPUT_DIR}/manifest.json`);
-console.log(`   - ${OUTPUT_DIR}/training-config.json`);
-console.log('\n🚀 Next steps:');
-console.log('   1. Review extracted code-comment pairs for quality');
-console.log('   2. Add real conversation logs from ~/.claude/sessions');
-console.log('   3. Scale: aim for 50+ examples per tool');
-console.log('   4. Convert to Parquet for faster loading');
-console.log('   5. Launch LoRA fine-tuning on Qwen2.5-Coder-32B');
-console.log('   6. Deploy with vLLM and submit to OpenRouter');

scripts/update_context_window.py

@@ -1,190 +0,0 @@
-#!/usr/bin/env python3
-"""
-Update all configuration files to use 128K context window.
-Updates: manifest, training config, prepare_dataset, vLLM server, deploy scripts, docs.
-"""
-
-import json
-import re
-from pathlib import Path
-import argparse
-
-def update_json_file(path: Path, updates: dict):
-    """Update JSON file with key->value updates."""
-    if not path.exists():
-        print(f"   ⚠️  Not found: {path}")
-        return False
-
-    with open(path, 'r') as f:
-        data = json.load(f)
-
-    changed = False
-    for key, value in updates.items():
-        if key in data and data[key] != value:
-            data[key] = value
-            changed = True
-
-    if changed:
-        with open(path, 'w') as f:
-            json.dump(data, f, indent=2)
-        print(f"   ✅ Updated {path.name}")
-    else:
-        print(f"   ℹ️  {path.name} already up-to-date")
-    return changed
-
-def update_python_file(path: Path, old_pattern: str, new_value: str):
-    """Replace a constant in a Python file."""
-    if not path.exists():
-        print(f"   ⚠️  Not found: {path}")
-        return False
-
-    content = path.read_text()
-    if old_pattern in content:
-        new_content = content.replace(old_pattern, new_value)
-        path.write_text(new_content)
-        print(f"   ✅ Updated {path.name}")
-        return True
-    else:
-        print(f"   ℹ️  {path.name} - pattern not found, may be already updated")
-        return False
-
-def update_shell_script(path: Path, old_var: str, new_value: str):
-    """Update shell script variable."""
-    if not path.exists():
-        print(f"   ⚠️  Not found: {path}")
-        return False
-
-    content = path.read_text()
-    if old_var in content:
-        new_content = re.sub(
-            rf'{old_var}=.+',
-            f'{old_var}={new_value}',
-            content
-        )
-        path.write_text(new_content)
-        print(f"   ✅ Updated {path.name}")
-        return True
-    else:
-        print(f"   ℹ️  {path.name} - variable not found")
-        return False
-
-def update_markdown_file(path: Path, old_text: str, new_text: str):
-    """Update markdown documentation."""
-    if not path.exists():
-        print(f"   ⚠️  Not found: {path}")
-        return False
-
-    content = path.read_text()
-    if old_text in content:
-        new_content = content.replace(old_text, new_text)
-        path.write_text(new_content)
-        print(f"   ✅ Updated {path.name}")
-        return True
-    else:
-        print(f"   ℹ️  {path.name} - pattern not found")
-        return False
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--workspace", type=str, default=".")
-    args = parser.parse_args()
-
-    root = Path(args.workspace)
-
-    print("🚀 Updating context window to 128K (131072 tokens)")
-
-    # 1. Training manifest
-    manifest_path = root / "training-data/manifest.json"
-    update_json_file(manifest_path, {
-        "max_seq_length": 131072,
-        "context_length": 131072
-    })
-
-    # 2. Training config
-    training_config_path = root / "training-data/training-config.json"
-    update_json_file(training_config_path, {
-        "max_seq_length": 131072
-    })
-
-    # 3. Python scripts
-    prepare_script = root / "stack-2.9-training/prepare_dataset.py"
-    if prepare_script.exists():
-        content = prepare_script.read_text()
-        if "max_length=32768" in content:
-            new_content = content.replace("max_length=32768", "max_length=131072")
-            prepare_script.write_text(new_content)
-            print(f"   ✅ Updated prepare_dataset.py (max_length)")
-        else:
-            print(f"   ℹ️  prepare_dataset.py - already 128K or pattern not found")
-
-    # 4. vLLM server
-    vllm_script = root / "stack-2.9-deploy/vllm_server.py"
-    if vllm_script.exists():
-        content = vllm_script.read_text()
-        if "max_model_len" in content:
-            # Update max_model_len parameter
-            new_content = re.sub(
-                r'--max-model-len\s+\d+',
-                '--max-model-len 131072',
-                content
-            )
-            vllm_script.write_text(new_content)
-            print(f"   ✅ Updated vllm_server.py (--max-model-len)")
-        else:
-            print(f"   ℹ️  vllm_server.py - max_model_len not found directly, check manually")
-
-    # 5. Local deploy script
-    deploy_script = root / "stack-2.9-deploy/local_deploy.sh"
-    if deploy_script.exists():
-        content = deploy_script.read_text()
-        # Update any context-related env var
-        new_content = content.replace("MAX_MODEL_LEN=32768", "MAX_MODEL_LEN=131072") \
-                           .replace("max_model_len=32768", "max_model_len=131072")
-        if new_content != content:
-            deploy_script.write_text(new_content)
-            print(f"   ✅ Updated local_deploy.sh")
-        else:
-            print(f"   ℹ️  local_deploy.sh - no changes needed")
-
-    # 6. README.md performance table
-    readme_path = root / "README.md"
-    if readme_path.exists():
-        content = readme_path.read_text()
-        # Update context length from 32K to 128K
-        new_content = content.replace("32,768 tokens", "131,072 tokens (128K)") \
-                           .replace("32K tokens", "128K tokens")
-        if new_content != content:
-            readme_path.write_text(new_content)
-            print(f"   ✅ Updated README.md (context length)")
-        else:
-            print(f"   ℹ️  README.md - context length already correct")
-
-    # 7. Create configuration note
-    config_note = """# Context Window Configuration
-
-Stack 2.9 uses full 128K context window (131072 tokens) to provide complete repository awareness.
-
-## Settings
-- max_model_len: 131072
-- max_seq_length: 131072
-- block_size: 16 or 32 (adjust for memory/performance tradeoff)
-
-## Memory Requirements
-| Context | A100 80GB (4-bit) | H100 80GB (4-bit) |
-|---------|-------------------|-------------------|
-| 32K     | ~20GB             | ~18GB             |
-| 64K     | ~35GB             | ~32GB             |
-| 128K    | ~60GB             | ~55GB             |
-
-Throughput decreases slightly at longer contexts (~30% slower at 128K vs 32K) but provides full repository context.
-
-"""
-    note_path = root / "stack-2.9-docs/CONTEXT_CONFIG.md"
-    note_path.write_text(config_note)
-    print(f"   ✅ Created CONTEXT_CONFIG.md")
-
-    print("\n✅ Context window update complete!")
-    print("   All configs now set to 128K (131072 tokens)")
-
-if __name__ == "__main__":
-    main()