llm-debugger

LLM 调试器

系统地诊断和修复 LLM 输出问题。

故障分类

class FailureType(Enum):
    HALLUCINATION = "hallucination"
    FORMAT_VIOLATION = "format_violation"
    CONSTRAINT_BREAK = "constraint_break"
    REASONING_ERROR = "reasoning_error"
    TOOL_MISUSE = "tool_misuse"
    REFUSAL = "unexpected_refusal"
    INCOMPLETE = "incomplete_output"

根因分析

def diagnose_failure(input: str, output: str, expected: dict) -> dict:
    """Identify why LLM output failed"""

    issues = []

    # Check format
    if expected.get("format") == "json":
        try:
            json.loads(output)
        except:
            issues.append({
                "type": FailureType.FORMAT_VIOLATION,
                "details": "Invalid JSON output"
            })

    # Check required fields
    if expected.get("required_fields"):
        for field in expected["required_fields"]:
            if field not in output:
                issues.append({
                    "type": FailureType.INCOMPLETE,
                    "details": f"Missing required field: {field}"
                })

    # Check constraints
    if expected.get("max_length"):
        if len(output) > expected["max_length"]:
            issues.append({
                "type": FailureType.CONSTRAINT_BREAK,
                "details": f"Output too long: {len(output)} > {expected['max_length']}"
            })

    # Check for hallucination indicators
    if contains_hallucination_markers(output):
        issues.append({
            "type": FailureType.HALLUCINATION,
            "details": "Contains fabricated information"
        })

    return {
        "has_issues": len(issues) > 0,
        "issues": issues,
        "primary_issue": issues[0] if issues else None
    }

def contains_hallucination_markers(output: str) -> bool:
    """Detect common hallucination patterns"""
    markers = [
        r'According to.*that doesn\'t exist',
        r'In \d{4}.*before that year',
        r'contradicts itself',
    ]
    return any(re.search(marker, output, re.IGNORECASE) for marker in markers)

提示词修复

PROMPT_FIXES = {
    FailureType.FORMAT_VIOLATION: """
Add strict format instructions:
"Output MUST be valid JSON with this exact structure:
```json
{{"field1": "value", "field2": 123}}

请勿在 JSON 前后添加任何文本。” """,

FailureType.HALLUCINATION: """

添加依据上下文的指令： “仅根据提供的上下文作答。 如果上下文中没有相关信息，请说‘我没有这方面的信息。’ 切勿编造事实或细节。” """,

FailureType.CONSTRAINT_BREAK: """

添加明确的约束： “你的回答必须满足以下要求：

• 最多 200 个词
• 不提供代码示例
• 仅使用专业语气” """, FailureType.REASONING_ERROR: """ 添加逐步推理： “请逐步思考：

1. 首先，确定……
2. 然后，考虑……
3. 最后，得出结论……” """, }

def suggest_prompt_fix(diagnosis: dict, current_prompt: str) -> str: """根据诊断结果生成改进后的提示词""" primary_issue = diagnosis["primary_issue"] if not primary_issue: return current_prompt

fix = PROMPT_FIXES[primary_issue["type"]]

return f"""{current_prompt}

重要指令： {fix} """

## Tool Improvements

```python
def suggest_tool_fixes(diagnosis: dict, tool_schema: dict) -> dict:
    """Improve tool schema to prevent misuse"""

    fixes = {}

    for issue in diagnosis["issues"]:
        if issue["type"] == FailureType.TOOL_MISUSE:
            # Make required parameters more explicit
            fixes["parameters"] = {
                **tool_schema["parameters"],
                "description": "REQUIRED: " + tool_schema["parameters"].get("description", "")
            }

            # Add examples
            fixes["examples"] = [
                {
                    "params": {"query": "example search"},
                    "description": "Use for finding information"
                }
            ]

    return {**tool_schema, **fixes}

测试用例生成

def generate_test_cases(diagnosis: dict, failed_input: str, failed_output: str) -> List[dict]:
    """Create regression test cases from failures"""

    test_cases = []

    # Test case for the specific failure
    test_cases.append({
        "input": failed_input,
        "expected_behavior": "Should not " + diagnosis["primary_issue"]["details"],
        "validation": lambda output: not has_same_issue(output, diagnosis),
    })

    # Edge cases based on failure type
    if diagnosis["primary_issue"]["type"] == FailureType.FORMAT_VIOLATION:
        test_cases.append({
            "input": failed_input,
            "validation": lambda output: is_valid_json(output),
            "description": "Output must be valid JSON"
        })

    # Similar inputs that might trigger same issue
    similar_inputs = generate_similar_inputs(failed_input)
    for inp in similar_inputs:
        test_cases.append({
            "input": inp,
            "validation": lambda output: not has_same_issue(output, diagnosis),
        })

    return test_cases

调试工作流

def debug_llm_output(
    prompt: str,
    output: str,
    expected: dict,
    context: dict = {}
) -> dict:
    """Complete debugging workflow"""

    # 1. Diagnose issue
    diagnosis = diagnose_failure(prompt, output, expected)

    if not diagnosis["has_issues"]:
        return {"status": "ok", "diagnosis": diagnosis}

    # 2. Generate fixes
    fixed_prompt = suggest_prompt_fix(diagnosis, prompt)

    # 3. Create test cases
    test_cases = generate_test_cases(diagnosis, prompt, output)

    # 4. Test fix
    test_output = llm(fixed_prompt)
    fix_works = diagnose_failure(fixed_prompt, test_output, expected)["has_issues"] == False

    return {
        "status": "issues_found",
        "diagnosis": diagnosis,
        "fixes": {
            "prompt": fixed_prompt,
            "fix_verified": fix_works,
        },
        "test_cases": test_cases,
        "recommendations": generate_recommendations(diagnosis, context)
    }

交互式调试

def interactive_debug():
    """Interactive debugging session"""
    print("LLM Debugger - Interactive Mode")

    prompt = input("Enter prompt: ")
    expected_output = input("Expected output format (json/text): ")

    output = llm(prompt)
    print(f"\nGenerated output:\n{output}\n")

    if input("Does this look correct? (y/n): ").lower() == 'n':
        diagnosis = diagnose_failure(prompt, output, {"format": expected_output})

        print("\nDiagnosis:")
        for issue in diagnosis["issues"]:
            print(f"- {issue['type'].value}: {issue['details']}")

        print("\nSuggested fix:")
        print(suggest_prompt_fix(diagnosis, prompt))

最佳实践

1. 稳定复现：运行多次
2. 隔离变量：每次只测试一项提示词更改
3. 检查示例：少样本示例是否清晰？
4. 验证约束：约束是否足够明确？
5. 测试边界情况：边界条件
6. 记录所有信息：输入、输出和问题
7. 系统地迭代：跟踪哪些修复有效

输出检查清单

• 定义故障分类
• 根因分析器
• 提示词修复建议
• 工具架构改进
• 测试用例生成器
• 修复验证
• 调试工作流
• 建议引擎
• 交互式调试器
• 日志系统