log-analyzer

日志分析技能

概述

此技能可帮助你有效分析应用日志，以诊断问题、跟踪错误并了解系统行为。涵盖日志搜索、模式检测、结构化日志记录以及与监控工具集成。

日志分析理念

原则

1. 结构优先于文本：结构化日志更易于分析
2. 上下文很重要：包含相关元数据
3. 日志级别各有含义：使用适当的严重性级别
4. 关联是关键：关联不同服务中的相关事件

日志级别

级别	使用时机	示例
ERROR	出现故障，需要关注	数据库连接失败
WARN	出现意外情况，但已处理	失败后重试成功
INFO	正常运行的重要节点	用户已注册
DEBUG	详细的诊断信息	查询在 50ms 内执行完成
TRACE	非常详细的信息，通常处于禁用状态	函数进入/退出

结构化日志记录

Winston 配置（Node.js）

// src/lib/logger.ts
import winston from 'winston';

const logger = winston.createLogger({
  level: process.env.LOG_LEVEL || 'info',
  format: winston.format.combine(
    winston.format.timestamp(),
    winston.format.errors({ stack: true }),
    winston.format.json()
  ),
  defaultMeta: {
    service: 'my-app',
    environment: process.env.NODE_ENV,
  },
  transports: [
    // Console for development
    new winston.transports.Console({
      format: process.env.NODE_ENV === 'development'
        ? winston.format.combine(
            winston.format.colorize(),
            winston.format.simple()
          )
        : winston.format.json(),
    }),
  ],
});

// Add request context
export function createRequestLogger(requestId: string, userId?: string) {
  return logger.child({
    requestId,
    userId,
  });
}

export { logger };

Pino 日志记录器（高性能）

// src/lib/logger.ts
import pino from 'pino';

export const logger = pino({
  level: process.env.LOG_LEVEL || 'info',
  transport: process.env.NODE_ENV === 'development'
    ? {
        target: 'pino-pretty',
        options: {
          colorize: true,
          translateTime: 'SYS:standard',
        },
      }
    : undefined,
  base: {
    service: 'my-app',
    env: process.env.NODE_ENV,
  },
  redact: {
    paths: ['password', 'token', 'apiKey', '*.password', '*.token'],
    censor: '[REDACTED]',
  },
});

// Request-scoped logger
export function requestLogger(requestId: string, userId?: string) {
  return logger.child({ requestId, userId });
}

日志记录最佳实践

// DO: Include context
logger.info('User signed up', {
  userId: user.id,
  email: user.email,
  plan: 'free',
  source: 'web',
});

// DO: Log errors with stack traces
logger.error('Payment failed', {
  error: err.message,
  stack: err.stack,
  userId: user.id,
  amount: 99.99,
  provider: 'stripe',
});

// DO: Use appropriate levels
logger.debug('Database query', {
  query: 'SELECT * FROM users WHERE id = ?',
  duration: 45,
  rows: 1,
});

// DON'T: Log sensitive data
// BAD: logger.info('Login', { password: user.password })

// DON'T: Use string concatenation
// BAD: logger.info('User ' + user.id + ' signed up')

日志搜索与筛选

命令行（grep、jq）

# Search JSON logs with jq
cat logs.json | jq 'select(.level == "error")'

# Search for specific user
cat logs.json | jq 'select(.userId == "usr_123")'

# Search by time range
cat logs.json | jq 'select(.timestamp >= "2024-01-15T10:00:00")'

# Count errors by type
cat logs.json | jq 'select(.level == "error") | .error.code' | sort | uniq -c

# Extract specific fields
cat logs.json | jq '{timestamp, level, message, userId}'

# Search text logs
grep -i "error" logs.txt
grep -E "error|warn" logs.txt
grep -A5 "ERROR" logs.txt  # 5 lines after match
grep -B3 "ERROR" logs.txt  # 3 lines before match

Vercel 日志分析

# View live logs
vercel logs --follow

# Filter by level
vercel logs --level error

# Search specific timeframe
vercel logs --since 2h

# Filter by deployment
vercel logs --deployment-url https://myapp-abc123.vercel.app

# Output JSON for processing
vercel logs --output json | jq 'select(.level == "error")'

Supabase 日志分析

-- Query Postgres logs
SELECT *
FROM postgres_logs
WHERE timestamp > now() - interval '1 hour'
  AND error_severity = 'ERROR'
ORDER BY timestamp DESC
LIMIT 100;

-- Query auth logs
SELECT *
FROM auth.audit_log_entries
WHERE timestamp > now() - interval '24 hours'
  AND payload->>'action' = 'login'
ORDER BY timestamp DESC;

-- Find slow queries
SELECT
  query,
  calls,
  mean_exec_time,
  total_exec_time
FROM pg_stat_statements
ORDER BY mean_exec_time DESC
LIMIT 20;

模式检测

常见错误模式

// Error pattern analyzer
interface ErrorPattern {
  pattern: RegExp;
  category: string;
  severity: 'critical' | 'high' | 'medium' | 'low';
  suggestion: string;
}

const errorPatterns: ErrorPattern[] = [
  {
    pattern: /ECONNREFUSED|connection refused/i,
    category: 'connectivity',
    severity: 'critical',
    suggestion: 'Check if the target service is running and accessible',
  },
  {
    pattern: /timeout|ETIMEDOUT/i,
    category: 'timeout',
    severity: 'high',
    suggestion: 'Increase timeout or check for slow operations',
  },
  {
    pattern: /out of memory|heap|OOM/i,
    category: 'memory',
    severity: 'critical',
    suggestion: 'Increase memory allocation or fix memory leak',
  },
  {
    pattern: /rate.?limit|429|too many requests/i,
    category: 'rate-limiting',
    severity: 'medium',
    suggestion: 'Implement backoff strategy or increase rate limits',
  },
  {
    pattern: /unauthorized|401|invalid.?token/i,
    category: 'auth',
    severity: 'medium',
    suggestion: 'Check authentication credentials or token expiry',
  },
  {
    pattern: /not.?found|404/i,
    category: 'not-found',
    severity: 'low',
    suggestion: 'Verify resource exists or check URL',
  },
];

function categorizeError(message: string): ErrorPattern | null {
  for (const pattern of errorPatterns) {
    if (pattern.pattern.test(message)) {
      return pattern;
    }
  }
  return null;
}

日志聚合脚本

// scripts/analyze-logs.ts
import * as readline from 'readline';
import * as fs from 'fs';

interface LogEntry {
  timestamp: string;
  level: string;
  message: string;
  error?: {
    message: string;
    code?: string;
  };
  [key: string]: any;
}

interface LogSummary {
  totalEntries: number;
  byLevel: Record<string, number>;
  errorMessages: Record<string, number>;
  timeRange: {
    start: string;
    end: string;
  };
}

async function analyzeLogs(filePath: string): Promise<LogSummary> {
  const summary: LogSummary = {
    totalEntries: 0,
    byLevel: {},
    errorMessages: {},
    timeRange: { start: '', end: '' },
  };

  const fileStream = fs.createReadStream(filePath);
  const rl = readline.createInterface({
    input: fileStream,
    crlfDelay: Infinity,
  });

  for await (const line of rl) {
    try {
      const entry: LogEntry = JSON.parse(line);
      summary.totalEntries++;

      // Count by level
      summary.byLevel[entry.level] = (summary.byLevel[entry.level] || 0) + 1;

      // Track error messages
      if (entry.level === 'error' && entry.error?.message) {
        const msg = entry.error.message.substring(0, 100);
        summary.errorMessages[msg] = (summary.errorMessages[msg] || 0) + 1;
      }

      // Track time range
      if (!summary.timeRange.start || entry.timestamp < summary.timeRange.start) {
        summary.timeRange.start = entry.timestamp;
      }
      if (!summary.timeRange.end || entry.timestamp > summary.timeRange.end) {
        summary.timeRange.end = entry.timestamp;
      }
    } catch {
      // Skip non-JSON lines
    }
  }

  return summary;
}

// Usage
analyzeLogs('logs.json').then(console.log);

请求跟踪

请求 ID 中间件

// src/middleware.ts
import { NextResponse } from 'next/server';
import type { NextRequest } from 'next/server';
import { v4 as uuidv4 } from 'uuid';

export function middleware(request: NextRequest) {
  const requestId = request.headers.get('x-request-id') || uuidv4();

  const response = NextResponse.next();
  response.headers.set('x-request-id', requestId);

  return response;
}

跟踪上下文

// src/lib/tracing.ts
import { AsyncLocalStorage } from 'async_hooks';

interface TraceContext {
  requestId: string;
  userId?: string;
  startTime: number;
}

const asyncLocalStorage = new AsyncLocalStorage<TraceContext>();

export function withTrace<T>(
  context: TraceContext,
  fn: () => T | Promise<T>
): T | Promise<T> {
  return asyncLocalStorage.run(context, fn);
}

export function getTrace(): TraceContext | undefined {
  return asyncLocalStorage.getStore();
}

// Usage in logger
export function log(level: string, message: string, data?: object) {
  const trace = getTrace();
  console.log(JSON.stringify({
    timestamp: new Date().toISOString(),
    level,
    message,
    requestId: trace?.requestId,
    userId: trace?.userId,
    duration: trace ? Date.now() - trace.startTime : undefined,
    ...data,
  }));
}

错误跟踪集成

Sentry 集成

// src/lib/sentry.ts
import * as Sentry from '@sentry/nextjs';

Sentry.init({
  dsn: process.env.SENTRY_DSN,
  environment: process.env.NODE_ENV,
  tracesSampleRate: 0.1, // 10% of transactions
  beforeSend(event, hint) {
    // Filter out specific errors
    const error = hint.originalException;
    if (error instanceof Error && error.message.includes('Network')) {
      return null;
    }
    return event;
  },
});

// Capture with context
export function captureError(error: Error, context?: Record<string, any>) {
  Sentry.withScope((scope) => {
    if (context) {
      scope.setExtras(context);
    }
    Sentry.captureException(error);
  });
}

// Usage
try {
  await riskyOperation();
} catch (error) {
  captureError(error, {
    userId: user.id,
    action: 'payment',
    amount: 99.99,
  });
  throw error;
}

自定义错误边界日志记录

// src/components/error-boundary.tsx
'use client';

import { useEffect } from 'react';
import * as Sentry from '@sentry/nextjs';

export default function Error({
  error,
  reset,
}: {
  error: Error & { digest?: string };
  reset: () => void;
}) {
  useEffect(() => {
    // Log to error tracking
    Sentry.captureException(error);

    // Log to console with context
    console.error('Client Error:', {
      message: error.message,
      digest: error.digest,
      stack: error.stack,
      url: window.location.href,
      userAgent: navigator.userAgent,
    });
  }, [error]);

  return (
    <div>
      <h2>Something went wrong!</h2>
      <button onClick={reset}>Try again</button>
    </div>
  );
}

日志分析查询

常见调查查询

# Find all errors for a specific user
jq 'select(.userId == "usr_123" and .level == "error")' logs.json

# Find errors in the last hour
jq --arg time "$(date -d '1 hour ago' -Iseconds)" \
  'select(.timestamp > $time and .level == "error")' logs.json

# Group errors by message
jq -s 'group_by(.error.message) |
  map({message: .[0].error.message, count: length}) |
  sort_by(-.count)' logs.json

# Find slow requests (>1 second)
jq 'select(.duration > 1000)' logs.json

# Trace a specific request
jq 'select(.requestId == "req_abc123")' logs.json | sort_by(.timestamp)

# Find patterns in error messages
jq -r 'select(.level == "error") | .error.message' logs.json | \
  sort | uniq -c | sort -rn | head -20

类 SQL 日志分析

-- Using tools like clickhouse or DuckDB for log analysis

-- Error rate by hour
SELECT
  date_trunc('hour', timestamp) as hour,
  count(*) FILTER (WHERE level = 'error') as errors,
  count(*) as total,
  round(100.0 * count(*) FILTER (WHERE level = 'error') / count(*), 2) as error_rate
FROM logs
WHERE timestamp > now() - interval '24 hours'
GROUP BY 1
ORDER BY 1;

-- Top error messages
SELECT
  error_message,
  count(*) as occurrences,
  min(timestamp) as first_seen,
  max(timestamp) as last_seen
FROM logs
WHERE level = 'error'
  AND timestamp > now() - interval '24 hours'
GROUP BY error_message
ORDER BY occurrences DESC
LIMIT 10;

-- Slowest endpoints
SELECT
  path,
  count(*) as requests,
  avg(duration) as avg_duration,
  max(duration) as max_duration,
  percentile_cont(0.95) WITHIN GROUP (ORDER BY duration) as p95
FROM logs
WHERE timestamp > now() - interval '1 hour'
GROUP BY path
ORDER BY avg_duration DESC
LIMIT 10;

日志保留与清理

日志轮换配置

# /etc/logrotate.d/myapp
/var/log/myapp/*.log {
    daily
    rotate 14
    compress
    delaycompress
    missingok
    notifempty
    create 0640 www-data www-data
    sharedscripts
    postrotate
        systemctl reload myapp
    endscript
}

自动清理日志

// scripts/cleanup-logs.ts
import * as fs from 'fs';
import * as path from 'path';

const LOG_DIR = '/var/log/myapp';
const RETENTION_DAYS = 30;

function cleanupOldLogs() {
  const now = Date.now();
  const maxAge = RETENTION_DAYS * 24 * 60 * 60 * 1000;

  const files = fs.readdirSync(LOG_DIR);

  for (const file of files) {
    const filePath = path.join(LOG_DIR, file);
    const stats = fs.statSync(filePath);

    if (now - stats.mtime.getTime() > maxAge) {
      fs.unlinkSync(filePath);
      console.log(`Deleted: ${file}`);
    }
  }
}

cleanupOldLogs();

调试检查清单

调查问题时

• 确定时间范围
• 查找相关请求 ID
• 检查错误是否激增
• 查看受影响的用户/端点
• 检查模式是否发生变化
• 查看近期部署

日志质量检查

• 采用结构化 JSON 格式
• 使用适当的日志级别
• 包含请求 ID
• 对敏感数据进行脱敏
• 错误包含堆栈跟踪
• 包含相关上下文

何时使用此技能

在以下情况下调用此技能：

• 调查生产环境问题
• 设置日志基础设施
• 调试应用错误
• 分析性能问题
• 创建日志分析脚本
• 设置错误跟踪
• 实现请求跟踪
• 清理日志数据