用 eBPF off-CPU 分析揪出内核锁竞争：LinkedIn 的排障实录

# 安装 bcc-tools（Ubuntu/Debian）
sudo apt-get install bpfcc-tools linux-headers-$(uname -r)

# 查看目标进程的 off-CPU 时间分布
# -p 指定 PID，-f 显示完整调用栈
sudo offcputime-bpfcc -p <PID> -f

    comm             tid     total_onoffcpu    count
    mysqld           1234    8320              45
        futex_wait_queue_me
        mutex_lock
        __do_page_fault
        handle_mm_fault
        ...

#!/usr/bin/env python3
"""
trace_rwsem_wait.py — 追踪内核 rwsem 写锁等待时长
用法: sudo python3 trace_rwsem_wait.py <进程名>
需要: bcc-tools, 内核 4.9+
"""

from bcc import BPF
import sys
import time

if len(sys.argv) < 2:
    print("用法: sudo python3 trace_rwsem_wait.py <进程名>")
    sys.exit(1)

TARGET_COMM = sys.argv[1]

# eBPF C 代码：挂到 rwsem_write_lock 的入口和返回
bpf_text = """
#include <uapi/linux/ptrace.h>
#include <linux/sched.h>

struct key_t {
    u64 stack_id;
    u32 pid;
    char comm[16];
};

BPF_HASH(start, struct key_t, u64);
BPF_STACK_TRACE(stack_traces, 2048);
BPF_HASH(wait_time, struct key_t, u64);

int trace_entry(struct pt_regs *ctx) {
    struct key_t key = {};
    u64 ts = bpf_ktime_get_ns();

    u32 pid = bpf_get_current_pid_tgid() >> 32;
    if (pid == 0)
        return 0;

    key.pid = pid;
    bpf_get_current_comm(&key.comm, sizeof(key.comm));

    // 只追踪目标进程
    char target[TARGET_COMM_LEN];
    target[0] = 0;
    bpf_probe_read_kernel_str(target, TARGET_COMM_LEN, TARGET_COMM);
    if (key.comm[0] != target[0])
        return 0;

    key.stack_id = stack_traces.get_stackid(ctx, BPF_F_REUSE_STACKID);
    start.update(&key, &ts);
    return 0;
}

int trace_return(struct pt_regs *ctx) {
    struct key_t key = {};
    u64 *tsp, delta;

    u32 pid = bpf_get_current_pid_tgid() >> 32;
    key.pid = pid;
    bpf_get_current_comm(&key.comm, sizeof(key.comm));

    char target[TARGET_COMM_LEN];
    target[0] = 0;
    bpf_probe_read_kernel_str(target, TARGET_COMM_LEN, TARGET_COMM);
    if (key.comm[0] != target[0])
        return 0;

    // 用 0 作为 stack_id 查找 entry 时存的 timestamp
    key.stack_id = 0;
    tsp = start.lookup(&key);
    if (tsp == 0)
        return 0;

    delta = bpf_ktime_get_ns() - *tsp;
    start.delete(&key);

    // 重新获取真实 stack_id 用于聚合
    key.stack_id = stack_traces.get_stackid(ctx, BPF_F_REUSE_STACKID);
    wait_time.increment(&key, delta);
    return 0;
}
"""

# 替换目标进程名占位符
bpf_text = bpf_text.replace("TARGET_COMM_LEN", str(len(TARGET_COMM) + 1))
bpf_text = bpf_text.replace("TARGET_COMM", '"' + TARGET_COMM + '"')

b = BPF(text=bpf_text)

# 挂载 kprobe 到内核 rwsem 写锁函数
# 函数名因内核版本不同可能变化，常见为 down_write 或 rwsem_write_lock
try:
    b.attach_kprobe(event="down_write", fn_name="trace_entry")
    b.attach_kretprobe(event="down_write", fn_name="trace_return")
except Exception:
    # 较新内核可能用 rwsem_down_write_slowpath
    b.attach_kprobe(event="rwsem_down_write_slowpath", fn_name="trace_entry")
    b.attach_kretprobe(event="rwsem_down_write_slowpath", fn_name="trace_return")

print(f"追踪 {TARGET_COMM} 的 rwsem 写锁等待... Ctrl+C 停止\n")

# 定时输出结果
while True:
    try:
        time.sleep(5)
        print("\n=== rwsem 写锁等待时间 (微秒) ===")
        for k, v in b["wait_time"].items():
            stack = b["stack_traces"].get(k.stack_id)
            comm = k.comm.decode('utf-8', 'replace')
            print(f"  {comm}/{k.pid}  等待: {v/1000:.0f} us")
            if stack:
                for addr in stack:
                    sym = b.ksym(addr, show_module=True, show_offset=True)
                    print(f"    {sym}")
            print()
        b["wait_time"].clear()
    except KeyboardInterrupt:
        print("\n停止追踪。")
        break

# 追踪 mysqld 进程的 rwsem 写锁等待
sudo python3 trace_rwsem_wait.py mysqld

# 采集 30 秒 off-CPU 数据，输出折叠格式
sudo offcputime-bpfcc -p <PID> -f 30 > offcpu_folded.txt

# 生成火焰图（需要 flamegraph.pl）
git clone https://github.com/brendangregg/FlameGraph
perl FlameGraph/flamegraph.pl --title="off-CPU" --countname=us offcpu_folded.txt > offcpu.svg