Kernel調試追蹤技術之 Kprobe on ARM64
本題目目標
- 熟悉kprobe的由來、接口使用方式和基本原理
- 詳解ARM64 Kprobe的實現方式
- 思考探索kprobe可以解決哪些問題
- 簡介目前基於kprobe的工具
kprobe是什么？
kprobe 是一種動態調試機制，用於debugging，動態跟蹤，性能分析，動態修改內核行為等，2004年由IBM發布，是名為Dprobes工具集的底層實現機制[1][2]，2005年合入Linux kernel。probe的含義是像一個探針，可以不修改分析對象源碼的情況下，獲取Kernel的運行時信息。
kprobe的實現原理是把指定地址（探測點）的指令替換成一個可以讓cpu進入debug模式的指令，使執行路徑暫停，跳轉到probe 處理函數后收集、修改信息，再跳轉回來繼續執行。
X86中使用的是int3指令，ARM64中使用的是BRK指令進入debug monitor模式。
這種指令替換機制使得kprobe可以在大部分kernel的代碼段插入探測點，除了一些分支類的指令如br、exception、eret等。另外kprobe模塊本身的代碼不能probe。在pre_handler和post_handler中可以訪問、修改所有寄存器和全局變量，其變體jprobe可以檢查傳入參數，kretprobe可以檢查返回值。且不需要修改探測目標的源碼，方便用於生產系統的debug、性能分析、log記錄等。
kprobe一直在X86系統上使用，ARM64的平台支持在2015年合入kernel [8]。
kprobe代碼示例
-
sample/kprobe/kprobe_example.c// sample/kprobe/kprobe_example.c #include <linux/kernel.h> #include <linux/module.h> #include <linux/kprobes.h> #define MAX_SYMBOL_LEN 64 static char symbol[MAX_SYMBOL_LEN] = "_do_fork"; module_param_string(symbol, symbol, sizeof(symbol), 0644); /* For each probe you need to allocate a kprobe structure */ static struct kprobe kp = { .symbol_name = symbol, }; /* kprobe pre_handler: called just before the probed instruction is executed */ static int handler_pre(struct kprobe *p, struct pt_regs *regs) { pr_info("<%s> pre_handler: p->addr = 0x%p, pc = 0x%lx," " pstate = 0x%lx\n", p->symbol_name, p->addr, (long)regs->pc, (long)regs->pstate); /* A dump_stack() here will give a stack backtrace */ return 0; } /* kprobe post_handler: called after the probed instruction is executed */ static void handler_post(struct kprobe *p, struct pt_regs *regs, unsigned long flags) { pr_info("<%s> post_handler: p->addr = 0x%p, pstate = 0x%lx\n", p->symbol_name, p->addr, (long)regs->pstate); } static int handler_fault(struct kprobe *p, struct pt_regs *regs, int trapnr) { pr_info("fault_handler: p->addr = 0x%p, trap #%dn", p->addr, trapnr); /* Return 0 because we don't handle the fault. */ return 0; } static int __init kprobe_init(void) { int ret; kp.pre_handler = handler_pre; kp.post_handler = handler_post; kp.fault_handler = handler_fault; ret = register_kprobe(&kp); if (ret < 0) { pr_err("register_kprobe failed, returned %d\n", ret); return ret; } pr_info("Planted kprobe at %p\n", kp.addr); return 0; } static void __exit kprobe_exit(void) { unregister_kprobe(&kp); pr_info("kprobe at %p unregistered\n", kp.addr); } -
sample/kprobe/jprobe_example.c#include <linux/kernel.h> #include <linux/module.h> #include <linux/kprobes.h> /* Proxy routine having the same arguments as actual _do_fork() routine */ static long j_do_fork(unsigned long clone_flags, unsigned long stack_start, unsigned long stack_size, int __user *parent_tidptr, int __user *child_tidptr, unsigned long tls) { pr_info("jprobe: clone_flags = 0x%lx, stack_start = 0x%lx " "stack_size = 0x%lx\n", clone_flags, stack_start, stack_size); /* Always end with a call to jprobe_return(). */ jprobe_return(); return 0; } static struct jprobe my_jprobe = { .entry = j_do_fork, .kp = { .symbol_name = "_do_fork", }, }; static int __init jprobe_init(void) { int ret; ret = register_jprobe(&my_jprobe); if (ret < 0) { pr_err("register_jprobe failed, returned %d\n", ret); return -1; } pr_info("Planted jprobe at %p, handler addr %p\n", my_jprobe.kp.addr, my_jprobe.entry); return 0; } static void __exit jprobe_exit(void) { unregister_jprobe(&my_jprobe); pr_info("jprobe at %p unregistered\n", my_jprobe.kp.addr); } module_init(jprobe_init) module_exit(jprobe_exit) MODULE_LICENSE("GPL"); -
sample/kprobe/kretprobe_example.c#include <linux/kernel.h> #include <linux/module.h> #include <linux/kprobes.h> #include <linux/ktime.h> #include <linux/limits.h> #include <linux/sched.h> static char func_name[NAME_MAX] = "_do_fork"; module_param_string(func, func_name, NAME_MAX, S_IRUGO); MODULE_PARM_DESC(func, "Function to kretprobe; this module will report the" " function's execution time"); /* per-instance private data */ struct my_data { ktime_t entry_stamp; }; /* Here we use the entry_hanlder to timestamp function entry */ static int entry_handler(struct kretprobe_instance *ri, struct pt_regs *regs) { struct my_data *data; if (!current->mm) return 1; /* Skip kernel threads */ data = (struct my_data *)ri->data; data->entry_stamp = ktime_get(); return 0; } static int ret_handler(struct kretprobe_instance *ri, struct pt_regs *regs) { unsigned long retval = regs_return_value(regs); struct my_data *data = (struct my_data *)ri->data; s64 delta; ktime_t now; now = ktime_get(); delta = ktime_to_ns(ktime_sub(now, data->entry_stamp)); pr_info("%s returned %lu and took %lld ns to execute\n", func_name, retval, (long long)delta); return 0; } static struct kretprobe my_kretprobe = { .handler = ret_handler, .entry_handler = entry_handler, .data_size = sizeof(struct my_data), /* Probe up to 20 instances concurrently. */ .maxactive = 20, }; static int __init kretprobe_init(void) { int ret; my_kretprobe.kp.symbol_name = func_name; ret = register_kretprobe(&my_kretprobe); if (ret < 0) { pr_err("register_kretprobe failed, returned %d\n", ret); return -1; } pr_info("Planted return probe at %s: %p\n", my_kretprobe.kp.symbol_name, my_kretprobe.kp.addr); return 0; } static void __exit kretprobe_exit(void) { unregister_kretprobe(&my_kretprobe); pr_info("kretprobe at %p unregistered\n", my_kretprobe.kp.addr); /* nmissed > 0 suggests that maxactive was set too low. */ pr_info("Missed probing %d instances of %s\n", my_kretprobe.nmissed, my_kretprobe.kp.symbol_name); }
kprobe源碼和接口
源碼
./include/linux/kprobes.h # kprobe頭文件
./include/asm-generic/kprobes.h
./kernel/kprobes.c # kprobe核心實現
./arch/arm64/include/asm/kprobes.h
./arch/arm64/kernel/probes/kprobes.c # kprobe arm64支持
./arch/arm64/kernel/probes/kprobes_trampoline.S
./samples/kprobes/kprobe_example.c # kprobe使用例子程序
./samples/kprobes/jprobe_example.c
./samples/kprobes/kretprobe_example.c
對外接口
struct kprobe {
struct hlist_node hlist; // hash list保存所有kprobe，key是指令地址
struct list_head list; // 鏈接一個地址上注冊的多個kprobe
unsigned long nmissed; // 被臨時disabled的次數
kprobe_opcode_t *addr; // 探測點地址
const char *symbol_name; // 探測點函數名
unsigned int offset; // 探測點在函數內的偏移
~~~~ kprobe_pre_handler_t pre_handler; // 前處理函數
kprobe_post_handler_t post_handler; // 后處理函數
kprobe_fault_handler_t fault_handler; // 探測指令發生fault時的處理函數
kprobe_break_handler_t break_handler; // 在前三個handler里發生break trap的處理函數
/* Saved opcode (which has been replaced with breakpoint) */
kprobe_opcode_t opcode; // 被breakpoint替換了的操作碼
/* copy of the original instruction */
struct arch_specific_insn ainsn; // 保存平台相關的被探測指令和下一條指令
u32 flags; // 狀態標記
};
struct jprobe {
struct kprobe kp;
void *entry; /* probe handling code to jump to */
};
struct kretprobe {
struct kprobe kp;
kretprobe_handler_t handler;
kretprobe_handler_t entry_handler;
int maxactive;
int nmissed;
size_t data_size;
struct hlist_head free_instances;
raw_spinlock_t lock;
};
int register_kprobe(struct kprobe *p);
int register_kprobes(struct kprobe **kps, int num);
int register_jprobe(struct jprobe *p);
int register_jprobes(struct jprobe **jps, int num);
int register_kretprobe(struct kretprobe *rp);
int register_kretprobes(struct kretprobe **rps, int num);
int disable_kprobe(struct kprobe *kp);
int enable_kprobe(struct kprobe *kp);
void jprobe_return(void);
void unregister_jprobe(struct jprobe *p);
void unregister_jprobes(struct jprobe **jps, int num);
void unregister_kprobe(struct kprobe *p);
void unregister_kprobes(struct kprobe **kps, int num);
void unregister_kretprobe(struct kretprobe *rp);
void unregister_kretprobes(struct kretprobe **rps, int num);
kprobe的使用比較簡單，只需要指定探測點地址，或者使用符號名+偏移的方式，定義xxx_handler，注冊即可。注冊后探測指令被替換，可以使用kprobe_enable/disable函數動態開關。
jprobe是kprobe的一種方便檢查函數參數的實現方式，通過定義一個和探測函數相同原型的函數實現。
kretprobe用於檢查函數返回值，可以定義函數入口handler和返回時的handler。
可以參看kernel源碼中的samples/kprobes/下的例子程序。
kprobe的實現及Arm64支持
kprobe的管理
kprobe可以支持大量的探測點，為了快速查詢和插入，使用哈希鏈表管理所有的kprobes，hash的key值是探測點的地址值。
static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
struct kprobe *get_kprobe(void *addr)
{
struct hlist_head *head;
struct kprobe *p;
head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
hlist_for_each_entry_rcu(p, head, hlist) {
if (p->addr == addr)
return p;
}
return NULL;
}
int register_kprobe(struct kprobe *p)
{..
INIT_HLIST_NODE(&p->hlist);
hlist_add_head_rcu(&p->hlist,
&kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
..}
kprobe的注冊
kprobe注冊時傳入的是struct kprobe結構體，里面包含指令地址或者函數名地址和函數內偏移，傳入地址需要先檢查是否在代碼段里，且不在blacklist里，blacklist包含不能probe的函數，主要是kprobe本身的函數。
然后調用arch_prepare_kprobe解碼指令，看指令是否是一些分支等特殊指令，需要特別處理。如果是正常可以probe的指令，調用arch_prepare_ss_slot把探測點的指令備份到slot page里，把下一條指令存入struct arch_probe_insn結構的restore成員里，在post_handler之后恢復執行。
arch_prepare_krpobe無誤后把kprobe加入kprobe_table哈希鏈表。
然后調用arch_arm_kprobe替換探測點指令為BRK64_OPCODE_KPROBES指令。
kprobe的觸發和處理
kprobe的觸發和處理是通過brk exception和single step 單步exception執行的，每次的處理函數中會修改被異常中斷的上下文（struct pt_regs）的指令寄存器，實現執行流的跳轉。
異常機制依賴於CPU的體系結構，所以這部分代碼是在arch/arm64/kernel/probes/kprobes.c 中實現的，這也是各個體系結構支持kprobe的主要功能。
異常處理的注冊
異常處理的注冊在arch/arm64/kernel/debug-monitors.c， 是arm64的通用debug模塊，kgdb也基於這個模塊。
static int __init debug_traps_init(void)
{
hook_debug_fault_code(DBG_ESR_EVT_HWSS, single_step_handler, SIGTRAP, // 單步異常處理函數
TRAP_TRACE, "single-step handler");
hook_debug_fault_code(DBG_ESR_EVT_BRK, brk_handler, SIGTRAP, // 斷點異常處理函數
TRAP_BRKPT, "ptrace BRK handler");
return 0;
}
hook_debug_fault_code是替換arch/arm64/mm/fault.c 中的debug_fault_info異常表項：
/*
* __refdata because early_brk64 is __init, but the reference to it is
* clobbered at arch_initcall time.
* See traps.c and debug-monitors.c:debug_traps_init().
*/
static struct fault_info __refdata debug_fault_info[] = {
{ do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" },
{ do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" },
{ do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" },
{ do_bad, SIGBUS, 0, "unknown 3" },
{ do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" },
{ do_bad, SIGTRAP, 0, "aarch32 vector catch" },
{ early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" },
{ do_bad, SIGBUS, 0, "unknown 7" },
};
void __init hook_debug_fault_code(int nr,
int (*fn)(unsigned long, unsigned int, struct pt_regs *),
int sig, int code, const char *name)
{
BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info));
debug_fault_info[nr].fn = fn;
debug_fault_info[nr].sig = sig;
debug_fault_info[nr].code = code;
debug_fault_info[nr].name = name;
}
BRK和SS異常執行流程
arch/arm64/kernel/entry.S:
el1_dbg:
bl do_debug_exception
fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr);
inf->fn(addr, esr, regs)
brk_handler()
kprobe_breakpoint_handler()
single_step_handler()
kprobe_single_step_handler()
breakpoint斷點執行流程主要任務是執行pre_handler，把slot中保存的原指令設置進regs上下文的指令寄存器里，這樣退出brk異常后，會單步執行被probe的指令。
單步執行探測點的指令后，會觸發單步異常，進入single_step_handler，調用kprobe_single_step_handler，主要任務是恢復執行路徑，調用用戶注冊的post_handler。
首先檢查當前單步異常的指令地址是否是之前設定的下一條指令地址；然后關閉單步狀態，即只執行一次單步。
最后設置寄存器上下文中的指令寄存器為探測點指令的下一條指令。調用post_handler，結束這個kprobe的工作。
其它未詳細介紹的部分
- 被替換的指令放在哪里？
slot page，使用了module_alloc分配可以執行的內存頁
- 一個探測點由多個probe注冊怎么處理
aggrprobe
- SMP、中斷、搶占時可能有kprobe重入，如何處理
- 實現了
reenter檢查機制，允許probe嵌套
- 實現了
- kprobe的性能
- break指令導致CPU執行停止，時間開銷較大
- x86實現了優化機制，使用jmp指令替換int3 這種break指令，速度提升10倍；ARM64中未實現。
kprobes: Kprobes jump optimization support[5]
kprobe 的問題
- probe函數的定義要非常小心，否則會引起kernel panic或其它異常，比如probe的一個函數臨界區被調用，而你的probe handler是睡眠的，就會報錯
- 需要依賴kernel源碼樹或build文件，編寫kernel 模塊
kprobe的優勢
- 可以不重編kernel對生產系統進行探測，在PC和Server中比較有意義
- 可以動態觀察、修改幾乎任意代碼路徑的狀態，比Ftrace有更強的定制性
kprobe適用場景
1. 觀察 Observability
- kprobe觀察任一點的寄存器狀態，全局變量，局部變量，jprobe觀察函數參數，kretprobe觀察返回值
- 函數調用，profiling等
- 這塊Ftrace可以替代
- 數據收集
- 例如：打印genpool的bitmap，觀察ION carveout的使用情況
2. 篡改 Hacking
- error-injection
- 修改函數返回值或者判斷條件，使代碼進入錯誤處理路徑，提高測試覆蓋率
- data-injection
- 動態修改數據，仿真測試數據，比如注入溫度傳感器溫度，模仿各種測試值測試
- 實驗：對pvt做溫度抽樣值注入，測試各種溫度下的計算問題
- 動態補丁
3. 調試
4. 其它？
kprobe的應用
1. trace_kprobe
kernel ftrace子系統基於kprobe實現了kprobe_event，可以probe任意函數。
優點是無需寫代碼，方便簡單的函數trace，可以獲得函數參數值和返回值
缺點是不能做動態修改
使用方法參看kernel/Documentation/trace/kprobetrace.txt
-
kprobe_event使用說明和示例
24 Synopsis of kprobe_events 25 ------------------------- 26 p[:[GRP/]EVENT] [MOD:]SYM[+offs]|MEMADDR [FETCHARGS] : Set a probe 27 r[MAXACTIVE][:[GRP/]EVENT] [MOD:]SYM[+0] [FETCHARGS] : Set a return probe 28 -:[GRP/]EVENT : Clear a probe 29 30 GRP : Group name. If omitted, use "kprobes" for it. 31 EVENT : Event name. If omitted, the event name is generated 32 based on SYM+offs or MEMADDR. 33 MOD : Module name which has given SYM. 34 SYM[+offs] : Symbol+offset where the probe is inserted. 35 MEMADDR : Address where the probe is inserted. 36 MAXACTIVE : Maximum number of instances of the specified function that 37 can be probed simultaneously, or 0 for the default value 38 as defined in Documentation/kprobes.txt section 1.3.1. 39 40 FETCHARGS : Arguments. Each probe can have up to 128 args. 41 %REG : Fetch register REG 42 @ADDR : Fetch memory at ADDR (ADDR should be in kernel) 43 @SYM[+|-offs] : Fetch memory at SYM +|- offs (SYM should be a data symbol) 44 $stackN : Fetch Nth entry of stack (N >= 0) 45 $stack : Fetch stack address. 46 $retval : Fetch return value.(*) 47 $comm : Fetch current task comm. 48 +|-offs(FETCHARG) : Fetch memory at FETCHARG +|- offs address.(**) 49 NAME=FETCHARG : Set NAME as the argument name of FETCHARG. 50 FETCHARG:TYPE : Set TYPE as the type of FETCHARG. Currently, basic types 51 (u8/u16/u32/u64/s8/s16/s32/s64), hexadecimal types 52 (x8/x16/x32/x64), "string" and bitfield are supported. 53 54 (*) only for return probe. 55 (**) this is useful for fetching a field of data structures. echo 'p:myprobe do_sys_open filename=+0(%x1):string flags=%x2 mode=%x3' > /sys/kernel/debug/tracing/kprobe_events echo 'r:myretprobe do_sys_open $retval ' >> /sys/kernel/debug/tracing/kprobe_events echo > trace echo 1 > events/kprobes/enable root@x3dvbj3-hynix2G-2666:/sys/kernel/debug/tracing# cat trace # tracer: nop # # _-----=> irqs-off # / _----=> need-resched # | / _---=> hardirq/softirq # || / _--=> preempt-depth # ||| / delay # TASK-PID CPU# |||| TIMESTAMP FUNCTION # | | | |||| | | sh-1499 [002] d... 1902.985663: myprobe: (do_sys_open+0x0/0x208) filename="/etc/passwd" flags=0xa0000 mode=0x0 sh-1499 [002] d..1 1902.985730: myretprobe: (SyS_openat+0x3c/0x50 <- do_sys_open) arg1=0x3 ps-12596 [001] d... 1903.694056: myprobe: (do_sys_open+0x0/0x208) filename="/etc/ld.so.cache" flags=0xa0000 mode=0x0 grep-12597 [002] d... 1903.694056: myprobe: (do_sys_open+0x0/0x208) filename="/etc/ld.so.cache" flags=0xa0000 mode=0x0 grep-12597 [002] d..1 1903.694103: myretprobe: (SyS_openat+0x3c/0x50 <- do_sys_open) arg1=0x3 ps-12596 [001] d..1 1903.694103: myretprobe: (SyS_openat+0x3c/0x50 <- do_sys_open) arg1=0x3 grep-12597 [002] d... 1903.694147: myprobe: (do_sys_open+0x0/0x208) filename="/lib/libm.so.6" flags=0xa0000 mode=0x0 ps-12596 [001] d... 1903.694148: myprobe: (do_sys_open+0x0/0x208) filename="/lib/libm.so.6" flags=0xa0000 mode=0x0 grep-12597 [002] d..1 1903.694179: myretprobe: (SyS_openat+0x3c/0x50 <- do_sys_open) arg1=0x3 ps-12596 [001] d..1 1903.694179: myretprobe: (SyS_openat+0x3c/0x50 <- do_sys_open) arg1=0x3 grep-12597 [002] d... 1903.694364: myprobe: (do_sys_open+0x0/0x208) filename="/lib/libresolv.so.2" flags=0xa0000 mode=0x0 ps-12596 [001] d... 1903.694381: myprobe: (do_sys_open+0x0/0x208) filename="/lib/libresolv.so.2" flags=0xa0000 mode=0x0 grep-12597 [002] d..1 1903.694398: myretprobe: (SyS_openat+0x3c/0x50 <- do_sys_open) arg1=0x3 ps-12596 [001] d..1 1903.694413: myretprobe: (SyS_openat+0x3c/0x50 <- do_sys_open) arg1=0x3
通過offset和類型打印，實現結構體內部成員的打印，但是需要知道寄存器和參數的對應關系和結構體成員的偏移。[13]提到了新的function_event機制，可以直接傳遞參數名。
-
獲取數據結構偏移的例子：打印
ip_rcv的網絡設備名和收發包數$ aarch64-linux-gnu-gdb vmlinux (gdb) ptype/o struct net_device /* offset | size */ type = struct net_device { /* 0 | 16 */ char name[16]; //設備名 /* 16 | 16 */ struct hlist_node { /* 16 | 8 */ struct hlist_node *next; /* 24 | 8 */ struct hlist_node **pprev; ... /* 272 | 184 */ struct net_device_stats { /* 272 | 8 */ unsigned long rx_packets; /* 280 | 8 */ unsigned long tx_packets; /* 288 | 8 */ unsigned long rx_bytes; (gdb) print (int)&((struct net_device *)0)->stats $3 = 272 cd /sys/kernel/debug/tracing/ echo 'p:net ip_rcv name=+0(%x1):string rx_pkts=+272(%x1):u64 tx_pkts=+280(%x1):u64 ' > kprobe_events echo 1 > events/kprobes/enable root@x3dvbj3-hynix2G-2666:/sys/kernel/debug/tracing# cat trace # tracer: nop # # _-----=> irqs-off # / _----=> need-resched # | / _---=> hardirq/softirq # || / _--=> preempt-depth # ||| / delay # TASK-PID CPU# |||| TIMESTAMP FUNCTION # | | | |||| | | <idle>-0 [000] d.s1 837.456030: net: (ip_rcv+0x0/0x3c8) name="eth0" rx_pkts=2445 tx_pkts=131 dropbear-2776 [000] d.s1 837.457538: net: (ip_rcv+0x0/0x3c8) name="eth0" rx_pkts=2446 tx_pkts=133 <idle>-0 [000] d.s1 837.662158: net: (ip_rcv+0x0/0x3c8) name="eth0" rx_pkts=2447 tx_pkts=133 <idle>-0 [000] d.s1 837.668020: net: (ip_rcv+0x0/0x3c8) name="eth0" rx_pkts=2448 tx_pkts=135 root@x3dvbj3-hynix2G-2666:/sys/kernel/debug/tracing# ifconfig eth0 Link encap:Ethernet HWaddr 00:70:64:7b:59:23 inet addr:192.168.1.10 Bcast:192.168.1.255 Mask:255.255.255.0 inet6 addr: fe80::270:64ff:fe7b:5923/64 Scope:Link UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 RX packets:2450 errors:0 dropped:0 overruns:0 frame:0 TX packets:136 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:1000 RX bytes:196959 (192.3 KiB) TX bytes:16262 (15.8 KiB) Interrupt:44 Base address:0x4000
2. systemtap
systemap 實現了一種腳本語言，把腳本翻譯成C代碼，編譯成kernel 模塊，注冊成kprobe handler，對kernel進行探測[12]。
由Redhat開發，在Redhat系統里運行比較穩定，其它環境易用性較差。
-
Systemtap腳本示例
# cat inode-watch.stp probe kernel.function ("vfs_write"), kernel.function ("vfs_read") { if (@defined($file->f_path->dentry)) { dev_nr = $file->f_path->dentry->d_inode->i_sb->s_dev inode_nr = $file->f_path->dentry->d_inode->i_ino } else { dev_nr = $file->f_dentry->d_inode->i_sb->s_dev inode_nr = $file->f_dentry->d_inode->i_ino } if (dev_nr == ($1 << 20 | $2) # major/minor device && inode_nr == $3) printf ("%s(%d) %s 0x%x/%u\n", execname(), pid(), ppfunc(), dev_nr, inode_nr) } # stat -c "%D %i" /etc/crontab fd03 133099 # stap inode-watch.stp 0xfd 3 133099 more(30789) vfs_read 0xfd00003/133099 more(30789) vfs_read 0xfd00003/133099
3. perf probe
perf 的probe命令提供了添加動態探測點的功能， 參看 kernel/tools/perf/Documentation/perf-probe.txt
236 EXAMPLES
237 --------
238 Display which lines in schedule() can be probed:
239
240 ./perf probe --line schedule
241
242 Add a probe on schedule() function 12th line with recording cpu local variable:
243
244 ./perf probe schedule:12 cpu
245 or
246 ./perf probe --add='schedule:12 cpu'
247
248 Add one or more probes which has the name start with "schedule".
249
250 ./perf probe schedule*
251 or
252 ./perf probe --add='schedule*'
253
254 Add probes on lines in schedule() function which calls update_rq_clock().
255
256 ./perf probe 'schedule;update_rq_clock*
4. eBPF
eBPF (Extended Berkeley Packet Filter)是最近比較流行的內核探測技術，通過在Kernel中運行一個虛擬機，執行用戶傳入的字節碼，實現對kernel內部信息的探測能力，比較安全易用。
其動態探測部分是基於kprobe和kretprobe實現的，可以實現對任意函數的探測、數據處理等能力。
Appendix
ARM64 BRK 指令
IMM16是自定義的，用於標識是哪種brk用途，比如kgdb或者kprobe。主要使用的幾個BRK立即數定義如下
#define FAULT_BRK_IMM 0x100
#define KGDB_DYN_DBG_BRK_IMM 0x400
#define KGDB_COMPILED_DBG_BRK_IMM 0x401
#define BUG_BRK_IMM 0x800
#define BRK64_ESR_KPROBES 0x0004
#define BRK64_ESR_UPROBES 0x0005
#define AARCH64_BREAK_MON 0xd4200000
#define BRK64_OPCODE_KPROBES (AARCH64_BREAK_MON | (BRK64_ESR_KPROBES << 5))
#define BRK64_OPCODE_UPROBES (AARCH64_BREAK_MON | (BRK64_ESR_UPROBES << 5))
#define AARCH64_BREAK_FAULT (AARCH64_BREAK_MON | (FAULT_BRK_IMM << 5))
#define AARCH64_BREAK_KGDB_DYN_DBG (AARCH64_BREAK_MON | (KGDB_DYN_DBG_BRK_IMM << 5))
ABI
寫kprobe驅動要知道函數參數和寄存器的對應關系，需要了解對應體系架構的ABI （Application Binary Interface）
X86 and X86_64 ABI :
1. User-level applications use as integer registers for passing the sequence
%rdi, %rsi, %rdx, %rcx, %r8 and %r9. The kernel interface uses %rdi,
%rsi, %rdx, %r10, %r8 and %r9.
2. A system-call is done via the syscall instruction. The kernel destroys
registers %rcx and %r11.
3. The number of the syscall has to be passed in register %rax.
4. System-calls are limited to six arguments, no argument is passed directly on
the stack.
5. Returning from the syscall, register %rax contains the result of the
system-call. A value in the range between -4095 and -1 indicates an error,
it is -errno.
6. Only values of class INTEGER or class MEMORY are passed to the kernel.
ARM64 ABI : ARM64 Parameters in general-purpose registers
Reference
- 2004, Kernel debugging with Kprobes, IBM
- 2005, An introduction to KProbes [LWN.net]
- 2006, Kprobes: User space probes support [LWN.net]
- 2009, tracing: kprobe-based event tracer [LWN.net]
- 2010, kprobes: Kprobes jump optimization support [LWN.net]
- 2012, ftrace, kprobes: Ftrace-based kprobe optimization [LWN.net]
- 2015, Early kprobe: enable kprobes at very early booting LWN.net Wang Nan
- 2015, arm64: Add kernel probes (kprobes) support [LWN.net]
- 2016, Kprobes Event Tracing on Armv8 - Linaro
- Kernel Probes (Kprobes) [kernel/Documentation]
- Kprobe-based Event Tracing
- https://sourceware.org/systemtap/tutorial.pdf
- https://events19.linuxfoundation.org/wp-content/uploads/2017/12/oss-eu-2018-fun-with-dynamic-trace-events_steven-rostedt.pdf
文章標題：Kernel調試追蹤技術之 Kprobe on ARM64
本文作者：hpyu
本文鏈接：https://www.cnblogs.com/hpyu/articles/14257305.html
歡迎轉載，請注明原文鏈接