I'm trying to evaluate the content of in and outgoing tcp packages with eBPF kprobes using libbpf. Therefore I attached a Kprobe to tcp_sendmsg and tcp_recvmsg. As of my knowledge, the package payload is in the second parameter struct msghdr *msg ->(struct iov_iter*) msg_iter ->(struct iovec*) iov ->(void*) iov_base.
when I try to read the length of the iovec (iov_length) it is 1, while msg_iter->count has the correct length of the payload.
Reading the payload-length from iov_base works, but the data then seems to be corrupt. I guess that the data has to be fully flushed into the iov_iter before reading.
How can I read the package-payload correctly inside a kprobe?
eBPF-kprobe on tcp_sendmsg to reproduce this behaviour:
#include"vmlinux.h"
#include<bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
const char fmt_info1[] = "segs: %d, offset: %d ,count:%d\n";
const char fmt_info2[] = "iov_length: %d, read_length: %d, data-readsuccess: %d\n";
const char fmt_notify_data[] = "iter_type: %d, Data: ";
const char fmt_data[] = "%02x ";
const char fmt_end[] = "\n";
unsigned char databuf[256];
SEC("kprobe/tcp_sendmsg")
int probe_tcp_sendmsg(struct pt_regs *ctx){
struct msghdr *msg = (struct msghdr *)PT_REGS_PARM2(ctx);
struct iov_iter *iter= &(msg->msg_iter);
struct iovec *iter_data = (struct iovec *)&(iter->iov);
int read_success=-1;
long unsigned int iov_seg_count;//count of iovec's
unsigned char iov_type;//0 is iter_iovec
unsigned long iov_count;//total data length in iter
unsigned long iov_length;//length of current iovec
unsigned long iov_offset;//iovec's to skip in iter
unsigned long read_length;//final length in the data buffer
read_success=bpf_probe_read(&iov_type, sizeof(unsigned char), &(iter->iter_type));
if(read_success!=0) {
return 0;
}
read_success=bpf_probe_read(&iov_seg_count, sizeof(long unsigned int), &(iter->nr_segs));
if(read_success!=0) {
return 0;
}
read_success=bpf_probe_read(&iov_count, sizeof(unsigned long), &(iter->count));
if(read_success!=0) {
return 0;
}
read_success=bpf_probe_read(&iov_offset, sizeof(unsigned long), &(iter->iov_offset));
if(read_success!=0) {
return 0;
}
void* startaddress;
read_success=bpf_probe_read(&startaddress, sizeof(void*), &(iter_data->iov_base));
if(read_success!=0){
return 0;
}
read_success=bpf_probe_read(&iov_length, sizeof(size_t), &(iter_data->iov_len));
if(read_success!=0){
return 0;
}
if(iov_length< 256){//change against iov_count
read_length=iov_length;//change against iov_count
}else{
read_length=256;
}
read_success=bpf_probe_read(databuf, read_length, startaddress);
bpf_trace_printk(fmt_info1, sizeof(fmt_info1),iov_seg_count, iov_offset, iov_count);
bpf_trace_printk(fmt_info2, sizeof(fmt_info2),iov_length, read_length, read_success);
bpf_trace_printk(fmt_notify_data, sizeof(fmt_notify_data), iov_type);
for(unsigned long i=0;i<read_length;++i){
bpf_trace_printk(fmt_data,sizeof(fmt_data), databuf[i]);
}
bpf_trace_printk(fmt_end, sizeof(fmt_end));
return 0;
}
char LICENSE[] SEC("license") = "Dual BSD/GPL";
vmlinux.h is created with:
bpftool btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h
Then the ebpf programm is compiled with:
clang -g -O3 -target bpf -D__TARGET_ARCH_x86 -c bpf_ip_tcp.c -o bpf_ip_tcp.o
Then the skelleton is created with:
bpftool gen skeleton bpf_ip_tcp.o name bpf_ip_tcp > bpf_ip_tcp.h
Then the loader is compiled with:
clang -Wall -Wextra -g3 main.c -o ebpf-example -lbpf
My loader looks like this:
#include <sys/resource.h>
#include "bpf_ip_tcp.h"
int main(){
struct rlimit rlim_new = {
.rlim_cur = RLIM_INFINITY,
.rlim_max = RLIM_INFINITY,
};
if (setrlimit(RLIMIT_MEMLOCK, &rlim_new)) {
exit(1);
}
struct bpf_ip_tcp *skel = bpf_ip_tcp__open();
bpf_ip_tcp__load(skel);
bpf_ip_tcp__attach(skel);
while(1==1){};
return 0;
}
Results:
Output in cat /sys/kernel/tracing/trace_pipe while using wget:
...
wget-22674 [006] d...1 46410.754999: bpf_trace_printk: segs: 1, offset: 0 ,count:400
wget-22674 [006] d...1 46410.755059: bpf_trace_printk: iov_length: 1, read_length: 1, data-readsuccess: 0
wget-22674 [006] d...1 46410.755061: bpf_trace_printk: iter_type: 0, Data:
wget-22674 [006] d...1 46410.755061: bpf_trace_printk: 40
...
Expected is either multiple segments or iov_length equal to count.
System: Ubuntu 22.04 LTS, Kernel: 5.15.0 .
