Is it supposed to work on armv7 and 5.10.x kernel? about ply

[wkz]

To answer your initial question: Yes, I see no reason why that should not work.

My guess is that the script files are failing because they rely on the names of kernel symbols that may have changed. A good reminder for me to get rid of them, or convert them to use stable tracepoints instead. 😄

As for the do_sys_open script, that should definitely give you some output. I just copied it verbatim and ran it in my arm test sysroot and got the expected results (with a 5.5 kernel). Once I have the energy to update the test sysroots to a newer kernel I can have a look at this.

[RobertBerger]

In case you think that helps I can give you remote access to my system.
Just let me know how I can help.

[RobertBerger]

from above we can see that kretprobe works

Just for the fun of it, let's try something similar with kprobe:

# ply 'kprobe:vfs_read { printf("%v(%v)\n", comm, uid); }'
ply: active
sshd           (   0)
sshd           (   0)
sshd           (   0)
sh             (   0)
sh             (   0)
sh             (   0)
sshd           (   0)
sshd           (   0)
sh             (   0)
sshd           (   0)
sh             (   0)
cat            (   0)
cat            (   0)
sshd           (   0)
sshd           (   0)
sshd           (   0)
cat            (   0)
sshd           (   0)

seems to work as well

[RobertBerger]

tracepoint

# ply 'tracepoint:net/netif_receive_skb {printf("skbaddr=%p len=%u\n",data->skbaddr, data->len);}'
ply: active
skbaddr=0xc5ab5540 len=76
skbaddr=0xc5ab5540 len=88
skbaddr=0xc56f8a80 len=265
skbaddr=0xc5ab5900 len=168

[RobertBerger]

# cat count-syscalls.ply 
#!/usr/bin/env ply

kprobe:SyS_*
{
    @syscalls[caller] = count();
}

Maybe this strace helps?

# strace ply ./count-syscalls.ply
execve("/usr/sbin/ply", ["ply", "./count-syscalls.ply"], 0xbee72dc4 /* 14 vars */) = 0
brk(NULL)                               = 0x4d3000
mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6f72000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=100660, ...}) = 0
mmap2(NULL, 100660, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb6f31000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/libply.so.0", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\240[\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=93480, ...}) = 0
mmap2(NULL, 157616, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb6f0a000
mprotect(0xb6f1f000, 61440, PROT_NONE)  = 0
mmap2(0xb6f2e000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0xb6f2e000
close(3)                                = 0
openat(AT_FDCWD, "/lib/tls/v7l/neon/vfp/libc.so.6", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls/v7l/neon/vfp", 0xbe95e170) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/tls/v7l/neon/libc.so.6", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls/v7l/neon", 0xbe95e170) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/tls/v7l/vfp/libc.so.6", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls/v7l/vfp", 0xbe95e170)  = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/tls/v7l/libc.so.6", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls/v7l", 0xbe95e170)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/tls/neon/vfp/libc.so.6", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls/neon/vfp", 0xbe95e170) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/tls/neon/libc.so.6", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls/neon", 0xbe95e170)     = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/tls/vfp/libc.so.6", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls/vfp", 0xbe95e170)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/tls/libc.so.6", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls", 0xbe95e170)          = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/v7l/neon/vfp/libc.so.6", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/v7l/neon/vfp", 0xbe95e170) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/v7l/neon/libc.so.6", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/v7l/neon", 0xbe95e170)     = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/v7l/vfp/libc.so.6", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/v7l/vfp", 0xbe95e170)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/v7l/libc.so.6", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/v7l", 0xbe95e170)          = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/neon/vfp/libc.so.6", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/neon/vfp", 0xbe95e170)     = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/neon/libc.so.6", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/neon", 0xbe95e170)         = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/vfp/libc.so.6", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/vfp", 0xbe95e170)          = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/libc.so.6", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\3\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0I\305\4A4\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=948008, ...}) = 0
mmap2(NULL, 1014540, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb6e12000
mprotect(0xb6ef5000, 65536, PROT_NONE)  = 0
mmap2(0xb6f05000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xe3000) = 0xb6f05000
mmap2(0xb6f08000, 6924, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb6f08000
close(3)                                = 0
set_tls(0xb6f73100)                     = 0
mprotect(0xb6f05000, 4096, PROT_READ)   = 0
mprotect(0xb6f2e000, 4096, PROT_READ)   = 0
mprotect(0x4d1000, 4096, PROT_READ)     = 0
mprotect(0xb6f74000, 4096, PROT_READ)   = 0
munmap(0xb6f31000, 100660)              = 0
brk(NULL)                               = 0x4d3000
brk(0x4f4000)                           = 0x4f4000
clock_gettime64(CLOCK_MONOTONIC_RAW, {tv_sec=4058, tv_nsec=341889149}) = 0
clock_gettime64(CLOCK_REALTIME, {tv_sec=1610822915, tv_nsec=462357585}) = 0
clock_gettime64(CLOCK_REALTIME_COARSE, {tv_sec=1610822915, tv_nsec=787654291}) = 0
openat(AT_FDCWD, "/sys/devices/system/cpu/online", O_RDONLY|O_CLOEXEC) = 3
read(3, "0-3\n", 8192)                  = 4
close(3)                                = 0
access("./count-syscalls.ply", R_OK)    = 0
openat(AT_FDCWD, "./count-syscalls.ply", O_RDONLY) = 3
getpid()                                = 1032
stat64("/var/tmp/ply-ksyms", {st_mode=S_IFREG|0644, st_size=4297088, ...}) = 0
openat(AT_FDCWD, "/var/tmp/ply-ksyms", O_RDWR) = 4
mmap2(NULL, 4297088, PROT_READ|PROT_WRITE, MAP_SHARED, 4, 0) = 0xb69f8000
stat64("/proc", {st_mode=S_IFDIR|0555, st_size=0, ...}) = 0
stat64("/var/tmp/ply-ksyms", {st_mode=S_IFREG|0644, st_size=4297088, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0755, st_size=70, ...}) = 0
read(3, "#!/usr/bin/env ply\n\nkprobe:SyS_*"..., 8192) = 70
read(3, "", 4096)                       = 0
read(3, "", 8192)                       = 0
ugetrlimit(RLIMIT_MEMLOCK, {rlim_cur=64*1024, rlim_max=64*1024}) = 0
prlimit64(0, RLIMIT_MEMLOCK, {rlim_cur=RLIM64_INFINITY, rlim_max=RLIM64_INFINITY}, NULL) = 0
bpf(BPF_MAP_CREATE, {map_type=BPF_MAP_TYPE_HASH, key_size=4, value_size=4, max_entries=1024, map_flags=0, inner_map_fd=0, map_name="", map_ifindex=0, btf_fd=0, btf_key_type_id=0, btf_value_type_id=0, btf_vmlinux_value_type_id=0}, 120) = 5
bpf(BPF_PROG_LOAD, {prog_type=BPF_PROG_TYPE_KPROBE, insn_cnt=34, insns=0x4d5238, license="GPL", log_level=0, log_size=0, log_buf=NULL, kern_version=KERNEL_VERSION(5, 10, 0), prog_flags=0, prog_name="", prog_ifindex=0, expected_attach_type=BPF_CGROUP_INET_INGRESS, prog_btf_fd=0, func_info_rec_size=0, func_info=NULL, func_info_cnt=0, line_info_rec_size=0, line_info=NULL, line_info_cnt=0, attach_btf_id=0, attach_prog_fd=0}, 120) = 6
openat(AT_FDCWD, "/sys/kernel/debug/tracing/kprobe_events", O_RDWR|O_CREAT|O_APPEND, 0666) = 7
fstat64(7, {st_mode=S_IFREG|0644, st_size=0, ...}) = 0
openat(AT_FDCWD, "/sys/kernel/debug/tracing/events/ply1032", O_RDONLY|O_NONBLOCK|O_LARGEFILE|O_CLOEXEC|O_DIRECTORY) = -1 ENOENT (No such file or directory)
close(7)                                = 0
close(6)                                = 0
close(5)                                = 0
munmap(0xb69f8000, 4297088)             = 0
close(4)                                = 0
fstat64(1, {st_mode=S_IFCHR|0600, st_rdev=makedev(0xcf, 0x11), ...}) = 0
ioctl(1, TCGETS, {B115200 opost isig icanon echo ...}) = 0
write(1, "ERR:-22\n", 8ERR:-22
)                = 8
exit_group(1)                           = ?
+++ exited with 1 +++

[RobertBerger]

are those the correct settings?

# zcat /proc/config.gz | grep CONFIG_BPF              
CONFIG_BPF=y
CONFIG_BPF_SYSCALL=y
# CONFIG_BPF_JIT_ALWAYS_ON is not set
# CONFIG_BPF_PRELOAD is not set
# CONFIG_BPFILTER is not set
CONFIG_BPF_JIT=y
CONFIG_BPF_EVENTS=y

Maybe CONFIG_BPF_JIT=y does some nasty things. I saw a similar issue with CONFIG_BPF_JIT_ALWAYS_ON before.

[RobertBerger]

just for completeness some more possibly relevant kernel settings

# zcat /proc/config.gz | grep KRETPROBE
CONFIG_KRETPROBES=y
CONFIG_HAVE_KRETPROBES=y
root@multi-v7-ml:~/projects/ply-examples# zcat /proc/config.gz | grep KPROBE
CONFIG_KPROBES=y
CONFIG_HAVE_KPROBES=y
CONFIG_KPROBE_EVENTS=y
# CONFIG_KPROBE_EVENT_GEN_TEST is not set
# CONFIG_ARM_KPROBES_TEST is not set
# CONFIG_KPROBES_SANITY_TEST is not set
root@multi-v7-ml:~/projects/ply-examples# zcat /proc/config.gz | grep _PERF_
CONFIG_HAVE_PERF_EVENTS=y
CONFIG_PERF_USE_VMALLOC=y
CONFIG_PERF_EVENTS=y
# CONFIG_DEBUG_PERF_USE_VMALLOC is not set
CONFIG_HW_PERF_EVENTS=y
CONFIG_HAVE_PERF_REGS=y
CONFIG_HAVE_PERF_USER_STACK_DUMP=y

[RobertBerger]

Now with respect to the system calls. If I do something like this it seems better:

ply 'k:do_sys_!(perf_event_open) { @[caller] = count(); }'
ply: active
^Cply: deactivating

@:
{ do_sys_openat2 }: 70
{ do_sys_poll }: 237

cat /proc/kallsyms | grep _sys_open
c0383cc4 W compat_sys_open_by_handle_at
c05958b4 t do_sys_openat2
c0595bcc T do_sys_open
c0595c98 T __se_sys_open
c0595d60 T __se_sys_openat
c0595e2c T __se_sys_openat2
c05c94d0 T __se_sys_open_tree
c062d720 T __se_sys_open_by_handle_at
c06494a8 t proc_sys_open

# ply 'k:__se_sys!(perf_event_open) { @[caller] = count(); }'
ply: active
^Cply: deactivating

@:
{ __se_sys_clone }: 1
{ __se_sys_exit_group }: 1
{ __se_sys_newuname }: 1
{ __se_sys_sysinfo }: 1
{ __se_sys_pipe }: 1
{ __se_sys_dup2 }: 1
{ __se_sys_getsockname }: 1
{ __se_sys_sendto }: 1
{ __se_sys_recvfrom }: 1
{ __se_sys_wait4 }: 2
{ __se_sys_setpgid }: 2
{ __se_sys_getrlimit }: 2
{ __se_sys_munmap }: 2
{ __se_sys_execve }: 2
{ __se_sys_llseek }: 3
{ __se_sys_fcntl64 }: 4
{ __se_sys_access }: 8
{ __se_sys_pselect6_time32 }: 9
{ __se_sys_futex_time32 }: 10
{ __se_sys_mprotect }: 12
{ __se_sys_mmap_pgoff }: 14
{ __se_sys_close }: 14
{ __se_sys_brk }: 18
{ __se_sys_bpf }: 52
{ __se_sys_stat64 }: 52
{ __se_sys_rt_sigaction }: 78
{ __se_sys_poll }: 108
{ __se_sys_perf_event_open }: 168
{ __se_sys_openat }: 212
{ __se_sys_clock_nanosleep }: 224
{ __se_sys_ioctl }: 261
{ __se_sys_fstat64 }: 295
{ __se_sys_write }: 477
{ __se_sys_select }: 493
{ __se_sys_read }: 626
{ __se_sys_rt_sigprocmask }: 1184
{ __se_sys_clock_gettime }: 1935

# ply 'k:sys_open!(perf_event_open) { @[caller] = count(); }'
ply: active
^Cply: deactivating

@:
{ __se_sys_openat }: 400
root@multi-v7-ml:~/projects/ply-examples# ply 'k:open!(perf_event_open) { @[caller] = count(); }'
ply: active
^Cply: deactivating

@:
{ open_exec }: 7