LTTng » LTTng-tools

lttng-tools commit cfa9a5a2b4a96e0d6a9eeddd2622a6d7c173b7ac
lttng-ust commit 5b14aab38abe095cf5885aba2f0a1cdaed36729b
userspace rcu commit: 69ee3e06681039891a04b98efea87a34ce84c6cb

1. running lttng-sessiond as root, no verbose output.

1. launching short-lived apps in loops:

for a in $(seq 1 45); do (for n in $(seq 1 100); do (./TestApp_100perSecOnly 3 np > /dev/null &); sleep 1; done); done &

1. and just after that, in parallel:

for a in $(seq 1 10000); do lttng create; lttng enable-event -u -a; lttng start; lttng destroy; done

gets us, after a couple of minutes:

lttng-sessiond: ust-app.c:4586: add_event_ust_registry: Assertion `ua_chan' failed.
Aborted (core dumped)

info threads:

(gdb) info threads
  Id   Target Id         Frame 
  10   Thread 0x7f627b970700 (LWP 16556) 0x00007f627d1535ad in recvmsg () at ../sysdeps/unix/syscall-template.S:82
  9    Thread 0x7f627896a700 (LWP 16580) 0x00007f627ce8ae33 in *__GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=10) at ../sysdeps/unix/sysv/linux/poll.c:87
  8    Thread 0x7f627a16d700 (LWP 16559) 0x00007f627ce960d3 in epoll_wait ()
    at ../sysdeps/unix/syscall-template.S:82
  7    Thread 0x7f627a96e700 (LWP 16558) 0x00007f627ce960d3 in epoll_wait ()
    at ../sysdeps/unix/syscall-template.S:82
  6    Thread 0x7f627e5eb980 (LWP 16550) 0x00007f627d14ce75 in pthread_join (threadid=140060915054336, 
    thread_return=0x7ffff75476d0) at pthread_join.c:89
  5    Thread 0x7f627b16f700 (LWP 16557) syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:39
  4    Thread 0x7f627c171700 (LWP 16555) 0x00007f627ce960d3 in epoll_wait ()
    at ../sysdeps/unix/syscall-template.S:82
  3    Thread 0x7f6273fff700 (LWP 16922) 0x00007f627ce960d3 in epoll_wait ()
    at ../sysdeps/unix/syscall-template.S:82
  2    Thread 0x7f627916b700 (LWP 16561) 0x00007f627ce960d3 in epoll_wait ()
    at ../sysdeps/unix/syscall-template.S:82
* 1    Thread 0x7f627996c700 (LWP 16560) 0x00007f627cded475 in *__GI_raise (sig=<optimized out>)
    at ../nptl/sysdeps/unix/sysv/linux/raise.c:64

backtrace:

(gdb) bt full
#0  0x00007f627cded475 in *__GI_raise (sig=<optimized out>) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64
        pid = <optimized out>
        selftid = <optimized out>
#1  0x00007f627cdf06f0 in *__GI_abort () at abort.c:92
        act = {__sigaction_handler = {sa_handler = 0x444b64, sa_sigaction = 0x444b64}, sa_mask = {__val = {
              140060979657771, 140060923443008, 4586, 140060923443248, 140060978717510, 206158430232, 
              140060923443264, 140060923443040, 140060978615864, 206158430256, 140060923443288, 140060829900528, 
              140060829679616, 10, 4475748, 140737342897715}}, sa_flags = 2096127089, sa_restorer = 0x444b50}
        sigs = {__val = {32, 0 <repeats 15 times>}}
#2  0x00007f627cde6621 in *__GI___assert_fail (assertion=0x444b64 "ua_chan", file=<optimized out>, line=4586, 
    function=0x448350 "add_event_ust_registry") at assert.c:81
        buf = 0x7f6274035ef0 "lttng-sessiond: ust-app.c:4586: add_event_ust_registry: Assertion `ua_chan' failed.\n" 
#3  0x0000000000423a0b in add_event_ust_registry (sock=52, sobjd=5, cobjd=6, 
    name=0x7f627996b9b0 "com_ericsson_cba_trace_testapp100_hightraf:HundredPerSec", 
    sig=0x7f627403ac70 "const char*, my_Time, int, PID, int, OnePs , int, TenPs , int, HundredPs, float , unit_float, const char *, unit_string", nr_fields=7, fields=0x7f62740c0f40, loglevel=13, model_emf_uri=0x0) at ust-app.c:4586
        ret = 32610
        ret_code = 0
        event_id = 0
        chan_reg_key = 140060829920368
        app = 0x7f6274003e60
        ua_chan = 0x0
        ua_sess = 0x0
        registry = 0x0
        __func__ = "add_event_ust_registry" 
        __PRETTY_FUNCTION__ = "add_event_ust_registry" 
#4  0x0000000000423f07 in ust_app_recv_notify (sock=52) at ust-app.c:4678
        loglevel = 13
        sig = 0x7f627403ac70 "const char*, my_Time, int, PID, int, OnePs , int, TenPs , int, HundredPs, float , unit_float, const char *, unit_string" 
        model_emf_uri = 0x0
        sobjd = 5
        cobjd = 6
        name = "com_ericsson_cba_trace_testapp100_hightraf:HundredPerSec", '\000' <repeats 199 times>
        nr_fields = 7
        fields = 0x7f62740c0f40
        ret = 0
        cmd = USTCTL_NOTIFY_CMD_EVENT
        __func__ = "ust_app_recv_notify" 
        __PRETTY_FUNCTION__ = "ust_app_recv_notify" 
#5  0x0000000000425b77 in ust_thread_manage_notify (data=0x0) at ust-thread.c:134
        i = 0
        ret = 0
        pollfd = 52
        revents = 1
        nb_fd = 1
        events = {epfd = 22, nb_fd = 14, alloc_size = 16, init_size = 2, events = 0xd0e210}
        __func__ = "ust_thread_manage_notify" 
#6  0x00007f627d14bb50 in start_thread (arg=<optimized out>) at pthread_create.c:304
        __res = <optimized out>
        pd = 0x7f627996c700
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140060923447040, -1529329965889502832, 140060982075808, 
                140060923447744, 140061003857984, 3, 1441795450841317776, 1441785572462412176}, 
              mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, 
              canceltype = 0}}}
        not_first_call = <optimized out>
        freesize = <optimized out>
        __PRETTY_FUNCTION__ = "start_thread" 
#7  0x00007f627ce95a7d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
No locals.
#8  0x0000000000000000 in ?? ()
No symbol table info available.