Project

General

Profile

Bug #546 » gdb_printous.log

gdb printout of relayd, sessiond and consumerd when hanging occurs - Tan le tran, 05/31/2013 10:58 AM

 
[SC-1:Node16 Fri May 31 08:09:31/cluster/dumps/may31_backup] # dir
total 66748
-rw------- 1 root root 78225408 May 31 01:59 lttng-sessiond.542.SC-2.core
-rw------- 1 root root 34070528 May 31 01:59 lttng-relayd.10773.SC-2.core
-rw------- 1 root root 124252160 May 31 01:59 lttng-consumerd.989.SC-2.core
[SC-1:Node16 Fri May 31 08:09:32/cluster/dumps/may31_backup] #
[SC-1:Node16 Fri May 31 08:09:33/cluster/dumps/may31_backup] #
[SC-1:Node16 Fri May 31 08:09:33/cluster/dumps/may31_backup] #
[SC-1:Node16 Fri May 31 08:09:33/cluster/dumps/may31_backup] #
[SC-1:Node16 Fri May 31 08:09:34/cluster/dumps/may31_backup] # which relayd
which: no relayd in (/sbin:/usr/sbin:/usr/local/sbin:/root/bin:/usr/local/bin:/usr/bin:/bin:/usr/X11R6/bin:/usr/games:/opt/coremw/bin:/usr/lib/mit/bin:/usr/lib/mit/sbin:/opt/trace/bin/)
[SC-1:Node16 Fri May 31 08:09:43/cluster/dumps/may31_backup] # which lttng-relayd
/usr/bin/lttng-relayd
[SC-1:Node16 Fri May 31 08:10:02/cluster/dumps/may31_backup] # gdb /usr/bin/lttng-relayd lttng-relayd.10773.SC-2.core
GNU gdb (GDB) SUSE (7.3-0.6.1)
Copyright (C) 2011 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-suse-linux".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /usr/bin/lttng-relayd...done.
[New LWP 10773]
[New LWP 10774]
[New LWP 10775]
[New LWP 10776]
[New LWP 10795]
Missing separate debuginfo for /lib64/librt.so.1
Try: zypper install -C "debuginfo(build-id)=44612b93c19e6567318299411987b113d2387081"
Missing separate debuginfo for /lib64/libuuid.so.1
Try: zypper install -C "debuginfo(build-id)=f9998d18c497b2047b48e0702daa82204e2a944d"
Missing separate debuginfo for /lib64/libpopt.so.0
Try: zypper install -C "debuginfo(build-id)=895e7fadb3a3371343391d34280543ea4df24a03"
Missing separate debuginfo for /lib64/libpthread.so.0
Try: zypper install -C "debuginfo(build-id)=09dae90d04b1e2e43758ce58845f026b4085aec9"
Missing separate debuginfo for /lib64/libc.so.6
Try: zypper install -C "debuginfo(build-id)=2f3040b47ab477519b3045352432f77c5bec1561"
Missing separate debuginfo for /lib64/ld-linux-x86-64.so.2
Try: zypper install -C "debuginfo(build-id)=181860a35c8e9a0456dd6675f85c2eb0f062e956"
Missing separate debuginfo for
Try: zypper install -C "debuginfo(build-id)=31b3276dc0b305d25a213860d7d5a67601b1bb7a"
Missing separate debuginfo for /lib64/librt.so.1
Try: zypper install -C "debuginfo(build-id)=44612b93c19e6567318299411987b113d2387081"
Missing separate debuginfo for /lib64/libuuid.so.1
Try: zypper install -C "debuginfo(build-id)=f9998d18c497b2047b48e0702daa82204e2a944d"
Missing separate debuginfo for /lib64/libpopt.so.0
Try: zypper install -C "debuginfo(build-id)=895e7fadb3a3371343391d34280543ea4df24a03"
Missing separate debuginfo for /lib64/libpthread.so.0
Try: zypper install -C "debuginfo(build-id)=09dae90d04b1e2e43758ce58845f026b4085aec9"
[Thread debugging using libthread_db enabled]
Missing separate debuginfo for /lib64/libc.so.6
Try: zypper install -C "debuginfo(build-id)=2f3040b47ab477519b3045352432f77c5bec1561"
Missing separate debuginfo for /lib64/ld-linux-x86-64.so.2
Try: zypper install -C "debuginfo(build-id)=181860a35c8e9a0456dd6675f85c2eb0f062e956"
Core was generated by `lttng-relayd -C tcp://192.168.0.2:55804 -D tcp://192.168.0.2:58751 -o /storage/'.
Program terminated with signal 6, Aborted.
#0 0x00007f2bc540cf95 in pthread_join () from /lib64/libpthread.so.0
(gdb) bt
#0 0x00007f2bc540cf95 in pthread_join () from /lib64/libpthread.so.0
#1 0x0000000000407fdc in main (argc=7, argv=0x7fffcfaf70b8) at main.c:2142
(gdb) bt full
#0 0x00007f2bc540cf95 in pthread_join () from /lib64/libpthread.so.0
No symbol table info available.
#1 0x0000000000407fdc in main (argc=7, argv=0x7fffcfaf70b8) at main.c:2142
ret = 0
status = 0x40de00
__func__ = "main"
(gdb) thread apply all bt full

Thread 5 (Thread 0x7f2bc388a700 (LWP 10795)):
#0 0x00007f2bc5164679 in syscall () from /lib64/libc.so.6
No symbol table info available.
#1 0x00007f2bc5a32669 in call_rcu_wait (crdp=0x61d170) at urcu-call-rcu-impl.h:215
No locals.
#2 0x00007f2bc5a32bef in call_rcu_thread (arg=0x61d170) at urcu-call-rcu-impl.h:302
cbcount = 9
cbs = 0x0
cbs_tail = 0x61bef0
crdp = 0x61d170
rhp = 0x61bef0
rt = 0
ret = 0
__func__ = "call_rcu_thread"
#3 0x00007f2bc540c7b6 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#4 0x00007f2bc5167c5d in clone () from /lib64/libc.so.6
No symbol table info available.
#5 0x0000000000000000 in ?? ()
No symbol table info available.

Thread 4 (Thread 0x7f2bc408b700 (LWP 10776)):
#0 0x00007f2bc51682e3 in epoll_wait () from /lib64/libc.so.6
No symbol table info available.
#1 0x000000000040db79 in compat_epoll_wait (events=0x7f2bc408ac30, timeout=-1) at compat-epoll.c:223
ret = 6378976
new_size = 0
__func__ = "compat_epoll_wait"
#2 0x0000000000403723 in relay_thread_listener (data=0x0) at main.c:519
i = 1
ret = 0
pollfd = 8
err = -1
val = 1
revents = 1
nb_fd = 1
events = {epfd = 10, nb_fd = 3, alloc_size = 3, init_size = 3, events = 0x61a120}
control_sock = 0x61a510
data_sock = 0x61a550
__func__ = "relay_thread_listener"
__PRETTY_FUNCTION__ = "relay_thread_listener"
#3 0x00007f2bc540c7b6 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#4 0x00007f2bc5167c5d in clone () from /lib64/libc.so.6
No symbol table info available.
#5 0x0000000000000000 in ?? ()
No symbol table info available.

Thread 3 (Thread 0x7f2bc488c700 (LWP 10775)):
#0 0x00007f2bc51682e3 in epoll_wait () from /lib64/libc.so.6
No symbol table info available.
#1 0x000000000040db79 in compat_epoll_wait (events=0x7f2bc488bc20, timeout=-1) at compat-epoll.c:223
ret = 0
new_size = 4
__func__ = "compat_epoll_wait"
#2 0x0000000000407301 in relay_thread_worker (data=0x0) at main.c:1809
idx = -1
i = 2
seen_control = 0
---Type <return> to continue, or q <return> to quit---
last_notdel_data_fd = -1
ret = 0
err = -1
last_seen_data_fd = -1
nb_fd = 2
relay_connection = 0x61beb0
events = {epfd = 9, nb_fd = 3, alloc_size = 4, init_size = 2, events = 0x620f70}
relay_connections_ht = 0x616010
node = 0x0
iter = {iter = {node = 0x0, next = 0x0}}
streams_ht = 0x61b070
recv_hdr = {circuit_id = 0, data_size = 1152921504606846976, cmd = 117440512, cmd_version = 0}
__func__ = "relay_thread_worker"
#3 0x00007f2bc540c7b6 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#4 0x00007f2bc5167c5d in clone () from /lib64/libc.so.6
No symbol table info available.
#5 0x0000000000000000 in ?? ()
No symbol table info available.

Thread 2 (Thread 0x7f2bc508d700 (LWP 10774)):
#0 0x00007f2bc5164679 in syscall () from /lib64/libc.so.6
No symbol table info available.
#1 0x000000000040c3c5 in futex_nto1_wait (futex=0x6155e0) at futex.c:87
__func__ = "futex_nto1_wait"
#2 0x00000000004040da in relay_thread_dispatcher (data=0x0) at main.c:680
ret = 96
node = 0x0
relay_cmd = 0x620ec0
__func__ = "relay_thread_dispatcher"
#3 0x00007f2bc540c7b6 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#4 0x00007f2bc5167c5d in clone () from /lib64/libc.so.6
No symbol table info available.
#5 0x0000000000000000 in ?? ()
No symbol table info available.

Thread 1 (Thread 0x7f2bc647e900 (LWP 10773)):
#0 0x00007f2bc540cf95 in pthread_join () from /lib64/libpthread.so.0
No symbol table info available.
#1 0x0000000000407fdc in main (argc=7, argv=0x7fffcfaf70b8) at main.c:2142
ret = 0
status = 0x40de00
__func__ = "main"
(gdb)
(gdb) quit
[SC-1:Node16 Fri May 31 08:11:41/cluster/dumps/may31_backup] #
[SC-1:Node16 Fri May 31 08:11:41/cluster/dumps/may31_backup] #
[SC-1:Node16 Fri May 31 08:11:42/cluster/dumps/may31_backup] #
[SC-1:Node16 Fri May 31 08:11:42/cluster/dumps/may31_backup] #
[SC-1:Node16 Fri May 31 08:11:42/cluster/dumps/may31_backup] # dir
total 66748
-rw------- 1 root root 78225408 May 31 01:59 lttng-sessiond.542.SC-2.core
-rw------- 1 root root 34070528 May 31 01:59 lttng-relayd.10773.SC-2.core
-rw------- 1 root root 124252160 May 31 01:59 lttng-consumerd.989.SC-2.core
[SC-1:Node16 Fri May 31 08:11:43/cluster/dumps/may31_backup] #
[SC-1:Node16 Fri May 31 08:11:44/cluster/dumps/may31_backup] #
[SC-1:Node16 Fri May 31 08:11:44/cluster/dumps/may31_backup] #
[SC-1:Node16 Fri May 31 08:11:44/cluster/dumps/may31_backup] # which lttng-sessiond
/usr/bin/lttng-sessiond
[SC-1:Node16 Fri May 31 08:11:55/cluster/dumps/may31_backup] #
[SC-1:Node16 Fri May 31 08:11:56/cluster/dumps/may31_backup] # gdb /usr/bin/lttng-sessiond lttng-sessiond.542.SC-2.core
GNU gdb (GDB) SUSE (7.3-0.6.1)
Copyright (C) 2011 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-suse-linux".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /usr/bin/lttng-sessiond...done.
[New LWP 542]
[New LWP 546]
[New LWP 1009]
Missing separate debuginfo for /lib64/librt.so.1
Try: zypper install -C "debuginfo(build-id)=44612b93c19e6567318299411987b113d2387081"
Missing separate debuginfo for /lib64/libuuid.so.1
Try: zypper install -C "debuginfo(build-id)=f9998d18c497b2047b48e0702daa82204e2a944d"
Missing separate debuginfo for /lib64/libdl.so.2
Try: zypper install -C "debuginfo(build-id)=3e4f6bfee9fdf77ca975b77b8c325347d9228bb8"
Missing separate debuginfo for /lib64/libpopt.so.0
Try: zypper install -C "debuginfo(build-id)=895e7fadb3a3371343391d34280543ea4df24a03"
Missing separate debuginfo for /lib64/libpthread.so.0
Try: zypper install -C "debuginfo(build-id)=09dae90d04b1e2e43758ce58845f026b4085aec9"
Missing separate debuginfo for /lib64/libc.so.6
Try: zypper install -C "debuginfo(build-id)=2f3040b47ab477519b3045352432f77c5bec1561"
Missing separate debuginfo for /lib64/ld-linux-x86-64.so.2
Try: zypper install -C "debuginfo(build-id)=181860a35c8e9a0456dd6675f85c2eb0f062e956"
Missing separate debuginfo for
Try: zypper install -C "debuginfo(build-id)=31b3276dc0b305d25a213860d7d5a67601b1bb7a"
Missing separate debuginfo for /lib64/librt.so.1
Try: zypper install -C "debuginfo(build-id)=44612b93c19e6567318299411987b113d2387081"
Missing separate debuginfo for /lib64/libuuid.so.1
Try: zypper install -C "debuginfo(build-id)=f9998d18c497b2047b48e0702daa82204e2a944d"
Missing separate debuginfo for /lib64/libdl.so.2
Try: zypper install -C "debuginfo(build-id)=3e4f6bfee9fdf77ca975b77b8c325347d9228bb8"
Missing separate debuginfo for /lib64/libpopt.so.0
Try: zypper install -C "debuginfo(build-id)=895e7fadb3a3371343391d34280543ea4df24a03"
Missing separate debuginfo for /lib64/libpthread.so.0
Try: zypper install -C "debuginfo(build-id)=09dae90d04b1e2e43758ce58845f026b4085aec9"
[Thread debugging using libthread_db enabled]
Missing separate debuginfo for /lib64/libc.so.6
Try: zypper install -C "debuginfo(build-id)=2f3040b47ab477519b3045352432f77c5bec1561"
Missing separate debuginfo for /lib64/ld-linux-x86-64.so.2
Try: zypper install -C "debuginfo(build-id)=181860a35c8e9a0456dd6675f85c2eb0f062e956"
Core was generated by `lttng-sessiond -d'.
Program terminated with signal 6, Aborted.
#0 0x00007f781c63bf95 in pthread_join () from /lib64/libpthread.so.0
(gdb) bt
#0 0x00007f781c63bf95 in pthread_join () from /lib64/libpthread.so.0
#1 0x000000000042fbdd in main (argc=2, argv=0x7fffc44cea88) at main.c:4421
(gdb) bt full
#0 0x00007f781c63bf95 in pthread_join () from /lib64/libpthread.so.0
No symbol table info available.
#1 0x000000000042fbdd in main (argc=2, argv=0x7fffc44cea88) at main.c:4421
ret = 0
status = 0x0
home_path = 0x7f781c2f1e55 "H\211\302H\213\005\361\300\063"
env_app_timeout = 0x0
__func__ = "main"
(gdb) thread apply all bt full

Thread 3 (Thread 0x7f78137fe700 (LWP 1009)):
#0 0x00007f781c393679 in syscall () from /lib64/libc.so.6
No symbol table info available.
#1 0x00007f781d0aa669 in call_rcu_wait (crdp=0x68e630) at urcu-call-rcu-impl.h:215
No locals.
#2 0x00007f781d0aabef in call_rcu_thread (arg=0x68e630) at urcu-call-rcu-impl.h:302
cbcount = 1
cbs = 0x0
cbs_tail = 0x6d7c80
crdp = 0x68e630
rhp = 0x6d7c80
rt = 0
ret = 0
__func__ = "call_rcu_thread"
#3 0x00007f781c63b7b6 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#4 0x00007f781c396c5d in clone () from /lib64/libc.so.6
No symbol table info available.
#5 0x0000000000000000 in ?? ()
No symbol table info available.

Thread 2 (Thread 0x7f781b6a9700 (LWP 546)):
#0 0x00007f781c642aad in connect () from /lib64/libpthread.so.0
No symbol table info available.
#1 0x0000000000431d47 in lttcomm_connect_inet_sock (sock=0x721ce0) at inet.c:102
ret = 0
closeret = 0
__func__ = "lttcomm_connect_inet_sock"
#2 0x0000000000438d94 in relayd_connect (rsock=0x721ce0) at relayd.c:357
__PRETTY_FUNCTION__ = "relayd_connect"
__func__ = "relayd_connect"
#3 0x0000000000411593 in create_connect_relayd (uri=0x6d42f4, relayd_sock=0x7f781b6a8848) at cmd.c:506
ret = 0
rsock = 0x721ce0
__func__ = "create_connect_relayd"
#4 0x00000000004118e6 in send_consumer_relayd_socket (domain=2, session=0x72c2c0, relayd_uri=0x6d42f4, consumer=0x6d12c0, consumer_sock=0x73a850) at cmd.c:557
ret = 10
rsock = 0x0
__PRETTY_FUNCTION__ = "send_consumer_relayd_socket"
#5 0x0000000000411ad8 in send_consumer_relayd_sockets (domain=2, session=0x72c2c0, consumer=0x6d12c0, sock=0x73a850) at cmd.c:642
ret = 10
__PRETTY_FUNCTION__ = "send_consumer_relayd_sockets"
#6 0x0000000000411c5f in cmd_setup_relayd (session=0x72c2c0) at cmd.c:684
ret = 10
usess = 0x6bb6a0
ksess = 0x0
socket = 0x73a850
iter = {iter = {node = 0x73a878, next = 0x67e930}}
__PRETTY_FUNCTION__ = "cmd_setup_relayd"
__func__ = "cmd_setup_relayd"
#7 0x000000000042b440 in process_client_msg (cmd_ctx=0x6ec940, sock=31, sock_error=0x7f781b6a8b58) at main.c:2656
ret = 0
need_tracing_session = 1
need_domain = 1
__func__ = "process_client_msg"
#8 0x000000000042d14a in thread_manage_clients (data=0x0) at main.c:3457
sock = 31
ret = 8780
i = 1
---Type <return> to continue, or q <return> to quit---
pollfd = 8
err = -1
sock_error = 0
revents = 1
nb_fd = 1
cmd_ctx = 0x6ec940
events = {epfd = 17, nb_fd = 2, alloc_size = 2, init_size = 2, events = 0x6682a0}
__func__ = "thread_manage_clients"
#9 0x00007f781c63b7b6 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#10 0x00007f781c396c5d in clone () from /lib64/libc.so.6
No symbol table info available.
#11 0x0000000000000000 in ?? ()
No symbol table info available.

Thread 1 (Thread 0x7f781daf5a00 (LWP 542)):
#0 0x00007f781c63bf95 in pthread_join () from /lib64/libpthread.so.0
No symbol table info available.
#1 0x000000000042fbdd in main (argc=2, argv=0x7fffc44cea88) at main.c:4421
ret = 0
status = 0x0
home_path = 0x7f781c2f1e55 "H\211\302H\213\005\361\300\063"
env_app_timeout = 0x0
__func__ = "main"
(gdb)
(gdb)
(gdb) quit
[SC-1:Node16 Fri May 31 08:12:30/cluster/dumps/may31_backup] #
[SC-1:Node16 Fri May 31 08:12:57/cluster/dumps/may31_backup] #
[SC-1:Node16 Fri May 31 08:12:57/cluster/dumps/may31_backup] # dir
total 66748
-rw------- 1 root root 78225408 May 31 01:59 lttng-sessiond.542.SC-2.core
-rw------- 1 root root 34070528 May 31 01:59 lttng-relayd.10773.SC-2.core
-rw------- 1 root root 124252160 May 31 01:59 lttng-consumerd.989.SC-2.core
[SC-1:Node16 Fri May 31 08:13:04/cluster/dumps/may31_backup] #
[SC-1:Node16 Fri May 31 08:13:04/cluster/dumps/may31_backup] # gdp /usr/lib64/lttng/libexec/lttng-consumerd lttng-consumerd.989.SC-2.core
-bash: gdp: command not found
[SC-1:Node16 Fri May 31 08:13:34/cluster/dumps/may31_backup] # gdb /usr/lib64/lttng/libexec/lttng-consumerd lttng-consumerd.989.SC-2.core
GNU gdb (GDB) SUSE (7.3-0.6.1)
Copyright (C) 2011 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-suse-linux".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /usr/lib64/lttng/libexec/lttng-consumerd...done.
[New LWP 989]
[New LWP 991]
[New LWP 992]
[New LWP 995]
[New LWP 993]
[New LWP 1010]
[New LWP 1388]
Missing separate debuginfo for /lib64/libuuid.so.1
Try: zypper install -C "debuginfo(build-id)=f9998d18c497b2047b48e0702daa82204e2a944d"
Missing separate debuginfo for /lib64/librt.so.1
Try: zypper install -C "debuginfo(build-id)=44612b93c19e6567318299411987b113d2387081"
Missing separate debuginfo for /lib64/libpopt.so.0
Try: zypper install -C "debuginfo(build-id)=895e7fadb3a3371343391d34280543ea4df24a03"
Missing separate debuginfo for /lib64/libpthread.so.0
Try: zypper install -C "debuginfo(build-id)=09dae90d04b1e2e43758ce58845f026b4085aec9"
Missing separate debuginfo for /lib64/libc.so.6
Try: zypper install -C "debuginfo(build-id)=2f3040b47ab477519b3045352432f77c5bec1561"
Missing separate debuginfo for /lib64/ld-linux-x86-64.so.2
Try: zypper install -C "debuginfo(build-id)=181860a35c8e9a0456dd6675f85c2eb0f062e956"
Missing separate debuginfo for
Try: zypper install -C "debuginfo(build-id)=31b3276dc0b305d25a213860d7d5a67601b1bb7a"
Missing separate debuginfo for /lib64/libuuid.so.1
Try: zypper install -C "debuginfo(build-id)=f9998d18c497b2047b48e0702daa82204e2a944d"
Missing separate debuginfo for /lib64/librt.so.1
Try: zypper install -C "debuginfo(build-id)=44612b93c19e6567318299411987b113d2387081"
Missing separate debuginfo for /lib64/libpopt.so.0
Try: zypper install -C "debuginfo(build-id)=895e7fadb3a3371343391d34280543ea4df24a03"
Missing separate debuginfo for /lib64/libpthread.so.0
Try: zypper install -C "debuginfo(build-id)=09dae90d04b1e2e43758ce58845f026b4085aec9"
[Thread debugging using libthread_db enabled]
Missing separate debuginfo for /lib64/libc.so.6
Try: zypper install -C "debuginfo(build-id)=2f3040b47ab477519b3045352432f77c5bec1561"
Missing separate debuginfo for /lib64/ld-linux-x86-64.so.2
Try: zypper install -C "debuginfo(build-id)=181860a35c8e9a0456dd6675f85c2eb0f062e956"
Core was generated by `lttng-consumerd --quiet -u --consumerd-cmd-sock /var/run/lttng/ustconsumerd64/c'.
Program terminated with signal 6, Aborted.
#0 0x00007f04a6272f95 in pthread_join () from /lib64/libpthread.so.0
(gdb) bt
#0 0x00007f04a6272f95 in pthread_join () from /lib64/libpthread.so.0
#1 0x0000000000404224 in main (argc=7, argv=0x7fffba08b4e8) at lttng-consumerd.c:444
(gdb) bt full
#0 0x00007f04a6272f95 in pthread_join () from /lib64/libpthread.so.0
No symbol table info available.
#1 0x0000000000404224 in main (argc=7, argv=0x7fffba08b4e8) at lttng-consumerd.c:444
ret = 0
status = 0x0
__func__ = "main"
(gdb) thread apply all bt full

Thread 7 (Thread 0x7f04a2343700 (LWP 1388)):
#0 0x00007f04a5f27abd in do_sigwaitinfo () from /lib64/libc.so.6
No symbol table info available.
#1 0x00007f04a5f27b8d in sigwaitinfo () from /lib64/libc.so.6
No symbol table info available.
#2 0x00007f04a66b96a0 in sig_thread (arg=0x0) at ring_buffer_frontend.c:403
mask = {__val = {60129542144, 0 <repeats 15 times>}}
info = {si_signo = 36, si_errno = 0, si_code = 0, _sifields = {_pad = {989, 0, -1677667600, 32516, 0 <repeats 24 times>}, _kill = {si_pid = 989, si_uid = 0}, _timer = {si_tid = 989,
si_overrun = 0, si_sigval = {sival_int = -1677667600, sival_ptr = 0x7f049c00d2f0}}, _rt = {si_pid = 989, si_uid = 0, si_sigval = {sival_int = -1677667600, sival_ptr = 0x7f049c00d2f0}},
_sigchld = {si_pid = 989, si_uid = 0, si_status = -1677667600, si_utime = 0, si_stime = 0}, _sigfault = {si_addr = 0x3dd}, _sigpoll = {si_band = 989, si_fd = -1677667600}}}
signr = 36
__func__ = "sig_thread"
#3 0x00007f04a62727b6 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#4 0x00007f04a5fcdc5d in clone () from /lib64/libc.so.6
No symbol table info available.
#5 0x0000000000000000 in ?? ()
No symbol table info available.

Thread 6 (Thread 0x7f04a2b44700 (LWP 1010)):
#0 0x00007f04a5fca679 in syscall () from /lib64/libc.so.6
No symbol table info available.
#1 0x00007f04a6ce5d2d in wait_gp () at urcu.c:214
No locals.
#2 0x00007f04a6ce5eb8 in update_counter_and_wait () at urcu.c:269
qsreaders = {next = 0x7f04a5ae1680, prev = 0x7f04a2b44680}
wait_loops = 100
index = 0x7f04a6eea0c0
tmp = 0x7f04a4adf600
#3 0x00007f04a6ce605c in synchronize_rcu_mb () at urcu.c:321
No locals.
#4 0x00007f04a6ce6a50 in call_rcu_thread (arg=0x636d70) at urcu-call-rcu-impl.h:279
cbcount = 16
cbs = 0x7f049c00bdf8
cbs_tail = 0x7f049c004f88
crdp = 0x636d70
rhp = 0x7f049c021d38
rt = 0
ret = 0
__func__ = "call_rcu_thread"
#5 0x00007f04a62727b6 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#6 0x00007f04a5fcdc5d in clone () from /lib64/libc.so.6
No symbol table info available.
#7 0x0000000000000000 in ?? ()
No symbol table info available.

Thread 5 (Thread 0x7f04a4adf700 (LWP 993)):
#0 0x00007f04a627970d in write () from /lib64/libpthread.so.0
No symbol table info available.
#1 0x000000000040745b in lttng_consumer_on_read_subbuffer_mmap (ctx=0x634d10, stream=0x7f049c004290, len=11878, padding=410) at consumer.c:1426
mmap_offset = 0
mmap_base = 0x7f04a724d000
ret = 24
written = 0
orig_offset = 0
outfd = 24
relayd = 0x7f049c00edb0
relayd_hang_up = 0
---Type <return> to continue, or q <return> to quit---
__PRETTY_FUNCTION__ = "lttng_consumer_on_read_subbuffer_mmap"
__func__ = "lttng_consumer_on_read_subbuffer_mmap"
#2 0x000000000041c959 in lttng_ustconsumer_read_subbuffer (stream=0x7f049c004290, ctx=0x634d10) at ust-consumer.c:1308
len = 12288
subbuf_size = 11878
padding = 410
err = 0
ret = 0
dummy = 0 '\000'
ustream = 0x7f049c004250
__PRETTY_FUNCTION__ = "lttng_ustconsumer_read_subbuffer"
__func__ = "lttng_ustconsumer_read_subbuffer"
#3 0x000000000040bf2e in lttng_consumer_read_subbuffer (stream=0x7f049c004290, ctx=0x634d10) at consumer.c:2999
ret = 11825
__PRETTY_FUNCTION__ = "lttng_consumer_read_subbuffer"
#4 0x000000000040a2c9 in consumer_thread_data_poll (data=0x634d10) at consumer.c:2473
num_rdy = 11
num_hup = 0
high_prio = 0
ret = 108
i = 100
pollfd = 0x63ede0
local_stream = 0x640160
new_stream = 0x0
nb_fd = 108
ctx = 0x634d10
len = 11825
__func__ = "consumer_thread_data_poll"
#5 0x00007f04a62727b6 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#6 0x00007f04a5fcdc5d in clone () from /lib64/libc.so.6
No symbol table info available.
#7 0x0000000000000000 in ?? ()
No symbol table info available.

Thread 4 (Thread 0x7f04a3add700 (LWP 995)):
#0 0x00007f04a5f27abd in do_sigwaitinfo () from /lib64/libc.so.6
No symbol table info available.
#1 0x00007f04a5f27b8d in sigwaitinfo () from /lib64/libc.so.6
No symbol table info available.
#2 0x000000000040d418 in consumer_timer_metadata_thread (data=0x634d10) at consumer-timer.c:232
signr = 45
mask = {__val = {26388279066624, 0 <repeats 15 times>}}
info = {si_signo = 45, si_errno = 0, si_code = 0, _sifields = {_pad = {989, 0, -1677672992, 32516, 0 <repeats 24 times>}, _kill = {si_pid = 989, si_uid = 0}, _timer = {si_tid = 989,
si_overrun = 0, si_sigval = {sival_int = -1677672992, sival_ptr = 0x7f049c00bde0}}, _rt = {si_pid = 989, si_uid = 0, si_sigval = {sival_int = -1677672992, sival_ptr = 0x7f049c00bde0}},
_sigchld = {si_pid = 989, si_uid = 0, si_status = -1677672992, si_utime = 0, si_stime = 0}, _sigfault = {si_addr = 0x3dd}, _sigpoll = {si_band = 989, si_fd = -1677672992}}}
ctx = 0x634d10
__func__ = "consumer_timer_metadata_thread"
#3 0x00007f04a62727b6 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#4 0x00007f04a5fcdc5d in clone () from /lib64/libc.so.6
No symbol table info available.
#5 0x0000000000000000 in ?? ()
No symbol table info available.

Thread 3 (Thread 0x7f04a52e0700 (LWP 992)):
#0 0x00007f04a5fce2e3 in epoll_wait () from /lib64/libc.so.6
No symbol table info available.
#1 0x0000000000419b59 in compat_epoll_wait (events=0x7f04a52dfbc0, timeout=-1) at compat-epoll.c:223
ret = 6507792
---Type <return> to continue, or q <return> to quit---
new_size = 0
__func__ = "compat_epoll_wait"
#2 0x0000000000409371 in consumer_thread_metadata_poll (data=0x634d10) at consumer.c:2156
ret = 1
i = 1
pollfd = 64
revents = 16
nb_fd = 1
stream = 0x7f049c004f70
iter = {iter = {node = 0x7f049c004f78, next = 0x0}}
node = 0x7f049c004f70
events = {epfd = 21, nb_fd = 1, alloc_size = 2, init_size = 2, events = 0x7f049c026310}
ctx = 0x634d10
len = -11
__func__ = "consumer_thread_metadata_poll"
__PRETTY_FUNCTION__ = "consumer_thread_metadata_poll"
#3 0x00007f04a62727b6 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#4 0x00007f04a5fcdc5d in clone () from /lib64/libc.so.6
No symbol table info available.
#5 0x0000000000000000 in ?? ()
No symbol table info available.

Thread 2 (Thread 0x7f04a5ae1700 (LWP 991)):
#0 0x00007f04a5fce2e3 in epoll_wait () from /lib64/libc.so.6
No symbol table info available.
#1 0x0000000000419b59 in compat_epoll_wait (events=0x7f04a5ae0b80, timeout=-1) at compat-epoll.c:223
ret = 2
new_size = 0
__func__ = "compat_epoll_wait"
#2 0x000000000040ac26 in consumer_thread_channel_poll (data=0x634d10) at consumer.c:2672
ret = 24
i = 1
pollfd = 12
revents = 1
nb_fd = 1
chan = 0x0
iter = {iter = {node = 0x7f049c022df0, next = 0x635c10}}
node = 0x7f049c022de8
events = {epfd = 19, nb_fd = 1, alloc_size = 2, init_size = 2, events = 0x636c00}
ctx = 0x634d10
channel_ht = 0x635120
__func__ = "consumer_thread_channel_poll"
__PRETTY_FUNCTION__ = "consumer_thread_channel_poll"
#3 0x00007f04a62727b6 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#4 0x00007f04a5fcdc5d in clone () from /lib64/libc.so.6
No symbol table info available.
#5 0x0000000000000000 in ?? ()
No symbol table info available.

Thread 1 (Thread 0x7f04a72fc980 (LWP 989)):
#0 0x00007f04a6272f95 in pthread_join () from /lib64/libpthread.so.0
No symbol table info available.
#1 0x0000000000404224 in main (argc=7, argv=0x7fffba08b4e8) at lttng-consumerd.c:444
ret = 0
status = 0x0
__func__ = "main"
(gdb)
(gdb)
(gdb) quit
[SC-1:Node16 Fri May 31 08:14:04/cluster/dumps/may31_backup] #
[SC-1:Node16 Fri May 31 08:14:04/cluster/dumps/may31_backup] #
(2-2/19)