[SC-1:Node16 Thu May 30 10:27:27/storage/no-backup/coremw/var/log/saflog] # cd /cluster/dumps [SC-1:Node16 Thu May 30 10:27:43/cluster/dumps] # dir total 1269020 drwxr-xr-x 2 root root 4096 Apr 22 10:28 old -rw------- 1 root root 80896000 May 30 02:55 lttng-sessiond.21491.SC-1.core -rw------- 1 root root 111173632 May 30 02:55 lttng-consumerd.29753.SC-1.core -rw------- 1 root root 84004864 May 30 03:34 lttng-sessiond.17162.SC-1.core -rw------- 1 root root 328339456 May 30 03:34 lttng-consumerd.19614.SC-1.core -rw------- 1 root root 84135936 May 30 04:22 lttng-sessiond.6931.SC-1.core -rw------- 1 root root 388345856 May 30 04:22 lttng-consumerd.11144.SC-1.core -rw------- 1 root root 83673088 May 30 04:49 lttng-sessiond.29775.SC-1.core -rw------- 1 root root 236560384 May 30 04:49 lttng-consumerd.5847.SC-1.core -rw------- 1 root root 82186240 May 30 05:10 lttng-sessiond.29115.SC-1.core -rw------- 1 root root 203612160 May 30 05:10 lttng-consumerd.4096.SC-1.core -rw------- 1 root root 351506432 May 30 09:24 lttng-consumerd.13398.PL-3.core [SC-1:Node16 Thu May 30 10:27:44/cluster/dumps] # gdb /usr/lib64/lttng/libexec/lttng-consumerd lttng-consumerd.13398.PL-3.core GNU gdb (GDB) SUSE (7.3-0.6.1) Copyright (C) 2011 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-suse-linux". For bug reporting instructions, please see: ... Reading symbols from /usr/lib64/lttng/libexec/lttng-consumerd...done. [New LWP 13398] [New LWP 13400] [New LWP 13404] [New LWP 13403] [New LWP 13402] [New LWP 13528] [New LWP 13401] [New LWP 13405] Missing separate debuginfo for /lib64/libuuid.so.1 Try: zypper install -C "debuginfo(build-id)=f9998d18c497b2047b48e0702daa82204e2a944d" Missing separate debuginfo for /lib64/librt.so.1 Try: zypper install -C "debuginfo(build-id)=44612b93c19e6567318299411987b113d2387081" Missing separate debuginfo for /lib64/libpopt.so.0 Try: zypper install -C "debuginfo(build-id)=895e7fadb3a3371343391d34280543ea4df24a03" Missing separate debuginfo for /lib64/libpthread.so.0 Try: zypper install -C "debuginfo(build-id)=09dae90d04b1e2e43758ce58845f026b4085aec9" Missing separate debuginfo for /lib64/libc.so.6 Try: zypper install -C "debuginfo(build-id)=2f3040b47ab477519b3045352432f77c5bec1561" Missing separate debuginfo for /lib64/ld-linux-x86-64.so.2 Try: zypper install -C "debuginfo(build-id)=181860a35c8e9a0456dd6675f85c2eb0f062e956" Missing separate debuginfo for Try: zypper install -C "debuginfo(build-id)=31b3276dc0b305d25a213860d7d5a67601b1bb7a" Missing separate debuginfo for /lib64/libuuid.so.1 Try: zypper install -C "debuginfo(build-id)=f9998d18c497b2047b48e0702daa82204e2a944d" Missing separate debuginfo for /lib64/librt.so.1 Try: zypper install -C "debuginfo(build-id)=44612b93c19e6567318299411987b113d2387081" Missing separate debuginfo for /lib64/libpopt.so.0 Try: zypper install -C "debuginfo(build-id)=895e7fadb3a3371343391d34280543ea4df24a03" Missing separate debuginfo for /lib64/libpthread.so.0 Try: zypper install -C "debuginfo(build-id)=09dae90d04b1e2e43758ce58845f026b4085aec9" [Thread debugging using libthread_db enabled] Missing separate debuginfo for /lib64/libc.so.6 Try: zypper install -C "debuginfo(build-id)=2f3040b47ab477519b3045352432f77c5bec1561" Missing separate debuginfo for /lib64/ld-linux-x86-64.so.2 Try: zypper install -C "debuginfo(build-id)=181860a35c8e9a0456dd6675f85c2eb0f062e956" Core was generated by `lttng-consumerd --quiet -u --consumerd-cmd-sock /var/run/lttng/ustconsumerd64/c'. Program terminated with signal 6, Aborted. #0 0x00007f7cc5a17f95 in pthread_join () from /lib64/libpthread.so.0 (gdb) bt #0 0x00007f7cc5a17f95 in pthread_join () from /lib64/libpthread.so.0 #1 0x00000000004041fc in main (argc=7, argv=0x7fffde128068) at lttng-consumerd.c:437 (gdb) bt full #0 0x00007f7cc5a17f95 in pthread_join () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00000000004041fc in main (argc=7, argv=0x7fffde128068) at lttng-consumerd.c:437 ret = 0 status = 0x420d40 __func__ = "main" (gdb) thread apply all bt full Thread 8 (Thread 0x7f7cc2a81700 (LWP 13405)): #0 0x00007f7cc56ccabd in do_sigwaitinfo () from /lib64/libc.so.6 No symbol table info available. #1 0x00007f7cc56ccb8d in sigwaitinfo () from /lib64/libc.so.6 No symbol table info available. #2 0x00007f7cc5e5e6a0 in sig_thread (arg=0x0) at ring_buffer_frontend.c:403 mask = {__val = {60129542144, 0 }} info = {si_signo = 34, si_errno = 0, si_code = -2, _sifields = {_pad = {2585, 0, -1102331936, 32636, 0 }, _kill = {si_pid = 2585, si_uid = 0}, _timer = {si_tid = 2585, si_overrun = 0, si_sigval = { sival_int = -1102331936, sival_ptr = 0x7f7cbe4bbfe0}}, _rt = {si_pid = 2585, si_uid = 0, si_sigval = {sival_int = -1102331936, sival_ptr = 0x7f7cbe4bbfe0}}, _sigchld = {si_pid = 2585, si_uid = 0, si_status = -1102331936, si_utime = 0, si_stime = 0}, _sigfault = {si_addr = 0xa19}, _sigpoll = {si_band = 2585, si_fd = -1102331936}}} signr = 34 __func__ = "sig_thread" #3 0x00007f7cc5a177b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00007f7cc5772c5d in clone () from /lib64/libc.so.6 No symbol table info available. #5 0x0000000000000000 in ?? () No symbol table info available. Thread 7 (Thread 0x7f7cc4a85700 (LWP 13401)): #0 0x00007f7cc57732e3 in epoll_wait () from /lib64/libc.so.6 No symbol table info available. #1 0x0000000000419c0d in compat_epoll_wait (events=0x7f7cc4a84bc0, timeout=-1) at compat-epoll.c:223 ret = 0 new_size = 2 __func__ = "compat_epoll_wait" #2 0x000000000040940e in consumer_thread_metadata_poll (data=0x634d10) at consumer.c:2159 ret = 1 i = 1 pollfd = 11031 revents = 16 nb_fd = 1 stream = 0x7f7cbd26f4f0 ---Type to continue, or q to quit--- iter = {iter = {node = 0x7f7cbd26f4f8, next = 0x94e150}} node = 0x7f7cbd26f4f0 events = {epfd = 20, nb_fd = 2, alloc_size = 2, init_size = 2, events = 0x989500} ctx = 0x634d10 len = -11 __func__ = "consumer_thread_metadata_poll" __PRETTY_FUNCTION__ = "consumer_thread_metadata_poll" #3 0x00007f7cc5a177b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00007f7cc5772c5d in clone () from /lib64/libc.so.6 No symbol table info available. #5 0x0000000000000000 in ?? () No symbol table info available. Thread 6 (Thread 0x7f7cc223e700 (LWP 13528)): #0 0x00007f7cc576f679 in syscall () from /lib64/libc.so.6 No symbol table info available. #1 0x00007f7cc648b669 in call_rcu_wait (crdp=0x7f7cbc0037c0) at urcu-call-rcu-impl.h:215 No locals. #2 0x00007f7cc648bbef in call_rcu_thread (arg=0x7f7cbc0037c0) at urcu-call-rcu-impl.h:302 cbcount = 8 cbs = 0x0 cbs_tail = 0x7f7cbd3769d8 crdp = 0x7f7cbc0037c0 rhp = 0x7f7cbd3769d8 rt = 0 ret = 0 __func__ = "call_rcu_thread" #3 0x00007f7cc5a177b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00007f7cc5772c5d in clone () from /lib64/libc.so.6 No symbol table info available. #5 0x0000000000000000 in ?? () No symbol table info available. ---Type to continue, or q to quit--- Thread 5 (Thread 0x7f7cc4284700 (LWP 13402)): #0 0x00007f7cc5769786 in poll () from /lib64/libc.so.6 No symbol table info available. #1 0x0000000000409e45 in consumer_thread_data_poll (data=0x634d10) at consumer.c:2381 num_rdy = 9 num_hup = 0 high_prio = 0 ret = 12 i = 21 pollfd = 0x9b8fa0 local_stream = 0x9d1c70 new_stream = 0x7f7cbd449380 nb_fd = 12 ctx = 0x634d10 len = -11 __func__ = "consumer_thread_data_poll" #2 0x00007f7cc5a177b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #3 0x00007f7cc5772c5d in clone () from /lib64/libc.so.6 No symbol table info available. #4 0x0000000000000000 in ?? () No symbol table info available. Thread 4 (Thread 0x7f7cc3a83700 (LWP 13403)): #0 0x00007f7cc5769786 in poll () from /lib64/libc.so.6 No symbol table info available. #1 0x0000000000406431 in lttng_consumer_poll_socket (consumer_sockpoll=0x7f7cc3a82bf0) at consumer.c:994 num_rdy = 1 __func__ = "lttng_consumer_poll_socket" #2 0x000000000040bb50 in consumer_thread_sessiond_poll (data=0x634d10) at consumer.c:2923 sock = 22 client_socket = -1 ret = 1 ---Type to continue, or q to quit--- consumer_sockpoll = {{fd = 8, events = 3, revents = 0}, {fd = 22, events = 3, revents = 0}} ctx = 0x634d10 __func__ = "consumer_thread_sessiond_poll" #3 0x00007f7cc5a177b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00007f7cc5772c5d in clone () from /lib64/libc.so.6 No symbol table info available. #5 0x0000000000000000 in ?? () No symbol table info available. Thread 3 (Thread 0x7f7cc3282700 (LWP 13404)): #0 0x00007f7cc56ccabd in do_sigwaitinfo () from /lib64/libc.so.6 No symbol table info available. #1 0x00007f7cc56ccb8d in sigwaitinfo () from /lib64/libc.so.6 No symbol table info available. #2 0x000000000040d4b8 in consumer_timer_metadata_thread (data=0x634d10) at consumer-timer.c:232 signr = 44 mask = {__val = {26388279066624, 0 }} info = {si_signo = 44, si_errno = 0, si_code = -2, _sifields = {_pad = {2703, 0, -1100515760, 32636, 0 }, _kill = {si_pid = 2703, si_uid = 0}, _timer = {si_tid = 2703, si_overrun = 0, si_sigval = { sival_int = -1100515760, sival_ptr = 0x7f7cbe677650}}, _rt = {si_pid = 2703, si_uid = 0, si_sigval = {sival_int = -1100515760, sival_ptr = 0x7f7cbe677650}}, _sigchld = {si_pid = 2703, si_uid = 0, si_status = -1100515760, si_utime = 0, si_stime = 0}, _sigfault = {si_addr = 0xa8f}, _sigpoll = {si_band = 2703, si_fd = -1100515760}}} ctx = 0x634d10 __func__ = "consumer_timer_metadata_thread" #3 0x00007f7cc5a177b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00007f7cc5772c5d in clone () from /lib64/libc.so.6 No symbol table info available. #5 0x0000000000000000 in ?? () No symbol table info available. Thread 2 (Thread 0x7f7cc5286700 (LWP 13400)): #0 0x00007f7cc57732e3 in epoll_wait () from /lib64/libc.so.6 No symbol table info available. #1 0x0000000000419c0d in compat_epoll_wait (events=0x7f7cc5285b80, timeout=-1) at compat-epoll.c:223 ---Type to continue, or q to quit--- ret = 0 new_size = 2 __func__ = "compat_epoll_wait" #2 0x000000000040acc3 in consumer_thread_channel_poll (data=0x634d10) at consumer.c:2675 ret = 0 i = 1 pollfd = 10919 revents = 16 nb_fd = 1 chan = 0x7f7cbe682560 iter = {iter = {node = 0x7f7cbe683720, next = 0x635c30}} node = 0x7f7cbe683718 events = {epfd = 19, nb_fd = 2, alloc_size = 2, init_size = 2, events = 0x822660} ctx = 0x634d10 channel_ht = 0x635120 __func__ = "consumer_thread_channel_poll" __PRETTY_FUNCTION__ = "consumer_thread_channel_poll" #3 0x00007f7cc5a177b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00007f7cc5772c5d in clone () from /lib64/libc.so.6 No symbol table info available. #5 0x0000000000000000 in ?? () No symbol table info available. Thread 1 (Thread 0x7f7cc6aa1980 (LWP 13398)): #0 0x00007f7cc5a17f95 in pthread_join () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00000000004041fc in main (argc=7, argv=0x7fffde128068) at lttng-consumerd.c:437 ret = 0 status = 0x420d40 __func__ = "main" (gdb) (gdb) (gdb) quit [SC-1:Node16 Thu May 30 10:28:41/cluster/dumps] #