[SC-1:Node16 Mon May 13 14:12:19/cluster/dumps] # dir total 5740 drwxr-xr-x 2 root root 4096 Apr 22 10:28 old -rw------- 1 root root 61763584 May 13 09:23 lttng-consumerd.13141.SC-1.core -rw------- 1 root root 61763584 May 13 14:10 lttng-consumerd.10982.SC-1.core [SC-1:Node16 Mon May 13 14:12:21/cluster/dumps] # [SC-1:Node16 Mon May 13 14:12:21/cluster/dumps] # [SC-1:Node16 Mon May 13 14:12:22/cluster/dumps] # gdb /usr/lib64/lttng/libexec/lttng-consumerd lttng-consumerd.10982.SC-1.core GNU gdb (GDB) SUSE (7.3-0.6.1) Copyright (C) 2011 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-suse-linux". For bug reporting instructions, please see: ... Reading symbols from /usr/lib64/lttng/libexec/lttng-consumerd...Reading symbols from /usr/lib/debug/usr/lib64/lttng/libexec/lttng-consumerd.debug...done. done. [New LWP 10995] [New LWP 10996] [New LWP 11089] [New LWP 10982] [New LWP 10993] [New LWP 10994] [New LWP 10997] [New LWP 12237] Missing separate debuginfo for /lib64/libuuid.so.1 Try: zypper install -C "debuginfo(build-id)=f9998d18c497b2047b48e0702daa82204e2a944d" Missing separate debuginfo for /lib64/librt.so.1 Try: zypper install -C "debuginfo(build-id)=44612b93c19e6567318299411987b113d2387081" Missing separate debuginfo for /lib64/libpopt.so.0 Try: zypper install -C "debuginfo(build-id)=895e7fadb3a3371343391d34280543ea4df24a03" Missing separate debuginfo for /lib64/libpthread.so.0 Try: zypper install -C "debuginfo(build-id)=09dae90d04b1e2e43758ce58845f026b4085aec9" Missing separate debuginfo for /lib64/libc.so.6 Try: zypper install -C "debuginfo(build-id)=2f3040b47ab477519b3045352432f77c5bec1561" Missing separate debuginfo for /lib64/ld-linux-x86-64.so.2 Try: zypper install -C "debuginfo(build-id)=181860a35c8e9a0456dd6675f85c2eb0f062e956" Missing separate debuginfo for Try: zypper install -C "debuginfo(build-id)=31b3276dc0b305d25a213860d7d5a67601b1bb7a" Missing separate debuginfo for /lib64/libuuid.so.1 Try: zypper install -C "debuginfo(build-id)=f9998d18c497b2047b48e0702daa82204e2a944d" Missing separate debuginfo for /lib64/librt.so.1 Try: zypper install -C "debuginfo(build-id)=44612b93c19e6567318299411987b113d2387081" Missing separate debuginfo for /lib64/libpopt.so.0 Try: zypper install -C "debuginfo(build-id)=895e7fadb3a3371343391d34280543ea4df24a03" Missing separate debuginfo for /lib64/libpthread.so.0 Try: zypper install -C "debuginfo(build-id)=09dae90d04b1e2e43758ce58845f026b4085aec9" [Thread debugging using libthread_db enabled] Missing separate debuginfo for /lib64/libc.so.6 Try: zypper install -C "debuginfo(build-id)=2f3040b47ab477519b3045352432f77c5bec1561" Missing separate debuginfo for /lib64/ld-linux-x86-64.so.2 Try: zypper install -C "debuginfo(build-id)=181860a35c8e9a0456dd6675f85c2eb0f062e956" Core was generated by `lttng-consumerd --verbose -u --consumerd-cmd-sock /var/run/lttng/ustconsumerd64'. Program terminated with signal 11, Segmentation fault. #0 0x00007f79958c72fc in ustctl_flush_buffer (stream=0x63c4e0, producer_active=0) at ustctl.c:1425 1425 ustctl.c: No such file or directory. in ustctl.c (gdb) bt #0 0x00007f79958c72fc in ustctl_flush_buffer (stream=0x63c4e0, producer_active=0) at ustctl.c:1425 #1 0x000000000041c57b in lttng_ustconsumer_on_stream_hangup (stream=0x647590) at ust-consumer.c:1196 #2 0x000000000040a5e9 in consumer_thread_data_poll (data=0x633d10) at consumer.c:2500 #3 0x00007f799549f7b6 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f79951fac5d in clone () from /lib64/libc.so.6 #5 0x0000000000000000 in ?? () (gdb) bt full #0 0x00007f79958c72fc in ustctl_flush_buffer (stream=0x63c4e0, producer_active=0) at ustctl.c:1425 buf = 0x7f799218c000 consumer_chan = 0x63c160 __PRETTY_FUNCTION__ = "ustctl_flush_buffer" #1 0x000000000041c57b in lttng_ustconsumer_on_stream_hangup (stream=0x647590) at ust-consumer.c:1196 __PRETTY_FUNCTION__ = "lttng_ustconsumer_on_stream_hangup" #2 0x000000000040a5e9 in consumer_thread_data_poll (data=0x633d10) at consumer.c:2500 num_rdy = 12 num_hup = 0 high_prio = 0 ret = 36 i = 0 pollfd = 0x7f798c0013f0 local_stream = 0x7f798c0032a0 new_stream = 0x648630 nb_fd = 36 ctx = 0x633d10 len = 3211 __func__ = "consumer_thread_data_poll" #3 0x00007f799549f7b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00007f79951fac5d in clone () from /lib64/libc.so.6 No symbol table info available. #5 0x0000000000000000 in ?? () No symbol table info available. (gdb) thread apply all bt full Thread 8 (Thread 0x7f7991574700 (LWP 12237)): #0 0x00007f7995154abd in do_sigwaitinfo () from /lib64/libc.so.6 No symbol table info available. #1 0x00007f7995154b8d in sigwaitinfo () from /lib64/libc.so.6 No symbol table info available. #2 0x00007f79958e6688 in sig_thread (arg=0x0) at ring_buffer_frontend.c:403 mask = {__val = {60129542144, 0 }} info = {si_signo = 34, si_errno = 0, si_code = -2, _sifields = {_pad = {5, 0, 6544816, 0 }, _kill = {si_pid = 5, si_uid = 0}, _timer = {si_tid = 5, si_overrun = 0, si_sigval = {sival_int = 6544816, sival_ptr = 0x63ddb0}}, _rt = {si_pid = 5, si_uid = 0, si_sigval = {sival_int = 6544816, sival_ptr = 0x63ddb0}}, _sigchld = {si_pid = 5, si_uid = 0, si_status = 6544816, si_utime = 0, si_stime = 0}, _sigfault = {si_addr = 0x5}, _sigpoll = {si_band = 5, si_fd = 6544816}}} signr = 34 __func__ = "sig_thread" #3 0x00007f799549f7b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00007f79951fac5d in clone () from /lib64/libc.so.6 No symbol table info available. #5 0x0000000000000000 in ?? () No symbol table info available. Thread 7 (Thread 0x7f7992d0e700 (LWP 10997)): #0 0x00007f7995154abd in do_sigwaitinfo () from /lib64/libc.so.6 No symbol table info available. #1 0x00007f7995154b8d in sigwaitinfo () from /lib64/libc.so.6 No symbol table info available. #2 0x000000000040d600 in consumer_timer_metadata_thread (data=0x633d10) at consumer-timer.c:232 signr = 44 mask = {__val = {26388279066624, 0 }} info = {si_signo = 44, si_errno = 0, si_code = -2, _sifields = {_pad = {7, 0, 6554064, 0 }, _kill = {si_pid = 7, si_uid = 0}, _timer = {si_tid = 7, si_overrun = 0, si_sigval = {sival_int = 6554064, sival_ptr = 0x6401d0}}, _rt = {si_pid = 7, si_uid = 0, si_sigval = {sival_int = 6554064, sival_ptr = 0x6401d0}}, _sigchld = {si_pid = 7, si_uid = 0, si_status = 6554064, si_utime = 0, si_stime = 0}, _sigfault = {si_addr = 0x7}, _sigpoll = {si_band = 7, si_fd = 6554064}}} ---Type to continue, or q to quit--- ctx = 0x633d10 __func__ = "consumer_timer_metadata_thread" #3 0x00007f799549f7b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00007f79951fac5d in clone () from /lib64/libc.so.6 No symbol table info available. #5 0x0000000000000000 in ?? () No symbol table info available. Thread 6 (Thread 0x7f7994511700 (LWP 10994)): #0 0x00007f79951fb2e3 in epoll_wait () from /lib64/libc.so.6 No symbol table info available. #1 0x0000000000419bf5 in compat_epoll_wait (events=0x7f7994510bc0, timeout=-1) at compat-epoll.c:225 ret = 6503696 new_size = 0 __func__ = "compat_epoll_wait" #2 0x0000000000409403 in consumer_thread_metadata_poll (data=0x633d10) at consumer.c:2153 ret = 1 i = 1 pollfd = 107 revents = 1 nb_fd = 1 stream = 0x63be40 iter = {iter = {node = 0x63be48, next = 0x635ac0}} node = 0x63be40 events = {epfd = 20, nb_fd = 3, alloc_size = 4, init_size = 2, events = 0x63c120} ctx = 0x633d10 len = 2213 __func__ = "consumer_thread_metadata_poll" __PRETTY_FUNCTION__ = "consumer_thread_metadata_poll" #3 0x00007f799549f7b6 in start_thread () from /lib64/libpthread.so.0 ---Type to continue, or q to quit--- No symbol table info available. #4 0x00007f79951fac5d in clone () from /lib64/libc.so.6 No symbol table info available. #5 0x0000000000000000 in ?? () No symbol table info available. Thread 5 (Thread 0x7f7994d12700 (LWP 10993)): #0 0x00007f79951fb2e3 in epoll_wait () from /lib64/libc.so.6 No symbol table info available. #1 0x0000000000419bf5 in compat_epoll_wait (events=0x7f7994d11b80, timeout=-1) at compat-epoll.c:225 ret = 0 new_size = 0 __func__ = "compat_epoll_wait" #2 0x000000000040ae0d in consumer_thread_channel_poll (data=0x633d10) at consumer.c:2679 ret = 0 i = 1 pollfd = 110 revents = 16 nb_fd = 1 chan = 0x646370 iter = {iter = {node = 0x647530, next = 0x638a70}} node = 0x647528 events = {epfd = 18, nb_fd = 3, alloc_size = 4, init_size = 2, events = 0x640190} ctx = 0x633d10 channel_ht = 0x6341a0 __func__ = "consumer_thread_channel_poll" __PRETTY_FUNCTION__ = "consumer_thread_channel_poll" #3 0x00007f799549f7b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00007f79951fac5d in clone () from /lib64/libc.so.6 No symbol table info available. ---Type to continue, or q to quit--- #5 0x0000000000000000 in ?? () No symbol table info available. Thread 4 (Thread 0x7f7996525980 (LWP 10982)): #0 0x00007f799549ff95 in pthread_join () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00000000004041fc in main (argc=7, argv=0x7fffe43ba398) at lttng-consumerd.c:437 ret = 0 status = 0x4202a0 __func__ = "main" Thread 3 (Thread 0x7f7991d75700 (LWP 11089)): #0 0x00007f79951f7679 in syscall () from /lib64/libc.so.6 No symbol table info available. #1 0x00007f7995f11730 in call_rcu_wait (crdp=) at urcu-call-rcu-impl.h:215 No locals. #2 call_rcu_thread (arg=) at urcu-call-rcu-impl.h:302 cbcount = 18446744073709551615 cbs = 0x0 crdp = 0x7f798c0030b0 rt = 0 ret = __func__ = "call_rcu_thread" #3 0x00007f799549f7b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00007f79951fac5d in clone () from /lib64/libc.so.6 No symbol table info available. #5 0x0000000000000000 in ?? () No symbol table info available. Thread 2 (Thread 0x7f799350f700 (LWP 10996)): ---Type to continue, or q to quit--- #0 0x00007f79951f1786 in poll () from /lib64/libc.so.6 No symbol table info available. #1 0x00000000004063ab in lttng_consumer_poll_socket (consumer_sockpoll=0x7f799350ebf0) at consumer.c:988 num_rdy = 1 __func__ = "lttng_consumer_poll_socket" #2 0x000000000040bc9a in consumer_thread_sessiond_poll (data=0x633d10) at consumer.c:2927 sock = 21 client_socket = -1 ret = 1 consumer_sockpoll = {{fd = 4, events = 3, revents = 0}, {fd = 21, events = 3, revents = 0}} ctx = 0x633d10 __func__ = "consumer_thread_sessiond_poll" #3 0x00007f799549f7b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00007f79951fac5d in clone () from /lib64/libc.so.6 No symbol table info available. #5 0x0000000000000000 in ?? () No symbol table info available. Thread 1 (Thread 0x7f7993d10700 (LWP 10995)): #0 0x00007f79958c72fc in ustctl_flush_buffer (stream=0x63c4e0, producer_active=0) at ustctl.c:1425 buf = 0x7f799218c000 consumer_chan = 0x63c160 __PRETTY_FUNCTION__ = "ustctl_flush_buffer" #1 0x000000000041c57b in lttng_ustconsumer_on_stream_hangup (stream=0x647590) at ust-consumer.c:1196 __PRETTY_FUNCTION__ = "lttng_ustconsumer_on_stream_hangup" #2 0x000000000040a5e9 in consumer_thread_data_poll (data=0x633d10) at consumer.c:2500 num_rdy = 12 num_hup = 0 high_prio = 0 ret = 36 ---Type to continue, or q to quit--- i = 0 pollfd = 0x7f798c0013f0 local_stream = 0x7f798c0032a0 new_stream = 0x648630 nb_fd = 36 ctx = 0x633d10 len = 3211 __func__ = "consumer_thread_data_poll" #3 0x00007f799549f7b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00007f79951fac5d in clone () from /lib64/libc.so.6 No symbol table info available. #5 0x0000000000000000 in ?? () No symbol table info available. (gdb) (gdb) (gdb) (gdb) (gdb) quit [SC-1:Node16 Mon May 13 14:13:49/cluster/dumps] # [SC-1:Node16 Mon May 13 14:13:53/cluster/dumps] #