[SC-1:Xen43 Tue Jun 04 11:30:45/cluster/dumps] # [SC-1:Xen43 Tue Jun 04 11:30:45/cluster/dumps] # dir total 111192 -rw------- 1 root root 83173376 Jun 4 09:47 lttng-sessiond.15805.SC-1.core -rw------- 1 root root 34070528 Jun 4 09:47 lttng-relayd.8084.SC-1.core -rw------- 1 root root 164085760 Jun 4 09:47 lttng-consumerd.18814.SC-1.core -rw------- 1 root root 19070976 Jun 4 11:04 trace_c.1400.SC-1.core [SC-1:Xen43 Tue Jun 04 11:30:46/cluster/dumps] # [SC-1:Xen43 Tue Jun 04 11:30:47/cluster/dumps] # which sessiond which: no sessiond in (/sbin:/usr/sbin:/usr/local/sbin:/root/bin:/usr/local/bin:/usr/bin:/bin:/usr/X11R6/bin:/usr/games:/opt/coremw/bin:/usr/lib/mit/bin:/usr/lib/mit/sbin:/opt/trace/bin/) [SC-1:Xen43 Tue Jun 04 11:30:54/cluster/dumps] # [SC-1:Xen43 Tue Jun 04 11:30:56/cluster/dumps] # which lttng-sessiond /usr/bin/lttng-sessiond [SC-1:Xen43 Tue Jun 04 11:31:00/cluster/dumps] # [SC-1:Xen43 Tue Jun 04 11:31:03/cluster/dumps] # gdb /usr/bin/lttng-sessiond lttng-sessiond.15805.SC-1.core GNU gdb (GDB) SUSE (7.3-0.6.1) Copyright (C) 2011 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-suse-linux". For bug reporting instructions, please see: ... Reading symbols from /usr/bin/lttng-sessiond...done. [New LWP 15805] [New LWP 15809] [New LWP 18826] Missing separate debuginfo for /lib64/librt.so.1 Try: zypper install -C "debuginfo(build-id)=44612b93c19e6567318299411987b113d2387081" Missing separate debuginfo for /lib64/libuuid.so.1 Try: zypper install -C "debuginfo(build-id)=f9998d18c497b2047b48e0702daa82204e2a944d" Missing separate debuginfo for /lib64/libdl.so.2 Try: zypper install -C "debuginfo(build-id)=3e4f6bfee9fdf77ca975b77b8c325347d9228bb8" Missing separate debuginfo for /lib64/libpopt.so.0 Try: zypper install -C "debuginfo(build-id)=895e7fadb3a3371343391d34280543ea4df24a03" Missing separate debuginfo for /lib64/libpthread.so.0 Try: zypper install -C "debuginfo(build-id)=09dae90d04b1e2e43758ce58845f026b4085aec9" Missing separate debuginfo for /lib64/libc.so.6 Try: zypper install -C "debuginfo(build-id)=2f3040b47ab477519b3045352432f77c5bec1561" Missing separate debuginfo for /lib64/ld-linux-x86-64.so.2 Try: zypper install -C "debuginfo(build-id)=181860a35c8e9a0456dd6675f85c2eb0f062e956" Missing separate debuginfo for Try: zypper install -C "debuginfo(build-id)=31b3276dc0b305d25a213860d7d5a67601b1bb7a" Missing separate debuginfo for /lib64/librt.so.1 Try: zypper install -C "debuginfo(build-id)=44612b93c19e6567318299411987b113d2387081" Missing separate debuginfo for /lib64/libuuid.so.1 Try: zypper install -C "debuginfo(build-id)=f9998d18c497b2047b48e0702daa82204e2a944d" Missing separate debuginfo for /lib64/libdl.so.2 Try: zypper install -C "debuginfo(build-id)=3e4f6bfee9fdf77ca975b77b8c325347d9228bb8" Missing separate debuginfo for /lib64/libpopt.so.0 Try: zypper install -C "debuginfo(build-id)=895e7fadb3a3371343391d34280543ea4df24a03" Missing separate debuginfo for /lib64/libpthread.so.0 Try: zypper install -C "debuginfo(build-id)=09dae90d04b1e2e43758ce58845f026b4085aec9" [Thread debugging using libthread_db enabled] Missing separate debuginfo for /lib64/libc.so.6 Try: zypper install -C "debuginfo(build-id)=2f3040b47ab477519b3045352432f77c5bec1561" Missing separate debuginfo for /lib64/ld-linux-x86-64.so.2 Try: zypper install -C "debuginfo(build-id)=181860a35c8e9a0456dd6675f85c2eb0f062e956" Core was generated by `lttng-sessiond -vvv'. Program terminated with signal 6, Aborted. #0 0x00007f2926fcdf95 in pthread_join () from /lib64/libpthread.so.0 (gdb) bt #0 0x00007f2926fcdf95 in pthread_join () from /lib64/libpthread.so.0 #1 0x000000000042fc45 in main (argc=2, argv=0x7fffa91da2c8) at main.c:4421 (gdb) bt full #0 0x00007f2926fcdf95 in pthread_join () from /lib64/libpthread.so.0 No symbol table info available. #1 0x000000000042fc45 in main (argc=2, argv=0x7fffa91da2c8) at main.c:4421 ret = 0 status = 0x0 home_path = 0x7f2926c83e55 "H\211\302H\213\005\361\300\063" env_app_timeout = 0x0 __func__ = "main" (gdb) thread apply all bt full Thread 3 (Thread 0x7f2922834700 (LWP 18826)): #0 0x00007f2926d25679 in syscall () from /lib64/libc.so.6 No symbol table info available. #1 0x00007f2927a3c669 in call_rcu_wait (crdp=0x67d090) at urcu-call-rcu-impl.h:215 No locals. #2 0x00007f2927a3cbef in call_rcu_thread (arg=0x67d090) at urcu-call-rcu-impl.h:302 cbcount = 1 cbs = 0x0 cbs_tail = 0x791590 crdp = 0x67d090 rhp = 0x791590 rt = 0 ret = 0 __func__ = "call_rcu_thread" #3 0x00007f2926fcd7b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00007f2926d28c5d in clone () from /lib64/libc.so.6 No symbol table info available. #5 0x0000000000000000 in ?? () No symbol table info available. Thread 2 (Thread 0x7f292603b700 (LWP 15809)): #0 0x00007f2926fd4aad in connect () from /lib64/libpthread.so.0 No symbol table info available. #1 0x0000000000431daf in lttcomm_connect_inet_sock (sock=0x822d00) at inet.c:102 ret = 4524128 closeret = 0 __func__ = "lttcomm_connect_inet_sock" #2 0x0000000000438dfc in relayd_connect (rsock=0x822d00) at relayd.c:357 ---Type to continue, or q to quit--- __PRETTY_FUNCTION__ = "relayd_connect" __func__ = "relayd_connect" #3 0x0000000000411593 in create_connect_relayd (uri=0x772bb4, relayd_sock=0x7f292603a848) at cmd.c:506 ret = 0 rsock = 0x822d00 __func__ = "create_connect_relayd" #4 0x00000000004118e6 in send_consumer_relayd_socket (domain=2, session=0x744910, relayd_uri=0x772bb4, consumer=0x76fb80, consumer_sock=0x86ef80) at cmd.c:557 ret = 10 rsock = 0x0 __PRETTY_FUNCTION__ = "send_consumer_relayd_socket" #5 0x0000000000411ad8 in send_consumer_relayd_sockets (domain=2, session=0x744910, consumer=0x76fb80, sock=0x86ef80) at cmd.c:642 ret = 10 __PRETTY_FUNCTION__ = "send_consumer_relayd_sockets" #6 0x0000000000411c5f in cmd_setup_relayd (session=0x744910) at cmd.c:684 ret = 10 usess = 0x73d980 ksess = 0x0 socket = 0x86ef80 iter = {iter = {node = 0x86efa8, next = 0x0}} __PRETTY_FUNCTION__ = "cmd_setup_relayd" __func__ = "cmd_setup_relayd" #7 0x000000000042b4a8 in process_client_msg (cmd_ctx=0x683cd0, sock=34, sock_error=0x7f292603ab58) at main.c:2656 ret = 0 need_tracing_session = 1 need_domain = 1 __func__ = "process_client_msg" #8 0x000000000042d1b2 in thread_manage_clients (data=0x0) at main.c:3457 sock = 34 ret = 8780 i = 1 ---Type to continue, or q to quit--- pollfd = 11 err = -1 sock_error = 0 revents = 1 nb_fd = 1 cmd_ctx = 0x683cd0 events = {epfd = 20, nb_fd = 2, alloc_size = 2, init_size = 2, events = 0x666610} __func__ = "thread_manage_clients" #9 0x00007f2926fcd7b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #10 0x00007f2926d28c5d in clone () from /lib64/libc.so.6 No symbol table info available. #11 0x0000000000000000 in ?? () No symbol table info available. Thread 1 (Thread 0x7f2928487a00 (LWP 15805)): #0 0x00007f2926fcdf95 in pthread_join () from /lib64/libpthread.so.0 No symbol table info available. #1 0x000000000042fc45 in main (argc=2, argv=0x7fffa91da2c8) at main.c:4421 ret = 0 status = 0x0 home_path = 0x7f2926c83e55 "H\211\302H\213\005\361\300\063" env_app_timeout = 0x0 __func__ = "main" (gdb) (gdb) quit [SC-1:Xen43 Tue Jun 04 11:31:39/cluster/dumps] # [SC-1:Xen43 Tue Jun 04 11:31:40/cluster/dumps] # dir total 111192 -rw------- 1 root root 83173376 Jun 4 09:47 lttng-sessiond.15805.SC-1.core -rw------- 1 root root 34070528 Jun 4 09:47 lttng-relayd.8084.SC-1.core -rw------- 1 root root 164085760 Jun 4 09:47 lttng-consumerd.18814.SC-1.core -rw------- 1 root root 19070976 Jun 4 11:04 trace_c.1400.SC-1.core [SC-1:Xen43 Tue Jun 04 11:31:42/cluster/dumps] # [SC-1:Xen43 Tue Jun 04 11:31:42/cluster/dumps] # which lttng-relayd /usr/bin/lttng-relayd [SC-1:Xen43 Tue Jun 04 11:32:08/cluster/dumps] # [SC-1:Xen43 Tue Jun 04 11:32:09/cluster/dumps] # gdb /usr/bin/lttng-relayd lttng-relayd.8084.SC-1.core GNU gdb (GDB) SUSE (7.3-0.6.1) Copyright (C) 2011 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-suse-linux". For bug reporting instructions, please see: ... Reading symbols from /usr/bin/lttng-relayd...done. [New LWP 8084] [New LWP 8085] [New LWP 8087] [New LWP 8086] [New LWP 8505] Missing separate debuginfo for /lib64/librt.so.1 Try: zypper install -C "debuginfo(build-id)=44612b93c19e6567318299411987b113d2387081" Missing separate debuginfo for /lib64/libuuid.so.1 Try: zypper install -C "debuginfo(build-id)=f9998d18c497b2047b48e0702daa82204e2a944d" Missing separate debuginfo for /lib64/libpopt.so.0 Try: zypper install -C "debuginfo(build-id)=895e7fadb3a3371343391d34280543ea4df24a03" Missing separate debuginfo for /lib64/libpthread.so.0 Try: zypper install -C "debuginfo(build-id)=09dae90d04b1e2e43758ce58845f026b4085aec9" Missing separate debuginfo for /lib64/libc.so.6 Try: zypper install -C "debuginfo(build-id)=2f3040b47ab477519b3045352432f77c5bec1561" Missing separate debuginfo for /lib64/ld-linux-x86-64.so.2 Try: zypper install -C "debuginfo(build-id)=181860a35c8e9a0456dd6675f85c2eb0f062e956" Missing separate debuginfo for Try: zypper install -C "debuginfo(build-id)=31b3276dc0b305d25a213860d7d5a67601b1bb7a" Missing separate debuginfo for /lib64/librt.so.1 Try: zypper install -C "debuginfo(build-id)=44612b93c19e6567318299411987b113d2387081" Missing separate debuginfo for /lib64/libuuid.so.1 Try: zypper install -C "debuginfo(build-id)=f9998d18c497b2047b48e0702daa82204e2a944d" Missing separate debuginfo for /lib64/libpopt.so.0 Try: zypper install -C "debuginfo(build-id)=895e7fadb3a3371343391d34280543ea4df24a03" Missing separate debuginfo for /lib64/libpthread.so.0 Try: zypper install -C "debuginfo(build-id)=09dae90d04b1e2e43758ce58845f026b4085aec9" [Thread debugging using libthread_db enabled] Missing separate debuginfo for /lib64/libc.so.6 Try: zypper install -C "debuginfo(build-id)=2f3040b47ab477519b3045352432f77c5bec1561" Missing separate debuginfo for /lib64/ld-linux-x86-64.so.2 Try: zypper install -C "debuginfo(build-id)=181860a35c8e9a0456dd6675f85c2eb0f062e956" Core was generated by `lttng-relayd -C tcp://192.168.0.1:53241 -D tcp://192.168.0.1:58399 -o /storage/'. Program terminated with signal 6, Aborted. #0 0x00007f2f9518af95 in pthread_join () from /lib64/libpthread.so.0 (gdb) bt #0 0x00007f2f9518af95 in pthread_join () from /lib64/libpthread.so.0 #1 0x0000000000407fdc in main (argc=7, argv=0x7fff73e2b398) at main.c:2142 (gdb) bt full #0 0x00007f2f9518af95 in pthread_join () from /lib64/libpthread.so.0 No symbol table info available. #1 0x0000000000407fdc in main (argc=7, argv=0x7fff73e2b398) at main.c:2142 ret = 0 status = 0x40de00 __func__ = "main" (gdb) thread apply all bt full Thread 5 (Thread 0x7f2f93608700 (LWP 8505)): #0 0x00007f2f94ee2679 in syscall () from /lib64/libc.so.6 No symbol table info available. #1 0x00007f2f957b0669 in call_rcu_wait (crdp=0x61c170) at urcu-call-rcu-impl.h:215 No locals. #2 0x00007f2f957b0bef in call_rcu_thread (arg=0x61c170) at urcu-call-rcu-impl.h:302 cbcount = 3 cbs = 0x0 cbs_tail = 0x61b0c0 crdp = 0x61c170 rhp = 0x61b0c0 rt = 0 ret = 0 __func__ = "call_rcu_thread" #3 0x00007f2f9518a7b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00007f2f94ee5c5d in clone () from /lib64/libc.so.6 No symbol table info available. #5 0x0000000000000000 in ?? () No symbol table info available. Thread 4 (Thread 0x7f2f9460a700 (LWP 8086)): #0 0x00007f2f94ee62e3 in epoll_wait () from /lib64/libc.so.6 No symbol table info available. #1 0x000000000040db79 in compat_epoll_wait (events=0x7f2f94609c20, timeout=-1) at compat-epoll.c:223 ret = 0 new_size = 4 __func__ = "compat_epoll_wait" #2 0x0000000000407301 in relay_thread_worker (data=0x0) at main.c:1809 ---Type to continue, or q to quit--- idx = -1 i = 2 seen_control = 0 last_notdel_data_fd = -1 ret = 0 err = -1 last_seen_data_fd = -1 nb_fd = 2 relay_connection = 0x61b080 events = {epfd = 7, nb_fd = 3, alloc_size = 4, init_size = 2, events = 0x61cb40} relay_connections_ht = 0x616010 node = 0x0 iter = {iter = {node = 0x0, next = 0x0}} streams_ht = 0x61a9c0 recv_hdr = {circuit_id = 0, data_size = 1152921504606846976, cmd = 117440512, cmd_version = 0} __func__ = "relay_thread_worker" #3 0x00007f2f9518a7b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00007f2f94ee5c5d in clone () from /lib64/libc.so.6 No symbol table info available. #5 0x0000000000000000 in ?? () No symbol table info available. Thread 3 (Thread 0x7f2f93e09700 (LWP 8087)): #0 0x00007f2f94ee62e3 in epoll_wait () from /lib64/libc.so.6 No symbol table info available. #1 0x000000000040db79 in compat_epoll_wait (events=0x7f2f93e08c30, timeout=-1) at compat-epoll.c:223 ret = 6378976 new_size = 0 __func__ = "compat_epoll_wait" ---Type to continue, or q to quit--- #2 0x0000000000403723 in relay_thread_listener (data=0x0) at main.c:519 i = 1 ret = 0 pollfd = 9 err = -1 val = 1 revents = 1 nb_fd = 1 events = {epfd = 10, nb_fd = 3, alloc_size = 3, init_size = 3, events = 0x61afa0} control_sock = 0x61af00 data_sock = 0x61af60 __func__ = "relay_thread_listener" __PRETTY_FUNCTION__ = "relay_thread_listener" #3 0x00007f2f9518a7b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00007f2f94ee5c5d in clone () from /lib64/libc.so.6 No symbol table info available. #5 0x0000000000000000 in ?? () No symbol table info available. Thread 2 (Thread 0x7f2f94e0b700 (LWP 8085)): #0 0x00007f2f94ee2679 in syscall () from /lib64/libc.so.6 No symbol table info available. #1 0x000000000040c3c5 in futex_nto1_wait (futex=0x6155e0) at futex.c:87 __func__ = "futex_nto1_wait" #2 0x00000000004040da in relay_thread_dispatcher (data=0x0) at main.c:680 ret = 96 node = 0x0 relay_cmd = 0x61ca90 __func__ = "relay_thread_dispatcher" ---Type to continue, or q to quit--- #3 0x00007f2f9518a7b6 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00007f2f94ee5c5d in clone () from /lib64/libc.so.6 No symbol table info available. #5 0x0000000000000000 in ?? () No symbol table info available. Thread 1 (Thread 0x7f2f961fc900 (LWP 8084)): #0 0x00007f2f9518af95 in pthread_join () from /lib64/libpthread.so.0 No symbol table info available. #1 0x0000000000407fdc in main (argc=7, argv=0x7fff73e2b398) at main.c:2142 ret = 0 status = 0x40de00 __func__ = "main" (gdb) (gdb) quit [SC-1:Xen43 Tue Jun 04 11:32:40/cluster/dumps] # [SC-1:Xen43 Tue Jun 04 11:32:40/cluster/dumps] # [SC-1:Xen43 Tue Jun 04 11:32:41/cluster/dumps] #