[SC-1:Node16 Thu May 30 08:48:43/storage/no-backup/coremw/var/log/saflog] # tail -f saLogSystem_20130530_074957.log 1142004 08:48:50 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1142005 08:48:50 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. : : 1142470 08:49:14 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1142471 08:49:14 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. 1142472 08:49:14 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1142473 08:49:14 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. ^C [SC-1:Node16 Thu May 30 08:49:14/storage/no-backup/coremw/var/log/saflog] # [SC-1:Node16 Thu May 30 08:49:15/storage/no-backup/coremw/var/log/saflog] # netstat -etanp Active Internet connections (servers and established) Proto Recv-Q Send-Q Local Address Foreign Address State User Inode PID/Program name tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 0 41846 1291/rpcbind tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 0 45359 3415/sshd tcp 0 0 0.0.0.0:41783 0.0.0.0:* LISTEN 0 22281 2972/rpc.mountd tcp 0 0 192.168.0.1:55160 0.0.0.0:* LISTEN 0 85628284 20579/lttng-relayd tcp 0 0 0.0.0.0:59993 0.0.0.0:* LISTEN 0 23725 - tcp 0 0 0.0.0.0:34138 0.0.0.0:* LISTEN 0 22269 2972/rpc.mountd tcp 0 0 192.168.0.1:59963 0.0.0.0:* LISTEN 0 85628285 20579/lttng-relayd tcp 0 0 192.168.0.1:1022 0.0.0.0:* LISTEN 0 23818 3418/sshd tcp 0 0 0.0.0.0:2049 0.0.0.0:* LISTEN 0 20001 - tcp 0 0 192.168.0.1:1128 0.0.0.0:* LISTEN 0 43210 2754/alarmd tcp 0 0 0.0.0.0:51273 0.0.0.0:* LISTEN 0 22293 2972/rpc.mountd tcp 0 0 192.168.0.1:1130 0.0.0.0:* LISTEN 0 42095 2781/noded tcp 0 0 192.168.0.1:1131 0.0.0.0:* LISTEN 0 22658 2781/noded tcp 0 0 192.168.0.1:35400 192.168.0.8:1022 ESTABLISHED 0 47070172 24320/ssh tcp 0 0 192.168.0.1:53166 192.168.0.7:1022 ESTABLISHED 0 47070161 24294/ssh tcp 0 0 192.168.0.1:40795 192.168.0.9:1022 ESTABLISHED 0 47070176 24346/ssh tcp 0 0 192.168.0.1:50340 192.168.0.2:7788 ESTABLISHED 0 41919 - tcp 0 0 10.121.235.16:22 10.120.20.175:58861 ESTABLISHED 0 44191462 11518/1 tcp 0 0 192.168.0.1:22 192.168.0.1:32901 ESTABLISHED 0 85591684 10742/sshd: root@no tcp 0 0 192.168.0.1:36392 192.168.0.2:1022 ESTABLISHED 0 47070136 24128/ssh tcp 0 0 192.168.0.1:60645 192.168.0.5:1022 ESTABLISHED 0 47071617 24242/ssh tcp 0 0 192.168.0.1:35860 192.168.0.1:1022 ESTABLISHED 0 47070125 24100/ssh tcp 0 0 192.168.0.1:48100 192.168.0.4:1022 ESTABLISHED 0 47070146 24216/ssh tcp 0 3432 10.121.235.16:22 142.133.180.21:50505 ESTABLISHED 0 86295706 26192/2 tcp 0 0 192.168.0.1:45302 192.168.0.3:1022 ESTABLISHED 0 47071615 24163/ssh tcp 0 0 192.168.0.1:58550 192.168.0.6:1022 ESTABLISHED 0 47070158 24268/ssh tcp 0 0 192.168.0.1:1022 192.168.0.1:35860 ESTABLISHED 0 47071595 24101/sshd: root@no tcp 0 0 192.168.0.1:59963 192.168.0.3:39695 ESTABLISHED 0 85635462 20579/lttng-relayd tcp 0 0 192.168.0.1:22 192.168.0.1:33615 ESTABLISHED 0 85780147 6849/sshd: root@not tcp 0 4 192.168.0.1:55160 192.168.0.3:56944 ESTABLISHED 0 85636188 20579/lttng-relayd tcp 0 0 192.168.0.1:32901 192.168.0.1:22 ESTABLISHED 0 85606085 2649/Trace_FT tcp 0 0 192.168.0.1:7788 192.168.0.2:33797 ESTABLISHED 0 45254 - tcp 0 0 192.168.0.1:48776 192.168.0.10:1022 ESTABLISHED 0 47070179 24372/ssh tcp 0 0 192.168.0.1:33615 192.168.0.1:22 ESTABLISHED 0 85786631 31444/Trace_FT tcp 0 0 :::111 :::* LISTEN 0 41851 1291/rpcbind tcp 0 0 :::57393 :::* LISTEN 0 22287 2972/rpc.mountd tcp 0 0 :::58675 :::* LISTEN 0 23727 - tcp 0 0 :::22 :::* LISTEN 0 45362 3415/sshd tcp 0 0 :::40439 :::* LISTEN 0 22299 2972/rpc.mountd tcp 0 0 :::2049 :::* LISTEN 0 23715 - tcp 0 0 :::43655 :::* LISTEN 0 22275 2972/rpc.mountd [SC-1:Node16 Thu May 30 08:49:22/storage/no-backup/coremw/var/log/saflog] # netstat -etanp |grep lttng tcp 0 0 192.168.0.1:55160 0.0.0.0:* LISTEN 0 85628284 20579/lttng-relayd tcp 0 0 192.168.0.1:59963 0.0.0.0:* LISTEN 0 85628285 20579/lttng-relayd tcp 0 0 192.168.0.1:59963 192.168.0.3:39695 ESTABLISHED 0 85635462 20579/lttng-relayd tcp 0 0 192.168.0.1:55160 192.168.0.3:56944 ESTABLISHED 0 85636188 20579/lttng-relayd [SC-1:Node16 Thu May 30 08:49:28/storage/no-backup/coremw/var/log/saflog] # ssh PL-3 lttng list Available tracing sessions: 1) STAB002_3 (net://192.168.0.1:55160/./ [data: 59963]) [inactive] Use lttng list for more details [SC-1:Node16 Thu May 30 08:50:56/storage/no-backup/coremw/var/log/saflog] # ssh PL-3 ps -ef |grep lttng root 11850 1 0 May29 ? 00:00:58 lttng-sessiond -d root 13398 11850 2 May29 ? 00:22:38 lttng-consumerd --quiet -u --consumerd-cmd-sock /var/run/lttng/ustconsumerd64/command --consumerd-err-sock /var/run/lttng/ustconsumerd64/error [SC-1:Node16 Thu May 30 08:51:15/storage/no-backup/coremw/var/log/saflog] # ssh PL-3 netstat -etanp Active Internet connections (servers and established) Proto Recv-Q Send-Q Local Address Foreign Address State User Inode PID/Program name tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 0 11375 1654/rpcbind tcp 0 0 192.168.0.3:22 0.0.0.0:* LISTEN 0 8323 3055/sshd tcp 0 0 192.168.0.3:1022 0.0.0.0:* LISTEN 0 11452 3059/sshd tcp 0 0 192.168.0.3:1128 0.0.0.0:* LISTEN 0 29943 2764/alarmd tcp 0 0 192.168.0.3:1130 0.0.0.0:* LISTEN 0 11431 2783/noded tcp 0 0 192.168.0.3:1131 0.0.0.0:* LISTEN 0 8296 2783/noded tcp 0 0 192.168.0.3:39695 192.168.0.1:59963 ESTABLISHED 0 3384062 13398/lttng-consume tcp 0 0 192.168.0.3:1022 192.168.0.1:44750 ESTABLISHED 0 3693934 22271/sshd: root@no tcp 0 0 192.168.0.3:1022 192.168.0.1:45302 ESTABLISHED 0 1421946 12887/sshd: root@no tcp 0 0 192.168.0.3:56944 192.168.0.1:55160 ESTABLISHED 0 3384060 13398/lttng-consume tcp 0 0 :::111 :::* LISTEN 0 11380 1654/rpcbind [SC-1:Node16 Thu May 30 08:51:33/storage/no-backup/coremw/var/log/saflog] # ssh PL-3 netstat -etanp |grep lttng tcp 0 0 192.168.0.3:39695 192.168.0.1:59963 ESTABLISHED 0 3384062 13398/lttng-consume tcp 0 0 192.168.0.3:56944 192.168.0.1:55160 ESTABLISHED 0 3384060 13398/lttng-consume [SC-1:Node16 Thu May 30 08:51:41/storage/no-backup/coremw/var/log/saflog] # : : [SC-1:Node16 Thu May 30 09:11:07/storage/no-backup/coremw/var/log/saflog] # dir /storage/no-backup/tracecc-apr9010500/STAB002_3/ total 20 -rw-r--r-- 1 root root 411 May 30 05:54 session_metadata drwxr-x--- 3 root root 4096 May 30 05:54 PL-4 drwxr-x--- 3 root root 4096 May 30 05:54 SC-2 drwxr-x--- 3 root root 4096 May 30 05:54 SC-1 drwxr-x--- 3 root root 4096 May 30 05:54 PL-3 [SC-1:Node16 Thu May 30 09:11:47/storage/no-backup/coremw/var/log/saflog] # du -h /storage/no-backup/tracecc-apr9010500/STAB002_3/ 6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid/0/64-bit 6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid/0 6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid 6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust 6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid/0/64-bit 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid/0 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3 5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid/0/64-bit 5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid/0 5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid 5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust 5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid/0/64-bit 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid/0 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4 24M /storage/no-backup/tracecc-apr9010500/STAB002_3/ [SC-1:Node16 Thu May 30 09:11:53/storage/no-backup/coremw/var/log/saflog] # du -h /storage/no-backup/tracecc-apr9010500/STAB002_4 94M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-1/ust/uid/0/64-bit 94M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-1/ust/uid/0 94M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-1/ust/uid 94M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-1/ust 94M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-1 92M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-3/ust/uid/0/64-bit 92M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-3/ust/uid/0 92M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-3/ust/uid 93M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-3/ust 93M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-3 91M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-2/ust/uid/0/64-bit 91M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-2/ust/uid/0 91M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-2/ust/uid 91M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-2/ust 91M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-2 92M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-4/ust/uid/0/64-bit 92M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-4/ust/uid/0 92M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-4/ust/uid 92M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-4/ust 92M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-4 368M /storage/no-backup/tracecc-apr9010500/STAB002_4 [SC-1:Node16 Thu May 30 09:12:04/storage/no-backup/coremw/var/log/saflog] # du -h /storage/no-backup/tracecc-apr9010500/STAB002_5 95M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-1/ust/uid/0/64-bit 95M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-1/ust/uid/0 95M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-1/ust/uid 95M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-1/ust 95M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-1 93M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-3/ust/uid/0/64-bit 93M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-3/ust/uid/0 93M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-3/ust/uid 93M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-3/ust 93M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-3 93M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-2/ust/uid/0/64-bit 93M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-2/ust/uid/0 93M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-2/ust/uid 93M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-2/ust 93M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-2 94M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-4/ust/uid/0/64-bit 94M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-4/ust/uid/0 94M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-4/ust/uid 94M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-4/ust 94M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-4 374M /storage/no-backup/tracecc-apr9010500/STAB002_5 [SC-1:Node16 Thu May 30 09:12:08/storage/no-backup/coremw/var/log/saflog] # du -h /storage/no-backup/tracecc-apr9010500/STAB002_3 6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid/0/64-bit 6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid/0 6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid 6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust 6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid/0/64-bit 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid/0 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3 5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid/0/64-bit 5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid/0 5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid 5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust 5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid/0/64-bit 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid/0 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust 5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4 24M /storage/no-backup/tracecc-apr9010500/STAB002_3 [SC-1:Node16 Thu May 30 09:14:58/storage/no-backup/coremw/var/log/saflog] # du -k /storage/no-backup/tracecc-apr9010500/STAB002_3 6048 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid/0/64-bit 6052 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid/0 6056 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid 6060 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust 6064 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1 6016 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid/0/64-bit 6020 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid/0 6024 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid 6028 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust 6032 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3 5896 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid/0/64-bit 5900 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid/0 5904 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid 5908 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust 5912 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2 5956 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid/0/64-bit 5960 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid/0 5964 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid 5968 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust 5972 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4 23988 /storage/no-backup/tracecc-apr9010500/STAB002_3 [SC-1:Node16 Thu May 30 09:15:14/storage/no-backup/coremw/var/log/saflog] # du -k /storage/no-backup/tracecc-apr9010500/STAB002_3 6048 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid/0/64-bit 6052 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid/0 6056 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid 6060 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust 6064 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1 6016 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid/0/64-bit 6020 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid/0 6024 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid 6028 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust 6032 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3 5896 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid/0/64-bit 5900 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid/0 5904 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid 5908 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust 5912 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2 5956 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid/0/64-bit 5960 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid/0 5964 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid 5968 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust 5972 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4 23988 /storage/no-backup/tracecc-apr9010500/STAB002_3 [SC-1:Node16 Thu May 30 09:15:17/storage/no-backup/coremw/var/log/saflog] # [SC-1:Node16 Thu May 30 09:21:18/storage/no-backup/coremw/var/log/saflog] # dir total 162788 -rw-r----- 1 root root 0 May 24 09:05 saLogNotification_20130524_090540.log -rw-r----- 1 root root 41200 May 24 09:07 saLogSystem_20130524_090540.log -rw-r----- 1 root root 15600 May 24 09:07 saLogAlarm_20130524_090540.log -rw-r--r-- 1 root root 143 May 24 09:09 saLogSystem_20130528_195920.cfg -rw-r----- 1 root root 0 May 24 09:09 saLogNotification_20130524_090934.log -rw-r--r-- 1 root root 138 May 24 09:09 saLogNotification.cfg -rw-r--r-- 1 root root 138 May 24 09:09 saLogAlarm.cfg -rw-r----- 1 root root 133256 May 28 15:16 saLogSystem_20130524_090934_20130528_195920.log -rw-r--r-- 1 root root 144 May 28 19:59 saLogSystem_20130528_195930.cfg -rw-r----- 1 root root 0 May 28 19:59 saLogSystem_20130528_195920_20130528_195930.log -rw-r--r-- 1 root root 145 May 28 19:59 saLogSystem.cfg -rw-r--r-- 1 root root 1230873 May 29 08:03 saflog_may29_073x.log -rw-r----- 1 root root 28800 May 29 11:10 saLogAlarm_20130524_090934.log -rw-r----- 1 root root 9866713 May 29 14:30 saLogSystem_20130528_195930_20130529_143006.log -rw-r----- 1 root root 9999996 May 29 16:03 saLogSystem_20130529_143006_20130529_160334.log -rw-r----- 1 root root 9999911 May 29 17:38 saLogSystem_20130529_160334_20130529_173848.log -rw-r----- 1 root root 9999828 May 29 19:14 saLogSystem_20130529_173848_20130529_191401.log -rw-r----- 1 root root 9999873 May 29 20:47 saLogSystem_20130529_191401_20130529_204757.log -rw-r----- 1 root root 9999945 May 29 22:28 saLogSystem_20130529_204757_20130529_222823.log -rw-r----- 1 root root 9999893 May 30 00:10 saLogSystem_20130529_222823_20130530_001055.log -rw-r----- 1 root root 9999929 May 30 01:46 saLogSystem_20130530_001055_20130530_014617.log -rw-r----- 1 root root 9999871 May 30 03:26 saLogSystem_20130530_014617_20130530_032645.log -rw-r----- 1 root root 9999871 May 30 05:01 saLogSystem_20130530_032645_20130530_050114.log -rw-r----- 1 root root 9999881 May 30 05:50 saLogSystem_20130530_050114_20130530_055019.log -rw-r----- 1 root root 9999983 May 30 06:20 saLogSystem_20130530_055019_20130530_062004.log -rw-r----- 1 root root 9999894 May 30 06:48 saLogSystem_20130530_062004_20130530_064836.log -rw-r----- 1 root root 9999989 May 30 07:18 saLogSystem_20130530_064836_20130530_071836.log -rw-r----- 1 root root 9999960 May 30 07:49 saLogSystem_20130530_071836_20130530_074957.log -rw-r----- 1 root root 9999871 May 30 08:50 saLogSystem_20130530_074957_20130530_085050.log -rw-r----- 1 root root 5022752 May 30 09:21 saLogSystem_20130530_085050.log [SC-1:Node16 Thu May 30 09:21:25/storage/no-backup/coremw/var/log/saflog] # tail -f saLogSystem_20130530_085050.log 1180545 09:21:33 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1180546 09:21:33 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. 1180547 09:21:33 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1180548 09:21:33 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. 1180549 09:21:33 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1180550 09:21:33 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. 1180551 09:21:33 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1180552 09:21:33 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. 1180553 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1180554 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. 1180555 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1180556 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. 1180557 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1180558 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. 1180559 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1180560 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. 1180561 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1180562 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. 1180563 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1180564 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. 1180565 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1180566 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. 1180567 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1180568 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. 1180569 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1180570 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. 1180571 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1180572 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. 1180573 09:21:35 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1180574 09:21:35 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. ^C [SC-1:Node16 Thu May 30 09:21:36/storage/no-backup/coremw/var/log/saflog] # head -n 4 saLogSystem_20130530_085050.log 1144353 08:50:50 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. 1144354 08:50:50 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1144355 08:50:50 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. 1144356 08:50:50 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 [SC-1:Node16 Thu May 30 09:21:54/storage/no-backup/coremw/var/log/saflog] # head -n 4 saLogSystem_20130530_074957_20130530_085050.log 1072632 07:49:57 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1072633 07:49:57 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. 1072634 07:49:57 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3 1072635 07:49:57 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1. [SC-1:Node16 Thu May 30 09:22:08/storage/no-backup/coremw/var/log/saflog] # ssh PL-3 PL-3:~ # ps -ef |grep lttng root 11850 1 0 May29 ? 00:01:00 lttng-sessiond -d root 13398 11850 2 May29 ? 00:23:33 lttng-consumerd --quiet -u --consumerd-cmd-sock /var/run/lttng/ustconsumerd64/command --consumerd-err-sock /var/run/lttng/ustconsumerd64/error root 24261 24227 0 09:22 pts/0 00:00:00 grep lttng PL-3:~ # netstat -etanp |grep lttng tcp 0 0 192.168.0.3:39695 192.168.0.1:59963 ESTABLISHED 0 3384062 13398/lttng-consume tcp 0 0 192.168.0.3:56944 192.168.0.1:55160 ESTABLISHED 0 3384060 13398/lttng-consume PL-3:~ # ulimit -Sa core file size (blocks, -c) 0 data seg size (kbytes, -d) unlimited scheduling priority (-e) 0 file size (blocks, -f) unlimited pending signals (-i) 193117 max locked memory (kbytes, -l) 64 max memory size (kbytes, -m) unlimited open files (-n) 1024 pipe size (512 bytes, -p) 8 POSIX message queues (bytes, -q) 819200 real-time priority (-r) 0 stack size (kbytes, -s) 8192 cpu time (seconds, -t) unlimited max user processes (-u) 193117 virtual memory (kbytes, -v) unlimited file locks (-x) unlimited PL-3:~ # ulimit -Sc unlimitted -bash: ulimit: unlimitted: invalid number PL-3:~ # ulimit -Sc unlimited PL-3:~ # ulimit -Sa core file size (blocks, -c) unlimited data seg size (kbytes, -d) unlimited scheduling priority (-e) 0 file size (blocks, -f) unlimited pending signals (-i) 193117 max locked memory (kbytes, -l) 64 max memory size (kbytes, -m) unlimited open files (-n) 1024 pipe size (512 bytes, -p) 8 POSIX message queues (bytes, -q) 819200 real-time priority (-r) 0 stack size (kbytes, -s) 8192 cpu time (seconds, -t) unlimited max user processes (-u) 193117 virtual memory (kbytes, -v) unlimited file locks (-x) unlimited PL-3:~ # PL-3:~ # kill -SIGABRT 13398 PL-3:~ # ps -ef |grep lttng root 11850 1 0 May29 ? 00:01:00 lttng-sessiond -d root 13398 11850 2 May29 ? 00:23:36 lttng-consumerd --quiet -u --consumerd-cmd-sock /var/run/lttng/ustconsumerd64/command --consumerd-err-sock /var/run/lttng/ustconsumerd64/error root 24342 24227 0 09:23 pts/0 00:00:00 grep lttng PL-3:~ # dir /cluster/dumps/ total 1269020 -rw------- 1 root root 388345856 May 30 04:22 lttng-consumerd.11144.SC-1.core -rw------- 1 root root 351506432 May 30 09:24 lttng-consumerd.13398.PL-3.core -rw------- 1 root root 328339456 May 30 03:34 lttng-consumerd.19614.SC-1.core -rw------- 1 root root 111173632 May 30 02:55 lttng-consumerd.29753.SC-1.core -rw------- 1 root root 203612160 May 30 05:10 lttng-consumerd.4096.SC-1.core -rw------- 1 root root 236560384 May 30 04:49 lttng-consumerd.5847.SC-1.core -rw------- 1 root root 84004864 May 30 03:34 lttng-sessiond.17162.SC-1.core -rw------- 1 root root 80896000 May 30 02:55 lttng-sessiond.21491.SC-1.core -rw------- 1 root root 82186240 May 30 05:10 lttng-sessiond.29115.SC-1.core -rw------- 1 root root 83673088 May 30 04:49 lttng-sessiond.29775.SC-1.core -rw------- 1 root root 84135936 May 30 04:22 lttng-sessiond.6931.SC-1.core drwxr-xr-x 2 root root 4096 Apr 22 10:28 old PL-3:~ # kill -SIGABRT 11850 -bash: kill: (11850) - No such process PL-3:~ # ps -ef |grep lttng root 24375 1 0 09:24 ? 00:00:00 lttng-sessiond -d root 24631 24227 0 09:24 pts/0 00:00:00 grep lttng PL-3:~ # PL-3:~ # #--- sessiond got killed by TraceEA becaluse it failed the health check PL-3:~ # tail -n 20 /var/log/messages May 30 09:13:32 PL-3 osafimmnd[5344]: NO Ccb 339816 COMMITTED (TraceC) May 30 09:14:18 PL-3 osafimmnd[5344]: NO Ccb 339817 COMMITTED (TraceC) May 30 09:15:03 PL-3 osafimmnd[5344]: NO Ccb 339818 COMMITTED (TraceC) May 30 09:15:48 PL-3 osafimmnd[5344]: NO Ccb 339819 COMMITTED (TraceC) May 30 09:16:33 PL-3 osafimmnd[5344]: NO Ccb 339820 COMMITTED (TraceC) May 30 09:17:18 PL-3 osafimmnd[5344]: NO Ccb 339821 COMMITTED (TraceC) May 30 09:18:03 PL-3 osafimmnd[5344]: NO Ccb 339822 COMMITTED (TraceC) May 30 09:18:48 PL-3 osafimmnd[5344]: NO Ccb 339823 COMMITTED (TraceC) May 30 09:19:33 PL-3 osafimmnd[5344]: NO Ccb 339824 COMMITTED (TraceC) May 30 09:20:18 PL-3 osafimmnd[5344]: NO Ccb 339825 COMMITTED (TraceC) May 30 09:21:03 PL-3 osafimmnd[5344]: NO Ccb 339826 COMMITTED (TraceC) May 30 09:21:48 PL-3 osafimmnd[5344]: NO Ccb 339827 COMMITTED (TraceC) May 30 09:22:33 PL-3 osafimmnd[5344]: NO Ccb 339828 COMMITTED (TraceC) May 30 09:23:18 PL-3 osafimmnd[5344]: NO Ccb 339829 COMMITTED (TraceC) May 30 09:24:03 PL-3 osafimmnd[5344]: NO Ccb 339830 COMMITTED (TraceC) May 30 09:24:07 PL-3 TraceEa: SessionD healthcheck failed: LTTNG_HEALTH_CONSUMER ! May 30 09:24:07 PL-3 TraceEa: sessiond faulted - Trying to Stop LTTNG May 30 09:24:07 PL-3 logger: lttng-sessiond is stopped May 30 09:24:09 PL-3 osafimmnd[5344]: NO Ccb 339888 COMMITTED (TraceCLI) May 30 09:24:09 PL-3 logger: lttng-sessiond is started PL-3:~ # netstat -etanp |grep lttng PL-3:~ # PL-3:~ # exit logout Connection to PL-3 closed. [SC-1:Node16 Thu May 30 09:25:40/storage/no-backup/coremw/var/log/saflog] # [SC-1:Node16 Thu May 30 09:25:41/storage/no-backup/coremw/var/log/saflog] # netstat -etanp |grep lttng [SC-1:Node16 Thu May 30 09:25:52/storage/no-backup/coremw/var/log/saflog] # tracecc-profile-list [SC-1:Node16 Thu May 30 09:26:03/storage/no-backup/coremw/var/log/saflog] # [SC-1:Node16 Thu May 30 09:41:50/storage/no-backup/coremw/var/log/saflog] # dir /cluster/dumps/ total 1269020 drwxr-xr-x 2 root root 4096 Apr 22 10:28 old -rw------- 1 root root 80896000 May 30 02:55 lttng-sessiond.21491.SC-1.core -rw------- 1 root root 111173632 May 30 02:55 lttng-consumerd.29753.SC-1.core -rw------- 1 root root 84004864 May 30 03:34 lttng-sessiond.17162.SC-1.core -rw------- 1 root root 328339456 May 30 03:34 lttng-consumerd.19614.SC-1.core -rw------- 1 root root 84135936 May 30 04:22 lttng-sessiond.6931.SC-1.core -rw------- 1 root root 388345856 May 30 04:22 lttng-consumerd.11144.SC-1.core -rw------- 1 root root 83673088 May 30 04:49 lttng-sessiond.29775.SC-1.core -rw------- 1 root root 236560384 May 30 04:49 lttng-consumerd.5847.SC-1.core -rw------- 1 root root 82186240 May 30 05:10 lttng-sessiond.29115.SC-1.core -rw------- 1 root root 203612160 May 30 05:10 lttng-consumerd.4096.SC-1.core -rw------- 1 root root 351506432 May 30 09:24 lttng-consumerd.13398.PL-3.core [SC-1:Node16 Thu May 30 09:41:55/storage/no-backup/coremw/var/log/saflog] #