|
|
|
[SC-1:Node16 Thu May 30 08:48:43/storage/no-backup/coremw/var/log/saflog] # tail -f saLogSystem_20130530_074957.log
|
|
1142004 08:48:50 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1142005 08:48:50 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
:
|
|
:
|
|
1142470 08:49:14 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1142471 08:49:14 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
1142472 08:49:14 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1142473 08:49:14 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
^C
|
|
[SC-1:Node16 Thu May 30 08:49:14/storage/no-backup/coremw/var/log/saflog] #
|
|
[SC-1:Node16 Thu May 30 08:49:15/storage/no-backup/coremw/var/log/saflog] # netstat -etanp
|
|
Active Internet connections (servers and established)
|
|
Proto Recv-Q Send-Q Local Address Foreign Address State User Inode PID/Program name
|
|
tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 0 41846 1291/rpcbind
|
|
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 0 45359 3415/sshd
|
|
tcp 0 0 0.0.0.0:41783 0.0.0.0:* LISTEN 0 22281 2972/rpc.mountd
|
|
tcp 0 0 192.168.0.1:55160 0.0.0.0:* LISTEN 0 85628284 20579/lttng-relayd
|
|
tcp 0 0 0.0.0.0:59993 0.0.0.0:* LISTEN 0 23725 -
|
|
tcp 0 0 0.0.0.0:34138 0.0.0.0:* LISTEN 0 22269 2972/rpc.mountd
|
|
tcp 0 0 192.168.0.1:59963 0.0.0.0:* LISTEN 0 85628285 20579/lttng-relayd
|
|
tcp 0 0 192.168.0.1:1022 0.0.0.0:* LISTEN 0 23818 3418/sshd
|
|
tcp 0 0 0.0.0.0:2049 0.0.0.0:* LISTEN 0 20001 -
|
|
tcp 0 0 192.168.0.1:1128 0.0.0.0:* LISTEN 0 43210 2754/alarmd
|
|
tcp 0 0 0.0.0.0:51273 0.0.0.0:* LISTEN 0 22293 2972/rpc.mountd
|
|
tcp 0 0 192.168.0.1:1130 0.0.0.0:* LISTEN 0 42095 2781/noded
|
|
tcp 0 0 192.168.0.1:1131 0.0.0.0:* LISTEN 0 22658 2781/noded
|
|
tcp 0 0 192.168.0.1:35400 192.168.0.8:1022 ESTABLISHED 0 47070172 24320/ssh
|
|
tcp 0 0 192.168.0.1:53166 192.168.0.7:1022 ESTABLISHED 0 47070161 24294/ssh
|
|
tcp 0 0 192.168.0.1:40795 192.168.0.9:1022 ESTABLISHED 0 47070176 24346/ssh
|
|
tcp 0 0 192.168.0.1:50340 192.168.0.2:7788 ESTABLISHED 0 41919 -
|
|
tcp 0 0 10.121.235.16:22 10.120.20.175:58861 ESTABLISHED 0 44191462 11518/1
|
|
tcp 0 0 192.168.0.1:22 192.168.0.1:32901 ESTABLISHED 0 85591684 10742/sshd: root@no
|
|
tcp 0 0 192.168.0.1:36392 192.168.0.2:1022 ESTABLISHED 0 47070136 24128/ssh
|
|
tcp 0 0 192.168.0.1:60645 192.168.0.5:1022 ESTABLISHED 0 47071617 24242/ssh
|
|
tcp 0 0 192.168.0.1:35860 192.168.0.1:1022 ESTABLISHED 0 47070125 24100/ssh
|
|
tcp 0 0 192.168.0.1:48100 192.168.0.4:1022 ESTABLISHED 0 47070146 24216/ssh
|
|
tcp 0 3432 10.121.235.16:22 142.133.180.21:50505 ESTABLISHED 0 86295706 26192/2
|
|
tcp 0 0 192.168.0.1:45302 192.168.0.3:1022 ESTABLISHED 0 47071615 24163/ssh
|
|
tcp 0 0 192.168.0.1:58550 192.168.0.6:1022 ESTABLISHED 0 47070158 24268/ssh
|
|
tcp 0 0 192.168.0.1:1022 192.168.0.1:35860 ESTABLISHED 0 47071595 24101/sshd: root@no
|
|
tcp 0 0 192.168.0.1:59963 192.168.0.3:39695 ESTABLISHED 0 85635462 20579/lttng-relayd
|
|
tcp 0 0 192.168.0.1:22 192.168.0.1:33615 ESTABLISHED 0 85780147 6849/sshd: root@not
|
|
tcp 0 4 192.168.0.1:55160 192.168.0.3:56944 ESTABLISHED 0 85636188 20579/lttng-relayd
|
|
tcp 0 0 192.168.0.1:32901 192.168.0.1:22 ESTABLISHED 0 85606085 2649/Trace_FT
|
|
tcp 0 0 192.168.0.1:7788 192.168.0.2:33797 ESTABLISHED 0 45254 -
|
|
tcp 0 0 192.168.0.1:48776 192.168.0.10:1022 ESTABLISHED 0 47070179 24372/ssh
|
|
tcp 0 0 192.168.0.1:33615 192.168.0.1:22 ESTABLISHED 0 85786631 31444/Trace_FT
|
|
tcp 0 0 :::111 :::* LISTEN 0 41851 1291/rpcbind
|
|
tcp 0 0 :::57393 :::* LISTEN 0 22287 2972/rpc.mountd
|
|
tcp 0 0 :::58675 :::* LISTEN 0 23727 -
|
|
tcp 0 0 :::22 :::* LISTEN 0 45362 3415/sshd
|
|
tcp 0 0 :::40439 :::* LISTEN 0 22299 2972/rpc.mountd
|
|
tcp 0 0 :::2049 :::* LISTEN 0 23715 -
|
|
tcp 0 0 :::43655 :::* LISTEN 0 22275 2972/rpc.mountd
|
|
[SC-1:Node16 Thu May 30 08:49:22/storage/no-backup/coremw/var/log/saflog] # netstat -etanp |grep lttng
|
|
tcp 0 0 192.168.0.1:55160 0.0.0.0:* LISTEN 0 85628284 20579/lttng-relayd
|
|
tcp 0 0 192.168.0.1:59963 0.0.0.0:* LISTEN 0 85628285 20579/lttng-relayd
|
|
tcp 0 0 192.168.0.1:59963 192.168.0.3:39695 ESTABLISHED 0 85635462 20579/lttng-relayd
|
|
tcp 0 0 192.168.0.1:55160 192.168.0.3:56944 ESTABLISHED 0 85636188 20579/lttng-relayd
|
|
[SC-1:Node16 Thu May 30 08:49:28/storage/no-backup/coremw/var/log/saflog] # ssh PL-3 lttng list
|
|
Available tracing sessions:
|
|
1) STAB002_3 (net://192.168.0.1:55160/./ [data: 59963]) [inactive]
|
|
|
|
Use lttng list <session_name> for more details
|
|
[SC-1:Node16 Thu May 30 08:50:56/storage/no-backup/coremw/var/log/saflog] # ssh PL-3 ps -ef |grep lttng
|
|
root 11850 1 0 May29 ? 00:00:58 lttng-sessiond -d
|
|
root 13398 11850 2 May29 ? 00:22:38 lttng-consumerd --quiet -u --consumerd-cmd-sock /var/run/lttng/ustconsumerd64/command --consumerd-err-sock /var/run/lttng/ustconsumerd64/error
|
|
[SC-1:Node16 Thu May 30 08:51:15/storage/no-backup/coremw/var/log/saflog] # ssh PL-3 netstat -etanp
|
|
Active Internet connections (servers and established)
|
|
Proto Recv-Q Send-Q Local Address Foreign Address State User Inode PID/Program name
|
|
tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 0 11375 1654/rpcbind
|
|
tcp 0 0 192.168.0.3:22 0.0.0.0:* LISTEN 0 8323 3055/sshd
|
|
tcp 0 0 192.168.0.3:1022 0.0.0.0:* LISTEN 0 11452 3059/sshd
|
|
tcp 0 0 192.168.0.3:1128 0.0.0.0:* LISTEN 0 29943 2764/alarmd
|
|
tcp 0 0 192.168.0.3:1130 0.0.0.0:* LISTEN 0 11431 2783/noded
|
|
tcp 0 0 192.168.0.3:1131 0.0.0.0:* LISTEN 0 8296 2783/noded
|
|
tcp 0 0 192.168.0.3:39695 192.168.0.1:59963 ESTABLISHED 0 3384062 13398/lttng-consume
|
|
tcp 0 0 192.168.0.3:1022 192.168.0.1:44750 ESTABLISHED 0 3693934 22271/sshd: root@no
|
|
tcp 0 0 192.168.0.3:1022 192.168.0.1:45302 ESTABLISHED 0 1421946 12887/sshd: root@no
|
|
tcp 0 0 192.168.0.3:56944 192.168.0.1:55160 ESTABLISHED 0 3384060 13398/lttng-consume
|
|
tcp 0 0 :::111 :::* LISTEN 0 11380 1654/rpcbind
|
|
[SC-1:Node16 Thu May 30 08:51:33/storage/no-backup/coremw/var/log/saflog] # ssh PL-3 netstat -etanp |grep lttng
|
|
tcp 0 0 192.168.0.3:39695 192.168.0.1:59963 ESTABLISHED 0 3384062 13398/lttng-consume
|
|
tcp 0 0 192.168.0.3:56944 192.168.0.1:55160 ESTABLISHED 0 3384060 13398/lttng-consume
|
|
[SC-1:Node16 Thu May 30 08:51:41/storage/no-backup/coremw/var/log/saflog] #
|
|
:
|
|
:
|
|
[SC-1:Node16 Thu May 30 09:11:07/storage/no-backup/coremw/var/log/saflog] # dir /storage/no-backup/tracecc-apr9010500/STAB002_3/
|
|
total 20
|
|
-rw-r--r-- 1 root root 411 May 30 05:54 session_metadata
|
|
drwxr-x--- 3 root root 4096 May 30 05:54 PL-4
|
|
drwxr-x--- 3 root root 4096 May 30 05:54 SC-2
|
|
drwxr-x--- 3 root root 4096 May 30 05:54 SC-1
|
|
drwxr-x--- 3 root root 4096 May 30 05:54 PL-3
|
|
[SC-1:Node16 Thu May 30 09:11:47/storage/no-backup/coremw/var/log/saflog] # du -h /storage/no-backup/tracecc-apr9010500/STAB002_3/
|
|
6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid/0/64-bit
|
|
6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid/0
|
|
6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid
|
|
6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust
|
|
6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid/0/64-bit
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid/0
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3
|
|
5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid/0/64-bit
|
|
5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid/0
|
|
5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid
|
|
5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust
|
|
5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid/0/64-bit
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid/0
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4
|
|
24M /storage/no-backup/tracecc-apr9010500/STAB002_3/
|
|
[SC-1:Node16 Thu May 30 09:11:53/storage/no-backup/coremw/var/log/saflog] # du -h /storage/no-backup/tracecc-apr9010500/STAB002_4
|
|
94M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-1/ust/uid/0/64-bit
|
|
94M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-1/ust/uid/0
|
|
94M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-1/ust/uid
|
|
94M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-1/ust
|
|
94M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-1
|
|
92M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-3/ust/uid/0/64-bit
|
|
92M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-3/ust/uid/0
|
|
92M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-3/ust/uid
|
|
93M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-3/ust
|
|
93M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-3
|
|
91M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-2/ust/uid/0/64-bit
|
|
91M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-2/ust/uid/0
|
|
91M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-2/ust/uid
|
|
91M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-2/ust
|
|
91M /storage/no-backup/tracecc-apr9010500/STAB002_4/SC-2
|
|
92M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-4/ust/uid/0/64-bit
|
|
92M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-4/ust/uid/0
|
|
92M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-4/ust/uid
|
|
92M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-4/ust
|
|
92M /storage/no-backup/tracecc-apr9010500/STAB002_4/PL-4
|
|
368M /storage/no-backup/tracecc-apr9010500/STAB002_4
|
|
[SC-1:Node16 Thu May 30 09:12:04/storage/no-backup/coremw/var/log/saflog] # du -h /storage/no-backup/tracecc-apr9010500/STAB002_5
|
|
95M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-1/ust/uid/0/64-bit
|
|
95M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-1/ust/uid/0
|
|
95M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-1/ust/uid
|
|
95M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-1/ust
|
|
95M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-1
|
|
93M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-3/ust/uid/0/64-bit
|
|
93M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-3/ust/uid/0
|
|
93M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-3/ust/uid
|
|
93M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-3/ust
|
|
93M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-3
|
|
93M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-2/ust/uid/0/64-bit
|
|
93M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-2/ust/uid/0
|
|
93M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-2/ust/uid
|
|
93M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-2/ust
|
|
93M /storage/no-backup/tracecc-apr9010500/STAB002_5/SC-2
|
|
94M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-4/ust/uid/0/64-bit
|
|
94M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-4/ust/uid/0
|
|
94M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-4/ust/uid
|
|
94M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-4/ust
|
|
94M /storage/no-backup/tracecc-apr9010500/STAB002_5/PL-4
|
|
374M /storage/no-backup/tracecc-apr9010500/STAB002_5
|
|
[SC-1:Node16 Thu May 30 09:12:08/storage/no-backup/coremw/var/log/saflog] # du -h /storage/no-backup/tracecc-apr9010500/STAB002_3
|
|
6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid/0/64-bit
|
|
6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid/0
|
|
6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid
|
|
6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust
|
|
6.0M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid/0/64-bit
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid/0
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3
|
|
5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid/0/64-bit
|
|
5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid/0
|
|
5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid
|
|
5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust
|
|
5.8M /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid/0/64-bit
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid/0
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust
|
|
5.9M /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4
|
|
24M /storage/no-backup/tracecc-apr9010500/STAB002_3
|
|
[SC-1:Node16 Thu May 30 09:14:58/storage/no-backup/coremw/var/log/saflog] # du -k /storage/no-backup/tracecc-apr9010500/STAB002_3
|
|
6048 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid/0/64-bit
|
|
6052 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid/0
|
|
6056 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid
|
|
6060 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust
|
|
6064 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1
|
|
6016 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid/0/64-bit
|
|
6020 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid/0
|
|
6024 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid
|
|
6028 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust
|
|
6032 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3
|
|
5896 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid/0/64-bit
|
|
5900 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid/0
|
|
5904 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid
|
|
5908 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust
|
|
5912 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2
|
|
5956 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid/0/64-bit
|
|
5960 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid/0
|
|
5964 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid
|
|
5968 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust
|
|
5972 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4
|
|
23988 /storage/no-backup/tracecc-apr9010500/STAB002_3
|
|
[SC-1:Node16 Thu May 30 09:15:14/storage/no-backup/coremw/var/log/saflog] # du -k /storage/no-backup/tracecc-apr9010500/STAB002_3
|
|
6048 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid/0/64-bit
|
|
6052 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid/0
|
|
6056 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust/uid
|
|
6060 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1/ust
|
|
6064 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-1
|
|
6016 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid/0/64-bit
|
|
6020 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid/0
|
|
6024 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust/uid
|
|
6028 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3/ust
|
|
6032 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-3
|
|
5896 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid/0/64-bit
|
|
5900 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid/0
|
|
5904 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust/uid
|
|
5908 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2/ust
|
|
5912 /storage/no-backup/tracecc-apr9010500/STAB002_3/SC-2
|
|
5956 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid/0/64-bit
|
|
5960 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid/0
|
|
5964 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust/uid
|
|
5968 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4/ust
|
|
5972 /storage/no-backup/tracecc-apr9010500/STAB002_3/PL-4
|
|
23988 /storage/no-backup/tracecc-apr9010500/STAB002_3
|
|
[SC-1:Node16 Thu May 30 09:15:17/storage/no-backup/coremw/var/log/saflog] #
|
|
[SC-1:Node16 Thu May 30 09:21:18/storage/no-backup/coremw/var/log/saflog] # dir
|
|
total 162788
|
|
-rw-r----- 1 root root 0 May 24 09:05 saLogNotification_20130524_090540.log
|
|
-rw-r----- 1 root root 41200 May 24 09:07 saLogSystem_20130524_090540.log
|
|
-rw-r----- 1 root root 15600 May 24 09:07 saLogAlarm_20130524_090540.log
|
|
-rw-r--r-- 1 root root 143 May 24 09:09 saLogSystem_20130528_195920.cfg
|
|
-rw-r----- 1 root root 0 May 24 09:09 saLogNotification_20130524_090934.log
|
|
-rw-r--r-- 1 root root 138 May 24 09:09 saLogNotification.cfg
|
|
-rw-r--r-- 1 root root 138 May 24 09:09 saLogAlarm.cfg
|
|
-rw-r----- 1 root root 133256 May 28 15:16 saLogSystem_20130524_090934_20130528_195920.log
|
|
-rw-r--r-- 1 root root 144 May 28 19:59 saLogSystem_20130528_195930.cfg
|
|
-rw-r----- 1 root root 0 May 28 19:59 saLogSystem_20130528_195920_20130528_195930.log
|
|
-rw-r--r-- 1 root root 145 May 28 19:59 saLogSystem.cfg
|
|
-rw-r--r-- 1 root root 1230873 May 29 08:03 saflog_may29_073x.log
|
|
-rw-r----- 1 root root 28800 May 29 11:10 saLogAlarm_20130524_090934.log
|
|
-rw-r----- 1 root root 9866713 May 29 14:30 saLogSystem_20130528_195930_20130529_143006.log
|
|
-rw-r----- 1 root root 9999996 May 29 16:03 saLogSystem_20130529_143006_20130529_160334.log
|
|
-rw-r----- 1 root root 9999911 May 29 17:38 saLogSystem_20130529_160334_20130529_173848.log
|
|
-rw-r----- 1 root root 9999828 May 29 19:14 saLogSystem_20130529_173848_20130529_191401.log
|
|
-rw-r----- 1 root root 9999873 May 29 20:47 saLogSystem_20130529_191401_20130529_204757.log
|
|
-rw-r----- 1 root root 9999945 May 29 22:28 saLogSystem_20130529_204757_20130529_222823.log
|
|
-rw-r----- 1 root root 9999893 May 30 00:10 saLogSystem_20130529_222823_20130530_001055.log
|
|
-rw-r----- 1 root root 9999929 May 30 01:46 saLogSystem_20130530_001055_20130530_014617.log
|
|
-rw-r----- 1 root root 9999871 May 30 03:26 saLogSystem_20130530_014617_20130530_032645.log
|
|
-rw-r----- 1 root root 9999871 May 30 05:01 saLogSystem_20130530_032645_20130530_050114.log
|
|
-rw-r----- 1 root root 9999881 May 30 05:50 saLogSystem_20130530_050114_20130530_055019.log
|
|
-rw-r----- 1 root root 9999983 May 30 06:20 saLogSystem_20130530_055019_20130530_062004.log
|
|
-rw-r----- 1 root root 9999894 May 30 06:48 saLogSystem_20130530_062004_20130530_064836.log
|
|
-rw-r----- 1 root root 9999989 May 30 07:18 saLogSystem_20130530_064836_20130530_071836.log
|
|
-rw-r----- 1 root root 9999960 May 30 07:49 saLogSystem_20130530_071836_20130530_074957.log
|
|
-rw-r----- 1 root root 9999871 May 30 08:50 saLogSystem_20130530_074957_20130530_085050.log
|
|
-rw-r----- 1 root root 5022752 May 30 09:21 saLogSystem_20130530_085050.log
|
|
[SC-1:Node16 Thu May 30 09:21:25/storage/no-backup/coremw/var/log/saflog] # tail -f saLogSystem_20130530_085050.log
|
|
1180545 09:21:33 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1180546 09:21:33 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
1180547 09:21:33 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1180548 09:21:33 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
1180549 09:21:33 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1180550 09:21:33 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
1180551 09:21:33 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1180552 09:21:33 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
1180553 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1180554 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
1180555 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1180556 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
1180557 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1180558 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
1180559 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1180560 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
1180561 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1180562 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
1180563 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1180564 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
1180565 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1180566 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
1180567 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1180568 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
1180569 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1180570 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
1180571 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1180572 09:21:34 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
1180573 09:21:35 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1180574 09:21:35 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
^C
|
|
[SC-1:Node16 Thu May 30 09:21:36/storage/no-backup/coremw/var/log/saflog] # head -n 4 saLogSystem_20130530_085050.log
|
|
1144353 08:50:50 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
1144354 08:50:50 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1144355 08:50:50 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
1144356 08:50:50 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
[SC-1:Node16 Thu May 30 09:21:54/storage/no-backup/coremw/var/log/saflog] # head -n 4 saLogSystem_20130530_074957_20130530_085050.log
|
|
1072632 07:49:57 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1072633 07:49:57 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
1072634 07:49:57 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Invoke lttng_data_pending for session STAB002_3
|
|
1072635 07:49:57 05/30/2013 IN TraceP_PL-3 "DBG: deactivatingSessionInProgress: Reply recieved for lttng_data_pending for session STAB002_3: 1.
|
|
[SC-1:Node16 Thu May 30 09:22:08/storage/no-backup/coremw/var/log/saflog] # ssh PL-3
|
|
PL-3:~ # ps -ef |grep lttng
|
|
root 11850 1 0 May29 ? 00:01:00 lttng-sessiond -d
|
|
root 13398 11850 2 May29 ? 00:23:33 lttng-consumerd --quiet -u --consumerd-cmd-sock /var/run/lttng/ustconsumerd64/command --consumerd-err-sock /var/run/lttng/ustconsumerd64/error
|
|
root 24261 24227 0 09:22 pts/0 00:00:00 grep lttng
|
|
PL-3:~ # netstat -etanp |grep lttng
|
|
tcp 0 0 192.168.0.3:39695 192.168.0.1:59963 ESTABLISHED 0 3384062 13398/lttng-consume
|
|
tcp 0 0 192.168.0.3:56944 192.168.0.1:55160 ESTABLISHED 0 3384060 13398/lttng-consume
|
|
PL-3:~ # ulimit -Sa
|
|
core file size (blocks, -c) 0
|
|
data seg size (kbytes, -d) unlimited
|
|
scheduling priority (-e) 0
|
|
file size (blocks, -f) unlimited
|
|
pending signals (-i) 193117
|
|
max locked memory (kbytes, -l) 64
|
|
max memory size (kbytes, -m) unlimited
|
|
open files (-n) 1024
|
|
pipe size (512 bytes, -p) 8
|
|
POSIX message queues (bytes, -q) 819200
|
|
real-time priority (-r) 0
|
|
stack size (kbytes, -s) 8192
|
|
cpu time (seconds, -t) unlimited
|
|
max user processes (-u) 193117
|
|
virtual memory (kbytes, -v) unlimited
|
|
file locks (-x) unlimited
|
|
PL-3:~ # ulimit -Sc unlimitted
|
|
-bash: ulimit: unlimitted: invalid number
|
|
PL-3:~ # ulimit -Sc unlimited
|
|
PL-3:~ # ulimit -Sa
|
|
core file size (blocks, -c) unlimited
|
|
data seg size (kbytes, -d) unlimited
|
|
scheduling priority (-e) 0
|
|
file size (blocks, -f) unlimited
|
|
pending signals (-i) 193117
|
|
max locked memory (kbytes, -l) 64
|
|
max memory size (kbytes, -m) unlimited
|
|
open files (-n) 1024
|
|
pipe size (512 bytes, -p) 8
|
|
POSIX message queues (bytes, -q) 819200
|
|
real-time priority (-r) 0
|
|
stack size (kbytes, -s) 8192
|
|
cpu time (seconds, -t) unlimited
|
|
max user processes (-u) 193117
|
|
virtual memory (kbytes, -v) unlimited
|
|
file locks (-x) unlimited
|
|
PL-3:~ #
|
|
PL-3:~ # kill -SIGABRT 13398
|
|
PL-3:~ # ps -ef |grep lttng
|
|
root 11850 1 0 May29 ? 00:01:00 lttng-sessiond -d
|
|
root 13398 11850 2 May29 ? 00:23:36 lttng-consumerd --quiet -u --consumerd-cmd-sock /var/run/lttng/ustconsumerd64/command --consumerd-err-sock /var/run/lttng/ustconsumerd64/error
|
|
root 24342 24227 0 09:23 pts/0 00:00:00 grep lttng
|
|
PL-3:~ # dir /cluster/dumps/
|
|
total 1269020
|
|
-rw------- 1 root root 388345856 May 30 04:22 lttng-consumerd.11144.SC-1.core
|
|
-rw------- 1 root root 351506432 May 30 09:24 lttng-consumerd.13398.PL-3.core
|
|
-rw------- 1 root root 328339456 May 30 03:34 lttng-consumerd.19614.SC-1.core
|
|
-rw------- 1 root root 111173632 May 30 02:55 lttng-consumerd.29753.SC-1.core
|
|
-rw------- 1 root root 203612160 May 30 05:10 lttng-consumerd.4096.SC-1.core
|
|
-rw------- 1 root root 236560384 May 30 04:49 lttng-consumerd.5847.SC-1.core
|
|
-rw------- 1 root root 84004864 May 30 03:34 lttng-sessiond.17162.SC-1.core
|
|
-rw------- 1 root root 80896000 May 30 02:55 lttng-sessiond.21491.SC-1.core
|
|
-rw------- 1 root root 82186240 May 30 05:10 lttng-sessiond.29115.SC-1.core
|
|
-rw------- 1 root root 83673088 May 30 04:49 lttng-sessiond.29775.SC-1.core
|
|
-rw------- 1 root root 84135936 May 30 04:22 lttng-sessiond.6931.SC-1.core
|
|
drwxr-xr-x 2 root root 4096 Apr 22 10:28 old
|
|
PL-3:~ # kill -SIGABRT 11850
|
|
-bash: kill: (11850) - No such process
|
|
PL-3:~ # ps -ef |grep lttng
|
|
root 24375 1 0 09:24 ? 00:00:00 lttng-sessiond -d
|
|
root 24631 24227 0 09:24 pts/0 00:00:00 grep lttng
|
|
PL-3:~ #
|
|
PL-3:~ # #--- sessiond got killed by TraceEA becaluse it failed the health check
|
|
PL-3:~ # tail -n 20 /var/log/messages
|
|
May 30 09:13:32 PL-3 osafimmnd[5344]: NO Ccb 339816 COMMITTED (TraceC)
|
|
May 30 09:14:18 PL-3 osafimmnd[5344]: NO Ccb 339817 COMMITTED (TraceC)
|
|
May 30 09:15:03 PL-3 osafimmnd[5344]: NO Ccb 339818 COMMITTED (TraceC)
|
|
May 30 09:15:48 PL-3 osafimmnd[5344]: NO Ccb 339819 COMMITTED (TraceC)
|
|
May 30 09:16:33 PL-3 osafimmnd[5344]: NO Ccb 339820 COMMITTED (TraceC)
|
|
May 30 09:17:18 PL-3 osafimmnd[5344]: NO Ccb 339821 COMMITTED (TraceC)
|
|
May 30 09:18:03 PL-3 osafimmnd[5344]: NO Ccb 339822 COMMITTED (TraceC)
|
|
May 30 09:18:48 PL-3 osafimmnd[5344]: NO Ccb 339823 COMMITTED (TraceC)
|
|
May 30 09:19:33 PL-3 osafimmnd[5344]: NO Ccb 339824 COMMITTED (TraceC)
|
|
May 30 09:20:18 PL-3 osafimmnd[5344]: NO Ccb 339825 COMMITTED (TraceC)
|
|
May 30 09:21:03 PL-3 osafimmnd[5344]: NO Ccb 339826 COMMITTED (TraceC)
|
|
May 30 09:21:48 PL-3 osafimmnd[5344]: NO Ccb 339827 COMMITTED (TraceC)
|
|
May 30 09:22:33 PL-3 osafimmnd[5344]: NO Ccb 339828 COMMITTED (TraceC)
|
|
May 30 09:23:18 PL-3 osafimmnd[5344]: NO Ccb 339829 COMMITTED (TraceC)
|
|
May 30 09:24:03 PL-3 osafimmnd[5344]: NO Ccb 339830 COMMITTED (TraceC)
|
|
May 30 09:24:07 PL-3 TraceEa: SessionD healthcheck failed: LTTNG_HEALTH_CONSUMER !
|
|
May 30 09:24:07 PL-3 TraceEa: sessiond faulted - Trying to Stop LTTNG
|
|
May 30 09:24:07 PL-3 logger: lttng-sessiond is stopped
|
|
May 30 09:24:09 PL-3 osafimmnd[5344]: NO Ccb 339888 COMMITTED (TraceCLI)
|
|
May 30 09:24:09 PL-3 logger: lttng-sessiond is started
|
|
PL-3:~ # netstat -etanp |grep lttng
|
|
PL-3:~ #
|
|
PL-3:~ # exit
|
|
logout
|
|
Connection to PL-3 closed.
|
|
[SC-1:Node16 Thu May 30 09:25:40/storage/no-backup/coremw/var/log/saflog] #
|
|
[SC-1:Node16 Thu May 30 09:25:41/storage/no-backup/coremw/var/log/saflog] # netstat -etanp |grep lttng
|
|
[SC-1:Node16 Thu May 30 09:25:52/storage/no-backup/coremw/var/log/saflog] # tracecc-profile-list
|
|
[SC-1:Node16 Thu May 30 09:26:03/storage/no-backup/coremw/var/log/saflog] #
|
|
[SC-1:Node16 Thu May 30 09:41:50/storage/no-backup/coremw/var/log/saflog] # dir /cluster/dumps/
|
|
total 1269020
|
|
drwxr-xr-x 2 root root 4096 Apr 22 10:28 old
|
|
-rw------- 1 root root 80896000 May 30 02:55 lttng-sessiond.21491.SC-1.core
|
|
-rw------- 1 root root 111173632 May 30 02:55 lttng-consumerd.29753.SC-1.core
|
|
-rw------- 1 root root 84004864 May 30 03:34 lttng-sessiond.17162.SC-1.core
|
|
-rw------- 1 root root 328339456 May 30 03:34 lttng-consumerd.19614.SC-1.core
|
|
-rw------- 1 root root 84135936 May 30 04:22 lttng-sessiond.6931.SC-1.core
|
|
-rw------- 1 root root 388345856 May 30 04:22 lttng-consumerd.11144.SC-1.core
|
|
-rw------- 1 root root 83673088 May 30 04:49 lttng-sessiond.29775.SC-1.core
|
|
-rw------- 1 root root 236560384 May 30 04:49 lttng-consumerd.5847.SC-1.core
|
|
-rw------- 1 root root 82186240 May 30 05:10 lttng-sessiond.29115.SC-1.core
|
|
-rw------- 1 root root 203612160 May 30 05:10 lttng-consumerd.4096.SC-1.core
|
|
-rw------- 1 root root 351506432 May 30 09:24 lttng-consumerd.13398.PL-3.core
|
|
[SC-1:Node16 Thu May 30 09:41:55/storage/no-backup/coremw/var/log/saflog] #
|
|
|
|
|