An open CheckMK agent is publicly available.
This could leak sensitive information such as :
https://docs.checkmk.com/latest/en/wato_monitoringagents.html
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09acddd55095
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (cvmfs,1305864,11528,0.0,450) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,79408,4732,0.0,454) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1096,0.0,458) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,788792,2364,0.0,516) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,788792,2620,0.0,527) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (cvmfs,788792,2368,0.0,549) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (cvmfs,788792,2404,0.0,556) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,788792,2328,0.0,563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,564) [kworker/0:1H] (cvmfs,788656,5056,0.0,570) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (cvmfs,788792,2368,0.0,575) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,3074196,36232,0.0,583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,788792,2388,0.0,588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,76900,2324,0.0,592) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1088,0.0,596) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (root,55840,21320,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,0.0,707) [kworker/u34:2] (root,0,0,1.1,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8280,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1226) [kworker/3:2] (root,0,0,0.0,1261) [kworker/5:0] (root,0,0,0.0,1305) [bond0] (root,0,0,0.0,1383) [kworker/4:2] (root,10079220,279452,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323804,70072,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,747024,15792,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1378836,37752,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,3062) [kworker/11:0] (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,591424,14560,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,4724) [kworker/7:2] (root,0,0,0.0,5761) [kworker/4:1] (root,0,0,0.0,6240) [kworker/u34:1] (root,0,0,0.0,6455) [kworker/u33:2] (root,0,0,0.0,6599) [kworker/12:2] (root,0,0,0.0,6615) [kworker/u32:1] (root,0,0,0.0,6679) [kworker/15:0] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,7432) [kworker/2:1] (root,0,0,0.0,8563) [kworker/2:2] (root,0,0,0.0,9295) [kworker/4:0] (root,108056,360,0.0,9910) sleep 5 (root,8632,660,0.0,9911) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,9912) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,9929) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,9930) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,10210) [kworker/9:2] (root,0,0,0.0,10763) [kworker/1:0] (root,0,0,0.0,11669) [kworker/1:2] (root,0,0,0.0,12074) [kworker/0:1] (root,0,0,0.0,13640) [kworker/12:1] (root,0,0,0.0,14034) [kworker/6:2] (cvmfs,858848,27512,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,857736,29216,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,3143140,62408,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,857732,27088,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,857752,27092,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (root,0,0,0.0,15023) [kworker/15:2] (cvmfs,857732,27052,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,857608,30884,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,859856,28672,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,857892,27044,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11304,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6772,0.0,15641) condor_shared_port (condor,3812196,3700300,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,17359) [kworker/u32:2] (root,0,0,0.0,18001) [kworker/9:1] (root,0,0,0.0,18559) [kworker/8:1] (root,0,0,0.0,18642) [kworker/11:1] (root,0,0,0.0,19217) [kworker/10:0] (root,0,0,0.0,20148) [kworker/13:1] (root,0,0,0.0,21023) [kworker/14:2] (root,0,0,0.0,22150) [kworker/14:1] (root,0,0,0.0,22151) [kworker/8:0] (root,0,0,0.0,23383) [kworker/3:1] (root,0,0,0.0,23635) [kworker/6:1] (root,0,0,0.0,23852) [kworker/u33:1] (root,0,0,0.0,24599) [kworker/11:2] (root,0,0,0.0,26358) [kworker/2:0] (root,0,0,0.0,26776) [kworker/10:2] (root,0,0,0.0,28106) [kworker/0:0] (postfix,89912,4032,0.0,28724) pickup -l -t unix -u (root,0,0,0.0,30321) [kworker/7:1] (root,0,0,0.0,30687) [kworker/5:1] (root,0,0,0.0,31855) [kworker/13:2] (root,0,0,0.0,32388) [kworker/6:0]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac663a89bb
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,348) [kworker/11:2] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (cvmfs,1305864,11528,0.0,450) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,79408,4732,0.0,454) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1096,0.0,458) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,788792,2364,0.0,516) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (root,0,0,0.0,518) [kworker/2:1] (cvmfs,788792,2620,0.0,527) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (cvmfs,788792,2368,0.0,549) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (cvmfs,788792,2404,0.0,556) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,788792,2328,0.0,563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,564) [kworker/0:1H] (cvmfs,788656,5056,0.0,570) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (cvmfs,788792,2368,0.0,575) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,3074196,36232,0.0,583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,788792,2388,0.0,588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,76900,2324,0.0,592) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1088,0.0,596) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (root,109132,57256,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.1,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1151) [kworker/15:1] (root,0,0,0.0,1305) [bond0] (root,0,0,0.0,1383) [kworker/4:2] (root,10079220,279316,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323804,70072,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,747000,35188,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1378836,36608,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,0,0,0.0,3452) [kworker/6:0] (root,591424,14560,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,4724) [kworker/7:2] (root,0,0,0.0,5716) [kworker/0:1] (root,0,0,0.0,6271) [kworker/4:1] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,8089) [kworker/u32:0] (root,0,0,0.0,8997) [kworker/13:0] (root,0,0,0.0,9409) [kworker/9:2] (root,0,0,0.0,9861) [kworker/3:0] (root,0,0,0.0,10819) [kworker/u32:1] (root,0,0,0.0,11368) [kworker/8:2] (root,0,0,0.0,11432) [kworker/1:1] (root,0,0,0.0,11788) [kworker/0:0] (root,0,0,0.0,11789) [kworker/8:1] (root,0,0,0.0,12476) [kworker/12:0] (cvmfs,858848,27076,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,857732,29216,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,3143136,62404,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,857732,27088,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,14873) [kworker/u33:1] (cvmfs,857744,27092,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,857732,27064,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,857596,30772,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,859856,28168,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,857776,27012,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11304,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6772,0.0,15641) condor_shared_port (condor,3719472,3607656,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,16420) [kworker/8:0] (root,0,0,0.0,17574) [kworker/12:2] (root,0,0,0.0,17732) [kworker/11:0] (root,0,0,0.0,17847) [kworker/14:2] (postfix,89912,4032,0.0,18648) pickup -l -t unix -u (root,0,0,0.0,18957) [kworker/10:1] (root,0,0,0.0,21245) [kworker/u33:0] (root,0,0,0.0,21782) [kworker/0:2] (root,0,0,0.0,22854) [kworker/6:2] (root,0,0,0.0,23899) [kworker/14:0] (root,108056,356,0.0,24296) sleep 5 (root,8632,664,0.0,24297) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,24298) /bin/bash /bin/check_mk_agent (root,51732,1688,0.0,24315) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,24316) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,24320) [kworker/u34:0] (root,0,0,0.0,25238) [kworker/15:0] (root,0,0,0.0,26456) [kworker/2:2] (root,0,0,0.0,27813) [kworker/10:0] (root,0,0,0.0,28882) [kworker/7:0] (root,0,0,0.0,28948) [kworker/1:0] (root,0,0,0.0,30382) [kworker/9:1] (root,0,0,0.0,30597) [kworker/3:1] (root,0,0,0.0,30687) [kworker/5:1] (root,0,0,0.0,30791) [kworker/5:0] (root,0,0,0.0,30816) [kworker/u34:1] (root,0,0,0.0,31855) [kworker/13:2]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09acffbebca1
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (cvmfs,1305864,11528,0.0,450) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,79408,4732,0.0,454) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1096,0.0,458) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,788792,2364,0.0,516) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,788792,2620,0.0,527) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (cvmfs,788792,2368,0.0,549) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (cvmfs,788792,2404,0.0,556) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,788792,2328,0.0,563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,564) [kworker/0:1H] (cvmfs,788656,5056,0.0,570) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (cvmfs,788792,2368,0.0,575) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,3074196,36232,0.0,583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,788792,2388,0.0,588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,76900,2324,0.0,592) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1088,0.0,596) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (root,47648,12496,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.1,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1151) [kworker/15:1] (root,0,0,0.0,1305) [bond0] (root,10079220,279236,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323804,70072,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,747028,11220,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (root,0,0,0.0,2392) [kworker/8:0] (root,0,0,0.0,2461) [kworker/3:0] (root,0,0,0.0,2575) [kworker/3:1] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1376928,37684,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2821) [kworker/u32:0] (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,591424,14560,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,4791) [kworker/9:1] (root,0,0,0.0,4889) [kworker/u33:2] (root,0,0,0.0,6030) [kworker/13:2] (root,0,0,0.0,6856) [kworker/0:0] (root,0,0,0.0,7208) [kworker/7:0] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,7612) [kworker/5:2] (root,0,0,0.0,7933) [kworker/u32:1] (root,0,0,0.0,8212) [kworker/u34:0] (root,0,0,0.0,9890) [kworker/1:1] (root,0,0,0.0,11246) [kworker/2:1] (root,0,0,0.0,11506) [kworker/14:2] (root,0,0,0.0,11743) [kworker/2:0] (root,0,0,0.0,12382) [kworker/4:0] (root,0,0,0.0,13277) [kworker/1:2] (root,0,0,0.0,14298) [kworker/7:2] (cvmfs,858320,27224,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,857736,29216,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,3143136,62400,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,857732,27088,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,857732,27092,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,857736,27064,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,15182) [kworker/5:0] (cvmfs,857596,30708,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,859856,27900,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,858848,27844,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11304,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6772,0.0,15641) condor_shared_port (condor,3630008,3518168,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,16445) [kworker/11:1] (root,0,0,0.0,17574) [kworker/12:2] (root,0,0,0.0,18728) [kworker/15:0] (root,0,0,0.0,18965) [kworker/7:1] (root,0,0,0.0,19740) [kworker/u33:1] (root,0,0,0.0,20637) [kworker/14:0] (root,0,0,0.0,21281) [kworker/11:0] (root,0,0,0.0,22705) [kworker/0:2] (root,0,0,0.0,23515) [kworker/10:0] (root,0,0,0.0,24219) [kworker/9:0] (root,0,0,0.0,25079) [kworker/6:1] (root,0,0,0.0,25532) [kworker/11:2] (root,0,0,0.0,26520) [kworker/8:2] (root,108056,356,0.0,26856) sleep 5 (root,8632,664,0.0,26857) timeout -s 9 360 check_mk_agent (root,11696,1532,0.0,26858) /bin/bash /bin/check_mk_agent (root,51732,1696,0.0,26875) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,26876) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,27514) [kworker/4:1] (root,0,0,0.0,28206) [kworker/u34:2] (root,0,0,0.0,28940) [kworker/10:1] (root,0,0,0.0,31835) [kworker/12:1] (root,0,0,0.0,32181) [kworker/6:0] (root,0,0,0.0,32457) [kworker/13:0] (postfix,89912,4032,0.0,32565) pickup -l -t unix -u
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac4f8d59c8
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (cvmfs,1305864,11528,0.0,450) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,79408,4732,0.0,454) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1096,0.0,458) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,788792,2364,0.0,516) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,788792,2620,0.0,527) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (cvmfs,788792,2368,0.0,549) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (cvmfs,788792,2404,0.0,556) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,788792,2328,0.0,563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,564) [kworker/0:1H] (cvmfs,788656,5056,0.0,570) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (cvmfs,788792,2368,0.0,575) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,3074196,36232,0.0,583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,788792,2388,0.0,588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,76900,2324,0.0,592) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1088,0.0,596) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (root,39456,8304,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.1,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1151) [kworker/15:1] (root,0,0,0.0,1305) [bond0] (root,10079220,279228,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,0,0,0.0,1533) [kworker/13:1] (root,323804,70072,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,747028,8752,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (root,0,0,0.0,2575) [kworker/3:1] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1376928,37680,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,591424,14560,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,6246) [kworker/u32:0] (root,0,0,0.0,7208) [kworker/7:0] (root,0,0,0.0,7213) [kworker/11:2] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,7612) [kworker/5:2] (root,0,0,0.0,7617) [kworker/15:2] (root,0,0,0.0,8023) [kworker/2:0] (root,0,0,0.0,8212) [kworker/u34:0] (root,0,0,0.0,9183) [kworker/4:0] (root,0,0,0.0,9188) [kworker/13:2] (root,0,0,0.0,10839) [kworker/3:2] (root,0,0,0.0,11246) [kworker/2:1] (root,0,0,0.0,12477) [kworker/u33:0] (root,0,0,0.0,12733) [kworker/0:0] (root,0,0,0.0,13277) [kworker/1:2] (root,108056,356,0.0,13315) sleep 5 (root,8632,660,0.0,13330) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,13331) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,13349) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,13350) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (ganglia,0,0,0.0,13351) [nvidia-smi] (cvmfs,858320,27224,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,857732,29216,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,3143136,62400,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,857732,27088,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,857736,27096,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,857736,27064,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,857596,30716,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,859856,27900,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,858848,27672,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11304,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6772,0.0,15641) condor_shared_port (condor,3619748,3507904,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,16418) [kworker/10:0] (root,0,0,0.0,16445) [kworker/11:1] (root,0,0,0.0,17574) [kworker/12:2] (root,0,0,0.0,18407) [kworker/6:1] (postfix,89912,4032,0.0,19295) pickup -l -t unix -u (root,0,0,0.0,19740) [kworker/u33:1] (root,0,0,0.0,19769) [kworker/12:0] (root,0,0,0.0,20637) [kworker/14:0] (root,0,0,0.0,20762) [kworker/u32:2] (root,0,0,0.0,21103) [kworker/14:1] (root,0,0,0.0,21937) [kworker/u34:2] (root,0,0,0.0,22705) [kworker/0:2] (root,0,0,0.0,23873) [kworker/5:1] (root,0,0,0.0,24219) [kworker/9:0] (root,0,0,0.0,26474) [kworker/8:0] (root,0,0,0.0,27040) [kworker/9:1] (root,0,0,0.0,27514) [kworker/4:1] (root,0,0,0.0,28210) [kworker/7:1] (root,0,0,0.0,28940) [kworker/10:1] (root,0,0,0.0,29432) [kworker/1:1] (root,0,0,0.0,29813) [kworker/8:1] (root,0,0,0.0,32181) [kworker/6:0] (root,0,0,0.0,32457) [kworker/13:0]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09acfae26354
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (cvmfs,1305864,11528,0.0,450) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,79408,4732,0.0,454) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1096,0.0,458) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,788792,2364,0.0,516) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,788792,2620,0.0,527) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,535) [kworker/3:2] (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (cvmfs,788792,2368,0.0,549) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (cvmfs,788792,2404,0.0,556) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,788792,2328,0.0,563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,564) [kworker/0:1H] (cvmfs,788656,5056,0.0,570) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (cvmfs,788792,2368,0.0,575) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,3074196,36232,0.0,583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,788792,2388,0.0,588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,76900,2324,0.0,592) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1088,0.0,596) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (root,80416,43112,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.2,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1182) [kworker/13:1] (root,0,0,0.0,1305) [bond0] (postfix,89912,4032,0.0,1405) pickup -l -t unix -u (root,10079220,279120,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,747028,27568,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1853) [kworker/6:2] (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (root,0,0,0.0,2337) [kworker/1:1] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1376928,36652,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,591424,14560,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,5039) [kworker/8:0] (root,0,0,0.0,5694) [kworker/9:2] (root,0,0,0.0,5878) [kworker/3:0] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,7495) [kworker/u33:2] (root,0,0,0.0,8249) [kworker/u32:0] (root,0,0,0.0,8571) [kworker/14:1] (root,0,0,0.0,8865) [kworker/u33:0] (root,0,0,0.0,9704) [kworker/11:0] (root,0,0,0.0,9836) [kworker/2:1] (root,0,0,0.0,11015) [kworker/2:0] (root,0,0,0.0,12425) [kworker/5:0] (root,0,0,0.0,12492) [kworker/9:1] (root,0,0,0.0,12921) [kworker/7:0] (root,0,0,0.0,13418) [kworker/9:0] (cvmfs,858320,27064,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,857732,29132,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,14523) [kworker/u32:2] (cvmfs,3143140,62384,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,857732,27088,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,14867) [kworker/7:2] (cvmfs,857744,27096,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,857744,27060,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,857600,30628,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,859856,27760,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,858848,27544,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11304,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6772,0.0,15641) condor_shared_port (condor,3531056,3419144,0.0,15652) condor_startd (root,0,0,0.0,15688) [kworker/0:2] (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,15908) [kworker/12:0] (root,0,0,0.0,16966) [kworker/0:1] (root,0,0,0.0,18941) [kworker/1:0] (root,0,0,0.0,19590) [kworker/10:0] (root,0,0,0.0,19627) [kworker/u32:1] (root,0,0,0.0,20390) [kworker/15:1] (root,0,0,0.0,20816) [kworker/15:0] (root,0,0,0.0,20913) [kworker/10:2] (root,0,0,0.0,21307) [kworker/4:0] (root,0,0,0.0,21754) [kworker/10:1] (root,0,0,0.0,22060) [kworker/11:2] (root,108056,360,0.0,22241) sleep 5 (root,8632,664,0.0,22242) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,22243) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,22260) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,22261) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,24793) [kworker/13:2] (root,0,0,0.0,25963) [kworker/11:1] (root,0,0,0.0,26160) [kworker/2:2] (root,0,0,0.0,26331) [kworker/u34:1] (root,0,0,0.0,27034) [kworker/14:2] (root,0,0,0.0,27250) [kworker/4:2] (root,0,0,0.0,27601) [kworker/8:1] (root,0,0,0.0,30470) [kworker/6:0] (root,0,0,0.0,31477) [kworker/12:1] (root,0,0,0.0,32532) [kworker/u34:0] (root,0,0,0.0,32613) [kworker/5:2]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09acce3b8cd8
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (cvmfs,1305864,11528,0.0,450) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,79408,4732,0.0,454) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1096,0.0,458) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,788792,2364,0.0,516) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,788792,2620,0.0,527) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (cvmfs,788792,2368,0.0,549) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (cvmfs,788792,2404,0.0,556) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,788792,2328,0.0,563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,564) [kworker/0:1H] (cvmfs,788656,5056,0.0,570) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (cvmfs,788792,2368,0.0,575) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,3074196,36232,0.0,583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,788792,2388,0.0,588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,76900,2324,0.0,592) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1088,0.0,596) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (root,129568,78372,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.2,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,950) [kworker/10:2] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1305) [bond0] (root,10079220,278964,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,747028,48604,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1375920,36692,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,591424,14560,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,5081) [kworker/u34:0] (root,0,0,0.0,5184) [kworker/15:1] (root,0,0,0.0,5395) [kworker/8:2] (root,0,0,0.0,5509) [kworker/u32:0] (root,0,0,0.0,5614) [kworker/4:1] (root,0,0,0.0,5674) [kworker/0:0] (root,0,0,0.0,7231) [kworker/9:2] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,7322) [kworker/7:0] (root,0,0,0.0,7483) [kworker/5:0] (root,0,0,0.0,9469) [kworker/13:0] (root,0,0,0.0,10015) [kworker/12:0] (root,0,0,0.0,10203) [kworker/6:2] (root,0,0,0.0,11118) [kworker/14:1] (root,0,0,0.0,13137) [kworker/1:2] (root,0,0,0.0,13277) [kworker/5:2] (cvmfs,858040,27044,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,857736,28900,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,3143136,62384,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,857732,27088,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,14867) [kworker/7:2] (cvmfs,857744,27092,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (root,0,0,0.0,15010) [kworker/u34:2] (cvmfs,857744,27060,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,15292) [kworker/6:0] (cvmfs,857616,30564,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,859856,27340,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,858848,27364,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11304,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6772,0.0,15641) condor_shared_port (condor,3438200,3326276,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,16444) [kworker/2:2] (root,0,0,0.0,16575) [kworker/9:1] (root,0,0,0.0,17364) [kworker/u33:2] (root,0,0,0.0,17594) [kworker/u32:2] (root,108056,360,0.0,19715) sleep 5 (root,8632,660,0.0,19716) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,19717) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,19734) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,944,0.0,19735) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,20126) [kworker/u33:1] (root,0,0,0.0,20744) [kworker/u32:1] (root,0,0,0.0,21284) [kworker/8:1] (root,0,0,0.0,21558) [kworker/3:2] (root,0,0,0.0,22033) [kworker/10:0] (postfix,89912,4036,0.0,22066) pickup -l -t unix -u (root,0,0,0.0,22164) [kworker/15:2] (root,0,0,0.0,22279) [kworker/4:0] (root,0,0,0.0,22411) [kworker/6:1] (root,0,0,0.0,22548) [kworker/12:1] (root,0,0,0.0,23430) [kworker/7:1] (root,0,0,0.0,23768) [kworker/2:1] (root,0,0,0.0,24845) [kworker/14:0] (root,0,0,0.0,26211) [kworker/12:2] (root,0,0,0.0,28558) [kworker/3:1] (root,0,0,0.0,28986) [kworker/1:0] (root,0,0,0.0,29375) [kworker/13:2] (root,0,0,0.0,31043) [kworker/0:1] (root,0,0,0.0,31127) [kworker/11:1] (root,0,0,0.0,31951) [kworker/9:0] (root,0,0,0.0,32744) [kworker/11:2]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac1e9e48f3
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (cvmfs,1305864,11528,0.0,450) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,79408,4732,0.0,454) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1096,0.0,458) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,788792,2364,0.0,516) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,788792,2620,0.0,527) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (cvmfs,788792,2368,0.0,549) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (cvmfs,788792,2404,0.0,556) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,788792,2328,0.0,563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,564) [kworker/0:1H] (cvmfs,788656,5056,0.0,570) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (cvmfs,788792,2368,0.0,575) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,3074196,36232,0.0,583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,788792,2388,0.0,588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,76900,2324,0.0,592) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1088,0.0,596) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (root,80416,36620,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.2,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1008) [kworker/15:1] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1305) [bond0] (root,10079220,278892,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,108056,352,0.0,1510) sleep 5 (root,8632,660,0.0,1511) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,1512) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,1529) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,944,0.0,1530) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,0,0,0.0,1629) [kworker/14:2] (root,0,0,0.0,1821) [kworker/u34:2] (root,747028,23448,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1375392,36260,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,0,0,0.0,3846) [kworker/11:2] (root,0,0,0.0,3849) [kworker/2:0] (root,591424,14560,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,4009) [kworker/6:0] (root,0,0,0.0,4625) [kworker/10:0] (root,0,0,0.0,5614) [kworker/4:1] (root,0,0,0.0,6650) [kworker/12:0] (root,0,0,0.0,7111) [kworker/11:1] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,7872) [kworker/7:1] (root,0,0,0.0,8028) [kworker/u33:0] (root,0,0,0.0,9204) [kworker/u34:1] (root,0,0,0.0,9468) [kworker/0:1] (root,0,0,0.0,10419) [kworker/u32:0] (root,0,0,0.0,10450) [kworker/15:0] (root,0,0,0.0,12205) [kworker/5:1] (root,0,0,0.0,12334) [kworker/14:0] (root,0,0,0.0,13656) [kworker/8:2] (root,0,0,0.0,14291) [kworker/0:0] (cvmfs,857892,27000,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,857732,28744,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,14558) [kworker/7:2] (cvmfs,3143136,62384,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,857732,27088,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,857744,27088,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (root,0,0,0.0,14998) [kworker/9:2] (cvmfs,857732,27048,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,857596,30440,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,15352) [kworker/3:1] (cvmfs,859856,26916,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (root,0,0,0.0,15500) [kworker/10:2] (cvmfs,858848,27196,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11304,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6772,0.0,15641) condor_shared_port (condor,3351684,3239808,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,16528) [kworker/4:0] (root,0,0,0.0,17021) [kworker/2:1] (root,0,0,0.0,17273) [kworker/u33:2] (root,0,0,0.0,17871) [kworker/3:2] (root,0,0,0.0,18467) [kworker/5:0] (root,0,0,0.0,18538) [kworker/1:1] (root,0,0,0.0,18878) [kworker/1:0] (root,0,0,0.0,20573) [kworker/13:0] (root,0,0,0.0,22037) [kworker/13:2] (root,0,0,0.0,22548) [kworker/12:1] (root,0,0,0.0,23197) [kworker/13:1] (root,0,0,0.0,26897) [kworker/7:0] (root,0,0,0.0,29593) [kworker/6:2] (root,0,0,0.0,29767) [kworker/8:1] (postfix,89912,4032,0.0,29973) pickup -l -t unix -u (root,0,0,0.0,30383) [kworker/6:1] (root,0,0,0.0,30525) [kworker/u32:1] (root,0,0,0.0,30526) [kworker/9:0] (root,0,0,0.0,32363) [kworker/2:2] (root,0,0,0.0,32365) [kworker/5:2]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09acedaa7ed0
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (cvmfs,1305864,11528,0.0,450) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,79408,4732,0.0,454) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1096,0.0,458) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,788792,2364,0.0,516) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,788792,2620,0.0,527) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (cvmfs,788792,2368,0.0,549) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (cvmfs,788792,2404,0.0,556) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,788792,2328,0.0,563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,564) [kworker/0:1H] (cvmfs,788656,5056,0.0,570) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (cvmfs,788792,2368,0.0,575) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,3074196,36232,0.0,583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,788792,2388,0.0,588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,76900,2324,0.0,592) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1088,0.0,596) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (root,121376,70816,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.2,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1305) [bond0] (root,10079220,278780,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,0,0,0.0,1511) [kworker/10:0] (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,747028,43988,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (root,0,0,0.0,2558) [kworker/2:0] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1375920,37256,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,591424,14560,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,4878) [kworker/3:0] (root,0,0,0.0,5614) [kworker/4:1] (root,0,0,0.0,5967) [kworker/u33:0] (root,0,0,0.0,7156) [kworker/3:2] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,7500) [kworker/14:1] (root,0,0,0.0,8501) [kworker/5:0] (root,0,0,0.0,9005) [kworker/11:2] (root,0,0,0.0,9228) [kworker/1:1] (root,0,0,0.0,10062) [kworker/12:0] (root,0,0,0.0,11940) [kworker/u32:2] (root,0,0,0.0,12716) [kworker/0:2] (root,0,0,0.0,13499) [kworker/6:2] (cvmfs,858848,27644,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,857732,28556,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,3144252,62420,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,857732,27024,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,14802) [kworker/13:0] (cvmfs,857752,27096,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (root,0,0,0.0,15089) [kworker/10:2] (cvmfs,857732,27056,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,857596,30400,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,858848,27912,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (root,0,0,0.0,15568) [kworker/11:1] (cvmfs,858848,27016,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11300,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6772,0.0,15641) condor_shared_port (condor,3261220,3148740,0.0,15652) condor_startd (root,0,0,0.0,15741) [kworker/6:0] (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,16373) [kworker/8:1] (postfix,89912,4028,0.0,17696) pickup -l -t unix -u (root,0,0,0.0,19071) [kworker/9:1] (root,0,0,0.0,20915) [kworker/u33:2] (root,0,0,0.0,21826) [kworker/13:2] (root,0,0,0.0,21913) [kworker/2:1] (root,0,0,0.0,21953) [kworker/7:0] (root,0,0,0.0,22548) [kworker/12:1] (root,0,0,0.0,22573) [kworker/0:1] (root,0,0,0.0,23823) [kworker/u34:2] (root,0,0,0.0,23910) [kworker/7:1] (root,0,0,0.0,24194) [kworker/4:2] (root,0,0,0.0,24530) [kworker/15:2] (root,0,0,0.0,24896) [kworker/15:1] (root,0,0,0.0,25009) [kworker/8:0] (root,0,0,0.0,26635) [kworker/4:0] (root,0,0,0.0,29400) [kworker/14:2] (root,0,0,0.0,29985) [kworker/14:0] (root,0,0,0.0,30107) [kworker/7:2] (root,108056,360,0.0,30770) sleep 5 (root,8632,656,0.0,30771) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,30772) /bin/bash /bin/check_mk_agent (root,51732,1688,0.0,30789) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,30790) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,31014) [kworker/1:2] (root,0,0,0.0,31198) [kworker/9:0] (root,0,0,0.0,32010) [kworker/u34:0] (root,0,0,0.0,32224) [kworker/u32:0] (root,0,0,0.0,32612) [kworker/5:1]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac789e144b
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,311) [kworker/3:2] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (cvmfs,1305864,11528,0.0,450) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,79408,4732,0.0,454) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1096,0.0,458) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,788792,2364,0.0,516) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,788792,2620,0.0,527) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (cvmfs,788792,2368,0.0,549) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (cvmfs,788792,2404,0.0,556) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,788792,2328,0.0,563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,564) [kworker/0:1H] (cvmfs,788656,5056,0.0,570) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (cvmfs,788792,2368,0.0,575) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,3074196,36232,0.0,583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,788792,2388,0.0,588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,76900,2324,0.0,592) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1088,0.0,596) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (root,64032,24052,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.2,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,900) [kworker/u33:2] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1305) [bond0] (root,10079220,278788,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,0,0,0.0,1493) [kworker/10:0] (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,747028,16768,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1375392,36704,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2698) [kworker/7:2] (root,0,0,0.0,2956) [kworker/13:0] (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,591424,14560,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,4874) [kworker/11:2] (root,0,0,0.0,5371) [kworker/3:0] (root,0,0,0.0,5441) [kworker/14:1] (root,0,0,0.0,5575) [kworker/4:1] (root,0,0,0.0,5899) [kworker/14:2] (root,0,0,0.0,6477) [kworker/1:1] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,7827) [kworker/4:2] (root,0,0,0.0,8356) [kworker/6:1] (root,0,0,0.0,8970) [kworker/8:1] (root,0,0,0.0,9001) [kworker/u33:1] (root,0,0,0.0,9871) [kworker/2:2] (root,0,0,0.0,11038) [kworker/15:0] (postfix,89912,4032,0.0,11564) pickup -l -t unix -u (root,0,0,0.0,12065) [kworker/10:1] (root,0,0,0.0,13158) [kworker/u32:0] (root,0,0,0.0,13834) [kworker/0:0] (cvmfs,858848,27552,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,857732,28364,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,14574) [kworker/1:0] (cvmfs,3143724,62728,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,857816,27036,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,857732,27092,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,857744,27060,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,857596,30248,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,858848,27452,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,858320,27400,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11300,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6772,0.0,15641) condor_shared_port (condor,3165100,3052652,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,16147) [kworker/12:0] (root,0,0,0.0,16399) [kworker/0:2] (root,0,0,0.0,17187) [kworker/9:0] (root,0,0,0.0,17243) [kworker/13:1] (root,0,0,0.0,17609) [kworker/5:2] (root,0,0,0.0,18944) [kworker/8:2] (root,0,0,0.0,18965) [kworker/9:1] (root,0,0,0.0,21589) [kworker/11:1] (root,0,0,0.0,21953) [kworker/7:0] (root,0,0,0.0,22400) [kworker/12:2] (root,0,0,0.0,22548) [kworker/12:1] (root,0,0,0.0,25400) [kworker/u34:1] (root,0,0,0.0,26621) [kworker/15:1] (root,0,0,0.0,28287) [kworker/6:0] (root,0,0,0.0,28426) [kworker/u34:2] (root,108056,360,0.0,29056) sleep 5 (root,8632,664,0.0,29057) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,29058) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,29075) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,944,0.0,29076) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,29370) [kworker/u32:2] (root,0,0,0.0,29835) [kworker/2:1] (root,0,0,0.0,31238) [kworker/5:0]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09acf8e754e9
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (cvmfs,1305864,11528,0.0,450) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,79408,4732,0.0,454) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1096,0.0,458) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,788792,2364,0.0,516) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,788792,2620,0.0,527) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (cvmfs,788792,2368,0.0,549) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (cvmfs,788792,2404,0.0,556) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,788792,2328,0.0,563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,564) [kworker/0:1H] (cvmfs,788656,5056,0.0,570) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (cvmfs,788792,2368,0.0,575) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,3074196,36232,0.0,583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,788792,2388,0.0,588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,76900,2324,0.0,592) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1088,0.0,596) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (root,104992,59592,0.0,654) /usr/lib/systemd/systemd-journald (root,0,0,0.0,664) [kworker/7:1] (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.2,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1305) [bond0] (root,10079220,278568,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,0,0,0.0,1664) [kworker/15:1] (root,0,0,0.0,1767) [kworker/11:0] (root,747028,36616,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1374888,36196,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2622) [kworker/13:2] (root,0,0,0.0,2703) [kworker/2:0] (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,0,0,0.0,3159) [kworker/1:0] (root,591424,14560,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,4408) [kworker/3:0] (root,0,0,0.0,4695) [kworker/u33:1] (root,0,0,0.0,5718) [kworker/9:1] (root,0,0,0.0,5792) [kworker/7:2] (root,0,0,0.0,5905) [kworker/u32:1] (root,0,0,0.0,5978) [kworker/u34:1] (root,0,0,0.0,6490) [kworker/0:1] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,8153) [kworker/4:2] (root,0,0,0.0,8633) [kworker/7:0] (root,0,0,0.0,9215) [kworker/6:0] (root,0,0,0.0,9917) [kworker/8:2] (root,0,0,0.0,10111) [kworker/10:1] (root,0,0,0.0,11278) [kworker/4:0] (root,0,0,0.0,12122) [kworker/5:2] (root,108056,356,0.0,12611) sleep 5 (root,8632,660,0.0,12612) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,12613) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,12630) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,12631) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (cvmfs,858848,27152,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,857732,28204,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,3143724,62548,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,857732,27088,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,14827) [kworker/12:0] (root,0,0,0.0,14956) [kworker/6:1] (cvmfs,857744,27092,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (root,0,0,0.0,15027) [kworker/4:1] (cvmfs,857744,27056,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,857608,30160,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,858848,27080,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (root,0,0,0.0,15572) [kworker/10:0] (cvmfs,858320,27088,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11300,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6772,0.0,15641) condor_shared_port (condor,3074072,2962236,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,17898) [kworker/14:1] (root,0,0,0.0,18485) [kworker/13:1] (root,0,0,0.0,18810) [kworker/9:2] (root,0,0,0.0,20357) [kworker/15:2] (root,0,0,0.0,20595) [kworker/u32:0] (root,0,0,0.0,20803) [kworker/14:0] (root,0,0,0.0,21499) [kworker/8:0] (root,0,0,0.0,22548) [kworker/12:1] (root,0,0,0.0,23186) [kworker/0:0] (root,0,0,0.0,27076) [kworker/2:1] (root,0,0,0.0,27508) [kworker/1:2] (root,0,0,0.0,28149) [kworker/3:2] (root,0,0,0.0,29252) [kworker/11:2] (root,0,0,0.0,30045) [kworker/u34:2] (postfix,89912,4032,0.0,30342) pickup -l -t unix -u (root,0,0,0.0,30346) [kworker/u32:2] (root,0,0,0.0,32325) [kworker/u33:0] (root,0,0,0.0,32613) [kworker/5:1]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac7c809a6e
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (cvmfs,1305864,11528,0.0,450) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,79408,4732,0.0,454) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1096,0.0,458) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,788792,2364,0.0,516) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,788792,2620,0.0,527) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (cvmfs,788792,2368,0.0,549) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (cvmfs,788792,2404,0.0,556) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,788792,2328,0.0,563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,564) [kworker/0:1H] (cvmfs,788656,5056,0.0,570) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (cvmfs,788792,2368,0.0,575) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,3074196,36232,0.0,583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,788792,2388,0.0,588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,76900,2324,0.0,592) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1088,0.0,596) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (root,55840,16056,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.3,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1305) [bond0] (root,10079220,263304,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,747028,12948,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (root,0,0,0.0,1945) [kworker/2:1] (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1376928,38120,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,0,0,0.0,3623) [kworker/6:0] (root,591424,14560,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,4784) [kworker/11:0] (root,0,0,0.0,4850) [kworker/12:2] (root,0,0,0.0,4906) [kworker/2:0] (root,0,0,0.0,5034) [kworker/7:1] (root,0,0,0.0,5221) [kworker/1:2] (root,0,0,0.0,5508) [kworker/10:1] (postfix,89912,4028,0.0,5756) pickup -l -t unix -u (root,0,0,0.0,5836) [kworker/5:1] (root,0,0,0.0,6095) [kworker/4:0] (root,0,0,0.0,7236) [kworker/9:1H] (root,108056,360,0.0,7449) sleep 5 (root,8632,660,0.0,7450) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,7451) /bin/bash /bin/check_mk_agent (root,51732,1696,0.0,7468) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,7469) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,7615) [kworker/10:0] (root,0,0,0.0,8201) [kworker/u33:0] (root,0,0,0.0,8208) [kworker/3:1] (root,0,0,0.0,8414) [kworker/13:1] (root,0,0,0.0,9068) [kworker/8:0] (root,0,0,0.0,9247) [kworker/3:2] (root,0,0,0.0,9627) [kworker/13:0] (root,0,0,0.0,10587) [kworker/0:1] (root,0,0,0.0,11202) [kworker/5:2] (root,0,0,0.0,11587) [kworker/9:2] (root,0,0,0.0,11705) [kworker/15:0] (cvmfs,858848,26980,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,857732,28012,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,3143724,62060,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,857732,27088,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,857744,27092,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,857732,27064,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,857596,30072,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,858848,26864,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,858040,27216,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11300,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6772,0.0,15641) condor_shared_port (condor,2987012,2875092,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,16145) [kworker/8:2] (root,0,0,0.0,16160) [kworker/11:2] (root,0,0,0.0,16805) [kworker/15:2] (root,0,0,0.0,19108) [kworker/4:1] (root,0,0,0.0,19583) [kworker/7:2] (root,0,0,0.0,21411) [kworker/0:0] (root,0,0,0.0,21604) [kworker/4:2] (root,0,0,0.0,22309) [kworker/u34:0] (root,0,0,0.0,22427) [kworker/12:0] (root,0,0,0.0,23155) [kworker/14:0] (root,0,0,0.0,24049) [kworker/1:0] (root,0,0,0.0,24359) [kworker/6:1] (root,0,0,0.0,24610) [kworker/u32:0] (root,0,0,0.0,24977) [kworker/9:0] (root,0,0,0.0,25980) [kworker/u32:1] (root,0,0,0.0,27277) [kworker/13:2] (root,0,0,0.0,27367) [kworker/u34:1] (root,0,0,0.0,29762) [kworker/14:1] (root,0,0,0.0,29862) [kworker/u33:1] (root,0,0,0.0,32707) [kworker/10:2]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09accf29c770
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (cvmfs,1305864,11528,0.0,450) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,79408,4732,0.0,454) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1096,0.0,458) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,788792,2364,0.0,516) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,788792,2620,0.0,527) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (cvmfs,788792,2368,0.0,549) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (cvmfs,788792,2404,0.0,556) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,788792,2328,0.0,563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,564) [kworker/0:1H] (cvmfs,788656,5056,0.0,570) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (cvmfs,788792,2368,0.0,575) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,3074196,36232,0.0,583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,788792,2388,0.0,588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,76900,2324,0.0,592) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1088,0.0,596) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (root,105024,52816,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.3,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1305) [bond0] (root,10079220,262836,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,746964,32464,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (root,0,0,0.0,2404) [kworker/8:2] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1376928,37028,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2825) [kworker/9:1] (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,0,0,0.0,3319) [kworker/6:2] (root,0,0,0.0,3551) [kworker/15:0] (root,591424,14560,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,4164) [kworker/u33:0] (root,0,0,0.0,5333) [kworker/2:0] (root,0,0,0.0,5707) [kworker/14:2] (root,0,0,0.0,6041) [kworker/10:0] (root,0,0,0.0,6944) [kworker/3:0] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,8054) [kworker/6:1] (root,0,0,0.0,8811) [kworker/1:1] (root,0,0,0.0,10363) [kworker/u32:0] (root,0,0,0.0,10841) [kworker/14:0] (root,0,0,0.0,10971) [kworker/4:2] (root,0,0,0.0,11030) [kworker/9:0] (root,0,0,0.0,11150) [kworker/11:1] (root,0,0,0.0,13659) [kworker/12:0] (root,0,0,0.0,13892) [kworker/3:1] (cvmfs,858320,27120,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,857736,27868,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,3143444,62340,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,857732,27088,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,14961) [kworker/u32:2] (cvmfs,857732,27092,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,857732,27064,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,15199) [kworker/5:1] (cvmfs,857596,30012,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,858320,27188,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,858040,27104,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11300,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6772,0.0,15641) condor_shared_port (condor,2896740,2784840,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,16011) [kworker/5:2] (root,0,0,0.0,16087) [kworker/u34:0] (root,0,0,0.0,16499) [kworker/2:1] (root,0,0,0.0,16725) [kworker/10:2] (root,0,0,0.0,17502) [kworker/0:2] (root,108056,356,0.0,17810) sleep 5 (root,8632,664,0.0,17811) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,17812) /bin/bash /bin/check_mk_agent (root,51732,1688,0.0,17829) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,17830) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (postfix,89912,4032,0.0,18240) pickup -l -t unix -u (root,0,0,0.0,18952) [kworker/u34:2] (root,0,0,0.0,19108) [kworker/4:1] (root,0,0,0.0,19518) [kworker/12:1] (root,0,0,0.0,19557) [kworker/13:1] (root,0,0,0.0,20028) [kworker/0:0] (root,0,0,0.0,20886) [kworker/8:0] (root,0,0,0.0,21055) [kworker/u33:1] (root,0,0,0.0,21530) [kworker/7:1] (root,0,0,0.0,21867) [kworker/12:2] (root,0,0,0.0,23088) [kworker/15:1] (root,0,0,0.0,23237) [kworker/6:0] (root,0,0,0.0,23428) [kworker/2:2] (root,0,0,0.0,23451) [kworker/4:0] (root,0,0,0.0,25060) [kworker/14:1] (root,0,0,0.0,25274) [kworker/13:2] (root,0,0,0.0,26905) [kworker/7:2] (root,0,0,0.0,28747) [kworker/11:0] (root,0,0,0.0,28943) [kworker/1:0]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac40684a07
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,344) [kworker/u34:2] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,440) [kworker/u32:1] (cvmfs,1305864,11528,0.0,450) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,79408,4732,0.0,454) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1096,0.0,458) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,788792,2364,0.0,516) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,788792,2620,0.0,527) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (cvmfs,788792,2368,0.0,549) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (cvmfs,788792,2404,0.0,556) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,788792,2328,0.0,563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,564) [kworker/0:1H] (cvmfs,788656,5056,0.0,570) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (cvmfs,788792,2368,0.0,575) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,3074196,36232,0.0,583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,788792,2388,0.0,588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,76900,2324,0.0,592) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1088,0.0,596) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (root,39456,7924,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,108056,360,0.0,684) sleep 5 (root,8632,660,0.0,685) timeout -s 9 360 check_mk_agent (root,11696,1532,0.0,695) /bin/bash /bin/check_mk_agent (root,51732,1688,0.0,712) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,713) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,1.3,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,877) [kworker/13:2] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1305) [bond0] (root,10079220,256404,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,747028,8120,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (root,0,0,0.0,2085) [kworker/5:1] (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2249) [kworker/u32:0] (root,0,0,0.0,2296) [kworker/3:1H] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1375920,37092,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2758) [kworker/5:0] (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,591424,14560,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,4041) [kworker/14:1] (root,0,0,0.0,4927) [kworker/2:1] (root,0,0,0.0,6554) [kworker/6:2] (root,0,0,0.0,6913) [kworker/u33:2] (root,0,0,0.0,7117) [kworker/8:2] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,9331) [kworker/14:0] (root,0,0,0.0,10618) [kworker/1:0] (root,0,0,0.0,10797) [kworker/11:2] (root,0,0,0.0,11444) [kworker/3:2] (postfix,89912,4032,0.0,11907) pickup -l -t unix -u (root,0,0,0.0,12477) [kworker/9:1] (root,0,0,0.0,12979) [kworker/3:0] (root,0,0,0.0,14077) [kworker/7:2] (cvmfs,858320,26684,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,857732,27732,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,3143296,62392,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,857732,27088,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,14916) [kworker/8:1] (root,0,0,0.0,14924) [kworker/2:0] (cvmfs,857744,27088,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,857732,27064,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,857600,29928,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,858320,26900,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,857892,26988,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11300,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6772,0.0,15641) condor_shared_port (condor,2803732,2691256,0.0,15652) condor_startd (root,0,0,0.0,15681) [kworker/11:1] (root,0,0,0.0,15804) [kworker/7:0] (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,17153) [kworker/0:0] (root,0,0,0.0,17767) [kworker/0:2] (root,0,0,0.0,18315) [kworker/1:2] (root,0,0,0.0,18751) [kworker/9:2] (root,0,0,0.0,19518) [kworker/12:1] (root,0,0,0.0,20039) [kworker/10:1] (root,0,0,0.0,20568) [kworker/11:0] (root,0,0,0.0,21352) [kworker/15:2] (root,0,0,0.0,23237) [kworker/6:0] (root,0,0,0.0,23257) [kworker/7:1] (root,0,0,0.0,24534) [kworker/15:1] (root,0,0,0.0,24779) [kworker/10:2] (root,0,0,0.0,25571) [kworker/12:0] (root,0,0,0.0,26076) [kworker/15:0] (root,0,0,0.0,28228) [kworker/4:1] (root,0,0,0.0,28800) [kworker/12:2] (root,0,0,0.0,30378) [kworker/13:1] (root,0,0,0.0,30761) [kworker/u33:1] (root,0,0,0.0,31167) [kworker/4:2] (root,0,0,0.0,31595) [kworker/14:2] (root,0,0,0.0,32338) [kworker/u34:0]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09aca6691792
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (cvmfs,1305864,11528,0.0,450) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,79408,4732,0.0,454) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1096,0.0,458) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,788792,2364,0.0,516) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,788792,2620,0.0,527) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (cvmfs,788792,2368,0.0,549) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (cvmfs,788792,2404,0.0,556) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,788792,2328,0.0,563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,564) [kworker/0:1H] (cvmfs,788656,5056,0.0,570) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (cvmfs,788792,2368,0.0,575) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,3074196,36232,0.0,583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,788792,2388,0.0,588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,76900,2324,0.0,592) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1088,0.0,596) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (root,80416,41464,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.3,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1305) [bond0] (root,10079220,252448,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,747028,25920,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (root,0,0,0.0,2364) [kworker/u34:2] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1375920,36028,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,0,0,0.0,3454) [kworker/15:1] (root,591424,14560,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,4684) [kworker/1:2] (root,0,0,0.0,6286) [kworker/9:2] (root,0,0,0.0,6692) [kworker/10:2] (root,0,0,0.0,6980) [kworker/3:2] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,10575) [kworker/7:2] (root,0,0,0.0,13411) [kworker/13:2] (root,0,0,0.0,13623) [kworker/5:0] (root,0,0,0.0,13648) [kworker/2:1] (root,0,0,0.0,14168) [kworker/u33:0] (cvmfs,858040,26804,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,857732,27548,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,14637) [kworker/11:2] (cvmfs,3143180,62336,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,857732,27088,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,14839) [kworker/0:0] (cvmfs,857736,27092,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,857732,27052,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,857600,29860,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,858040,26836,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,857776,27020,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11300,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6772,0.0,15641) condor_shared_port (condor,2711784,2599712,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,16186) [kworker/15:0] (root,0,0,0.0,16316) [kworker/4:1] (root,0,0,0.0,17336) [kworker/14:0] (root,0,0,0.0,17382) [kworker/2:0] (root,0,0,0.0,18118) [kworker/8:2] (root,0,0,0.0,18651) [kworker/6:1] (root,0,0,0.0,19674) [kworker/8:0] (root,0,0,0.0,20323) [kworker/u34:1] (root,0,0,0.0,20869) [kworker/1:0] (root,0,0,0.0,21397) [kworker/9:1] (root,0,0,0.0,22506) [kworker/12:1] (root,0,0,0.0,24500) [kworker/u33:1] (postfix,89912,4032,0.0,25007) pickup -l -t unix -u (root,0,0,0.0,26065) [kworker/u32:1] (root,0,0,0.0,26557) [kworker/2:2] (root,0,0,0.0,27268) [kworker/6:2] (root,0,0,0.0,27969) [kworker/10:0] (root,0,0,0.0,28225) [kworker/12:0] (root,0,0,0.0,29287) [kworker/3:0] (root,0,0,0.0,30095) [kworker/3:1] (root,0,0,0.0,30292) [kworker/4:0] (root,0,0,0.0,30337) [kworker/7:0] (root,0,0,0.0,30378) [kworker/13:1] (root,0,0,0.0,30394) [kworker/0:1] (root,108056,360,0.0,30657) sleep 5 (root,8632,660,0.0,30658) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,30659) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,30676) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,30677) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,30829) [kworker/u34:0] (root,0,0,0.0,30888) [kworker/u32:0] (root,0,0,0.0,30920) [kworker/5:2] (root,0,0,0.0,32472) [kworker/14:2] (root,0,0,0.0,32487) [kworker/11:1]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac2b4aef60
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (cvmfs,1305864,11528,0.0,450) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,79408,4732,0.0,454) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1096,0.0,458) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,0,0,0.0,489) [kworker/u34:2] (cvmfs,788792,2364,0.0,516) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,788792,2620,0.0,527) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (cvmfs,788792,2368,0.0,549) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (cvmfs,788792,2404,0.0,556) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,788792,2328,0.0,563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,564) [kworker/0:1H] (cvmfs,788656,5056,0.0,570) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (cvmfs,788792,2368,0.0,575) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,3074196,36232,0.0,583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,788792,2388,0.0,588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,76900,2264,0.0,592) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1088,0.0,596) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (root,121376,75748,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.3,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1305) [bond0] (root,0,0,0.0,1388) [kworker/1:1] (root,10079220,249612,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,747028,46436,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (root,0,0,0.0,1921) [kworker/u34:1] (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1374964,35920,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,591424,14560,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,3975) [kworker/6:2] (root,0,0,0.0,4112) [kworker/15:0] (root,0,0,0.0,5434) [kworker/3:1] (root,0,0,0.0,6392) [kworker/13:2] (root,0,0,0.0,6980) [kworker/3:2] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,8948) [kworker/8:0] (root,0,0,0.0,9952) [kworker/u33:0] (root,0,0,0.0,10525) [kworker/14:2] (root,0,0,0.0,11033) [kworker/9:0] (root,0,0,0.0,11691) [kworker/2:1] (root,0,0,0.0,12605) [kworker/8:1] (root,0,0,0.0,12916) [kworker/u32:1] (root,8632,664,0.0,13168) timeout -s 9 360 check_mk_agent (root,11696,1532,0.0,13169) /bin/bash /bin/check_mk_agent (root,108056,360,0.0,13188) sleep 5 (root,51732,1692,0.0,13189) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,13190) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,13509) [kworker/2:2] (root,0,0,0.0,14094) [kworker/14:1] (cvmfs,857776,26828,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,857732,27352,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,14542) [kworker/11:2] (cvmfs,3143136,62268,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,857732,27088,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,14942) [kworker/9:1] (root,0,0,0.0,14960) [kworker/8:2] (cvmfs,857736,27092,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (root,0,0,0.0,15070) [kworker/3:0] (cvmfs,857744,27008,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,857608,29708,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,857776,26788,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,857752,26864,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11300,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6772,0.0,15641) condor_shared_port (condor,2619044,2507160,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,16199) [kworker/0:0] (root,0,0,0.0,16852) [kworker/5:1] (root,0,0,0.0,17973) [kworker/5:2] (postfix,89912,4032,0.0,18221) pickup -l -t unix -u (root,0,0,0.0,18454) [kworker/2:0] (root,0,0,0.0,18547) [kworker/u34:0] (root,0,0,0.0,18693) [kworker/6:1] (root,0,0,0.0,19034) [kworker/15:1] (root,0,0,0.0,19568) [kworker/u32:2] (root,0,0,0.0,19825) [kworker/u33:2] (root,0,0,0.0,20159) [kworker/u32:0] (root,0,0,0.0,20964) [kworker/12:0] (root,0,0,0.0,22506) [kworker/12:1] (root,0,0,0.0,23507) [kworker/10:1] (root,0,0,0.0,23690) [kworker/10:0] (root,0,0,0.0,26325) [kworker/11:1] (root,0,0,0.0,26468) [kworker/13:1] (root,0,0,0.0,26774) [kworker/4:1] (root,0,0,0.0,28161) [kworker/1:2] (root,0,0,0.0,29283) [kworker/0:2] (root,0,0,0.0,30292) [kworker/4:0] (root,0,0,0.0,30337) [kworker/7:0] (root,0,0,0.0,30394) [kworker/7:1]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac93b33e75
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,428) [kworker/11:0] (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (root,0,0,0.0,564) [kworker/0:1H] (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (root,64032,29492,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,0.0,716) [kworker/2:2] (root,0,0,1.4,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1305) [bond0] (root,10079220,248720,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,714260,19776,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (cvmfs,66484,1108,0.0,2045) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,82712,8200,0.0,2051) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1092,0.0,2055) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2263) [kworker/1:0] (root,0,0,0.0,2296) [kworker/3:1H] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1262616,39020,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2931) [kworker/0:2] (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,0,0,0.0,3152) [kworker/10:1] (root,591424,14492,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,3975) [kworker/6:2] (root,0,0,0.0,5763) [kworker/7:0] (root,0,0,0.0,6392) [kworker/13:2] (root,0,0,0.0,6394) [kworker/4:1] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,9172) [kworker/15:1] (root,0,0,0.0,9589) [kworker/14:0] (root,0,0,0.0,9848) [kworker/3:0] (root,0,0,0.0,10166) [kworker/11:2] (root,0,0,0.0,10972) [kworker/u32:2] (root,0,0,0.0,11203) [kworker/14:2] (root,0,0,0.0,11501) [kworker/8:2] (root,0,0,0.0,12018) [kworker/u32:1] (root,0,0,0.0,12522) [kworker/1:2] (root,108056,356,0.0,12574) sleep 5 (root,8632,660,0.0,12575) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,12576) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,12593) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,12594) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,12662) [kworker/7:2] (root,0,0,0.0,12799) [kworker/8:1] (cvmfs,66484,1108,0.0,14343) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,797628,24340,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66484,1108,0.0,14498) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,796516,25820,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,66484,1108,0.0,14639) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,3180236,61700,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,66484,1112,0.0,14789) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,77012,2552,0.0,14793) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1092,0.0,14797) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,796512,23656,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,66484,1108,0.0,14963) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,796516,23680,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,66484,1108,0.0,15152) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,796524,23656,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,66484,1108,0.0,15294) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,796376,26860,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,66484,1108,0.0,15435) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,796820,23812,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,66484,1108,0.0,15576) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,797628,23932,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,15615) [kworker/6:1] (condor,132908,11300,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6772,0.0,15641) condor_shared_port (condor,2527164,2415280,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,15961) [kworker/u34:2] (root,0,0,0.0,17245) [kworker/0:0] (root,0,0,0.0,17973) [kworker/5:2] (root,0,0,0.0,18028) [kworker/15:0] (root,0,0,0.0,18435) [kworker/5:1] (root,0,0,0.0,19474) [kworker/2:1] (root,0,0,0.0,19504) [kworker/10:2] (root,0,0,0.0,19588) [kworker/u34:0] (root,0,0,0.0,20131) [kworker/u33:0] (root,0,0,0.0,20615) [kworker/u33:1] (root,0,0,0.0,22034) [kworker/3:1] (root,0,0,0.0,25386) [kworker/14:1] (root,0,0,0.0,26047) [kworker/12:2] (root,0,0,0.0,26717) [kworker/13:1] (root,0,0,0.0,27104) [kworker/7:1] (root,0,0,0.0,29892) [kworker/0:1] (root,0,0,0.0,31237) [kworker/9:1] (postfix,89912,4028,0.0,31445) pickup -l -t unix -u (root,0,0,0.0,31573) [kworker/4:2] (root,0,0,0.0,31758) [kworker/12:1] (root,0,0,0.0,32218) [kworker/15:2] (root,0,0,0.0,32673) [kworker/9:0]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09acb580c88f
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (root,0,0,0.0,564) [kworker/0:1H] (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (root,113184,64232,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.4,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,108056,356,0.0,1090) sleep 5 (root,8632,664,0.0,1091) timeout -s 9 360 check_mk_agent (root,11696,1532,0.0,1092) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,1114) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,1115) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,1305) [bond0] (root,0,0,0.0,1347) [kworker/11:1] (root,10077168,248216,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,730616,38860,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (cvmfs,66484,1108,0.0,2045) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,82712,8200,0.0,2051) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1092,0.0,2055) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2119) [kworker/u33:1] (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1262616,37864,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,0,0,0.0,3320) [kworker/12:2] (root,591424,14492,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,3982) [kworker/8:0] (root,0,0,0.0,4187) [kworker/u33:0] (root,0,0,0.0,4207) [kworker/6:0] (root,0,0,0.0,4363) [kworker/u34:2] (root,0,0,0.0,5349) [kworker/7:1] (root,0,0,0.0,5812) [kworker/10:1] (root,0,0,0.0,6392) [kworker/13:2] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,7570) [kworker/0:1] (root,0,0,0.0,10369) [kworker/6:2] (root,0,0,0.0,12529) [kworker/0:0] (root,0,0,0.0,13691) [kworker/13:0] (root,0,0,0.0,13766) [kworker/15:1] (cvmfs,66484,1108,0.0,14343) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,797628,24340,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66484,1108,0.0,14498) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,796512,25820,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,66484,1108,0.0,14639) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,3180236,61688,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (root,0,0,0.0,14669) [kworker/9:0] (cvmfs,66484,1112,0.0,14789) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,77012,2552,0.0,14793) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1092,0.0,14797) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,796512,23656,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,14918) [kworker/9:2] (cvmfs,66484,1108,0.0,14963) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,796512,23684,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (root,0,0,0.0,15003) [kworker/12:1] (cvmfs,66484,1108,0.0,15152) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,796524,23660,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,66484,1108,0.0,15294) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,796388,26768,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,66484,1108,0.0,15435) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,796672,23556,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,66484,1108,0.0,15576) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,797628,23820,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11296,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6764,0.0,15641) condor_shared_port (condor,2433072,2320632,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,16402) [kworker/5:0] (root,0,0,0.0,17973) [kworker/5:2] (root,0,0,0.0,18298) [kworker/8:1] (root,0,0,0.0,18769) [kworker/10:2] (root,0,0,0.0,20523) [kworker/11:2] (root,0,0,0.0,21143) [kworker/4:1] (root,0,0,0.0,21165) [kworker/u32:1] (root,0,0,0.0,21438) [kworker/u32:2] (root,0,0,0.0,21708) [kworker/15:2] (root,0,0,0.0,21794) [kworker/4:0] (root,0,0,0.0,22296) [kworker/1:2] (root,0,0,0.0,22305) [kworker/u34:0] (root,0,0,0.0,23630) [kworker/2:1] (root,0,0,0.0,23789) [kworker/12:0] (root,0,0,0.0,23791) [kworker/11:0] (postfix,89912,4032,0.0,24398) pickup -l -t unix -u (root,0,0,0.0,25604) [kworker/2:0] (root,0,0,0.0,26957) [kworker/4:2] (root,0,0,0.0,29087) [kworker/3:2] (root,0,0,0.0,29747) [kworker/1:1] (root,0,0,0.0,30685) [kworker/7:0] (root,0,0,0.0,30898) [kworker/3:1] (root,0,0,0.0,31579) [kworker/14:1] (root,0,0,0.0,31653) [kworker/14:0] (root,0,0,0.0,31687) [kworker/u32:0]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac3f01a8bd
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (root,0,0,0.0,564) [kworker/0:1H] (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (root,55840,19708,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.4,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1305) [bond0] (root,10077168,247820,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,673280,14340,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (cvmfs,66484,1108,0.0,2045) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,82712,8200,0.0,2051) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1092,0.0,2055) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1262616,36688,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2746) [kworker/11:2] (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,0,0,0.0,3143) [kworker/14:2] (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,591424,14492,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,4795) [kworker/6:0] (root,0,0,0.0,6392) [kworker/13:2] (root,0,0,0.0,6725) [kworker/6:2] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,7565) [kworker/14:1] (root,108056,360,0.0,8936) sleep 5 (root,8632,660,0.0,8937) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,8938) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,8955) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,8956) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,9297) [kworker/14:0] (root,0,0,0.0,10664) [kworker/13:1] (root,0,0,0.0,10867) [kworker/3:1] (root,0,0,0.0,11081) [kworker/3:2] (root,0,0,0.0,11230) [kworker/7:0] (root,0,0,0.0,11407) [kworker/11:0] (root,0,0,0.0,11689) [kworker/4:2] (root,0,0,0.0,13395) [kworker/8:1] (root,0,0,0.0,13938) [kworker/0:2] (cvmfs,66484,1108,0.0,14343) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,797628,23772,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66484,1108,0.0,14498) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,796512,25820,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,66484,1108,0.0,14639) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,3180236,61692,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,66484,1112,0.0,14789) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,77012,2552,0.0,14793) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1092,0.0,14797) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,796512,23656,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,66484,1108,0.0,14963) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,796512,23676,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,66484,1108,0.0,15152) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,796512,23664,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,66484,1108,0.0,15294) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,796380,26708,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,66484,1108,0.0,15435) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,797628,23936,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,66484,1108,0.0,15576) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,797628,23644,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11296,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6764,0.0,15641) condor_shared_port (condor,2345540,2233608,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,16172) [kworker/9:0] (root,0,0,0.0,16345) [kworker/10:1] (root,0,0,0.0,16532) [kworker/u34:1] (root,0,0,0.0,16868) [kworker/2:2] (root,0,0,0.0,16973) [kworker/12:0] (root,0,0,0.0,17269) [kworker/u33:2] (root,0,0,0.0,17670) [kworker/15:1] (root,0,0,0.0,17922) [kworker/10:0] (root,0,0,0.0,19663) [kworker/1:2] (root,0,0,0.0,19687) [kworker/5:2] (root,0,0,0.0,20392) [kworker/2:0] (root,0,0,0.0,21214) [kworker/4:0] (root,0,0,0.0,21450) [kworker/8:2] (root,0,0,0.0,21940) [kworker/u33:0] (root,0,0,0.0,22470) [kworker/u34:2] (root,0,0,0.0,23906) [kworker/7:1] (root,0,0,0.0,24383) [kworker/u32:1] (root,0,0,0.0,24597) [kworker/5:0] (root,0,0,0.0,25267) [kworker/9:1] (root,0,0,0.0,26095) [kworker/12:2] (root,0,0,0.0,26995) [kworker/6:1] (root,0,0,0.0,27664) [kworker/15:2] (root,0,0,0.0,27997) [kworker/4:1] (root,0,0,0.0,28630) [kworker/0:1] (root,0,0,0.0,29787) [kworker/u32:2] (postfix,89912,4028,0.0,31045) pickup -l -t unix -u (root,0,0,0.0,32213) [kworker/1:0]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac3f5454bc
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (root,0,0,0.0,564) [kworker/0:1H] (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (root,96800,52540,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.5,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,958) [kworker/6:2] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1305) [bond0] (root,10077168,247140,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,0,0,0.0,1647) [kworker/5:2] (root,681492,32232,0.0,1826) /usr/sbin/rsyslogd -n (root,0,0,0.0,1833) [kworker/8:0] (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (cvmfs,66484,1108,0.0,2045) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,82712,8200,0.0,2051) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1092,0.0,2055) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (root,0,0,0.0,2575) [kworker/2:2] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1262616,35556,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,0,0,0.0,3211) [kworker/0:1] (root,0,0,0.0,3450) [kworker/13:1] (root,591424,14492,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,5528) [kworker/8:1] (root,0,0,0.0,6606) [kworker/7:2] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,7476) [kworker/2:1] (root,0,0,0.0,8764) [kworker/1:0] (root,0,0,0.0,8929) [kworker/4:0] (root,0,0,0.0,8974) [kworker/u32:0] (root,0,0,0.0,9115) [kworker/3:2] (root,0,0,0.0,10162) [kworker/14:2] (root,0,0,0.0,11826) [kworker/u34:2] (root,0,0,0.0,12868) [kworker/0:0] (root,0,0,0.0,13671) [kworker/12:1] (root,0,0,0.0,14026) [kworker/u34:0] (cvmfs,66484,1108,0.0,14343) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,797628,23548,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66484,1108,0.0,14498) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,796512,25820,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,66484,1108,0.0,14639) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,3180236,61688,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,66484,1112,0.0,14789) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,77012,2552,0.0,14793) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1092,0.0,14797) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,796512,23656,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,66484,1108,0.0,14963) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,796524,23672,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,66484,1108,0.0,15152) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,796524,23656,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,66484,1108,0.0,15294) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,796388,26596,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,66484,1108,0.0,15435) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,797628,23640,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (root,0,0,0.0,15533) [kworker/u32:2] (cvmfs,66484,1108,0.0,15576) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,797100,23932,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11292,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6764,0.0,15641) condor_shared_port (condor,2250084,2137608,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,17590) [kworker/9:1] (root,0,0,0.0,18168) [kworker/3:1] (root,0,0,0.0,21342) [kworker/4:1] (root,0,0,0.0,21687) [kworker/7:0] (root,0,0,0.0,21729) [kworker/12:2] (root,0,0,0.0,21820) [kworker/u33:0] (root,0,0,0.0,21859) [kworker/15:2] (root,0,0,0.0,22088) [kworker/14:1] (root,0,0,0.0,22272) [kworker/10:2] (root,0,0,0.0,23198) [kworker/9:2] (root,0,0,0.0,24116) [kworker/5:0] (postfix,89912,4032,0.0,24684) pickup -l -t unix -u (root,0,0,0.0,25020) [kworker/10:1] (root,0,0,0.0,25106) [kworker/5:1] (root,0,0,0.0,25916) [kworker/15:0] (root,0,0,0.0,27743) [kworker/11:1] (root,108056,360,0.0,28359) sleep 5 (root,8632,660,0.0,28360) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,28361) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,28378) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,28379) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,28446) [kworker/1:1] (root,0,0,0.0,29241) [kworker/4:2] (root,0,0,0.0,29255) [kworker/6:0] (root,0,0,0.0,29326) [kworker/11:2] (root,0,0,0.0,30971) [kworker/u33:2] (root,0,0,0.0,32143) [kworker/13:0]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac7efe04a7
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,330) [kworker/7:0] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (root,0,0,0.0,564) [kworker/0:1H] (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (root,47648,9736,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.5,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1768,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1305) [bond0] (root,10077168,246564,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,0,0,0.0,1643) [kworker/5:0] (root,632340,9284,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (cvmfs,66484,1108,0.0,2045) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,82712,8200,0.0,2051) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1092,0.0,2055) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1262616,34372,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2931) [kworker/11:0] (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,0,0,0.0,3218) [kworker/10:1] (root,0,0,0.0,3477) [kworker/4:0] (root,0,0,0.0,3659) [kworker/0:0] (root,0,0,0.0,3808) [kworker/u33:0] (root,0,0,0.0,3892) [kworker/9:2] (root,591424,14492,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,4420) [kworker/u34:0] (root,0,0,0.0,5290) [kworker/15:1] (postfix,89912,4032,0.0,5618) pickup -l -t unix -u (root,108056,356,0.0,6622) sleep 5 (root,8632,660,0.0,6623) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,6624) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,6641) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,6642) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,7882) [kworker/8:2] (root,0,0,0.0,9115) [kworker/3:2] (root,0,0,0.0,9762) [kworker/4:1] (root,0,0,0.0,10370) [kworker/14:1] (root,0,0,0.0,11025) [kworker/9:0] (root,0,0,0.0,14119) [kworker/10:0] (root,0,0,0.0,14314) [kworker/12:1] (cvmfs,66484,1108,0.0,14343) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,797100,23868,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66484,1108,0.0,14498) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,796512,25816,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,66484,1108,0.0,14639) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,3180236,60072,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,66484,1112,0.0,14789) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,77012,2552,0.0,14793) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1092,0.0,14797) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,796512,23656,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,66484,1108,0.0,14963) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,796532,23672,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,66484,1108,0.0,15152) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,796512,23652,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,66484,1108,0.0,15294) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,796396,26480,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,15363) [kworker/3:1] (cvmfs,66484,1108,0.0,15435) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,797100,23940,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,66484,1108,0.0,15576) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,797100,23672,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11292,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6764,0.0,15641) condor_shared_port (condor,2161272,2049232,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,15896) [kworker/6:0] (root,0,0,0.0,16062) [kworker/u34:1] (root,0,0,0.0,16136) [kworker/13:1] (root,0,0,0.0,18696) [kworker/u32:0] (root,0,0,0.0,19087) [kworker/8:1] (root,0,0,0.0,19395) [kworker/1:0] (root,0,0,0.0,19852) [kworker/6:1] (root,0,0,0.0,20507) [kworker/u32:1] (root,0,0,0.0,20543) [kworker/1:2] (root,0,0,0.0,20886) [kworker/15:0] (root,0,0,0.0,23661) [kworker/11:1] (root,0,0,0.0,23792) [kworker/2:2] (root,0,0,0.0,24433) [kworker/5:1] (root,0,0,0.0,26060) [kworker/15:2] (root,0,0,0.0,26431) [kworker/13:2] (root,0,0,0.0,28373) [kworker/7:2] (root,0,0,0.0,29675) [kworker/2:0] (root,0,0,0.0,30082) [kworker/14:2] (root,0,0,0.0,30084) [kworker/12:0] (root,0,0,0.0,32251) [kworker/0:2] (root,0,0,0.0,32293) [kworker/u33:2]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09acc93d7840
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (root,0,0,0.0,564) [kworker/0:1H] (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (root,88608,44612,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.5,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1764,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1104) [kworker/u33:1] (root,0,0,0.0,1305) [bond0] (root,10077168,246064,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,0,0,0.0,1663) [kworker/2:1] (root,640532,27452,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (cvmfs,66484,1108,0.0,2045) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,82712,8200,0.0,2051) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1092,0.0,2055) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1262616,33276,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2928) [kworker/u32:1] (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,0,0,0.0,3708) [kworker/11:2] (root,0,0,0.0,3897) [kworker/u34:0] (root,591424,14488,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,4339) [kworker/10:2] (root,0,0,0.0,4621) [kworker/0:2] (root,0,0,0.0,4633) [kworker/7:2] (root,0,0,0.0,5427) [kworker/2:2] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,9287) [kworker/9:2] (root,0,0,0.0,10336) [kworker/u32:2] (root,0,0,0.0,12850) [kworker/9:1] (root,0,0,0.0,13711) [kworker/14:1] (cvmfs,66484,1108,0.0,14343) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,797100,23440,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66484,1108,0.0,14498) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,796512,25820,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,66484,1108,0.0,14639) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,3180236,61696,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,66484,1112,0.0,14789) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,77012,2552,0.0,14793) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1092,0.0,14797) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,796512,23656,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,66484,1108,0.0,14963) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,796516,23676,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,66484,1108,0.0,15152) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,796532,23648,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,66484,1108,0.0,15294) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,796396,26388,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,66484,1108,0.0,15435) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,797100,23480,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,66484,1108,0.0,15576) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,796820,23860,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11292,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6764,0.0,15641) condor_shared_port (condor,2069600,1957680,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,16112) [kworker/3:0] (root,0,0,0.0,16892) [kworker/4:2] (root,0,0,0.0,18201) [kworker/0:0] (root,0,0,0.0,19527) [kworker/1:1] (root,0,0,0.0,19917) [kworker/11:1] (root,0,0,0.0,20355) [kworker/u33:0] (root,0,0,0.0,20588) [kworker/14:0] (root,0,0,0.0,20848) [kworker/8:2] (root,0,0,0.0,21014) [kworker/12:2] (root,0,0,0.0,21438) [kworker/8:1] (root,0,0,0.0,21472) [kworker/8:0] (root,108056,356,0.0,22809) sleep 5 (root,8632,664,0.0,22810) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,22811) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,22828) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,944,0.0,22829) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,23148) [kworker/5:2] (postfix,89912,4028,0.0,23504) pickup -l -t unix -u (root,0,0,0.0,24148) [kworker/6:1] (root,0,0,0.0,24403) [kworker/u34:2] (root,0,0,0.0,24516) [kworker/3:2] (root,0,0,0.0,24872) [kworker/10:0] (root,0,0,0.0,25412) [kworker/5:1] (root,0,0,0.0,26500) [kworker/7:0] (root,0,0,0.0,27686) [kworker/13:1] (root,0,0,0.0,28038) [kworker/15:0] (root,0,0,0.0,28874) [kworker/4:1] (root,0,0,0.0,29406) [kworker/13:0] (root,0,0,0.0,29997) [kworker/3:1] (root,0,0,0.0,30545) [kworker/1:0] (root,0,0,0.0,31355) [kworker/6:2] (root,0,0,0.0,32015) [kworker/12:1] (root,0,0,0.0,32059) [kworker/15:1]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac708ad2ec
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (root,0,0,0.0,564) [kworker/0:1H] (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (root,129568,79232,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.6,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,108056,356,0.0,775) sleep 5 (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,8632,660,0.0,847) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,848) /bin/bash /bin/check_mk_agent (root,51732,1676,0.0,865) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,866) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,942) [kworker/u32:0] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1764,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1240) [kworker/u34:0] (root,0,0,0.0,1305) [bond0] (root,10077168,245524,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,0,0,0.0,1499) [kworker/9:0] (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,648724,48524,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (cvmfs,66484,1108,0.0,2045) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,82712,8200,0.0,2051) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1092,0.0,2055) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (root,0,0,0.0,2344) [kworker/5:0] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1262616,32052,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2751) [kworker/7:0] (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,0,0,0.0,3615) [kworker/10:2] (root,591424,14488,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,4399) [kworker/9:1] (root,0,0,0.0,4660) [kworker/0:1] (root,0,0,0.0,5947) [kworker/u32:1] (root,0,0,0.0,6320) [kworker/4:0] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,7792) [kworker/11:0] (root,0,0,0.0,8514) [kworker/15:2] (postfix,89912,4028,0.0,8853) pickup -l -t unix -u (root,0,0,0.0,9169) [kworker/10:1] (root,0,0,0.0,9932) [kworker/3:0] (root,0,0,0.0,11298) [kworker/2:1] (root,0,0,0.0,11407) [kworker/0:2] (root,0,0,0.0,11690) [kworker/u33:2] (cvmfs,66484,1108,0.0,14343) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,796820,23660,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (root,0,0,0.0,14432) [kworker/11:2] (cvmfs,66484,1108,0.0,14498) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,796512,25820,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,66484,1108,0.0,14639) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,3180236,61592,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (root,0,0,0.0,14701) [kworker/6:0] (cvmfs,66484,1112,0.0,14789) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,77012,2552,0.0,14793) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1092,0.0,14797) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,796512,23656,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,66484,1108,0.0,14963) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,796512,23640,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (root,0,0,0.0,15115) [kworker/8:2] (cvmfs,66484,1108,0.0,15152) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,796512,23652,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,66484,1108,0.0,15294) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,796376,26348,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,66484,1108,0.0,15435) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,796820,23464,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,66484,1108,0.0,15576) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,796820,23540,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11292,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6764,0.0,15641) condor_shared_port (condor,1978716,1866252,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,17686) [kworker/12:0] (root,0,0,0.0,17885) [kworker/5:2] (root,0,0,0.0,20526) [kworker/9:2] (root,0,0,0.0,20586) [kworker/u34:1] (root,0,0,0.0,21917) [kworker/7:1] (root,0,0,0.0,22151) [kworker/3:2] (root,0,0,0.0,22230) [kworker/13:2] (root,0,0,0.0,22896) [kworker/14:0] (root,0,0,0.0,23233) [kworker/4:1] (root,0,0,0.0,24148) [kworker/6:1] (root,0,0,0.0,24485) [kworker/8:0] (root,0,0,0.0,24683) [kworker/14:2] (root,0,0,0.0,25556) [kworker/12:2] (root,0,0,0.0,27518) [kworker/2:2] (root,0,0,0.0,27867) [kworker/1:2] (root,0,0,0.0,28306) [kworker/13:0] (root,0,0,0.0,28817) [kworker/5:1] (root,0,0,0.0,29298) [kworker/4:2] (root,0,0,0.0,31400) [kworker/1:0] (root,0,0,0.0,31543) [kworker/13:1] (root,0,0,0.0,31781) [kworker/u32:2] (root,0,0,0.0,31899) [kworker/15:1] (root,537976,194360,23.6,32333) puppet agent: applying configuration (root,0,0,0.0,32416) [kworker/u33:0]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09acb57995a9
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (root,0,0,0.0,564) [kworker/0:1H] (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (root,72224,34240,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.6,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1764,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1305) [bond0] (root,10077168,244964,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,591380,21596,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (cvmfs,66484,1108,0.0,2045) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,82712,8200,0.0,2051) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1092,0.0,2055) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2133) [kworker/3:1] (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1262616,31212,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2751) [kworker/7:0] (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,0,0,0.0,3207) [kworker/7:1] (root,591424,14488,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,4039) [kworker/10:2] (root,0,0,0.0,6152) [kworker/2:1] (root,0,0,0.0,6660) [kworker/5:1] (root,0,0,0.0,7219) [kworker/0:2] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,9860) [kworker/2:2] (root,0,0,0.0,11540) [kworker/14:2] (root,0,0,0.0,12113) [kworker/6:0] (root,0,0,0.0,12688) [kworker/1:1] (root,0,0,0.0,12728) [kworker/4:1] (root,0,0,0.0,14069) [kworker/11:1] (cvmfs,66484,1108,0.0,14343) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,796596,23652,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66484,1108,0.0,14498) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,796512,25820,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,66484,1108,0.0,14639) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,3180236,61660,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,66484,1112,0.0,14789) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,77012,2552,0.0,14793) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1092,0.0,14797) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,796512,23656,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,66484,1108,0.0,14963) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,796524,23680,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (root,0,0,0.0,15083) [kworker/u32:0] (cvmfs,66484,1108,0.0,15152) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,796512,23652,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,66484,1108,0.0,15294) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,796536,26336,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,66484,1108,0.0,15435) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,796596,23576,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,66484,1108,0.0,15576) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,796672,23584,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11288,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6764,0.0,15641) condor_shared_port (condor,1886752,1774268,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (postfix,89912,4028,0.0,16156) pickup -l -t unix -u (root,0,0,0.0,16228) [kworker/3:0] (root,0,0,0.0,16457) [kworker/15:1] (root,0,0,0.0,17675) [kworker/10:1] (root,0,0,0.0,18730) [kworker/u33:1] (root,0,0,0.0,18824) [kworker/15:0] (root,0,0,0.0,19287) [kworker/4:0] (root,0,0,0.0,20003) [kworker/0:1] (root,0,0,0.0,22230) [kworker/13:2] (root,0,0,0.0,22841) [kworker/9:1] (root,0,0,0.0,22896) [kworker/14:0] (root,0,0,0.0,23929) [kworker/8:1] (root,0,0,0.0,24094) [kworker/8:2] (root,0,0,0.0,24148) [kworker/6:1] (root,0,0,0.0,24670) [kworker/1:0] (root,0,0,0.0,25533) [kworker/1:2] (root,0,0,0.0,25556) [kworker/12:2] (root,0,0,0.0,26117) [kworker/5:2] (root,0,0,0.0,26378) [kworker/9:2] (root,0,0,0.0,26740) [kworker/u32:1] (root,108056,360,0.0,27853) sleep 5 (root,8632,660,0.0,27854) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,27855) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,27872) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,27873) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,27995) [kworker/11:2] (root,0,0,0.0,29108) [kworker/13:1] (root,0,0,0.0,29812) [kworker/u34:1] (root,0,0,0.0,30971) [kworker/u34:2] (root,0,0,0.0,32115) [kworker/u33:2] (root,0,0,0.0,32731) [kworker/12:0]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac8f1c8fd9
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (root,0,0,0.0,564) [kworker/0:1H] (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (root,113184,67064,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.6,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,957) [kworker/11:2] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1764,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1305) [bond0] (root,10077168,244536,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,599572,40744,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,0,0,0.0,1845) [kworker/15:2] (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (cvmfs,66484,1108,0.0,2045) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,82712,8200,0.0,2051) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1092,0.0,2055) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1255712,36888,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,591424,14488,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,4660) [kworker/2:2] (root,0,0,0.0,4759) [kworker/9:2] (root,0,0,0.0,6392) [kworker/7:1] (root,108056,356,0.0,7100) sleep 5 (root,8632,664,0.0,7101) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,7102) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,7119) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,7120) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,8774) [kworker/u34:0] (root,0,0,0.0,10574) [kworker/1:2] (root,0,0,0.0,10665) [kworker/2:1] (root,0,0,0.0,11082) [kworker/u32:0] (root,0,0,0.0,11667) [kworker/2:0] (root,0,0,0.0,12420) [kworker/u32:1] (root,0,0,0.0,12493) [kworker/4:0] (root,0,0,0.0,13395) [kworker/15:1] (root,0,0,0.0,13863) [kworker/12:0] (cvmfs,66484,1108,0.0,14343) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,796820,23584,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66484,1108,0.0,14498) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,796512,25820,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,14591) [kworker/0:1] (cvmfs,66484,1108,0.0,14639) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,3180236,61652,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,66484,1112,0.0,14789) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,77012,2552,0.0,14793) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1092,0.0,14797) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,796512,23656,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,66484,1108,0.0,14963) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,796596,23648,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,66484,1108,0.0,15152) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,796524,23660,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,15266) [kworker/u33:0] (cvmfs,66484,1108,0.0,15294) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,796396,26340,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,66484,1108,0.0,15435) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,797628,24436,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (root,0,0,0.0,15496) [kworker/u34:2] (cvmfs,66484,1108,0.0,15576) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,796556,23620,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,132908,11288,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6764,0.0,15641) condor_shared_port (condor,1791644,1679148,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,16547) [kworker/12:1] (root,0,0,0.0,16552) [kworker/3:0] (root,0,0,0.0,18378) [kworker/0:0] (root,0,0,0.0,18390) [kworker/u33:2] (root,0,0,0.0,18747) [kworker/1:0] (root,0,0,0.0,19075) [kworker/15:0] (root,0,0,0.0,19607) [kworker/11:1] (root,0,0,0.0,19744) [kworker/6:2] (root,0,0,0.0,20727) [kworker/10:1] (root,0,0,0.0,22230) [kworker/13:2] (root,0,0,0.0,22476) [kworker/7:2] (root,0,0,0.0,24503) [kworker/13:0] (root,0,0,0.0,26290) [kworker/14:1] (root,0,0,0.0,27685) [kworker/4:1] (root,0,0,0.0,27861) [kworker/9:0] (root,0,0,0.0,28042) [kworker/14:0] (root,0,0,0.0,28239) [kworker/8:2] (postfix,89912,4028,0.0,28791) pickup -l -t unix -u (root,0,0,0.0,29446) [kworker/6:1] (root,0,0,0.0,29734) [kworker/3:2] (root,0,0,0.0,30817) [kworker/5:1] (root,0,0,0.0,31451) [kworker/6:0] (root,0,0,0.0,31733) [kworker/5:2] (root,0,0,0.0,31745) [kworker/8:0] (root,0,0,0.0,31892) [kworker/10:2] (root,0,0,0.0,32155) [kworker/3:1]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac778fd2c8
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43824,4200,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [scsi_eh_0] (root,0,0,0.0,405) [scsi_tmf_0] (root,0,0,0.0,406) [scsi_eh_1] (root,0,0,0.0,407) [scsi_tmf_1] (root,0,0,0.0,408) [scsi_eh_2] (root,0,0,0.0,409) [scsi_tmf_2] (root,0,0,0.0,410) [scsi_eh_3] (root,0,0,0.0,411) [scsi_tmf_3] (root,0,0,0.0,415) [scsi_eh_4] (root,0,0,0.0,416) [scsi_tmf_4] (root,0,0,0.0,417) [scsi_eh_5] (root,0,0,0.0,418) [scsi_tmf_5] (root,0,0,0.0,419) [scsi_eh_6] (root,0,0,0.0,420) [scsi_tmf_6] (root,0,0,0.0,421) [scsi_eh_7] (root,0,0,0.0,422) [scsi_tmf_7] (root,0,0,0.0,423) [scsi_eh_8] (root,0,0,0.0,424) [scsi_tmf_8] (root,0,0,0.0,425) [scsi_eh_9] (root,0,0,0.0,426) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (root,0,0,0.0,564) [kworker/0:1H] (root,0,0,0.0,573) [jbd2/dm-0-8] (root,0,0,0.0,574) [ext4-rsv-conver] (root,64032,22084,0.0,654) /usr/lib/systemd/systemd-journald (root,45556,2020,0.0,677) /usr/lib/systemd/systemd-udevd (root,0,0,1.6,731) [kipmi0] (root,0,0,0.0,739) [kworker/10:1H] (root,0,0,0.0,742) [kworker/6:1H] (root,0,0,0.0,806) [kvm-irqfd-clean] (root,0,0,0.0,810) [kworker/11:1H] (root,198572,1432,0.0,818) /usr/sbin/lvmetad -f (root,0,0,0.0,821) [kworker/1:1H] (root,0,0,0.0,881) [kworker/13:1H] (root,0,0,0.0,882) [kworker/5:1H] (root,0,0,0.0,883) [kworker/4:1H] (root,0,0,0.0,884) [kworker/7:1H] (root,0,0,0.0,886) [kworker/8:1H] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,890) [kdmflush] (root,0,0,0.0,891) [bioset] (root,0,0,0.0,893) [kdmflush] (root,0,0,0.0,894) [bioset] (root,0,0,0.0,896) [kdmflush] (root,0,0,0.0,898) [bioset] (root,0,0,0.0,909) [jbd2/sda1-8] (root,0,0,0.0,910) [ext4-rsv-conver] (root,0,0,0.0,931) [kworker/14:1H] (root,0,0,0.0,933) [kworker/15:1H] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,945) [jbd2/dm-2-8] (root,0,0,0.0,946) [ext4-rsv-conver] (root,0,0,0.0,959) [nv_queue] (root,0,0,0.0,960) [nv_queue] (root,0,0,0.0,961) [kworker/2:1H] (root,0,0,0.0,973) [UVM global queu] (root,0,0,0.0,974) [UVM deferred re] (root,0,0,0.0,975) [UVM Tools Event] (root,0,0,0.0,984) [nvidia-modeset/] (root,0,0,0.0,985) [nvidia-modeset/] (root,0,0,0.0,1026) [jbd2/dm-4-8] (root,0,0,0.0,1027) [ext4-rsv-conver] (root,0,0,0.0,1029) [kworker/12:1H] (root,0,0,0.0,1034) [jbd2/dm-5-8] (root,0,0,0.0,1035) [ext4-rsv-conver] (root,52900,2940,0.0,1058) /usr/sbin/smartd -n -q never (root,21716,1432,0.0,1061) /usr/sbin/irqbalance --foreground (dbus,58220,2408,0.0,1064) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612240,8216,0.0,1066) /usr/lib/polkit-1/polkitd --no-debug (root,26384,1764,0.0,1068) /usr/lib/systemd/systemd-logind (node_ex+,727344,13168,0.0,1080) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,0,0,0.0,1305) [bond0] (root,0,0,0.0,1357) [kworker/6:0] (root,0,0,0.0,1422) [kworker/u34:2] (root,10077168,243968,0.1,1439) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,323672,70004,0.0,1559) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4328,0.0,1560) /usr/sbin/sshd -D (root,574288,17628,0.0,1562) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,0,0,0.0,1566) [kworker/4:2] (root,27168,1052,0.0,1580) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1590) [irq/98-nvidia] (root,0,0,0.0,1591) [nvidia] (root,0,0,0.2,1592) [nv_queue] (root,550420,15200,0.0,1826) /usr/sbin/rsyslogd -n (root,115704,1928,0.1,1843) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1851) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,9180,0.0,1884) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209416,8196,0.0,1894) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1901) [irq/99-nvidia] (root,0,0,0.0,1902) [nvidia] (root,0,0,0.2,1903) [nv_queue] (root,0,0,0.0,2038) [kworker/2:1] (cvmfs,66484,1108,0.0,2045) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,82712,8200,0.0,2051) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1092,0.0,2055) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,89808,2164,0.0,2108) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2111) qmgr -l -t unix -u (root,0,0,0.0,2189) [irq/100-nvidia] (root,0,0,0.0,2190) [nvidia] (root,0,0,0.2,2191) [nv_queue] (root,0,0,0.0,2232) [irq/101-nvidia] (root,0,0,0.0,2233) [nvidia] (root,0,0,0.1,2234) [nv_queue] (root,0,0,0.0,2235) [irq/102-nvidia] (root,0,0,0.0,2236) [nvidia] (root,0,0,0.1,2237) [nv_queue] (root,0,0,0.0,2238) [irq/103-nvidia] (root,0,0,0.0,2239) [nvidia] (root,0,0,0.1,2240) [nv_queue] (root,0,0,0.0,2241) [irq/104-nvidia] (root,0,0,0.0,2242) [nvidia] (root,0,0,0.2,2243) [nv_queue] (root,0,0,0.0,2244) [irq/105-nvidia] (root,0,0,0.0,2245) [nvidia] (root,0,0,0.2,2246) [nv_queue] (root,0,0,0.0,2296) [kworker/3:1H] (ntp,29960,2172,0.0,2604) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1255712,35476,0.0,2616) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2826) [kworker/8:1] (root,126420,1644,0.0,3133) /usr/sbin/crond -n (root,110208,864,0.0,3144) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3149) /usr/sbin/gmond (root,0,0,0.0,3710) [kworker/11:2] (root,591424,14488,0.2,3957) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,5105) [kworker/1:1] (root,0,0,0.0,5332) [kworker/0:2] (root,0,0,0.0,5463) [kworker/2:2] (root,0,0,0.0,7236) [kworker/9:1H] (root,0,0,0.0,7560) [kworker/15:0] (root,537956,193492,33.4,8220) puppet agent: applying configuration (root,108056,356,0.0,8565) sleep 5 (root,11688,1348,0.0,8755) sh -c /usr/bin/pip --version 2>&1 (root,52352,6412,0.0,8756) /usr/bin/python2 /usr/bin/pip --version (root,8632,664,0.0,8757) timeout -s 9 360 check_mk_agent (root,11696,1532,0.0,8758) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,8775) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,8776) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,9085) [kworker/10:1] (root,0,0,0.0,9509) [kworker/u32:1] (postfix,89912,4032,0.0,10331) pickup -l -t unix -u (root,0,0,0.0,10351) [kworker/3:1] (root,0,0,0.0,10526) [kworker/u33:2] (root,0,0,0.0,11094) [kworker/3:2] (root,0,0,0.0,11638) [kworker/0:0] (root,0,0,0.0,12086) [kworker/1:0] (root,0,0,0.0,12533) [kworker/5:2] (root,0,0,0.0,12944) [kworker/5:0] (root,0,0,0.0,13863) [kworker/12:0] (cvmfs,66484,1108,0.0,14343) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,796596,23620,0.0,14348) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66484,1108,0.0,14498) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,796516,25820,0.0,14501) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,66484,1108,0.0,14639) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,3180236,61636,0.0,14642) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,66484,1112,0.0,14789) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,77012,2552,0.0,14793) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66480,1092,0.0,14797) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,796512,23656,0.0,14801) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,66484,1108,0.0,14963) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,796524,23680,0.0,14966) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,66484,1108,0.0,15152) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,796524,23660,0.0,15156) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,66484,1108,0.0,15294) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,796380,26336,0.0,15297) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,66484,1108,0.0,15435) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,797628,23924,0.0,15438) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (root,0,0,0.0,15485) [kworker/11:0] (cvmfs,66484,1108,0.0,15576) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,796820,23564,0.0,15579) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,74844,8036,0.0,15636) /usr/sbin/condor_master -f (root,26984,5900,0.0,15637) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6764,0.0,15641) condor_shared_port (root,0,0,0.0,15642) [kworker/13:0] (condor,1637040,1583564,0.0,15652) condor_startd (condor,34080820,17168,0.2,15805) GPUs_MONITOR (root,0,0,0.0,15807) [UVM GPU1 BH] (root,0,0,0.0,15808) [UVM GPU2 BH] (root,0,0,0.0,15809) [UVM GPU3 BH] (root,0,0,0.0,15810) [UVM GPU4 BH] (root,0,0,0.0,15811) [UVM GPU5 BH] (root,0,0,0.0,15812) [UVM GPU6 BH] (root,0,0,0.0,15813) [UVM GPU7 BH] (root,0,0,0.0,15814) [UVM GPU8 BH] (root,0,0,0.0,16415) [kworker/7:0] (root,0,0,0.0,16831) [kworker/u34:0] (root,0,0,0.0,19004) [kworker/14:2] (root,0,0,0.0,19236) [kworker/12:1] (root,0,0,0.0,20403) [kworker/6:1] (root,0,0,0.0,20409) [kworker/9:1] (root,0,0,0.0,21334) [kworker/4:1] (root,0,0,0.0,21690) [kworker/0:1] (root,0,0,0.0,22230) [kworker/13:2] (root,0,0,0.0,22239) [kworker/10:0] (root,0,0,0.0,23767) [kworker/u32:0] (root,0,0,0.0,24881) [kworker/15:2] (root,0,0,0.0,27861) [kworker/9:0] (root,0,0,0.0,29016) [kworker/14:0] (root,0,0,0.0,30016) [kworker/8:2] (root,0,0,0.0,30922) [kworker/8:0] (root,0,0,0.0,31322) [kworker/7:1] (root,0,0,0.0,31896) [kworker/11:1] (root,0,0,0.0,32231) [kworker/2:0] (root,0,0,0.0,32387) [kworker/u33:1]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac7330a372
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43796,3756,0.0,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.2,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,259) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,405) [scsi_eh_0] (root,0,0,0.0,406) [scsi_tmf_0] (root,0,0,0.0,407) [scsi_eh_1] (root,0,0,0.0,408) [scsi_tmf_1] (root,0,0,0.0,409) [scsi_eh_2] (root,0,0,0.0,410) [scsi_tmf_2] (root,0,0,0.0,411) [scsi_eh_3] (root,0,0,0.0,412) [scsi_tmf_3] (root,0,0,0.0,416) [scsi_eh_4] (root,0,0,0.0,417) [scsi_tmf_4] (root,0,0,0.0,418) [scsi_eh_5] (root,0,0,0.0,419) [scsi_tmf_5] (root,0,0,0.0,420) [scsi_eh_6] (root,0,0,0.0,421) [scsi_tmf_6] (root,0,0,0.0,422) [scsi_eh_7] (root,0,0,0.0,423) [scsi_tmf_7] (root,0,0,0.0,424) [scsi_eh_8] (root,0,0,0.0,425) [scsi_tmf_8] (root,0,0,0.0,426) [scsi_eh_9] (root,0,0,0.0,427) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,551) [kdmflush] (root,0,0,0.0,552) [bioset] (root,0,0,0.0,562) [kdmflush] (root,0,0,0.0,563) [bioset] (root,0,0,0.0,573) [kworker/0:1H] (root,0,0,0.0,582) [jbd2/dm-0-8] (root,0,0,0.0,583) [ext4-rsv-conver] (root,0,0,0.0,593) [kworker/u34:2] (root,88608,47940,0.0,663) /usr/lib/systemd/systemd-journald (root,45572,1984,0.0,687) /usr/lib/systemd/systemd-udevd (root,0,0,0.0,738) [kworker/3:1H] (root,0,0,0.0,739) [kworker/2:1H] (root,0,0,1.3,740) [kipmi0] (root,190376,1404,0.0,799) /usr/sbin/lvmetad -f (root,0,0,0.0,800) [kvm-irqfd-clean] (root,0,0,0.0,832) [kworker/10:2] (root,0,0,0.0,879) [kdmflush] (root,0,0,0.0,880) [bioset] (root,0,0,0.0,881) [kdmflush] (root,0,0,0.0,882) [bioset] (root,0,0,0.0,884) [kdmflush] (root,0,0,0.0,885) [bioset] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,898) [kworker/11:1H] (root,0,0,0.0,899) [kworker/5:1H] (root,0,0,0.0,900) [kworker/13:1H] (postfix,89912,4032,0.0,909) pickup -l -t unix -u (root,0,0,0.0,911) [jbd2/sda1-8] (root,0,0,0.0,912) [ext4-rsv-conver] (root,0,0,0.0,925) [kworker/14:1H] (root,0,0,0.0,931) [kworker/8:1H] (root,0,0,0.0,936) [jbd2/dm-3-8] (root,0,0,0.0,937) [ext4-rsv-conver] (root,0,0,0.0,939) [kworker/12:1H] (root,0,0,0.0,945) [nv_queue] (root,0,0,0.0,946) [nv_queue] (root,0,0,0.0,957) [kworker/1:1H] (root,0,0,0.0,960) [jbd2/dm-2-8] (root,0,0,0.0,961) [ext4-rsv-conver] (root,0,0,0.0,981) [UVM global queu] (root,0,0,0.0,982) [UVM deferred re] (root,0,0,0.0,984) [UVM Tools Event] (root,0,0,0.0,992) [jbd2/dm-5-8] (root,0,0,0.0,993) [ext4-rsv-conver] (root,0,0,0.0,994) [nvidia-modeset/] (root,0,0,0.0,995) [nvidia-modeset/] (root,0,0,0.0,1002) [jbd2/dm-4-8] (root,0,0,0.0,1003) [ext4-rsv-conver] (root,52900,2944,0.0,1023) /usr/sbin/smartd -n -q never (root,26384,1764,0.0,1024) /usr/lib/systemd/systemd-logind (node_ex+,727600,15788,0.1,1030) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,21788,1428,0.0,1034) /usr/sbin/irqbalance --foreground (polkitd,612240,8212,0.0,1052) /usr/lib/polkit-1/polkitd --no-debug (dbus,58220,2428,0.0,1063) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (root,0,0,0.0,1114) [kworker/6:1H] (root,0,0,0.0,1157) [kworker/7:1H] (root,0,0,0.0,1292) [bond0] (root,10080252,225368,0.0,1412) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,113004,4336,0.0,1572) /usr/sbin/sshd -D (root,574288,17376,0.0,1573) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,928,0.0,1587) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1603) [irq/98-nvidia] (root,0,0,0.0,1604) [nvidia] (root,0,0,0.1,1605) [nv_queue] (root,0,0,0.0,1607) [kworker/u32:2] (root,323672,69976,0.0,1826) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,377112,29008,0.0,1832) /usr/sbin/rsyslogd -n (root,115708,1924,0.0,1862) /bin/bash /sbin/condor_inspector (root,209016,10840,0.0,1887) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1899) [irq/99-nvidia] (root,0,0,0.0,1900) [nvidia] (root,0,0,0.1,1901) [nv_queue] (root,358544,9244,0.0,1908) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209264,8160,0.0,1916) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1931) [kworker/3:1] (cvmfs,66484,1104,0.0,1989) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,82684,8208,0.0,2021) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1088,0.0,2027) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,0,0,0.0,2177) [irq/100-nvidia] (root,0,0,0.0,2178) [nvidia] (root,0,0,0.1,2179) [nv_queue] (root,89808,2168,0.0,2216) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2219) qmgr -l -t unix -u (root,0,0,0.0,2224) [irq/101-nvidia] (root,0,0,0.0,2225) [nvidia] (root,0,0,0.0,2226) [nv_queue] (root,0,0,0.0,2227) [irq/102-nvidia] (root,0,0,0.0,2228) [nvidia] (root,0,0,0.0,2229) [nv_queue] (root,0,0,0.0,2230) [irq/103-nvidia] (root,0,0,0.0,2231) [nvidia] (root,0,0,0.0,2232) [nv_queue] (root,0,0,0.0,2233) [irq/104-nvidia] (root,0,0,0.0,2234) [nvidia] (root,0,0,0.1,2235) [nv_queue] (root,0,0,0.0,2237) [irq/105-nvidia] (root,0,0,0.0,2238) [nvidia] (root,0,0,0.1,2239) [nv_queue] (root,0,0,0.0,2304) [kworker/4:1H] (root,0,0,0.0,2406) [kworker/6:1] (ntp,29960,2168,0.0,2642) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1252080,27316,0.0,2649) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2905) [kworker/9:0] (root,126392,1600,0.0,3168) /usr/sbin/crond -n (root,110208,856,0.0,3174) /sbin/agetty --noclear tty1 linux (ganglia,220220,8144,0.0,3178) /usr/sbin/gmond (slot-1,46172,6612,0.0,3456) condor_starter -f -a slot1_11 pyglidein1-submit.icecube.wisc.edu (root,0,0,0.0,3466) [kworker/11:2] (slot-1,11704,1564,0.0,3504) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_3456/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443116,23076,0.0,3547) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1464,0.0,4407) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1552,0.0,4500) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358196,47736,0.0,4508) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1424,0.0,4515) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_3456/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_3456/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_3456/CORSIKA000000.0.log (slot-1,109076,29772,99.9,4516) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_3456/dcors0/corsika.0.Linux (root,0,0,0.0,4739) [kworker/13:1] (root,0,0,0.0,5338) [kworker/6:2] (root,591424,14476,0.1,6042) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,6700) [kworker/14:0] (root,0,0,0.0,7588) [kworker/2:2] (slot-1,46180,6616,0.0,7676) condor_starter -f -a slot1_5 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,7680) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_7676/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443084,23148,0.0,7720) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1456,0.0,8112) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.11.1 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1556,0.0,8209) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.11.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13551108,976172,70.2,8216) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,46184,6628,0.0,8310) condor_starter -f -a slot1_2 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1564,0.0,8314) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8310/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443080,23340,0.0,8369) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (cvmfs,66488,1108,0.0,8444) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,732096,23208,0.0,8447) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66488,1108,0.0,8597) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,796516,25096,0.0,8612) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (slot-1,46172,6608,0.0,8784) condor_starter -f -a slot1_12 pyglidein1-submit.icecube.wisc.edu (slot-1,46172,6608,0.0,8803) condor_starter -f -a slot1_13 pyglidein1-submit.icecube.wisc.edu (cvmfs,66488,1112,0.0,8821) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (slot-1,11704,1564,0.0,8886) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8784/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11704,1560,0.0,8901) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8803/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (cvmfs,3774112,115844,0.1,8922) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (slot-1,443116,23076,0.0,8940) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443120,22912,0.0,8980) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (cvmfs,66488,1116,0.0,9069) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,77040,2560,0.0,9073) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66484,1092,0.0,9077) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,870384,23700,0.0,9081) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,66488,1104,0.0,9336) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,796520,23684,0.0,9339) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,66488,1104,0.0,9544) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,796528,23664,0.0,9583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,66488,1108,0.0,9721) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,796688,26396,0.0,9822) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (slot-1,11700,1460,0.0,9952) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.11.1 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (cvmfs,66488,1108,0.0,9968) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,796536,23524,0.0,9971) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,66488,1104,0.0,10109) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,10110) [kworker/15:1H] (slot-1,11704,1552,0.0,10116) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.11.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13584260,987784,70.5,10124) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11700,1464,0.0,10126) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (cvmfs,796824,23692,0.0,10127) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,10156) [kworker/5:0] (slot-1,11704,1544,0.0,10232) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358200,47740,0.0,10239) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (condor,74780,7768,0.0,10273) /usr/sbin/condor_master -f (root,27644,6480,0.0,10274) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6592,0.0,10278) condor_shared_port (condor,773936,720084,0.0,10288) condor_startd (slot-1,11704,1424,0.0,10359) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8803/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8803/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8803/CORSIKA000000.0.log (slot-1,109076,29292,99.9,10360) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8803/dcors0/corsika.0.Linux (slot-1,11700,1460,0.0,10365) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1548,0.0,10460) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358196,47732,0.0,10467) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (condor,34080820,16588,0.1,10569) GPUs_MONITOR (root,0,0,0.0,10571) [UVM GPU1 BH] (root,0,0,0.0,10572) [UVM GPU2 BH] (root,0,0,0.0,10573) [UVM GPU3 BH] (root,0,0,0.0,10574) [UVM GPU4 BH] (root,0,0,0.0,10575) [UVM GPU5 BH] (root,0,0,0.0,10576) [UVM GPU6 BH] (root,0,0,0.0,10577) [UVM GPU7 BH] (root,0,0,0.0,10578) [UVM GPU8 BH] (slot-1,11704,1424,0.0,10591) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8784/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8784/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8784/CORSIKA000000.0.log (slot-1,109076,29296,99.9,10592) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8784/dcors0/corsika.0.Linux (slot-1,46176,6612,0.0,10949) condor_starter -f -a slot1_1 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1564,0.0,10953) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_10949/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443084,23136,0.0,10993) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (root,0,0,0.0,11089) [kworker/4:0] (slot-1,11700,1468,0.0,11261) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.11.1 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1552,0.0,11370) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.11.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13986188,1396376,68.1,11377) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,12389) [kworker/14:1] (condor,66440,8324,0.0,12542) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (slot-1,11692,1572,0.0,12597) /bin/sh glidein_start.sh (slot-1,61812,6476,0.0,12867) glideinExec/sbin/condor_master -f -r 20135 (slot-1,12084,1892,0.0,12868) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_d85cb2f3-47fb-46f6-b823-0d04bd2cbf02.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20240911%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240911T014002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=f6ee32e18e084ade0d82b67cc8df727c2a1feab8b009e9332a6d35aa486ae72b (slot-1,25100,5228,0.0,12880) condor_procd -A /scratch/condor/dir_12542/glidein/log.1b86c92c-6fdf-11ef-8619-0cc47a857a10/procd_address -L /scratch/condor/dir_12542/glidein/log.1b86c92c-6fdf-11ef-8619-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,64128,9216,0.0,12907) condor_startd -f (root,0,0,0.0,13871) [kworker/8:0] (root,0,0,0.0,13996) [kworker/0:2] (root,0,0,0.0,15243) [kworker/15:0] (root,0,0,0.0,15823) [kworker/u33:1] (root,0,0,0.0,16148) [kworker/9:1H] (slot-1,46176,6612,0.0,16261) condor_starter -f -a slot1_4 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1564,0.0,16265) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_16261/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443080,23224,0.0,16305) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (root,0,0,0.0,17236) [kworker/11:0] (slot-1,11700,1468,0.0,17480) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.11.1 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1552,0.0,17578) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.11.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13560720,961536,70.5,17585) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,18130) [kworker/9:2] (root,0,0,0.0,20135) [kworker/2:0] (root,0,0,0.0,20221) [kworker/15:2] (root,0,0,0.0,20449) [kworker/u32:1] (root,0,0,0.0,20832) [kworker/u32:0] (slot-1,4368,356,0.0,20876) sleep 300 (root,0,0,0.0,20881) [kworker/4:1] (slot-1,46176,6616,0.0,21439) condor_starter -f -a slot1_8 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1564,0.0,21476) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_21439/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,444140,23432,0.0,21547) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1468,0.0,21623) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1552,0.0,21716) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,0,0,99.5,21723) [python] <defunct> (root,0,0,0.0,22239) [kworker/7:1] (root,0,0,0.0,22334) [kworker/9:1] (slot-1,46176,6616,0.0,23959) condor_starter -f -a slot1_6 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,23963) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_23959/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443084,23228,0.0,24003) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (root,0,0,0.0,24121) [kworker/5:2] (root,0,0,0.0,24237) [kworker/u33:0] (root,0,0,0.0,24341) [kworker/12:1] (slot-1,11700,1468,0.0,24458) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.11.1 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1556,0.0,24555) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.11.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13547816,951076,71.2,24562) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,46172,6608,0.0,24623) condor_starter -f -a slot1_9 pyglidein1-submit.icecube.wisc.edu (slot-1,46172,6612,0.0,24646) condor_starter -f -a slot1_10 pyglidein1-submit.icecube.wisc.edu (root,0,0,0.0,24661) [kworker/13:0] (slot-1,11704,1568,0.0,24686) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24623/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443120,22928,0.0,24727) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11704,1568,0.0,24729) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24646/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443116,22916,0.0,24772) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (root,0,0,0.0,25385) [kworker/15:1] (root,0,0,0.0,25409) [kworker/12:0] (slot-1,11700,1460,0.0,25692) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1552,0.0,25785) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358200,47740,0.0,25792) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1420,0.0,25906) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24623/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24623/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24623/CORSIKA000000.0.log (slot-1,109076,29912,99.9,25907) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24623/dcors0/corsika.0.Linux (slot-1,11700,1460,0.0,26019) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1548,0.0,26112) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358196,47732,0.0,26119) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1424,0.0,26126) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24646/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24646/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24646/CORSIKA000000.0.log (slot-1,109076,29196,99.9,26127) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24646/dcors0/corsika.0.Linux (root,0,0,0.0,26884) [kworker/1:2] (root,0,0,0.0,28016) [kworker/10:1] (root,0,0,0.0,28061) [kworker/1:1] (root,0,0,0.0,28249) [kworker/3:0] (root,0,0,0.0,29465) [kworker/10:1H] (root,0,0,0.0,29646) [kworker/u34:1] (root,0,0,0.0,30015) [kworker/8:2] (root,0,0,0.0,30566) [kworker/14:2] (root,108056,360,0.0,32001) sleep 5 (root,8632,660,0.0,32002) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,32003) /bin/bash /bin/check_mk_agent (root,51732,1696,0.0,32020) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,32021) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,32044) [kworker/7:0] (slot-1,46180,6616,0.0,32321) condor_starter -f -a slot1_3 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,32325) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_32321/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,444144,23396,0.0,32365) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1460,0.0,32370) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.9.2/ /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=29010700 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,11704,1552,0.0,32467) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.9.2//env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=29010700 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (root,0,0,0.0,32473) [kworker/0:0] (slot-1,19998240,7364572,74.4,32474) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=29010700 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac5e59c896
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43796,3756,0.0,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.2,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,259) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,405) [scsi_eh_0] (root,0,0,0.0,406) [scsi_tmf_0] (root,0,0,0.0,407) [scsi_eh_1] (root,0,0,0.0,408) [scsi_tmf_1] (root,0,0,0.0,409) [scsi_eh_2] (root,0,0,0.0,410) [scsi_tmf_2] (root,0,0,0.0,411) [scsi_eh_3] (root,0,0,0.0,412) [scsi_tmf_3] (root,0,0,0.0,416) [scsi_eh_4] (root,0,0,0.0,417) [scsi_tmf_4] (root,0,0,0.0,418) [scsi_eh_5] (root,0,0,0.0,419) [scsi_tmf_5] (root,0,0,0.0,420) [scsi_eh_6] (root,0,0,0.0,421) [scsi_tmf_6] (root,0,0,0.0,422) [scsi_eh_7] (root,0,0,0.0,423) [scsi_tmf_7] (root,0,0,0.0,424) [scsi_eh_8] (root,0,0,0.0,425) [scsi_tmf_8] (root,0,0,0.0,426) [scsi_eh_9] (root,0,0,0.0,427) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,551) [kdmflush] (root,0,0,0.0,552) [bioset] (root,0,0,0.0,562) [kdmflush] (root,0,0,0.0,563) [bioset] (root,0,0,0.0,573) [kworker/0:1H] (root,0,0,0.0,582) [jbd2/dm-0-8] (root,0,0,0.0,583) [ext4-rsv-conver] (root,47648,11472,0.0,663) /usr/lib/systemd/systemd-journald (root,45572,1984,0.0,687) /usr/lib/systemd/systemd-udevd (root,0,0,0.0,738) [kworker/3:1H] (root,0,0,0.0,739) [kworker/2:1H] (root,0,0,1.3,740) [kipmi0] (root,190376,1404,0.0,799) /usr/sbin/lvmetad -f (root,0,0,0.0,800) [kvm-irqfd-clean] (root,0,0,0.0,879) [kdmflush] (root,0,0,0.0,880) [bioset] (root,0,0,0.0,881) [kdmflush] (root,0,0,0.0,882) [bioset] (root,0,0,0.0,884) [kdmflush] (root,0,0,0.0,885) [bioset] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,898) [kworker/11:1H] (root,0,0,0.0,899) [kworker/5:1H] (root,0,0,0.0,900) [kworker/13:1H] (root,0,0,0.0,911) [jbd2/sda1-8] (root,0,0,0.0,912) [ext4-rsv-conver] (root,0,0,0.0,925) [kworker/14:1H] (root,0,0,0.0,931) [kworker/8:1H] (root,0,0,0.0,936) [jbd2/dm-3-8] (root,0,0,0.0,937) [ext4-rsv-conver] (root,0,0,0.0,939) [kworker/12:1H] (root,0,0,0.0,945) [nv_queue] (root,0,0,0.0,946) [nv_queue] (root,0,0,0.0,957) [kworker/1:1H] (root,0,0,0.0,960) [jbd2/dm-2-8] (root,0,0,0.0,961) [ext4-rsv-conver] (root,0,0,0.0,981) [UVM global queu] (root,0,0,0.0,982) [UVM deferred re] (root,0,0,0.0,984) [UVM Tools Event] (root,0,0,0.0,992) [jbd2/dm-5-8] (root,0,0,0.0,993) [ext4-rsv-conver] (root,0,0,0.0,994) [nvidia-modeset/] (root,0,0,0.0,995) [nvidia-modeset/] (root,0,0,0.0,1002) [jbd2/dm-4-8] (root,0,0,0.0,1003) [ext4-rsv-conver] (root,52900,2944,0.0,1023) /usr/sbin/smartd -n -q never (root,26384,1764,0.0,1024) /usr/lib/systemd/systemd-logind (node_ex+,727600,15900,0.1,1030) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,21788,1428,0.0,1034) /usr/sbin/irqbalance --foreground (polkitd,612240,8204,0.0,1052) /usr/lib/polkit-1/polkitd --no-debug (dbus,58220,2428,0.0,1063) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (root,0,0,0.0,1114) [kworker/6:1H] (root,0,0,0.0,1157) [kworker/7:1H] (root,0,0,0.0,1292) [bond0] (root,10080252,225144,0.0,1412) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,113004,4336,0.0,1572) /usr/sbin/sshd -D (root,574288,17376,0.0,1573) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,928,0.0,1587) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1603) [irq/98-nvidia] (root,0,0,0.0,1604) [nvidia] (root,0,0,0.1,1605) [nv_queue] (root,323672,69976,0.0,1826) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,336152,8924,0.0,1832) /usr/sbin/rsyslogd -n (root,115708,1924,0.0,1862) /bin/bash /sbin/condor_inspector (root,209016,10840,0.0,1887) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1899) [irq/99-nvidia] (root,0,0,0.0,1900) [nvidia] (root,0,0,0.1,1901) [nv_queue] (root,358544,9244,0.0,1908) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209264,8160,0.0,1916) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (cvmfs,66484,1104,0.0,1989) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,82684,8208,0.0,2021) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1088,0.0,2027) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,0,0,0.0,2040) [kworker/0:2] (root,0,0,0.0,2177) [irq/100-nvidia] (root,0,0,0.0,2178) [nvidia] (root,0,0,0.1,2179) [nv_queue] (root,89808,2168,0.0,2216) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2219) qmgr -l -t unix -u (root,0,0,0.0,2224) [irq/101-nvidia] (root,0,0,0.0,2225) [nvidia] (root,0,0,0.1,2226) [nv_queue] (root,0,0,0.0,2227) [irq/102-nvidia] (root,0,0,0.0,2228) [nvidia] (root,0,0,0.0,2229) [nv_queue] (root,0,0,0.0,2230) [irq/103-nvidia] (root,0,0,0.0,2231) [nvidia] (root,0,0,0.0,2232) [nv_queue] (root,0,0,0.0,2233) [irq/104-nvidia] (root,0,0,0.0,2234) [nvidia] (root,0,0,0.1,2235) [nv_queue] (root,0,0,0.0,2237) [irq/105-nvidia] (root,0,0,0.0,2238) [nvidia] (root,0,0,0.1,2239) [nv_queue] (root,0,0,0.0,2304) [kworker/4:1H] (ntp,29960,2168,0.0,2642) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1252080,26400,0.0,2649) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2863) [kworker/u33:0] (root,0,0,0.0,2905) [kworker/9:0] (slot-1,46180,6612,0.0,2930) condor_starter -f -a slot1_2 pyglidein1-submit.icecube.wisc.edu (root,126392,1600,0.0,3168) /usr/sbin/crond -n (slot-1,11704,1568,0.0,3172) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_2930/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (root,110208,856,0.0,3174) /sbin/agetty --noclear tty1 linux (ganglia,220220,8144,0.0,3178) /usr/sbin/gmond (slot-1,443120,23100,0.0,3214) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1460,0.0,3250) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.9.2/ /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=29890650 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,11704,1556,0.0,3351) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.9.2//env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=29890650 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,16204740,3434152,70.7,3358) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=29890650 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,46172,6612,0.0,3456) condor_starter -f -a slot1_11 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1564,0.0,3504) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_3456/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443116,22932,0.0,3547) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (root,0,0,0.0,4221) [kworker/7:1] (slot-1,11700,1464,0.0,4407) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1552,0.0,4500) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358196,47736,0.0,4508) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1424,0.0,4515) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_3456/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_3456/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_3456/CORSIKA000000.0.log (slot-1,109076,29772,99.9,4516) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_3456/dcors0/corsika.0.Linux (slot-1,4368,356,0.0,4829) sleep 300 (root,0,0,0.0,5057) [kworker/u32:1] (root,591424,14472,0.1,6042) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,6176) [kworker/4:2] (root,108056,360,0.0,6988) sleep 5 (root,8632,664,0.0,7009) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,7010) /bin/bash /bin/check_mk_agent (root,51732,1696,0.0,7027) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,944,0.0,7028) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,7103) [kworker/0:0] (root,0,0,0.0,7775) [kworker/8:2] (root,0,0,0.0,7785) [kworker/0:1] (root,0,0,0.0,7975) [kworker/1:0] (root,0,0,0.0,8326) [kworker/u34:1] (cvmfs,66488,1108,0.0,8444) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,732096,23156,0.0,8447) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66488,1108,0.0,8597) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,796516,24924,0.0,8612) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (slot-1,46172,6608,0.0,8784) condor_starter -f -a slot1_12 pyglidein1-submit.icecube.wisc.edu (slot-1,46172,6608,0.0,8803) condor_starter -f -a slot1_13 pyglidein1-submit.icecube.wisc.edu (cvmfs,66488,1112,0.0,8821) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (slot-1,11704,1564,0.0,8886) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8784/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11704,1560,0.0,8901) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8803/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (cvmfs,3774124,115112,0.1,8922) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (slot-1,443116,23076,0.0,8940) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443120,22848,0.0,8980) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (cvmfs,66488,1116,0.0,9069) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,77040,2560,0.0,9073) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66484,1092,0.0,9077) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,796516,23644,0.0,9081) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,9305) [kworker/5:2] (cvmfs,66488,1104,0.0,9336) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,796520,23692,0.0,9339) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,66488,1104,0.0,9544) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,796528,23668,0.0,9583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,66488,1108,0.0,9721) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,796540,26432,0.0,9822) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,9913) [kworker/7:2] (cvmfs,66488,1108,0.0,9968) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,796676,23592,0.0,9971) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,66488,1104,0.0,10109) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,10110) [kworker/15:1H] (slot-1,11700,1464,0.0,10126) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (cvmfs,796676,23608,0.0,10127) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,10156) [kworker/5:0] (slot-1,11704,1544,0.0,10232) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358200,47740,0.0,10239) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (condor,74780,7768,0.0,10273) /usr/sbin/condor_master -f (root,27644,6480,0.0,10274) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6592,0.0,10278) condor_shared_port (condor,677288,623444,0.0,10288) condor_startd (slot-1,11704,1424,0.0,10359) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8803/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8803/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8803/CORSIKA000000.0.log (slot-1,109076,29292,99.9,10360) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8803/dcors0/corsika.0.Linux (slot-1,11700,1460,0.0,10365) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1548,0.0,10460) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358196,47732,0.0,10467) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (condor,34080820,16588,0.1,10569) GPUs_MONITOR (root,0,0,0.0,10571) [UVM GPU1 BH] (root,0,0,0.0,10572) [UVM GPU2 BH] (root,0,0,0.0,10573) [UVM GPU3 BH] (root,0,0,0.0,10574) [UVM GPU4 BH] (root,0,0,0.0,10575) [UVM GPU5 BH] (root,0,0,0.0,10576) [UVM GPU6 BH] (root,0,0,0.0,10577) [UVM GPU7 BH] (root,0,0,0.0,10578) [UVM GPU8 BH] (slot-1,11704,1424,0.0,10591) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8784/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8784/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8784/CORSIKA000000.0.log (slot-1,109076,29296,99.9,10592) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8784/dcors0/corsika.0.Linux (condor,66444,8328,0.0,12542) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (slot-1,11692,1572,0.0,12597) /bin/sh glidein_start.sh (slot-1,61812,6476,0.0,12867) glideinExec/sbin/condor_master -f -r 20135 (slot-1,11988,1844,0.0,12868) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_d85cb2f3-47fb-46f6-b823-0d04bd2cbf02.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20240911%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240911T014002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=f6ee32e18e084ade0d82b67cc8df727c2a1feab8b009e9332a6d35aa486ae72b (slot-1,25100,5228,0.0,12880) condor_procd -A /scratch/condor/dir_12542/glidein/log.1b86c92c-6fdf-11ef-8619-0cc47a857a10/procd_address -L /scratch/condor/dir_12542/glidein/log.1b86c92c-6fdf-11ef-8619-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,64128,9204,0.0,12907) condor_startd -f (root,0,0,0.0,12962) [kworker/3:0] (root,0,0,0.0,13653) [kworker/15:1] (root,0,0,0.0,13969) [kworker/4:1] (root,0,0,0.0,14415) [kworker/12:0] (root,0,0,0.0,14797) [kworker/11:2] (root,0,0,0.0,14919) [kworker/u34:0] (root,0,0,0.0,16148) [kworker/9:1H] (root,0,0,0.0,16751) [kworker/13:2] (root,0,0,0.0,17076) [kworker/6:0] (root,0,0,0.0,19063) [kworker/1:2] (root,0,0,0.0,19702) [kworker/10:1] (root,0,0,0.0,20236) [kworker/u32:0] (root,0,0,0.0,21256) [kworker/6:1] (slot-1,46176,6616,0.0,21439) condor_starter -f -a slot1_8 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1564,0.0,21476) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_21439/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443116,23328,0.0,21547) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1468,0.0,21623) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1552,0.0,21716) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,0,0,99.5,21723) [python] <defunct> (root,0,0,0.0,21970) [kworker/3:2] (root,0,0,0.0,23669) [kworker/2:0] (root,0,0,0.0,24341) [kworker/12:1] (slot-1,46172,6608,0.0,24623) condor_starter -f -a slot1_9 pyglidein1-submit.icecube.wisc.edu (slot-1,46172,6612,0.0,24646) condor_starter -f -a slot1_10 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,24686) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24623/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443120,23044,0.0,24727) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11704,1568,0.0,24729) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24646/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443116,22824,0.0,24772) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1460,0.0,25692) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1552,0.0,25785) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358200,47740,0.0,25792) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1420,0.0,25906) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24623/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24623/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24623/CORSIKA000000.0.log (slot-1,109076,29912,99.9,25907) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24623/dcors0/corsika.0.Linux (slot-1,11700,1460,0.0,26019) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1548,0.0,26112) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358196,47732,0.0,26119) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1424,0.0,26126) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24646/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24646/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24646/CORSIKA000000.0.log (slot-1,109076,29196,99.9,26127) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_24646/dcors0/corsika.0.Linux (root,0,0,0.0,26248) [kworker/11:0] (root,0,0,0.0,26682) [kworker/14:2] (root,0,0,0.0,27102) [kworker/15:0] (root,0,0,0.0,27217) [kworker/14:1] (slot-1,46180,6616,0.0,27282) condor_starter -f -a slot1_1 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,27286) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_27282/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (postfix,89912,4032,0.0,27315) pickup -l -t unix -u (slot-1,444144,23352,0.0,27326) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1460,0.0,27438) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.9.2/ /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=30160000 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,11704,1552,0.0,27535) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.9.2//env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=30160000 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,15418296,2745216,67.3,27542) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=30160000 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (root,0,0,0.0,28630) [kworker/14:0] (root,0,0,0.0,28790) [kworker/9:1] (root,0,0,0.0,29048) [kworker/2:1] (root,0,0,0.0,29465) [kworker/10:1H] (slot-1,46180,6616,0.0,30882) condor_starter -f -a slot1_6 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1564,0.0,30886) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_30882/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,444140,23264,0.0,30926) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1464,0.0,30936) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.9.2/ /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=30160590 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,11704,1548,0.0,31034) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.9.2//env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=30160590 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,14696468,2051360,71.2,31041) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=30160590 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (root,0,0,0.0,31259) [kworker/10:2] (root,0,0,0.0,31484) [kworker/8:1] (root,0,0,0.0,32110) [kworker/13:0] (slot-1,46180,6616,0.0,32321) condor_starter -f -a slot1_3 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,32325) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_32321/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443120,23220,0.0,32365) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1460,0.0,32370) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.9.2/ /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=29010700 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,11704,1552,0.0,32467) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.9.2//env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=29010700 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,14145352,1578884,71.6,32474) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=29010700 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (root,0,0,0.0,32567) [kworker/u33:2]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac6d5d6d04
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43796,4176,0.0,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.2,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,259) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,405) [scsi_eh_0] (root,0,0,0.0,406) [scsi_tmf_0] (root,0,0,0.0,407) [scsi_eh_1] (root,0,0,0.0,408) [scsi_tmf_1] (root,0,0,0.0,409) [scsi_eh_2] (root,0,0,0.0,410) [scsi_tmf_2] (root,0,0,0.0,411) [scsi_eh_3] (root,0,0,0.0,412) [scsi_tmf_3] (root,0,0,0.0,416) [scsi_eh_4] (root,0,0,0.0,417) [scsi_tmf_4] (root,0,0,0.0,418) [scsi_eh_5] (root,0,0,0.0,419) [scsi_tmf_5] (root,0,0,0.0,420) [scsi_eh_6] (root,0,0,0.0,421) [scsi_tmf_6] (root,0,0,0.0,422) [scsi_eh_7] (root,0,0,0.0,423) [scsi_tmf_7] (root,0,0,0.0,424) [scsi_eh_8] (root,0,0,0.0,425) [scsi_tmf_8] (root,0,0,0.0,426) [scsi_eh_9] (root,0,0,0.0,427) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,551) [kdmflush] (root,0,0,0.0,552) [bioset] (root,0,0,0.0,562) [kdmflush] (root,0,0,0.0,563) [bioset] (root,0,0,0.0,573) [kworker/0:1H] (root,0,0,0.0,582) [jbd2/dm-0-8] (root,0,0,0.0,583) [ext4-rsv-conver] (root,88608,48264,0.0,663) /usr/lib/systemd/systemd-journald (root,45572,2012,0.0,687) /usr/lib/systemd/systemd-udevd (root,0,0,0.0,738) [kworker/3:1H] (root,0,0,0.0,739) [kworker/2:1H] (root,0,0,1.3,740) [kipmi0] (root,190376,1428,0.0,799) /usr/sbin/lvmetad -f (root,0,0,0.0,800) [kvm-irqfd-clean] (root,0,0,0.0,869) [kworker/0:2] (root,0,0,0.0,879) [kdmflush] (root,0,0,0.0,880) [bioset] (root,0,0,0.0,881) [kdmflush] (root,0,0,0.0,882) [bioset] (root,0,0,0.0,884) [kdmflush] (root,0,0,0.0,885) [bioset] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,898) [kworker/11:1H] (root,0,0,0.0,899) [kworker/5:1H] (root,0,0,0.0,900) [kworker/13:1H] (root,0,0,0.0,911) [jbd2/sda1-8] (root,0,0,0.0,912) [ext4-rsv-conver] (root,0,0,0.0,925) [kworker/14:1H] (root,0,0,0.0,931) [kworker/8:1H] (root,0,0,0.0,936) [jbd2/dm-3-8] (root,0,0,0.0,937) [ext4-rsv-conver] (root,0,0,0.0,939) [kworker/12:1H] (root,0,0,0.0,945) [nv_queue] (root,0,0,0.0,946) [nv_queue] (root,0,0,0.0,957) [kworker/1:1H] (root,0,0,0.0,960) [jbd2/dm-2-8] (root,0,0,0.0,961) [ext4-rsv-conver] (root,0,0,0.0,981) [UVM global queu] (root,0,0,0.0,982) [UVM deferred re] (root,0,0,0.0,984) [UVM Tools Event] (root,0,0,0.0,992) [jbd2/dm-5-8] (root,0,0,0.0,993) [ext4-rsv-conver] (root,0,0,0.0,994) [nvidia-modeset/] (root,0,0,0.0,995) [nvidia-modeset/] (root,0,0,0.0,1002) [jbd2/dm-4-8] (root,0,0,0.0,1003) [ext4-rsv-conver] (root,52900,2944,0.0,1023) /usr/sbin/smartd -n -q never (root,26384,1764,0.0,1024) /usr/lib/systemd/systemd-logind (node_ex+,727600,15776,0.1,1030) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,21788,1432,0.0,1034) /usr/sbin/irqbalance --foreground (polkitd,612240,8204,0.0,1052) /usr/lib/polkit-1/polkitd --no-debug (dbus,58220,2432,0.0,1063) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (root,0,0,0.0,1114) [kworker/6:1H] (root,0,0,0.0,1145) [kworker/u32:0] (root,0,0,0.0,1157) [kworker/7:1H] (root,0,0,0.0,1281) [kworker/u33:0] (root,0,0,0.0,1292) [bond0] (root,10080252,224268,0.0,1412) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,113004,4336,0.0,1572) /usr/sbin/sshd -D (root,574288,17588,0.0,1573) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1587) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1603) [irq/98-nvidia] (root,0,0,0.0,1604) [nvidia] (root,0,0,0.1,1605) [nv_queue] (root,323672,69976,0.0,1826) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,339248,29584,0.0,1832) /usr/sbin/rsyslogd -n (root,115708,1924,0.0,1862) /bin/bash /sbin/condor_inspector (root,209016,10708,0.0,1887) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,591424,14548,0.1,1894) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,1899) [irq/99-nvidia] (root,0,0,0.0,1900) [nvidia] (root,0,0,0.1,1901) [nv_queue] (root,358544,9244,0.0,1908) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209264,8140,0.0,1916) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (cvmfs,66484,1108,0.0,1989) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,82684,8208,0.0,2021) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1092,0.0,2027) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,0,0,0.0,2177) [irq/100-nvidia] (root,0,0,0.0,2178) [nvidia] (root,0,0,0.1,2179) [nv_queue] (root,89808,2168,0.0,2216) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2219) qmgr -l -t unix -u (root,0,0,0.0,2224) [irq/101-nvidia] (root,0,0,0.0,2225) [nvidia] (root,0,0,0.0,2226) [nv_queue] (root,0,0,0.0,2227) [irq/102-nvidia] (root,0,0,0.0,2228) [nvidia] (root,0,0,0.1,2229) [nv_queue] (root,0,0,0.0,2230) [irq/103-nvidia] (root,0,0,0.0,2231) [nvidia] (root,0,0,0.0,2232) [nv_queue] (root,0,0,0.0,2233) [irq/104-nvidia] (root,0,0,0.0,2234) [nvidia] (root,0,0,0.1,2235) [nv_queue] (root,0,0,0.0,2237) [irq/105-nvidia] (root,0,0,0.0,2238) [nvidia] (root,0,0,0.1,2239) [nv_queue] (root,0,0,0.0,2304) [kworker/4:1H] (ntp,29960,2172,0.0,2642) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1250172,26812,0.0,2649) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2905) [kworker/9:0] (root,126392,1604,0.0,3168) /usr/sbin/crond -n (root,110208,860,0.0,3174) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.0,3178) /usr/sbin/gmond (root,0,0,0.0,3863) [kworker/15:0] (slot-1,0,0,50.4,5581) [python] <defunct> (root,0,0,0.0,7103) [kworker/0:0] (root,0,0,0.0,7183) [kworker/14:2] (slot-1,1213796,880528,99.9,7470) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.Corsika5ComponentGenerator --args (slot-1,11704,1428,0.0,7472) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_10375/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_10375/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_10375/CORSIKA000000.5626.log (slot-1,162068,66732,1.4,7473) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_10375/dcors0/corsika.0.Linux (slot-1,11704,1420,0.0,7474) sh -c gzip -9 -c > ./DAT000000.gz (slot-1,4640,788,0.0,7475) gzip -9 -c (root,0,0,0.0,7775) [kworker/8:2] (root,0,0,0.0,8216) [kworker/7:1] (cvmfs,66488,1112,0.0,8444) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,732096,22988,0.0,8447) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66488,1112,0.0,8597) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,796516,24776,0.0,8612) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,66488,1116,0.0,8821) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,3774084,101324,0.1,8922) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,66488,1120,0.0,9069) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,77040,2548,0.0,9073) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66484,1096,0.0,9077) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,796516,23644,0.0,9081) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,66488,1108,0.0,9336) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,796528,23676,0.0,9339) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,66488,1108,0.0,9544) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,796528,23656,0.0,9583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,66488,1112,0.0,9721) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,796424,26388,0.0,9822) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,9913) [kworker/7:2] (cvmfs,66488,1112,0.0,9968) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,796516,23260,0.0,9971) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,66488,1108,0.0,10109) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,10110) [kworker/15:1H] (cvmfs,796600,23584,0.0,10127) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,74780,8028,0.0,10273) /usr/sbin/condor_master -f (root,27644,6560,0.1,10274) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6764,0.0,10278) condor_shared_port (condor,576916,523464,0.0,10288) condor_startd (slot-1,46176,6612,0.0,10375) condor_starter -f -a slot1_9 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,10420) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_10375/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,375528,22928,0.0,10462) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1464,0.0,10512) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env /cvmfs/icecube.opensciencegrid.org/users/mhuennefeld/software/icecube/py3-v4.3.0/v1.11.0-rc1/icetray.HEAD.r94427c54.Linux-x86_64.gcc-13.1.0 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.Corsika5ComponentGenerator --args (condor,34080820,17172,0.1,10569) GPUs_MONITOR (root,0,0,0.0,10571) [UVM GPU1 BH] (root,0,0,0.0,10572) [UVM GPU2 BH] (root,0,0,0.0,10573) [UVM GPU3 BH] (root,0,0,0.0,10574) [UVM GPU4 BH] (root,0,0,0.0,10575) [UVM GPU5 BH] (root,0,0,0.0,10576) [UVM GPU6 BH] (root,0,0,0.0,10577) [UVM GPU7 BH] (root,0,0,0.0,10578) [UVM GPU8 BH] (slot-1,11704,1552,0.0,10621) bash /cvmfs/icecube.opensciencegrid.org/users/mhuennefeld/software/icecube/py3-v4.3.0/v1.11.0-rc1/icetray.HEAD.r94427c54.Linux-x86_64.gcc-13.1.0/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.Corsika5ComponentGenerator --args (slot-1,382384,49740,0.0,10628) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.Corsika5ComponentGenerator --args (slot-1,0,0,0.9,10632) [python] <defunct> (root,0,0,0.0,11282) [kworker/1:2] (root,0,0,0.0,12372) [kworker/14:0] (condor,66440,8708,0.0,12542) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (slot-1,11692,1572,0.0,12597) /bin/sh glidein_start.sh (root,0,0,0.0,12702) [kworker/11:2] (slot-1,61812,6472,0.0,12867) glideinExec/sbin/condor_master -f -r 20135 (slot-1,11988,1796,0.0,12868) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_d85cb2f3-47fb-46f6-b823-0d04bd2cbf02.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20240911%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240911T014002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=f6ee32e18e084ade0d82b67cc8df727c2a1feab8b009e9332a6d35aa486ae72b (slot-1,25100,5228,0.0,12880) condor_procd -A /scratch/condor/dir_12542/glidein/log.1b86c92c-6fdf-11ef-8619-0cc47a857a10/procd_address -L /scratch/condor/dir_12542/glidein/log.1b86c92c-6fdf-11ef-8619-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,64128,9216,0.0,12907) condor_startd -f (root,0,0,0.0,13653) [kworker/15:1] (root,0,0,0.0,13750) [kworker/9:2] (postfix,89912,4028,0.0,14957) pickup -l -t unix -u (root,0,0,0.0,15044) [kworker/8:0] (root,0,0,0.0,16060) [kworker/5:2] (root,0,0,0.0,16148) [kworker/9:1H] (root,0,0,0.0,16278) [kworker/5:0] (root,0,0,0.0,16289) [kworker/12:0] (root,0,0,0.0,16360) [kworker/u32:2] (root,0,0,0.0,17051) [kworker/10:1] (root,0,0,0.0,17197) [kworker/5:1] (root,0,0,0.0,17937) [kworker/10:2] (slot-1,46180,6616,0.0,18073) condor_starter -f -a slot1_1 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,18077) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_18073/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443116,23316,0.0,18117) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1468,0.0,18295) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.9.2/ /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=28370710 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,11704,1548,0.0,18558) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.9.2//env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=28370710 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,14004632,1379620,68.8,18565) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=28370710 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (root,0,0,0.0,19966) [kworker/2:2] (root,0,0,0.0,21207) [kworker/14:1] (root,0,0,0.0,21256) [kworker/6:1] (root,0,0,0.0,21594) [kworker/2:0] (slot-1,4368,360,0.0,21877) sleep 300 (root,0,0,0.0,22315) [kworker/1:1] (root,108056,360,0.0,22562) sleep 5 (root,8632,664,0.0,22563) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,22564) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,22581) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,22582) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,23495) [kworker/u33:1] (root,0,0,0.0,24321) [kworker/u34:2] (root,0,0,0.0,24341) [kworker/12:1] (root,0,0,0.0,24401) [kworker/4:1] (root,0,0,0.0,25322) [kworker/1:0] (root,0,0,0.0,26248) [kworker/11:0] (slot-1,0,0,3.9,26298) [python] <defunct> (root,0,0,0.0,26401) [kworker/6:0] (root,0,0,0.0,27287) [kworker/3:2] (root,0,0,0.0,27548) [kworker/3:1] (root,0,0,0.0,27649) [kworker/u34:0] (root,0,0,0.0,27679) [kworker/13:1] (slot-1,46180,6612,0.0,29387) condor_starter -f -a slot1_2 pyglidein1-submit.icecube.wisc.edu (root,0,0,0.0,29465) [kworker/10:1H] (slot-1,11704,1568,0.0,29515) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_29387/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,444144,23292,0.0,29555) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1468,0.0,29768) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.9.2/ /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=28370870 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,11704,1552,0.0,29865) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.9.2//env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=28370870 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,14740892,2158608,71.7,29872) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=28370870 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (root,0,0,0.0,30191) [kworker/10:0] (root,0,0,0.0,31905) [kworker/13:0] (root,0,0,0.0,32115) [kworker/4:2] (slot-1,0,0,11.3,32611) [python] <defunct>
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac40722ae7
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43796,4176,0.0,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.2,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,259) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,405) [scsi_eh_0] (root,0,0,0.0,406) [scsi_tmf_0] (root,0,0,0.0,407) [scsi_eh_1] (root,0,0,0.0,408) [scsi_tmf_1] (root,0,0,0.0,409) [scsi_eh_2] (root,0,0,0.0,410) [scsi_tmf_2] (root,0,0,0.0,411) [scsi_eh_3] (root,0,0,0.0,412) [scsi_tmf_3] (root,0,0,0.0,416) [scsi_eh_4] (root,0,0,0.0,417) [scsi_tmf_4] (root,0,0,0.0,418) [scsi_eh_5] (root,0,0,0.0,419) [scsi_tmf_5] (root,0,0,0.0,420) [scsi_eh_6] (root,0,0,0.0,421) [scsi_tmf_6] (root,0,0,0.0,422) [scsi_eh_7] (root,0,0,0.0,423) [scsi_tmf_7] (root,0,0,0.0,424) [scsi_eh_8] (root,0,0,0.0,425) [scsi_tmf_8] (root,0,0,0.0,426) [scsi_eh_9] (root,0,0,0.0,427) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,551) [kdmflush] (root,0,0,0.0,552) [bioset] (root,0,0,0.0,562) [kdmflush] (root,0,0,0.0,563) [bioset] (root,0,0,0.0,573) [kworker/0:1H] (root,0,0,0.0,582) [jbd2/dm-0-8] (root,0,0,0.0,583) [ext4-rsv-conver] (root,47648,11532,0.0,663) /usr/lib/systemd/systemd-journald (root,45572,2028,0.0,687) /usr/lib/systemd/systemd-udevd (root,0,0,0.0,738) [kworker/3:1H] (root,0,0,0.0,739) [kworker/2:1H] (root,0,0,1.3,740) [kipmi0] (root,190376,1428,0.0,799) /usr/sbin/lvmetad -f (root,0,0,0.0,800) [kvm-irqfd-clean] (root,0,0,0.0,879) [kdmflush] (root,0,0,0.0,880) [bioset] (root,0,0,0.0,881) [kdmflush] (root,0,0,0.0,882) [bioset] (root,0,0,0.0,884) [kdmflush] (root,0,0,0.0,885) [bioset] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,898) [kworker/11:1H] (root,0,0,0.0,899) [kworker/5:1H] (root,0,0,0.0,900) [kworker/13:1H] (root,0,0,0.0,911) [jbd2/sda1-8] (root,0,0,0.0,912) [ext4-rsv-conver] (root,0,0,0.0,925) [kworker/14:1H] (root,0,0,0.0,931) [kworker/8:1H] (root,0,0,0.0,936) [jbd2/dm-3-8] (root,0,0,0.0,937) [ext4-rsv-conver] (root,0,0,0.0,939) [kworker/12:1H] (root,0,0,0.0,945) [nv_queue] (root,0,0,0.0,946) [nv_queue] (root,0,0,0.0,957) [kworker/1:1H] (root,0,0,0.0,960) [jbd2/dm-2-8] (root,0,0,0.0,961) [ext4-rsv-conver] (root,0,0,0.0,981) [UVM global queu] (root,0,0,0.0,982) [UVM deferred re] (root,0,0,0.0,984) [UVM Tools Event] (root,0,0,0.0,992) [jbd2/dm-5-8] (root,0,0,0.0,993) [ext4-rsv-conver] (root,0,0,0.0,994) [nvidia-modeset/] (root,0,0,0.0,995) [nvidia-modeset/] (root,0,0,0.0,1002) [jbd2/dm-4-8] (root,0,0,0.0,1003) [ext4-rsv-conver] (root,0,0,0.0,1006) [kworker/1:2] (root,52900,2944,0.0,1023) /usr/sbin/smartd -n -q never (root,26384,1760,0.0,1024) /usr/lib/systemd/systemd-logind (node_ex+,727600,15788,0.1,1030) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,21788,1432,0.0,1034) /usr/sbin/irqbalance --foreground (polkitd,612240,8204,0.0,1052) /usr/lib/polkit-1/polkitd --no-debug (dbus,58220,2432,0.0,1063) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (root,0,0,0.0,1114) [kworker/6:1H] (root,0,0,0.0,1157) [kworker/7:1H] (root,0,0,0.0,1292) [bond0] (root,10080252,221448,0.0,1412) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,113004,4336,0.0,1572) /usr/sbin/sshd -D (root,574288,17588,0.0,1573) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1587) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1603) [irq/98-nvidia] (root,0,0,0.0,1604) [nvidia] (root,0,0,0.1,1605) [nv_queue] (root,0,0,0.0,1622) [kworker/15:1] (root,323672,69976,0.0,1826) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,298288,9020,0.0,1832) /usr/sbin/rsyslogd -n (root,115708,1924,0.0,1862) /bin/bash /sbin/condor_inspector (root,209084,10840,0.0,1887) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,591424,14548,0.1,1894) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,1899) [irq/99-nvidia] (root,0,0,0.0,1900) [nvidia] (root,0,0,0.1,1901) [nv_queue] (root,358544,9184,0.0,1908) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209412,8192,0.0,1916) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (cvmfs,66484,1108,0.0,1989) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,82684,8208,0.0,2021) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1092,0.0,2027) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,0,0,0.0,2116) [kworker/13:2] (root,0,0,0.0,2177) [irq/100-nvidia] (root,0,0,0.0,2178) [nvidia] (root,0,0,0.1,2179) [nv_queue] (root,89808,2168,0.0,2216) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2219) qmgr -l -t unix -u (root,0,0,0.0,2224) [irq/101-nvidia] (root,0,0,0.0,2225) [nvidia] (root,0,0,0.0,2226) [nv_queue] (root,0,0,0.0,2227) [irq/102-nvidia] (root,0,0,0.0,2228) [nvidia] (root,0,0,0.1,2229) [nv_queue] (root,0,0,0.0,2230) [irq/103-nvidia] (root,0,0,0.0,2231) [nvidia] (root,0,0,0.0,2232) [nv_queue] (root,0,0,0.0,2233) [irq/104-nvidia] (root,0,0,0.0,2234) [nvidia] (root,0,0,0.1,2235) [nv_queue] (root,0,0,0.0,2237) [irq/105-nvidia] (root,0,0,0.0,2238) [nvidia] (root,0,0,0.1,2239) [nv_queue] (root,0,0,0.0,2304) [kworker/4:1H] (root,0,0,0.0,2366) [kworker/4:2] (ntp,29960,2172,0.0,2642) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1250172,25776,0.0,2649) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2905) [kworker/9:0] (root,126392,1604,0.0,3168) /usr/sbin/crond -n (root,110208,860,0.0,3174) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.0,3178) /usr/sbin/gmond (root,0,0,0.0,4597) [kworker/10:0] (root,0,0,0.0,4971) [kworker/u32:1] (root,0,0,0.0,5467) [kworker/6:0] (slot-1,1133516,800048,99.9,5581) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.Corsika5ComponentGenerator --args (slot-1,11704,1420,0.0,5583) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_10375/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_10375/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_10375/CORSIKA000000.1407.log (slot-1,162068,66688,1.9,5584) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_10375/dcors0/corsika.0.Linux (slot-1,11704,1416,0.0,5585) sh -c gzip -9 -c > ./DAT000000.gz (slot-1,4640,788,0.0,5586) gzip -9 -c (root,0,0,0.0,5755) [kworker/u34:0] (root,0,0,0.0,7676) [kworker/10:1] (root,0,0,0.0,7775) [kworker/8:2] (root,0,0,0.0,8355) [kworker/12:0] (cvmfs,66488,1112,0.0,8444) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,731568,23252,0.0,8447) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66488,1112,0.0,8597) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,796516,24600,0.0,8612) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,66488,1116,0.0,8821) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,3773672,94996,0.1,8922) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,66488,1120,0.0,9069) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,77040,2548,0.0,9073) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66484,1096,0.0,9077) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,796516,23644,0.0,9081) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,66488,1108,0.0,9336) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,796516,23684,0.0,9339) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,66488,1108,0.0,9544) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,796528,23656,0.0,9583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,66488,1112,0.0,9721) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,796688,26384,0.0,9822) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,9846) [kworker/u34:2] (cvmfs,66488,1112,0.0,9968) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,731568,23172,0.0,9971) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,66488,1108,0.0,10109) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,10110) [kworker/15:1H] (cvmfs,796560,23576,0.0,10127) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,74780,8028,0.0,10273) /usr/sbin/condor_master -f (root,27644,6560,0.1,10274) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6764,0.0,10278) condor_shared_port (condor,477760,424300,0.0,10288) condor_startd (slot-1,46176,6612,0.0,10375) condor_starter -f -a slot1_9 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,10420) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_10375/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,375528,22748,0.0,10462) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1464,0.0,10512) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env /cvmfs/icecube.opensciencegrid.org/users/mhuennefeld/software/icecube/py3-v4.3.0/v1.11.0-rc1/icetray.HEAD.r94427c54.Linux-x86_64.gcc-13.1.0 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.Corsika5ComponentGenerator --args (condor,34080820,17172,0.1,10569) GPUs_MONITOR (root,0,0,0.0,10571) [UVM GPU1 BH] (root,0,0,0.0,10572) [UVM GPU2 BH] (root,0,0,0.0,10573) [UVM GPU3 BH] (root,0,0,0.0,10574) [UVM GPU4 BH] (root,0,0,0.0,10575) [UVM GPU5 BH] (root,0,0,0.0,10576) [UVM GPU6 BH] (root,0,0,0.0,10577) [UVM GPU7 BH] (root,0,0,0.0,10578) [UVM GPU8 BH] (slot-1,11704,1552,0.0,10621) bash /cvmfs/icecube.opensciencegrid.org/users/mhuennefeld/software/icecube/py3-v4.3.0/v1.11.0-rc1/icetray.HEAD.r94427c54.Linux-x86_64.gcc-13.1.0/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.Corsika5ComponentGenerator --args (slot-1,382384,49568,0.0,10628) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.Corsika5ComponentGenerator --args (slot-1,0,0,2.6,10632) [python] <defunct> (root,0,0,0.0,10824) [kworker/7:1] (root,0,0,0.0,11369) [kworker/u32:0] (condor,66428,8700,0.0,12542) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (slot-1,11692,1572,0.0,12597) /bin/sh glidein_start.sh (root,0,0,0.0,12696) [kworker/u33:0] (root,0,0,0.0,12739) [kworker/12:2] (slot-1,61812,6544,0.0,12867) glideinExec/sbin/condor_master -f -r 20135 (slot-1,11892,1740,0.0,12868) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_d85cb2f3-47fb-46f6-b823-0d04bd2cbf02.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20240911%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240911T014002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=f6ee32e18e084ade0d82b67cc8df727c2a1feab8b009e9332a6d35aa486ae72b (slot-1,25100,5260,0.0,12880) condor_procd -A /scratch/condor/dir_12542/glidein/log.1b86c92c-6fdf-11ef-8619-0cc47a857a10/procd_address -L /scratch/condor/dir_12542/glidein/log.1b86c92c-6fdf-11ef-8619-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,64128,9408,0.0,12907) condor_startd -f (root,0,0,0.0,13186) [kworker/5:2] (root,0,0,0.0,14613) [kworker/8:0] (root,0,0,0.0,15171) [kworker/15:0] (root,0,0,0.0,15767) [kworker/11:1] (root,0,0,0.0,15882) [kworker/14:1] (root,0,0,0.0,16148) [kworker/9:1H] (root,0,0,0.0,16804) [kworker/0:2] (root,0,0,0.0,17713) [kworker/4:1] (root,0,0,0.0,18956) [kworker/4:0] (slot-1,46172,6608,0.0,19261) condor_starter -f -a slot1_10 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,19309) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_19261/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443120,22868,0.0,19349) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1464,0.0,20830) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1552,0.0,20923) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358200,47736,0.0,20930) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1420,0.0,21045) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_19261/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_19261/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_19261/CORSIKA000000.0.log (slot-1,109076,29872,99.9,21046) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_19261/dcors0/corsika.0.Linux (root,0,0,0.0,21256) [kworker/6:1] (root,0,0,0.0,21532) [kworker/2:1] (root,0,0,0.0,21566) [kworker/0:0] (root,0,0,0.0,21615) [kworker/9:2] (root,0,0,0.0,22780) [kworker/5:0] (postfix,89912,4032,0.0,23807) pickup -l -t unix -u (root,0,0,0.0,23843) [kworker/11:2] (root,0,0,0.0,24238) [kworker/1:1] (slot-1,4368,360,0.0,25873) sleep 300 (root,0,0,0.0,25903) [kworker/2:0] (slot-1,0,0,11.2,26298) [python] <defunct> (root,0,0,0.0,28189) [kworker/2:2] (root,0,0,0.0,29230) [kworker/0:1] (root,0,0,0.0,29242) [kworker/14:0] (root,0,0,0.0,29340) [kworker/13:1] (root,0,0,0.0,29465) [kworker/10:1H] (root,0,0,0.0,29600) [kworker/3:1] (root,428820,95628,30.0,30222) puppet agent: applying configuration (root,108056,352,0.0,30566) sleep 5 (root,11688,1352,0.0,30586) sh -c /usr/bin/rpm -qa --nosignature --nodigest --qf '%{NAME} %|EPOCH?{%{EPOCH}}:{0}| %{VERSION} %{RELEASE} %{ARCH}\n' 2>&1 (root,93684,23876,0.0,30587) /usr/bin/rpm -qa --nosignature --nodigest --qf %{NAME} %|EPOCH?{%{EPOCH}}:{0}| %{VERSION} %{RELEASE} %{ARCH}\n (root,8632,664,0.0,30588) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,30589) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,30606) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,30607) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,30685) [kworker/u33:2] (root,0,0,0.0,30916) [kworker/3:0] (root,0,0,0.0,30998) [kworker/7:0] (root,0,0,0.0,32244) [kworker/1:0]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac1c538bfd
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43796,4176,0.0,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.2,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,259) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,405) [scsi_eh_0] (root,0,0,0.0,406) [scsi_tmf_0] (root,0,0,0.0,407) [scsi_eh_1] (root,0,0,0.0,408) [scsi_tmf_1] (root,0,0,0.0,409) [scsi_eh_2] (root,0,0,0.0,410) [scsi_tmf_2] (root,0,0,0.0,411) [scsi_eh_3] (root,0,0,0.0,412) [scsi_tmf_3] (root,0,0,0.0,416) [scsi_eh_4] (root,0,0,0.0,417) [scsi_tmf_4] (root,0,0,0.0,418) [scsi_eh_5] (root,0,0,0.0,419) [scsi_tmf_5] (root,0,0,0.0,420) [scsi_eh_6] (root,0,0,0.0,421) [scsi_tmf_6] (root,0,0,0.0,422) [scsi_eh_7] (root,0,0,0.0,423) [scsi_tmf_7] (root,0,0,0.0,424) [scsi_eh_8] (root,0,0,0.0,425) [scsi_tmf_8] (root,0,0,0.0,426) [scsi_eh_9] (root,0,0,0.0,427) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,551) [kdmflush] (root,0,0,0.0,552) [bioset] (root,0,0,0.0,562) [kdmflush] (root,0,0,0.0,563) [bioset] (root,0,0,0.0,573) [kworker/0:1H] (root,0,0,0.0,582) [jbd2/dm-0-8] (root,0,0,0.0,583) [ext4-rsv-conver] (root,88608,47372,0.0,663) /usr/lib/systemd/systemd-journald (root,45572,2028,0.0,687) /usr/lib/systemd/systemd-udevd (root,0,0,0.0,738) [kworker/3:1H] (root,0,0,0.0,739) [kworker/2:1H] (root,0,0,1.3,740) [kipmi0] (root,190376,1428,0.0,799) /usr/sbin/lvmetad -f (root,0,0,0.0,800) [kvm-irqfd-clean] (root,0,0,0.0,879) [kdmflush] (root,0,0,0.0,880) [bioset] (root,0,0,0.0,881) [kdmflush] (root,0,0,0.0,882) [bioset] (root,0,0,0.0,884) [kdmflush] (root,0,0,0.0,885) [bioset] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,898) [kworker/11:1H] (root,0,0,0.0,899) [kworker/5:1H] (root,0,0,0.0,900) [kworker/13:1H] (root,0,0,0.0,911) [jbd2/sda1-8] (root,0,0,0.0,912) [ext4-rsv-conver] (root,0,0,0.0,925) [kworker/14:1H] (root,0,0,0.0,931) [kworker/8:1H] (root,0,0,0.0,936) [jbd2/dm-3-8] (root,0,0,0.0,937) [ext4-rsv-conver] (root,0,0,0.0,939) [kworker/12:1H] (root,0,0,0.0,945) [nv_queue] (root,0,0,0.0,946) [nv_queue] (root,0,0,0.0,957) [kworker/1:1H] (root,0,0,0.0,960) [jbd2/dm-2-8] (root,0,0,0.0,961) [ext4-rsv-conver] (root,0,0,0.0,981) [UVM global queu] (root,0,0,0.0,982) [UVM deferred re] (root,0,0,0.0,984) [UVM Tools Event] (root,0,0,0.0,992) [jbd2/dm-5-8] (root,0,0,0.0,993) [ext4-rsv-conver] (root,0,0,0.0,994) [nvidia-modeset/] (root,0,0,0.0,995) [nvidia-modeset/] (root,0,0,0.0,1002) [jbd2/dm-4-8] (root,0,0,0.0,1003) [ext4-rsv-conver] (root,0,0,0.0,1006) [kworker/1:2] (root,52900,2944,0.0,1023) /usr/sbin/smartd -n -q never (root,26384,1760,0.0,1024) /usr/lib/systemd/systemd-logind (node_ex+,727600,16000,0.1,1030) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,21788,1432,0.0,1034) /usr/sbin/irqbalance --foreground (polkitd,612240,8200,0.0,1052) /usr/lib/polkit-1/polkitd --no-debug (dbus,58220,2432,0.0,1063) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (root,0,0,0.0,1114) [kworker/6:1H] (root,0,0,0.0,1157) [kworker/7:1H] (root,0,0,0.0,1252) [kworker/5:0] (root,0,0,0.0,1292) [bond0] (root,10080252,220584,0.0,1412) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,113004,4336,0.0,1572) /usr/sbin/sshd -D (root,574288,17588,0.0,1573) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1587) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1603) [irq/98-nvidia] (root,0,0,0.0,1604) [nvidia] (root,0,0,0.1,1605) [nv_queue] (root,323672,69976,0.0,1826) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,309508,28720,0.0,1832) /usr/sbin/rsyslogd -n (root,115708,1924,0.0,1862) /bin/bash /sbin/condor_inspector (root,209084,10840,0.0,1887) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,591424,14548,0.1,1894) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,1899) [irq/99-nvidia] (root,0,0,0.0,1900) [nvidia] (root,0,0,0.1,1901) [nv_queue] (root,358544,9184,0.0,1908) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209412,8192,0.0,1916) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (cvmfs,66484,1108,0.0,1989) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,82684,8208,0.0,2021) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1092,0.0,2027) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,0,0,0.0,2177) [irq/100-nvidia] (root,0,0,0.0,2178) [nvidia] (root,0,0,0.1,2179) [nv_queue] (root,89808,2168,0.0,2216) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2219) qmgr -l -t unix -u (root,0,0,0.0,2224) [irq/101-nvidia] (root,0,0,0.0,2225) [nvidia] (root,0,0,0.0,2226) [nv_queue] (root,0,0,0.0,2227) [irq/102-nvidia] (root,0,0,0.0,2228) [nvidia] (root,0,0,0.1,2229) [nv_queue] (root,0,0,0.0,2230) [irq/103-nvidia] (root,0,0,0.0,2231) [nvidia] (root,0,0,0.0,2232) [nv_queue] (root,0,0,0.0,2233) [irq/104-nvidia] (root,0,0,0.0,2234) [nvidia] (root,0,0,0.1,2235) [nv_queue] (root,0,0,0.0,2237) [irq/105-nvidia] (root,0,0,0.0,2238) [nvidia] (root,0,0,0.1,2239) [nv_queue] (root,0,0,0.0,2304) [kworker/4:1H] (root,0,0,0.0,2439) [kworker/15:0] (ntp,29960,2172,0.0,2642) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1250172,24924,0.0,2649) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2905) [kworker/9:0] (root,126392,1604,0.0,3168) /usr/sbin/crond -n (root,110208,860,0.0,3174) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.0,3178) /usr/sbin/gmond (root,0,0,0.0,4066) [kworker/u33:1] (root,0,0,0.0,4592) [kworker/10:2] (root,0,0,0.0,4994) [kworker/13:2] (root,0,0,0.0,5916) [kworker/u32:1] (slot-1,1584044,1250640,99.9,6026) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.Corsika5ComponentGenerator --args (slot-1,11704,1420,0.0,6028) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_17426/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_17426/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_17426/CORSIKA000000.2713.log (slot-1,162068,65912,0.7,6029) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_17426/dcors0/corsika.0.Linux (slot-1,11704,1420,0.0,6030) sh -c gzip -9 -c > ./DAT000000.gz (slot-1,4640,784,0.0,6031) gzip -9 -c (root,0,0,0.0,6075) [kworker/2:0] (root,0,0,0.0,6434) [kworker/0:1] (slot-1,4368,360,0.0,6842) sleep 300 (root,0,0,0.0,8225) [kworker/5:1] (cvmfs,66488,1112,0.0,8444) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,657700,22984,0.0,8447) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66488,1112,0.0,8597) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,796520,24452,0.0,8612) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,8636) [kworker/8:0] (root,0,0,0.0,8714) [kworker/u34:0] (cvmfs,66488,1116,0.0,8821) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,3249484,87540,0.1,8922) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,66488,1120,0.0,9069) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,77040,2504,0.0,9073) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66484,1096,0.0,9077) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,796516,23644,0.0,9081) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,66488,1108,0.0,9336) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,796528,23680,0.0,9339) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,66488,1108,0.0,9544) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,796528,23660,0.0,9583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,66488,1112,0.0,9721) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,796540,26356,0.0,9822) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,66488,1112,0.0,9968) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,731568,22928,0.0,9971) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,66488,1108,0.0,10109) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,10110) [kworker/15:1H] (cvmfs,796520,23592,0.0,10127) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,74780,8028,0.0,10273) /usr/sbin/condor_master -f (root,27644,6544,0.0,10274) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6756,0.0,10278) condor_shared_port (condor,381172,327696,0.0,10288) condor_startd (condor,34080820,17172,0.1,10569) GPUs_MONITOR (root,0,0,0.0,10571) [UVM GPU1 BH] (root,0,0,0.0,10572) [UVM GPU2 BH] (root,0,0,0.0,10573) [UVM GPU3 BH] (root,0,0,0.0,10574) [UVM GPU4 BH] (root,0,0,0.0,10575) [UVM GPU5 BH] (root,0,0,0.0,10576) [UVM GPU6 BH] (root,0,0,0.0,10577) [UVM GPU7 BH] (root,0,0,0.0,10578) [UVM GPU8 BH] (root,0,0,0.0,10839) [kworker/12:0] (root,0,0,0.0,11299) [kworker/13:1] (root,0,0,0.0,11302) [kworker/3:0] (root,0,0,0.0,11330) [kworker/u34:2] (slot-1,46172,6612,0.0,11526) condor_starter -f -a slot1_14 pyglidein1-submit.icecube.wisc.edu (root,0,0,0.0,11594) [kworker/1:1] (slot-1,11704,1572,0.0,11599) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_11526/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443116,22800,0.0,11655) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (postfix,89912,4028,0.0,11668) pickup -l -t unix -u (slot-1,46120,6580,0.8,12303) condor_starter -f -a slot1_1 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.8,12307) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_12303/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443116,27744,6.0,12347) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1472,0.0,12352) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1556,0.0,12445) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,487292,273748,111,12452) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (condor,66440,8716,0.0,12542) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (root,108056,360,0.0,12561) sleep 5 (root,8632,660,0.0,12562) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,12563) /bin/bash /bin/check_mk_agent (root,51732,1696,0.0,12580) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,12581) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (slot-1,11692,1572,0.0,12597) /bin/sh glidein_start.sh (root,0,0,0.0,12726) [kworker/13:0] (slot-1,61812,6544,0.0,12867) glideinExec/sbin/condor_master -f -r 20135 (slot-1,11892,1680,0.0,12868) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_d85cb2f3-47fb-46f6-b823-0d04bd2cbf02.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20240911%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240911T014002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=f6ee32e18e084ade0d82b67cc8df727c2a1feab8b009e9332a6d35aa486ae72b (slot-1,25100,5260,0.0,12880) condor_procd -A /scratch/condor/dir_12542/glidein/log.1b86c92c-6fdf-11ef-8619-0cc47a857a10/procd_address -L /scratch/condor/dir_12542/glidein/log.1b86c92c-6fdf-11ef-8619-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,64128,9404,0.0,12907) condor_startd -f (root,0,0,0.0,13261) [kworker/3:1] (slot-1,11700,1456,0.0,14584) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1552,0.0,14785) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358196,47732,0.0,14792) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1420,0.0,14799) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_11526/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_11526/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_11526/CORSIKA000000.0.log (slot-1,109076,29876,99.9,14800) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_11526/dcors0/corsika.0.Linux (root,0,0,0.0,14895) [kworker/15:1] (root,0,0,0.0,15857) [kworker/15:2] (root,0,0,0.0,16112) [kworker/u33:2] (root,0,0,0.0,16148) [kworker/9:1H] (root,0,0,0.0,16713) [kworker/11:0] (root,0,0,0.0,16804) [kworker/0:2] (slot-1,46176,6608,0.0,17426) condor_starter -f -a slot1_12 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1572,0.0,17661) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_17426/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,375528,22852,0.0,17705) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1464,0.0,17707) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env /cvmfs/icecube.opensciencegrid.org/users/mhuennefeld/software/icecube/py3-v4.3.0/v1.11.0-rc1/icetray.HEAD.r94427c54.Linux-x86_64.gcc-13.1.0 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.Corsika5ComponentGenerator --args (root,0,0,0.0,17713) [kworker/4:1] (slot-1,11704,1548,0.0,17806) bash /cvmfs/icecube.opensciencegrid.org/users/mhuennefeld/software/icecube/py3-v4.3.0/v1.11.0-rc1/icetray.HEAD.r94427c54.Linux-x86_64.gcc-13.1.0/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.Corsika5ComponentGenerator --args (slot-1,382384,49652,0.0,17813) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.Corsika5ComponentGenerator --args (slot-1,0,0,3.3,17814) [python] <defunct> (root,0,0,0.0,18076) [kworker/4:0] (root,0,0,0.0,19041) [kworker/6:2] (slot-1,46172,6608,0.0,19261) condor_starter -f -a slot1_10 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,19309) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_19261/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443120,22748,0.0,19349) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (root,0,0,0.0,19750) [kworker/3:2] (root,0,0,0.0,19790) [kworker/8:1] (slot-1,0,0,3.3,20735) [python] <defunct> (slot-1,11700,1464,0.0,20830) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1552,0.0,20923) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358200,47736,0.0,20930) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (root,0,0,0.0,21020) [kworker/u32:2] (slot-1,11704,1420,0.0,21045) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_19261/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_19261/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_19261/CORSIKA000000.0.log (slot-1,109076,29872,99.9,21046) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_19261/dcors0/corsika.0.Linux (root,0,0,0.0,22807) [kworker/8:2] (slot-1,0,0,1.1,23061) [python] <defunct> (root,0,0,0.0,24331) [kworker/u32:0] (root,0,0,0.0,26034) [kworker/14:0] (root,0,0,0.0,26938) [kworker/6:1] (root,0,0,0.0,27100) [kworker/10:0] (slot-1,46180,6612,0.0,27579) condor_starter -f -a slot1_6 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,27583) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_27579/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,444140,23216,0.0,27623) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1464,0.0,27736) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.9.2/ /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=28370780 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (root,0,0,0.0,27828) [kworker/9:1] (slot-1,11704,1552,0.0,27833) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.9.2//env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=28370780 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,15132996,2499344,73.1,27840) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=28370780 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (root,0,0,0.0,28517) [kworker/14:2] (root,0,0,0.0,29465) [kworker/10:1H] (root,0,0,0.0,29691) [kworker/2:1] (root,0,0,0.0,30051) [kworker/11:2] (root,0,0,0.0,30998) [kworker/7:0] (root,0,0,0.0,31144) [kworker/7:2] (root,0,0,0.0,31565) [kworker/12:1]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09acbe1e13e1
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43796,4176,0.0,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.2,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,259) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,405) [scsi_eh_0] (root,0,0,0.0,406) [scsi_tmf_0] (root,0,0,0.0,407) [scsi_eh_1] (root,0,0,0.0,408) [scsi_tmf_1] (root,0,0,0.0,409) [scsi_eh_2] (root,0,0,0.0,410) [scsi_tmf_2] (root,0,0,0.0,411) [scsi_eh_3] (root,0,0,0.0,412) [scsi_tmf_3] (root,0,0,0.0,416) [scsi_eh_4] (root,0,0,0.0,417) [scsi_tmf_4] (root,0,0,0.0,418) [scsi_eh_5] (root,0,0,0.0,419) [scsi_tmf_5] (root,0,0,0.0,420) [scsi_eh_6] (root,0,0,0.0,421) [scsi_tmf_6] (root,0,0,0.0,422) [scsi_eh_7] (root,0,0,0.0,423) [scsi_tmf_7] (root,0,0,0.0,424) [scsi_eh_8] (root,0,0,0.0,425) [scsi_tmf_8] (root,0,0,0.0,426) [scsi_eh_9] (root,0,0,0.0,427) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,551) [kdmflush] (root,0,0,0.0,552) [bioset] (root,0,0,0.0,562) [kdmflush] (root,0,0,0.0,563) [bioset] (root,0,0,0.0,573) [kworker/0:1H] (root,0,0,0.0,582) [jbd2/dm-0-8] (root,0,0,0.0,583) [ext4-rsv-conver] (root,47648,10756,0.0,663) /usr/lib/systemd/systemd-journald (root,45572,2028,0.0,687) /usr/lib/systemd/systemd-udevd (root,0,0,0.0,738) [kworker/3:1H] (root,0,0,0.0,739) [kworker/2:1H] (root,0,0,1.4,740) [kipmi0] (root,190376,1428,0.0,799) /usr/sbin/lvmetad -f (root,0,0,0.0,800) [kvm-irqfd-clean] (slot-1,46176,6612,0.0,854) condor_starter -f -a slot1_3 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1564,0.0,858) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_854/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (root,0,0,0.0,865) [kworker/1:0] (root,0,0,0.0,879) [kdmflush] (root,0,0,0.0,880) [bioset] (root,0,0,0.0,881) [kdmflush] (root,0,0,0.0,882) [bioset] (root,0,0,0.0,884) [kdmflush] (root,0,0,0.0,885) [bioset] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,898) [kworker/11:1H] (root,0,0,0.0,899) [kworker/5:1H] (root,0,0,0.0,900) [kworker/13:1H] (slot-1,443080,23128,0.0,909) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,46172,6608,0.0,910) condor_starter -f -a slot1_4 pyglidein1-submit.icecube.wisc.edu (root,0,0,0.0,911) [jbd2/sda1-8] (root,0,0,0.0,912) [ext4-rsv-conver] (slot-1,11704,1572,0.0,916) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_910/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (root,0,0,0.0,925) [kworker/14:1H] (root,0,0,0.0,931) [kworker/8:1H] (root,0,0,0.0,936) [jbd2/dm-3-8] (root,0,0,0.0,937) [ext4-rsv-conver] (root,0,0,0.0,939) [kworker/12:1H] (root,0,0,0.0,945) [nv_queue] (root,0,0,0.0,946) [nv_queue] (root,0,0,0.0,957) [kworker/1:1H] (root,0,0,0.0,960) [jbd2/dm-2-8] (root,0,0,0.0,961) [ext4-rsv-conver] (slot-1,443080,23320,0.0,966) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (root,0,0,0.0,981) [UVM global queu] (root,0,0,0.0,982) [UVM deferred re] (root,0,0,0.0,984) [UVM Tools Event] (root,0,0,0.0,992) [jbd2/dm-5-8] (root,0,0,0.0,993) [ext4-rsv-conver] (root,0,0,0.0,994) [nvidia-modeset/] (root,0,0,0.0,995) [nvidia-modeset/] (root,0,0,0.0,1002) [jbd2/dm-4-8] (root,0,0,0.0,1003) [ext4-rsv-conver] (root,0,0,0.0,1006) [kworker/1:2] (root,52900,2944,0.0,1023) /usr/sbin/smartd -n -q never (root,26384,1760,0.0,1024) /usr/lib/systemd/systemd-logind (node_ex+,727344,15888,0.1,1030) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,21788,1432,0.0,1034) /usr/sbin/irqbalance --foreground (polkitd,612240,8196,0.0,1052) /usr/lib/polkit-1/polkitd --no-debug (dbus,58220,2432,0.0,1063) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (root,0,0,0.0,1114) [kworker/6:1H] (root,0,0,0.0,1157) [kworker/7:1H] (root,0,0,0.0,1292) [bond0] (root,10080252,219872,0.0,1412) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,113004,4336,0.0,1572) /usr/sbin/sshd -D (root,574288,17588,0.0,1573) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1587) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1603) [irq/98-nvidia] (root,0,0,0.0,1604) [nvidia] (root,0,0,0.1,1605) [nv_queue] (slot-1,11700,1456,0.0,1632) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.11.1 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1556,0.0,1729) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.11.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13622848,1038132,71.0,1736) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,323672,69976,0.0,1826) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,268548,8308,0.0,1832) /usr/sbin/rsyslogd -n (slot-1,11700,1456,0.0,1851) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.11.1 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,115708,1924,0.0,1862) /bin/bash /sbin/condor_inspector (root,209084,10840,0.0,1887) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,591424,14544,0.1,1894) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,1899) [irq/99-nvidia] (root,0,0,0.0,1900) [nvidia] (root,0,0,0.0,1901) [nv_queue] (root,358544,9168,0.0,1908) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209412,8192,0.0,1916) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (slot-1,11704,1552,0.0,1959) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.11.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13580164,986828,71.4,1966) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (cvmfs,66484,1108,0.0,1989) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,82684,8208,0.0,2021) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1092,0.0,2027) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,0,0,0.0,2177) [irq/100-nvidia] (root,0,0,0.0,2178) [nvidia] (root,0,0,0.0,2179) [nv_queue] (root,89808,2168,0.0,2216) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2219) qmgr -l -t unix -u (root,0,0,0.0,2224) [irq/101-nvidia] (root,0,0,0.0,2225) [nvidia] (root,0,0,0.0,2226) [nv_queue] (root,0,0,0.0,2227) [irq/102-nvidia] (root,0,0,0.0,2228) [nvidia] (root,0,0,0.0,2229) [nv_queue] (root,0,0,0.0,2230) [irq/103-nvidia] (root,0,0,0.0,2231) [nvidia] (root,0,0,0.0,2232) [nv_queue] (root,0,0,0.0,2233) [irq/104-nvidia] (root,0,0,0.0,2234) [nvidia] (root,0,0,0.1,2235) [nv_queue] (root,0,0,0.0,2237) [irq/105-nvidia] (root,0,0,0.0,2238) [nvidia] (root,0,0,0.1,2239) [nv_queue] (root,0,0,0.0,2304) [kworker/4:1H] (ntp,29960,2172,0.0,2642) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1249164,24516,0.0,2649) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,126392,1604,0.0,3168) /usr/sbin/crond -n (root,110208,860,0.0,3174) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.0,3178) /usr/sbin/gmond (root,0,0,0.0,3835) [kworker/4:2] (slot-1,1052448,719084,99.8,6026) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.Corsika5ComponentGenerator --args (slot-1,11704,1420,0.0,6028) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_17426/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_17426/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_17426/CORSIKA000000.2713.log (slot-1,162068,66620,1.2,6029) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_17426/dcors0/corsika.0.Linux (slot-1,11704,1420,0.0,6030) sh -c gzip -9 -c > ./DAT000000.gz (slot-1,4640,784,0.0,6031) gzip -9 -c (root,0,0,0.0,6143) [kworker/10:1] (root,0,0,0.0,7147) [kworker/5:2] (root,0,0,0.0,7689) [kworker/9:2] (root,0,0,0.0,8216) [kworker/8:0] (root,0,0,0.0,8225) [kworker/5:1] (cvmfs,66488,1112,0.0,8444) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,657420,22900,0.0,8447) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66488,1112,0.0,8597) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,796516,24280,0.0,8612) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,8783) [kworker/12:2] (cvmfs,66488,1116,0.0,8821) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,2856348,79608,0.1,8922) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,66488,1120,0.0,9069) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,77040,2504,0.0,9073) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66484,1096,0.0,9077) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,796516,23644,0.0,9081) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,9263) [kworker/5:0] (cvmfs,66488,1108,0.0,9336) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,796516,23684,0.0,9339) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (slot-1,46176,6612,0.0,9440) condor_starter -f -a slot1_2 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,9444) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_9440/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443084,23328,0.0,9484) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (cvmfs,66488,1108,0.0,9544) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,796528,23656,0.0,9583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,9691) [kworker/u32:1] (cvmfs,66488,1112,0.0,9721) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,796464,26316,0.0,9822) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,66488,1112,0.0,9968) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,731288,23092,0.0,9971) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (slot-1,11700,1468,0.0,10046) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.11.1 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.PolyplopiaModule --args (cvmfs,66488,1108,0.0,10109) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,10110) [kworker/15:1H] (cvmfs,796528,23588,0.0,10127) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (slot-1,11700,1460,0.0,10230) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.11.1 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (condor,74780,8028,0.0,10273) /usr/sbin/condor_master -f (root,27644,6544,0.0,10274) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6752,0.0,10278) condor_shared_port (condor,285600,232204,0.0,10288) condor_startd (root,0,0,0.0,10301) [kworker/0:1] (slot-1,11704,1552,0.0,10331) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.11.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13622000,1026524,71.9,10338) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1552,0.0,10401) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.11.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.PolyplopiaModule --args (slot-1,14688812,2134272,96.9,10408) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.PolyplopiaModule --args (condor,34080820,17172,0.1,10569) GPUs_MONITOR (root,0,0,0.0,10571) [UVM GPU1 BH] (root,0,0,0.0,10572) [UVM GPU2 BH] (root,0,0,0.0,10573) [UVM GPU3 BH] (root,0,0,0.0,10574) [UVM GPU4 BH] (root,0,0,0.0,10575) [UVM GPU5 BH] (root,0,0,0.0,10576) [UVM GPU6 BH] (root,0,0,0.0,10577) [UVM GPU7 BH] (root,0,0,0.0,10578) [UVM GPU8 BH] (root,0,0,0.0,11111) [kworker/9:1] (slot-1,46172,6612,0.0,11526) condor_starter -f -a slot1_14 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1572,0.0,11599) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_11526/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443116,23124,0.0,11655) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (root,0,0,0.0,11734) [kworker/6:0] (root,0,0,0.0,12339) [kworker/14:0] (condor,66440,8712,0.0,12542) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (root,0,0,0.0,12562) [kworker/u34:2] (slot-1,11692,1572,0.0,12597) /bin/sh glidein_start.sh (slot-1,61812,6544,0.0,12867) glideinExec/sbin/condor_master -f -r 20135 (slot-1,11792,1620,0.0,12868) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_d85cb2f3-47fb-46f6-b823-0d04bd2cbf02.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20240911%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240911T014002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=f6ee32e18e084ade0d82b67cc8df727c2a1feab8b009e9332a6d35aa486ae72b (slot-1,25100,5236,0.0,12880) condor_procd -A /scratch/condor/dir_12542/glidein/log.1b86c92c-6fdf-11ef-8619-0cc47a857a10/procd_address -L /scratch/condor/dir_12542/glidein/log.1b86c92c-6fdf-11ef-8619-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,64128,9404,0.0,12907) condor_startd -f (root,0,0,0.0,13606) [kworker/3:2] (root,0,0,0.0,14178) [kworker/15:0] (slot-1,11700,1456,0.0,14584) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (root,0,0,0.0,14612) [kworker/11:2] (root,0,0,0.0,14707) [kworker/3:1] (slot-1,11704,1552,0.0,14785) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358196,47732,0.0,14792) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1420,0.0,14799) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_11526/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_11526/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_11526/CORSIKA000000.0.log (slot-1,109076,29876,99.8,14800) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_11526/dcors0/corsika.0.Linux (slot-1,46172,6612,0.0,15177) condor_starter -f -a slot1_9 pyglidein1-submit.icecube.wisc.edu (slot-1,46172,6612,0.0,15180) condor_starter -f -a slot1_10 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,15232) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15177/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443120,23068,0.0,15274) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11704,1572,0.0,15276) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15180/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443120,23004,0.0,15318) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,4368,360,0.0,15607) sleep 300 (root,0,0,0.0,15629) [kworker/u33:1] (root,0,0,0.0,15831) [kworker/7:1] (root,0,0,0.0,16148) [kworker/9:1H] (root,0,0,0.0,16462) [kworker/u32:2] (slot-1,11700,1460,0.0,16496) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (root,0,0,0.0,16504) [kworker/11:0] (slot-1,11704,1552,0.0,16589) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358196,47732,0.0,16596) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (root,0,0,0.0,16804) [kworker/0:2] (slot-1,11704,1424,0.0,16967) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15180/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15180/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15180/CORSIKA000000.0.log (slot-1,109076,29884,99.8,16968) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15180/dcors0/corsika.0.Linux (slot-1,46176,6608,0.0,17426) condor_starter -f -a slot1_12 pyglidein1-submit.icecube.wisc.edu (root,0,0,0.0,17443) [kworker/13:2] (root,0,0,0.0,17474) [kworker/2:2] (slot-1,11700,1456,0.0,17480) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1548,0.0,17575) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358196,47732,0.0,17582) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1572,0.0,17661) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_17426/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11704,1420,0.0,17694) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15177/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15177/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15177/CORSIKA000000.0.log (slot-1,109076,29916,99.8,17695) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15177/dcors0/corsika.0.Linux (slot-1,375528,22920,0.0,17705) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1464,0.0,17707) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env /cvmfs/icecube.opensciencegrid.org/users/mhuennefeld/software/icecube/py3-v4.3.0/v1.11.0-rc1/icetray.HEAD.r94427c54.Linux-x86_64.gcc-13.1.0 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.Corsika5ComponentGenerator --args (slot-1,11704,1548,0.0,17806) bash /cvmfs/icecube.opensciencegrid.org/users/mhuennefeld/software/icecube/py3-v4.3.0/v1.11.0-rc1/icetray.HEAD.r94427c54.Linux-x86_64.gcc-13.1.0/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.Corsika5ComponentGenerator --args (slot-1,382384,49652,0.0,17813) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.Corsika5ComponentGenerator --args (slot-1,0,0,9.3,17814) [python] <defunct> (root,0,0,0.0,17910) [kworker/3:0] (root,0,0,0.0,18204) [kworker/u33:2] (root,0,0,0.0,19243) [kworker/7:0] (root,0,0,0.0,19823) [kworker/11:1] (root,0,0,0.0,19873) [kworker/u34:1] (slot-1,46116,6580,0.2,20364) condor_starter -f -a slot1_5 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,20368) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_20364/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443120,27752,3.0,20408) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1472,0.0,20517) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1552,0.0,20613) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,891476,564152,95.0,20620) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,108056,360,0.0,20732) sleep 5 (root,8632,660,0.0,20733) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,20734) /bin/bash /bin/check_mk_agent (slot-1,0,0,10.3,20735) [python] <defunct> (root,51732,1696,0.0,20752) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,20753) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,20955) [kworker/8:2] (root,0,0,0.0,21246) [kworker/12:0] (slot-1,46176,6612,0.0,21977) condor_starter -f -a slot1_7 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1564,0.0,21981) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_21977/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443080,23348,0.0,22023) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,0,0,3.3,23061) [python] <defunct> (slot-1,11700,1468,0.0,23091) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.11.1 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1548,0.0,23189) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.11.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13585524,995904,69.7,23196) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,24289) [kworker/u32:0] (root,0,0,0.0,25359) [kworker/2:0] (root,0,0,0.0,25419) [kworker/13:1] (root,0,0,0.0,26339) [kworker/4:0] (root,0,0,0.0,27100) [kworker/10:0] (root,0,0,0.0,27240) [kworker/15:2] (slot-1,46180,6612,0.0,27579) condor_starter -f -a slot1_6 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,27583) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_27579/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,444140,23328,0.0,27623) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1464,0.0,27736) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.9.2/ /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=28370780 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,11704,1552,0.0,27833) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.9.2//env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=28370780 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,14566060,2019384,72.2,27840) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=28370780 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (root,0,0,0.0,28517) [kworker/14:2] (root,0,0,0.0,28787) [kworker/6:2] (root,0,0,0.0,29465) [kworker/10:1H] (postfix,89912,4028,0.0,30034) pickup -l -t unix -u (slot-1,46176,6616,0.0,32394) condor_starter -f -a slot1_1 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,32398) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_32394/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443084,24328,0.0,32438) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac010b6b36
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43796,4176,0.0,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.2,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,100) [kworker/0:1] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,143) [kworker/15:1] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,259) [kauditd] (slot-1,11700,1464,0.0,305) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.11.1 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1552,0.0,402) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.11.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,405) [scsi_eh_0] (root,0,0,0.0,406) [scsi_tmf_0] (root,0,0,0.0,407) [scsi_eh_1] (root,0,0,0.0,408) [scsi_tmf_1] (root,0,0,0.0,409) [scsi_eh_2] (root,0,0,0.0,410) [scsi_tmf_2] (root,0,0,0.0,411) [scsi_eh_3] (root,0,0,0.0,412) [scsi_tmf_3] (root,0,0,0.0,416) [scsi_eh_4] (root,0,0,0.0,417) [scsi_tmf_4] (root,0,0,0.0,418) [scsi_eh_5] (root,0,0,0.0,419) [scsi_tmf_5] (root,0,0,0.0,420) [scsi_eh_6] (root,0,0,0.0,421) [scsi_tmf_6] (root,0,0,0.0,422) [scsi_eh_7] (root,0,0,0.0,423) [scsi_tmf_7] (root,0,0,0.0,424) [scsi_eh_8] (root,0,0,0.0,425) [scsi_tmf_8] (root,0,0,0.0,426) [scsi_eh_9] (root,0,0,0.0,427) [scsi_tmf_9] (slot-1,13656224,1097344,70.2,430) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,551) [kdmflush] (root,0,0,0.0,552) [bioset] (root,0,0,0.0,562) [kdmflush] (root,0,0,0.0,563) [bioset] (root,0,0,0.0,573) [kworker/0:1H] (root,0,0,0.0,582) [jbd2/dm-0-8] (root,0,0,0.0,583) [ext4-rsv-conver] (root,92708,48612,0.0,663) /usr/lib/systemd/systemd-journald (root,0,0,0.0,686) [kworker/7:2] (root,45572,2028,0.0,687) /usr/lib/systemd/systemd-udevd (root,0,0,0.0,738) [kworker/3:1H] (root,0,0,0.0,739) [kworker/2:1H] (root,0,0,1.5,740) [kipmi0] (root,0,0,0.0,758) [kworker/8:2] (root,190376,1428,0.0,799) /usr/sbin/lvmetad -f (root,0,0,0.0,800) [kvm-irqfd-clean] (root,0,0,0.0,838) [kworker/6:2] (slot-1,11700,1468,0.0,873) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.11.1 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,879) [kdmflush] (root,0,0,0.0,880) [bioset] (root,0,0,0.0,881) [kdmflush] (root,0,0,0.0,882) [bioset] (root,0,0,0.0,884) [kdmflush] (root,0,0,0.0,885) [bioset] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,898) [kworker/11:1H] (root,0,0,0.0,899) [kworker/5:1H] (root,0,0,0.0,900) [kworker/13:1H] (root,0,0,0.0,911) [jbd2/sda1-8] (root,0,0,0.0,912) [ext4-rsv-conver] (root,0,0,0.0,925) [kworker/14:1H] (root,0,0,0.0,931) [kworker/8:1H] (root,0,0,0.0,934) [kworker/11:2] (root,0,0,0.0,936) [jbd2/dm-3-8] (root,0,0,0.0,937) [ext4-rsv-conver] (root,0,0,0.0,939) [kworker/12:1H] (root,0,0,0.0,945) [nv_queue] (root,0,0,0.0,946) [nv_queue] (root,0,0,0.0,957) [kworker/1:1H] (root,0,0,0.0,960) [jbd2/dm-2-8] (root,0,0,0.0,961) [ext4-rsv-conver] (root,0,0,0.0,981) [UVM global queu] (root,0,0,0.0,982) [UVM deferred re] (root,0,0,0.0,984) [UVM Tools Event] (root,0,0,0.0,992) [jbd2/dm-5-8] (root,0,0,0.0,993) [ext4-rsv-conver] (root,0,0,0.0,994) [nvidia-modeset/] (root,0,0,0.0,995) [nvidia-modeset/] (slot-1,11704,1548,0.0,1001) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.11.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,1002) [jbd2/dm-4-8] (root,0,0,0.0,1003) [ext4-rsv-conver] (slot-1,13590380,1008456,69.5,1010) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,52900,2944,0.0,1023) /usr/sbin/smartd -n -q never (root,26384,1760,0.0,1024) /usr/lib/systemd/systemd-logind (node_ex+,727344,15984,0.2,1030) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,21788,1432,0.0,1034) /usr/sbin/irqbalance --foreground (polkitd,612240,8196,0.0,1052) /usr/lib/polkit-1/polkitd --no-debug (dbus,58220,2432,0.0,1063) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (root,0,0,0.0,1114) [kworker/6:1H] (root,0,0,0.0,1157) [kworker/7:1H] (root,0,0,0.0,1292) [bond0] (root,10080252,218860,0.0,1412) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,113004,4336,0.0,1572) /usr/sbin/sshd -D (root,574288,17588,0.0,1573) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1587) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1603) [irq/98-nvidia] (root,0,0,0.0,1604) [nvidia] (root,0,0,0.1,1605) [nv_queue] (root,0,0,0.0,1803) [kworker/3:2] (root,323672,69976,0.0,1826) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,271824,28748,0.0,1832) /usr/sbin/rsyslogd -n (root,115708,1924,0.0,1862) /bin/bash /sbin/condor_inspector (root,209084,10764,0.0,1887) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,591424,14536,0.1,1894) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,1899) [irq/99-nvidia] (root,0,0,0.0,1900) [nvidia] (root,0,0,0.1,1901) [nv_queue] (root,358544,9168,0.0,1908) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209412,8192,0.0,1916) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (cvmfs,66484,1108,0.0,1989) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,82688,8208,0.0,2021) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1092,0.0,2027) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,0,0,0.0,2177) [irq/100-nvidia] (root,0,0,0.0,2178) [nvidia] (root,0,0,0.1,2179) [nv_queue] (root,89808,2168,0.0,2216) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2219) qmgr -l -t unix -u (root,0,0,0.0,2224) [irq/101-nvidia] (root,0,0,0.0,2225) [nvidia] (root,0,0,0.0,2226) [nv_queue] (root,0,0,0.0,2227) [irq/102-nvidia] (root,0,0,0.0,2228) [nvidia] (root,0,0,0.1,2229) [nv_queue] (root,0,0,0.0,2230) [irq/103-nvidia] (root,0,0,0.0,2231) [nvidia] (root,0,0,0.1,2232) [nv_queue] (root,0,0,0.0,2233) [irq/104-nvidia] (root,0,0,0.0,2234) [nvidia] (root,0,0,0.1,2235) [nv_queue] (root,0,0,0.0,2237) [irq/105-nvidia] (root,0,0,0.0,2238) [nvidia] (root,0,0,0.1,2239) [nv_queue] (root,0,0,0.0,2304) [kworker/4:1H] (slot-1,46176,6612,0.0,2641) condor_starter -f -a slot1_3 pyglidein1-submit.icecube.wisc.edu (ntp,29960,2172,0.0,2642) /usr/sbin/ntpd -u ntp:ntp -g (slot-1,11704,1568,0.0,2646) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_2641/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (cvmfs,1248636,23180,0.0,2649) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (slot-1,443084,23308,0.0,2700) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1456,0.0,3136) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.11.1 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,126392,1604,0.0,3168) /usr/sbin/crond -n (root,110208,860,0.0,3174) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.0,3178) /usr/sbin/gmond (root,0,0,0.0,3219) [kworker/u33:0] (root,0,0,0.0,3235) [kworker/14:0] (slot-1,11704,1556,0.0,3238) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.11.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13517740,930380,70.6,3245) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,3559) [kworker/4:2] (slot-1,46176,6616,0.0,4694) condor_starter -f -a slot1_4 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,4698) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_4694/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443080,23180,0.0,4740) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1460,0.0,5140) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.11.1 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1552,0.0,5239) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.11.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13842756,1265180,71.7,5246) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,5278) [kworker/7:1] (root,0,0,0.0,5301) [kworker/4:1] (postfix,89912,4032,0.0,5683) pickup -l -t unix -u (root,0,0,0.0,6200) [kworker/u34:0] (root,0,0,0.0,7689) [kworker/9:2] (root,0,0,0.0,7995) [kworker/8:0] (cvmfs,66488,1112,0.0,8444) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,657272,22884,0.0,8447) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66488,1112,0.0,8597) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,722648,23964,0.0,8612) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (slot-1,46172,6608,0.0,8666) condor_starter -f -a slot1_5 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1564,0.0,8670) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_8666/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443084,23144,0.0,8710) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (root,0,0,0.0,8783) [kworker/12:2] (cvmfs,66488,1116,0.0,8821) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,2594016,64604,0.1,8922) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (slot-1,11700,1460,0.0,8954) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.11.1 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1552,0.0,9051) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.11.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13549848,951000,70.2,9058) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (cvmfs,66488,1120,0.0,9069) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,76904,2324,0.0,9073) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66484,1096,0.0,9077) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,796516,23644,0.0,9081) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,66488,1108,0.0,9336) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,796520,23684,0.0,9339) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,66488,1108,0.0,9544) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,722648,23576,0.0,9583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (root,0,0,0.0,9646) [kworker/u32:0] (cvmfs,66488,1112,0.0,9721) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,722652,25308,0.0,9822) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,66488,1112,0.0,9968) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,657272,22812,0.0,9971) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,66488,1108,0.0,10109) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,10110) [kworker/15:1H] (cvmfs,722668,23464,0.0,10127) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,74780,8028,0.0,10273) /usr/sbin/condor_master -f (root,27644,6544,0.1,10274) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6752,0.0,10278) condor_shared_port (condor,180796,127364,0.0,10288) condor_startd (root,0,0,0.0,10327) [kworker/0:2] (condor,34080820,17172,0.2,10569) GPUs_MONITOR (root,0,0,0.0,10571) [UVM GPU1 BH] (root,0,0,0.0,10572) [UVM GPU2 BH] (root,0,0,0.0,10573) [UVM GPU3 BH] (root,0,0,0.0,10574) [UVM GPU4 BH] (root,0,0,0.0,10575) [UVM GPU5 BH] (root,0,0,0.0,10576) [UVM GPU6 BH] (root,0,0,0.0,10577) [UVM GPU7 BH] (root,0,0,0.0,10578) [UVM GPU8 BH] (condor,66440,8712,0.0,12542) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (slot-1,11692,1572,0.0,12597) /bin/sh glidein_start.sh (slot-1,61812,6544,0.0,12867) glideinExec/sbin/condor_master -f -r 20135 (slot-1,11696,1552,0.0,12868) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_d85cb2f3-47fb-46f6-b823-0d04bd2cbf02.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20240911%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240911T014002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=f6ee32e18e084ade0d82b67cc8df727c2a1feab8b009e9332a6d35aa486ae72b (slot-1,25100,5236,0.0,12880) condor_procd -A /scratch/condor/dir_12542/glidein/log.1b86c92c-6fdf-11ef-8619-0cc47a857a10/procd_address -L /scratch/condor/dir_12542/glidein/log.1b86c92c-6fdf-11ef-8619-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,64128,9276,0.0,12907) condor_startd -f (root,0,0,0.0,12937) [kworker/12:0] (root,0,0,0.0,13415) [kworker/14:2] (root,0,0,0.0,14825) [kworker/9:0] (slot-1,46172,6612,0.0,15177) condor_starter -f -a slot1_9 pyglidein1-submit.icecube.wisc.edu (slot-1,46172,6612,0.0,15180) condor_starter -f -a slot1_10 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,15232) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15177/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443120,23072,0.0,15274) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11704,1572,0.0,15276) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15180/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443120,22880,0.0,15318) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (root,0,0,0.0,16148) [kworker/9:1H] (slot-1,11700,1460,0.0,16496) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1552,0.0,16589) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358196,47732,0.0,16596) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (root,0,0,0.0,16748) [kworker/1:1] (root,0,0,0.0,16836) [kworker/15:0] (slot-1,11704,1424,0.0,16967) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15180/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15180/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15180/CORSIKA000000.0.log (slot-1,109076,29884,99.6,16968) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15180/dcors0/corsika.0.Linux (slot-1,11700,1456,0.0,17480) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1548,0.0,17575) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358196,47732,0.0,17582) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1420,0.0,17694) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15177/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15177/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15177/CORSIKA000000.0.log (slot-1,109076,29916,99.7,17695) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_15177/dcors0/corsika.0.Linux (root,0,0,0.0,18787) [kworker/u34:2] (root,0,0,0.0,18798) [kworker/2:0] (root,0,0,0.0,18818) [kworker/6:0] (root,0,0,0.0,19511) [kworker/2:1] (slot-1,4368,360,0.0,20514) sleep 300 (slot-1,46176,6608,0.0,20620) condor_starter -f -a slot1_7 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1564,0.0,20624) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_20620/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443084,23328,0.0,20664) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (root,0,0,0.0,20763) [kworker/10:0] (slot-1,11700,1468,0.0,20890) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.11.1 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1552,0.0,20987) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.11.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13557148,960260,70.3,20994) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,21898) [kworker/11:1] (root,0,0,0.0,22019) [kworker/3:1] (root,0,0,0.0,22082) [kworker/5:2] (root,0,0,0.0,23058) [kworker/u32:2] (root,0,0,0.0,23102) [kworker/10:2] (slot-1,46116,6580,0.0,23562) condor_starter -f -a slot1_11 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,23566) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_23562/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443116,27764,0.4,23606) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (root,0,0,0.0,24129) [kworker/7:0] (slot-1,11700,1460,0.0,24251) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/icetray-env /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2/ python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1548,0.0,24344) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.8.2//env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,358196,142724,1.4,24351) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.CorsikaGenerator --args (slot-1,11704,1424,0.0,24360) /bin/sh -c /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_23562/dcors0/corsika.0.Linux < /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_23562/dcors0/INPUTS > /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_23562/CORSIKA000000.0.log (slot-1,109076,29848,98.9,24361) /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_23562/dcors0/corsika.0.Linux (root,0,0,0.0,25419) [kworker/13:1] (root,536640,193772,17.3,25450) puppet agent: applying configuration (root,0,0,0.0,25902) [kworker/u33:1] (root,350780,48568,6.1,26115) /usr/bin/python /usr/bin/yum check-update (root,0,0,0.0,26271) [kworker/8:1] (root,108056,352,0.0,26375) sleep 5 (root,8632,664,0.0,26378) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,26379) /bin/bash /bin/check_mk_agent (root,51732,1696,0.0,26396) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,26397) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,26661) [kworker/6:1] (slot-1,46180,6612,0.0,27579) condor_starter -f -a slot1_6 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,27583) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_27579/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443116,23224,0.0,27623) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1464,0.0,27736) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.9.2/ /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=28370780 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,11704,1552,0.0,27833) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.9.2//env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=28370780 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,13694184,1017212,75.0,27840) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.2/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=28370780 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=500 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (root,0,0,0.0,29170) [kworker/13:0] (slot-1,46180,6616,0.0,29204) condor_starter -f -a slot1_2 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,29208) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_29204/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443080,23100,0.0,29248) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (root,0,0,0.0,29465) [kworker/10:1H] (slot-1,11700,1468,0.0,29480) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.11.1 python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1552,0.0,29577) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.11.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13642328,1065708,71.0,29584) python /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1/RHEL_7_x86_64/lib/python3.10/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,30544) [kworker/1:0] (root,0,0,0.0,31567) [kworker/u32:1] (root,0,0,0.0,31759) [kworker/5:0] (slot-1,46176,6612,0.0,31939) condor_starter -f -a slot1_1 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,32081) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_31939/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443084,23144,0.0,32124) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (root,0,0,0.0,32262) [kworker/10:1] (slot-1,46176,6608,0.0,32274) condor_starter -f -a slot1_8 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,32278) /bin/sh /scratch/condor/dir_12542/glidein/execute.1b86c92c-6fdf-11ef-8619-0cc47a857a10/dir_32274/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443084,23192,0.0,32319) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (root,0,0,0.0,32602) [kworker/u34:1]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac39501fce
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43796,4176,0.1,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.3,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,100) [kworker/0:1] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,104) [kworker/10:1] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,133) [kworker/1:1] (root,0,0,0.0,138) [kworker/5:1] (root,0,0,0.0,140) [kworker/12:1] (root,0,0,0.0,142) [kworker/14:1] (root,0,0,0.0,143) [kworker/15:1] (root,0,0,0.0,144) [kworker/13:1] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,259) [kauditd] (root,0,0,0.0,403) [ata_sff] (root,0,0,0.0,404) [kworker/4:2] (root,0,0,0.0,405) [scsi_eh_0] (root,0,0,0.0,406) [scsi_tmf_0] (root,0,0,0.0,407) [scsi_eh_1] (root,0,0,0.0,408) [scsi_tmf_1] (root,0,0,0.0,409) [scsi_eh_2] (root,0,0,0.0,410) [scsi_tmf_2] (root,0,0,0.0,411) [scsi_eh_3] (root,0,0,0.0,412) [scsi_tmf_3] (root,0,0,0.0,416) [scsi_eh_4] (root,0,0,0.0,417) [scsi_tmf_4] (root,0,0,0.0,418) [scsi_eh_5] (root,0,0,0.0,419) [scsi_tmf_5] (root,0,0,0.0,420) [scsi_eh_6] (root,0,0,0.0,421) [scsi_tmf_6] (root,0,0,0.0,422) [scsi_eh_7] (root,0,0,0.0,423) [scsi_tmf_7] (root,0,0,0.0,424) [scsi_eh_8] (root,0,0,0.0,425) [scsi_tmf_8] (root,0,0,0.0,426) [scsi_eh_9] (root,0,0,0.0,427) [scsi_tmf_9] (root,0,0,0.0,436) [ttm_swap] (root,0,0,0.0,457) [kworker/u32:0] (root,0,0,0.0,551) [kdmflush] (root,0,0,0.0,552) [bioset] (root,0,0,0.0,562) [kdmflush] (root,0,0,0.0,563) [bioset] (root,0,0,0.0,573) [kworker/0:1H] (root,0,0,0.0,582) [jbd2/dm-0-8] (root,0,0,0.0,583) [ext4-rsv-conver] (root,55836,21372,0.0,663) /usr/lib/systemd/systemd-journald (root,0,0,0.0,686) [kworker/7:2] (root,45572,2028,0.0,687) /usr/lib/systemd/systemd-udevd (root,0,0,0.0,738) [kworker/3:1H] (root,0,0,0.0,739) [kworker/2:1H] (root,0,0,1.6,740) [kipmi0] (root,0,0,0.0,758) [kworker/8:2] (root,190376,1428,0.0,799) /usr/sbin/lvmetad -f (root,0,0,0.0,800) [kvm-irqfd-clean] (root,0,0,0.0,879) [kdmflush] (root,0,0,0.0,880) [bioset] (root,0,0,0.0,881) [kdmflush] (root,0,0,0.0,882) [bioset] (root,0,0,0.0,884) [kdmflush] (root,0,0,0.0,885) [bioset] (root,0,0,0.0,887) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,898) [kworker/11:1H] (root,0,0,0.0,899) [kworker/5:1H] (root,0,0,0.0,900) [kworker/13:1H] (root,0,0,0.0,911) [jbd2/sda1-8] (root,0,0,0.0,912) [ext4-rsv-conver] (root,0,0,0.0,925) [kworker/14:1H] (root,0,0,0.0,931) [kworker/8:1H] (root,0,0,0.0,934) [kworker/11:2] (root,0,0,0.0,936) [jbd2/dm-3-8] (root,0,0,0.0,937) [ext4-rsv-conver] (root,0,0,0.0,939) [kworker/12:1H] (root,0,0,0.0,945) [nv_queue] (root,0,0,0.0,946) [nv_queue] (root,0,0,0.0,957) [kworker/1:1H] (root,0,0,0.0,960) [jbd2/dm-2-8] (root,0,0,0.0,961) [ext4-rsv-conver] (root,0,0,0.0,968) [kworker/6:2] (root,0,0,0.0,981) [UVM global queu] (root,0,0,0.0,982) [UVM deferred re] (root,0,0,0.0,984) [UVM Tools Event] (root,0,0,0.0,992) [jbd2/dm-5-8] (root,0,0,0.0,993) [ext4-rsv-conver] (root,0,0,0.0,994) [nvidia-modeset/] (root,0,0,0.0,995) [nvidia-modeset/] (root,0,0,0.0,1002) [jbd2/dm-4-8] (root,0,0,0.0,1003) [ext4-rsv-conver] (root,52900,2944,0.0,1023) /usr/sbin/smartd -n -q never (root,26384,1760,0.0,1024) /usr/lib/systemd/systemd-logind (node_ex+,727088,14684,0.2,1030) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (root,21788,1432,0.0,1034) /usr/sbin/irqbalance --foreground (polkitd,612240,8196,0.0,1052) /usr/lib/polkit-1/polkitd --no-debug (dbus,58220,2432,0.0,1063) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (root,0,0,0.0,1114) [kworker/6:1H] (root,0,0,0.0,1157) [kworker/7:1H] (root,0,0,0.0,1292) [bond0] (root,10070568,191752,0.1,1412) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,113004,4336,0.0,1572) /usr/sbin/sshd -D (root,574288,17588,0.0,1573) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,27168,1052,0.0,1587) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1603) [irq/98-nvidia] (root,0,0,0.0,1604) [nvidia] (root,0,0,0.2,1605) [nv_queue] (root,323672,69912,0.0,1826) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,239056,13456,0.0,1832) /usr/sbin/rsyslogd -n (root,0,0,0.0,1855) [kworker/5:0] (root,115708,1912,0.0,1862) /bin/bash /sbin/condor_inspector (root,208916,10764,0.0,1887) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,591424,14536,0.2,1894) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,1899) [irq/99-nvidia] (root,0,0,0.0,1900) [nvidia] (root,0,0,0.2,1901) [nv_queue] (root,358544,8732,0.0,1908) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,209412,8192,0.0,1916) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (cvmfs,66484,1108,0.0,1989) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,82696,8208,0.0,2021) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1092,0.0,2027) /usr/bin/cvmfs2 __cachemgr__ . 10 11 41943040000 20971520000 0 3 -1 : (root,0,0,0.0,2177) [irq/100-nvidia] (root,0,0,0.0,2178) [nvidia] (root,0,0,0.2,2179) [nv_queue] (root,89808,2168,0.0,2216) /usr/libexec/postfix/master -w (postfix,89980,4048,0.0,2219) qmgr -l -t unix -u (root,0,0,0.0,2224) [irq/101-nvidia] (root,0,0,0.0,2225) [nvidia] (root,0,0,0.1,2226) [nv_queue] (root,0,0,0.0,2227) [irq/102-nvidia] (root,0,0,0.0,2228) [nvidia] (root,0,0,0.1,2229) [nv_queue] (root,0,0,0.0,2230) [irq/103-nvidia] (root,0,0,0.0,2231) [nvidia] (root,0,0,0.2,2232) [nv_queue] (root,0,0,0.0,2233) [irq/104-nvidia] (root,0,0,0.0,2234) [nvidia] (root,0,0,0.2,2235) [nv_queue] (root,0,0,0.0,2237) [irq/105-nvidia] (root,0,0,0.0,2238) [nvidia] (root,0,0,0.2,2239) [nv_queue] (root,0,0,0.0,2304) [kworker/4:1H] (ntp,29960,2172,0.0,2642) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,1248356,22900,0.0,2649) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,126392,1604,0.0,3168) /usr/sbin/crond -n (root,110208,860,0.0,3174) /sbin/agetty --noclear tty1 linux (ganglia,220220,8416,0.1,3178) /usr/sbin/gmond (root,0,0,0.0,4249) [kworker/u34:1] (root,0,0,0.0,4666) [kworker/1:2] (root,0,0,0.0,4997) [kworker/14:0] (root,0,0,0.0,6105) [kworker/u33:2] (root,0,0,0.0,7205) [kworker/3:2] (cvmfs,66488,1112,0.0,8444) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,657196,22872,0.0,8447) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66488,1112,0.0,8597) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,722648,23848,0.0,8612) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,66488,1116,0.0,8821) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,2264476,60712,0.1,8922) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,66488,1120,0.0,9069) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,76904,2072,0.0,9073) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,66484,1096,0.0,9077) /usr/bin/cvmfs2 __cachemgr__ . 9 10 1048576000 524288000 0 3 -1 : (cvmfs,722648,23568,0.0,9081) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,66488,1108,0.0,9336) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,722648,23516,0.0,9339) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (root,0,0,0.0,9460) [kworker/15:2] (postfix,89912,4032,0.0,9523) pickup -l -t unix -u (cvmfs,66488,1108,0.0,9544) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,722652,23520,0.0,9583) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,66488,1112,0.0,9721) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,722648,25312,0.0,9822) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,66488,1112,0.0,9968) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,657196,22676,0.0,9971) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (condor,66440,8676,0.0,10029) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (slot-1,11692,1572,0.0,10037) /bin/sh glidein_start.sh (cvmfs,66488,1108,0.0,10109) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (root,0,0,0.0,10110) [kworker/15:1H] (cvmfs,657156,22736,0.0,10127) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (condor,74780,8020,0.0,10273) /usr/sbin/condor_master -f (root,26984,5940,0.0,10274) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6748,0.0,10278) condor_shared_port (condor,103036,49604,0.0,10288) condor_startd (slot-1,61812,6532,0.0,10297) glideinExec/sbin/condor_master -f -r 20135 (slot-1,11696,1496,0.0,10298) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_75448ad4-faf3-4f6f-a829-d0f52e1a5a1c.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20240910%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240910T120002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=246830c42aecea5262479aa7b36ef2c8cc145248bdd100bd1d32036be9f1245e (slot-1,24108,3400,0.0,10300) condor_procd -A /scratch/condor/dir_10029/glidein/log.a95fe418-6f6c-11ef-8cb7-0cc47a857a10/procd_address -L /scratch/condor/dir_10029/glidein/log.a95fe418-6f6c-11ef-8cb7-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,62436,7164,0.0,10301) condor_startd -f (condor,34080820,17172,0.2,10569) GPUs_MONITOR (root,0,0,0.0,10571) [UVM GPU1 BH] (root,0,0,0.0,10572) [UVM GPU2 BH] (root,0,0,0.0,10573) [UVM GPU3 BH] (root,0,0,0.0,10574) [UVM GPU4 BH] (root,0,0,0.0,10575) [UVM GPU5 BH] (root,0,0,0.0,10576) [UVM GPU6 BH] (root,0,0,0.0,10577) [UVM GPU7 BH] (root,0,0,0.0,10578) [UVM GPU8 BH] (root,125488,1136,0.0,11289) /usr/sbin/anacron -s (root,0,0,0.0,11720) [kworker/6:1] (root,0,0,0.0,12191) [kworker/12:0] (root,0,0,0.0,13404) [kworker/3:0] (root,0,0,0.0,14595) [kworker/10:2] (root,0,0,0.0,15773) [kworker/12:2] (root,0,0,0.0,16148) [kworker/9:1H] (slot-1,4368,360,0.0,16236) sleep 300 (root,108056,360,0.0,17023) sleep 5 (root,8632,660,0.0,17024) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,17025) /bin/bash /bin/check_mk_agent (root,51732,1688,0.0,17042) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,17043) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,17600) [kworker/u32:1] (root,0,0,0.0,18300) [kworker/13:2] (root,0,0,0.0,18762) [kworker/u34:0] (root,0,0,0.0,19185) [kworker/9:0] (root,0,0,0.0,19491) [kworker/8:0] (root,0,0,0.0,21150) [kworker/9:2] (root,0,0,0.0,23151) [kworker/2:2] (root,0,0,0.0,24375) [kworker/4:1] (root,0,0,0.0,26920) [kworker/2:0] (root,0,0,0.0,28436) [kworker/0:2] (root,113284,1484,0.0,28675) /bin/bash /bin/run-parts /etc/cron.daily (root,352072,21892,0.0,28688) /usr/bin/python -tt /usr/sbin/yum-cron (root,113644,976,0.0,28689) awk -v progname=/etc/cron.daily/0yum-daily.cron progname { .... print progname ":\n" .... progname=""; ... } ... { print; } (root,0,0,0.0,28895) [kworker/7:1] (root,0,0,0.0,29465) [kworker/10:1H] (root,0,0,0.0,30987) [kworker/u33:1] (root,0,0,0.0,31980) [kworker/11:0]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b18312c263d0ab09ac0a6af0b3
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2 AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43668,4112,0.0,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.2,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,17) [kworker/u33:0] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,126) [kmpath_rdacd] (root,0,0,0.0,127) [kaluad] (root,0,0,0.0,129) [kworker/4:1] (root,0,0,0.0,132) [kworker/3:1] (root,0,0,0.0,135) [kworker/9:1] (root,0,0,0.0,138) [kworker/12:1] (root,0,0,0.0,139) [kworker/13:1] (root,0,0,0.0,144) [kpsmoused] (root,0,0,0.0,145) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,310) [kauditd] (root,0,0,0.0,402) [ata_sff] (root,0,0,0.0,403) [scsi_eh_0] (root,0,0,0.0,404) [scsi_tmf_0] (root,0,0,0.0,405) [scsi_eh_1] (root,0,0,0.0,406) [scsi_tmf_1] (root,0,0,0.0,407) [scsi_eh_2] (root,0,0,0.0,408) [scsi_tmf_2] (root,0,0,0.0,409) [scsi_eh_3] (root,0,0,0.0,410) [scsi_tmf_3] (root,0,0,0.0,414) [scsi_eh_4] (root,0,0,0.0,415) [scsi_tmf_4] (root,0,0,0.0,416) [scsi_eh_5] (root,0,0,0.0,417) [scsi_tmf_5] (root,0,0,0.0,418) [scsi_eh_6] (root,0,0,0.0,419) [scsi_tmf_6] (root,0,0,0.0,420) [scsi_eh_7] (root,0,0,0.0,421) [scsi_tmf_7] (root,0,0,0.0,422) [scsi_eh_8] (root,0,0,0.0,423) [scsi_tmf_8] (root,0,0,0.0,424) [scsi_eh_9] (root,0,0,0.0,425) [scsi_tmf_9] (root,0,0,0.0,430) [kworker/u33:9] (root,0,0,0.0,434) [ttm_swap] (root,0,0,0.0,512) [kworker/u34:2] (root,0,0,0.0,541) [kdmflush] (root,0,0,0.0,542) [bioset] (root,0,0,0.0,552) [kdmflush] (root,0,0,0.0,553) [bioset] (root,0,0,0.0,556) [kworker/0:1H] (root,0,0,0.0,570) [jbd2/dm-0-8] (root,0,0,0.0,571) [ext4-rsv-conver] (root,55840,18456,0.0,652) /usr/lib/systemd/systemd-journald (root,0,0,0.0,676) [kworker/7:2] (root,45612,2068,0.0,681) /usr/lib/systemd/systemd-udevd (root,0,0,1.4,728) [kipmi0] (root,0,0,0.0,738) [kworker/4:1H] (root,0,0,0.0,743) [kworker/15:1H] (root,0,0,0.0,754) [kworker/14:2] (root,0,0,0.0,789) [kvm-irqfd-clean] (root,190376,1432,0.0,790) /usr/sbin/lvmetad -f (root,0,0,0.0,860) [kworker/8:1H] (root,0,0,0.0,861) [kworker/1:1H] (root,0,0,0.0,868) [kworker/12:1H] (root,0,0,0.0,869) [kworker/5:1H] (root,0,0,0.0,874) [jbd2/sda1-8] (root,0,0,0.0,875) [ext4-rsv-conver] (root,0,0,0.0,878) [kdmflush] (root,0,0,0.0,879) [bioset] (root,0,0,0.0,880) [kdmflush] (root,0,0,0.0,882) [bioset] (root,0,0,0.0,883) [kdmflush] (root,0,0,0.0,884) [bioset] (root,0,0,0.0,886) [kdmflush] (root,0,0,0.0,888) [bioset] (root,0,0,0.0,901) [kworker/14:1H] (root,0,0,0.0,914) [kworker/13:1H] (root,0,0,0.0,915) [kworker/7:1H] (root,0,0,0.0,924) [jbd2/dm-3-8] (root,0,0,0.0,925) [ext4-rsv-conver] (root,0,0,0.0,928) [kworker/6:1H] (root,0,0,0.0,929) [jbd2/dm-5-8] (root,0,0,0.0,930) [ext4-rsv-conver] (root,0,0,0.0,935) [jbd2/dm-4-8] (root,0,0,0.0,936) [ext4-rsv-conver] (root,0,0,0.0,937) [jbd2/dm-2-8] (root,0,0,0.0,938) [ext4-rsv-conver] (root,0,0,0.0,957) [kworker/1:2] (root,0,0,0.0,963) [nv_queue] (root,0,0,0.0,964) [nv_queue] (node_ex+,727344,15432,0.3,976) /usr/local/bin/node_exporter --collector.disable-defaults --collector.cpu --collector.cpufreq --collector.diskstats --collector.dmi --collector.edac --collector.filefd --collector.filesystem --collector.hwmon --collector.loadavg --collector.meminfo --collector.netclass --collector.netdev --collector.netstat --collector.nfs --collector.os --collector.processes --collector.schedstat --collector.sockstat --collector.softnet --collector.stat --collector.thermal_zone --collector.time --collector.timex --collector.udp_queues --collector.uname --collector.vmstat (dbus,58216,2400,0.0,981) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (root,26384,1768,0.0,998) /usr/lib/systemd/systemd-logind (root,21788,1420,0.0,1005) /usr/sbin/irqbalance --foreground (polkitd,612240,8096,0.0,1017) /usr/lib/polkit-1/polkitd --no-debug (root,52900,2944,0.0,1067) /usr/sbin/smartd -n -q never (root,0,0,0.0,1187) [UVM global queu] (root,0,0,0.0,1188) [UVM deferred re] (root,0,0,0.0,1190) [UVM Tools Event] (root,0,0,0.0,1227) [nvidia-modeset/] (root,0,0,0.0,1228) [nvidia-modeset/] (root,0,0,0.0,1238) [bond0] (root,10070568,185580,0.1,1390) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,239124,11840,0.0,1553) /usr/sbin/rsyslogd -n (root,209048,10580,0.0,1562) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,801948,3160,0.0,1564) /usr/sbin/collectd (root,404692,82876,0.0,1586) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,113004,4336,0.0,1591) /usr/sbin/sshd -D (root,27168,1056,0.0,1592) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,358544,8308,0.0,1593) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,574288,17592,0.0,1595) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,0,0,0.0,1601) [irq/98-nvidia] (root,0,0,0.0,1602) [nvidia] (root,0,0,0.2,1603) [nv_queue] (root,209196,8104,0.0,1604) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (cvmfs,66484,1108,0.0,2065) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,115612,1812,0.0,2078) /bin/bash /sbin/condor_inspector (cvmfs,79408,4728,0.0,2084) /usr/bin/cvmfs2 __cachemgr__ . 9 10 41943040000 20971520000 0 3 -1 : (cvmfs,66480,1092,0.0,2089) /usr/bin/cvmfs2 __cachemgr__ . 9 10 41943040000 20971520000 0 3 -1 : (root,591424,14540,0.1,2103) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,2116) [irq/99-nvidia] (root,0,0,0.0,2117) [nvidia] (root,0,0,0.1,2118) [nv_queue] (root,89808,2184,0.0,2188) /usr/libexec/postfix/master -w (postfix,90084,4084,0.0,2190) qmgr -l -t unix -u (root,0,0,0.0,2196) [irq/100-nvidia] (root,0,0,0.0,2197) [nvidia] (root,0,0,0.1,2198) [nv_queue] (root,0,0,0.0,2200) [irq/101-nvidia] (root,0,0,0.0,2201) [nvidia] (root,0,0,0.0,2202) [nv_queue] (root,0,0,0.0,2203) [irq/102-nvidia] (root,0,0,0.0,2204) [nvidia] (root,0,0,0.1,2205) [nv_queue] (root,0,0,0.0,2206) [irq/103-nvidia] (root,0,0,0.0,2207) [nvidia] (root,0,0,0.1,2208) [nv_queue] (root,0,0,0.0,2209) [irq/104-nvidia] (root,0,0,0.0,2210) [nvidia] (root,0,0,0.1,2211) [nv_queue] (root,0,0,0.0,2212) [kworker/9:1H] (root,0,0,0.0,2248) [kworker/u34:1] (root,0,0,0.0,2251) [irq/105-nvidia] (root,0,0,0.0,2252) [nvidia] (root,0,0,0.1,2253) [nv_queue] (root,0,0,0.0,2311) [kworker/2:1H] (ntp,29960,2148,0.0,2453) /usr/sbin/ntpd -u ntp:ntp -g (cvmfs,878868,22756,0.0,2510) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,66484,1108,0.0,2841) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,66484,1108,0.0,2869) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,66484,1104,0.0,2876) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,66484,1108,0.0,2931) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66484,1108,0.0,2936) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,66484,1104,0.0,2952) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,66484,1108,0.0,2979) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,722656,23516,0.0,2982) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,66484,1104,0.0,3006) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,731020,22896,0.0,3014) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,657152,22888,0.0,3034) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,66484,1116,0.0,3051) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,77616,2816,0.0,3056) /usr/bin/cvmfs2 __cachemgr__ . 10 11 1048576000 524288000 0 3 -1 : (cvmfs,66480,1096,0.0,3060) /usr/bin/cvmfs2 __cachemgr__ . 10 11 1048576000 524288000 0 3 -1 : (cvmfs,3314540,60996,0.1,3063) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,722688,23444,0.0,3078) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,796376,25204,0.0,3093) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,722648,23708,0.0,3108) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,3121) [kworker/3:1H] (cvmfs,796512,23624,0.0,3125) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,722664,23448,0.0,3135) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (condor,132884,11656,0.0,3152) /usr/sbin/condor_master -f (root,126392,1612,0.0,3156) /usr/sbin/crond -n (root,110208,864,0.0,3162) /sbin/agetty --noclear tty1 linux (ganglia,220196,8404,0.0,3166) /usr/sbin/gmond (root,27116,6076,0.0,3171) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,60712,6764,0.0,3172) condor_shared_port (condor,155552,43968,0.0,3175) condor_startd (condor,34080820,17168,0.2,3268) GPUs_MONITOR (root,0,0,0.0,3270) [UVM GPU1 BH] (root,0,0,0.0,3271) [UVM GPU2 BH] (root,0,0,0.0,3272) [UVM GPU3 BH] (root,0,0,0.0,3273) [UVM GPU4 BH] (root,0,0,0.0,3274) [UVM GPU5 BH] (root,0,0,0.0,3275) [UVM GPU6 BH] (root,0,0,0.0,3279) [UVM GPU7 BH] (root,0,0,0.0,3290) [UVM GPU8 BH] (root,0,0,0.0,3624) [kworker/11:0] (root,0,0,0.0,3713) [kworker/6:0] (root,0,0,0.0,3944) [kworker/0:0] (root,0,0,0.0,4723) [kworker/2:1] (root,0,0,0.0,5134) [kworker/8:1] (root,0,0,0.0,5573) [kworker/15:1] (root,0,0,0.0,7828) [kworker/5:1] (root,0,0,0.0,8146) [kworker/u32:2] (root,0,0,0.0,9276) [kworker/u32:0] (root,0,0,0.0,10180) [kworker/1:0] (root,0,0,0.0,11841) [kworker/0:1] (root,0,0,0.0,13010) [kworker/10:0] (root,0,0,0.0,13502) [kworker/10:1] (root,0,0,0.0,15604) [kworker/12:2] (root,0,0,0.0,16299) [kworker/10:1H] (root,0,0,0.0,16575) [kworker/13:0] (root,0,0,0.0,17105) [kworker/11:1H] (root,0,0,0.0,17806) [kworker/5:0] (root,0,0,0.0,18203) [kworker/8:0] (root,0,0,0.0,18729) [kworker/8:2] (root,0,0,0.0,19716) [kworker/u32:1] (root,0,0,0.0,21999) [kworker/11:2] (root,125488,1128,0.0,22883) /usr/sbin/anacron -s (root,0,0,0.0,22929) [kworker/7:1] (condor,66440,8704,0.0,23491) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (slot-1,11692,1572,0.0,23493) /bin/sh glidein_start.sh (root,0,0,0.0,23669) [kworker/2:2] (slot-1,61812,6544,0.0,23723) glideinExec/sbin/condor_master -f -r 20135 (slot-1,11696,1540,0.0,23724) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_6df31f59-e113-4b51-ab7f-a19b12fb49a7.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20240116%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240116T193838Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=2b08344e06b6dee455039140fee5a7618cb8b22cc1e2d222312dbbc29486ab11 (slot-1,24616,4384,0.0,23726) condor_procd -A /scratch/condor/dir_23491/glidein/log.dbf831a4-b4a6-11ee-921f-0cc47a857a10/procd_address -L /scratch/condor/dir_23491/glidein/log.dbf831a4-b4a6-11ee-921f-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,62944,7996,0.0,23727) condor_startd -f (root,0,0,0.0,23746) [kworker/15:0] (slot-1,46172,6608,0.0,24093) condor_starter -f -a slot1_3 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,24097) /bin/sh /scratch/condor/dir_23491/glidein/execute.dbf831a4-b4a6-11ee-921f-0cc47a857a10/dir_24093/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443120,23220,0.0,24137) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1460,0.0,24142) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.9.1/ /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.1/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=26133980 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=10 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,11704,1548,0.0,24239) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.9.1//env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.1/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=26133980 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=10 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,17690196,1218032,78.8,24246) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.1/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=26133980 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=10 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (root,0,0,0.0,24945) [kworker/14:1] (root,0,0,0.0,25263) [kworker/12:0] (slot-1,4368,360,0.0,25273) sleep 300 (root,108056,360,0.0,25813) sleep 15 (root,8632,664,0.0,25814) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,25815) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,25832) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,25833) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,26878) [kworker/11:1] (root,0,0,0.0,26987) [kworker/4:2] (root,0,0,0.0,28296) [kworker/6:2] (slot-1,46176,6608,0.0,28495) condor_starter -f -a slot1_1 pyglidein1-submit.icecube.wisc.edu (slot-1,46176,6608,0.0,28499) condor_starter -f -a slot1_4 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1568,0.0,28500) /bin/sh /scratch/condor/dir_23491/glidein/execute.dbf831a4-b4a6-11ee-921f-0cc47a857a10/dir_28495/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443116,27744,0.0,28540) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11704,1564,0.0,28647) /bin/sh /scratch/condor/dir_23491/glidein/execute.dbf831a4-b4a6-11ee-921f-0cc47a857a10/dir_28499/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443116,27748,0.0,28705) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1460,0.0,28725) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.9.1/ /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.1/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=26134270 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=10 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (root,113284,1484,0.0,28816) /bin/bash /bin/run-parts /etc/cron.daily (root,352072,21892,0.0,28820) /usr/bin/python -tt /usr/sbin/yum-cron (root,113644,980,0.0,28821) awk -v progname=/etc/cron.daily/0yum-daily.cron progname { .... print progname ":\n" .... progname=""; ... } ... { print; } (slot-1,11704,1552,0.0,28825) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.9.1//env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.1/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=26134270 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=10 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,17628160,1319424,70.5,28832) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.1/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=26134270 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=10 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,11700,1464,0.0,28890) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.9.1/ /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.1/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=26134310 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=10 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (root,0,0,0.0,29025) [kworker/9:2] (slot-1,11704,1552,0.0,29035) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.9.1//env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.1/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=26134310 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=10 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,17595316,1335100,77.6,29042) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.1/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=26134310 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=10 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (postfix,89912,4028,0.0,31222) pickup -l -t unix -u (slot-1,46172,6608,0.0,31987) condor_starter -f -a slot1_2 pyglidein1-submit.icecube.wisc.edu (slot-1,46172,6608,0.0,31989) condor_starter -f -a slot1_5 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1564,0.0,31992) /bin/sh /scratch/condor/dir_23491/glidein/execute.dbf831a4-b4a6-11ee-921f-0cc47a857a10/dir_31987/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443120,23216,0.0,32033) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11704,1564,0.0,32036) /bin/sh /scratch/condor/dir_23491/glidein/execute.dbf831a4-b4a6-11ee-921f-0cc47a857a10/dir_31989/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/v2.7.1 -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,443116,23140,0.0,32076) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1464,0.0,32085) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.9.1/ /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.1/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=26133590 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=10 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,11704,1552,0.0,32182) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.9.1//env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.1/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=26133590 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=10 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,17735448,1215020,72.1,32189) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.1/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=26133590 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=10 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,11700,1460,0.0,32190) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/icetray-env icetray/v1.9.1/ /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.1/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=26133630 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=10 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,11704,1552,0.0,32287) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/RHEL_7_x86_64/metaprojects/icetray/v1.9.1//env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.1/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=26133630 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=10 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (slot-1,15838328,1124132,67.2,32294) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.3.0/metaprojects/icetray/v1.9.1/simprod-scripts/resources/scripts/SnowSuite/3-Snowstorm.py --seed=26133630 --infile=gen_poly_MuonProp_NuGenCCNC.i3.zst --outfile=hits_NuGenCCNC.i3.zst --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --events-per-model=10 --domoversizefactor=3 --cpu=False --UseI3PropagatorService=False --config_file=snowstorm.json --summaryfile=clsim_summary_NuGenCCNC.json --log-level=WARN (root,0,0,0.0,32475) [kworker/3:0]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b1a45a4f05aae993daa7766345
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2.chtc.wisc.edu AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43800,2928,0.0,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.2,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,259) [kauditd] (slot-1,13568,1964,0.0,339) zstd -d -qq (root,0,0,0.0,411) [ata_sff] (root,0,0,0.0,412) [scsi_eh_0] (root,0,0,0.0,413) [scsi_tmf_0] (root,0,0,0.0,414) [scsi_eh_1] (root,0,0,0.0,415) [scsi_tmf_1] (root,0,0,0.0,416) [scsi_eh_2] (root,0,0,0.0,417) [scsi_tmf_2] (root,0,0,0.0,418) [scsi_eh_3] (root,0,0,0.0,419) [scsi_tmf_3] (root,0,0,0.0,423) [scsi_eh_4] (root,0,0,0.0,424) [scsi_tmf_4] (root,0,0,0.0,425) [scsi_eh_5] (root,0,0,0.0,426) [scsi_tmf_5] (root,0,0,0.0,427) [scsi_eh_6] (root,0,0,0.0,428) [scsi_tmf_6] (root,0,0,0.0,429) [scsi_eh_7] (root,0,0,0.0,430) [scsi_tmf_7] (root,0,0,0.0,431) [scsi_eh_8] (root,0,0,0.0,432) [scsi_tmf_8] (root,0,0,0.0,433) [scsi_eh_9] (root,0,0,0.0,434) [scsi_tmf_9] (root,0,0,0.0,442) [ttm_swap] (root,0,0,0.0,543) [kdmflush] (root,0,0,0.0,544) [bioset] (root,0,0,0.0,554) [kdmflush] (root,0,0,0.0,555) [bioset] (root,0,0,0.0,558) [kworker/0:1H] (root,0,0,0.0,572) [jbd2/dm-0-8] (root,0,0,0.0,573) [ext4-rsv-conver] (root,126404,68476,0.0,653) /usr/lib/systemd/systemd-journald (root,45604,656,0.0,679) /usr/lib/systemd/systemd-udevd (root,0,0,1.3,730) [kipmi0] (root,0,0,0.0,735) [kworker/14:1H] (root,0,0,0.0,737) [kworker/6:1H] (root,0,0,0.0,738) [kworker/5:1H] (root,0,0,0.0,805) [kvm-irqfd-clean] (root,198572,408,0.0,815) /usr/sbin/lvmetad -f (root,0,0,0.0,868) [kworker/1:1H] (root,0,0,0.0,872) [kworker/2:1H] (root,0,0,0.0,880) [kdmflush] (root,0,0,0.0,881) [bioset] (root,0,0,0.0,883) [kdmflush] (root,0,0,0.0,884) [bioset] (root,0,0,0.0,885) [kdmflush] (root,0,0,0.0,887) [bioset] (root,0,0,0.0,889) [kdmflush] (root,0,0,0.0,890) [bioset] (root,0,0,0.0,897) [kworker/13:1H] (root,0,0,0.0,898) [kworker/7:1H] (root,0,0,0.0,899) [kworker/15:1H] (root,0,0,0.0,907) [jbd2/sda1-8] (root,0,0,0.0,908) [ext4-rsv-conver] (root,0,0,0.0,929) [kworker/9:1H] (root,0,0,0.0,930) [kworker/12:1H] (root,0,0,0.0,931) [kworker/4:1H] (root,0,0,0.0,938) [jbd2/dm-5-8] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,941) [ext4-rsv-conver] (root,0,0,0.0,947) [jbd2/dm-4-8] (root,0,0,0.0,948) [ext4-rsv-conver] (root,0,0,0.0,951) [jbd2/dm-2-8] (root,0,0,0.0,952) [ext4-rsv-conver] (root,0,0,0.0,976) [nv_queue] (root,0,0,0.0,977) [nv_queue] (polkitd,612240,1104,0.0,1012) /usr/lib/polkit-1/polkitd --no-debug (root,52900,592,0.0,1015) /usr/sbin/smartd -n -q never (root,21788,508,0.0,1027) /usr/sbin/irqbalance --foreground (dbus,58220,892,0.0,1034) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (root,26752,1116,0.0,1057) /usr/lib/systemd/systemd-logind (root,0,0,0.0,1099) [UVM global queu] (root,0,0,0.0,1100) [UVM deferred re] (root,0,0,0.0,1101) [UVM Tools Event] (root,0,0,0.0,1132) [nvidia-modeset/] (root,0,0,0.0,1133) [nvidia-modeset/] (root,0,0,0.0,1199) [kworker/u34:0] (root,0,0,0.0,1262) [bond0] (root,10058844,156364,0.0,1292) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,302284,39096,0.0,1570) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,574288,2312,0.0,1574) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,113000,8,0.0,1576) /usr/sbin/sshd -D (root,0,0,0.0,1581) [irq/98-nvidia] (root,0,0,0.0,1582) [nvidia] (root,0,0,0.2,1583) [nv_queue] (root,799816,1912,0.0,1823) /usr/sbin/collectd (root,740652,40828,0.0,1845) /usr/sbin/rsyslogd -n (root,591424,5112,0.1,1871) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,115700,964,0.0,1874) /bin/bash /sbin/condor_inspector (root,209064,1612,0.0,1887) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,1248,0.0,1906) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1913) [irq/99-nvidia] (root,0,0,0.0,1914) [nvidia] (root,0,0,0.2,1915) [nv_queue] (root,209332,2212,0.0,1920) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,89804,228,0.0,2118) /usr/libexec/postfix/master -w (postfix,90080,544,0.0,2121) qmgr -l -t unix -u (cvmfs,1310644,23108,0.0,2210) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2309) [kworker/10:2] (root,0,0,0.0,2330) [irq/100-nvidia] (root,0,0,0.0,2331) [nvidia] (root,0,0,0.2,2332) [nv_queue] (root,0,0,0.0,2503) [kworker/12:2] (cvmfs,852208,7180,0.0,2588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,853372,9092,0.0,2607) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,852564,13236,0.0,2626) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,852844,13332,0.0,2645) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,852268,11492,0.0,2664) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,852208,5708,0.0,2757) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,5072972,136652,0.1,2792) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (root,0,0,0.0,2817) [irq/101-nvidia] (root,0,0,0.0,2818) [nvidia] (root,0,0,0.1,2819) [nv_queue] (cvmfs,852132,21596,0.0,2830) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,852256,13364,0.0,2850) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,2870) [irq/102-nvidia] (root,0,0,0.0,2871) [nvidia] (root,0,0,0.1,2872) [nv_queue] (root,126388,452,0.0,2874) /usr/sbin/crond -n (condor,75276,3096,0.0,2875) /usr/sbin/condor_master -f (ntp,29960,788,0.0,2877) /usr/sbin/ntpd -u ntp:ntp -g (root,27828,4304,0.0,2878) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (root,0,0,0.0,2879) [irq/103-nvidia] (root,0,0,0.0,2880) [nvidia] (root,0,0,0.1,2881) [nv_queue] (condor,64000,2836,0.0,2882) condor_shared_port -f (root,0,0,0.0,2883) [kworker/10:1H] (root,0,0,0.0,2884) [irq/104-nvidia] (root,0,0,0.0,2885) [nvidia] (root,0,0,0.2,2886) [nv_queue] (condor,4329484,4201544,0.1,2887) condor_startd -f (root,0,0,0.0,2908) [irq/105-nvidia] (root,0,0,0.0,2909) [nvidia] (root,0,0,0.2,2910) [nv_queue] (root,110208,8,0.0,3014) /sbin/agetty --noclear tty1 linux (condor,34082884,508,0.1,3821) GPUs_MONITOR (root,0,0,28.2,3824) [UVM GPU1 BH] (root,0,0,0.0,3825) [UVM GPU2 BH] (root,0,0,0.0,3827) [UVM GPU3 BH] (root,0,0,0.0,3828) [UVM GPU4 BH] (root,0,0,0.0,3830) [UVM GPU5 BH] (root,0,0,0.0,3831) [UVM GPU6 BH] (root,0,0,69.4,3832) [UVM GPU7 BH] (root,0,0,0.0,3833) [UVM GPU8 BH] (root,0,0,0.0,4069) [kworker/8:1H] (root,0,0,0.0,4366) [kworker/3:1H] (root,27168,4,0.0,5419) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,6100) [kworker/0:0] (slot-1,46168,6580,0.0,6137) condor_starter -f -a slot1_9 pyglidein1-submit.icecube.wisc.edu (slot-1,11832,1540,0.0,6141) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/lfischer/oscnext_hnl_meta/build//env-shell.sh python /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_6137/condor_exec.exe --seed 12465 --outfile gsiftp://gridftp-users.icecube.wisc.edu/data/ana/BSM/HNL/MC/190613/Gen/02001-03000/Gen_02465.i3.zst --Emin 2.0 --Emax 10000 --index 2.0 --nEvents 500000 -m 0.1 (slot-1,965656,288272,99.2,6261) python /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_6137/condor_exec.exe --seed 12465 --outfile gsiftp://gridftp-users.icecube.wisc.edu/data/ana/BSM/HNL/MC/190613/Gen/02001-03000/Gen_02465.i3.zst --Emin 2.0 --Emax 10000 --index 2.0 --nEvents 500000 -m 0.1 (ganglia,220196,2864,0.0,6311) /usr/sbin/gmond (slot-1,10903860,354164,0.1,6554) python /scratch/condor/dir_21435/glidein/execute.bde4dab8-997d-11ed-81a0-0cc47a857a10/dir_5868/condor_exec.exe -i /data/ana/BSM/HNL/MC/190611/Gen/03001-04000/Gen_03934.i3.zst -o /data/ana/BSM/HNL/MC/190611/Phot/03001-04000/Phot_03934.i3.zst -r 1 -l 3934 -t -m spice_3.2.1 --holeice angsens/as.flasher_p1_0.30_p2_-1 --osg True (root,0,0,0.0,7036) [kworker/3:2] (root,0,0,0.0,7208) [kworker/5:2] (root,0,0,0.0,7392) [kworker/u34:2] (root,0,0,0.0,7896) [kworker/11:1H] (root,0,0,0.0,8778) [kworker/14:0] (root,0,0,0.0,9372) [kworker/13:2] (root,0,0,0.0,9677) [kworker/6:0] (root,0,0,0.0,9712) [kworker/4:2] (root,0,0,0.0,10864) [kworker/7:0] (slot-1,46156,6596,0.0,13289) condor_starter -f -a slot1_5 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1512,0.0,13397) bash /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_13289/condor_exec.exe (slot-1,96860,8624,0.0,13476) condor_starter -f -a slot1_1 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1516,0.0,13484) bash /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_13476/condor_exec.exe (slot-1,11832,1548,0.0,13496) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_D_140000_000566_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141173/NuMu_D_141173_000566_step2.i3.zst --filenr 566 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_13289/2023-04-08_15-52-52 --gridftp --runnumber 141173 --icemodel spice_bfr-v2/err_s+.00_adust+.00_aice+.10 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling (slot-1,13601260,1156684,113,13500) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_D_140000_000566_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141173/NuMu_D_141173_000566_step2.i3.zst --filenr 566 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_13289/2023-04-08_15-52-52 --gridftp --runnumber 141173 --icemodel spice_bfr-v2/err_s+.00_adust+.00_aice+.10 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling (slot-1,11832,1544,0.0,13581) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuE/NuE_A_120000_000299_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/121173/NuE_A_121173_000299_step2.i3.zst --filenr 299 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_13476/2023-04-08_15-52-38 --gridftp --runnumber 121173 --icemodel spice_bfr-v2/err_s+.00_adust+.00_aice+.10 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling (slot-1,11202692,960224,1.1,13585) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuE/NuE_A_120000_000299_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/121173/NuE_A_121173_000299_step2.i3.zst --filenr 299 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_13476/2023-04-08_15-52-38 --gridftp --runnumber 121173 --icemodel spice_bfr-v2/err_s+.00_adust+.00_aice+.10 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling (slot-1,13568,2168,0.0,13902) zstd -d -qq (slot-1,96860,8620,0.0,13920) condor_starter -f -a slot1_7 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1516,0.0,14143) bash /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_13920/condor_exec.exe (slot-1,11832,1548,0.0,14240) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000292_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141174/NuMu_C_141174_000292_step2.i3.zst --filenr 292 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_13920/2023-04-08_15-53-16 --gridftp --runnumber 141174 --icemodel spice_bfr-v2/err_s+.00_adust+.00_aice-.10 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling (slot-1,11132740,880120,4.7,14245) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000292_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141174/NuMu_C_141174_000292_step2.i3.zst --filenr 292 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_13920/2023-04-08_15-53-16 --gridftp --runnumber 141174 --icemodel spice_bfr-v2/err_s+.00_adust+.00_aice-.10 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling (postfix,89908,4028,0.0,14272) pickup -l -t unix -u (root,0,0,0.0,14563) [kworker/10:1] (slot-1,46156,6596,0.0,14663) condor_starter -f -a slot1_2 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1516,0.0,14807) bash /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_14663/condor_exec.exe (root,0,0,0.0,14879) [kworker/4:0] (slot-1,11832,1544,0.0,14933) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000589_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141174/NuMu_C_141174_000589_step2.i3.zst --filenr 589 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_14663/2023-04-08_15-53-16 --gridftp --runnumber 141174 --icemodel spice_bfr-v2/err_s+.00_adust+.00_aice-.10 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling (slot-1,13306524,828780,117,14937) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000589_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141174/NuMu_C_141174_000589_step2.i3.zst --filenr 589 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_14663/2023-04-08_15-53-16 --gridftp --runnumber 141174 --icemodel spice_bfr-v2/err_s+.00_adust+.00_aice-.10 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling (slot-1,13568,2116,0.0,14963) zstd -d -qq (slot-1,17150796,208152,0.4,14989) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000136_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141156/NuMu_B_141156_000136_step2.i3.zst --filenr 136 --tmpdir /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_14769/2023-02-02_02-26-39 --gridftp --runnumber 141156 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,15530208,4762208,0.7,15169) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000633_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_B_141159_000633_step2.i3.zst --filenr 633 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_14465/2023-03-04_17-59-09 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (cvmfs,73892,2772,0.0,15170) /usr/bin/cvmfs2 __cachemgr__ . 7 8 41943040000 20971520000 0 3 -1 : (cvmfs,60968,12,0.0,15174) /usr/bin/cvmfs2 __cachemgr__ . 7 8 41943040000 20971520000 0 3 -1 : (cvmfs,967072,11260,0.0,15179) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,72156,1784,0.0,15354) /usr/bin/cvmfs2 __cachemgr__ . 7 8 1048576000 524288000 0 3 -1 : (cvmfs,60968,32,0.0,15358) /usr/bin/cvmfs2 __cachemgr__ . 7 8 1048576000 524288000 0 3 -1 : (cvmfs,811276,10932,0.0,15366) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (slot-1,13878336,286540,0.5,15370) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000776_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141157/NuMu_B_141157_000776_step2.i3.zst --filenr 776 --tmpdir /scratch/condor/dir_12927/glidein/execute.f02ec7ea-ae0e-11ed-990f-0cc47a857a10/dir_14937/2023-02-16_08-41-23 --gridftp --runnumber 141157 --icemodel spice_3.2.1 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling --geant4-muons --particle-history (cvmfs,811224,2744,0.0,15380) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,811224,628,0.0,15393) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,811276,7832,0.0,15406) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,811276,6876,0.0,15410) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,811276,9304,0.0,15435) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,5011736,24224,0.0,15448) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,811276,6312,0.0,15461) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,811140,15636,0.0,15474) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,16097) [kworker/5:0] (condor,67144,4328,0.0,16806) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (slot-1,11692,184,0.0,16826) /bin/sh glidein_start.sh (slot-1,61812,3148,0.0,16978) glideinExec/sbin/condor_master -f -r 20135 (slot-1,12084,1044,0.0,16979) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_48e309b5-e8b3-4abb-8450-b7d984513eb1.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20230329%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230329T100002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=0476f77bd4e0c7462d757b637f0a5f03fc2655b1ff93a5de63cf102da4edfa90 (slot-1,24804,3748,0.0,16981) condor_procd -A /scratch/condor/dir_16806/glidein/log.be51bf96-ce18-11ed-8d74-0cc47a857a10/procd_address -L /scratch/condor/dir_16806/glidein/log.be51bf96-ce18-11ed-8d74-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,63500,5596,0.0,16982) condor_startd -f (root,0,0,0.0,17004) [kworker/u32:0] (slot-1,46236,4032,0.0,17047) python ../../clsim_gpu_test.py -n 1 (slot-1,11700,232,0.0,17113) /bin/sh /cvmfs/icecube.opensciencegrid.org/py2-v2/RHEL_7_x86_64/metaprojects/simulation/V05-00-07/env-shell.sh /cvmfs/icecube.opensciencegrid.org/py2-v2/RHEL_7_x86_64/metaprojects/simulation/V05-00-07/clsim/resources/scripts/benchmark.py -n 1 (slot-1,10332404,187684,0.0,17121) python /cvmfs/icecube.opensciencegrid.org/py2-v2/RHEL_7_x86_64/metaprojects/simulation/V05-00-07/clsim/resources/scripts/benchmark.py -n 1 (slot-1,11523760,234960,0.7,17379) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /data/sim/DeepCore/2018/pass2/gcd/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise_newStringPositions.i3 --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuE/NuE_D_120000_000009_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/121170/NuE_D_121170_000009_step2.i3.zst --filenr 9 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_17274/2023-03-29_04-41-40 --gridftp --runnumber 121170 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t (slot-1,10858068,371384,0.1,18091) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /data/sim/DeepCore/2018/pass2/gcd/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise_newStringPositions.i3 --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuE/NuE_B_120000_000014_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/121170/NuE_B_121170_000014_step2.i3.zst --filenr 14 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_17810/2023-03-29_04-41-40 --gridftp --runnumber 121170 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t (root,0,0,0.0,18130) [kworker/8:1] (root,0,0,0.0,18282) [kworker/2:1] (root,0,0,0.0,19311) [kworker/15:2] (root,0,0,0.0,19395) [kworker/3:0] (root,0,0,0.0,19654) [kworker/11:1] (root,0,0,0.0,19903) [kworker/14:1] (root,0,0,0.0,19921) [kworker/13:1] (slot-1,13628156,622676,0.1,20601) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_A_140000_000115_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_A_141159_000115_step2.i3.zst --filenr 115 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_20078/2023-03-01_14-44-52 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (root,0,0,0.0,20695) [kworker/11:0] (root,0,0,0.0,20710) [kworker/2:2] (root,0,0,0.0,21299) [kworker/9:0] (slot-1,11183212,326236,0.8,21544) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000037_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_C_141159_000037_step2.i3.zst --filenr 37 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_21019/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (root,0,0,0.0,22339) [kworker/12:0] (root,0,0,0.0,22426) [kworker/7:1] (root,0,0,0.0,22738) [kworker/u33:1] (slot-1,0,0,0.5,22845) [python] <defunct> (slot-1,46156,6596,0.0,22866) condor_starter -f -a slot1_3 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1516,0.0,22870) bash /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_22866/condor_exec.exe (slot-1,11832,1548,0.0,22967) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000572_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141173/NuMu_B_141173_000572_step2.i3.zst --filenr 572 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_22866/2023-04-08_15-52-52 --gridftp --runnumber 141173 --icemodel spice_bfr-v2/err_s+.00_adust+.00_aice+.10 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling (slot-1,14104600,1613276,108,22971) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000572_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141173/NuMu_B_141173_000572_step2.i3.zst --filenr 572 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_22866/2023-04-08_15-52-52 --gridftp --runnumber 141173 --icemodel spice_bfr-v2/err_s+.00_adust+.00_aice+.10 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling (slot-1,13568,2060,0.1,23030) zstd -d -qq (slot-1,0,0,0.4,23553) [python] <defunct> (root,0,0,0.0,23689) [kworker/6:2] (root,0,0,0.0,24003) [kworker/1:2] (root,0,0,0.0,24655) [kworker/7:2] (root,0,0,0.0,24890) [kworker/u33:0] (slot-1,0,0,0.0,25319) [zstd] <defunct> (root,0,0,0.0,26056) [kworker/9:2] (root,0,0,0.0,26550) [kworker/5:1] (slot-1,46176,6608,0.0,26551) condor_starter -f -a slot1_4 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1512,0.0,26555) bash /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_26551/condor_exec.exe (slot-1,11832,1548,0.0,26652) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_D_140000_000570_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141173/NuMu_D_141173_000570_step2.i3.zst --filenr 570 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_26551/2023-04-08_15-52-52 --gridftp --runnumber 141173 --icemodel spice_bfr-v2/err_s+.00_adust+.00_aice+.10 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling (slot-1,13513708,1096740,118,26656) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_D_140000_000570_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141173/NuMu_D_141173_000570_step2.i3.zst --filenr 570 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_26551/2023-04-08_15-52-52 --gridftp --runnumber 141173 --icemodel spice_bfr-v2/err_s+.00_adust+.00_aice+.10 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling (slot-1,13568,2008,0.0,26680) zstd -d -qq (slot-1,10858484,124128,0.1,27083) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuE/NuE_B_120000_000000_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/121173/NuE_B_121173_000000_step2.i3.zst --filenr 0 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_26865/2023-03-31_15-42-36 --gridftp --runnumber 121173 --icemodel spice_bfr-v2/err_s+.00_adust+.00_aice+.10 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling (slot-1,11137260,665448,0.2,27632) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuE/NuE_A_120000_000008_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/121174/NuE_A_121174_000008_step2.i3.zst --filenr 8 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_27414/2023-03-31_15-42-48 --gridftp --runnumber 121174 --icemodel spice_bfr-v2/err_s+.00_adust+.00_aice-.10 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling (root,0,0,0.0,27773) [kworker/6:1] (root,0,0,0.0,27902) [kworker/0:1] (root,0,0,0.0,27957) [kworker/u32:1] (root,0,0,0.0,28315) [kworker/15:0] (root,0,0,0.0,28361) [kworker/u32:2] (root,0,0,0.0,29368) [kworker/1:0] (root,0,0,0.0,29662) [kworker/4:1] (slot-1,4368,360,0.0,29667) sleep 300 (slot-1,0,0,0.0,29724) [python] <defunct> (slot-1,0,0,0.0,29885) [zstd] <defunct> (slot-1,46156,6596,0.0,30978) condor_starter -f -a slot1_8 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1516,0.0,30997) bash /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_30978/condor_exec.exe (root,108056,360,0.0,31043) sleep 15 (root,8632,664,0.0,31044) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,31045) /bin/bash /bin/check_mk_agent (root,51732,1696,0.0,31061) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,31062) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (slot-1,11832,1548,0.0,31098) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000589_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141173/NuMu_C_141173_000589_step2.i3.zst --filenr 589 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_30978/2023-04-08_15-52-52 --gridftp --runnumber 141173 --icemodel spice_bfr-v2/err_s+.00_adust+.00_aice+.10 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling (slot-1,13180332,709648,116,31102) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000589_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141173/NuMu_C_141173_000589_step2.i3.zst --filenr 589 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_30978/2023-04-08_15-52-52 --gridftp --runnumber 141173 --icemodel spice_bfr-v2/err_s+.00_adust+.00_aice+.10 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling (slot-1,13568,2112,0.0,31145) zstd -d -qq (root,0,0,0.0,32133) [kworker/8:0] (slot-1,46156,6596,0.0,32504) condor_starter -f -a slot1_6 pyglidein1-submit.icecube.wisc.edu (slot-1,11704,1516,0.0,32508) bash /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_32504/condor_exec.exe (slot-1,11832,1548,0.0,32605) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_D_140000_000553_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141174/NuMu_D_141174_000553_step2.i3.zst --filenr 553 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_32504/2023-04-08_15-53-16 --gridftp --runnumber 141174 --icemodel spice_bfr-v2/err_s+.00_adust+.00_aice-.10 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling (slot-1,13748104,1328872,111,32609) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_D_140000_000553_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141174/NuMu_D_141174_000553_step2.i3.zst --filenr 553 --tmpdir /scratch/condor/dir_16806/glidein/execute.be51bf96-ce18-11ed-8d74-0cc47a857a10/dir_32504/2023-04-08_15-53-16 --gridftp --runnumber 141174 --icemodel spice_bfr-v2/err_s+.00_adust+.00_aice-.10 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b1a45a4f05aae993dafef1bcc6
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2.chtc.wisc.edu AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43800,3120,0.0,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.2,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,259) [kauditd] (root,0,0,0.0,411) [ata_sff] (root,0,0,0.0,412) [scsi_eh_0] (root,0,0,0.0,413) [scsi_tmf_0] (root,0,0,0.0,414) [scsi_eh_1] (root,0,0,0.0,415) [scsi_tmf_1] (root,0,0,0.0,416) [scsi_eh_2] (root,0,0,0.0,417) [scsi_tmf_2] (root,0,0,0.0,418) [scsi_eh_3] (root,0,0,0.0,419) [scsi_tmf_3] (root,0,0,0.0,423) [scsi_eh_4] (root,0,0,0.0,424) [scsi_tmf_4] (root,0,0,0.0,425) [scsi_eh_5] (root,0,0,0.0,426) [scsi_tmf_5] (root,0,0,0.0,427) [scsi_eh_6] (root,0,0,0.0,428) [scsi_tmf_6] (root,0,0,0.0,429) [scsi_eh_7] (root,0,0,0.0,430) [scsi_tmf_7] (root,0,0,0.0,431) [scsi_eh_8] (root,0,0,0.0,432) [scsi_tmf_8] (root,0,0,0.0,433) [scsi_eh_9] (root,0,0,0.0,434) [scsi_tmf_9] (root,0,0,0.0,442) [ttm_swap] (root,0,0,0.0,448) [kworker/1:1] (root,0,0,0.0,543) [kdmflush] (root,0,0,0.0,544) [bioset] (root,0,0,0.0,554) [kdmflush] (root,0,0,0.0,555) [bioset] (root,0,0,0.0,558) [kworker/0:1H] (root,0,0,0.0,572) [jbd2/dm-0-8] (root,0,0,0.0,573) [ext4-rsv-conver] (root,88612,43480,0.0,653) /usr/lib/systemd/systemd-journald (root,45604,656,0.0,679) /usr/lib/systemd/systemd-udevd (root,0,0,1.4,730) [kipmi0] (root,0,0,0.0,735) [kworker/14:1H] (root,0,0,0.0,737) [kworker/6:1H] (root,0,0,0.0,738) [kworker/5:1H] (root,0,0,0.0,805) [kvm-irqfd-clean] (root,198572,564,0.0,815) /usr/sbin/lvmetad -f (root,0,0,0.0,868) [kworker/1:1H] (root,0,0,0.0,872) [kworker/2:1H] (root,0,0,0.0,880) [kdmflush] (root,0,0,0.0,881) [bioset] (root,0,0,0.0,883) [kdmflush] (root,0,0,0.0,884) [bioset] (root,0,0,0.0,885) [kdmflush] (root,0,0,0.0,887) [bioset] (root,0,0,0.0,889) [kdmflush] (root,0,0,0.0,890) [bioset] (root,0,0,0.0,897) [kworker/13:1H] (root,0,0,0.0,898) [kworker/7:1H] (root,0,0,0.0,899) [kworker/15:1H] (root,0,0,0.0,907) [jbd2/sda1-8] (root,0,0,0.0,908) [ext4-rsv-conver] (root,0,0,0.0,929) [kworker/9:1H] (root,0,0,0.0,930) [kworker/12:1H] (root,0,0,0.0,931) [kworker/4:1H] (root,0,0,0.0,938) [jbd2/dm-5-8] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,941) [ext4-rsv-conver] (root,0,0,0.0,947) [jbd2/dm-4-8] (root,0,0,0.0,948) [ext4-rsv-conver] (root,0,0,0.0,951) [jbd2/dm-2-8] (root,0,0,0.0,952) [ext4-rsv-conver] (root,0,0,0.0,976) [nv_queue] (root,0,0,0.0,977) [nv_queue] (polkitd,612240,1076,0.0,1012) /usr/lib/polkit-1/polkitd --no-debug (root,52900,592,0.0,1015) /usr/sbin/smartd -n -q never (root,21788,508,0.0,1027) /usr/sbin/irqbalance --foreground (dbus,58220,900,0.0,1034) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (root,26752,1356,0.0,1057) /usr/lib/systemd/systemd-logind (root,0,0,0.0,1079) [kworker/12:1] (root,0,0,0.0,1099) [UVM global queu] (root,0,0,0.0,1100) [UVM deferred re] (root,0,0,0.0,1101) [UVM Tools Event] (root,0,0,0.0,1132) [nvidia-modeset/] (root,0,0,0.0,1133) [nvidia-modeset/] (root,0,0,0.0,1262) [bond0] (root,10058840,122424,0.0,1292) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,302284,45412,0.0,1570) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,574288,2332,0.0,1574) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,113000,8,0.0,1576) /usr/sbin/sshd -D (root,0,0,0.0,1581) [irq/98-nvidia] (root,0,0,0.0,1582) [nvidia] (root,0,0,0.2,1583) [nv_queue] (root,0,0,0.0,1751) [kworker/u33:0] (root,799816,2012,0.0,1823) /usr/sbin/collectd (root,740688,25572,0.0,1845) /usr/sbin/rsyslogd -n (root,591424,5636,0.1,1871) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,115700,964,0.0,1874) /bin/bash /sbin/condor_inspector (root,209064,1688,0.0,1887) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,1304,0.0,1906) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1913) [irq/99-nvidia] (root,0,0,0.0,1914) [nvidia] (root,0,0,0.2,1915) [nv_queue] (root,209332,2228,0.0,1920) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,89804,228,0.0,2118) /usr/libexec/postfix/master -w (postfix,90080,780,0.0,2121) qmgr -l -t unix -u (cvmfs,1303024,30340,0.0,2210) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2330) [irq/100-nvidia] (root,0,0,0.0,2331) [nvidia] (root,0,0,0.1,2332) [nv_queue] (root,0,0,0.0,2501) [kworker/0:2] (cvmfs,852204,25500,0.0,2588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,852268,25152,0.0,2607) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,852260,25416,0.0,2626) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,852260,25464,0.0,2645) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,852260,25468,0.0,2664) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,852204,25724,0.0,2757) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,5052716,128152,0.0,2792) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (root,0,0,0.0,2817) [irq/101-nvidia] (root,0,0,0.0,2818) [nvidia] (root,0,0,0.1,2819) [nv_queue] (cvmfs,852124,27044,0.0,2830) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,852256,25620,0.0,2850) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,2870) [irq/102-nvidia] (root,0,0,0.0,2871) [nvidia] (root,0,0,0.1,2872) [nv_queue] (root,126388,432,0.0,2874) /usr/sbin/crond -n (condor,75276,3088,0.0,2875) /usr/sbin/condor_master -f (ntp,29960,776,0.0,2877) /usr/sbin/ntpd -u ntp:ntp -g (root,27828,4744,0.0,2878) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (root,0,0,0.0,2879) [irq/103-nvidia] (root,0,0,0.0,2880) [nvidia] (root,0,0,0.1,2881) [nv_queue] (condor,63876,2780,0.0,2882) condor_shared_port -f (root,0,0,0.0,2883) [kworker/10:1H] (root,0,0,0.0,2884) [irq/104-nvidia] (root,0,0,0.0,2885) [nvidia] (root,0,0,0.2,2886) [nv_queue] (condor,3109764,3034684,0.1,2887) condor_startd -f (root,0,0,0.0,2908) [irq/105-nvidia] (root,0,0,0.0,2909) [nvidia] (root,0,0,0.2,2910) [nv_queue] (root,110208,8,0.0,3014) /sbin/agetty --noclear tty1 linux (condor,34082884,508,0.1,3821) GPUs_MONITOR (root,0,0,0.0,3824) [UVM GPU1 BH] (root,0,0,0.0,3825) [UVM GPU2 BH] (root,0,0,0.0,3827) [UVM GPU3 BH] (root,0,0,0.0,3828) [UVM GPU4 BH] (root,0,0,0.0,3830) [UVM GPU5 BH] (root,0,0,0.0,3831) [UVM GPU6 BH] (root,0,0,56.9,3832) [UVM GPU7 BH] (root,0,0,0.0,3833) [UVM GPU8 BH] (root,0,0,0.0,4069) [kworker/8:1H] (root,0,0,0.0,4293) [kworker/13:1] (root,0,0,0.0,4366) [kworker/3:1H] (root,0,0,0.0,4444) [kworker/4:1] (root,27168,1056,0.0,5419) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,5727) [kworker/9:2] (root,0,0,0.0,5865) [kworker/1:0] (root,0,0,0.0,5959) [kworker/5:0] (ganglia,220196,8316,0.0,6311) /usr/sbin/gmond (slot-1,10903860,637224,0.3,6554) python /scratch/condor/dir_21435/glidein/execute.bde4dab8-997d-11ed-81a0-0cc47a857a10/dir_5868/condor_exec.exe -i /data/ana/BSM/HNL/MC/190611/Gen/03001-04000/Gen_03934.i3.zst -o /data/ana/BSM/HNL/MC/190611/Phot/03001-04000/Phot_03934.i3.zst -r 1 -l 3934 -t -m spice_3.2.1 --holeice angsens/as.flasher_p1_0.30_p2_-1 --osg True (root,0,0,0.0,6810) [kworker/8:1] (root,0,0,0.0,7533) [kworker/9:1] (root,0,0,0.0,7629) [kworker/0:1] (root,0,0,0.0,7896) [kworker/11:1H] (root,0,0,0.0,8115) [kworker/15:2] (root,0,0,0.0,10511) [kworker/u32:1] (root,0,0,0.0,10866) [kworker/6:2] (root,0,0,0.0,12468) [kworker/10:1] (root,0,0,0.0,12758) [kworker/14:0] (root,0,0,0.0,12780) [kworker/u34:2] (root,0,0,0.0,13950) [kworker/4:2] (root,0,0,0.0,14558) [kworker/11:0] (slot-1,17150796,6844324,0.8,14989) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000136_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141156/NuMu_B_141156_000136_step2.i3.zst --filenr 136 --tmpdir /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_14769/2023-02-02_02-26-39 --gridftp --runnumber 141156 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,15530208,5192288,5.3,15169) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000633_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_B_141159_000633_step2.i3.zst --filenr 633 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_14465/2023-03-04_17-59-09 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (cvmfs,73892,4552,0.0,15170) /usr/bin/cvmfs2 __cachemgr__ . 7 8 41943040000 20971520000 0 3 -1 : (cvmfs,60968,720,0.0,15174) /usr/bin/cvmfs2 __cachemgr__ . 7 8 41943040000 20971520000 0 3 -1 : (cvmfs,967072,26816,0.0,15179) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,71648,2188,0.0,15354) /usr/bin/cvmfs2 __cachemgr__ . 7 8 1048576000 524288000 0 3 -1 : (cvmfs,60968,720,0.0,15358) /usr/bin/cvmfs2 __cachemgr__ . 7 8 1048576000 524288000 0 3 -1 : (cvmfs,811276,21816,0.0,15366) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (slot-1,13878336,3613124,1.3,15370) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000776_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141157/NuMu_B_141157_000776_step2.i3.zst --filenr 776 --tmpdir /scratch/condor/dir_12927/glidein/execute.f02ec7ea-ae0e-11ed-990f-0cc47a857a10/dir_14937/2023-02-16_08-41-23 --gridftp --runnumber 141157 --icemodel spice_3.2.1 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling --geant4-muons --particle-history (cvmfs,811224,21844,0.0,15380) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,811224,22104,0.0,15393) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,811276,21928,0.0,15406) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,811276,21692,0.0,15410) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,811276,21760,0.0,15435) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,5011736,124484,0.0,15448) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,811276,21860,0.0,15461) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,811140,23332,0.0,15474) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,17158) [kworker/15:1] (root,0,0,0.0,17852) [kworker/u34:1] (root,0,0,0.0,17978) [kworker/6:1] (root,0,0,0.0,18651) [kworker/u33:2] (root,0,0,0.0,19449) [kworker/8:0] (root,0,0,0.0,19657) [kworker/5:1] (root,521640,187596,21.0,20045) puppet agent: applying configuration (root,176296,2060,0.0,20443) /usr/sbin/CROND -n (root,113288,1432,0.0,20446) /bin/bash /usr/local/sbin/thermal_shutdown.sh 36 (root,113288,588,0.0,20455) /bin/bash /usr/local/sbin/thermal_shutdown.sh 36 (root,112324,680,0.0,20456) timeout 20 ipmitool sdr type temperature (root,112812,1004,0.0,20457) grep -v ^\(CPU\|PCH\|Peripheral\|Planar\) (root,113644,976,0.0,20458) awk /degrees C/{ ...temp = $(NF-2); ...if (temp ~ /^[0-9]+$/ && temp > 0) { ....print temp ...} ..} (root,126844,928,0.0,20459) sort -n (root,108064,676,0.0,20460) head -n 1 (root,24320,1684,0.0,20461) ipmitool sdr type temperature (root,0,0,0.0,20484) [kworker/11:2] (root,108056,356,0.0,20563) sleep 15 (root,8632,660,0.0,20595) timeout -s 9 360 check_mk_agent (root,115408,1436,0.0,20596) sh -c rsync -ni rsync://repos.icecube.wisc.edu/puppet/ipmicfg_1.23.0_general_20151106.zip /opt/puppetlabs/server/data/puppetserver/installer/supermicro/ipmicfg_1.23.0_general_20151106.zip | grep . (root,11696,1536,0.0,20599) /bin/bash /bin/check_mk_agent (slot-1,13628156,3307676,0.5,20601) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_A_140000_000115_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_A_141159_000115_step2.i3.zst --filenr 115 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_20078/2023-03-01_14-44-52 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (root,121208,1212,0.0,20610) rsync -ni rsync://repos.icecube.wisc.edu/puppet/ipmicfg_1.23.0_general_20151106.zip /opt/puppetlabs/server/data/puppetserver/installer/supermicro/ipmicfg_1.23.0_general_20151106.zip (root,112812,956,0.0,20611) grep . (root,51732,1692,0.0,20618) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,20619) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,21308) [kworker/3:0] (postfix,89908,4024,0.0,21426) pickup -l -t unix -u (slot-1,11183212,970120,3.5,21544) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000037_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_C_141159_000037_step2.i3.zst --filenr 37 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_21019/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (root,0,0,0.0,22234) [kworker/u32:2] (root,0,0,0.0,22646) [kworker/2:2] (root,0,0,0.0,22835) [kworker/12:2] (root,0,0,0.0,22938) [kworker/4:0] (root,0,0,0.0,25748) [kworker/11:1] (root,0,0,0.0,26209) [kworker/7:1] (root,0,0,0.0,26540) [kworker/13:2] (root,0,0,0.0,29588) [kworker/14:2] (slot-1,0,0,0.0,29724) [python] <defunct> (slot-1,0,0,0.0,29885) [zstd] <defunct> (root,0,0,0.0,31217) [kworker/7:0] (root,0,0,0.0,31404) [kworker/3:1] (root,0,0,0.0,31748) [kworker/10:2] (root,0,0,0.0,32040) [kworker/2:1] (root,0,0,0.0,32609) [kworker/15:0]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b1a45a4f05aae993daf5f9bd5d
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2.chtc.wisc.edu AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43800,3008,0.0,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.2,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,259) [kauditd] (root,0,0,0.0,411) [ata_sff] (root,0,0,0.0,412) [scsi_eh_0] (root,0,0,0.0,413) [scsi_tmf_0] (root,0,0,0.0,414) [scsi_eh_1] (root,0,0,0.0,415) [scsi_tmf_1] (root,0,0,0.0,416) [scsi_eh_2] (root,0,0,0.0,417) [scsi_tmf_2] (root,0,0,0.0,418) [scsi_eh_3] (root,0,0,0.0,419) [scsi_tmf_3] (root,0,0,0.0,423) [scsi_eh_4] (root,0,0,0.0,424) [scsi_tmf_4] (root,0,0,0.0,425) [scsi_eh_5] (root,0,0,0.0,426) [scsi_tmf_5] (root,0,0,0.0,427) [scsi_eh_6] (root,0,0,0.0,428) [scsi_tmf_6] (root,0,0,0.0,429) [scsi_eh_7] (root,0,0,0.0,430) [scsi_tmf_7] (root,0,0,0.0,431) [scsi_eh_8] (root,0,0,0.0,432) [scsi_tmf_8] (root,0,0,0.0,433) [scsi_eh_9] (root,0,0,0.0,434) [scsi_tmf_9] (root,0,0,0.0,442) [ttm_swap] (root,0,0,0.0,543) [kdmflush] (root,0,0,0.0,544) [bioset] (root,0,0,0.0,554) [kdmflush] (root,0,0,0.0,555) [bioset] (root,0,0,0.0,558) [kworker/0:1H] (root,0,0,0.0,572) [jbd2/dm-0-8] (root,0,0,0.0,573) [ext4-rsv-conver] (root,0,0,0.0,588) [kworker/4:2] (root,0,0,0.0,640) [kworker/13:1] (root,56588,15480,0.0,653) /usr/lib/systemd/systemd-journald (root,45604,656,0.0,679) /usr/lib/systemd/systemd-udevd (root,0,0,1.4,730) [kipmi0] (root,0,0,0.0,735) [kworker/14:1H] (root,0,0,0.0,737) [kworker/6:1H] (root,0,0,0.0,738) [kworker/5:1H] (root,0,0,0.0,805) [kvm-irqfd-clean] (root,198572,564,0.0,815) /usr/sbin/lvmetad -f (root,0,0,0.0,868) [kworker/1:1H] (root,0,0,0.0,872) [kworker/2:1H] (root,0,0,0.0,880) [kdmflush] (root,0,0,0.0,881) [bioset] (root,0,0,0.0,883) [kdmflush] (root,0,0,0.0,884) [bioset] (root,0,0,0.0,885) [kdmflush] (root,0,0,0.0,887) [bioset] (root,0,0,0.0,889) [kdmflush] (root,0,0,0.0,890) [bioset] (root,0,0,0.0,897) [kworker/13:1H] (root,0,0,0.0,898) [kworker/7:1H] (root,0,0,0.0,899) [kworker/15:1H] (root,0,0,0.0,907) [jbd2/sda1-8] (root,0,0,0.0,908) [ext4-rsv-conver] (root,0,0,0.0,921) [kworker/u34:2] (root,0,0,0.0,929) [kworker/9:1H] (root,0,0,0.0,930) [kworker/12:1H] (root,0,0,0.0,931) [kworker/4:1H] (root,0,0,0.0,938) [jbd2/dm-5-8] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,941) [ext4-rsv-conver] (root,0,0,0.0,947) [jbd2/dm-4-8] (root,0,0,0.0,948) [ext4-rsv-conver] (root,0,0,0.0,951) [jbd2/dm-2-8] (root,0,0,0.0,952) [ext4-rsv-conver] (root,0,0,0.0,976) [nv_queue] (root,0,0,0.0,977) [nv_queue] (polkitd,612240,1092,0.0,1012) /usr/lib/polkit-1/polkitd --no-debug (root,52900,592,0.0,1015) /usr/sbin/smartd -n -q never (root,21788,508,0.0,1027) /usr/sbin/irqbalance --foreground (dbus,58220,900,0.0,1034) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (root,26620,1244,0.0,1057) /usr/lib/systemd/systemd-logind (root,0,0,0.0,1099) [UVM global queu] (root,0,0,0.0,1100) [UVM deferred re] (root,0,0,0.0,1101) [UVM Tools Event] (root,0,0,0.0,1132) [nvidia-modeset/] (root,0,0,0.0,1133) [nvidia-modeset/] (root,0,0,0.0,1262) [bond0] (root,10058840,112780,0.0,1292) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,0,0,0.0,1514) [kworker/1:1] (root,302152,39068,0.0,1570) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,574288,2332,0.0,1574) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,113000,8,0.0,1576) /usr/sbin/sshd -D (root,0,0,0.0,1581) [irq/98-nvidia] (root,0,0,0.0,1582) [nvidia] (root,0,0,0.2,1583) [nv_queue] (root,799816,1900,0.0,1823) /usr/sbin/collectd (root,740688,10740,0.0,1845) /usr/sbin/rsyslogd -n (root,591424,5636,0.1,1871) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,115700,964,0.0,1874) /bin/bash /sbin/condor_inspector (root,209064,1692,0.0,1887) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,1304,0.0,1906) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1913) [irq/99-nvidia] (root,0,0,0.0,1914) [nvidia] (root,0,0,0.2,1915) [nv_queue] (root,209332,2228,0.0,1920) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,89804,228,0.0,2118) /usr/libexec/postfix/master -w (postfix,90080,780,0.0,2121) qmgr -l -t unix -u (cvmfs,73348,2784,0.0,2130) /usr/bin/cvmfs2 __cachemgr__ . 7 8 41943040000 20971520000 0 3 -1 : (cvmfs,60968,0,0.0,2134) /usr/bin/cvmfs2 __cachemgr__ . 7 8 41943040000 20971520000 0 3 -1 : (cvmfs,946300,11256,0.0,2210) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,92204,8,0.0,2214) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2330) [irq/100-nvidia] (root,0,0,0.0,2331) [nvidia] (root,0,0,0.1,2332) [nv_queue] (cvmfs,790988,5476,0.0,2588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,92152,4,0.0,2592) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,733164,2784,0.0,2607) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,92204,4,0.0,2611) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,791344,4360,0.0,2626) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,92204,4,0.0,2630) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,791624,4604,0.0,2645) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,92204,28,0.0,2649) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,791036,5616,0.0,2664) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,92204,4,0.0,2668) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,790988,5664,0.0,2757) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,92152,4,0.0,2761) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,5089796,130256,0.0,2792) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,92208,112,0.0,2796) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (root,0,0,0.0,2817) [irq/101-nvidia] (root,0,0,0.0,2818) [nvidia] (root,0,0,0.1,2819) [nv_queue] (cvmfs,72164,1780,0.0,2822) /usr/bin/cvmfs2 __cachemgr__ . 7 8 1048576000 524288000 0 3 -1 : (cvmfs,60968,0,0.0,2826) /usr/bin/cvmfs2 __cachemgr__ . 7 8 1048576000 524288000 0 3 -1 : (cvmfs,790912,7492,0.0,2830) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,92204,60,0.0,2834) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,791036,5708,0.0,2850) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,92204,16,0.0,2854) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,2870) [irq/102-nvidia] (root,0,0,0.0,2871) [nvidia] (root,0,0,0.1,2872) [nv_queue] (root,126388,432,0.0,2874) /usr/sbin/crond -n (condor,75276,3088,0.0,2875) /usr/sbin/condor_master -f (ntp,29960,776,0.0,2877) /usr/sbin/ntpd -u ntp:ntp -g (root,27828,4744,0.0,2878) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (root,0,0,0.0,2879) [irq/103-nvidia] (root,0,0,0.0,2880) [nvidia] (root,0,0,0.1,2881) [nv_queue] (condor,63876,2768,0.0,2882) condor_shared_port -f (root,0,0,0.0,2883) [kworker/10:1H] (root,0,0,0.0,2884) [irq/104-nvidia] (root,0,0,0.0,2885) [nvidia] (root,0,0,0.2,2886) [nv_queue] (condor,2918884,2843744,0.1,2887) condor_startd -f (root,0,0,0.0,2908) [irq/105-nvidia] (root,0,0,0.0,2909) [nvidia] (root,0,0,0.2,2910) [nv_queue] (root,110208,8,0.0,3014) /sbin/agetty --noclear tty1 linux (root,0,0,0.0,3176) [kworker/15:1] (condor,34082884,508,0.1,3821) GPUs_MONITOR (root,0,0,0.0,3824) [UVM GPU1 BH] (root,0,0,0.0,3825) [UVM GPU2 BH] (root,0,0,0.0,3827) [UVM GPU3 BH] (root,0,0,0.0,3828) [UVM GPU4 BH] (root,0,0,0.0,3830) [UVM GPU5 BH] (root,0,0,0.0,3831) [UVM GPU6 BH] (root,0,0,54.0,3832) [UVM GPU7 BH] (root,0,0,0.0,3833) [UVM GPU8 BH] (root,0,0,0.0,4069) [kworker/8:1H] (root,0,0,0.0,4366) [kworker/3:1H] (root,27168,1056,0.0,5419) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (ganglia,220196,8316,0.0,6311) /usr/sbin/gmond (slot-1,10903860,637224,0.3,6554) python /scratch/condor/dir_21435/glidein/execute.bde4dab8-997d-11ed-81a0-0cc47a857a10/dir_5868/condor_exec.exe -i /data/ana/BSM/HNL/MC/190611/Gen/03001-04000/Gen_03934.i3.zst -o /data/ana/BSM/HNL/MC/190611/Phot/03001-04000/Phot_03934.i3.zst -r 1 -l 3934 -t -m spice_3.2.1 --holeice angsens/as.flasher_p1_0.30_p2_-1 --osg True (root,0,0,0.0,7641) [kworker/11:1] (root,0,0,0.0,7896) [kworker/11:1H] (root,0,0,0.0,8895) [kworker/3:2] (root,0,0,0.0,9388) [kworker/5:2] (root,0,0,0.0,9478) [kworker/3:1] (root,0,0,0.0,9487) [kworker/14:0] (root,0,0,0.0,10395) [kworker/1:0] (root,0,0,0.0,11180) [kworker/7:0] (root,0,0,0.0,11283) [kworker/0:2] (root,0,0,0.0,11592) [kworker/u32:2] (root,0,0,0.0,11961) [kworker/5:0] (root,0,0,0.0,12874) [kworker/9:0] (root,0,0,0.0,13765) [kworker/4:1] (root,0,0,0.0,13929) [kworker/0:1] (root,0,0,0.0,14075) [kworker/13:0] (root,0,0,0.0,14080) [kworker/7:1] (slot-1,96860,8608,0.0,14465) condor_starter -f -a slot1_7 sub-1.icecube.wisc.edu (root,0,0,0.0,14478) [kworker/12:0] (slot-1,17150796,6844324,1.0,14989) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000136_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141156/NuMu_B_141156_000136_step2.i3.zst --filenr 136 --tmpdir /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_14769/2023-02-02_02-26-39 --gridftp --runnumber 141156 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,11704,1516,0.0,15065) bash /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_14465/condor_exec.exe (slot-1,11832,1548,0.0,15165) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000633_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_B_141159_000633_step2.i3.zst --filenr 633 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_14465/2023-03-04_17-59-09 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,15530208,5192288,51.6,15169) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000633_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_B_141159_000633_step2.i3.zst --filenr 633 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_14465/2023-03-04_17-59-09 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,13878336,3613124,1.7,15370) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000776_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141157/NuMu_B_141157_000776_step2.i3.zst --filenr 776 --tmpdir /scratch/condor/dir_12927/glidein/execute.f02ec7ea-ae0e-11ed-990f-0cc47a857a10/dir_14937/2023-02-16_08-41-23 --gridftp --runnumber 141157 --icemodel spice_3.2.1 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling --geant4-muons --particle-history (root,0,0,0.0,15657) [kworker/u32:0] (root,0,0,0.0,16144) [kworker/12:1] (root,0,0,0.0,16858) [kworker/8:0] (root,0,0,0.0,17734) [kworker/15:0] (condor,67124,8192,0.0,17888) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (slot-1,11692,1572,0.0,18171) /bin/sh glidein_start.sh (slot-1,61812,6476,0.0,18396) glideinExec/sbin/condor_master -f -r 20135 (slot-1,11896,1688,0.0,18397) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_8304c479-8653-4c2c-b7a7-e9fcb4de7eca.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20230228%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230228T122002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=9c819c9848f05cc8c492f7148f595d4f3d21502fb8bf030d2e2fad526a1d2e09 (postfix,89908,4028,0.0,18422) pickup -l -t unix -u (slot-1,24680,4416,0.0,18439) condor_procd -A /scratch/condor/dir_17888/glidein/log.7e1c014c-b762-11ed-b5dc-0cc47a857a10/procd_address -L /scratch/condor/dir_17888/glidein/log.7e1c014c-b762-11ed-b5dc-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,63392,8160,0.0,18474) condor_startd -f (slot-1,4368,360,0.0,19731) sleep 300 (root,0,0,0.0,20175) [kworker/4:0] (slot-1,13628156,3307676,1.2,20601) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_A_140000_000115_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_A_141159_000115_step2.i3.zst --filenr 115 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_20078/2023-03-01_14-44-52 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (root,0,0,0.0,20635) [kworker/9:2] (slot-1,11183212,970120,7.0,21544) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000037_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_C_141159_000037_step2.i3.zst --filenr 37 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_21019/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (root,0,0,0.0,22304) [kworker/u33:0] (root,0,0,0.0,22418) [kworker/2:1] (root,0,0,0.0,22734) [kworker/6:0] (root,0,0,0.0,22835) [kworker/12:2] (root,108056,360,0.0,23229) sleep 15 (root,8632,664,0.0,23230) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,23231) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,23247) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,944,0.0,23248) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,23628) [kworker/10:0] (root,0,0,0.0,24185) [kworker/11:0] (root,0,0,0.0,24441) [kworker/14:2] (root,0,0,0.0,24553) [kworker/10:1] (root,0,0,0.0,24962) [kworker/u33:2] (root,0,0,0.0,26928) [kworker/6:2] (root,0,0,0.0,27813) [kworker/2:2] (root,0,0,0.0,27969) [kworker/u34:1] (slot-1,0,0,0.0,29724) [python] <defunct> (root,0,0,0.0,29780) [kworker/2:0] (slot-1,0,0,0.0,29885) [zstd] <defunct> (root,0,0,0.0,32031) [kworker/8:1]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b1a45a4f05aae993da9a65ecd5
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2.chtc.wisc.edu AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43800,3108,0.0,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.2,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,259) [kauditd] (root,0,0,0.0,411) [ata_sff] (root,0,0,0.0,412) [scsi_eh_0] (root,0,0,0.0,413) [scsi_tmf_0] (root,0,0,0.0,414) [scsi_eh_1] (root,0,0,0.0,415) [scsi_tmf_1] (root,0,0,0.0,416) [scsi_eh_2] (root,0,0,0.0,417) [scsi_tmf_2] (root,0,0,0.0,418) [scsi_eh_3] (root,0,0,0.0,419) [scsi_tmf_3] (root,0,0,0.0,423) [scsi_eh_4] (root,0,0,0.0,424) [scsi_tmf_4] (root,0,0,0.0,425) [scsi_eh_5] (root,0,0,0.0,426) [scsi_tmf_5] (root,0,0,0.0,427) [scsi_eh_6] (root,0,0,0.0,428) [scsi_tmf_6] (root,0,0,0.0,429) [scsi_eh_7] (root,0,0,0.0,430) [scsi_tmf_7] (root,0,0,0.0,431) [scsi_eh_8] (root,0,0,0.0,432) [scsi_tmf_8] (root,0,0,0.0,433) [scsi_eh_9] (root,0,0,0.0,434) [scsi_tmf_9] (root,0,0,0.0,442) [ttm_swap] (root,0,0,0.0,543) [kdmflush] (root,0,0,0.0,544) [bioset] (root,0,0,0.0,554) [kdmflush] (root,0,0,0.0,555) [bioset] (root,0,0,0.0,558) [kworker/0:1H] (root,0,0,0.0,572) [jbd2/dm-0-8] (root,0,0,0.0,573) [ext4-rsv-conver] (root,126320,73420,0.0,653) /usr/lib/systemd/systemd-journald (root,45604,656,0.0,679) /usr/lib/systemd/systemd-udevd (root,0,0,0.0,685) [kworker/1:1] (root,0,0,1.4,730) [kipmi0] (root,0,0,0.0,735) [kworker/14:1H] (root,0,0,0.0,737) [kworker/6:1H] (root,0,0,0.0,738) [kworker/5:1H] (root,0,0,0.0,805) [kvm-irqfd-clean] (root,198572,564,0.0,815) /usr/sbin/lvmetad -f (root,0,0,0.0,868) [kworker/1:1H] (root,0,0,0.0,872) [kworker/2:1H] (root,0,0,0.0,880) [kdmflush] (root,0,0,0.0,881) [bioset] (root,0,0,0.0,883) [kdmflush] (root,0,0,0.0,884) [bioset] (root,0,0,0.0,885) [kdmflush] (root,0,0,0.0,887) [bioset] (root,0,0,0.0,889) [kdmflush] (root,0,0,0.0,890) [bioset] (root,0,0,0.0,897) [kworker/13:1H] (root,0,0,0.0,898) [kworker/7:1H] (root,0,0,0.0,899) [kworker/15:1H] (root,0,0,0.0,907) [jbd2/sda1-8] (root,0,0,0.0,908) [ext4-rsv-conver] (root,0,0,0.0,929) [kworker/9:1H] (root,0,0,0.0,930) [kworker/12:1H] (root,0,0,0.0,931) [kworker/4:1H] (root,0,0,0.0,938) [jbd2/dm-5-8] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,941) [ext4-rsv-conver] (root,0,0,0.0,947) [jbd2/dm-4-8] (root,0,0,0.0,948) [ext4-rsv-conver] (root,0,0,0.0,951) [jbd2/dm-2-8] (root,0,0,0.0,952) [ext4-rsv-conver] (root,0,0,0.0,974) [kworker/10:1] (root,0,0,0.0,976) [nv_queue] (root,0,0,0.0,977) [nv_queue] (polkitd,612240,1092,0.0,1012) /usr/lib/polkit-1/polkitd --no-debug (root,52900,592,0.0,1015) /usr/sbin/smartd -n -q never (root,21788,508,0.0,1027) /usr/sbin/irqbalance --foreground (dbus,58220,900,0.0,1034) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (root,26620,1244,0.0,1057) /usr/lib/systemd/systemd-logind (root,0,0,0.0,1099) [UVM global queu] (root,0,0,0.0,1100) [UVM deferred re] (root,0,0,0.0,1101) [UVM Tools Event] (root,0,0,0.0,1112) [kworker/15:1] (root,0,0,0.0,1132) [nvidia-modeset/] (root,0,0,0.0,1133) [nvidia-modeset/] (root,0,0,0.0,1191) [kworker/u32:0] (root,0,0,0.0,1262) [bond0] (root,10058840,112656,0.0,1292) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,302152,35968,0.0,1570) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,574288,2332,0.0,1574) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,113000,8,0.0,1576) /usr/sbin/sshd -D (root,0,0,0.0,1581) [irq/98-nvidia] (root,0,0,0.0,1582) [nvidia] (root,0,0,0.2,1583) [nv_queue] (root,799816,1904,0.0,1823) /usr/sbin/collectd (root,740560,43196,0.0,1845) /usr/sbin/rsyslogd -n (root,591424,5644,0.1,1871) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,115700,964,0.0,1874) /bin/bash /sbin/condor_inspector (root,209064,1696,0.0,1887) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,1304,0.0,1906) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1913) [irq/99-nvidia] (root,0,0,0.0,1914) [nvidia] (root,0,0,0.2,1915) [nv_queue] (root,209332,2228,0.0,1920) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,89804,228,0.0,2118) /usr/libexec/postfix/master -w (postfix,90080,780,0.0,2121) qmgr -l -t unix -u (cvmfs,73348,2784,0.0,2130) /usr/bin/cvmfs2 __cachemgr__ . 7 8 41943040000 20971520000 0 3 -1 : (cvmfs,60968,0,0.0,2134) /usr/bin/cvmfs2 __cachemgr__ . 7 8 41943040000 20971520000 0 3 -1 : (cvmfs,946300,9096,0.0,2210) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,92204,8,0.0,2214) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2330) [irq/100-nvidia] (root,0,0,0.0,2331) [nvidia] (root,0,0,0.1,2332) [nv_queue] (cvmfs,790996,5424,0.0,2588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,92152,4,0.0,2592) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,733164,2012,0.0,2607) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,92204,4,0.0,2611) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,729532,3244,0.0,2626) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,92204,4,0.0,2630) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,791624,4332,0.0,2645) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,92204,28,0.0,2649) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,791036,5576,0.0,2664) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,92204,4,0.0,2668) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,790984,5624,0.0,2757) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,92152,4,0.0,2761) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,5089712,135164,0.0,2792) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,92208,112,0.0,2796) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (root,0,0,0.0,2817) [irq/101-nvidia] (root,0,0,0.0,2818) [nvidia] (root,0,0,0.1,2819) [nv_queue] (cvmfs,72164,1780,0.0,2822) /usr/bin/cvmfs2 __cachemgr__ . 7 8 1048576000 524288000 0 3 -1 : (cvmfs,60968,0,0.0,2826) /usr/bin/cvmfs2 __cachemgr__ . 7 8 1048576000 524288000 0 3 -1 : (cvmfs,790900,7484,0.0,2830) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,92204,60,0.0,2834) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,791036,5708,0.0,2850) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,92204,16,0.0,2854) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,2870) [irq/102-nvidia] (root,0,0,0.0,2871) [nvidia] (root,0,0,0.1,2872) [nv_queue] (root,126388,432,0.0,2874) /usr/sbin/crond -n (condor,75276,3088,0.0,2875) /usr/sbin/condor_master -f (ntp,29960,776,0.0,2877) /usr/sbin/ntpd -u ntp:ntp -g (root,27828,4744,0.0,2878) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (root,0,0,0.0,2879) [irq/103-nvidia] (root,0,0,0.0,2880) [nvidia] (root,0,0,0.1,2881) [nv_queue] (condor,63876,2756,0.0,2882) condor_shared_port -f (root,0,0,0.0,2883) [kworker/10:1H] (root,0,0,0.0,2884) [irq/104-nvidia] (root,0,0,0.0,2885) [nvidia] (root,0,0,0.2,2886) [nv_queue] (condor,2723756,2648588,0.1,2887) condor_startd -f (root,0,0,0.0,2908) [irq/105-nvidia] (root,0,0,0.0,2909) [nvidia] (root,0,0,0.2,2910) [nv_queue] (root,110208,8,0.0,3014) /sbin/agetty --noclear tty1 linux (root,0,0,0.0,3196) [kworker/6:0] (root,0,0,0.0,3456) [kworker/5:2] (condor,34082884,508,0.1,3821) GPUs_MONITOR (root,0,0,0.0,3824) [UVM GPU1 BH] (root,0,0,0.0,3825) [UVM GPU2 BH] (root,0,0,0.0,3827) [UVM GPU3 BH] (root,0,0,0.0,3828) [UVM GPU4 BH] (root,0,0,0.0,3830) [UVM GPU5 BH] (root,0,0,0.0,3831) [UVM GPU6 BH] (root,0,0,50.7,3832) [UVM GPU7 BH] (root,0,0,0.0,3833) [UVM GPU8 BH] (root,0,0,0.0,4069) [kworker/8:1H] (root,0,0,0.0,4366) [kworker/3:1H] (root,27168,1056,0.0,5419) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,5468) [kworker/u34:2] (ganglia,220196,8408,0.0,6311) /usr/sbin/gmond (slot-1,10903860,656756,0.4,6554) python /scratch/condor/dir_21435/glidein/execute.bde4dab8-997d-11ed-81a0-0cc47a857a10/dir_5868/condor_exec.exe -i /data/ana/BSM/HNL/MC/190611/Gen/03001-04000/Gen_03934.i3.zst -o /data/ana/BSM/HNL/MC/190611/Phot/03001-04000/Phot_03934.i3.zst -r 1 -l 3934 -t -m spice_3.2.1 --holeice angsens/as.flasher_p1_0.30_p2_-1 --osg True (root,0,0,0.0,7832) [kworker/11:1] (root,0,0,0.0,7896) [kworker/11:1H] (root,0,0,0.0,9451) [kworker/14:0] (root,0,0,0.0,10488) [kworker/9:0] (root,0,0,0.0,11091) [kworker/5:0] (root,0,0,0.0,11242) [kworker/7:2] (root,0,0,0.0,11695) [kworker/13:1] (root,0,0,0.0,11868) [kworker/2:1] (root,0,0,0.0,13242) [kworker/15:2] (root,0,0,0.0,13251) [kworker/u33:2] (root,0,0,0.0,14098) [kworker/3:1] (root,0,0,0.0,14271) [kworker/2:2] (slot-1,17150796,6863848,1.2,14989) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000136_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141156/NuMu_B_141156_000136_step2.i3.zst --filenr 136 --tmpdir /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_14769/2023-02-02_02-26-39 --gridftp --runnumber 141156 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,13878336,3632588,2.5,15370) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000776_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141157/NuMu_B_141157_000776_step2.i3.zst --filenr 776 --tmpdir /scratch/condor/dir_12927/glidein/execute.f02ec7ea-ae0e-11ed-990f-0cc47a857a10/dir_14937/2023-02-16_08-41-23 --gridftp --runnumber 141157 --icemodel spice_3.2.1 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling --geant4-muons --particle-history (root,0,0,0.0,15664) [kworker/0:0] (root,0,0,0.0,16061) [kworker/7:1] (condor,67120,8152,0.0,17888) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (slot-1,11692,1572,0.0,18171) /bin/sh glidein_start.sh (slot-1,61812,6540,0.0,18396) glideinExec/sbin/condor_master -f -r 20135 (slot-1,11696,1512,0.0,18397) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_8304c479-8653-4c2c-b7a7-e9fcb4de7eca.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20230228%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230228T122002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=9c819c9848f05cc8c492f7148f595d4f3d21502fb8bf030d2e2fad526a1d2e09 (slot-1,24680,4376,0.0,18439) condor_procd -A /scratch/condor/dir_17888/glidein/log.7e1c014c-b762-11ed-b5dc-0cc47a857a10/procd_address -L /scratch/condor/dir_17888/glidein/log.7e1c014c-b762-11ed-b5dc-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,63236,8184,0.0,18474) condor_startd -f (root,0,0,0.0,18520) [kworker/13:2] (root,0,0,0.0,18575) [kworker/11:2] (root,0,0,0.0,18624) [kworker/8:2] (root,0,0,0.0,19713) [kworker/u34:1] (root,0,0,0.0,19968) [kworker/u33:1] (root,0,0,0.0,20012) [kworker/0:1] (root,0,0,0.0,20143) [kworker/u32:2] (slot-1,46156,6576,0.0,20149) condor_starter -f -a slot1_2 sub-1.icecube.wisc.edu (slot-1,11704,1516,0.0,20154) bash /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_20149/condor_exec.exe (slot-1,11832,1548,0.0,20253) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000066_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_B_141159_000066_step2.i3.zst --filenr 66 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_20149/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,14949492,2455860,108,20257) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000066_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_B_141159_000066_step2.i3.zst --filenr 66 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_20149/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,13568,2164,0.0,20283) zstd -d -qq (slot-1,46172,6588,0.0,20907) condor_starter -f -a slot1_4 sub-1.icecube.wisc.edu (slot-1,46172,6592,0.0,20908) condor_starter -f -a slot1_5 sub-1.icecube.wisc.edu (slot-1,46172,6588,0.0,20947) condor_starter -f -a slot1_6 sub-1.icecube.wisc.edu (slot-1,46172,6588,0.0,21019) condor_starter -f -a slot1_7 sub-1.icecube.wisc.edu (slot-1,46172,6592,0.0,21078) condor_starter -f -a slot1_8 sub-1.icecube.wisc.edu (root,0,0,0.0,21108) [kworker/3:0] (slot-1,11704,1512,0.0,21130) bash /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_20907/condor_exec.exe (slot-1,11832,1548,0.0,21230) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000034_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_C_141159_000034_step2.i3.zst --filenr 34 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_20907/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,13928932,1435080,110,21234) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000034_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_C_141159_000034_step2.i3.zst --filenr 34 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_20907/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,11704,1516,0.0,21236) bash /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_20908/condor_exec.exe (slot-1,11832,1548,0.0,21335) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000035_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_C_141159_000035_step2.i3.zst --filenr 35 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_20908/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,13787140,1316048,110,21339) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000035_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_C_141159_000035_step2.i3.zst --filenr 35 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_20908/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,11704,1516,0.0,21340) bash /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_20947/condor_exec.exe (slot-1,11832,1544,0.0,21438) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000036_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_C_141159_000036_step2.i3.zst --filenr 36 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_20947/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,13696328,1194404,110,21442) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000036_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_C_141159_000036_step2.i3.zst --filenr 36 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_20947/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,11704,1512,0.0,21443) bash /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_21019/condor_exec.exe (slot-1,11832,1544,0.0,21540) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000037_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_C_141159_000037_step2.i3.zst --filenr 37 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_21019/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,14188572,1663580,109,21544) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000037_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_C_141159_000037_step2.i3.zst --filenr 37 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_21019/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,11704,1516,0.0,21546) bash /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_21078/condor_exec.exe (slot-1,11832,1548,0.0,21645) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000038_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_C_141159_000038_step2.i3.zst --filenr 38 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_21078/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,13576520,1105400,110,21649) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000038_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_C_141159_000038_step2.i3.zst --filenr 38 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_21078/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,13568,2140,0.0,21705) zstd -d -qq (slot-1,13568,2136,0.0,21836) zstd -d -qq (root,0,0,0.0,22030) [kworker/12:2] (root,0,0,0.0,22051) [kworker/6:1] (slot-1,13568,2140,0.0,22199) zstd -d -qq (slot-1,13568,2136,0.0,22256) zstd -d -qq (slot-1,13568,2136,0.0,22257) zstd -d -qq (root,0,0,0.0,22973) [kworker/9:2] (root,0,0,0.0,23021) [kworker/12:1] (root,0,0,0.0,23652) [kworker/4:0] (postfix,89908,4028,0.0,24016) pickup -l -t unix -u (root,0,0,0.0,25349) [kworker/3:2] (root,0,0,0.0,27596) [kworker/4:1] (slot-1,46156,6576,0.0,28440) condor_starter -f -a slot1_3 sub-1.icecube.wisc.edu (slot-1,11704,1512,0.0,28444) bash /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_28440/condor_exec.exe (slot-1,11832,1548,0.0,28541) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000067_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_B_141159_000067_step2.i3.zst --filenr 67 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_28440/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,15979176,3540432,108,28545) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000067_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_B_141159_000067_step2.i3.zst --filenr 67 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_28440/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,13568,2060,0.0,28660) zstd -d -qq (slot-1,4368,360,0.0,28664) sleep 300 (root,0,0,0.0,28872) [kworker/14:2] (root,108056,356,0.0,29429) sleep 15 (root,0,0,0.0,29461) [kworker/9:1] (root,8632,664,0.0,29509) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,29510) /bin/bash /bin/check_mk_agent (root,51732,1696,0.0,29526) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,944,0.0,29527) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (slot-1,0,0,0.0,29724) [python] <defunct> (slot-1,0,0,0.0,29885) [zstd] <defunct> (slot-1,46156,6576,0.0,30643) condor_starter -f -a slot1_1 sub-1.icecube.wisc.edu (slot-1,11704,1512,0.0,30647) bash /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_30643/condor_exec.exe (slot-1,11832,1548,0.0,30744) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_D_140000_000082_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_D_141159_000082_step2.i3.zst --filenr 82 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_30643/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,13368624,934324,113,30748) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_D_140000_000082_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141159/NuMu_D_141159_000082_step2.i3.zst --filenr 82 --tmpdir /scratch/condor/dir_17888/glidein/execute.7e1c014c-b762-11ed-b5dc-0cc47a857a10/dir_30643/2023-02-28_06-03-11 --gridftp --runnumber 141159 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,13568,2168,0.0,30943) zstd -d -qq (root,0,0,0.0,31745) [kworker/1:0] (root,0,0,0.0,32545) [kworker/8:0] (root,0,0,0.0,32720) [kworker/10:2]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b1a45a4f05aae993da12ab7393
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2.chtc.wisc.edu AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43800,3108,0.0,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.2,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,259) [kauditd] (root,0,0,0.0,411) [ata_sff] (root,0,0,0.0,412) [scsi_eh_0] (root,0,0,0.0,413) [scsi_tmf_0] (root,0,0,0.0,414) [scsi_eh_1] (root,0,0,0.0,415) [scsi_tmf_1] (root,0,0,0.0,416) [scsi_eh_2] (root,0,0,0.0,417) [scsi_tmf_2] (root,0,0,0.0,418) [scsi_eh_3] (root,0,0,0.0,419) [scsi_tmf_3] (root,0,0,0.0,423) [scsi_eh_4] (root,0,0,0.0,424) [scsi_tmf_4] (root,0,0,0.0,425) [scsi_eh_5] (root,0,0,0.0,426) [scsi_tmf_5] (root,0,0,0.0,427) [scsi_eh_6] (root,0,0,0.0,428) [scsi_tmf_6] (root,0,0,0.0,429) [scsi_eh_7] (root,0,0,0.0,430) [scsi_tmf_7] (root,0,0,0.0,431) [scsi_eh_8] (root,0,0,0.0,432) [scsi_tmf_8] (root,0,0,0.0,433) [scsi_eh_9] (root,0,0,0.0,434) [scsi_tmf_9] (root,0,0,0.0,442) [ttm_swap] (root,0,0,0.0,543) [kdmflush] (root,0,0,0.0,544) [bioset] (root,0,0,0.0,554) [kdmflush] (root,0,0,0.0,555) [bioset] (root,0,0,0.0,558) [kworker/0:1H] (root,0,0,0.0,572) [jbd2/dm-0-8] (root,0,0,0.0,573) [ext4-rsv-conver] (root,105740,53224,0.0,653) /usr/lib/systemd/systemd-journald (root,0,0,0.0,663) [kworker/10:1] (root,45604,656,0.0,679) /usr/lib/systemd/systemd-udevd (root,0,0,1.4,730) [kipmi0] (root,0,0,0.0,735) [kworker/14:1H] (root,0,0,0.0,737) [kworker/6:1H] (root,0,0,0.0,738) [kworker/5:1H] (root,0,0,0.0,805) [kvm-irqfd-clean] (root,198572,564,0.0,815) /usr/sbin/lvmetad -f (root,0,0,0.0,868) [kworker/1:1H] (root,0,0,0.0,872) [kworker/2:1H] (root,0,0,0.0,880) [kdmflush] (root,0,0,0.0,881) [bioset] (root,0,0,0.0,883) [kdmflush] (root,0,0,0.0,884) [bioset] (root,0,0,0.0,885) [kdmflush] (root,0,0,0.0,887) [bioset] (root,0,0,0.0,889) [kdmflush] (root,0,0,0.0,890) [bioset] (root,0,0,0.0,897) [kworker/13:1H] (root,0,0,0.0,898) [kworker/7:1H] (root,0,0,0.0,899) [kworker/15:1H] (root,0,0,0.0,907) [jbd2/sda1-8] (root,0,0,0.0,908) [ext4-rsv-conver] (root,0,0,0.0,929) [kworker/9:1H] (root,0,0,0.0,930) [kworker/12:1H] (root,0,0,0.0,931) [kworker/4:1H] (root,0,0,0.0,938) [jbd2/dm-5-8] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,941) [ext4-rsv-conver] (root,0,0,0.0,947) [jbd2/dm-4-8] (root,0,0,0.0,948) [ext4-rsv-conver] (root,0,0,0.0,951) [jbd2/dm-2-8] (root,0,0,0.0,952) [ext4-rsv-conver] (root,0,0,0.0,976) [nv_queue] (root,0,0,0.0,977) [nv_queue] (polkitd,612240,1092,0.0,1012) /usr/lib/polkit-1/polkitd --no-debug (root,52900,592,0.0,1015) /usr/sbin/smartd -n -q never (root,21788,508,0.0,1027) /usr/sbin/irqbalance --foreground (dbus,58220,900,0.0,1034) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (root,26620,1244,0.0,1057) /usr/lib/systemd/systemd-logind (root,0,0,0.0,1099) [UVM global queu] (root,0,0,0.0,1100) [UVM deferred re] (root,0,0,0.0,1101) [UVM Tools Event] (root,0,0,0.0,1132) [nvidia-modeset/] (root,0,0,0.0,1133) [nvidia-modeset/] (root,0,0,0.0,1262) [bond0] (root,10058840,112276,0.0,1292) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,302152,39092,0.0,1570) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,574288,2332,0.0,1574) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,113000,8,0.0,1576) /usr/sbin/sshd -D (root,0,0,0.0,1581) [irq/98-nvidia] (root,0,0,0.0,1582) [nvidia] (root,0,0,0.2,1583) [nv_queue] (root,0,0,0.0,1702) [kworker/5:2] (root,799816,1904,0.0,1823) /usr/sbin/collectd (root,740624,30444,0.0,1845) /usr/sbin/rsyslogd -n (root,591424,5628,0.1,1871) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,115700,964,0.0,1874) /bin/bash /sbin/condor_inspector (root,209064,1708,0.0,1887) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,1232,0.0,1906) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1913) [irq/99-nvidia] (root,0,0,0.0,1914) [nvidia] (root,0,0,0.2,1915) [nv_queue] (root,209332,2228,0.0,1920) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,89804,228,0.0,2118) /usr/libexec/postfix/master -w (postfix,90080,780,0.0,2121) qmgr -l -t unix -u (cvmfs,73348,2784,0.0,2130) /usr/bin/cvmfs2 __cachemgr__ . 7 8 41943040000 20971520000 0 3 -1 : (cvmfs,60968,0,0.0,2134) /usr/bin/cvmfs2 __cachemgr__ . 7 8 41943040000 20971520000 0 3 -1 : (cvmfs,942668,7792,0.0,2210) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,92204,8,0.0,2214) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2330) [irq/100-nvidia] (root,0,0,0.0,2331) [nvidia] (root,0,0,0.1,2332) [nv_queue] (root,0,0,0.0,2406) [kworker/10:0] (cvmfs,790996,5296,0.0,2588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,92152,4,0.0,2592) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,729532,4932,0.0,2607) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,92204,4,0.0,2611) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,729532,2260,0.0,2626) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,92204,4,0.0,2630) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,791080,4304,0.0,2645) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,92204,28,0.0,2649) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,791036,5480,0.0,2664) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,92204,4,0.0,2668) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,790984,5532,0.0,2757) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,92152,4,0.0,2761) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,5089696,133404,0.0,2792) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,92208,112,0.0,2796) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (root,0,0,0.0,2817) [irq/101-nvidia] (root,0,0,0.0,2818) [nvidia] (root,0,0,0.1,2819) [nv_queue] (cvmfs,72164,1780,0.0,2822) /usr/bin/cvmfs2 __cachemgr__ . 7 8 1048576000 524288000 0 3 -1 : (cvmfs,60968,0,0.0,2826) /usr/bin/cvmfs2 __cachemgr__ . 7 8 1048576000 524288000 0 3 -1 : (cvmfs,790912,7404,0.0,2830) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,92204,60,0.0,2834) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,791036,5708,0.0,2850) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,92204,16,0.0,2854) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,2870) [irq/102-nvidia] (root,0,0,0.0,2871) [nvidia] (root,0,0,0.1,2872) [nv_queue] (root,126388,432,0.0,2874) /usr/sbin/crond -n (condor,75276,3080,0.0,2875) /usr/sbin/condor_master -f (ntp,29960,776,0.0,2877) /usr/sbin/ntpd -u ntp:ntp -g (root,27828,4744,0.0,2878) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (root,0,0,0.0,2879) [irq/103-nvidia] (root,0,0,0.0,2880) [nvidia] (root,0,0,0.1,2881) [nv_queue] (condor,63876,2700,0.0,2882) condor_shared_port -f (root,0,0,0.0,2883) [kworker/10:1H] (root,0,0,0.0,2884) [irq/104-nvidia] (root,0,0,0.0,2885) [nvidia] (root,0,0,0.2,2886) [nv_queue] (condor,2399184,2324080,0.1,2887) condor_startd -f (root,0,0,0.0,2908) [irq/105-nvidia] (root,0,0,0.0,2909) [nvidia] (root,0,0,0.2,2910) [nv_queue] (root,110208,8,0.0,3014) /sbin/agetty --noclear tty1 linux (root,0,0,0.0,3632) [kworker/1:1] (condor,34082884,508,0.1,3821) GPUs_MONITOR (root,0,0,0.0,3823) [kworker/u32:1] (root,0,0,0.0,3824) [UVM GPU1 BH] (root,0,0,0.0,3825) [UVM GPU2 BH] (root,0,0,0.0,3827) [UVM GPU3 BH] (root,0,0,0.0,3828) [UVM GPU4 BH] (root,0,0,0.0,3830) [UVM GPU5 BH] (root,0,0,0.0,3831) [UVM GPU6 BH] (root,0,0,43.6,3832) [UVM GPU7 BH] (root,0,0,0.0,3833) [UVM GPU8 BH] (root,0,0,0.0,4069) [kworker/8:1H] (root,0,0,0.0,4366) [kworker/3:1H] (root,0,0,0.0,4626) [kworker/9:1] (root,0,0,0.0,5244) [kworker/2:0] (root,0,0,0.0,5361) [kworker/4:1] (root,27168,1056,0.0,5419) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,108056,356,0.0,5453) sleep 15 (root,8632,660,0.0,5454) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,5455) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,5471) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,5472) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (ganglia,220196,8408,0.0,6311) /usr/sbin/gmond (slot-1,10903860,656756,0.6,6554) python /scratch/condor/dir_21435/glidein/execute.bde4dab8-997d-11ed-81a0-0cc47a857a10/dir_5868/condor_exec.exe -i /data/ana/BSM/HNL/MC/190611/Gen/03001-04000/Gen_03934.i3.zst -o /data/ana/BSM/HNL/MC/190611/Phot/03001-04000/Phot_03934.i3.zst -r 1 -l 3934 -t -m spice_3.2.1 --holeice angsens/as.flasher_p1_0.30_p2_-1 --osg True (root,0,0,0.0,7285) [kworker/u32:2] (root,0,0,0.0,7343) [kworker/6:0] (root,0,0,0.0,7896) [kworker/11:1H] (root,0,0,0.0,8170) [kworker/8:1] (root,0,0,0.0,8678) [kworker/10:2] (root,0,0,0.0,9406) [kworker/11:0] (root,0,0,0.0,9974) [kworker/0:2] (root,0,0,0.0,10704) [kworker/13:2] (condor,67120,8192,0.0,12927) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (slot-1,11692,1572,0.0,12946) /bin/sh glidein_start.sh (slot-1,61812,6540,0.0,13268) glideinExec/sbin/condor_master -f -r 20135 (slot-1,11796,1648,0.0,13269) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_f4a462e7-ac13-4738-aa77-4a911fd4f608.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20230216%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230216T153003Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=be70c88d2c0111e01891fea452edadd7462dc960e030e91f45ee4cbf8ef94214 (slot-1,24684,4420,0.0,13271) condor_procd -A /scratch/condor/dir_12927/glidein/log.f02ec7ea-ae0e-11ed-990f-0cc47a857a10/procd_address -L /scratch/condor/dir_12927/glidein/log.f02ec7ea-ae0e-11ed-990f-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,63236,8348,0.0,13272) condor_startd -f (root,0,0,0.0,13607) [kworker/14:0] (root,0,0,0.0,14273) [kworker/4:0] (slot-1,96856,8608,0.0,14937) condor_starter -f -a slot1_7 sub-1.icecube.wisc.edu (root,0,0,0.0,14975) [kworker/9:0] (slot-1,17150796,6863848,1.8,14989) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000136_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141156/NuMu_B_141156_000136_step2.i3.zst --filenr 136 --tmpdir /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_14769/2023-02-02_02-26-39 --gridftp --runnumber 141156 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,11704,1512,0.0,15257) bash /scratch/condor/dir_12927/glidein/execute.f02ec7ea-ae0e-11ed-990f-0cc47a857a10/dir_14937/condor_exec.exe (slot-1,11832,1544,0.0,15366) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000776_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141157/NuMu_B_141157_000776_step2.i3.zst --filenr 776 --tmpdir /scratch/condor/dir_12927/glidein/execute.f02ec7ea-ae0e-11ed-990f-0cc47a857a10/dir_14937/2023-02-16_08-41-23 --gridftp --runnumber 141157 --icemodel spice_3.2.1 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling --geant4-muons --particle-history (slot-1,13878336,3632588,8.3,15370) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000776_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141157/NuMu_B_141157_000776_step2.i3.zst --filenr 776 --tmpdir /scratch/condor/dir_12927/glidein/execute.f02ec7ea-ae0e-11ed-990f-0cc47a857a10/dir_14937/2023-02-16_08-41-23 --gridftp --runnumber 141157 --icemodel spice_3.2.1 --efficiency 1.2 --crossenergy 30 -t --simple-hqe-scaling --geant4-muons --particle-history (root,0,0,0.0,15590) [kworker/15:0] (root,0,0,0.0,16107) [kworker/1:2] (root,0,0,0.0,17727) [kworker/2:2] (root,0,0,0.0,18720) [kworker/5:0] (root,0,0,0.0,18820) [kworker/u34:1] (root,0,0,0.0,19199) [kworker/15:1] (root,0,0,0.0,21235) [kworker/13:1] (root,0,0,0.0,21297) [kworker/u32:0] (root,0,0,0.0,21368) [kworker/12:0] (root,0,0,0.0,21446) [kworker/0:0] (root,0,0,0.0,21518) [kworker/u33:1] (root,0,0,0.0,21873) [kworker/3:2] (root,0,0,0.0,22139) [kworker/12:1] (postfix,89908,4024,0.0,22317) pickup -l -t unix -u (root,0,0,0.0,26167) [kworker/3:0] (root,0,0,0.0,26319) [kworker/14:2] (root,0,0,0.0,26966) [kworker/11:1] (root,0,0,0.0,27988) [kworker/7:1] (root,0,0,0.0,28023) [kworker/4:2] (root,0,0,0.0,29682) [kworker/u33:0] (slot-1,0,0,0.0,29724) [python] <defunct> (root,0,0,0.0,29883) [kworker/u34:2] (slot-1,0,0,0.0,29885) [zstd] <defunct> (root,0,0,0.0,29948) [kworker/6:1] (slot-1,4368,356,0.0,30248) sleep 300 (root,0,0,0.0,30607) [kworker/8:0] (root,0,0,0.0,30754) [kworker/7:2]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b1a45a4f05aae993da79c0f5f8
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2.chtc.wisc.edu AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43800,3108,0.0,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.2,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,259) [kauditd] (root,0,0,0.0,411) [ata_sff] (root,0,0,0.0,412) [scsi_eh_0] (root,0,0,0.0,413) [scsi_tmf_0] (root,0,0,0.0,414) [scsi_eh_1] (root,0,0,0.0,415) [scsi_tmf_1] (root,0,0,0.0,416) [scsi_eh_2] (root,0,0,0.0,417) [scsi_tmf_2] (root,0,0,0.0,418) [scsi_eh_3] (root,0,0,0.0,419) [scsi_tmf_3] (root,0,0,0.0,423) [scsi_eh_4] (root,0,0,0.0,424) [scsi_tmf_4] (root,0,0,0.0,425) [scsi_eh_5] (root,0,0,0.0,426) [scsi_tmf_5] (root,0,0,0.0,427) [scsi_eh_6] (root,0,0,0.0,428) [scsi_tmf_6] (root,0,0,0.0,429) [scsi_eh_7] (root,0,0,0.0,430) [scsi_tmf_7] (root,0,0,0.0,431) [scsi_eh_8] (root,0,0,0.0,432) [scsi_tmf_8] (root,0,0,0.0,433) [scsi_eh_9] (root,0,0,0.0,434) [scsi_tmf_9] (root,0,0,0.0,442) [ttm_swap] (root,0,0,0.0,543) [kdmflush] (root,0,0,0.0,544) [bioset] (root,0,0,0.0,554) [kdmflush] (root,0,0,0.0,555) [bioset] (root,0,0,0.0,558) [kworker/0:1H] (root,0,0,0.0,572) [jbd2/dm-0-8] (root,0,0,0.0,573) [ext4-rsv-conver] (root,81164,40172,0.0,653) /usr/lib/systemd/systemd-journald (root,45604,656,0.0,679) /usr/lib/systemd/systemd-udevd (root,0,0,1.5,730) [kipmi0] (root,0,0,0.0,735) [kworker/14:1H] (root,0,0,0.0,737) [kworker/6:1H] (root,0,0,0.0,738) [kworker/5:1H] (root,0,0,0.0,805) [kvm-irqfd-clean] (root,198572,564,0.0,815) /usr/sbin/lvmetad -f (root,0,0,0.0,868) [kworker/1:1H] (root,0,0,0.0,872) [kworker/2:1H] (root,0,0,0.0,880) [kdmflush] (root,0,0,0.0,881) [bioset] (root,0,0,0.0,883) [kdmflush] (root,0,0,0.0,884) [bioset] (root,0,0,0.0,885) [kdmflush] (root,0,0,0.0,887) [bioset] (root,0,0,0.0,889) [kdmflush] (root,0,0,0.0,890) [bioset] (root,0,0,0.0,897) [kworker/13:1H] (root,0,0,0.0,898) [kworker/7:1H] (root,0,0,0.0,899) [kworker/15:1H] (root,0,0,0.0,907) [jbd2/sda1-8] (root,0,0,0.0,908) [ext4-rsv-conver] (root,0,0,0.0,929) [kworker/9:1H] (root,0,0,0.0,930) [kworker/12:1H] (root,0,0,0.0,931) [kworker/4:1H] (root,0,0,0.0,938) [jbd2/dm-5-8] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,941) [ext4-rsv-conver] (root,0,0,0.0,947) [jbd2/dm-4-8] (root,0,0,0.0,948) [ext4-rsv-conver] (root,0,0,0.0,951) [jbd2/dm-2-8] (root,0,0,0.0,952) [ext4-rsv-conver] (root,0,0,0.0,976) [nv_queue] (root,0,0,0.0,977) [nv_queue] (polkitd,612240,1092,0.0,1012) /usr/lib/polkit-1/polkitd --no-debug (root,52900,592,0.0,1015) /usr/sbin/smartd -n -q never (root,21788,508,0.0,1027) /usr/sbin/irqbalance --foreground (dbus,58220,900,0.0,1034) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (root,26620,1244,0.0,1057) /usr/lib/systemd/systemd-logind (root,0,0,0.0,1099) [UVM global queu] (root,0,0,0.0,1100) [UVM deferred re] (root,0,0,0.0,1101) [UVM Tools Event] (root,0,0,0.0,1132) [nvidia-modeset/] (root,0,0,0.0,1133) [nvidia-modeset/] (root,0,0,0.0,1217) [kworker/4:1] (root,0,0,0.0,1262) [bond0] (root,0,0,0.0,1278) [kworker/5:1] (root,10058476,110344,0.0,1292) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,302152,35948,0.0,1570) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,574288,2332,0.0,1574) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,113000,8,0.0,1576) /usr/sbin/sshd -D (root,0,0,0.0,1581) [irq/98-nvidia] (root,0,0,0.0,1582) [nvidia] (root,0,0,0.2,1583) [nv_queue] (root,799816,1900,0.0,1823) /usr/sbin/collectd (root,658768,22948,0.0,1845) /usr/sbin/rsyslogd -n (root,591424,5616,0.1,1871) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,115700,964,0.0,1874) /bin/bash /sbin/condor_inspector (root,209064,1668,0.0,1887) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,1288,0.0,1906) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1913) [irq/99-nvidia] (root,0,0,0.0,1914) [nvidia] (root,0,0,0.2,1915) [nv_queue] (root,209332,2228,0.0,1920) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,89804,228,0.0,2118) /usr/libexec/postfix/master -w (postfix,90080,780,0.0,2121) qmgr -l -t unix -u (cvmfs,73348,2784,0.0,2130) /usr/bin/cvmfs2 __cachemgr__ . 7 8 41943040000 20971520000 0 3 -1 : (cvmfs,60968,0,0.0,2134) /usr/bin/cvmfs2 __cachemgr__ . 7 8 41943040000 20971520000 0 3 -1 : (cvmfs,942668,9304,0.0,2210) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,92204,8,0.0,2214) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2262) [kworker/14:2] (root,0,0,0.0,2330) [irq/100-nvidia] (root,0,0,0.0,2331) [nvidia] (root,0,0,0.1,2332) [nv_queue] (cvmfs,791028,5096,0.0,2588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,92152,4,0.0,2592) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,729532,3248,0.0,2607) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,92204,4,0.0,2611) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,727624,3420,0.0,2626) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,92204,4,0.0,2630) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,792152,4684,0.0,2645) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,92204,28,0.0,2649) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,791036,5304,0.0,2664) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,92204,4,0.0,2668) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,790996,5392,0.0,2757) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,92152,4,0.0,2761) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,5089700,124204,0.0,2792) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,92208,112,0.0,2796) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (root,0,0,0.0,2817) [irq/101-nvidia] (root,0,0,0.0,2818) [nvidia] (root,0,0,0.1,2819) [nv_queue] (cvmfs,72164,1780,0.0,2822) /usr/bin/cvmfs2 __cachemgr__ . 7 8 1048576000 524288000 0 3 -1 : (cvmfs,60968,0,0.0,2826) /usr/bin/cvmfs2 __cachemgr__ . 7 8 1048576000 524288000 0 3 -1 : (cvmfs,790900,7276,0.0,2830) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,92204,60,0.0,2834) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,791036,5672,0.0,2850) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,92204,16,0.0,2854) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,2870) [irq/102-nvidia] (root,0,0,0.0,2871) [nvidia] (root,0,0,0.1,2872) [nv_queue] (root,126388,432,0.0,2874) /usr/sbin/crond -n (condor,75276,3080,0.0,2875) /usr/sbin/condor_master -f (ntp,29960,776,0.0,2877) /usr/sbin/ntpd -u ntp:ntp -g (root,27828,4744,0.0,2878) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (root,0,0,0.0,2879) [irq/103-nvidia] (root,0,0,0.0,2880) [nvidia] (root,0,0,0.1,2881) [nv_queue] (condor,63816,2668,0.0,2882) condor_shared_port -f (root,0,0,0.0,2883) [kworker/10:1H] (root,0,0,0.0,2884) [irq/104-nvidia] (root,0,0,0.0,2885) [nvidia] (root,0,0,0.2,2886) [nv_queue] (condor,1922744,1847440,0.1,2887) condor_startd -f (root,0,0,0.0,2908) [irq/105-nvidia] (root,0,0,0.0,2909) [nvidia] (root,0,0,0.2,2910) [nv_queue] (root,110208,8,0.0,3014) /sbin/agetty --noclear tty1 linux (slot-1,46168,6584,0.0,3728) condor_starter -f -a slot1_5 sub-1.icecube.wisc.edu (slot-1,11832,1544,0.0,3737) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/lfischer/oscnext_hnl_meta/build//env-shell.sh python /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_3728/condor_exec.exe -i /data/ana/BSM/HNL/MC/190610/Gen/07001-08000/Gen_07395.i3.zst -o /data/ana/BSM/HNL/MC/190610_work_around_directory_permission_issue/Phot/07001-08000/Phot_07395.i3.zst -r 1 -l 7395 -t -m spice_3.2.1 --holeice angsens/as.flasher_p1_0.30_p2_-1 --osg True (condor,34082884,508,0.2,3821) GPUs_MONITOR (root,0,0,0.0,3824) [UVM GPU1 BH] (root,0,0,0.0,3825) [UVM GPU2 BH] (root,0,0,0.0,3827) [UVM GPU3 BH] (root,0,0,0.0,3828) [UVM GPU4 BH] (root,0,0,0.0,3830) [UVM GPU5 BH] (root,0,0,0.0,3831) [UVM GPU6 BH] (root,0,0,29.2,3832) [UVM GPU7 BH] (root,0,0,0.0,3833) [UVM GPU8 BH] (slot-1,12618872,632808,105,3869) python /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_3728/condor_exec.exe -i /data/ana/BSM/HNL/MC/190610/Gen/07001-08000/Gen_07395.i3.zst -o /data/ana/BSM/HNL/MC/190610_work_around_directory_permission_issue/Phot/07001-08000/Phot_07395.i3.zst -r 1 -l 7395 -t -m spice_3.2.1 --holeice angsens/as.flasher_p1_0.30_p2_-1 --osg True (root,0,0,0.0,3878) [kworker/13:2] (root,0,0,0.0,4069) [kworker/8:1H] (root,0,0,0.0,4366) [kworker/3:1H] (root,0,0,0.0,4730) [kworker/6:2] (root,0,0,0.0,5025) [kworker/3:0] (root,27168,1056,0.0,5419) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (ganglia,220196,8408,0.0,6311) /usr/sbin/gmond (slot-1,10903860,656756,1.8,6554) python /scratch/condor/dir_21435/glidein/execute.bde4dab8-997d-11ed-81a0-0cc47a857a10/dir_5868/condor_exec.exe -i /data/ana/BSM/HNL/MC/190611/Gen/03001-04000/Gen_03934.i3.zst -o /data/ana/BSM/HNL/MC/190611/Phot/03001-04000/Phot_03934.i3.zst -r 1 -l 3934 -t -m spice_3.2.1 --holeice angsens/as.flasher_p1_0.30_p2_-1 --osg True (root,0,0,0.0,7163) [kworker/1:0] (root,0,0,0.0,7319) [kworker/11:1] (root,0,0,0.0,7896) [kworker/11:1H] (root,0,0,0.0,9111) [kworker/10:1] (root,0,0,0.0,9172) [kworker/u32:2] (root,0,0,0.0,9269) [kworker/9:1] (postfix,89908,4028,0.0,9658) pickup -l -t unix -u (slot-1,46156,6564,0.0,9763) condor_starter -f -a slot1_9 sub-1.icecube.wisc.edu (slot-1,11704,1528,0.0,9769) bash /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_9763/condor_exec.exe (slot-1,11832,1528,0.0,9845) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/Oscillation/software/oscNext_meta/releases/V01-00-07/build__py2-v3.1.1__osgvo-el7/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/Oscillation/software/oscNext_meta/releases/V01-00-07/src/oscNext/resources/scripts/run_oscNext.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --inputfile /data/ana/LE/oscNext//pass2/genie/level5_v02.00/141156/oscNext_genie_level5_v02.00_pass2.141156.000214_split/oscNext_genie_level5_v02.00_pass2.141156.000214__split_000031.i3.zst --outputfile /data/ana/LE/oscNext//pass2/genie/level6_v02.00/141156/oscNext_genie_level6_v02.00_pass2.141156.000214_split/oscNext_genie_level6_v02.00_pass2.141156.000214__split_000031.i3.zst --file-num 214 --tmp-dir /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_9763/2023-02-07_23-55-35 --gridftp --metadata data_type:genie stage:level6 file_number:214 pass_num:2 run_id:0 --levels 6 --datatype genie --dataset 141156 --fix-sim-headers (slot-1,7747428,6634440,99.5,9853) python /cvmfs/icecube.opensciencegrid.org/users/Oscillation/software/oscNext_meta/releases/V01-00-07/src/oscNext/resources/scripts/run_oscNext.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --inputfile /data/ana/LE/oscNext//pass2/genie/level5_v02.00/141156/oscNext_genie_level5_v02.00_pass2.141156.000214_split/oscNext_genie_level5_v02.00_pass2.141156.000214__split_000031.i3.zst --outputfile /data/ana/LE/oscNext//pass2/genie/level6_v02.00/141156/oscNext_genie_level6_v02.00_pass2.141156.000214_split/oscNext_genie_level6_v02.00_pass2.141156.000214__split_000031.i3.zst --file-num 214 --tmp-dir /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_9763/2023-02-07_23-55-35 --gridftp --metadata data_type:genie stage:level6 file_number:214 pass_num:2 run_id:0 --levels 6 --datatype genie --dataset 141156 --fix-sim-headers (slot-1,46168,6576,0.0,9986) condor_starter -f -a slot1_4 sub-1.icecube.wisc.edu (slot-1,11832,1544,0.0,9991) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/lfischer/oscnext_hnl_meta/build//env-shell.sh python /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_9986/condor_exec.exe -i /data/ana/BSM/HNL/MC/190610/Gen/09001-10000/Gen_09240.i3.zst -o /data/ana/BSM/HNL/MC/190610_work_around_directory_permission_issue/Phot/09001-10000/Phot_09240.i3.zst -r 1 -l 9240 -t -m spice_3.2.1 --holeice angsens/as.flasher_p1_0.30_p2_-1 --osg True (slot-1,12615972,636068,105,10109) python /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_9986/condor_exec.exe -i /data/ana/BSM/HNL/MC/190610/Gen/09001-10000/Gen_09240.i3.zst -o /data/ana/BSM/HNL/MC/190610_work_around_directory_permission_issue/Phot/09001-10000/Phot_09240.i3.zst -r 1 -l 9240 -t -m spice_3.2.1 --holeice angsens/as.flasher_p1_0.30_p2_-1 --osg True (root,0,0,0.0,10115) [kworker/u32:1] (root,0,0,0.0,10706) [kworker/12:0] (slot-1,46168,6588,0.0,11211) condor_starter -f -a slot1_3 sub-1.icecube.wisc.edu (slot-1,11832,1552,0.0,11216) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/lfischer/oscnext_hnl_meta/build//env-shell.sh python /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_11211/condor_exec.exe -i /data/ana/BSM/HNL/MC/190610/Gen/09001-10000/Gen_09275.i3.zst -o /data/ana/BSM/HNL/MC/190610_work_around_directory_permission_issue/Phot/09001-10000/Phot_09275.i3.zst -r 1 -l 9275 -t -m spice_3.2.1 --holeice angsens/as.flasher_p1_0.30_p2_-1 --osg True (slot-1,12667452,626244,105,11334) python /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_11211/condor_exec.exe -i /data/ana/BSM/HNL/MC/190610/Gen/09001-10000/Gen_09275.i3.zst -o /data/ana/BSM/HNL/MC/190610_work_around_directory_permission_issue/Phot/09001-10000/Phot_09275.i3.zst -r 1 -l 9275 -t -m spice_3.2.1 --holeice angsens/as.flasher_p1_0.30_p2_-1 --osg True (root,0,0,0.0,12407) [kworker/13:0] (root,0,0,0.0,12601) [kworker/10:2] (root,113284,1488,0.0,12663) /bin/bash /bin/run-parts /etc/cron.daily (root,352000,21852,0.0,12667) /usr/bin/python -tt /usr/sbin/yum-cron (root,113644,976,0.0,12668) awk -v progname=/etc/cron.daily/0yum-daily.cron progname { .... print progname ":\n" .... progname=""; ... } ... { print; } (root,0,0,0.0,13031) [kworker/6:1] (root,0,0,0.0,13895) [kworker/8:2] (root,0,0,0.0,14337) [kworker/0:2] (slot-1,46116,6560,0.0,14585) condor_starter -f -a slot1_2 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1568,0.0,14629) /bin/sh /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_14585/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,96864,8632,0.0,14769) condor_starter -f -a slot1_7 sub-1.icecube.wisc.edu (slot-1,11704,1516,0.0,14859) bash /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_14769/condor_exec.exe (slot-1,4372,408,0.0,14942) sleep 599 (slot-1,11832,1544,0.0,14985) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000136_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141156/NuMu_B_141156_000136_step2.i3.zst --filenr 136 --tmpdir /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_14769/2023-02-02_02-26-39 --gridftp --runnumber 141156 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (slot-1,17150796,6863848,7.3,14989) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_B_140000_000136_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141156/NuMu_B_141156_000136_step2.i3.zst --filenr 136 --tmpdir /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_14769/2023-02-02_02-26-39 --gridftp --runnumber 141156 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 30 -t --geant4-muons --particle-history (root,0,0,0.0,16131) [kworker/u33:2] (root,0,0,0.0,16176) [kworker/14:1] (root,0,0,0.0,16445) [kworker/u34:1] (slot-1,46168,6580,0.0,16685) condor_starter -f -a slot1_8 sub-1.icecube.wisc.edu (slot-1,11832,1552,0.0,16689) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/lfischer/oscnext_hnl_meta/build//env-shell.sh python /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_16685/condor_exec.exe -i /data/ana/BSM/HNL/MC/190610/Gen/07001-08000/Gen_07031.i3.zst -o /data/ana/BSM/HNL/MC/190610_work_around_directory_permission_issue/Phot/07001-08000/Phot_07031.i3.zst -r 1 -l 7031 -t -m spice_3.2.1 --holeice angsens/as.flasher_p1_0.30_p2_-1 --osg True (slot-1,12622216,643712,105,16808) python /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_16685/condor_exec.exe -i /data/ana/BSM/HNL/MC/190610/Gen/07001-08000/Gen_07031.i3.zst -o /data/ana/BSM/HNL/MC/190610_work_around_directory_permission_issue/Phot/07001-08000/Phot_07031.i3.zst -r 1 -l 7031 -t -m spice_3.2.1 --holeice angsens/as.flasher_p1_0.30_p2_-1 --osg True (slot-1,4368,360,0.0,19056) sleep 300 (root,0,0,0.0,19494) [kworker/15:2] (root,0,0,0.0,19495) [kworker/0:0] (root,0,0,0.0,19533) [kworker/5:2] (root,0,0,0.0,20183) [kworker/u32:0] (root,0,0,0.0,20423) [kworker/7:2] (root,125488,1136,0.0,20471) /usr/sbin/anacron -s (root,0,0,0.0,20539) [kworker/2:2] (root,0,0,0.0,20566) [kworker/4:0] (root,0,0,0.0,20723) [kworker/15:1] (root,0,0,0.0,22953) [kworker/12:2] (root,0,0,0.0,23177) [kworker/13:1] (root,0,0,0.0,23684) [kworker/1:1] (root,0,0,0.0,24848) [kworker/2:1] (slot-1,46168,6588,0.0,25191) condor_starter -f -a slot1_6 sub-1.icecube.wisc.edu (slot-1,11832,1552,0.0,25196) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/lfischer/oscnext_hnl_meta/build//env-shell.sh python /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_25191/condor_exec.exe -i /data/ana/BSM/HNL/MC/190610/Gen/06001-07000/Gen_06278.i3.zst -o /data/ana/BSM/HNL/MC/190610_work_around_directory_permission_issue/Phot/06001-07000/Phot_06278.i3.zst -r 1 -l 6278 -t -m spice_3.2.1 --holeice angsens/as.flasher_p1_0.30_p2_-1 --osg True (slot-1,12603252,698416,105,25314) python /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_25191/condor_exec.exe -i /data/ana/BSM/HNL/MC/190610/Gen/06001-07000/Gen_06278.i3.zst -o /data/ana/BSM/HNL/MC/190610_work_around_directory_permission_issue/Phot/06001-07000/Phot_06278.i3.zst -r 1 -l 6278 -t -m spice_3.2.1 --holeice angsens/as.flasher_p1_0.30_p2_-1 --osg True (root,0,0,0.0,25603) [kworker/7:1] (root,0,0,0.0,26369) [kworker/u34:0] (root,0,0,0.0,27020) [kworker/1:2] (condor,67120,8188,0.0,27173) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (slot-1,11692,1576,0.0,27192) /bin/sh glidein_start.sh (slot-1,61812,6548,0.0,27254) glideinExec/sbin/condor_master -f -r 20135 (slot-1,11896,1656,0.0,27255) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_9cb90bf5-cb85-4da7-a3f3-5a4039a3c550.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20230204%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230204T110002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=b9f3f1aee8638d37aafe3ce876e19df9089e7bce384d784a3bfac9a5e9513c71 (slot-1,25232,5444,0.1,27257) condor_procd -A /scratch/condor/dir_27173/glidein/log.44bbaef6-a47b-11ed-8320-0cc47a857a10/procd_address -L /scratch/condor/dir_27173/glidein/log.44bbaef6-a47b-11ed-8320-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,64628,9604,0.0,27258) condor_startd -f (root,108056,356,0.0,27272) sleep 15 (root,8632,660,0.0,27280) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,27281) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,27297) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,27298) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,28314) [kworker/5:0] (root,0,0,0.0,28457) [kworker/u33:0] (root,0,0,0.0,29095) [kworker/9:0] (root,0,0,0.0,29102) [kworker/7:0] (slot-1,0,0,0.2,29724) [python] <defunct> (slot-1,0,0,0.0,29885) [zstd] <defunct> (slot-1,46168,6580,0.0,30018) condor_starter -f -a slot1_1 sub-1.icecube.wisc.edu (slot-1,11832,1548,0.0,30022) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/lfischer/oscnext_hnl_meta/build//env-shell.sh python /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_30018/condor_exec.exe -i /data/ana/BSM/HNL/MC/190610/Gen/07001-08000/Gen_07286.i3.zst -o /data/ana/BSM/HNL/MC/190610_work_around_directory_permission_issue/Phot/07001-08000/Phot_07286.i3.zst -r 1 -l 7286 -t -m spice_3.2.1 --holeice angsens/as.flasher_p1_0.30_p2_-1 --osg True (slot-1,12654496,679644,105,30141) python /scratch/condor/dir_27173/glidein/execute.44bbaef6-a47b-11ed-8320-0cc47a857a10/dir_30018/condor_exec.exe -i /data/ana/BSM/HNL/MC/190610/Gen/07001-08000/Gen_07286.i3.zst -o /data/ana/BSM/HNL/MC/190610_work_around_directory_permission_issue/Phot/07001-08000/Phot_07286.i3.zst -r 1 -l 7286 -t -m spice_3.2.1 --holeice angsens/as.flasher_p1_0.30_p2_-1 --osg True (root,0,0,0.0,30495) [kworker/3:1] (root,0,0,0.0,30901) [kworker/11:2] (root,0,0,0.0,30903) [kworker/8:1]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b1a45a4f05aae993dadfc36d86
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2.chtc.wisc.edu AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43800,3108,0.0,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.2,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,259) [kauditd] (root,0,0,0.0,411) [ata_sff] (root,0,0,0.0,412) [scsi_eh_0] (root,0,0,0.0,413) [scsi_tmf_0] (root,0,0,0.0,414) [scsi_eh_1] (root,0,0,0.0,415) [scsi_tmf_1] (root,0,0,0.0,416) [scsi_eh_2] (root,0,0,0.0,417) [scsi_tmf_2] (root,0,0,0.0,418) [scsi_eh_3] (root,0,0,0.0,419) [scsi_tmf_3] (root,0,0,0.0,423) [scsi_eh_4] (root,0,0,0.0,424) [scsi_tmf_4] (root,0,0,0.0,425) [scsi_eh_5] (root,0,0,0.0,426) [scsi_tmf_5] (root,0,0,0.0,427) [scsi_eh_6] (root,0,0,0.0,428) [scsi_tmf_6] (root,0,0,0.0,429) [scsi_eh_7] (root,0,0,0.0,430) [scsi_tmf_7] (root,0,0,0.0,431) [scsi_eh_8] (root,0,0,0.0,432) [scsi_tmf_8] (root,0,0,0.0,433) [scsi_eh_9] (root,0,0,0.0,434) [scsi_tmf_9] (root,0,0,0.0,442) [ttm_swap] (root,0,0,0.0,543) [kdmflush] (root,0,0,0.0,544) [bioset] (root,0,0,0.0,554) [kdmflush] (root,0,0,0.0,555) [bioset] (root,0,0,0.0,558) [kworker/0:1H] (root,0,0,0.0,572) [jbd2/dm-0-8] (root,0,0,0.0,573) [ext4-rsv-conver] (root,116916,66436,0.0,653) /usr/lib/systemd/systemd-journald (root,45604,656,0.0,679) /usr/lib/systemd/systemd-udevd (root,0,0,1.5,730) [kipmi0] (root,0,0,0.0,735) [kworker/14:1H] (root,0,0,0.0,737) [kworker/6:1H] (root,0,0,0.0,738) [kworker/5:1H] (root,0,0,0.0,805) [kvm-irqfd-clean] (root,198572,564,0.0,815) /usr/sbin/lvmetad -f (root,0,0,0.0,868) [kworker/1:1H] (root,0,0,0.0,872) [kworker/2:1H] (root,0,0,0.0,880) [kdmflush] (root,0,0,0.0,881) [bioset] (root,0,0,0.0,883) [kdmflush] (root,0,0,0.0,884) [bioset] (root,0,0,0.0,885) [kdmflush] (root,0,0,0.0,887) [bioset] (root,0,0,0.0,889) [kdmflush] (root,0,0,0.0,890) [bioset] (root,0,0,0.0,897) [kworker/13:1H] (root,0,0,0.0,898) [kworker/7:1H] (root,0,0,0.0,899) [kworker/15:1H] (root,0,0,0.0,907) [jbd2/sda1-8] (root,0,0,0.0,908) [ext4-rsv-conver] (root,0,0,0.0,929) [kworker/9:1H] (root,0,0,0.0,930) [kworker/12:1H] (root,0,0,0.0,931) [kworker/4:1H] (root,0,0,0.0,938) [jbd2/dm-5-8] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,941) [ext4-rsv-conver] (root,0,0,0.0,947) [jbd2/dm-4-8] (root,0,0,0.0,948) [ext4-rsv-conver] (root,0,0,0.0,951) [jbd2/dm-2-8] (root,0,0,0.0,952) [ext4-rsv-conver] (root,0,0,0.0,976) [nv_queue] (root,0,0,0.0,977) [nv_queue] (polkitd,612240,1092,0.0,1012) /usr/lib/polkit-1/polkitd --no-debug (root,52900,592,0.0,1015) /usr/sbin/smartd -n -q never (root,21788,508,0.0,1027) /usr/sbin/irqbalance --foreground (dbus,58220,900,0.0,1034) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (root,26620,1244,0.0,1057) /usr/lib/systemd/systemd-logind (root,0,0,0.0,1099) [UVM global queu] (root,0,0,0.0,1100) [UVM deferred re] (root,0,0,0.0,1101) [UVM Tools Event] (root,0,0,0.0,1132) [nvidia-modeset/] (root,0,0,0.0,1133) [nvidia-modeset/] (root,0,0,0.0,1262) [bond0] (root,10058476,109256,0.0,1292) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,302152,27936,0.0,1570) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,574288,2332,0.0,1574) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,113000,8,0.0,1576) /usr/sbin/sshd -D (root,0,0,0.0,1581) [irq/98-nvidia] (root,0,0,0.0,1582) [nvidia] (root,0,0,0.2,1583) [nv_queue] (root,799816,1844,0.0,1823) /usr/sbin/collectd (root,634152,38880,0.0,1845) /usr/sbin/rsyslogd -n (root,591424,5644,0.1,1871) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,115700,964,0.0,1874) /bin/bash /sbin/condor_inspector (root,209064,1620,0.0,1887) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,1280,0.0,1906) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1913) [irq/99-nvidia] (root,0,0,0.0,1914) [nvidia] (root,0,0,0.2,1915) [nv_queue] (root,209332,2228,0.0,1920) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,89804,224,0.0,2118) /usr/libexec/postfix/master -w (postfix,90080,780,0.0,2121) qmgr -l -t unix -u (cvmfs,73348,2784,0.0,2130) /usr/bin/cvmfs2 __cachemgr__ . 7 8 41943040000 20971520000 0 3 -1 : (cvmfs,60968,0,0.0,2134) /usr/bin/cvmfs2 __cachemgr__ . 7 8 41943040000 20971520000 0 3 -1 : (cvmfs,940760,8244,0.0,2210) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,92204,8,0.0,2214) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2330) [irq/100-nvidia] (root,0,0,0.0,2331) [nvidia] (root,0,0,0.2,2332) [nv_queue] (root,0,0,0.0,2475) [kworker/1:2] (cvmfs,790988,5012,0.0,2588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,92152,4,0.0,2592) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,729532,2640,0.0,2607) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,92204,4,0.0,2611) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,727624,2992,0.0,2626) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,92204,4,0.0,2630) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,791624,4736,0.0,2645) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,92204,28,0.0,2649) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,791040,5228,0.0,2664) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,92204,4,0.0,2668) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,790988,5288,0.0,2757) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,92152,4,0.0,2761) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,5078796,114804,0.0,2792) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,92208,112,0.0,2796) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (root,0,0,0.0,2817) [irq/101-nvidia] (root,0,0,0.0,2818) [nvidia] (root,0,0,0.1,2819) [nv_queue] (cvmfs,72164,1780,0.0,2822) /usr/bin/cvmfs2 __cachemgr__ . 7 8 1048576000 524288000 0 3 -1 : (cvmfs,60968,0,0.0,2826) /usr/bin/cvmfs2 __cachemgr__ . 7 8 1048576000 524288000 0 3 -1 : (cvmfs,790904,7204,0.0,2830) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,92204,60,0.0,2834) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,791036,5660,0.0,2850) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,92204,16,0.0,2854) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,2870) [irq/102-nvidia] (root,0,0,0.0,2871) [nvidia] (root,0,0,0.1,2872) [nv_queue] (root,126388,432,0.0,2874) /usr/sbin/crond -n (condor,75276,3080,0.0,2875) /usr/sbin/condor_master -f (ntp,29960,776,0.0,2877) /usr/sbin/ntpd -u ntp:ntp -g (root,27828,4744,0.0,2878) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (root,0,0,0.0,2879) [irq/103-nvidia] (root,0,0,0.0,2880) [nvidia] (root,0,0,0.1,2881) [nv_queue] (condor,63816,2652,0.0,2882) condor_shared_port -f (root,0,0,0.0,2883) [kworker/10:1H] (root,0,0,0.0,2884) [irq/104-nvidia] (root,0,0,0.0,2885) [nvidia] (root,0,0,0.2,2886) [nv_queue] (condor,1651448,1576120,0.1,2887) condor_startd -f (root,0,0,0.0,2908) [irq/105-nvidia] (root,0,0,0.0,2909) [nvidia] (root,0,0,0.2,2910) [nv_queue] (root,110208,8,0.0,3014) /sbin/agetty --noclear tty1 linux (root,0,0,0.0,3344) [kworker/2:2] (root,0,0,0.0,3443) [kworker/u32:1] (condor,34082884,508,0.2,3821) GPUs_MONITOR (root,0,0,0.0,3824) [UVM GPU1 BH] (root,0,0,0.0,3825) [UVM GPU2 BH] (root,0,0,0.0,3827) [UVM GPU3 BH] (root,0,0,0.0,3828) [UVM GPU4 BH] (root,0,0,0.0,3830) [UVM GPU5 BH] (root,0,0,0.0,3831) [UVM GPU6 BH] (root,0,0,18.2,3832) [UVM GPU7 BH] (root,0,0,0.0,3833) [UVM GPU8 BH] (root,0,0,0.0,4021) [kworker/14:2] (root,0,0,0.0,4069) [kworker/8:1H] (root,0,0,0.0,4366) [kworker/3:1H] (postfix,89908,4024,0.0,4904) pickup -l -t unix -u (root,0,0,0.0,5239) [kworker/9:0] (root,27168,1056,0.0,5419) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,5946) [kworker/0:0] (ganglia,220196,8408,0.0,6311) /usr/sbin/gmond (root,0,0,0.0,6849) [kworker/9:1] (root,0,0,0.0,7217) [kworker/10:1] (root,0,0,0.0,7354) [kworker/13:1] (root,0,0,0.0,7600) [kworker/5:0] (root,0,0,0.0,7896) [kworker/11:1H] (root,0,0,0.0,7905) [kworker/13:2] (root,0,0,0.0,8891) [kworker/7:1] (root,0,0,0.0,9745) [kworker/12:0] (root,0,0,0.0,9917) [kworker/4:1] (root,0,0,0.0,9922) [kworker/5:2] (root,0,0,0.0,11694) [kworker/6:2] (root,0,0,0.0,11844) [kworker/8:1] (slot-1,46176,6592,0.0,12826) condor_starter -f -a slot1_2 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1564,0.0,12830) /bin/sh /scratch/condor/dir_21435/glidein/execute.bde4dab8-997d-11ed-81a0-0cc47a857a10/dir_12826/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,390336,22052,0.3,12866) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11700,1464,0.0,12869) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//icetray-env icetray/v1.4.1 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=5 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000005.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000005.i3.json (slot-1,11704,1548,0.0,12906) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/RHEL_7_x86_64/metaprojects/icetray/v1.4.1/env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=5 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000005.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000005.i3.json (slot-1,13146280,1257320,0.0,12913) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=5 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000005.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000005.i3.json (slot-1,10750956,714748,2.0,12927) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=5 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000005.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000005.i3.json (slot-1,10750956,707508,1.8,12928) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=5 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000005.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000005.i3.json (slot-1,10750960,712260,1.8,12929) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=5 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000005.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000005.i3.json (slot-1,10746112,704640,1.8,12930) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=5 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000005.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000005.i3.json (slot-1,10750960,711952,1.8,12931) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=5 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000005.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000005.i3.json (slot-1,10750960,712456,1.8,12932) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=5 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000005.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000005.i3.json (slot-1,10750960,712276,1.8,12933) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=5 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000005.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000005.i3.json (slot-1,10750964,710068,1.9,12934) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=5 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000005.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000005.i3.json (slot-1,233504,65620,0.5,12935) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65728,0.5,12936) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65632,0.5,12937) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65684,0.5,12938) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65640,0.5,12939) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65440,0.5,12940) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65616,0.5,12941) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65664,0.5,12942) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (root,0,0,0.0,13005) [kworker/15:2] (root,0,0,0.0,13237) [kworker/u33:0] (root,0,0,0.0,13330) [kworker/1:1] (slot-1,4368,360,0.0,13811) sleep 300 (root,0,0,0.0,14516) [kworker/2:0] (root,0,0,0.0,15007) [kworker/11:2] (root,0,0,0.0,16106) [kworker/u34:1] (root,0,0,0.0,16585) [kworker/11:1] (root,0,0,0.0,17008) [kworker/u34:0] (root,108056,356,0.0,17113) sleep 15 (root,8632,664,0.0,17140) timeout -s 9 360 check_mk_agent (root,11696,1532,0.0,17141) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,17157) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,944,0.0,17158) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,17650) [kworker/4:0] (root,0,0,0.0,18611) [kworker/0:2] (root,0,0,0.0,19437) [kworker/8:0] (root,0,0,0.0,19673) [kworker/3:0] (condor,67144,8224,0.0,21435) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (slot-1,11692,1576,0.0,21623) /bin/sh glidein_start.sh (slot-1,61812,6540,0.0,21776) glideinExec/sbin/condor_master -f -r 20135 (slot-1,12092,1864,0.0,21777) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_4626fa64-7aae-4c5d-9624-a7ed601982dd.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20230121%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230121T112002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=ca1ae6f4195b23963d27698b87ee30bc76aacc90f5e06dd7b22dfcc7ed77cc73 (slot-1,24608,4456,0.0,21779) condor_procd -A /scratch/condor/dir_21435/glidein/log.bde4dab8-997d-11ed-81a0-0cc47a857a10/procd_address -L /scratch/condor/dir_21435/glidein/log.bde4dab8-997d-11ed-81a0-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,62932,8056,0.0,21781) condor_startd -f (root,0,0,0.0,23743) [kworker/15:0] (root,0,0,0.0,24175) [kworker/14:1] (root,0,0,0.0,24220) [kworker/7:0] (root,0,0,0.0,24353) [kworker/6:1] (root,0,0,0.0,24573) [kworker/3:1] (slot-1,46176,6592,0.0,25002) condor_starter -f -a slot1_1 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1568,0.0,25006) /bin/sh /scratch/condor/dir_21435/glidein/execute.bde4dab8-997d-11ed-81a0-0cc47a857a10/dir_25002/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,390336,22056,0.3,25042) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11700,1464,0.0,25149) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//icetray-env icetray/v1.4.1 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=0 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000000.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000000.i3.json (slot-1,11704,1548,0.0,25186) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/RHEL_7_x86_64/metaprojects/icetray/v1.4.1/env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=0 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000000.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000000.i3.json (slot-1,13211816,1277804,0.0,25193) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=0 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000000.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000000.i3.json (root,0,0,0.0,25217) [kworker/u33:1] (slot-1,10746108,698540,1.9,25518) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=0 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000000.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000000.i3.json (slot-1,10746108,696408,2.0,25519) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=0 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000000.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000000.i3.json (slot-1,10746108,702528,1.8,25520) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=0 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000000.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000000.i3.json (slot-1,10746108,704004,1.8,25521) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=0 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000000.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000000.i3.json (slot-1,233504,65536,0.5,25522) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,10746108,696352,1.9,25523) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=0 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000000.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000000.i3.json (slot-1,233504,65536,0.5,25524) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,10746112,703424,1.9,25525) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=0 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000000.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000000.i3.json (slot-1,10746112,696124,1.9,25526) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=0 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000000.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000000.i3.json (slot-1,10750956,709784,1.9,25527) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=0 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000000.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000000.i3.json (slot-1,233504,65708,0.5,25528) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65436,0.5,25529) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65500,0.5,25530) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65628,0.5,25531) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65584,0.5,25532) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65648,0.5,25533) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (root,0,0,0.0,26215) [kworker/u34:2] (root,0,0,0.0,26464) [kworker/5:1] (root,0,0,0.0,28463) [kworker/12:2] (root,0,0,0.0,28499) [kworker/10:0] (root,0,0,0.0,32582) [kworker/u32:2] (root,0,0,0.0,32618) [kworker/4:2] (root,0,0,0.0,32641) [kworker/11:0]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b1a45a4f05aae993da0afef3b4
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2.chtc.wisc.edu AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43800,3108,0.0,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.2,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,259) [kauditd] (root,0,0,0.0,378) [kworker/u32:1] (root,0,0,0.0,411) [ata_sff] (root,0,0,0.0,412) [scsi_eh_0] (root,0,0,0.0,413) [scsi_tmf_0] (root,0,0,0.0,414) [scsi_eh_1] (root,0,0,0.0,415) [scsi_tmf_1] (root,0,0,0.0,416) [scsi_eh_2] (root,0,0,0.0,417) [scsi_tmf_2] (root,0,0,0.0,418) [scsi_eh_3] (root,0,0,0.0,419) [scsi_tmf_3] (root,0,0,0.0,423) [scsi_eh_4] (root,0,0,0.0,424) [scsi_tmf_4] (root,0,0,0.0,425) [scsi_eh_5] (root,0,0,0.0,426) [scsi_tmf_5] (root,0,0,0.0,427) [scsi_eh_6] (root,0,0,0.0,428) [scsi_tmf_6] (root,0,0,0.0,429) [scsi_eh_7] (root,0,0,0.0,430) [scsi_tmf_7] (root,0,0,0.0,431) [scsi_eh_8] (root,0,0,0.0,432) [scsi_tmf_8] (root,0,0,0.0,433) [scsi_eh_9] (root,0,0,0.0,434) [scsi_tmf_9] (root,0,0,0.0,442) [ttm_swap] (root,0,0,0.0,543) [kdmflush] (root,0,0,0.0,544) [bioset] (root,0,0,0.0,554) [kdmflush] (root,0,0,0.0,555) [bioset] (root,0,0,0.0,558) [kworker/0:1H] (root,0,0,0.0,572) [jbd2/dm-0-8] (root,0,0,0.0,573) [ext4-rsv-conver] (root,47252,10132,0.0,653) /usr/lib/systemd/systemd-journald (root,45604,656,0.0,679) /usr/lib/systemd/systemd-udevd (root,0,0,1.5,730) [kipmi0] (root,0,0,0.0,735) [kworker/14:1H] (root,0,0,0.0,737) [kworker/6:1H] (root,0,0,0.0,738) [kworker/5:1H] (root,0,0,0.0,805) [kvm-irqfd-clean] (root,198572,564,0.0,815) /usr/sbin/lvmetad -f (root,0,0,0.0,868) [kworker/1:1H] (root,0,0,0.0,872) [kworker/2:1H] (root,0,0,0.0,880) [kdmflush] (root,0,0,0.0,881) [bioset] (root,0,0,0.0,883) [kdmflush] (root,0,0,0.0,884) [bioset] (root,0,0,0.0,885) [kdmflush] (root,0,0,0.0,887) [bioset] (root,0,0,0.0,889) [kdmflush] (root,0,0,0.0,890) [bioset] (root,0,0,0.0,897) [kworker/13:1H] (root,0,0,0.0,898) [kworker/7:1H] (root,0,0,0.0,899) [kworker/15:1H] (root,0,0,0.0,907) [jbd2/sda1-8] (root,0,0,0.0,908) [ext4-rsv-conver] (root,0,0,0.0,929) [kworker/9:1H] (root,0,0,0.0,930) [kworker/12:1H] (root,0,0,0.0,931) [kworker/4:1H] (root,0,0,0.0,938) [jbd2/dm-5-8] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,941) [ext4-rsv-conver] (root,0,0,0.0,947) [jbd2/dm-4-8] (root,0,0,0.0,948) [ext4-rsv-conver] (root,0,0,0.0,951) [jbd2/dm-2-8] (root,0,0,0.0,952) [ext4-rsv-conver] (root,0,0,0.0,976) [nv_queue] (root,0,0,0.0,977) [nv_queue] (polkitd,612240,1092,0.0,1012) /usr/lib/polkit-1/polkitd --no-debug (root,52900,592,0.0,1015) /usr/sbin/smartd -n -q never (root,21788,508,0.0,1027) /usr/sbin/irqbalance --foreground (dbus,58220,896,0.0,1034) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (root,26620,1244,0.0,1057) /usr/lib/systemd/systemd-logind (root,0,0,0.0,1099) [UVM global queu] (root,0,0,0.0,1100) [UVM deferred re] (root,0,0,0.0,1101) [UVM Tools Event] (root,0,0,0.0,1132) [nvidia-modeset/] (root,0,0,0.0,1133) [nvidia-modeset/] (root,0,0,0.0,1262) [bond0] (root,10058476,105832,0.0,1292) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,302152,39068,0.0,1570) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,574288,2332,0.0,1574) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,113000,8,0.0,1576) /usr/sbin/sshd -D (root,0,0,0.0,1581) [irq/98-nvidia] (root,0,0,0.0,1582) [nvidia] (root,0,0,0.2,1583) [nv_queue] (root,0,0,0.0,1750) [kworker/5:2] (root,799816,1840,0.0,1823) /usr/sbin/collectd (root,494928,7124,0.0,1845) /usr/sbin/rsyslogd -n (root,591424,5624,0.1,1871) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,115700,960,0.0,1874) /bin/bash /sbin/condor_inspector (root,209064,1660,0.0,1887) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,1248,0.0,1906) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1913) [irq/99-nvidia] (root,0,0,0.0,1914) [nvidia] (root,0,0,0.2,1915) [nv_queue] (root,209332,2228,0.0,1920) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,89804,224,0.0,2118) /usr/libexec/postfix/master -w (postfix,90080,664,0.0,2121) qmgr -l -t unix -u (cvmfs,73348,2784,0.0,2130) /usr/bin/cvmfs2 __cachemgr__ . 7 8 41943040000 20971520000 0 3 -1 : (cvmfs,60968,0,0.0,2134) /usr/bin/cvmfs2 __cachemgr__ . 7 8 41943040000 20971520000 0 3 -1 : (cvmfs,942668,8436,0.0,2210) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,92204,8,0.0,2214) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2285) [kworker/13:1] (root,0,0,0.0,2330) [irq/100-nvidia] (root,0,0,0.0,2331) [nvidia] (root,0,0,0.2,2332) [nv_queue] (cvmfs,790996,4780,0.0,2588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,92152,4,0.0,2592) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,727624,3380,0.0,2607) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,92204,4,0.0,2611) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,727624,1984,0.0,2626) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,92204,4,0.0,2630) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,791196,4308,0.0,2645) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,92204,28,0.0,2649) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,791036,4952,0.0,2664) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,92204,4,0.0,2668) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,790984,5104,0.0,2757) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,92152,4,0.0,2761) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,5078828,112964,0.0,2792) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,92208,112,0.0,2796) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (root,0,0,0.0,2817) [irq/101-nvidia] (root,0,0,0.0,2818) [nvidia] (root,0,0,0.1,2819) [nv_queue] (cvmfs,72164,1768,0.0,2822) /usr/bin/cvmfs2 __cachemgr__ . 7 8 1048576000 524288000 0 3 -1 : (cvmfs,60968,0,0.0,2826) /usr/bin/cvmfs2 __cachemgr__ . 7 8 1048576000 524288000 0 3 -1 : (cvmfs,790900,6924,0.0,2830) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,92204,60,0.0,2834) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,791036,5648,0.0,2850) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,92204,16,0.0,2854) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,2870) [irq/102-nvidia] (root,0,0,0.0,2871) [nvidia] (root,0,0,0.1,2872) [nv_queue] (root,126388,432,0.0,2874) /usr/sbin/crond -n (condor,75276,3080,0.0,2875) /usr/sbin/condor_master -f (ntp,29960,776,0.0,2877) /usr/sbin/ntpd -u ntp:ntp -g (root,27828,4744,0.0,2878) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (root,0,0,0.0,2879) [irq/103-nvidia] (root,0,0,0.0,2880) [nvidia] (root,0,0,0.1,2881) [nv_queue] (condor,63756,2624,0.0,2882) condor_shared_port -f (root,0,0,0.0,2883) [kworker/10:1H] (root,0,0,0.0,2884) [irq/104-nvidia] (root,0,0,0.0,2885) [nvidia] (root,0,0,0.2,2886) [nv_queue] (condor,1258584,1183188,0.1,2887) condor_startd -f (root,0,0,0.0,2908) [irq/105-nvidia] (root,0,0,0.0,2909) [nvidia] (root,0,0,0.2,2910) [nv_queue] (root,110208,8,0.0,3014) /sbin/agetty --noclear tty1 linux (condor,34082884,508,0.2,3821) GPUs_MONITOR (root,0,0,0.0,3824) [UVM GPU1 BH] (root,0,0,0.0,3825) [UVM GPU2 BH] (root,0,0,0.0,3827) [UVM GPU3 BH] (root,0,0,0.0,3828) [UVM GPU4 BH] (root,0,0,0.0,3830) [UVM GPU5 BH] (root,0,0,0.0,3831) [UVM GPU6 BH] (root,0,0,0.0,3832) [UVM GPU7 BH] (root,0,0,0.0,3833) [UVM GPU8 BH] (root,0,0,0.0,4069) [kworker/8:1H] (root,0,0,0.0,4334) [kworker/8:2] (root,0,0,0.0,4366) [kworker/3:1H] (root,0,0,0.0,5038) [kworker/15:0] (ganglia,220196,8408,0.1,6311) /usr/sbin/gmond (root,0,0,0.0,7896) [kworker/11:1H] (root,0,0,0.0,8014) [kworker/0:1] (root,0,0,0.0,9155) [kworker/11:1] (root,0,0,0.0,9617) [kworker/5:0] (root,0,0,0.0,11422) [kworker/4:1] (root,0,0,0.0,11511) [kworker/0:0] (root,0,0,0.0,11583) [kworker/9:1] (root,0,0,0.0,11691) [kworker/10:0] (postfix,89908,4024,0.0,13024) pickup -l -t unix -u (root,0,0,0.0,13820) [kworker/8:0] (root,0,0,0.0,14190) [kworker/13:0] (root,0,0,0.0,14237) [kworker/u34:0] (root,0,0,0.0,14396) [kworker/u32:0] (root,0,0,0.0,16238) [kworker/11:2] (root,0,0,0.0,16472) [kworker/2:2] (root,0,0,0.0,16498) [kworker/12:0] (root,0,0,0.0,16861) [kworker/6:0] (root,0,0,0.0,17965) [kworker/9:0] (slot-1,4368,356,0.0,18788) sleep 300 (root,27168,1048,0.0,18855) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,19038) [kworker/2:1] (root,0,0,0.0,19919) [kworker/u33:0] (root,0,0,0.0,19920) [kworker/1:1] (slot-1,46176,6596,0.0,20814) condor_starter -f -a slot1_4 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1564,0.0,20818) /bin/sh /scratch/condor/dir_21435/glidein/execute.bde4dab8-997d-11ed-81a0-0cc47a857a10/dir_20814/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,390336,22056,0.4,20855) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11700,1460,0.0,20959) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//icetray-env icetray/v1.4.1 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=1 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000001.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000001.i3.json (slot-1,11704,1548,0.0,20995) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/RHEL_7_x86_64/metaprojects/icetray/v1.4.1/env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=1 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000001.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000001.i3.json (slot-1,13146280,1222872,0.0,21106) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=1 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000001.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000001.i3.json (root,0,0,0.0,21236) [kworker/10:2] (root,0,0,0.0,21242) [kworker/3:0] (condor,67116,8180,0.0,21435) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (root,0,0,0.0,21480) [kworker/1:0] (slot-1,11692,1576,0.0,21623) /bin/sh glidein_start.sh (slot-1,61812,6540,0.0,21776) glideinExec/sbin/condor_master -f -r 20135 (slot-1,11796,1576,0.0,21777) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_4626fa64-7aae-4c5d-9624-a7ed601982dd.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20230121%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230121T112002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=ca1ae6f4195b23963d27698b87ee30bc76aacc90f5e06dd7b22dfcc7ed77cc73 (slot-1,24608,4396,0.0,21779) condor_procd -A /scratch/condor/dir_21435/glidein/log.bde4dab8-997d-11ed-81a0-0cc47a857a10/procd_address -L /scratch/condor/dir_21435/glidein/log.bde4dab8-997d-11ed-81a0-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,62932,7876,0.0,21781) condor_startd -f (root,108056,360,0.0,21800) sleep 15 (root,8632,664,0.0,21803) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,21804) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,21820) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,21821) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (slot-1,10750956,707396,2.3,22480) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=1 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000001.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000001.i3.json (slot-1,10750956,704232,2.5,22481) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=1 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000001.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000001.i3.json (slot-1,10750960,703616,2.3,22482) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=1 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000001.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000001.i3.json (slot-1,10750960,705352,2.5,22483) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=1 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000001.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000001.i3.json (slot-1,10750960,706220,2.3,22484) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=1 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000001.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000001.i3.json (slot-1,10750960,712356,2.3,22485) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=1 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000001.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000001.i3.json (slot-1,10750960,704936,2.3,22486) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=1 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000001.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000001.i3.json (slot-1,10750960,707272,2.3,22487) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=1 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/metaprojects/icetray/v1.5.0/ice-models/resources/models/ANGSENS/angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022262.000001.i3.zst --summary-file=Summary_IC86.2020_corsika.022262.000001.i3.json (slot-1,233504,65536,0.7,22488) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65592,0.7,22489) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65708,0.7,22490) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65736,0.7,22491) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65660,0.7,22492) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65592,0.7,22493) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65676,0.7,22494) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65708,0.7,22495) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (root,0,0,0.0,22727) [kworker/2:0] (root,0,0,0.0,22972) [kworker/9:2] (root,0,0,0.0,24700) [kworker/7:1] (root,0,0,0.0,24763) [kworker/15:2] (slot-1,46176,6592,0.0,25105) condor_starter -f -a slot1_1 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1564,0.0,25109) /bin/sh /scratch/condor/dir_21435/glidein/execute.bde4dab8-997d-11ed-81a0-0cc47a857a10/dir_25105/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,390336,22020,0.5,25145) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11700,1464,0.0,25149) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1//icetray-env icetray/v1.5.0 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=9 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=../angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022251.000009.i3.zst --summary-file=Summary_IC86.2020_corsika.022251.000009.i3.json (slot-1,11704,1552,0.0,25242) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1/RHEL_7_x86_64/metaprojects/icetray/v1.5.0/env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=9 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=../angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022251.000009.i3.zst --summary-file=Summary_IC86.2020_corsika.022251.000009.i3.json (slot-1,13185864,1224128,0.0,25249) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=9 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=../angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022251.000009.i3.zst --summary-file=Summary_IC86.2020_corsika.022251.000009.i3.json (root,0,0,0.0,25493) [kworker/7:2] (slot-1,10790548,714480,2.9,25557) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=9 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=../angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022251.000009.i3.zst --summary-file=Summary_IC86.2020_corsika.022251.000009.i3.json (slot-1,10790548,711180,3.1,25566) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=9 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=../angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022251.000009.i3.zst --summary-file=Summary_IC86.2020_corsika.022251.000009.i3.json (slot-1,10790548,704864,2.9,25575) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=9 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=../angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022251.000009.i3.zst --summary-file=Summary_IC86.2020_corsika.022251.000009.i3.json (slot-1,10790548,707172,3.1,25576) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=9 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=../angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022251.000009.i3.zst --summary-file=Summary_IC86.2020_corsika.022251.000009.i3.json (slot-1,10785704,704620,2.9,25579) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=9 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=../angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022251.000009.i3.zst --summary-file=Summary_IC86.2020_corsika.022251.000009.i3.json (slot-1,10790552,700096,2.9,25582) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=9 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=../angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022251.000009.i3.zst --summary-file=Summary_IC86.2020_corsika.022251.000009.i3.json (slot-1,10790552,706392,2.9,25585) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=9 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=../angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022251.000009.i3.zst --summary-file=Summary_IC86.2020_corsika.022251.000009.i3.json (slot-1,10790552,714036,2.9,25586) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.1//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=9 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=../angsens_unified/as.set50_p0=-0.27_p1=-0.042 --oversize=1 --outfile=Hits_IC86.2020_corsika.022251.000009.i3.zst --summary-file=Summary_IC86.2020_corsika.022251.000009.i3.json (slot-1,233504,65628,0.9,25590) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65744,0.9,25591) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65644,0.9,25592) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65564,0.9,25593) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65548,0.9,25594) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65740,0.9,25595) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65648,0.9,25596) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (slot-1,233504,65536,0.9,25597) /cvmfs/icecube.opensciencegrid.org/users/kmeagher/corsika-77401/corsika77401Linux_SIBYLL_gheisha (root,0,0,0.0,25913) [kworker/3:1] (root,0,0,0.0,26025) [kworker/4:0] (root,0,0,0.0,26973) [kworker/10:1] (root,0,0,0.0,26985) [kworker/12:1] (root,0,0,0.0,27076) [kworker/u33:1] (root,0,0,0.0,27241) [kworker/3:2] (root,0,0,0.0,28548) [kworker/u34:2] (root,0,0,0.0,29024) [kworker/6:1] (root,0,0,0.0,31321) [kworker/14:2] (root,0,0,0.0,32122) [kworker/14:0]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b1a45a4f05aae993da216a0655
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2.chtc.wisc.edu AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43800,2932,0.0,1) /usr/lib/systemd/systemd --switched-root --system --deserialize 22 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.2,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,112) [kswapd0] (root,0,0,0.0,113) [kswapd1] (root,0,0,0.0,114) [ksmd] (root,0,0,0.0,115) [crypto] (root,0,0,0.0,123) [kthrotld] (root,0,0,0.0,128) [kmpath_rdacd] (root,0,0,0.0,129) [kaluad] (root,0,0,0.0,145) [kpsmoused] (root,0,0,0.0,146) [ipv6_addrconf] (root,0,0,0.0,159) [deferwq] (root,0,0,0.0,259) [kauditd] (root,0,0,0.0,346) [kworker/10:0] (root,0,0,0.0,411) [ata_sff] (root,0,0,0.0,412) [scsi_eh_0] (root,0,0,0.0,413) [scsi_tmf_0] (root,0,0,0.0,414) [scsi_eh_1] (root,0,0,0.0,415) [scsi_tmf_1] (root,0,0,0.0,416) [scsi_eh_2] (root,0,0,0.0,417) [scsi_tmf_2] (root,0,0,0.0,418) [scsi_eh_3] (root,0,0,0.0,419) [scsi_tmf_3] (root,0,0,0.0,423) [scsi_eh_4] (root,0,0,0.0,424) [scsi_tmf_4] (root,0,0,0.0,425) [scsi_eh_5] (root,0,0,0.0,426) [scsi_tmf_5] (root,0,0,0.0,427) [scsi_eh_6] (root,0,0,0.0,428) [scsi_tmf_6] (root,0,0,0.0,429) [scsi_eh_7] (root,0,0,0.0,430) [scsi_tmf_7] (root,0,0,0.0,431) [scsi_eh_8] (root,0,0,0.0,432) [scsi_tmf_8] (root,0,0,0.0,433) [scsi_eh_9] (root,0,0,0.0,434) [scsi_tmf_9] (root,0,0,0.0,442) [ttm_swap] (root,0,0,0.0,543) [kdmflush] (root,0,0,0.0,544) [bioset] (root,0,0,0.0,554) [kdmflush] (root,0,0,0.0,555) [bioset] (root,0,0,0.0,558) [kworker/0:1H] (root,0,0,0.0,572) [jbd2/dm-0-8] (root,0,0,0.0,573) [ext4-rsv-conver] (root,133292,81272,0.0,653) /usr/lib/systemd/systemd-journald (root,45604,656,0.0,679) /usr/lib/systemd/systemd-udevd (root,0,0,1.5,730) [kipmi0] (root,0,0,0.0,735) [kworker/14:1H] (root,0,0,0.0,737) [kworker/6:1H] (root,0,0,0.0,738) [kworker/5:1H] (root,0,0,0.0,805) [kvm-irqfd-clean] (root,198572,424,0.0,815) /usr/sbin/lvmetad -f (root,0,0,0.0,840) [kworker/11:2] (root,0,0,0.0,868) [kworker/1:1H] (root,0,0,0.0,872) [kworker/2:1H] (root,0,0,0.0,880) [kdmflush] (root,0,0,0.0,881) [bioset] (root,0,0,0.0,883) [kdmflush] (root,0,0,0.0,884) [bioset] (root,0,0,0.0,885) [kdmflush] (root,0,0,0.0,887) [bioset] (root,0,0,0.0,889) [kdmflush] (root,0,0,0.0,890) [bioset] (root,0,0,0.0,897) [kworker/13:1H] (root,0,0,0.0,898) [kworker/7:1H] (root,0,0,0.0,899) [kworker/15:1H] (root,0,0,0.0,907) [jbd2/sda1-8] (root,0,0,0.0,908) [ext4-rsv-conver] (root,0,0,0.0,929) [kworker/9:1H] (root,0,0,0.0,930) [kworker/12:1H] (root,0,0,0.0,931) [kworker/4:1H] (root,0,0,0.0,938) [jbd2/dm-5-8] (root,0,0,0.0,939) [jbd2/dm-3-8] (root,0,0,0.0,940) [ext4-rsv-conver] (root,0,0,0.0,941) [ext4-rsv-conver] (root,0,0,0.0,947) [jbd2/dm-4-8] (root,0,0,0.0,948) [ext4-rsv-conver] (root,0,0,0.0,951) [jbd2/dm-2-8] (root,0,0,0.0,952) [ext4-rsv-conver] (root,0,0,0.0,976) [nv_queue] (root,0,0,0.0,977) [nv_queue] (polkitd,612240,1092,0.0,1012) /usr/lib/polkit-1/polkitd --no-debug (root,52900,592,0.0,1015) /usr/sbin/smartd -n -q never (root,21788,508,0.0,1027) /usr/sbin/irqbalance --foreground (dbus,58220,896,0.0,1034) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (root,26516,1072,0.0,1057) /usr/lib/systemd/systemd-logind (root,0,0,0.0,1099) [UVM global queu] (root,0,0,0.0,1100) [UVM deferred re] (root,0,0,0.0,1101) [UVM Tools Event] (root,0,0,0.0,1132) [nvidia-modeset/] (root,0,0,0.0,1133) [nvidia-modeset/] (root,0,0,0.0,1262) [bond0] (root,10056420,83192,0.0,1292) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,0,0,0.0,1376) [kworker/u34:1] (root,302152,39092,0.0,1570) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,574288,2216,0.0,1574) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,113000,8,0.0,1576) /usr/sbin/sshd -D (root,0,0,0.0,1581) [irq/98-nvidia] (root,0,0,0.0,1582) [nvidia] (root,0,0,0.2,1583) [nv_queue] (root,0,0,0.0,1686) [kworker/0:2] (slot-1,46172,6588,0.0,1699) condor_starter -f -a slot1_9 sub-1.icecube.wisc.edu (slot-1,11836,1636,0.0,1711) /bin/bash /scratch/condor/dir_29440/glidein/execute.baaf730e-9184-11ed-a0c1-0cc47a857a10/dir_1699/condor_exec.exe --point 0_0.199526_0.000000_0.031628_0.063169_0.000000_3.141593 --use_fastMC True --Emax 9976 --sim Platinum_Split_Plus_Tau --max_sim -1 --fast_scaling 0.50 --realization_file /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/scripts/jobs/Data/Data_IC86.All_cut_everything_platinum_LT_240747841.npz --outdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fits/FitToData/AIRS_mceq121_2011_sib_HG_E3/FS_0.50/n_seeds_4//Ut4sq_0.003981/ --fastmc_dir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/FastMC/ --n_seeds 4 --osg True --fluxdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fluxes/AIRS_mceq121_2011_sib_HG_E3/Ut4sq_0.003981/ --years 7.634 --change_tol 1e-20 --grad_tol 1e-20 --inject_fit False --inject_fit_file None --systematics 11001110 --barr 00111111 --multisim 11 --astro 011 --xs 111 --bayes False (slot-1,46168,6580,0.0,1814) condor_starter -f -a slot1_10 sub-1.icecube.wisc.edu (root,799816,1844,0.0,1823) /usr/sbin/collectd (root,445728,46564,0.0,1845) /usr/sbin/rsyslogd -n (slot-1,2877160,2605064,95.7,1855) python3 -u Brazil.py --point 0_0.199526_0.000000_0.031628_0.063169_0.000000_3.141593 --use_fastMC True --Emax 9976 --sim Platinum_Split_Plus_Tau --max_sim -1 --fast_scaling 0.50 --realization_file /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/scripts/jobs/Data/Data_IC86.All_cut_everything_platinum_LT_240747841.npz --outdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fits/FitToData/AIRS_mceq121_2011_sib_HG_E3/FS_0.50/n_seeds_4//Ut4sq_0.003981/ --fastmc_dir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/FastMC/ --n_seeds 4 --osg True --fluxdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fluxes/AIRS_mceq121_2011_sib_HG_E3/Ut4sq_0.003981/ --years 7.634 --change_tol 1e-20 --grad_tol 1e-20 --inject_fit False --inject_fit_file None --systematics 11001110 --barr 00111111 --multisim 11 --astro 011 --xs 111 --bayes False (slot-1,46168,6580,0.0,1858) condor_starter -f -a slot1_11 sub-1.icecube.wisc.edu (slot-1,46168,6576,0.0,1864) condor_starter -f -a slot1_12 sub-1.icecube.wisc.edu (root,591424,4716,0.1,1871) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,115700,956,0.0,1874) /bin/bash /sbin/condor_inspector (slot-1,11836,1636,0.0,1876) /bin/bash /scratch/condor/dir_29440/glidein/execute.baaf730e-9184-11ed-a0c1-0cc47a857a10/dir_1814/condor_exec.exe --point 0_0.316228_0.000000_0.031628_0.063169_0.000000_3.141593 --use_fastMC True --Emax 9976 --sim Platinum_Split_Plus_Tau --max_sim -1 --fast_scaling 0.50 --realization_file /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/scripts/jobs/Data/Data_IC86.All_cut_everything_platinum_LT_240747841.npz --outdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fits/FitToData/AIRS_mceq121_2011_sib_HG_E3/FS_0.50/n_seeds_-1//Ut4sq_0.003981/ --fastmc_dir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/FastMC/ --n_seeds -1 --osg True --fluxdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fluxes/AIRS_mceq121_2011_sib_HG_E3/Ut4sq_0.003981/ --years 7.634 --change_tol 1e-20 --grad_tol 1e-20 --inject_fit False --inject_fit_file None --systematics 11001110 --barr 00111111 --multisim 11 --astro 011 --xs 111 --bayes False (slot-1,46168,6580,0.0,1879) condor_starter -f -a slot1_13 sub-1.icecube.wisc.edu (root,209064,1608,0.0,1887) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,1076,0.0,1906) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,1913) [irq/99-nvidia] (root,0,0,0.0,1914) [nvidia] (root,0,0,0.2,1915) [nv_queue] (root,209332,2208,0.0,1920) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (slot-1,11836,1640,0.0,1972) /bin/bash /scratch/condor/dir_29440/glidein/execute.baaf730e-9184-11ed-a0c1-0cc47a857a10/dir_1858/condor_exec.exe --point 0_0.630957_0.000000_0.031628_0.063169_0.000000_3.141593 --use_fastMC True --Emax 9976 --sim Platinum_Split_Plus_Tau --max_sim -1 --fast_scaling 0.50 --realization_file /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/scripts/jobs/Data/Data_IC86.All_cut_everything_platinum_LT_240747841.npz --outdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fits/FitToData/AIRS_mceq121_2011_sib_HG_E3/FS_0.50/n_seeds_4//Ut4sq_0.003981/ --fastmc_dir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/FastMC/ --n_seeds 4 --osg True --fluxdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fluxes/AIRS_mceq121_2011_sib_HG_E3/Ut4sq_0.003981/ --years 7.634 --change_tol 1e-20 --grad_tol 1e-20 --inject_fit False --inject_fit_file None --systematics 11001110 --barr 00111111 --multisim 11 --astro 011 --xs 111 --bayes False (slot-1,2877156,2605072,95.6,2054) python3 -u Brazil.py --point 0_0.316228_0.000000_0.031628_0.063169_0.000000_3.141593 --use_fastMC True --Emax 9976 --sim Platinum_Split_Plus_Tau --max_sim -1 --fast_scaling 0.50 --realization_file /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/scripts/jobs/Data/Data_IC86.All_cut_everything_platinum_LT_240747841.npz --outdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fits/FitToData/AIRS_mceq121_2011_sib_HG_E3/FS_0.50/n_seeds_-1//Ut4sq_0.003981/ --fastmc_dir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/FastMC/ --n_seeds -1 --osg True --fluxdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fluxes/AIRS_mceq121_2011_sib_HG_E3/Ut4sq_0.003981/ --years 7.634 --change_tol 1e-20 --grad_tol 1e-20 --inject_fit False --inject_fit_file None --systematics 11001110 --barr 00111111 --multisim 11 --astro 011 --xs 111 --bayes False (slot-1,11836,1640,0.0,2055) /bin/bash /scratch/condor/dir_29440/glidein/execute.baaf730e-9184-11ed-a0c1-0cc47a857a10/dir_1864/condor_exec.exe --point 0_1.000000_0.000000_0.031628_0.063169_0.000000_3.141593 --use_fastMC True --Emax 9976 --sim Platinum_Split_Plus_Tau --max_sim -1 --fast_scaling 0.50 --realization_file /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/scripts/jobs/Data/Data_IC86.All_cut_everything_platinum_LT_240747841.npz --outdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fits/FitToData/AIRS_mceq121_2011_sib_HG_E3/FS_0.50/n_seeds_4//Ut4sq_0.003981/ --fastmc_dir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/FastMC/ --n_seeds 4 --osg True --fluxdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fluxes/AIRS_mceq121_2011_sib_HG_E3/Ut4sq_0.003981/ --years 7.634 --change_tol 1e-20 --grad_tol 1e-20 --inject_fit False --inject_fit_file None --systematics 11001110 --barr 00111111 --multisim 11 --astro 011 --xs 111 --bayes False (root,89804,224,0.0,2118) /usr/libexec/postfix/master -w (postfix,90080,664,0.0,2121) qmgr -l -t unix -u (cvmfs,73348,2772,0.0,2130) /usr/bin/cvmfs2 __cachemgr__ . 7 8 41943040000 20971520000 0 3 -1 : (cvmfs,60968,0,0.0,2134) /usr/bin/cvmfs2 __cachemgr__ . 7 8 41943040000 20971520000 0 3 -1 : (cvmfs,866020,4388,0.0,2210) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,92204,8,0.0,2214) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (slot-1,2877156,2605064,95.8,2241) python3 -u Brazil.py --point 0_0.630957_0.000000_0.031628_0.063169_0.000000_3.141593 --use_fastMC True --Emax 9976 --sim Platinum_Split_Plus_Tau --max_sim -1 --fast_scaling 0.50 --realization_file /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/scripts/jobs/Data/Data_IC86.All_cut_everything_platinum_LT_240747841.npz --outdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fits/FitToData/AIRS_mceq121_2011_sib_HG_E3/FS_0.50/n_seeds_4//Ut4sq_0.003981/ --fastmc_dir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/FastMC/ --n_seeds 4 --osg True --fluxdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fluxes/AIRS_mceq121_2011_sib_HG_E3/Ut4sq_0.003981/ --years 7.634 --change_tol 1e-20 --grad_tol 1e-20 --inject_fit False --inject_fit_file None --systematics 11001110 --barr 00111111 --multisim 11 --astro 011 --xs 111 --bayes False (slot-1,2877156,2605160,95.7,2291) python3 -u Brazil.py --point 0_1.000000_0.000000_0.031628_0.063169_0.000000_3.141593 --use_fastMC True --Emax 9976 --sim Platinum_Split_Plus_Tau --max_sim -1 --fast_scaling 0.50 --realization_file /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/scripts/jobs/Data/Data_IC86.All_cut_everything_platinum_LT_240747841.npz --outdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fits/FitToData/AIRS_mceq121_2011_sib_HG_E3/FS_0.50/n_seeds_4//Ut4sq_0.003981/ --fastmc_dir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/FastMC/ --n_seeds 4 --osg True --fluxdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fluxes/AIRS_mceq121_2011_sib_HG_E3/Ut4sq_0.003981/ --years 7.634 --change_tol 1e-20 --grad_tol 1e-20 --inject_fit False --inject_fit_file None --systematics 11001110 --barr 00111111 --multisim 11 --astro 011 --xs 111 --bayes False (slot-1,11836,1636,0.0,2292) /bin/bash /scratch/condor/dir_29440/glidein/execute.baaf730e-9184-11ed-a0c1-0cc47a857a10/dir_1879/condor_exec.exe --point 0_1.258925_0.000000_0.031628_0.063169_0.000000_3.141593 --use_fastMC True --Emax 9976 --sim Platinum_Split_Plus_Tau --max_sim -1 --fast_scaling 0.50 --realization_file /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/scripts/jobs/Data/Data_IC86.All_cut_everything_platinum_LT_240747841.npz --outdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fits/FitToData/AIRS_mceq121_2011_sib_HG_E3/FS_0.50/n_seeds_4//Ut4sq_0.003981/ --fastmc_dir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/FastMC/ --n_seeds 4 --osg True --fluxdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fluxes/AIRS_mceq121_2011_sib_HG_E3/Ut4sq_0.003981/ --years 7.634 --change_tol 1e-20 --grad_tol 1e-20 --inject_fit False --inject_fit_file None --systematics 11001110 --barr 00111111 --multisim 11 --astro 011 --xs 111 --bayes False (slot-1,46168,6580,0.0,2293) condor_starter -f -a slot1_15 sub-1.icecube.wisc.edu (root,0,0,0.0,2330) [irq/100-nvidia] (root,0,0,0.0,2331) [nvidia] (root,0,0,0.2,2332) [nv_queue] (root,0,0,0.0,2391) [kworker/9:0] (slot-1,2877156,2605068,95.7,2417) python3 -u Brazil.py --point 0_1.258925_0.000000_0.031628_0.063169_0.000000_3.141593 --use_fastMC True --Emax 9976 --sim Platinum_Split_Plus_Tau --max_sim -1 --fast_scaling 0.50 --realization_file /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/scripts/jobs/Data/Data_IC86.All_cut_everything_platinum_LT_240747841.npz --outdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fits/FitToData/AIRS_mceq121_2011_sib_HG_E3/FS_0.50/n_seeds_4//Ut4sq_0.003981/ --fastmc_dir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/FastMC/ --n_seeds 4 --osg True --fluxdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fluxes/AIRS_mceq121_2011_sib_HG_E3/Ut4sq_0.003981/ --years 7.634 --change_tol 1e-20 --grad_tol 1e-20 --inject_fit False --inject_fit_file None --systematics 11001110 --barr 00111111 --multisim 11 --astro 011 --xs 111 --bayes False (slot-1,11836,1636,0.0,2543) /bin/bash /scratch/condor/dir_29440/glidein/execute.baaf730e-9184-11ed-a0c1-0cc47a857a10/dir_2293/condor_exec.exe --point 0_2.511886_0.000000_0.031628_0.063169_0.000000_3.141593 --use_fastMC True --Emax 9976 --sim Platinum_Split_Plus_Tau --max_sim -1 --fast_scaling 0.50 --realization_file /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/scripts/jobs/Data/Data_IC86.All_cut_everything_platinum_LT_240747841.npz --outdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fits/FitToData/AIRS_mceq121_2011_sib_HG_E3/FS_0.50/n_seeds_4//Ut4sq_0.003981/ --fastmc_dir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/FastMC/ --n_seeds 4 --osg True --fluxdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fluxes/AIRS_mceq121_2011_sib_HG_E3/Ut4sq_0.003981/ --years 7.634 --change_tol 1e-20 --grad_tol 1e-20 --inject_fit False --inject_fit_file None --systematics 11001110 --barr 00111111 --multisim 11 --astro 011 --xs 111 --bayes False (cvmfs,790984,4464,0.0,2588) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,92152,4,0.0,2592) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,727624,1668,0.0,2607) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,92204,4,0.0,2611) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,726616,1784,0.0,2626) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,92204,4,0.0,2630) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,726616,2072,0.0,2645) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,92204,28,0.0,2649) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,791040,4500,0.0,2664) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,92204,4,0.0,2668) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,790988,4772,0.0,2757) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,92152,4,0.0,2761) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,4226996,65472,0.0,2792) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,92208,112,0.0,2796) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (slot-1,2877160,2605156,95.8,2810) python3 -u Brazil.py --point 0_2.511886_0.000000_0.031628_0.063169_0.000000_3.141593 --use_fastMC True --Emax 9976 --sim Platinum_Split_Plus_Tau --max_sim -1 --fast_scaling 0.50 --realization_file /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/scripts/jobs/Data/Data_IC86.All_cut_everything_platinum_LT_240747841.npz --outdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fits/FitToData/AIRS_mceq121_2011_sib_HG_E3/FS_0.50/n_seeds_4//Ut4sq_0.003981/ --fastmc_dir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/FastMC/ --n_seeds 4 --osg True --fluxdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fluxes/AIRS_mceq121_2011_sib_HG_E3/Ut4sq_0.003981/ --years 7.634 --change_tol 1e-20 --grad_tol 1e-20 --inject_fit False --inject_fit_file None --systematics 11001110 --barr 00111111 --multisim 11 --astro 011 --xs 111 --bayes False (root,0,0,0.0,2817) [irq/101-nvidia] (root,0,0,0.0,2818) [nvidia] (root,0,0,0.1,2819) [nv_queue] (cvmfs,72164,1768,0.0,2822) /usr/bin/cvmfs2 __cachemgr__ . 7 8 1048576000 524288000 0 3 -1 : (cvmfs,60968,0,0.0,2826) /usr/bin/cvmfs2 __cachemgr__ . 7 8 1048576000 524288000 0 3 -1 : (cvmfs,790900,6416,0.0,2830) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,92204,60,0.0,2834) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,791036,5264,0.0,2850) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,92204,16,0.0,2854) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (root,0,0,0.0,2870) [irq/102-nvidia] (root,0,0,0.0,2871) [nvidia] (root,0,0,0.1,2872) [nv_queue] (root,126388,432,0.0,2874) /usr/sbin/crond -n (condor,75276,3064,0.0,2875) /usr/sbin/condor_master -f (ntp,29960,776,0.0,2877) /usr/sbin/ntpd -u ntp:ntp -g (root,27828,4744,0.0,2878) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (root,0,0,0.0,2879) [irq/103-nvidia] (root,0,0,0.0,2880) [nvidia] (root,0,0,0.1,2881) [nv_queue] (condor,63756,2560,0.0,2882) condor_shared_port -f (root,0,0,0.0,2883) [kworker/10:1H] (root,0,0,0.0,2884) [irq/104-nvidia] (root,0,0,0.0,2885) [nvidia] (root,0,0,0.2,2886) [nv_queue] (condor,797332,719572,0.2,2887) condor_startd -f (root,0,0,0.0,2908) [irq/105-nvidia] (root,0,0,0.0,2909) [nvidia] (root,0,0,0.2,2910) [nv_queue] (root,0,0,0.0,2951) [kworker/4:0] (root,110208,8,0.0,3014) /sbin/agetty --noclear tty1 linux (ganglia,220196,2748,0.1,3019) /usr/sbin/gmond (condor,34082884,508,0.2,3821) GPUs_MONITOR (root,0,0,0.0,3824) [UVM GPU1 BH] (root,0,0,0.0,3825) [UVM GPU2 BH] (root,0,0,0.0,3827) [UVM GPU3 BH] (root,0,0,0.0,3828) [UVM GPU4 BH] (root,0,0,0.0,3830) [UVM GPU5 BH] (root,0,0,0.0,3831) [UVM GPU6 BH] (root,0,0,0.0,3832) [UVM GPU7 BH] (root,0,0,0.0,3833) [UVM GPU8 BH] (root,0,0,0.0,4069) [kworker/8:1H] (root,0,0,0.0,4335) [kworker/2:2] (root,0,0,0.0,4366) [kworker/3:1H] (root,0,0,0.0,4499) [kworker/8:2] (root,27168,8,0.0,5048) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,5386) [kworker/2:1] (slot-1,46164,6572,0.0,6969) condor_starter -f -a slot1_16 sub-1.icecube.wisc.edu (slot-1,11836,1636,0.0,6975) /bin/bash /scratch/condor/dir_29440/glidein/execute.baaf730e-9184-11ed-a0c1-0cc47a857a10/dir_6969/condor_exec.exe --point 0_6.309573_0.000000_0.050140_0.063217_0.000000_3.141593 --use_fastMC True --Emax 9976 --sim Platinum_Split_Plus_Tau --max_sim -1 --fast_scaling 0.50 --realization_file /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/scripts/jobs/Data/Data_IC86.All_cut_everything_platinum_LT_240747841.npz --outdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fits/FitToData/AIRS_mceq121_2011_sib_HG_E3/FS_0.50/n_seeds_4//Ut4sq_0.003981/ --fastmc_dir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/FastMC/ --n_seeds 4 --osg True --fluxdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fluxes/AIRS_mceq121_2011_sib_HG_E3/Ut4sq_0.003981/ --years 7.634 --change_tol 1e-20 --grad_tol 1e-20 --inject_fit False --inject_fit_file None --systematics 11001110 --barr 00111111 --multisim 11 --astro 011 --xs 111 --bayes False (slot-1,2877156,2623524,97.5,7100) python3 -u Brazil.py --point 0_6.309573_0.000000_0.050140_0.063217_0.000000_3.141593 --use_fastMC True --Emax 9976 --sim Platinum_Split_Plus_Tau --max_sim -1 --fast_scaling 0.50 --realization_file /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/scripts/jobs/Data/Data_IC86.All_cut_everything_platinum_LT_240747841.npz --outdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fits/FitToData/AIRS_mceq121_2011_sib_HG_E3/FS_0.50/n_seeds_4//Ut4sq_0.003981/ --fastmc_dir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/FastMC/ --n_seeds 4 --osg True --fluxdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fluxes/AIRS_mceq121_2011_sib_HG_E3/Ut4sq_0.003981/ --years 7.634 --change_tol 1e-20 --grad_tol 1e-20 --inject_fit False --inject_fit_file None --systematics 11001110 --barr 00111111 --multisim 11 --astro 011 --xs 111 --bayes False (root,0,0,0.0,7894) [kworker/0:0] (root,0,0,0.0,7895) [kworker/9:1] (root,0,0,0.0,7896) [kworker/11:1H] (root,0,0,0.0,7963) [kworker/4:2] (root,0,0,0.0,9037) [kworker/5:1] (root,0,0,0.0,10444) [kworker/11:0] (root,0,0,0.0,11559) [kworker/u32:1] (root,0,0,0.0,12237) [kworker/12:0] (root,0,0,0.0,12546) [kworker/u32:0] (root,0,0,0.2,12547) [kworker/u34:2] (slot-1,4368,360,0.0,12965) sleep 300 (root,0,0,0.0,14209) [kworker/13:1] (root,0,0,0.0,14777) [kworker/4:1] (root,0,0,0.0,14892) [kworker/13:2] (root,0,0,0.0,15061) [kworker/u33:1] (root,0,0,0.0,15450) [kworker/15:2] (root,0,0,0.0,16634) [kworker/15:0] (root,0,0,0.0,16641) [kworker/14:0] (root,0,0,0.0,16644) [kworker/5:2] (root,0,0,0.0,18103) [kworker/14:2] (root,0,0,0.0,18573) [kworker/14:1] (root,0,0,0.0,18902) [kworker/1:1] (slot-1,46176,6580,0.0,18913) condor_starter -f -a slot1_2 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1564,0.0,18918) /bin/sh /scratch/condor/dir_29440/glidein/execute.baaf730e-9184-11ed-a0c1-0cc47a857a10/dir_18913/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,457664,26148,0.5,18954) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11700,1460,0.0,19171) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/icetray-env /cvmfs/icecube.opensciencegrid.org/users/akochocki/icetray/background_update python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1548,0.0,19267) bash /cvmfs/icecube.opensciencegrid.org/users/akochocki/icetray/background_update/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13192860,719828,98.0,19274) python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,46176,6584,0.0,19404) condor_starter -f -a slot1_6 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1564,0.0,19408) /bin/sh /scratch/condor/dir_29440/glidein/execute.baaf730e-9184-11ed-a0c1-0cc47a857a10/dir_19404/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,457660,26156,0.5,19445) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11700,1460,0.0,19481) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/icetray-env /cvmfs/icecube.opensciencegrid.org/users/akochocki/icetray/background_update python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1548,0.0,19576) bash /cvmfs/icecube.opensciencegrid.org/users/akochocki/icetray/background_update/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13260372,808748,98.0,19583) python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,20106) [kworker/u33:2] (root,0,0,0.0,20214) [kworker/u34:0] (root,0,0,0.0,20262) [kworker/15:1] (slot-1,46172,6576,0.1,20475) condor_starter -f -a slot1_14 sub-1.icecube.wisc.edu (slot-1,11836,1632,0.0,20480) /bin/bash /scratch/condor/dir_29440/glidein/execute.baaf730e-9184-11ed-a0c1-0cc47a857a10/dir_20475/condor_exec.exe --point 0_2.511886_0.000000_0.056264_0.063238_0.000000_3.141593 --use_fastMC True --Emax 9976 --sim Platinum_Split_Plus_Tau --max_sim -1 --fast_scaling 0.50 --realization_file /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/scripts/jobs/Data/Data_IC86.All_cut_everything_platinum_LT_240747841.npz --outdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fits/FitToData/AIRS_mceq121_2011_sib_HG_E3/FS_0.50/n_seeds_4//Ut4sq_0.003981/ --fastmc_dir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/FastMC/ --n_seeds 4 --osg True --fluxdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fluxes/AIRS_mceq121_2011_sib_HG_E3/Ut4sq_0.003981/ --years 7.634 --change_tol 1e-20 --grad_tol 1e-20 --inject_fit False --inject_fit_file None --systematics 11001110 --barr 00111111 --multisim 11 --astro 011 --xs 111 --bayes False (slot-1,2877160,2623520,72.7,20599) python3 -u Brazil.py --point 0_2.511886_0.000000_0.056264_0.063238_0.000000_3.141593 --use_fastMC True --Emax 9976 --sim Platinum_Split_Plus_Tau --max_sim -1 --fast_scaling 0.50 --realization_file /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/scripts/jobs/Data/Data_IC86.All_cut_everything_platinum_LT_240747841.npz --outdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fits/FitToData/AIRS_mceq121_2011_sib_HG_E3/FS_0.50/n_seeds_4//Ut4sq_0.003981/ --fastmc_dir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/FastMC/ --n_seeds 4 --osg True --fluxdir /data/ana/SterileNeutrino/IC86/HighEnergy/Analysis/th34/Fluxes/AIRS_mceq121_2011_sib_HG_E3/Ut4sq_0.003981/ --years 7.634 --change_tol 1e-20 --grad_tol 1e-20 --inject_fit False --inject_fit_file None --systematics 11001110 --barr 00111111 --multisim 11 --astro 011 --xs 111 --bayes False (root,0,0,0.0,20643) [kworker/1:0] (root,0,0,0.0,20747) [kworker/9:2] (postfix,89908,4024,0.0,20799) pickup -l -t unix -u (slot-1,46176,6584,0.1,20816) condor_starter -f -a slot1_1 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1564,0.1,20825) /bin/sh /scratch/condor/dir_29440/glidein/execute.baaf730e-9184-11ed-a0c1-0cc47a857a10/dir_20816/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,457664,26132,1.8,20862) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,46116,6560,0.2,20870) condor_starter -f -a slot1_3 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1564,0.1,20877) /bin/sh /scratch/condor/dir_29440/glidein/execute.baaf730e-9184-11ed-a0c1-0cc47a857a10/dir_20870/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,457668,26148,2.5,20913) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,46116,6560,0.2,20914) condor_starter -f -a slot1_4 sub-simprod-2.icecube.wisc.edu (root,108056,356,0.0,21011) sleep 15 (slot-1,46176,6580,0.3,21017) condor_starter -f -a slot1_5 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1568,0.1,21018) /bin/sh /scratch/condor/dir_29440/glidein/execute.baaf730e-9184-11ed-a0c1-0cc47a857a10/dir_20914/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,457664,26140,2.6,21054) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,46116,6560,0.2,21063) condor_starter -f -a slot1_7 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1568,0.0,21066) /bin/sh /scratch/condor/dir_29440/glidein/execute.baaf730e-9184-11ed-a0c1-0cc47a857a10/dir_21017/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,457664,26200,2.6,21103) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,46116,6560,0.3,21106) condor_starter -f -a slot1_8 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1564,0.0,21112) /bin/sh /scratch/condor/dir_29440/glidein/execute.baaf730e-9184-11ed-a0c1-0cc47a857a10/dir_21063/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,457664,26156,3.1,21148) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11704,1564,0.0,21152) /bin/sh /scratch/condor/dir_29440/glidein/execute.baaf730e-9184-11ed-a0c1-0cc47a857a10/dir_21106/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,389940,25876,2.9,21188) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11700,1456,0.0,21190) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/icetray-env /cvmfs/icecube.opensciencegrid.org/users/akochocki/icetray/background_update python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,78196,7876,3.0,21270) globus-url-copy gsiftp://gridftp-scratch.icecube.wisc.edu/mnt/tank/simprod/22268/14390/corsika_partprop.i3.zst file:/scratch/condor/dir_29440/glidein/execute.baaf730e-9184-11ed-a0c1-0cc47a857a10/dir_21106/corsika_partprop.i3.zst (slot-1,11704,1544,0.0,21295) bash /cvmfs/icecube.opensciencegrid.org/users/akochocki/icetray/background_update/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,10879916,553392,93.5,21302) python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11700,1460,0.0,21320) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/icetray-env /cvmfs/icecube.opensciencegrid.org/users/akochocki/icetray/background_update python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1548,0.0,21414) bash /cvmfs/icecube.opensciencegrid.org/users/akochocki/icetray/background_update/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,10837020,510556,101,21421) python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11700,1456,0.0,21454) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/icetray-env /cvmfs/icecube.opensciencegrid.org/users/akochocki/icetray/background_update python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1548,0.0,21548) bash /cvmfs/icecube.opensciencegrid.org/users/akochocki/icetray/background_update/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,10837084,510304,104,21555) python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11700,1456,0.0,21584) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/icetray-env /cvmfs/icecube.opensciencegrid.org/users/akochocki/icetray/background_update python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,21641) [kworker/13:0] (slot-1,11700,1460,0.0,21656) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/icetray-env /cvmfs/icecube.opensciencegrid.org/users/akochocki/icetray/background_update python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1548,0.0,21710) bash /cvmfs/icecube.opensciencegrid.org/users/akochocki/icetray/background_update/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,414060,160444,122,21717) python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1548,0.0,21784) bash /cvmfs/icecube.opensciencegrid.org/users/akochocki/icetray/background_update/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,414060,160744,103,21791) python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,8632,660,0.0,21811) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,21812) /bin/bash /bin/check_mk_agent (root,51732,1696,0.0,21828) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,944,0.0,21829) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,23165) [kworker/u32:2] (root,0,0,0.0,23735) [kworker/u33:0] (root,0,0,0.0,23929) [kworker/3:1] (root,0,0,0.0,25166) [kworker/12:2] (root,0,0,0.0,25191) [kworker/1:2] (root,0,0,0.0,25223) [kworker/7:2] (root,0,0,0.0,25977) [kworker/6:1] (root,0,0,0.0,27206) [kworker/8:0] (root,0,0,0.0,28601) [kworker/10:2] (root,0,0,0.0,29133) [kworker/3:2] (root,0,0,0.0,29217) [kworker/7:0] (condor,67120,8148,0.0,29440) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (slot-1,11692,1572,0.0,29461) /bin/sh glidein_start.sh (slot-1,61812,6520,0.0,29522) glideinExec/sbin/condor_master -f -r 20135 (slot-1,11696,1492,0.0,29523) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_8327446b-e527-406a-89c5-ec69474a2ce2.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20230111%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230111T075002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=8a14067be352b39c5c76ac9d0c8bc765f09afac87c3459e138850c6b9c1dfb2c (slot-1,24640,4528,0.3,29525) condor_procd -A /scratch/condor/dir_29440/glidein/log.baaf730e-9184-11ed-a0c1-0cc47a857a10/procd_address -L /scratch/condor/dir_29440/glidein/log.baaf730e-9184-11ed-a0c1-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,64072,9396,0.2,29526) condor_startd -f (root,0,0,0.0,32378) [kworker/6:0] (root,0,0,0.0,32671) [kworker/5:0]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b1a45a4f05aae993da2c297655
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2.chtc.wisc.edu AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43872,3104,0.0,1) /usr/lib/systemd/systemd --system --deserialize 17 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.1,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,109) [kswapd0] (root,0,0,0.0,110) [kswapd1] (root,0,0,0.0,111) [ksmd] (root,0,0,0.0,112) [khugepaged] (root,0,0,0.0,113) [crypto] (root,0,0,0.0,121) [kthrotld] (root,0,0,0.0,125) [kmpath_rdacd] (root,0,0,0.0,126) [kaluad] (root,0,0,0.0,143) [kpsmoused] (root,0,0,0.0,144) [ipv6_addrconf] (root,0,0,0.0,158) [deferwq] (root,0,0,0.0,203) [kauditd] (root,0,0,0.0,397) [ata_sff] (root,0,0,0.0,405) [scsi_eh_0] (root,0,0,0.0,406) [scsi_tmf_0] (root,0,0,0.0,407) [scsi_eh_1] (root,0,0,0.0,408) [scsi_tmf_1] (root,0,0,0.0,409) [scsi_eh_2] (root,0,0,0.0,410) [scsi_tmf_2] (root,0,0,0.0,411) [scsi_eh_3] (root,0,0,0.0,412) [scsi_tmf_3] (root,0,0,0.0,416) [scsi_eh_4] (root,0,0,0.0,417) [scsi_tmf_4] (root,0,0,0.0,418) [scsi_eh_5] (root,0,0,0.0,419) [scsi_tmf_5] (root,0,0,0.0,420) [scsi_eh_6] (root,0,0,0.0,421) [scsi_tmf_6] (root,0,0,0.0,422) [scsi_eh_7] (root,0,0,0.0,423) [scsi_tmf_7] (root,0,0,0.0,424) [scsi_eh_8] (root,0,0,0.0,425) [scsi_tmf_8] (root,0,0,0.0,426) [scsi_eh_9] (root,0,0,0.0,427) [scsi_tmf_9] (root,0,0,0.0,434) [ttm_swap] (root,0,0,0.0,535) [kdmflush] (root,0,0,0.0,536) [bioset] (root,0,0,0.0,546) [kdmflush] (root,0,0,0.0,547) [bioset] (root,0,0,0.0,550) [kworker/0:1H] (root,0,0,0.0,552) [kworker/15:1H] (root,0,0,0.0,553) [kworker/1:1H] (root,0,0,0.0,568) [jbd2/dm-0-8] (root,0,0,0.0,569) [ext4-rsv-conver] (root,80420,41768,0.0,650) /usr/lib/systemd/systemd-journald (root,0,0,1.2,727) [kipmi0] (root,0,0,0.0,735) [kworker/13:1H] (root,0,0,0.0,742) [kworker/7:1H] (root,0,0,0.0,765) [kworker/9:1H] (root,0,0,0.0,784) [kworker/3:1H] (root,0,0,0.0,792) [kvm-irqfd-clean] (root,198572,380,0.0,793) /usr/sbin/lvmetad -f (root,0,0,0.0,867) [kdmflush] (root,0,0,0.0,868) [bioset] (root,0,0,0.0,869) [kdmflush] (root,0,0,0.0,871) [bioset] (root,0,0,0.0,873) [kdmflush] (root,0,0,0.0,874) [bioset] (root,0,0,0.0,876) [kdmflush] (root,0,0,0.0,877) [bioset] (root,0,0,0.0,886) [kworker/5:1H] (root,0,0,0.0,887) [kworker/4:1H] (root,0,0,0.0,888) [kworker/12:1H] (root,0,0,0.0,889) [kworker/8:1H] (root,0,0,0.0,901) [jbd2/sda1-8] (root,0,0,0.0,902) [ext4-rsv-conver] (postfix,89908,4032,0.0,915) pickup -l -t unix -u (root,0,0,0.0,937) [kworker/14:1H] (root,0,0,0.0,947) [jbd2/dm-3-8] (root,0,0,0.0,948) [ext4-rsv-conver] (root,0,0,0.0,970) [jbd2/dm-2-8] (root,0,0,0.0,971) [ext4-rsv-conver] (root,0,0,0.0,972) [kworker/11:0] (root,0,0,0.0,978) [jbd2/dm-4-8] (root,0,0,0.0,979) [ext4-rsv-conver] (root,0,0,0.0,1018) [jbd2/dm-5-8] (root,0,0,0.0,1019) [ext4-rsv-conver] (dbus,58232,1032,0.0,1038) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612356,1212,0.0,1065) /usr/lib/polkit-1/polkitd --no-debug (root,21716,528,0.0,1077) /usr/sbin/irqbalance --foreground (root,52900,592,0.0,1078) /usr/sbin/smartd -n -q never (root,27360,1076,0.0,1081) /usr/lib/systemd/systemd-logind (root,0,0,0.0,1089) [kworker/6:1H] (condor,67120,8148,0.0,1111) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (slot-1,11692,1576,0.0,1130) /bin/sh glidein_start.sh (slot-1,61812,6528,0.0,1191) glideinExec/sbin/condor_master -f -r 20135 (slot-1,11696,1500,0.0,1192) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_68e6507c-3c1f-4ec8-a9a7-738df522ef99.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20221205%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20221205T202002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=64ea60c5692585e962a9c6a861d97d8ec098886ce4fb393f1de577ad0e69abb3 (slot-1,24416,3784,0.0,1194) condor_procd -A /scratch/condor/dir_1111/glidein/log.6938f76e-74da-11ed-ac6c-0cc47a857a10/procd_address -L /scratch/condor/dir_1111/glidein/log.6938f76e-74da-11ed-ac6c-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,63256,8196,0.0,1195) condor_startd -f (root,0,0,0.0,1268) [bond0] (root,0,0,0.0,1278) [kworker/2:1H] (root,10059916,168464,0.0,1379) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,0,0,0.0,1556) [kworker/11:1H] (root,112984,360,0.0,1648) /usr/sbin/sshd -D (root,27168,4,0.0,1653) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1856) [kworker/4:2] (root,801948,2128,0.0,1895) /usr/sbin/collectd (root,729392,26040,0.0,1921) /usr/sbin/rsyslogd -n (root,115708,968,0.0,1948) /bin/bash /sbin/condor_inspector (root,591424,5740,0.2,1985) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,2070) [kworker/12:0] (cvmfs,1372468,37020,0.0,2140) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,89788,228,0.0,2488) /usr/libexec/postfix/master -w (postfix,90064,676,0.0,2538) qmgr -l -t unix -u (cvmfs,852768,29348,0.0,2548) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,852192,32864,0.0,2563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,852060,31152,0.0,2615) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,853296,29780,0.0,2635) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (root,0,0,0.0,2643) [kworker/3:1] (cvmfs,925772,33380,0.0,2660) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,925808,32520,0.0,2749) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,926084,31148,0.0,2779) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,925780,33212,0.0,2798) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,4225540,124780,0.6,2830) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (root,126392,452,0.0,2859) /usr/sbin/crond -n (ntp,29960,844,0.0,2868) /usr/sbin/ntpd -u ntp:ntp -g (slot-1,46152,6572,0.0,2976) condor_starter -f -a slot1_2 sub-1.icecube.wisc.edu (slot-1,46152,6572,0.0,2982) condor_starter -f -a slot1_3 sub-1.icecube.wisc.edu (slot-1,11704,1516,0.0,3167) bash /scratch/condor/dir_1111/glidein/execute.6938f76e-74da-11ed-ac6c-0cc47a857a10/dir_2976/condor_exec.exe (root,574284,2336,0.0,3281) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,93236,4,0.0,3339) login -- i3admin (ganglia,220728,3592,0.0,3343) /usr/sbin/gmond (slot-1,11832,1548,0.0,3404) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuE/NuE_D_120000_000035_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/121148/NuE_D_121148_000035_step2.i3.zst --filenr 35 --tmpdir /scratch/condor/dir_1111/glidein/execute.6938f76e-74da-11ed-ac6c-0cc47a857a10/dir_2976/2022-12-05_13-27-14 --gridftp --runnumber 121148 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 1e+04 -t --simple-hqe-scaling (slot-1,13290904,800404,116,3408) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuE/NuE_D_120000_000035_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/121148/NuE_D_121148_000035_step2.i3.zst --filenr 35 --tmpdir /scratch/condor/dir_1111/glidein/execute.6938f76e-74da-11ed-ac6c-0cc47a857a10/dir_2976/2022-12-05_13-27-14 --gridftp --runnumber 121148 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 1e+04 -t --simple-hqe-scaling (slot-1,46152,6576,0.0,3409) condor_starter -f -a slot1_5 sub-1.icecube.wisc.edu (slot-1,11704,1512,0.0,3411) bash /scratch/condor/dir_1111/glidein/execute.6938f76e-74da-11ed-ac6c-0cc47a857a10/dir_2982/condor_exec.exe (slot-1,11832,1548,0.0,3519) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000029_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141148/NuMu_C_141148_000029_step2.i3.zst --filenr 29 --tmpdir /scratch/condor/dir_1111/glidein/execute.6938f76e-74da-11ed-ac6c-0cc47a857a10/dir_2982/2022-12-05_13-27-18 --gridftp --runnumber 141148 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 1e+04 -t --simple-hqe-scaling (slot-1,13250276,836624,113,3523) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000029_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141148/NuMu_C_141148_000029_step2.i3.zst --filenr 29 --tmpdir /scratch/condor/dir_1111/glidein/execute.6938f76e-74da-11ed-ac6c-0cc47a857a10/dir_2982/2022-12-05_13-27-18 --gridftp --runnumber 141148 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 1e+04 -t --simple-hqe-scaling (root,0,0,0.0,3622) [kworker/u32:1] (slot-1,46152,6576,0.0,3668) condor_starter -f -a slot1_7 sub-1.icecube.wisc.edu (slot-1,46152,6576,0.0,3740) condor_starter -f -a slot1_8 sub-1.icecube.wisc.edu (slot-1,11704,1516,0.0,3741) bash /scratch/condor/dir_1111/glidein/execute.6938f76e-74da-11ed-ac6c-0cc47a857a10/dir_3409/condor_exec.exe (root,0,0,0.0,3788) [kworker/8:1] (slot-1,11832,1548,0.0,3897) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000031_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141148/NuMu_C_141148_000031_step2.i3.zst --filenr 31 --tmpdir /scratch/condor/dir_1111/glidein/execute.6938f76e-74da-11ed-ac6c-0cc47a857a10/dir_3409/2022-12-05_13-27-18 --gridftp --runnumber 141148 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 1e+04 -t --simple-hqe-scaling (slot-1,13284012,869484,113,3906) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuMu/NuMu_C_140000_000031_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/141148/NuMu_C_141148_000031_step2.i3.zst --filenr 31 --tmpdir /scratch/condor/dir_1111/glidein/execute.6938f76e-74da-11ed-ac6c-0cc47a857a10/dir_3409/2022-12-05_13-27-18 --gridftp --runnumber 141148 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 1e+04 -t --simple-hqe-scaling (root,0,0,0.0,3912) [kworker/9:1] (root,0,0,0.0,3925) [kworker/4:0] (slot-1,11704,1516,0.0,4064) bash /scratch/condor/dir_1111/glidein/execute.6938f76e-74da-11ed-ac6c-0cc47a857a10/dir_3668/condor_exec.exe (slot-1,11832,1548,0.0,4228) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuE/NuE_D_120000_000039_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/121148/NuE_D_121148_000039_step2.i3.zst --filenr 39 --tmpdir /scratch/condor/dir_1111/glidein/execute.6938f76e-74da-11ed-ac6c-0cc47a857a10/dir_3668/2022-12-05_13-27-14 --gridftp --runnumber 121148 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 1e+04 -t --simple-hqe-scaling (slot-1,13342556,839960,116,4234) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuE/NuE_D_120000_000039_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/121148/NuE_D_121148_000039_step2.i3.zst --filenr 39 --tmpdir /scratch/condor/dir_1111/glidein/execute.6938f76e-74da-11ed-ac6c-0cc47a857a10/dir_3668/2022-12-05_13-27-14 --gridftp --runnumber 121148 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 1e+04 -t --simple-hqe-scaling (slot-1,11704,1516,0.0,4271) bash /scratch/condor/dir_1111/glidein/execute.6938f76e-74da-11ed-ac6c-0cc47a857a10/dir_3740/condor_exec.exe (slot-1,11832,1548,0.0,4375) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/build_py3-v4.2.1/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuE/NuE_C_120000_000040_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/121148/NuE_C_121148_000040_step2.i3.zst --filenr 40 --tmpdir /scratch/condor/dir_1111/glidein/execute.6938f76e-74da-11ed-ac6c-0cc47a857a10/dir_3740/2022-12-05_13-27-14 --gridftp --runnumber 121148 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 1e+04 -t --simple-hqe-scaling (slot-1,13986552,1501856,111,4380) python /cvmfs/icecube.opensciencegrid.org/users/stuttard/icetray/oscNext/src/oscnext_scripts/step_2_clsim_setCrossE.py --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz --infile /data/sim/DeepCore/2018/pass2/genie/step1/NuE/NuE_C_120000_000040_step1.zst --outfile /data/sim/DeepCore/2018//pass2/genie/step2/121148/NuE_C_121148_000040_step2.i3.zst --filenr 40 --tmpdir /scratch/condor/dir_1111/glidein/execute.6938f76e-74da-11ed-ac6c-0cc47a857a10/dir_3740/2022-12-05_13-27-14 --gridftp --runnumber 121148 --icemodel spice_bfr-v2 --efficiency 1.2 --crossenergy 1e+04 -t --simple-hqe-scaling (slot-1,13568,1988,0.0,4399) zstd -d -qq (slot-1,13568,2152,0.0,4557) zstd -d -qq (slot-1,13568,1996,0.0,4641) zstd -d -qq (slot-1,13568,1996,0.0,4642) zstd -d -qq (slot-1,13568,2168,0.1,4665) zstd -d -qq (root,0,0,0.0,4830) [kworker/u34:0] (root,0,0,0.0,4987) [kworker/9:0] (root,0,0,0.0,6204) [kworker/3:0] (root,0,0,0.0,6217) [kworker/14:2] (root,0,0,0.0,7161) [kworker/13:0] (root,0,0,0.0,7310) [kworker/11:2] (root,0,0,0.0,7435) [kworker/10:0] (root,0,0,0.0,7439) [kworker/10:1] (root,0,0,0.0,8324) [kworker/15:1] (root,0,0,0.0,8695) [bioset] (root,0,0,0.0,8696) [xfsalloc] (root,0,0,0.0,8697) [xfs_mru_cache] (root,0,0,0.0,8724) [bioset] (root,0,0,0.0,9415) [kworker/1:2] (root,0,0,0.0,9457) [kworker/0:0] (root,0,0,0.0,10579) [kworker/2:1] (root,0,0,0.0,11159) [kworker/14:1] (root,0,0,0.0,12372) [kworker/15:2] (root,0,0,0.0,12625) [kworker/7:0] (root,0,0,0.0,12966) [kworker/0:2] (i3admin,12752,0,0.0,14064) -bash (root,131516,4,0.0,14081) sudo su - (root,81988,0,0.0,14082) su - (root,12748,8,0.0,14083) -bash (root,102988,876,0.0,15151) /sbin/dhclient -1 -q -lf /var/lib/dhclient/dhclient--bond0.lease -pf /var/run/dhclient-bond0.pid -H gzk-2 bond0 (root,0,0,0.0,15737) [kworker/1:1] (slot-1,4368,360,0.0,16125) sleep 300 (root,0,0,0.0,16330) [kworker/2:2] (root,0,0,0.0,16648) [kworker/9:2] (root,0,0,0.0,17044) [kworker/10:1H] (root,0,0,0.0,17060) [kworker/u33:0] (root,0,0,0.0,17360) [kworker/6:0] (root,302344,41668,0.0,17833) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,0,0,0.0,18076) [nv_queue] (root,0,0,0.0,18077) [nv_queue] (root,0,0,0.0,18099) [UVM global queu] (root,0,0,0.0,18100) [UVM deferred re] (root,0,0,0.0,18102) [UVM Tools Event] (root,0,0,0.0,19742) [kworker/2:0] (root,108056,360,0.0,19963) sleep 15 (root,8632,664,0.0,19964) timeout -s 9 360 check_mk_agent (root,11696,1532,0.0,19965) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,19981) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,944,0.0,19982) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,23270) [kworker/u34:2] (root,209064,1464,0.0,23403) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,1512,0.0,23405) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,212132,2520,0.0,23409) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,24029) [kworker/5:2] (root,0,0,0.0,25512) [irq/98-nvidia] (root,0,0,0.0,25513) [nvidia] (root,0,0,0.6,25514) [nv_queue] (root,0,0,0.0,25545) [irq/99-nvidia] (root,0,0,0.0,25546) [nvidia] (root,0,0,0.5,25547) [nv_queue] (root,0,0,0.0,25555) [irq/100-nvidia] (root,0,0,0.0,25556) [nvidia] (root,0,0,0.4,25557) [nv_queue] (root,0,0,0.0,25558) [irq/101-nvidia] (root,0,0,0.0,25559) [nvidia] (root,0,0,0.5,25560) [nv_queue] (root,0,0,0.0,25561) [irq/102-nvidia] (root,0,0,0.0,25562) [nvidia] (root,0,0,0.4,25563) [nv_queue] (root,0,0,0.0,25564) [irq/103-nvidia] (root,0,0,0.0,25565) [nvidia] (root,0,0,0.4,25566) [nv_queue] (root,0,0,0.0,25567) [irq/104-nvidia] (root,0,0,0.0,25568) [nvidia] (root,0,0,0.4,25569) [nv_queue] (root,0,0,0.0,25570) [irq/105-nvidia] (root,0,0,0.0,25571) [nvidia] (root,0,0,0.4,25572) [nv_queue] (condor,75284,3272,0.0,25684) /usr/sbin/condor_master -f (root,27892,4864,0.0,25693) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,63760,2748,0.0,25695) condor_shared_port -f (condor,1509540,1449640,0.0,25724) condor_startd -f (condor,34059816,8296,0.2,25915) GPUs_MONITOR (root,0,0,0.0,25917) [UVM GPU1 BH] (root,0,0,0.0,25918) [UVM GPU2 BH] (root,0,0,0.0,25919) [UVM GPU3 BH] (root,0,0,0.0,25920) [UVM GPU4 BH] (root,0,0,0.0,25921) [UVM GPU5 BH] (root,0,0,0.0,25922) [UVM GPU6 BH] (root,0,0,0.0,25923) [UVM GPU7 BH] (root,0,0,0.0,25924) [UVM GPU8 BH] (root,0,0,0.0,26312) [kworker/6:2] (cvmfs,60916,720,0.0,27819) /usr/bin/cvmfs2 __cachemgr__ . 8 9 41943040000 20971520000 0 3 -1 : (cvmfs,72240,3060,0.0,27822) /usr/bin/cvmfs2 __cachemgr__ . 8 9 41943040000 20971520000 0 3 -1 : (cvmfs,1327456,28220,0.0,27854) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,27877) [kworker/5:1] (cvmfs,60916,720,0.0,27919) /usr/bin/cvmfs2 __cachemgr__ . 8 9 1048576000 524288000 0 3 -1 : (cvmfs,72096,2936,0.0,27923) /usr/bin/cvmfs2 __cachemgr__ . 8 9 1048576000 524288000 0 3 -1 : (cvmfs,884792,27748,0.0,27954) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,884796,27732,0.0,27967) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,884796,29608,0.0,27983) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,4151740,112576,0.0,27991) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,811200,23808,0.0,28006) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,811080,23564,0.0,28019) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,811200,29284,0.0,28032) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,811200,23400,0.0,28045) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,884816,24900,0.0,28058) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,28268) [kworker/7:2] (root,0,0,0.0,29372) [kworker/u32:0] (root,0,0,0.0,29450) [kworker/12:1] (root,0,0,0.0,29707) [kworker/8:2] (root,0,0,0.0,29756) [kworker/u33:2] (root,0,0,0.0,30391) [kworker/13:2] (root,45364,760,0.0,31760) /usr/lib/systemd/systemd-udevd
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b1a45a4f05aae993da0ac64bf7
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2.chtc.wisc.edu AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43872,3104,0.0,1) /usr/lib/systemd/systemd --system --deserialize 17 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.1,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,109) [kswapd0] (root,0,0,0.0,110) [kswapd1] (root,0,0,0.0,111) [ksmd] (root,0,0,0.0,112) [khugepaged] (root,0,0,0.0,113) [crypto] (root,0,0,0.0,121) [kthrotld] (root,0,0,0.0,125) [kmpath_rdacd] (root,0,0,0.0,126) [kaluad] (root,0,0,0.0,143) [kpsmoused] (root,0,0,0.0,144) [ipv6_addrconf] (root,0,0,0.0,158) [deferwq] (root,0,0,0.0,203) [kauditd] (root,0,0,0.0,397) [ata_sff] (root,0,0,0.0,405) [scsi_eh_0] (root,0,0,0.0,406) [scsi_tmf_0] (root,0,0,0.0,407) [scsi_eh_1] (root,0,0,0.0,408) [scsi_tmf_1] (root,0,0,0.0,409) [scsi_eh_2] (root,0,0,0.0,410) [scsi_tmf_2] (root,0,0,0.0,411) [scsi_eh_3] (root,0,0,0.0,412) [scsi_tmf_3] (root,0,0,0.0,416) [scsi_eh_4] (root,0,0,0.0,417) [scsi_tmf_4] (root,0,0,0.0,418) [scsi_eh_5] (root,0,0,0.0,419) [scsi_tmf_5] (root,0,0,0.0,420) [scsi_eh_6] (root,0,0,0.0,421) [scsi_tmf_6] (root,0,0,0.0,422) [scsi_eh_7] (root,0,0,0.0,423) [scsi_tmf_7] (root,0,0,0.0,424) [scsi_eh_8] (root,0,0,0.0,425) [scsi_tmf_8] (root,0,0,0.0,426) [scsi_eh_9] (root,0,0,0.0,427) [scsi_tmf_9] (root,0,0,0.0,434) [ttm_swap] (root,0,0,0.0,535) [kdmflush] (root,0,0,0.0,536) [bioset] (root,0,0,0.0,546) [kdmflush] (root,0,0,0.0,547) [bioset] (root,0,0,0.0,550) [kworker/0:1H] (root,0,0,0.0,552) [kworker/15:1H] (root,0,0,0.0,553) [kworker/1:1H] (root,0,0,0.0,568) [jbd2/dm-0-8] (root,0,0,0.0,569) [ext4-rsv-conver] (root,80420,40776,0.0,650) /usr/lib/systemd/systemd-journald (root,0,0,1.2,727) [kipmi0] (root,0,0,0.0,735) [kworker/13:1H] (root,0,0,0.0,742) [kworker/7:1H] (root,0,0,0.0,765) [kworker/9:1H] (root,0,0,0.0,784) [kworker/3:1H] (root,0,0,0.0,792) [kvm-irqfd-clean] (root,198572,380,0.0,793) /usr/sbin/lvmetad -f (root,0,0,0.0,807) [kworker/15:0] (root,0,0,0.0,862) [kworker/u32:0] (root,0,0,0.0,867) [kdmflush] (root,0,0,0.0,868) [bioset] (root,0,0,0.0,869) [kdmflush] (root,0,0,0.0,871) [bioset] (root,0,0,0.0,873) [kdmflush] (root,0,0,0.0,874) [bioset] (root,0,0,0.0,876) [kdmflush] (root,0,0,0.0,877) [bioset] (root,0,0,0.0,886) [kworker/5:1H] (root,0,0,0.0,887) [kworker/4:1H] (root,0,0,0.0,888) [kworker/12:1H] (root,0,0,0.0,889) [kworker/8:1H] (root,0,0,0.0,901) [jbd2/sda1-8] (root,0,0,0.0,902) [ext4-rsv-conver] (root,0,0,0.0,937) [kworker/14:1H] (root,0,0,0.0,947) [jbd2/dm-3-8] (root,0,0,0.0,948) [ext4-rsv-conver] (root,0,0,0.0,970) [jbd2/dm-2-8] (root,0,0,0.0,971) [ext4-rsv-conver] (root,0,0,0.0,978) [jbd2/dm-4-8] (root,0,0,0.0,979) [ext4-rsv-conver] (root,0,0,0.0,1018) [jbd2/dm-5-8] (root,0,0,0.0,1019) [ext4-rsv-conver] (dbus,58232,1032,0.0,1038) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612356,1212,0.0,1065) /usr/lib/polkit-1/polkitd --no-debug (root,21716,528,0.0,1077) /usr/sbin/irqbalance --foreground (root,52900,592,0.0,1078) /usr/sbin/smartd -n -q never (root,27360,1076,0.0,1081) /usr/lib/systemd/systemd-logind (root,0,0,0.0,1089) [kworker/6:1H] (root,0,0,0.0,1268) [bond0] (root,0,0,0.0,1278) [kworker/2:1H] (root,10059916,167784,0.0,1379) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,0,0,0.0,1556) [kworker/11:1H] (root,112984,360,0.0,1648) /usr/sbin/sshd -D (root,27168,4,0.0,1653) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,801948,2108,0.0,1895) /usr/sbin/collectd (root,726388,25016,0.0,1921) /usr/sbin/rsyslogd -n (root,115708,968,0.0,1948) /bin/bash /sbin/condor_inspector (root,591424,5728,0.2,1985) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (cvmfs,1370560,37372,0.0,2140) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2485) [kworker/1:0] (root,89788,228,0.0,2488) /usr/libexec/postfix/master -w (postfix,90064,680,0.0,2538) qmgr -l -t unix -u (cvmfs,852768,29000,0.0,2548) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,852180,32868,0.0,2563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,852064,31164,0.0,2615) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,852768,29676,0.0,2635) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,925772,33360,0.0,2660) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,925800,32520,0.0,2749) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,925936,31240,0.0,2779) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,925780,33208,0.0,2798) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,4225504,122324,0.6,2830) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (root,126392,452,0.0,2859) /usr/sbin/crond -n (ntp,29960,844,0.0,2868) /usr/sbin/ntpd -u ntp:ntp -g (root,574284,2336,0.0,3281) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,93236,4,0.0,3339) login -- i3admin (ganglia,220728,3592,0.0,3343) /usr/sbin/gmond (root,0,0,0.0,3354) [kworker/8:1] (root,0,0,0.0,4260) [kworker/11:2] (root,0,0,0.0,4595) [kworker/u32:1] (root,0,0,0.0,5538) [kworker/6:0] (root,0,0,0.0,5542) [kworker/2:2] (root,108056,360,0.0,5635) sleep 15 (root,8632,660,0.0,5636) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,5637) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,5653) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,5654) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,6126) [kworker/u32:2] (root,0,0,0.0,6223) [kworker/13:1] (root,0,0,0.0,6384) [kworker/2:1] (root,0,0,0.0,6557) [kworker/11:0] (root,0,0,0.0,7307) [kworker/0:2] (root,0,0,0.0,8247) [kworker/5:0] (root,0,0,0.0,8695) [bioset] (root,0,0,0.0,8696) [xfsalloc] (root,0,0,0.0,8697) [xfs_mru_cache] (root,0,0,0.0,8724) [bioset] (root,0,0,0.0,10141) [kworker/7:1] (root,0,0,0.0,11160) [kworker/1:1] (root,0,0,0.0,12705) [kworker/0:0] (i3admin,12752,0,0.0,14064) -bash (root,131516,4,0.0,14081) sudo su - (root,81988,0,0.0,14082) su - (root,12748,8,0.0,14083) -bash (root,0,0,0.0,14627) [kworker/6:1] (root,102988,876,0.0,15151) /sbin/dhclient -1 -q -lf /var/lib/dhclient/dhclient--bond0.lease -pf /var/run/dhclient-bond0.pid -H gzk-2 bond0 (root,0,0,0.0,15812) [kworker/14:0] (root,0,0,0.0,17044) [kworker/10:1H] (postfix,89908,4024,0.0,17134) pickup -l -t unix -u (root,0,0,0.0,17177) [kworker/u33:0] (root,302344,40680,0.0,17833) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,0,0,0.0,17893) [kworker/8:0] (root,0,0,0.0,17915) [kworker/9:1] (root,0,0,0.0,18018) [kworker/13:2] (root,0,0,0.0,18076) [nv_queue] (root,0,0,0.0,18077) [nv_queue] (root,0,0,0.0,18099) [UVM global queu] (root,0,0,0.0,18100) [UVM deferred re] (root,0,0,0.0,18102) [UVM Tools Event] (root,0,0,0.0,18654) [kworker/6:2] (root,0,0,0.0,18951) [kworker/9:2] (root,0,0,0.0,18954) [kworker/1:2] (root,0,0,0.0,19018) [kworker/5:1] (root,0,0,0.0,21141) [kworker/4:0] (root,0,0,0.0,21660) [kworker/2:0] (root,209064,1500,0.0,23403) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,1424,0.0,23405) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,212132,2520,0.0,23409) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,24118) [kworker/15:1] (root,0,0,0.0,24319) [kworker/u34:1] (root,0,0,0.0,25403) [kworker/3:0] (root,0,0,0.0,25512) [irq/98-nvidia] (root,0,0,0.0,25513) [nvidia] (root,0,0,0.6,25514) [nv_queue] (root,0,0,0.0,25545) [irq/99-nvidia] (root,0,0,0.0,25546) [nvidia] (root,0,0,0.5,25547) [nv_queue] (root,0,0,0.0,25555) [irq/100-nvidia] (root,0,0,0.0,25556) [nvidia] (root,0,0,0.4,25557) [nv_queue] (root,0,0,0.0,25558) [irq/101-nvidia] (root,0,0,0.0,25559) [nvidia] (root,0,0,0.5,25560) [nv_queue] (root,0,0,0.0,25561) [irq/102-nvidia] (root,0,0,0.0,25562) [nvidia] (root,0,0,0.4,25563) [nv_queue] (root,0,0,0.0,25564) [irq/103-nvidia] (root,0,0,0.0,25565) [nvidia] (root,0,0,0.4,25566) [nv_queue] (root,0,0,0.0,25567) [irq/104-nvidia] (root,0,0,0.0,25568) [nvidia] (root,0,0,0.4,25569) [nv_queue] (root,0,0,0.0,25570) [irq/105-nvidia] (root,0,0,0.0,25571) [nvidia] (root,0,0,0.4,25572) [nv_queue] (condor,75284,3164,0.0,25684) /usr/sbin/condor_master -f (root,27892,4864,0.0,25693) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,63760,2676,0.0,25695) condor_shared_port -f (condor,1350916,1291036,0.0,25724) condor_startd -f (condor,34059816,8296,0.1,25915) GPUs_MONITOR (root,0,0,0.0,25917) [UVM GPU1 BH] (root,0,0,0.0,25918) [UVM GPU2 BH] (root,0,0,0.0,25919) [UVM GPU3 BH] (root,0,0,0.0,25920) [UVM GPU4 BH] (root,0,0,0.0,25921) [UVM GPU5 BH] (root,0,0,0.0,25922) [UVM GPU6 BH] (root,0,0,0.0,25923) [UVM GPU7 BH] (root,0,0,0.0,25924) [UVM GPU8 BH] (root,0,0,0.0,25943) [kworker/10:2] (root,0,0,0.0,25976) [kworker/12:1] (root,0,0,0.0,26120) [kworker/14:2] (root,0,0,0.0,27510) [kworker/3:1] (cvmfs,60916,720,0.0,27819) /usr/bin/cvmfs2 __cachemgr__ . 8 9 41943040000 20971520000 0 3 -1 : (cvmfs,72108,2860,0.0,27822) /usr/bin/cvmfs2 __cachemgr__ . 8 9 41943040000 20971520000 0 3 -1 : (cvmfs,1327456,28220,0.0,27854) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,60916,720,0.0,27919) /usr/bin/cvmfs2 __cachemgr__ . 8 9 1048576000 524288000 0 3 -1 : (cvmfs,72096,2936,0.0,27923) /usr/bin/cvmfs2 __cachemgr__ . 8 9 1048576000 524288000 0 3 -1 : (cvmfs,884792,27752,0.0,27954) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,884796,27732,0.0,27967) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,884796,29608,0.0,27983) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,4151740,112576,0.0,27991) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,811200,23824,0.0,28006) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,811080,23564,0.0,28019) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,811200,29288,0.0,28032) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,811200,23400,0.0,28045) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,884816,24900,0.0,28058) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,28273) [kworker/4:2] (root,0,0,0.0,29335) [kworker/12:0] (root,0,0,0.0,29853) [kworker/u33:2] (root,0,0,0.0,30802) [kworker/7:2] (root,45364,760,0.0,31760) /usr/lib/systemd/systemd-udevd (root,0,0,0.0,32065) [kworker/u34:2] (root,0,0,0.0,32528) [kworker/10:1]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b1a45a4f05aae993da887d29b6
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2.chtc.wisc.edu AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43872,3104,0.0,1) /usr/lib/systemd/systemd --system --deserialize 17 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.1,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,109) [kswapd0] (root,0,0,0.0,110) [kswapd1] (root,0,0,0.0,111) [ksmd] (root,0,0,0.0,112) [khugepaged] (root,0,0,0.0,113) [crypto] (root,0,0,0.0,121) [kthrotld] (root,0,0,0.0,125) [kmpath_rdacd] (root,0,0,0.0,126) [kaluad] (root,0,0,0.0,143) [kpsmoused] (root,0,0,0.0,144) [ipv6_addrconf] (root,0,0,0.0,158) [deferwq] (root,0,0,0.0,203) [kauditd] (root,0,0,0.0,397) [ata_sff] (root,0,0,0.0,405) [scsi_eh_0] (root,0,0,0.0,406) [scsi_tmf_0] (root,0,0,0.0,407) [scsi_eh_1] (root,0,0,0.0,408) [scsi_tmf_1] (root,0,0,0.0,409) [scsi_eh_2] (root,0,0,0.0,410) [scsi_tmf_2] (root,0,0,0.0,411) [scsi_eh_3] (root,0,0,0.0,412) [scsi_tmf_3] (root,0,0,0.0,416) [scsi_eh_4] (root,0,0,0.0,417) [scsi_tmf_4] (root,0,0,0.0,418) [scsi_eh_5] (root,0,0,0.0,419) [scsi_tmf_5] (root,0,0,0.0,420) [scsi_eh_6] (root,0,0,0.0,421) [scsi_tmf_6] (root,0,0,0.0,422) [scsi_eh_7] (root,0,0,0.0,423) [scsi_tmf_7] (root,0,0,0.0,424) [scsi_eh_8] (root,0,0,0.0,425) [scsi_tmf_8] (root,0,0,0.0,426) [scsi_eh_9] (root,0,0,0.0,427) [scsi_tmf_9] (root,0,0,0.0,434) [ttm_swap] (root,0,0,0.0,535) [kdmflush] (root,0,0,0.0,536) [bioset] (root,0,0,0.0,546) [kdmflush] (root,0,0,0.0,547) [bioset] (root,0,0,0.0,550) [kworker/0:1H] (root,0,0,0.0,552) [kworker/15:1H] (root,0,0,0.0,553) [kworker/1:1H] (root,0,0,0.0,568) [jbd2/dm-0-8] (root,0,0,0.0,569) [ext4-rsv-conver] (root,80420,39280,0.0,650) /usr/lib/systemd/systemd-journald (root,0,0,1.2,727) [kipmi0] (root,0,0,0.0,735) [kworker/13:1H] (root,0,0,0.0,742) [kworker/7:1H] (root,0,0,0.0,765) [kworker/9:1H] (root,0,0,0.0,784) [kworker/3:1H] (root,0,0,0.0,792) [kvm-irqfd-clean] (root,198572,380,0.0,793) /usr/sbin/lvmetad -f (root,0,0,0.0,862) [kworker/u34:2] (root,0,0,0.0,867) [kdmflush] (root,0,0,0.0,868) [bioset] (root,0,0,0.0,869) [kdmflush] (root,0,0,0.0,871) [bioset] (root,0,0,0.0,873) [kdmflush] (root,0,0,0.0,874) [bioset] (root,0,0,0.0,876) [kdmflush] (root,0,0,0.0,877) [bioset] (root,0,0,0.0,886) [kworker/5:1H] (root,0,0,0.0,887) [kworker/4:1H] (root,0,0,0.0,888) [kworker/12:1H] (root,0,0,0.0,889) [kworker/8:1H] (root,0,0,0.0,901) [jbd2/sda1-8] (root,0,0,0.0,902) [ext4-rsv-conver] (root,0,0,0.0,937) [kworker/14:1H] (root,0,0,0.0,947) [jbd2/dm-3-8] (root,0,0,0.0,948) [ext4-rsv-conver] (root,0,0,0.0,970) [jbd2/dm-2-8] (root,0,0,0.0,971) [ext4-rsv-conver] (root,0,0,0.0,978) [jbd2/dm-4-8] (root,0,0,0.0,979) [ext4-rsv-conver] (root,0,0,0.0,1018) [jbd2/dm-5-8] (root,0,0,0.0,1019) [ext4-rsv-conver] (dbus,58232,1032,0.0,1038) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612356,1212,0.0,1065) /usr/lib/polkit-1/polkitd --no-debug (root,21716,528,0.0,1077) /usr/sbin/irqbalance --foreground (root,52900,592,0.0,1078) /usr/sbin/smartd -n -q never (root,27360,1076,0.0,1081) /usr/lib/systemd/systemd-logind (root,0,0,0.0,1089) [kworker/6:1H] (root,0,0,0.0,1218) [kworker/0:1] (root,0,0,0.0,1268) [bond0] (root,0,0,0.0,1278) [kworker/2:1H] (root,10059916,167020,0.0,1379) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,0,0,0.0,1556) [kworker/11:1H] (root,112984,360,0.0,1648) /usr/sbin/sshd -D (root,27168,4,0.0,1653) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,801948,2116,0.0,1895) /usr/sbin/collectd (root,726376,24076,0.0,1921) /usr/sbin/rsyslogd -n (root,115708,968,0.0,1948) /bin/bash /sbin/condor_inspector (root,591424,5684,0.2,1985) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (cvmfs,1369024,35992,0.0,2140) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,89788,228,0.0,2488) /usr/libexec/postfix/master -w (postfix,90064,680,0.0,2538) qmgr -l -t unix -u (cvmfs,852264,28932,0.0,2548) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,852224,32812,0.0,2563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,852064,31160,0.0,2615) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,852340,29276,0.0,2635) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,925772,33332,0.0,2660) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,925796,32476,0.0,2749) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,925860,31216,0.0,2779) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,925780,33196,0.0,2798) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,4192724,116344,0.7,2830) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (root,126392,432,0.0,2859) /usr/sbin/crond -n (ntp,29960,844,0.0,2868) /usr/sbin/ntpd -u ntp:ntp -g (root,0,0,0.0,3064) [kworker/14:0] (root,574284,2336,0.0,3281) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,93236,4,0.0,3339) login -- i3admin (ganglia,220728,3592,0.0,3343) /usr/sbin/gmond (root,0,0,0.0,3613) [kworker/2:0] (root,0,0,0.0,4755) [kworker/u33:2] (root,113284,1484,0.0,5972) /bin/bash /bin/run-parts /etc/cron.daily (root,352000,21860,0.0,5976) /usr/bin/python -tt /usr/sbin/yum-cron (root,113644,980,0.0,5977) awk -v progname=/etc/cron.daily/0yum-daily.cron progname { .... print progname ":\n" .... progname=""; ... } ... { print; } (root,0,0,0.0,7694) [kworker/10:0] (root,0,0,0.0,8574) [kworker/11:1] (root,0,0,0.0,8695) [bioset] (root,0,0,0.0,8696) [xfsalloc] (root,0,0,0.0,8697) [xfs_mru_cache] (root,0,0,0.0,8724) [bioset] (root,0,0,0.0,9604) [kworker/8:2] (root,0,0,0.0,10789) [kworker/15:1] (root,0,0,0.0,10841) [kworker/15:2] (root,0,0,0.0,11542) [kworker/u34:1] (root,0,0,0.0,11902) [kworker/12:2] (root,0,0,0.0,12053) [kworker/u32:2] (root,0,0,0.0,12214) [kworker/9:1] (root,0,0,0.0,13597) [kworker/10:2] (root,0,0,0.0,13830) [kworker/5:0] (root,0,0,0.0,13885) [kworker/4:2] (i3admin,12752,0,0.0,14064) -bash (root,131516,4,0.0,14081) sudo su - (root,81988,0,0.0,14082) su - (root,12748,8,0.0,14083) -bash (root,0,0,0.0,14406) [kworker/13:2] (root,102988,872,0.0,15151) /sbin/dhclient -1 -q -lf /var/lib/dhclient/dhclient--bond0.lease -pf /var/run/dhclient-bond0.pid -H gzk-2 bond0 (root,125488,1132,0.0,15264) /usr/sbin/anacron -s (root,0,0,0.0,15805) [kworker/3:2] (root,0,0,0.0,15983) [kworker/12:1] (root,0,0,0.0,16308) [kworker/5:2] (root,0,0,0.0,16535) [kworker/14:1] (root,0,0,0.0,16905) [kworker/6:2] (root,0,0,0.0,17044) [kworker/10:1H] (postfix,89908,4028,0.0,17282) pickup -l -t unix -u (root,302344,41660,0.0,17833) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,0,0,0.0,18076) [nv_queue] (root,0,0,0.0,18077) [nv_queue] (root,0,0,0.0,18099) [UVM global queu] (root,0,0,0.0,18100) [UVM deferred re] (root,0,0,0.0,18102) [UVM Tools Event] (root,0,0,0.0,18308) [kworker/6:0] (root,0,0,0.0,18390) [kworker/1:1] (root,0,0,0.0,18642) [kworker/4:0] (root,0,0,0.0,19160) [kworker/8:1] (root,0,0,0.0,19209) [kworker/8:0] (root,0,0,0.0,19385) [kworker/9:0] (root,0,0,0.0,20000) [kworker/5:1] (root,0,0,0.0,20087) [kworker/u33:1] (root,0,0,0.0,21390) [kworker/u32:0] (root,0,0,0.0,22415) [kworker/7:2] (root,0,0,0.0,22955) [kworker/1:2] (root,209064,1616,0.0,23403) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,1516,0.0,23405) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,212132,2520,0.0,23409) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,24171) [kworker/4:1] (root,0,0,0.0,24172) [kworker/13:0] (root,0,0,0.0,24499) [kworker/13:1] (root,0,0,0.0,25064) [kworker/7:0] (root,0,0,0.0,25512) [irq/98-nvidia] (root,0,0,0.0,25513) [nvidia] (root,0,0,0.8,25514) [nv_queue] (root,0,0,0.0,25545) [irq/99-nvidia] (root,0,0,0.0,25546) [nvidia] (root,0,0,0.6,25547) [nv_queue] (root,0,0,0.0,25555) [irq/100-nvidia] (root,0,0,0.0,25556) [nvidia] (root,0,0,0.5,25557) [nv_queue] (root,0,0,0.0,25558) [irq/101-nvidia] (root,0,0,0.0,25559) [nvidia] (root,0,0,0.6,25560) [nv_queue] (root,0,0,0.0,25561) [irq/102-nvidia] (root,0,0,0.0,25562) [nvidia] (root,0,0,0.5,25563) [nv_queue] (root,0,0,0.0,25564) [irq/103-nvidia] (root,0,0,0.0,25565) [nvidia] (root,0,0,0.5,25566) [nv_queue] (root,0,0,0.0,25567) [irq/104-nvidia] (root,0,0,0.0,25568) [nvidia] (root,0,0,0.5,25569) [nv_queue] (root,0,0,0.0,25570) [irq/105-nvidia] (root,0,0,0.0,25571) [nvidia] (root,0,0,0.5,25572) [nv_queue] (condor,75284,3160,0.0,25684) /usr/sbin/condor_master -f (root,27892,4864,0.0,25693) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,63760,2648,0.0,25695) condor_shared_port -f (condor,1039628,979680,0.1,25724) condor_startd -f (condor,34059816,8296,0.1,25915) GPUs_MONITOR (root,0,0,0.0,25917) [UVM GPU1 BH] (root,0,0,0.0,25918) [UVM GPU2 BH] (root,0,0,0.0,25919) [UVM GPU3 BH] (root,0,0,0.0,25920) [UVM GPU4 BH] (root,0,0,0.0,25921) [UVM GPU5 BH] (root,0,0,0.0,25922) [UVM GPU6 BH] (root,0,0,0.0,25923) [UVM GPU7 BH] (root,0,0,0.0,25924) [UVM GPU8 BH] (root,108056,360,0.0,25979) sleep 15 (root,8632,660,0.0,25980) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,25981) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,25997) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,25998) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,27196) [kworker/3:0] (cvmfs,60916,720,0.0,27819) /usr/bin/cvmfs2 __cachemgr__ . 8 9 41943040000 20971520000 0 3 -1 : (cvmfs,71340,2088,0.0,27822) /usr/bin/cvmfs2 __cachemgr__ . 8 9 41943040000 20971520000 0 3 -1 : (cvmfs,1327456,28232,0.0,27854) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,60916,720,0.0,27919) /usr/bin/cvmfs2 __cachemgr__ . 8 9 1048576000 524288000 0 3 -1 : (cvmfs,71852,2464,0.0,27923) /usr/bin/cvmfs2 __cachemgr__ . 8 9 1048576000 524288000 0 3 -1 : (cvmfs,884792,27768,0.0,27954) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,884796,27736,0.0,27967) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,884796,29624,0.0,27983) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,4151740,112584,0.0,27991) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,811200,23824,0.0,28006) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,811080,23568,0.0,28019) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,811200,29300,0.0,28032) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,811200,23416,0.0,28045) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,884816,24916,0.0,28058) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (root,0,0,0.0,29419) [kworker/2:1] (root,0,0,0.0,30292) [kworker/u32:1] (root,45364,760,0.0,31760) /usr/lib/systemd/systemd-udevd (root,0,0,0.0,32676) [kworker/11:2] (root,0,0,0.0,32679) [kworker/0:2] (root,0,0,0.0,32711) [kworker/7:1]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b1a45a4f05aae993da968ef63d
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2.chtc.wisc.edu AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43872,3104,0.0,1) /usr/lib/systemd/systemd --system --deserialize 17 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.1,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,109) [kswapd0] (root,0,0,0.0,110) [kswapd1] (root,0,0,0.0,111) [ksmd] (root,0,0,0.0,112) [khugepaged] (root,0,0,0.0,113) [crypto] (root,0,0,0.0,121) [kthrotld] (root,0,0,0.0,125) [kmpath_rdacd] (root,0,0,0.0,126) [kaluad] (root,0,0,0.0,143) [kpsmoused] (root,0,0,0.0,144) [ipv6_addrconf] (root,0,0,0.0,158) [deferwq] (root,0,0,0.0,203) [kauditd] (root,0,0,0.0,354) [kworker/1:1] (root,0,0,0.0,397) [ata_sff] (root,0,0,0.0,405) [scsi_eh_0] (root,0,0,0.0,406) [scsi_tmf_0] (root,0,0,0.0,407) [scsi_eh_1] (root,0,0,0.0,408) [scsi_tmf_1] (root,0,0,0.0,409) [scsi_eh_2] (root,0,0,0.0,410) [scsi_tmf_2] (root,0,0,0.0,411) [scsi_eh_3] (root,0,0,0.0,412) [scsi_tmf_3] (root,0,0,0.0,415) [kworker/14:2] (root,0,0,0.0,416) [scsi_eh_4] (root,0,0,0.0,417) [scsi_tmf_4] (root,0,0,0.0,418) [scsi_eh_5] (root,0,0,0.0,419) [scsi_tmf_5] (root,0,0,0.0,420) [scsi_eh_6] (root,0,0,0.0,421) [scsi_tmf_6] (root,0,0,0.0,422) [scsi_eh_7] (root,0,0,0.0,423) [scsi_tmf_7] (root,0,0,0.0,424) [scsi_eh_8] (root,0,0,0.0,425) [scsi_tmf_8] (root,0,0,0.0,426) [scsi_eh_9] (root,0,0,0.0,427) [scsi_tmf_9] (root,0,0,0.0,434) [ttm_swap] (root,0,0,0.0,535) [kdmflush] (root,0,0,0.0,536) [bioset] (root,0,0,0.0,546) [kdmflush] (root,0,0,0.0,547) [bioset] (root,0,0,0.0,550) [kworker/0:1H] (root,0,0,0.0,552) [kworker/15:1H] (root,0,0,0.0,553) [kworker/1:1H] (root,0,0,0.0,568) [jbd2/dm-0-8] (root,0,0,0.0,569) [ext4-rsv-conver] (root,0,0,0.0,600) [kworker/9:1] (root,39460,6116,0.0,650) /usr/lib/systemd/systemd-journald (root,0,0,1.2,727) [kipmi0] (root,0,0,0.0,735) [kworker/13:1H] (root,0,0,0.0,742) [kworker/7:1H] (root,0,0,0.0,765) [kworker/9:1H] (root,0,0,0.0,784) [kworker/3:1H] (root,0,0,0.0,792) [kvm-irqfd-clean] (root,198572,380,0.0,793) /usr/sbin/lvmetad -f (root,0,0,0.0,867) [kdmflush] (root,0,0,0.0,868) [bioset] (root,0,0,0.0,869) [kdmflush] (root,0,0,0.0,871) [bioset] (root,0,0,0.0,873) [kdmflush] (root,0,0,0.0,874) [bioset] (root,0,0,0.0,876) [kdmflush] (root,0,0,0.0,877) [bioset] (root,0,0,0.0,886) [kworker/5:1H] (root,0,0,0.0,887) [kworker/4:1H] (root,0,0,0.0,888) [kworker/12:1H] (root,0,0,0.0,889) [kworker/8:1H] (root,0,0,0.0,901) [jbd2/sda1-8] (root,0,0,0.0,902) [ext4-rsv-conver] (root,0,0,0.0,937) [kworker/14:1H] (root,0,0,0.0,947) [jbd2/dm-3-8] (root,0,0,0.0,948) [ext4-rsv-conver] (root,0,0,0.0,949) [kworker/6:0] (root,0,0,0.0,970) [jbd2/dm-2-8] (root,0,0,0.0,971) [ext4-rsv-conver] (root,0,0,0.0,978) [jbd2/dm-4-8] (root,0,0,0.0,979) [ext4-rsv-conver] (root,0,0,0.0,1018) [jbd2/dm-5-8] (root,0,0,0.0,1019) [ext4-rsv-conver] (dbus,58232,1032,0.0,1038) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612356,1212,0.0,1065) /usr/lib/polkit-1/polkitd --no-debug (root,21716,528,0.0,1077) /usr/sbin/irqbalance --foreground (root,52900,592,0.0,1078) /usr/sbin/smartd -n -q never (root,27360,1076,0.0,1081) /usr/lib/systemd/systemd-logind (root,0,0,0.0,1089) [kworker/6:1H] (root,0,0,0.0,1268) [bond0] (root,0,0,0.0,1278) [kworker/2:1H] (root,10059916,120680,0.0,1379) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,0,0,0.0,1556) [kworker/11:1H] (root,112984,248,0.0,1648) /usr/sbin/sshd -D (root,27168,4,0.0,1653) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,801948,2100,0.0,1895) /usr/sbin/collectd (root,720396,6056,0.0,1921) /usr/sbin/rsyslogd -n (root,115708,968,0.0,1948) /bin/bash /sbin/condor_inspector (root,591424,5652,0.2,1985) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (cvmfs,1372396,9288,0.0,2140) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2369) [kworker/0:2] (root,89788,228,0.0,2488) /usr/libexec/postfix/master -w (postfix,90064,528,0.0,2538) qmgr -l -t unix -u (cvmfs,856140,1836,0.0,2548) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,852112,5176,0.0,2563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,852008,4880,0.0,2615) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,853224,4212,0.0,2635) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,925700,5944,0.0,2660) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,925736,10216,0.0,2749) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,925788,6344,0.0,2779) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,925704,8292,0.0,2798) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,4225948,117488,0.7,2830) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (root,126392,432,0.0,2859) /usr/sbin/crond -n (ntp,29960,768,0.0,2868) /usr/sbin/ntpd -u ntp:ntp -g (root,125488,1136,0.0,2873) /usr/sbin/anacron -s (root,0,0,0.0,3046) [kworker/3:2] (root,113284,1488,0.0,3049) /bin/bash /bin/run-parts /etc/cron.daily (root,352000,21848,0.0,3053) /usr/bin/python -tt /usr/sbin/yum-cron (root,113644,976,0.0,3054) awk -v progname=/etc/cron.daily/0yum-daily.cron progname { .... print progname ":\n" .... progname=""; ... } ... { print; } (root,0,0,0.0,3263) [kworker/0:0] (root,574284,2336,0.0,3281) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,93236,4,0.0,3339) login -- i3admin (ganglia,220728,3592,0.0,3343) /usr/sbin/gmond (root,0,0,0.0,4054) [kworker/u32:0] (root,0,0,0.0,4167) [kworker/5:1] (root,0,0,0.0,6210) [kworker/15:2] (root,0,0,0.0,6345) [kworker/1:0] (root,0,0,0.0,6720) [kworker/5:2] (root,0,0,0.0,7016) [kworker/2:0] (root,0,0,0.0,7487) [kworker/7:1] (root,0,0,0.0,7971) [kworker/0:1] (root,0,0,0.0,8695) [bioset] (root,0,0,0.0,8696) [xfsalloc] (root,0,0,0.0,8697) [xfs_mru_cache] (root,0,0,0.0,8724) [bioset] (postfix,89912,4032,0.0,9625) trivial-rewrite -n rewrite -t unix -u (postfix,90116,4276,0.0,9626) smtp -t unix -u (root,0,0,0.0,10052) [kworker/8:0] (root,0,0,0.0,10379) [kworker/u34:1] (root,0,0,0.0,10395) [kworker/12:2] (postfix,89948,4028,0.0,10815) bounce -z -n defer -t unix -u (root,108056,356,0.0,10906) sleep 15 (root,8632,660,0.0,10947) timeout -s 9 360 check_mk_agent (root,11696,1532,0.0,10948) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,10964) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,10965) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (postfix,89908,4024,0.0,12162) pickup -l -t unix -u (root,0,0,0.0,12221) [kworker/5:0] (root,0,0,0.0,12360) [kworker/14:0] (root,0,0,0.0,12797) [kworker/13:0] (i3admin,12752,0,0.0,14064) -bash (root,131516,4,0.0,14081) sudo su - (root,81988,0,0.0,14082) su - (root,12748,8,0.0,14083) -bash (root,102988,876,0.0,15151) /sbin/dhclient -1 -q -lf /var/lib/dhclient/dhclient--bond0.lease -pf /var/run/dhclient-bond0.pid -H gzk-2 bond0 (root,0,0,0.0,16168) [kworker/u34:2] (root,0,0,0.0,16506) [kworker/u33:1] (root,0,0,0.0,17044) [kworker/10:1H] (cvmfs,60820,4,0.0,17587) /usr/bin/cvmfs2 __cachemgr__ . 8 9 41943040000 20971520000 0 3 -1 : (cvmfs,73228,2724,0.0,17590) /usr/bin/cvmfs2 __cachemgr__ . 8 9 41943040000 20971520000 0 3 -1 : (cvmfs,1327384,4,0.0,17594) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,60820,4,0.0,17682) /usr/bin/cvmfs2 __cachemgr__ . 8 9 1048576000 524288000 0 3 -1 : (cvmfs,72048,1400,0.0,17685) /usr/bin/cvmfs2 __cachemgr__ . 8 9 1048576000 524288000 0 3 -1 : (cvmfs,811128,8,0.0,17692) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,810992,52,0.0,17705) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,811008,4,0.0,17718) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,811128,4,0.0,17732) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,737280,8,0.0,17745) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,811128,12,0.0,17760) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,2901176,2056,0.0,17764) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,737256,28,0.0,17783) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,810992,16,0.0,17796) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,302344,41632,0.0,17833) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,0,0,0.0,18076) [nv_queue] (root,0,0,0.0,18077) [nv_queue] (root,0,0,0.0,18099) [UVM global queu] (root,0,0,0.0,18100) [UVM deferred re] (root,0,0,0.0,18102) [UVM Tools Event] (root,0,0,0.0,19324) [kworker/u33:2] (root,0,0,0.0,20645) [kworker/12:0] (root,0,0,0.0,21194) [kworker/6:2] (root,0,0,0.0,21831) [kworker/3:0] (root,0,0,0.0,22146) [kworker/11:0] (root,0,0,0.0,22506) [kworker/13:2] (root,0,0,0.0,23263) [kworker/7:0] (root,209064,1640,0.0,23403) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,1436,0.0,23405) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,212132,2512,0.0,23409) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,23571) [kworker/12:1] (root,0,0,0.0,24418) [kworker/9:0] (root,0,0,0.0,25252) [kworker/2:2] (root,0,0,0.0,25512) [irq/98-nvidia] (root,0,0,0.0,25513) [nvidia] (root,0,0,1.2,25514) [nv_queue] (root,0,0,0.0,25545) [irq/99-nvidia] (root,0,0,0.0,25546) [nvidia] (root,0,0,0.9,25547) [nv_queue] (root,0,0,0.0,25555) [irq/100-nvidia] (root,0,0,0.0,25556) [nvidia] (root,0,0,0.9,25557) [nv_queue] (root,0,0,0.0,25558) [irq/101-nvidia] (root,0,0,0.0,25559) [nvidia] (root,0,0,0.9,25560) [nv_queue] (root,0,0,0.0,25561) [irq/102-nvidia] (root,0,0,0.0,25562) [nvidia] (root,0,0,0.7,25563) [nv_queue] (root,0,0,0.0,25564) [irq/103-nvidia] (root,0,0,0.0,25565) [nvidia] (root,0,0,0.7,25566) [nv_queue] (root,0,0,0.0,25567) [irq/104-nvidia] (root,0,0,0.0,25568) [nvidia] (root,0,0,0.8,25569) [nv_queue] (root,0,0,0.0,25570) [irq/105-nvidia] (root,0,0,0.0,25571) [nvidia] (root,0,0,0.8,25572) [nv_queue] (condor,75284,3160,0.0,25684) /usr/sbin/condor_master -f (root,27892,4704,0.0,25693) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,63760,2620,0.0,25695) condor_shared_port -f (condor,650220,590200,0.1,25724) condor_startd -f (root,0,0,0.0,25824) [kworker/10:1] (condor,34059816,8296,0.1,25915) GPUs_MONITOR (root,0,0,0.0,25917) [UVM GPU1 BH] (root,0,0,0.0,25918) [UVM GPU2 BH] (root,0,0,0.0,25919) [UVM GPU3 BH] (root,0,0,0.0,25920) [UVM GPU4 BH] (root,0,0,0.0,25921) [UVM GPU5 BH] (root,0,0,0.0,25922) [UVM GPU6 BH] (root,0,0,0.0,25923) [UVM GPU7 BH] (root,0,0,0.0,25924) [UVM GPU8 BH] (root,0,0,0.0,27734) [kworker/11:1] (root,0,0,0.0,29087) [kworker/15:1] (root,0,0,0.0,29464) [kworker/8:1] (root,0,0,0.0,30272) [kworker/4:1] (root,0,0,0.0,31005) [kworker/u32:2] (root,0,0,0.0,31144) [kworker/4:0] (root,0,0,0.0,31676) [kworker/10:2] (root,45364,760,0.0,31760) /usr/lib/systemd/systemd-udevd
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b1a45a4f05aae993da5fda8533
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2.chtc.wisc.edu AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43872,3156,0.0,1) /usr/lib/systemd/systemd --system --deserialize 17 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.1,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,109) [kswapd0] (root,0,0,0.0,110) [kswapd1] (root,0,0,0.0,111) [ksmd] (root,0,0,0.0,112) [khugepaged] (root,0,0,0.0,113) [crypto] (root,0,0,0.0,121) [kthrotld] (root,0,0,0.0,125) [kmpath_rdacd] (root,0,0,0.0,126) [kaluad] (root,0,0,0.0,143) [kpsmoused] (root,0,0,0.0,144) [ipv6_addrconf] (root,0,0,0.0,158) [deferwq] (root,0,0,0.0,203) [kauditd] (root,0,0,0.0,397) [ata_sff] (root,0,0,0.0,405) [scsi_eh_0] (root,0,0,0.0,406) [scsi_tmf_0] (root,0,0,0.0,407) [scsi_eh_1] (root,0,0,0.0,408) [scsi_tmf_1] (root,0,0,0.0,409) [scsi_eh_2] (root,0,0,0.0,410) [scsi_tmf_2] (root,0,0,0.0,411) [scsi_eh_3] (root,0,0,0.0,412) [scsi_tmf_3] (root,0,0,0.0,416) [scsi_eh_4] (root,0,0,0.0,417) [scsi_tmf_4] (root,0,0,0.0,418) [scsi_eh_5] (root,0,0,0.0,419) [scsi_tmf_5] (root,0,0,0.0,420) [scsi_eh_6] (root,0,0,0.0,421) [scsi_tmf_6] (root,0,0,0.0,422) [scsi_eh_7] (root,0,0,0.0,423) [scsi_tmf_7] (root,0,0,0.0,424) [scsi_eh_8] (root,0,0,0.0,425) [scsi_tmf_8] (root,0,0,0.0,426) [scsi_eh_9] (root,0,0,0.0,427) [scsi_tmf_9] (root,0,0,0.0,434) [ttm_swap] (root,0,0,0.0,535) [kdmflush] (root,0,0,0.0,536) [bioset] (root,0,0,0.0,546) [kdmflush] (root,0,0,0.0,547) [bioset] (root,0,0,0.0,550) [kworker/0:1H] (root,0,0,0.0,552) [kworker/15:1H] (root,0,0,0.0,553) [kworker/1:1H] (root,0,0,0.0,568) [jbd2/dm-0-8] (root,0,0,0.0,569) [ext4-rsv-conver] (root,47652,8996,0.0,650) /usr/lib/systemd/systemd-journald (root,0,0,1.2,727) [kipmi0] (root,0,0,0.0,735) [kworker/13:1H] (root,0,0,0.0,742) [kworker/7:1H] (root,0,0,0.0,765) [kworker/9:1H] (root,0,0,0.0,784) [kworker/3:1H] (root,0,0,0.0,792) [kvm-irqfd-clean] (root,198572,448,0.0,793) /usr/sbin/lvmetad -f (root,0,0,0.0,867) [kdmflush] (root,0,0,0.0,868) [bioset] (root,0,0,0.0,869) [kdmflush] (root,0,0,0.0,871) [bioset] (root,0,0,0.0,873) [kdmflush] (root,0,0,0.0,874) [bioset] (root,0,0,0.0,876) [kdmflush] (root,0,0,0.0,877) [bioset] (root,0,0,0.0,886) [kworker/5:1H] (root,0,0,0.0,887) [kworker/4:1H] (root,0,0,0.0,888) [kworker/12:1H] (root,0,0,0.0,889) [kworker/8:1H] (root,0,0,0.0,901) [jbd2/sda1-8] (root,0,0,0.0,902) [ext4-rsv-conver] (root,0,0,0.0,937) [kworker/14:1H] (root,0,0,0.0,947) [jbd2/dm-3-8] (root,0,0,0.0,948) [ext4-rsv-conver] (root,0,0,0.0,970) [jbd2/dm-2-8] (root,0,0,0.0,971) [ext4-rsv-conver] (root,0,0,0.0,978) [jbd2/dm-4-8] (root,0,0,0.0,979) [ext4-rsv-conver] (root,0,0,0.0,1018) [jbd2/dm-5-8] (root,0,0,0.0,1019) [ext4-rsv-conver] (dbus,58232,1376,0.0,1038) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612356,1212,0.0,1065) /usr/lib/polkit-1/polkitd --no-debug (root,21716,692,0.0,1077) /usr/sbin/irqbalance --foreground (root,52900,592,0.0,1078) /usr/sbin/smartd -n -q never (root,27360,1040,0.0,1081) /usr/lib/systemd/systemd-logind (root,0,0,0.0,1089) [kworker/6:1H] (root,0,0,0.0,1268) [bond0] (root,0,0,0.0,1278) [kworker/2:1H] (root,10057868,122684,0.0,1379) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,0,0,0.0,1556) [kworker/11:1H] (root,112984,360,0.0,1648) /usr/sbin/sshd -D (root,27168,4,0.0,1653) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,801948,1976,0.0,1895) /usr/sbin/collectd (root,720420,10096,0.0,1921) /usr/sbin/rsyslogd -n (root,115708,996,0.0,1948) /bin/bash /sbin/condor_inspector (root,591424,5700,0.2,1985) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (cvmfs,1370488,3764,0.0,2140) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,89788,232,0.0,2488) /usr/libexec/postfix/master -w (postfix,90064,516,0.0,2538) qmgr -l -t unix -u (cvmfs,854232,2476,0.0,2548) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,852112,4844,0.0,2563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,852008,4556,0.0,2615) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,852696,4340,0.0,2635) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,925700,5568,0.0,2660) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,925736,9756,0.0,2749) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,926292,4048,0.0,2779) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,925704,7928,0.0,2798) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (cvmfs,4231284,108280,0.7,2830) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (root,126392,432,0.0,2859) /usr/sbin/crond -n (ntp,29960,768,0.0,2868) /usr/sbin/ntpd -u ntp:ntp -g (root,0,0,0.0,3256) [kworker/2:2] (root,574284,2336,0.0,3281) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,93236,4,0.0,3339) login -- i3admin (ganglia,220728,3880,0.0,3343) /usr/sbin/gmond (root,0,0,0.0,3550) [kworker/9:0] (root,0,0,0.0,5014) [kworker/6:0] (root,0,0,0.0,5038) [kworker/12:2] (root,0,0,0.0,5583) [kworker/7:0] (root,125488,1132,0.0,5898) /usr/sbin/anacron -s (root,0,0,0.0,6416) [kworker/8:1] (root,113284,1484,0.0,7540) /bin/bash /bin/run-parts /etc/cron.daily (root,352000,21848,0.0,7544) /usr/bin/python -tt /usr/sbin/yum-cron (root,113644,980,0.0,7545) awk -v progname=/etc/cron.daily/0yum-daily.cron progname { .... print progname ":\n" .... progname=""; ... } ... { print; } (root,0,0,0.0,7920) [kworker/2:1] (root,0,0,0.0,8545) [kworker/u33:0] (root,0,0,0.0,8695) [bioset] (root,0,0,0.0,8696) [xfsalloc] (root,0,0,0.0,8697) [xfs_mru_cache] (root,0,0,0.0,8724) [bioset] (root,0,0,0.0,9189) [kworker/1:1] (root,0,0,0.0,9289) [kworker/4:2] (root,0,0,0.0,10486) [kworker/0:0] (root,0,0,0.0,10675) [kworker/u32:2] (root,0,0,0.0,10760) [kworker/12:1] (root,0,0,0.0,10953) [kworker/10:1] (root,0,0,0.0,11159) [kworker/4:0] (slot-1,46112,6556,0.0,11764) condor_starter -f -a slot1_6 sub-simprod.icecube.wisc.edu (slot-1,11704,1564,0.0,11837) /bin/sh /scratch/condor/dir_28813/glidein/execute.faca298a-5add-11ed-bc28-0cc47a857a10/dir_11764/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,457588,21528,0.0,11873) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1456,0.0,11970) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//icetray-env icetray/v1.4.1 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/detector.py --outputfile=IC86.2020_corsika.022183.050647.i3.zst --MCType=corsika_weighted --SkipKeys=SignalI3MCPEs,BackgroundI3MCPESeriesMap,I3MCPESeriesMap,I3MCPESeriesMapWithoutNoise --MCPrescale=1000 --LowMem --HistogramFilename=histo.pkl --RunID=18350648 --nproc=100000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --inputfile=Hits_IC86.2020_corsika.022183.050647.i3.zst --seed=22183 --procnum=50647 (slot-1,11704,1544,0.0,12063) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/RHEL_7_x86_64/metaprojects/icetray/v1.4.1/env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/detector.py --outputfile=IC86.2020_corsika.022183.050647.i3.zst --MCType=corsika_weighted --SkipKeys=SignalI3MCPEs,BackgroundI3MCPESeriesMap,I3MCPESeriesMap,I3MCPESeriesMapWithoutNoise --MCPrescale=1000 --LowMem --HistogramFilename=histo.pkl --RunID=18350648 --nproc=100000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --inputfile=Hits_IC86.2020_corsika.022183.050647.i3.zst --seed=22183 --procnum=50647 (slot-1,1694232,1307500,96.5,12071) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/detector.py --outputfile=IC86.2020_corsika.022183.050647.i3.zst --MCType=corsika_weighted --SkipKeys=SignalI3MCPEs,BackgroundI3MCPESeriesMap,I3MCPESeriesMap,I3MCPESeriesMapWithoutNoise --MCPrescale=1000 --LowMem --HistogramFilename=histo.pkl --RunID=18350648 --nproc=100000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --inputfile=Hits_IC86.2020_corsika.022183.050647.i3.zst --seed=22183 --procnum=50647 (root,0,0,0.0,12785) [kworker/u34:0] (root,0,0,0.0,13204) [kworker/9:2] (root,0,0,0.0,14051) [kworker/6:2] (i3admin,12752,0,0.0,14064) -bash (root,131516,4,0.0,14081) sudo su - (root,81988,0,0.0,14082) su - (root,12748,8,0.0,14083) -bash (root,0,0,0.0,14219) [kworker/u33:1] (postfix,89908,4024,0.0,14771) pickup -l -t unix -u (root,102988,876,0.0,15151) /sbin/dhclient -1 -q -lf /var/lib/dhclient/dhclient--bond0.lease -pf /var/run/dhclient-bond0.pid -H gzk-2 bond0 (root,0,0,0.0,15437) [kworker/13:1] (root,0,0,0.0,16133) [kworker/15:1] (slot-1,4368,356,0.0,16163) sleep 300 (root,0,0,0.0,17044) [kworker/10:1H] (root,0,0,0.0,17114) [kworker/12:0] (root,108056,356,0.0,17299) sleep 15 (root,8632,660,0.0,17304) timeout -s 9 360 check_mk_agent (root,11696,1540,0.0,17305) /bin/bash /bin/check_mk_agent (root,51732,1696,0.0,17321) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,944,0.0,17322) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (cvmfs,60820,4,0.0,17587) /usr/bin/cvmfs2 __cachemgr__ . 8 9 41943040000 20971520000 0 3 -1 : (cvmfs,73228,2728,0.0,17590) /usr/bin/cvmfs2 __cachemgr__ . 8 9 41943040000 20971520000 0 3 -1 : (cvmfs,1327384,4,0.0,17594) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,60820,4,0.0,17682) /usr/bin/cvmfs2 __cachemgr__ . 8 9 1048576000 524288000 0 3 -1 : (cvmfs,72048,1816,0.0,17685) /usr/bin/cvmfs2 __cachemgr__ . 8 9 1048576000 524288000 0 3 -1 : (cvmfs,811128,8,0.0,17692) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,810992,52,0.0,17705) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,811008,4,0.0,17718) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,811128,4,0.0,17732) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,737280,8,0.0,17745) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,811128,12,0.0,17760) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,2901176,2056,0.0,17764) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,737256,28,0.0,17783) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,810992,16,0.0,17796) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,302344,39428,0.0,17833) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,0,0,0.0,18076) [nv_queue] (root,0,0,0.0,18077) [nv_queue] (root,0,0,0.0,18099) [UVM global queu] (root,0,0,0.0,18100) [UVM deferred re] (root,0,0,0.0,18102) [UVM Tools Event] (root,0,0,0.0,18669) [kworker/9:1] (root,0,0,0.0,18700) [kworker/10:2] (root,0,0,0.0,20027) [kworker/14:0] (root,0,0,0.0,20530) [kworker/8:2] (root,0,0,0.0,20668) [kworker/6:1] (root,0,0,0.0,21089) [kworker/u34:2] (root,0,0,0.0,21547) [kworker/13:2] (root,0,0,0.0,22400) [kworker/3:0] (root,0,0,0.0,22410) [kworker/7:2] (slot-1,46112,6556,0.0,22481) condor_starter -f -a slot1_8 sub-simprod.icecube.wisc.edu (slot-1,11704,1564,0.0,22485) /bin/sh /scratch/condor/dir_28813/glidein/execute.faca298a-5add-11ed-bc28-0cc47a857a10/dir_22481/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,457588,21604,0.0,22522) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1460,0.0,22661) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//icetray-env icetray/v1.4.1 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/detector.py --outputfile=IC86.2020_corsika.022183.049584.i3.zst --MCType=corsika_weighted --SkipKeys=SignalI3MCPEs,BackgroundI3MCPESeriesMap,I3MCPESeriesMap,I3MCPESeriesMapWithoutNoise --MCPrescale=1000 --LowMem --HistogramFilename=histo.pkl --RunID=18349585 --nproc=100000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --inputfile=Hits_IC86.2020_corsika.022183.049584.i3.zst --seed=22183 --procnum=49584 (slot-1,11704,1548,0.0,22756) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/RHEL_7_x86_64/metaprojects/icetray/v1.4.1/env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/detector.py --outputfile=IC86.2020_corsika.022183.049584.i3.zst --MCType=corsika_weighted --SkipKeys=SignalI3MCPEs,BackgroundI3MCPESeriesMap,I3MCPESeriesMap,I3MCPESeriesMapWithoutNoise --MCPrescale=1000 --LowMem --HistogramFilename=histo.pkl --RunID=18349585 --nproc=100000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --inputfile=Hits_IC86.2020_corsika.022183.049584.i3.zst --seed=22183 --procnum=49584 (slot-1,1701524,1314828,96.6,22763) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/detector.py --outputfile=IC86.2020_corsika.022183.049584.i3.zst --MCType=corsika_weighted --SkipKeys=SignalI3MCPEs,BackgroundI3MCPESeriesMap,I3MCPESeriesMap,I3MCPESeriesMapWithoutNoise --MCPrescale=1000 --LowMem --HistogramFilename=histo.pkl --RunID=18349585 --nproc=100000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --inputfile=Hits_IC86.2020_corsika.022183.049584.i3.zst --seed=22183 --procnum=49584 (slot-1,46112,6556,0.0,22933) condor_starter -f -a slot1_9 sub-simprod.icecube.wisc.edu (slot-1,11704,1568,0.0,23031) /bin/sh /scratch/condor/dir_28813/glidein/execute.faca298a-5add-11ed-bc28-0cc47a857a10/dir_22933/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,457588,21472,0.0,23070) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs (slot-1,11700,1460,0.0,23075) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//icetray-env icetray/v1.4.1 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/detector.py --outputfile=IC86.2020_corsika.022183.049643.i3.zst --MCType=corsika_weighted --SkipKeys=SignalI3MCPEs,BackgroundI3MCPESeriesMap,I3MCPESeriesMap,I3MCPESeriesMapWithoutNoise --MCPrescale=1000 --LowMem --HistogramFilename=histo.pkl --RunID=18349644 --nproc=100000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --inputfile=Hits_IC86.2020_corsika.022183.049643.i3.zst --seed=22183 --procnum=49643 (slot-1,11704,1548,0.0,23168) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/RHEL_7_x86_64/metaprojects/icetray/v1.4.1/env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/detector.py --outputfile=IC86.2020_corsika.022183.049643.i3.zst --MCType=corsika_weighted --SkipKeys=SignalI3MCPEs,BackgroundI3MCPESeriesMap,I3MCPESeriesMap,I3MCPESeriesMapWithoutNoise --MCPrescale=1000 --LowMem --HistogramFilename=histo.pkl --RunID=18349644 --nproc=100000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --inputfile=Hits_IC86.2020_corsika.022183.049643.i3.zst --seed=22183 --procnum=49643 (slot-1,1698056,1311120,96.7,23175) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/detector.py --outputfile=IC86.2020_corsika.022183.049643.i3.zst --MCType=corsika_weighted --SkipKeys=SignalI3MCPEs,BackgroundI3MCPESeriesMap,I3MCPESeriesMap,I3MCPESeriesMapWithoutNoise --MCPrescale=1000 --LowMem --HistogramFilename=histo.pkl --RunID=18349644 --nproc=100000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --inputfile=Hits_IC86.2020_corsika.022183.049643.i3.zst --seed=22183 --procnum=49643 (root,209140,1588,0.0,23403) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,1524,0.0,23405) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,212132,2520,0.0,23409) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,23564) [kworker/1:0] (slot-1,46176,6592,0.0,23677) condor_starter -f -a slot1_3 sub-simprod-2.icecube.wisc.edu (slot-1,46176,6588,0.0,23678) condor_starter -f -a slot1_4 sub-simprod-2.icecube.wisc.edu (slot-1,46176,6588,0.0,23680) condor_starter -f -a slot1_5 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1568,0.0,23727) /bin/sh /scratch/condor/dir_28813/glidein/execute.faca298a-5add-11ed-bc28-0cc47a857a10/dir_23678/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11704,1568,0.0,23729) /bin/sh /scratch/condor/dir_28813/glidein/execute.faca298a-5add-11ed-bc28-0cc47a857a10/dir_23677/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11704,1568,0.0,23751) /bin/sh /scratch/condor/dir_28813/glidein/execute.faca298a-5add-11ed-bc28-0cc47a857a10/dir_23680/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,457940,21932,0.1,23801) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,457940,21928,0.2,23803) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,457940,21940,0.2,23839) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11700,1532,0.0,24160) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/icetray-env icetray/v1.4.1 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=14181 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (root,0,0,0.0,24180) [kworker/u32:1] (slot-1,11700,1532,0.0,24208) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/icetray-env icetray/v1.4.1 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=13554 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,11836,1556,0.0,24325) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/RHEL_7_x86_64/metaprojects/icetray/v1.4.1/env-shell.sh /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=14181 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,13349932,948624,89.5,24332) python3 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=14181 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,11836,1556,0.0,24356) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/RHEL_7_x86_64/metaprojects/icetray/v1.4.1/env-shell.sh /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=13554 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,13467216,1033308,89.7,24363) python3 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=13554 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,11700,1532,0.0,24364) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/icetray-env icetray/v1.4.1 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=13545 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,11832,1584,0.0,24457) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/RHEL_7_x86_64/metaprojects/icetray/v1.4.1/env-shell.sh /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=13545 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,13337616,911700,89.8,24464) python3 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=13545 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,46172,6588,0.0,24810) condor_starter -f -a slot1_2 sub-1.icecube.wisc.edu (slot-1,11704,1512,0.0,24814) bash /scratch/condor/dir_28813/glidein/execute.faca298a-5add-11ed-bc28-0cc47a857a10/dir_24810/condor_exec.exe 16 dat047 bfr-v2-x5-best (slot-1,11704,1556,0.0,24831) bash ./llh.sh 16 ./ bfr-v2-x5-best (root,0,0,0.0,25244) [kworker/14:2] (slot-1,46180,6592,0.0,25292) condor_starter -f -a slot1_1 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1568,0.0,25296) /bin/sh /scratch/condor/dir_28813/glidein/execute.faca298a-5add-11ed-bc28-0cc47a857a10/dir_25292/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,390072,21884,0.4,25332) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11700,1464,0.0,25335) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//icetray-env icetray/v1.4.1 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=39581 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=as.9 --oversize=5 --outfile=Hits_IC86.2020_corsika.022183.039581.i3.zst --summary-file=Summary_IC86.2020_corsika.022183.039581.i3.json (slot-1,44392608,3141792,71.6,25341) ./llh/llh 16 (slot-1,11704,1552,0.0,25439) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/RHEL_7_x86_64/metaprojects/icetray/v1.4.1/env-shell.sh /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=39581 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=as.9 --oversize=5 --outfile=Hits_IC86.2020_corsika.022183.039581.i3.zst --summary-file=Summary_IC86.2020_corsika.022183.039581.i3.json (slot-1,12421088,756744,13.0,25446) python3 /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0//metaprojects/icetray/v1.4.1//simprod-scripts/resources/scripts/clsim_server_corsika.py --UseGPUs --proc-id=39581 --ntrays=8 --nevents=300000 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --min-energy=30000 --max-energy=1000000 --power-law-index=-2.65,-2.6,-2.6,-2.6,-2.6 --normalizations=5,2.25,1.1,1.2,1 --lower-cutoff-type=EnergyPerParticle --upper-cutoff-type=EnergyPerParticle --mu-bias-target=0.001 --efficiency=1 --bulk-ice=spice_bfr-v2 --hole-ice=as.9 --oversize=5 --outfile=Hits_IC86.2020_corsika.022183.039581.i3.zst --summary-file=Summary_IC86.2020_corsika.022183.039581.i3.json (root,0,0,0.2,25512) [irq/98-nvidia] (root,0,0,0.0,25513) [nvidia] (root,0,0,8.7,25514) [nv_queue] (root,0,0,0.2,25545) [irq/99-nvidia] (root,0,0,0.0,25546) [nvidia] (root,0,0,6.8,25547) [nv_queue] (root,0,0,0.2,25555) [irq/100-nvidia] (root,0,0,0.0,25556) [nvidia] (root,0,0,7.0,25557) [nv_queue] (root,0,0,0.2,25558) [irq/101-nvidia] (root,0,0,0.0,25559) [nvidia] (root,0,0,6.8,25560) [nv_queue] (root,0,0,0.3,25561) [irq/102-nvidia] (root,0,0,0.0,25562) [nvidia] (root,0,0,4.7,25563) [nv_queue] (root,0,0,0.3,25564) [irq/103-nvidia] (root,0,0,0.0,25565) [nvidia] (root,0,0,4.6,25566) [nv_queue] (root,0,0,0.3,25567) [irq/104-nvidia] (root,0,0,0.0,25568) [nvidia] (root,0,0,5.7,25569) [nv_queue] (root,0,0,0.2,25570) [irq/105-nvidia] (root,0,0,0.0,25571) [nvidia] (root,0,0,5.7,25572) [nv_queue] (condor,75284,7164,0.0,25684) /usr/sbin/condor_master -f (root,27760,6440,0.1,25693) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (condor,63696,6256,0.0,25695) condor_shared_port -f (condor,152152,95168,0.1,25724) condor_startd -f (condor,34059816,17048,0.1,25915) GPUs_MONITOR (root,0,0,0.0,25917) [UVM GPU1 BH] (root,0,0,0.0,25918) [UVM GPU2 BH] (root,0,0,0.0,25919) [UVM GPU3 BH] (root,0,0,0.0,25920) [UVM GPU4 BH] (root,0,0,0.0,25921) [UVM GPU5 BH] (root,0,0,0.0,25922) [UVM GPU6 BH] (root,0,0,0.0,25923) [UVM GPU7 BH] (root,0,0,0.0,25924) [UVM GPU8 BH] (root,0,0,0.0,26721) [kworker/5:1] (root,0,0,0.0,26723) [kworker/15:0] (root,0,0,0.0,26830) [kworker/3:1] (root,0,0,0.0,27930) [kworker/11:0] (condor,67120,8164,0.0,28813) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (slot-1,11692,1576,0.0,28839) /bin/sh glidein_start.sh (slot-1,61812,6544,0.0,28900) glideinExec/sbin/condor_master -f -r 20135 (slot-1,11796,1580,0.0,28901) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_9ea3b152-d36c-4d28-b080-454637b4bbfe.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20221102%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20221102T184003Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=2a632c4a33ea029695dd7552fa2179872156b743d2a95aa2d1a696144add192b (slot-1,24904,4912,0.0,28903) condor_procd -A /scratch/condor/dir_28813/glidein/log.faca298a-5add-11ed-bc28-0cc47a857a10/procd_address -L /scratch/condor/dir_28813/glidein/log.faca298a-5add-11ed-bc28-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,63388,8588,0.0,28904) condor_startd -f (root,0,0,0.0,29087) [kworker/11:1] (root,0,0,0.0,30791) [kworker/5:2] (root,0,0,0.0,31716) [kworker/u32:0] (root,45364,1028,0.0,31760) /usr/lib/systemd/systemd-udevd (root,0,0,0.0,32538) [kworker/0:1]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b1a45a4f05aae993da2084aa3e
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2.chtc.wisc.edu AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43800,3512,0.0,1) /usr/lib/systemd/systemd --system --deserialize 17 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.1,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,109) [kswapd0] (root,0,0,0.0,110) [kswapd1] (root,0,0,0.0,111) [ksmd] (root,0,0,0.0,112) [khugepaged] (root,0,0,0.0,113) [crypto] (root,0,0,0.0,121) [kthrotld] (root,0,0,0.0,125) [kmpath_rdacd] (root,0,0,0.0,126) [kaluad] (root,0,0,0.0,143) [kpsmoused] (root,0,0,0.0,144) [ipv6_addrconf] (root,0,0,0.0,158) [deferwq] (root,0,0,0.0,203) [kauditd] (root,0,0,0.0,397) [ata_sff] (root,0,0,0.0,405) [scsi_eh_0] (root,0,0,0.0,406) [scsi_tmf_0] (root,0,0,0.0,407) [scsi_eh_1] (root,0,0,0.0,408) [scsi_tmf_1] (root,0,0,0.0,409) [scsi_eh_2] (root,0,0,0.0,410) [scsi_tmf_2] (root,0,0,0.0,411) [scsi_eh_3] (root,0,0,0.0,412) [scsi_tmf_3] (root,0,0,0.0,416) [scsi_eh_4] (root,0,0,0.0,417) [scsi_tmf_4] (root,0,0,0.0,418) [scsi_eh_5] (root,0,0,0.0,419) [scsi_tmf_5] (root,0,0,0.0,420) [scsi_eh_6] (root,0,0,0.0,421) [scsi_tmf_6] (root,0,0,0.0,422) [scsi_eh_7] (root,0,0,0.0,423) [scsi_tmf_7] (root,0,0,0.0,424) [scsi_eh_8] (root,0,0,0.0,425) [scsi_tmf_8] (root,0,0,0.0,426) [scsi_eh_9] (root,0,0,0.0,427) [scsi_tmf_9] (root,0,0,0.0,434) [ttm_swap] (root,0,0,0.0,535) [kdmflush] (root,0,0,0.0,536) [bioset] (root,0,0,0.0,546) [kdmflush] (root,0,0,0.0,547) [bioset] (root,0,0,0.0,550) [kworker/0:1H] (root,0,0,0.0,552) [kworker/15:1H] (root,0,0,0.0,553) [kworker/1:1H] (root,0,0,0.0,568) [jbd2/dm-0-8] (root,0,0,0.0,569) [ext4-rsv-conver] (root,47652,9352,0.0,650) /usr/lib/systemd/systemd-journald (root,0,0,1.1,727) [kipmi0] (root,0,0,0.0,735) [kworker/13:1H] (root,0,0,0.0,742) [kworker/7:1H] (root,0,0,0.0,765) [kworker/9:1H] (root,0,0,0.0,784) [kworker/3:1H] (root,0,0,0.0,792) [kvm-irqfd-clean] (root,198572,552,0.0,793) /usr/sbin/lvmetad -f (root,0,0,0.0,867) [kdmflush] (root,0,0,0.0,868) [bioset] (root,0,0,0.0,869) [kdmflush] (root,0,0,0.0,871) [bioset] (root,0,0,0.0,873) [kdmflush] (root,0,0,0.0,874) [bioset] (root,0,0,0.0,876) [kdmflush] (root,0,0,0.0,877) [bioset] (root,0,0,0.0,886) [kworker/5:1H] (root,0,0,0.0,887) [kworker/4:1H] (root,0,0,0.0,888) [kworker/12:1H] (root,0,0,0.0,889) [kworker/8:1H] (root,0,0,0.0,901) [jbd2/sda1-8] (root,0,0,0.0,902) [ext4-rsv-conver] (root,0,0,0.0,917) [nv_queue] (root,0,0,0.0,918) [nv_queue] (root,0,0,0.0,937) [kworker/14:1H] (root,0,0,0.0,947) [jbd2/dm-3-8] (root,0,0,0.0,948) [ext4-rsv-conver] (root,0,0,0.0,950) [UVM global queu] (root,0,0,0.0,951) [UVM deferred re] (root,0,0,0.0,953) [UVM Tools Event] (root,0,0,0.0,963) [nvidia-modeset/] (root,0,0,0.0,964) [nvidia-modeset/] (root,0,0,0.0,970) [jbd2/dm-2-8] (root,0,0,0.0,971) [ext4-rsv-conver] (root,0,0,0.0,978) [jbd2/dm-4-8] (root,0,0,0.0,979) [ext4-rsv-conver] (root,0,0,0.0,1018) [jbd2/dm-5-8] (root,0,0,0.0,1019) [ext4-rsv-conver] (dbus,58232,1356,0.0,1038) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612356,1956,0.0,1065) /usr/lib/polkit-1/polkitd --no-debug (root,21788,592,0.0,1077) /usr/sbin/irqbalance --foreground (root,52900,1072,0.0,1078) /usr/sbin/smartd -n -q never (root,27360,1704,0.0,1081) /usr/lib/systemd/systemd-logind (root,0,0,0.0,1089) [kworker/6:1H] (root,0,0,0.0,1268) [bond0] (root,0,0,0.0,1278) [kworker/2:1H] (root,10057836,281124,0.0,1379) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,0,0,0.0,1556) [kworker/11:1H] (root,0,0,0.0,1637) [kworker/8:2] (root,112984,1356,0.0,1648) /usr/sbin/sshd -D (root,27168,184,0.0,1653) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1655) [irq/98-nvidia] (root,0,0,0.0,1656) [nvidia] (root,0,0,0.0,1657) [nv_queue] (root,801948,2488,0.0,1895) /usr/sbin/collectd (root,717536,12324,0.0,1921) /usr/sbin/rsyslogd -n (root,115708,1208,0.0,1948) /bin/bash /sbin/condor_inspector (root,591424,7640,0.2,1985) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,1989) [irq/99-nvidia] (root,0,0,0.0,1990) [nvidia] (root,0,0,0.1,1991) [nv_queue] (root,0,0,0.0,2109) [kworker/12:2] (cvmfs,1368952,39672,0.0,2140) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2303) [irq/100-nvidia] (root,0,0,0.0,2304) [nvidia] (root,0,0,0.1,2305) [nv_queue] (root,89788,1216,0.0,2488) /usr/libexec/postfix/master -w (slot-1,11700,1456,0.0,2507) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/icetray-env combo/V01-01-00 python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (postfix,90064,1012,0.0,2538) qmgr -l -t unix -u (cvmfs,853224,21088,0.0,2548) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,852108,17004,0.0,2563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,852000,22152,0.0,2615) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,856140,17716,0.0,2635) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,851968,30512,0.0,2660) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (slot-1,11704,1556,0.0,2714) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/RHEL_7_x86_64/metaprojects/combo/V01-01-00/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13389932,1003564,145,2721) python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,2741) [kworker/4:1] (cvmfs,852004,20352,0.0,2749) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (slot-1,46176,6596,0.0,2758) condor_starter -f -a slot1_6 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1564,0.0,2762) /bin/sh /scratch/condor/dir_18808/glidein/execute.400afbda-4a6e-11ed-ba83-0cc47a857a10/dir_2758/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (cvmfs,854096,18980,0.0,2779) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,851972,15556,0.0,2798) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (slot-1,390080,21916,0.1,2808) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (root,0,0,0.0,2823) [irq/101-nvidia] (root,0,0,0.0,2824) [nvidia] (root,0,0,0.0,2825) [nv_queue] (cvmfs,2980464,132060,0.9,2830) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (root,0,0,0.0,2851) [irq/102-nvidia] (root,0,0,0.0,2852) [nvidia] (root,0,0,0.0,2853) [nv_queue] (root,126392,520,0.0,2859) /usr/sbin/crond -n (condor,75280,3992,0.2,2862) /usr/sbin/condor_master -f (root,0,0,0.0,2867) [kworker/u32:1] (ntp,29960,1092,0.0,2868) /usr/sbin/ntpd -u ntp:ntp -g (root,0,0,0.0,2897) [irq/103-nvidia] (root,0,0,0.0,2899) [nvidia] (root,0,0,0.0,2900) [nv_queue] (root,28076,6268,0.0,2931) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (root,0,0,0.0,2983) [irq/104-nvidia] (root,0,0,0.0,2984) [nvidia] (root,0,0,0.0,2985) [nv_queue] (root,0,0,0.0,3169) [irq/105-nvidia] (root,0,0,0.0,3170) [nvidia] (root,0,0,0.2,3171) [nv_queue] (root,0,0,0.0,3238) [kworker/10:1] (root,574284,12812,0.0,3281) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,93236,704,0.0,3339) login -- i3admin (ganglia,220192,3072,0.0,3343) /usr/sbin/gmond (root,0,0,0.0,3411) [kworker/4:0] (root,0,0,0.0,3664) [kworker/u33:0] (slot-1,11700,1460,0.0,4422) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/icetray-env combo/V01-01-00 python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1556,0.0,4515) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/RHEL_7_x86_64/metaprojects/combo/V01-01-00/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13268920,880148,145,4522) python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,5029) [kworker/0:0] (root,0,0,0.0,5546) [kworker/5:0] (slot-1,46176,6592,0.0,7588) condor_starter -f -a slot1_7 sub-simprod-2.icecube.wisc.edu (slot-1,46176,6592,0.0,7642) condor_starter -f -a slot1_2 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1568,0.0,7647) /bin/sh /scratch/condor/dir_18808/glidein/execute.400afbda-4a6e-11ed-ba83-0cc47a857a10/dir_7642/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11704,1564,0.0,7681) /bin/sh /scratch/condor/dir_18808/glidein/execute.400afbda-4a6e-11ed-ba83-0cc47a857a10/dir_7588/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,390336,21976,0.1,7684) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,390336,22032,0.1,7721) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (root,0,0,0.0,8062) [kworker/10:2] (postfix,90116,4280,0.0,8968) smtp -t unix -u (root,0,0,0.0,9097) [kworker/1:2] (slot-1,4368,360,0.0,10385) sleep 300 (root,0,0,0.0,10755) [kworker/5:1] (postfix,89948,4028,0.0,11606) bounce -z -n defer -t unix -u (root,0,0,0.0,12076) [kworker/9:0] (root,0,0,0.0,12179) [kworker/7:1] (root,0,0,0.0,12207) [kworker/2:0] (root,0,0,0.0,12816) [kworker/3:1] (root,0,0,0.0,12821) [kworker/13:0] (root,108056,356,0.0,12904) sleep 15 (root,8632,660,0.0,12940) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,12941) /bin/bash /bin/check_mk_agent (root,51732,1700,0.0,12957) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,12958) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,13982) [kworker/u32:2] (slot-1,46176,6592,0.0,14011) condor_starter -f -a slot1_3 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1564,0.0,14015) /bin/sh /scratch/condor/dir_18808/glidein/execute.400afbda-4a6e-11ed-ba83-0cc47a857a10/dir_14011/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,390336,22040,0.1,14051) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (i3admin,12752,1936,0.0,14064) -bash (root,131516,1188,0.0,14081) sudo su - (root,81988,80,0.0,14082) su - (root,12748,724,0.0,14083) -bash (slot-1,46176,6600,0.0,14338) condor_starter -f -a slot1_8 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1568,0.0,14342) /bin/sh /scratch/condor/dir_18808/glidein/execute.400afbda-4a6e-11ed-ba83-0cc47a857a10/dir_14338/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,390336,22012,0.1,14378) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (root,0,0,0.0,14434) [kworker/8:0] (root,0,0,0.0,14870) [kworker/1:0] (root,102988,928,0.0,15151) /sbin/dhclient -1 -q -lf /var/lib/dhclient/dhclient--bond0.lease -pf /var/run/dhclient-bond0.pid -H gzk-2 bond0 (slot-1,46176,6596,0.0,15230) condor_starter -f -a slot1_5 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1568,0.0,15234) /bin/sh /scratch/condor/dir_18808/glidein/execute.400afbda-4a6e-11ed-ba83-0cc47a857a10/dir_15230/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,390080,21924,0.1,15270) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (root,0,0,0.0,15451) [kworker/0:1] (condor,63816,3264,0.0,15544) condor_shared_port -f (slot-1,11700,1460,0.0,15636) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/icetray-env combo/V01-01-00 python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1552,0.0,15729) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/RHEL_7_x86_64/metaprojects/combo/V01-01-00/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13281888,922416,119,15736) python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,15750) [kworker/12:0] (root,0,0,0.0,17044) [kworker/10:1H] (root,0,0,0.0,17057) [kworker/u34:1] (cvmfs,60820,24,0.0,17587) /usr/bin/cvmfs2 __cachemgr__ . 8 9 41943040000 20971520000 0 3 -1 : (cvmfs,73228,2968,0.0,17590) /usr/bin/cvmfs2 __cachemgr__ . 8 9 41943040000 20971520000 0 3 -1 : (cvmfs,1327384,32268,0.0,17594) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,60820,716,0.0,17682) /usr/bin/cvmfs2 __cachemgr__ . 8 9 1048576000 524288000 0 3 -1 : (cvmfs,72048,2608,0.0,17685) /usr/bin/cvmfs2 __cachemgr__ . 8 9 1048576000 524288000 0 3 -1 : (cvmfs,811128,30316,0.0,17692) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,810992,18704,0.0,17705) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,811008,17340,0.0,17718) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,811128,18056,0.0,17732) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,737280,17536,0.0,17745) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,811128,17240,0.0,17760) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,2901176,97336,0.0,17764) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,737256,27776,0.0,17783) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,810992,17272,0.0,17796) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,302212,50792,0.0,17833) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,0,0,0.0,18772) [kworker/14:2] (condor,67120,8168,0.0,18808) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (root,0,0,0.0,18909) [kworker/15:2] (slot-1,11692,1572,0.0,18918) /bin/sh glidein_start.sh (slot-1,11700,1456,0.0,18956) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/icetray-env combo/V01-01-00 python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,61812,6548,0.0,18979) glideinExec/sbin/condor_master -f -r 20135 (slot-1,11796,1652,0.0,18980) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_c949a814-0c19-455b-90f8-3e9ac02cd116.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20221012%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20221012T204002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=5ec5a100574c801fd91ddb3c766a007032f746560376cb85aa460593f2d3ad87 (slot-1,24712,4496,0.0,18982) condor_procd -A /scratch/condor/dir_18808/glidein/log.400afbda-4a6e-11ed-ba83-0cc47a857a10/procd_address -L /scratch/condor/dir_18808/glidein/log.400afbda-4a6e-11ed-ba83-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,63252,8332,0.0,18983) condor_startd -f (slot-1,11704,1548,0.0,19053) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/RHEL_7_x86_64/metaprojects/combo/V01-01-00/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13389880,1009428,147,19060) python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11700,1460,0.0,19613) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/icetray-env combo/V01-01-00 python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1552,0.0,19706) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/RHEL_7_x86_64/metaprojects/combo/V01-01-00/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13390884,1020648,154,19713) python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,20120) [kworker/7:2] (root,0,0,0.0,20301) [kworker/3:0] (root,0,0,0.0,20440) [kworker/9:2] (postfix,89908,4028,0.0,21882) pickup -l -t unix -u (root,0,0,0.0,22050) [kworker/11:0] (root,0,0,0.0,22197) [kworker/15:1] (root,0,0,0.0,22434) [kworker/11:1] (condor,2405880,2345784,0.1,23351) condor_startd -f (root,209064,7412,0.0,23403) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,10984,0.0,23405) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,212132,9344,0.0,23409) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (condor,34058616,13168,0.1,23591) GPUs_MONITOR (root,0,0,0.0,23593) [UVM GPU1 BH] (root,0,0,0.0,23594) [UVM GPU2 BH] (root,0,0,0.0,23595) [UVM GPU3 BH] (root,0,0,0.0,23596) [UVM GPU4 BH] (root,0,0,0.0,23597) [UVM GPU5 BH] (root,0,0,0.0,23598) [UVM GPU6 BH] (root,0,0,0.0,23599) [UVM GPU7 BH] (root,0,0,0.0,23600) [UVM GPU8 BH] (slot-1,46176,6592,0.0,23680) condor_starter -f -a slot1_4 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1564,0.0,23773) /bin/sh /scratch/condor/dir_18808/glidein/execute.400afbda-4a6e-11ed-ba83-0cc47a857a10/dir_23680/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,390336,22020,0.1,23813) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (root,0,0,0.0,24576) [kworker/6:2] (slot-1,46176,6600,0.0,25938) condor_starter -f -a slot1_1 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1568,0.0,25944) /bin/sh /scratch/condor/dir_18808/glidein/execute.400afbda-4a6e-11ed-ba83-0cc47a857a10/dir_25938/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,390080,21960,0.1,25980) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11700,1460,0.0,26027) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/icetray-env combo/V01-01-00 python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1556,0.0,26120) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/RHEL_7_x86_64/metaprojects/combo/V01-01-00/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13389992,1011220,136,26127) python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,26659) [kworker/u34:0] (slot-1,11700,1456,0.0,26720) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/icetray-env combo/V01-01-00 python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1556,0.0,26813) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/RHEL_7_x86_64/metaprojects/combo/V01-01-00/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13389736,1003260,136,26820) python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,27039) [kworker/2:1] (slot-1,11700,1460,0.0,27889) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/icetray-env combo/V01-01-00 python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,11704,1556,0.0,27983) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.1.1/RHEL_7_x86_64/metaprojects/combo/V01-01-00/env-shell.sh python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (slot-1,13268916,884780,136,27990) python /cvmfs/icecube.opensciencegrid.org/iceprod/master/RHEL_7_x86_64/lib/python3.8/site-packages/iceprod/core/exe_helper.py --classname icecube.simprod.modules.ClSim --args (root,0,0,0.0,28808) [kworker/14:0] (root,0,0,0.0,31259) [kworker/9:1] (root,0,0,0.0,31330) [kworker/6:0] (root,45364,1480,0.0,31760) /usr/lib/systemd/systemd-udevd (root,0,0,0.0,31901) [kworker/1:1] (root,0,0,0.0,32191) [kworker/10:0] (root,0,0,0.0,32202) [kworker/13:1] (root,0,0,0.0,32641) [kworker/u33:2]
Severity: high
Fingerprint: 03cb82e6f6a6b45342c4bbcb32628cb644bc00b1a45a4f05aae993da4dbb64d2
Found public CheckMk agent: Version: 1.2.8p26 AgentOS: linux Hostname: gzk-2.chtc.wisc.edu AgentDirectory: /etc/check-mk-agent DataDirectory: /var/lib/check_mk_agent SpoolDirectory: /var/lib/check_mk_agent/spool PluginsDirectory: /usr/share/check-mk-agent/plugins LocalDirectory: /usr/share/check-mk-agent/local Found process list through CheckMk: (root,43800,3800,0.0,1) /usr/lib/systemd/systemd --system --deserialize 17 (root,0,0,0.0,2) [kthreadd] (root,0,0,0.0,4) [kworker/0:0H] (root,0,0,0.0,6) [ksoftirqd/0] (root,0,0,0.0,7) [migration/0] (root,0,0,0.0,8) [rcu_bh] (root,0,0,0.1,9) [rcu_sched] (root,0,0,0.0,10) [lru-add-drain] (root,0,0,0.0,11) [watchdog/0] (root,0,0,0.0,12) [watchdog/1] (root,0,0,0.0,13) [migration/1] (root,0,0,0.0,14) [ksoftirqd/1] (root,0,0,0.0,16) [kworker/1:0H] (root,0,0,0.0,18) [watchdog/2] (root,0,0,0.0,19) [migration/2] (root,0,0,0.0,20) [ksoftirqd/2] (root,0,0,0.0,22) [kworker/2:0H] (root,0,0,0.0,23) [watchdog/3] (root,0,0,0.0,24) [migration/3] (root,0,0,0.0,25) [ksoftirqd/3] (root,0,0,0.0,27) [kworker/3:0H] (root,0,0,0.0,28) [watchdog/4] (root,0,0,0.0,29) [migration/4] (root,0,0,0.0,30) [ksoftirqd/4] (root,0,0,0.0,32) [kworker/4:0H] (root,0,0,0.0,34) [watchdog/5] (root,0,0,0.0,35) [migration/5] (root,0,0,0.0,36) [ksoftirqd/5] (root,0,0,0.0,38) [kworker/5:0H] (root,0,0,0.0,39) [watchdog/6] (root,0,0,0.0,40) [migration/6] (root,0,0,0.0,41) [ksoftirqd/6] (root,0,0,0.0,43) [kworker/6:0H] (root,0,0,0.0,44) [watchdog/7] (root,0,0,0.0,45) [migration/7] (root,0,0,0.0,46) [ksoftirqd/7] (root,0,0,0.0,48) [kworker/7:0H] (root,0,0,0.0,49) [watchdog/8] (root,0,0,0.0,50) [migration/8] (root,0,0,0.0,51) [ksoftirqd/8] (root,0,0,0.0,53) [kworker/8:0H] (root,0,0,0.0,54) [watchdog/9] (root,0,0,0.0,55) [migration/9] (root,0,0,0.0,56) [ksoftirqd/9] (root,0,0,0.0,58) [kworker/9:0H] (root,0,0,0.0,59) [watchdog/10] (root,0,0,0.0,60) [migration/10] (root,0,0,0.0,61) [ksoftirqd/10] (root,0,0,0.0,63) [kworker/10:0H] (root,0,0,0.0,64) [watchdog/11] (root,0,0,0.0,65) [migration/11] (root,0,0,0.0,66) [ksoftirqd/11] (root,0,0,0.0,68) [kworker/11:0H] (root,0,0,0.0,69) [watchdog/12] (root,0,0,0.0,70) [migration/12] (root,0,0,0.0,71) [ksoftirqd/12] (root,0,0,0.0,73) [kworker/12:0H] (root,0,0,0.0,74) [watchdog/13] (root,0,0,0.0,75) [migration/13] (root,0,0,0.0,76) [ksoftirqd/13] (root,0,0,0.0,78) [kworker/13:0H] (root,0,0,0.0,79) [watchdog/14] (root,0,0,0.0,80) [migration/14] (root,0,0,0.0,81) [ksoftirqd/14] (root,0,0,0.0,83) [kworker/14:0H] (root,0,0,0.0,84) [watchdog/15] (root,0,0,0.0,85) [migration/15] (root,0,0,0.0,86) [ksoftirqd/15] (root,0,0,0.0,88) [kworker/15:0H] (root,0,0,0.0,91) [kdevtmpfs] (root,0,0,0.0,92) [netns] (root,0,0,0.0,93) [khungtaskd] (root,0,0,0.0,94) [writeback] (root,0,0,0.0,95) [kintegrityd] (root,0,0,0.0,96) [bioset] (root,0,0,0.0,97) [bioset] (root,0,0,0.0,98) [bioset] (root,0,0,0.0,99) [kblockd] (root,0,0,0.0,101) [md] (root,0,0,0.0,102) [edac-poller] (root,0,0,0.0,103) [watchdogd] (root,0,0,0.0,109) [kswapd0] (root,0,0,0.0,110) [kswapd1] (root,0,0,0.0,111) [ksmd] (root,0,0,0.0,112) [khugepaged] (root,0,0,0.0,113) [crypto] (root,0,0,0.0,121) [kthrotld] (root,0,0,0.0,125) [kmpath_rdacd] (root,0,0,0.0,126) [kaluad] (root,0,0,0.0,143) [kpsmoused] (root,0,0,0.0,144) [ipv6_addrconf] (root,0,0,0.0,158) [deferwq] (root,0,0,0.0,203) [kauditd] (slot-1,11700,1532,0.0,335) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/icetray-env icetray/v1.4.1 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=8903 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (root,0,0,0.0,397) [ata_sff] (root,0,0,0.0,405) [scsi_eh_0] (root,0,0,0.0,406) [scsi_tmf_0] (root,0,0,0.0,407) [scsi_eh_1] (root,0,0,0.0,408) [scsi_tmf_1] (root,0,0,0.0,409) [scsi_eh_2] (root,0,0,0.0,410) [scsi_tmf_2] (root,0,0,0.0,411) [scsi_eh_3] (root,0,0,0.0,412) [scsi_tmf_3] (root,0,0,0.0,416) [scsi_eh_4] (root,0,0,0.0,417) [scsi_tmf_4] (root,0,0,0.0,418) [scsi_eh_5] (root,0,0,0.0,419) [scsi_tmf_5] (root,0,0,0.0,420) [scsi_eh_6] (root,0,0,0.0,421) [scsi_tmf_6] (root,0,0,0.0,422) [scsi_eh_7] (root,0,0,0.0,423) [scsi_tmf_7] (root,0,0,0.0,424) [scsi_eh_8] (root,0,0,0.0,425) [scsi_tmf_8] (root,0,0,0.0,426) [scsi_eh_9] (root,0,0,0.0,427) [scsi_tmf_9] (root,0,0,0.0,434) [ttm_swap] (slot-1,11836,1560,0.0,452) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/RHEL_7_x86_64/metaprojects/icetray/v1.4.1/env-shell.sh /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=8903 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,13446076,1085768,95.1,459) python3 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=8903 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (root,0,0,0.0,535) [kdmflush] (root,0,0,0.0,536) [bioset] (root,0,0,0.0,546) [kdmflush] (root,0,0,0.0,547) [bioset] (root,0,0,0.0,550) [kworker/0:1H] (root,0,0,0.0,552) [kworker/15:1H] (root,0,0,0.0,553) [kworker/1:1H] (root,0,0,0.0,568) [jbd2/dm-0-8] (root,0,0,0.0,569) [ext4-rsv-conver] (root,47652,11276,0.0,650) /usr/lib/systemd/systemd-journald (root,0,0,1.1,727) [kipmi0] (root,0,0,0.0,735) [kworker/13:1H] (root,0,0,0.0,742) [kworker/7:1H] (root,0,0,0.0,765) [kworker/9:1H] (root,0,0,0.0,784) [kworker/3:1H] (root,0,0,0.0,792) [kvm-irqfd-clean] (root,198572,644,0.0,793) /usr/sbin/lvmetad -f (root,0,0,0.0,867) [kdmflush] (root,0,0,0.0,868) [bioset] (root,0,0,0.0,869) [kdmflush] (root,0,0,0.0,871) [bioset] (root,0,0,0.0,873) [kdmflush] (root,0,0,0.0,874) [bioset] (root,0,0,0.0,876) [kdmflush] (root,0,0,0.0,877) [bioset] (root,0,0,0.0,879) [kworker/1:2] (root,0,0,0.0,886) [kworker/5:1H] (root,0,0,0.0,887) [kworker/4:1H] (root,0,0,0.0,888) [kworker/12:1H] (root,0,0,0.0,889) [kworker/8:1H] (root,0,0,0.0,901) [jbd2/sda1-8] (root,0,0,0.0,902) [ext4-rsv-conver] (root,0,0,0.0,917) [nv_queue] (root,0,0,0.0,918) [nv_queue] (root,0,0,0.0,937) [kworker/14:1H] (root,0,0,0.0,947) [jbd2/dm-3-8] (root,0,0,0.0,948) [ext4-rsv-conver] (root,0,0,0.0,950) [UVM global queu] (root,0,0,0.0,951) [UVM deferred re] (root,0,0,0.0,953) [UVM Tools Event] (root,0,0,0.0,963) [nvidia-modeset/] (root,0,0,0.0,964) [nvidia-modeset/] (root,0,0,0.0,970) [jbd2/dm-2-8] (root,0,0,0.0,971) [ext4-rsv-conver] (root,0,0,0.0,978) [jbd2/dm-4-8] (root,0,0,0.0,979) [ext4-rsv-conver] (root,0,0,0.0,1018) [jbd2/dm-5-8] (root,0,0,0.0,1019) [ext4-rsv-conver] (dbus,58232,1488,0.0,1038) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation (polkitd,612356,2140,0.0,1065) /usr/lib/polkit-1/polkitd --no-debug (root,21788,600,0.0,1077) /usr/sbin/irqbalance --foreground (root,52900,1072,0.0,1078) /usr/sbin/smartd -n -q never (root,27360,1968,0.0,1081) /usr/lib/systemd/systemd-logind (root,0,0,0.0,1089) [kworker/6:1H] (root,0,0,0.0,1268) [bond0] (root,0,0,0.0,1278) [kworker/2:1H] (root,10057836,285940,0.0,1379) /opt/APC/PowerChute/jre1.8.0_91/bin/java -Xrs -Xms32m -Xmx64m -Dlog4j.configurationFile=log4j2.xml -cp .:./lib/log4j-core-2.2.jar:./lib/log4j-api-2.2.jar:./lib/vijava5120121125.jar:./lib/jasypt-1.9.0.jar:./lib/dom4j-1.6.1.jar:./lib/jetty-continuation-9.1.3.v20140225.jar:./lib/jetty-http-9.1.3.v20140225.jar:./lib/jetty-io-9.1.3.v20140225.jar:./lib/jetty-security-9.1.3.v20140225.jar:./lib/jetty-server-9.1.3.v20140225.jar:./lib/jetty-servlet-9.1.3.v20140225.jar:./lib/jetty-util-9.1.3.v20140225.jar:./lib/jetty-webapp-9.1.3.v20140225.jar:./lib/jetty-xml-9.1.3.v20140225.jar:./lib/servlet-api-3.1.jar:./lib/m11.jar:./lib/commons-collections-3.2.1.jar:./lib/commons-configuration-1.6.jar:./lib/commons-lang-2.5.jar:./lib/commons-logging-1.1.1.jar:./lib/commons-codec-1.4.jar:./comp/pcns.jar:./comp/Notifier.jar:./lib/snmp4j-2.4.3.jar:./lib/snmp4j-agent-2.4.2.jar:./lib/bcprov-jdk15on-151.jar:./lib/bcpkix-jdk15on-151.jar:./lib/json_simple-1.1.jar com.apcc.m11.arch.application.Application everything (root,0,0,0.0,1556) [kworker/11:1H] (root,112984,1716,0.0,1648) /usr/sbin/sshd -D (root,27168,192,0.0,1653) /usr/sbin/xinetd -stayalive -pidfile /var/run/xinetd.pid (root,0,0,0.0,1655) [irq/98-nvidia] (root,0,0,0.0,1656) [nvidia] (root,0,0,0.1,1657) [nv_queue] (slot-1,46180,6596,0.0,1847) condor_starter -f -a slot1_5 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1564,0.0,1851) /bin/sh /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_1847/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,457940,22048,0.2,1887) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (root,801948,4320,0.0,1895) /usr/sbin/collectd (slot-1,11700,1532,0.0,1903) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/icetray-env icetray/v1.4.1 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=9164 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (root,717496,13052,0.0,1921) /usr/sbin/rsyslogd -n (root,115708,1208,0.0,1948) /bin/bash /sbin/condor_inspector (root,591424,8468,0.2,1985) /usr/bin/python2 -s /usr/bin/fail2ban-server -xf start (root,0,0,0.0,1989) [irq/99-nvidia] (root,0,0,0.0,1990) [nvidia] (root,0,0,0.1,1991) [nv_queue] (slot-1,11836,1556,0.0,2019) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/RHEL_7_x86_64/metaprojects/icetray/v1.4.1/env-shell.sh /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=9164 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,13459208,1127952,99.0,2026) python3 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=9164 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (cvmfs,1372396,39524,0.0,2140) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (root,0,0,0.0,2303) [irq/100-nvidia] (root,0,0,0.0,2304) [nvidia] (root,0,0,0.1,2305) [nv_queue] (root,89788,1272,0.0,2488) /usr/libexec/postfix/master -w (postfix,90064,1344,0.0,2538) qmgr -l -t unix -u (cvmfs,852416,25120,0.0,2548) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,852112,17024,0.0,2563) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,852000,22068,0.0,2615) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,854232,22440,0.0,2635) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,851968,30440,0.0,2660) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,851992,20636,0.0,2749) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,853088,19028,0.0,2779) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,851984,15512,0.0,2798) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,0,0,0.0,2823) [irq/101-nvidia] (root,0,0,0.0,2824) [nvidia] (root,0,0,0.0,2825) [nv_queue] (cvmfs,2986336,140020,0.8,2830) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (root,0,0,0.0,2851) [irq/102-nvidia] (root,0,0,0.0,2852) [nvidia] (root,0,0,0.0,2853) [nv_queue] (root,126392,536,0.0,2859) /usr/sbin/crond -n (condor,75280,4040,0.3,2862) /usr/sbin/condor_master -f (ntp,29960,1108,0.0,2868) /usr/sbin/ntpd -u ntp:ntp -g (root,0,0,0.0,2897) [irq/103-nvidia] (root,0,0,0.0,2899) [nvidia] (root,0,0,0.0,2900) [nv_queue] (root,27944,6208,0.0,2931) condor_procd -A /var/run/condor/procd_pipe -L /var/log/condor/ProcLog -R 1000000 -S 60 -C 995 (root,0,0,0.0,2933) [kworker/11:2] (root,0,0,0.0,2983) [irq/104-nvidia] (root,0,0,0.0,2984) [nvidia] (root,0,0,0.1,2985) [nv_queue] (root,0,0,0.0,3169) [irq/105-nvidia] (root,0,0,0.0,3170) [nvidia] (root,0,0,0.2,3171) [nv_queue] (root,574284,17628,0.0,3281) /usr/bin/python2 -Es /usr/sbin/tuned -l -P (root,93236,784,0.0,3339) login -- i3admin (ganglia,220192,3168,0.0,3343) /usr/sbin/gmond (root,0,0,0.0,3805) [kworker/12:1] (root,0,0,0.0,5155) [kworker/5:2] (root,0,0,0.0,6085) [kworker/3:1] (root,0,0,0.0,6088) [kworker/8:2] (root,0,0,0.0,6488) [kworker/u33:2] (root,0,0,0.0,6841) [kworker/7:2] (postfix,89892,4024,0.0,6937) pickup -l -t unix -u (slot-1,46148,6540,0.0,7436) condor_starter -f -a slot1_11 sub-1.icecube.wisc.edu (slot-1,11700,1548,0.0,7440) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/vbasu/meta-projects/combo3/build/env-shell.sh python /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_7436/condor_exec.exe --outfile gsiftp://gridftp-users.icecube.wisc.edu/data/user/nlad/Condor/IceCube/Outputs/flavor/MESE_NewCuts/NuTau/22017/L5_022017_000284.i3.zst --infiles gsiftp://gridftp-users.icecube.wisc.edu/data/user/abalagopalv/diffuse/SnowStormNuGen/New_Cuts_Aug2022/L5_NewIce_022017_000284_waveform_DPscores.i3.zst --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --icemodel Spice_3.2.1 (slot-1,5346108,4965960,97.0,7550) python /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_7436/condor_exec.exe --outfile gsiftp://gridftp-users.icecube.wisc.edu/data/user/nlad/Condor/IceCube/Outputs/flavor/MESE_NewCuts/NuTau/22017/L5_022017_000284.i3.zst --infiles gsiftp://gridftp-users.icecube.wisc.edu/data/user/abalagopalv/diffuse/SnowStormNuGen/New_Cuts_Aug2022/L5_NewIce_022017_000284_waveform_DPscores.i3.zst --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --icemodel Spice_3.2.1 (root,0,0,0.0,8010) [kworker/11:0] (root,0,0,0.0,8115) [kworker/13:1] (condor,67144,7500,0.0,8162) condor_starter -f -local-name slot_type_1 -a slot1_1 gzk-cm-el7.icecube.wisc.edu (slot-1,11692,1568,0.0,8181) /bin/sh glidein_start.sh (slot-1,61812,6324,0.0,8334) glideinExec/sbin/condor_master -f -r 20135 (slot-1,12088,1900,0.0,8335) /bin/bash ../log_shipper.sh https://pyglidein-logging-gzk.s3.us-east-1.amazonaws.com/GZK_a5c64759-2618-4eb2-b049-dc634716958b.tar.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAITTJEMXDWMU5MQGQ%2F20220914%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20220914T212002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=088c30848b5a84eae835d6ff8216412690f608c47e4340a7e8fa084150f45dbb (slot-1,25364,5216,0.0,8337) condor_procd -A /scratch/condor/dir_8162/glidein/log.4233f378-3473-11ed-8dc7-0cc47a857a10/procd_address -L /scratch/condor/dir_8162/glidein/log.4233f378-3473-11ed-8dc7-0cc47a857a10/ProcLog -R 1000000 -S 60 -C 1002 (slot-1,64568,9520,0.0,8338) condor_startd -f (root,0,0,0.0,8471) [kworker/3:2] (root,0,0,0.0,8538) [kworker/4:1] (root,0,0,0.0,8910) [kworker/9:2] (root,0,0,0.0,9930) [kworker/2:0] (root,0,0,0.0,9933) [kworker/12:2] (root,0,0,0.0,9995) [kworker/6:1] (root,0,0,0.0,10761) [kworker/7:1] (root,0,0,0.0,11264) [kworker/14:0] (slot-1,46148,6540,0.0,11412) condor_starter -f -a slot1_16 sub-1.icecube.wisc.edu (root,0,0,0.0,11422) [kworker/10:2] (slot-1,11700,1548,0.0,11500) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/vbasu/meta-projects/combo3/build/env-shell.sh python /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_11412/condor_exec.exe --outfile gsiftp://gridftp-users.icecube.wisc.edu/data/user/nlad/Condor/IceCube/Outputs/flavor/MESE_NewCuts/NuTau/22017/L5_022017_000418.i3.zst --infiles gsiftp://gridftp-users.icecube.wisc.edu/data/user/abalagopalv/diffuse/SnowStormNuGen/New_Cuts_Aug2022/L5_NewIce_022017_000418_waveform_DPscores.i3.zst --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --icemodel Spice_3.2.1 (slot-1,5339672,5051552,97.6,11611) python /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_11412/condor_exec.exe --outfile gsiftp://gridftp-users.icecube.wisc.edu/data/user/nlad/Condor/IceCube/Outputs/flavor/MESE_NewCuts/NuTau/22017/L5_022017_000418.i3.zst --infiles gsiftp://gridftp-users.icecube.wisc.edu/data/user/abalagopalv/diffuse/SnowStormNuGen/New_Cuts_Aug2022/L5_NewIce_022017_000418_waveform_DPscores.i3.zst --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --icemodel Spice_3.2.1 (root,0,0,0.0,12460) [kworker/0:2] (root,0,0,0.0,12872) [kworker/15:2] (root,0,0,0.0,13506) [kworker/u32:0] (root,0,0,0.0,14003) [kworker/1:1] (i3admin,12752,1936,0.0,14064) -bash (root,131516,1516,0.0,14081) sudo su - (root,81988,128,0.0,14082) su - (root,12748,824,0.0,14083) -bash (slot-1,46176,6596,0.0,14710) condor_starter -f -a slot1_8 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1568,0.0,14717) /bin/sh /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_14710/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,457940,22048,0.2,14754) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,46148,6540,0.0,14866) condor_starter -f -a slot1_10 sub-1.icecube.wisc.edu (slot-1,11700,1540,0.0,14870) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/vbasu/meta-projects/combo3/build/env-shell.sh python /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_14866/condor_exec.exe --outfile gsiftp://gridftp-users.icecube.wisc.edu/data/user/nlad/Condor/IceCube/Outputs/flavor/MESE_NewCuts/NuE/22014/L5_022014_000061.i3.zst --infiles gsiftp://gridftp-users.icecube.wisc.edu/data/user/abalagopalv/diffuse/SnowStormNuGen/New_Cuts_Aug2022/L5_NewIce_022014_000061_waveform_DPscores.i3.zst --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --icemodel Spice_3.2.1 (slot-1,11700,1532,0.0,14941) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/icetray-env icetray/v1.4.1 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=7750 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,5345368,4965312,97.6,14982) python /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_14866/condor_exec.exe --outfile gsiftp://gridftp-users.icecube.wisc.edu/data/user/nlad/Condor/IceCube/Outputs/flavor/MESE_NewCuts/NuE/22014/L5_022014_000061.i3.zst --infiles gsiftp://gridftp-users.icecube.wisc.edu/data/user/abalagopalv/diffuse/SnowStormNuGen/New_Cuts_Aug2022/L5_NewIce_022014_000061_waveform_DPscores.i3.zst --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --icemodel Spice_3.2.1 (slot-1,11836,1560,0.0,15038) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/RHEL_7_x86_64/metaprojects/icetray/v1.4.1/env-shell.sh /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=7750 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,13425212,1045180,91.3,15045) python3 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=7750 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (root,102988,1140,0.0,15151) /sbin/dhclient -1 -q -lf /var/lib/dhclient/dhclient--bond0.lease -pf /var/run/dhclient-bond0.pid -H gzk-2 bond0 (slot-1,46144,6552,0.0,15272) condor_starter -f -a slot1_12 sub-1.icecube.wisc.edu (slot-1,11700,1548,0.0,15276) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/vbasu/meta-projects/combo3/build/env-shell.sh python /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_15272/condor_exec.exe --outfile gsiftp://gridftp-users.icecube.wisc.edu/data/user/nlad/Condor/IceCube/Outputs/flavor/MESE_NewCuts/NuTau/22017/L5_022017_000197.i3.zst --infiles gsiftp://gridftp-users.icecube.wisc.edu/data/user/abalagopalv/diffuse/SnowStormNuGen/New_Cuts_Aug2022/L5_NewIce_022017_000197_waveform_DPscores.i3.zst --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --icemodel Spice_3.2.1 (slot-1,5356588,4976584,97.1,15387) python /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_15272/condor_exec.exe --outfile gsiftp://gridftp-users.icecube.wisc.edu/data/user/nlad/Condor/IceCube/Outputs/flavor/MESE_NewCuts/NuTau/22017/L5_022017_000197.i3.zst --infiles gsiftp://gridftp-users.icecube.wisc.edu/data/user/abalagopalv/diffuse/SnowStormNuGen/New_Cuts_Aug2022/L5_NewIce_022017_000197_waveform_DPscores.i3.zst --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --icemodel Spice_3.2.1 (condor,63756,3312,0.0,15544) condor_shared_port -f (root,0,0,0.0,15657) [kworker/u34:2] (root,0,0,0.0,17044) [kworker/10:1H] (root,0,0,0.0,17435) [kworker/u34:1] (cvmfs,60820,24,0.0,17587) /usr/bin/cvmfs2 __cachemgr__ . 8 9 41943040000 20971520000 0 3 -1 : (cvmfs,73228,3028,0.0,17590) /usr/bin/cvmfs2 __cachemgr__ . 8 9 41943040000 20971520000 0 3 -1 : (cvmfs,1327384,32304,0.0,17594) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 config-osg.opensciencegrid.org /cvmfs/config-osg.opensciencegrid.org (cvmfs,60820,716,0.0,17682) /usr/bin/cvmfs2 __cachemgr__ . 8 9 1048576000 524288000 0 3 -1 : (cvmfs,72048,2608,0.0,17685) /usr/bin/cvmfs2 __cachemgr__ . 8 9 1048576000 524288000 0 3 -1 : (cvmfs,811128,30500,0.0,17692) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 oasis.opensciencegrid.org /cvmfs/oasis.opensciencegrid.org (cvmfs,810992,20052,0.0,17705) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 xenon.opensciencegrid.org /cvmfs/xenon.opensciencegrid.org (cvmfs,811008,17440,0.0,17718) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 sft.cern.ch /cvmfs/sft.cern.ch (cvmfs,811128,18068,0.0,17732) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 ara.opensciencegrid.org /cvmfs/ara.opensciencegrid.org (cvmfs,737280,17644,0.0,17745) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 cms.cern.ch /cvmfs/cms.cern.ch (cvmfs,811128,17248,0.0,17760) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 spt.opensciencegrid.org /cvmfs/spt.opensciencegrid.org (cvmfs,2901176,97496,0.0,17764) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.opensciencegrid.org /cvmfs/icecube.opensciencegrid.org (cvmfs,737256,27776,0.0,17783) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 icecube.osgstorage.org /cvmfs/icecube.osgstorage.org (cvmfs,810992,17296,0.0,17796) /usr/bin/cvmfs2 -o ro,allow_other,_netdev,system_mount,fsname=cvmfs2,allow_other,grab_mountpoint,uid=996,gid=993 singularity.opensciencegrid.org /cvmfs/singularity.opensciencegrid.org (root,302212,43964,0.0,17833) /opt/puppetlabs/puppet/bin/ruby /opt/puppetlabs/puppet/bin/puppet agent --no-daemonize (root,0,0,0.0,19556) [kworker/9:1] (root,0,0,0.0,20420) [kworker/14:2] (root,0,0,0.0,21543) [kworker/13:0] (root,0,0,0.0,22071) [kworker/u32:1] (root,0,0,0.0,22321) [kworker/u33:0] (root,0,0,0.0,22675) [kworker/7:0] (slot-1,46144,6548,0.0,22814) condor_starter -f -a slot1_14 sub-1.icecube.wisc.edu (slot-1,11700,1544,0.0,22818) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/vbasu/meta-projects/combo3/build/env-shell.sh python /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_22814/condor_exec.exe --outfile gsiftp://gridftp-users.icecube.wisc.edu/data/user/nlad/Condor/IceCube/Outputs/flavor/MESE_NewCuts/NuE/22013/L5_022013_000273.i3.zst --infiles gsiftp://gridftp-users.icecube.wisc.edu/data/user/abalagopalv/diffuse/SnowStormNuGen/New_Cuts_Aug2022/L5_NewIce_022013_000273_waveform_DPscores.i3.zst --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --icemodel Spice_3.2.1 (slot-1,5338640,4958804,97.4,23033) python /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_22814/condor_exec.exe --outfile gsiftp://gridftp-users.icecube.wisc.edu/data/user/nlad/Condor/IceCube/Outputs/flavor/MESE_NewCuts/NuE/22013/L5_022013_000273.i3.zst --infiles gsiftp://gridftp-users.icecube.wisc.edu/data/user/abalagopalv/diffuse/SnowStormNuGen/New_Cuts_Aug2022/L5_NewIce_022013_000273_waveform_DPscores.i3.zst --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --icemodel Spice_3.2.1 (condor,1620228,1560132,0.1,23351) condor_startd -f (slot-1,4368,360,0.0,23380) sleep 300 (root,209100,7468,0.0,23403) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,358544,10976,0.0,23405) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,212132,9356,0.0,23409) /usr/bin/python /usr/local/sbin/check_mk_agent_daemon.py (root,0,0,0.0,23547) [kworker/10:0] (condor,34058616,13168,0.1,23591) GPUs_MONITOR (root,0,0,0.0,23593) [UVM GPU1 BH] (root,0,0,0.0,23594) [UVM GPU2 BH] (root,0,0,0.0,23595) [UVM GPU3 BH] (root,0,0,0.0,23596) [UVM GPU4 BH] (root,0,0,0.0,23597) [UVM GPU5 BH] (root,0,0,0.0,23598) [UVM GPU6 BH] (root,0,0,0.0,23599) [UVM GPU7 BH] (root,0,0,0.0,23600) [UVM GPU8 BH] (slot-1,46176,6592,0.0,25317) condor_starter -f -a slot1_7 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1568,0.0,25321) /bin/sh /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_25317/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,457940,22044,0.2,25357) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11700,1532,0.0,25602) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/icetray-env icetray/v1.4.1 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=5157 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,11836,1560,0.0,25828) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/RHEL_7_x86_64/metaprojects/icetray/v1.4.1/env-shell.sh /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=5157 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,13447068,1070152,90.1,25835) python3 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=5157 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (root,0,0,0.0,25848) [kworker/1:0] (root,0,0,0.0,26572) [kworker/2:2] (root,0,0,0.0,26578) [kworker/u32:2] (root,0,0,0.0,26622) [kworker/5:0] (root,108056,360,0.0,26693) sleep 15 (root,8632,664,0.0,26735) timeout -s 9 360 check_mk_agent (root,11696,1536,0.0,26736) /bin/bash /bin/check_mk_agent (root,51732,1692,0.0,26752) ps ax -o user,vsz,rss,pcpu,pid,command --columns 10000 (root,13324,948,0.0,26753) sed -e 1d -e s/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) / (root,0,0,0.0,26972) [kworker/10:1] (slot-1,46176,6588,0.0,28215) condor_starter -f -a slot1_2 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1564,0.0,28307) /bin/sh /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_28215/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,457684,21900,0.2,28343) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11700,1532,0.0,28524) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/icetray-env icetray/v1.4.1 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=10000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=7793 --seed=22162 --summaryfile=summary.json --keep-pure-background --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,11836,1560,0.0,28618) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/RHEL_7_x86_64/metaprojects/icetray/v1.4.1/env-shell.sh /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=10000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=7793 --seed=22162 --summaryfile=summary.json --keep-pure-background --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,13792772,1396420,96.5,28625) python3 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=10000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=7793 --seed=22162 --summaryfile=summary.json --keep-pure-background --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (root,0,0,0.0,28705) [kworker/0:1] (slot-1,46176,6592,0.0,28708) condor_starter -f -a slot1_3 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1568,0.0,28714) /bin/sh /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_28708/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,457940,22052,0.2,28750) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11700,1528,0.0,28768) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/icetray-env icetray/v1.4.1 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=9247 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,11836,1560,0.0,28865) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/RHEL_7_x86_64/metaprojects/icetray/v1.4.1/env-shell.sh /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=9247 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,13450588,1072612,94.7,28872) python3 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=9247 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (root,0,0,0.0,29614) [kworker/15:0] (slot-1,46144,6552,0.0,29799) condor_starter -f -a slot1_9 sub-1.icecube.wisc.edu (root,0,0,0.0,29800) [kworker/14:1] (slot-1,11700,1544,0.0,29803) /bin/sh /cvmfs/icecube.opensciencegrid.org/users/vbasu/meta-projects/combo3/build/env-shell.sh python /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_29799/condor_exec.exe --outfile gsiftp://gridftp-users.icecube.wisc.edu/data/user/nlad/Condor/IceCube/Outputs/flavor/MESE_NewCuts/NuMu/22010/L5_022010_000266.i3.zst --infiles gsiftp://gridftp-users.icecube.wisc.edu/data/user/abalagopalv/diffuse/SnowStormNuGen/New_Cuts_Aug2022/L5_NewIce_022010_000266_waveform_DPscores.i3.zst --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --icemodel Spice_3.2.1 (root,0,0,0.0,29805) [kworker/8:1] (slot-1,5348196,4968340,97.3,29915) python /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_29799/condor_exec.exe --outfile gsiftp://gridftp-users.icecube.wisc.edu/data/user/nlad/Condor/IceCube/Outputs/flavor/MESE_NewCuts/NuMu/22010/L5_022010_000266.i3.zst --infiles gsiftp://gridftp-users.icecube.wisc.edu/data/user/abalagopalv/diffuse/SnowStormNuGen/New_Cuts_Aug2022/L5_NewIce_022010_000266_waveform_DPscores.i3.zst --gcdfile /cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --icemodel Spice_3.2.1 (slot-1,46176,6592,0.0,30662) condor_starter -f -a slot1_4 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1564,0.0,30666) /bin/sh /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_30662/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,457940,22044,0.2,30702) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11700,1532,0.0,30951) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/icetray-env icetray/v1.4.1 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=6656 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,11836,1560,0.0,31044) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/RHEL_7_x86_64/metaprojects/icetray/v1.4.1/env-shell.sh /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=6656 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,13449860,1066224,97.3,31051) python3 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_3.2.1 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=6656 --seed=22164 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (root,45364,1740,0.0,31760) /usr/lib/systemd/systemd-udevd (root,0,0,0.0,31951) [kworker/6:2] (slot-1,46172,6584,0.0,32159) condor_starter -f -a slot1_6 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1564,0.0,32220) /bin/sh /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_32159/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,457684,21852,0.2,32256) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,11700,1532,0.0,32263) /bin/bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/icetray-env icetray/v1.4.1 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_bfr-v2 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=7070 --seed=22184 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,11836,1560,0.0,32356) bash /cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/RHEL_7_x86_64/metaprojects/icetray/v1.4.1/env-shell.sh /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_bfr-v2 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=7070 --seed=22184 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (slot-1,13419392,1031524,72.9,32363) python3 /cvmfs/icecube.opensciencegrid.org/users/juancarlos/simulation/clsim-mod.py --efficiency=1 --gcdfile=/cvmfs/icecube.opensciencegrid.org/data/GCD/GeoCalibDetectorStatus_2020.Run134142.Pass2_V0.i3.gz --holeiceparametrization=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ANGSENS/angsens_flasher/as.9 --IceModel=spice_bfr-v2 --IceModelLocation=/cvmfs/icecube.opensciencegrid.org/py3-v4.2.0/metaprojects/icetray/v1.4.1/ice-models/resources/models/ICEMODEL/ --inputfilelist=nugen.i3.zst --nproc=20000 --outputfile=signalhits.i3.zst --oversize=1 --procnum=7070 --seed=22184 --summaryfile=summary.json --keep-pure-background --no-PropagateMuons --UseGSLRNG --UseGPUs --EnableHistogram --HistogramFilename=clsim_histo.pkl (root,0,0,0.0,32700) [kworker/4:2] (slot-1,46176,6592,0.0,32717) condor_starter -f -a slot1_1 sub-simprod-2.icecube.wisc.edu (slot-1,11704,1568,0.0,32721) /bin/sh /scratch/condor/dir_8162/glidein/execute.4233f378-3473-11ed-8dc7-0cc47a857a10/dir_32717/condor_exec.exe -s /cvmfs/icecube.opensciencegrid.org/iceprod/master -x x509up_u891 --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug (slot-1,457944,22048,0.2,32757) python -m iceprod.core.i3exec --url https://iceprod2-api.icecube.wisc.edu --cfgfile task.cfg --offline --offline_transfer True --gzip-logs --debug