Started by user Pau Espin Pedrol Running as SYSTEM Building remotely on osmo-gsm-tester-prod (osmo-gsm-tester) in workspace /home/jenkins/workspace/osmo-gsm-tester_manual-run No credentials specified > git rev-parse --is-inside-work-tree # timeout=10 Fetching changes from the remote Git repository > git config remote.origin.url https://gerrit.osmocom.org/osmo-gsm-tester # timeout=10 Cleaning workspace > git rev-parse --verify HEAD # timeout=10 Resetting working tree > git reset --hard # timeout=10 > git clean -fdx # timeout=10 Fetching upstream changes from https://gerrit.osmocom.org/osmo-gsm-tester > git --version # timeout=10 > git fetch --tags --progress https://gerrit.osmocom.org/osmo-gsm-tester +refs/heads/*:refs/remotes/origin/* > git rev-parse refs/remotes/origin/pespin/tmp^{commit} # timeout=10 > git rev-parse refs/remotes/origin/refs/heads/pespin/tmp^{commit} # timeout=10 Checking out Revision 0fa9d33c3f4de2bdc039f0fe92a42fb80f6f7259 (refs/remotes/origin/pespin/tmp) > git config core.sparsecheckout # timeout=10 > git checkout -f 0fa9d33c3f4de2bdc039f0fe92a42fb80f6f7259 Commit message: "bts_oc2g: Fix ready_for_pcu() waiting forever" > git rev-list --no-walk 19155245a4819e0e666c8e4938a43bfc5fd0969e # timeout=10 Copied 2 artifacts from "osmo-gsm-tester_build-osmo-nitb" build number 6700 Copied 2 artifacts from "osmo-gsm-tester_build-osmo-trx" build number 1690 Copied 2 artifacts from "osmo-gsm-tester_build-osmo-bts" build number 4055 Copied 2 artifacts from "osmo-gsm-tester_build-osmo-bts-sysmo" build number 4070 Copied 2 artifacts from "osmo-gsm-tester_build-osmo-pcu-sysmo" build number 2425 Copied 2 artifacts from "osmo-gsm-tester_build-osmo-pcu" build number 2387 Copied 2 artifacts from "osmo-gsm-tester_build-osmo-hlr" build number 3995 Copied 2 artifacts from "osmo-gsm-tester_build-osmo-msc" build number 14835 Copied 2 artifacts from "osmo-gsm-tester_build-osmo-bsc" build number 8837 Copied 2 artifacts from "osmo-gsm-tester_build-osmo-mgw" build number 4084 Copied 2 artifacts from "osmo-gsm-tester_build-osmo-stp" build number 4457 Copied 2 artifacts from "osmo-gsm-tester_build-osmo-sgsn" build number 14154 Copied 2 artifacts from "osmo-gsm-tester_build-osmo-ggsn" build number 3036 Copied 2 artifacts from "osmo-gsm-tester_build-osmocom-bb" build number 1968 Copied 2 artifacts from "osmo-gsm-tester_build-osmo-bts-oc2g" build number 1562 Copied 2 artifacts from "osmo-gsm-tester_build-osmo-pcu-oc2g" build number 1149 [osmo-gsm-tester_manual-run] $ /bin/sh /tmp/jenkins1585243220340283460.sh + export OSMO_GSM_TESTER_CONF=/home/jenkins/workspace/osmo-gsm-tester_manual-run/osmo-gsm-tester/example + unlink osmo-gsm-tester/example/resources.conf unlink: cannot unlink 'osmo-gsm-tester/example/resources.conf': No such file or directory + true + ln -s resources.conf.prod osmo-gsm-tester/example/resources.conf + PATH=/home/jenkins/workspace/osmo-gsm-tester_manual-run/osmo-gsm-tester/src:/usr/local/bin:/usr/bin:/bin:/usr/games ./osmo-gsm-tester/contrib/jenkins-run.sh + base=/home/jenkins/workspace/osmo-gsm-tester_manual-run + date +%F %T + time_start=2020-04-10 02:19:39 + trial_dir_prefix=trial- + rm -rf trial-273 trial-273-bin.tgz trial-273-run.tgz + trial_dir=trial-274 + mkdir -p trial-274 + mv osmo-bsc.build-8837.tgz osmo-bts-oc2g.build-1562.tgz osmo-bts-sysmo.build-4070.tgz osmo-bts.build-4055.tgz osmo-ggsn.build-3036.tgz osmo-hlr.build-3995.tgz osmo-mgw.build-4084.tgz osmo-msc.build-14835.tgz osmo-nitb.build-6700.tgz osmo-pcu-oc2g.build-1149.tgz osmo-pcu-sysmo.build-2425.tgz osmo-pcu.build-2387.tgz osmo-sgsn.build-14154.tgz osmo-stp.build-4457.tgz osmo-trx.build-1690.tgz osmocom-bb.build-1968.tgz trial-274 + cat osmo-bsc.build-8837.md5 osmo-bts-oc2g.build-1562.md5 osmo-bts-sysmo.build-4070.md5 osmo-bts.build-4055.md5 osmo-ggsn.build-3036.md5 osmo-hlr.build-3995.md5 osmo-mgw.build-4084.md5 osmo-msc.build-14835.md5 osmo-nitb.build-6700.md5 osmo-pcu-oc2g.build-1149.md5 osmo-pcu-sysmo.build-2425.md5 osmo-pcu.build-2387.md5 osmo-sgsn.build-14154.md5 osmo-stp.build-4457.md5 osmo-trx.build-1690.md5 osmocom-bb.build-1968.md5 + rm osmo-bsc.build-8837.md5 osmo-bts-oc2g.build-1562.md5 osmo-bts-sysmo.build-4070.md5 osmo-bts.build-4055.md5 osmo-ggsn.build-3036.md5 osmo-hlr.build-3995.md5 osmo-mgw.build-4084.md5 osmo-msc.build-14835.md5 osmo-nitb.build-6700.md5 osmo-pcu-oc2g.build-1149.md5 osmo-pcu-sysmo.build-2425.md5 osmo-pcu.build-2387.md5 osmo-sgsn.build-14154.md5 osmo-stp.build-4457.md5 osmo-trx.build-1690.md5 osmocom-bb.build-1968.md5 + exit_code=1 + which osmo-gsm-tester.py + python3 -u /home/jenkins/workspace/osmo-gsm-tester_manual-run/osmo-gsm-tester/src/osmo-gsm-tester.py trial-274 -s gprs:oc2g -T -t =ping.py combinations: ['gprs:oc2g'] series: None trial: 'trial-274' tests: ['=ping.py'] ['ping.py'] -------------------------------------------------------------------------------------------- trial-274 -------------------------------------------------------------------------------------------- 02:19:41.405934 tst trial-274: Detailed log at /home/jenkins/workspace/osmo-gsm-tester_manual-run/trial-274/run.2020-04-10_02-19-41/log --------------------------------------------------------------------- trial-274 gprs:oc2g --------------------------------------------------------------------- 02:19:42.804629 tst gprs:oc2g: reserving resources in /var/tmp/osmo-gsm-tester/state ... 02:19:42.824563 tst gprs:oc2g: Reserving 1 x bts (candidates: 11) 02:19:42.847928 tst gprs:oc2g: Reserving 9 x ip_address (candidates: 9) 02:19:42.898390 tst gprs:oc2g: Reserving 4 x modem (candidates: 10) ---------------------------------------------- trial-274 gprs:oc2g ping.py ---------------------------------------------- 02:19:43.360681 tst gprs:oc2g: Using 1 x ip_address (candidates: 9) 02:19:43.374352 tst gprs:oc2g: Using 1 x bts (candidates: 1) 02:19:44.656258 tst gprs:oc2g: using LAC 6018 02:19:44.674468 tst gprs:oc2g: using RAC 153 02:19:44.691577 tst gprs:oc2g: using CellId 6018 02:19:44.708207 tst gprs:oc2g: using BVCI 6019 02:19:44.711028 tst gprs:oc2g: Using 1 x ip_address (candidates: 9) 02:19:44.726125 tst gprs:oc2g: Using 1 x ip_address (candidates: 9) 02:19:44.739818 tst gprs:oc2g: Using 1 x ip_address (candidates: 9) 02:19:44.753352 tst gprs:oc2g: Using 1 x ip_address (candidates: 9) 02:19:44.766957 tst gprs:oc2g: Using 1 x ip_address (candidates: 9) 02:19:44.780455 tst gprs:oc2g: Using 1 x ip_address (candidates: 9) 02:19:44.794447 tst gprs:oc2g: Using 1 x ip_address (candidates: 9) 02:19:44.807885 tst gprs:oc2g: Using 1 x modem (candidates: 4) 02:19:44.897154 tst ping.py:19: start network... 02:19:45.107617 run osmo-hlr_10.42.42.2: Starting osmo-hlr 02:19:46.500280 run create_hlr_db(pid=10369): Launched 02:19:47.505762 run create_hlr_db(pid=10369): Terminated: ok {rc=0} 02:19:47.524776 run pcap-recorder_any(filters='host 10.42.42.2')(pid=10371): Launched 02:19:47.542943 run osmo-hlr_10.42.42.2(pid=10372): Launched 02:19:47.544444 run osmo-stp_10.42.42.5: Starting osmo-stp 02:19:49.929308 run pcap-recorder_any(filters='host 10.42.42.5 and port not 22')(pid=10373): Launched 02:19:49.947991 run osmo-stp_10.42.42.5(pid=10374): Launched 02:19:49.950865 run osmo-ggsn_10.42.42.6: Starting osmo-ggsn 02:19:51.304203 run pcap-recorder_any(filters='host 10.42.42.6')(pid=10375): Launched 02:19:51.307145 run osmo-ggsn_10.42.42.6: Setting RPATH for osmo-ggsn 02:19:51.327352 run patchelf(pid=10376): Launched 02:19:52.333284 run patchelf(pid=10376): Terminated: ok {rc=0} 02:19:52.335552 run osmo-ggsn_10.42.42.6: Applying CAP_NET_ADMIN capability to osmo-ggsn 02:19:52.356049 run osmo-gsm-tester_setcap_net_admin.sh(pid=10377): Launched 02:19:53.362344 run osmo-gsm-tester_setcap_net_admin.sh(pid=10377): Terminated: ok {rc=0} 02:19:53.380710 run osmo-ggsn_10.42.42.6(pid=10380): Launched 02:19:53.383382 run osmo-sgsn_10.42.42.7: Starting osmo-sgsn 02:19:57.624869 run pcap-recorder_any(filters='host 10.42.42.7')(pid=10607): Launched 02:19:57.642989 run osmo-sgsn_10.42.42.7(pid=10608): Launched 02:19:57.645000 run osmo-msc_10.42.42.8: Starting osmo-msc 02:20:02.200252 run pcap-recorder_any(filters='host 10.42.42.8 and port not 22')(pid=10612): Launched 02:20:02.219615 run osmo-msc_10.42.42.8(pid=10613): Launched 02:20:02.222291 run osmo-mgw_10.42.42.3: Starting osmo-mgw 02:20:03.680553 run pcap-recorder_any(filters='host 10.42.42.3 and port not 22')(pid=10614): Launched 02:20:03.698439 run osmo-mgw_10.42.42.3(pid=10615): Launched 02:20:03.700592 run osmo-mgw_10.42.42.4: Starting osmo-mgw 02:20:04.033301 run pcap-recorder_any(filters='host 10.42.42.4 and port not 22')(pid=10616): Launched 02:20:04.052357 run osmo-mgw_10.42.42.4(pid=10617): Launched 02:20:04.054684 run osmo-bsc_10.42.42.9: Starting osmo-bsc 02:20:08.083251 run pcap-recorder_any(filters='host 10.42.42.9 and port not 22')(pid=10618): Launched 02:20:08.101707 run osmo-bsc_10.42.42.9(pid=10619): Launched 02:20:08.104477 run osmo-bts-oc2g: Starting OsmoBtsOC2G to connect to osmo-bsc_10.42.42.9 02:20:08.822415 run rm-remote-dir(pid=10620): Launched 02:20:09.829363 run rm-remote-dir(pid=10620): Terminated: ok {rc=0} 02:20:09.849150 run mk-remote-dir(pid=10623): Launched 02:20:10.854891 run mk-remote-dir(pid=10623): Terminated: ok {rc=0} 02:20:10.873686 run scp-inst-to-remote(pid=10624): Launched 02:20:16.881599 run scp-inst-to-remote(pid=10624): Terminated: ok {rc=0} 02:20:16.901080 run mk-remote-dir(pid=10626): Launched 02:20:17.906605 run mk-remote-dir(pid=10626): Terminated: ok {rc=0} 02:20:17.925523 run scp-cfg-to-remote(pid=10627): Launched 02:20:18.935528 run scp-cfg-to-remote(pid=10627): Terminated: ok {rc=0} 02:20:18.942692 tst ping.py:29: ERR: NameError: name 'proc' is not defined [trial-274↪gprs:oc2g↪ping.py:29] 02:20:18.947567 tst ping.py:29: TRACEBACK: Traceback (most recent call last): File "/home/jenkins/workspace/osmo-gsm-tester_manual-run/osmo-gsm-tester/src/osmo_gsm_tester/test.py", line 64, in run self.path) File "/home/jenkins/workspace/osmo-gsm-tester_manual-run/osmo-gsm-tester/src/osmo_gsm_tester/util.py", line 367, in run_python_file spec.loader.exec_module( importlib.util.module_from_spec(spec) ) File "", line 673, in exec_module File "", line 222, in _call_with_frames_removed File "/home/jenkins/workspace/osmo-gsm-tester_manual-run/osmo-gsm-tester/example/suites/gprs/ping.py", line 29, in bts.start() File "/home/jenkins/workspace/osmo-gsm-tester_manual-run/osmo-gsm-tester/src/osmo_gsm_tester/bts_oc2g.py", line 133, in start self.suite_run.remember_to_stop(proc, keepalive) NameError: name 'proc' is not defined [trial-274↪gprs:oc2g↪ping.py:29] 02:20:18.950764 tst ping.py:29: Test FAILED (35.6 sec) 02:20:18.956161 run pcap-recorder_any(filters='host 10.42.42.2')(pid=10371): Terminating (SIGTERM) 02:20:18.957723 run osmo-hlr_10.42.42.2(pid=10372): Terminating (SIGTERM) 02:20:18.958765 run pcap-recorder_any(filters='host 10.42.42.5 and port not 22')(pid=10373): Terminating (SIGTERM) 02:20:18.960063 run osmo-stp_10.42.42.5(pid=10374): Terminating (SIGTERM) 02:20:18.960858 run pcap-recorder_any(filters='host 10.42.42.6')(pid=10375): Terminating (SIGTERM) 02:20:18.962140 run osmo-ggsn_10.42.42.6(pid=10380): Terminating (SIGTERM) 02:20:18.970510 run pcap-recorder_any(filters='host 10.42.42.7')(pid=10607): Terminating (SIGTERM) 02:20:18.971774 run osmo-sgsn_10.42.42.7(pid=10608): Terminating (SIGTERM) 02:20:18.972430 run pcap-recorder_any(filters='host 10.42.42.8 and port not 22')(pid=10612): Terminating (SIGTERM) 02:20:18.973072 run osmo-msc_10.42.42.8(pid=10613): Terminating (SIGTERM) 02:20:18.974356 run pcap-recorder_any(filters='host 10.42.42.3 and port not 22')(pid=10614): Terminating (SIGTERM) 02:20:18.975857 run osmo-mgw_10.42.42.3(pid=10615): Terminating (SIGTERM) 02:20:18.989246 run pcap-recorder_any(filters='host 10.42.42.4 and port not 22')(pid=10616): Terminating (SIGTERM) 02:20:19.004056 run osmo-mgw_10.42.42.4(pid=10617): Terminating (SIGTERM) 02:20:19.017483 run pcap-recorder_any(filters='host 10.42.42.9 and port not 22')(pid=10618): Terminating (SIGTERM) 02:20:19.020128 run osmo-bsc_10.42.42.9(pid=10619): Terminating (SIGTERM) 02:20:19.025762 --- ParallelTerminationStrategy: PID 10371 died... 02:20:19.031968 run pcap-recorder_any(filters='host 10.42.42.2')(pid=10371): Terminated: ok {rc=0} 02:20:19.033771 --- ParallelTerminationStrategy: PID 10373 died... 02:20:19.037568 run pcap-recorder_any(filters='host 10.42.42.5 and port not 22')(pid=10373): Terminated: ok {rc=0} 02:20:19.051445 --- ParallelTerminationStrategy: PID 10375 died... 02:20:19.055801 run pcap-recorder_any(filters='host 10.42.42.6')(pid=10375): Terminated: ok {rc=0} 02:20:19.060523 --- ParallelTerminationStrategy: PID 10607 died... 02:20:19.065449 run pcap-recorder_any(filters='host 10.42.42.7')(pid=10607): Terminated: ok {rc=0} 02:20:19.067002 --- ParallelTerminationStrategy: PID 10612 died... 02:20:19.071350 run pcap-recorder_any(filters='host 10.42.42.8 and port not 22')(pid=10612): Terminated: ok {rc=0} 02:20:19.072854 --- ParallelTerminationStrategy: PID 10614 died... 02:20:19.080069 run pcap-recorder_any(filters='host 10.42.42.3 and port not 22')(pid=10614): Terminated: ok {rc=0} 02:20:19.081549 --- ParallelTerminationStrategy: PID 10615 died... 02:20:19.083077 run osmo-mgw_10.42.42.3(pid=10615): Terminated {rc=15} 02:20:19.086914 --- ParallelTerminationStrategy: PID 10616 died... 02:20:19.088494 run pcap-recorder_any(filters='host 10.42.42.4 and port not 22')(pid=10616): Terminated: ok {rc=0} 02:20:19.095409 --- ParallelTerminationStrategy: PID 10617 died... 02:20:19.096872 run osmo-mgw_10.42.42.4(pid=10617): Terminated {rc=15} 02:20:19.099662 --- ParallelTerminationStrategy: PID 10618 died... 02:20:19.103343 run pcap-recorder_any(filters='host 10.42.42.9 and port not 22')(pid=10618): Terminated: ok {rc=0} 02:20:19.275418 --- ParallelTerminationStrategy: PID 10372 died... 02:20:19.276883 run osmo-hlr_10.42.42.2(pid=10372): Terminated: ok {rc=0} 02:20:19.278055 --- ParallelTerminationStrategy: PID 10374 died... 02:20:19.279230 run osmo-stp_10.42.42.5(pid=10374): Terminated: ok {rc=0} 02:20:19.280269 --- ParallelTerminationStrategy: PID 10380 died... 02:20:19.281448 run osmo-ggsn_10.42.42.6(pid=10380): Terminated: ok {rc=0} 02:20:20.531215 --- ParallelTerminationStrategy: PID 10608 died... 02:20:20.532786 run osmo-sgsn_10.42.42.7(pid=10608): Terminated: ok {rc=0} 02:20:22.522490 --- ParallelTerminationStrategy: PID 10613 died... 02:20:22.524057 run osmo-msc_10.42.42.8(pid=10613): Terminated: ok {rc=0} 02:20:22.525367 --- ParallelTerminationStrategy: PID 10619 died... 02:20:22.526928 run osmo-bsc_10.42.42.9(pid=10619): Terminated: ok {rc=0} 02:20:22.536118 bus /gobi_11: Setting Powered False --------------------------------------------------------------------- trial-274 gprs:oc2g FAIL --------------------------------------------------------------------- 02:20:23.810649 tst trial-274: Storing JUnit report in /home/jenkins/workspace/osmo-gsm-tester_manual-run/trial-274/run.2020-04-10_02-19-41/trial-274.xml -------------------------------------------------------------------------------------------- trial-274 FAIL -------------------------------------------------------------------------------------------- 02:20:23.912008 tst trial-274: trial-274: FAIL, 1 suites failed FAIL: gprs:oc2g (fail: 1, skip: 4) skip: cs_paging_gprs_active.py skip: iperf3.py skip: iperf3m4.py FAIL: ping.py (35.6 sec) NameError: name 'proc' is not defined skip: ping_idle_ping.py + rm -rf trial-274/inst + cd trial-274 + command -v journalctl + [ ! -z /bin/journalctl ] + readlink last_run + journalctl -u ofono -o short-precise --since 2020-04-10 02:19:39 + readlink last_run + tar czf /home/jenkins/workspace/osmo-gsm-tester_manual-run/trial-274-run.tgz run.2020-04-10_02-19-41 + tar czf /home/jenkins/workspace/osmo-gsm-tester_manual-run/trial-274-bin.tgz checksums.md5 osmo-bsc.build-8837.tgz osmo-bts-oc2g.build-1562.tgz osmo-bts-sysmo.build-4070.tgz osmo-bts.build-4055.tgz osmo-ggsn.build-3036.tgz osmo-hlr.build-3995.tgz osmo-mgw.build-4084.tgz osmo-msc.build-14835.tgz osmo-nitb.build-6700.tgz osmo-pcu-oc2g.build-1149.tgz osmo-pcu-sysmo.build-2425.tgz osmo-pcu.build-2387.tgz osmo-sgsn.build-14154.tgz osmo-stp.build-4457.tgz osmo-trx.build-1690.tgz osmocom-bb.build-1968.tgz + exit 1 Build step 'Execute shell' marked build as failure Archiving artifacts Recording test results Finished: FAILURE