{ "_class" : "hudson.tasks.junit.TestResult", "testActions" : [ ], "duration" : 1393.0, "empty" : False, "failCount" : 5, "passCount" : 2, "skipCount" : 0, "suites" : [ { "cases" : [ { "testActions" : [ ], "age" : 5, "className" : "nitb_sms:sysmo", "duration" : 5.0, "errorDetails" : None, "errorStackTrace" : "osmo-nitb_10.42.42.2(pid=20869): Process ended prematurely: osmo-nitb_10.42.42.2(pid=20869) [trial-2067↪nitb_sms:sysmo↪mo_mt_sms.py:11↪mo_mt_sms.py↪osmo-nitb_10.42.42.2↪osmo-nitb_10.42.42.2(pid=20869)]", "failedSince" : 2063, "name" : "mo_mt_sms.py", "properties" : { }, "skipped" : False, "skippedMessage" : None, "status" : "FAILED", "stderr" : "Traceback (most recent call last):\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/core/test.py\", line 76, in run\u000a self.path)\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/core/util.py\", line 414, in run_python_file\u000a spec.loader.exec_module( importlib.util.module_from_spec(spec) )\u000a File \"\", line 728, in exec_module\u000a File \"\", line 219, in _call_with_frames_removed\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/sysmocom/suites/nitb_sms/mo_mt_sms.py\", line 11, in \u000a nitb.start()\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/obj/nitb_osmo.py\", line 67, in start\u000a self.ctrl.connect()\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/obj/osmo_ctrl.py\", line 92, in connect\u000a MainLoop.wait(self.try_connect, timestep=3, timeout=timeout)\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/core/event_loop.py\", line 108, in wait\u000a if not self.wait_no_raise(condition, condition_args, condition_kwargs, timeout, timestep):\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/core/event_loop.py\", line 98, in wait_no_raise\u000a self.poll(may_block=True)\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/core/event_loop.py\", line 86, in poll\u000a self.deferred_handling.handle_queue()\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/core/event_loop.py\", line 33, in handle_queue\u000a handler(*args, **kwargs)\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/testenv.py\", line 142, in poll\u000a raise log_module.Error('Process ended prematurely: %s' % proc.name())\u000aosmo_gsm_tester.core.log.Error: osmo-nitb_10.42.42.2(pid=20869): Process ended prematurely: osmo-nitb_10.42.42.2(pid=20869) [trial-2067↪nitb_sms:sysmo↪mo_mt_sms.py:11↪mo_mt_sms.py↪osmo-nitb_10.42.42.2↪osmo-nitb_10.42.42.2(pid=20869)]\u000a", "stdout" : "\u000a----------------------------------------------\u000atrial-2067 nitb_sms:sysmo mo_mt_sms.py\u000a----------------------------------------------\u000a12:00:23.734225 tst nitb_sms:sysmo: Using 1 x ip_address (candidates: 1)\u000a12:00:23.745166 tst nitb_sms:sysmo: Using 1 x bts (candidates: 1)\u000a12:00:24.316187 tst mo_mt_sms.py: using LAC 3964\u000a12:00:24.426764 tst mo_mt_sms.py: using RAC 139\u000a12:00:24.537458 tst mo_mt_sms.py: using CellId 3964\u000a12:00:24.648699 tst mo_mt_sms.py: using BVCI 3965\u000a12:00:24.670483 tst nitb_sms:sysmo: Using 1 x modem (candidates: 2)\u000a12:00:24.867660 tst nitb_sms:sysmo: Using 1 x modem (candidates: 2)\u000a12:00:25.138945 tst mo_mt_sms.py:9: start nitb and bts...\u000a12:00:25.262034 run osmo-nitb_10.42.42.2: Starting osmo-nitb\u000a12:00:27.498251 run pcap-recorder_any(filters='host 10.42.42.2 and port not 22')(pid=20868): Launched\u000a12:00:27.794993 run osmo-nitb_10.42.42.2(pid=20869): Launched\u000a12:00:27.965841 run osmo-nitb_10.42.42.2(pid=20869): ERR: Terminated: ERROR {rc=2} [trial-2067↪nitb_sms:sysmo↪mo_mt_sms.py:11↪mo_mt_sms.py↪osmo-nitb_10.42.42.2↪osmo-nitb_10.42.42.2(pid=20869)]\u000a12:00:28.089734 run osmo-nitb_10.42.42.2(pid=20869): stdout: \u000a| (launched: 2022-09-08_12:00:27.648943) \u000a12:00:28.214083 run osmo-nitb_10.42.42.2(pid=20869): stderr: \u000a| (launched: 2022-09-08_12:00:27.693149)\u000a| ======================================================================\u000a| ERROR: You should not run osmo-nitb. It is ancient, unmaintained and\u000a| obsolete for several years. If you really want to ignore years of hard work on\u000a| the new post-NITB stack (osmo-bsc, osmo-mgw, osmo-msc, osmo-hlr, osmo-stp),\u000a| you may ues the '--yes-i-really-want-to-run-prehistoric-software' command line\u000a| argument. DON'T EVER BOTHER THE DEVELOPERS IF YOU ENCOUNTER PROBLEMS WITH IT!!!\u000a| Use https://osmocom.org/projects/cellular-infrastructure/wiki/Osmocom_Network_In_The_Box\u000a| ====================================================================== \u000a12:00:28.333735 run osmo-nitb_10.42.42.2(pid=20869): stdout: \u000a| (launched: 2022-09-08_12:00:27.648943) \u000a12:00:28.452520 run osmo-nitb_10.42.42.2(pid=20869): stderr: \u000a| (launched: 2022-09-08_12:00:27.693149)\u000a| ======================================================================\u000a| ERROR: You should not run osmo-nitb. It is ancient, unmaintained and\u000a| obsolete for several years. If you really want to ignore years of hard work on\u000a| the new post-NITB stack (osmo-bsc, osmo-mgw, osmo-msc, osmo-hlr, osmo-stp),\u000a| you may ues the '--yes-i-really-want-to-run-prehistoric-software' command line\u000a| argument. DON'T EVER BOTHER THE DEVELOPERS IF YOU ENCOUNTER PROBLEMS WITH IT!!!\u000a| Use https://osmocom.org/projects/cellular-infrastructure/wiki/Osmocom_Network_In_The_Box\u000a| ====================================================================== \u000a12:00:28.504456 tst mo_mt_sms.py:11: ERR: Error: osmo-nitb_10.42.42.2(pid=20869): Process ended prematurely: osmo-nitb_10.42.42.2(pid=20869) [trial-2067↪nitb_sms:sysmo↪mo_mt_sms.py:11↪mo_mt_sms.py↪osmo-nitb_10.42.42.2↪osmo-nitb_10.42.42.2(pid=20869)] [trial-2067↪nitb_sms:sysmo↪mo_mt_sms.py:11]\u000a12:00:28.512421 tst mo_mt_sms.py:11: Test FAILED (4.8 sec)\u000a12:00:28.547078 run pcap-recorder_any(filters='host 10.42.42.2 and port not 22')(pid=20868): Terminating (SIGTERM)\u000a12:00:28.724223 --- ParallelTerminationStrategy: PID 20868 died...\u000a12:00:28.779653 run pcap-recorder_any(filters='host 10.42.42.2 and port not 22')(pid=20868): Terminated: ok {rc=0}\u000a12:00:28.868284 bus /sierra_3: Setting Powered False\u000a12:00:29.975240 bus /sierra_3: Setting Powered False\u000a12:00:31.087063 bus /sierra_2: Setting Powered False\u000a" } ], "duration" : 10.0, "enclosingBlockNames" : [ ], "enclosingBlocks" : [ ], "id" : "0", "name" : "nitb_sms_sysmo", "nodeId" : None, "properties" : { }, "stderr" : None, "stdout" : None, "timestamp" : "2022-09-08T12:00:23" }, { "cases" : [ { "testActions" : [ ], "age" : 0, "className" : "sms:sysmo", "duration" : 114.0, "errorDetails" : None, "errorStackTrace" : None, "failedSince" : 0, "name" : "mo_mt_sms.py", "properties" : { }, "skipped" : False, "skippedMessage" : None, "status" : "PASSED", "stderr" : None, "stdout" : "\u000a----------------------------------------------\u000atrial-2067 sms:sysmo mo_mt_sms.py\u000a----------------------------------------------\u000a12:00:32.941671 tst sms:sysmo: Using 1 x ip_address (candidates: 6)\u000a12:00:32.953378 tst sms:sysmo: Using 1 x bts (candidates: 1)\u000a12:00:33.080281 tst mo_mt_sms.py: using LAC 3965\u000a12:00:33.203155 tst mo_mt_sms.py: using RAC 140\u000a12:00:33.327535 tst mo_mt_sms.py: using CellId 3965\u000a12:00:33.453511 tst mo_mt_sms.py: using BVCI 3966\u000a12:00:33.477571 tst sms:sysmo: Using 1 x ip_address (candidates: 6)\u000a12:00:33.488756 tst sms:sysmo: Using 1 x ip_address (candidates: 6)\u000a12:00:33.499935 tst sms:sysmo: Using 1 x ip_address (candidates: 6)\u000a12:00:33.511192 tst sms:sysmo: Using 1 x ip_address (candidates: 6)\u000a12:00:33.522424 tst sms:sysmo: Using 1 x ip_address (candidates: 6)\u000a12:00:33.533739 tst sms:sysmo: Using 1 x modem (candidates: 2)\u000a12:00:33.647394 tst sms:sysmo: Using 1 x modem (candidates: 2)\u000a12:00:33.847405 run osmo-hlr_10.42.42.2: Starting osmo-hlr\u000a12:00:35.547841 run create_hlr_db(pid=20873): Launched\u000a12:00:35.718412 bus /sierra_2: Setting Powered False\u000a12:00:36.739665 run create_hlr_db(pid=20873): Terminated: ok {rc=0}\u000a12:00:37.137618 run pcap-recorder_any(filters='host 10.42.42.2')(pid=20875): Launched\u000a12:00:37.436123 run osmo-hlr_10.42.42.2(pid=20876): Launched\u000a12:00:37.543873 run osmo-stp_10.42.42.5: Starting osmo-stp\u000a12:00:40.052307 run pcap-recorder_any(filters='host 10.42.42.5 and port not 22')(pid=20878): Launched\u000a12:00:40.349356 run osmo-stp_10.42.42.5(pid=20879): Launched\u000a12:00:40.455179 run osmo-msc_10.42.42.6: Starting osmo-msc\u000a12:00:45.724814 run pcap-recorder_any(filters='host 10.42.42.6 and port not 22')(pid=20880): Launched\u000a12:00:46.023902 run osmo-msc_10.42.42.6(pid=20881): Launched\u000a12:00:49.184031 run osmo-mgw_10.42.42.3: Starting osmo-mgw\u000a12:00:50.882779 run pcap-recorder_any(filters='host 10.42.42.3 and port not 22')(pid=20882): Launched\u000a12:00:51.182095 run osmo-mgw_10.42.42.3(pid=20883): Launched\u000a12:00:51.292209 run osmo-mgw_10.42.42.4: Starting osmo-mgw\u000a12:00:51.800177 run pcap-recorder_any(filters='host 10.42.42.4 and port not 22')(pid=20884): Launched\u000a12:00:52.098587 run osmo-mgw_10.42.42.4(pid=20885): Launched\u000a12:00:52.207586 run osmo-bsc_10.42.42.7: Starting osmo-bsc\u000a12:00:56.750105 run pcap-recorder_any(filters='host 10.42.42.7 and port not 22')(pid=20886): Launched\u000a12:00:57.051336 run osmo-bsc_10.42.42.7(pid=20887): Launched\u000a12:01:03.323006 run osmo-bts-sysmo: Starting sysmoBTS to connect to osmo-bsc_10.42.42.7\u000a12:01:04.277144 run rm-remote-dir(pid=20888): Launched\u000a12:01:06.473283 run rm-remote-dir(pid=20888): Terminated: ok {rc=0}\u000a12:01:06.813777 run mk-remote-dir(pid=20889): Launched\u000a12:01:08.007902 run mk-remote-dir(pid=20889): Terminated: ok {rc=0}\u000a12:01:08.300618 run scp-inst-to-remote(pid=20890): Launched\u000a12:01:20.512512 run scp-inst-to-remote(pid=20890): Terminated: ok {rc=0}\u000a12:01:20.849517 run mk-remote-dir(pid=20892): Launched\u000a12:01:22.040695 run mk-remote-dir(pid=20892): Terminated: ok {rc=0}\u000a12:01:22.329418 run scp-cfg-to-remote(pid=20893): Launched\u000a12:01:23.511166 run scp-cfg-to-remote(pid=20893): Terminated: ok {rc=0}\u000a12:01:23.821709 run reload-dsp-firmware(pid=20895): Launched\u000a12:01:26.001236 run reload-dsp-firmware(pid=20895): Terminated: ok {rc=0}\u000a12:01:26.304832 run osmo-bts-sysmo(pid=20896): Launched\u000a12:01:30.550633 tst mo_mt_sms.py: using MSISDN 3980\u000a12:01:30.717702 bus /sierra_2: Setting Powered True\u000a12:01:34.172035 run osmo-hlr_10.42.42.2: Add subscriber {algo=1, algo_str='comp128v1', imsi='901700000015252', msisdn='3980', subscriber_id=1}\u000a12:01:34.306401 tst mo_mt_sms.py: using MSISDN 3981\u000a12:01:34.471876 bus /sierra_3: Setting Powered True\u000a12:01:37.962500 run osmo-hlr_10.42.42.2: Add subscriber {algo=1, algo_str='comp128v1', imsi='901700000015253', msisdn='3981', subscriber_id=2}\u000a12:01:38.228220 bus /sierra_2: Setting Online False\u000a12:01:39.406860 bus /sierra_2: Setting Powered False\u000a12:01:45.164218 bus /sierra_2: Setting Powered True\u000a12:01:46.517079 bus /sierra_2: Setting Online True\u000a12:01:48.976383 tst /sierra_2: Connect to ('901', '70')\u000a12:01:49.283712 bus /sierra_3: Setting Online False\u000a12:01:50.460933 bus /sierra_3: Setting Powered False\u000a12:01:56.162802 bus /sierra_3: Setting Powered True\u000a12:01:57.506556 bus /sierra_3: Setting Online True\u000a12:02:00.015081 tst /sierra_3: Connect to ('901', '70')\u000a12:02:00.171868 tst /sierra_2: Manufacturer: 'Sierra Wireless, Incorporated', Model: 'MC7304', Revision: 'SWI9X15C_06.03.32.04 r28324 CNSHZ-AR-BUILD 2015/05/25 01:09:15', Serial: '356853054228053'\u000a12:02:00.292611 tst /sierra_3: Manufacturer: 'Sierra Wireless, Incorporated', Model: 'MC7304', Revision: 'SWI9X15C_06.03.32.04 r28324 CNSHZ-AR-BUILD 2015/05/25 01:09:15', Serial: '356853054229028'\u000a12:02:00.486085 tst mo_mt_sms.py:35: waiting for modems to attach...\u000a12:02:04.225129 tst /sierra_2: Already registered with network ('901', '70')\u000a12:02:15.780607 tst /sierra_3: Already registered with network ('901', '70')\u000a12:02:20.382293 tst /sierra_2: sending sms to MSISDN 3981 {sms='message nr. 1, from /sierra_2, to /sierra_3, from 3980, to 3981'}\u000a12:02:26.191115 tst /sierra_3: Incoming SMS: 'message nr. 1, from /sierra_2, to /sierra_3, from 3980, to 3981'\u000a12:02:26.517939 tst /sierra_3: SMS received as expected: 'message nr. 1, from /sierra_2, to /sierra_3, from 3980, to 3981'\u000a12:02:26.646197 tst mo_mt_sms.py: Test passed (113.7 sec)\u000a12:02:26.695074 run pcap-recorder_any(filters='host 10.42.42.2')(pid=20875): Terminating (SIGTERM)\u000a12:02:26.737985 run osmo-hlr_10.42.42.2(pid=20876): Terminating (SIGTERM)\u000a12:02:26.782191 run pcap-recorder_any(filters='host 10.42.42.5 and port not 22')(pid=20878): Terminating (SIGTERM)\u000a12:02:26.826245 run osmo-stp_10.42.42.5(pid=20879): Terminating (SIGTERM)\u000a12:02:26.890563 run pcap-recorder_any(filters='host 10.42.42.6 and port not 22')(pid=20880): Terminating (SIGTERM)\u000a12:02:26.936253 run osmo-msc_10.42.42.6(pid=20881): Terminating (SIGTERM)\u000a12:02:26.979409 run pcap-recorder_any(filters='host 10.42.42.3 and port not 22')(pid=20882): Terminating (SIGTERM)\u000a12:02:27.023368 run osmo-mgw_10.42.42.3(pid=20883): Terminating (SIGTERM)\u000a12:02:27.066783 run pcap-recorder_any(filters='host 10.42.42.4 and port not 22')(pid=20884): Terminating (SIGTERM)\u000a12:02:27.112019 run osmo-mgw_10.42.42.4(pid=20885): Terminating (SIGTERM)\u000a12:02:27.163800 run pcap-recorder_any(filters='host 10.42.42.7 and port not 22')(pid=20886): Terminating (SIGTERM)\u000a12:02:27.213077 run osmo-bsc_10.42.42.7(pid=20887): Terminating (SIGTERM)\u000a12:02:27.256078 run osmo-bts-sysmo(pid=20896): Terminating (SIGTERM)\u000a12:02:27.266180 --- ParallelTerminationStrategy: PID 20875 died...\u000a12:02:27.324637 run pcap-recorder_any(filters='host 10.42.42.2')(pid=20875): Terminated: ok {rc=0}\u000a12:02:27.335497 --- ParallelTerminationStrategy: PID 20876 died...\u000a12:02:27.393674 run osmo-hlr_10.42.42.2(pid=20876): Terminated: ok {rc=0}\u000a12:02:27.404566 --- ParallelTerminationStrategy: PID 20878 died...\u000a12:02:27.462520 run pcap-recorder_any(filters='host 10.42.42.5 and port not 22')(pid=20878): Terminated: ok {rc=0}\u000a12:02:27.473306 --- ParallelTerminationStrategy: PID 20879 died...\u000a12:02:27.533057 run osmo-stp_10.42.42.5(pid=20879): Terminated: ok {rc=0}\u000a12:02:27.543431 --- ParallelTerminationStrategy: PID 20880 died...\u000a12:02:27.599441 run pcap-recorder_any(filters='host 10.42.42.6 and port not 22')(pid=20880): Terminated: ok {rc=0}\u000a12:02:27.609751 --- ParallelTerminationStrategy: PID 20881 died...\u000a12:02:27.665926 run osmo-msc_10.42.42.6(pid=20881): Terminated: ok {rc=0}\u000a12:02:27.675955 --- ParallelTerminationStrategy: PID 20882 died...\u000a12:02:27.732530 run pcap-recorder_any(filters='host 10.42.42.3 and port not 22')(pid=20882): Terminated: ok {rc=0}\u000a12:02:27.742600 --- ParallelTerminationStrategy: PID 20883 died...\u000a12:02:27.799245 run osmo-mgw_10.42.42.3(pid=20883): Terminated {rc=15}\u000a12:02:27.809291 --- ParallelTerminationStrategy: PID 20884 died...\u000a12:02:27.865742 run pcap-recorder_any(filters='host 10.42.42.4 and port not 22')(pid=20884): Terminated: ok {rc=0}\u000a12:02:27.875801 --- ParallelTerminationStrategy: PID 20885 died...\u000a12:02:27.932864 run osmo-mgw_10.42.42.4(pid=20885): Terminated {rc=15}\u000a12:02:27.943073 --- ParallelTerminationStrategy: PID 20886 died...\u000a12:02:27.999501 run pcap-recorder_any(filters='host 10.42.42.7 and port not 22')(pid=20886): Terminated: ok {rc=0}\u000a12:02:28.009793 --- ParallelTerminationStrategy: PID 20887 died...\u000a12:02:28.065958 run osmo-bsc_10.42.42.7(pid=20887): Terminated: ok {rc=0}\u000a12:02:28.076073 --- ParallelTerminationStrategy: PID 20896 died...\u000a12:02:28.132511 run osmo-bts-sysmo(pid=20896): Terminated: ok {rc=0}\u000a12:02:28.262065 bus /sierra_3: Setting Online False\u000a12:02:29.383751 bus /sierra_3: Setting Powered False\u000a12:02:39.955653 bus /sierra_3: Setting Powered False\u000a12:02:41.062393 bus /sierra_3: Setting Powered False\u000a12:02:42.219150 bus /sierra_2: Setting Online False\u000a12:02:43.339232 bus /sierra_2: Setting Powered False\u000a12:02:49.955485 bus /sierra_2: Setting Powered False\u000a" } ], "duration" : 139.0, "enclosingBlockNames" : [ ], "enclosingBlocks" : [ ], "id" : "1", "name" : "sms_sysmo", "nodeId" : None, "properties" : { }, "stderr" : None, "stdout" : None, "timestamp" : "2022-09-08T12:00:32" }, { "cases" : [ { "testActions" : [ ], "age" : 5, "className" : "gprs:sysmo", "duration" : 225.0, "errorDetails" : None, "errorStackTrace" : "'sum_sent'", "failedSince" : 2063, "name" : "cs_paging_gprs_active.py", "properties" : { }, "skipped" : False, "skippedMessage" : None, "status" : "FAILED", "stderr" : "Traceback (most recent call last):\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/core/test.py\", line 76, in run\u000a self.path)\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/core/util.py\", line 414, in run_python_file\u000a spec.loader.exec_module( importlib.util.module_from_spec(spec) )\u000a File \"\", line 728, in exec_module\u000a File \"\", line 219, in _call_with_frames_removed\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/sysmocom/suites/gprs/cs_paging_gprs_active.py\", line 38, in \u000a setup_run_iperf3_test_parallel(2, ready_cb=ready_cb_place_voicecall)\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/sysmocom/suites/gprs/lib/testlib.py\", line 105, in setup_run_iperf3_test_parallel\u000a print_results(clients[i], servers[i])\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/sysmocom/suites/gprs/lib/testlib.py\", line 5, in print_results\u000a cli.print_results()\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/obj/iperf3.py\", line 342, in print_results\u000a print_result_node_tcp(self.get_results(), 'client')\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/obj/iperf3.py\", line 72, in print_result_node_tcp\u000a raise e\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/obj/iperf3.py\", line 65, in print_result_node_tcp\u000a sent = result['end']['sum_sent']\u000aKeyError: 'sum_sent'\u000a", "stdout" : "\u000a----------------------------------------------\u000atrial-2067 gprs:sysmo cs_paging_gprs_active.py\u000a----------------------------------------------\u000a12:02:52.061673 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:02:52.073087 tst gprs:sysmo: Using 1 x bts (candidates: 1)\u000a12:02:52.203530 tst cs_paging_gprs_active.py: using LAC 3966\u000a12:02:52.331173 tst cs_paging_gprs_active.py: using RAC 141\u000a12:02:52.457129 tst cs_paging_gprs_active.py: using CellId 3966\u000a12:02:52.584021 tst cs_paging_gprs_active.py: using BVCI 3967\u000a12:02:52.608809 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:02:52.619934 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:02:52.630995 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:02:52.641998 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:02:52.653090 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:02:52.664212 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:02:52.675411 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:02:52.686440 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:02:52.698060 tst gprs:sysmo: Using 1 x modem (candidates: 4)\u000a12:02:52.818004 tst gprs:sysmo: Using 1 x modem (candidates: 4)\u000a12:02:53.095823 tst cs_paging_gprs_active.py:38: start iperfv3 server 10.42.42.10:5003...\u000a12:02:53.223298 run iperf3-srv_10.42.42.10: Starting iperf3-srv\u000a12:02:53.540396 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=20907): Launched\u000a12:02:53.806400 run iperf3-srv_10.42.42.10(pid=20908): Launched\u000a12:02:53.991494 tst cs_paging_gprs_active.py:38: start iperfv3 server 10.42.42.10:5004...\u000a12:02:54.117930 run iperf3-srv_10.42.42.10: Starting iperf3-srv\u000a12:02:54.433595 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=20909): Launched\u000a12:02:54.698945 run iperf3-srv_10.42.42.10(pid=20910): Launched\u000a12:02:54.884180 tst cs_paging_gprs_active.py:38: start network...\u000a12:02:55.010248 run osmo-hlr_10.42.42.2: Starting osmo-hlr\u000a12:02:55.499541 run create_hlr_db(pid=20911): Launched\u000a12:02:55.674398 bus /sierra_2: Setting Powered False\u000a12:02:56.696923 run create_hlr_db(pid=20911): Terminated: ok {rc=0}\u000a12:02:57.103662 run pcap-recorder_any(filters='host 10.42.42.2')(pid=20913): Launched\u000a12:02:57.411124 run osmo-hlr_10.42.42.2(pid=20914): Launched\u000a12:02:57.521101 run osmo-stp_10.42.42.5: Starting osmo-stp\u000a12:02:58.050448 run pcap-recorder_any(filters='host 10.42.42.5 and port not 22')(pid=20916): Launched\u000a12:02:58.355191 run osmo-stp_10.42.42.5(pid=20917): Launched\u000a12:02:58.464046 run osmo-ggsn_10.42.42.6: Starting osmo-ggsn\u000a12:02:59.994493 run pcap-recorder_any(filters='host 10.42.42.6')(pid=20918): Launched\u000a12:03:00.102256 run osmo-ggsn_10.42.42.6: Setting RPATH for osmo-ggsn\u000a12:03:00.370119 run patchelf(pid=20919): Launched\u000a12:03:01.551331 run patchelf(pid=20919): Terminated: ok {rc=0}\u000a12:03:01.662853 run osmo-ggsn_10.42.42.6: Applying CAP_NET_ADMIN capability to osmo-ggsn\u000a12:03:01.937143 run osmo-gsm-tester_setcap_net_admin.sh(pid=20920): Launched\u000a12:03:03.118623 run osmo-gsm-tester_setcap_net_admin.sh(pid=20920): Terminated: ok {rc=0}\u000a12:03:03.428863 run osmo-ggsn_10.42.42.6(pid=20923): Launched\u000a12:03:03.693462 run osmo-sgsn_10.42.42.7: Starting osmo-sgsn\u000a12:03:08.652675 run pcap-recorder_any(filters='host 10.42.42.7')(pid=20950): Launched\u000a12:03:08.959849 run osmo-sgsn_10.42.42.7(pid=20951): Launched\u000a12:03:09.073020 run osmo-msc_10.42.42.8: Starting osmo-msc\u000a12:03:09.652721 run pcap-recorder_any(filters='host 10.42.42.8 and port not 22')(pid=20952): Launched\u000a12:03:09.960697 run osmo-msc_10.42.42.8(pid=20953): Launched\u000a12:03:13.123154 run osmo-mgw_10.42.42.3: Starting osmo-mgw\u000a12:03:13.656455 run pcap-recorder_any(filters='host 10.42.42.3 and port not 22')(pid=20954): Launched\u000a12:03:13.963680 run osmo-mgw_10.42.42.3(pid=20955): Launched\u000a12:03:14.082903 run osmo-mgw_10.42.42.4: Starting osmo-mgw\u000a12:03:14.604781 run pcap-recorder_any(filters='host 10.42.42.4 and port not 22')(pid=20956): Launched\u000a12:03:14.912603 run osmo-mgw_10.42.42.4(pid=20957): Launched\u000a12:03:15.033325 run osmo-bsc_10.42.42.9: Starting osmo-bsc\u000a12:03:15.716980 run pcap-recorder_any(filters='host 10.42.42.9 and port not 22')(pid=20958): Launched\u000a12:03:16.047395 run osmo-bsc_10.42.42.9(pid=20959): Launched\u000a12:03:22.325740 run osmo-bts-sysmo: Starting sysmoBTS to connect to osmo-bsc_10.42.42.9\u000a12:03:22.909093 run rm-remote-dir(pid=20960): Launched\u000a12:03:25.111109 run rm-remote-dir(pid=20960): Terminated: ok {rc=0}\u000a12:03:25.462103 run mk-remote-dir(pid=20961): Launched\u000a12:03:26.661468 run mk-remote-dir(pid=20961): Terminated: ok {rc=0}\u000a12:03:26.957998 run scp-inst-to-remote(pid=20962): Launched\u000a12:03:39.175332 run scp-inst-to-remote(pid=20962): Terminated: ok {rc=0}\u000a12:03:39.522269 run mk-remote-dir(pid=20964): Launched\u000a12:03:40.718859 run mk-remote-dir(pid=20964): Terminated: ok {rc=0}\u000a12:03:41.012711 run scp-cfg-to-remote(pid=20965): Launched\u000a12:03:42.199568 run scp-cfg-to-remote(pid=20965): Terminated: ok {rc=0}\u000a12:03:42.517052 run reload-dsp-firmware(pid=20967): Launched\u000a12:03:44.701337 run reload-dsp-firmware(pid=20967): Terminated: ok {rc=0}\u000a12:03:45.014037 run osmo-bts-sysmo(pid=20968): Launched\u000a12:03:49.328015 tst cs_paging_gprs_active.py:38: Waiting for bts to be ready...\u000a12:03:51.340332 run rm-remote-dir(pid=20969): Launched\u000a12:03:53.533282 run rm-remote-dir(pid=20969): Terminated: ok {rc=0}\u000a12:03:53.869936 run mk-remote-dir(pid=20970): Launched\u000a12:03:56.063698 run mk-remote-dir(pid=20970): Terminated: ok {rc=0}\u000a12:03:56.351025 run scp-inst-to-bts(pid=20971): Launched\u000a12:04:09.547909 run scp-inst-to-bts(pid=20971): Terminated: ok {rc=0}\u000a12:04:09.863979 run mk-remote-run-dir(pid=20973): Launched\u000a12:04:12.045074 run mk-remote-run-dir(pid=20973): Terminated: ok {rc=0}\u000a12:04:12.314282 run scp-cfg-to-sysmobts(pid=20974): Launched\u000a12:04:14.506603 run scp-cfg-to-sysmobts(pid=20974): Terminated: ok {rc=0}\u000a12:04:14.843606 run osmo-pcu(pid=20976): Launched\u000a12:04:14.972854 tst cs_paging_gprs_active.py: using MSISDN 3982\u000a12:04:15.143584 bus /gobi_6: Setting Powered True\u000a12:04:18.670216 run osmo-hlr_10.42.42.2: Add subscriber {algo=1, algo_str='comp128v1', imsi='901700000010978', msisdn='3982', subscriber_id=1}\u000a12:04:18.958148 bus /gobi_6: Setting Online False\u000a12:04:20.143603 bus /gobi_6: Setting Powered False\u000a12:04:27.228641 bus /gobi_6: Setting Powered True\u000a12:04:28.596328 bus /gobi_6: Setting Online True\u000a12:04:31.031252 tst /gobi_6: Connect to ('901', '70')\u000a12:04:31.302532 tst /gobi_6: Manufacturer: 'Quectel', Model: 'EC20', Revision: 'EC20EQAR02A13E2G 1 [Jul 5 2017 10:49:17]', Serial: '868323024273648'\u000a12:04:31.430236 tst cs_paging_gprs_active.py: using MSISDN 3983\u000a12:04:31.649643 bus /gobi_0: Setting Powered True\u000a12:04:35.134137 run osmo-hlr_10.42.42.2: Add subscriber {algo=1, algo_str='comp128v1', imsi='901700000023847', msisdn='3983', subscriber_id=2}\u000a12:04:35.401509 bus /gobi_0: Setting Online False\u000a12:04:36.594147 bus /gobi_0: Setting Powered False\u000a12:04:43.364599 bus /gobi_0: Setting Powered True\u000a12:04:44.768534 bus /gobi_0: Setting Online True\u000a12:04:46.469274 tst /gobi_6: Already registered with network ('901', '70')\u000a12:04:47.245209 tst /gobi_0: Connect to ('901', '70')\u000a12:04:47.522788 tst /gobi_0: Manufacturer: 'Quectel', Model: 'EC20', Revision: 'EC20EQAR02A05E2G 1 [Mar 14 2016 19:33:01]', Serial: '868323024271873'\u000a12:04:47.723618 tst cs_paging_gprs_active.py:38: waiting for modems to attach...\u000a12:05:01.893097 tst /gobi_0: Already registered with network ('901', '70')\u000a12:05:07.333732 tst cs_paging_gprs_active.py:38: waiting for modems to attach to data services...\u000a12:05:10.371232 tst /gobi_6: context activated {apn='inet46', path='/gobi_6/context2', properties={AccessPointName='inet46', Active=True, AuthenticationMethod='chap', IPv6.Settings={}, Name='Internet', Password='', Protocol='ip', Settings={Address='176.16.46.1', DomainNameServers=['192.168.100.1', '8.8.8.8'], Gateway='176.16.46.2', Interface='r433306bbc2604b', Method='static', Netmask='255.255.255.252'}, Type='internet', Username='ogt'}, user='ogt'}\u000a12:05:10.571800 tst cs_paging_gprs_active.py:38: Setting up data plan for \"'/gobi_6/context2'\"\u000a12:05:10.953341 run move_netns(pid=20985): Launched\u000a12:05:12.150061 run move_netns(pid=20985): Terminated: ok {rc=0}\u000a12:05:12.509734 run ifup(pid=20988): Launched\u000a12:05:13.705435 run ifup(pid=20988): Terminated: ok {rc=0}\u000a12:05:14.056066 run dhcp(pid=20991): Launched\u000a12:05:15.238952 run dhcp(pid=20991): Terminated: ok {rc=0}\u000a12:05:18.308181 tst /gobi_0: context activated {apn='inet46', path='/gobi_0/context3', properties={AccessPointName='inet46', Active=True, AuthenticationMethod='chap', IPv6.Settings={}, Name='Internet', Password='', Protocol='ip', Settings={Address='176.16.46.2', DomainNameServers=['192.168.100.1', '8.8.8.8'], Gateway='176.16.46.1', Interface='r82dcf628584c1c', Method='static', Netmask='255.255.255.252'}, Type='internet', Username='ogt'}, user='ogt'}\u000a12:05:18.507204 tst cs_paging_gprs_active.py:38: Setting up data plan for \"'/gobi_0/context3'\"\u000a12:05:18.875369 run move_netns(pid=21009): Launched\u000a12:05:20.056940 run move_netns(pid=21009): Terminated: ok {rc=0}\u000a12:05:20.376733 run ifup(pid=21012): Launched\u000a12:05:21.559688 run ifup(pid=21012): Terminated: ok {rc=0}\u000a12:05:21.882953 run dhcp(pid=21015): Launched\u000a12:05:23.065380 run dhcp(pid=21015): Terminated: ok {rc=0}\u000a12:05:23.257067 tst cs_paging_gprs_active.py:38: Running iperf3 client to 10.42.42.10:5003 through \"'/gobi_6/context2'\"\u000a12:05:23.386024 run iperf3-cli_10.42.42.10: Preparing iperf3-client connecting to 10.42.42.10:5003 (proto=tcp,time=60s)\u000a12:05:23.771495 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21031): Launched\u000a12:05:24.023657 tst cs_paging_gprs_active.py:38: Running iperf3 client to 10.42.42.10:5004 through \"'/gobi_0/context3'\"\u000a12:05:24.161703 run iperf3-cli_10.42.42.10: Preparing iperf3-client connecting to 10.42.42.10:5004 (proto=tcp,time=60s)\u000a12:05:24.561263 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21034): Launched\u000a12:05:24.893468 run iperf3-cli_10.42.42.10(pid=21037): Launched\u000a12:05:25.177038 run iperf3-cli_10.42.42.10(pid=21040): Launched\u000a12:05:25.400711 tst cs_paging_gprs_active.py:13: waiting a few secs to make sure iperf3 test is running\u000a12:05:27.662423 tst cs_paging_gprs_active.py:16: iperf3 running, let's place a call\u000a12:05:32.542421 tst cs_paging_gprs_active.py:22: dial success\u000a12:05:35.313151 tst cs_paging_gprs_active.py:28: answer success, call established and ongoing\u000a12:05:41.896195 tst cs_paging_gprs_active.py:35: hangup success\u000a12:06:34.180504 run iperf3-cli_10.42.42.10(pid=21037): ERR: Terminated: ERROR {rc=1} [trial-2067↪gprs:sysmo↪cs_paging_gprs_active.py:38↪cs_paging_gprs_active.py↪iperf3-srv_10.42.42.10↪iperf3-cli_10.42.42.10↪iperf3-cli_10.42.42.10(pid=21037)]\u000a12:06:34.322600 run iperf3-cli_10.42.42.10(pid=21037): stdout: \u000a| \u0009\u0009\u0009\u0009\"bits_per_second\":\u00090,\u000a| \u0009\u0009\u0009\u0009\"retransmits\":\u00090,\u000a| \u0009\u0009\u0009\u0009\"omitted\":\u0009false\u000a| \u0009\u0009\u0009}\u000a| \u0009\u0009}],\u000a| \u0009\"end\":\u0009{\u000a| \u0009},\u000a| \u0009\"error\":\u0009\"error - control socket has closed unexpectedly\"\u000a| }\u000a| child exited with 1 \u000a12:06:34.465134 run iperf3-cli_10.42.42.10(pid=21037): stderr: \u000a| (launched: 2022-09-08_12:05:24.778295) \u000a12:06:35.648989 run iperf3-cli_10.42.42.10(pid=21040): ERR: Terminated: ERROR {rc=1} [trial-2067↪gprs:sysmo↪cs_paging_gprs_active.py:38↪cs_paging_gprs_active.py↪iperf3-srv_10.42.42.10↪iperf3-cli_10.42.42.10↪iperf3-cli_10.42.42.10(pid=21040)]\u000a12:06:35.778635 run iperf3-cli_10.42.42.10(pid=21040): stdout: \u000a| \u0009\u0009\u0009\u0009\"bits_per_second\":\u00090,\u000a| \u0009\u0009\u0009\u0009\"retransmits\":\u00090,\u000a| \u0009\u0009\u0009\u0009\"omitted\":\u0009false\u000a| \u0009\u0009\u0009}\u000a| \u0009\u0009}],\u000a| \u0009\"end\":\u0009{\u000a| \u0009},\u000a| \u0009\"error\":\u0009\"error - control socket has closed unexpectedly\"\u000a| }\u000a| child exited with 1 \u000a12:06:35.906512 run iperf3-cli_10.42.42.10(pid=21040): stderr: \u000a| (launched: 2022-09-08_12:05:25.061346) \u000a12:06:36.023652 run iperf3-srv_10.42.42.10(pid=20908): Terminating (SIGINT)\u000a12:06:36.353816 run iperf3-srv_10.42.42.10(pid=20908): Terminated {rc=1}\u000a12:06:36.540271 tst cs_paging_gprs_active.py:38: Results for 10.42.42.10:5003 through \"'/gobi_6/context2'\"\u000a12:06:36.590985 tst cs_paging_gprs_active.py:38: ERR: KeyError: 'sum_sent' [trial-2067↪gprs:sysmo↪cs_paging_gprs_active.py:38]\u000a12:06:36.597738 tst cs_paging_gprs_active.py:38: Test FAILED (224.6 sec)\u000a12:06:36.634347 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=20907): Terminating (SIGTERM)\u000a12:06:36.675359 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=20909): Terminating (SIGTERM)\u000a12:06:36.716197 run iperf3-srv_10.42.42.10(pid=20910): Terminating (SIGTERM)\u000a12:06:36.757098 run pcap-recorder_any(filters='host 10.42.42.2')(pid=20913): Terminating (SIGTERM)\u000a12:06:36.798560 run osmo-hlr_10.42.42.2(pid=20914): Terminating (SIGTERM)\u000a12:06:36.841623 run pcap-recorder_any(filters='host 10.42.42.5 and port not 22')(pid=20916): Terminating (SIGTERM)\u000a12:06:36.886541 run osmo-stp_10.42.42.5(pid=20917): Terminating (SIGTERM)\u000a12:06:36.940624 run pcap-recorder_any(filters='host 10.42.42.6')(pid=20918): Terminating (SIGTERM)\u000a12:06:36.982015 run osmo-ggsn_10.42.42.6(pid=20923): Terminating (SIGTERM)\u000a12:06:37.030145 run pcap-recorder_any(filters='host 10.42.42.7')(pid=20950): Terminating (SIGTERM)\u000a12:06:37.076043 run osmo-sgsn_10.42.42.7(pid=20951): Terminating (SIGTERM)\u000a12:06:37.119927 run pcap-recorder_any(filters='host 10.42.42.8 and port not 22')(pid=20952): Terminating (SIGTERM)\u000a12:06:37.165229 run osmo-msc_10.42.42.8(pid=20953): Terminating (SIGTERM)\u000a12:06:37.209134 run pcap-recorder_any(filters='host 10.42.42.3 and port not 22')(pid=20954): Terminating (SIGTERM)\u000a12:06:37.257869 run osmo-mgw_10.42.42.3(pid=20955): Terminating (SIGTERM)\u000a12:06:37.318790 run pcap-recorder_any(filters='host 10.42.42.4 and port not 22')(pid=20956): Terminating (SIGTERM)\u000a12:06:37.360197 run osmo-mgw_10.42.42.4(pid=20957): Terminating (SIGTERM)\u000a12:06:37.401599 run pcap-recorder_any(filters='host 10.42.42.9 and port not 22')(pid=20958): Terminating (SIGTERM)\u000a12:06:37.442409 run osmo-bsc_10.42.42.9(pid=20959): Terminating (SIGTERM)\u000a12:06:37.484227 run osmo-bts-sysmo(pid=20968): Terminating (SIGTERM)\u000a12:06:37.528370 run osmo-pcu(pid=20976): Terminating (SIGTERM)\u000a12:06:37.572152 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21031): Terminating (SIGTERM)\u000a12:06:37.720975 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21031)-killSignals.SIGTERM(pid=21060): Launched\u000a12:06:38.820160 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21031)-killSignals.SIGTERM(pid=21060): Terminated: ok {rc=0}\u000a12:06:38.872144 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21034): Terminating (SIGTERM)\u000a12:06:39.051755 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21034)-killSignals.SIGTERM(pid=21065): Launched\u000a12:06:40.146970 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21034)-killSignals.SIGTERM(pid=21065): Terminated: ok {rc=0}\u000a12:06:40.160888 --- ParallelTerminationStrategy: PID 20907 died...\u000a12:06:40.216105 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=20907): Terminated: ok {rc=0}\u000a12:06:40.225785 --- ParallelTerminationStrategy: PID 20909 died...\u000a12:06:40.280935 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=20909): Terminated: ok {rc=0}\u000a12:06:40.290677 --- ParallelTerminationStrategy: PID 20910 died...\u000a12:06:40.344363 run iperf3-srv_10.42.42.10(pid=20910): Terminated {rc=256}\u000a12:06:40.354123 --- ParallelTerminationStrategy: PID 20913 died...\u000a12:06:40.408136 run pcap-recorder_any(filters='host 10.42.42.2')(pid=20913): Terminated: ok {rc=0}\u000a12:06:40.417809 --- ParallelTerminationStrategy: PID 20914 died...\u000a12:06:40.471683 run osmo-hlr_10.42.42.2(pid=20914): Terminated: ok {rc=0}\u000a12:06:40.481560 --- ParallelTerminationStrategy: PID 20916 died...\u000a12:06:40.535522 run pcap-recorder_any(filters='host 10.42.42.5 and port not 22')(pid=20916): Terminated: ok {rc=0}\u000a12:06:40.545390 --- ParallelTerminationStrategy: PID 20917 died...\u000a12:06:40.599515 run osmo-stp_10.42.42.5(pid=20917): Terminated: ok {rc=0}\u000a12:06:40.609324 --- ParallelTerminationStrategy: PID 20918 died...\u000a12:06:40.664296 run pcap-recorder_any(filters='host 10.42.42.6')(pid=20918): Terminated: ok {rc=0}\u000a12:06:40.674553 --- ParallelTerminationStrategy: PID 20923 died...\u000a12:06:40.728749 run osmo-ggsn_10.42.42.6(pid=20923): Terminated: ok {rc=0}\u000a12:06:40.739196 --- ParallelTerminationStrategy: PID 20950 died...\u000a12:06:40.793370 run pcap-recorder_any(filters='host 10.42.42.7')(pid=20950): Terminated: ok {rc=0}\u000a12:06:40.803676 --- ParallelTerminationStrategy: PID 20951 died...\u000a12:06:40.857586 run osmo-sgsn_10.42.42.7(pid=20951): Terminated: ok {rc=0}\u000a12:06:40.867557 --- ParallelTerminationStrategy: PID 20952 died...\u000a12:06:40.921475 run pcap-recorder_any(filters='host 10.42.42.8 and port not 22')(pid=20952): Terminated: ok {rc=0}\u000a12:06:40.931543 --- ParallelTerminationStrategy: PID 20953 died...\u000a12:06:40.985633 run osmo-msc_10.42.42.8(pid=20953): Terminated: ok {rc=0}\u000a12:06:40.996020 --- ParallelTerminationStrategy: PID 20954 died...\u000a12:06:41.050348 run pcap-recorder_any(filters='host 10.42.42.3 and port not 22')(pid=20954): Terminated: ok {rc=0}\u000a12:06:41.060175 --- ParallelTerminationStrategy: PID 20955 died...\u000a12:06:41.114161 run osmo-mgw_10.42.42.3(pid=20955): Terminated {rc=15}\u000a12:06:41.124010 --- ParallelTerminationStrategy: PID 20956 died...\u000a12:06:41.177726 run pcap-recorder_any(filters='host 10.42.42.4 and port not 22')(pid=20956): Terminated: ok {rc=0}\u000a12:06:41.187679 --- ParallelTerminationStrategy: PID 20957 died...\u000a12:06:41.241334 run osmo-mgw_10.42.42.4(pid=20957): Terminated {rc=15}\u000a12:06:41.251390 --- ParallelTerminationStrategy: PID 20958 died...\u000a12:06:41.305961 run pcap-recorder_any(filters='host 10.42.42.9 and port not 22')(pid=20958): Terminated: ok {rc=0}\u000a12:06:41.315799 --- ParallelTerminationStrategy: PID 20959 died...\u000a12:06:41.369862 run osmo-bsc_10.42.42.9(pid=20959): Terminated: ok {rc=0}\u000a12:06:41.379695 --- ParallelTerminationStrategy: PID 20968 died...\u000a12:06:41.433519 run osmo-bts-sysmo(pid=20968): Terminated: ok {rc=0}\u000a12:06:41.443574 --- ParallelTerminationStrategy: PID 20976 died...\u000a12:06:41.497596 run osmo-pcu(pid=20976): Terminated: ok {rc=0}\u000a12:06:47.363897 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21031): Terminating (SIGINT)\u000a12:06:47.510610 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21031)-killSignals.SIGINT(pid=21068): Launched\u000a12:06:48.600612 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21031)-killSignals.SIGINT(pid=21068): Terminated: ok {rc=0}\u000a12:06:48.645817 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21034): Terminating (SIGINT)\u000a12:06:48.794093 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21034)-killSignals.SIGINT(pid=21071): Launched\u000a12:06:49.883966 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21034)-killSignals.SIGINT(pid=21071): Terminated: ok {rc=0}\u000a12:06:55.754398 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21031): Terminating (SIGKILL)\u000a12:06:55.905656 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21031)-killSignals.SIGUSR1(pid=21074): Launched\u000a12:06:56.995263 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21031)-killSignals.SIGUSR1(pid=21074): Terminated: ok {rc=0}\u000a12:06:57.040540 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21034): Terminating (SIGKILL)\u000a12:06:57.191564 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21034)-killSignals.SIGUSR1(pid=21077): Launched\u000a12:06:58.286991 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21034)-killSignals.SIGUSR1(pid=21077): Terminated: ok {rc=0}\u000a12:06:58.564248 bus /gobi_0: Setting Online False\u000a12:07:01.347829 bus /gobi_0: Setting Powered False\u000a12:07:16.027591 bus /gobi_0: Setting Powered False\u000a12:07:17.134000 bus /gobi_0: Setting Powered False\u000a12:07:18.287179 bus /gobi_6: Setting Online False\u000a12:07:19.427941 bus /gobi_6: Setting Powered False\u000a12:07:27.974599 bus /gobi_6: Setting Powered False\u000a" }, { "testActions" : [ ], "age" : 5, "className" : "gprs:sysmo", "duration" : 202.0, "errorDetails" : None, "errorStackTrace" : "'sum_sent'", "failedSince" : 2063, "name" : "iperf3.py", "properties" : { }, "skipped" : False, "skippedMessage" : None, "status" : "FAILED", "stderr" : "Traceback (most recent call last):\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/core/test.py\", line 76, in run\u000a self.path)\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/core/util.py\", line 414, in run_python_file\u000a spec.loader.exec_module( importlib.util.module_from_spec(spec) )\u000a File \"\", line 728, in exec_module\u000a File \"\", line 219, in _call_with_frames_removed\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/sysmocom/suites/gprs/iperf3.py\", line 8, in \u000a setup_run_iperf3_test_parallel(1)\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/sysmocom/suites/gprs/lib/testlib.py\", line 105, in setup_run_iperf3_test_parallel\u000a print_results(clients[i], servers[i])\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/sysmocom/suites/gprs/lib/testlib.py\", line 5, in print_results\u000a cli.print_results()\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/obj/iperf3.py\", line 342, in print_results\u000a print_result_node_tcp(self.get_results(), 'client')\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/obj/iperf3.py\", line 72, in print_result_node_tcp\u000a raise e\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/obj/iperf3.py\", line 65, in print_result_node_tcp\u000a sent = result['end']['sum_sent']\u000aKeyError: 'sum_sent'\u000a", "stdout" : "\u000a----------------------------------------------\u000atrial-2067 gprs:sysmo iperf3.py\u000a----------------------------------------------\u000a12:07:29.145703 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:07:29.157557 tst gprs:sysmo: Using 1 x bts (candidates: 1)\u000a12:07:29.296637 tst iperf3.py: using LAC 3967\u000a12:07:29.434199 tst iperf3.py: using RAC 142\u000a12:07:29.561792 tst iperf3.py: using CellId 3967\u000a12:07:29.687413 tst iperf3.py: using BVCI 3968\u000a12:07:29.712348 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:07:29.723596 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:07:29.734593 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:07:29.745843 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:07:29.756939 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:07:29.768117 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:07:29.779890 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:07:29.791146 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:07:29.802411 tst gprs:sysmo: Using 1 x modem (candidates: 4)\u000a12:07:30.110289 tst iperf3.py:8: start iperfv3 server 10.42.42.10:5003...\u000a12:07:30.247539 run iperf3-srv_10.42.42.10: Starting iperf3-srv\u000a12:07:30.587851 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21080): Launched\u000a12:07:30.870677 run iperf3-srv_10.42.42.10(pid=21081): Launched\u000a12:07:31.069887 tst iperf3.py:8: start network...\u000a12:07:31.206246 run osmo-hlr_10.42.42.2: Starting osmo-hlr\u000a12:07:31.722526 run create_hlr_db(pid=21082): Launched\u000a12:07:31.911420 bus /gobi_6: Setting Powered False\u000a12:07:32.932936 run create_hlr_db(pid=21082): Terminated: ok {rc=0}\u000a12:07:33.344987 run pcap-recorder_any(filters='host 10.42.42.2')(pid=21084): Launched\u000a12:07:33.658625 run osmo-hlr_10.42.42.2(pid=21085): Launched\u000a12:07:33.769519 run osmo-stp_10.42.42.5: Starting osmo-stp\u000a12:07:34.301497 run pcap-recorder_any(filters='host 10.42.42.5 and port not 22')(pid=21086): Launched\u000a12:07:34.609515 run osmo-stp_10.42.42.5(pid=21087): Launched\u000a12:07:34.718457 run osmo-ggsn_10.42.42.6: Starting osmo-ggsn\u000a12:07:35.251825 run pcap-recorder_any(filters='host 10.42.42.6')(pid=21088): Launched\u000a12:07:35.359842 run osmo-ggsn_10.42.42.6: Setting RPATH for osmo-ggsn\u000a12:07:35.628423 run patchelf(pid=21089): Launched\u000a12:07:36.810507 run patchelf(pid=21089): Terminated: ok {rc=0}\u000a12:07:36.922418 run osmo-ggsn_10.42.42.6: Applying CAP_NET_ADMIN capability to osmo-ggsn\u000a12:07:37.200812 run osmo-gsm-tester_setcap_net_admin.sh(pid=21090): Launched\u000a12:07:38.381919 run osmo-gsm-tester_setcap_net_admin.sh(pid=21090): Terminated: ok {rc=0}\u000a12:07:38.691826 run osmo-ggsn_10.42.42.6(pid=21093): Launched\u000a12:07:38.881853 run osmo-sgsn_10.42.42.7: Starting osmo-sgsn\u000a12:07:39.517255 run pcap-recorder_any(filters='host 10.42.42.7')(pid=21112): Launched\u000a12:07:39.848863 run osmo-sgsn_10.42.42.7(pid=21113): Launched\u000a12:07:39.968604 run osmo-msc_10.42.42.8: Starting osmo-msc\u000a12:07:40.569195 run pcap-recorder_any(filters='host 10.42.42.8 and port not 22')(pid=21114): Launched\u000a12:07:40.900610 run osmo-msc_10.42.42.8(pid=21115): Launched\u000a12:07:44.074331 run osmo-mgw_10.42.42.3: Starting osmo-mgw\u000a12:07:44.644470 run pcap-recorder_any(filters='host 10.42.42.3 and port not 22')(pid=21116): Launched\u000a12:07:44.974852 run osmo-mgw_10.42.42.3(pid=21117): Launched\u000a12:07:45.092476 run osmo-mgw_10.42.42.4: Starting osmo-mgw\u000a12:07:45.652529 run pcap-recorder_any(filters='host 10.42.42.4 and port not 22')(pid=21118): Launched\u000a12:07:45.983248 run osmo-mgw_10.42.42.4(pid=21119): Launched\u000a12:07:46.100402 run osmo-bsc_10.42.42.9: Starting osmo-bsc\u000a12:07:46.786232 run pcap-recorder_any(filters='host 10.42.42.9 and port not 22')(pid=21120): Launched\u000a12:07:47.117596 run osmo-bsc_10.42.42.9(pid=21121): Launched\u000a12:07:53.394793 run osmo-bts-sysmo: Starting sysmoBTS to connect to osmo-bsc_10.42.42.9\u000a12:07:53.976672 run rm-remote-dir(pid=21122): Launched\u000a12:07:56.177730 run rm-remote-dir(pid=21122): Terminated: ok {rc=0}\u000a12:07:56.528012 run mk-remote-dir(pid=21123): Launched\u000a12:07:57.727297 run mk-remote-dir(pid=21123): Terminated: ok {rc=0}\u000a12:07:58.023013 run scp-inst-to-remote(pid=21124): Launched\u000a12:08:10.239817 run scp-inst-to-remote(pid=21124): Terminated: ok {rc=0}\u000a12:08:10.585408 run mk-remote-dir(pid=21126): Launched\u000a12:08:11.782785 run mk-remote-dir(pid=21126): Terminated: ok {rc=0}\u000a12:08:12.077989 run scp-cfg-to-remote(pid=21127): Launched\u000a12:08:13.275035 run scp-cfg-to-remote(pid=21127): Terminated: ok {rc=0}\u000a12:08:13.613996 run reload-dsp-firmware(pid=21129): Launched\u000a12:08:15.808861 run reload-dsp-firmware(pid=21129): Terminated: ok {rc=0}\u000a12:08:16.140713 run osmo-bts-sysmo(pid=21130): Launched\u000a12:08:20.453197 tst iperf3.py:8: Waiting for bts to be ready...\u000a12:08:22.059958 run rm-remote-dir(pid=21131): Launched\u000a12:08:24.250338 run rm-remote-dir(pid=21131): Terminated: ok {rc=0}\u000a12:08:24.586611 run mk-remote-dir(pid=21132): Launched\u000a12:08:26.781268 run mk-remote-dir(pid=21132): Terminated: ok {rc=0}\u000a12:08:27.068976 run scp-inst-to-bts(pid=21133): Launched\u000a12:08:40.275783 run scp-inst-to-bts(pid=21133): Terminated: ok {rc=0}\u000a12:08:40.610679 run mk-remote-run-dir(pid=21135): Launched\u000a12:08:42.804368 run mk-remote-run-dir(pid=21135): Terminated: ok {rc=0}\u000a12:08:43.091956 run scp-cfg-to-sysmobts(pid=21136): Launched\u000a12:08:45.285300 run scp-cfg-to-sysmobts(pid=21136): Terminated: ok {rc=0}\u000a12:08:45.620744 run osmo-pcu(pid=21138): Launched\u000a12:08:45.750911 tst iperf3.py: using MSISDN 3984\u000a12:08:45.922960 bus /gobi_6: Setting Powered True\u000a12:08:49.429825 run osmo-hlr_10.42.42.2: Add subscriber {algo=1, algo_str='comp128v1', imsi='901700000010978', msisdn='3984', subscriber_id=1}\u000a12:08:49.717794 bus /gobi_6: Setting Online False\u000a12:08:50.913511 bus /gobi_6: Setting Powered False\u000a12:08:58.203273 bus /gobi_6: Setting Powered True\u000a12:08:59.594063 bus /gobi_6: Setting Online True\u000a12:09:02.073782 tst /gobi_6: Connect to ('901', '70')\u000a12:09:02.340491 tst /gobi_6: Manufacturer: 'Quectel', Model: 'EC20', Revision: 'EC20EQAR02A13E2G 1 [Jul 5 2017 10:49:17]', Serial: '868323024273648'\u000a12:09:02.526347 tst iperf3.py:8: waiting for modems to attach...\u000a12:09:09.560292 tst /gobi_6: Already registered with network ('901', '70')\u000a12:09:19.999606 tst iperf3.py:8: waiting for modems to attach to data services...\u000a12:09:29.878579 tst /gobi_6: context activated {apn='inet46', path='/gobi_6/context3', properties={AccessPointName='inet46', Active=True, AuthenticationMethod='chap', IPv6.Settings={}, Name='Internet', Password='', Protocol='ip', Settings={Address='176.16.46.1', DomainNameServers=['192.168.100.1', '8.8.8.8'], Gateway='176.16.46.2', Interface='r433306bbc2604b', Method='static', Netmask='255.255.255.252'}, Type='internet', Username='ogt'}, user='ogt'}\u000a12:09:30.078172 tst iperf3.py:8: Setting up data plan for \"'/gobi_6/context3'\"\u000a12:09:30.449116 run move_netns(pid=21141): Launched\u000a12:09:31.634903 run move_netns(pid=21141): Terminated: ok {rc=0}\u000a12:09:31.963116 run ifup(pid=21144): Launched\u000a12:09:33.144766 run ifup(pid=21144): Terminated: ok {rc=0}\u000a12:09:33.469109 run dhcp(pid=21147): Launched\u000a12:09:34.650991 run dhcp(pid=21147): Terminated: ok {rc=0}\u000a12:09:34.845412 tst iperf3.py:8: Running iperf3 client to 10.42.42.10:5003 through \"'/gobi_6/context3'\"\u000a12:09:34.976010 run iperf3-cli_10.42.42.10: Preparing iperf3-client connecting to 10.42.42.10:5003 (proto=tcp,time=60s)\u000a12:09:35.362272 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21163): Launched\u000a12:09:35.701174 run iperf3-cli_10.42.42.10(pid=21166): Launched\u000a12:10:50.028284 run iperf3-cli_10.42.42.10(pid=21166): ERR: Terminated: ERROR {rc=1} [trial-2067↪gprs:sysmo↪iperf3.py:8↪iperf3.py↪iperf3-srv_10.42.42.10↪iperf3-cli_10.42.42.10↪iperf3-cli_10.42.42.10(pid=21166)]\u000a12:10:50.159238 run iperf3-cli_10.42.42.10(pid=21166): stdout: \u000a| \u0009\u0009\u0009\u0009\"bits_per_second\":\u00090,\u000a| \u0009\u0009\u0009\u0009\"retransmits\":\u00090,\u000a| \u0009\u0009\u0009\u0009\"omitted\":\u0009false\u000a| \u0009\u0009\u0009}\u000a| \u0009\u0009}],\u000a| \u0009\"end\":\u0009{\u000a| \u0009},\u000a| \u0009\"error\":\u0009\"error - control socket has closed unexpectedly\"\u000a| }\u000a| child exited with 1 \u000a12:10:50.291191 run iperf3-cli_10.42.42.10(pid=21166): stderr: \u000a| (launched: 2022-09-08_12:09:35.580850) \u000a12:10:50.410857 run iperf3-srv_10.42.42.10(pid=21081): Terminating (SIGINT)\u000a12:10:50.746264 run iperf3-srv_10.42.42.10(pid=21081): Terminated {rc=1}\u000a12:10:50.939216 tst iperf3.py:8: Results for 10.42.42.10:5003 through \"'/gobi_6/context3'\"\u000a12:10:50.997637 tst iperf3.py:8: ERR: KeyError: 'sum_sent' [trial-2067↪gprs:sysmo↪iperf3.py:8]\u000a12:10:51.004954 tst iperf3.py:8: Test FAILED (201.9 sec)\u000a12:10:51.041340 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21080): Terminating (SIGTERM)\u000a12:10:51.083698 run pcap-recorder_any(filters='host 10.42.42.2')(pid=21084): Terminating (SIGTERM)\u000a12:10:51.125351 run osmo-hlr_10.42.42.2(pid=21085): Terminating (SIGTERM)\u000a12:10:51.168488 run pcap-recorder_any(filters='host 10.42.42.5 and port not 22')(pid=21086): Terminating (SIGTERM)\u000a12:10:51.211952 run osmo-stp_10.42.42.5(pid=21087): Terminating (SIGTERM)\u000a12:10:51.255032 run pcap-recorder_any(filters='host 10.42.42.6')(pid=21088): Terminating (SIGTERM)\u000a12:10:51.297651 run osmo-ggsn_10.42.42.6(pid=21093): Terminating (SIGTERM)\u000a12:10:51.345774 run pcap-recorder_any(filters='host 10.42.42.7')(pid=21112): Terminating (SIGTERM)\u000a12:10:51.400532 run osmo-sgsn_10.42.42.7(pid=21113): Terminating (SIGTERM)\u000a12:10:51.444372 run pcap-recorder_any(filters='host 10.42.42.8 and port not 22')(pid=21114): Terminating (SIGTERM)\u000a12:10:51.501156 run osmo-msc_10.42.42.8(pid=21115): Terminating (SIGTERM)\u000a12:10:51.545532 run pcap-recorder_any(filters='host 10.42.42.3 and port not 22')(pid=21116): Terminating (SIGTERM)\u000a12:10:51.591116 run osmo-mgw_10.42.42.3(pid=21117): Terminating (SIGTERM)\u000a12:10:51.633825 run pcap-recorder_any(filters='host 10.42.42.4 and port not 22')(pid=21118): Terminating (SIGTERM)\u000a12:10:51.674793 run osmo-mgw_10.42.42.4(pid=21119): Terminating (SIGTERM)\u000a12:10:51.715960 run pcap-recorder_any(filters='host 10.42.42.9 and port not 22')(pid=21120): Terminating (SIGTERM)\u000a12:10:51.756670 run osmo-bsc_10.42.42.9(pid=21121): Terminating (SIGTERM)\u000a12:10:51.797947 run osmo-bts-sysmo(pid=21130): Terminating (SIGTERM)\u000a12:10:51.838788 run osmo-pcu(pid=21138): Terminating (SIGTERM)\u000a12:10:51.880025 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21163): Terminating (SIGTERM)\u000a12:10:52.026290 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21163)-killSignals.SIGTERM(pid=21189): Launched\u000a12:10:53.117144 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21163)-killSignals.SIGTERM(pid=21189): Terminated: ok {rc=0}\u000a12:10:53.131054 --- ParallelTerminationStrategy: PID 21080 died...\u000a12:10:53.185326 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21080): Terminated: ok {rc=0}\u000a12:10:53.195214 --- ParallelTerminationStrategy: PID 21084 died...\u000a12:10:53.249133 run pcap-recorder_any(filters='host 10.42.42.2')(pid=21084): Terminated: ok {rc=0}\u000a12:10:53.259072 --- ParallelTerminationStrategy: PID 21085 died...\u000a12:10:53.313380 run osmo-hlr_10.42.42.2(pid=21085): Terminated: ok {rc=0}\u000a12:10:53.323566 --- ParallelTerminationStrategy: PID 21086 died...\u000a12:10:53.377620 run pcap-recorder_any(filters='host 10.42.42.5 and port not 22')(pid=21086): Terminated: ok {rc=0}\u000a12:10:53.387527 --- ParallelTerminationStrategy: PID 21087 died...\u000a12:10:53.441045 run osmo-stp_10.42.42.5(pid=21087): Terminated: ok {rc=0}\u000a12:10:53.450778 --- ParallelTerminationStrategy: PID 21088 died...\u000a12:10:53.504478 run pcap-recorder_any(filters='host 10.42.42.6')(pid=21088): Terminated: ok {rc=0}\u000a12:10:53.514221 --- ParallelTerminationStrategy: PID 21093 died...\u000a12:10:53.568129 run osmo-ggsn_10.42.42.6(pid=21093): Terminated: ok {rc=0}\u000a12:10:53.577920 --- ParallelTerminationStrategy: PID 21112 died...\u000a12:10:53.632376 run pcap-recorder_any(filters='host 10.42.42.7')(pid=21112): Terminated: ok {rc=0}\u000a12:10:53.642114 --- ParallelTerminationStrategy: PID 21113 died...\u000a12:10:53.695818 run osmo-sgsn_10.42.42.7(pid=21113): Terminated: ok {rc=0}\u000a12:10:53.705617 --- ParallelTerminationStrategy: PID 21114 died...\u000a12:10:53.759701 run pcap-recorder_any(filters='host 10.42.42.8 and port not 22')(pid=21114): Terminated: ok {rc=0}\u000a12:10:53.769491 --- ParallelTerminationStrategy: PID 21115 died...\u000a12:10:53.823173 run osmo-msc_10.42.42.8(pid=21115): Terminated: ok {rc=0}\u000a12:10:53.832992 --- ParallelTerminationStrategy: PID 21116 died...\u000a12:10:53.887078 run pcap-recorder_any(filters='host 10.42.42.3 and port not 22')(pid=21116): Terminated: ok {rc=0}\u000a12:10:53.896926 --- ParallelTerminationStrategy: PID 21117 died...\u000a12:10:53.951468 run osmo-mgw_10.42.42.3(pid=21117): Terminated {rc=15}\u000a12:10:53.961232 --- ParallelTerminationStrategy: PID 21118 died...\u000a12:10:54.015091 run pcap-recorder_any(filters='host 10.42.42.4 and port not 22')(pid=21118): Terminated: ok {rc=0}\u000a12:10:54.025043 --- ParallelTerminationStrategy: PID 21119 died...\u000a12:10:54.078882 run osmo-mgw_10.42.42.4(pid=21119): Terminated {rc=15}\u000a12:10:54.088672 --- ParallelTerminationStrategy: PID 21120 died...\u000a12:10:54.142528 run pcap-recorder_any(filters='host 10.42.42.9 and port not 22')(pid=21120): Terminated: ok {rc=0}\u000a12:10:54.152391 --- ParallelTerminationStrategy: PID 21121 died...\u000a12:10:54.206450 run osmo-bsc_10.42.42.9(pid=21121): Terminated: ok {rc=0}\u000a12:10:54.216464 --- ParallelTerminationStrategy: PID 21130 died...\u000a12:10:54.270792 run osmo-bts-sysmo(pid=21130): Terminated: ok {rc=0}\u000a12:10:54.280562 --- ParallelTerminationStrategy: PID 21138 died...\u000a12:10:54.334830 run osmo-pcu(pid=21138): Terminated: ok {rc=0}\u000a12:11:00.201338 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21163): Terminating (SIGINT)\u000a12:11:00.353349 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21163)-killSignals.SIGINT(pid=21195): Launched\u000a12:11:01.443752 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21163)-killSignals.SIGINT(pid=21195): Terminated: ok {rc=0}\u000a12:11:07.313241 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21163): Terminating (SIGKILL)\u000a12:11:07.466405 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21163)-killSignals.SIGUSR1(pid=21199): Launched\u000a12:11:08.561439 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21163)-killSignals.SIGUSR1(pid=21199): Terminated: ok {rc=0}\u000a12:11:08.801385 bus /gobi_6: Setting Online False\u000a12:11:11.824484 bus /gobi_6: Setting Powered False\u000a12:11:27.221491 bus /gobi_6: Setting Powered False\u000a" }, { "testActions" : [ ], "age" : 5, "className" : "gprs:sysmo", "duration" : 297.0, "errorDetails" : None, "errorStackTrace" : "'sum_sent'", "failedSince" : 2063, "name" : "iperf3m4.py", "properties" : { }, "skipped" : False, "skippedMessage" : None, "status" : "FAILED", "stderr" : "Traceback (most recent call last):\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/core/test.py\", line 76, in run\u000a self.path)\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/core/util.py\", line 414, in run_python_file\u000a spec.loader.exec_module( importlib.util.module_from_spec(spec) )\u000a File \"\", line 728, in exec_module\u000a File \"\", line 219, in _call_with_frames_removed\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/sysmocom/suites/gprs/iperf3m4.py\", line 8, in \u000a setup_run_iperf3_test_parallel(4)\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/sysmocom/suites/gprs/lib/testlib.py\", line 105, in setup_run_iperf3_test_parallel\u000a print_results(clients[i], servers[i])\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/sysmocom/suites/gprs/lib/testlib.py\", line 5, in print_results\u000a cli.print_results()\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/obj/iperf3.py\", line 342, in print_results\u000a print_result_node_tcp(self.get_results(), 'client')\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/obj/iperf3.py\", line 72, in print_result_node_tcp\u000a raise e\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/obj/iperf3.py\", line 65, in print_result_node_tcp\u000a sent = result['end']['sum_sent']\u000aKeyError: 'sum_sent'\u000a", "stdout" : "\u000a----------------------------------------------\u000atrial-2067 gprs:sysmo iperf3m4.py\u000a----------------------------------------------\u000a12:11:28.384621 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:11:28.396039 tst gprs:sysmo: Using 1 x bts (candidates: 1)\u000a12:11:28.519635 tst iperf3m4.py: using LAC 3968\u000a12:11:28.638268 tst iperf3m4.py: using RAC 143\u000a12:11:28.757337 tst iperf3m4.py: using CellId 3968\u000a12:11:28.877363 tst iperf3m4.py: using BVCI 3969\u000a12:11:28.901006 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:11:28.911734 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:11:28.922233 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:11:28.932780 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:11:28.943428 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:11:28.953824 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:11:28.964558 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:11:28.975079 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:11:28.985667 tst gprs:sysmo: Using 1 x modem (candidates: 4)\u000a12:11:29.104475 tst gprs:sysmo: Using 1 x modem (candidates: 4)\u000a12:11:29.225655 tst gprs:sysmo: Using 1 x modem (candidates: 4)\u000a12:11:29.343283 tst gprs:sysmo: Using 1 x modem (candidates: 4)\u000a12:11:29.636006 tst iperf3m4.py:8: start iperfv3 server 10.42.42.10:5003...\u000a12:11:29.772812 run iperf3-srv_10.42.42.10: Starting iperf3-srv\u000a12:11:30.112555 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21202): Launched\u000a12:11:30.395136 run iperf3-srv_10.42.42.10(pid=21203): Launched\u000a12:11:30.594876 tst iperf3m4.py:8: start iperfv3 server 10.42.42.10:5004...\u000a12:11:30.731313 run iperf3-srv_10.42.42.10: Starting iperf3-srv\u000a12:11:31.070373 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21204): Launched\u000a12:11:31.353300 run iperf3-srv_10.42.42.10(pid=21205): Launched\u000a12:11:31.551804 tst iperf3m4.py:8: start iperfv3 server 10.42.42.10:5005...\u000a12:11:31.687552 run iperf3-srv_10.42.42.10: Starting iperf3-srv\u000a12:11:32.026211 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21206): Launched\u000a12:11:32.308308 run iperf3-srv_10.42.42.10(pid=21207): Launched\u000a12:11:32.507044 tst iperf3m4.py:8: start iperfv3 server 10.42.42.10:5006...\u000a12:11:32.642921 run iperf3-srv_10.42.42.10: Starting iperf3-srv\u000a12:11:32.981362 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21208): Launched\u000a12:11:33.262119 run iperf3-srv_10.42.42.10(pid=21209): Launched\u000a12:11:33.460123 tst iperf3m4.py:8: start network...\u000a12:11:33.595416 run osmo-hlr_10.42.42.2: Starting osmo-hlr\u000a12:11:34.119829 run create_hlr_db(pid=21210): Launched\u000a12:11:34.307885 bus /gobi_6: Setting Powered False\u000a12:11:35.330104 run create_hlr_db(pid=21210): Terminated: ok {rc=0}\u000a12:11:35.741645 run pcap-recorder_any(filters='host 10.42.42.2')(pid=21212): Launched\u000a12:11:36.050013 run osmo-hlr_10.42.42.2(pid=21213): Launched\u000a12:11:36.159546 run osmo-stp_10.42.42.5: Starting osmo-stp\u000a12:11:36.692023 run pcap-recorder_any(filters='host 10.42.42.5 and port not 22')(pid=21214): Launched\u000a12:11:37.001427 run osmo-stp_10.42.42.5(pid=21215): Launched\u000a12:11:37.109491 run osmo-ggsn_10.42.42.6: Starting osmo-ggsn\u000a12:11:37.642130 run pcap-recorder_any(filters='host 10.42.42.6')(pid=21216): Launched\u000a12:11:37.750184 run osmo-ggsn_10.42.42.6: Setting RPATH for osmo-ggsn\u000a12:11:38.016156 run patchelf(pid=21217): Launched\u000a12:11:39.198578 run patchelf(pid=21217): Terminated: ok {rc=0}\u000a12:11:39.310350 run osmo-ggsn_10.42.42.6: Applying CAP_NET_ADMIN capability to osmo-ggsn\u000a12:11:39.587049 run osmo-gsm-tester_setcap_net_admin.sh(pid=21218): Launched\u000a12:11:40.768594 run osmo-gsm-tester_setcap_net_admin.sh(pid=21218): Terminated: ok {rc=0}\u000a12:11:41.079368 run osmo-ggsn_10.42.42.6(pid=21221): Launched\u000a12:11:41.312334 run osmo-sgsn_10.42.42.7: Starting osmo-sgsn\u000a12:11:41.873770 run pcap-recorder_any(filters='host 10.42.42.7')(pid=21238): Launched\u000a12:11:42.180746 run osmo-sgsn_10.42.42.7(pid=21239): Launched\u000a12:11:42.293902 run osmo-msc_10.42.42.8: Starting osmo-msc\u000a12:11:42.856918 run pcap-recorder_any(filters='host 10.42.42.8 and port not 22')(pid=21240): Launched\u000a12:11:43.166389 run osmo-msc_10.42.42.8(pid=21241): Launched\u000a12:11:46.328458 run osmo-mgw_10.42.42.3: Starting osmo-mgw\u000a12:11:46.876278 run pcap-recorder_any(filters='host 10.42.42.3 and port not 22')(pid=21242): Launched\u000a12:11:47.183575 run osmo-mgw_10.42.42.3(pid=21243): Launched\u000a12:11:47.291398 run osmo-mgw_10.42.42.4: Starting osmo-mgw\u000a12:11:47.812179 run pcap-recorder_any(filters='host 10.42.42.4 and port not 22')(pid=21244): Launched\u000a12:11:48.120509 run osmo-mgw_10.42.42.4(pid=21245): Launched\u000a12:11:48.228230 run osmo-bsc_10.42.42.9: Starting osmo-bsc\u000a12:11:48.869239 run pcap-recorder_any(filters='host 10.42.42.9 and port not 22')(pid=21246): Launched\u000a12:11:49.178072 run osmo-bsc_10.42.42.9(pid=21247): Launched\u000a12:11:55.437998 run osmo-bts-sysmo: Starting sysmoBTS to connect to osmo-bsc_10.42.42.9\u000a12:11:55.984198 run rm-remote-dir(pid=21248): Launched\u000a12:11:58.175327 run rm-remote-dir(pid=21248): Terminated: ok {rc=0}\u000a12:11:58.502877 run mk-remote-dir(pid=21249): Launched\u000a12:11:59.690314 run mk-remote-dir(pid=21249): Terminated: ok {rc=0}\u000a12:11:59.968047 run scp-inst-to-remote(pid=21250): Launched\u000a12:12:12.181031 run scp-inst-to-remote(pid=21250): Terminated: ok {rc=0}\u000a12:12:12.527308 run mk-remote-dir(pid=21252): Launched\u000a12:12:13.725142 run mk-remote-dir(pid=21252): Terminated: ok {rc=0}\u000a12:12:14.019130 run scp-cfg-to-remote(pid=21253): Launched\u000a12:12:15.216796 run scp-cfg-to-remote(pid=21253): Terminated: ok {rc=0}\u000a12:12:15.557714 run reload-dsp-firmware(pid=21255): Launched\u000a12:12:17.754583 run reload-dsp-firmware(pid=21255): Terminated: ok {rc=0}\u000a12:12:18.084912 run osmo-bts-sysmo(pid=21256): Launched\u000a12:12:22.399990 tst iperf3m4.py:8: Waiting for bts to be ready...\u000a12:12:24.045193 run rm-remote-dir(pid=21257): Launched\u000a12:12:26.242062 run rm-remote-dir(pid=21257): Terminated: ok {rc=0}\u000a12:12:26.577295 run mk-remote-dir(pid=21258): Launched\u000a12:12:28.769683 run mk-remote-dir(pid=21258): Terminated: ok {rc=0}\u000a12:12:29.058192 run scp-inst-to-bts(pid=21259): Launched\u000a12:12:42.263798 run scp-inst-to-bts(pid=21259): Terminated: ok {rc=0}\u000a12:12:42.579493 run mk-remote-run-dir(pid=21261): Launched\u000a12:12:44.762330 run mk-remote-run-dir(pid=21261): Terminated: ok {rc=0}\u000a12:12:45.034081 run scp-cfg-to-sysmobts(pid=21262): Launched\u000a12:12:47.216206 run scp-cfg-to-sysmobts(pid=21262): Terminated: ok {rc=0}\u000a12:12:47.531375 run osmo-pcu(pid=21264): Launched\u000a12:12:47.659298 tst iperf3m4.py: using MSISDN 3985\u000a12:12:47.831240 bus /gobi_6: Setting Powered True\u000a12:12:51.335436 run osmo-hlr_10.42.42.2: Add subscriber {algo=1, algo_str='comp128v1', imsi='901700000010978', msisdn='3985', subscriber_id=1}\u000a12:12:51.607981 bus /gobi_6: Setting Online False\u000a12:12:52.802368 bus /gobi_6: Setting Powered False\u000a12:13:00.203440 bus /gobi_6: Setting Powered True\u000a12:13:01.565566 bus /gobi_6: Setting Online True\u000a12:13:03.999762 tst /gobi_6: Connect to ('901', '70')\u000a12:13:04.276040 tst /gobi_6: Manufacturer: 'Quectel', Model: 'EC20', Revision: 'EC20EQAR02A13E2G 1 [Jul 5 2017 10:49:17]', Serial: '868323024273648'\u000a12:13:04.408913 tst iperf3m4.py: using MSISDN 3986\u000a12:13:04.679627 bus /gobi_0: Setting Powered True\u000a12:13:08.134264 run osmo-hlr_10.42.42.2: Add subscriber {algo=1, algo_str='comp128v1', imsi='901700000023847', msisdn='3986', subscriber_id=2}\u000a12:13:08.419310 bus /gobi_0: Setting Online False\u000a12:13:09.613159 bus /gobi_0: Setting Powered False\u000a12:13:16.389001 bus /gobi_0: Setting Powered True\u000a12:13:17.764557 bus /gobi_0: Setting Online True\u000a12:13:19.586365 tst /gobi_6: Already registered with network ('901', '70')\u000a12:13:20.320389 tst /gobi_0: Connect to ('901', '70')\u000a12:13:20.602968 tst /gobi_0: Manufacturer: 'Quectel', Model: 'EC20', Revision: 'EC20EQAR02A05E2G 1 [Mar 14 2016 19:33:01]', Serial: '868323024271873'\u000a12:13:20.734532 tst iperf3m4.py: using MSISDN 3987\u000a12:13:20.964104 bus /gobi_1: Setting Powered True\u000a12:13:24.464786 run osmo-hlr_10.42.42.2: Add subscriber {algo=1, algo_str='comp128v1', imsi='901700000023848', msisdn='3987', subscriber_id=3}\u000a12:13:24.738010 bus /gobi_1: Setting Online False\u000a12:13:25.923743 bus /gobi_1: Setting Powered False\u000a12:13:33.224246 bus /gobi_1: Setting Powered True\u000a12:13:34.597021 bus /gobi_1: Setting Online True\u000a12:13:35.662311 tst /gobi_0: Already registered with network ('901', '70')\u000a12:13:37.277102 tst /gobi_1: Connect to ('901', '70')\u000a12:13:37.544259 tst /gobi_1: Manufacturer: 'Quectel', Model: 'EC20', Revision: 'EC20EQAR02A05E2G 1 [Mar 14 2016 19:33:01]', Serial: '868323024272103'\u000a12:13:37.667139 tst iperf3m4.py: using MSISDN 3988\u000a12:13:37.835635 bus /gobi_4: Setting Powered True\u000a12:13:41.490774 run osmo-hlr_10.42.42.2: Add subscriber {algo=1, algo_str='comp128v1', imsi='901700000023849', msisdn='3988', subscriber_id=4}\u000a12:13:41.765016 bus /gobi_4: Setting Online False\u000a12:13:42.959797 bus /gobi_4: Setting Powered False\u000a12:13:50.226870 bus /gobi_4: Setting Powered True\u000a12:13:51.612541 bus /gobi_4: Setting Online True\u000a12:13:54.094963 tst /gobi_4: Connect to ('901', '70')\u000a12:13:54.382104 tst /gobi_4: Manufacturer: 'Quectel', Model: 'EC20', Revision: 'EC20EQAR02A08E2G 1 [Jul 21 2016 10:00:50]', Serial: '868323025161511'\u000a12:13:54.581523 tst iperf3m4.py:8: waiting for modems to attach...\u000a12:13:55.139563 tst /gobi_1: Already registered with network ('901', '70')\u000a12:14:10.059830 tst /gobi_4: Already registered with network ('901', '70')\u000a12:14:15.201781 tst iperf3m4.py:8: waiting for modems to attach to data services...\u000a12:14:18.241199 tst /gobi_6: context activated {apn='inet46', path='/gobi_6/context2', properties={AccessPointName='inet46', Active=True, AuthenticationMethod='chap', IPv6.Settings={}, Name='Internet', Password='', Protocol='ip', Settings={Address='176.16.46.1', DomainNameServers=['192.168.100.1', '8.8.8.8'], Gateway='176.16.46.2', Interface='r433306bbc2604b', Method='static', Netmask='255.255.255.252'}, Type='internet', Username='ogt'}, user='ogt'}\u000a12:14:18.441883 tst iperf3m4.py:8: Setting up data plan for \"'/gobi_6/context2'\"\u000a12:14:18.818467 run move_netns(pid=21268): Launched\u000a12:14:20.013709 run move_netns(pid=21268): Terminated: ok {rc=0}\u000a12:14:20.366648 run ifup(pid=21271): Launched\u000a12:14:21.550188 run ifup(pid=21271): Terminated: ok {rc=0}\u000a12:14:21.885075 run dhcp(pid=21274): Launched\u000a12:14:23.080306 run dhcp(pid=21274): Terminated: ok {rc=0}\u000a12:14:26.211202 tst /gobi_0: context activated {apn='inet46', path='/gobi_0/context2', properties={AccessPointName='inet46', Active=True, AuthenticationMethod='chap', IPv6.Settings={}, Name='Internet', Password='', Protocol='ip', Settings={Address='176.16.46.2', DomainNameServers=['192.168.100.1', '8.8.8.8'], Gateway='176.16.46.1', Interface='r82dcf628584c1c', Method='static', Netmask='255.255.255.252'}, Type='internet', Username='ogt'}, user='ogt'}\u000a12:14:26.399843 tst iperf3m4.py:8: Setting up data plan for \"'/gobi_0/context2'\"\u000a12:14:26.763701 run move_netns(pid=21292): Launched\u000a12:14:27.958439 run move_netns(pid=21292): Terminated: ok {rc=0}\u000a12:14:28.303717 run ifup(pid=21295): Launched\u000a12:14:29.499821 run ifup(pid=21295): Terminated: ok {rc=0}\u000a12:14:29.847564 run dhcp(pid=21298): Launched\u000a12:14:31.043002 run dhcp(pid=21298): Terminated: ok {rc=0}\u000a12:14:39.430561 tst /gobi_1: context activated {apn='inet46', path='/gobi_1/context12', properties={AccessPointName='inet46', Active=True, AuthenticationMethod='chap', IPv6.Settings={}, Name='Internet', Password='', Protocol='ip', Settings={Address='176.16.46.3', DomainNameServers=['192.168.100.1', '8.8.8.8'], Gateway='176.16.46.1', Interface='rf303c5eb721dad', Method='static', Netmask='255.255.255.248'}, Type='internet', Username='ogt'}, user='ogt'}\u000a12:14:39.630190 tst iperf3m4.py:8: Setting up data plan for \"'/gobi_1/context12'\"\u000a12:14:40.008769 run move_netns(pid=21316): Launched\u000a12:14:41.203594 run move_netns(pid=21316): Terminated: ok {rc=0}\u000a12:14:41.555073 run ifup(pid=21319): Launched\u000a12:14:42.750593 run ifup(pid=21319): Terminated: ok {rc=0}\u000a12:14:43.094092 run dhcp(pid=21322): Launched\u000a12:14:44.288607 run dhcp(pid=21322): Terminated: ok {rc=0}\u000a12:14:47.398290 tst /gobi_4: context activated {apn='inet46', path='/gobi_4/context2', properties={AccessPointName='inet46', Active=True, AuthenticationMethod='chap', IPv6.Settings={}, Name='Internet', Password='', Protocol='ip', Settings={Address='176.16.46.4', DomainNameServers=['192.168.100.1', '8.8.8.8'], Gateway='176.16.46.5', Interface='r9d788e55295d1b', Method='static', Netmask='255.255.255.252'}, Type='internet', Username='ogt'}, user='ogt'}\u000a12:14:47.598515 tst iperf3m4.py:8: Setting up data plan for \"'/gobi_4/context2'\"\u000a12:14:47.972558 run move_netns(pid=21344): Launched\u000a12:14:49.167644 run move_netns(pid=21344): Terminated: ok {rc=0}\u000a12:14:49.511099 run ifup(pid=21347): Launched\u000a12:14:50.695102 run ifup(pid=21347): Terminated: ok {rc=0}\u000a12:14:51.022092 run dhcp(pid=21350): Launched\u000a12:14:52.204665 run dhcp(pid=21350): Terminated: ok {rc=0}\u000a12:14:52.398498 tst iperf3m4.py:8: Running iperf3 client to 10.42.42.10:5003 through \"'/gobi_6/context2'\"\u000a12:14:52.528723 run iperf3-cli_10.42.42.10: Preparing iperf3-client connecting to 10.42.42.10:5003 (proto=tcp,time=60s)\u000a12:14:52.914061 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21366): Launched\u000a12:14:53.168421 tst iperf3m4.py:8: Running iperf3 client to 10.42.42.10:5004 through \"'/gobi_0/context2'\"\u000a12:14:53.307840 run iperf3-cli_10.42.42.10: Preparing iperf3-client connecting to 10.42.42.10:5004 (proto=tcp,time=60s)\u000a12:14:53.709624 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21369): Launched\u000a12:14:53.962766 tst iperf3m4.py:8: Running iperf3 client to 10.42.42.10:5005 through \"'/gobi_1/context12'\"\u000a12:14:54.102251 run iperf3-cli_10.42.42.10: Preparing iperf3-client connecting to 10.42.42.10:5005 (proto=tcp,time=60s)\u000a12:14:54.505434 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21372): Launched\u000a12:14:54.758418 tst iperf3m4.py:8: Running iperf3 client to 10.42.42.10:5006 through \"'/gobi_4/context2'\"\u000a12:14:54.897262 run iperf3-cli_10.42.42.10: Preparing iperf3-client connecting to 10.42.42.10:5006 (proto=tcp,time=60s)\u000a12:14:55.297856 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21375): Launched\u000a12:14:55.633982 run iperf3-cli_10.42.42.10(pid=21378): Launched\u000a12:14:55.918859 run iperf3-cli_10.42.42.10(pid=21381): Launched\u000a12:14:56.208202 run iperf3-cli_10.42.42.10(pid=21384): Launched\u000a12:14:56.498429 run iperf3-cli_10.42.42.10(pid=21387): Launched\u000a12:16:19.844408 run iperf3-cli_10.42.42.10(pid=21378): ERR: Terminated: ERROR {rc=1} [trial-2067↪gprs:sysmo↪iperf3m4.py:8↪iperf3m4.py↪iperf3-srv_10.42.42.10↪iperf3-cli_10.42.42.10↪iperf3-cli_10.42.42.10(pid=21378)]\u000a12:16:19.980002 run iperf3-cli_10.42.42.10(pid=21378): stdout: \u000a| \u0009\u0009\u0009\u0009\"bits_per_second\":\u00090,\u000a| \u0009\u0009\u0009\u0009\"retransmits\":\u00090,\u000a| \u0009\u0009\u0009\u0009\"omitted\":\u0009false\u000a| \u0009\u0009\u0009}\u000a| \u0009\u0009}],\u000a| \u0009\"end\":\u0009{\u000a| \u0009},\u000a| \u0009\"error\":\u0009\"error - control socket has closed unexpectedly\"\u000a| }\u000a| child exited with 1 \u000a12:16:20.113772 run iperf3-cli_10.42.42.10(pid=21378): stderr: \u000a| (launched: 2022-09-08_12:14:55.515951) \u000a12:16:21.302189 run iperf3-cli_10.42.42.10(pid=21381): ERR: Terminated: ERROR {rc=1} [trial-2067↪gprs:sysmo↪iperf3m4.py:8↪iperf3m4.py↪iperf3-srv_10.42.42.10↪iperf3-cli_10.42.42.10↪iperf3-cli_10.42.42.10(pid=21381)]\u000a12:16:21.436576 run iperf3-cli_10.42.42.10(pid=21381): stdout: \u000a| \u0009\u0009\u0009\u0009\"bits_per_second\":\u00090,\u000a| \u0009\u0009\u0009\u0009\"retransmits\":\u00090,\u000a| \u0009\u0009\u0009\u0009\"omitted\":\u0009false\u000a| \u0009\u0009\u0009}\u000a| \u0009\u0009}],\u000a| \u0009\"end\":\u0009{\u000a| \u0009},\u000a| \u0009\"error\":\u0009\"error - control socket has closed unexpectedly\"\u000a| }\u000a| child exited with 1 \u000a12:16:21.570684 run iperf3-cli_10.42.42.10(pid=21381): stderr: \u000a| (launched: 2022-09-08_12:14:55.801521) \u000a12:16:22.767473 run iperf3-cli_10.42.42.10(pid=21384): ERR: Terminated: ERROR {rc=1} [trial-2067↪gprs:sysmo↪iperf3m4.py:8↪iperf3m4.py↪iperf3-srv_10.42.42.10↪iperf3-cli_10.42.42.10↪iperf3-cli_10.42.42.10(pid=21384)]\u000a12:16:22.904946 run iperf3-cli_10.42.42.10(pid=21384): stdout: \u000a| \u0009\u0009\u0009\u0009\"bits_per_second\":\u00090,\u000a| \u0009\u0009\u0009\u0009\"retransmits\":\u00090,\u000a| \u0009\u0009\u0009\u0009\"omitted\":\u0009false\u000a| \u0009\u0009\u0009}\u000a| \u0009\u0009}],\u000a| \u0009\"end\":\u0009{\u000a| \u0009},\u000a| \u0009\"error\":\u0009\"error - control socket has closed unexpectedly\"\u000a| }\u000a| child exited with 1 \u000a12:16:23.041323 run iperf3-cli_10.42.42.10(pid=21384): stderr: \u000a| (launched: 2022-09-08_12:14:56.087095) \u000a12:16:24.237555 run iperf3-cli_10.42.42.10(pid=21387): ERR: Terminated: ERROR {rc=1} [trial-2067↪gprs:sysmo↪iperf3m4.py:8↪iperf3m4.py↪iperf3-srv_10.42.42.10↪iperf3-cli_10.42.42.10↪iperf3-cli_10.42.42.10(pid=21387)]\u000a12:16:24.372865 run iperf3-cli_10.42.42.10(pid=21387): stdout: \u000a| \u0009\u0009\u0009\u0009\"bits_per_second\":\u00090,\u000a| \u0009\u0009\u0009\u0009\"retransmits\":\u00090,\u000a| \u0009\u0009\u0009\u0009\"omitted\":\u0009false\u000a| \u0009\u0009\u0009}\u000a| \u0009\u0009}],\u000a| \u0009\"end\":\u0009{\u000a| \u0009},\u000a| \u0009\"error\":\u0009\"error - control socket has closed unexpectedly\"\u000a| }\u000a| child exited with 1 \u000a12:16:24.506316 run iperf3-cli_10.42.42.10(pid=21387): stderr: \u000a| (launched: 2022-09-08_12:14:56.377632) \u000a12:16:24.628427 run iperf3-srv_10.42.42.10(pid=21203): Terminating (SIGINT)\u000a12:16:24.966664 run iperf3-srv_10.42.42.10(pid=21203): Terminated {rc=1}\u000a12:16:25.166682 tst iperf3m4.py:8: Results for 10.42.42.10:5003 through \"'/gobi_6/context2'\"\u000a12:16:25.225067 tst iperf3m4.py:8: ERR: KeyError: 'sum_sent' [trial-2067↪gprs:sysmo↪iperf3m4.py:8]\u000a12:16:25.232006 tst iperf3m4.py:8: Test FAILED (296.9 sec)\u000a12:16:25.269516 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21202): Terminating (SIGTERM)\u000a12:16:25.311985 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21204): Terminating (SIGTERM)\u000a12:16:25.353839 run iperf3-srv_10.42.42.10(pid=21205): Terminating (SIGTERM)\u000a12:16:25.396412 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21206): Terminating (SIGTERM)\u000a12:16:25.438433 run iperf3-srv_10.42.42.10(pid=21207): Terminating (SIGTERM)\u000a12:16:25.480936 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21208): Terminating (SIGTERM)\u000a12:16:25.523604 run iperf3-srv_10.42.42.10(pid=21209): Terminating (SIGTERM)\u000a12:16:25.565657 run pcap-recorder_any(filters='host 10.42.42.2')(pid=21212): Terminating (SIGTERM)\u000a12:16:25.608160 run osmo-hlr_10.42.42.2(pid=21213): Terminating (SIGTERM)\u000a12:16:25.651464 run pcap-recorder_any(filters='host 10.42.42.5 and port not 22')(pid=21214): Terminating (SIGTERM)\u000a12:16:25.694859 run osmo-stp_10.42.42.5(pid=21215): Terminating (SIGTERM)\u000a12:16:25.757834 run pcap-recorder_any(filters='host 10.42.42.6')(pid=21216): Terminating (SIGTERM)\u000a12:16:25.801100 run osmo-ggsn_10.42.42.6(pid=21221): Terminating (SIGTERM)\u000a12:16:25.864673 run pcap-recorder_any(filters='host 10.42.42.7')(pid=21238): Terminating (SIGTERM)\u000a12:16:25.928722 run osmo-sgsn_10.42.42.7(pid=21239): Terminating (SIGTERM)\u000a12:16:25.991089 run pcap-recorder_any(filters='host 10.42.42.8 and port not 22')(pid=21240): Terminating (SIGTERM)\u000a12:16:26.052432 run osmo-msc_10.42.42.8(pid=21241): Terminating (SIGTERM)\u000a12:16:26.099766 run pcap-recorder_any(filters='host 10.42.42.3 and port not 22')(pid=21242): Terminating (SIGTERM)\u000a12:16:26.143596 run osmo-mgw_10.42.42.3(pid=21243): Terminating (SIGTERM)\u000a12:16:26.187521 run pcap-recorder_any(filters='host 10.42.42.4 and port not 22')(pid=21244): Terminating (SIGTERM)\u000a12:16:26.231474 run osmo-mgw_10.42.42.4(pid=21245): Terminating (SIGTERM)\u000a12:16:26.274626 run pcap-recorder_any(filters='host 10.42.42.9 and port not 22')(pid=21246): Terminating (SIGTERM)\u000a12:16:26.322089 run osmo-bsc_10.42.42.9(pid=21247): Terminating (SIGTERM)\u000a12:16:26.367138 run osmo-bts-sysmo(pid=21256): Terminating (SIGTERM)\u000a12:16:26.412659 run osmo-pcu(pid=21264): Terminating (SIGTERM)\u000a12:16:26.455494 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21366): Terminating (SIGTERM)\u000a12:16:26.612839 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21366)-killSignals.SIGTERM(pid=21422): Launched\u000a12:16:27.704877 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21366)-killSignals.SIGTERM(pid=21422): Terminated: ok {rc=0}\u000a12:16:27.751075 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21369): Terminating (SIGTERM)\u000a12:16:27.904689 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21369)-killSignals.SIGTERM(pid=21427): Launched\u000a12:16:28.999863 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21369)-killSignals.SIGTERM(pid=21427): Terminated: ok {rc=0}\u000a12:16:29.047928 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21372): Terminating (SIGTERM)\u000a12:16:29.208359 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21372)-killSignals.SIGTERM(pid=21430): Launched\u000a12:16:30.303402 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21372)-killSignals.SIGTERM(pid=21430): Terminated: ok {rc=0}\u000a12:16:30.351861 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21375): Terminating (SIGTERM)\u000a12:16:30.512537 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21375)-killSignals.SIGTERM(pid=21433): Launched\u000a12:16:31.608861 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21375)-killSignals.SIGTERM(pid=21433): Terminated: ok {rc=0}\u000a12:16:31.623397 --- ParallelTerminationStrategy: PID 21202 died...\u000a12:16:31.680652 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21202): Terminated: ok {rc=0}\u000a12:16:31.690827 --- ParallelTerminationStrategy: PID 21204 died...\u000a12:16:31.747549 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21204): Terminated: ok {rc=0}\u000a12:16:31.757507 --- ParallelTerminationStrategy: PID 21205 died...\u000a12:16:31.814036 run iperf3-srv_10.42.42.10(pid=21205): Terminated {rc=256}\u000a12:16:31.824276 --- ParallelTerminationStrategy: PID 21206 died...\u000a12:16:31.880538 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21206): Terminated: ok {rc=0}\u000a12:16:31.890929 --- ParallelTerminationStrategy: PID 21207 died...\u000a12:16:31.947411 run iperf3-srv_10.42.42.10(pid=21207): Terminated {rc=256}\u000a12:16:31.957615 --- ParallelTerminationStrategy: PID 21208 died...\u000a12:16:32.014299 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21208): Terminated: ok {rc=0}\u000a12:16:32.024316 --- ParallelTerminationStrategy: PID 21209 died...\u000a12:16:32.080791 run iperf3-srv_10.42.42.10(pid=21209): Terminated {rc=256}\u000a12:16:32.090808 --- ParallelTerminationStrategy: PID 21212 died...\u000a12:16:32.147348 run pcap-recorder_any(filters='host 10.42.42.2')(pid=21212): Terminated: ok {rc=0}\u000a12:16:32.157352 --- ParallelTerminationStrategy: PID 21213 died...\u000a12:16:32.214347 run osmo-hlr_10.42.42.2(pid=21213): Terminated: ok {rc=0}\u000a12:16:32.224388 --- ParallelTerminationStrategy: PID 21214 died...\u000a12:16:32.280732 run pcap-recorder_any(filters='host 10.42.42.5 and port not 22')(pid=21214): Terminated: ok {rc=0}\u000a12:16:32.291036 --- ParallelTerminationStrategy: PID 21215 died...\u000a12:16:32.347530 run osmo-stp_10.42.42.5(pid=21215): Terminated: ok {rc=0}\u000a12:16:32.357654 --- ParallelTerminationStrategy: PID 21216 died...\u000a12:16:32.414318 run pcap-recorder_any(filters='host 10.42.42.6')(pid=21216): Terminated: ok {rc=0}\u000a12:16:32.424561 --- ParallelTerminationStrategy: PID 21221 died...\u000a12:16:32.481054 run osmo-ggsn_10.42.42.6(pid=21221): Terminated: ok {rc=0}\u000a12:16:32.491414 --- ParallelTerminationStrategy: PID 21238 died...\u000a12:16:32.548260 run pcap-recorder_any(filters='host 10.42.42.7')(pid=21238): Terminated: ok {rc=0}\u000a12:16:32.558281 --- ParallelTerminationStrategy: PID 21239 died...\u000a12:16:32.615021 run osmo-sgsn_10.42.42.7(pid=21239): Terminated: ok {rc=0}\u000a12:16:32.625215 --- ParallelTerminationStrategy: PID 21240 died...\u000a12:16:32.682156 run pcap-recorder_any(filters='host 10.42.42.8 and port not 22')(pid=21240): Terminated: ok {rc=0}\u000a12:16:32.692263 --- ParallelTerminationStrategy: PID 21241 died...\u000a12:16:32.748864 run osmo-msc_10.42.42.8(pid=21241): Terminated: ok {rc=0}\u000a12:16:32.759004 --- ParallelTerminationStrategy: PID 21242 died...\u000a12:16:32.815666 run pcap-recorder_any(filters='host 10.42.42.3 and port not 22')(pid=21242): Terminated: ok {rc=0}\u000a12:16:32.825733 --- ParallelTerminationStrategy: PID 21243 died...\u000a12:16:32.882639 run osmo-mgw_10.42.42.3(pid=21243): Terminated {rc=15}\u000a12:16:32.892716 --- ParallelTerminationStrategy: PID 21244 died...\u000a12:16:32.949488 run pcap-recorder_any(filters='host 10.42.42.4 and port not 22')(pid=21244): Terminated: ok {rc=0}\u000a12:16:32.959638 --- ParallelTerminationStrategy: PID 21245 died...\u000a12:16:33.016987 run osmo-mgw_10.42.42.4(pid=21245): Terminated {rc=15}\u000a12:16:33.027078 --- ParallelTerminationStrategy: PID 21246 died...\u000a12:16:33.083914 run pcap-recorder_any(filters='host 10.42.42.9 and port not 22')(pid=21246): Terminated: ok {rc=0}\u000a12:16:33.094256 --- ParallelTerminationStrategy: PID 21247 died...\u000a12:16:33.151037 run osmo-bsc_10.42.42.9(pid=21247): Terminated: ok {rc=0}\u000a12:16:33.161108 --- ParallelTerminationStrategy: PID 21256 died...\u000a12:16:33.217850 run osmo-bts-sysmo(pid=21256): Terminated: ok {rc=0}\u000a12:16:33.228025 --- ParallelTerminationStrategy: PID 21264 died...\u000a12:16:33.285572 run osmo-pcu(pid=21264): Terminated: ok {rc=0}\u000a12:16:39.154267 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21366): Terminating (SIGINT)\u000a12:16:39.309667 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21366)-killSignals.SIGINT(pid=21436): Launched\u000a12:16:40.404915 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21366)-killSignals.SIGINT(pid=21436): Terminated: ok {rc=0}\u000a12:16:40.453447 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21369): Terminating (SIGINT)\u000a12:16:40.613845 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21369)-killSignals.SIGINT(pid=21439): Launched\u000a12:16:41.708654 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21369)-killSignals.SIGINT(pid=21439): Terminated: ok {rc=0}\u000a12:16:41.757246 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21372): Terminating (SIGINT)\u000a12:16:41.916333 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21372)-killSignals.SIGINT(pid=21442): Launched\u000a12:16:43.011078 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21372)-killSignals.SIGINT(pid=21442): Terminated: ok {rc=0}\u000a12:16:43.059868 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21375): Terminating (SIGINT)\u000a12:16:43.215292 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21375)-killSignals.SIGINT(pid=21445): Launched\u000a12:16:44.305955 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21375)-killSignals.SIGINT(pid=21445): Terminated: ok {rc=0}\u000a12:16:50.176527 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21366): Terminating (SIGKILL)\u000a12:16:50.342896 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21366)-killSignals.SIGUSR1(pid=21448): Launched\u000a12:16:51.434046 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21366)-killSignals.SIGUSR1(pid=21448): Terminated: ok {rc=0}\u000a12:16:51.480013 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21369): Terminating (SIGKILL)\u000a12:16:51.632717 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21369)-killSignals.SIGUSR1(pid=21451): Launched\u000a12:16:52.724058 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21369)-killSignals.SIGUSR1(pid=21451): Terminated: ok {rc=0}\u000a12:16:52.769859 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21372): Terminating (SIGKILL)\u000a12:16:52.922247 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21372)-killSignals.SIGUSR1(pid=21454): Launched\u000a12:16:54.012502 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21372)-killSignals.SIGUSR1(pid=21454): Terminated: ok {rc=0}\u000a12:16:54.058627 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21375): Terminating (SIGKILL)\u000a12:16:54.209302 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21375)-killSignals.SIGUSR1(pid=21457): Launched\u000a12:16:55.300985 run pcap-recorder_any(filters='host 10.42.42.10 and port not 22')(pid=21375)-killSignals.SIGUSR1(pid=21457): Terminated: ok {rc=0}\u000a12:16:55.539496 bus /gobi_4: Setting Online False\u000a12:16:56.677312 bus /gobi_4: Setting Powered False\u000a12:17:06.977899 bus /gobi_4: Setting Powered False\u000a12:17:08.085825 bus /gobi_4: Setting Powered False\u000a12:17:09.238187 bus /gobi_1: Setting Online False\u000a12:17:10.373666 bus /gobi_1: Setting Powered False\u000a12:17:18.084487 bus /gobi_1: Setting Powered False\u000a12:17:19.190688 bus /gobi_1: Setting Powered False\u000a12:17:20.342609 bus /gobi_0: Setting Online False\u000a12:17:21.478233 bus /gobi_0: Setting Powered False\u000a12:17:29.976993 bus /gobi_0: Setting Powered False\u000a12:17:31.083720 bus /gobi_0: Setting Powered False\u000a12:17:32.235462 bus /gobi_6: Setting Online False\u000a12:17:33.348859 bus /gobi_6: Setting Powered False\u000a12:17:41.052366 bus /gobi_6: Setting Powered False\u000a" }, { "testActions" : [ ], "age" : 0, "className" : "gprs:sysmo", "duration" : 147.0, "errorDetails" : None, "errorStackTrace" : None, "failedSince" : 0, "name" : "ping.py", "properties" : { }, "skipped" : False, "skippedMessage" : None, "status" : "PASSED", "stderr" : None, "stdout" : "(launched: 2022-09-08_12:19:42.075900)\u000a21548: waiting for 21549\u000aPING 10.42.42.6 (10.42.42.6) 56(84) bytes of data.\u000a64 bytes from 10.42.42.6: icmp_seq=1 ttl=64 time=614 ms\u000a64 bytes from 10.42.42.6: icmp_seq=2 ttl=64 time=532 ms\u000a64 bytes from 10.42.42.6: icmp_seq=3 ttl=64 time=509 ms\u000a64 bytes from 10.42.42.6: icmp_seq=4 ttl=64 time=492 ms\u000a64 bytes from 10.42.42.6: icmp_seq=5 ttl=64 time=511 ms\u000a64 bytes from 10.42.42.6: icmp_seq=6 ttl=64 time=507 ms\u000a64 bytes from 10.42.42.6: icmp_seq=7 ttl=64 time=488 ms\u000a64 bytes from 10.42.42.6: icmp_seq=8 ttl=64 time=484 ms\u000a64 bytes from 10.42.42.6: icmp_seq=9 ttl=64 time=485 ms\u000a64 bytes from 10.42.42.6: icmp_seq=10 ttl=64 time=462 ms\u000a\u000a--- 10.42.42.6 ping statistics ---\u000a10 packets transmitted, 10 received, 0% packet loss, time 18ms\u000artt min/avg/max/mdev = 461.661/508.363/613.770/39.585 ms\u000achild exited with 0\u000a" }, { "testActions" : [ ], "age" : 1, "className" : "gprs:sysmo", "duration" : 160.0, "errorDetails" : None, "errorStackTrace" : "ping1(pid=21659): launch_sync(): local process exited with status 1 [trial-2067↪gprs:sysmo↪ping_idle_ping.py:61↪ping_idle_ping.py↪/gobi_6↪ping1(pid=21659)]", "failedSince" : 2067, "name" : "ping_idle_ping.py", "properties" : { }, "skipped" : False, "skippedMessage" : None, "status" : "REGRESSION", "stderr" : "Traceback (most recent call last):\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/core/test.py\", line 76, in run\u000a self.path)\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/core/util.py\", line 414, in run_python_file\u000a spec.loader.exec_module( importlib.util.module_from_spec(spec) )\u000a File \"\", line 728, in exec_module\u000a File \"\", line 219, in _call_with_frames_removed\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/sysmocom/suites/gprs/ping_idle_ping.py\", line 61, in \u000a proc = ms.run_netns_wait('ping1', ('ping', '-c', '10', ggsn.addr()))\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/obj/ms_ofono.py\", line 691, in run_netns_wait\u000a proc.launch_sync()\u000a File \"/home/jenkins/workspace/osmo-gsm-tester_gerrit/osmo-gsm-tester/src/osmo_gsm_tester/core/process.py\", line 216, in launch_sync\u000a raise self.RunError('launch_sync()')\u000aosmo_gsm_tester.core.log.Error: ping1(pid=21659): launch_sync(): local process exited with status 1 [trial-2067↪gprs:sysmo↪ping_idle_ping.py:61↪ping_idle_ping.py↪/gobi_6↪ping1(pid=21659)]\u000a", "stdout" : "\u000a----------------------------------------------\u000atrial-2067 gprs:sysmo ping_idle_ping.py\u000a----------------------------------------------\u000a12:20:30.081796 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:20:30.093738 tst gprs:sysmo: Using 1 x bts (candidates: 1)\u000a12:20:30.224222 tst ping_idle_ping.py: using LAC 3970\u000a12:20:30.350047 tst ping_idle_ping.py: using RAC 145\u000a12:20:30.479342 tst ping_idle_ping.py: using CellId 3970\u000a12:20:30.603345 tst ping_idle_ping.py: using BVCI 3971\u000a12:20:30.628030 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:20:30.639215 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:20:30.650216 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:20:30.661311 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:20:30.672282 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:20:30.683508 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:20:30.695322 tst gprs:sysmo: Using 1 x ip_address (candidates: 9)\u000a12:20:30.706312 tst gprs:sysmo: Using 1 x modem (candidates: 4)\u000a12:20:30.998841 tst ping_idle_ping.py:24: start network...\u000a12:20:31.132930 run osmo-hlr_10.42.42.2: Starting osmo-hlr\u000a12:20:31.648079 run create_hlr_db(pid=21574): Launched\u000a12:20:31.834199 bus /gobi_6: Setting Powered False\u000a12:20:32.855130 run create_hlr_db(pid=21574): Terminated: ok {rc=0}\u000a12:20:33.265484 run pcap-recorder_any(filters='host 10.42.42.2')(pid=21576): Launched\u000a12:20:33.570023 run osmo-hlr_10.42.42.2(pid=21577): Launched\u000a12:20:33.678245 run osmo-stp_10.42.42.5: Starting osmo-stp\u000a12:20:34.203630 run pcap-recorder_any(filters='host 10.42.42.5 and port not 22')(pid=21578): Launched\u000a12:20:34.508282 run osmo-stp_10.42.42.5(pid=21579): Launched\u000a12:20:34.614871 run osmo-ggsn_10.42.42.6: Starting osmo-ggsn\u000a12:20:35.142177 run pcap-recorder_any(filters='host 10.42.42.6')(pid=21580): Launched\u000a12:20:35.249308 run osmo-ggsn_10.42.42.6: Setting RPATH for osmo-ggsn\u000a12:20:35.515597 run patchelf(pid=21581): Launched\u000a12:20:36.695157 run patchelf(pid=21581): Terminated: ok {rc=0}\u000a12:20:36.805028 run osmo-ggsn_10.42.42.6: Applying CAP_NET_ADMIN capability to osmo-ggsn\u000a12:20:37.076568 run osmo-gsm-tester_setcap_net_admin.sh(pid=21582): Launched\u000a12:20:38.266521 run osmo-gsm-tester_setcap_net_admin.sh(pid=21582): Terminated: ok {rc=0}\u000a12:20:38.593356 run osmo-ggsn_10.42.42.6(pid=21585): Launched\u000a12:20:38.829248 run osmo-sgsn_10.42.42.7: Starting osmo-sgsn\u000a12:20:39.401077 run pcap-recorder_any(filters='host 10.42.42.7')(pid=21607): Launched\u000a12:20:39.722163 run osmo-sgsn_10.42.42.7(pid=21608): Launched\u000a12:20:39.835824 run osmo-msc_10.42.42.8: Starting osmo-msc\u000a12:20:40.395235 run pcap-recorder_any(filters='host 10.42.42.8 and port not 22')(pid=21609): Launched\u000a12:20:40.702796 run osmo-msc_10.42.42.8(pid=21610): Launched\u000a12:20:43.864705 run osmo-mgw_10.42.42.3: Starting osmo-mgw\u000a12:20:44.392686 run pcap-recorder_any(filters='host 10.42.42.3 and port not 22')(pid=21611): Launched\u000a12:20:44.699853 run osmo-mgw_10.42.42.3(pid=21612): Launched\u000a12:20:44.807625 run osmo-mgw_10.42.42.4: Starting osmo-mgw\u000a12:20:45.325923 run pcap-recorder_any(filters='host 10.42.42.4 and port not 22')(pid=21613): Launched\u000a12:20:45.633601 run osmo-mgw_10.42.42.4(pid=21614): Launched\u000a12:20:45.741647 run osmo-bsc_10.42.42.9: Starting osmo-bsc\u000a12:20:46.380496 run pcap-recorder_any(filters='host 10.42.42.9 and port not 22')(pid=21615): Launched\u000a12:20:46.690954 run osmo-bsc_10.42.42.9(pid=21616): Launched\u000a12:20:52.949756 run osmo-bts-sysmo: Starting sysmoBTS to connect to osmo-bsc_10.42.42.9\u000a12:20:53.494863 run rm-remote-dir(pid=21617): Launched\u000a12:20:55.684836 run rm-remote-dir(pid=21617): Terminated: ok {rc=0}\u000a12:20:56.012804 run mk-remote-dir(pid=21618): Launched\u000a12:20:57.200402 run mk-remote-dir(pid=21618): Terminated: ok {rc=0}\u000a12:20:57.476386 run scp-inst-to-remote(pid=21619): Launched\u000a12:21:09.684733 run scp-inst-to-remote(pid=21619): Terminated: ok {rc=0}\u000a12:21:10.027054 run mk-remote-dir(pid=21621): Launched\u000a12:21:11.224531 run mk-remote-dir(pid=21621): Terminated: ok {rc=0}\u000a12:21:11.517682 run scp-cfg-to-remote(pid=21622): Launched\u000a12:21:12.712754 run scp-cfg-to-remote(pid=21622): Terminated: ok {rc=0}\u000a12:21:13.051428 run reload-dsp-firmware(pid=21624): Launched\u000a12:21:15.247274 run reload-dsp-firmware(pid=21624): Terminated: ok {rc=0}\u000a12:21:15.576991 run osmo-bts-sysmo(pid=21625): Launched\u000a12:21:19.889296 tst ping_idle_ping.py:36: Waiting for bts to be ready...\u000a12:21:21.530687 run rm-remote-dir(pid=21626): Launched\u000a12:21:23.722588 run rm-remote-dir(pid=21626): Terminated: ok {rc=0}\u000a12:21:24.054907 run mk-remote-dir(pid=21627): Launched\u000a12:21:26.237751 run mk-remote-dir(pid=21627): Terminated: ok {rc=0}\u000a12:21:26.506057 run scp-inst-to-bts(pid=21628): Launched\u000a12:21:39.697797 run scp-inst-to-bts(pid=21628): Terminated: ok {rc=0}\u000a12:21:40.011612 run mk-remote-run-dir(pid=21631): Launched\u000a12:21:42.192933 run mk-remote-run-dir(pid=21631): Terminated: ok {rc=0}\u000a12:21:42.460275 run scp-cfg-to-sysmobts(pid=21632): Launched\u000a12:21:44.639531 run scp-cfg-to-sysmobts(pid=21632): Terminated: ok {rc=0}\u000a12:21:44.950382 run osmo-pcu(pid=21634): Launched\u000a12:21:45.068816 tst ping_idle_ping.py: using MSISDN 3990\u000a12:21:45.247220 bus /gobi_6: Setting Powered True\u000a12:21:48.761924 run osmo-hlr_10.42.42.2: Add subscriber {algo=1, algo_str='comp128v1', imsi='901700000010978', msisdn='3990', subscriber_id=1}\u000a12:21:49.035308 bus /gobi_6: Setting Online False\u000a12:21:50.228343 bus /gobi_6: Setting Powered False\u000a12:21:57.202252 bus /gobi_6: Setting Powered True\u000a12:21:58.607077 bus /gobi_6: Setting Online True\u000a12:22:01.011173 tst /gobi_6: Connect to ('901', '70')\u000a12:22:01.288601 tst /gobi_6: Manufacturer: 'Quectel', Model: 'EC20', Revision: 'EC20EQAR02A13E2G 1 [Jul 5 2017 10:49:17]', Serial: '868323024273648'\u000a12:22:01.475058 tst ping_idle_ping.py:47: waiting for modems to attach...\u000a12:22:08.048265 tst /gobi_6: Already registered with network ('901', '70')\u000a12:22:19.916718 tst ping_idle_ping.py:51: waiting for modems to attach to data services...\u000a12:22:43.918517 tst /gobi_6: context activated {apn='inet46', path='/gobi_6/context2', properties={AccessPointName='inet46', Active=True, AuthenticationMethod='chap', IPv6.Settings={}, Name='Internet', Password='', Protocol='ip', Settings={Address='176.16.46.1', DomainNameServers=['192.168.100.1', '8.8.8.8'], Gateway='176.16.46.2', Interface='r433306bbc2604b', Method='static', Netmask='255.255.255.252'}, Type='internet', Username='ogt'}, user='ogt'}\u000a12:22:44.115798 tst ping_idle_ping.py:56: Setting up data plane for \"'/gobi_6/context2'\"\u000a12:22:44.497949 run move_netns(pid=21637): Launched\u000a12:22:45.692824 run move_netns(pid=21637): Terminated: ok {rc=0}\u000a12:22:46.042547 run ifup(pid=21640): Launched\u000a12:22:47.238063 run ifup(pid=21640): Terminated: ok {rc=0}\u000a12:22:47.586950 run dhcp(pid=21643): Launched\u000a12:22:48.770066 run dhcp(pid=21643): Terminated: ok {rc=0}\u000a12:22:48.961262 tst ping_idle_ping.py:60: [1] Running 10 ping requests for \"'/gobi_6/context2'\"\u000a12:22:49.306102 run ping1(pid=21659): Launched\u000a12:23:09.530912 run ping1(pid=21659): ERR: Terminated: ERROR {rc=1} [trial-2067↪gprs:sysmo↪ping_idle_ping.py:61↪ping_idle_ping.py↪/gobi_6↪ping1(pid=21659)]\u000a12:23:09.668304 run ping1(pid=21659): stdout: \u000a| (launched: 2022-09-08_12:22:49.140188)\u000a| 21660: waiting for 21661\u000a| PING 10.42.42.6 (10.42.42.6) 56(84) bytes of data.\u000a| \u000a| --- 10.42.42.6 ping statistics ---\u000a| 10 packets transmitted, 0 received, 100% packet loss, time 204ms\u000a| \u000a| child exited with 1 \u000a12:23:09.806443 run ping1(pid=21659): stderr: \u000a| (launched: 2022-09-08_12:22:49.187567) \u000a12:23:09.861444 tst ping_idle_ping.py:61: ERR: Error: ping1(pid=21659): launch_sync(): local process exited with status 1 [trial-2067↪gprs:sysmo↪ping_idle_ping.py:61↪ping_idle_ping.py↪/gobi_6↪ping1(pid=21659)] [trial-2067↪gprs:sysmo↪ping_idle_ping.py:61]\u000a12:23:09.868406 tst ping_idle_ping.py:61: Test FAILED (159.8 sec)\u000a12:23:09.905283 run pcap-recorder_any(filters='host 10.42.42.2')(pid=21576): Terminating (SIGTERM)\u000a12:23:09.947827 run osmo-hlr_10.42.42.2(pid=21577): Terminating (SIGTERM)\u000a12:23:09.990852 run pcap-recorder_any(filters='host 10.42.42.5 and port not 22')(pid=21578): Terminating (SIGTERM)\u000a12:23:10.036640 run osmo-stp_10.42.42.5(pid=21579): Terminating (SIGTERM)\u000a12:23:10.101770 run pcap-recorder_any(filters='host 10.42.42.6')(pid=21580): Terminating (SIGTERM)\u000a12:23:10.145882 run osmo-ggsn_10.42.42.6(pid=21585): Terminating (SIGTERM)\u000a12:23:10.238645 run pcap-recorder_any(filters='host 10.42.42.7')(pid=21607): Terminating (SIGTERM)\u000a12:23:10.288469 run osmo-sgsn_10.42.42.7(pid=21608): Terminating (SIGTERM)\u000a12:23:10.334358 run pcap-recorder_any(filters='host 10.42.42.8 and port not 22')(pid=21609): Terminating (SIGTERM)\u000a12:23:10.377543 run osmo-msc_10.42.42.8(pid=21610): Terminating (SIGTERM)\u000a12:23:10.419807 run pcap-recorder_any(filters='host 10.42.42.3 and port not 22')(pid=21611): Terminating (SIGTERM)\u000a12:23:10.460589 run osmo-mgw_10.42.42.3(pid=21612): Terminating (SIGTERM)\u000a12:23:10.501691 run pcap-recorder_any(filters='host 10.42.42.4 and port not 22')(pid=21613): Terminating (SIGTERM)\u000a12:23:10.542212 run osmo-mgw_10.42.42.4(pid=21614): Terminating (SIGTERM)\u000a12:23:10.583362 run pcap-recorder_any(filters='host 10.42.42.9 and port not 22')(pid=21615): Terminating (SIGTERM)\u000a12:23:10.624246 run osmo-bsc_10.42.42.9(pid=21616): Terminating (SIGTERM)\u000a12:23:10.664918 run osmo-bts-sysmo(pid=21625): Terminating (SIGTERM)\u000a12:23:10.706050 run osmo-pcu(pid=21634): Terminating (SIGTERM)\u000a12:23:10.715895 --- ParallelTerminationStrategy: PID 21576 died...\u000a12:23:10.771244 run pcap-recorder_any(filters='host 10.42.42.2')(pid=21576): Terminated: ok {rc=0}\u000a12:23:10.781773 --- ParallelTerminationStrategy: PID 21577 died...\u000a12:23:10.839163 run osmo-hlr_10.42.42.2(pid=21577): Terminated: ok {rc=0}\u000a12:23:10.849789 --- ParallelTerminationStrategy: PID 21578 died...\u000a12:23:10.943508 run pcap-recorder_any(filters='host 10.42.42.5 and port not 22')(pid=21578): Terminated: ok {rc=0}\u000a12:23:10.971556 --- ParallelTerminationStrategy: PID 21579 died...\u000a12:23:11.046159 run osmo-stp_10.42.42.5(pid=21579): Terminated: ok {rc=0}\u000a12:23:11.056810 --- ParallelTerminationStrategy: PID 21580 died...\u000a12:23:11.114861 run pcap-recorder_any(filters='host 10.42.42.6')(pid=21580): Terminated: ok {rc=0}\u000a12:23:11.125221 --- ParallelTerminationStrategy: PID 21585 died...\u000a12:23:11.183076 run osmo-ggsn_10.42.42.6(pid=21585): Terminated: ok {rc=0}\u000a12:23:11.193499 --- ParallelTerminationStrategy: PID 21607 died...\u000a12:23:11.251481 run pcap-recorder_any(filters='host 10.42.42.7')(pid=21607): Terminated: ok {rc=0}\u000a12:23:11.262064 --- ParallelTerminationStrategy: PID 21609 died...\u000a12:23:11.320507 run pcap-recorder_any(filters='host 10.42.42.8 and port not 22')(pid=21609): Terminated: ok {rc=0}\u000a12:23:11.331569 --- ParallelTerminationStrategy: PID 21610 died...\u000a12:23:11.390381 run osmo-msc_10.42.42.8(pid=21610): Terminated: ok {rc=0}\u000a12:23:11.401289 --- ParallelTerminationStrategy: PID 21611 died...\u000a12:23:11.460390 run pcap-recorder_any(filters='host 10.42.42.3 and port not 22')(pid=21611): Terminated: ok {rc=0}\u000a12:23:11.471938 --- ParallelTerminationStrategy: PID 21612 died...\u000a12:23:11.531311 run osmo-mgw_10.42.42.3(pid=21612): Terminated {rc=15}\u000a12:23:11.542189 --- ParallelTerminationStrategy: PID 21613 died...\u000a12:23:11.601839 run pcap-recorder_any(filters='host 10.42.42.4 and port not 22')(pid=21613): Terminated: ok {rc=0}\u000a12:23:11.612879 --- ParallelTerminationStrategy: PID 21614 died...\u000a12:23:11.672178 run osmo-mgw_10.42.42.4(pid=21614): Terminated {rc=15}\u000a12:23:11.683160 --- ParallelTerminationStrategy: PID 21615 died...\u000a12:23:11.742887 run pcap-recorder_any(filters='host 10.42.42.9 and port not 22')(pid=21615): Terminated: ok {rc=0}\u000a12:23:11.753910 --- ParallelTerminationStrategy: PID 21616 died...\u000a12:23:11.813419 run osmo-bsc_10.42.42.9(pid=21616): Terminated: ok {rc=0}\u000a12:23:11.824438 --- ParallelTerminationStrategy: PID 21625 died...\u000a12:23:11.884079 run osmo-bts-sysmo(pid=21625): Terminated: ok {rc=0}\u000a12:23:11.895189 --- ParallelTerminationStrategy: PID 21634 died...\u000a12:23:11.954287 run osmo-pcu(pid=21634): Terminated: ok {rc=0}\u000a12:23:12.133061 --- ParallelTerminationStrategy: PID 21608 died...\u000a12:23:12.192455 run osmo-sgsn_10.42.42.7(pid=21608): Terminated: ok {rc=0}\u000a12:23:12.418659 bus /gobi_6: Setting Online False\u000a12:23:16.343470 bus /gobi_6: Setting Powered False\u000a12:23:33.134040 bus /gobi_6: Setting Powered False\u000a" } ], "duration" : 1244.0, "enclosingBlockNames" : [ ], "enclosingBlocks" : [ ], "id" : "2", "name" : "gprs_sysmo", "nodeId" : None, "properties" : { }, "stderr" : None, "stdout" : None, "timestamp" : "2022-09-08T12:02:51" } ] }