name: func-target
run_id: commands[0]
env OS_TEST_HTTP_PROXY: http://squid.internal:3128/
env TEST_NAME_SERVER: 10.245.160.2
env TEST_VIP_RANGE: 172.16.154.230:172.16.154.250
env TEST_NETAPP_SAN_USERNAME: vsadmin
env OS_PASSWORD: aiv4thu8Iewo
env TEST_NET_ID: 94a87d6a-6d82-4931-878d-aedcf0e62962
env TEST_ZAZA_BUG_LP1987332: 1
env OS_PURESTORAGE_API_TOKEN: f8173d96-c387-bed8-23fa-2056038890f9
env TEST_IRONIC_DEPLOY_INITRD: http://10.245.161.162/swift/v1/images/ironic-python-agent.initramfs
env TEST_NETAPP_POOL_NAME_SEARCH_PATTERN: zaza
env HOME: /home/ubuntu
env TEST_CIDR_EXT: 172.16.0.0/16
env TEST_VIP: 172.16.154.230
env OS_PROJECT_DOMAIN_NAME: user
env OS_PURESTORAGE_SAN_IP: 10.246.112.11
env OS_AUTH_VERSION: 3
env TEST_NETAPP_SAN_IP: 10.245.168.123
env OS_IDENTITY_API_VERSION: 3
env TEST_GRAFANA_PLUGIN_PIECHART_URL: http://10.245.161.162:80/swift/v1/grafana_plugins/grafana-piechart-panel-1.6.2.zip
env OS_REGION_NAME: serverstack
env TEST_ARISTA_IMAGE_REMOTE: http://10.245.161.162/swift/v1/images/arista-cvx-virt-test.qcow2
env TEST_FIP_START: 172.16.154.200
env TEST_HTTP_PROXY: http://squid.internal:3128/
env TEST_CIDR_END: 172.16.154.254
env TEST_DEPLOY_TIMEOUT: 7200
env OS_AUTH_URL: http://10.245.161.156:5000/v3
env TEST_NETAPP_SAN_PASSWORD: Ubuntu123
env TEST_NVIDIA_VGPU_HOST_SW: http://10.245.161.162/swift/v1/images/nvidia-vgpu-ubuntu-510_510.47.03_amd64.deb
env OS_PROJECT_NAME: zuul-tests
env OS_USERNAME: zuul-tests
env OS_VIP00: 172.16.154.230
env TEST_NETAPP_STORAGE_TYPE: ontap_cluster
env LANG: C.UTF-8
env OS_VIP01: 172.16.154.231
env OS_VIP10: 172.16.154.240
env TEST_FIP_RANGE: 172.16.154.200:172.16.154.229
env TEST_REGISTRY_PREFIX: 10.245.164.190:5000
env OS_VIP02: 172.16.154.232
env OS_VIP11: 172.16.154.241
env OS_VIP20: 172.16.154.250
env TEST_SWIFT_IP: 10.245.161.162
env OS_VIP03: 172.16.154.233
env OS_VIP12: 172.16.154.242
env OS_VIP04: 172.16.154.234
env OS_VIP13: 172.16.154.243
env TEST_GATEWAY: 172.16.0.1
env TEST_VIP00: 172.16.154.230
env OS_VIP05: 172.16.154.235
env OS_VIP14: 172.16.154.244
env TEST_GRAFANA_PLUGIN_VONAGE_URL: http://10.245.161.162:80/swift/v1/grafana_plugins/vonage-status-panel-1.0.11.zip
env TEST_TRILIO_LICENSE: /tmp/tvault-license.lic
env TEST_VIP01: 172.16.154.231
env TEST_VIP10: 172.16.154.240
env OS_USER_DOMAIN_NAME: user
env OS_VIP06: 172.16.154.236
env OS_VIP15: 172.16.154.245
env TEST_NETAPP_SAN_VSERVER: svm01
env TEST_VIP02: 172.16.154.232
env TEST_VIP11: 172.16.154.241
env TEST_VIP20: 172.16.154.250
env OS_VIP07: 172.16.154.237
env OS_VIP16: 172.16.154.246
env TEST_FIP_END: 172.16.154.229
env TEST_IRONIC_RAW_BM_IMAGE: http://10.245.161.162/swift/v1/images/baremetal-ubuntu-focal.img
env TEST_VIP03: 172.16.154.233
env TEST_VIP12: 172.16.154.242
env OS_VIP08: 172.16.154.238
env OS_VIP17: 172.16.154.247
env TEST_NETAPP_SAN_PORT: 80
env TEST_VIP04: 172.16.154.234
env TEST_VIP13: 172.16.154.243
env OS_VIP09: 172.16.154.239
env OS_VIP18: 172.16.154.248
env TEST_VIP05: 172.16.154.235
env TEST_VIP14: 172.16.154.244
env OS_VIP19: 172.16.154.249
env TEST_NETAPP_ROOT_VOL_AGGR_NAME: openstack
env TEST_VIP06: 172.16.154.236
env TEST_VIP15: 172.16.154.245
env TEST_IRONIC_DEPLOY_VMLINUZ: http://10.245.161.162/swift/v1/images/ironic-python-agent.kernel
env TEST_VIP07: 172.16.154.237
env TEST_VIP16: 172.16.154.246
env TEST_MAX_RESOLVE_COUNT: 3
env TEST_VIP08: 172.16.154.238
env TEST_VIP17: 172.16.154.247
env TEST_VIP09: 172.16.154.239
env TEST_VIP18: 172.16.154.248
env TEST_MODEL_CONSTRAINTS: cores=2
env TEST_VIP19: 172.16.154.249
env PATH: /home/ubuntu/src/review.opendev.org/openstack/charm-hacluster/.tox/func-target/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin
env PIP_DISABLE_PIP_VERSION_CHECK: 1
env PYTHONIOENCODING: utf-8
env VIRTUAL_ENV: /home/ubuntu/src/review.opendev.org/openstack/charm-hacluster/.tox/func-target
env PYTHONHASHSEED: 0
env CHARM_DIR: /home/ubuntu/src/review.opendev.org/openstack/charm-hacluster/.tox/func-target
env TOX_ENV_NAME: func-target
env TOX_WORK_DIR: /home/ubuntu/src/review.opendev.org/openstack/charm-hacluster/.tox
env TOX_ENV_DIR: /home/ubuntu/src/review.opendev.org/openstack/charm-hacluster/.tox/func-target
metadata pid: 22747
cwd: /home/ubuntu/src/review.opendev.org/openstack/charm-hacluster
allow: /home/ubuntu/src/review.opendev.org/openstack/charm-hacluster/.tox/func-target/bin/*:charmcraft:/home/ubuntu/src/review.opendev.org/openstack/charm-hacluster/rename.sh
cmd: functest-run-suite --keep-model --bundle jammy-yoga
exit_code: 0

standard error:
2024-01-11 23:02:32 [INFO] Could not determine correct model aliasfrom tests.yaml, using default
2024-01-11 23:02:42 [INFO] Deploying bundle '/home/ubuntu/src/review.opendev.org/openstack/charm-hacluster/tests/bundles/jammy-yoga.yaml' on to 'zaza-1aeeac3c0667' model
2024-01-11 23:02:43 [INFO] Rendered template '<Template 'local-charm-overlay.yaml.j2'>' to file '/tmp/tmp7_kpcfh3/local-charm-overlay.yaml'
2024-01-11 23:02:44 [INFO] Deploying overlay '/tmp/tmp7_kpcfh3/local-charm-overlay.yaml' on to 'zaza-1aeeac3c0667' model
2024-01-11 23:02:46 [INFO] Located charm "keystone" in charm-hub, channel yoga/stable
2024-01-11 23:02:47 [INFO] Located charm "mysql-router" in charm-hub, channel latest/edge
2024-01-11 23:02:47 [INFO] Located charm "mysql-innodb-cluster" in charm-hub, channel latest/edge
2024-01-11 23:02:50 [INFO] Executing changes:
2024-01-11 23:02:50 [INFO] - upload charm keystone from charm-hub for series jammy from channel yoga/edge with architecture=amd64
2024-01-11 23:02:51 [INFO] - deploy application keystone from charm-hub on jammy with yoga/edge
2024-01-11 23:02:52 [INFO] added resource policyd-override
2024-01-11 23:02:52 [INFO] - upload charm /home/ubuntu/src/review.opendev.org/openstack/charm-hacluster/hacluster.charm for series jammy with architecture=amd64
2024-01-11 23:02:52 [INFO] - deploy application keystone-hacluster on jammy
2024-01-11 23:02:53 [INFO] - upload charm mysql-router from charm-hub for series jammy from channel latest/edge with architecture=amd64
2024-01-11 23:02:55 [INFO] - deploy application keystone-mysql-router from charm-hub on jammy with latest/edge using mysql-router
2024-01-11 23:02:56 [INFO] - upload charm mysql-innodb-cluster from charm-hub for series jammy from channel latest/edge with architecture=amd64
2024-01-11 23:02:58 [INFO] - deploy application mysql-innodb-cluster from charm-hub on jammy with latest/edge
2024-01-11 23:02:59 [INFO] added resource mysql-shell
2024-01-11 23:02:59 [INFO] added resource mysqld-exporter
2024-01-11 23:02:59 [INFO] - add new machine 0
2024-01-11 23:02:59 [INFO] - add new machine 1
2024-01-11 23:02:59 [INFO] - add new machine 2
2024-01-11 23:02:59 [INFO] - add new machine 3
2024-01-11 23:02:59 [INFO] - add new machine 4
2024-01-11 23:02:59 [INFO] - add new machine 5
2024-01-11 23:02:59 [INFO] - add relation keystone:shared-db - keystone-mysql-router:shared-db
2024-01-11 23:03:00 [INFO] - add relation keystone-hacluster:ha - keystone:ha
2024-01-11 23:03:00 [INFO] - add relation keystone-mysql-router:db-router - mysql-innodb-cluster:db-router
2024-01-11 23:03:00 [INFO] - add unit keystone/0 to new machine 3
2024-01-11 23:03:00 [INFO] - add unit keystone/1 to new machine 4
2024-01-11 23:03:00 [INFO] - add unit keystone/2 to new machine 5
2024-01-11 23:03:00 [INFO] - add unit mysql-innodb-cluster/0 to new machine 0
2024-01-11 23:03:01 [INFO] - add unit mysql-innodb-cluster/1 to new machine 1
2024-01-11 23:03:01 [INFO] - add unit mysql-innodb-cluster/2 to new machine 2
2024-01-11 23:03:01 [INFO] Deploy of bundle completed.
2024-01-11 23:03:01 [INFO] Waiting for environment to settle
2024-01-11 23:03:01 [INFO] Timeout for deployment to settle set to: 7200
2024-01-11 23:03:01 [INFO] Maximum resolve count for deployment set to: 3
2024-01-11 23:03:01 [INFO] Waiting for application states to reach targeted states.
2024-01-11 23:03:01 [INFO] Waiting for an application to be present
2024-01-11 23:03:01 [INFO] Now checking workload status and status messages
2024-01-11 23:08:38 [INFO] Application mysql-innodb-cluster is ready.
2024-01-11 23:13:01 [INFO] Applications left: keystone, keystone-mysql-router, keystone-hacluster
2024-01-11 23:19:42 [INFO] Application keystone-hacluster is ready.
2024-01-11 23:20:24 [INFO] Application keystone-mysql-router is ready.
2024-01-11 23:22:26 [INFO] Application keystone is ready.
2024-01-11 23:22:26 [INFO] All applications reached approved status, number of units (where relevant), and workload status message checks.
2024-01-11 23:22:26 [INFO] Waiting for zaza-1aeeac3c0667 to settle
2024-01-11 23:22:26 [INFO] Model zaza-1aeeac3c0667 has settled
2024-01-11 23:22:26 [INFO] configuring zaza-1aeeac3c0667
2024-01-11 23:22:35 [INFO] looking at application: {'name': 'keystone', 'type': {'pkg': 'keystone', 'origin_setting': 'openstack-origin'}}
2024-01-11 23:22:47 [INFO] looking at application: {'name': 'keystone', 'type': {'pkg': 'keystone', 'origin_setting': 'openstack-origin'}}
2024-01-11 23:22:59 [INFO] looking at application: {'name': 'keystone', 'type': {'pkg': 'keystone', 'origin_setting': 'openstack-origin'}}
2024-01-11 23:23:04 [INFO] looking at application: {'name': 'keystone', 'type': {'pkg': 'keystone', 'origin_setting': 'openstack-origin'}}
2024-01-11 23:23:05 [INFO] Using keystone API V3 (or later) for overcloud auth
scp: /usr/local/share/ca-certificates/keystone_juju_ca_cert.crt: No such file or directory
2024-01-11 23:23:38 [INFO] Finished configuring zaza-1aeeac3c0667
2024-01-11 23:23:38 [INFO] Testing zaza-1aeeac3c0667
2024-01-11 23:23:38 [INFO] ## Running Test zaza.openstack.charm_tests.hacluster.tests.HaclusterTest ##
2024-01-11 23:23:39 [WARNING] Could not find application under test
2024-01-11 23:23:43 [INFO] looking at application: {'name': 'keystone', 'type': {'pkg': 'keystone', 'origin_setting': 'openstack-origin'}}
2024-01-11 23:23:44 [INFO] Using keystone API V3 (or later) for overcloud auth
scp: /usr/local/share/ca-certificates/keystone_juju_ca_cert.crt: No such file or directory
2024-01-11 23:23:44 [INFO] test_900_action_cleanup (zaza.openstack.charm_tests.hacluster.tests.HaclusterTest)
2024-01-11 23:23:44 [INFO] The services can be cleaned up.
2024-01-11 23:23:44 [INFO]  ... 
2024-01-11 23:23:45 [INFO] ok
2024-01-11 23:23:45 [INFO] test_910_pause_and_resume (zaza.openstack.charm_tests.hacluster.tests.HaclusterTest)
2024-01-11 23:23:45 [INFO] The services can be paused and resumed.
2024-01-11 23:23:45 [INFO]  ... 
2024-01-11 23:23:50 [INFO] Testing pause resume
2024-01-11 23:23:54 [INFO] ok
2024-01-11 23:23:54 [INFO] test_920_put_in_maintenance (zaza.openstack.charm_tests.hacluster.tests.HaclusterTest)
2024-01-11 23:23:54 [INFO] Put pacemaker in maintenance mode.
2024-01-11 23:23:54 [INFO]  ... 
2024-01-11 23:23:54 [INFO] Setting config to {'maintenance-mode': 'true'}
2024-01-11 23:23:54 [INFO] Waiting for application states to reach targeted states.
2024-01-11 23:23:54 [INFO] Waiting for an application to be present
2024-01-11 23:23:54 [WARNING] DEPRECATION: Application keystone-hacluster uses 'workload-status-message'; please use 'workload-status-message-prefix' instead.
2024-01-11 23:23:54 [INFO] Now checking workload status and status messages
2024-01-11 23:23:54 [INFO] Application keystone is ready.
2024-01-11 23:23:54 [INFO] Application keystone-mysql-router is ready.
2024-01-11 23:23:54 [INFO] Application mysql-innodb-cluster is ready.
2024-01-11 23:24:14 [INFO] Application keystone-hacluster is ready.
2024-01-11 23:24:14 [INFO] All applications reached approved status, number of units (where relevant), and workload status message checks.
2024-01-11 23:24:14 [INFO] Setting config to {'maintenance-mode': 'false'}
2024-01-11 23:24:14 [INFO] Waiting for application states to reach targeted states.
2024-01-11 23:24:14 [INFO] Waiting for an application to be present
2024-01-11 23:24:14 [WARNING] DEPRECATION: Application keystone-hacluster uses 'workload-status-message'; please use 'workload-status-message-prefix' instead.
2024-01-11 23:24:14 [INFO] Now checking workload status and status messages
2024-01-11 23:24:15 [INFO] Application keystone-mysql-router is ready.
2024-01-11 23:24:15 [INFO] Application mysql-innodb-cluster is ready.
2024-01-11 23:24:19 [INFO] Application keystone is ready.
2024-01-11 23:27:02 [INFO] Application keystone-hacluster is ready.
2024-01-11 23:27:02 [INFO] All applications reached approved status, number of units (where relevant), and workload status message checks.
2024-01-11 23:27:02 [INFO] ok
2024-01-11 23:27:02 [INFO] ----------------------------------------------------------------------
2024-01-11 23:27:02 [INFO] Ran 3 tests in 203.765s
2024-01-11 23:27:02 [INFO] OK
2024-01-11 23:27:02 [INFO] ## Running Test zaza.openstack.charm_tests.hacluster.tests.HaclusterScaleBackAndForthTest ##
2024-01-11 23:27:03 [WARNING] Could not find application under test
2024-01-11 23:27:03 [INFO] looking at application: {'name': 'keystone', 'type': {'pkg': 'keystone', 'origin_setting': 'openstack-origin'}}
2024-01-11 23:27:04 [INFO] Using keystone API V3 (or later) for overcloud auth
scp: /usr/local/share/ca-certificates/keystone_juju_ca_cert.crt: No such file or directory
2024-01-11 23:27:05 [INFO] test_930_scaleback (zaza.openstack.charm_tests.hacluster.tests.HaclusterScaleBackAndForthTest)
2024-01-11 23:27:05 [INFO] Remove one unit, recalculate quorum and re-add one unit.
2024-01-11 23:27:05 [INFO]  ... 
2024-01-11 23:27:14 [INFO] Pausing unit keystone-hacluster/2
2024-01-11 23:27:21 [INFO] Removing keystone/1
2024-01-11 23:48:54 [INFO] Waiting for model to settle
2024-01-11 23:48:58 [INFO] Checking that corosync considers at least one node to be offline
2024-01-11 23:48:58 [INFO] Updating corosync ring
2024-01-11 23:49:02 [INFO] Checking that corosync considers all nodes to be online
2024-01-11 23:49:03 [INFO] Re-adding an hacluster unit
2024-01-11 23:49:04 [INFO] Waiting for model to settle
2024-01-12 00:05:28 [INFO] Updating corosync ring - workaround for lp:1874719
2024-01-12 00:05:29 [INFO] Checking that corosync considers all nodes to be online
2024-01-12 00:05:30 [INFO] ok
2024-01-12 00:05:30 [INFO] ----------------------------------------------------------------------
2024-01-12 00:05:30 [INFO] Ran 1 test in 2307.911s
2024-01-12 00:05:30 [INFO] OK
2024-01-12 00:05:30 [INFO] Finished testing zaza-1aeeac3c0667
2024-01-12 00:05:30 [INFO] Events:
  Configure zaza.openstack.charm_tests.keystone.setup.add_demo_user:
    Start: 1705015346.601599
    Finish: 1705015418.7070355
    Elapsed Time: 72.10543656349182
    PCT Of Run Time: 2
  Deploy Bundle:
    Start: 1705014162.3291306
    Finish: 1705014181.3879497
    Elapsed Time: 19.05881905555725
    PCT Of Run Time: 1
  Prepare Environment:
    Start: 1705014152.8663752
    Finish: 1705014162.3286111
    Elapsed Time: 9.462235927581787
    PCT Of Run Time: 1
  Test zaza.openstack.charm_tests.hacluster.tests.HaclusterScaleBackAndForthTest:
    Start: 1705015622.4749086
    Finish: 1705017930.386602
    Elapsed Time: 2307.9116933345795
    PCT Of Run Time: 62
  Test zaza.openstack.charm_tests.hacluster.tests.HaclusterTest:
    Start: 1705015418.707727
    Finish: 1705015622.4747803
    Elapsed Time: 203.7670533657074
    PCT Of Run Time: 6
  Wait for Deployment:
    Start: 1705014181.3879683
    Finish: 1705015346.600172
    Elapsed Time: 1165.2122037410736
    PCT Of Run Time: 31
Metadata: {}