name: func-target run_id: commands[0] env OS_TEST_HTTP_PROXY: http://squid.internal:3128/ env TEST_NAME_SERVER: 10.245.160.2 env TEST_VIP_RANGE: 172.16.187.230:172.16.187.250 env TEST_NETAPP_SAN_USERNAME: vsadmin env OS_PASSWORD: aiv4thu8Iewo env TEST_NET_ID: 94a87d6a-6d82-4931-878d-aedcf0e62962 env TEST_ZAZA_BUG_LP1987332: 1 env OS_PURESTORAGE_API_TOKEN: f8173d96-c387-bed8-23fa-2056038890f9 env TEST_IRONIC_DEPLOY_INITRD: http://10.245.161.162/swift/v1/images/ironic-python-agent.initramfs env TEST_NETAPP_POOL_NAME_SEARCH_PATTERN: zaza env HOME: /home/ubuntu env TEST_CIDR_EXT: 172.16.0.0/16 env TEST_VIP: 172.16.187.230 env OS_PROJECT_DOMAIN_NAME: user env OS_PURESTORAGE_SAN_IP: 10.246.112.11 env OS_AUTH_VERSION: 3 env TEST_NETAPP_SAN_IP: 10.245.168.123 env OS_IDENTITY_API_VERSION: 3 env TEST_GRAFANA_PLUGIN_PIECHART_URL: http://10.245.161.162:80/swift/v1/grafana_plugins/grafana-piechart-panel-1.6.2.zip env OS_REGION_NAME: serverstack env TEST_ARISTA_IMAGE_REMOTE: http://10.245.161.162/swift/v1/images/arista-cvx-virt-test.qcow2 env TEST_FIP_START: 172.16.187.200 env TEST_HTTP_PROXY: http://squid.internal:3128/ env TEST_CIDR_END: 172.16.187.254 env TEST_DEPLOY_TIMEOUT: 7200 env OS_AUTH_URL: http://10.245.161.156:5000/v3 env TEST_NETAPP_SAN_PASSWORD: Ubuntu123 env TEST_NVIDIA_VGPU_HOST_SW: http://10.245.161.162/swift/v1/images/nvidia-vgpu-ubuntu-510_510.47.03_amd64.deb env OS_PROJECT_NAME: zuul-tests env OS_USERNAME: zuul-tests env OS_VIP00: 172.16.187.230 env TEST_NETAPP_STORAGE_TYPE: ontap_cluster env LANG: C.UTF-8 env OS_VIP01: 172.16.187.231 env OS_VIP10: 172.16.187.240 env TEST_FIP_RANGE: 172.16.187.200:172.16.187.229 env TEST_REGISTRY_PREFIX: 10.245.164.190:5000 env OS_VIP02: 172.16.187.232 env OS_VIP11: 172.16.187.241 env OS_VIP20: 172.16.187.250 env TEST_SWIFT_IP: 10.245.161.162 env OS_VIP03: 172.16.187.233 env OS_VIP12: 172.16.187.242 env OS_VIP04: 172.16.187.234 env OS_VIP13: 172.16.187.243 env TEST_GATEWAY: 172.16.0.1 env TEST_VIP00: 172.16.187.230 env OS_VIP05: 172.16.187.235 env OS_VIP14: 172.16.187.244 env TEST_GRAFANA_PLUGIN_VONAGE_URL: http://10.245.161.162:80/swift/v1/grafana_plugins/vonage-status-panel-1.0.11.zip env TEST_TRILIO_LICENSE: /tmp/tvault-license.lic env TEST_VIP01: 172.16.187.231 env TEST_VIP10: 172.16.187.240 env OS_USER_DOMAIN_NAME: user env OS_VIP06: 172.16.187.236 env OS_VIP15: 172.16.187.245 env TEST_NETAPP_SAN_VSERVER: svm01 env TEST_VIP02: 172.16.187.232 env TEST_VIP11: 172.16.187.241 env TEST_VIP20: 172.16.187.250 env OS_VIP07: 172.16.187.237 env OS_VIP16: 172.16.187.246 env TEST_FIP_END: 172.16.187.229 env TEST_IRONIC_RAW_BM_IMAGE: http://10.245.161.162/swift/v1/images/baremetal-ubuntu-focal.img env TEST_VIP03: 172.16.187.233 env TEST_VIP12: 172.16.187.242 env OS_VIP08: 172.16.187.238 env OS_VIP17: 172.16.187.247 env TEST_NETAPP_SAN_PORT: 80 env TEST_VIP04: 172.16.187.234 env TEST_VIP13: 172.16.187.243 env OS_VIP09: 172.16.187.239 env OS_VIP18: 172.16.187.248 env TEST_VIP05: 172.16.187.235 env TEST_VIP14: 172.16.187.244 env OS_VIP19: 172.16.187.249 env TEST_NETAPP_ROOT_VOL_AGGR_NAME: openstack env TEST_VIP06: 172.16.187.236 env TEST_VIP15: 172.16.187.245 env TEST_IRONIC_DEPLOY_VMLINUZ: http://10.245.161.162/swift/v1/images/ironic-python-agent.kernel env TEST_VIP07: 172.16.187.237 env TEST_VIP16: 172.16.187.246 env TEST_MAX_RESOLVE_COUNT: 3 env TEST_VIP08: 172.16.187.238 env TEST_VIP17: 172.16.187.247 env TEST_VIP09: 172.16.187.239 env TEST_VIP18: 172.16.187.248 env TEST_MODEL_CONSTRAINTS: cores=2 env TEST_VIP19: 172.16.187.249 env PATH: /home/ubuntu/src/review.opendev.org/openstack/charm-ceph-osd/.tox/func-target/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin env PIP_DISABLE_PIP_VERSION_CHECK: 1 env PYTHONIOENCODING: utf-8 env VIRTUAL_ENV: /home/ubuntu/src/review.opendev.org/openstack/charm-ceph-osd/.tox/func-target env PYTHONHASHSEED: 0 env CHARM_DIR: /home/ubuntu/src/review.opendev.org/openstack/charm-ceph-osd/.tox/func-target env TOX_ENV_NAME: func-target env TOX_WORK_DIR: /home/ubuntu/src/review.opendev.org/openstack/charm-ceph-osd/.tox env TOX_ENV_DIR: /home/ubuntu/src/review.opendev.org/openstack/charm-ceph-osd/.tox/func-target metadata pid: 42170 cwd: /home/ubuntu/src/review.opendev.org/openstack/charm-ceph-osd allow: /home/ubuntu/src/review.opendev.org/openstack/charm-ceph-osd/.tox/func-target/bin/*:charmcraft:/home/ubuntu/src/review.opendev.org/openstack/charm-ceph-osd/rename.sh cmd: functest-run-suite --keep-model --bundle jammy-bobcat exit_code: 0 standard error: 2024-03-22 17:42:43 [INFO] Could not determine correct model aliasfrom tests.yaml, using default 2024-03-22 17:42:52 [INFO] Deploying bundle '/home/ubuntu/src/review.opendev.org/openstack/charm-ceph-osd/tests/bundles/jammy-bobcat.yaml' on to 'zaza-282a57c8c3a0' model 2024-03-22 17:42:56 [INFO] Located charm "ceph-mon" in charm-hub, channel latest/edge 2024-03-22 17:42:57 [INFO] Located charm "ceph-osd" in charm-hub, channel latest/edge 2024-03-22 17:42:58 [INFO] Located charm "cinder" in charm-hub, channel latest/edge 2024-03-22 17:42:59 [INFO] Located charm "cinder-ceph" in charm-hub, channel latest/edge 2024-03-22 17:43:00 [INFO] Located charm "mysql-router" in charm-hub, channel 8.0/edge 2024-03-22 17:43:01 [INFO] Located charm "glance" in charm-hub, channel latest/edge 2024-03-22 17:43:01 [INFO] Located charm "mysql-router" in charm-hub, channel 8.0/edge 2024-03-22 17:43:03 [INFO] Located charm "keystone" in charm-hub, channel latest/edge 2024-03-22 17:43:03 [INFO] Located charm "mysql-router" in charm-hub, channel 8.0/edge 2024-03-22 17:43:04 [INFO] Located charm "mysql-innodb-cluster" in charm-hub, channel 8.0/edge 2024-03-22 17:43:05 [INFO] Located charm "nova-cloud-controller" in charm-hub, channel latest/edge 2024-03-22 17:43:06 [INFO] Located charm "mysql-router" in charm-hub, channel 8.0/edge 2024-03-22 17:43:07 [INFO] Located charm "nova-compute" in charm-hub, channel latest/edge 2024-03-22 17:43:08 [INFO] Located charm "placement" in charm-hub, channel latest/edge 2024-03-22 17:43:09 [INFO] Located charm "mysql-router" in charm-hub, channel 8.0/edge 2024-03-22 17:43:09 [INFO] Located charm "rabbitmq-server" in charm-hub, channel 3.9/stable 2024-03-22 17:43:26 [INFO] Executing changes: 2024-03-22 17:43:26 [INFO] - upload charm ceph-mon from charm-hub for series jammy from channel latest/edge with architecture=amd64 2024-03-22 17:43:28 [INFO] - deploy application ceph-mon from charm-hub on jammy with latest/edge 2024-03-22 17:43:29 [INFO] added resource alert-rules 2024-03-22 17:43:29 [INFO] - upload charm ceph-osd from charm-hub for series jammy from channel latest/edge with architecture=amd64 2024-03-22 17:43:30 [INFO] - deploy application ceph-osd from charm-hub on jammy with latest/edge 2024-03-22 17:43:31 [INFO] - upload charm cinder from charm-hub for series jammy from channel latest/edge with architecture=amd64 2024-03-22 17:43:33 [INFO] - deploy application cinder from charm-hub on jammy with latest/edge 2024-03-22 17:43:34 [INFO] added resource policyd-override 2024-03-22 17:43:34 [INFO] - expose all endpoints of cinder and allow access from CIDRs 0.0.0.0/0 and ::/0 2024-03-22 17:43:34 [INFO] - upload charm cinder-ceph from charm-hub for series jammy from channel latest/edge with architecture=amd64 2024-03-22 17:43:36 [INFO] - deploy application cinder-ceph from charm-hub on jammy with latest/edge 2024-03-22 17:43:36 [INFO] - upload charm mysql-router from charm-hub for series jammy from channel 8.0/edge with architecture=amd64 2024-03-22 17:43:38 [INFO] - deploy application cinder-mysql-router from charm-hub on jammy with 8.0/edge using mysql-router 2024-03-22 17:43:38 [INFO] - upload charm glance from charm-hub for series jammy from channel latest/edge with architecture=amd64 2024-03-22 17:43:40 [INFO] - deploy application glance from charm-hub on jammy with latest/edge 2024-03-22 17:43:41 [INFO] added resource policyd-override 2024-03-22 17:43:41 [INFO] - expose all endpoints of glance and allow access from CIDRs 0.0.0.0/0 and ::/0 2024-03-22 17:43:41 [INFO] - deploy application glance-mysql-router from charm-hub on jammy with 8.0/edge using mysql-router 2024-03-22 17:43:41 [INFO] - upload charm keystone from charm-hub for series jammy from channel latest/edge with architecture=amd64 2024-03-22 17:43:44 [INFO] - deploy application keystone from charm-hub on jammy with latest/edge 2024-03-22 17:43:46 [INFO] added resource policyd-override 2024-03-22 17:43:46 [INFO] - expose all endpoints of keystone and allow access from CIDRs 0.0.0.0/0 and ::/0 2024-03-22 17:43:46 [INFO] - deploy application keystone-mysql-router from charm-hub on jammy with 8.0/edge using mysql-router 2024-03-22 17:43:47 [INFO] - upload charm mysql-innodb-cluster from charm-hub for series jammy from channel 8.0/edge with architecture=amd64 2024-03-22 17:43:48 [INFO] - deploy application mysql-innodb-cluster from charm-hub on jammy with 8.0/edge 2024-03-22 17:43:49 [INFO] added resource mysql-shell 2024-03-22 17:43:49 [INFO] added resource mysqld-exporter 2024-03-22 17:43:49 [INFO] - upload charm nova-cloud-controller from charm-hub for series jammy from channel latest/edge with architecture=amd64 2024-03-22 17:43:50 [INFO] - deploy application nova-cloud-controller from charm-hub on jammy with latest/edge 2024-03-22 17:43:51 [INFO] added resource policyd-override 2024-03-22 17:43:51 [INFO] - expose all endpoints of nova-cloud-controller and allow access from CIDRs 0.0.0.0/0 and ::/0 2024-03-22 17:43:51 [INFO] - deploy application nova-cloud-controller-mysql-router from charm-hub on jammy with 8.0/edge using mysql-router 2024-03-22 17:43:51 [INFO] - upload charm nova-compute from charm-hub for series jammy from channel latest/edge with architecture=amd64 2024-03-22 17:43:53 [INFO] - deploy application nova-compute from charm-hub on jammy with latest/edge 2024-03-22 17:43:54 [INFO] - upload charm placement from charm-hub for series jammy from channel latest/edge with architecture=amd64 2024-03-22 17:43:58 [INFO] - deploy application placement from charm-hub on jammy with latest/edge 2024-03-22 17:43:58 [INFO] - deploy application placement-mysql-router from charm-hub on jammy with 8.0/edge using mysql-router 2024-03-22 17:43:58 [INFO] - upload charm rabbitmq-server from charm-hub for series jammy from channel 3.9/edge with architecture=amd64 2024-03-22 17:44:00 [INFO] - deploy application rabbitmq-server from charm-hub on jammy with 3.9/edge 2024-03-22 17:44:00 [INFO] - add new machine 0 2024-03-22 17:44:00 [INFO] - add new machine 1 2024-03-22 17:44:00 [INFO] - add new machine 2 2024-03-22 17:44:01 [INFO] - add new machine 3 2024-03-22 17:44:01 [INFO] - add new machine 4 2024-03-22 17:44:01 [INFO] - add new machine 5 2024-03-22 17:44:01 [INFO] - add new machine 6 2024-03-22 17:44:01 [INFO] - add new machine 7 2024-03-22 17:44:01 [INFO] - add new machine 8 2024-03-22 17:44:01 [INFO] - add new machine 9 2024-03-22 17:44:02 [INFO] - add new machine 10 2024-03-22 17:44:02 [INFO] - add new machine 11 2024-03-22 17:44:02 [INFO] - add new machine 12 2024-03-22 17:44:02 [INFO] - add new machine 13 2024-03-22 17:44:02 [INFO] - add new machine 14 2024-03-22 17:44:02 [INFO] - add new machine 15 2024-03-22 17:44:03 [INFO] - add relation nova-compute:amqp - rabbitmq-server:amqp 2024-03-22 17:44:03 [INFO] - add relation nova-compute:image-service - glance:image-service 2024-03-22 17:44:03 [INFO] - add relation nova-compute:ceph - ceph-mon:client 2024-03-22 17:44:03 [INFO] - add relation keystone:shared-db - keystone-mysql-router:shared-db 2024-03-22 17:44:03 [INFO] - add relation keystone-mysql-router:db-router - mysql-innodb-cluster:db-router 2024-03-22 17:44:03 [INFO] - add relation glance:shared-db - glance-mysql-router:shared-db 2024-03-22 17:44:03 [INFO] - add relation glance-mysql-router:db-router - mysql-innodb-cluster:db-router 2024-03-22 17:44:03 [INFO] - add relation glance:identity-service - keystone:identity-service 2024-03-22 17:44:04 [INFO] - add relation glance:amqp - rabbitmq-server:amqp 2024-03-22 17:44:04 [INFO] - add relation glance:ceph - ceph-mon:client 2024-03-22 17:44:04 [INFO] - add relation cinder:shared-db - cinder-mysql-router:shared-db 2024-03-22 17:44:04 [INFO] - add relation cinder-mysql-router:db-router - mysql-innodb-cluster:db-router 2024-03-22 17:44:04 [INFO] - add relation cinder:identity-service - keystone:identity-service 2024-03-22 17:44:04 [INFO] - add relation cinder:amqp - rabbitmq-server:amqp 2024-03-22 17:44:04 [INFO] - add relation cinder:image-service - glance:image-service 2024-03-22 17:44:04 [INFO] - add relation cinder-ceph:storage-backend - cinder:storage-backend 2024-03-22 17:44:04 [INFO] - add relation cinder-ceph:ceph - ceph-mon:client 2024-03-22 17:44:05 [INFO] - add relation ceph-osd:mon - ceph-mon:osd 2024-03-22 17:44:05 [INFO] - add relation nova-cloud-controller:shared-db - nova-cloud-controller-mysql-router:shared-db 2024-03-22 17:44:05 [INFO] - add relation nova-cloud-controller-mysql-router:db-router - mysql-innodb-cluster:db-router 2024-03-22 17:44:05 [INFO] - add relation nova-cloud-controller:identity-service - keystone:identity-service 2024-03-22 17:44:05 [INFO] - add relation nova-cloud-controller:amqp - rabbitmq-server:amqp 2024-03-22 17:44:05 [INFO] - add relation nova-cloud-controller:cloud-compute - nova-compute:cloud-compute 2024-03-22 17:44:05 [INFO] - add relation nova-cloud-controller:image-service - glance:image-service 2024-03-22 17:44:05 [INFO] - add relation placement:shared-db - placement-mysql-router:shared-db 2024-03-22 17:44:05 [INFO] - add relation placement-mysql-router:db-router - mysql-innodb-cluster:db-router 2024-03-22 17:44:05 [INFO] - add relation placement - keystone 2024-03-22 17:44:06 [INFO] - add relation placement - nova-cloud-controller 2024-03-22 17:44:06 [INFO] - add relation cinder-ceph:ceph-access - nova-compute:ceph-access 2024-03-22 17:44:06 [INFO] - add unit ceph-mon/0 to new machine 6 2024-03-22 17:44:06 [INFO] - add unit ceph-mon/1 to new machine 7 2024-03-22 17:44:06 [INFO] - add unit ceph-mon/2 to new machine 8 2024-03-22 17:44:06 [INFO] - add unit ceph-osd/0 to new machine 3 2024-03-22 17:44:08 [INFO] - add unit ceph-osd/1 to new machine 4 2024-03-22 17:44:09 [INFO] - add unit ceph-osd/2 to new machine 5 2024-03-22 17:44:10 [INFO] - add unit cinder/0 to new machine 13 2024-03-22 17:44:10 [INFO] - add unit glance/0 to new machine 12 2024-03-22 17:44:10 [INFO] - add unit keystone/0 to new machine 10 2024-03-22 17:44:10 [INFO] - add unit mysql-innodb-cluster/0 to new machine 0 2024-03-22 17:44:10 [INFO] - add unit mysql-innodb-cluster/1 to new machine 1 2024-03-22 17:44:10 [INFO] - add unit mysql-innodb-cluster/2 to new machine 2 2024-03-22 17:44:11 [INFO] - add unit nova-cloud-controller/0 to new machine 14 2024-03-22 17:44:11 [INFO] - add unit nova-compute/0 to new machine 11 2024-03-22 17:44:11 [INFO] - add unit placement/0 to new machine 15 2024-03-22 17:44:11 [INFO] - add unit rabbitmq-server/0 to new machine 9 2024-03-22 17:44:11 [INFO] Deploy of bundle completed. 2024-03-22 17:44:11 [INFO] Waiting for environment to settle 2024-03-22 17:44:11 [INFO] Timeout for deployment to settle set to: 7200 2024-03-22 17:44:11 [INFO] Maximum resolve count for deployment set to: 3 2024-03-22 17:44:11 [INFO] Waiting for application states to reach targeted states. 2024-03-22 17:44:12 [INFO] Waiting for an application to be present 2024-03-22 17:44:12 [INFO] Now checking workload status and status messages 2024-03-22 17:49:07 [INFO] Application rabbitmq-server is ready. 2024-03-22 17:49:57 [INFO] Application mysql-innodb-cluster is ready. 2024-03-22 17:51:47 [INFO] Application ceph-mon is ready. 2024-03-22 17:52:50 [INFO] Application ceph-osd is ready. 2024-03-22 17:54:12 [INFO] Applications left: placement, keystone-mysql-router, glance, cinder-ceph, glance-mysql-router, nova-compute, cinder, keystone, nova-cloud-controller-mysql-router, placement-mysql-router, nova-cloud-controller, cinder-mysql-router 2024-03-22 18:00:51 [INFO] Application nova-compute is ready. 2024-03-22 18:04:12 [INFO] Applications left: placement, keystone-mysql-router, glance, cinder-ceph, glance-mysql-router, cinder, keystone, nova-cloud-controller-mysql-router, placement-mysql-router, nova-cloud-controller, cinder-mysql-router 2024-03-22 18:04:36 [INFO] Application glance-mysql-router is ready. 2024-03-22 18:05:55 [INFO] Application placement-mysql-router is ready. 2024-03-22 18:07:42 [INFO] Application keystone-mysql-router is ready. 2024-03-22 18:08:12 [INFO] Application glance is ready. 2024-03-22 18:09:09 [INFO] Application cinder-mysql-router is ready. 2024-03-22 18:09:11 [INFO] Application cinder-ceph is ready. 2024-03-22 18:10:34 [INFO] Application cinder is ready. 2024-03-22 18:11:03 [INFO] Application keystone is ready. 2024-03-22 18:11:27 [INFO] Application placement is ready. 2024-03-22 18:12:27 [INFO] Application nova-cloud-controller-mysql-router is ready. 2024-03-22 18:14:13 [INFO] Applications left: nova-cloud-controller 2024-03-22 18:19:51 [INFO] Application nova-cloud-controller is ready. 2024-03-22 18:19:51 [INFO] All applications reached approved status, number of units (where relevant), and workload status message checks. 2024-03-22 18:19:51 [INFO] Waiting for zaza-282a57c8c3a0 to settle 2024-03-22 18:19:51 [INFO] Model zaza-282a57c8c3a0 has settled 2024-03-22 18:19:51 [INFO] configuring zaza-282a57c8c3a0 2024-03-22 18:19:51 [INFO] Finished configuring zaza-282a57c8c3a0 2024-03-22 18:19:51 [INFO] Testing zaza-282a57c8c3a0 2024-03-22 18:19:51 [INFO] ## Running Test zaza.openstack.charm_tests.ceph.tests.CephLowLevelTest ## 2024-03-22 18:19:53 [INFO] test_pg_tuning (zaza.openstack.charm_tests.ceph.tests.CephLowLevelTest) 2024-03-22 18:19:53 [INFO] ... 2024-03-22 18:19:55 [INFO] ok 2024-03-22 18:19:55 [INFO] test_processes (zaza.openstack.charm_tests.ceph.tests.CephLowLevelTest) 2024-03-22 18:19:55 [INFO] Verify Ceph processes. 2024-03-22 18:19:55 [INFO] ... 2024-03-22 18:19:55 [INFO] Checking ceph-mon and ceph-osd processes... 2024-03-22 18:19:58 [INFO] ok 2024-03-22 18:19:58 [INFO] test_services (zaza.openstack.charm_tests.ceph.tests.CephLowLevelTest) 2024-03-22 18:19:58 [INFO] Verify the ceph services. 2024-03-22 18:19:58 [INFO] ... 2024-03-22 18:19:58 [INFO] Checking ceph-osd and ceph-mon services... 2024-03-22 18:20:00 [INFO] ok 2024-03-22 18:20:00 [INFO] ---------------------------------------------------------------------- 2024-03-22 18:20:00 [INFO] Ran 3 tests in 8.351s 2024-03-22 18:20:00 [INFO] OK 2024-03-22 18:20:00 [INFO] ## Running Test zaza.openstack.charm_tests.ceph.tests.CephTest ## 2024-03-22 18:20:02 [INFO] test_blacklist (zaza.openstack.charm_tests.ceph.tests.CephTest) 2024-03-22 18:20:02 [INFO] Check the blacklist action. 2024-03-22 18:20:02 [INFO] ... 2024-03-22 18:20:02 [INFO] Checking blacklist-add-disk and blacklist-remove-disk actions... 2024-03-22 18:20:19 [INFO] ok 2024-03-22 18:20:19 [INFO] test_blocked_when_non_pristine_disk_appears (zaza.openstack.charm_tests.ceph.tests.CephTest) 2024-03-22 18:20:19 [INFO] Test blocked state with non-pristine disk. 2024-03-22 18:20:19 [INFO] ... 2024-03-22 18:20:30 [INFO] looking at application: {'name': 'ceph-mon', 'type': {'pkg': 'ceph-common', 'origin_setting': 'source'}} 2024-03-22 18:20:34 [WARNING] Skipping pristine disk test for focal and higher 2024-03-22 18:20:34 [INFO] ok 2024-03-22 18:20:34 [INFO] test_cache_device (zaza.openstack.charm_tests.ceph.tests.CephTest) 2024-03-22 18:20:34 [INFO] Test replacing a disk in use. 2024-03-22 18:20:34 [INFO] ... 2024-03-22 18:20:34 [INFO] Running add-disk action with a caching device 2024-03-22 18:20:34 [INFO] Removing single disk from each OSD 2024-03-22 18:20:59 [INFO] Removing device /dev/vdc from unit ceph-osd/0 2024-03-22 18:21:22 [INFO] Removing device /dev/vdc from unit ceph-osd/1 2024-03-22 18:21:40 [INFO] Removing device /dev/vdc from unit ceph-osd/2 2024-03-22 18:21:40 [INFO] Waiting for application states to reach targeted states. 2024-03-22 18:21:40 [INFO] Waiting for an application to be present 2024-03-22 18:21:40 [INFO] Now checking workload status and status messages 2024-03-22 18:21:40 [INFO] Application glance is ready. 2024-03-22 18:21:40 [INFO] Application ceph-osd is ready. 2024-03-22 18:21:40 [INFO] Application nova-compute is ready. 2024-03-22 18:21:40 [INFO] Application cinder is ready. 2024-03-22 18:21:40 [INFO] Application rabbitmq-server is ready. 2024-03-22 18:21:40 [INFO] Application keystone is ready. 2024-03-22 18:21:41 [INFO] Application nova-cloud-controller-mysql-router is ready. 2024-03-22 18:21:41 [INFO] Application mysql-innodb-cluster is ready. 2024-03-22 18:21:41 [INFO] Application nova-cloud-controller is ready. 2024-03-22 18:21:41 [INFO] Application placement is ready. 2024-03-22 18:21:41 [INFO] Application keystone-mysql-router is ready. 2024-03-22 18:21:41 [INFO] Application cinder-ceph is ready. 2024-03-22 18:21:41 [INFO] Application placement-mysql-router is ready. 2024-03-22 18:21:41 [INFO] Application cinder-mysql-router is ready. 2024-03-22 18:21:41 [INFO] Application glance-mysql-router is ready. 2024-03-22 18:21:51 [INFO] Application ceph-mon is ready. 2024-03-22 18:21:51 [INFO] All applications reached approved status, number of units (where relevant), and workload status message checks. 2024-03-22 18:21:51 [INFO] Recycling previously removed disks 2024-03-22 18:21:51 [INFO] Found device /dev/vdc on unit ceph-osd/0 2024-03-22 18:22:02 [INFO] Found device /dev/vdc on unit ceph-osd/1 2024-03-22 18:22:12 [INFO] Found device /dev/vdc on unit ceph-osd/2 2024-03-22 18:22:21 [INFO] Waiting for application states to reach targeted states. 2024-03-22 18:22:21 [INFO] Waiting for an application to be present 2024-03-22 18:22:21 [INFO] Now checking workload status and status messages 2024-03-22 18:22:22 [INFO] Application glance is ready. 2024-03-22 18:22:22 [INFO] Application nova-compute is ready. 2024-03-22 18:22:22 [INFO] Application cinder is ready. 2024-03-22 18:22:22 [INFO] Application rabbitmq-server is ready. 2024-03-22 18:22:22 [INFO] Application keystone is ready. 2024-03-22 18:22:22 [INFO] Application nova-cloud-controller-mysql-router is ready. 2024-03-22 18:22:22 [INFO] Application mysql-innodb-cluster is ready. 2024-03-22 18:22:22 [INFO] Application nova-cloud-controller is ready. 2024-03-22 18:22:22 [INFO] Application placement is ready. 2024-03-22 18:22:22 [INFO] Application keystone-mysql-router is ready. 2024-03-22 18:22:22 [INFO] Application cinder-ceph is ready. 2024-03-22 18:22:22 [INFO] Application placement-mysql-router is ready. 2024-03-22 18:22:22 [INFO] Application cinder-mysql-router is ready. 2024-03-22 18:22:22 [INFO] Application glance-mysql-router is ready. 2024-03-22 18:22:24 [INFO] Application ceph-osd is ready. 2024-03-22 18:22:41 [INFO] Application ceph-mon is ready. 2024-03-22 18:22:41 [INFO] All applications reached approved status, number of units (where relevant), and workload status message checks. 2024-03-22 18:22:41 [INFO] Removing previously added OSDs 2024-03-22 18:22:41 [INFO] Removing block device /dev/vdc from unit ceph-osd/0 2024-03-22 18:23:06 [INFO] Removing block device /dev/vdc from unit ceph-osd/1 2024-03-22 18:23:18 [INFO] Removing block device /dev/vdc from unit ceph-osd/2 2024-03-22 18:23:41 [INFO] Waiting for application states to reach targeted states. 2024-03-22 18:23:41 [INFO] Waiting for an application to be present 2024-03-22 18:23:41 [INFO] Now checking workload status and status messages 2024-03-22 18:23:42 [INFO] Application glance is ready. 2024-03-22 18:23:42 [INFO] Application ceph-osd is ready. 2024-03-22 18:23:42 [INFO] Application nova-compute is ready. 2024-03-22 18:23:42 [INFO] Application cinder is ready. 2024-03-22 18:23:42 [INFO] Application rabbitmq-server is ready. 2024-03-22 18:23:42 [INFO] Application keystone is ready. 2024-03-22 18:23:42 [INFO] Application nova-cloud-controller-mysql-router is ready. 2024-03-22 18:23:42 [INFO] Application mysql-innodb-cluster is ready. 2024-03-22 18:23:42 [INFO] Application nova-cloud-controller is ready. 2024-03-22 18:23:42 [INFO] Application placement is ready. 2024-03-22 18:23:42 [INFO] Application keystone-mysql-router is ready. 2024-03-22 18:23:42 [INFO] Application cinder-ceph is ready. 2024-03-22 18:23:42 [INFO] Application placement-mysql-router is ready. 2024-03-22 18:23:42 [INFO] Application cinder-mysql-router is ready. 2024-03-22 18:23:42 [INFO] Application glance-mysql-router is ready. 2024-03-22 18:23:54 [INFO] Application ceph-mon is ready. 2024-03-22 18:23:54 [INFO] All applications reached approved status, number of units (where relevant), and workload status message checks. 2024-03-22 18:23:54 [INFO] Finally adding back OSDs 2024-03-22 18:24:21 [INFO] Waiting for application states to reach targeted states. 2024-03-22 18:24:21 [INFO] Waiting for an application to be present 2024-03-22 18:24:21 [INFO] Now checking workload status and status messages 2024-03-22 18:24:22 [INFO] Application glance is ready. 2024-03-22 18:24:22 [INFO] Application nova-compute is ready. 2024-03-22 18:24:22 [INFO] Application cinder is ready. 2024-03-22 18:24:22 [INFO] Application rabbitmq-server is ready. 2024-03-22 18:24:22 [INFO] Application keystone is ready. 2024-03-22 18:24:22 [INFO] Application nova-cloud-controller-mysql-router is ready. 2024-03-22 18:24:22 [INFO] Application mysql-innodb-cluster is ready. 2024-03-22 18:24:22 [INFO] Application nova-cloud-controller is ready. 2024-03-22 18:24:22 [INFO] Application placement is ready. 2024-03-22 18:24:22 [INFO] Application keystone-mysql-router is ready. 2024-03-22 18:24:22 [INFO] Application cinder-ceph is ready. 2024-03-22 18:24:22 [INFO] Application placement-mysql-router is ready. 2024-03-22 18:24:22 [INFO] Application cinder-mysql-router is ready. 2024-03-22 18:24:22 [INFO] Application glance-mysql-router is ready. 2024-03-22 18:24:44 [INFO] Application ceph-mon is ready. 2024-03-22 18:27:26 [INFO] Application ceph-osd is ready. 2024-03-22 18:27:26 [INFO] All applications reached approved status, number of units (where relevant), and workload status message checks. 2024-03-22 18:27:27 [INFO] ok 2024-03-22 18:27:27 [INFO] test_ceph_check_osd_pools (zaza.openstack.charm_tests.ceph.tests.CephTest) 2024-03-22 18:27:27 [INFO] Check OSD pools. 2024-03-22 18:27:27 [INFO] ... 2024-03-22 18:27:27 [INFO] Checking pools on ceph units... 2024-03-22 18:27:35 [INFO] looking at application: {'name': 'keystone', 'type': {'pkg': 'keystone', 'origin_setting': 'openstack-origin'}} 2024-03-22 18:27:38 [INFO] ok 2024-03-22 18:27:38 [INFO] test_ceph_encryption (zaza.openstack.charm_tests.ceph.tests.CephTest) 2024-03-22 18:27:38 [INFO] Test Ceph encryption. 2024-03-22 18:27:38 [INFO] ... 2024-03-22 18:27:47 [INFO] looking at application: {'name': 'ceph-mon', 'type': {'pkg': 'ceph-common', 'origin_setting': 'source'}} 2024-03-22 18:27:49 [WARNING] Skipping encryption test for Mitaka and higher 2024-03-22 18:27:49 [INFO] ok 2024-03-22 18:27:49 [INFO] test_ceph_pool_creation_with_text_file (zaza.openstack.charm_tests.ceph.tests.CephTest) 2024-03-22 18:27:49 [INFO] Check the creation of a pool and a text file. 2024-03-22 18:27:49 [INFO] ... 2024-03-22 18:27:52 [INFO] ok 2024-03-22 18:27:52 [INFO] test_list_disks (zaza.openstack.charm_tests.ceph.tests.CephTest) 2024-03-22 18:27:52 [INFO] Test the list-disks action. 2024-03-22 18:27:52 [INFO] ... 2024-03-22 18:27:52 [INFO] Checking list-disks action... 2024-03-22 18:27:56 [INFO] ok 2024-03-22 18:27:56 [INFO] test_pause_and_resume (zaza.openstack.charm_tests.ceph.tests.CephTest) 2024-03-22 18:27:56 [INFO] The services can be paused and resumed. 2024-03-22 18:27:56 [INFO] ... 2024-03-22 18:27:56 [INFO] Checking pause and resume actions... 2024-03-22 18:27:56 [INFO] ok 2024-03-22 18:27:56 [INFO] ---------------------------------------------------------------------- 2024-03-22 18:27:56 [INFO] Ran 8 tests in 475.682s 2024-03-22 18:27:56 [INFO] OK 2024-03-22 18:27:56 [INFO] ## Running Test zaza.openstack.charm_tests.ceph.osd.tests.SecurityTest ## 2024-03-22 18:27:56 [INFO] test_osd_security_checklist (zaza.openstack.charm_tests.ceph.osd.tests.SecurityTest) 2024-03-22 18:27:56 [INFO] Verify expected state with security-checklist. 2024-03-22 18:27:56 [INFO] ... 2024-03-22 18:27:56 [INFO] Running `security-checklist` action on Ceph OSD leader unit 2024-03-22 18:27:57 [INFO] ok 2024-03-22 18:27:57 [INFO] ---------------------------------------------------------------------- 2024-03-22 18:27:57 [INFO] Ran 1 test in 1.177s 2024-03-22 18:27:57 [INFO] OK 2024-03-22 18:27:57 [INFO] ## Running Test zaza.openstack.charm_tests.ceph.osd.tests.ServiceTest ## 2024-03-22 18:27:57 [INFO] test_active_after_pristine_block (zaza.openstack.charm_tests.ceph.osd.tests.ServiceTest) 2024-03-22 18:27:57 [INFO] Test if we can get back to active state after pristine block. 2024-03-22 18:27:57 [INFO] ... 2024-03-22 18:27:58 [INFO] Setting Non-pristine status 2024-03-22 18:27:58 [INFO] Waiting for application states to reach targeted states. 2024-03-22 18:27:59 [INFO] Waiting for an application to be present 2024-03-22 18:27:59 [INFO] Now checking workload status and status messages 2024-03-22 18:27:59 [INFO] Application glance is ready. 2024-03-22 18:27:59 [INFO] Application ceph-mon is ready. 2024-03-22 18:27:59 [INFO] Application ceph-osd is ready. 2024-03-22 18:27:59 [INFO] Application nova-compute is ready. 2024-03-22 18:27:59 [INFO] Application cinder is ready. 2024-03-22 18:27:59 [INFO] Application rabbitmq-server is ready. 2024-03-22 18:27:59 [INFO] Application keystone is ready. 2024-03-22 18:27:59 [INFO] Application nova-cloud-controller-mysql-router is ready. 2024-03-22 18:27:59 [INFO] Application mysql-innodb-cluster is ready. 2024-03-22 18:27:59 [INFO] Application nova-cloud-controller is ready. 2024-03-22 18:27:59 [INFO] Application placement is ready. 2024-03-22 18:27:59 [INFO] Application keystone-mysql-router is ready. 2024-03-22 18:27:59 [INFO] Application cinder-ceph is ready. 2024-03-22 18:27:59 [INFO] Application placement-mysql-router is ready. 2024-03-22 18:28:00 [INFO] Application cinder-mysql-router is ready. 2024-03-22 18:28:00 [INFO] Application glance-mysql-router is ready. 2024-03-22 18:28:00 [INFO] All applications reached approved status, number of units (where relevant), and workload status message checks. 2024-03-22 18:28:00 [INFO] Running update-status action 2024-03-22 18:28:02 [INFO] Wait for idle/ready status 2024-03-22 18:28:02 [INFO] Waiting for application states to reach targeted states. 2024-03-22 18:28:02 [INFO] Waiting for an application to be present 2024-03-22 18:28:02 [INFO] Now checking workload status and status messages 2024-03-22 18:28:03 [INFO] Application glance is ready. 2024-03-22 18:28:03 [INFO] Application ceph-mon is ready. 2024-03-22 18:28:03 [INFO] Application ceph-osd is ready. 2024-03-22 18:28:03 [INFO] Application nova-compute is ready. 2024-03-22 18:28:03 [INFO] Application cinder is ready. 2024-03-22 18:28:03 [INFO] Application rabbitmq-server is ready. 2024-03-22 18:28:03 [INFO] Application keystone is ready. 2024-03-22 18:28:03 [INFO] Application nova-cloud-controller-mysql-router is ready. 2024-03-22 18:28:03 [INFO] Application mysql-innodb-cluster is ready. 2024-03-22 18:28:03 [INFO] Application nova-cloud-controller is ready. 2024-03-22 18:28:03 [INFO] Application placement is ready. 2024-03-22 18:28:03 [INFO] Application keystone-mysql-router is ready. 2024-03-22 18:28:03 [INFO] Application cinder-ceph is ready. 2024-03-22 18:28:03 [INFO] Application placement-mysql-router is ready. 2024-03-22 18:28:03 [INFO] Application cinder-mysql-router is ready. 2024-03-22 18:28:03 [INFO] Application glance-mysql-router is ready. 2024-03-22 18:28:03 [INFO] All applications reached approved status, number of units (where relevant), and workload status message checks. 2024-03-22 18:28:05 [INFO] ok 2024-03-22 18:28:05 [INFO] test_start_specific (zaza.openstack.charm_tests.ceph.osd.tests.ServiceTest) 2024-03-22 18:28:05 [INFO] Start only specified ceph-osd service. 2024-03-22 18:28:05 [INFO] ... 2024-03-22 18:28:06 [INFO] Stopping all running ceph-osd services 2024-03-22 18:28:08 [WARNING] run_in_model is deprecated. 2024-03-22 18:28:09 [INFO] Running 'service start=6 on ceph-osd/0 unit 2024-03-22 18:28:14 [INFO] ok 2024-03-22 18:28:14 [INFO] test_start_stop_all_by_keyword (zaza.openstack.charm_tests.ceph.osd.tests.ServiceTest) 2024-03-22 18:28:14 [INFO] Start and Stop all ceph-osd services using keyword 'all'. 2024-03-22 18:28:14 [INFO] ... 2024-03-22 18:28:15 [INFO] Running 'service stop=all' action on ceph-osd/0 unit 2024-03-22 18:28:18 [INFO] Running 'service start=all' action on ceph-osd/0 unit 2024-03-22 18:28:23 [INFO] ok 2024-03-22 18:28:23 [INFO] test_start_stop_all_by_list (zaza.openstack.charm_tests.ceph.osd.tests.ServiceTest) 2024-03-22 18:28:23 [INFO] Start and Stop all ceph-osd services using explicit list. 2024-03-22 18:28:23 [INFO] ... 2024-03-22 18:28:27 [INFO] Running 'service stop=0,6' action on ceph-osd/0 unit 2024-03-22 18:28:31 [INFO] Running 'service start=0,6' action on ceph-osd/0 unit 2024-03-22 18:28:37 [INFO] ok 2024-03-22 18:28:37 [INFO] test_stop_specific (zaza.openstack.charm_tests.ceph.osd.tests.ServiceTest) 2024-03-22 18:28:37 [INFO] Stop only specified ceph-osd service. 2024-03-22 18:28:37 [INFO] ... 2024-03-22 18:28:38 [INFO] Running 'service stop=6 on ceph-osd/0 unit 2024-03-22 18:28:42 [INFO] ok 2024-03-22 18:28:42 [INFO] ---------------------------------------------------------------------- 2024-03-22 18:28:42 [INFO] Ran 5 tests in 45.180s 2024-03-22 18:28:42 [INFO] OK 2024-03-22 18:28:42 [INFO] ## Running Test zaza.charm_tests.lifecycle.tests.UpgradeCharmsToPath;ceph-osd ## 2024-03-22 18:28:42 [INFO] Performing a charm upgrade on: ['ceph-osd'] Added local charm "ceph-osd", revision 0, to the model Leaving endpoints in "alpha": cluster, mon, nrpe-external-master, public, secrets-storage 2024-03-22 18:28:44 [INFO] ## Running Test zaza.openstack.charm_tests.ceph.tests.CephLowLevelTest ## 2024-03-22 18:28:46 [INFO] test_pg_tuning (zaza.openstack.charm_tests.ceph.tests.CephLowLevelTest) 2024-03-22 18:28:46 [INFO] ... 2024-03-22 18:28:49 [INFO] ok 2024-03-22 18:28:49 [INFO] test_processes (zaza.openstack.charm_tests.ceph.tests.CephLowLevelTest) 2024-03-22 18:28:49 [INFO] Verify Ceph processes. 2024-03-22 18:28:49 [INFO] ... 2024-03-22 18:28:49 [INFO] Checking ceph-mon and ceph-osd processes... 2024-03-22 18:28:56 [INFO] ok 2024-03-22 18:28:56 [INFO] test_services (zaza.openstack.charm_tests.ceph.tests.CephLowLevelTest) 2024-03-22 18:28:56 [INFO] Verify the ceph services. 2024-03-22 18:28:56 [INFO] ... 2024-03-22 18:28:56 [INFO] Checking ceph-osd and ceph-mon services... 2024-03-22 18:28:59 [INFO] ok 2024-03-22 18:28:59 [INFO] ---------------------------------------------------------------------- 2024-03-22 18:28:59 [INFO] Ran 3 tests in 15.010s 2024-03-22 18:28:59 [INFO] OK 2024-03-22 18:28:59 [INFO] ## Running Test zaza.openstack.charm_tests.ceph.tests.CephTest ## 2024-03-22 18:29:01 [INFO] test_blacklist (zaza.openstack.charm_tests.ceph.tests.CephTest) 2024-03-22 18:29:01 [INFO] Check the blacklist action. 2024-03-22 18:29:01 [INFO] ... 2024-03-22 18:29:01 [INFO] Checking blacklist-add-disk and blacklist-remove-disk actions... 2024-03-22 18:29:17 [INFO] ok 2024-03-22 18:29:17 [INFO] test_blocked_when_non_pristine_disk_appears (zaza.openstack.charm_tests.ceph.tests.CephTest) 2024-03-22 18:29:17 [INFO] Test blocked state with non-pristine disk. 2024-03-22 18:29:17 [INFO] ... 2024-03-22 18:29:30 [INFO] looking at application: {'name': 'ceph-mon', 'type': {'pkg': 'ceph-common', 'origin_setting': 'source'}} 2024-03-22 18:29:32 [WARNING] Skipping pristine disk test for focal and higher 2024-03-22 18:29:32 [INFO] ok 2024-03-22 18:29:32 [INFO] test_cache_device (zaza.openstack.charm_tests.ceph.tests.CephTest) 2024-03-22 18:29:32 [INFO] Test replacing a disk in use. 2024-03-22 18:29:32 [INFO] ... 2024-03-22 18:29:32 [INFO] Running add-disk action with a caching device 2024-03-22 18:29:32 [INFO] Removing single disk from each OSD 2024-03-22 18:29:48 [INFO] Removing device /dev/vdc from unit ceph-osd/0 2024-03-22 18:30:03 [INFO] Removing device /dev/vdc from unit ceph-osd/1 2024-03-22 18:30:17 [INFO] Removing device /dev/vdc from unit ceph-osd/2 2024-03-22 18:30:17 [INFO] Waiting for application states to reach targeted states. 2024-03-22 18:30:17 [INFO] Waiting for an application to be present 2024-03-22 18:30:17 [INFO] Now checking workload status and status messages 2024-03-22 18:30:18 [INFO] Application glance is ready. 2024-03-22 18:30:18 [INFO] Application ceph-osd is ready. 2024-03-22 18:30:18 [INFO] Application nova-compute is ready. 2024-03-22 18:30:18 [INFO] Application cinder is ready. 2024-03-22 18:30:18 [INFO] Application rabbitmq-server is ready. 2024-03-22 18:30:18 [INFO] Application keystone is ready. 2024-03-22 18:30:18 [INFO] Application nova-cloud-controller-mysql-router is ready. 2024-03-22 18:30:18 [INFO] Application mysql-innodb-cluster is ready. 2024-03-22 18:30:18 [INFO] Application nova-cloud-controller is ready. 2024-03-22 18:30:18 [INFO] Application placement is ready. 2024-03-22 18:30:18 [INFO] Application keystone-mysql-router is ready. 2024-03-22 18:30:18 [INFO] Application cinder-ceph is ready. 2024-03-22 18:30:18 [INFO] Application placement-mysql-router is ready. 2024-03-22 18:30:18 [INFO] Application cinder-mysql-router is ready. 2024-03-22 18:30:18 [INFO] Application glance-mysql-router is ready. 2024-03-22 18:30:29 [INFO] Application ceph-mon is ready. 2024-03-22 18:30:29 [INFO] All applications reached approved status, number of units (where relevant), and workload status message checks. 2024-03-22 18:30:29 [INFO] Recycling previously removed disks 2024-03-22 18:30:29 [INFO] Found device /dev/vdc on unit ceph-osd/0 2024-03-22 18:30:40 [INFO] Found device /dev/vdc on unit ceph-osd/1 2024-03-22 18:30:48 [INFO] Found device /dev/vdc on unit ceph-osd/2 2024-03-22 18:30:58 [INFO] Waiting for application states to reach targeted states. 2024-03-22 18:30:58 [INFO] Waiting for an application to be present 2024-03-22 18:30:58 [INFO] Now checking workload status and status messages 2024-03-22 18:30:58 [INFO] Application glance is ready. 2024-03-22 18:30:58 [INFO] Application nova-compute is ready. 2024-03-22 18:30:58 [INFO] Application cinder is ready. 2024-03-22 18:30:58 [INFO] Application rabbitmq-server is ready. 2024-03-22 18:30:58 [INFO] Application keystone is ready. 2024-03-22 18:30:58 [INFO] Application nova-cloud-controller-mysql-router is ready. 2024-03-22 18:30:58 [INFO] Application mysql-innodb-cluster is ready. 2024-03-22 18:30:58 [INFO] Application nova-cloud-controller is ready. 2024-03-22 18:30:59 [INFO] Application placement is ready. 2024-03-22 18:30:59 [INFO] Application keystone-mysql-router is ready. 2024-03-22 18:30:59 [INFO] Application cinder-ceph is ready. 2024-03-22 18:30:59 [INFO] Application placement-mysql-router is ready. 2024-03-22 18:30:59 [INFO] Application cinder-mysql-router is ready. 2024-03-22 18:30:59 [INFO] Application glance-mysql-router is ready. 2024-03-22 18:31:20 [INFO] Application ceph-mon is ready. 2024-03-22 18:33:37 [INFO] Application ceph-osd is ready. 2024-03-22 18:33:37 [INFO] All applications reached approved status, number of units (where relevant), and workload status message checks. 2024-03-22 18:33:37 [INFO] Removing previously added OSDs 2024-03-22 18:33:37 [INFO] Removing block device /dev/vdc from unit ceph-osd/0 2024-03-22 18:33:54 [INFO] Removing block device /dev/vdc from unit ceph-osd/1 2024-03-22 18:34:07 [INFO] Removing block device /dev/vdc from unit ceph-osd/2 2024-03-22 18:34:21 [INFO] Waiting for application states to reach targeted states. 2024-03-22 18:34:21 [INFO] Waiting for an application to be present 2024-03-22 18:34:21 [INFO] Now checking workload status and status messages 2024-03-22 18:34:22 [INFO] Application glance is ready. 2024-03-22 18:34:22 [INFO] Application ceph-osd is ready. 2024-03-22 18:34:22 [INFO] Application nova-compute is ready. 2024-03-22 18:34:22 [INFO] Application cinder is ready. 2024-03-22 18:34:22 [INFO] Application rabbitmq-server is ready. 2024-03-22 18:34:22 [INFO] Application keystone is ready. 2024-03-22 18:34:22 [INFO] Application nova-cloud-controller-mysql-router is ready. 2024-03-22 18:34:22 [INFO] Application mysql-innodb-cluster is ready. 2024-03-22 18:34:22 [INFO] Application nova-cloud-controller is ready. 2024-03-22 18:34:22 [INFO] Application placement is ready. 2024-03-22 18:34:22 [INFO] Application keystone-mysql-router is ready. 2024-03-22 18:34:22 [INFO] Application cinder-ceph is ready. 2024-03-22 18:34:22 [INFO] Application placement-mysql-router is ready. 2024-03-22 18:34:22 [INFO] Application cinder-mysql-router is ready. 2024-03-22 18:34:22 [INFO] Application glance-mysql-router is ready. 2024-03-22 18:34:34 [INFO] Application ceph-mon is ready. 2024-03-22 18:34:34 [INFO] All applications reached approved status, number of units (where relevant), and workload status message checks. 2024-03-22 18:34:34 [INFO] Finally adding back OSDs 2024-03-22 18:35:01 [INFO] Waiting for application states to reach targeted states. 2024-03-22 18:35:01 [INFO] Waiting for an application to be present 2024-03-22 18:35:01 [INFO] Now checking workload status and status messages 2024-03-22 18:35:02 [INFO] Application glance is ready. 2024-03-22 18:35:02 [INFO] Application nova-compute is ready. 2024-03-22 18:35:02 [INFO] Application cinder is ready. 2024-03-22 18:35:02 [INFO] Application rabbitmq-server is ready. 2024-03-22 18:35:02 [INFO] Application nova-cloud-controller-mysql-router is ready. 2024-03-22 18:35:02 [INFO] Application mysql-innodb-cluster is ready. 2024-03-22 18:35:02 [INFO] Application nova-cloud-controller is ready. 2024-03-22 18:35:02 [INFO] Application placement is ready. 2024-03-22 18:35:02 [INFO] Application keystone-mysql-router is ready. 2024-03-22 18:35:02 [INFO] Application cinder-ceph is ready. 2024-03-22 18:35:02 [INFO] Application placement-mysql-router is ready. 2024-03-22 18:35:02 [INFO] Application cinder-mysql-router is ready. 2024-03-22 18:35:02 [INFO] Application glance-mysql-router is ready. 2024-03-22 18:35:03 [INFO] Application keystone is ready. 2024-03-22 18:35:23 [INFO] Application ceph-mon is ready. 2024-03-22 18:38:55 [INFO] Application ceph-osd is ready. 2024-03-22 18:38:55 [INFO] All applications reached approved status, number of units (where relevant), and workload status message checks. 2024-03-22 18:38:56 [INFO] ok 2024-03-22 18:38:56 [INFO] test_ceph_check_osd_pools (zaza.openstack.charm_tests.ceph.tests.CephTest) 2024-03-22 18:38:56 [INFO] Check OSD pools. 2024-03-22 18:38:56 [INFO] ... 2024-03-22 18:38:56 [INFO] Checking pools on ceph units... 2024-03-22 18:39:05 [INFO] looking at application: {'name': 'keystone', 'type': {'pkg': 'keystone', 'origin_setting': 'openstack-origin'}} 2024-03-22 18:39:06 [INFO] ok 2024-03-22 18:39:06 [INFO] test_ceph_encryption (zaza.openstack.charm_tests.ceph.tests.CephTest) 2024-03-22 18:39:06 [INFO] Test Ceph encryption. 2024-03-22 18:39:06 [INFO] ... 2024-03-22 18:39:17 [INFO] looking at application: {'name': 'ceph-mon', 'type': {'pkg': 'ceph-common', 'origin_setting': 'source'}} 2024-03-22 18:39:19 [WARNING] Skipping encryption test for Mitaka and higher 2024-03-22 18:39:19 [INFO] ok 2024-03-22 18:39:19 [INFO] test_ceph_pool_creation_with_text_file (zaza.openstack.charm_tests.ceph.tests.CephTest) 2024-03-22 18:39:19 [INFO] Check the creation of a pool and a text file. 2024-03-22 18:39:19 [INFO] ... 2024-03-22 18:39:21 [INFO] ok 2024-03-22 18:39:21 [INFO] test_list_disks (zaza.openstack.charm_tests.ceph.tests.CephTest) 2024-03-22 18:39:21 [INFO] Test the list-disks action. 2024-03-22 18:39:21 [INFO] ... 2024-03-22 18:39:21 [INFO] Checking list-disks action... 2024-03-22 18:39:25 [INFO] ok 2024-03-22 18:39:25 [INFO] test_pause_and_resume (zaza.openstack.charm_tests.ceph.tests.CephTest) 2024-03-22 18:39:25 [INFO] The services can be paused and resumed. 2024-03-22 18:39:25 [INFO] ... 2024-03-22 18:39:25 [INFO] Checking pause and resume actions... 2024-03-22 18:39:25 [INFO] ok 2024-03-22 18:39:25 [INFO] ---------------------------------------------------------------------- 2024-03-22 18:39:25 [INFO] Ran 8 tests in 625.379s 2024-03-22 18:39:25 [INFO] OK 2024-03-22 18:39:25 [INFO] ## Running Test zaza.openstack.charm_tests.ceph.osd.tests.SecurityTest ## 2024-03-22 18:39:25 [INFO] test_osd_security_checklist (zaza.openstack.charm_tests.ceph.osd.tests.SecurityTest) 2024-03-22 18:39:25 [INFO] Verify expected state with security-checklist. 2024-03-22 18:39:25 [INFO] ... 2024-03-22 18:39:25 [INFO] Running `security-checklist` action on Ceph OSD leader unit 2024-03-22 18:39:26 [INFO] ok 2024-03-22 18:39:26 [INFO] ---------------------------------------------------------------------- 2024-03-22 18:39:26 [INFO] Ran 1 test in 1.392s 2024-03-22 18:39:26 [INFO] OK 2024-03-22 18:39:26 [INFO] ## Running Test zaza.openstack.charm_tests.ceph.osd.tests.ServiceTest ## 2024-03-22 18:39:26 [INFO] test_active_after_pristine_block (zaza.openstack.charm_tests.ceph.osd.tests.ServiceTest) 2024-03-22 18:39:26 [INFO] Test if we can get back to active state after pristine block. 2024-03-22 18:39:26 [INFO] ... 2024-03-22 18:39:27 [INFO] Setting Non-pristine status 2024-03-22 18:39:27 [INFO] Waiting for application states to reach targeted states. 2024-03-22 18:39:27 [INFO] Waiting for an application to be present 2024-03-22 18:39:27 [INFO] Now checking workload status and status messages 2024-03-22 18:39:28 [INFO] Application glance is ready. 2024-03-22 18:39:28 [INFO] Application ceph-mon is ready. 2024-03-22 18:39:28 [INFO] Application ceph-osd is ready. 2024-03-22 18:39:28 [INFO] Application nova-compute is ready. 2024-03-22 18:39:28 [INFO] Application cinder is ready. 2024-03-22 18:39:28 [INFO] Application rabbitmq-server is ready. 2024-03-22 18:39:28 [INFO] Application keystone is ready. 2024-03-22 18:39:28 [INFO] Application nova-cloud-controller-mysql-router is ready. 2024-03-22 18:39:28 [INFO] Application mysql-innodb-cluster is ready. 2024-03-22 18:39:28 [INFO] Application nova-cloud-controller is ready. 2024-03-22 18:39:28 [INFO] Application placement is ready. 2024-03-22 18:39:28 [INFO] Application keystone-mysql-router is ready. 2024-03-22 18:39:28 [INFO] Application cinder-ceph is ready. 2024-03-22 18:39:28 [INFO] Application placement-mysql-router is ready. 2024-03-22 18:39:28 [INFO] Application cinder-mysql-router is ready. 2024-03-22 18:39:28 [INFO] Application glance-mysql-router is ready. 2024-03-22 18:39:28 [INFO] All applications reached approved status, number of units (where relevant), and workload status message checks. 2024-03-22 18:39:28 [INFO] Running update-status action 2024-03-22 18:39:31 [INFO] Wait for idle/ready status 2024-03-22 18:39:31 [INFO] Waiting for application states to reach targeted states. 2024-03-22 18:39:31 [INFO] Waiting for an application to be present 2024-03-22 18:39:31 [INFO] Now checking workload status and status messages 2024-03-22 18:39:31 [INFO] Application glance is ready. 2024-03-22 18:39:31 [INFO] Application ceph-mon is ready. 2024-03-22 18:39:32 [INFO] Application ceph-osd is ready. 2024-03-22 18:39:32 [INFO] Application nova-compute is ready. 2024-03-22 18:39:32 [INFO] Application cinder is ready. 2024-03-22 18:39:32 [INFO] Application rabbitmq-server is ready. 2024-03-22 18:39:32 [INFO] Application keystone is ready. 2024-03-22 18:39:32 [INFO] Application nova-cloud-controller-mysql-router is ready. 2024-03-22 18:39:32 [INFO] Application mysql-innodb-cluster is ready. 2024-03-22 18:39:32 [INFO] Application nova-cloud-controller is ready. 2024-03-22 18:39:32 [INFO] Application placement is ready. 2024-03-22 18:39:32 [INFO] Application keystone-mysql-router is ready. 2024-03-22 18:39:32 [INFO] Application cinder-ceph is ready. 2024-03-22 18:39:32 [INFO] Application placement-mysql-router is ready. 2024-03-22 18:39:32 [INFO] Application cinder-mysql-router is ready. 2024-03-22 18:39:32 [INFO] Application glance-mysql-router is ready. 2024-03-22 18:39:32 [INFO] All applications reached approved status, number of units (where relevant), and workload status message checks. 2024-03-22 18:39:34 [INFO] ok 2024-03-22 18:39:34 [INFO] test_start_specific (zaza.openstack.charm_tests.ceph.osd.tests.ServiceTest) 2024-03-22 18:39:34 [INFO] Start only specified ceph-osd service. 2024-03-22 18:39:34 [INFO] ... 2024-03-22 18:39:35 [INFO] Stopping all running ceph-osd services 2024-03-22 18:39:38 [INFO] Running 'service start=9 on ceph-osd/0 unit 2024-03-22 18:39:43 [INFO] ok 2024-03-22 18:39:43 [INFO] test_start_stop_all_by_keyword (zaza.openstack.charm_tests.ceph.osd.tests.ServiceTest) 2024-03-22 18:39:43 [INFO] Start and Stop all ceph-osd services using keyword 'all'. 2024-03-22 18:39:43 [INFO] ... 2024-03-22 18:39:44 [INFO] Running 'service stop=all' action on ceph-osd/0 unit 2024-03-22 18:39:48 [INFO] Running 'service start=all' action on ceph-osd/0 unit 2024-03-22 18:39:52 [INFO] ok 2024-03-22 18:39:52 [INFO] test_start_stop_all_by_list (zaza.openstack.charm_tests.ceph.osd.tests.ServiceTest) 2024-03-22 18:39:52 [INFO] Start and Stop all ceph-osd services using explicit list. 2024-03-22 18:39:52 [INFO] ... 2024-03-22 18:39:53 [INFO] Running 'service stop=0,9' action on ceph-osd/0 unit 2024-03-22 18:39:57 [INFO] Running 'service start=0,9' action on ceph-osd/0 unit 2024-03-22 18:40:02 [INFO] ok 2024-03-22 18:40:02 [INFO] test_stop_specific (zaza.openstack.charm_tests.ceph.osd.tests.ServiceTest) 2024-03-22 18:40:02 [INFO] Stop only specified ceph-osd service. 2024-03-22 18:40:02 [INFO] ... 2024-03-22 18:40:03 [INFO] Running 'service stop=9 on ceph-osd/0 unit 2024-03-22 18:40:08 [INFO] ok 2024-03-22 18:40:08 [INFO] ---------------------------------------------------------------------- 2024-03-22 18:40:08 [INFO] Ran 5 tests in 41.710s 2024-03-22 18:40:08 [INFO] OK 2024-03-22 18:40:08 [INFO] Finished testing zaza-282a57c8c3a0 2024-03-22 18:40:08 [INFO] Events: Deploy Bundle: Start: 1711129372.8679736 Finish: 1711129451.985032 Elapsed Time: 79.1170585155487 PCT Of Run Time: 3 Prepare Environment: Start: 1711129363.5243196 Finish: 1711129372.866862 Elapsed Time: 9.34254240989685 PCT Of Run Time: 1 Test zaza.charm_tests.lifecycle.tests.UpgradeCharmsToPath;ceph-osd: Start: 1711132122.9270134 Finish: 1711132124.834104 Elapsed Time: 1.907090663909912 PCT Of Run Time: 1 Test zaza.openstack.charm_tests.ceph.osd.tests.SecurityTest: Start: 1711132765.2276602 Finish: 1711132766.6215322 Elapsed Time: 1.3938720226287842 PCT Of Run Time: 1 Test zaza.openstack.charm_tests.ceph.osd.tests.ServiceTest: Start: 1711132766.621585 Finish: 1711132808.3328402 Elapsed Time: 41.71125531196594 PCT Of Run Time: 2 Test zaza.openstack.charm_tests.ceph.tests.CephLowLevelTest: Start: 1711132124.8342896 Finish: 1711132139.8467083 Elapsed Time: 15.012418746948242 PCT Of Run Time: 1 Test zaza.openstack.charm_tests.ceph.tests.CephTest: Start: 1711132139.846837 Finish: 1711132765.227529 Elapsed Time: 625.3806920051575 PCT Of Run Time: 19 Wait for Deployment: Start: 1711129451.9851213 Finish: 1711131591.5297105 Elapsed Time: 2139.544589281082 PCT Of Run Time: 63 Metadata: {}