#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import os
import re
import time
import allure

from oslo_log import log as logging

from configs import config
from tests import exceptions
from lib.common.utils import test_utils
from lib import exceptions as lib_exc

CONF = config.CONF
LOG = logging.getLogger(__name__)


def _get_task_state(body):
    return body.get("OS-EXT-STS:task_state", None)


@allure.step("Wait for server status")
# NOTE(afazekas): This function needs to know a token and a subject.
def wait_for_server_status(
    client,
    server_id,
    status,
    ready_wait=True,
    extra_timeout=0,
    raise_on_error=True,
    request_id=None,
):
    """Waits for a server to reach a given status."""

    # NOTE(afazekas): UNKNOWN status possible on ERROR
    # or in a very early stage.
    body = client.show_server(server_id)["server"]
    old_status = server_status = body["status"]
    old_task_state = task_state = _get_task_state(body)
    start_time = int(time.time())
    timeout = client.build_timeout + extra_timeout
    metadata = None
    while True:
        # NOTE(afazekas): Now the BUILD status only reached
        # between the UNKNOWN->ACTIVE transition.
        # TODO(afazekas): enumerate and validate the stable status set
        if status == "BUILD" and server_status != "UNKNOWN":
            return body
        if server_status == status:
            if ready_wait:
                if status == "BUILD":
                    return body
                all_metadata_ok = True
                if CONF.compute.validate_metadata and status == "ACTIVE":
                    # Validate metadata status
                    metadata = body["metadata"]
                    failure_domain_pattern = (
                        r"^([0-9A-Z]{1,4}-)?[0-9A-Z]{1,4}-[0-9A-Z]{1,4}-[0-9A-Z]{1,4}(-[0-9A-Z]{1,4})?$"
                    )
                    keys_to_check = ["cmdb", "ipdb", "ping", "ssh", "dns_lookup", "failure_domain"]
                    # If flava cloud, "key to check" set for flava metadata keys
                    if CONF.common.target_cloud == "flava":
                        keys_to_check = ["dns_lookup", "ping", "topos", "dns_lookup", "failure_domain"]
                    if all(key in metadata for key in keys_to_check):
                        for key in keys_to_check:
                            if metadata.get(key) != "OK":
                                # If "failure_domain" metadata validated format, passed
                                if key == "failure_domain" and re.match(failure_domain_pattern, metadata.get(key)):
                                    continue
                                details = f"Error occurred in the follow metadata: {key} = {metadata.get(key)}"
                                if request_id:
                                    details += " Request ID of server operation performed before"
                                    details += (
                                        " checking the server status %s." % request_id
                                    )
                                raise exceptions.BuildErrorException(
                                    details, server_id=server_id
                                )
                    else:
                        all_metadata_ok = False
                        LOG.debug(
                            "Server status is ACTIVE, but waiting for metadata "
                            "information. %s",
                            metadata,
                        )
                # NOTE(afazekas): The instance is in "ready for action state"
                # when no task in progress
                if all_metadata_ok and task_state is None:
                    # without state api extension 3 sec usually enough
                    time.sleep(CONF.compute.ready_wait)
                    return body
            else:
                return body

        time.sleep(client.build_interval)
        body = client.show_server(server_id)["server"]
        server_status = body["status"]
        task_state = _get_task_state(body)
        if (server_status != old_status) or (task_state != old_task_state):
            LOG.info(
                'State transition "%s" ==> "%s" after %d second wait',
                "/".join((old_status, str(old_task_state))),
                "/".join((server_status, str(task_state))),
                time.time() - start_time,
            )
        if (server_status == "ERROR") and raise_on_error:
            details = ""
            if "fault" in body:
                details += "Fault: %s." % body["fault"]
            if request_id:
                details += " Request ID of server operation performed before"
                details += " checking the server status %s." % request_id
            raise exceptions.BuildErrorException(details, server_id=server_id)

        timed_out = int(time.time()) - start_time >= timeout

        if timed_out:
            expected_task_state = "None" if ready_wait else "n/a"
            message = (
                "Server %(server_id)s failed to reach %(status)s "
                'status and task state "%(expected_task_state)s" '
                "and metadata 'OK' "
                "within the required time (%(timeout)s s)."
                % {
                    "server_id": server_id,
                    "status": status,
                    "expected_task_state": expected_task_state,
                    "timeout": timeout,
                }
            )
            if request_id:
                message += " Request ID of server operation performed before"
                message += " checking the server status %s." % request_id
            message += " Current status: %s." % server_status
            message += " Current task state: %s." % task_state
            if metadata:
                message += " Current metadata: %s." % metadata
            caller = test_utils.find_test_caller()
            if caller:
                message = "(%s) %s" % (caller, message)
            raise lib_exc.TimeoutException(message)
        old_status = server_status
        old_task_state = task_state


@allure.step("Wait for server termination")
def wait_for_server_termination(client, server_id, ignore_error=False):
    """Waits for server to reach termination."""
    try:
        body = client.show_server(server_id)["server"]
    except lib_exc.NotFound:
        return
    old_status = body["status"]
    old_task_state = _get_task_state(body)
    start_time = int(time.time())
    while True:
        time.sleep(client.build_interval)
        try:
            body = client.show_server(server_id)["server"]
        except lib_exc.NotFound:
            return
        server_status = body["status"]
        task_state = _get_task_state(body)
        if (server_status != old_status) or (task_state != old_task_state):
            LOG.info(
                'State transition "%s" ==> "%s" after %d second wait',
                "/".join((old_status, str(old_task_state))),
                "/".join((server_status, str(task_state))),
                time.time() - start_time,
            )
        if server_status == "ERROR" and not ignore_error:
            raise lib_exc.DeleteErrorException(
                "Server %s failed to delete and is in ERROR status" % server_id
            )

        if server_status == "SOFT_DELETED":
            # Soft-deleted instances need to be forcibly deleted to
            # prevent some test cases from failing.
            LOG.debug("Automatically force-deleting soft-deleted server %s", server_id)
            try:
                client.force_delete_server(server_id)
            except lib_exc.NotFound:
                # The instance may have been deleted so ignore
                # NotFound exception
                return

        if int(time.time()) - start_time >= client.build_timeout:
            raise lib_exc.TimeoutException
        old_status = server_status
        old_task_state = task_state


def wait_for_image_status(client, image_id, status):
    """Waits for an image to reach a given status.

    The client should have a show_image(image_id) method to get the image.
    The client should also have build_interval and build_timeout attributes.
    """
    show_image = client.show_image

    current_status = "An unknown status"
    start = int(time.time())
    while int(time.time()) - start < client.build_timeout:
        image = show_image(image_id)
        # Compute image client returns response wrapped in 'image' element
        # which is not the case with Glance image client.
        if "image" in image:
            image = image["image"]

        current_status = image["status"]
        if current_status == status:
            return
        if current_status.lower() == "killed":
            raise exceptions.ImageKilledException(image_id=image_id, status=status)
        if current_status.lower() == "error":
            raise exceptions.AddImageException(image_id=image_id)

        time.sleep(client.build_interval)

    message = (
        "Image %(image_id)s failed to reach %(status)s state "
        "(current state %(current_status)s) within the required "
        "time (%(timeout)s s)."
        % {
            "image_id": image_id,
            "status": status,
            "current_status": current_status,
            "timeout": client.build_timeout,
        }
    )
    caller = test_utils.find_test_caller()
    if caller:
        message = "(%s) %s" % (caller, message)
    raise lib_exc.TimeoutException(message)


def wait_for_image_tasks_status(client, image_id, status):
    """Waits for an image tasks to reach a given status."""
    pending_tasks = []
    start = int(time.time())
    while int(time.time()) - start < client.build_timeout:
        tasks = client.show_image_tasks(image_id)["tasks"]

        pending_tasks = [task for task in tasks if task["status"] != status]
        if not pending_tasks:
            return tasks
        time.sleep(client.build_interval)

    message = (
        "Image %(image_id)s tasks: %(pending_tasks)s "
        "failed to reach %(status)s state within the required "
        "time (%(timeout)s s)."
        % {
            "image_id": image_id,
            "pending_tasks": pending_tasks,
            "status": status,
            "timeout": client.build_timeout,
        }
    )
    caller = test_utils.find_test_caller()
    if caller:
        message = "(%s) %s" % (caller, message)
    raise lib_exc.TimeoutException(message)


def wait_for_tasks_status(client, task_id, status):
    start = int(time.time())
    while int(time.time()) - start < client.build_timeout:
        task = client.show_tasks(task_id)
        if task["status"] == status:
            return task
        time.sleep(client.build_interval)
    message = (
        "Task %(task_id)s tasks: "
        "failed to reach %(status)s state within the required "
        "time (%(timeout)s s)."
        % {"task_id": task_id, "status": status, "timeout": client.build_timeout}
    )
    caller = test_utils.find_test_caller()
    if caller:
        message = "(%s) %s" % (caller, message)
    raise lib_exc.TimeoutException(message)


def wait_for_image_imported_to_stores(client, image_id, stores=None):
    """Waits for an image to be imported to all requested stores.

    Short circuits to fail if the serer reports failure of any store.
    If stores is None, just wait for status==active.

    The client should also have build_interval and build_timeout attributes.
    """

    exc_cls = lib_exc.TimeoutException
    start = int(time.time())

    # NOTE(danms): Don't wait for stores that are read-only as those
    # will never complete
    try:
        store_info = client.info_stores()["stores"]
        stores = ",".join(
            sorted(
                [
                    store["id"]
                    for store in store_info
                    if store.get("read-only") != "true"
                    and (not stores or store["id"] in stores.split(","))
                ]
            )
        )
    except lib_exc.NotFound:
        # If multi-store is not enabled, then we can not resolve which
        # ones are read-only, and stores must have been passed as None
        # anyway for us to succeed. If not, then we should raise right
        # now and avoid waiting since we will never see the stores
        # appear.
        if stores is not None:
            raise lib_exc.TimeoutException(
                "Image service has no store support; "
                "cowardly refusing to wait for them."
            )

    while int(time.time()) - start < client.build_timeout:
        image = client.show_image(image_id)
        if image["status"] == "active" and (
            stores is None or image["stores"] == stores
        ):
            return
        if image.get("os_glance_failed_import"):
            exc_cls = lib_exc.OtherRestClientException
            break

        time.sleep(client.build_interval)

    message = "Image %s failed to import on stores: %s" % (
        image_id,
        str(image.get("os_glance_failed_import")),
    )
    caller = test_utils.find_test_caller()
    if caller:
        message = "(%s) %s" % (caller, message)
    raise exc_cls(message)


def wait_for_image_copied_to_stores(client, image_id):
    """Waits for an image to be copied on all requested stores.

    The client should also have build_interval and build_timeout attributes.
    This return the list of stores where copy is failed.
    """

    start = int(time.time())
    store_left = []
    while int(time.time()) - start < client.build_timeout:
        image = client.show_image(image_id)
        store_left = image.get("os_glance_importing_to_stores")
        # NOTE(danms): If os_glance_importing_to_stores is None, then
        # we've raced with the startup of the task and should continue
        # to wait.
        if store_left is not None and not store_left:
            return image["os_glance_failed_import"]
        if image["status"].lower() == "killed":
            raise exceptions.ImageKilledException(
                image_id=image_id, status=image["status"]
            )

        time.sleep(client.build_interval)

    message = "Image %s failed to finish the copy operation " "on stores: %s" % (
        image_id,
        str(store_left),
    )
    caller = test_utils.find_test_caller()
    if caller:
        message = "(%s) %s" % (caller, message)
    raise lib_exc.TimeoutException(message)


def wait_for_volume_resource_status(
    client, resource_id, status, server_id=None, servers_client=None
):
    """Waits for a volume resource to reach a given status.

    This function is a common function for volume, snapshot and backup
    resources. The function extracts the name of the desired resource from
    the client class name of the resource.

    If server_id and servers_client are provided, dump the console for that
    server on failure.
    """
    resource_name = re.findall(
        r"(volume|group-snapshot|snapshot|backup|group)", client.resource_type
    )[-1].replace("-", "_")
    show_resource = getattr(client, "show_" + resource_name)
    resource_status = show_resource(resource_id)[resource_name]["status"]
    resource_display_name = show_resource(resource_id)[resource_name]["name"]
    start = int(time.time())

    while resource_status != status:
        time.sleep(client.build_interval)
        resource_status = show_resource(resource_id)[resource_name]["status"]

        if resource_status == "error" and resource_status != status:
            raise exceptions.VolumeResourceBuildErrorException(
                resource_name=resource_name, resource_id=resource_id
            )
        if resource_name == "volume" and resource_status == "error_restoring":
            raise exceptions.VolumeRestoreErrorException(volume_id=resource_id)
        if resource_status == "error_extending" and resource_status != status:
            raise exceptions.VolumeExtendErrorException(volume_id=resource_id)

        if int(time.time()) - start >= client.build_timeout:
            if server_id and servers_client:
                console_output = servers_client.get_console_output(server_id)["output"]
                LOG.debug(f"Console output for {server_id}\nbody=\n{console_output}")
            message = (
                f"{resource_display_name}(id={resource_id}) failed to reach %s status (current {status}) "
                f"within the required time ({client.build_timeout} s)."
            )
            raise lib_exc.TimeoutException(message)

    LOG.info(
        f"{resource_display_name}(id={resource_id}) reached {status} after waiting for {time.time() - start} seconds"
    )


def wait_for_volume_attachment_create(client, volume_id, server_id):
    """Waits for a volume attachment to be created at a given volume."""
    start = int(time.time())
    while True:
        attachments = client.show_volume(volume_id)["volume"]["attachments"]
        found = [a for a in attachments if a["server_id"] == server_id]
        if found:
            LOG.info(
                "Attachment %s created for volume %s to server %s after "
                "waiting for %f seconds",
                found[0]["attachment_id"],
                volume_id,
                server_id,
                time.time() - start,
            )
            return found[0]
        time.sleep(client.build_interval)
        if int(time.time()) - start >= client.build_timeout:
            message = (
                "Failed to attach volume %s to server %s "
                "within the required time (%s s)."
                % (volume_id, server_id, client.build_timeout)
            )
            raise lib_exc.TimeoutException(message)


def wait_for_volume_attachment_remove(client, volume_id, attachment_id):
    """Waits for a volume attachment to be removed from a given volume."""
    start = int(time.time())
    attachments = client.show_volume(volume_id)["volume"]["attachments"]
    while any(attachment_id == a["attachment_id"] for a in attachments):
        time.sleep(client.build_interval)
        if int(time.time()) - start >= client.build_timeout:
            message = (
                "Failed to remove attachment %s from volume %s "
                "within the required time (%s s)."
                % (attachment_id, volume_id, client.build_timeout)
            )
            raise lib_exc.TimeoutException(message)
        attachments = client.show_volume(volume_id)["volume"]["attachments"]
    LOG.info(
        "Attachment %s removed from volume %s after waiting for %f " "seconds",
        attachment_id,
        volume_id,
        time.time() - start,
    )


def wait_for_volume_attachment_remove_from_server(client, server_id, volume_id):
    """Waits for a volume to be removed from a given server.

    This waiter checks the compute API if the volume attachment is removed.
    """
    start = int(time.time())

    try:
        volumes = client.list_volume_attachments(server_id)["volumeAttachments"]
    except lib_exc.NotFound:
        # Ignore 404s on detach in case the server is deleted or the volume
        # is already detached.
        return

    while any(volume for volume in volumes if volume["volumeId"] == volume_id):
        time.sleep(client.build_interval)

        timed_out = int(time.time()) - start >= client.build_timeout
        if timed_out:
            console_output = client.get_console_output(server_id)["output"]
            LOG.debug("Console output for %s\nbody=\n%s", server_id, console_output)
            message = (
                "Volume %s failed to detach from server %s within "
                "the required time (%s s) from the compute API "
                "perspective" % (volume_id, server_id, client.build_timeout)
            )
            raise lib_exc.TimeoutException(message)
        try:
            volumes = client.list_volume_attachments(server_id)["volumeAttachments"]
        except lib_exc.NotFound:
            # Ignore 404s on detach in case the server is deleted or the volume
            # is already detached.
            return
    return


def wait_for_volume_migration(client, volume_id, new_host):
    """Waits for a Volume to move to a new host."""
    body = client.show_volume(volume_id)["volume"]
    host = body["os-vol-host-attr:host"]
    migration_status = body["migration_status"]
    start = int(time.time())

    # new_host is hostname@backend while current_host is hostname@backend#type
    while migration_status != "success" or new_host not in host:
        time.sleep(client.build_interval)
        body = client.show_volume(volume_id)["volume"]
        host = body["os-vol-host-attr:host"]
        migration_status = body["migration_status"]

        if migration_status == "error":
            message = "volume %s failed to migrate." % (volume_id)
            raise lib_exc.TempestException(message)

        if int(time.time()) - start >= client.build_timeout:
            message = (
                "Volume %s failed to migrate to %s (current %s) "
                "within the required time (%s s)."
                % (volume_id, new_host, host, client.build_timeout)
            )
            raise lib_exc.TimeoutException(message)


def wait_for_volume_retype(client, volume_id, new_volume_type):
    """Waits for a Volume to have a new volume type."""
    body = client.show_volume(volume_id)["volume"]
    current_volume_type = body["volume_type"]
    start = int(time.time())

    while current_volume_type != new_volume_type:
        time.sleep(client.build_interval)
        body = client.show_volume(volume_id)["volume"]
        current_volume_type = body["volume_type"]

        if int(time.time()) - start >= client.build_timeout:
            message = (
                "Volume %s failed to reach %s volume type (current %s) "
                "within the required time (%s s)."
                % (
                    volume_id,
                    new_volume_type,
                    current_volume_type,
                    client.build_timeout,
                )
            )
            raise lib_exc.TimeoutException(message)


def wait_for_qos_operations(client, qos_id, operation, args=None):
    """Waits for a qos operations to be completed.

    NOTE : operation value is required for  wait_for_qos_operations()
    operation = 'qos-key' / 'disassociate' / 'disassociate-all'
    args = keys[] when operation = 'qos-key'
    args = volume-type-id disassociated when operation = 'disassociate'
    args = None when operation = 'disassociate-all'
    """
    start_time = int(time.time())
    while True:
        if operation == "qos-key-unset":
            body = client.show_qos(qos_id)["qos_specs"]
            if not any(key in body["specs"] for key in args):
                return
        elif operation == "disassociate":
            body = client.show_association_qos(qos_id)["qos_associations"]
            if not any(args in body[i]["id"] for i in range(0, len(body))):
                return
        elif operation == "disassociate-all":
            body = client.show_association_qos(qos_id)["qos_associations"]
            if not body:
                return
        else:
            msg = " operation value is either not defined or incorrect."
            raise lib_exc.UnprocessableEntity(msg)

        if int(time.time()) - start_time >= client.build_timeout:
            raise lib_exc.TimeoutException
        time.sleep(client.build_interval)


def wait_for_interface_status(client, server_id, port_id, status):
    """Waits for an interface to reach a given status."""
    body = client.show_interface(server_id, port_id)["interfaceAttachment"]
    interface_status = body["port_state"]
    start = int(time.time())

    while interface_status != status:
        time.sleep(client.build_interval)
        body = client.show_interface(server_id, port_id)["interfaceAttachment"]
        interface_status = body["port_state"]

        timed_out = int(time.time()) - start >= client.build_timeout

        if interface_status != status and timed_out:
            message = (
                "Interface %s failed to reach %s status "
                "(current %s) within the required time (%s s)."
                % (port_id, status, interface_status, client.build_timeout)
            )
            raise lib_exc.TimeoutException(message)

    return body


def wait_for_interface_detach(client, server_id, port_id, detach_request_id):
    """Waits for an interface to be detached from a server."""

    def _get_detach_event_results():
        # NOTE(gibi): The obvious choice for this waiter would be to wait
        # until the interface disappears from the client.list_interfaces()
        # response. However that response is based on the binding status of the
        # port in Neutron. Nova deallocates the port resources _after the port
        # was  unbound in Neutron. This can cause that the naive waiter would
        # return before the port is fully deallocated. Wait instead of the
        # os-instance-action to succeed as that is recorded after both the
        # port is fully deallocated.
        events = client.show_instance_action(server_id, detach_request_id)[
            "instanceAction"
        ].get("events", [])
        return [
            event["result"]
            for event in events
            if event["event"] == "compute_detach_interface"
        ]

    detach_event_results = _get_detach_event_results()

    start = int(time.time())

    while "Success" not in detach_event_results:
        time.sleep(client.build_interval)
        detach_event_results = _get_detach_event_results()
        if "Success" in detach_event_results:
            return client.show_instance_action(server_id, detach_request_id)[
                "instanceAction"
            ]

        timed_out = int(time.time()) - start >= client.build_timeout
        if timed_out:
            message = (
                "Interface %s failed to detach from server %s within "
                "the required time (%s s)" % (port_id, server_id, client.build_timeout)
            )
            raise lib_exc.TimeoutException(message)


def wait_for_server_floating_ip(
    servers_client, server, floating_ip, wait_for_disassociate=False
):
    """Wait for floating IP association or disassociation.

    :param servers_client: The servers client to use when querying the server's
    floating IPs.
    :param server: The server JSON dict on which to wait.
    :param floating_ip: The floating IP JSON dict on which to wait.
    :param wait_for_disassociate: Boolean indiating whether to wait for
    disassociation instead of association.
    """

    def _get_floating_ip_in_server_addresses(floating_ip, server):
        for addresses in server["addresses"].values():
            for address in addresses:
                if (
                    address["OS-EXT-IPS:type"] == "floating"
                    and address["addr"] == floating_ip["floating_ip_address"]
                ):
                    return address
        return None

    start_time = int(time.time())
    while True:
        server = servers_client.show_server(server["id"])["server"]
        address = _get_floating_ip_in_server_addresses(floating_ip, server)
        if address is None and wait_for_disassociate:
            return None
        if not wait_for_disassociate and address:
            return address

        if int(time.time()) - start_time >= servers_client.build_timeout:
            if wait_for_disassociate:
                msg = (
                    "Floating ip %s failed to disassociate from server %s "
                    "in time." % (floating_ip, server["id"])
                )
            else:
                msg = (
                    "Floating ip %s failed to associate with server %s "
                    "in time." % (floating_ip, server["id"])
                )
            raise lib_exc.TimeoutException(msg)
        time.sleep(servers_client.build_interval)


def wait_for_ping(server_ip, timeout=30, interval=1):
    """Waits for an address to become pingable"""
    start_time = int(time.time())
    while int(time.time()) - start_time < timeout:
        response = os.system("ping -c 1 " + server_ip)
        if response == 0:
            return
        time.sleep(interval)
    raise lib_exc.TimeoutException()


def wait_for_port_status(client, port_id, status):
    """Wait for a port reach a certain status : ["BUILD" | "DOWN" | "ACTIVE"]
    :param client: The network client to use when querying the port's
    status
    :param status: A string to compare the current port status-to.
    :param port_id: The uuid of the port we would like queried for status.
    """
    start_time = time.time()
    while time.time() - start_time <= client.build_timeout:
        result = client.show_port(port_id)
        if result["port"]["status"].lower() == status.lower():
            return result
        time.sleep(client.build_interval)
    raise lib_exc.TimeoutException


def wait_for_ssh(ssh_client, timeout=30):
    """Waits for SSH connection to become usable"""
    start_time = int(time.time())
    while int(time.time()) - start_time < timeout:
        try:
            ssh_client.validate_authentication()
            return
        except lib_exc.SSHTimeout:
            pass
    raise lib_exc.TimeoutException()


def wait_for_caching(client, cache_client, image_id):
    """Waits until image is cached"""
    start = int(time.time())
    while int(time.time()) - start < client.build_timeout:
        caching = cache_client.list_cache()
        output = [image["image_id"] for image in caching["cached_images"]]
        if output and image_id in output:
            return caching

        time.sleep(client.build_interval)

    message = "Image %s failed to cache in time." % image_id
    caller = test_utils.find_test_caller()
    if caller:
        message = "(%s) %s" % (caller, message)
    raise lib_exc.TimeoutException(message)


def wait_for_object_create(object_client, container_name, object_name, interval=1):
    """Waits for created object to become available"""
    start_time = time.time()
    while time.time() - start_time < object_client.build_timeout:
        try:
            return object_client.get_object(container_name, object_name)
        except lib_exc.NotFound:
            time.sleep(interval)
    message = "Object %s failed to create within the required time (%s s)." % (
        object_name,
        object_client.build_timeout,
    )
    raise lib_exc.TimeoutException(message)


def wait_for_lb_termination(lb_client, lb_id):
    """Waits for loadbalancer to reach termination (until not found deleted lb)"""
    start_time = time.time()
    while time.time() - start_time < lb_client.build_timeout:
        time.sleep(lb_client.build_interval)
        try:
            lb_client.show_lb(lb_id)
        except lib_exc.NotFound:
            return
    raise lib_exc.TimeoutException(
        f"LB({lb_id}) failed to delete in timeout({lb_client.build_timeout})"
    )


def wait_for_gslb_instance_ready(gslb_client, instance_id, timeout=60):
    """Waits for GSLB instance to be ready (status.status == completed)

    After creating or updating an instance, there may be a background job running.
    This waiter polls the instance until status.status becomes 'completed'.
    """
    start_time = time.time()
    # Initial wait to allow the job to start
    time.sleep(5)
    while time.time() - start_time < timeout:
        try:
            instance = gslb_client.show_instance(instance_id)
            status = instance.get("status", {}).get("status")
            if status == "completed":
                return
            # Status is not completed yet (e.g., 'running', 'pending'), wait and retry
            time.sleep(gslb_client.build_interval)
        except lib_exc.Conflict:
            # Job is still running, wait and retry
            time.sleep(gslb_client.build_interval)
        except lib_exc.NotFound:
            # Instance not found, might still be creating
            time.sleep(gslb_client.build_interval)
    raise lib_exc.TimeoutException(
        f"GSLB({instance_id}) failed to become ready in timeout({timeout})"
    )


def wait_for_gslb_termination(gslb_client, instance_id):
    """Waits for GSLB instance to reach termination (until not found deleted instance)"""
    start_time = time.time()
    while time.time() - start_time < gslb_client.build_timeout:
        time.sleep(gslb_client.build_interval)
        try:
            gslb_client.show_instance(instance_id)
        except lib_exc.NotFound:
            return
    raise lib_exc.TimeoutException(
        f"GSLB({instance_id}) failed to delete in timeout({gslb_client.build_timeout})"
    )


def wait_for_routing_gateway_termination(vpc_client, routing_gateway_name):
    """Waits for routing gateway to reach termination (until not found deleted routing gateway)"""
    start_time = time.time()
    while time.time() - start_time < vpc_client.build_timeout:
        time.sleep(vpc_client.build_interval)
        try:
            vpc_client.read_routing_gateway(routing_gateway_name)["metadata"]["name"]
        except lib_exc.NotFound:
            return
    raise lib_exc.TimeoutException(
        f"Routing gateway({routing_gateway_name}) failed to delete in timeout({vpc_client.build_timeout})"
    )


def wait_for_lb_running_status(
    lb_client, lb_id, is_running=True, lb_running_status_timeout_sec=360
):
    """Waits for loadbalancer is_running true or false status"""
    start_time = time.time()
    while time.time() - start_time < int(lb_running_status_timeout_sec):
        time.sleep(lb_client.build_interval)
        lb_status = lb_client.show_lb_status(lb_id)["status"]
        # Need to check current job's processing changed 'False' too.
        if (
            lb_status["is_running"] == is_running
            and not lb_status["job"]["is_processing"]
        ):
            return
    raise lib_exc.TimeoutException(
        f"LB({lb_id}) failed to reach 'is_running: {is_running}' in timeout({int(lb_running_status_timeout_sec)})"
    )


def wait_for_lb_realserver_is_drained(
    lb_client, lb_id, condition_id, hostname, check_drained=True
):
    """
    Waits for loadbalancer's target drain server state through checking
    whether it exists or not by get drainservers
    """
    start_time = time.time()
    while time.time() - start_time < lb_client.build_timeout:
        time.sleep(lb_client.build_interval)
        lb_drain_list = lb_client.list_drain_servers(lb_id)["conditions"]
        # There is no drain server list. no drain state server
        if len(lb_drain_list) == 0 and not check_drained:
            return
        if len(lb_drain_list) == 0 and check_drained:
            continue
        # Check target drain condition_id & hostname in list
        found_target_drainserver = False
        for condition in lb_drain_list:
            if condition["id"] == condition_id and condition["hostname"] == hostname:
                found_target_drainserver = True
                break
        # If found_target_drainserver, the host server for lb is 'drain' state
        if check_drained and found_target_drainserver:
            return
        if not check_drained and not found_target_drainserver:
            return

    err_msg = f"LB({lb_id}) failed to wait for until "
    if check_drained:
        err_msg += f"exist condtion_id({condition_id}) & server({hostname}) in drain servers list"
    else:
        err_msg += f"not exist condtion_id({condition_id}) & server({hostname}) in drain servers list"

    raise lib_exc.TimeoutException(err_msg)


def wait_for_lb_realserver_up_state(
    lb_client, lb_id, real_server_name, time_out, interval=10
):
    """Waits for loadbalancer realserver state is 'up' state"""
    start_time = time.time()
    while time.time() - start_time < time_out:
        time.sleep(interval)
        ports_status = lb_client.show_lb_status(lb_id)["status"]["ports"]
        for port_status in ports_status:
            for real_server_info in port_status["realservers"]:
                # Checked that the realserver state is "up"
                if (
                    real_server_info["hostname"] == real_server_name
                    and real_server_info["is_up"]
                ):
                    return True
    # Failed to checked that the realserver state is "up" in time_out.
    return False


def wait_for_routing_gateway_creation(vpc_client, created_routing_gateway_name):
    """Waits for routing gateway to be created"""
    start_time = time.time()
    while time.time() - start_time < vpc_client.build_timeout:
        time.sleep(vpc_client.build_interval)
        read_created_routing_gateway = vpc_client.read_routing_gateway(
            created_routing_gateway_name
        )
        if (
            read_created_routing_gateway["metadata"]["name"]
            == created_routing_gateway_name
        ):
            return
    raise lib_exc.TimeoutException(
        f"Routing gateway({created_routing_gateway_name}) failed to create in timeout({vpc_client.build_timeout})"
    )


def wait_for_routing_gateway_available(vpc_client, routing_gateway_name):
    """Waits for routing gateway to be available"""
    start_time = time.time()
    while time.time() - start_time < vpc_client.build_timeout:
        time.sleep(vpc_client.build_interval)
        # Get routing gateway's status data
        routing_gateway_status = vpc_client.read_routing_gateway(routing_gateway_name)[
            "status"
        ]
        # From the status data, fetch a info if the routing gateway is available
        available_condition = [
            condition
            for condition in routing_gateway_status["conditions"]
            if condition["type"] == "Available"
        ]
        for data in available_condition:
            if data["status"] == "True":
                return
    raise lib_exc.TimeoutException(
        f"Routing gateway({routing_gateway_name}) failed to be available in timeout({vpc_client.build_timeout})"
    )


def wait_for_acl_policy_termination(vpc_client, acl_policy_name):
    """Waits for acl_policy to reach termination (until not found deleted the acl_policy)"""
    start_time = time.time()
    while time.time() - start_time < vpc_client.build_timeout:
        time.sleep(vpc_client.build_interval)
        try:
            vpc_client.read_acl_policy(acl_policy_name)["metadata"]["name"]
        except lib_exc.NotFound:
            return
    raise lib_exc.TimeoutException(
        f"Routing gateway({acl_policy_name}) failed to delete in timeout({vpc_client.build_timeout})"
    )


def wait_for_acl_policy_creation(vpc_client, created_acl_policy_name):
    """Waits for acl policy to be created"""
    start_time = time.time()
    while time.time() - start_time < vpc_client.build_timeout:
        time.sleep(vpc_client.build_interval)
        read_created_acl_policy = vpc_client.read_acl_policy(created_acl_policy_name)
        if read_created_acl_policy["metadata"]["name"] == created_acl_policy_name:
            return
    raise lib_exc.TimeoutException(
        f"Acl policy({created_acl_policy_name}) failed to create in timeout({vpc_client.build_timeout})"
    )


@allure.step("Wait for function status")
def wait_for_function_status(client, function_name, status):
    """Waits for a function to reach a given status."""
    current_status = "initial status"
    start_time = time.time()
    while time.time() - start_time < client.build_timeout:
        time.sleep(client.build_interval)

        function = client.show_function(function_name)
        if status == "active":
            if (
                function["lastOperation"]["type"] not in ("build", "deploy")
                or function["lastOperation"]["state"] != "complete"
            ):
                continue
        current_status = function["status"]["phase"]
        if current_status == status:
            return function

    message = (
        f"Function {function_name} failed to reach {status} state "
        f"(current state {current_status}) within the required "
        f"time ({client.build_timeout} s)."
    )

    caller = test_utils.find_test_caller()
    if caller:
        message = f"({caller}) {message}"
    raise lib_exc.TimeoutException(message)


@allure.step("Wait for function termination")
def wait_for_function_termination(client, function_name, ignore_error=False):
    """Waits for a function to reach termination."""
    try:
        body = client.show_function(function_name)
    except lib_exc.NotFound:
        return
    old_phase = body["status"]["phase"]
    start_time = int(time.time())
    while True:
        time.sleep(client.build_interval)
        try:
            body = client.show_function(function_name)
        except lib_exc.NotFound:
            return
        function_phase = body["status"]["phase"]
        if function_phase != old_phase:
            LOG.info(
                'Phase transition "{old_phase}" ==> "{function_phase}" after {time.time() - start_time} second wait'
            )
        if function_phase == "unhealthy" and not ignore_error:
            LOG.info(
                'Phase transition "{old_phase}" ==> "{function_phase}" after {time.time() - start_time} second wait'
            )
            raise lib_exc.DeleteErrorException(
                f"Function {function_name} failed to delete and is in Unhealthy phase"
            )

        if int(time.time()) - start_time >= client.build_timeout:
            raise lib_exc.TimeoutException
        old_phase = function_phase


@allure.step("Wait for function endpoint")
def wait_for_function_endpoint(client, function_name):
    """Waits for a function endpoint to be available."""
    start_time = time.time()
    while time.time() - start_time < client.build_timeout:
        time.sleep(client.build_interval)
        function = client.show_function(function_name)
        endpoint = function.get("endpoint")
        if endpoint and endpoint.strip():
            return function

    message = (
        f"Function {function_name} endpoint not available "
        f"within the required time ({client.build_timeout} s)."
    )
    caller = test_utils.find_test_caller()
    if caller:
        message = f"({caller}) {message}"
    raise lib_exc.TimeoutException(message)


def wait_for_function_logs_to_return(client, search_method, function_name, **kwargs):
    """Waits until logs are available for the specified function or a timeout occurs."""

    start = int(time.time())
    while int(time.time()) - start < client.build_timeout:
        logs = search_method(function_name, **kwargs)
        if logs and "logs" in logs:
            return logs["logs"]
        time.sleep(client.build_interval)

    message = "Timeout waiting for logs from function '{function_name}'."
    caller = test_utils.find_test_caller()
    if caller:
        message = f"({caller}) {message}"
    raise lib_exc.TimeoutException(message)


def wait_for_searching_histories_from_function(client, function_name, wait_until_count=1):
    """Waits until trigger histories are available for the specified function or a timeout occurs."""
    start = int(time.time())
    while int(time.time()) - start < client.build_timeout:
        histories = client.search_trigger_histories(function_name)
        if histories and histories["total"] >= wait_until_count:
            return histories
        time.sleep(client.build_interval)

    message = f"Timeout waiting for histories from function '{function_name}'."
    caller = test_utils.find_test_caller()
    if caller:
        message = f"({caller}) {message}"
    raise lib_exc.TimeoutException(message)


def wait_for_nat_gateway_termination(vpc_client, nat_gateway_name):
    """Waits for NAT gateway to reach termination (until not found deleted NAT gateway)"""
    start_time = time.time()
    while time.time() - start_time < vpc_client.build_timeout:
        time.sleep(vpc_client.build_interval)
        try:
            vpc_client.read_nat_gateway(nat_gateway_name)["metadata"]["name"]
        except lib_exc.NotFound:
            return
    raise lib_exc.TimeoutException(
        f"NAT gateway({nat_gateway_name}) failed to delete in timeout({vpc_client.build_timeout})"
    )


def wait_for_nat_gateway_creation(vpc_client, created_nat_gateway_name):
    """Waits for NAT gateway to be created"""
    start_time = time.time()
    while time.time() - start_time < vpc_client.build_timeout:
        time.sleep(vpc_client.build_interval)
        read_created_nat_gateway = vpc_client.read_nat_gateway(created_nat_gateway_name)
        if read_created_nat_gateway["metadata"]["name"] == created_nat_gateway_name:
            return
    raise lib_exc.TimeoutException(
        f"NAT gateway({created_nat_gateway_name}) failed to create in timeout({vpc_client.build_timeout})"
    )


def wait_for_nat_gateway_available(vpc_client, nat_gateway_name):
    """Waits for NAT gateway to be available"""
    start_time = time.time()
    while time.time() - start_time < vpc_client.build_timeout:
        time.sleep(vpc_client.build_interval)
        # Get NAT gateway's status data
        nat_gateway_status = vpc_client.read_nat_gateway(nat_gateway_name)["status"]
        # From the status data, fetch a info if the NAT gateway is available
        if any(
            condition["status"] == "True"
            for condition in nat_gateway_status["conditions"]
            if condition["type"] == "Available"
        ):
            return
    raise lib_exc.TimeoutException(
        f"NAT gateway({nat_gateway_name}) failed to be available in timeout({vpc_client.build_timeout})"
    )


def wait_for_reserved_ip_creation(vpc_client, created_reserved_ip_name):
    """Waits for reserved IP to be created"""
    start_time = time.time()
    while time.time() - start_time < vpc_client.build_timeout:
        time.sleep(vpc_client.build_interval)
        read_created_reserved_ip = vpc_client.read_reserved_ip(created_reserved_ip_name)
        if (
            read_created_reserved_ip["metadata"]["name"] == created_reserved_ip_name
            and read_created_reserved_ip.get("status", {}).get("address") is not None
        ):
            return
    raise lib_exc.TimeoutException(
        f"Reserved IP({created_reserved_ip_name}) failed to create in timeout({vpc_client.build_timeout})"
    )


def wait_for_reserved_ip_termination(vpc_client, reserved_ip_name):
    """Waits for reserved_ip to reach termination (until not found deleted reserved_ip)"""
    start_time = time.time()
    while time.time() - start_time < vpc_client.build_timeout:
        time.sleep(vpc_client.build_interval)
        try:
            vpc_client.read_reserved_ip(reserved_ip_name)["metadata"]["name"]
        except lib_exc.NotFound:
            return
    raise lib_exc.TimeoutException(
        f"Reserved_ip({reserved_ip_name}) failed to delete in timeout({vpc_client.build_timeout})"
    )


def wait_for_vpc_creation(vpc_client, created_vpc_name):
    """Waits for VPC to be created"""
    start_time = time.time()
    while time.time() - start_time < vpc_client.build_timeout:
        time.sleep(vpc_client.build_interval)
        read_created_vpc = vpc_client.read_vpc(created_vpc_name)

        status = read_created_vpc.get("status", {}).get("status")
        if status in ["PartialActive", "Active"]:
            return
    raise lib_exc.TimeoutException(
        f"VPC({created_vpc_name}) failed to create in timeout({vpc_client.build_timeout})"
    )


def wait_for_vpc_termination(vpc_client, vpc_name):
    """Waits for vpc to reach termination (until not found deleted vpc)"""
    start_time = time.time()
    while time.time() - start_time < vpc_client.build_timeout:
        time.sleep(vpc_client.build_interval)
        try:
            vpc_client.read_vpc(vpc_name)["metadata"]["name"]
        except lib_exc.NotFound:
            return
    raise lib_exc.TimeoutException(
        f"VPC({vpc_name}) failed to delete in timeout({vpc_client.build_timeout})"
    )


def wait_for_external_ip_creation(vpc_client, created_external_ip_name):
    """Waits for external IP to be created"""
    start_time = time.time()
    while time.time() - start_time < vpc_client.build_timeout:
        time.sleep(vpc_client.build_interval)
        read_created_external_ip = vpc_client.read_external_ip(created_external_ip_name)
        if read_created_external_ip["metadata"]["name"] == created_external_ip_name:
            return
    raise lib_exc.TimeoutException(
        f"External IP({created_external_ip_name}) failed to create in timeout({vpc_client.build_timeout})"
    )


def wait_for_external_ip_termination(
    vpc_client, external_ip_name, reserved_ip_name=None
):
    """Waits for external IP to reach termination (until not found deleted external IP)"""
    start_time = time.time()
    while time.time() - start_time < vpc_client.build_timeout:
        time.sleep(vpc_client.build_interval)
        try:
            vpc_client.read_external_ip(external_ip_name)
        except lib_exc.NotFound:
            # Check if a lock is release or not by reading reservedIP
            if reserved_ip_name:
                attached_rip_metadata = vpc_client.read_reserved_ip(reserved_ip_name)[
                    "metadata"
                ]
                # Get the label dictionary safely, defaulting to an empty dictionary if no label is present
                labels = attached_rip_metadata.get("labels", {})
                # Check if the reservedIP is still locked by another resource
                if not labels.get("vpc.lycorp.co.jp/address_locked_by", False):
                    return
                # Check if the IP is still locked by another resource
                locked_by = labels.get("vpc.lycorp.co.jp/address_locked_by", "")
                if external_ip_name not in locked_by:
                    return
            else:
                return
    raise lib_exc.TimeoutException(
        f"External_ip({external_ip_name}) failed to delete in timeout({vpc_client.build_timeout})"
    )


def wait_for_vpc_gateways_replica_ready(
    vpc_client, vpc_name, nat_ready_replicas, vpc_ready_replicas
):
    """Waits for Flava VPC gateways' replica is updated"""
    start_time = time.time()
    while time.time() - start_time < vpc_client.build_timeout:
        time.sleep(vpc_client.build_interval)
        read_updated_vpc = vpc_client.read_vpc(vpc_name)
        if (
            read_updated_vpc["status"]["status"] in ["PartialActive", "Active"]
            and read_updated_vpc["status"]["natGateway"]["readyReplicas"]
            == nat_ready_replicas
            and read_updated_vpc["status"]["vpcGateway"]["readyReplicas"]
            == vpc_ready_replicas
        ):
            return
    raise lib_exc.TimeoutException(
        f"External_ip({vpc_name}) failed to update in timeout({vpc_client.build_timeout})"
    )


def wait_for_detach_network_from_vpc(vpc_client, vpc_name):
    """Waits for detaching network from the VPC"""
    start_time = time.time()
    while time.time() - start_time < vpc_client.build_timeout:
        time.sleep(vpc_client.build_interval)
        try:
            vpc_spec = vpc_client.read_vpc(vpc_name)["spec"]
            if "networks" not in vpc_spec:
                return
        except lib_exc.NotFound:
            return
    raise lib_exc.TimeoutException(
        f"VPC({vpc_name}) failed detach all networks in timeout({vpc_client.build_timeout})"
    )


@allure.step("Wait for cluster status")
def wait_for_cluster_status(client, cluster_id, status):
    """Waits for a cluster to reach a given status."""
    current_status = "initial status"
    start_time = time.time()
    while time.time() - start_time < client.build_timeout:
        time.sleep(client.build_interval)

        cluster = client.fetch_cluster(cluster_id)
        current_status = cluster["status"]["state"]
        if current_status == status:
            return cluster

    message = (
        f"Cluster {cluster_id} failed to reach {status} state "
        f"(current state {current_status}) within the required "
        f"time ({client.build_timeout} s)."
    )

    caller = test_utils.find_test_caller()
    if caller:
        message = f"({caller}) {message}"
    raise lib_exc.TimeoutException(message)


@allure.step("Wait for cluster termination")
def wait_for_cluster_termination(client, cluster_id):
    """Waits for a cluster to reach termination."""
    start_time = time.time()
    while time.time() - start_time < client.build_timeout:
        time.sleep(client.build_interval)
        try:
            client.fetch_cluster(cluster_id)
        except lib_exc.NotFound:
            return
    raise lib_exc.TimeoutException(
        f"Cluster({cluster_id}) failed to delete in timeout({client.build_timeout})"
    )


@allure.step("Wait for node pool status")
def wait_for_node_pool_status(client, cluster_id, node_pool_name, status):
    """Waits for a node pool to reach a given status."""
    current_status = "initial status"
    start_time = time.time()
    while time.time() - start_time < client.build_timeout:
        time.sleep(client.build_interval)

        node_pool = client.fetch_node_pool(cluster_id, node_pool_name)
        current_status = node_pool["status"]["state"]
        if current_status == status:
            return node_pool

    message = (
        f"Node pool {node_pool_name} failed to reach {status} state "
        f"(current state {current_status}) within the required "
        f"time ({client.build_timeout} s)."
    )

    caller = test_utils.find_test_caller()
    if caller:
        message = f"({caller}) {message}"
    raise lib_exc.TimeoutException(message)


@allure.step("Wait for node pool termination")
def wait_for_node_pool_termination(client, cluster_id, node_pool_name):
    """Waits for a node pool to reach termination."""
    start_time = time.time()
    while time.time() - start_time < client.build_timeout:
        time.sleep(client.build_interval)
        try:
            client.fetch_node_pool(cluster_id, node_pool_name)
        except lib_exc.NotFound:
            return
    raise lib_exc.TimeoutException(
        f"Node Pool({node_pool_name}) failed to delete in timeout({client.build_timeout})"
    )


@allure.step("Wait for EgressProxy status")
def wait_for_egress_proxy_status(client, name, status):
    """Waits for EgressProxy to reach a given status."""
    current_status = "initial status"
    start_time = time.time()
    while time.time() - start_time < client.build_timeout:
        time.sleep(client.build_interval)

        egress_proxy = client.show_egress_proxy(name)
        current_status = egress_proxy["status"]["phase"]
        if current_status == status:
            return egress_proxy

    message = (
        f"EgressProxy {name} failed to reach {status} state "
        f"(current state {current_status}) within the required "
        f"time ({client.build_timeout} s)."
    )

    caller = test_utils.find_test_caller()
    if caller:
        message = f"({caller}) {message}"
    raise lib_exc.TimeoutException(message)


@allure.step("Wait for EgressProxy termination")
def wait_for_egress_proxy_termination(client, name):
    """Waits for EgressProxy to reach termination."""
    start_time = time.time()
    while time.time() - start_time < client.build_timeout:
        time.sleep(client.build_interval)
        try:
            client.show_egress_proxy(name)
        except lib_exc.NotFound:
            return
    raise lib_exc.TimeoutException(
        f"EgressProxy({name}) failed to delete in timeout({client.build_timeout})"
    )

@allure.step("Wait for Service Account Credentials termination")
def wait_for_sa_credentials_termination(client, credential_id):
    """Waits for Service Account Credentials to reach termination."""
    start_time = time.time()
    while time.time() - start_time < client.build_timeout:
        time.sleep(client.build_interval)
        credentials = client.list_sa_credential()["credentials"]
        # If not found in list, return
        if not [i for i in credentials if i["id"] == credential_id]:
            return
    raise lib_exc.TimeoutException(
        f"Service Account Credentials({credential_id}) failed to delete in timeout({client.build_timeout})"
    )