# Copyright 2012 OpenStack Foundation # Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import copy import importlib import inspect import sys from debtcollector import removals from oslo_log import log as logging import testtools from lib import auth from lib.common.utils import misc from lib import exceptions from lib.services import compute from lib.services import identity from lib.services import image from lib.services import network from lib.services import object_storage from lib.services import placement from lib.services import volume from lib.services import loadbalancer from lib.services import vpc from lib.services import faas from lib.services import kaas from lib.services import egress_proxy from lib.services import container_registry from lib.services import flava_pulsar from lib.services import dns from lib.services import gslb from lib.services import langfuse from lib.services import servicemap from lib.services import redis from lib.services import api_gateway from lib.services import rollouts from lib.services import cloud_blueprint LOG = logging.getLogger(__name__) def tempest_modules(): """Dict of service client modules available in Tempest. Provides a dict of stable service modules available in Tempest, with ``service_version`` as key, and the module object as value. """ return { 'compute': compute, 'placement': placement, 'identity.v2': identity.v2, 'identity.v3': identity.v3, 'image.v2': image.v2, 'network': network, 'object-storage': object_storage, 'volume.v3': volume.v3, # Add new LB service client 'loadbalancer': loadbalancer, 'vpc': vpc, 'faas': faas, 'kaas': kaas, 'egressproxy': egress_proxy, 'container_registry': container_registry, 'flava_pulsar': flava_pulsar, 'dns': dns, 'gslb': gslb, 'langfuse': langfuse, 'servicemap': servicemap, 'redis': redis, 'api_gateway': api_gateway, 'rollouts': rollouts, 'cloud_blueprint': cloud_blueprint, } def available_modules(): """Set of service client modules available in Tempest and plugins Set of stable service clients from Tempest and service clients exposed by plugins. This set of available modules can be used for automatic configuration. :raise PluginRegistrationException: if a plugin exposes a service_version already defined by Tempest or another plugin. Examples:: from tempest import config params = {} for service_version in available_modules(): service = service_version.split('.')[0] params[service] = config.service_client_config(service) service_clients = ServiceClients(creds, identity_uri, client_parameters=params) """ extra_service_versions = set([]) _tempest_modules = set(tempest_modules()) plugin_services = ClientsRegistry().get_service_clients() name_conflicts = [] for plugin_name in plugin_services: plug_service_versions = set([x['service_version'] for x in plugin_services[plugin_name]]) # If a plugin exposes a duplicate service_version raise an exception if plug_service_versions: if not plug_service_versions.isdisjoint(extra_service_versions): detailed_error = ( 'Plugin %s is trying to register a service %s already ' 'claimed by another one' % (plugin_name, extra_service_versions & plug_service_versions)) name_conflicts.append(exceptions.PluginRegistrationException( name=plugin_name, detailed_error=detailed_error)) extra_service_versions |= plug_service_versions if name_conflicts: LOG.error( 'Failed to list available modules due to name conflicts: %s', name_conflicts) raise testtools.MultipleExceptions(*name_conflicts) return _tempest_modules | extra_service_versions @misc.singleton class ClientsRegistry(object): """Registry of all service clients available from plugins""" def __init__(self): self._service_clients = {} def register_service_client(self, plugin_name, service_client_data): if plugin_name in self._service_clients: detailed_error = 'Clients for plugin %s already registered' raise exceptions.PluginRegistrationException( name=plugin_name, detailed_error=detailed_error % plugin_name) self._service_clients[plugin_name] = service_client_data LOG.debug("Successfully registered plugin %s in the service client " "registry with configuration: %s", plugin_name, service_client_data) def get_service_clients(self): return self._service_clients class ClientsFactory(object): """Builds service clients for a service client module This class implements the logic of feeding service client parameters to service clients from a specific module. It allows setting the parameters once and obtaining new instances of the clients without the need of passing any parameter. ClientsFactory can be used directly, or consumed via the `ServiceClients` class, which manages the authorization part. """ def __init__(self, module_path, client_names, auth_provider, **kwargs): """Initialises the client factory :param module_path: Path to module that includes all service clients. All service client classes must be exposed by a single module. If they are separated in different modules, defining __all__ in the root module can help, similar to what is done by service clients in tempest. :param client_names: List or set of names of the service client classes. :param auth_provider: The auth provider used to initialise client. :param kwargs: Parameters to be passed to all clients. Parameters values can be overwritten when clients are initialised, but parameters cannot be deleted. :raise ImportError: if the specified module_path cannot be imported Example:: # Get credentials and an auth_provider clients = ClientsFactory( module_path='my_service.my_service_clients', client_names=['ServiceClient1', 'ServiceClient2'], auth_provider=auth_provider, service='my_service', region='region1') my_api_client = clients.MyApiClient() my_api_client_region2 = clients.MyApiClient(region='region2') """ # Import the module. If it's not importable, the raised exception # provides good enough information about what happened _module = importlib.import_module(module_path) # If any of the classes is not in the module we fail for class_name in client_names: # TODO(andreaf) This always passes all parameters to all clients. # In future to allow clients to specify the list of parameters # that they accept based out of a list of standard ones. # Obtain the class klass = self._get_class(_module, class_name) final_kwargs = copy.copy(kwargs) # Set the function as an attribute of the factory setattr(self, class_name, self._get_partial_class( klass, auth_provider, final_kwargs)) def _get_partial_class(self, klass, auth_provider, kwargs): # Define a function that returns a new class instance by # combining default kwargs with extra ones def partial_class(alias=None, **later_kwargs): """Returns a callable the initialises a service client Builds a callable that accepts kwargs, which are passed through to the __init__ of the service client, along with a set of defaults set in factory at factory __init__ time. Original args in the service client can only be passed as kwargs. It accepts one extra parameter 'alias' compared to the original service client. When alias is provided, the returned callable will also set an attribute called with a name defined in 'alias', which contains the instance of the service client. :param alias: str Name of the attribute set on the factory once the callable is invoked which contains the initialised service client. If None, no attribute is set. :param later_kwargs: kwargs passed through to the service client __init__ on top of defaults set at factory level. """ kwargs.update(later_kwargs) _client = klass(auth_provider=auth_provider, **kwargs) if alias: setattr(self, alias, _client) return _client return partial_class @classmethod def _get_class(cls, module, class_name): klass = getattr(module, class_name, None) if not klass: msg = 'Invalid class name, %s is not found in %s' raise AttributeError(msg % (class_name, module)) if not inspect.isclass(klass): msg = 'Expected a class, got %s of type %s instead' raise TypeError(msg % (klass, type(klass))) return klass class ServiceClients(object): """Service client provider class The ServiceClients object provides a useful means for tests to access service clients configured for a specified set of credentials. It hides some of the complexity from the authorization and configuration layers. Examples:: # johndoe is a tempest.lib.auth.Credentials type instance johndoe_clients = clients.ServiceClients(johndoe, identity_uri) # List servers in default region johndoe_servers_client = johndoe_clients.compute.ServersClient() johndoe_servers = johndoe_servers_client.list_servers() # List servers in Region B johndoe_servers_client_B = johndoe_clients.compute.ServersClient( region='B') johndoe_servers = johndoe_servers_client_B.list_servers() """ # NOTE(andreaf) This class does not depend on tempest configuration # and its meant for direct consumption by external clients such as tempest # plugins. Tempest provides a wrapper class, `clients.Manager`, that # initialises this class using values from tempest CONF object. The wrapper # class should only be used by tests hosted in Tempest. @removals.removed_kwarg('client_parameters') def __init__(self, credentials, identity_uri, region=None, scope=None, disable_ssl_certificate_validation=True, ca_certs=None, trace_requests='', client_parameters=None, proxy_url=None): """Service Clients provider Instantiate a `ServiceClients` object, from a set of credentials and an identity URI. The identity version is inferred from the credentials object. Optionally auth scope can be provided. A few parameters can be given a value which is applied as default for all service clients: region, dscv, ca_certs, trace_requests. Parameters dscv, ca_certs and trace_requests all apply to the auth provider as well as any service clients provided by this manager. Any other client parameter should be set via ClientsRegistry. Client parameter used to be set via client_parameters, but this is deprecated, and it is actually already not honoured anymore: https://launchpad.net/bugs/1680915. The list of available parameters is defined in the service clients interfaces. For reference, most clients will accept 'region', 'service', 'endpoint_type', 'build_timeout' and 'build_interval', which are all inherited from RestClient. The `config` module in Tempest exposes an helper function `service_client_config` that can be used to extract from configuration a dictionary ready to be injected in kwargs. Exceptions are: - Token clients for 'identity' must be given an 'auth_url' parameter - Volume client for 'volume' accepts 'default_volume_size' - Servers client from 'compute' accepts 'enable_instance_password' If Tempest configuration is used, parameters will be loaded in the Registry automatically for all service client (Tempest stable ones and plugins). Examples:: identity_params = config.service_client_config('identity') params = { 'identity': identity_params, 'compute': {'region': 'region2'}} manager = lib_manager.Manager( my_creds, identity_uri, client_parameters=params) :param credentials: An instance of `auth.Credentials` :param identity_uri: URI of the identity API. This should be a mandatory parameter, and it will so soon. :param region: Default value of region for service clients. :param scope: default scope for tokens produced by the auth provider :param disable_ssl_certificate_validation: Applies to auth and to all service clients. :param ca_certs: Applies to auth and to all service clients. :param trace_requests: Applies to auth and to all service clients. :param client_parameters: Dictionary with parameters for service clients. Keys of the dictionary are the service client service name, as declared in `service_clients.available_modules()` except for the version. Values are dictionaries of parameters that are going to be passed to all clients in the service client module. :param proxy_url: Applies to auth and to all service clients, set a proxy url for the clients to use. """ self._registered_services = set([]) self.credentials = credentials self.identity_uri = identity_uri if not identity_uri: raise exceptions.InvalidCredentials( 'ServiceClients requires a non-empty identity_uri.') self.region = region # Check if passed or default credentials are valid if not self.credentials.is_valid(): raise exceptions.InvalidCredentials(credentials) # Get the identity classes matching the provided credentials # TODO(andreaf) Define a new interface in Credentials to get # the API version from an instance identity = [(k, auth.IDENTITY_VERSION[k][1]) for k in auth.IDENTITY_VERSION.keys() if isinstance(self.credentials, auth.IDENTITY_VERSION[k][0])] # Zero matches or more than one are both not valid. if len(identity) != 1: msg = "Zero or %d ambiguous auth provider found. identity: %s, " \ "credentials: %s" % (len(identity), identity, credentials) raise exceptions.InvalidCredentials(msg) self.auth_version, auth_provider_class = identity[0] self.dscv = disable_ssl_certificate_validation self.ca_certs = ca_certs self.trace_requests = trace_requests self.proxy_url = proxy_url if self.credentials.project_id or self.credentials.project_name: scope = 'project' elif self.credentials.system: scope = 'system' elif self.credentials.domain_id or self.credentials.domain_name: scope = 'domain' else: scope = 'project' # Creates an auth provider for the credentials self.auth_provider = auth_provider_class( self.credentials, self.identity_uri, scope=scope, disable_ssl_certificate_validation=self.dscv, ca_certs=self.ca_certs, trace_requests=self.trace_requests, proxy_url=proxy_url) # Setup some defaults for client parameters of registered services client_parameters = client_parameters or {} self.parameters = {} # Parameters are provided for unversioned services all_modules = available_modules() unversioned_services = set( [x.split('.')[0] for x in all_modules]) for service in unversioned_services: self.parameters[service] = self._setup_parameters( client_parameters.pop(service, {})) # Check that no client parameters was supplied for unregistered clients if client_parameters: raise exceptions.UnknownServiceClient( services=list(client_parameters.keys())) # Register service clients from the registry (__tempest__ and plugins) clients_registry = ClientsRegistry() plugin_service_clients = clients_registry.get_service_clients() registration_errors = [] for plugin in plugin_service_clients: service_clients = plugin_service_clients[plugin] # Each plugin returns a list of service client parameters for service_client in service_clients: # NOTE(andreaf) If a plugin cannot register, stop the # registration process, log some details to help # troubleshooting, and re-raise try: self.register_service_client_module(**service_client) except Exception: registration_errors.append(sys.exc_info()) LOG.exception( 'Failed to register service client from plugin %s ' 'with parameters %s', plugin, service_client) if registration_errors: raise testtools.MultipleExceptions(*registration_errors) def register_service_client_module(self, name, service_version, module_path, client_names, **kwargs): """Register a service client module Initiates a client factory for the specified module, using this class auth_provider, and accessible via a `name` attribute in the service client. :param name: Name used to access the client :param service_version: Name of the service complete with version. Used to track registered services. When a plugin implements it, it can be used by other plugins to obtain their configuration. :param module_path: Path to module that includes all service clients. All service client classes must be exposed by a single module. If they are separated in different modules, defining __all__ in the root module can help, similar to what is done by service clients in tempest. :param client_names: List or set of names of service client classes. :param kwargs: Extra optional parameters to be passed to all clients. ServiceClient provides defaults for region, dscv, ca_certs, http proxies and trace_requests. :raise ServiceClientRegistrationException: if the provided name is already in use or if service_version is already registered. :raise ImportError: if module_path cannot be imported. """ if hasattr(self, name): using_name = getattr(self, name) detailed_error = 'Module name already in use: %s' % using_name raise exceptions.ServiceClientRegistrationException( name=name, service_version=service_version, module_path=module_path, client_names=client_names, detailed_error=detailed_error) if service_version in self.registered_services: detailed_error = 'Service %s already registered.' % service_version raise exceptions.ServiceClientRegistrationException( name=name, service_version=service_version, module_path=module_path, client_names=client_names, detailed_error=detailed_error) params = dict(region=self.region, disable_ssl_certificate_validation=self.dscv, ca_certs=self.ca_certs, trace_requests=self.trace_requests, proxy_url=self.proxy_url) params.update(kwargs) # Instantiate the client factory _factory = ClientsFactory(module_path=module_path, client_names=client_names, auth_provider=self.auth_provider, **params) # Adds the client factory to the service_client setattr(self, name, _factory) # Add the name of the new service in self.SERVICES for discovery self._registered_services.add(service_version) @property def registered_services(self): return self._registered_services def _setup_parameters(self, parameters): """Setup default values for client parameters Region by default is the region passed as an __init__ parameter. Checks that no parameter for an unknown service is provided. """ _parameters = {} # Use region from __init__ if self.region: _parameters['region'] = self.region # Update defaults with specified parameters _parameters.update(parameters) # If any parameter is left, parameters for an unknown service were # provided as input. Fail rather than ignore silently. return _parameters