Merge pull request #1 from stfc/hypervisor-fixes

Hypervisor fixes

.gitignore

@@ -159,4 +159,6 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-.idea/
+.idea/
+
+.vscode/settings.json

docs/user_docs/query_docs/SERVERS.md

@@ -30,7 +30,7 @@ from enums.query.props.server_properties import ServerProperties
 | Property Enum            | Type         | Aliases                       | Description                                                                                                                                                                                                                                          |
 |--------------------------|--------------|-------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | FLAVOR_ID                | `string`     | `None`                        | The ID of the Flavor the Server is using                                                                                                                                                                                                             |
-| HYPERVISOR_ID            | `string`     | `None`                        | The ID of the Hypervisor the Server is being hosted on                                                                                                                                                                                               |
+| HYPERVISOR_NAME          | `string`     | `None`                        | Name of the Hypervisor the Server is being hosted on                                                                                                                                                                                               |
 | IMAGE_ID                 | `string`     | `None`                        | The ID of the Image the Server is using                                                                                                                                                                                                              |
 | PROJECT_ID               | `string`     | `None`                        | The ID of the Project the Server is associated with                                                                                                                                                                                                  |
 | SERVER_CREATION_DATE     | `string` (x) | "created_at"                  | Timestamp of when the server was created.                                                                                                                                                                                                            |
@@ -69,7 +69,7 @@ The following shared-common properties are listed below (as well as the Query ob
 | ServerProperties.PROJECT_ID    | ProjectProperties.PROJECT_ID       | Many-to-One | `ServerQuery` to `ProjectQuery`    | [PROJECTS.md](PROJECTS.md)       |
 | ServerProperties.FLAVOR_ID     | FlavorProperties.FLAVOR_ID         | Many-to-One | `ServerQuery` to `FlavorQuery`     | [FLAVORS.md](FLAVORS.md)         |
 | ServerProperties.IMAGE_ID      | ImageProperties.IMAGE_ID           | Many-to-One | `ServerQuery` to `ImageQuery`      | [IMAGES.md](IMAGES.md)           |
-| ServerProperties.HYPERVISOR_ID | HypervisorProperties.HYPERVISOR_ID | Many-to-One | `ServerQuery` to `HypervisorQuery` | [HYPERVISORS.md](HYPERVISORS.md) |
+| ServerProperties.HYPERVISOR_NAME | HypervisorProperties.HYPERVISOR_NAME | Many-to-One | `ServerQuery` to `HypervisorQuery` | [HYPERVISORS.md](HYPERVISORS.md) |
 
 
 

openstackquery/enums/props/hypervisor_properties.py

@@ -12,7 +12,7 @@ class HypervisorProperties(PropEnum):
     An enum class for all hypervisor properties
     """
 
-    HYPERVISOR_CURRENT_WORKLOAD = auto()
+    # HYPERVISOR_CURRENT_WORKLOAD = auto()
     HYPERVISOR_DISK_FREE = auto()
     HYPERVISOR_DISK_SIZE = auto()
     HYPERVISOR_DISK_USED = auto()
@@ -22,7 +22,7 @@ class HypervisorProperties(PropEnum):
     HYPERVISOR_MEMORY_SIZE = auto()
     HYPERVISOR_MEMORY_USED = auto()
     HYPERVISOR_NAME = auto()
-    HYPERVISOR_SERVER_COUNT = auto()
+    # HYPERVISOR_SERVER_COUNT = auto() # Deprecated, use server query
     HYPERVISOR_STATE = auto()
     HYPERVISOR_STATUS = auto()
     HYPERVISOR_VCPUS = auto()
@@ -35,10 +35,10 @@ def _get_aliases() -> Dict:
         A method that returns all valid string alias mappings
         """
         return {
-            HypervisorProperties.HYPERVISOR_CURRENT_WORKLOAD: [
-                "current_workload",
-                "workload",
-            ],
+            # HypervisorProperties.HYPERVISOR_CURRENT_WORKLOAD: [
+            #     "current_workload",
+            #     "workload",
+            # ],
             HypervisorProperties.HYPERVISOR_DISK_FREE: [
                 "local_disk_free",
                 "free_disk_gb",
@@ -57,7 +57,7 @@ def _get_aliases() -> Dict:
                 "memory_mb_used",
             ],
             HypervisorProperties.HYPERVISOR_NAME: ["name", "host_name"],
-            HypervisorProperties.HYPERVISOR_SERVER_COUNT: ["running_vms"],
+            # HypervisorProperties.HYPERVISOR_SERVER_COUNT: ["running_vms"],
             HypervisorProperties.HYPERVISOR_STATE: ["state"],
             HypervisorProperties.HYPERVISOR_STATUS: ["status"],
             HypervisorProperties.HYPERVISOR_VCPUS: ["vcpus"],
@@ -74,23 +74,39 @@ def get_prop_mapping(prop) -> Optional[PropFunc]:
         :param prop: A HypervisorProperty Enum for which a function may exist for
         """
         mapping = {
-            HypervisorProperties.HYPERVISOR_CURRENT_WORKLOAD: lambda a: a[
-                "current_workload"
+            # HypervisorProperties.HYPERVISOR_CURRENT_WORKLOAD: lambda a: a[
+            #     "current_workload"
+            # ],
+            HypervisorProperties.HYPERVISOR_DISK_FREE: lambda a: a.resources["DISK_GB"][
+                "free"
+            ],
+            HypervisorProperties.HYPERVISOR_DISK_SIZE: lambda a: a.resources["DISK_GB"][
+                "total"
+            ],
+            HypervisorProperties.HYPERVISOR_DISK_USED: lambda a: a.resources["DISK_GB"][
+                "usage"
             ],
-            HypervisorProperties.HYPERVISOR_DISK_FREE: lambda a: a["free_disk_gb"],
-            HypervisorProperties.HYPERVISOR_DISK_SIZE: lambda a: a["local_gb"],
-            HypervisorProperties.HYPERVISOR_DISK_USED: lambda a: a["local_gb_used"],
             HypervisorProperties.HYPERVISOR_ID: lambda a: a["id"],
             HypervisorProperties.HYPERVISOR_IP: lambda a: a["host_ip"],
-            HypervisorProperties.HYPERVISOR_MEMORY_FREE: lambda a: a["free_ram_mb"],
-            HypervisorProperties.HYPERVISOR_MEMORY_SIZE: lambda a: a["memory_mb"],
-            HypervisorProperties.HYPERVISOR_MEMORY_USED: lambda a: a["memory_mb_used"],
+            HypervisorProperties.HYPERVISOR_MEMORY_FREE: lambda a: a.resources[
+                "MEMORY_MB"
+            ]["free"],
+            HypervisorProperties.HYPERVISOR_MEMORY_SIZE: lambda a: a.resources[
+                "MEMORY_MB"
+            ]["total"],
+            HypervisorProperties.HYPERVISOR_MEMORY_USED: lambda a: a.resources[
+                "MEMORY_MB"
+            ]["usage"],
             HypervisorProperties.HYPERVISOR_NAME: lambda a: a["name"],
-            HypervisorProperties.HYPERVISOR_SERVER_COUNT: lambda a: a["runnning_vms"],
+            # HypervisorProperties.HYPERVISOR_SERVER_COUNT: lambda a: a["runnning_vms"],
             HypervisorProperties.HYPERVISOR_STATE: lambda a: a["state"],
             HypervisorProperties.HYPERVISOR_STATUS: lambda a: a["status"],
-            HypervisorProperties.HYPERVISOR_VCPUS: lambda a: a["vcpus"],
-            HypervisorProperties.HYPERVISOR_VCPUS_USED: lambda a: a["vcpus_used"],
+            HypervisorProperties.HYPERVISOR_VCPUS: lambda a: a.resources["VCPU"][
+                "total"
+            ],
+            HypervisorProperties.HYPERVISOR_VCPUS_USED: lambda a: a.resources["VCPU"][
+                "usage"
+            ],
             HypervisorProperties.HYPERVISOR_DISABLED_REASON: lambda a: a["service"][
                 "disabled_reason"
             ],

openstackquery/enums/props/server_properties.py

@@ -11,7 +11,7 @@ class ServerProperties(PropEnum):
     """
 
     FLAVOR_ID = auto()
-    HYPERVISOR_ID = auto()
+    HYPERVISOR_NAME = auto()
     IMAGE_ID = auto()
     PROJECT_ID = auto()
     SERVER_CREATION_DATE = auto()
@@ -29,7 +29,7 @@ def _get_aliases():
         A method that returns all valid string alias mappings
         """
         return {
-            ServerProperties.HYPERVISOR_ID: ["host_id", "hv_id"],
+            ServerProperties.HYPERVISOR_NAME: ["hv_name", "hypervisor_name"],
             ServerProperties.SERVER_CREATION_DATE: ["created_at"],
             ServerProperties.SERVER_DESCRIPTION: [
                 "description",
@@ -53,7 +53,7 @@ def get_prop_mapping(prop):
         """
         mapping = {
             ServerProperties.USER_ID: lambda a: a["user_id"],
-            ServerProperties.HYPERVISOR_ID: lambda a: a["host_id"],
+            ServerProperties.HYPERVISOR_NAME: lambda a: a["hypervisor_hostname"],
             ServerProperties.SERVER_ID: lambda a: a["id"],
             ServerProperties.SERVER_NAME: lambda a: a["name"],
             ServerProperties.SERVER_DESCRIPTION: lambda a: a["description"],

openstackquery/mappings/hypervisor_mapping.py

@@ -35,7 +35,7 @@ def get_chain_mappings():
         Should return a dictionary containing property pairs mapped to query mappings.
         This is used to define how to chain results from this query to other possible queries
         """
-        return {HypervisorProperties.HYPERVISOR_ID: ServerProperties.HYPERVISOR_ID}
+        return {HypervisorProperties.HYPERVISOR_NAME: ServerProperties.HYPERVISOR_NAME}
 
     @staticmethod
     def get_runner_mapping() -> Type[RunnerWrapper]:
@@ -81,8 +81,8 @@ def get_client_side_handlers() -> QueryClientSideHandlers:
             HypervisorProperties.HYPERVISOR_MEMORY_FREE,
             HypervisorProperties.HYPERVISOR_VCPUS,
             HypervisorProperties.HYPERVISOR_VCPUS_USED,
-            HypervisorProperties.HYPERVISOR_SERVER_COUNT,
-            HypervisorProperties.HYPERVISOR_CURRENT_WORKLOAD,
+            # HypervisorProperties.HYPERVISOR_SERVER_COUNT, # Deprecated, use server query
+            # HypervisorProperties.HYPERVISOR_CURRENT_WORKLOAD,
         ]
 
         return QueryClientSideHandlers(

openstackquery/mappings/server_mapping.py

@@ -47,7 +47,7 @@ def get_chain_mappings():
             ServerProperties.PROJECT_ID: ProjectProperties.PROJECT_ID,
             ServerProperties.FLAVOR_ID: FlavorProperties.FLAVOR_ID,
             ServerProperties.IMAGE_ID: ImageProperties.IMAGE_ID,
-            ServerProperties.HYPERVISOR_ID: HypervisorProperties.HYPERVISOR_ID,
+            ServerProperties.HYPERVISOR_NAME: HypervisorProperties.HYPERVISOR_NAME,
         }
 
     @staticmethod

openstackquery/query_blocks/result.py

@@ -36,7 +36,7 @@ def get_prop(self, prop: PropEnum) -> PropValue:
         """
         try:
             return self._prop_enum_cls.get_prop_mapping(prop)(self._obj_result)
-        except AttributeError:
+        except (AttributeError, KeyError):
             return self._default_prop_value
 
     def update_forwarded_properties(self, forwarded_props: Dict[str, PropValue]):

openstackquery/runners/hypervisor_runner.py

@@ -1,8 +1,11 @@
+import json
 import logging
-from typing import Optional, List
+from typing import Dict, List, Optional
+
 from openstack.compute.v2.hypervisor import Hypervisor
+from osc_placement.http import SessionClient as PlacementClient
 
-from openstackquery.aliases import ServerSideFilters, OpenstackResourceObj
+from openstackquery.aliases import OpenstackResourceObj, ServerSideFilters
 from openstackquery.openstack_connection import OpenstackConnection
 from openstackquery.runners.runner_utils import RunnerUtils
 from openstackquery.runners.runner_wrapper import RunnerWrapper
@@ -26,6 +29,49 @@ def parse_meta_params(self, conn: OpenstackConnection, **kwargs):
         logger.debug("HypervisorQuery has no meta-params available")
         return super().parse_meta_params(conn, **kwargs)
 
+    def _populate_placement_info(
+        self, conn: OpenstackConnection, hypervisors: List
+    ) -> List:
+        """
+        Adds resource usage stats to the hypervisors
+        :param conn: Openstack connecion
+        :param hypervisors: List of hypervisors
+        :return: List of hypervisors with additional resource usage stats
+        """
+        client = PlacementClient(
+            api_version="1.6",
+            session=conn.session,
+            ks_filter={"service_type": "placement"},
+        )
+
+        for hypervisor in hypervisors:
+            hypervisor.resources = self._get_usage_info(conn, client, hypervisor)
+
+        return hypervisors
+
+    def _get_usage_info(
+        self, conn: OpenstackConnection, client: PlacementClient, hypervisor: Hypervisor
+    ) -> Dict:
+        """
+        Get usage stats from the openstack placement api
+        :param conn: Openstack connection
+        :param client: osc_placement session client
+        :param hypervisor: Openstack hypervisor
+        :return: resource usage for the hypervisor
+        """
+        resources = conn.placement.resource_provider_inventories(hypervisor.id)
+        usages = client.request("get", f"/resource_providers/{hypervisor.id}/usages")
+        usages = json.loads(usages.text).get("usages")
+        usage_info = {}
+        for i in resources:
+            usage_info[i.resource_class] = {
+                "total": i.total,
+                "usage": usages.get(i.resource_class),
+                "free": i.total - usages.get(i.resource_class),
+            }
+
+        return usage_info
+
     # pylint: disable=unused-argument
     def run_query(
         self,
@@ -49,6 +95,8 @@ def run_query(
             "running openstacksdk command conn.compute.hypervisors(%s)",
             ",".join(f"{key}={value}" for key, value in filter_kwargs.items()),
         )
-        return RunnerUtils.run_paginated_query(
+        hypervisors = RunnerUtils.run_paginated_query(
             conn.compute.hypervisors, self._page_marker_prop_func, filter_kwargs
         )
+
+        return self._populate_placement_info(conn, hypervisors)

requirements.txt

@@ -3,3 +3,4 @@ pylint
 openstacksdk
 pre-commit
 tabulate
+osc-placement

setup.py

@@ -7,7 +7,7 @@
 
 setup(
     name="openstackquery",
-    version="0.1.1",
+    version="0.1.2",
     author="Anish Mudaraddi",
     author_email="<anish.mudaraddi@stfc.ac.uk>",
     description=DESCRIPTION,

tests/enums/props/test_hypervisor_properties.py

@@ -37,26 +37,6 @@ def test_get_marker_prop_func(mock_get_prop_mapping):
     assert val == mock_get_prop_mapping.return_value
 
 
-@pytest.mark.parametrize(
-    "val",
-    [
-        "hypervisor_current_workload",
-        "Hypervisor_Current_Workload",
-        "HyPeRvIsOr_CuRrEnT_wOrKlOaD",
-        "current_workload",
-        "workload",
-    ],
-)
-def test_hypervisor_current_workload_serialization(val):
-    """
-    Tests that variants of HYPERVISOR_CURRENT_WORKLOAD can be serialized
-    """
-    assert (
-        HypervisorProperties.from_string(val)
-        is HypervisorProperties.HYPERVISOR_CURRENT_WORKLOAD
-    )
-
-
 @pytest.mark.parametrize(
     "val",
     [
@@ -210,25 +190,6 @@ def test_hypervisor_name_serialization(val):
     assert HypervisorProperties.from_string(val) is HypervisorProperties.HYPERVISOR_NAME
 
 
-@pytest.mark.parametrize(
-    "val",
-    [
-        "hypervisor_server_count",
-        "Hypervisor_Server_Count",
-        "HyPeRvIsOr_SeRvEr_CoUnT",
-        "running_vms",
-    ],
-)
-def test_hypervisor_server_count_serialization(val):
-    """
-    Tests that variants of HYPERVISOR_SERVER_COUNT can be serialized
-    """
-    assert (
-        HypervisorProperties.from_string(val)
-        is HypervisorProperties.HYPERVISOR_SERVER_COUNT
-    )
-
-
 @pytest.mark.parametrize(
     "val",
     [

tests/enums/props/test_server_properties.py

@@ -45,13 +45,14 @@ def test_flavor_id_serialization(val):
 
 
 @pytest.mark.parametrize(
-    "val", ["hypervisor_id", "Hypervisor_ID", "HyPerVisor_ID", "host_id", "hv_id"]
+    "val",
+    ["hypervisor_name", "Hypervisor_NAME", "HyPerVisor_NamE", "hv_name", "HV_name"],
 )
 def test_hypervisor_id_serialization(val):
     """
-    Tests that variants of HYPERVISOR_ID can be serialized
+    Tests that variants of HYPERVISOR_NAME can be serialized
     """
-    assert ServerProperties.from_string(val) is ServerProperties.HYPERVISOR_ID
+    assert ServerProperties.from_string(val) is ServerProperties.HYPERVISOR_NAME
 
 
 @pytest.mark.parametrize("val", ["image_id", "Image_ID", "ImaGe_iD"])

tests/mappings/test_hypervisor_mapping.py

@@ -85,8 +85,6 @@ def test_client_side_handlers_integer(client_side_test_mappings):
         HypervisorProperties.HYPERVISOR_MEMORY_FREE,
         HypervisorProperties.HYPERVISOR_VCPUS,
         HypervisorProperties.HYPERVISOR_VCPUS_USED,
-        HypervisorProperties.HYPERVISOR_SERVER_COUNT,
-        HypervisorProperties.HYPERVISOR_CURRENT_WORKLOAD,
     ]
     handler = HypervisorMapping.get_client_side_handlers().integer_handler
     mappings = {
@@ -103,7 +101,7 @@ def test_get_chain_mappings():
     Tests get_chain_mapping outputs correctly
     """
     expected_mappings = {
-        HypervisorProperties.HYPERVISOR_ID: ServerProperties.HYPERVISOR_ID,
+        HypervisorProperties.HYPERVISOR_NAME: ServerProperties.HYPERVISOR_NAME,
     }
 
     assert HypervisorMapping.get_chain_mappings() == expected_mappings

tests/mappings/test_server_mapping.py

@@ -210,7 +210,7 @@ def test_get_chain_mappings():
         ServerProperties.PROJECT_ID: ProjectProperties.PROJECT_ID,
         ServerProperties.FLAVOR_ID: FlavorProperties.FLAVOR_ID,
         ServerProperties.IMAGE_ID: ImageProperties.IMAGE_ID,
-        ServerProperties.HYPERVISOR_ID: HypervisorProperties.HYPERVISOR_ID,
+        ServerProperties.HYPERVISOR_NAME: HypervisorProperties.HYPERVISOR_NAME,
     }
 
     assert ServerMapping.get_chain_mappings() == expected_mappings