Structure change to support PowerFlex 5.0

.github/workflows/pytest.yml

@@ -36,7 +36,7 @@ jobs:
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
       - name: Run Unit Test and Generate report  
         run: |
-          coverage run -m pytest -v tests/test_*.py
+          coverage run -m pytest -v tests/
       - name: Upload Coverage to Codecov
         uses: codecov/codecov-action@v5
         if: ${{ matrix.python-version == '3.12' }}

.gitignore

@@ -4,3 +4,9 @@ __pycache__
 # pytest coverage
 .coverage
 htmlcov
+
+build/
+dist/
+*.egg-info
+
+test.py

PyPowerFlex/__init__.py

@@ -21,10 +21,11 @@
 
 from PyPowerFlex import configuration
 from PyPowerFlex import exceptions
-from PyPowerFlex import objects
-from PyPowerFlex import token
+from PyPowerFlex import powerflex_token
 from PyPowerFlex import utils
-
+from PyPowerFlex.objects import common
+from PyPowerFlex.objects import gen1
+from PyPowerFlex.objects import gen2
 
 __all__ = [
     'PowerFlexClient'
@@ -39,6 +40,7 @@ class PowerFlexClient:
     access to the various storage entities available in the PowerFlex system.
     """
     __slots__ = (
+        # gen1
         '__is_initialized',
         'configuration',
         'token',
@@ -60,7 +62,9 @@ class PowerFlexClient:
         'managed_device',
         'deployment',
         'firmware_repository',
-        'host'
+        'host',
+        # gen2
+        'storage_node',
     )
 
     def __init__(self,
@@ -80,7 +84,7 @@ def __init__(self,
                                                          certificate_path,
                                                          timeout,
                                                          log_level)
-        self.token = token.Token()
+        self.token = powerflex_token.PowerFlexToken()
         self.__is_initialized = False
 
     def __getattr__(self, item):
@@ -98,36 +102,60 @@ def initialize(self):
         Raises:
             PowerFlexClientException: If the PowerFlex API version is lower than 3.0.
         """
+        # common objects here
+        self.add_objects_common()
         self.configuration.validate()
-        self.__add_storage_entity('device', objects.Device)
-        self.__add_storage_entity('fault_set', objects.FaultSet)
-        self.__add_storage_entity('protection_domain',
-                                  objects.ProtectionDomain)
-        self.__add_storage_entity('sdc', objects.Sdc)
-        self.__add_storage_entity('sds', objects.Sds)
-        self.__add_storage_entity('sdt', objects.Sdt)
-        self.__add_storage_entity('snapshot_policy', objects.SnapshotPolicy)
-        self.__add_storage_entity('storage_pool', objects.StoragePool)
-        self.__add_storage_entity('acceleration_pool',
-                                  objects.AccelerationPool)
-        self.__add_storage_entity('system', objects.System)
-        self.__add_storage_entity('volume', objects.Volume)
-        self.__add_storage_entity('utility', objects.PowerFlexUtility)
-        self.__add_storage_entity(
-            'replication_consistency_group',
-            objects.ReplicationConsistencyGroup)
-        self.__add_storage_entity('replication_pair', objects.ReplicationPair)
-        self.__add_storage_entity('service_template', objects.ServiceTemplate)
-        self.__add_storage_entity('managed_device', objects.ManagedDevice)
-        self.__add_storage_entity('deployment', objects.Deployment)
-        self.__add_storage_entity(
-            'firmware_repository',
-            objects.FirmwareRepository)
-        self.__add_storage_entity('host', objects.Host)
+
         utils.init_logger(self.configuration.log_level)
         if version.parse(self.system.api_version()) < version.Version('3.0'):
             raise exceptions.PowerFlexClientException(
                 'PowerFlex (VxFlex OS) versions lower than '
                 '3.0 are not supported.'
             )
+
+        if version.parse(self.system.api_version()) > version.Version('3.0') and \
+           version.parse(self.system.api_version()) < version.Version('5.0'):
+            self.add_objects_gen1()
+        elif version.parse(self.system.api_version()) >= version.Version('5.0'):
+            self.add_objects_gen2()
         self.__is_initialized = True
+
+    def add_objects_common(self):
+        """Add common objects here."""
+        self.__add_storage_entity('system', common.System)
+        self.__add_storage_entity('sdc', common.Sdc)
+        self.__add_storage_entity('sdt', common.Sdt)
+        self.__add_storage_entity('host', common.Host)
+        self.__add_storage_entity('utility', common.PowerFlexUtility)
+
+
+    def add_objects_gen1(self):
+        """Add gen1 objects here."""
+        self.__add_storage_entity('device', gen1.Device)
+        self.__add_storage_entity(
+            'fault_set', gen1.FaultSet)
+        self.__add_storage_entity('protection_domain',
+                                  gen1.ProtectionDomain)
+        self.__add_storage_entity('sds', gen1.Sds)
+        self.__add_storage_entity(
+            'snapshot_policy', gen1.SnapshotPolicy)
+        self.__add_storage_entity('storage_pool', gen1.StoragePool)
+        self.__add_storage_entity('acceleration_pool',
+                                  gen1.AccelerationPool)
+        self.__add_storage_entity('volume', gen1.Volume)
+        self.__add_storage_entity(
+            'replication_consistency_group',
+            gen1.ReplicationConsistencyGroup)
+        self.__add_storage_entity('replication_pair', gen1.ReplicationPair)
+        self.__add_storage_entity('service_template', gen1.ServiceTemplate)
+        self.__add_storage_entity('managed_device', gen1.ManagedDevice)
+        self.__add_storage_entity('deployment', gen1.Deployment)
+        self.__add_storage_entity(
+            'firmware_repository',
+            gen1.FirmwareRepository)
+
+    def add_objects_gen2(self):
+        """Add gen2 objects here."""
+        self.__add_storage_entity('storage_node', gen2.StorageNode)
+        self.__add_storage_entity('protection_domain', gen2.ProtectionDomain)
+        self.__add_storage_entity('storage_pool', gen2.StoragePool)

PyPowerFlex/base_client.py

@@ -21,10 +21,11 @@
 
 import requests
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
-
+from marshmallow import EXCLUDE, Schema
 from PyPowerFlex import exceptions
 from PyPowerFlex import utils
 
+
 requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
 LOG = logging.getLogger(__name__)
 
@@ -110,21 +111,27 @@ def get_auth_headers(self, request_type=None):
             'content-type': 'application/json'
         }
 
-    def send_request(self, method, url, params=None, **url_params):
+    def send_request(self, method, url, params=None, use_base_url=True, **url_params):
         """
         Send a request to the PowerFlex API.
 
         Args:
             method (str): The HTTP method.
             url (str): The URL.
+            use_base_url (bool, optional): Whether to use the base URL. Defaults to True.
             params (dict): The parameters.
             url_params (dict): The URL parameters.
 
         Returns:
             Response: The response object.
         """
         params = params or {}
-        request_url = f"{self.base_url}{url.format(**url_params)}"
+        use_base_url = True if use_base_url is None else use_base_url
+        if use_base_url:
+            request_url = f"{self.base_url}{url.format(**url_params)}"
+        else:
+            request_url = f"{self.base_url.removesuffix('/api')}{url.format(**url_params)}"
+
         version = self.login()
         request_params = {
             'headers': self.get_auth_headers(method),
@@ -157,19 +164,21 @@ def send_get_request(self, url, params=None, **url_params):
         response = self.send_request(self.GET, url, params, **url_params)
         return response, response.json()
 
-    def send_post_request(self, url, params=None, **url_params):
+    def send_post_request(self, url, use_base_url=True, params=None, **url_params):
         """
         Send a POST request to the PowerFlex API.
 
         Args:
             url (str): The URL.
+            use_base_url (bool, optional): Whether to use the base URL. Defaults to True.
             params (dict): The parameters.
             url_params (dict): The URL parameters.
 
         Returns:
             tuple: The response object and the response content.
         """
-        response = self.send_request(self.POST, url, params, **url_params)
+        response = self.send_request(
+            self.POST, url, params, use_base_url, ** url_params)
         return response, response.json()
 
     def send_put_request(self, url, params=None, **url_params):
@@ -373,6 +382,7 @@ class EntityRequest(Request):
     base_type_special_action_url = '/types/{entity}/instances/action/{action}'
     query_mdm_cluster_url = '/instances/{entity}/queryMdmCluster'
     list_statistics_url = '/types/{entity}/instances/action/{action}'
+    metrics_query_url = '/dtapi/rest/v1/metrics/query'
     service_template_url = '/V1/ServiceTemplate'
     managed_device_url = '/V1/ManagedDevice'
     deployment_url = '/V1/Deployment'
@@ -420,6 +430,9 @@ def _delete_entity(self, entity_id, params=None):
             entity_id (str): The ID of the entity.
             params (dict, optional): Parameters for the entity.
 
+        Returns:
+            dict: The response from the API.
+
         Raises:
             PowerFlexFailDeleting: If the entity fails to be deleted.
         """
@@ -435,6 +448,7 @@ def _delete_entity(self, entity_id, params=None):
                                                    response)
             LOG.error(exc.message)
             raise exc
+        return response
 
     def _rename_entity(self, action, entity_id, params=None):
         """
@@ -446,7 +460,7 @@ def _rename_entity(self, action, entity_id, params=None):
             params (dict, optional): Parameters for the entity.
 
         Returns:
-            dict: The renamed entity.
+            dict: The response from the API.
 
         Raises:
             PowerFlexFailRenaming: If the entity fails to be renamed.
@@ -461,8 +475,7 @@ def _rename_entity(self, action, entity_id, params=None):
                                                    response)
             LOG.error(exc.message)
             raise exc
-
-        return self.get(entity_id=entity_id)
+        return response
 
     def get(self, entity_id=None, filter_fields=None, fields=None):
         """
@@ -576,6 +589,7 @@ def _perform_entity_operation_based_on_action(
                 self.entity, entity_id, action, response)
             LOG.error(exc.message)
             raise exc
+        return response
 
     def _query_selected_statistics(self, action, params=None):
         """
@@ -606,3 +620,50 @@ def _query_selected_statistics(self, action, params=None):
             LOG.error(exc.message)
             raise exc
         return response
+
+    def query_metrics(self, resource_type, ids=None, metrics=None):
+        """Query PowerFlex resource metrics.
+
+        :param resource_type: str
+        :param ids: list
+        :param metrics: list
+        :return: dict
+        """
+
+        params = {
+            'resource_type': resource_type
+        }
+        if ids is not None:
+            params['ids'] = ids
+        if metrics is not None:
+            params['metrics'] = metrics
+
+        r, response = self.send_post_request(self.metrics_query_url,
+                                             use_base_url=False,
+                                             params=params)
+        if r.status_code != requests.codes.ok:
+            msg = (
+                f"Failed to query {resource_type} statistics. "
+                f"Error: {response}"
+            )
+            LOG.error(msg)
+            raise exceptions.PowerFlexClientException(msg)
+
+        return response
+
+class BaseSchema(Schema):
+    """Base schema."""
+    # pylint: disable=too-few-public-methods
+
+    def on_bind_field(self, field_name, field_obj):
+        field_obj.data_key = camelcase(field_obj.data_key or field_name)
+
+    class Meta:
+        """Meta class."""
+        unknown = EXCLUDE
+
+
+def camelcase(s):
+    """Convert snake case to camel case."""
+    parts = iter(s.split("_"))
+    return next(parts) + "".join(i.title() for i in parts)

PyPowerFlex/constants.py

@@ -17,6 +17,7 @@
 
 # pylint: disable=too-few-public-methods
 
+
 class StoragePoolConstants:
     """
     This class holds constants related to StoragePool.
@@ -240,6 +241,10 @@ class StoragePoolConstants:
     DEFAULT_STATISTICS_PROPERTIES_ABOVE_3_5 = [
         "thinCapacityAllocatedInKm", "thinUserDataCapacityInKb"]
 
+    DEFAULT_QUERY_METRICS = [
+        ""
+    ]
+
 
 class VolumeConstants:
     """
@@ -365,3 +370,47 @@ class SnapshotPolicyConstants:
         "numOfExpiredButLockedSnapshots",
         "numOfSrcVols",
         "srcVolIds"]
+
+
+class StorageNodeConstants:
+    """
+    This class holds statistics constants related to StorageNode.
+    """
+    DEFAULT_STATISTICS_METRICS = [
+        "storage_fe_write_latency",
+        "device_local_read_bandwidth",
+        "device_local_read_iops",
+        "device_remote_write_bandwidth",
+        "device_remote_write_iops",
+        "total_device_write_bandwidth",
+        "total_device_write_iops",
+        "avg_device_write_io_size",
+        "storage_fe_trim_latency",
+        "device_local_write_bandwidth",
+        "device_local_write_iops",
+        "avg_device_write_latency",
+        "storage_fe_trim_bandwidth",
+        "storage_fe_trim_iops",
+        "avg_fe_trim_io_size",
+        "avg_device_pmem_write_latency",
+        "device_remote_read_bandwidth",
+        "device_remote_read_iops",
+        "avg_device_pmem_read_latency",
+        "storage_fe_write_bandwidth",
+        "storage_fe_write_iops",
+        "avg_fe_write_io_size",
+        "storage_fe_read_bandwidth",
+        "storage_fe_read_iops",
+        "avg_fe_read_io_size",
+        "total_device_pmem_write_bandwidth",
+        "total_device_pmem_write_iops",
+        "avg_device_pmem_write_io_size",
+        "total_device_pmem_read_bandwidth",
+        "total_device_pmem_read_iops",
+        "avg_device_pmem_read_io_size",
+        "total_device_read_bandwidth",
+        "total_device_read_iops",
+        "avg_device_read_io_size",
+        "raw_total",
+        "storage_fe_read_latency",
+        "avg_device_read_latency"]

PyPowerFlex/exceptions.py

@@ -130,3 +130,11 @@ def __init__(self, entity, entity_id, action, response=None):
         self.response = response
         if response:
             self.message = f"{self.message} Error: {response}"
+
+
+def nonupdatable_exception(field, entity, entity_id=None):
+    """Return a PowerFlexClientException for non-updatable fields."""
+    msg = (
+        f'{field} cannot be updated after creation for PowerFlex {entity} {entity_id}'
+    )
+    return PowerFlexClientException(msg)