Release 0.2.18

Enhancements:
* [API-123] Add keep_for option for snap operation
* [API-122] Add job for NFS share creation
* [API-121] Add retry for drop connection error.
* [API-120] performance enhancement for VNX API.

README.rst

@@ -10,7 +10,7 @@ StorOps: The Python Library for VNX & Unity
 .. image:: https://img.shields.io/pypi/v/storops.svg
     :target: https://pypi.python.org/pypi/storops
 
-VERSION: 0.2.17
+VERSION: 0.2.18
 
 A minimalist Python library to manage VNX/Unity systems.
 This document lies in the source code and go with the release.

requirements.txt

@@ -5,7 +5,7 @@ paramiko>=1.13.0
 python-dateutil>=2.4.2
 PyYAML>=3.11
 requests!=2.9.0,>=2.8.1
-retryz>=0.1.7
+retryz>=0.1.8
 cachez>=0.1.0
 six>=1.9.0
 bitmath>=1.3.0

storops/exception.py

@@ -424,11 +424,15 @@ class VNXLockRequiredException(VNXException):
 
 @cli_exception
 class VNXSpNotAvailableError(VNXException):
-    error_message = ('End of data stream',
-                     'connection refused',
+    error_message = ('connection refused',
                      'A network error occurred while trying to connect')
 
 
+@cli_exception
+class VNXDropConnectionError(VNXException):
+    error_message = 'End of data stream'
+
+
 @cli_exception
 class VNXNotSupportedError(VNXException):
     error_message = 'commands are not supported by the target storage system'
@@ -970,3 +974,22 @@ class VNXNasCommandNoError(VNXException):
 @cli_exception
 class VNXMoverInterfaceNotExistsError(VNXMoverInterfaceError):
     error_regex = 'network interface .* does not exist'
+
+
+class UnityJobException(UnityException):
+    """Unity Job exception.
+
+    Any job related exception should inherit this exception."""
+    pass
+
+
+class JobTimeoutException(UnityJobException):
+    message = "Timeout when waiting for job completion."
+
+
+class JobStateError(UnityJobException):
+    message = "Job failed in {state}."
+
+    def __init__(self, **kwargs):
+        self.message = JobStateError.message.format(**kwargs)
+        super(JobStateError, self).__init__(self.message)

storops/unity/resource/job.py

@@ -15,14 +15,96 @@
 #    under the License.
 from __future__ import unicode_literals
 
+import retryz
+from storops import exception as ex
 from storops.unity.resource import UnityResource, UnityResourceList, \
     UnityAttributeResource
+from storops.unity.enums import FSSupportedProtocolEnum
 
+import storops
+from storops.unity import enums
 __author__ = 'Cedric Zhuang'
 
 
 class UnityJob(UnityResource):
-    pass
+    @classmethod
+    def create_nfs_share(cls, cli, pool, nas_server, name, size,
+                         is_thin=None,
+                         tiering_policy=None, async=True):
+        pool_clz = storops.unity.resource.pool.UnityPool
+        nas_server_clz = storops.unity.resource.nas_server.UnityNasServer
+
+        pool = pool_clz.get(cli, pool)
+        nas_server = nas_server_clz.get(cli, nas_server)
+        proto = FSSupportedProtocolEnum.NFS
+
+        job_req_body = {
+            'description': 'Creating Filesystem and share',
+            'tasks': []
+        }
+        task_body = {
+            'action': 'createFilesystem',
+            'description': 'Create File System',
+            'name': 'CreateNewFilesystem',
+            'object': 'storageResource',
+            'parametersIn': {
+                'name': name,
+                'description': '',
+                'fsParameters': {},
+                'nfsShareCreate': []
+            }
+        }
+        fs_parameters = {
+            'pool': pool,
+            'nasServer': nas_server,
+            'supportedProtocols': proto,
+            'isThinEnabled': is_thin,
+            'size': size,
+            'fastVPParameters': {
+                'tieringPolicy': tiering_policy
+            }
+        }
+        nfs_share_create = {
+            'name': name,
+            'path': '/',
+
+        }
+        task_body['parametersIn']['fsParameters'] = cli.make_body(
+            fs_parameters)
+        task_body['parametersIn']['nfsShareCreate'].append(
+            cli.make_body(nfs_share_create))
+        job_req_body['tasks'].append(task_body)
+
+        resp = cli.post(cls().resource_class,
+                        **job_req_body)
+        resp.raise_if_err()
+        job = cls(_id=resp.resource_id, cli=cli)
+        if not async:
+            job.wait_job_completion()
+        return job
+
+    def check_errors(self):
+        if self.state == enums.JobStateEnum.COMPLETED:
+            return True
+        elif self.state in (enums.JobStateEnum.FAILED,
+                            enums.JobStateEnum.ROLLING_BACK,
+                            enums.JobStateEnum.COMPLETED_WITH_ERROR):
+            raise ex.JobStateError(state=self.state.name)
+        return False
+
+    def wait_job_completion(self, **kwargs):
+        interval = kwargs.pop('interval', 5)
+        timeout = kwargs.pop('timeout', 3600)
+
+        @retryz.retry(timeout=timeout, wait=interval, on_return=False)
+        def _do_update():
+            self.update()
+            return self.check_errors()
+
+        try:
+            _do_update()
+        except retryz.RetryTimeoutError:
+            raise ex.JobTimeoutException()
 
 
 class UnityJobList(UnityResourceList):
@@ -59,3 +141,24 @@ class UnityLocalizedMessageList(UnityResourceList):
     @classmethod
     def get_resource_class(cls):
         return UnityLocalizedMessage
+
+
+def wait_job_completion(job, **kwargs):
+    interval = kwargs.pop('interval', 5)
+    timeout = kwargs.pop('timeout', 3600)
+
+    @retryz.retry(timeout=timeout, wait=interval, on_return=False)
+    def _do_update():
+        job.update()
+        if job.state == enums.JobStateEnum.COMPLETED:
+            return True
+        elif job.state in (enums.JobStateEnum.FAILED,
+                           enums.JobStateEnum.ROLLING_BACK,
+                           enums.JobStateEnum.COMPLETED_WITH_ERROR):
+            raise ex.JobStateError(state=job.state.name)
+        return False
+
+    try:
+        _do_update()
+    except retryz.RetryTimeoutError:
+        raise ex.JobTimeoutException()

storops/unity/resource/pool.py

@@ -53,6 +53,13 @@ def create_lun(self, lun_name=None, size_gb=1, sp=None, host_access=None,
                                snap_schedule=snap_schedule,
                                iolimit_policy=iolimit_policy)
 
+    def create_nfs_share(self, nas_server, name, size, is_thin=None,
+                         tiering_policy=None):
+        clz = storops.unity.resource.job.UnityJob
+        return clz.create_nfs_share(
+            self._cli, self, nas_server, name, size,
+            is_thin, tiering_policy, False)
+
 
 class UnityPoolList(UnityResourceList):
     @classmethod

storops/vnx/block_cli.py

@@ -521,7 +521,7 @@ def get_snap(self, name=None, res=None):
 
     @command
     def create_snap(self, res_id, snap_name,
-                    allow_rw=True, auto_delete=False):
+                    allow_rw=True, auto_delete=False, keep_for=None):
         cmd = ['snap', '-create']
         try:
             cmd += int_var('-res', res_id)
@@ -533,6 +533,8 @@ def create_snap(self, res_id, snap_name,
         cmd += text_var('-name', snap_name)
         cmd += yes_no_var('-allowReadWrite', allow_rw)
         cmd += yes_no_var('-allowAutoDelete', auto_delete)
+        cmd += text_var('-keepFor', keep_for)
+
         return cmd
 
     @command
@@ -550,13 +552,14 @@ def copy_snap(self, src_name, tgt_name,
 
     @command
     def modify_snap(self, name, new_name=None, desc=None,
-                    auto_delete=None, allow_rw=None):
+                    auto_delete=None, allow_rw=None, keep_for=None):
         opt = []
         if new_name is not None and name != new_name:
             opt += text_var('-name', new_name)
         opt += text_var('-descr', desc)
         opt += yes_no_var('-allowAutoDelete', auto_delete)
         opt += yes_no_var('-allowReadWrite', allow_rw)
+        opt += text_var('-keepFor', keep_for)
         if len(opt) > 0:
             cmd = ['snap', '-modify', '-id', name] + opt
         else:
@@ -826,18 +829,18 @@ def execute(self, params, ip=None):
         if params is not None and len(params) > 0:
             if ip is None:
                 ip = self.ip
-            cmd = self._heart_beat.get_cmd_prefix(ip) + params
-            output = self._heart_beat.execute_cmd(ip, cmd)
+            output = self.do(ip, params)
         else:
             log.info('no command to execute.  return empty.')
             output = ''
         return output
 
-    def execute_dual(self, params):
-        def do(sp_ip):
-            cmd_to_exec = self._heart_beat.get_cmd_prefix(sp_ip) + params
-            return self._heart_beat.execute_cmd(sp_ip, cmd_to_exec)
+    @retry(on_error=ex.VNXDropConnectionError)
+    def do(self, ip, params):
+        cmd = self._heart_beat.get_cmd_prefix(ip) + params
+        return self._heart_beat.execute_cmd(ip, cmd)
 
+    def execute_dual(self, params):
         ip_list = self._heart_beat.get_all_alive_sps_ip()
         if not self._heart_beat.is_all_sps_alive():
             raise ex.VNXSPDownError(
@@ -847,5 +850,5 @@ def do(sp_ip):
         output = []
         if params is not None and len(params) > 0:
             pool = ThreadPool(len(ip_list))
-            output = pool.map(do, ip_list)
+            output = pool.map(lambda ip: self.do(ip, params), ip_list)
         return tuple(output)

storops/vnx/heart_beat.py

@@ -59,6 +59,15 @@ def available(self, available):
         self._available = available
         self.timestamp = datetime.now()
 
+    def is_updated_within(self, seconds):
+        current = datetime.now()
+        if self.timestamp is None:
+            ret = False
+        else:
+            delta = current - self.timestamp
+            ret = delta.total_seconds() < seconds
+        return ret
+
     @property
     def latency(self):
         return self._latency.value()
@@ -78,7 +87,7 @@ def __str__(self):
         return self.__repr__()
 
 
-class _NodeInfoMap(object):
+class NodeInfoMap(object):
     def __init__(self):
         self._map = dict()
 
@@ -131,15 +140,15 @@ def __init__(self, username=None, password=None, scope=0,
                                             sec_file=sec_file,
                                             timeout=timeout,
                                             naviseccli=naviseccli)
-        self._node_map = _NodeInfoMap()
+        self._node_map = NodeInfoMap()
         self._interval = interval
         self._heartbeat_thread = None
         if interval > 0:
             self._heartbeat_thread = daemon(self._run)
         self.command_count = 0
 
     def reset(self):
-        self._node_map = _NodeInfoMap()
+        self._node_map = NodeInfoMap()
         self.command_count = 0
 
     def _get_sp_by_category(self):
@@ -215,8 +224,10 @@ def execute_cmd(self, ip, cmd):
                 'cannot authenticate with user {}.'.format(self._username))
         out = self.execute_naviseccli(cmd)
         try:
-            ex.check_error(out, ex.VNXSpNotAvailableError,
-                           ex.VNXCredentialError)
+            ex.check_error(out,
+                           ex.VNXSpNotAvailableError,
+                           ex.VNXCredentialError,
+                           ex.VNXDropConnectionError)
             available = True
             latency = time() - start
         except ex.VNXSpNotAvailableError:
@@ -238,7 +249,9 @@ def execute_cmd(self, ip, cmd):
 
     def _ping_sp(self, ip):
         try:
-            self.execute_cmd(ip, self.get_agent(ip))
+            node = self._node_map.get_node_by_ip(ip)
+            if node and not node.is_updated_within(self.interval):
+                self.execute_cmd(ip, self.get_agent(ip))
         except OSError:
             log.debug('skip heartbeat, naviseccli not available.')
         except ex.VNXSPDownError:

storops/vnx/resource/snap.py

@@ -78,10 +78,10 @@ def copy(self, new_name,
         return VNXSnap(name=new_name, cli=self._cli)
 
     def modify(self, new_name=None, desc=None,
-               auto_delete=None, allow_rw=None):
+               auto_delete=None, allow_rw=None, keep_for=None):
         name = self._get_name()
         out = self._cli.modify_snap(name, new_name, desc, auto_delete,
-                                    allow_rw, poll=self.poll)
+                                    allow_rw, keep_for, poll=self.poll)
         ex.raise_if_err(out, 'failed to modify snap {}.'.format(name),
                         default=ex.VNXModifySnapError)
         if new_name is not None: