Merge pull request #152 from EmmaZhu/master

Ehance error handling and add support to download from one Azure Blob/File to multiple streams

BreakingChanges.txt

@@ -1,3 +1,7 @@
+Tracking Breaking Changes since 0.8.1
+
+   - Change to overwrite destination's metadata with source's metadata in async copying instead of keeping destination's metadata
+
 Tracking Breaking Changes since 0.7.1
 
   - Changed following callbacks to asynchronized methods:

README.md

@@ -1,4 +1,4 @@
-# Microsoft Azure Storage Data Movement Library (0.8.1)
+# Microsoft Azure Storage Data Movement Library (0.9.0)
 
 The Microsoft Azure Storage Data Movement Library designed for high-performance uploading, downloading and copying Azure Storage Blob and File. This library is based on the core data movement framework that powers [AzCopy](https://azure.microsoft.com/documentation/articles/storage-use-azcopy/).
 

changelog.txt

@@ -1,3 +1,11 @@
+2018.10.25 Version 0.9.0
+ * All Services on both .Net Framework and .Net Core
+   - Upgrade azure storage client library to 9.3.2
+   - Enhance exception handling during enumeration to add more detailed error messages
+   - Disable transational MD5 on sync copying when using HTTPS protocol
+   - Change to overwrite destination's metadata with source's metadata in async copying instead of keeping destination's metadata
+   - Add support for download from the same Azure Blob/Azure File to different Stream intances with multiple transfer jobs
+
 2018.07.31 Version 0.8.1
  * All Services on both .Net Framework and .Net Core
    - Upgrade azure storage client library to 9.3.0

lib/DataMovement.csproj

@@ -58,8 +58,8 @@
       <HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll</HintPath>
       <Private>True</Private>
     </Reference>
-    <Reference Include="Microsoft.WindowsAzure.Storage, Version=9.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\packages\WindowsAzure.Storage.9.3.0\lib\net45\Microsoft.WindowsAzure.Storage.dll</HintPath>
+    <Reference Include="Microsoft.WindowsAzure.Storage, Version=9.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <HintPath>..\packages\WindowsAzure.Storage.9.3.2\lib\net45\Microsoft.WindowsAzure.Storage.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="Newtonsoft.Json, Version=10.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">

lib/DirectoryTransferContext.cs

@@ -34,7 +34,7 @@ public DirectoryTransferContext(TransferCheckpoint checkpoint)
         /// Initializes a new instance of the <see cref="DirectoryTransferContext" /> class.
         /// </summary>
         /// <param name="journalStream">The stream into which the transfer journal info will be written into. 
-        /// It can resume the previours paused transfer from its journal stream.</param>
+        /// It can resume the previous paused transfer from its journal stream.</param>
         public DirectoryTransferContext(Stream journalStream)
             :base(journalStream)
         {

lib/TransferControllers/AsyncCopyControllers/BlobAsyncCopyController.cs

@@ -88,6 +88,11 @@ protected override Task<string> DoStartCopyAsync()
         {
             AccessCondition destAccessCondition = Utils.GenerateConditionWithCustomerCondition(this.destLocation.AccessCondition);
 
+            // To copy from source to blob, DataMovement Library should overwrite destination's properties and meta datas.
+            // Clear destination's meta data here to avoid using destination's meta data.
+            // Please reference to https://docs.microsoft.com/en-us/rest/api/storageservices/Copy-Blob.
+            this.destBlob.Metadata.Clear();
+
             if (null != this.SourceUri)
             {
                 return this.destBlob.StartCopyAsync(

lib/TransferControllers/AsyncCopyControllers/FileAsyncCopyController.cs

@@ -76,6 +76,11 @@ protected override Task DoFetchDestAttributesAsync()
 
         protected override Task<string> DoStartCopyAsync()
         {
+            // To copy from source to blob, DataMovement Library should overwrite destination's properties and meta datas.
+            // Clear destination's meta data here to avoid using destination's meta data.
+            // Please reference to https://docs.microsoft.com/en-us/rest/api/storageservices/copy-file.
+            this.destFile.Metadata.Clear();
+
             OperationContext operationContext = Utils.GenerateOperationContext(this.TransferContext);
             if (null != this.SourceUri)
             {

lib/TransferEnumerators/AzureBlobEntry.cs

@@ -33,5 +33,10 @@ public CloudBlob Blob
             get;
             private set;
         }
+
+        public override string ToString()
+        {
+            return Blob.ConvertToString();
+        }
     }
 }

lib/TransferEnumerators/AzureFileEntry.cs

@@ -35,5 +35,10 @@ public CloudFile File
             get;
             private set;
         }
+
+        public override string ToString()
+        {
+            return File.ConvertToString();
+        }
     }
 }

lib/TransferEnumerators/AzureFileEnumerator.cs

@@ -230,7 +230,7 @@ private IEnumerable<TransferEntry> EnumerateLocationRecursive(CancellationToken
                         string errorMessage = string.Format(
                             CultureInfo.CurrentCulture,
                             Resources.FailedToEnumerateDirectory,
-                            this.location.FileDirectory.SnapshotQualifiedUri.AbsoluteUri,
+                            directory.SnapshotQualifiedUri.AbsoluteUri,
                             string.Empty);
 
                         TransferException exception =

lib/TransferEnumerators/EnumerateDirectoryHelper.cs

@@ -229,7 +229,11 @@ private static IEnumerable<string> InternalEnumerateFiles(
                     // since the OS can block access to some paths. Getting files from a location
                     // will force path discovery checks which will indicate whether or not the user
                     // is authorized to access the directory.
-                    LongPathDirectory.GetFiles(folder);
+                    foreach (var fileItem in LongPathDirectory.EnumerateFileSystemEntries(folder, "*", SearchOption.TopDirectoryOnly))
+                    {
+                        // Just try to get the first item in directly to check whether has permission to access the directory.
+                        break;
+                    }
                 }
                 catch (SecurityException)
                 {

lib/TransferEnumerators/FileEntry.cs

@@ -33,5 +33,10 @@ public string FullPath
             get;
             private set;
         }
+
+        public override string ToString()
+        {
+            return FullPath;
+        }
     }
 }

lib/TransferJobs/MultipleObjectsTransfer.cs

@@ -386,11 +386,18 @@ private void ListNewTransfers(CancellationToken cancellationToken)
 
                     this.shouldTransferQueue.EnqueueJob(async () =>
                     {
-                        SingleObjectTransfer candidate = this.CreateTransfer(entry);
+                        try
+                        {
+                            SingleObjectTransfer candidate = this.CreateTransfer(entry);
 
-                        bool shouldTransfer = shouldTransferCallback == null || await shouldTransferCallback(candidate.Source.Instance, candidate.Destination.Instance);
+                            bool shouldTransfer = shouldTransferCallback == null || await shouldTransferCallback(candidate.Source.Instance, candidate.Destination.Instance);
 
-                        return new Tuple<SingleObjectTransfer, TransferEntry>(shouldTransfer ? candidate : null, entry);
+                            return new Tuple<SingleObjectTransfer, TransferEntry>(shouldTransfer ? candidate : null, entry);
+                        }
+                        catch (Exception ex)
+                        {
+                            throw new TransferException(TransferErrorCode.FailToEnumerateDirectory, string.Format(CultureInfo.CurrentCulture, "Error happens when handling entry {0}: {1}", entry.ToString(), ex.Message), ex);
+                        }
                     });
                 }
             }

lib/TransferJobs/StreamLocation.cs

@@ -9,13 +9,17 @@ namespace Microsoft.WindowsAzure.Storage.DataMovement
     using System;
     using System.Globalization;
     using System.IO;
+    using System.Runtime.CompilerServices;
     using System.Runtime.Serialization;
 
 #if !BINARY_SERIALIZATION
     [DataContract]
 #endif
     internal class StreamLocation : TransferLocation
     {
+        private static ConditionalWeakTable<Stream, string> StreamIdTable = new ConditionalWeakTable<Stream, string>();
+        string streamLocationIdentity = string.Empty;
+
         /// <summary>
         /// Initializes a new instance of the <see cref="StreamLocation"/> class.
         /// </summary>
@@ -28,6 +32,17 @@ public StreamLocation(Stream stream)
             }
 
             this.Stream = stream;
+
+            string identity = null;
+            if (StreamIdTable.TryGetValue(stream, out identity))
+            {
+                this.streamLocationIdentity = identity;
+            }
+            else
+            {
+                this.streamLocationIdentity = Guid.NewGuid().ToString();
+                StreamIdTable.Add(stream, this.streamLocationIdentity);
+            }
         }
 
 #if !BINARY_SERIALIZATION
@@ -105,7 +120,7 @@ public override void Validate()
         //     A string that represents the transfer location.
         public override string ToString()
         {
-            return this.Stream.ToString();
+            return streamLocationIdentity;
         }
     }
 }