Windows Azure Blob(4) - Upload large file using blocks

One of the target scenarios for Windows Azure Blob is to enable efficient upload of blobs that are many GBs in size. This is provided by Windows Azure Blob through the following steps:

      ·         Break the Blob (e.g., Movie.avi) to be uploaded into contiguous blocks. For example, a 10GB movie can be broken up into 2500 blocks, each of size 4MB.

·         Give each block a unique ID/name. 

·         PUT each block into the cloud.  

·         After all of the blocks are stored in Windows Azure Storage, then we commit the list of blocks uploaded to represent the blob name they were associated with. Then the HTTP header contains the list of blocks to be committed for this blob. When this operation succeeds, the list of blocks, in the order in which they were listed, now represents the readable version of the blob. The blob can then be read using the GET blob commands described above.

 

The following figure shows adding blocks to the Windows Azure Blob data concepts.

 

 

        static void Main(string[] args)
        {
            long BlockSize = 1024 * 1024;

            FileStream fs = new FileStream("d:\\2.mp3", FileMode.Open);
            byte[] value = new byte[fs.Length];
            fs.Read(value, 0, (int)fs.Length);
            Stream stream = new MemoryStream(value, false);

            long blockSize = BlockSize;
            long startPosition = stream.Position;
            long length = stream.Length - startPosition;
            int numBlocks = (int)Math.Ceiling((double)length / blockSize);
            string[] blockIds = new string[numBlocks];

            //Break the Blob
            for (int i = 0; i < numBlocks; ++i)
            {
                //Give each block a unique ID/name
                string blockId = Convert.ToBase64String(System.BitConverter.GetBytes(i));
                blockIds[i] = blockId;

                if (stream.CanSeek)
                    stream.Position = startPosition + i * blockSize;
                long blockLength = Math.Min(blockSize, length - stream.Position);

                //PUT each block into the cloud
                Uri uri = new Uri("http://yangazuretest.blob.core.windows.net/container1/002.mp3?comp=block&blockid=" + blockId + "&timeout=30");
                HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(uri);
                request.Timeout = 30000;
                request.ReadWriteTimeout = 30000;
                request.Method = "PUT";
                request.Headers.Add("x-ms-date", DateTime.UtcNow.ToString("R", CultureInfo.InvariantCulture));
                request.ContentType = "text/plain; charset=UTF-8";
                request.ContentLength = blockLength;

                string accountName = "your account name";
                string base64Key = "your key";
                byte[] key = Convert.FromBase64String(base64Key);
                ResourceUriComponents uriComponents = new ResourceUriComponents(accountName, "container", null);

                SharedKeyCredentials credentials = new SharedKeyCredentials(accountName, key);
                credentials.SignRequest(request, uriComponents);

                bool retval = false;

                using (Stream requestStream = request.GetRequestStream())
                {
                    CopyStream(stream, requestStream, blockLength);
                    using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
                    {
                        if (response.StatusCode == HttpStatusCode.Created)
                        {
                            retval = true;
                        }
                        else
                        {
                            retval = false;
                        }

                        response.Close();
                    }
                    requestStream.Close();
                }

            }

        }

        static void CopyStream(Stream sourceStream, Stream destinationStream, long length)
        {
            const int BufferSize = 0x10000;
            byte[] buffer = new byte[BufferSize];
            int n = 0;
            long amountLeft = length;

            do
            {
                amountLeft -= n;
                n = sourceStream.Read(buffer, 0, (int)Math.Min(BufferSize, amountLeft));
                if (n > 0)
                {
                    destinationStream.Write(buffer, 0, n);
                }

            } while (n > 0);
        }