Skip to content

Commit c7f1639

Browse files
SanyyazzzSanyyazzz
committed
add new method
1 parent 0bacba6 commit c7f1639

5 files changed

Lines changed: 143 additions & 16 deletions

File tree

‎ManagedCode.Storage.Client/IStorageClient.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ public interface IStorageClient
1313
Task<Result<BlobMetadata>> UploadFile(FileInfo fileInfo, string apiUrl, string contentName, CancellationToken cancellationToken = default);
1414
Task<Result<BlobMetadata>> UploadFile(byte[] bytes, string apiUrl, string contentName, CancellationToken cancellationToken = default);
1515
Task<Result<BlobMetadata>> UploadFile(string base64, string apiUrl, string contentName, CancellationToken cancellationToken = default);
16-
Task<Result> UploadLargeFile(Stream file, string сreateApiUrl, string uploadApiUrl, string completeApiUrl, Action<double>? onProgressChanged, CancellationToken cancellationToken = default);
16+
Task<Result> UploadLargeFileUsingStream(Stream file, string сreateApiUrl, string uploadApiUrl, string completeApiUrl, Action<double>? onProgressChanged, CancellationToken cancellationToken = default);
17+
Task<Result> UploadLargeFileUsingMerge(Stream file, string uploadApiUrl, string mergeApiUrl, Action<double>? onProgressChanged, CancellationToken cancellationToken);
1718
Task<Result<LocalFile>> DownloadFile(string fileName, string apiUrl, string? path = null, CancellationToken cancellationToken = default);
1819
}

‎ManagedCode.Storage.Client/StorageClient.cs‎

Lines changed: 66 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System;
33
using System.Collections.Generic;
44
using System.IO;
5+
using System.Linq;
56
using System.Net;
67
using System.Net.Http;
78
using System.Net.Http.Json;
@@ -124,7 +125,7 @@ public async Task<Result<LocalFile>> DownloadFile(string fileName, string apiUrl
124125
}
125126
}
126127

127-
public async Task<Result> UploadLargeFile(Stream file,
128+
public async Task<Result> UploadLargeFileUsingStream(Stream file,
128129
string сreateApiUrl,
129130
string uploadApiUrl,
130131
string completeApiUrl,
@@ -186,4 +187,68 @@ public async Task<Result> UploadLargeFile(Stream file,
186187

187188
return await mergeResult.Content.ReadFromJsonAsync<Result>(cancellationToken: cancellationToken);
188189
}
190+
191+
public async Task<Result> UploadLargeFileUsingMerge(Stream file,
192+
string uploadApiUrl,
193+
string mergeApiUrl,
194+
Action<double>? onProgressChanged,
195+
CancellationToken cancellationToken)
196+
{
197+
var bufferSize = 4096000; //TODO: chunk size get from config
198+
var buffer = new byte[bufferSize];
199+
int bytesRead;
200+
int chunkIndex = 1;
201+
var fileCRC = Crc32Helper.Calculate(file);
202+
var partOfProgress = file.Length / bufferSize;
203+
204+
var semaphore = new SemaphoreSlim(0, 4);
205+
var tasks = new List<Task<HttpResponseMessage>>();
206+
while ((bytesRead = await file.ReadAsync(buffer, 0, buffer.Length, cancellationToken)) > 0)
207+
{
208+
var task = Task.Run(() =>
209+
{
210+
try
211+
{
212+
semaphore.WaitAsync(cancellationToken);
213+
using (var memoryStream = new MemoryStream(buffer, 0, bytesRead))
214+
{
215+
var content = new StreamContent(memoryStream);
216+
217+
using (var chunk = new MultipartFormDataContent())
218+
{
219+
chunk.Add(content, "chunk", "file");
220+
chunk.Add(new StringContent(chunkIndex.ToString()), "Payload.ChunkIndex");
221+
chunk.Add(new StringContent(bufferSize.ToString()), "Payload.ChunkSize");
222+
chunk.Add(new StringContent(fileCRC.ToString()), "Payload.FullCRC");
223+
224+
var result = _httpClient.PostAsync(uploadApiUrl, chunk, cancellationToken);
225+
onProgressChanged?.Invoke(partOfProgress * chunkIndex);
226+
227+
return result;
228+
}
229+
}
230+
}
231+
finally
232+
{
233+
semaphore.Release();
234+
}
235+
}, cancellationToken);
236+
237+
tasks.Add(task);
238+
chunkIndex++;
239+
}
240+
241+
var tasksResult = await Task.WhenAll(tasks.ToArray());
242+
var blobNames = tasksResult
243+
.Select(async x =>
244+
{
245+
var content = await x.Content.ReadFromJsonAsync<Result<string>>(cancellationToken: cancellationToken);
246+
return content.Value;
247+
});
248+
249+
var mergeResult = await _httpClient.PostAsync(mergeApiUrl, JsonContent.Create(
250+
new {fileCrc = fileCRC, blobNames = blobNames}), cancellationToken);
251+
252+
return await mergeResult.Content.ReadFromJsonAsync<Result>(cancellationToken: cancellationToken);
253+
}
189254
}

‎ManagedCode.Storage.IntegrationTests/Constants/ApiEndpoints.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ public static class Base
88
{
99
public const string UploadFile = "{0}/upload";
1010
public const string UploadCreateFile = "{0}/upload-chunks/create";
11-
public const string UploadFileChunks = "{0}/upload-chunks";
11+
public const string UploadFileChunksUsingStream = "{0}/upload-chunks-stream";
12+
public const string UploadFileChunksUsingMerge = "{0}/upload-chunks-merge";
1213
public const string UploadFileComplete = "{0}/upload-chunks/complete";
1314
public const string DownloadFile = "{0}/download";
1415
}

‎ManagedCode.Storage.IntegrationTests/TestApp/Controllers/AzureTestController.cs‎

Lines changed: 39 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,14 @@ public AzureTestController(IAzureStorage storage) : base(storage)
2121
_storage = storage;
2222
}
2323

24-
[HttpPost("upload-chunks/create")]
24+
[HttpPost("upload-chunks-stream/create")]
2525
public async Task<Result<BlobMetadata>> CreateFile([FromBody] long fileSize, CancellationToken cancellationToken)
2626
{
2727
return await Storage.UploadAsync(new MemoryStream(new byte[fileSize]), cancellationToken);
2828
}
2929

30-
[HttpPost("upload-chunks/upload")]
31-
public async Task<Result> UploadChunks([FromForm] FileUploadPayload file, CancellationToken cancellationToken)
30+
[HttpPost("upload-chunks-stream/upload")]
31+
public async Task<Result> UploadChunksUsingStream([FromForm] FileUploadPayload file, CancellationToken cancellationToken)
3232
{
3333
using (var stream = new BlobStream(_storage.StorageClient.GetPageBlobClient(file.Payload.BlobName)))
3434
{
@@ -40,15 +40,47 @@ public async Task<Result> UploadChunks([FromForm] FileUploadPayload file, Cancel
4040
{
4141
await stream.WriteAsync(bytes, offset, bytesRead, cancellationToken);
4242
}
43-
44-
//check crc
43+
}
44+
45+
return Result.Succeed();
46+
}
47+
48+
[HttpPost("upload-chunks-merge/upload")]
49+
public async Task<Result<string>> UploadChunksUsingMerge([FromForm] FileUploadPayload file, CancellationToken cancellationToken)
50+
{
51+
var uploadResult = await _storage.UploadToStorageAsync(file.File,
52+
new UploadOptions($"{file.File.Name}_{file.Payload.ChunkIndex}", $"{file.File.Name}_directory" ),
53+
cancellationToken: cancellationToken);
54+
55+
if (uploadResult.IsSuccess)
56+
{
57+
return Result.Succeed(uploadResult.Value.FullName);
58+
}
59+
60+
return Result.Fail();
61+
}
62+
63+
[HttpPost("upload-chunks-merge/complete")]
64+
public async Task<Result> UploadChunksUsingMergeComplete(uint fileCrc, List<string> blobNames, CancellationToken cancellationToken)
65+
{
66+
using (var memoryStream = new MemoryStream())
67+
{
68+
foreach (var blobName in blobNames)
69+
{
70+
var file = await _storage.DownloadAsync(blobName, cancellationToken: cancellationToken);
71+
72+
using (Stream stream = file.Value.FileStream)
73+
{
74+
await memoryStream.CopyToAsync(stream, cancellationToken);
75+
}
76+
}
4577
}
4678

4779
return Result.Succeed();
4880
}
4981

50-
[HttpPost("upload-chunks/complete")]
51-
public async Task<bool> UploadComplete([FromBody] uint fileCrc, string blobName)
82+
[HttpPost("upload-chunks-stream/complete")]
83+
public async Task<bool> UploadChunksUsingStramComplete([FromBody] uint fileCrc, string blobName)
5284
{
5385
using (var stream = new BlobStream(_storage.StorageClient.GetPageBlobClient(blobName)))
5486
{

‎ManagedCode.Storage.IntegrationTests/Tests/BaseUploadControllerTests.cs‎

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,14 @@ namespace ManagedCode.Storage.IntegrationTests.Tests;
1010
public abstract class BaseUploadControllerTests : BaseControllerTests
1111
{
1212
private readonly string _uploadEndpoint;
13-
private readonly string _uploadChunksEndpoint;
14-
13+
private readonly string _uploadChunksStreamEndpoint;
14+
private readonly string _uploadChunksMergeEndpoint;
15+
1516
protected BaseUploadControllerTests(StorageTestApplication testApplication, string apiEndpoint) : base(testApplication, apiEndpoint)
1617
{
1718
_uploadEndpoint = string.Format(ApiEndpoints.Base.UploadFile, ApiEndpoint);
18-
_uploadChunksEndpoint = string.Format(ApiEndpoints.Base.UploadFileChunks, ApiEndpoint);
19+
_uploadChunksStreamEndpoint = string.Format(ApiEndpoints.Base.UploadFileChunksUsingStream, ApiEndpoint);
20+
_uploadChunksMergeEndpoint = string.Format(ApiEndpoints.Base.UploadFileChunksUsingMerge, ApiEndpoint);
1921
}
2022

2123
[Fact]
@@ -116,7 +118,7 @@ public async Task UploadFileFromBase64String_WhenFileValid_ReturnSuccess()
116118
}
117119

118120
[Fact]
119-
public async Task UploadFileInChunks_WhenFileValid_ReturnSuccess()
121+
public async Task UploadFileInChunksUsingStream_WhenFileValid_ReturnSuccess()
120122
{
121123
// Arrange
122124
var storageClient = GetStorageClient();
@@ -127,11 +129,37 @@ public async Task UploadFileInChunks_WhenFileValid_ReturnSuccess()
127129
FileHelper.GenerateLocalFile(localFile, 200);
128130

129131
// Act
130-
var result = await storageClient.UploadLargeFile(localFile.FileStream, _uploadChunksEndpoint + "/create",
131-
_uploadChunksEndpoint + "/upload", _uploadChunksEndpoint + "/complete", null, new CancellationToken());
132+
var result = await storageClient.UploadLargeFileUsingStream(localFile.FileStream, _uploadChunksStreamEndpoint + "/create",
133+
_uploadChunksStreamEndpoint + "/upload", _uploadChunksStreamEndpoint + "/complete", null, new CancellationToken());
134+
135+
// Assert
136+
result.IsSuccess.Should().BeTrue();
137+
//result.Value.Should().NotBeNull();
138+
}
139+
140+
[Fact]
141+
public async Task UploadFileInChunksUsingMerge_WhenFileValid_ReturnSuccess()
142+
{
143+
// Arrange
144+
var storageClient = GetStorageClient();
145+
var fileName = "test.txt";
146+
var contentName = "file";
147+
148+
await using var localFile = LocalFile.FromRandomNameWithExtension(".txt");
149+
FileHelper.GenerateLocalFile(localFile, 200);
150+
151+
//Act
152+
var result = await storageClient.UploadLargeFileUsingMerge(localFile.FileStream,
153+
_uploadChunksMergeEndpoint + "/upload",
154+
_uploadChunksMergeEndpoint + "/complete",
155+
null,
156+
new CancellationToken());
132157

133158
// Assert
134159
result.IsSuccess.Should().BeTrue();
135160
//result.Value.Should().NotBeNull();
136161
}
162+
163+
//Task<Result> UploadLargeFileUsingMerge(Stream file, string uploadApiUrl, string mergeApiUrl, Action<double>? onProgressChanged, CancellationToken cancellationToken);
164+
137165
}

0 commit comments

Comments
 (0)