Skip to content

Commit 39fb34b

Browse files
SanyyazzzSanyyazzz
committed
add method
1 parent d293502 commit 39fb34b

10 files changed

Lines changed: 132 additions & 235 deletions

File tree

‎ManagedCode.Storage.Client/IStorageClient.cs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ public interface IStorageClient
1313
Task<Result<BlobMetadata>> UploadFile(FileInfo fileInfo, string apiUrl, string contentName, CancellationToken cancellationToken = default);
1414
Task<Result<BlobMetadata>> UploadFile(byte[] bytes, string apiUrl, string contentName, CancellationToken cancellationToken = default);
1515
Task<Result<BlobMetadata>> UploadFile(string base64, string apiUrl, string contentName, CancellationToken cancellationToken = default);
16-
Task<Result> UploadLargeFileUsingStream(Stream file, string сreateApiUrl, string uploadApiUrl, string completeApiUrl, Action<double>? onProgressChanged, CancellationToken cancellationToken = default);
17-
Task<Result> UploadLargeFileUsingMerge(Stream file, string uploadApiUrl, string mergeApiUrl, Action<double>? onProgressChanged, CancellationToken cancellationToken);
16+
Task<Result<uint>> UploadLargeFile(Stream file, string uploadApiUrl, string completeApiUrl, Action<double>? onProgressChanged, CancellationToken cancellationToken = default);
1817
Task<Result<LocalFile>> DownloadFile(string fileName, string apiUrl, string? path = null, CancellationToken cancellationToken = default);
1918
}

‎ManagedCode.Storage.Client/ManagedCode.Storage.Client.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
<ItemGroup>
2222
<ProjectReference Include="..\ManagedCode.Storage.Core\ManagedCode.Storage.Core.csproj" />
23+
<ProjectReference Include="..\ManagedCode.Storage.Server\ManagedCode.Storage.Server.csproj" />
2324
</ItemGroup>
2425

2526
<ItemGroup>

‎ManagedCode.Storage.Client/StorageClient.cs‎

Lines changed: 27 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,20 @@
1111
using ManagedCode.Communication;
1212
using ManagedCode.Storage.Core.Helpers;
1313
using ManagedCode.Storage.Core.Models;
14+
using ManagedCode.Storage.Server;
15+
using Microsoft.Extensions.Configuration;
1416

1517
namespace ManagedCode.Storage.Client;
1618

1719
public class StorageClient : IStorageClient
1820
{
1921
private readonly HttpClient _httpClient;
22+
private readonly IConfiguration _configuration;
2023

21-
public StorageClient(HttpClient httpClient)
24+
public StorageClient(HttpClient httpClient, IConfiguration configuration)
2225
{
2326
_httpClient = httpClient;
27+
_configuration = configuration;
2428
}
2529

2630
public async Task<Result<BlobMetadata>> UploadFile(Stream stream, string apiUrl, string contentName, CancellationToken cancellationToken = default)
@@ -125,130 +129,53 @@ public async Task<Result<LocalFile>> DownloadFile(string fileName, string apiUrl
125129
}
126130
}
127131

128-
public async Task<Result> UploadLargeFileUsingStream(Stream file,
129-
string сreateApiUrl,
132+
public async Task<Result<uint>> UploadLargeFile(Stream file,
130133
string uploadApiUrl,
131134
string completeApiUrl,
132135
Action<double>? onProgressChanged,
133-
CancellationToken cancellationToken)
136+
CancellationToken cancellationToken = default)
134137
{
135-
var bufferSize = 4096000; //TODO: chunk size get from config
138+
int bufferSize = Int32.Parse(_configuration.GetSection("ChunkSize").Value);
136139
var buffer = new byte[bufferSize];
137-
int bytesRead;
138140
int chunkIndex = 1;
139-
var fileCRC = Crc32Helper.Calculate(file);
141+
uint fileCRC = 123214;
140142
var partOfProgress = file.Length / bufferSize;
141-
142-
var createdFileResponse = await _httpClient.PostAsync(сreateApiUrl, JsonContent.Create(file.Length), cancellationToken);
143-
var createdFile = await createdFileResponse.Content.ReadFromJsonAsync<Result<BlobMetadata>>(cancellationToken: cancellationToken);
143+
var fileName = "file" + Guid.NewGuid();
144144

145145
var semaphore = new SemaphoreSlim(0, 4);
146146
var tasks = new List<Task>();
147+
int bytesRead;
147148
while ((bytesRead = await file.ReadAsync(buffer, 0, buffer.Length, cancellationToken)) > 0)
148149
{
149-
var task = Task.Run(() =>
150+
var task = Task.Run(async () =>
150151
{
151-
try
152+
using (var memoryStream = new MemoryStream(buffer, 0, bytesRead))
152153
{
153-
semaphore.WaitAsync(cancellationToken);
154-
using (var memoryStream = new MemoryStream(buffer, 0, bytesRead))
154+
var content = new StreamContent(memoryStream);
155+
using (var formData = new MultipartFormDataContent())
155156
{
156-
var content = new StreamContent(memoryStream);
157-
158-
using (var chunk = new MultipartFormDataContent())
159-
{
160-
chunk.Add(content, "chunk", createdFile.Value.FullName);
161-
chunk.Add(new StringContent(createdFile.Value.FullName), "Payload.BlobName");
162-
chunk.Add(new StringContent(chunkIndex.ToString()), "Payload.ChunkIndex");
163-
chunk.Add(new StringContent(bufferSize.ToString()), "Payload.ChunkSize");
164-
chunk.Add(new StringContent(fileCRC.ToString()), "Payload.FullCRC");
165-
166-
_httpClient.PostAsync(uploadApiUrl, chunk, cancellationToken);
167-
}
168-
169-
onProgressChanged?.Invoke(partOfProgress * chunkIndex);
157+
formData.Add(content, "File", fileName);
158+
formData.Add(new StringContent(chunkIndex.ToString()), "Payload.ChunkIndex");
159+
formData.Add(new StringContent(bufferSize.ToString()), "Payload.ChunkSize");
160+
formData.Add(new StringContent(fileCRC.ToString()), "Payload.FullCRC");
161+
await _httpClient.PostAsync(uploadApiUrl, formData, cancellationToken);
170162
}
171163
}
172-
finally
173-
{
174-
semaphore.Release();
175-
}
164+
165+
semaphore.Release();
176166
}, cancellationToken);
177-
167+
168+
await semaphore.WaitAsync(cancellationToken);
178169
tasks.Add(task);
179-
170+
onProgressChanged?.Invoke(partOfProgress * chunkIndex);
180171
chunkIndex++;
181172
}
182173

183174
await Task.WhenAll(tasks.ToArray());
184175

185176
var mergeResult = await _httpClient.PostAsync(completeApiUrl, JsonContent.Create(
186-
new {fileCrc = fileCRC, blobName = createdFile.Value.FullName}), cancellationToken);
187-
188-
return await mergeResult.Content.ReadFromJsonAsync<Result>(cancellationToken: cancellationToken);
189-
}
190-
191-
public async Task<Result> UploadLargeFileUsingMerge(Stream file,
192-
string uploadApiUrl,
193-
string mergeApiUrl,
194-
Action<double>? onProgressChanged,
195-
CancellationToken cancellationToken)
196-
{
197-
var bufferSize = 4096000; //TODO: chunk size get from config
198-
var buffer = new byte[bufferSize];
199-
int bytesRead;
200-
int chunkIndex = 1;
201-
var fileCRC = Crc32Helper.Calculate(file);
202-
var partOfProgress = file.Length / bufferSize;
203-
204-
var semaphore = new SemaphoreSlim(0, 4);
205-
var tasks = new List<Task<HttpResponseMessage>>();
206-
while ((bytesRead = await file.ReadAsync(buffer, 0, buffer.Length, cancellationToken)) > 0)
207-
{
208-
var task = Task.Run(() =>
209-
{
210-
try
211-
{
212-
semaphore.WaitAsync(cancellationToken);
213-
using (var memoryStream = new MemoryStream(buffer, 0, bytesRead))
214-
{
215-
var content = new StreamContent(memoryStream);
216-
217-
using (var chunk = new MultipartFormDataContent())
218-
{
219-
chunk.Add(content, "chunk");
220-
chunk.Add(new StringContent(chunkIndex.ToString()), "Payload.ChunkIndex");
221-
chunk.Add(new StringContent(bufferSize.ToString()), "Payload.ChunkSize");
222-
chunk.Add(new StringContent(fileCRC.ToString()), "Payload.FullCRC");
223-
224-
var result = _httpClient.PostAsync(uploadApiUrl, chunk, cancellationToken);
225-
onProgressChanged?.Invoke(partOfProgress * chunkIndex);
226-
227-
return result;
228-
}
229-
}
230-
}
231-
finally
232-
{
233-
semaphore.Release();
234-
}
235-
}, cancellationToken);
236-
237-
tasks.Add(task);
238-
chunkIndex++;
239-
}
240-
241-
var tasksResult = await Task.WhenAll(tasks.ToArray());
242-
var blobNames = tasksResult
243-
.Select(async x =>
244-
{
245-
var content = await x.Content.ReadFromJsonAsync<Result<string>>(cancellationToken: cancellationToken);
246-
return content.Value;
247-
});
248-
249-
var mergeResult = await _httpClient.PostAsync(mergeApiUrl, JsonContent.Create(
250-
new {fileCrc = fileCRC, blobNames = blobNames}), cancellationToken);
177+
fileName), cancellationToken);
251178

252-
return await mergeResult.Content.ReadFromJsonAsync<Result>(cancellationToken: cancellationToken);
179+
return await mergeResult.Content.ReadFromJsonAsync<Result<uint>>(cancellationToken: cancellationToken);
253180
}
254181
}

‎ManagedCode.Storage.Core/Helpers/Crc32Helper.cs‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -37,19 +37,6 @@ public static uint Calculate(byte[] bytes)
3737
return ~crcValue;
3838
}
3939

40-
public static uint Calculate(Stream stream)
41-
{
42-
var bytes = StreamToByteArray(stream);
43-
uint crcValue = 0xffffffff;
44-
45-
foreach (byte by in bytes)
46-
{
47-
byte tableIndex = (byte)(((crcValue) & 0xff) ^ by);
48-
crcValue = Crc32Table[tableIndex] ^ (crcValue >> 8);
49-
}
50-
return ~crcValue;
51-
}
52-
5340
public static uint CalculateFileCRC(string filePath)
5441
{
5542
uint crcValue = 0xffffffff;

‎ManagedCode.Storage.IntegrationTests/Constants/ApiEndpoints.cs‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,7 @@ public static class ApiEndpoints
77
public static class Base
88
{
99
public const string UploadFile = "{0}/upload";
10-
public const string UploadCreateFile = "{0}/upload-chunks/create";
11-
public const string UploadFileChunksUsingStream = "{0}/upload-chunks-stream";
12-
public const string UploadFileChunksUsingMerge = "{0}/upload-chunks-merge";
13-
public const string UploadFileComplete = "{0}/upload-chunks/complete";
10+
public const string UploadLargeFile = "{0}/upload-chunks";
1411
public const string DownloadFile = "{0}/download";
1512
}
1613
}

‎ManagedCode.Storage.IntegrationTests/TestApp/Controllers/AzureTestController.cs‎

Lines changed: 0 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -14,85 +14,7 @@ namespace ManagedCode.Storage.IntegrationTests.TestApp.Controllers;
1414
[ApiController]
1515
public class AzureTestController : BaseTestController<IAzureStorage>
1616
{
17-
private readonly IAzureStorage _storage;
18-
1917
public AzureTestController(IAzureStorage storage) : base(storage)
2018
{
21-
_storage = storage;
22-
}
23-
24-
[HttpPost("upload-chunks-stream/create")]
25-
public async Task<Result<BlobMetadata>> CreateFile([FromBody] long fileSize, CancellationToken cancellationToken)
26-
{
27-
return await Storage.UploadAsync(new MemoryStream(new byte[fileSize]), cancellationToken);
28-
}
29-
30-
[HttpPost("upload-chunks-stream/upload")]
31-
public async Task<Result> UploadChunksUsingStream([FromForm] FileUploadPayload file, CancellationToken cancellationToken)
32-
{
33-
using (var stream = new BlobStream(_storage.StorageClient.GetPageBlobClient(file.Payload.BlobName)))
34-
{
35-
byte[] bytes = new byte[file.Payload.ChunkSize];
36-
int bytesRead = 0;
37-
int offset = (file.Payload.ChunkIndex - 1) * file.Payload.ChunkSize;
38-
39-
while ((bytesRead = await file.File.OpenReadStream().ReadAsync(bytes, 0, bytes.Length, cancellationToken)) > 0)
40-
{
41-
await stream.WriteAsync(bytes, offset, bytesRead, cancellationToken);
42-
}
43-
}
44-
45-
return Result.Succeed();
46-
}
47-
48-
[HttpPost("upload-chunks-merge/upload")]
49-
public async Task<Result<string>> UploadChunksUsingMerge([FromForm] FileUploadPayload file, CancellationToken cancellationToken)
50-
{
51-
var uploadResult = await _storage.UploadToStorageAsync(file.File,
52-
new UploadOptions($"{file.File.Name}_{file.Payload.ChunkIndex}", $"{file.File.Name}_directory" ),
53-
cancellationToken: cancellationToken);
54-
55-
if (uploadResult.IsSuccess)
56-
{
57-
return Result.Succeed(uploadResult.Value.FullName);
58-
}
59-
60-
return Result.Fail();
61-
}
62-
63-
[HttpPost("upload-chunks-merge/complete")]
64-
public async Task<Result<BlobMetadata>> UploadChunksUsingMergeComplete(uint fileCrc, List<string> blobNames, CancellationToken cancellationToken)
65-
{
66-
using (var memoryStream = new MemoryStream())
67-
{
68-
foreach (var blobName in blobNames)
69-
{
70-
var file = await _storage.DownloadAsync(blobName, cancellationToken: cancellationToken);
71-
72-
using (Stream stream = file.Value.FileStream)
73-
{
74-
await memoryStream.CopyToAsync(stream, cancellationToken);
75-
}
76-
}
77-
78-
var result = await _storage.UploadAsync(memoryStream, cancellationToken: cancellationToken);
79-
80-
if (result.IsSuccess)
81-
{
82-
return Result.Succeed(result.Value);
83-
}
84-
}
85-
86-
return Result.Fail();
87-
}
88-
89-
[HttpPost("upload-chunks-stream/complete")]
90-
public async Task<bool> UploadChunksUsingStramComplete([FromBody] uint fileCrc, string blobName)
91-
{
92-
using (var stream = new BlobStream(_storage.StorageClient.GetPageBlobClient(blobName)))
93-
{
94-
uint blobCrc = Crc32Helper.Calculate(stream);
95-
return blobCrc == fileCrc;
96-
}
9719
}
9820
}

0 commit comments

Comments
 (0)