From 2d009ebc85bc2ddc58b87c58661317991f3bb98f Mon Sep 17 00:00:00 2001 From: George Drak Date: Mon, 14 Dec 2020 17:25:11 +0500 Subject: [PATCH 1/3] rework storage modules --- Sitko.Core.sln | 15 - .../FileSystemStorage.cs | 20 +- .../ImageSharp/ImageSharpProvider.cs | 96 ---- .../Sitko.Core.Storage.Proxy.csproj | 36 -- .../StaticFiles/RangeHelper.cs | 131 ----- .../StaticFiles/StorageFileContext.cs | 506 ------------------ .../StaticFiles/StorageFileResponseContext.cs | 29 - .../StaticFiles/StorageMiddleware.cs | 75 --- .../StorageProxyModule.cs | 97 ---- src/Sitko.Core.Storage.S3/S3Storage.cs | 19 +- .../Cache/BaseStorageCache.cs | 20 +- .../Cache/FileStorageCache.cs | 2 +- src/Sitko.Core.Storage/Cache/IStorageCache.cs | 5 +- .../Cache/InMemoryStorageCache.cs | 3 +- src/Sitko.Core.Storage/DownloadResult.cs | 29 + src/Sitko.Core.Storage/FileDownloadResult.cs | 27 - src/Sitko.Core.Storage/IStorage.cs | 11 +- src/Sitko.Core.Storage/IStorageNode.cs | 8 - src/Sitko.Core.Storage/IsExternalInit.cs | 24 + src/Sitko.Core.Storage/Storage.cs | 76 +-- src/Sitko.Core.Storage/StorageFolder.cs | 33 -- src/Sitko.Core.Storage/StorageItem.cs | 31 +- src/Sitko.Core.Storage/StorageItemInfo.cs | 23 + src/Sitko.Core.Storage/StorageNode.cs | 77 +++ tests/Sitko.Core.Storage.Tests/BasicTests.cs | 31 +- 25 files changed, 253 insertions(+), 1171 deletions(-) delete mode 100644 src/Sitko.Core.Storage.Proxy/ImageSharp/ImageSharpProvider.cs delete mode 100644 src/Sitko.Core.Storage.Proxy/Sitko.Core.Storage.Proxy.csproj delete mode 100644 src/Sitko.Core.Storage.Proxy/StaticFiles/RangeHelper.cs delete mode 100644 src/Sitko.Core.Storage.Proxy/StaticFiles/StorageFileContext.cs delete mode 100644 src/Sitko.Core.Storage.Proxy/StaticFiles/StorageFileResponseContext.cs delete mode 100644 src/Sitko.Core.Storage.Proxy/StaticFiles/StorageMiddleware.cs delete mode 100644 src/Sitko.Core.Storage.Proxy/StorageProxyModule.cs create mode 100644 src/Sitko.Core.Storage/DownloadResult.cs delete mode 100644 src/Sitko.Core.Storage/FileDownloadResult.cs delete mode 100644 src/Sitko.Core.Storage/IStorageNode.cs create mode 100644 src/Sitko.Core.Storage/IsExternalInit.cs delete mode 100644 src/Sitko.Core.Storage/StorageFolder.cs create mode 100644 src/Sitko.Core.Storage/StorageItemInfo.cs create mode 100644 src/Sitko.Core.Storage/StorageNode.cs diff --git a/Sitko.Core.sln b/Sitko.Core.sln index e936e367..00b6dcdf 100644 --- a/Sitko.Core.sln +++ b/Sitko.Core.sln @@ -97,8 +97,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Sitko.Core.Repository.Entit EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Sitko.Core.Repository.Search", "src\Sitko.Core.Repository.Search\Sitko.Core.Repository.Search.csproj", "{F5530B6B-B1E4-4829-A0ED-486E2269C1BC}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Sitko.Core.Storage.Proxy", "src\Sitko.Core.Storage.Proxy\Sitko.Core.Storage.Proxy.csproj", "{7DB70746-EF0B-4B42-B022-67B0B8F31350}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Sitko.Core.Caching", "src\Sitko.Core.Caching\Sitko.Core.Caching.csproj", "{9BC6D2DE-1EBC-47A5-8137-495548793D4B}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Sitko.Core.ElasticStack", "src\Sitko.Core.ElasticStack\Sitko.Core.ElasticStack.csproj", "{BC09505A-1735-4E3C-9CB7-F4CF54684F55}" @@ -649,18 +647,6 @@ Global {F5530B6B-B1E4-4829-A0ED-486E2269C1BC}.Release|x64.Build.0 = Release|Any CPU {F5530B6B-B1E4-4829-A0ED-486E2269C1BC}.Release|x86.ActiveCfg = Release|Any CPU {F5530B6B-B1E4-4829-A0ED-486E2269C1BC}.Release|x86.Build.0 = Release|Any CPU - {7DB70746-EF0B-4B42-B022-67B0B8F31350}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {7DB70746-EF0B-4B42-B022-67B0B8F31350}.Debug|Any CPU.Build.0 = Debug|Any CPU - {7DB70746-EF0B-4B42-B022-67B0B8F31350}.Debug|x64.ActiveCfg = Debug|Any CPU - {7DB70746-EF0B-4B42-B022-67B0B8F31350}.Debug|x64.Build.0 = Debug|Any CPU - {7DB70746-EF0B-4B42-B022-67B0B8F31350}.Debug|x86.ActiveCfg = Debug|Any CPU - {7DB70746-EF0B-4B42-B022-67B0B8F31350}.Debug|x86.Build.0 = Debug|Any CPU - {7DB70746-EF0B-4B42-B022-67B0B8F31350}.Release|Any CPU.ActiveCfg = Release|Any CPU - {7DB70746-EF0B-4B42-B022-67B0B8F31350}.Release|Any CPU.Build.0 = Release|Any CPU - {7DB70746-EF0B-4B42-B022-67B0B8F31350}.Release|x64.ActiveCfg = Release|Any CPU - {7DB70746-EF0B-4B42-B022-67B0B8F31350}.Release|x64.Build.0 = Release|Any CPU - {7DB70746-EF0B-4B42-B022-67B0B8F31350}.Release|x86.ActiveCfg = Release|Any CPU - {7DB70746-EF0B-4B42-B022-67B0B8F31350}.Release|x86.Build.0 = Release|Any CPU {9BC6D2DE-1EBC-47A5-8137-495548793D4B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9BC6D2DE-1EBC-47A5-8137-495548793D4B}.Debug|Any CPU.Build.0 = Debug|Any CPU {9BC6D2DE-1EBC-47A5-8137-495548793D4B}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -943,7 +929,6 @@ Global {8D0A84B9-249A-4A18-9321-43A1B26ABE3C} = {109B331E-71B9-483C-8FC1-13B10C92A7F1} {AC57B296-8861-4D67-B64C-3B2EDFF6BA9B} = {109B331E-71B9-483C-8FC1-13B10C92A7F1} {F5530B6B-B1E4-4829-A0ED-486E2269C1BC} = {109B331E-71B9-483C-8FC1-13B10C92A7F1} - {7DB70746-EF0B-4B42-B022-67B0B8F31350} = {109B331E-71B9-483C-8FC1-13B10C92A7F1} {9BC6D2DE-1EBC-47A5-8137-495548793D4B} = {109B331E-71B9-483C-8FC1-13B10C92A7F1} {BC09505A-1735-4E3C-9CB7-F4CF54684F55} = {109B331E-71B9-483C-8FC1-13B10C92A7F1} {DE7A5982-CFCA-4DBA-B446-5FBD22F406F5} = {109B331E-71B9-483C-8FC1-13B10C92A7F1} diff --git a/src/Sitko.Core.Storage.FileSystem/FileSystemStorage.cs b/src/Sitko.Core.Storage.FileSystem/FileSystemStorage.cs index f7ee986a..fa7d8de7 100644 --- a/src/Sitko.Core.Storage.FileSystem/FileSystemStorage.cs +++ b/src/Sitko.Core.Storage.FileSystem/FileSystemStorage.cs @@ -83,9 +83,9 @@ protected override Task DoDeleteAllAsync() return Task.CompletedTask; } - protected override async Task DoGetFileAsync(string path) + protected override async Task DoGetFileAsync(string path) { - FileDownloadResult? result = null; + StorageItemInfo? result = null; var fullPath = Path.Combine(_storagePath, path); var metaDataPath = GetMetaDataPath(fullPath); var fileInfo = new FileInfo(fullPath); @@ -99,21 +99,20 @@ protected override Task DoDeleteAllAsync() metadata = await File.ReadAllTextAsync(metaDataPath); } - - result = new FileDownloadResult(metadata, fileInfo.Length, fileInfo.LastWriteTimeUtc, - fileInfo.OpenRead()); + result = new StorageItemInfo(metadata, fileInfo.Length, fileInfo.LastWriteTimeUtc, + () => new FileStream(fullPath, FileMode.Open)); } return result; } - protected override Task DoBuildStorageTreeAsync() + protected override Task DoBuildStorageTreeAsync() { return ListFolderAsync("/"); } - private async Task ListFolderAsync(string path) + private async Task ListFolderAsync(string path) { var fullPath = path == "/" ? _storagePath : Path.Combine(_storagePath, path.Trim('/')); List? children = null; @@ -145,15 +144,14 @@ private async Task ListFolderAsync(string path) var item = CreateStorageItem(PreparePath(Path.Combine(path, file.Name))!.Trim('/'), file.LastWriteTimeUtc, file.Length, - metadata, - physicalPath: file.FullName); + metadata); - children.Add(item); + children.Add(StorageNode.CreateStorageItem(item)); } } } - return new StorageFolder(path == "/" ? "/" : Path.GetFileNameWithoutExtension(path), + return StorageNode.CreateDirectory(path == "/" ? "/" : Path.GetFileNameWithoutExtension(path), PreparePath(Path.Combine(_storagePath, path)), children); } diff --git a/src/Sitko.Core.Storage.Proxy/ImageSharp/ImageSharpProvider.cs b/src/Sitko.Core.Storage.Proxy/ImageSharp/ImageSharpProvider.cs deleted file mode 100644 index 30370e17..00000000 --- a/src/Sitko.Core.Storage.Proxy/ImageSharp/ImageSharpProvider.cs +++ /dev/null @@ -1,96 +0,0 @@ -using System; -using System.IO; -using System.Net; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Http.Extensions; -using SixLabors.ImageSharp.Web; -using SixLabors.ImageSharp.Web.Providers; -using SixLabors.ImageSharp.Web.Resolvers; - -namespace Sitko.Core.Storage.Proxy.ImageSharp -{ - public abstract class ImageSharpStorageProvider : IImageProvider - { - protected static readonly char[] _slashChars = {'\\', '/'}; - private Func? _match; - private readonly FormatUtilities _formatUtilities; - - protected ImageSharpStorageProvider(FormatUtilities formatUtilities) - { - _formatUtilities = formatUtilities; - } - - public abstract Task GetAsync(HttpContext context); - public ProcessingBehavior ProcessingBehavior => ProcessingBehavior.All; - - - public Func Match - { - get => _match ?? IsMatch; - set => _match = value; - } - - private bool IsMatch(HttpContext context) - { - return true; - } - - public bool IsValidRequest(HttpContext context) - => _formatUtilities.GetExtensionFromUri(context.Request.GetDisplayUrl()) != null; - } - - public class ImageSharpStorageProvider : ImageSharpStorageProvider - where TStorageOptions : StorageOptions - { - private readonly IStorage _storage; - - public ImageSharpStorageProvider(IStorage storage, FormatUtilities formatUtilities) : base( - formatUtilities) - { - _storage = storage; - } - - public override async Task GetAsync(HttpContext context) - { - var key = WebUtility.UrlDecode(context.Request.Path.Value).TrimStart(_slashChars); - - bool imageExists = await _storage.IsFileExistsAsync(key); - -#pragma warning disable 8603 - return !imageExists ? null : new ImageSharpStorageResolver(_storage, key); -#pragma warning restore 8603 - } - } - - public class ImageSharpStorageResolver : IImageResolver where TStorageOptions : StorageOptions - { - private readonly IStorage _storage; - private readonly string _imagePath; - - public ImageSharpStorageResolver(IStorage storage, string key) - { - _storage = storage; - _imagePath = key; - } - - public async Task GetMetaDataAsync() - { - var fileInfo = await _storage.GetFileAsync(_imagePath); - if (fileInfo == null) - { - return new ImageMetadata(DateTime.UtcNow, 0); - } - - return new ImageMetadata(fileInfo.LastModified.DateTime, fileInfo.FileSize); - } - - public async Task OpenReadAsync() - { - var file = await _storage.GetFileAsync(_imagePath); -#pragma warning disable 8603 - return file?.Stream; -#pragma warning restore 8603 - } - } -} diff --git a/src/Sitko.Core.Storage.Proxy/Sitko.Core.Storage.Proxy.csproj b/src/Sitko.Core.Storage.Proxy/Sitko.Core.Storage.Proxy.csproj deleted file mode 100644 index 69623505..00000000 --- a/src/Sitko.Core.Storage.Proxy/Sitko.Core.Storage.Proxy.csproj +++ /dev/null @@ -1,36 +0,0 @@ - - - - netcoreapp3.1;net5.0 - enable - latest - - MIT - George Drak - Sitko.Ru - Sitko.Core - Sitko.Core is a set of libraries to help build .NET Core applications fast - Sitko.Core is a set of libraries to help build .NET Core applications fast - Copyright © Sitko.ru 2020 - https://github.com/sitkoru/Sitko.Core - packageIcon.png - true - true - snupkg - true - - - - - - - - - - - - - - - - diff --git a/src/Sitko.Core.Storage.Proxy/StaticFiles/RangeHelper.cs b/src/Sitko.Core.Storage.Proxy/StaticFiles/RangeHelper.cs deleted file mode 100644 index 27482d96..00000000 --- a/src/Sitko.Core.Storage.Proxy/StaticFiles/RangeHelper.cs +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Diagnostics; -using System.Linq; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Http.Headers; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Primitives; -using Microsoft.Net.Http.Headers; -#pragma warning disable 8629 -#pragma warning disable 8603 -#pragma warning disable 8619 - -namespace Sitko.Core.Storage.Proxy.StaticFiles -{ - /// - /// Provides a parser for the Range Header in an . - /// - internal static class RangeHelper - { - /// - /// Returns the normalized form of the requested range if the Range Header in the is valid. - /// - /// The associated with the request. - /// The associated with the given . - /// The total length of the file representation requested. - /// The . - /// A boolean value which represents if the contain a single valid - /// range request. A which represents the normalized form of the - /// range parsed from the or null if it cannot be normalized. - /// If the Range header exists but cannot be parsed correctly, or if the provided length is 0, then the range request cannot be satisfied (status 416). - /// This results in (true,null) return values. - public static (bool isRangeRequest, RangeItemHeaderValue range) ParseRange( - HttpContext context, - RequestHeaders requestHeaders, - long length, - ILogger logger) - { - var rawRangeHeader = context.Request.Headers[HeaderNames.Range]; - if (StringValues.IsNullOrEmpty(rawRangeHeader)) - { - logger.LogTrace("Range header's value is empty."); - return (false, null); - } - - // Perf: Check for a single entry before parsing it - if (rawRangeHeader.Count > 1 || rawRangeHeader[0].IndexOf(',') >= 0) - { - logger.LogDebug("Multiple ranges are not supported."); - - // The spec allows for multiple ranges but we choose not to support them because the client may request - // very strange ranges (e.g. each byte separately, overlapping ranges, etc.) that could negatively - // impact the server. Ignore the header and serve the response normally. - return (false, null); - } - - var rangeHeader = requestHeaders.Range; - if (rangeHeader == null) - { - logger.LogDebug("Range header's value is invalid."); - // Invalid - return (false, null); - } - - // Already verified above - Debug.Assert(rangeHeader.Ranges.Count == 1); - - var ranges = rangeHeader.Ranges; - if (ranges == null) - { - logger.LogDebug("Range header's value is invalid."); - return (false, null); - } - - if (ranges.Count == 0) - { - return (true, null); - } - - if (length == 0) - { - return (true, null); - } - - // Normalize the ranges - var range = NormalizeRange(ranges.SingleOrDefault(), length); - - // Return the single range - return (true, range); - } - - // Internal for testing - internal static RangeItemHeaderValue NormalizeRange(RangeItemHeaderValue range, long length) - { - var start = range.From; - var end = range.To; - - // X-[Y] - if (start.HasValue) - { - if (start.Value >= length) - { - // Not satisfiable, skip/discard. - return null; - } - - if (!end.HasValue || end.Value >= length) - { - end = length - 1; - } - } - else - { - // suffix range "-X" e.g. the last X bytes, resolve - if (end.Value == 0) - { - // Not satisfiable, skip/discard. - return null; - } - - var bytes = Math.Min(end.Value, length); - start = length - bytes; - end = start + bytes - 1; - } - - return new RangeItemHeaderValue(start, end); - } - } -} diff --git a/src/Sitko.Core.Storage.Proxy/StaticFiles/StorageFileContext.cs b/src/Sitko.Core.Storage.Proxy/StaticFiles/StorageFileContext.cs deleted file mode 100644 index bcd2e7aa..00000000 --- a/src/Sitko.Core.Storage.Proxy/StaticFiles/StorageFileContext.cs +++ /dev/null @@ -1,506 +0,0 @@ -using System; -using System.Diagnostics; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Http.Extensions; -using Microsoft.AspNetCore.Http.Features; -using Microsoft.AspNetCore.Http.Headers; -using Microsoft.AspNetCore.StaticFiles; -using Microsoft.Extensions.Logging; -using Microsoft.Net.Http.Headers; - -namespace Sitko.Core.Storage.Proxy.StaticFiles -{ - public struct StorageFileContext - { - private const int StreamCopyBufferSize = 64 * 1024; - - private readonly HttpContext _context; - private readonly StorageFileOptions _options; - private readonly HttpRequest _request; - private readonly HttpResponse _response; - private readonly ILogger _logger; - private readonly StorageItem _record; - private readonly string _contentType; - - private EntityTagHeaderValue? _etag; - private RequestHeaders? _requestHeaders; - private ResponseHeaders? _responseHeaders; - private RangeItemHeaderValue? _range; - - private long _length; - private readonly PathString _subPath; - private DateTimeOffset _lastModified; - - private PreconditionState _ifMatchState; - private PreconditionState _ifNoneMatchState; - private PreconditionState _ifModifiedSinceState; - private PreconditionState _ifUnmodifiedSinceState; - - private RequestType _requestType; - - public StorageFileContext(HttpContext context, StorageFileOptions options, ILogger logger, - StorageItem record, - string contentType, PathString subPath) - { - _context = context; - _options = options; - _request = context.Request; - _response = context.Response; - _logger = logger; - _record = record; - string method = _request.Method; - _contentType = contentType; - _etag = null; - _requestHeaders = null; - _responseHeaders = null; - _range = null; - - _length = record.FileSize; - _subPath = subPath; - _lastModified = new DateTimeOffset(); - _ifMatchState = PreconditionState.Unspecified; - _ifNoneMatchState = PreconditionState.Unspecified; - _ifModifiedSinceState = PreconditionState.Unspecified; - _ifUnmodifiedSinceState = PreconditionState.Unspecified; - - if (HttpMethods.IsGet(method)) - { - _requestType = RequestType.IsGet; - } - else if (HttpMethods.IsHead(method)) - { - _requestType = RequestType.IsHead; - } - else - { - _requestType = RequestType.Unspecified; - } - } - - private RequestHeaders RequestHeaders => _requestHeaders ??= _request.GetTypedHeaders(); - - private ResponseHeaders ResponseHeaders => _responseHeaders ??= _response.GetTypedHeaders(); - - public bool IsHeadMethod => _requestType.HasFlag(RequestType.IsHead); - - public bool IsGetMethod => _requestType.HasFlag(RequestType.IsGet); - - public bool IsRangeRequest - { - get => _requestType.HasFlag(RequestType.IsRange); - private set - { - if (value) - { - _requestType |= RequestType.IsRange; - } - else - { - _requestType &= ~RequestType.IsRange; - } - } - } - - public string SubPath => _subPath.Value; - - public bool LookupFileInfo() - { - _length = _record.FileSize; - - DateTimeOffset last = _record.LastModified; - // Truncate to the second. - _lastModified = - new DateTimeOffset(last.Year, last.Month, last.Day, last.Hour, last.Minute, last.Second, - last.Offset).ToUniversalTime(); - - long etagHash = _lastModified.ToFileTime() ^ _length; - _etag = new EntityTagHeaderValue('\"' + Convert.ToString(etagHash, 16) + '\"'); - return true; - } - - public void ComprehendRequestHeaders() - { - ComputeIfMatch(); - - ComputeIfModifiedSince(); - - ComputeRange(); - - ComputeIfRange(); - } - - private void ComputeIfMatch() - { - var requestHeaders = RequestHeaders; - - // 14.24 If-Match - var ifMatch = requestHeaders.IfMatch; - if (ifMatch?.Count > 0) - { - _ifMatchState = PreconditionState.PreconditionFailed; - foreach (var etag in ifMatch) - { - if (etag.Equals(EntityTagHeaderValue.Any) || etag.Compare(_etag, true)) - { - _ifMatchState = PreconditionState.ShouldProcess; - break; - } - } - } - - // 14.26 If-None-Match - var ifNoneMatch = requestHeaders.IfNoneMatch; - if (ifNoneMatch?.Count > 0) - { - _ifNoneMatchState = PreconditionState.ShouldProcess; - foreach (var etag in ifNoneMatch) - { - if (etag.Equals(EntityTagHeaderValue.Any) || etag.Compare(_etag, useStrongComparison: true)) - { - _ifNoneMatchState = PreconditionState.NotModified; - break; - } - } - } - } - - private void ComputeIfModifiedSince() - { - var requestHeaders = RequestHeaders; - var now = DateTimeOffset.UtcNow; - - // 14.25 If-Modified-Since - var ifModifiedSince = requestHeaders.IfModifiedSince; - if (ifModifiedSince.HasValue && ifModifiedSince <= now) - { - bool modified = ifModifiedSince < _lastModified; - _ifModifiedSinceState = modified ? PreconditionState.ShouldProcess : PreconditionState.NotModified; - } - - // 14.28 If-Unmodified-Since - var ifUnmodifiedSince = requestHeaders.IfUnmodifiedSince; - if (ifUnmodifiedSince.HasValue && ifUnmodifiedSince <= now) - { - bool unmodified = ifUnmodifiedSince >= _lastModified; - _ifUnmodifiedSinceState = - unmodified ? PreconditionState.ShouldProcess : PreconditionState.PreconditionFailed; - } - } - - private void ComputeIfRange() - { - // 14.27 If-Range - var ifRangeHeader = RequestHeaders.IfRange; - if (ifRangeHeader != null) - { - // If the validator given in the If-Range header field matches the - // current validator for the selected representation of the target - // resource, then the server SHOULD process the Range header field as - // requested. If the validator does not match, the server MUST ignore - // the Range header field. - if (ifRangeHeader.LastModified.HasValue) - { - if (_lastModified > ifRangeHeader.LastModified) - { - IsRangeRequest = false; - } - } - else if (_etag != null && ifRangeHeader.EntityTag != null && - !ifRangeHeader.EntityTag.Compare(_etag, useStrongComparison: true)) - { - IsRangeRequest = false; - } - } - } - - private void ComputeRange() - { - // 14.35 Range - // http://tools.ietf.org/html/draft-ietf-httpbis-p5-range-24 - - // A server MUST ignore a Range header field received with a request method other - // than GET. - if (!IsGetMethod) - { - return; - } - - (var isRangeRequest, var range) = RangeHelper.ParseRange(_context, RequestHeaders, _length, _logger); - - _range = range; - IsRangeRequest = isRangeRequest; - } - - public void ApplyResponseHeaders(int statusCode) - { - _response.StatusCode = statusCode; - if (statusCode < 400) - { - // these headers are returned for 200, 206, and 304 - // they are not returned for 412 and 416 - if (!string.IsNullOrEmpty(_contentType)) - { - _response.ContentType = _contentType; - } - - var responseHeaders = ResponseHeaders; - responseHeaders.LastModified = _lastModified; - responseHeaders.ETag = _etag; - responseHeaders.Headers[HeaderNames.AcceptRanges] = "bytes"; - } - - if (statusCode == StatusCodes.Status200OK) - { - // this header is only returned here for 200 - // it already set to the returned range for 206 - // it is not returned for 304, 412, and 416 - _response.ContentLength = _length; - } - - _options.OnPrepareResponse?.Invoke(new StorageFileResponseContext(_context, _record)); - } - - public PreconditionState GetPreconditionState() - => GetMaxPreconditionState(_ifMatchState, _ifNoneMatchState, _ifModifiedSinceState, - _ifUnmodifiedSinceState); - - private static PreconditionState GetMaxPreconditionState(params PreconditionState[] states) - { - PreconditionState max = PreconditionState.Unspecified; - for (int i = 0; i < states.Length; i++) - { - if (states[i] > max) - { - max = states[i]; - } - } - - return max; - } - - public Task SendStatusAsync(int statusCode) - { - ApplyResponseHeaders(statusCode); - - return Task.CompletedTask; - } - - public async Task ServeStaticFile(HttpContext context, RequestDelegate next) - { - ComprehendRequestHeaders(); - switch (GetPreconditionState()) - { - case PreconditionState.Unspecified: - case PreconditionState.ShouldProcess: - if (IsHeadMethod) - { - await SendStatusAsync(StatusCodes.Status200OK); - return; - } - - try - { - if (IsRangeRequest) - { - await SendRangeAsync(); - return; - } - - await SendAsync(); - _logger.LogDebug("File {Path} served", SubPath); - return; - } - catch (FileNotFoundException) - { - context.Response.Clear(); - } - - await next(context); - return; - case PreconditionState.NotModified: - _logger.LogDebug("File {Path} not modified", SubPath); - await SendStatusAsync(StatusCodes.Status304NotModified); - return; - case PreconditionState.PreconditionFailed: - _logger.LogDebug("File {Path} precondition failed", SubPath); - await SendStatusAsync(StatusCodes.Status412PreconditionFailed); - return; - default: - var exception = new NotImplementedException(GetPreconditionState().ToString()); - Debug.Fail(exception.ToString()); - throw exception; - } - } - - public async Task SendAsync() - { - SetCompressionMode(); - ApplyResponseHeaders(StatusCodes.Status200OK); - - var sendFile = _context.Features.Get(); - // ReSharper disable once ConditionIsAlwaysTrueOrFalse - if (sendFile != null && !string.IsNullOrEmpty(_record.PhysicalPath)) - { - await sendFile.SendFileAsync(_record.PhysicalPath, 0, _length, CancellationToken.None); - return; - } - - try - { - await using var readStream = _record.Stream; - // Larger StreamCopyBufferSize is required because in case of FileStream readStream isn't going to be buffering - await StreamCopyOperation.CopyToAsync(readStream, _response.Body, _length, StreamCopyBufferSize, - _context.RequestAborted); - } - catch (OperationCanceledException ex) - { - _logger.LogError(ex, "File {Path} write cancelled", SubPath); - // Don't throw this exception, it's most likely caused by the client disconnecting. - // However, if it was cancelled for any other reason we need to prevent empty responses. - _context.Abort(); - } - } - - // When there is only a single range the bytes are sent directly in the body. - internal async Task SendRangeAsync() - { - if (_range == null) - { - // 14.16 Content-Range - A server sending a response with status code 416 (Requested range not satisfiable) - // SHOULD include a Content-Range field with a byte-range-resp-spec of "*". The instance-length specifies - // the current length of the selected resource. e.g. */length - ResponseHeaders.ContentRange = new ContentRangeHeaderValue(_length); - ApplyResponseHeaders(StatusCodes.Status416RangeNotSatisfiable); - - _logger.LogError("File {Path} range not satisfiable", SubPath); - return; - } - - ResponseHeaders.ContentRange = ComputeContentRange(_range, out var start, out var length); - _response.ContentLength = length; - SetCompressionMode(); - ApplyResponseHeaders(StatusCodes.Status206PartialContent); - - var sendFile = _context.Features.Get(); - // ReSharper disable once ConditionIsAlwaysTrueOrFalse - if (sendFile != null && !string.IsNullOrEmpty(_record.PhysicalPath)) - { - await sendFile.SendFileAsync(_record.PhysicalPath, start, length, CancellationToken.None); - return; - } - - try - { - await using var readStream = _record.Stream; - if (readStream == null) - { - throw new Exception("Empty strem"); - } - - if (!readStream.CanSeek) - { - await using var memoryStream = new MemoryStream(); - await StreamCopyOperation.CopyToAsync(readStream, memoryStream, _length, _context.RequestAborted); - await SendRangeAsync(memoryStream, start, length); - } - else - { - await SendRangeAsync(readStream, start, length); - } - } - catch (OperationCanceledException ex) - { - _logger.LogError(ex, "File {Path} write cancelled", SubPath); - // Don't throw this exception, it's most likely caused by the client disconnecting. - // However, if it was cancelled for any other reason we need to prevent empty responses. - _context.Abort(); - } - } - - private Task SendRangeAsync(Stream stream, long start, long length) - { - stream.Seek(start, SeekOrigin.Begin); - _logger.LogDebug("Copying file range {Range} for file {File}", - _response.Headers[HeaderNames.ContentRange], SubPath); - return StreamCopyOperation.CopyToAsync(stream, _response.Body, length, _context.RequestAborted); - } - - // Note: This assumes ranges have been normalized to absolute byte offsets. - private ContentRangeHeaderValue ComputeContentRange(RangeItemHeaderValue range, out long start, out long length) - { - // ReSharper disable once PossibleInvalidOperationException - start = range!.From!.Value; - // ReSharper disable once PossibleInvalidOperationException - long end = range!.To!.Value; - length = end - start + 1; - return new ContentRangeHeaderValue(start, end, _length); - } - - // Only called when we expect to serve the body. - private void SetCompressionMode() - { - var responseCompressionFeature = _context.Features.Get(); - // ReSharper disable once ConditionIsAlwaysTrueOrFalse - if (responseCompressionFeature != null) - { - responseCompressionFeature.Mode = _options.HttpsCompression; - } - } - - public enum PreconditionState : byte - { - Unspecified, - NotModified, - ShouldProcess, - PreconditionFailed - } - - [Flags] - private enum RequestType : byte - { - Unspecified = 0b_000, - IsHead = 0b_001, - IsGet = 0b_010, - IsRange = 0b_100, - } - } - - public class StorageFileOptions - { - /// - /// Used to map files to content-types. - /// - public IContentTypeProvider? ContentTypeProvider { get; set; } - - /// - /// The default content type for a request if the ContentTypeProvider cannot determine one. - /// None is provided by default, so the client must determine the format themselves. - /// http://www.w3.org/Protocols/rfc2616/rfc2616-sec7.html#sec7 - /// - public string? DefaultContentType { get; set; } - - /// - /// If the file is not a recognized content-type should it be served? - /// Default: false. - /// - public bool ServeUnknownFileTypes { get; set; } - - /// - /// Indicates if files should be compressed for HTTPS requests when the Response Compression middleware is available. - /// The default value is . - /// - /// - /// Enabling compression on HTTPS requests for remotely manipulable content may expose security problems. - /// - public HttpsCompressionMode HttpsCompression { get; set; } = HttpsCompressionMode.Compress; - - /// - /// Called after the status code and headers have been set, but before the body has been written. - /// This can be used to add or change the response headers. - /// - public Action? OnPrepareResponse { get; set; } - } -} diff --git a/src/Sitko.Core.Storage.Proxy/StaticFiles/StorageFileResponseContext.cs b/src/Sitko.Core.Storage.Proxy/StaticFiles/StorageFileResponseContext.cs deleted file mode 100644 index 421c837f..00000000 --- a/src/Sitko.Core.Storage.Proxy/StaticFiles/StorageFileResponseContext.cs +++ /dev/null @@ -1,29 +0,0 @@ -using System; -using Microsoft.AspNetCore.Http; - -namespace Sitko.Core.Storage.Proxy.StaticFiles -{ - public class StorageFileResponseContext - { - /// - /// Constructs the . - /// - /// The request and response information. - /// The file to be served. - public StorageFileResponseContext(HttpContext context, StorageItem file) - { - Context = context ?? throw new ArgumentNullException(nameof(context)); - File = file; - } - - /// - /// The request and response information. - /// - public HttpContext Context { get; } - - /// - /// The file to be served. - /// - public StorageItem File { get; } - } -} diff --git a/src/Sitko.Core.Storage.Proxy/StaticFiles/StorageMiddleware.cs b/src/Sitko.Core.Storage.Proxy/StaticFiles/StorageMiddleware.cs deleted file mode 100644 index d30ffe8b..00000000 --- a/src/Sitko.Core.Storage.Proxy/StaticFiles/StorageMiddleware.cs +++ /dev/null @@ -1,75 +0,0 @@ -using System; -using System.Net; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.StaticFiles; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -namespace Sitko.Core.Storage.Proxy.StaticFiles -{ - public class StorageMiddleware where TStorageOptions : StorageOptions - { - private readonly RequestDelegate _next; - private readonly IStorage _storage; - private readonly ILogger _logger; - private readonly IContentTypeProvider _contentTypeProvider; - private readonly StorageFileOptions _options; - - public StorageMiddleware(RequestDelegate next, IOptions options, - IStorage storage, - ILogger> logger) - { - _next = next ?? throw new ArgumentNullException(nameof(next)); - _options = options.Value; - _storage = storage; - _contentTypeProvider = _options.ContentTypeProvider ?? new FileExtensionContentTypeProvider(); - _logger = logger; - } - - public Task Invoke(HttpContext context) - { - if (!ValidateNoEndpoint(context)) - { - _logger.LogInformation("No endpoint"); - } - else if (!ValidateMethod(context)) - { - _logger.LogInformation("Method {Method} is not supported", context.Request.Method); - } - - var subPath = WebUtility.UrlDecode(context.Request.Path); - _contentTypeProvider.TryGetContentType(subPath, out var contentType); - - return TryServeStaticFile(context, contentType, subPath); - } - - // Return true because we only want to run if there is no endpoint. - private static bool ValidateNoEndpoint(HttpContext context) => context.GetEndpoint() == null; - - private static bool ValidateMethod(HttpContext context) - { - return context.Request.Method == "GET" || context.Request.Method == "HEAD"; - } - - private async Task TryServeStaticFile(HttpContext context, string contentType, string subPath) - { - var file = await _storage.GetFileAsync(subPath); - if (file == null) - { - _logger.LogWarning("File {File} not found", subPath); - } - else - { - var fileContext = - new StorageFileContext(context, _options, _logger, file, contentType, subPath); - // If we get here, we can try to serve the file - await fileContext.ServeStaticFile(context, _next); - return; - } - - - await _next(context); - } - } -} diff --git a/src/Sitko.Core.Storage.Proxy/StorageProxyModule.cs b/src/Sitko.Core.Storage.Proxy/StorageProxyModule.cs deleted file mode 100644 index c8dfef33..00000000 --- a/src/Sitko.Core.Storage.Proxy/StorageProxyModule.cs +++ /dev/null @@ -1,97 +0,0 @@ -using System; -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.StaticFiles; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Net.Http.Headers; -using Sitko.Core.App; -using Sitko.Core.Storage.Proxy.ImageSharp; -using Sitko.Core.Storage.Proxy.StaticFiles; -using Sitko.Core.App.Web; -using SixLabors.ImageSharp.Web.Caching; -using SixLabors.ImageSharp.Web.Commands; -using SixLabors.ImageSharp.Web.DependencyInjection; -using SixLabors.ImageSharp.Web.Middleware; -using SixLabors.ImageSharp.Web.Processors; - -namespace Sitko.Core.Storage.Proxy -{ - public class StorageProxyModule : BaseApplicationModule, - IWebApplicationModule where TStorageOptions : StorageOptions - { - private FileExtensionContentTypeProvider? _mimeTypeProvider; - - public StorageProxyModule(StorageProxyModuleConfig config, Application application) : base(config, application) - { - } - - public override void ConfigureServices(IServiceCollection services, IConfiguration configuration, - IHostEnvironment environment) - { - base.ConfigureServices(services, configuration, environment); - services.AddImageSharp(options => - { - options.Configuration = SixLabors.ImageSharp.Configuration.Default; - options.BrowserMaxAge = TimeSpan.FromDays(1); - options.CachedNameLength = 12; - Config.ConfigureImageSharpMiddleware?.Invoke(options); - }) - .SetRequestParser() - .Configure(options => - { - if (!string.IsNullOrEmpty(Config.ImageSharpCacheDir)) - { - options.CacheFolder = Config.ImageSharpCacheDir; - } - }) - .SetCache() - .SetCacheHash() - .AddProvider>() - .AddProcessor() - .AddProcessor() - .AddProcessor(); - - services.Configure(options => - { - options.OnPrepareResponse = ctx => - { - var headers = ctx.Context.Response.Headers; - var contentType = headers["Content-Type"]; - - if (contentType != "application/x-gzip" && ctx.File.FilePath != null && - !ctx.File.FilePath.EndsWith(".gz")) - { - return; - } - - var fileNameToTry = ctx.File.FilePath?.Substring(0, ctx.File.FilePath.Length - 3); - - if (_mimeTypeProvider != null && - _mimeTypeProvider.TryGetContentType(fileNameToTry, out var mimeType)) - { - headers.Add("Content-Encoding", "gzip"); - headers["Content-Type"] = mimeType; - } - - headers[HeaderNames.CacheControl] = "public,max-age=" + Config.MaxAgeHeader.TotalSeconds; - }; - }); - } - - public void ConfigureBeforeUseRouting(IConfiguration configuration, IHostEnvironment environment, - IApplicationBuilder appBuilder) - { - appBuilder.UseImageSharp(); - _mimeTypeProvider = new FileExtensionContentTypeProvider(); - appBuilder.UseMiddleware>(); - } - } - - public class StorageProxyModuleConfig - { - public Action? ConfigureImageSharpMiddleware { get; set; } - public string? ImageSharpCacheDir { get; set; } - public TimeSpan MaxAgeHeader { get; set; } = TimeSpan.FromDays(30); - } -} diff --git a/src/Sitko.Core.Storage.S3/S3Storage.cs b/src/Sitko.Core.Storage.S3/S3Storage.cs index 8a462033..da0d2015 100644 --- a/src/Sitko.Core.Storage.S3/S3Storage.cs +++ b/src/Sitko.Core.Storage.S3/S3Storage.cs @@ -195,7 +195,7 @@ protected override async Task DoDeleteAllAsync() return metaData; } - protected override async Task DoGetFileAsync(string path) + protected override async Task DoGetFileAsync(string path) { var fileResponse = await DownloadFileAsync(path); if (fileResponse == null) @@ -205,13 +205,13 @@ protected override async Task DoDeleteAllAsync() var metaData = await DownloadFileMetadataAsync(path); - return new FileDownloadResult(metaData, fileResponse.ContentLength, fileResponse.LastModified, - fileResponse.ResponseStream); + return new StorageItemInfo(metaData, fileResponse.ContentLength, fileResponse.LastModified, + () => fileResponse.ResponseStream); } - protected override async Task DoBuildStorageTreeAsync() + protected override async Task DoBuildStorageTreeAsync() { - var root = new StorageFolder("/", "/"); + var root = StorageNode.CreateDirectory("/", "/"); try { ListObjectsV2Request request = new ListObjectsV2Request {BucketName = _options.Bucket}; @@ -241,7 +241,7 @@ protected override async Task DoDeleteAllAsync() return root; } - private async Task AddObjectAsync(S3Object s3Object, StorageFolder root) + private async Task AddObjectAsync(S3Object s3Object, StorageNode root) { if (s3Object.Key.EndsWith(MetaDataExtension)) return; var parts = s3Object.Key.Split("/"); @@ -252,14 +252,15 @@ private async Task AddObjectAsync(S3Object s3Object, StorageFolder root) { var metadata = await DownloadFileMetadataAsync(s3Object.Key); var item = CreateStorageItem(s3Object.Key, s3Object.LastModified, s3Object.Size, metadata); - current.AddChild(item); + current.AddChild(StorageNode.CreateStorageItem(item)); } else { - var child = current.Children.OfType().FirstOrDefault(f => f.Name == part); + var child = current.Children.Where(n => n.Type == StorageNodeType.Directory) + .FirstOrDefault(f => f.Name == part); if (child == null) { - child = new StorageFolder(part, PreparePath(Path.Combine(current.FullPath, part))); + child = StorageNode.CreateDirectory(part, PreparePath(Path.Combine(current.FullPath, part))); current.AddChild(child); } diff --git a/src/Sitko.Core.Storage/Cache/BaseStorageCache.cs b/src/Sitko.Core.Storage/Cache/BaseStorageCache.cs index c8e2d5ff..58978de5 100644 --- a/src/Sitko.Core.Storage/Cache/BaseStorageCache.cs +++ b/src/Sitko.Core.Storage/Cache/BaseStorageCache.cs @@ -51,8 +51,8 @@ IEnumerator IEnumerable.GetEnumerator() return GetEnumerator(); } - public async Task GetOrAddItemAsync(string path, - Func> addItem) + public async Task GetOrAddItemAsync(string path, + Func> addItem) { if (_cache == null) { @@ -86,7 +86,7 @@ IEnumerator IEnumerable.GetEnumerator() return result; } - var stream = result.Stream; + var stream = result.GetStream(); await using (stream) { @@ -126,15 +126,14 @@ IEnumerator IEnumerable.GetEnumerator() return cacheEntry is null ? null - : new FileDownloadResult(cacheEntry.Metadata, cacheEntry.FileSize, cacheEntry.Date, - cacheEntry.OpenRead(), - cacheEntry.PhysicalPath); + : new StorageItemInfo(cacheEntry.Metadata, cacheEntry.FileSize, cacheEntry.Date, + () => cacheEntry.OpenRead()); } protected abstract void DisposeItem(TRecord deletedRecord); - protected abstract Task GetEntryAsync(FileDownloadResult item, Stream stream); + protected abstract Task GetEntryAsync(StorageItemInfo item, Stream stream); private string NormalizePath(string path) { @@ -147,7 +146,7 @@ private string NormalizePath(string path) return path; } - public Task GetItemAsync(string path) + public Task GetItemAsync(string path) { if (_cache == null) { @@ -158,9 +157,8 @@ private string NormalizePath(string path) return Task.FromResult(cacheEntry is null ? null - : new FileDownloadResult(cacheEntry.Metadata, cacheEntry.FileSize, cacheEntry.Date, - cacheEntry.OpenRead(), - cacheEntry.PhysicalPath)); + : new StorageItemInfo(cacheEntry.Metadata, cacheEntry.FileSize, cacheEntry.Date, + () => cacheEntry.OpenRead())); } public Task RemoveItemAsync(string path) diff --git a/src/Sitko.Core.Storage/Cache/FileStorageCache.cs b/src/Sitko.Core.Storage/Cache/FileStorageCache.cs index 7d5cdeb3..e2634d9e 100644 --- a/src/Sitko.Core.Storage/Cache/FileStorageCache.cs +++ b/src/Sitko.Core.Storage/Cache/FileStorageCache.cs @@ -80,7 +80,7 @@ public static string CreateMD5(string input) } } - protected override async Task GetEntryAsync(FileDownloadResult item, + protected override async Task GetEntryAsync(StorageItemInfo item, Stream stream) { var tempFileName = CreateMD5(Guid.NewGuid().ToString()); diff --git a/src/Sitko.Core.Storage/Cache/IStorageCache.cs b/src/Sitko.Core.Storage/Cache/IStorageCache.cs index 20413111..1c6fc1c5 100644 --- a/src/Sitko.Core.Storage/Cache/IStorageCache.cs +++ b/src/Sitko.Core.Storage/Cache/IStorageCache.cs @@ -6,9 +6,9 @@ namespace Sitko.Core.Storage.Cache { public interface IStorageCache : IAsyncDisposable { - Task GetItemAsync(string path); + Task GetItemAsync(string path); - Task GetOrAddItemAsync(string path, Func> addItem); + Task GetOrAddItemAsync(string path, Func> addItem); Task RemoveItemAsync(string path); Task ClearAsync(); @@ -27,7 +27,6 @@ public interface IStorageCacheRecord DateTimeOffset Date { get; } public Stream OpenRead(); - public string? PhysicalPath { get; } } public abstract class StorageCacheOptions diff --git a/src/Sitko.Core.Storage/Cache/InMemoryStorageCache.cs b/src/Sitko.Core.Storage/Cache/InMemoryStorageCache.cs index e5dab2f2..bf09dd90 100644 --- a/src/Sitko.Core.Storage/Cache/InMemoryStorageCache.cs +++ b/src/Sitko.Core.Storage/Cache/InMemoryStorageCache.cs @@ -16,7 +16,7 @@ protected override void DisposeItem(InMemoryStorageCacheRecord deletedRecord) { } - protected override Task GetEntryAsync(FileDownloadResult item, Stream stream) + protected override Task GetEntryAsync(StorageItemInfo item, Stream stream) { using var memoryStream = new MemoryStream(); stream.CopyTo(memoryStream); @@ -40,7 +40,6 @@ public class InMemoryStorageCacheRecord : IStorageCacheRecord public string? Metadata { get; } public long FileSize { get; } public DateTimeOffset Date { get; } - public string? PhysicalPath { get; } public InMemoryStorageCacheRecord(string? metadata, long fileSize, DateTimeOffset date, byte[] data) { diff --git a/src/Sitko.Core.Storage/DownloadResult.cs b/src/Sitko.Core.Storage/DownloadResult.cs new file mode 100644 index 00000000..77f9a0a4 --- /dev/null +++ b/src/Sitko.Core.Storage/DownloadResult.cs @@ -0,0 +1,29 @@ +using System; +using System.IO; +using System.Threading.Tasks; + +namespace Sitko.Core.Storage +{ + public record DownloadResult(StorageItem StorageItem, Stream Stream) : IDisposable, IAsyncDisposable + { + private bool _isDisposed; + + public async ValueTask DisposeAsync() + { + if (!_isDisposed) + { + _isDisposed = true; + await Stream.DisposeAsync(); + } + } + + public void Dispose() + { + if (!_isDisposed) + { + Stream.Dispose(); + _isDisposed = true; + } + } + }; +} diff --git a/src/Sitko.Core.Storage/FileDownloadResult.cs b/src/Sitko.Core.Storage/FileDownloadResult.cs deleted file mode 100644 index 1f37af36..00000000 --- a/src/Sitko.Core.Storage/FileDownloadResult.cs +++ /dev/null @@ -1,27 +0,0 @@ -using System; -using System.IO; - -namespace Sitko.Core.Storage -{ - public class FileDownloadResult - { - public Stream Stream { get; } - - public string? Metadata { get; } - - public long FileSize { get; } - public DateTimeOffset Date { get; } - - public string? PhysicalPath { get; } - - public FileDownloadResult(string? metadata, long fileSize, DateTimeOffset date, Stream stream, - string? physicalPath = null) - { - Metadata = metadata; - FileSize = fileSize; - Date = date; - Stream = stream; - PhysicalPath = physicalPath; - } - } -} diff --git a/src/Sitko.Core.Storage/IStorage.cs b/src/Sitko.Core.Storage/IStorage.cs index aaf82f2d..e0bb03a5 100644 --- a/src/Sitko.Core.Storage/IStorage.cs +++ b/src/Sitko.Core.Storage/IStorage.cs @@ -7,14 +7,15 @@ namespace Sitko.Core.Storage { public interface IStorage { - Task SaveFileAsync(Stream file, string fileName, string path, + Task SaveAsync(Stream file, string fileName, string path, object? metadata = null); - Task GetFileAsync(string path); - Task DeleteFileAsync(string filePath); - Task IsFileExistsAsync(string path); + Task GetAsync(string path); + Task DownloadAsync(string path); + Task DeleteAsync(string filePath); + Task IsExistsAsync(string path); Task DeleteAllAsync(); - Task> GetDirectoryContentsAsync(string path); + Task> GetDirectoryContentsAsync(string path); Uri PublicUri(StorageItem item); } diff --git a/src/Sitko.Core.Storage/IStorageNode.cs b/src/Sitko.Core.Storage/IStorageNode.cs deleted file mode 100644 index d3fb80df..00000000 --- a/src/Sitko.Core.Storage/IStorageNode.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Sitko.Core.Storage -{ - public interface IStorageNode - { - string Name { get; } - string FullPath { get; } - } -} diff --git a/src/Sitko.Core.Storage/IsExternalInit.cs b/src/Sitko.Core.Storage/IsExternalInit.cs new file mode 100644 index 00000000..60e6fcdb --- /dev/null +++ b/src/Sitko.Core.Storage/IsExternalInit.cs @@ -0,0 +1,24 @@ +// https://github.com/dotnet/roslyn/issues/45510 + +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#if NETSTANDARD2_0 || NETSTANDARD2_1 || NETCOREAPP2_0 || NETCOREAPP2_1 || NETCOREAPP2_2 || NETCOREAPP3_0 || NETCOREAPP3_1 || NET45 || NET451 || NET452 || NET6 || NET461 || NET462 || NET47 || NET471 || NET472 || NET48 + +using System.ComponentModel; + +// ReSharper disable once CheckNamespace +namespace System.Runtime.CompilerServices +{ + /// + /// Reserved to be used by the compiler for tracking metadata. + /// This class should not be used by developers in source code. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + internal static class IsExternalInit + { + } +} + +#endif diff --git a/src/Sitko.Core.Storage/Storage.cs b/src/Sitko.Core.Storage/Storage.cs index d723fa13..0a3f7598 100644 --- a/src/Sitko.Core.Storage/Storage.cs +++ b/src/Sitko.Core.Storage/Storage.cs @@ -14,7 +14,7 @@ public abstract class Storage : IStorage, IAsyncDisposable where T : Stora protected readonly ILogger> Logger; private readonly IStorageCache? _cache; private readonly T _options; - private StorageFolder? _tree; + private StorageNode? _tree; private DateTimeOffset? _treeLastBuild; protected Storage(T options, ILogger> logger, IStorageCache? cache) @@ -24,7 +24,7 @@ protected Storage(T options, ILogger> logger, IStorageCache? cache) _options = options; } - public async Task SaveFileAsync(Stream file, string fileName, string path, object? metadata = null) + public async Task SaveAsync(Stream file, string fileName, string path, object? metadata = null) { string destinationPath = GetDestinationPath(fileName, path); @@ -65,37 +65,37 @@ private string GetDestinationPath(string fileName, string path) return destinationPath; } + protected StorageItem CreateStorageItem(string path, StorageItemInfo storageItemInfo) + { + return CreateStorageItem(path, storageItemInfo.Date, storageItemInfo.FileSize, storageItemInfo.Metadata); + } + protected StorageItem CreateStorageItem(string destinationPath, DateTimeOffset date, long fileSize, - string? metadata, - Stream? stream = null, string? physicalPath = null) + string? metadata) { return CreateStorageItem(destinationPath, date, fileSize, string.IsNullOrEmpty(metadata) ? new StorageItemMetadata {FileName = Path.GetFileName(destinationPath)} - : JsonSerializer.Deserialize(metadata), stream, - physicalPath); + : JsonSerializer.Deserialize(metadata)!); } private StorageItem CreateStorageItem(string destinationPath, DateTimeOffset date, long fileSize, - StorageItemMetadata metadata, - Stream? stream = null, string? physicalPath = null) + StorageItemMetadata metadata) { var storageItem = new StorageItem { + Path = PreparePath(Path.GetDirectoryName(destinationPath))!, FileName = metadata.FileName ?? Path.GetFileName(destinationPath), LastModified = date, FileSize = fileSize, FilePath = destinationPath, - Path = PreparePath(Path.GetDirectoryName(destinationPath))!, - Metadata = metadata, - Stream = stream, - PhysicalPath = physicalPath + Metadata = metadata }; return storageItem; } @@ -105,9 +105,21 @@ private StorageItem CreateStorageItem(string destinationPath, protected abstract Task DoIsFileExistsAsync(StorageItem item); protected abstract Task DoDeleteAllAsync(); - protected abstract Task DoGetFileAsync(string path); + protected abstract Task DoGetFileAsync(string path); - public async Task DeleteFileAsync(string filePath) + public async Task DownloadAsync(string path) + { + var info = await GetStorageItemInfoAsync(path); + if (info != null) + { + var item = CreateStorageItem(path, info); + return new DownloadResult(item, info.GetStream()); + } + + return null; + } + + public async Task DeleteAsync(string filePath) { if (_cache != null) { @@ -119,14 +131,14 @@ public async Task DeleteFileAsync(string filePath) return result; } - public Task GetFileAsync(string path) + public Task GetAsync(string path) { - return GetFileInternalAsync(path); + return GetStorageItemInternalAsync(path); } - private async Task GetFileInternalAsync(string path) + private async Task GetStorageItemInfoAsync(string path) { - FileDownloadResult? result; + StorageItemInfo? result; if (_cache != null) { result = await _cache.GetOrAddItemAsync(path, async () => await DoGetFileAsync(path)); @@ -136,19 +148,20 @@ public async Task DeleteFileAsync(string filePath) result = await DoGetFileAsync(path); } - if (result != null) - { - return CreateStorageItem(path, result.Date, result.FileSize, result.Metadata, result.Stream, - result.PhysicalPath); - } + return result; + } - return null; + private async Task GetStorageItemInternalAsync(string path) + { + var result = await GetStorageItemInfoAsync(path); + + return result != null ? CreateStorageItem(path, result.Date, result.FileSize, result.Metadata) : null; } - public async Task IsFileExistsAsync(string path) + public async Task IsExistsAsync(string path) { - var result = await GetFileInternalAsync(path); + var result = await GetStorageItemInternalAsync(path); return result != null; } @@ -165,23 +178,24 @@ public async Task DeleteAllAsync() } - public async Task> GetDirectoryContentsAsync(string path) + public async Task> GetDirectoryContentsAsync(string path) { if (_tree == null || _treeLastBuild < DateTimeOffset.UtcNow.Subtract(TimeSpan.FromMinutes(30))) { await BuildStorageTreeAsync(); } - if (_tree == null) { return new List(); } + if (_tree == null) { return new List(); } var parts = PreparePath(path.Trim('/'))!.Split("/"); var current = _tree; foreach (var part in parts) { - current = current?.Children.OfType().FirstOrDefault(f => f.Name == part); + current = current?.Children.Where(n => n.Type == StorageNodeType.Directory) + .FirstOrDefault(f => f.Name == part); } - return current?.Children ?? new IStorageNode[0]; + return current?.Children ?? new StorageNode[0]; } private async Task BuildStorageTreeAsync() @@ -190,7 +204,7 @@ private async Task BuildStorageTreeAsync() _treeLastBuild = DateTimeOffset.UtcNow; } - protected abstract Task DoBuildStorageTreeAsync(); + protected abstract Task DoBuildStorageTreeAsync(); public Uri PublicUri(StorageItem item) { diff --git a/src/Sitko.Core.Storage/StorageFolder.cs b/src/Sitko.Core.Storage/StorageFolder.cs deleted file mode 100644 index 208923b9..00000000 --- a/src/Sitko.Core.Storage/StorageFolder.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System.Collections.Generic; - -namespace Sitko.Core.Storage -{ - public class StorageFolder : IStorageNode - { - private readonly List _children = new List(); - public string Name { get; } - public string FullPath { get; } - public IEnumerable Children => _children.ToArray(); - - public StorageFolder(string name, string fullPath, IEnumerable? children = null) - { - Name = name; - FullPath = fullPath; - if (children != null) - { - _children.AddRange(children); - } - } - - public void AddChild(IStorageNode child) - { - _children.Add(child); - } - - public void SetChildren(IEnumerable children) - { - _children.Clear(); - _children.AddRange(children); - } - } -} diff --git a/src/Sitko.Core.Storage/StorageItem.cs b/src/Sitko.Core.Storage/StorageItem.cs index cd071a44..c3888acb 100644 --- a/src/Sitko.Core.Storage/StorageItem.cs +++ b/src/Sitko.Core.Storage/StorageItem.cs @@ -1,38 +1,17 @@ using System; -using System.IO; -using System.Threading.Tasks; namespace Sitko.Core.Storage { - public sealed class StorageItem : IStorageNode, IAsyncDisposable + public sealed class StorageItem { - public Stream? Stream { get; internal set; } - public string? FileName { get; set; } public long FileSize { get; set; } public string? FilePath { get; set; } - public string Name => FileName; - public string FullPath => FilePath; public DateTimeOffset LastModified { get; set; } - public string? PhysicalPath { get; internal set; } - public string Path { get; set; } - public string? StorageFileName => FilePath?.Substring(FilePath.LastIndexOf('/') + 1); internal StorageItemMetadata? Metadata { get; set; } - public StorageItem() - { - } - - public StorageItem(StorageItem item) - { - FileName = item.FileName; - FileSize = item.FileSize; - FilePath = item.FilePath; - Path = item.Path; - } - public TMetadata? GetMetadata() where TMetadata : class { return Metadata?.GetData(); @@ -45,13 +24,5 @@ public string HumanSize return Helpers.HumanSize(FileSize); } } - - public async ValueTask DisposeAsync() - { - if (Stream != null) - { - await Stream.DisposeAsync(); - } - } } } diff --git a/src/Sitko.Core.Storage/StorageItemInfo.cs b/src/Sitko.Core.Storage/StorageItemInfo.cs new file mode 100644 index 00000000..76892b7b --- /dev/null +++ b/src/Sitko.Core.Storage/StorageItemInfo.cs @@ -0,0 +1,23 @@ +using System; +using System.IO; + +namespace Sitko.Core.Storage +{ + public class StorageItemInfo + { + public Func GetStream { get; } + + public string? Metadata { get; } + + public long FileSize { get; } + public DateTimeOffset Date { get; } + + public StorageItemInfo(string? metadata, long fileSize, DateTimeOffset date, Func getStream) + { + Metadata = metadata; + FileSize = fileSize; + Date = date; + GetStream = getStream; + } + } +} diff --git a/src/Sitko.Core.Storage/StorageNode.cs b/src/Sitko.Core.Storage/StorageNode.cs new file mode 100644 index 00000000..66d8f63d --- /dev/null +++ b/src/Sitko.Core.Storage/StorageNode.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Sitko.Core.Storage +{ + public sealed class StorageNode + { + public string Name { get; set; } = string.Empty; + public string FullPath { get; set; } = string.Empty; + public DateTimeOffset LastModified { get; set; } = DateTimeOffset.Now; + public long Size { get; set; } + public StorageNodeType Type { get; private set; } + private readonly List _children = new(); + public IEnumerable Children => _children.ToArray(); + public StorageItem? StorageItem { get; private set; } + + public static StorageNode CreateDirectory(string name, string fullPath, + IEnumerable? children = null) + { + var node = new StorageNode {Type = StorageNodeType.Directory, Name = name, FullPath = fullPath}; + if (children?.Any() == true) + { + node.SetChildren(children); + } + + return node; + } + + public static StorageNode CreateStorageItem(StorageItem storageItem) + { + return new() + { + Type = StorageNodeType.StorageItem, + Name = storageItem.FileName!, + FullPath = storageItem.FilePath!, + Size = storageItem.FileSize, + LastModified = storageItem.LastModified, + StorageItem = storageItem + }; + } + + public void AddChild(StorageNode child) + { + if (Type == StorageNodeType.Directory) + { + _children.Add(child); + Size += child.Size; + } + } + + public void SetChildren(IEnumerable children) + { + if (Type == StorageNodeType.Directory) + { + _children.Clear(); + _children.AddRange(children); + Size += children.Sum(s => s.Size); + } + } + + public string HumanSize + { + get + { + return Helpers.HumanSize(Size); + } + } + } + + + public enum StorageNodeType + { + Directory, + StorageItem + } +} diff --git a/tests/Sitko.Core.Storage.Tests/BasicTests.cs b/tests/Sitko.Core.Storage.Tests/BasicTests.cs index d2e5b4f9..83badab7 100644 --- a/tests/Sitko.Core.Storage.Tests/BasicTests.cs +++ b/tests/Sitko.Core.Storage.Tests/BasicTests.cs @@ -28,7 +28,7 @@ public async Task UploadFile() const string fileName = "file.txt"; await using (var file = File.Open("Data/file.txt", FileMode.Open)) { - uploaded = await storage.SaveFileAsync(file, fileName, "upload"); + uploaded = await storage.SaveAsync(file, fileName, "upload"); } Assert.NotNull(uploaded); @@ -51,19 +51,20 @@ public async Task DownloadFile() await using (var file = File.Open("Data/file.txt", FileMode.Open)) { fileLength = file.Length; - uploaded = await storage.SaveFileAsync(file, fileName, "upload"); + uploaded = await storage.SaveAsync(file, fileName, "upload"); } Assert.NotNull(uploaded); Assert.NotNull(uploaded.FilePath); - var downloaded = await storage.GetFileAsync(uploaded!.FilePath!); + var downloaded = await storage.DownloadAsync(uploaded!.FilePath!); Assert.NotNull(downloaded); - await using (downloaded!) + await using (downloaded) { - Assert.Equal(fileLength, downloaded?.FileSize); - Assert.Equal(fileName, downloaded?.FileName); + Assert.Equal(fileLength, downloaded?.StorageItem.FileSize); + Assert.Equal(fileLength, downloaded?.Stream.Length); + Assert.Equal(fileName, downloaded?.StorageItem.FileName); } } @@ -80,12 +81,12 @@ public async Task DeleteFile() const string fileName = "file.txt"; await using (var file = File.Open("Data/file.txt", FileMode.Open)) { - uploaded = await storage.SaveFileAsync(file, fileName, "upload"); + uploaded = await storage.SaveAsync(file, fileName, "upload"); } Assert.NotNull(uploaded); - var result = await storage.DeleteFileAsync(uploaded.FilePath!); + var result = await storage.DeleteAsync(uploaded.FilePath!); Assert.True(result); } @@ -99,7 +100,7 @@ public async Task DeleteFileError() Assert.NotNull(storage); - var result = await storage.DeleteFileAsync(Guid.NewGuid().ToString()); + var result = await storage.DeleteAsync(Guid.NewGuid().ToString()); Assert.False(result); } @@ -118,7 +119,7 @@ public async Task Traverse() var metaData = new FileMetaData(); await using (var file = File.Open("Data/file.txt", FileMode.Open)) { - uploaded = await storage.SaveFileAsync(file, fileName, "upload/dir1/dir2", metaData); + uploaded = await storage.SaveAsync(file, fileName, "upload/dir1/dir2", metaData); } Assert.NotNull(uploaded); @@ -128,7 +129,7 @@ public async Task Traverse() Assert.Single(uploadDirectoryContent); var first = uploadDirectoryContent.First(); Assert.NotNull(first); - Assert.IsType(first); + Assert.IsType(first); Assert.Equal("dir1", first.Name); var dir1DirectoryContent = await storage.GetDirectoryContentsAsync(first.FullPath); @@ -136,7 +137,7 @@ public async Task Traverse() Assert.Single(dir1DirectoryContent); var second = dir1DirectoryContent.First(); Assert.NotNull(second); - Assert.IsType(second); + Assert.IsType(second); Assert.Equal("dir2", second.Name); var dir2DirectoryContent = await storage.GetDirectoryContentsAsync(second.FullPath); @@ -144,13 +145,13 @@ public async Task Traverse() Assert.Single(dir2DirectoryContent); var fileNode = dir2DirectoryContent.First(); Assert.NotNull(fileNode); - Assert.IsType(fileNode); - if (fileNode is StorageItem item) + Assert.Equal(StorageNodeType.StorageItem, fileNode.Type); + if (fileNode.StorageItem != null) { Assert.Equal(uploaded.FilePath, fileNode.FullPath); Assert.Equal(fileName, fileNode.Name); - var itemMetaData = item.GetMetadata(); + var itemMetaData = fileNode.StorageItem.GetMetadata(); Assert.NotNull(itemMetaData); Assert.Equal(metaData.Id, itemMetaData.Id); } From 36d04cc6347b315f22df28a9360a5e1ce6635123 Mon Sep 17 00:00:00 2001 From: George Drak Date: Thu, 17 Dec 2020 11:39:20 +0500 Subject: [PATCH 2/3] Refactoring and docs --- .../FileSystemStorage.cs | 2 +- src/Sitko.Core.Storage.S3/S3Storage.cs | 2 +- .../Cache/BaseStorageCache.cs | 6 +- .../Cache/FileStorageCache.cs | 2 +- src/Sitko.Core.Storage/Cache/IStorageCache.cs | 4 +- .../Cache/InMemoryStorageCache.cs | 2 +- src/Sitko.Core.Storage/DownloadResult.cs | 21 ++++++- src/Sitko.Core.Storage/Helpers.cs | 4 +- src/Sitko.Core.Storage/IStorage.cs | 61 ++++++++++++++++++- src/Sitko.Core.Storage/Storage.cs | 13 ++-- src/Sitko.Core.Storage/StorageItem.cs | 47 +++++++++++--- .../StorageItemCollection.cs | 25 -------- src/Sitko.Core.Storage/StorageItemInfo.cs | 6 +- src/Sitko.Core.Storage/StorageNode.cs | 2 +- .../BaseS3StorageTestScope.cs | 1 + tests/Sitko.Core.Storage.Tests/BasicTests.cs | 28 +++++++++ 16 files changed, 171 insertions(+), 55 deletions(-) delete mode 100644 src/Sitko.Core.Storage/StorageItemCollection.cs diff --git a/src/Sitko.Core.Storage.FileSystem/FileSystemStorage.cs b/src/Sitko.Core.Storage.FileSystem/FileSystemStorage.cs index fa7d8de7..b21be52f 100644 --- a/src/Sitko.Core.Storage.FileSystem/FileSystemStorage.cs +++ b/src/Sitko.Core.Storage.FileSystem/FileSystemStorage.cs @@ -83,7 +83,7 @@ protected override Task DoDeleteAllAsync() return Task.CompletedTask; } - protected override async Task DoGetFileAsync(string path) + internal override async Task DoGetFileAsync(string path) { StorageItemInfo? result = null; var fullPath = Path.Combine(_storagePath, path); diff --git a/src/Sitko.Core.Storage.S3/S3Storage.cs b/src/Sitko.Core.Storage.S3/S3Storage.cs index da0d2015..2e6b83a9 100644 --- a/src/Sitko.Core.Storage.S3/S3Storage.cs +++ b/src/Sitko.Core.Storage.S3/S3Storage.cs @@ -195,7 +195,7 @@ protected override async Task DoDeleteAllAsync() return metaData; } - protected override async Task DoGetFileAsync(string path) + internal override async Task DoGetFileAsync(string path) { var fileResponse = await DownloadFileAsync(path); if (fileResponse == null) diff --git a/src/Sitko.Core.Storage/Cache/BaseStorageCache.cs b/src/Sitko.Core.Storage/Cache/BaseStorageCache.cs index 58978de5..c55899a0 100644 --- a/src/Sitko.Core.Storage/Cache/BaseStorageCache.cs +++ b/src/Sitko.Core.Storage/Cache/BaseStorageCache.cs @@ -51,7 +51,7 @@ IEnumerator IEnumerable.GetEnumerator() return GetEnumerator(); } - public async Task GetOrAddItemAsync(string path, + async Task IStorageCache.GetOrAddItemAsync(string path, Func> addItem) { if (_cache == null) @@ -133,7 +133,7 @@ IEnumerator IEnumerable.GetEnumerator() protected abstract void DisposeItem(TRecord deletedRecord); - protected abstract Task GetEntryAsync(StorageItemInfo item, Stream stream); + internal abstract Task GetEntryAsync(StorageItemInfo item, Stream stream); private string NormalizePath(string path) { @@ -146,7 +146,7 @@ private string NormalizePath(string path) return path; } - public Task GetItemAsync(string path) + Task IStorageCache.GetItemAsync(string path) { if (_cache == null) { diff --git a/src/Sitko.Core.Storage/Cache/FileStorageCache.cs b/src/Sitko.Core.Storage/Cache/FileStorageCache.cs index e2634d9e..3529f33a 100644 --- a/src/Sitko.Core.Storage/Cache/FileStorageCache.cs +++ b/src/Sitko.Core.Storage/Cache/FileStorageCache.cs @@ -80,7 +80,7 @@ public static string CreateMD5(string input) } } - protected override async Task GetEntryAsync(StorageItemInfo item, + internal override async Task GetEntryAsync(StorageItemInfo item, Stream stream) { var tempFileName = CreateMD5(Guid.NewGuid().ToString()); diff --git a/src/Sitko.Core.Storage/Cache/IStorageCache.cs b/src/Sitko.Core.Storage/Cache/IStorageCache.cs index 1c6fc1c5..5bc6cf62 100644 --- a/src/Sitko.Core.Storage/Cache/IStorageCache.cs +++ b/src/Sitko.Core.Storage/Cache/IStorageCache.cs @@ -6,9 +6,9 @@ namespace Sitko.Core.Storage.Cache { public interface IStorageCache : IAsyncDisposable { - Task GetItemAsync(string path); + internal Task GetItemAsync(string path); - Task GetOrAddItemAsync(string path, Func> addItem); + internal Task GetOrAddItemAsync(string path, Func> addItem); Task RemoveItemAsync(string path); Task ClearAsync(); diff --git a/src/Sitko.Core.Storage/Cache/InMemoryStorageCache.cs b/src/Sitko.Core.Storage/Cache/InMemoryStorageCache.cs index bf09dd90..dd885706 100644 --- a/src/Sitko.Core.Storage/Cache/InMemoryStorageCache.cs +++ b/src/Sitko.Core.Storage/Cache/InMemoryStorageCache.cs @@ -16,7 +16,7 @@ protected override void DisposeItem(InMemoryStorageCacheRecord deletedRecord) { } - protected override Task GetEntryAsync(StorageItemInfo item, Stream stream) + internal override Task GetEntryAsync(StorageItemInfo item, Stream stream) { using var memoryStream = new MemoryStream(); stream.CopyTo(memoryStream); diff --git a/src/Sitko.Core.Storage/DownloadResult.cs b/src/Sitko.Core.Storage/DownloadResult.cs index 77f9a0a4..8ccf5761 100644 --- a/src/Sitko.Core.Storage/DownloadResult.cs +++ b/src/Sitko.Core.Storage/DownloadResult.cs @@ -4,10 +4,29 @@ namespace Sitko.Core.Storage { - public record DownloadResult(StorageItem StorageItem, Stream Stream) : IDisposable, IAsyncDisposable + /// + /// Download file result with StorageItem and Stream + /// + public record DownloadResult : IDisposable, IAsyncDisposable { + /// + /// StorageItem with file info + /// + public StorageItem StorageItem { get; } + + /// + /// Stream with file data + /// + public Stream Stream { get; } + private bool _isDisposed; + public DownloadResult(StorageItem storageItem, Stream stream) + { + StorageItem = storageItem; + Stream = stream; + } + public async ValueTask DisposeAsync() { if (!_isDisposed) diff --git a/src/Sitko.Core.Storage/Helpers.cs b/src/Sitko.Core.Storage/Helpers.cs index 13ca67ec..8a0bb050 100644 --- a/src/Sitko.Core.Storage/Helpers.cs +++ b/src/Sitko.Core.Storage/Helpers.cs @@ -2,11 +2,11 @@ namespace Sitko.Core.Storage { - public static class Helpers + internal static class Helpers { private static readonly string[] _units = {"bytes", "KB", "MB", "GB", "TB", "PB"}; - public static string HumanSize(long fileSize) + internal static string HumanSize(long fileSize) { if (fileSize < 1) { diff --git a/src/Sitko.Core.Storage/IStorage.cs b/src/Sitko.Core.Storage/IStorage.cs index e0bb03a5..8d5121af 100644 --- a/src/Sitko.Core.Storage/IStorage.cs +++ b/src/Sitko.Core.Storage/IStorage.cs @@ -7,16 +7,71 @@ namespace Sitko.Core.Storage { public interface IStorage { + /// + /// Upload file to storage + /// + /// Stream of data to upload + /// Original file name + /// Path on storage to upload file into + /// Serializable object with file metadata + /// StorageItem with information about uploaded file Task SaveAsync(Stream file, string fileName, string path, object? metadata = null); - Task GetAsync(string path); - Task DownloadAsync(string path); + /// + /// Get uploaded file info without downloading file + /// + /// Full path to file in storage + /// StorageItem with information about uploaded file + Task GetAsync(string filePath); + + /// + /// Get uploaded file info and stream + /// + /// Full path to file in storage + /// DownloadResult with StorageItem and Stream + Task DownloadAsync(string filePath); + + /// + /// Delete file from storage + /// + /// Full path to file in storage + /// True if success Task DeleteAsync(string filePath); - Task IsExistsAsync(string path); + + /// + /// Check if file exists in storage + /// + /// Full path to file in storage + /// True if file exists + Task IsExistsAsync(string filePath); + + /// + /// Delete all files from storage. Specific to storage realization. + /// + /// Task DeleteAllAsync(); + + /// + /// List folders and files in specified path + /// + /// Path to list + /// List of StorageNode Task> GetDirectoryContentsAsync(string path); + + /// + /// Generate public uri for file + /// + /// StorageItem to generate uri for + /// Public URI Uri PublicUri(StorageItem item); + + /// + /// Generate public uri for file + /// + /// Path to file in storage + /// Public URI + Uri PublicUri(string filePath); } // ReSharper disable once UnusedTypeParameter diff --git a/src/Sitko.Core.Storage/Storage.cs b/src/Sitko.Core.Storage/Storage.cs index 0a3f7598..f4023f5d 100644 --- a/src/Sitko.Core.Storage/Storage.cs +++ b/src/Sitko.Core.Storage/Storage.cs @@ -65,7 +65,7 @@ private string GetDestinationPath(string fileName, string path) return destinationPath; } - protected StorageItem CreateStorageItem(string path, StorageItemInfo storageItemInfo) + internal StorageItem CreateStorageItem(string path, StorageItemInfo storageItemInfo) { return CreateStorageItem(path, storageItemInfo.Date, storageItemInfo.FileSize, storageItemInfo.Metadata); } @@ -95,7 +95,7 @@ private StorageItem CreateStorageItem(string destinationPath, LastModified = date, FileSize = fileSize, FilePath = destinationPath, - Metadata = metadata + MetadataJson = metadata.Data }; return storageItem; } @@ -105,7 +105,7 @@ private StorageItem CreateStorageItem(string destinationPath, protected abstract Task DoIsFileExistsAsync(StorageItem item); protected abstract Task DoDeleteAllAsync(); - protected abstract Task DoGetFileAsync(string path); + internal abstract Task DoGetFileAsync(string path); public async Task DownloadAsync(string path) { @@ -208,7 +208,12 @@ private async Task BuildStorageTreeAsync() public Uri PublicUri(StorageItem item) { - return new Uri($"{_options.PublicUri}/{item.FilePath}"); + return PublicUri(item.FilePath); + } + + public Uri PublicUri(string filePath) + { + return new Uri($"{_options.PublicUri}/{filePath}"); } private string GetStorageFileName(string fileName) diff --git a/src/Sitko.Core.Storage/StorageItem.cs b/src/Sitko.Core.Storage/StorageItem.cs index c3888acb..eb25bee4 100644 --- a/src/Sitko.Core.Storage/StorageItem.cs +++ b/src/Sitko.Core.Storage/StorageItem.cs @@ -1,22 +1,53 @@ using System; +using System.Text.Json; namespace Sitko.Core.Storage { - public sealed class StorageItem + public sealed record StorageItem { - public string? FileName { get; set; } + /// + /// Name of uploaded file + /// + public string? FileName { get; set; } = string.Empty; + + /// + /// Size of uploaded file + /// public long FileSize { get; set; } - public string? FilePath { get; set; } - public DateTimeOffset LastModified { get; set; } - public string Path { get; set; } - - internal StorageItemMetadata? Metadata { get; set; } + + /// + /// Full path to uploaded file in storage + /// + public string FilePath { get; set; } = string.Empty; + + /// + /// Last modified date of uploaded file + /// + public DateTimeOffset LastModified { get; set; } = DateTimeOffset.Now; + + /// + /// Path without file name to uploaded file in storage + /// + public string Path { get; set; } = string.Empty; + + /// + /// Uploaded file metadata JSON. Read-only. + /// + public string? MetadataJson { get; set; } + /// + /// Get uploaded file metadata mapped to object + /// + /// + /// Uploaded file metadata object (TMedatata) public TMetadata? GetMetadata() where TMetadata : class { - return Metadata?.GetData(); + return string.IsNullOrEmpty(MetadataJson) ? null : JsonSerializer.Deserialize(MetadataJson); } + /// + /// Uploaded file size in human-readable format + /// public string HumanSize { get diff --git a/src/Sitko.Core.Storage/StorageItemCollection.cs b/src/Sitko.Core.Storage/StorageItemCollection.cs deleted file mode 100644 index f675d483..00000000 --- a/src/Sitko.Core.Storage/StorageItemCollection.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Collections; -using System.Collections.Generic; - -namespace Sitko.Core.Storage -{ - public class StorageItemCollection : IEnumerable - { - private readonly List _files; - - public StorageItemCollection(List files) - { - _files = files; - } - - public IEnumerator GetEnumerator() - { - return _files.GetEnumerator(); - } - - IEnumerator IEnumerable.GetEnumerator() - { - return GetEnumerator(); - } - } -} diff --git a/src/Sitko.Core.Storage/StorageItemInfo.cs b/src/Sitko.Core.Storage/StorageItemInfo.cs index 76892b7b..27769ed5 100644 --- a/src/Sitko.Core.Storage/StorageItemInfo.cs +++ b/src/Sitko.Core.Storage/StorageItemInfo.cs @@ -1,9 +1,11 @@ using System; using System.IO; - +using System.Runtime.CompilerServices; +[assembly: InternalsVisibleTo("Sitko.Core.Storage.S3")] +[assembly: InternalsVisibleTo("Sitko.Core.Storage.FileSystem")] namespace Sitko.Core.Storage { - public class StorageItemInfo + internal class StorageItemInfo { public Func GetStream { get; } diff --git a/src/Sitko.Core.Storage/StorageNode.cs b/src/Sitko.Core.Storage/StorageNode.cs index 66d8f63d..7de6386f 100644 --- a/src/Sitko.Core.Storage/StorageNode.cs +++ b/src/Sitko.Core.Storage/StorageNode.cs @@ -4,7 +4,7 @@ namespace Sitko.Core.Storage { - public sealed class StorageNode + public sealed record StorageNode { public string Name { get; set; } = string.Empty; public string FullPath { get; set; } = string.Empty; diff --git a/tests/Sitko.Core.Storage.S3.Tests/BaseS3StorageTestScope.cs b/tests/Sitko.Core.Storage.S3.Tests/BaseS3StorageTestScope.cs index ea8c40c2..54d1fc77 100644 --- a/tests/Sitko.Core.Storage.S3.Tests/BaseS3StorageTestScope.cs +++ b/tests/Sitko.Core.Storage.S3.Tests/BaseS3StorageTestScope.cs @@ -1,5 +1,6 @@ using System; using System.Threading.Tasks; +using Sitko.Core.Storage.Cache; using Sitko.Core.Xunit; namespace Sitko.Core.Storage.S3.Tests diff --git a/tests/Sitko.Core.Storage.Tests/BasicTests.cs b/tests/Sitko.Core.Storage.Tests/BasicTests.cs index 83badab7..82aa62f7 100644 --- a/tests/Sitko.Core.Storage.Tests/BasicTests.cs +++ b/tests/Sitko.Core.Storage.Tests/BasicTests.cs @@ -156,6 +156,34 @@ public async Task Traverse() Assert.Equal(metaData.Id, itemMetaData.Id); } } + + [Fact] + public async Task Metadata() + { + var scope = await GetScopeAsync(); + + var storage = scope.Get>(); + + Assert.NotNull(storage); + + StorageItem uploaded; + const string fileName = "file.txt"; + var metaData = new FileMetaData(); + await using (var file = File.Open("Data/file.txt", FileMode.Open)) + { + uploaded = await storage.SaveAsync(file, fileName, "upload/dir1/dir2", metaData); + } + + Assert.NotNull(uploaded); + + var item = await storage.GetAsync(uploaded.FilePath); + + Assert.NotNull(item); + + var itemMetaData = item.GetMetadata(); + Assert.NotNull(itemMetaData); + Assert.Equal(metaData.Id, itemMetaData.Id); + } } public class FileMetaData From 50ae69730343a26613732c9e1b53df68bbe54baa Mon Sep 17 00:00:00 2001 From: George Drak Date: Thu, 17 Dec 2020 11:43:22 +0500 Subject: [PATCH 3/3] Add mime type to StorageItem --- src/Sitko.Core.Storage/Sitko.Core.Storage.csproj | 1 + src/Sitko.Core.Storage/Storage.cs | 7 ++++--- src/Sitko.Core.Storage/StorageItem.cs | 5 +++++ tests/Sitko.Core.Storage.Tests/BasicTests.cs | 1 + 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Sitko.Core.Storage/Sitko.Core.Storage.csproj b/src/Sitko.Core.Storage/Sitko.Core.Storage.csproj index 7fed6f12..8e5bf620 100644 --- a/src/Sitko.Core.Storage/Sitko.Core.Storage.csproj +++ b/src/Sitko.Core.Storage/Sitko.Core.Storage.csproj @@ -25,6 +25,7 @@ + diff --git a/src/Sitko.Core.Storage/Storage.cs b/src/Sitko.Core.Storage/Storage.cs index f4023f5d..88678daf 100644 --- a/src/Sitko.Core.Storage/Storage.cs +++ b/src/Sitko.Core.Storage/Storage.cs @@ -87,15 +87,16 @@ private StorageItem CreateStorageItem(string destinationPath, long fileSize, StorageItemMetadata metadata) { + var fileName = metadata.FileName ?? Path.GetFileName(destinationPath); var storageItem = new StorageItem { Path = PreparePath(Path.GetDirectoryName(destinationPath))!, - FileName = - metadata.FileName ?? Path.GetFileName(destinationPath), + FileName = fileName, LastModified = date, FileSize = fileSize, FilePath = destinationPath, - MetadataJson = metadata.Data + MetadataJson = metadata.Data, + MimeType = MimeMapping.MimeUtility.GetMimeMapping(fileName) }; return storageItem; } diff --git a/src/Sitko.Core.Storage/StorageItem.cs b/src/Sitko.Core.Storage/StorageItem.cs index eb25bee4..7f9ed587 100644 --- a/src/Sitko.Core.Storage/StorageItem.cs +++ b/src/Sitko.Core.Storage/StorageItem.cs @@ -14,6 +14,11 @@ public sealed record StorageItem /// Size of uploaded file /// public long FileSize { get; set; } + + /// + /// MimeType of uploaded file + /// + public string MimeType { get; set; } = string.Empty; /// /// Full path to uploaded file in storage diff --git a/tests/Sitko.Core.Storage.Tests/BasicTests.cs b/tests/Sitko.Core.Storage.Tests/BasicTests.cs index 82aa62f7..e34ffbd5 100644 --- a/tests/Sitko.Core.Storage.Tests/BasicTests.cs +++ b/tests/Sitko.Core.Storage.Tests/BasicTests.cs @@ -34,6 +34,7 @@ public async Task UploadFile() Assert.NotNull(uploaded); Assert.NotEqual(0, uploaded.FileSize); Assert.Equal(fileName, uploaded.FileName); + Assert.Equal("text/plain", uploaded.MimeType); } [Fact]