Skip to content

Commit

Permalink
Merge pull request #154 from sitkoru/storage-tree-build
Browse files Browse the repository at this point in the history 
Improve storage tree build
  • Loading branch information
@SonicGD
SonicGD authored Jan 11, 2021
2 parents 46321bf + 6cff1e4 commit b9e60b8
Show file tree
Hide file tree
Showing 8 changed files with 136 additions and 48 deletions.
19 changes: 7 additions & 12 deletions src/Sitko.Core.Storage.FileSystem/FileSystemStorage.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading.Tasks;
Expand Down Expand Up @@ -107,24 +106,24 @@ protected override Task DoDeleteAllAsync()
return result;
}

protected override Task<StorageNode?> DoBuildStorageTreeAsync()
protected override async Task<StorageNode?> DoBuildStorageTreeAsync()
{
return ListFolderAsync("/");
var root = StorageNode.CreateDirectory("/", "/");
await ListFolderAsync(root, "/");
return root;
}

private async Task<StorageNode?> ListFolderAsync(string path)
private async Task ListFolderAsync(StorageNode root, string path)
{
var fullPath = path == "/" ? _storagePath : Path.Combine(_storagePath, path.Trim('/'));
List<StorageNode>? children = null;
if (Directory.Exists(fullPath))
{
children = new List<StorageNode>();
foreach (var info in new DirectoryInfo(fullPath)
.EnumerateFileSystemInfos())
{
if (info is DirectoryInfo dir)
{
children.Add(await ListFolderAsync(PreparePath(Path.Combine(path, dir.Name))));
await ListFolderAsync(root, PreparePath(Path.Combine(path, dir.Name)));
}

if (info is FileInfo file)
Expand All @@ -146,14 +145,10 @@ protected override Task DoDeleteAllAsync()
file.Length,
metadata);

children.Add(StorageNode.CreateStorageItem(item));
root.AddItem(item);
}
}
}

return StorageNode.CreateDirectory(path == "/" ? "/" : Path.GetFileNameWithoutExtension(path),
PreparePath(Path.Combine(_storagePath, path)),
children);
}
}
}
1 change: 1 addition & 0 deletions src/Sitko.Core.Storage.S3/IS3StorageOptions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@ public interface IS3StorageOptions
string Bucket { get; }
string AccessKey { get; }
string SecretKey { get; }
string BucketPath { get; }
}
}
55 changes: 28 additions & 27 deletions src/Sitko.Core.Storage.S3/S3Storage.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
Expand Down Expand Up @@ -187,14 +188,19 @@ protected override async Task DoDeleteAllAsync()
var metaDataResponse = await DownloadFileAsync(GetMetaDataPath(filePath));
if (metaDataResponse != null)
{
var buffer = new MemoryStream();
await metaDataResponse.ResponseStream.CopyToAsync(buffer);
metaData = Encoding.UTF8.GetString(buffer.ToArray());
metaData = await DownloadStreamAsString(metaDataResponse.ResponseStream);
}

return metaData;
}

private async Task<string> DownloadStreamAsString(Stream stream)
{
await using var buffer = new MemoryStream();
await stream.CopyToAsync(buffer);
return Encoding.UTF8.GetString(buffer.ToArray());
}

internal override async Task<StorageItemInfo?> DoGetFileAsync(string path)
{
var fileResponse = await DownloadFileAsync(path);
Expand All @@ -214,18 +220,25 @@ protected override async Task DoDeleteAllAsync()
var root = StorageNode.CreateDirectory("/", "/");
try
{
ListObjectsV2Request request = new ListObjectsV2Request {BucketName = _options.Bucket};
var request = new ListObjectsV2Request {BucketName = _options.Bucket, Prefix = _options.BucketPath};
ListObjectsV2Response response;
var objects = new Dictionary<string, S3Object>();
do
{
Logger.LogDebug($"Get objects list from S3. Current objects count: {objects.Count}");
response = await _client.ListObjectsV2Async(request);
foreach (var s3Object in response.S3Objects)
{
await AddObjectAsync(s3Object, root);
objects.Add(s3Object.Key, s3Object);
}

request.ContinuationToken = response.NextContinuationToken;
} while (response.IsTruncated);

foreach (var s3Object in objects.Values)
{
await AddObjectAsync(s3Object, root, objects);
}
}
catch (AmazonS3Exception amazonS3Exception)
{
Expand All @@ -241,32 +254,20 @@ protected override async Task DoDeleteAllAsync()
return root;
}

private async Task AddObjectAsync(S3Object s3Object, StorageNode root)
private async Task AddObjectAsync(S3Object s3Object, StorageNode root, Dictionary<string, S3Object> s3Objects)
{
if (s3Object.Key.EndsWith(MetaDataExtension)) return;
var parts = s3Object.Key.Split("/");
var current = root;
foreach (var part in parts)
{
if (part == parts.Last())
{
var metadata = await DownloadFileMetadataAsync(s3Object.Key);
var item = CreateStorageItem(s3Object.Key, s3Object.LastModified, s3Object.Size, metadata);
current.AddChild(StorageNode.CreateStorageItem(item));
}
else
{
var child = current.Children.Where(n => n.Type == StorageNodeType.Directory)
.FirstOrDefault(f => f.Name == part);
if (child == null)
{
child = StorageNode.CreateDirectory(part, PreparePath(Path.Combine(current.FullPath, part)));
current.AddChild(child);
}

current = child;
}
string? metadata = null;
var metadataPath = GetMetaDataPath(s3Object.Key);
if (s3Objects.ContainsKey(metadataPath))
{
metadata = await DownloadFileMetadataAsync(s3Object.Key);
}

var item = CreateStorageItem(s3Object.Key, s3Object.LastModified, s3Object.Size, metadata);

root.AddItem(item);
}

public override ValueTask DisposeAsync()
Expand Down
7 changes: 7 additions & 0 deletions src/Sitko.Core.Storage/IStorage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,13 @@ Task<StorageItem> SaveAsync(Stream file, string fileName, string path,
/// <param name="path">Path to list</param>
/// <returns>List of StorageNode</returns>
Task<IEnumerable<StorageNode>> GetDirectoryContentsAsync(string path);

/// <summary>
/// Refreshes storage items tree and returns folders and files in specified path
/// </summary>
/// <param name="path">Path to list</param>
/// <returns>List of StorageNode</returns>
Task<IEnumerable<StorageNode>> RefreshDirectoryContentsAsync(string path);

/// <summary>
/// Generate public uri for file
Expand Down
35 changes: 26 additions & 9 deletions src/Sitko.Core.Storage/Storage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,11 @@ public async Task<StorageItem> SaveAsync(Stream file, string fileName, string pa
itemMetadata);

var result = await SaveStorageItemAsync(file, path, destinationPath, storageItem, itemMetadata);
await RebuildStorageTreeAsync();
if (_tree != null)
{
_tree.AddItem(storageItem);
}

return result;
}

Expand All @@ -67,7 +71,7 @@ private string GetDestinationPath(string fileName, string path)
return destinationPath;
}

internal StorageItem CreateStorageItem(string path, StorageItemInfo storageItemInfo)
private StorageItem CreateStorageItem(string path, StorageItemInfo storageItemInfo)
{
return CreateStorageItem(path, storageItemInfo.Date, storageItemInfo.FileSize, storageItemInfo.Metadata);
}
Expand Down Expand Up @@ -130,7 +134,11 @@ public async Task<bool> DeleteAsync(string filePath)
}

var result = await DoDeleteAsync(filePath);
await RebuildStorageTreeAsync();
if (_tree != null)
{
_tree.RemoveItem(filePath);
}

return result;
}

Expand Down Expand Up @@ -183,7 +191,7 @@ public async Task DeleteAllAsync()

public async Task<IEnumerable<StorageNode>> GetDirectoryContentsAsync(string path)
{
if (_tree == null || _treeLastBuild < DateTimeOffset.UtcNow.Subtract(TimeSpan.FromMinutes(30)))
if (_tree == null || _treeLastBuild < DateTimeOffset.UtcNow.Subtract(_options.StorageTreeCacheTimeout))
{
await BuildStorageTreeAsync();
}
Expand All @@ -201,21 +209,30 @@ public async Task<IEnumerable<StorageNode>> GetDirectoryContentsAsync(string pat
return current?.Children ?? new StorageNode[0];
}

private async Task RebuildStorageTreeAsync()
public async Task<IEnumerable<StorageNode>> RefreshDirectoryContentsAsync(string path)
{
if (_tree != null)
{
await BuildStorageTreeAsync();
}
await BuildStorageTreeAsync();
return await GetDirectoryContentsAsync(path);
}

private TaskCompletionSource<bool>? _treeBuildTaskSource = null;

private async Task BuildStorageTreeAsync()
{
if (_treeBuildTaskSource != null)
{
await _treeBuildTaskSource.Task;
return;
}

using (await _treeLock.LockAsync())
{
Logger.LogInformation("Start building storage tree");
_treeBuildTaskSource = new TaskCompletionSource<bool>();
_tree = await DoBuildStorageTreeAsync();
_treeLastBuild = DateTimeOffset.UtcNow;
_treeBuildTaskSource.SetResult(true);
_treeBuildTaskSource = null;
Logger.LogInformation("Done building storage tree");
}
}
Expand Down
63 changes: 63 additions & 0 deletions src/Sitko.Core.Storage/StorageNode.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;

namespace Sitko.Core.Storage
Expand Down Expand Up @@ -59,6 +60,68 @@ public void SetChildren(IEnumerable<StorageNode> children)
}
}

public void AddItem(StorageItem item)
{
var parts = item.FilePath.Split("/");
var current = this;
foreach (var part in parts)
{
if (part == parts.Last())
{
current.AddChild(CreateStorageItem(item));
}
else
{
var child = current.Children.Where(n => n.Type == StorageNodeType.Directory)
.FirstOrDefault(f => f.Name == part);
if (child == null)
{
child = CreateDirectory(part, PreparePath(Path.Combine(current.FullPath, part)));
current.AddChild(child);
}

current = child;
}
}
}

public void RemoveItem(StorageItem item)
{
RemoveItem(item.FilePath);
}

public void RemoveItem(string filePath)
{
var parts = filePath.Split("/");
var current = this;
foreach (var part in parts)
{
if (part == parts.Last())
{
var children = current._children.FirstOrDefault(c =>
c.Type == StorageNodeType.StorageItem && c.StorageItem!.FilePath == filePath);
if (children != null)
{
current._children.Remove(children);
}
}
else
{
var child = current.Children.Where(n => n.Type == StorageNodeType.Directory)
.FirstOrDefault(f => f.Name == part);
if (child == null)
{
return;
}
}
}
}

private string PreparePath(string path)
{
return path.Replace("\\", "/").Replace("//", "/");
}

public string HumanSize
{
get
Expand Down
2 changes: 2 additions & 0 deletions src/Sitko.Core.Storage/StorageOptions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ namespace Sitko.Core.Storage
public abstract class StorageOptions
{
public Uri? PublicUri { get; set; }

public TimeSpan StorageTreeCacheTimeout { get; set; } = TimeSpan.FromMinutes(30);

public Action<IHostEnvironment, IConfiguration, IServiceCollection>? ConfigureCache { get; protected set; }

Expand Down
2 changes: 2 additions & 0 deletions tests/Sitko.Core.Storage.S3.Tests/TestS3StorageSettings.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,7 @@ public class TestS3StorageSettings : StorageOptions, IS3StorageOptions
public string Bucket { get; set; }
public string AccessKey { get; set; }
public string SecretKey { get; set; }

public string BucketPath { get; } = string.Empty;
}
}

0 comments on commit b9e60b8

Please sign in to comment.