IlyaSobolev/DocSpace-buildtools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Storage: refactor

Browse Source
This commit is contained in:
Maksim Chegulov 2022-02-11 13:04:06 +03:00
parent 359d5efe8d
commit f11bf4c124
5 changed files with 90 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
common/ASC.Data.Storage/BaseStorage.cs
View File

@ -31,18 +31,19 @@ public abstract class BaseStorage : IDataStore
public virtual bool IsSupportInternalUri => true;
public virtual bool IsSupportedPreSignedUri => true;
public virtual bool IsSupportChunking => false;
internal string Modulename { get; set; }
internal DataList DataList { get; set; }
internal string Tenant { get; set; }
internal Dictionary<string, TimeSpan> DomainsExpires { get; set; }
= new Dictionary<string, TimeSpan>();
protected ILog Logger { get; set; }
internal string _modulename;
internal DataList _dataList;
internal string _tenant;
internal Dictionary<string, TimeSpan> _domainsExpires = new Dictionary<string, TimeSpan>();
protected readonly TempStream _tempStream;
protected readonly TenantManager _tenantManager;
protected readonly PathUtils _pathUtils;
protected readonly EmailValidationKeyProvider _emailValidationKeyProvider;
protected readonly IHttpContextAccessor _httpContextAccessor;
protected readonly IOptionsMonitor<ILog> _options;
protected readonly TempStream TempStream;
protected readonly TenantManager TenantManager;
protected readonly PathUtils TpathUtils;
protected readonly EmailValidationKeyProvider TemailValidationKeyProvider;
protected readonly IHttpContextAccessor HttpContextAccessor;
protected readonly IOptionsMonitor<ILog> Options;
public BaseStorage(
TempStream tempStream,
@ -53,18 +54,18 @@ public abstract class BaseStorage : IDataStore
IOptionsMonitor<ILog> options)
{
_tempStream = tempStream;
_tenantManager = tenantManager;
_pathUtils = pathUtils;
_emailValidationKeyProvider = emailValidationKeyProvider;
_options = options;
TempStream = tempStream;
TenantManager = tenantManager;
TpathUtils = pathUtils;
TemailValidationKeyProvider = emailValidationKeyProvider;
Options = options;
Logger = options.CurrentValue;
_httpContextAccessor = httpContextAccessor;
HttpContextAccessor = httpContextAccessor;
}
public TimeSpan GetExpire(string domain)
{
return _domainsExpires.ContainsKey(domain) ? _domainsExpires[domain] : _domainsExpires[string.Empty];
return DomainsExpires.ContainsKey(domain) ? DomainsExpires[domain] : DomainsExpires[string.Empty];
}
public Uri GetUri(string path)
@ -84,7 +85,7 @@ public abstract class BaseStorage : IDataStore
throw new ArgumentNullException(nameof(path));
}
if (string.IsNullOrEmpty(_tenant) && IsSupportInternalUri)
if (string.IsNullOrEmpty(Tenant) && IsSupportInternalUri)
{
return GetInternalUri(domain, path, expire, headers);
}
@ -106,17 +107,17 @@ public abstract class BaseStorage : IDataStore
var expireString = expire.TotalMinutes.ToString(CultureInfo.InvariantCulture);
int currentTenantId;
var currentTenant = _tenantManager.GetCurrentTenant(false);
var currentTenant = TenantManager.GetCurrentTenant(false);
if (currentTenant != null)
{
currentTenantId = currentTenant.TenantId;
}
else if (!TenantPath.TryGetTenant(_tenant, out currentTenantId))
else if (!TenantPath.TryGetTenant(Tenant, out currentTenantId))
{
currentTenantId = 0;
}
var auth = _emailValidationKeyProvider.GetEmailKey(currentTenantId, path.Replace('/', Path.DirectorySeparatorChar).Replace('\\', Path.DirectorySeparatorChar) + "." + headerAttr + "." + expireString);
var auth = TemailValidationKeyProvider.GetEmailKey(currentTenantId, path.Replace('/', Path.DirectorySeparatorChar).Replace('\\', Path.DirectorySeparatorChar) + "." + headerAttr + "." + expireString);
query = string.Format("{0}{1}={2}&{3}={4}",
path.Contains("?") ? "&" : "?",
Constants.QueryExpire,
@ -133,9 +134,9 @@ public abstract class BaseStorage : IDataStore
HttpUtility.UrlEncode(headerAttr));
}
var tenant = _tenant.Trim('/');
var vpath = _pathUtils.ResolveVirtualPath(_modulename, domain);
vpath = _pathUtils.ResolveVirtualPath(vpath, false);
var tenant = Tenant.Trim('/');
var vpath = TpathUtils.ResolveVirtualPath(Modulename, domain);
vpath = TpathUtils.ResolveVirtualPath(vpath, false);
vpath = string.Format(vpath, tenant);
var virtualPath = new Uri(vpath + "/", UriKind.RelativeOrAbsolute);
@ -358,7 +359,7 @@ public abstract class BaseStorage : IDataStore
{
if (QuotaController != null)
{
QuotaController.QuotaUsedAdd(_modulename, domain, _dataList.GetData(domain), size, quotaCheckFileSize);
QuotaController.QuotaUsedAdd(Modulename, domain, DataList.GetData(domain), size, quotaCheckFileSize);
}
}
@ -366,7 +367,7 @@ public abstract class BaseStorage : IDataStore
{
if (QuotaController != null)
{
QuotaController.QuotaUsedDelete(_modulename, domain, _dataList.GetData(domain), size);
QuotaController.QuotaUsedDelete(Modulename, domain, DataList.GetData(domain), size);
}
}

18
common/ASC.Data.Storage/DiscStorage/DiscDataStore.cs
View File

@ -39,25 +39,25 @@ public class DiscDataStore : BaseStorage
public override IDataStore Configure(string tenant, Handler handlerConfig, Module moduleConfig, IDictionary<string, string> props)
{
_tenant = tenant;
Tenant = tenant;
//Fill map path
_modulename = moduleConfig.Name;
_dataList = new DataList(moduleConfig);
Modulename = moduleConfig.Name;
DataList = new DataList(moduleConfig);
foreach (var domain in moduleConfig.Domain)
{
_mappedPaths.Add(domain.Name, new MappedPath(_pathUtils, tenant, moduleConfig.AppendTenantId, domain.Path, handlerConfig.GetProperties()));
_mappedPaths.Add(domain.Name, new MappedPath(TpathUtils, tenant, moduleConfig.AppendTenantId, domain.Path, handlerConfig.GetProperties()));
}
//Add default
_mappedPaths.Add(string.Empty, new MappedPath(_pathUtils, tenant, moduleConfig.AppendTenantId, PathUtils.Normalize(moduleConfig.Path), handlerConfig.GetProperties()));
_mappedPaths.Add(string.Empty, new MappedPath(TpathUtils, tenant, moduleConfig.AppendTenantId, PathUtils.Normalize(moduleConfig.Path), handlerConfig.GetProperties()));
//Make expires
_domainsExpires =
DomainsExpires =
moduleConfig.Domain.Where(x => x.Expires != TimeSpan.Zero).
ToDictionary(x => x.Name,
y => y.Expires);
_domainsExpires.Add(string.Empty, moduleConfig.Expires);
DomainsExpires.Add(string.Empty, moduleConfig.Expires);
var settings = moduleConfig.DisabledEncryption ? new EncryptionSettings() : _encryptionSettingsHelper.Load();
_crypt = _encryptionFactory.GetCrypt(moduleConfig.Name, settings);
@ -157,7 +157,7 @@ public class DiscDataStore : BaseStorage
{
Logger.Debug("Save " + path);
var buffered = _tempStream.GetBuffered(stream);
var buffered = TempStream.GetBuffered(stream);
if (QuotaController != null)
{
QuotaController.QuotaUsedCheck(buffered.Length);
@ -633,7 +633,7 @@ public class DiscDataStore : BaseStorage
if (QuotaController != null)
{
var size = GetUsedQuota(domain);
QuotaController.QuotaUsedSet(_modulename, domain, _dataList.GetData(domain), size);
QuotaController.QuotaUsedSet(Modulename, domain, DataList.GetData(domain), size);
}
return 0;

32
common/ASC.Data.Storage/GoogleCloud/GoogleCloudStorage.cs
View File

@ -51,26 +51,26 @@ public class GoogleCloudStorage : BaseStorage
public override IDataStore Configure(string tenant, Handler handlerConfig, Module moduleConfig, IDictionary<string, string> props)
{
_tenant = tenant;
Tenant = tenant;
if (moduleConfig != null)
{
_modulename = moduleConfig.Name;
_dataList = new DataList(moduleConfig);
Modulename = moduleConfig.Name;
DataList = new DataList(moduleConfig);
_domainsExpires = moduleConfig.Domain.Where(x => x.Expires != TimeSpan.Zero).ToDictionary(x => x.Name, y => y.Expires);
DomainsExpires = moduleConfig.Domain.Where(x => x.Expires != TimeSpan.Zero).ToDictionary(x => x.Name, y => y.Expires);
_domainsExpires.Add(string.Empty, moduleConfig.Expires);
DomainsExpires.Add(string.Empty, moduleConfig.Expires);
_domainsAcl = moduleConfig.Domain.ToDictionary(x => x.Name, y => GetGoogleCloudAcl(y.Acl));
_moduleAcl = GetGoogleCloudAcl(moduleConfig.Acl);
}
else
{
_modulename = string.Empty;
_dataList = null;
Modulename = string.Empty;
DataList = null;
_domainsExpires = new Dictionary<string, TimeSpan> { { string.Empty, TimeSpan.Zero } };
DomainsExpires = new Dictionary<string, TimeSpan> { { string.Empty, TimeSpan.Zero } };
_domainsAcl = new Dictionary<string, PredefinedObjectAcl>();
_moduleAcl = PredefinedObjectAcl.PublicRead;
}
@ -129,7 +129,7 @@ public class GoogleCloudStorage : BaseStorage
public Uri GetUriShared(string domain, string path)
{
return new Uri(SecureHelper.IsSecure(_httpContextAccessor.HttpContext, _options) ? _bucketSSlRoot : _bucketRoot, MakePath(domain, path));
return new Uri(SecureHelper.IsSecure(HttpContextAccessor.HttpContext, Options) ? _bucketSSlRoot : _bucketRoot, MakePath(domain, path));
}
public override Stream GetReadStream(string domain, string path)
@ -139,7 +139,7 @@ public class GoogleCloudStorage : BaseStorage
public override Stream GetReadStream(string domain, string path, int offset)
{
var tempStream = _tempStream.Create();
var tempStream = TempStream.Create();
using var storage = GetStorage();
@ -157,7 +157,7 @@ public class GoogleCloudStorage : BaseStorage
public override async Task<Stream> GetReadStreamAsync(string domain, string path, int offset)
{
var tempStream = _tempStream.Create();
var tempStream = TempStream.Create();
var storage = GetStorage();
@ -197,7 +197,7 @@ public class GoogleCloudStorage : BaseStorage
string contentDisposition, ACL acl, string contentEncoding = null, int cacheDays = 5)
{
var buffered = _tempStream.GetBuffered(stream);
var buffered = TempStream.GetBuffered(stream);
if (QuotaController != null)
{
@ -503,7 +503,7 @@ public class GoogleCloudStorage : BaseStorage
}
}
QuotaController.QuotaUsedSet(_modulename, domain, _dataList.GetData(domain), size);
QuotaController.QuotaUsedSet(Modulename, domain, DataList.GetData(domain), size);
return size;
}
@ -581,7 +581,7 @@ public class GoogleCloudStorage : BaseStorage
using var storage = GetStorage();
var objectKey = MakePath(domain, path);
var buffered = _tempStream.GetBuffered(stream);
var buffered = TempStream.GetBuffered(stream);
var uploadObjectOptions = new UploadObjectOptions
{
@ -803,8 +803,8 @@ public class GoogleCloudStorage : BaseStorage
else//Key combined from module+domain+filename
{
result = string.Format("{0}/{1}/{2}/{3}",
_tenant,
_modulename,
Tenant,
Modulename,
domain,
path);
}

30
common/ASC.Data.Storage/RackspaceCloud/RackspaceCloudStorage.cs
View File

@ -61,23 +61,23 @@ public class RackspaceCloudStorage : BaseStorage
public override IDataStore Configure(string tenant, Handler handlerConfig, Module moduleConfig, IDictionary<string, string> props)
{
_tenant = tenant;
Tenant = tenant;
if (moduleConfig != null)
{
_modulename = moduleConfig.Name;
_dataList = new DataList(moduleConfig);
Modulename = moduleConfig.Name;
DataList = new DataList(moduleConfig);
_domains.AddRange(moduleConfig.Domain.Select(x => string.Format("{0}/", x.Name)));
_domainsExpires = moduleConfig.Domain.Where(x => x.Expires != TimeSpan.Zero).ToDictionary(x => x.Name, y => y.Expires);
_domainsExpires.Add(string.Empty, moduleConfig.Expires);
DomainsExpires = moduleConfig.Domain.Where(x => x.Expires != TimeSpan.Zero).ToDictionary(x => x.Name, y => y.Expires);
DomainsExpires.Add(string.Empty, moduleConfig.Expires);
_domainsAcl = moduleConfig.Domain.ToDictionary(x => x.Name, y => y.Acl);
_moduleAcl = moduleConfig.Acl;
}
else
{
_modulename = string.Empty;
_dataList = null;
_domainsExpires = new Dictionary<string, TimeSpan> { { string.Empty, TimeSpan.Zero } };
Modulename = string.Empty;
DataList = null;
DomainsExpires = new Dictionary<string, TimeSpan> { { string.Empty, TimeSpan.Zero } };
_domainsAcl = new Dictionary<string, ACL>();
_moduleAcl = ACL.Auto;
}
@ -164,7 +164,7 @@ public class RackspaceCloudStorage : BaseStorage
public override Stream GetReadStream(string domain, string path, int offset)
{
var outputStream = _tempStream.Create();
var outputStream = TempStream.Create();
var client = GetClient();
@ -209,7 +209,7 @@ public class RackspaceCloudStorage : BaseStorage
string contentDisposition, ACL acl, string contentEncoding = null, int cacheDays = 5,
DateTime? deleteAt = null, long? deleteAfter = null)
{
var buffered = _tempStream.GetBuffered(stream);
var buffered = TempStream.GetBuffered(stream);
if (QuotaController != null)
{
@ -260,7 +260,7 @@ public class RackspaceCloudStorage : BaseStorage
try
{
using (var emptyStream = _tempStream.Create())
using (var emptyStream = TempStream.Create())
{
var headers = new Dictionary<string, string>
@ -539,7 +539,7 @@ public class RackspaceCloudStorage : BaseStorage
size += obj.Bytes;
}
QuotaController.QuotaUsedSet(_modulename, domain, _dataList.GetData(domain), size);
QuotaController.QuotaUsedSet(Modulename, domain, DataList.GetData(domain), size);
return size;
}
@ -716,8 +716,8 @@ public class RackspaceCloudStorage : BaseStorage
else//Key combined from module+domain+filename
{
result = string.Format("{0}/{1}/{2}/{3}",
_tenant,
_modulename,
Tenant,
Modulename,
domain,
path);
}
@ -744,7 +744,7 @@ public class RackspaceCloudStorage : BaseStorage
private Uri GetUriShared(string domain, string path)
{
return new Uri(string.Format("{0}{1}", SecureHelper.IsSecure(_httpContextAccessor?.HttpContext, _options) ? _cnameSSL : _cname, MakePath(domain, path)));
return new Uri(string.Format("{0}{1}", SecureHelper.IsSecure(HttpContextAccessor?.HttpContext, Options) ? _cnameSSL : _cname, MakePath(domain, path)));
}
private ACL GetDomainACL(string domain)

46
common/ASC.Data.Storage/S3/S3Storage.cs
View File

@ -62,12 +62,12 @@ public class S3Storage : BaseStorage
public Uri GetUriInternal(string path)
{
return new Uri(SecureHelper.IsSecure(_httpContextAccessor?.HttpContext, _options) ? _bucketSSlRoot : _bucketRoot, path);
return new Uri(SecureHelper.IsSecure(HttpContextAccessor?.HttpContext, Options) ? _bucketSSlRoot : _bucketRoot, path);
}
public Uri GetUriShared(string domain, string path)
{
return new Uri(SecureHelper.IsSecure(_httpContextAccessor?.HttpContext, _options) ? _bucketSSlRoot : _bucketRoot, MakePath(domain, path));
return new Uri(SecureHelper.IsSecure(HttpContextAccessor?.HttpContext, Options) ? _bucketSSlRoot : _bucketRoot, MakePath(domain, path));
}
public override Uri GetInternalUri(string domain, string path, TimeSpan expire, IEnumerable<string> headers)
@ -86,7 +86,7 @@ public class S3Storage : BaseStorage
BucketName = _bucket,
Expires = DateTime.UtcNow.Add(expire),
Key = MakePath(domain, path),
Protocol = SecureHelper.IsSecure(_httpContextAccessor?.HttpContext, _options) ? Protocol.HTTPS : Protocol.HTTP,
Protocol = SecureHelper.IsSecure(HttpContextAccessor?.HttpContext, Options) ? Protocol.HTTPS : Protocol.HTTP,
Verb = HttpVerb.GET
};
@ -208,7 +208,7 @@ public class S3Storage : BaseStorage
public Uri Save(string domain, string path, Stream stream, string contentType,
string contentDisposition, ACL acl, string contentEncoding = null, int cacheDays = 5)
{
var buffered = _tempStream.GetBuffered(stream);
var buffered = TempStream.GetBuffered(stream);
if (QuotaController != null)
{
QuotaController.QuotaUsedCheck(buffered.Length);
@ -590,7 +590,7 @@ public class S3Storage : BaseStorage
using var client = GetClient();
using var uploader = new TransferUtility(client);
var objectKey = MakePath(domain, path);
var buffered = _tempStream.GetBuffered(stream);
var buffered = TempStream.GetBuffered(stream);
var request = new TransferUtilityUploadRequest
{
BucketName = _bucket,
@ -748,11 +748,11 @@ public class S3Storage : BaseStorage
public override string GetUploadedUrl(string domain, string directoryPath)
{
if (_httpContextAccessor?.HttpContext != null)
if (HttpContextAccessor?.HttpContext != null)
{
var buket = _httpContextAccessor?.HttpContext.Request.Query["bucket"].FirstOrDefault();
var key = _httpContextAccessor?.HttpContext.Request.Query["key"].FirstOrDefault();
var etag = _httpContextAccessor?.HttpContext.Request.Query["etag"].FirstOrDefault();
var buket = HttpContextAccessor?.HttpContext.Request.Query["bucket"].FirstOrDefault();
var key = HttpContextAccessor?.HttpContext.Request.Query["key"].FirstOrDefault();
var etag = HttpContextAccessor?.HttpContext.Request.Query["etag"].FirstOrDefault();
var destkey = MakePath(domain, directoryPath) + "/";
if (!string.IsNullOrEmpty(buket) && !string.IsNullOrEmpty(key) && string.Equals(buket, _bucket) &&
@ -760,9 +760,9 @@ public class S3Storage : BaseStorage
{
var domainpath = key.Substring(MakePath(domain, string.Empty).Length);
var skipQuota = false;
if (_httpContextAccessor?.HttpContext.Session != null)
if (HttpContextAccessor?.HttpContext.Session != null)
{
_httpContextAccessor.HttpContext.Session.TryGetValue(etag, out var isCounted);
HttpContextAccessor.HttpContext.Session.TryGetValue(etag, out var isCounted);
skipQuota = isCounted != null;
}
//Add to quota controller
@ -773,7 +773,7 @@ public class S3Storage : BaseStorage
var size = GetFileSize(domain, domainpath);
QuotaUsedAdd(domain, size);
if (_httpContextAccessor?.HttpContext.Session != null)
if (HttpContextAccessor?.HttpContext.Session != null)
{
//TODO:
//HttpContext.Current.Session.Add(etag, size);
@ -910,7 +910,7 @@ public class S3Storage : BaseStorage
{
var objects = GetS3Objects(domain);
var size = objects.Sum(s3Object => s3Object.Size);
QuotaController.QuotaUsedSet(_modulename, domain, _dataList.GetData(domain), size);
QuotaController.QuotaUsedSet(Modulename, domain, DataList.GetData(domain), size);
return size;
}
@ -977,28 +977,28 @@ public class S3Storage : BaseStorage
public override IDataStore Configure(string tenant, Handler handlerConfig, Module moduleConfig, IDictionary<string, string> props)
{
_tenant = tenant;
Tenant = tenant;
if (moduleConfig != null)
{
_modulename = moduleConfig.Name;
_dataList = new DataList(moduleConfig);
Modulename = moduleConfig.Name;
DataList = new DataList(moduleConfig);
_domains.AddRange(moduleConfig.Domain.Select(x => string.Format("{0}/", x.Name)));
//Make expires
_domainsExpires = moduleConfig.Domain.Where(x => x.Expires != TimeSpan.Zero).ToDictionary(x => x.Name, y => y.Expires);
_domainsExpires.Add(string.Empty, moduleConfig.Expires);
DomainsExpires = moduleConfig.Domain.Where(x => x.Expires != TimeSpan.Zero).ToDictionary(x => x.Name, y => y.Expires);
DomainsExpires.Add(string.Empty, moduleConfig.Expires);
_domainsAcl = moduleConfig.Domain.ToDictionary(x => x.Name, y => GetS3Acl(y.Acl));
_moduleAcl = GetS3Acl(moduleConfig.Acl);
}
else
{
_modulename = string.Empty;
_dataList = null;
Modulename = string.Empty;
DataList = null;
//Make expires
_domainsExpires = new Dictionary<string, TimeSpan> { { string.Empty, TimeSpan.Zero } };
DomainsExpires = new Dictionary<string, TimeSpan> { { string.Empty, TimeSpan.Zero } };
_domainsAcl = new Dictionary<string, S3CannedACL>();
_moduleAcl = S3CannedACL.PublicRead;
@ -1255,8 +1255,8 @@ public class S3Storage : BaseStorage
else//Key combined from module+domain+filename
{
result = string.Format("{0}/{1}/{2}/{3}",
_tenant,
_modulename,
Tenant,
Modulename,
domain,
path);
}