IlyaSobolev/DocSpace-buildtools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #524 from ONLYOFFICE/feature/asc-storage-refactor

Browse Source 
Feature/asc storage refactor
This commit is contained in:
Alexey Bannov 2022-02-17 15:46:28 +03:00 committed by GitHub
commit 77cb2b5ccc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 6836 additions and 6621 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

668
common/ASC.Data.Storage/BaseStorage.cs
View File

@ -23,367 +23,371 @@
*
*/
namespace ASC.Data.Storage
namespace ASC.Data.Storage;
public abstract class BaseStorage : IDataStore
{
public abstract class BaseStorage : IDataStore
public IQuotaController QuotaController { get; set; }
public virtual bool IsSupportInternalUri => true;
public virtual bool IsSupportedPreSignedUri => true;
public virtual bool IsSupportChunking => false;
internal string Modulename { get; set; }
internal DataList DataList { get; set; }
internal string Tenant { get; set; }
internal Dictionary<string, TimeSpan> DomainsExpires { get; set; }
= new Dictionary<string, TimeSpan>();
protected ILog Logger { get; set; }
protected readonly TempStream TempStream;
protected readonly TenantManager TenantManager;
protected readonly PathUtils TpathUtils;
protected readonly EmailValidationKeyProvider TemailValidationKeyProvider;
protected readonly IHttpContextAccessor HttpContextAccessor;
protected readonly IOptionsMonitor<ILog> Options;
protected readonly IHttpClientFactory ClientFactory;
public BaseStorage(
TempStream tempStream,
TenantManager tenantManager,
PathUtils pathUtils,
EmailValidationKeyProvider emailValidationKeyProvider,
IHttpContextAccessor httpContextAccessor,
IOptionsMonitor<ILog> options,
IHttpClientFactory clientFactory)
{
protected ILog Log { get; set; }
protected TempStream TempStream { get; }
protected TenantManager TenantManager { get; }
protected PathUtils PathUtils { get; }
protected EmailValidationKeyProvider EmailValidationKeyProvider { get; }
protected IHttpContextAccessor HttpContextAccessor { get; }
protected IOptionsMonitor<ILog> Options { get; }
protected IHttpClientFactory ClientFactory { get; }
protected BaseStorage(
TempStream tempStream,
TenantManager tenantManager,
PathUtils pathUtils,
EmailValidationKeyProvider emailValidationKeyProvider,
IHttpContextAccessor httpContextAccessor,
IOptionsMonitor<ILog> options,
IHttpClientFactory clientFactory)
TempStream = tempStream;
TenantManager = tenantManager;
TpathUtils = pathUtils;
TemailValidationKeyProvider = emailValidationKeyProvider;
Options = options;
ClientFactory = clientFactory;
Logger = options.CurrentValue;
HttpContextAccessor = httpContextAccessor;
}
public TimeSpan GetExpire(string domain)
{
return DomainsExpires.ContainsKey(domain) ? DomainsExpires[domain] : DomainsExpires[string.Empty];
}
public Uri GetUri(string path)
{
return GetUri(string.Empty, path);
}
public Uri GetUri(string domain, string path)
{
return GetPreSignedUri(domain, path, TimeSpan.MaxValue, null);
}
public Uri GetPreSignedUri(string domain, string path, TimeSpan expire, IEnumerable<string> headers)
{
if (path == null)
{
TempStream = tempStream;
TenantManager = tenantManager;
PathUtils = pathUtils;
EmailValidationKeyProvider = emailValidationKeyProvider;
Options = options;
Log = options.CurrentValue;
HttpContextAccessor = httpContextAccessor;
ClientFactory = clientFactory;
throw new ArgumentNullException(nameof(path));
}
#region IDataStore Members
internal string _modulename;
internal DataList _dataList;
internal string _tenant;
internal Dictionary<string, TimeSpan> _domainsExpires = new Dictionary<string, TimeSpan>();
public IQuotaController QuotaController { get; set; }
public TimeSpan GetExpire(string domain)
if (string.IsNullOrEmpty(Tenant) && IsSupportInternalUri)
{
return _domainsExpires.ContainsKey(domain) ? _domainsExpires[domain] : _domainsExpires[string.Empty];
return GetInternalUri(domain, path, expire, headers);
}
public Uri GetUri(string path)
var headerAttr = string.Empty;
if (headers != null)
{
return GetUri(string.Empty, path);
headerAttr = string.Join("&", headers.Select(HttpUtility.UrlEncode));
}
public Uri GetUri(string domain, string path)
if (expire == TimeSpan.Zero || expire == TimeSpan.MinValue || expire == TimeSpan.MaxValue)
{
return GetPreSignedUri(domain, path, TimeSpan.MaxValue, null);
expire = GetExpire(domain);
}
public Uri GetPreSignedUri(string domain, string path, TimeSpan expire, IEnumerable<string> headers)
var query = string.Empty;
if (expire != TimeSpan.Zero && expire != TimeSpan.MinValue && expire != TimeSpan.MaxValue)
{
if (path == null)
var expireString = expire.TotalMinutes.ToString(CultureInfo.InvariantCulture);
int currentTenantId;
var currentTenant = TenantManager.GetCurrentTenant(false);
if (currentTenant != null)
{
throw new ArgumentNullException(nameof(path));
currentTenantId = currentTenant.TenantId;
}
else if (!TenantPath.TryGetTenant(Tenant, out currentTenantId))
{
currentTenantId = 0;
}
if (string.IsNullOrEmpty(_tenant) && IsSupportInternalUri)
{
return GetInternalUri(domain, path, expire, headers);
}
var headerAttr = string.Empty;
if (headers != null)
{
headerAttr = string.Join("&", headers.Select(HttpUtility.UrlEncode));
}
if (expire == TimeSpan.Zero || expire == TimeSpan.MinValue || expire == TimeSpan.MaxValue)
{
expire = GetExpire(domain);
}
var query = string.Empty;
if (expire != TimeSpan.Zero && expire != TimeSpan.MinValue && expire != TimeSpan.MaxValue)
{
var expireString = expire.TotalMinutes.ToString(CultureInfo.InvariantCulture);
int currentTenantId;
var currentTenant = TenantManager.GetCurrentTenant(false);
if (currentTenant != null)
{
currentTenantId = currentTenant.TenantId;
}
else if (!TenantPath.TryGetTenant(_tenant, out currentTenantId))
{
currentTenantId = 0;
}
var auth = EmailValidationKeyProvider.GetEmailKey(currentTenantId, path.Replace('/', Path.DirectorySeparatorChar).Replace('\\', Path.DirectorySeparatorChar) + "." + headerAttr + "." + expireString);
query = $"{(path.IndexOf('?') >= 0 ? "&" : "?")}{Constants.QUERY_EXPIRE}={expireString}&{Constants.QUERY_AUTH}={auth}";
}
if (!string.IsNullOrEmpty(headerAttr))
{
query += $"{(query.IndexOf('?') >= 0 ? "&" : "?")}{Constants.QUERY_HEADER}={HttpUtility.UrlEncode(headerAttr)}";
}
var tenant = _tenant.Trim('/');
var vpath = PathUtils.ResolveVirtualPath(_modulename, domain);
vpath = PathUtils.ResolveVirtualPath(vpath, false);
vpath = string.Format(vpath, tenant);
var virtualPath = new Uri(vpath + "/", UriKind.RelativeOrAbsolute);
var uri = virtualPath.IsAbsoluteUri ?
new MonoUri(virtualPath, virtualPath.LocalPath.TrimEnd('/') + EnsureLeadingSlash(path.Replace('\\', '/')) + query) :
new MonoUri(virtualPath.ToString().TrimEnd('/') + EnsureLeadingSlash(path.Replace('\\', '/')) + query, UriKind.Relative);
return uri;
var auth = TemailValidationKeyProvider.GetEmailKey(currentTenantId, path.Replace('/', Path.DirectorySeparatorChar).Replace('\\', Path.DirectorySeparatorChar) + "." + headerAttr + "." + expireString);
query = $"{(path.IndexOf('?') >= 0 ? "&" : "?")}{Constants.QueryExpire}={expireString}&{Constants.QueryAuth}={auth}";
}
public virtual bool IsSupportInternalUri
if (!string.IsNullOrEmpty(headerAttr))
{
get { return true; }
query += $"{(query.IndexOf('?') >= 0 ? "&" : "?")}{Constants.QueryHeader}={HttpUtility.UrlEncode(headerAttr)}";
}
public virtual Uri GetInternalUri(string domain, string path, TimeSpan expire, IEnumerable<string> headers)
var tenant = Tenant.Trim('/');
var vpath = TpathUtils.ResolveVirtualPath(Modulename, domain);
vpath = TpathUtils.ResolveVirtualPath(vpath, false);
vpath = string.Format(vpath, tenant);
var virtualPath = new Uri(vpath + "/", UriKind.RelativeOrAbsolute);
var uri = virtualPath.IsAbsoluteUri ?
new MonoUri(virtualPath, virtualPath.LocalPath.TrimEnd('/') + EnsureLeadingSlash(path.Replace('\\', '/')) + query) :
new MonoUri(virtualPath.ToString().TrimEnd('/') + EnsureLeadingSlash(path.Replace('\\', '/')) + query, UriKind.Relative);
return uri;
}
public virtual Uri GetInternalUri(string domain, string path, TimeSpan expire, IEnumerable<string> headers)
{
return null;
}
public abstract Stream GetReadStream(string domain, string path);
public abstract Stream GetReadStream(string domain, string path, int offset);
public abstract Task<Stream> GetReadStreamAsync(string domain, string path, int offset);
public abstract Uri Save(string domain, string path, Stream stream);
public abstract Uri Save(string domain, string path, Stream stream, ACL acl);
public Uri Save(string domain, string path, Stream stream, string attachmentFileName)
{
if (!string.IsNullOrEmpty(attachmentFileName))
{
return null;
return SaveWithAutoAttachment(domain, path, stream, attachmentFileName);
}
public abstract Stream GetReadStream(string domain, string path);
public abstract Stream GetReadStream(string domain, string path, int offset);
public abstract Task<Stream> GetReadStreamAsync(string domain, string path, int offset);
return Save(domain, path, stream);
}
public abstract Uri Save(string domain, string path, Stream stream);
public abstract Uri Save(string domain, string path, Stream stream, ACL acl);
protected abstract Uri SaveWithAutoAttachment(string domain, string path, Stream stream, string attachmentFileName);
public Uri Save(string domain, string path, Stream stream, string attachmentFileName)
public abstract Uri Save(string domain, string path, Stream stream, string contentType,
string contentDisposition);
public abstract Uri Save(string domain, string path, Stream stream, string contentEncoding, int cacheDays);
#region chunking
public virtual string InitiateChunkedUpload(string domain, string path)
{
throw new NotImplementedException();
}
public virtual string UploadChunk(string domain, string path, string uploadId, Stream stream, long defaultChunkSize, int chunkNumber, long chunkLength)
{
throw new NotImplementedException();
}
public virtual Uri FinalizeChunkedUpload(string domain, string path, string uploadId, Dictionary<int, string> eTags)
{
throw new NotImplementedException();
}
public virtual void AbortChunkedUpload(string domain, string path, string uploadId)
{
throw new NotImplementedException();
}
#endregion
public abstract void Delete(string domain, string path);
public abstract void DeleteFiles(string domain, string folderPath, string pattern, bool recursive);
public abstract void DeleteFiles(string domain, List<string> paths);
public abstract void DeleteFiles(string domain, string folderPath, DateTime fromDate, DateTime toDate);
public abstract void MoveDirectory(string srcdomain, string srcdir, string newdomain, string newdir);
public abstract Uri Move(string srcdomain, string srcpath, string newdomain, string newpath, bool quotaCheckFileSize = true);
public abstract Uri SaveTemp(string domain, out string assignedPath, Stream stream);
public abstract string[] ListDirectoriesRelative(string domain, string path, bool recursive);
public abstract string[] ListFilesRelative(string domain, string path, string pattern, bool recursive);
public abstract bool IsFile(string domain, string path);
public abstract Task<bool> IsFileAsync(string domain, string path);
public abstract bool IsDirectory(string domain, string path);
public abstract void DeleteDirectory(string domain, string path);
public abstract long GetFileSize(string domain, string path);
public abstract long GetDirectorySize(string domain, string path);
public abstract long ResetQuota(string domain);
public abstract long GetUsedQuota(string domain);
public abstract Uri Copy(string srcdomain, string path, string newdomain, string newpath);
public abstract void CopyDirectory(string srcdomain, string dir, string newdomain, string newdir);
public Stream GetReadStream(string path)
{
return GetReadStream(string.Empty, path);
}
public Uri Save(string path, Stream stream, string attachmentFileName)
{
return Save(string.Empty, path, stream, attachmentFileName);
}
public Uri Save(string path, Stream stream)
{
return Save(string.Empty, path, stream);
}
public void Delete(string path)
{
Delete(string.Empty, path);
}
public void DeleteFiles(string folderPath, string pattern, bool recursive)
{
DeleteFiles(string.Empty, folderPath, pattern, recursive);
}
public Uri Move(string srcpath, string newdomain, string newpath)
{
return Move(string.Empty, srcpath, newdomain, newpath);
}
public Uri SaveTemp(out string assignedPath, Stream stream)
{
return SaveTemp(string.Empty, out assignedPath, stream);
}
public string[] ListDirectoriesRelative(string path, bool recursive)
{
return ListDirectoriesRelative(string.Empty, path, recursive);
}
public Uri[] ListFiles(string path, string pattern, bool recursive)
{
return ListFiles(string.Empty, path, pattern, recursive);
}
public Uri[] ListFiles(string domain, string path, string pattern, bool recursive)
{
var filePaths = ListFilesRelative(domain, path, pattern, recursive);
return Array.ConvertAll(
filePaths,
x => GetUri(domain, CrossPlatform.PathCombine(PathUtils.Normalize(path), x)));
}
public bool IsFile(string path)
{
return IsFile(string.Empty, path);
}
public bool IsDirectory(string path)
{
return IsDirectory(string.Empty, path);
}
public void DeleteDirectory(string path)
{
DeleteDirectory(string.Empty, path);
}
public long GetFileSize(string path)
{
return GetFileSize(string.Empty, path);
}
public long GetDirectorySize(string path)
{
return GetDirectorySize(string.Empty, path);
}
public Uri Copy(string path, string newdomain, string newpath)
{
return Copy(string.Empty, path, newdomain, newpath);
}
public void CopyDirectory(string dir, string newdomain, string newdir)
{
CopyDirectory(string.Empty, dir, newdomain, newdir);
}
public virtual IDataStore Configure(string tenant, Handler handlerConfig, Module moduleConfig, IDictionary<string, string> props)
{
return this;
}
public IDataStore SetQuotaController(IQuotaController controller)
{
QuotaController = controller;
return this;
}
public abstract string SavePrivate(string domain, string path, Stream stream, DateTime expires);
public abstract void DeleteExpired(string domain, string path, TimeSpan oldThreshold);
public abstract string GetUploadForm(string domain, string directoryPath, string redirectTo, long maxUploadSize,
string contentType, string contentDisposition, string submitLabel);
public abstract string GetUploadedUrl(string domain, string directoryPath);
public abstract string GetUploadUrl();
public abstract string GetPostParams(string domain, string directoryPath, long maxUploadSize, string contentType,
string contentDisposition);
internal void QuotaUsedAdd(string domain, long size, bool quotaCheckFileSize = true)
{
if (QuotaController != null)
{
if (!string.IsNullOrEmpty(attachmentFileName))
{
return SaveWithAutoAttachment(domain, path, stream, attachmentFileName);
}
return Save(domain, path, stream);
}
protected abstract Uri SaveWithAutoAttachment(string domain, string path, Stream stream, string attachmentFileName);
public abstract Uri Save(string domain, string path, Stream stream, string contentType,
string contentDisposition);
public abstract Uri Save(string domain, string path, Stream stream, string contentEncoding, int cacheDays);
public virtual bool IsSupportedPreSignedUri
{
get
{
return true;
}
}
#region chunking
public virtual string InitiateChunkedUpload(string domain, string path)
{
throw new NotImplementedException();
}
public virtual string UploadChunk(string domain, string path, string uploadId, Stream stream, long defaultChunkSize, int chunkNumber, long chunkLength)
{
throw new NotImplementedException();
}
public virtual Uri FinalizeChunkedUpload(string domain, string path, string uploadId, Dictionary<int, string> eTags)
{
throw new NotImplementedException();
}
public virtual void AbortChunkedUpload(string domain, string path, string uploadId)
{
throw new NotImplementedException();
}
public virtual bool IsSupportChunking { get { return false; } }
#endregion
public abstract void Delete(string domain, string path);
public abstract void DeleteFiles(string domain, string folderPath, string pattern, bool recursive);
public abstract void DeleteFiles(string domain, List<string> paths);
public abstract void DeleteFiles(string domain, string folderPath, DateTime fromDate, DateTime toDate);
public abstract void MoveDirectory(string srcdomain, string srcdir, string newdomain, string newdir);
public abstract Uri Move(string srcdomain, string srcpath, string newdomain, string newpath, bool quotaCheckFileSize = true);
public abstract Uri SaveTemp(string domain, out string assignedPath, Stream stream);
public abstract string[] ListDirectoriesRelative(string domain, string path, bool recursive);
public abstract string[] ListFilesRelative(string domain, string path, string pattern, bool recursive);
public abstract bool IsFile(string domain, string path);
public abstract Task<bool> IsFileAsync(string domain, string path);
public abstract bool IsDirectory(string domain, string path);
public abstract void DeleteDirectory(string domain, string path);
public abstract long GetFileSize(string domain, string path);
public abstract long GetDirectorySize(string domain, string path);
public abstract long ResetQuota(string domain);
public abstract long GetUsedQuota(string domain);
public abstract Uri Copy(string srcdomain, string path, string newdomain, string newpath);
public abstract void CopyDirectory(string srcdomain, string dir, string newdomain, string newdir);
public Stream GetReadStream(string path)
{
return GetReadStream(string.Empty, path);
}
public Uri Save(string path, Stream stream, string attachmentFileName)
{
return Save(string.Empty, path, stream, attachmentFileName);
}
public Uri Save(string path, Stream stream)
{
return Save(string.Empty, path, stream);
}
public void Delete(string path)
{
Delete(string.Empty, path);
}
public void DeleteFiles(string folderPath, string pattern, bool recursive)
{
DeleteFiles(string.Empty, folderPath, pattern, recursive);
}
public Uri Move(string srcpath, string newdomain, string newpath)
{
return Move(string.Empty, srcpath, newdomain, newpath);
}
public Uri SaveTemp(out string assignedPath, Stream stream)
{
return SaveTemp(string.Empty, out assignedPath, stream);
}
public string[] ListDirectoriesRelative(string path, bool recursive)
{
return ListDirectoriesRelative(string.Empty, path, recursive);
}
public Uri[] ListFiles(string path, string pattern, bool recursive)
{
return ListFiles(string.Empty, path, pattern, recursive);
}
public Uri[] ListFiles(string domain, string path, string pattern, bool recursive)
{
var filePaths = ListFilesRelative(domain, path, pattern, recursive);
return Array.ConvertAll(
filePaths,
x => GetUri(domain, CrossPlatform.PathCombine(PathUtils.Normalize(path), x)));
}
public bool IsFile(string path)
{
return IsFile(string.Empty, path);
}
public bool IsDirectory(string path)
{
return IsDirectory(string.Empty, path);
}
public void DeleteDirectory(string path)
{
DeleteDirectory(string.Empty, path);
}
public long GetFileSize(string path)
{
return GetFileSize(string.Empty, path);
}
public long GetDirectorySize(string path)
{
return GetDirectorySize(string.Empty, path);
}
public Uri Copy(string path, string newdomain, string newpath)
{
return Copy(string.Empty, path, newdomain, newpath);
}
public void CopyDirectory(string dir, string newdomain, string newdir)
{
CopyDirectory(string.Empty, dir, newdomain, newdir);
}
public virtual IDataStore Configure(string tenant, Handler handlerConfig, Module moduleConfig, IDictionary<string, string> props)
{
return this;
}
public IDataStore SetQuotaController(IQuotaController controller)
{
QuotaController = controller;
return this;
}
public abstract string SavePrivate(string domain, string path, Stream stream, DateTime expires);
public abstract void DeleteExpired(string domain, string path, TimeSpan oldThreshold);
public abstract string GetUploadForm(string domain, string directoryPath, string redirectTo, long maxUploadSize,
string contentType, string contentDisposition, string submitLabel);
public abstract string GetUploadedUrl(string domain, string directoryPath);
public abstract string GetUploadUrl();
public abstract string GetPostParams(string domain, string directoryPath, long maxUploadSize, string contentType,
string contentDisposition);
#endregion
internal void QuotaUsedAdd(string domain, long size, bool quotaCheckFileSize = true)
{
if (QuotaController != null)
{
QuotaController.QuotaUsedAdd(_modulename, domain, _dataList.GetData(domain), size, quotaCheckFileSize);
}
}
internal void QuotaUsedDelete(string domain, long size)
{
if (QuotaController != null)
{
QuotaController.QuotaUsedDelete(_modulename, domain, _dataList.GetData(domain), size);
}
}
internal static string EnsureLeadingSlash(string str)
{
return "/" + str.TrimStart('/');
}
internal class MonoUri : Uri
{
public MonoUri(Uri baseUri, string relativeUri)
: base(baseUri, relativeUri)
{
}
public MonoUri(string uriString, UriKind uriKind)
: base(uriString, uriKind)
{
}
public override string ToString()
{
var s = base.ToString();
if (WorkContext.IsMono && s.StartsWith(UriSchemeFile + SchemeDelimiter))
{
return s.Substring(7);
}
return s;
}
QuotaController.QuotaUsedAdd(Modulename, domain, DataList.GetData(domain), size, quotaCheckFileSize);
}
}
}
internal void QuotaUsedDelete(string domain, long size)
{
if (QuotaController != null)
{
QuotaController.QuotaUsedDelete(Modulename, domain, DataList.GetData(domain), size);
}
}
internal static string EnsureLeadingSlash(string str)
{
return "/" + str.TrimStart('/');
}
internal class MonoUri : Uri
{
public MonoUri(Uri baseUri, string relativeUri)
: base(baseUri, relativeUri) { }
public MonoUri(string uriString, UriKind uriKind)
: base(uriString, uriKind) { }
public override string ToString()
{
var s = base.ToString();
if (WorkContext.IsMono && s.StartsWith(UriSchemeFile + SchemeDelimiter))
{
return s.Substring(7);
}
return s;
}
}
}

164
common/ASC.Data.Storage/ChunkedUploader/CommonChunkedUploadSession.cs
View File

@ -23,114 +23,102 @@
*
*/
namespace ASC.Core.ChunkedUploader
namespace ASC.Core.ChunkedUploader;
[Serializable]
public class CommonChunkedUploadSession : ICloneable
{
[Serializable]
public class CommonChunkedUploadSession : ICloneable
public string Id { get; set; }
public DateTime Created { get; set; }
public DateTime Expired { get; set; }
public string Location { get; set; }
public long BytesUploaded { get; set; }
public long BytesTotal { get; set; }
public int TenantId { get; set; }
public Guid UserId { get; set; }
public bool UseChunks { get; set; }
public string CultureName { get; set; }
public Dictionary<string, object> Items { get; set; } = new Dictionary<string, object>();
[JsonIgnore]
public string TempPath
{
public string Id { get; set; }
get => GetItemOrDefault<string>(TempPathKey);
set => Items[TempPathKey] = value;
}
public DateTime Created { get; set; }
[JsonIgnore]
public string UploadId
{
get => GetItemOrDefault<string>(UploadIdKey);
set => Items[UploadIdKey] = value;
}
public DateTime Expired { get; set; }
[JsonIgnore]
public string ChunksBuffer
{
get => GetItemOrDefault<string>(ChunksBufferKey);
set => Items[ChunksBufferKey] = value;
}
public string Location { get; set; }
private const string TempPathKey = "TempPath";
private const string UploadIdKey = "UploadId";
private const string ChunksBufferKey = "ChunksBuffer";
public long BytesUploaded { get; set; }
public CommonChunkedUploadSession(long bytesTotal)
{
Id = Guid.NewGuid().ToString("N");
Created = DateTime.UtcNow;
BytesUploaded = 0;
BytesTotal = bytesTotal;
UseChunks = true;
}
public long BytesTotal { get; set; }
public T GetItemOrDefault<T>(string key)
{
return Items.ContainsKey(key) && Items[key] is T t ? t : default;
}
public int TenantId { get; set; }
public virtual Stream Serialize()
{
return null;
}
public Guid UserId { get; set; }
public void TransformItems()
{
var newItems = new Dictionary<string, object>();
public bool UseChunks { get; set; }
public string CultureName { get; set; }
public Dictionary<string, object> Items { get; set; } = new Dictionary<string, object>();
private const string TempPathKey = "TempPath";
[JsonIgnore]
public string TempPath
foreach (var item in Items)
{
get { return GetItemOrDefault<string>(TempPathKey); }
set { Items[TempPathKey] = value; }
}
private const string UploadIdKey = "UploadId";
[JsonIgnore]
public string UploadId
{
get { return GetItemOrDefault<string>(UploadIdKey); }
set { Items[UploadIdKey] = value; }
}
private const string ChunksBufferKey = "ChunksBuffer";
[JsonIgnore]
public string ChunksBuffer
{
get { return GetItemOrDefault<string>(ChunksBufferKey); }
set { Items[ChunksBufferKey] = value; }
}
public CommonChunkedUploadSession(long bytesTotal)
{
Id = Guid.NewGuid().ToString("N");
Created = DateTime.UtcNow;
BytesUploaded = 0;
BytesTotal = bytesTotal;
UseChunks = true;
}
public T GetItemOrDefault<T>(string key)
{
return Items.ContainsKey(key) && Items[key] is T t ? t : default;
}
public virtual Stream Serialize()
{
return null;
}
public void TransformItems()
{
var newItems = new Dictionary<string, object>();
foreach(var item in Items)
if (item.Value != null)
{
if (item.Value != null)
{
if (item.Value is JsonElement)
{
var value = (JsonElement)item.Value;
if (value.ValueKind == JsonValueKind.String)
{
var value = (JsonElement)item.Value;
if (value.ValueKind == JsonValueKind.String)
{
newItems.Add(item.Key, item.Value.ToString());
}
if (value.ValueKind == JsonValueKind.Number)
{
newItems.Add(item.Key, Int32.Parse(item.Value.ToString()));
}
if (value.ValueKind == JsonValueKind.Array)
{
newItems.Add(item.Key, value.EnumerateArray().Select(o => o.ToString()).ToList());
}
newItems.Add(item.Key, item.Value.ToString());
}
else
if (value.ValueKind == JsonValueKind.Number)
{
newItems.Add(item.Key, item.Value);
newItems.Add(item.Key, Int32.Parse(item.Value.ToString()));
}
if (value.ValueKind == JsonValueKind.Array)
{
newItems.Add(item.Key, value.EnumerateArray().Select(o => o.ToString()).ToList());
}
}
else
{
newItems.Add(item.Key, item.Value);
}
}
Items = newItems;
}
Items = newItems;
}
public virtual object Clone()
{
return (CommonChunkedUploadSession)MemberwiseClone();
}
public virtual object Clone()
{
return (CommonChunkedUploadSession)MemberwiseClone();
}
}

299
common/ASC.Data.Storage/ChunkedUploader/CommonChunkedUploadSessionHolder.cs
View File

@ -23,159 +23,158 @@
*
*/
namespace ASC.Core.ChunkedUploader
{
public class CommonChunkedUploadSessionHolder
{
public static readonly TimeSpan SlidingExpiration = TimeSpan.FromHours(12);
namespace ASC.Core.ChunkedUploader;
private TempPath TempPath { get; }
private IOptionsMonitor<ILog> Option { get; }
public IDataStore DataStore { get; set; }
private string Domain { get; set; }
private long MaxChunkUploadSize { get; set; }
private const string StoragePath = "sessions";
public CommonChunkedUploadSessionHolder(
TempPath tempPath,
IOptionsMonitor<ILog> option,
IDataStore dataStore,
string domain,
long maxChunkUploadSize = 10 * 1024 * 1024)
public class CommonChunkedUploadSessionHolder
{
public IDataStore DataStore { get; set; }
public static readonly TimeSpan SlidingExpiration = TimeSpan.FromHours(12);
private readonly TempPath _tempPath;
private readonly IOptionsMonitor<ILog> _option;
private readonly string _domain;
private readonly long _maxChunkUploadSize;
private const string StoragePath = "sessions";
public CommonChunkedUploadSessionHolder(
TempPath tempPath,
IOptionsMonitor<ILog> option,
IDataStore dataStore,
string domain,
long maxChunkUploadSize = 10 * 1024 * 1024)
{
_tempPath = tempPath;
_option = option;
DataStore = dataStore;
_domain = domain;
_maxChunkUploadSize = maxChunkUploadSize;
}
public void DeleteExpired()
{
// clear old sessions
try
{
TempPath = tempPath;
Option = option;
DataStore = dataStore;
Domain = domain;
MaxChunkUploadSize = maxChunkUploadSize;
}
public void DeleteExpired()
{
// clear old sessions
try
{
DataStore.DeleteExpired(Domain, StoragePath, SlidingExpiration);
}
catch (Exception err)
{
Option.CurrentValue.Error(err);
}
}
public void Store(CommonChunkedUploadSession s)
{
using var stream = s.Serialize();
DataStore.SavePrivate(Domain, GetPathWithId(s.Id), stream, s.Expired);
}
public void Remove(CommonChunkedUploadSession s)
{
DataStore.Delete(Domain, GetPathWithId(s.Id));
}
public Stream GetStream(string sessionId)
DataStore.DeleteExpired(_domain, StoragePath, SlidingExpiration);
}
catch (Exception err)
{
return DataStore.GetReadStream(Domain, GetPathWithId(sessionId));
_option.CurrentValue.Error(err);
}
public void Init(CommonChunkedUploadSession chunkedUploadSession)
{
if (chunkedUploadSession.BytesTotal < MaxChunkUploadSize)
{
chunkedUploadSession.UseChunks = false;
return;
}
var tempPath = Guid.NewGuid().ToString();
var uploadId = DataStore.InitiateChunkedUpload(Domain, tempPath);
chunkedUploadSession.TempPath = tempPath;
chunkedUploadSession.UploadId = uploadId;
}
public void Finalize(CommonChunkedUploadSession uploadSession)
{
var tempPath = uploadSession.TempPath;
var uploadId = uploadSession.UploadId;
var eTags = uploadSession.GetItemOrDefault<List<string>>("ETag")
.Select((x, i) => new KeyValuePair<int, string>(i + 1, x))
.ToDictionary(x => x.Key, x => x.Value);
DataStore.FinalizeChunkedUpload(Domain, tempPath, uploadId, eTags);
}
}
public void Move(CommonChunkedUploadSession chunkedUploadSession, string newPath, bool quotaCheckFileSize = true)
{
DataStore.Move(Domain, chunkedUploadSession.TempPath, string.Empty, newPath, quotaCheckFileSize);
public void Store(CommonChunkedUploadSession s)
{
using var stream = s.Serialize();
DataStore.SavePrivate(_domain, GetPathWithId(s.Id), stream, s.Expired);
}
public void Remove(CommonChunkedUploadSession s)
{
DataStore.Delete(_domain, GetPathWithId(s.Id));
}
public Stream GetStream(string sessionId)
{
return DataStore.GetReadStream(_domain, GetPathWithId(sessionId));
}
public void Init(CommonChunkedUploadSession chunkedUploadSession)
{
if (chunkedUploadSession.BytesTotal < _maxChunkUploadSize)
{
chunkedUploadSession.UseChunks = false;
return;
}
public void Abort(CommonChunkedUploadSession uploadSession)
{
if (uploadSession.UseChunks)
{
var tempPath = uploadSession.TempPath;
var uploadId = uploadSession.UploadId;
DataStore.AbortChunkedUpload(Domain, tempPath, uploadId);
}
else if (!string.IsNullOrEmpty(uploadSession.ChunksBuffer))
{
File.Delete(uploadSession.ChunksBuffer);
}
}
public void UploadChunk(CommonChunkedUploadSession uploadSession, Stream stream, long length)
{
var tempPath = uploadSession.TempPath;
var uploadId = uploadSession.UploadId;
var chunkNumber = uploadSession.GetItemOrDefault<int>("ChunksUploaded") + 1;
var eTag = DataStore.UploadChunk(Domain, tempPath, uploadId, stream, MaxChunkUploadSize, chunkNumber, length);
uploadSession.Items["ChunksUploaded"] = chunkNumber;
uploadSession.BytesUploaded += length;
var eTags = uploadSession.GetItemOrDefault<List<string>>("ETag") ?? new List<string>();
eTags.Add(eTag);
uploadSession.Items["ETag"] = eTags;
}
public Stream UploadSingleChunk(CommonChunkedUploadSession uploadSession, Stream stream, long chunkLength)
{
if (uploadSession.BytesTotal == 0)
uploadSession.BytesTotal = chunkLength;
if (uploadSession.BytesTotal >= chunkLength)
{
//This is hack fixing strange behaviour of plupload in flash mode.
if (string.IsNullOrEmpty(uploadSession.ChunksBuffer))
{
uploadSession.ChunksBuffer = TempPath.GetTempFileName();
}
using (var bufferStream = new FileStream(uploadSession.ChunksBuffer, FileMode.Append))
{
stream.CopyTo(bufferStream);
}
uploadSession.BytesUploaded += chunkLength;
if (uploadSession.BytesTotal == uploadSession.BytesUploaded)
{
return new FileStream(uploadSession.ChunksBuffer, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite,
4096, FileOptions.DeleteOnClose);
}
}
return Stream.Null;
}
private string GetPathWithId(string id)
{
return CrossPlatform.PathCombine(StoragePath, id + ".session");
}
}
}
var tempPath = Guid.NewGuid().ToString();
var uploadId = DataStore.InitiateChunkedUpload(_domain, tempPath);
chunkedUploadSession.TempPath = tempPath;
chunkedUploadSession.UploadId = uploadId;
}
public void Finalize(CommonChunkedUploadSession uploadSession)
{
var tempPath = uploadSession.TempPath;
var uploadId = uploadSession.UploadId;
var eTags = uploadSession.GetItemOrDefault<List<string>>("ETag")
.Select((x, i) => new KeyValuePair<int, string>(i + 1, x))
.ToDictionary(x => x.Key, x => x.Value);
DataStore.FinalizeChunkedUpload(_domain, tempPath, uploadId, eTags);
}
public void Move(CommonChunkedUploadSession chunkedUploadSession, string newPath, bool quotaCheckFileSize = true)
{
DataStore.Move(_domain, chunkedUploadSession.TempPath, string.Empty, newPath, quotaCheckFileSize);
}
public void Abort(CommonChunkedUploadSession uploadSession)
{
if (uploadSession.UseChunks)
{
var tempPath = uploadSession.TempPath;
var uploadId = uploadSession.UploadId;
DataStore.AbortChunkedUpload(_domain, tempPath, uploadId);
}
else if (!string.IsNullOrEmpty(uploadSession.ChunksBuffer))
{
File.Delete(uploadSession.ChunksBuffer);
}
}
public void UploadChunk(CommonChunkedUploadSession uploadSession, Stream stream, long length)
{
var tempPath = uploadSession.TempPath;
var uploadId = uploadSession.UploadId;
var chunkNumber = uploadSession.GetItemOrDefault<int>("ChunksUploaded") + 1;
var eTag = DataStore.UploadChunk(_domain, tempPath, uploadId, stream, _maxChunkUploadSize, chunkNumber, length);
uploadSession.Items["ChunksUploaded"] = chunkNumber;
uploadSession.BytesUploaded += length;
var eTags = uploadSession.GetItemOrDefault<List<string>>("ETag") ?? new List<string>();
eTags.Add(eTag);
uploadSession.Items["ETag"] = eTags;
}
public Stream UploadSingleChunk(CommonChunkedUploadSession uploadSession, Stream stream, long chunkLength)
{
if (uploadSession.BytesTotal == 0)
uploadSession.BytesTotal = chunkLength;
if (uploadSession.BytesTotal >= chunkLength)
{
//This is hack fixing strange behaviour of plupload in flash mode.
if (string.IsNullOrEmpty(uploadSession.ChunksBuffer))
{
uploadSession.ChunksBuffer = _tempPath.GetTempFileName();
}
using (var bufferStream = new FileStream(uploadSession.ChunksBuffer, FileMode.Append))
{
stream.CopyTo(bufferStream);
}
uploadSession.BytesUploaded += chunkLength;
if (uploadSession.BytesTotal == uploadSession.BytesUploaded)
{
return new FileStream(uploadSession.ChunksBuffer, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite,
4096, FileOptions.DeleteOnClose);
}
}
return Stream.Null;
}
private string GetPathWithId(string id)
{
return CrossPlatform.PathCombine(StoragePath, id + ".session");
}
}

15
common/ASC.Data.Storage/Configuration/ACL.cs
View File

@ -24,12 +24,11 @@
*/
namespace ASC.Data.Storage.Configuration
namespace ASC.Data.Storage.Configuration;
public enum ACL
{
public enum ACL
{
Auto,
Read,
Private
}
}
Auto,
Read,
Private
}

135
common/ASC.Data.Storage/Configuration/Appender.cs
View File

@ -1,74 +1,71 @@
namespace ASC.Data.Storage.Configuration
namespace ASC.Data.Storage.Configuration;
public static class StorageConfigExtension
{
public static class StorageConfigExtension
public static void Register(DIHelper services)
{
public static void Register(DIHelper services)
{
services.TryAddSingleton(r => r.GetService<ConfigurationExtension>().GetSetting<Storage>("Storage"));
}
}
public class Storage
{
public IEnumerable<Appender> Appender { get; set; }
public IEnumerable<Handler> Handler { get; set; }
public IEnumerable<Module> Module { get; set; }
public Module GetModuleElement(string name)
{
return Module?.FirstOrDefault(r => r.Name == name);
}
public Handler GetHandler(string name)
{
return Handler?.FirstOrDefault(r => r.Name == name);
}
}
public class Appender
{
public string Name { get; set; }
public string Append { get; set; }
public string AppendSecure { get; set; }
public string Extensions { get; set; }
}
public class Handler
{
public string Name { get; set; }
public string Type { get; set; }
public IEnumerable<Properties> Property { get; set; }
public IDictionary<string, string> GetProperties()
{
if (Property == null || !Property.Any()) return new Dictionary<string, string>();
return Property.ToDictionary(r => r.Name, r => r.Value);
}
}
public class Properties
{
public string Name { get; set; }
public string Value { get; set; }
}
public class Module
{
public string Name { get; set; }
public string Data { get; set; }
public string Type { get; set; }
public string Path { get; set; }
public ACL Acl { get; set; } = ACL.Read;
public string VirtualPath { get; set; }
public TimeSpan Expires { get; set; }
public bool Visible { get; set; } = true;
public bool AppendTenantId { get; set; } = true;
public bool Public { get; set; }
public bool DisableMigrate { get; set; }
public bool Count { get; set; } = true;
public bool DisabledEncryption { get; set; }
public IEnumerable<Module> Domain { get; set; }
services.TryAddSingleton(r => r.GetService<ConfigurationExtension>().GetSetting<Storage>("Storage"));
}
}
public class Storage
{
public IEnumerable<Appender> Appender { get; set; }
public IEnumerable<Handler> Handler { get; set; }
public IEnumerable<Module> Module { get; set; }
public Module GetModuleElement(string name)
{
return Module?.FirstOrDefault(r => r.Name == name);
}
public Handler GetHandler(string name)
{
return Handler?.FirstOrDefault(r => r.Name == name);
}
}
public class Appender
{
public string Name { get; set; }
public string Append { get; set; }
public string AppendSecure { get; set; }
public string Extensions { get; set; }
}
public class Handler
{
public string Name { get; set; }
public string Type { get; set; }
public IEnumerable<Properties> Property { get; set; }
public IDictionary<string, string> GetProperties()
{
return Property == null || !Property.Any() ? new Dictionary<string, string>()
: Property.ToDictionary(r => r.Name, r => r.Value);
}
}
public class Properties
{
public string Name { get; set; }
public string Value { get; set; }
}
public class Module
{
public string Name { get; set; }
public string Data { get; set; }
public string Type { get; set; }
public string Path { get; set; }
public ACL Acl { get; set; } = ACL.Read;
public string VirtualPath { get; set; }
public TimeSpan Expires { get; set; }
public bool Visible { get; set; } = true;
public bool AppendTenantId { get; set; } = true;
public bool Public { get; set; }
public bool DisableMigrate { get; set; }
public bool Count { get; set; } = true;
public bool DisabledEncryption { get; set; }
public IEnumerable<Module> Domain { get; set; }
}

415
common/ASC.Data.Storage/Configuration/StorageSettings.cs
View File

@ -23,220 +23,229 @@
*
*/
namespace ASC.Data.Storage.Configuration
namespace ASC.Data.Storage.Configuration;
[Singletone(Additional = typeof(StorageSettingsExtension))]
public class BaseStorageSettingsListener
{
[Singletone(Additional = typeof(StorageSettingsExtension))]
public class BaseStorageSettingsListener
{
private IServiceProvider ServiceProvider { get; }
private volatile bool Subscribed;
private readonly object locker;
private readonly IServiceProvider _serviceProvider;
private readonly object _locker;
private volatile bool _subscribed;
public BaseStorageSettingsListener(IServiceProvider serviceProvider)
public BaseStorageSettingsListener(IServiceProvider serviceProvider)
{
_serviceProvider = serviceProvider;
_locker = new object();
}
public void Subscribe()
{
if (_subscribed)
{
ServiceProvider = serviceProvider;
locker = new object();
return;
}
public void Subscribe()
lock (_locker)
{
if (Subscribed) return;
lock (locker)
if (_subscribed)
{
if (Subscribed) return;
return;
}
Subscribed = true;
_subscribed = true;
ServiceProvider.GetService<ICacheNotify<ConsumerCacheItem>>().Subscribe((i) =>
_serviceProvider.GetService<ICacheNotify<ConsumerCacheItem>>().Subscribe((i) =>
{
using var scope = _serviceProvider.CreateScope();
var scopeClass = scope.ServiceProvider.GetService<BaseStorageSettingsListenerScope>();
var (storageSettingsHelper, settingsManager, cdnStorageSettings) = scopeClass;
var settings = settingsManager.LoadForTenant<StorageSettings>(i.TenantId);
if (i.Name == settings.Module)
{
using var scope = ServiceProvider.CreateScope();
storageSettingsHelper.Clear(settings);
}
var scopeClass = scope.ServiceProvider.GetService<BaseStorageSettingsListenerScope>();
var (storageSettingsHelper, settingsManager, cdnStorageSettings) = scopeClass;
var settings = settingsManager.LoadForTenant<StorageSettings>(i.TenantId);
if (i.Name == settings.Module)
{
storageSettingsHelper.Clear(settings);
}
var cdnSettings = settingsManager.LoadForTenant<CdnStorageSettings>(i.TenantId);
if (i.Name == cdnSettings.Module)
{
storageSettingsHelper.Clear(cdnSettings);
}
}, Common.Caching.CacheNotifyAction.Remove);
}
}
}
[Serializable]
public abstract class BaseStorageSettings<T> : ISettings where T : class, ISettings, new()
{
public string Module { get; set; }
public Dictionary<string, string> Props { get; set; }
public ISettings GetDefault(IServiceProvider serviceProvider)
{
return new T();
}
public virtual Func<DataStoreConsumer, DataStoreConsumer> Switch { get { return d => d; } }
internal ICacheNotify<DataStoreCacheItem> Cache { get; set; }
public abstract Guid ID { get; }
}
[Serializable]
public class StorageSettings : BaseStorageSettings<StorageSettings>
{
public override Guid ID
{
get { return new Guid("F13EAF2D-FA53-44F1-A6D6-A5AEDA46FA2B"); }
}
}
[Scope]
[Serializable]
public class CdnStorageSettings : BaseStorageSettings<CdnStorageSettings>
{
public override Guid ID
{
get { return new Guid("0E9AE034-F398-42FE-B5EE-F86D954E9FB2"); }
}
public override Func<DataStoreConsumer, DataStoreConsumer> Switch { get { return d => d.Cdn; } }
}
[Scope]
public class StorageSettingsHelper
{
private StorageFactoryConfig StorageFactoryConfig { get; }
private PathUtils PathUtils { get; }
private ICacheNotify<DataStoreCacheItem> Cache { get; }
private IOptionsMonitor<ILog> Options { get; }
private TenantManager TenantManager { get; }
private SettingsManager SettingsManager { get; }
private IHttpContextAccessor HttpContextAccessor { get; }
private ConsumerFactory ConsumerFactory { get; }
public StorageSettingsHelper(
BaseStorageSettingsListener baseStorageSettingsListener,
StorageFactoryConfig storageFactoryConfig,
PathUtils pathUtils,
ICacheNotify<DataStoreCacheItem> cache,
IOptionsMonitor<ILog> options,
TenantManager tenantManager,
SettingsManager settingsManager,
ConsumerFactory consumerFactory)
{
baseStorageSettingsListener.Subscribe();
StorageFactoryConfig = storageFactoryConfig;
PathUtils = pathUtils;
Cache = cache;
Options = options;
TenantManager = tenantManager;
SettingsManager = settingsManager;
ConsumerFactory = consumerFactory;
}
public StorageSettingsHelper(
BaseStorageSettingsListener baseStorageSettingsListener,
StorageFactoryConfig storageFactoryConfig,
PathUtils pathUtils,
ICacheNotify<DataStoreCacheItem> cache,
IOptionsMonitor<ILog> options,
TenantManager tenantManager,
SettingsManager settingsManager,
IHttpContextAccessor httpContextAccessor,
ConsumerFactory consumerFactory)
: this(baseStorageSettingsListener, storageFactoryConfig, pathUtils, cache, options, tenantManager, settingsManager, consumerFactory)
{
HttpContextAccessor = httpContextAccessor;
}
public bool Save<T>(BaseStorageSettings<T> baseStorageSettings) where T : class, ISettings, new()
{
ClearDataStoreCache();
return SettingsManager.Save(baseStorageSettings);
}
internal void ClearDataStoreCache()
{
var tenantId = TenantManager.GetCurrentTenant().TenantId.ToString();
var path = TenantPath.CreatePath(tenantId);
foreach (var module in StorageFactoryConfig.GetModuleList("", true))
{
Cache.Publish(new DataStoreCacheItem() { TenantId = path, Module = module }, Common.Caching.CacheNotifyAction.Remove);
}
}
public void Clear<T>(BaseStorageSettings<T> baseStorageSettings) where T : class, ISettings, new()
{
baseStorageSettings.Module = null;
baseStorageSettings.Props = null;
Save(baseStorageSettings);
}
public DataStoreConsumer DataStoreConsumer<T>(BaseStorageSettings<T> baseStorageSettings) where T : class, ISettings, new()
{
if (string.IsNullOrEmpty(baseStorageSettings.Module) || baseStorageSettings.Props == null) return new DataStoreConsumer();
var consumer = ConsumerFactory.GetByKey<DataStoreConsumer>(baseStorageSettings.Module);
if (!consumer.IsSet) return new DataStoreConsumer();
var dataStoreConsumer = (DataStoreConsumer)consumer.Clone();
foreach (var prop in baseStorageSettings.Props)
{
dataStoreConsumer[prop.Key] = prop.Value;
}
return dataStoreConsumer;
}
private IDataStore dataStore;
public IDataStore DataStore<T>(BaseStorageSettings<T> baseStorageSettings) where T : class, ISettings, new()
{
if (dataStore != null) return dataStore;
if (DataStoreConsumer(baseStorageSettings).HandlerType == null) return null;
return dataStore = ((IDataStore)
Activator.CreateInstance(DataStoreConsumer(baseStorageSettings).HandlerType, TenantManager, PathUtils, HttpContextAccessor, Options))
.Configure(TenantManager.GetCurrentTenant().TenantId.ToString(), null, null, DataStoreConsumer(baseStorageSettings));
}
}
[Scope]
public class BaseStorageSettingsListenerScope
{
private StorageSettingsHelper StorageSettingsHelper { get; }
private SettingsManager SettingsManager { get; }
private CdnStorageSettings CdnStorageSettings { get; }
public BaseStorageSettingsListenerScope(StorageSettingsHelper storageSettingsHelper, SettingsManager settingsManager, CdnStorageSettings cdnStorageSettings)
{
StorageSettingsHelper = storageSettingsHelper;
SettingsManager = settingsManager;
CdnStorageSettings = cdnStorageSettings;
}
public void Deconstruct(out StorageSettingsHelper storageSettingsHelper, out SettingsManager settingsManager, out CdnStorageSettings cdnStorageSettings)
{
storageSettingsHelper = StorageSettingsHelper;
settingsManager = SettingsManager;
cdnStorageSettings = CdnStorageSettings;
}
}
public static class StorageSettingsExtension
{
public static void Register(DIHelper services)
{
services.TryAdd<BaseStorageSettingsListenerScope>();
var cdnSettings = settingsManager.LoadForTenant<CdnStorageSettings>(i.TenantId);
if (i.Name == cdnSettings.Module)
{
storageSettingsHelper.Clear(cdnSettings);
}
}, Common.Caching.CacheNotifyAction.Remove);
}
}
}
[Serializable]
public abstract class BaseStorageSettings<T> : ISettings where T : class, ISettings, new()
{
public string Module { get; set; }
public Dictionary<string, string> Props { get; set; }
public virtual Func<DataStoreConsumer, DataStoreConsumer> Switch => d => d;
public abstract Guid ID { get; }
internal ICacheNotify<DataStoreCacheItem> Cache { get; set; }
public ISettings GetDefault(IServiceProvider serviceProvider)
{
return new T();
}
}
[Serializable]
public class StorageSettings : BaseStorageSettings<StorageSettings>
{
public override Guid ID => new Guid("F13EAF2D-FA53-44F1-A6D6-A5AEDA46FA2B");
}
[Scope]
[Serializable]
public class CdnStorageSettings : BaseStorageSettings<CdnStorageSettings>
{
public override Guid ID => new Guid("0E9AE034-F398-42FE-B5EE-F86D954E9FB2");
public override Func<DataStoreConsumer, DataStoreConsumer> Switch => d => d.Cdn;
}
[Scope]
public class StorageSettingsHelper
{
private readonly StorageFactoryConfig _storageFactoryConfig;
private readonly PathUtils _pathUtils;
private readonly ICacheNotify<DataStoreCacheItem> _cache;
private readonly IOptionsMonitor<ILog> _options;
private readonly TenantManager _tenantManager;
private readonly SettingsManager _settingsManager;
private readonly IHttpContextAccessor _httpContextAccessor;
private readonly ConsumerFactory _consumerFactory;
private IDataStore _dataStore;
public StorageSettingsHelper(
BaseStorageSettingsListener baseStorageSettingsListener,
StorageFactoryConfig storageFactoryConfig,
PathUtils pathUtils,
ICacheNotify<DataStoreCacheItem> cache,
IOptionsMonitor<ILog> options,
TenantManager tenantManager,
SettingsManager settingsManager,
ConsumerFactory consumerFactory)
{
baseStorageSettingsListener.Subscribe();
_storageFactoryConfig = storageFactoryConfig;
_pathUtils = pathUtils;
_cache = cache;
_options = options;
_tenantManager = tenantManager;
_settingsManager = settingsManager;
_consumerFactory = consumerFactory;
}
public StorageSettingsHelper(
BaseStorageSettingsListener baseStorageSettingsListener,
StorageFactoryConfig storageFactoryConfig,
PathUtils pathUtils,
ICacheNotify<DataStoreCacheItem> cache,
IOptionsMonitor<ILog> options,
TenantManager tenantManager,
SettingsManager settingsManager,
IHttpContextAccessor httpContextAccessor,
ConsumerFactory consumerFactory)
: this(baseStorageSettingsListener, storageFactoryConfig, pathUtils, cache, options, tenantManager, settingsManager, consumerFactory)
{
_httpContextAccessor = httpContextAccessor;
}
public bool Save<T>(BaseStorageSettings<T> baseStorageSettings) where T : class, ISettings, new()
{
ClearDataStoreCache();
return _settingsManager.Save(baseStorageSettings);
}
public void Clear<T>(BaseStorageSettings<T> baseStorageSettings) where T : class, ISettings, new()
{
baseStorageSettings.Module = null;
baseStorageSettings.Props = null;
Save(baseStorageSettings);
}
public DataStoreConsumer DataStoreConsumer<T>(BaseStorageSettings<T> baseStorageSettings) where T : class, ISettings, new()
{
if (string.IsNullOrEmpty(baseStorageSettings.Module) || baseStorageSettings.Props == null)
{
return new DataStoreConsumer();
}
var consumer = _consumerFactory.GetByKey<DataStoreConsumer>(baseStorageSettings.Module);
if (!consumer.IsSet)
{
return new DataStoreConsumer();
}
var _dataStoreConsumer = (DataStoreConsumer)consumer.Clone();
foreach (var prop in baseStorageSettings.Props)
{
_dataStoreConsumer[prop.Key] = prop.Value;
}
return _dataStoreConsumer;
}
public IDataStore DataStore<T>(BaseStorageSettings<T> baseStorageSettings) where T : class, ISettings, new()
{
if (_dataStore != null)
{
return _dataStore;
}
if (DataStoreConsumer(baseStorageSettings).HandlerType == null)
{
return null;
}
return _dataStore = ((IDataStore)
Activator.CreateInstance(DataStoreConsumer(baseStorageSettings).HandlerType, _tenantManager, _pathUtils, _httpContextAccessor, _options))
.Configure(_tenantManager.GetCurrentTenant().TenantId.ToString(), null, null, DataStoreConsumer(baseStorageSettings));
}
internal void ClearDataStoreCache()
{
var tenantId = _tenantManager.GetCurrentTenant().TenantId.ToString();
var path = TenantPath.CreatePath(tenantId);
foreach (var module in _storageFactoryConfig.GetModuleList("", true))
{
_cache.Publish(new DataStoreCacheItem() { TenantId = path, Module = module }, CacheNotifyAction.Remove);
}
}
}
[Scope]
public class BaseStorageSettingsListenerScope
{
private readonly StorageSettingsHelper _storageSettingsHelper;
private readonly SettingsManager _settingsManager;
private readonly CdnStorageSettings _cdnStorageSettings;
public BaseStorageSettingsListenerScope(StorageSettingsHelper storageSettingsHelper, SettingsManager settingsManager, CdnStorageSettings cdnStorageSettings)
{
_storageSettingsHelper = storageSettingsHelper;
_settingsManager = settingsManager;
_cdnStorageSettings = cdnStorageSettings;
}
public void Deconstruct(out StorageSettingsHelper storageSettingsHelper, out SettingsManager settingsManager, out CdnStorageSettings cdnStorageSettings)
{
storageSettingsHelper = _storageSettingsHelper;
settingsManager = _settingsManager;
cdnStorageSettings = this._cdnStorageSettings;
}
}
public static class StorageSettingsExtension
{
public static void Register(DIHelper services)
{
services.TryAdd<BaseStorageSettingsListenerScope>();
}
}

19
common/ASC.Data.Storage/Constants.cs
View File

@ -24,14 +24,13 @@
*/
namespace ASC.Data.Storage
namespace ASC.Data.Storage;
static class Constants
{
static class Constants
{
public const string CONFIG_DIR = "CONFIG_DIR";
public const string STORAGE_ROOT_PARAM = "$STORAGE_ROOT";
public const string QUERY_AUTH = "auth";
public const string QUERY_EXPIRE = "expire";
public const string QUERY_HEADER = "headers";
}
}
public const string ConfigDir = "CONFIG_DIR";
public const string StorageRootParam = "$STORAGE_ROOT";
public const string QueryAuth = "auth";
public const string QueryExpire = "expire";
public const string QueryHeader = "headers";
}

201
common/ASC.Data.Storage/CrossModuleTransferUtility.cs
View File

@ -23,100 +23,117 @@
*
*/
namespace ASC.Data.Storage
{
public class CrossModuleTransferUtility
{
private readonly ILog Log;
private readonly IDataStore source;
private readonly IDataStore destination;
private readonly long maxChunkUploadSize;
private readonly int chunksize;
private IOptionsMonitor<ILog> Option { get; }
private TempStream TempStream { get; }
private TempPath TempPath { get; }
namespace ASC.Data.Storage;
public CrossModuleTransferUtility(
IOptionsMonitor<ILog> option,
TempStream tempStream,
TempPath tempPath,
IDataStore source,
IDataStore destination)
public class CrossModuleTransferUtility
{
private readonly ILog _logger;
private readonly IDataStore _source;
private readonly IDataStore _destination;
private readonly long _maxChunkUploadSize;
private readonly int _chunkSize;
private readonly IOptionsMonitor<ILog> _option;
private readonly TempStream _tempStream;
private readonly TempPath _tempPath;
public CrossModuleTransferUtility(
IOptionsMonitor<ILog> option,
TempStream tempStream,
TempPath tempPath,
IDataStore source,
IDataStore destination)
{
_logger = option.Get("ASC.CrossModuleTransferUtility");
_option = option;
_tempStream = tempStream;
_tempPath = tempPath;
_source = source ?? throw new ArgumentNullException(nameof(source));
_destination = destination ?? throw new ArgumentNullException(nameof(destination));
_maxChunkUploadSize = 10 * 1024 * 1024;
_chunkSize = 5 * 1024 * 1024;
}
public void CopyFile(string srcDomain, string srcPath, string destDomain, string destPath)
{
if (srcDomain == null)
{
Log = option.Get("ASC.CrossModuleTransferUtility");
Option = option;
TempStream = tempStream;
TempPath = tempPath;
this.source = source ?? throw new ArgumentNullException(nameof(source));
this.destination = destination ?? throw new ArgumentNullException(nameof(destination));
maxChunkUploadSize = 10 * 1024 * 1024;
chunksize = 5 * 1024 * 1024;
throw new ArgumentNullException(nameof(srcDomain));
}
if (srcPath == null)
{
throw new ArgumentNullException(nameof(srcPath));
}
public void CopyFile(string srcDomain, string srcPath, string destDomain, string destPath)
{
if (srcDomain == null) throw new ArgumentNullException(nameof(srcDomain));
if (srcPath == null) throw new ArgumentNullException(nameof(srcPath));
if (destDomain == null) throw new ArgumentNullException(nameof(destDomain));
if (destPath == null) throw new ArgumentNullException(nameof(destPath));
using var stream = source.GetReadStream(srcDomain, srcPath);
if (stream.Length < maxChunkUploadSize)
{
destination.Save(destDomain, destPath, stream);
}
else
{
var session = new CommonChunkedUploadSession(stream.Length);
var holder = new CommonChunkedUploadSessionHolder(TempPath, Option, destination, destDomain);
holder.Init(session);
try
{
Stream memstream = null;
try
{
while (GetStream(stream, out memstream))
{
memstream.Seek(0, SeekOrigin.Begin);
holder.UploadChunk(session, memstream, chunksize);
memstream.Dispose();
}
}
finally
{
if (memstream != null)
{
memstream.Dispose();
}
}
holder.Finalize(session);
destination.Move(destDomain, session.TempPath, destDomain, destPath);
}
catch (Exception ex)
{
Log.Error("Copy File", ex);
holder.Abort(session);
}
}
}
private bool GetStream(Stream stream, out Stream memstream)
{
memstream = TempStream.Create();
var total = 0;
int readed;
const int portion = 2048;
var buffer = new byte[portion];
while ((readed = stream.Read(buffer, 0, portion)) > 0)
{
memstream.Write(buffer, 0, readed);
total += readed;
if (total >= chunksize) break;
}
return total > 0;
}
}
if (destDomain == null)
{
throw new ArgumentNullException(nameof(destDomain));
}
if (destPath == null)
{
throw new ArgumentNullException(nameof(destPath));
}
using var stream = _source.GetReadStream(srcDomain, srcPath);
if (stream.Length < _maxChunkUploadSize)
{
_destination.Save(destDomain, destPath, stream);
}
else
{
var session = new CommonChunkedUploadSession(stream.Length);
var holder = new CommonChunkedUploadSessionHolder(_tempPath, _option, _destination, destDomain);
holder.Init(session);
try
{
Stream memstream = null;
try
{
while (GetStream(stream, out memstream))
{
memstream.Seek(0, SeekOrigin.Begin);
holder.UploadChunk(session, memstream, _chunkSize);
memstream.Dispose();
}
}
finally
{
if (memstream != null)
{
memstream.Dispose();
}
}
holder.Finalize(session);
_destination.Move(destDomain, session.TempPath, destDomain, destPath);
}
catch (Exception ex)
{
_logger.Error("Copy File", ex);
holder.Abort(session);
}
}
}
private bool GetStream(Stream stream, out Stream memstream)
{
memstream = _tempStream.Create();
var total = 0;
int readed;
const int portion = 2048;
var buffer = new byte[portion];
while ((readed = stream.Read(buffer, 0, portion)) > 0)
{
memstream.Write(buffer, 0, readed);
total += readed;
if (total >= _chunkSize)
{
break;
}
}
return total > 0;
}
}

42
common/ASC.Data.Storage/DataList.cs
View File

@ -23,33 +23,33 @@
*
*/
namespace ASC.Data.Storage
namespace ASC.Data.Storage;
public class DataList : Dictionary<string, string>
{
public class DataList : Dictionary<string, string>
public DataList(Module config)
{
public DataList(Module config)
Add(string.Empty, config.Data);
if (config.Domain != null)
{
Add(string.Empty, config.Data);
if (config.Domain != null)
foreach (var domain in config.Domain)
{
foreach (var domain in config.Domain)
{
Add(domain.Name, domain.Data);
}
}
else
{
config.Domain = new List<Module>();
Add(domain.Name, domain.Data);
}
}
public string GetData(string name)
else
{
if (ContainsKey(name))
{
return this[name] ?? string.Empty;
}
return string.Empty;
config.Domain = new List<Module>();
}
}
}
public string GetData(string name)
{
if (ContainsKey(name))
{
return this[name] ?? string.Empty;
}
return string.Empty;
}
}

200
common/ASC.Data.Storage/DiscStorage/DiscDataHandler.cs
View File

@ -23,113 +23,113 @@
*
*/
namespace ASC.Data.Storage.DiscStorage
{
public class DiscDataHandler
{
private readonly string _physPath;
private readonly bool _checkAuth;
public DiscDataHandler(string physPath, bool checkAuth = true)
{
_physPath = physPath;
_checkAuth = checkAuth;
}
namespace ASC.Data.Storage.DiscStorage;
public async Task Invoke(HttpContext context)
{
//TODO
//if (_checkAuth && !Core.SecurityContext.IsAuthenticated)
//{
// context.Response.StatusCode = (int)HttpStatusCode.Forbidden;
// return;
//}
public class DiscDataHandler
{
private readonly string _physPath;
private readonly bool _checkAuth;
var path = CombinePath(_physPath, context);
if (File.Exists(path))
{
var lastwrite = File.GetLastWriteTime(path);
var etag = '"' + lastwrite.Ticks.ToString("X8", CultureInfo.InvariantCulture) + '"';
var notmodified = context.Request.Headers["If-None-Match"] == etag ||
context.Request.Headers["If-Modified-Since"] == lastwrite.ToString("R");
if (notmodified)
{
context.Response.StatusCode = (int)HttpStatusCode.NotModified;
}
else
{
if (File.Exists(path + ".gz"))
{
await context.Response.SendFileAsync(path + ".gz");
context.Response.Headers["Content-Encoding"] = "gzip";
}
else
{
await context.Response.SendFileAsync(path);
}
context.Response.ContentType = MimeMapping.GetMimeMapping(path);
//TODO
//context.Response.Cache.SetVaryByCustom("*");
//context.Response.Cache.SetAllowResponseInBrowserHistory(true);
//context.Response.Cache.SetExpires(DateTime.UtcNow.AddDays(1));
//context.Response.Cache.SetLastModified(lastwrite);
//context.Response.Cache.SetETag(etag);
//context.Response.Cache.SetCacheability(HttpCacheability.Public);
}
}
else
{
context.Response.StatusCode = (int)HttpStatusCode.NotFound;
}
}
static string CombinePath(string physPath, HttpContext requestContext)
{
var pathInfo = GetRouteValue("pathInfo").Replace('/', Path.DirectorySeparatorChar);
var path = CrossPlatform.PathCombine(physPath, pathInfo);
var tenant = GetRouteValue("0");
if (string.IsNullOrEmpty(tenant))
{
tenant = CrossPlatform.PathCombine(GetRouteValue("t1"), GetRouteValue("t2"), GetRouteValue("t3"));
}
path = path.Replace("{0}", tenant);
return path;
string GetRouteValue(string name)
{
return (requestContext.GetRouteValue(name) ?? "").ToString();
}
}
public DiscDataHandler(string physPath, bool checkAuth = true)
{
_physPath = physPath;
_checkAuth = checkAuth;
}
public static class DiscDataHandlerExtensions
public async Task Invoke(HttpContext context)
{
public static IEndpointRouteBuilder RegisterDiscDataHandler(this IEndpointRouteBuilder builder, string virtPath, string physPath, bool publicRoute = false)
{
if (virtPath != "/")
{
var handler = new DiscDataHandler(physPath, !publicRoute);
var url = virtPath + "{*pathInfo}";
//TODO
//if (_checkAuth && !Core.SecurityContext.IsAuthenticated)
//{
// context.Response.StatusCode = (int)HttpStatusCode.Forbidden;
// return;
//}
if (!builder.DataSources.Any(r => r.Endpoints.Any(e => e.DisplayName == url)))
var path = CombinePath(_physPath, context);
if (File.Exists(path))
{
var lastwrite = File.GetLastWriteTime(path);
var etag = '"' + lastwrite.Ticks.ToString("X8", CultureInfo.InvariantCulture) + '"';
var notmodified = context.Request.Headers["If-None-Match"] == etag ||
context.Request.Headers["If-Modified-Since"] == lastwrite.ToString("R");
if (notmodified)
{
context.Response.StatusCode = (int)HttpStatusCode.NotModified;
}
else
{
if (File.Exists(path + ".gz"))
{
await context.Response.SendFileAsync(path + ".gz");
context.Response.Headers["Content-Encoding"] = "gzip";
}
else
{
await context.Response.SendFileAsync(path);
}
context.Response.ContentType = MimeMapping.GetMimeMapping(path);
//TODO
//context.Response.Cache.SetVaryByCustom("*");
//context.Response.Cache.SetAllowResponseInBrowserHistory(true);
//context.Response.Cache.SetExpires(DateTime.UtcNow.AddDays(1));
//context.Response.Cache.SetLastModified(lastwrite);
//context.Response.Cache.SetETag(etag);
//context.Response.Cache.SetCacheability(HttpCacheability.Public);
}
}
else
{
context.Response.StatusCode = (int)HttpStatusCode.NotFound;
}
}
private static string CombinePath(string physPath, HttpContext requestContext)
{
var pathInfo = GetRouteValue("pathInfo").Replace('/', Path.DirectorySeparatorChar);
var path = CrossPlatform.PathCombine(physPath, pathInfo);
var tenant = GetRouteValue("0");
if (string.IsNullOrEmpty(tenant))
{
tenant = CrossPlatform.PathCombine(GetRouteValue("t1"), GetRouteValue("t2"), GetRouteValue("t3"));
}
path = path.Replace("{0}", tenant);
return path;
string GetRouteValue(string name)
{
return (requestContext.GetRouteValue(name) ?? "").ToString();
}
}
}
public static class DiscDataHandlerExtensions
{
public static IEndpointRouteBuilder RegisterDiscDataHandler(this IEndpointRouteBuilder builder, string virtPath, string physPath, bool publicRoute = false)
{
if (virtPath != "/")
{
var handler = new DiscDataHandler(physPath, !publicRoute);
var url = virtPath + "{*pathInfo}";
if (!builder.DataSources.Any(r => r.Endpoints.Any(e => e.DisplayName == url)))
{
builder.Map(url, handler.Invoke);
var newUrl = url.Replace("{0}", "{t1}/{t2}/{t3}");
if (newUrl != url)
{
builder.Map(url, handler.Invoke);
var newUrl = url.Replace("{0}", "{t1}/{t2}/{t3}");
if (newUrl != url)
{
builder.Map(url, handler.Invoke);
}
}
}
}
return builder;
}
}
}
return builder;
}
}

1357
common/ASC.Data.Storage/DiscStorage/DiscDataStore.cs
View File

File diff suppressed because it is too large Load Diff

59
common/ASC.Data.Storage/DiscStorage/MappedPath.cs
View File

@ -23,33 +23,34 @@
*
*/
namespace ASC.Data.Storage.DiscStorage
{
internal class MappedPath
{
public string PhysicalPath { get; set; }
private PathUtils PathUtils { get; }
private MappedPath(PathUtils pathUtils)
namespace ASC.Data.Storage.DiscStorage;
internal class MappedPath
{
public string PhysicalPath { get; set; }
private readonly PathUtils _pathUtils;
public MappedPath(PathUtils pathUtils, string tenant, bool appendTenant, string ppath, IDictionary<string, string> storageConfig) : this(pathUtils)
{
tenant = tenant.Trim('/');
ppath = _pathUtils.ResolvePhysicalPath(ppath, storageConfig);
PhysicalPath = ppath.IndexOf('{') == -1 && appendTenant ? CrossPlatform.PathCombine(ppath, tenant) : string.Format(ppath, tenant);
}
private MappedPath(PathUtils pathUtils)
{
_pathUtils = pathUtils;
}
public MappedPath AppendDomain(string domain)
{
domain = domain.Replace('.', '_'); //Domain prep. Remove dots
return new MappedPath(_pathUtils)
{
PathUtils = pathUtils;
}
public MappedPath(PathUtils pathUtils, string tenant, bool appendTenant, string ppath, IDictionary<string, string> storageConfig) : this(pathUtils)
{
tenant = tenant.Trim('/');
ppath = PathUtils.ResolvePhysicalPath(ppath, storageConfig);
PhysicalPath = ppath.IndexOf('{') == -1 && appendTenant ? CrossPlatform.PathCombine(ppath, tenant) : string.Format(ppath, tenant);
}
public MappedPath AppendDomain(string domain)
{
domain = domain.Replace('.', '_'); //Domain prep. Remove dots
return new MappedPath(PathUtils)
{
PhysicalPath = CrossPlatform.PathCombine(PhysicalPath, PathUtils.Normalize(domain, true)),
};
}
}
}
PhysicalPath = CrossPlatform.PathCombine(PhysicalPath, PathUtils.Normalize(domain, true)),
};
}
}

44
common/ASC.Data.Storage/Encryption/EncryptionFactory.cs
View File

@ -23,31 +23,31 @@
*
*/
namespace ASC.Data.Storage.Encryption
{
[Singletone]
public class EncryptionFactory
{
private IServiceProvider ServiceProvider { get; }
namespace ASC.Data.Storage.Encryption;
public EncryptionFactory(IServiceProvider serviceProvider)
[Singletone]
public class EncryptionFactory
{
private readonly IServiceScopeFactory _serviceScopeFactory;
public EncryptionFactory(IServiceScopeFactory serviceScopeFactory)
{
_serviceScopeFactory = serviceScopeFactory;
}
public ICrypt GetCrypt(string storageName, EncryptionSettings encryptionSettings)
{
ICrypt result = null;
using var scope = _serviceScopeFactory.CreateScope();
if (scope != null)
{
ServiceProvider = serviceProvider;
result = scope.ServiceProvider.GetService<ICrypt>();
}
public ICrypt GetCrypt(string storageName, EncryptionSettings encryptionSettings)
{
ICrypt result = null;
result ??= new FakeCrypt();
result.Init(storageName, encryptionSettings);
using var scope = ServiceProvider.CreateScope();
if (scope != null)
{
result = scope.ServiceProvider.GetService<ICrypt>();
}
result ??= new FakeCrypt();
result.Init(storageName, encryptionSettings);
return result;
}
}
return result;
}
}

674
common/ASC.Data.Storage/Encryption/EncryptionOperation.cs
View File

@ -25,348 +25,350 @@
using Tenant = ASC.Core.Tenants.Tenant;
namespace ASC.Data.Storage.Encryption
{
[Transient(Additional = typeof(EncryptionOperationExtension))]
public class EncryptionOperation : DistributedTaskProgress
{
private const string ConfigPath = "";
private bool HasErrors = false;
private const string ProgressFileName = "EncryptionProgress.tmp";
private IServiceProvider ServiceProvider { get; }
private EncryptionSettings EncryptionSettings { get; set; }
private bool IsEncryption { get; set; }
private bool UseProgressFile { get; set; }
private IEnumerable<string> Modules { get; set; }
private IEnumerable<Tenant> Tenants { get; set; }
private string ServerRootPath { get; set; }
public EncryptionOperation(IServiceProvider serviceProvider)
{
ServiceProvider = serviceProvider;
}
public void Init(EncryptionSettingsProto encryptionSettingsProto, string id)
namespace ASC.Data.Storage.Encryption;
[Transient(Additional = typeof(EncryptionOperationExtension))]
public class EncryptionOperation : DistributedTaskProgress
{
private const string ConfigPath = "";
private const string ProgressFileName = "EncryptionProgress.tmp";
private readonly IServiceScopeFactory _serviceScopeFactory;
private bool _hasErrors = false;
private EncryptionSettings _encryptionSettings;
private bool _isEncryption;
private bool _useProgressFile;
private IEnumerable<string> _modules;
private IEnumerable<Tenant> _tenants;
private string _serverRootPath;
public EncryptionOperation(IServiceScopeFactory serviceScopeFactory)
{
_serviceScopeFactory = serviceScopeFactory;
}
public void Init(EncryptionSettingsProto encryptionSettingsProto, string id)
{
Id = id;
_encryptionSettings = new EncryptionSettings(encryptionSettingsProto);
_isEncryption = _encryptionSettings.Status == EncryprtionStatus.EncryptionStarted;
_serverRootPath = encryptionSettingsProto.ServerRootPath;
}
protected override void DoJob()
{
using var scope = _serviceScopeFactory.CreateScope();
var scopeClass = scope.ServiceProvider.GetService<EncryptionOperationScope>();
var (log, encryptionSettingsHelper, tenantManager, notifyHelper, coreBaseSettings, storageFactoryConfig, storageFactory, configuration) = scopeClass;
notifyHelper.Init(_serverRootPath);
_tenants = tenantManager.GetTenants(false);
_modules = storageFactoryConfig.GetModuleList(ConfigPath, true);
_useProgressFile = Convert.ToBoolean(configuration["storage:encryption:progressfile"] ?? "true");
Percentage = 10;
PublishChanges();
try
{
Id = id;
EncryptionSettings = new EncryptionSettings(encryptionSettingsProto);
IsEncryption = EncryptionSettings.Status == EncryprtionStatus.EncryptionStarted;
ServerRootPath = encryptionSettingsProto.ServerRootPath;
}
protected override void DoJob()
{
using var scope = ServiceProvider.CreateScope();
var scopeClass = scope.ServiceProvider.GetService<EncryptionOperationScope>();
var (log, encryptionSettingsHelper, tenantManager, notifyHelper, coreBaseSettings, storageFactoryConfig, storageFactory, configuration) = scopeClass;
notifyHelper.Init(ServerRootPath);
Tenants = tenantManager.GetTenants(false);
Modules = storageFactoryConfig.GetModuleList(ConfigPath, true);
UseProgressFile = Convert.ToBoolean(configuration["storage:encryption:progressfile"] ?? "true");
Percentage = 10;
if (!coreBaseSettings.Standalone)
{
throw new NotSupportedException();
}
if (_encryptionSettings.Status == EncryprtionStatus.Encrypted || _encryptionSettings.Status == EncryprtionStatus.Decrypted)
{
log.Debug("Storage already " + _encryptionSettings.Status);
return;
}
Percentage = 30;
PublishChanges();
try
{
if (!coreBaseSettings.Standalone)
{
throw new NotSupportedException();
}
if (EncryptionSettings.Status == EncryprtionStatus.Encrypted || EncryptionSettings.Status == EncryprtionStatus.Decrypted)
{
log.Debug("Storage already " + EncryptionSettings.Status);
return;
foreach (var tenant in _tenants)
{
var dictionary = new Dictionary<string, DiscDataStore>();
foreach (var module in _modules)
{
dictionary.Add(module, (DiscDataStore)storageFactory.GetStorage(ConfigPath, tenant.TenantId.ToString(), module));
}
Percentage = 30;
PublishChanges();
foreach (var tenant in Tenants)
{
var dictionary = new Dictionary<string, DiscDataStore>();
foreach (var module in Modules)
{
dictionary.Add(module, (DiscDataStore)storageFactory.GetStorage(ConfigPath, tenant.TenantId.ToString(), module));
}
Parallel.ForEach(dictionary, (elem) =>
{
EncryptStore(tenant, elem.Key, elem.Value, storageFactoryConfig, log);
});
}
Percentage = 70;
PublishChanges();
if (!HasErrors)
{
DeleteProgressFiles(storageFactory);
SaveNewSettings(encryptionSettingsHelper, log);
}
Percentage = 90;
PublishChanges();
ActivateTenants(tenantManager, log, notifyHelper);
Percentage = 100;
PublishChanges();
}
catch (Exception e)
{
Exception = e;
log.Error(e);
}
}
private void EncryptStore(Tenant tenant, string module, DiscDataStore store, StorageFactoryConfig storageFactoryConfig, ILog log)
{
var domains = storageFactoryConfig.GetDomainList(ConfigPath, module).ToList();
domains.Add(string.Empty);
var progress = ReadProgress(store);
foreach (var domain in domains)
{
Parallel.ForEach(dictionary, (elem) =>
{
EncryptStore(tenant, elem.Key, elem.Value, storageFactoryConfig, log);
});
}
Percentage = 70;
PublishChanges();
if (!_hasErrors)
{
DeleteProgressFiles(storageFactory);
SaveNewSettings(encryptionSettingsHelper, log);
}
Percentage = 90;
PublishChanges();
ActivateTenants(tenantManager, log, notifyHelper);
Percentage = 100;
PublishChanges();
}
catch (Exception e)
{
Exception = e;
log.Error(e);
}
}
private void EncryptStore(Tenant tenant, string module, DiscDataStore store, StorageFactoryConfig storageFactoryConfig, ILog log)
{
var domains = storageFactoryConfig.GetDomainList(ConfigPath, module).ToList();
domains.Add(string.Empty);
var progress = ReadProgress(store);
foreach (var domain in domains)
{
var logParent = $"Tenant: {tenant.TenantAlias}, Module: {module}, Domain: {domain}";
var files = GetFiles(domains, progress, store, domain);
EncryptFiles(store, domain, files, logParent, log);
}
StepDone();
log.DebugFormat("Percentage: {0}", Percentage);
}
private List<string> ReadProgress(DiscDataStore store)
{
var encryptedFiles = new List<string>();
if (!UseProgressFile)
{
return encryptedFiles;
}
if (store.IsFile(string.Empty, ProgressFileName))
{
using var stream = store.GetReadStream(string.Empty, ProgressFileName);
using var reader = new StreamReader(stream);
string line;
while ((line = reader.ReadLine()) != null)
{
encryptedFiles.Add(line);
}
}
else
{
store.GetWriteStream(string.Empty, ProgressFileName).Close();
}
return encryptedFiles;
}
private IEnumerable<string> GetFiles(List<string> domains, List<string> progress, DiscDataStore targetStore, string targetDomain)
{
IEnumerable<string> files = targetStore.ListFilesRelative(targetDomain, "\\", "*.*", true);
var files = GetFiles(domains, progress, store, domain);
EncryptFiles(store, domain, files, logParent, log);
}
StepDone();
log.DebugFormat("Percentage: {0}", Percentage);
}
private List<string> ReadProgress(DiscDataStore store)
{
var encryptedFiles = new List<string>();
if (!_useProgressFile)
{
return encryptedFiles;
}
if (store.IsFile(string.Empty, ProgressFileName))
{
using var stream = store.GetReadStream(string.Empty, ProgressFileName);
using var reader = new StreamReader(stream);
string line;
while ((line = reader.ReadLine()) != null)
{
encryptedFiles.Add(line);
}
}
else
{
store.GetWriteStream(string.Empty, ProgressFileName).Close();
}
return encryptedFiles;
}
private IEnumerable<string> GetFiles(List<string> domains, List<string> progress, DiscDataStore targetStore, string targetDomain)
{
IEnumerable<string> files = targetStore.ListFilesRelative(targetDomain, "\\", "*.*", true);
if (progress.Count > 0)
{
files = files.Where(path => !progress.Contains(path));
}
if (!string.IsNullOrEmpty(targetDomain))
{
return files;
}
var notEmptyDomains = domains.Where(domain => !string.IsNullOrEmpty(domain));
if (notEmptyDomains.Any())
{
files = files.Where(path => notEmptyDomains.All(domain => !path.Contains(domain + Path.DirectorySeparatorChar)));
}
files = files.Where(path => !path.EndsWith(ProgressFileName));
return files;
}
private void EncryptFiles(DiscDataStore store, string domain, IEnumerable<string> files, string logParent, ILog log)
{
foreach (var file in files)
{
{
files = files.Where(path => !progress.Contains(path));
}
if (!string.IsNullOrEmpty(targetDomain))
{
return files;
}
var notEmptyDomains = domains.Where(domain => !string.IsNullOrEmpty(domain));
if (notEmptyDomains.Any())
{
files = files.Where(path => notEmptyDomains.All(domain => !path.Contains(domain + Path.DirectorySeparatorChar)));
}
files = files.Where(path => !path.EndsWith(ProgressFileName));
return files;
}
private void EncryptFiles(DiscDataStore store, string domain, IEnumerable<string> files, string logParent, ILog log)
{
foreach (var file in files)
{
var logItem = $"{logParent}, File: {file}";
log.Debug(logItem);
try
{
if (IsEncryption)
{
store.Encrypt(domain, file);
}
else
{
store.Decrypt(domain, file);
}
WriteProgress(store, file, UseProgressFile);
}
catch (Exception e)
{
HasErrors = true;
log.Error(logItem + " " + e.Message, e);
// ERROR_DISK_FULL: There is not enough space on the disk.
// if (e is IOException && e.HResult == unchecked((int)0x80070070)) break;
}
}
}
private void WriteProgress(DiscDataStore store, string file, bool useProgressFile)
{
if (!useProgressFile)
{
return;
}
using var stream = store.GetWriteStream(string.Empty, ProgressFileName, FileMode.Append);
using var writer = new StreamWriter(stream);
writer.WriteLine(file);
}
private void DeleteProgressFiles(StorageFactory storageFactory)
{
foreach (var tenant in Tenants)
{
foreach (var module in Modules)
{
var store = (DiscDataStore)storageFactory.GetStorage(ConfigPath, tenant.TenantId.ToString(), module);
if (store.IsFile(string.Empty, ProgressFileName))
{
store.Delete(string.Empty, ProgressFileName);
}
}
}
}
private void SaveNewSettings(EncryptionSettingsHelper encryptionSettingsHelper, ILog log)
{
if (IsEncryption)
{
EncryptionSettings.Status = EncryprtionStatus.Encrypted;
}
else
{
EncryptionSettings.Status = EncryprtionStatus.Decrypted;
EncryptionSettings.Password = string.Empty;
}
encryptionSettingsHelper.Save(EncryptionSettings);
log.Debug("Save new EncryptionSettings");
}
private void ActivateTenants(TenantManager tenantManager, ILog log, NotifyHelper notifyHelper)
{
foreach (var tenant in Tenants)
{
if (tenant.Status == TenantStatus.Encryption)
{
tenantManager.SetCurrentTenant(tenant);
tenant.SetStatus(TenantStatus.Active);
tenantManager.SaveTenant(tenant);
log.DebugFormat("Tenant {0} SetStatus Active", tenant.TenantAlias);
if (!HasErrors)
log.Debug(logItem);
try
{
if (_isEncryption)
{
store.Encrypt(domain, file);
}
else
{
store.Decrypt(domain, file);
}
WriteProgress(store, file, _useProgressFile);
}
catch (Exception e)
{
_hasErrors = true;
log.Error(logItem + " " + e.Message, e);
// ERROR_DISK_FULL: There is not enough space on the disk.
// if (e is IOException && e.HResult == unchecked((int)0x80070070)) break;
}
}
}
private void WriteProgress(DiscDataStore store, string file, bool useProgressFile)
{
if (!useProgressFile)
{
return;
}
using var stream = store.GetWriteStream(string.Empty, ProgressFileName, FileMode.Append);
using var writer = new StreamWriter(stream);
writer.WriteLine(file);
}
private void DeleteProgressFiles(StorageFactory storageFactory)
{
foreach (var tenant in _tenants)
{
foreach (var module in _modules)
{
var store = (DiscDataStore)storageFactory.GetStorage(ConfigPath, tenant.TenantId.ToString(), module);
if (store.IsFile(string.Empty, ProgressFileName))
{
store.Delete(string.Empty, ProgressFileName);
}
}
}
}
private void SaveNewSettings(EncryptionSettingsHelper encryptionSettingsHelper, ILog log)
{
if (_isEncryption)
{
_encryptionSettings.Status = EncryprtionStatus.Encrypted;
}
else
{
_encryptionSettings.Status = EncryprtionStatus.Decrypted;
_encryptionSettings.Password = string.Empty;
}
encryptionSettingsHelper.Save(_encryptionSettings);
log.Debug("Save new EncryptionSettings");
}
private void ActivateTenants(TenantManager tenantManager, ILog log, NotifyHelper notifyHelper)
{
foreach (var tenant in _tenants)
{
if (tenant.Status == TenantStatus.Encryption)
{
tenantManager.SetCurrentTenant(tenant);
tenant.SetStatus(TenantStatus.Active);
tenantManager.SaveTenant(tenant);
log.DebugFormat("Tenant {0} SetStatus Active", tenant.TenantAlias);
if (!_hasErrors)
{
if (_encryptionSettings.NotifyUsers)
{
if (EncryptionSettings.NotifyUsers)
{
if (IsEncryption)
{
notifyHelper.SendStorageEncryptionSuccess(tenant.TenantId);
}
else
{
notifyHelper.SendStorageDecryptionSuccess(tenant.TenantId);
}
log.DebugFormat("Tenant {0} SendStorageEncryptionSuccess", tenant.TenantAlias);
}
}
else
{
if (IsEncryption)
{
notifyHelper.SendStorageEncryptionError(tenant.TenantId);
}
else
{
notifyHelper.SendStorageDecryptionError(tenant.TenantId);
}
log.DebugFormat("Tenant {0} SendStorageEncryptionError", tenant.TenantAlias);
}
}
}
}
}
[Scope]
public class EncryptionOperationScope
{
private ILog Log { get; set; }
private EncryptionSettingsHelper EncryptionSettingsHelper { get; set; }
private TenantManager TenantManager { get; set; }
private NotifyHelper NotifyHelper { get; set; }
private CoreBaseSettings CoreBaseSettings { get; set; }
private StorageFactoryConfig StorageFactoryConfig { get; set; }
private StorageFactory StorageFactory { get; set; }
private IConfiguration Configuration { get; }
public EncryptionOperationScope(IOptionsMonitor<ILog> options,
StorageFactoryConfig storageFactoryConfig,
StorageFactory storageFactory,
TenantManager tenantManager,
CoreBaseSettings coreBaseSettings,
NotifyHelper notifyHelper,
EncryptionSettingsHelper encryptionSettingsHelper,
IConfiguration configuration)
{
Log = options.CurrentValue;
StorageFactoryConfig = storageFactoryConfig;
StorageFactory = storageFactory;
TenantManager = tenantManager;
CoreBaseSettings = coreBaseSettings;
NotifyHelper = notifyHelper;
EncryptionSettingsHelper = encryptionSettingsHelper;
Configuration = configuration;
}
public void Deconstruct(out ILog log, out EncryptionSettingsHelper encryptionSettingsHelper, out TenantManager tenantManager, out NotifyHelper notifyHelper, out CoreBaseSettings coreBaseSettings, out StorageFactoryConfig storageFactoryConfig, out StorageFactory storageFactory, out IConfiguration configuration)
{
log = Log;
encryptionSettingsHelper = EncryptionSettingsHelper;
tenantManager = TenantManager;
notifyHelper = NotifyHelper;
coreBaseSettings = CoreBaseSettings;
storageFactoryConfig = StorageFactoryConfig;
storageFactory = StorageFactory;
configuration = Configuration;
}
}
public static class EncryptionOperationExtension
{
public static void Register(DIHelper services)
{
services.TryAdd<EncryptionOperationScope>();
}
}
if (_isEncryption)
{
notifyHelper.SendStorageEncryptionSuccess(tenant.TenantId);
}
else
{
notifyHelper.SendStorageDecryptionSuccess(tenant.TenantId);
}
log.DebugFormat("Tenant {0} SendStorageEncryptionSuccess", tenant.TenantAlias);
}
}
else
{
if (_isEncryption)
{
notifyHelper.SendStorageEncryptionError(tenant.TenantId);
}
else
{
notifyHelper.SendStorageDecryptionError(tenant.TenantId);
}
log.DebugFormat("Tenant {0} SendStorageEncryptionError", tenant.TenantAlias);
}
}
}
}
}
[Scope]
public class EncryptionOperationScope
{
private readonly ILog _logger;
private readonly EncryptionSettingsHelper _encryptionSettingsHelper;
private readonly TenantManager _tenantManager;
private readonly NotifyHelper _notifyHelper;
private readonly CoreBaseSettings _coreBaseSettings;
private readonly StorageFactoryConfig _storageFactoryConfig;
private readonly StorageFactory _storageFactory;
private readonly IConfiguration _configuration;
public EncryptionOperationScope(IOptionsMonitor<ILog> options,
StorageFactoryConfig storageFactoryConfig,
StorageFactory storageFactory,
TenantManager tenantManager,
CoreBaseSettings coreBaseSettings,
NotifyHelper notifyHelper,
EncryptionSettingsHelper encryptionSettingsHelper,
IConfiguration configuration)
{
_logger = options.CurrentValue;
_storageFactoryConfig = storageFactoryConfig;
_storageFactory = storageFactory;
_tenantManager = tenantManager;
_coreBaseSettings = coreBaseSettings;
_notifyHelper = notifyHelper;
_encryptionSettingsHelper = encryptionSettingsHelper;
_configuration = configuration;
}
public void Deconstruct(out ILog log, out EncryptionSettingsHelper encryptionSettingsHelper, out TenantManager tenantManager, out NotifyHelper notifyHelper, out CoreBaseSettings coreBaseSettings, out StorageFactoryConfig storageFactoryConfig, out StorageFactory storageFactory, out IConfiguration configuration)
{
log = _logger;
encryptionSettingsHelper = _encryptionSettingsHelper;
tenantManager = _tenantManager;
notifyHelper = _notifyHelper;
coreBaseSettings = _coreBaseSettings;
storageFactoryConfig = _storageFactoryConfig;
storageFactory = _storageFactory;
configuration = _configuration;
}
}
public static class EncryptionOperationExtension
{
public static void Register(DIHelper services)
{
services.TryAdd<EncryptionOperationScope>();
}
}

53
common/ASC.Data.Storage/Encryption/EncryptionServiceClient.cs
View File

@ -23,33 +23,30 @@
*
*/
using CacheNotifyAction = ASC.Common.Caching.CacheNotifyAction;
namespace ASC.Data.Storage.Encryption
using CacheNotifyAction = ASC.Common.Caching.CacheNotifyAction;
namespace ASC.Data.Storage.Encryption;
[Scope]
public class EncryptionServiceClient : IEncryptionService
{
[Scope]
public class EncryptionServiceClient : IEncryptionService
{
private ICacheNotify<EncryptionSettingsProto> NotifySetting { get; }
private ICacheNotify<EncryptionStop> NotifyStop { get; }
public EncryptionServiceClient(
ICacheNotify<EncryptionSettingsProto> notifySetting, ICacheNotify<EncryptionStop> notifyStop)
{
NotifySetting = notifySetting;
NotifyStop = notifyStop;
}
public void Start(EncryptionSettingsProto encryptionSettingsProto)
{
NotifySetting.Publish(encryptionSettingsProto, CacheNotifyAction.Insert);
}
public void Stop()
{
NotifyStop.Publish(new EncryptionStop(), CacheNotifyAction.Insert);
}
}
private readonly ICacheNotify<EncryptionSettingsProto> _notifySetting;
private readonly ICacheNotify<EncryptionStop> _notifyStop;
public EncryptionServiceClient(
ICacheNotify<EncryptionSettingsProto> notifySetting, ICacheNotify<EncryptionStop> notifyStop)
{
_notifySetting = notifySetting;
_notifyStop = notifyStop;
}
public void Start(EncryptionSettingsProto encryptionSettingsProto)
{
_notifySetting.Publish(encryptionSettingsProto, CacheNotifyAction.Insert);
}
public void Stop()
{
_notifyStop.Publish(new EncryptionStop(), CacheNotifyAction.Insert);
}
}

143
common/ASC.Data.Storage/Encryption/EncryptionWorker.cs
View File

@ -23,74 +23,79 @@
*
*/
namespace ASC.Data.Storage.Encryption
{
[Singletone]
public class EncryptionWorker
{
private object Locker { get; }
private FactoryOperation FactoryOperation { get; }
private DistributedTaskQueue Queue { get; }
public EncryptionWorker(FactoryOperation factoryOperation, DistributedTaskQueueOptionsManager options)
{
Locker = new object();
FactoryOperation = factoryOperation;
Queue = options.Get<EncryptionOperation>();
}
public void Start(EncryptionSettingsProto encryptionSettings)
{
EncryptionOperation encryptionOperation;
lock (Locker)
{
if (Queue.GetTask<EncryptionOperation>(GetCacheId()) != null) return;
encryptionOperation = FactoryOperation.CreateOperation(encryptionSettings, GetCacheId());
Queue.QueueTask(encryptionOperation);
}
}
public void Stop()
{
Queue.CancelTask(GetCacheId());
}
public string GetCacheId()
{
return typeof(EncryptionOperation).FullName;
}
public double? GetEncryptionProgress()
{
var progress = Queue.GetTasks<EncryptionOperation>().FirstOrDefault();
return progress.Percentage;
}
}
[Singletone(Additional = typeof(FactoryOperationExtension))]
public class FactoryOperation
{
private IServiceProvider ServiceProvider { get; set; }
public FactoryOperation(IServiceProvider serviceProvider)
{
ServiceProvider = serviceProvider;
}
public EncryptionOperation CreateOperation(EncryptionSettingsProto encryptionSettings, string id)
{
var item = ServiceProvider.GetService<EncryptionOperation>();
item.Init(encryptionSettings, id);
return item;
}
}
public static class FactoryOperationExtension
{
public static void Register(DIHelper dIHelper)
namespace ASC.Data.Storage.Encryption;
[Singletone]
public class EncryptionWorker
{
private readonly object _locker;
private readonly FactoryOperation _factoryOperation;
private readonly DistributedTaskQueue _queue;
public EncryptionWorker(FactoryOperation factoryOperation, DistributedTaskQueueOptionsManager options)
{
_locker = new object();
_factoryOperation = factoryOperation;
_queue = options.Get<EncryptionOperation>();
}
public void Start(EncryptionSettingsProto encryptionSettings)
{
EncryptionOperation encryptionOperation;
lock (_locker)
{
dIHelper.TryAdd<EncryptionOperation>();
dIHelper.AddDistributedTaskQueueService<EncryptionOperation>(1);
}
}
if (_queue.GetTask<EncryptionOperation>(GetCacheId()) != null)
{
return;
}
encryptionOperation = _factoryOperation.CreateOperation(encryptionSettings, GetCacheId());
_queue.QueueTask(encryptionOperation);
}
}
public void Stop()
{
_queue.CancelTask(GetCacheId());
}
public string GetCacheId()
{
return typeof(EncryptionOperation).FullName;
}
public double? GetEncryptionProgress()
{
var progress = _queue.GetTasks<EncryptionOperation>().FirstOrDefault();
return progress.Percentage;
}
}
[Singletone(Additional = typeof(FactoryOperationExtension))]
public class FactoryOperation
{
private readonly IServiceProvider _serviceProvider;
public FactoryOperation(IServiceProvider serviceProvider)
{
_serviceProvider = serviceProvider;
}
public EncryptionOperation CreateOperation(EncryptionSettingsProto encryptionSettings, string id)
{
var item = _serviceProvider.GetService<EncryptionOperation>();
item.Init(encryptionSettings, id);
return item;
}
}
public static class FactoryOperationExtension
{
public static void Register(DIHelper dIHelper)
{
dIHelper.TryAdd<EncryptionOperation>();
dIHelper.AddDistributedTaskQueueService<EncryptionOperation>(1);
}
}

52
common/ASC.Data.Storage/Encryption/FakeCrypt.cs
View File

@ -1,32 +1,28 @@
namespace ASC.Data.Storage.Encryption
namespace ASC.Data.Storage.Encryption;
public class FakeCrypt : ICrypt
{
public class FakeCrypt : ICrypt
public byte Version => 1;
public void EncryptFile(string filePath)
{
public byte Version { get { return 1; } }
public void EncryptFile(string filePath)
{
return;
}
public void DecryptFile(string filePath)
{
return;
}
public Stream GetReadStream(string filePath)
{
return File.OpenRead(filePath);
}
public long GetFileSize(string filePath)
{
return new FileInfo(filePath).Length;
}
public void Init(string storageName, EncryptionSettings encryptionSettings)
{
}
return;
}
public void DecryptFile(string filePath)
{
return;
}
public Stream GetReadStream(string filePath)
{
return File.OpenRead(filePath);
}
public long GetFileSize(string filePath)
{
return new FileInfo(filePath).Length;
}
public void Init(string storageName, EncryptionSettings encryptionSettings) { }
}

13
common/ASC.Data.Storage/Encryption/IEncryptionService.cs
View File

@ -23,12 +23,11 @@
*
*/
namespace ASC.Data.Storage.Encryption
{
public interface IEncryptionService
{
void Start(EncryptionSettingsProto encryptionSettingsProto);
namespace ASC.Data.Storage.Encryption;
void Stop();
}
public interface IEncryptionService
{
void Start(EncryptionSettingsProto encryptionSettingsProto);
void Stop();
}

141
common/ASC.Data.Storage/Encryption/NotifyHelper.cs
View File

@ -23,75 +23,76 @@
*
*/
namespace ASC.Data.Storage.Encryption
namespace ASC.Data.Storage.Encryption;
[Scope]
public class NotifyHelper
{
[Scope]
public class NotifyHelper
{
private const string NotifyService = "ASC.Web.Studio.Core.Notify.StudioNotifyService, ASC.Web.Core";
private string ServerRootPath { get; set; }
private NotifyServiceClient NotifyServiceClient { get; set; }
private ILog Log { get; set; }
public NotifyHelper(IOptionsMonitor<ILog> option, NotifyServiceClient notifyServiceClient)
{
NotifyServiceClient = notifyServiceClient;
Log = option.CurrentValue;
}
public void Init(string serverRootPath)
{
ServerRootPath = serverRootPath;
}
public void SendStorageEncryptionStart(int tenantId)
{
SendStorageEncryptionNotification("SendStorageEncryptionStart", tenantId);
}
public void SendStorageEncryptionSuccess(int tenantId)
{
SendStorageEncryptionNotification("SendStorageEncryptionSuccess", tenantId);
}
public void SendStorageEncryptionError(int tenantId)
{
SendStorageEncryptionNotification("SendStorageEncryptionError", tenantId);
}
public void SendStorageDecryptionStart(int tenantId)
{
SendStorageEncryptionNotification("SendStorageDecryptionStart", tenantId);
}
public void SendStorageDecryptionSuccess(int tenantId)
{
SendStorageEncryptionNotification("SendStorageDecryptionSuccess", tenantId);
}
public void SendStorageDecryptionError(int tenantId)
{
SendStorageEncryptionNotification("SendStorageDecryptionError", tenantId);
}
private void SendStorageEncryptionNotification(string method, int tenantId)
{
var notifyInvoke = new NotifyInvoke()
{
Service = NotifyService,
Method = method,
Tenant = tenantId
};
notifyInvoke.Parameters.Add(ServerRootPath);
try
{
NotifyServiceClient.InvokeSendMethod(notifyInvoke);
}
catch (Exception error)
{
Log.Warn("Error while sending notification", error);
}
}
}
private const string NotifyService = "ASC.Web.Studio.Core.Notify.StudioNotifyService, ASC.Web.Core";
private string _serverRootPath;
private readonly NotifyServiceClient _notifyServiceClient;
private readonly ILog _logger;
public NotifyHelper(IOptionsMonitor<ILog> option, NotifyServiceClient notifyServiceClient)
{
_notifyServiceClient = notifyServiceClient;
_logger = option.CurrentValue;
}
public void Init(string serverRootPath)
{
_serverRootPath = serverRootPath;
}
public void SendStorageEncryptionStart(int tenantId)
{
SendStorageEncryptionNotification("SendStorageEncryptionStart", tenantId);
}
public void SendStorageEncryptionSuccess(int tenantId)
{
SendStorageEncryptionNotification("SendStorageEncryptionSuccess", tenantId);
}
public void SendStorageEncryptionError(int tenantId)
{
SendStorageEncryptionNotification("SendStorageEncryptionError", tenantId);
}
public void SendStorageDecryptionStart(int tenantId)
{
SendStorageEncryptionNotification("SendStorageDecryptionStart", tenantId);
}
public void SendStorageDecryptionSuccess(int tenantId)
{
SendStorageEncryptionNotification("SendStorageDecryptionSuccess", tenantId);
}
public void SendStorageDecryptionError(int tenantId)
{
SendStorageEncryptionNotification("SendStorageDecryptionError", tenantId);
}
private void SendStorageEncryptionNotification(string method, int tenantId)
{
var notifyInvoke = new NotifyInvoke()
{
Service = NotifyService,
Method = method,
Tenant = tenantId
};
notifyInvoke.Parameters.Add(_serverRootPath);
try
{
_notifyServiceClient.InvokeSendMethod(notifyInvoke);
}
catch (Exception error)
{
_logger.Warn("Error while sending notification", error);
}
}
}

78
common/ASC.Data.Storage/Extensions.cs
View File

@ -23,52 +23,60 @@
*
*/
namespace ASC.Data.Storage
{
public static class Extensions
{
private const int BufferSize = 2048;//NOTE: set to 2048 to fit in minimum tcp window
namespace ASC.Data.Storage;
public static Stream IronReadStream(this IDataStore store, TempStream tempStream, string domain, string path, int tryCount)
public static class Extensions
{
private const int BufferSize = 2048;//NOTE: set to 2048 to fit in minimum tcp window
public static Stream IronReadStream(this IDataStore store, TempStream tempStream, string domain, string path, int tryCount)
{
var ms = tempStream.Create();
IronReadToStream(store, domain, path, tryCount, ms);
ms.Seek(0, SeekOrigin.Begin);
return ms;
}
public static void IronReadToStream(this IDataStore store, string domain, string path, int tryCount, Stream readTo)
{
if (tryCount < 1)
{
var ms = tempStream.Create();
IronReadToStream(store, domain, path, tryCount, ms);
ms.Seek(0, SeekOrigin.Begin);
return ms;
throw new ArgumentOutOfRangeException(nameof(tryCount), "Must be greater or equal 1.");
}
public static void IronReadToStream(this IDataStore store, string domain, string path, int tryCount, Stream readTo)
if (!readTo.CanWrite)
{
if (tryCount < 1) throw new ArgumentOutOfRangeException(nameof(tryCount), "Must be greater or equal 1.");
if (!readTo.CanWrite) throw new ArgumentException("stream cannot be written", nameof(readTo));
throw new ArgumentException("stream cannot be written", nameof(readTo));
}
var tryCurrent = 0;
var offset = 0;
var tryCurrent = 0;
var offset = 0;
while (tryCurrent < tryCount)
while (tryCurrent < tryCount)
{
try
{
try
{
tryCurrent++;
using var stream = store.GetReadStream(domain, path, offset);
var buffer = new byte[BufferSize];
tryCurrent++;
using var stream = store.GetReadStream(domain, path, offset);
var buffer = new byte[BufferSize];
int readed;
while ((readed = stream.Read(buffer, 0, BufferSize)) > 0)
{
readTo.Write(buffer, 0, readed);
offset += readed;
}
break;
}
catch (Exception ex)
while ((readed = stream.Read(buffer, 0, BufferSize)) > 0)
{
if (tryCurrent >= tryCount)
{
throw new IOException("Can not read stream. Tries count: " + tryCurrent + ".", ex);
}
Thread.Sleep(tryCount * 50);
readTo.Write(buffer, 0, readed);
offset += readed;
}
break;
}
catch (Exception ex)
{
if (tryCurrent >= tryCount)
{
throw new IOException("Can not read stream. Tries count: " + tryCurrent + ".", ex);
}
Thread.Sleep(tryCount * 50);
}
}
}
}
}

1513
common/ASC.Data.Storage/GoogleCloud/GoogleCloudStorage.cs
View File

File diff suppressed because it is too large Load Diff

485
common/ASC.Data.Storage/IDataStore.cs
View File

@ -23,296 +23,295 @@
*
*/
namespace ASC.Data.Storage
namespace ASC.Data.Storage;
///<summary>
/// Interface for working with files
///</summary>
public interface IDataStore
{
IQuotaController QuotaController { get; set; }
TimeSpan GetExpire(string domain);
///<summary>
/// Interface for working with files
/// Get absolute Uri for html links to handler
///</summary>
public interface IDataStore
{
IQuotaController QuotaController { get; set; }
///<param name="path"></param>
///<returns></returns>
Uri GetUri(string path);
TimeSpan GetExpire(string domain);
///<summary>
/// Get absolute Uri for html links to handler
///</summary>
///<param name="domain"></param>
///<param name="path"></param>
///<returns></returns>
Uri GetUri(string domain, string path);
///<summary>
/// Get absolute Uri for html links to handler
///</summary>
///<param name="path"></param>
///<returns></returns>
Uri GetUri(string path);
/// <summary>
/// Get absolute Uri for html links to handler
/// </summary>
/// <param name="domain"></param>
/// <param name="path"></param>
/// <param name="expire"></param>
/// <param name="headers"></param>
/// <returns></returns>
Uri GetPreSignedUri(string domain, string path, TimeSpan expire, IEnumerable<string> headers);
///<summary>
/// Get absolute Uri for html links to handler
///</summary>
///<param name="domain"></param>
///<param name="path"></param>
///<returns></returns>
Uri GetUri(string domain, string path);
///<summary>
/// Supporting generate uri to the file
///</summary>
///<returns></returns>
bool IsSupportInternalUri { get; }
/// <summary>
/// Get absolute Uri for html links to handler
/// </summary>
/// <param name="domain"></param>
/// <param name="path"></param>
/// <param name="expire"></param>
/// <param name="headers"></param>
/// <returns></returns>
Uri GetPreSignedUri(string domain, string path, TimeSpan expire, IEnumerable<string> headers);
/// <summary>
/// Get absolute Uri for html links
/// </summary>
/// <param name="domain"></param>
/// <param name="path"></param>
/// <param name="expire"></param>
/// <param name="headers"></param>
/// <returns></returns>
Uri GetInternalUri(string domain, string path, TimeSpan expire, IEnumerable<string> headers);
///<summary>
/// Supporting generate uri to the file
///</summary>
///<returns></returns>
bool IsSupportInternalUri { get; }
///<summary>
/// A stream of read-only. In the case of the C3 stream NetworkStream general, and with him we have to work
/// Very carefully as a Jedi cutter groin lightsaber.
///</summary>
///<param name="domain"></param>
///<param name="path"></param>
///<returns></returns>
Stream GetReadStream(string domain, string path);
/// <summary>
/// Get absolute Uri for html links
/// </summary>
/// <param name="domain"></param>
/// <param name="path"></param>
/// <param name="expire"></param>
/// <param name="headers"></param>
/// <returns></returns>
Uri GetInternalUri(string domain, string path, TimeSpan expire, IEnumerable<string> headers);
Task<Stream> GetReadStreamAsync(string domain, string path, int offset);
///<summary>
/// A stream of read-only. In the case of the C3 stream NetworkStream general, and with him we have to work
/// Very carefully as a Jedi cutter groin lightsaber.
///</summary>
///<param name="domain"></param>
///<param name="path"></param>
///<returns></returns>
Stream GetReadStream(string domain, string path);
///<summary>
/// A stream of read-only. In the case of the C3 stream NetworkStream general, and with him we have to work
/// Very carefully as a Jedi cutter groin lightsaber.
///</summary>
///<param name="domain"></param>
///<param name="path"></param>
///<returns></returns>
Stream GetReadStream(string domain, string path, int offset);
Task<Stream> GetReadStreamAsync(string domain, string path, int offset);
///<summary>
/// Saves the contents of the stream in the repository.
///</ Summary>
/// <param Name="domain"> </param>
/// <param Name="path"> </param>
/// <param Name="stream"> flow. Is read from the current position! Desirable to set to 0 when the transmission MemoryStream instance </param>
/// <returns> </Returns>
Uri Save(string domain, string path, Stream stream);
///<summary>
/// A stream of read-only. In the case of the C3 stream NetworkStream general, and with him we have to work
/// Very carefully as a Jedi cutter groin lightsaber.
///</summary>
///<param name="domain"></param>
///<param name="path"></param>
///<returns></returns>
Stream GetReadStream(string domain, string path, int offset);
/// <summary>
/// Saves the contents of the stream in the repository.
/// </summary>
/// <param name="domain"></param>
/// <param name="path"></param>
/// <param name="stream"></param>
/// <param name="acl"></param>
/// <returns></returns>
Uri Save(string domain, string path, Stream stream, ACL acl);
///<summary>
/// Saves the contents of the stream in the repository.
///</ Summary>
/// <param Name="domain"> </param>
/// <param Name="path"> </param>
/// <param Name="stream"> flow. Is read from the current position! Desirable to set to 0 when the transmission MemoryStream instance </param>
/// <returns> </Returns>
Uri Save(string domain, string path, Stream stream);
/// <summary>
/// Saves the contents of the stream in the repository.
/// </summary>
/// <param name="domain"></param>
/// <param name="path"></param>
/// <param name="stream"></param>
/// <param name="attachmentFileName"></param>
/// <returns></returns>
Uri Save(string domain, string path, Stream stream, string attachmentFileName);
/// <summary>
/// Saves the contents of the stream in the repository.
/// </summary>
/// <param name="domain"></param>
/// <param name="path"></param>
/// <param name="stream"></param>
/// <param name="acl"></param>
/// <returns></returns>
Uri Save(string domain, string path, Stream stream, ACL acl);
/// <summary>
/// Saves the contents of the stream in the repository.
/// </summary>
/// <param name="domain"></param>
/// <param name="path"></param>
/// <param name="stream"></param>
/// <param name="contentType"></param>
/// <param name="contentDisposition"></param>
/// <returns></returns>
Uri Save(string domain, string path, Stream stream, string contentType, string contentDisposition);
/// <summary>
/// Saves the contents of the stream in the repository.
/// </summary>
/// <param name="domain"></param>
/// <param name="path"></param>
/// <param name="stream"></param>
/// <param name="attachmentFileName"></param>
/// <returns></returns>
Uri Save(string domain, string path, Stream stream, string attachmentFileName);
/// <summary>
/// Saves the contents of the stream in the repository.
/// </summary>
/// <param name="domain"></param>
/// <param name="path"></param>
/// <param name="stream"></param>
/// <param name="contentEncoding"></param>
/// <param name="cacheDays"></param>
/// <returns></returns>
Uri Save(string domain, string path, Stream stream, string contentEncoding, int cacheDays);
/// <summary>
/// Saves the contents of the stream in the repository.
/// </summary>
/// <param name="domain"></param>
/// <param name="path"></param>
/// <param name="stream"></param>
/// <param name="contentType"></param>
/// <param name="contentDisposition"></param>
/// <returns></returns>
Uri Save(string domain, string path, Stream stream, string contentType, string contentDisposition);
string InitiateChunkedUpload(string domain, string path);
/// <summary>
/// Saves the contents of the stream in the repository.
/// </summary>
/// <param name="domain"></param>
/// <param name="path"></param>
/// <param name="stream"></param>
/// <param name="contentEncoding"></param>
/// <param name="cacheDays"></param>
/// <returns></returns>
Uri Save(string domain, string path, Stream stream, string contentEncoding, int cacheDays);
string UploadChunk(string domain, string path, string uploadId, Stream stream, long defaultChunkSize, int chunkNumber, long chunkLength);
string InitiateChunkedUpload(string domain, string path);
Uri FinalizeChunkedUpload(string domain, string path, string uploadId, Dictionary<int, string> eTags);
string UploadChunk(string domain, string path, string uploadId, Stream stream, long defaultChunkSize, int chunkNumber, long chunkLength);
void AbortChunkedUpload(string domain, string path, string uploadId);
Uri FinalizeChunkedUpload(string domain, string path, string uploadId, Dictionary<int, string> eTags);
bool IsSupportChunking { get; }
void AbortChunkedUpload(string domain, string path, string uploadId);
bool IsSupportedPreSignedUri { get; }
bool IsSupportChunking { get; }
///<summary>
/// Deletes file
///</summary>
///<param name="domain"></param>
///<param name="path"></param>
void Delete(string domain, string path);
bool IsSupportedPreSignedUri { get; }
///<summary>
/// Deletes file by mask
///</summary>
///<param name="domain"></param>
///<param name="folderPath"></param>
///<param name="pattern">Wildcard mask (*.png)</param>
///<param name="recursive"></param>
void DeleteFiles(string domain, string folderPath, string pattern, bool recursive);
///<summary>
/// Deletes file
///</summary>
///<param name="domain"></param>
///<param name="path"></param>
void Delete(string domain, string path);
///<summary>
/// Deletes files
///</summary>
///<param name="domain"></param>
///<param name="listPaths"></param>
void DeleteFiles(string domain, List<string> paths);
///<summary>
/// Deletes file by mask
///</summary>
///<param name="domain"></param>
///<param name="folderPath"></param>
///<param name="pattern">Wildcard mask (*.png)</param>
///<param name="recursive"></param>
void DeleteFiles(string domain, string folderPath, string pattern, bool recursive);
///<summary>
/// Deletes file by last modified date
///</summary>
///<param name="domain"></param>
///<param name="folderPath"></param>
///<param name="fromDate"></param>
///<param name="toDate"></param>
void DeleteFiles(string domain, string folderPath, DateTime fromDate, DateTime toDate);
///<summary>
/// Deletes files
///</summary>
///<param name="domain"></param>
///<param name="listPaths"></param>
void DeleteFiles(string domain, List<string> paths);
///<summary>
/// Moves the contents of one directory to another. s3 for a very expensive procedure.
///</summary>
///<param name="srcdomain"></param>
///<param name="srcdir"></param>
///<param name="newdomain"></param>
///<param name="newdir"></param>
void MoveDirectory(string srcdomain, string srcdir, string newdomain, string newdir);
///<summary>
/// Deletes file by last modified date
///</summary>
///<param name="domain"></param>
///<param name="folderPath"></param>
///<param name="fromDate"></param>
///<param name="toDate"></param>
void DeleteFiles(string domain, string folderPath, DateTime fromDate, DateTime toDate);
///<summary>
/// Moves file
///</summary>
///<param name="srcdomain"></param>
///<param name="srcpath"></param>
///<param name="newdomain"></param>
///<param name="newpath"></param>
///<returns></returns>
Uri Move(string srcdomain, string srcpath, string newdomain, string newpath, bool quotaCheckFileSize = true);
///<summary>
/// Moves the contents of one directory to another. s3 for a very expensive procedure.
///</summary>
///<param name="srcdomain"></param>
///<param name="srcdir"></param>
///<param name="newdomain"></param>
///<param name="newdir"></param>
void MoveDirectory(string srcdomain, string srcdir, string newdomain, string newdir);
///<summary>
/// Saves the file in the temp. In fact, almost no different from the usual Save except that generates the file name itself. An inconvenient thing.
///</summary>
///<param name="domain"></param>
///<param name="assignedPath"></param>
///<param name="stream"></param>
///<returns></returns>
Uri SaveTemp(string domain, out string assignedPath, Stream stream);
///<summary>
/// Moves file
///</summary>
///<param name="srcdomain"></param>
///<param name="srcpath"></param>
///<param name="newdomain"></param>
///<param name="newpath"></param>
///<returns></returns>
Uri Move(string srcdomain, string srcpath, string newdomain, string newpath, bool quotaCheckFileSize = true);
/// <summary>
/// Returns a list of links to all subfolders
/// </summary>
/// <param name="domain"></param>
/// <param name="path"></param>
/// <param name="recursive">iterate subdirectories or not</param>
/// <returns></returns>
string[] ListDirectoriesRelative(string domain, string path, bool recursive);
///<summary>
/// Saves the file in the temp. In fact, almost no different from the usual Save except that generates the file name itself. An inconvenient thing.
///</summary>
///<param name="domain"></param>
///<param name="assignedPath"></param>
///<param name="stream"></param>
///<returns></returns>
Uri SaveTemp(string domain, out string assignedPath, Stream stream);
///<summary>
/// Returns a list of links to all files
///</summary>
///<param name="domain"></param>
///<param name="path"></param>
///<param name="pattern">Wildcard mask (*. jpg for example)</param>
///<param name="recursive">iterate subdirectories or not</param>
///<returns></returns>
Uri[] ListFiles(string domain, string path, string pattern, bool recursive);
/// <summary>
/// Returns a list of links to all subfolders
/// </summary>
/// <param name="domain"></param>
/// <param name="path"></param>
/// <param name="recursive">iterate subdirectories or not</param>
/// <returns></returns>
string[] ListDirectoriesRelative(string domain, string path, bool recursive);
///<summary>
/// Returns a list of relative paths for all files
///</summary>
///<param name="domain"></param>
///<param name="path"></param>
///<param name="pattern">Wildcard mask (*. jpg for example)</param>
///<param name="recursive">iterate subdirectories or not</param>
///<returns></returns>
string[] ListFilesRelative(string domain, string path, string pattern, bool recursive);
///<summary>
/// Returns a list of links to all files
///</summary>
///<param name="domain"></param>
///<param name="path"></param>
///<param name="pattern">Wildcard mask (*. jpg for example)</param>
///<param name="recursive">iterate subdirectories or not</param>
///<returns></returns>
Uri[] ListFiles(string domain, string path, string pattern, bool recursive);
///<summary>
/// Checks whether a file exists. On s3 it took long time.
///</summary>
///<param name="domain"></param>
///<param name="path"></param>
///<returns></returns>
bool IsFile(string domain, string path);
///<summary>
/// Returns a list of relative paths for all files
///</summary>
///<param name="domain"></param>
///<param name="path"></param>
///<param name="pattern">Wildcard mask (*. jpg for example)</param>
///<param name="recursive">iterate subdirectories or not</param>
///<returns></returns>
string[] ListFilesRelative(string domain, string path, string pattern, bool recursive);
Task<bool> IsFileAsync(string domain, string path);
///<summary>
/// Checks whether a file exists. On s3 it took long time.
///</summary>
///<param name="domain"></param>
///<param name="path"></param>
///<returns></returns>
bool IsFile(string domain, string path);
///<summary>
/// Checks whether a directory exists. On s3 it took long time.
///</summary>
///<param name="domain"></param>
///<param name="path"></param>
///<returns></returns>
bool IsDirectory(string domain, string path);
Task<bool> IsFileAsync(string domain, string path);
void DeleteDirectory(string domain, string path);
///<summary>
/// Checks whether a directory exists. On s3 it took long time.
///</summary>
///<param name="domain"></param>
///<param name="path"></param>
///<returns></returns>
bool IsDirectory(string domain, string path);
long GetFileSize(string domain, string path);
void DeleteDirectory(string domain, string path);
long GetDirectorySize(string domain, string path);
long GetFileSize(string domain, string path);
long ResetQuota(string domain);
long GetDirectorySize(string domain, string path);
long GetUsedQuota(string domain);
long ResetQuota(string domain);
Uri Copy(string srcdomain, string path, string newdomain, string newpath);
void CopyDirectory(string srcdomain, string dir, string newdomain, string newdir);
long GetUsedQuota(string domain);
Uri Copy(string srcdomain, string path, string newdomain, string newpath);
void CopyDirectory(string srcdomain, string dir, string newdomain, string newdir);
//Then there are restarted methods without domain. functionally identical to the top
//Then there are restarted methods without domain. functionally identical to the top
#pragma warning disable 1591
Stream GetReadStream(string path);
Uri Save(string path, Stream stream, string attachmentFileName);
Uri Save(string path, Stream stream);
void Delete(string path);
void DeleteFiles(string folderPath, string pattern, bool recursive);
Uri Move(string srcpath, string newdomain, string newpath);
Uri SaveTemp(out string assignedPath, Stream stream);
string[] ListDirectoriesRelative(string path, bool recursive);
Uri[] ListFiles(string path, string pattern, bool recursive);
bool IsFile(string path);
bool IsDirectory(string path);
void DeleteDirectory(string path);
long GetFileSize(string path);
long GetDirectorySize(string path);
Uri Copy(string path, string newdomain, string newpath);
void CopyDirectory(string dir, string newdomain, string newdir);
Stream GetReadStream(string path);
Uri Save(string path, Stream stream, string attachmentFileName);
Uri Save(string path, Stream stream);
void Delete(string path);
void DeleteFiles(string folderPath, string pattern, bool recursive);
Uri Move(string srcpath, string newdomain, string newpath);
Uri SaveTemp(out string assignedPath, Stream stream);
string[] ListDirectoriesRelative(string path, bool recursive);
Uri[] ListFiles(string path, string pattern, bool recursive);
bool IsFile(string path);
bool IsDirectory(string path);
void DeleteDirectory(string path);
long GetFileSize(string path);
long GetDirectorySize(string path);
Uri Copy(string path, string newdomain, string newpath);
void CopyDirectory(string dir, string newdomain, string newdir);
#pragma warning restore 1591
IDataStore Configure(string tenant, Handler handlerConfig, Module moduleConfig, IDictionary<string, string> props);
IDataStore SetQuotaController(IQuotaController controller);
IDataStore Configure(string tenant, Handler handlerConfig, Module moduleConfig, IDictionary<string, string> props);
IDataStore SetQuotaController(IQuotaController controller);
string SavePrivate(string domain, string path, Stream stream, DateTime expires);
void DeleteExpired(string domain, string path, TimeSpan oldThreshold);
string SavePrivate(string domain, string path, Stream stream, DateTime expires);
void DeleteExpired(string domain, string path, TimeSpan oldThreshold);
string GetUploadForm(string domain, string directoryPath, string redirectTo, long maxUploadSize,
string contentType, string contentDisposition, string submitLabel);
string GetUploadForm(string domain, string directoryPath, string redirectTo, long maxUploadSize,
string contentType, string contentDisposition, string submitLabel);
string GetUploadedUrl(string domain, string directoryPath);
string GetUploadUrl();
string GetUploadedUrl(string domain, string directoryPath);
string GetUploadUrl();
string GetPostParams(string domain, string directoryPath, long maxUploadSize, string contentType,
string contentDisposition);
}
}
string GetPostParams(string domain, string directoryPath, long maxUploadSize, string contentType,
string contentDisposition);
}

19
common/ASC.Data.Storage/IQuotaController.cs
View File

@ -24,17 +24,16 @@
*/
namespace ASC.Data.Storage
namespace ASC.Data.Storage;
public interface IQuotaController
{
public interface IQuotaController
{
//quotaCheckFileSize:hack for Backup bug 48873
void QuotaUsedAdd(string module, string domain, string dataTag, long size, bool quotaCheckFileSize = true);
//quotaCheckFileSize:hack for Backup bug 48873
void QuotaUsedAdd(string module, string domain, string dataTag, long size, bool quotaCheckFileSize = true);
void QuotaUsedDelete(string module, string domain, string dataTag, long size);
void QuotaUsedDelete(string module, string domain, string dataTag, long size);
void QuotaUsedSet(string module, string domain, string dataTag, long size);
void QuotaUsedSet(string module, string domain, string dataTag, long size);
void QuotaUsedCheck(long size);
}
}
void QuotaUsedCheck(long size);
}

25
common/ASC.Data.Storage/Migration/IService.cs
View File

@ -23,21 +23,20 @@
*
*/
namespace ASC.Data.Storage.Migration
namespace ASC.Data.Storage.Migration;
[ServiceContract]
public interface IService
{
[ServiceContract]
public interface IService
{
[OperationContract]
void Migrate(int tenant, StorageSettings storageSettings);
[OperationContract]
void Migrate(int tenant, StorageSettings storageSettings);
[OperationContract]
double GetProgress(int tenant);
[OperationContract]
double GetProgress(int tenant);
[OperationContract]
void StopMigrate();
[OperationContract]
void StopMigrate();
[OperationContract]
void UploadCdn(int tenant, string relativePath, string mappedPath, CdnStorageSettings cdnStorageSettings = null);
}
[OperationContract]
void UploadCdn(int tenant, string relativePath, string mappedPath, CdnStorageSettings cdnStorageSettings = null);
}

205
common/ASC.Data.Storage/Migration/MigrateServiceClient.cs
View File

@ -23,111 +23,110 @@
*
*/
namespace ASC.Data.Storage.Migration
{
[Singletone]
public class ServiceClientListener
{
private ICacheNotify<MigrationProgress> ProgressMigrationNotify { get; }
private IServiceProvider ServiceProvider { get; }
private ICache Cache { get; }
public ServiceClientListener(
ICacheNotify<MigrationProgress> progressMigrationNotify,
IServiceProvider serviceProvider,
ICache cache)
{
ProgressMigrationNotify = progressMigrationNotify;
ServiceProvider = serviceProvider;
Cache = cache;
ProgressListening();
}
public MigrationProgress GetProgress(int tenantId)
{
return Cache.Get<MigrationProgress>(GetCacheKey(tenantId));
}
private void ProgressListening()
{
ProgressMigrationNotify.Subscribe(n =>
{
var migrationProgress = new MigrationProgress
{
TenantId = n.TenantId,
Progress = n.Progress,
IsCompleted = n.IsCompleted,
Error = n.Error
};
Cache.Insert(GetCacheKey(n.TenantId), migrationProgress, DateTime.MaxValue);
},
Common.Caching.CacheNotifyAction.Insert);
}
private string GetCacheKey(int tenantId)
{
return typeof(MigrationProgress).FullName + tenantId;
}
}
namespace ASC.Data.Storage.Migration;
[Scope]
public class ServiceClient : IService
{
public ServiceClientListener ServiceClientListener { get; }
public ICacheNotify<MigrationCache> CacheMigrationNotify { get; }
public ICacheNotify<MigrationUploadCdn> UploadCdnMigrationNotify { get; }
public IServiceProvider ServiceProvider { get; }
public ServiceClient(
ServiceClientListener serviceClientListener,
ICacheNotify<MigrationCache> cacheMigrationNotify,
ICacheNotify<MigrationUploadCdn> uploadCdnMigrationNotify,
IServiceProvider serviceProvider)
{
ServiceClientListener = serviceClientListener;
CacheMigrationNotify = cacheMigrationNotify;
UploadCdnMigrationNotify = uploadCdnMigrationNotify;
ServiceProvider = serviceProvider;
}
public void Migrate(int tenant, StorageSettings storageSettings)
{
var storSettings = new StorSettings { Id = storageSettings.ID.ToString(), Module = storageSettings.Module };
CacheMigrationNotify.Publish(new MigrationCache
{
TenantId = tenant,
StorSettings = storSettings
},
[Singletone]
public class ServiceClientListener
{
private readonly ICacheNotify<MigrationProgress> _progressMigrationNotify;
private readonly IServiceProvider _serviceProvider;
private readonly ICache _cache;
public ServiceClientListener(
ICacheNotify<MigrationProgress> progressMigrationNotify,
IServiceProvider serviceProvider,
ICache cache)
{
_progressMigrationNotify = progressMigrationNotify;
_serviceProvider = serviceProvider;
_cache = cache;
ProgressListening();
}
public MigrationProgress GetProgress(int tenantId)
{
return _cache.Get<MigrationProgress>(GetCacheKey(tenantId));
}
private void ProgressListening()
{
_progressMigrationNotify.Subscribe(n =>
{
var migrationProgress = new MigrationProgress
{
TenantId = n.TenantId,
Progress = n.Progress,
IsCompleted = n.IsCompleted,
Error = n.Error
};
_cache.Insert(GetCacheKey(n.TenantId), migrationProgress, DateTime.MaxValue);
},
Common.Caching.CacheNotifyAction.Insert);
}
private string GetCacheKey(int tenantId)
{
return typeof(MigrationProgress).FullName + tenantId;
}
}
[Scope]
public class ServiceClient : IService
{
public ServiceClientListener ServiceClientListener { get; }
public ICacheNotify<MigrationCache> CacheMigrationNotify { get; }
public ICacheNotify<MigrationUploadCdn> UploadCdnMigrationNotify { get; }
public IServiceProvider ServiceProvider { get; }
public ServiceClient(
ServiceClientListener serviceClientListener,
ICacheNotify<MigrationCache> cacheMigrationNotify,
ICacheNotify<MigrationUploadCdn> uploadCdnMigrationNotify,
IServiceProvider serviceProvider)
{
ServiceClientListener = serviceClientListener;
CacheMigrationNotify = cacheMigrationNotify;
UploadCdnMigrationNotify = uploadCdnMigrationNotify;
ServiceProvider = serviceProvider;
}
public void Migrate(int tenant, StorageSettings storageSettings)
{
var storSettings = new StorSettings { Id = storageSettings.ID.ToString(), Module = storageSettings.Module };
CacheMigrationNotify.Publish(new MigrationCache
{
TenantId = tenant,
StorSettings = storSettings
},
Common.Caching.CacheNotifyAction.Insert);
}
public void UploadCdn(int tenantId, string relativePath, string mappedPath, CdnStorageSettings settings = null)
{
var cdnStorSettings = new CdnStorSettings { Id = settings.ID.ToString(), Module = settings.Module };
UploadCdnMigrationNotify.Publish(new MigrationUploadCdn
{
Tenant = tenantId,
RelativePath = relativePath,
MappedPath = mappedPath,
CdnStorSettings = cdnStorSettings
},
}
public void UploadCdn(int tenantId, string relativePath, string mappedPath, CdnStorageSettings settings = null)
{
var cdnStorSettings = new CdnStorSettings { Id = settings.ID.ToString(), Module = settings.Module };
UploadCdnMigrationNotify.Publish(new MigrationUploadCdn
{
Tenant = tenantId,
RelativePath = relativePath,
MappedPath = mappedPath,
CdnStorSettings = cdnStorSettings
},
Common.Caching.CacheNotifyAction.Insert);
}
public double GetProgress(int tenant)
{
var migrationProgress = ServiceClientListener.GetProgress(tenant);
return migrationProgress.Progress;
}
public void StopMigrate()
{
}
public double GetProgress(int tenant)
{
var migrationProgress = ServiceClientListener.GetProgress(tenant);
return migrationProgress.Progress;
}
public void StopMigrate()
{
CacheMigrationNotify.Publish(new MigrationCache(), Common.Caching.CacheNotifyAction.InsertOrUpdate);
}
}
}
}

138
common/ASC.Data.Storage/PathUtils.cs
View File

@ -23,86 +23,94 @@
*
*/
namespace ASC.Data.Storage
namespace ASC.Data.Storage;
[Singletone]
public class PathUtils
{
[Singletone]
public class PathUtils
public IHostEnvironment HostEnvironment { get; }
private readonly string _storageRoot;
private readonly IConfiguration _configuration;
private readonly IWebHostEnvironment _webHostEnvironment;
public PathUtils(IConfiguration configuration, IHostEnvironment hostEnvironment)
{
private string StorageRoot { get; }
private IConfiguration Configuration { get; }
public IHostEnvironment HostEnvironment { get; }
private IWebHostEnvironment WebHostEnvironment { get; }
_configuration = configuration;
HostEnvironment = hostEnvironment;
_storageRoot = _configuration[Constants.StorageRootParam];
}
public PathUtils(IConfiguration configuration, IHostEnvironment hostEnvironment)
{
Configuration = configuration;
HostEnvironment = hostEnvironment;
StorageRoot = Configuration[Constants.STORAGE_ROOT_PARAM];
}
public PathUtils(
IConfiguration configuration,
IHostEnvironment hostEnvironment,
IWebHostEnvironment webHostEnvironment) : this(configuration, hostEnvironment)
{
_webHostEnvironment = webHostEnvironment;
}
public PathUtils(IConfiguration configuration, IHostEnvironment hostEnvironment, IWebHostEnvironment webHostEnvironment) : this(configuration, hostEnvironment)
{
WebHostEnvironment = webHostEnvironment;
}
public static string Normalize(string path, bool addTailingSeparator = false)
{
path = path
.Replace('/', Path.DirectorySeparatorChar)
.Replace('\\', Path.DirectorySeparatorChar)
.Replace("\\\\", Path.DirectorySeparatorChar.ToString())
.Replace("//", Path.DirectorySeparatorChar.ToString())
.TrimEnd(Path.DirectorySeparatorChar);
public static string Normalize(string path, bool addTailingSeparator = false)
{
path = path
.Replace('/', Path.DirectorySeparatorChar)
.Replace('\\', Path.DirectorySeparatorChar)
.Replace("\\\\", Path.DirectorySeparatorChar.ToString())
.Replace("//", Path.DirectorySeparatorChar.ToString())
.TrimEnd(Path.DirectorySeparatorChar);
return addTailingSeparator && 0 < path.Length ? path + Path.DirectorySeparatorChar : path;
}
return addTailingSeparator && 0 < path.Length ? path + Path.DirectorySeparatorChar : path;
}
public string ResolveVirtualPath(string module, string domain)
{
public string ResolveVirtualPath(string module, string domain)
{
var url = $"~/storage/{module}/{(string.IsNullOrEmpty(domain) ? "root" : domain)}/";
return ResolveVirtualPath(url);
}
public string ResolveVirtualPath(string virtPath, bool addTrailingSlash = true)
return ResolveVirtualPath(url);
}
public string ResolveVirtualPath(string virtPath, bool addTrailingSlash = true)
{
if (virtPath == null)
{
if (virtPath == null)
{
virtPath = "";
}
if (virtPath.StartsWith('~') && !Uri.IsWellFormedUriString(virtPath, UriKind.Absolute))
{
var rootPath = "/";
if (!string.IsNullOrEmpty(WebHostEnvironment?.WebRootPath) && WebHostEnvironment?.WebRootPath.Length > 1)
{
rootPath = WebHostEnvironment?.WebRootPath.Trim('/');
}
virtPath = virtPath.Replace("~", rootPath);
}
if (addTrailingSlash)
{
virtPath += "/";
}
else
{
virtPath = virtPath.TrimEnd('/');
}
return virtPath.Replace("//", "/");
virtPath = "";
}
public string ResolvePhysicalPath(string physPath, IDictionary<string, string> storageConfig)
if (virtPath.StartsWith('~') && !Uri.IsWellFormedUriString(virtPath, UriKind.Absolute))
{
physPath = Normalize(physPath, false).TrimStart('~');
if (physPath.Contains(Constants.STORAGE_ROOT_PARAM))
var rootPath = "/";
if (!string.IsNullOrEmpty(_webHostEnvironment?.WebRootPath) && _webHostEnvironment?.WebRootPath.Length > 1)
{
physPath = physPath.Replace(Constants.STORAGE_ROOT_PARAM, StorageRoot ?? storageConfig[Constants.STORAGE_ROOT_PARAM]);
rootPath = _webHostEnvironment?.WebRootPath.Trim('/');
}
if (!Path.IsPathRooted(physPath))
{
physPath = Path.GetFullPath(CrossPlatform.PathCombine(HostEnvironment.ContentRootPath, physPath.Trim(Path.DirectorySeparatorChar)));
}
return physPath;
virtPath = virtPath.Replace("~", rootPath);
}
if (addTrailingSlash)
{
virtPath += "/";
}
else
{
virtPath = virtPath.TrimEnd('/');
}
return virtPath.Replace("//", "/");
}
public string ResolvePhysicalPath(string physPath, IDictionary<string, string> storageConfig)
{
physPath = Normalize(physPath, false).TrimStart('~');
if (physPath.Contains(Constants.StorageRootParam))
{
physPath = physPath.Replace(Constants.StorageRootParam, _storageRoot ?? storageConfig[Constants.StorageRootParam]);
}
if (!Path.IsPathRooted(physPath))
{
physPath = Path.GetFullPath(CrossPlatform.PathCombine(HostEnvironment.ContentRootPath, physPath.Trim(Path.DirectorySeparatorChar)));
}
return physPath;
}
}

137
common/ASC.Data.Storage/ProgressStream.cs
View File

@ -23,82 +23,67 @@
*
*/
namespace ASC.Data.Storage
namespace ASC.Data.Storage;
public class ProgressStream : Stream
{
public class ProgressStream : Stream
public override bool CanRead => _stream.CanRead;
public override bool CanSeek => _stream.CanSeek;
public override bool CanWrite => _stream.CanWrite;
public override long Length => _length;
public override long Position
{
private readonly Stream stream;
private long length = long.MaxValue;
public ProgressStream(Stream stream)
{
this.stream = stream ?? throw new ArgumentNullException(nameof(stream));
try
{
length = stream.Length;
}
catch (Exception) { }
}
public override bool CanRead
{
get { return stream.CanRead; }
}
public override bool CanSeek
{
get { return stream.CanSeek; }
}
public override bool CanWrite
{
get { return stream.CanWrite; }
}
public override long Length
{
get { return length; }
}
public override long Position
{
get { return stream.Position; }
set { stream.Position = value; }
}
public event Action<ProgressStream, int> OnReadProgress;
public void InvokeOnReadProgress(int progress)
{
OnReadProgress?.Invoke(this, progress);
}
public override void Flush()
{
stream.Flush();
}
public override long Seek(long offset, SeekOrigin origin)
{
return stream.Seek(offset, origin);
}
public override void SetLength(long value)
{
stream.SetLength(value);
length = value;
}
public override int Read(byte[] buffer, int offset, int count)
{
var readed = stream.Read(buffer, offset, count);
OnReadProgress(this, (int)(stream.Position / (double)length * 100));
return readed;
}
public override void Write(byte[] buffer, int offset, int count)
{
stream.Write(buffer, offset, count);
}
get => _stream.Position;
set => _stream.Position = value;
}
}
private readonly Stream _stream;
private long _length = long.MaxValue;
public ProgressStream(Stream stream)
{
_stream = stream ?? throw new ArgumentNullException(nameof(stream));
try
{
_length = stream.Length;
}
catch (Exception) { }
}
public event Action<ProgressStream, int> OnReadProgress;
public void InvokeOnReadProgress(int progress)
{
OnReadProgress?.Invoke(this, progress);
}
public override void Flush()
{
_stream.Flush();
}
public override long Seek(long offset, SeekOrigin origin)
{
return _stream.Seek(offset, origin);
}
public override void SetLength(long value)
{
_stream.SetLength(value);
_length = value;
}
public override int Read(byte[] buffer, int offset, int count)
{
var readed = _stream.Read(buffer, offset, count);
OnReadProgress(this, (int)(_stream.Position / (double)_length * 100));
return readed;
}
public override void Write(byte[] buffer, int offset, int count)
{
_stream.Write(buffer, offset, count);
}
}

1245
common/ASC.Data.Storage/RackspaceCloud/RackspaceCloudStorage.cs
View File

File diff suppressed because it is too large Load Diff

2193
common/ASC.Data.Storage/S3/S3Storage.cs
View File

File diff suppressed because it is too large Load Diff

192
common/ASC.Data.Storage/S3/S3UploadGuard.cs
View File

@ -23,117 +23,117 @@
*
*/
namespace ASC.Data.Storage.S3
namespace ASC.Data.Storage.S3;
public class S3UploadGuard
{
public class S3UploadGuard
public Configuration.Storage Storage { get; }
private readonly CoreSettings _coreSettings;
private string _accessKey;
private string _secretAccessKey;
private string _bucket;
private string _region;
private bool _configErrors;
private bool _configured;
public S3UploadGuard(CoreSettings coreSettings, Configuration.Storage storage)
{
private string accessKey;
private string secretAccessKey;
private string bucket;
private string region;
private bool configErrors;
private bool configured;
_coreSettings = coreSettings;
Storage = storage;
}
private CoreSettings CoreSettings { get; }
public Configuration.Storage Storage { get; }
public S3UploadGuard(CoreSettings coreSettings, Configuration.Storage storage)
public void DeleteExpiredUploadsAsync(TimeSpan trustInterval)
{
var task = new Task(() =>
{
CoreSettings = coreSettings;
Storage = storage;
DeleteExpiredUploads(trustInterval);
}, TaskCreationOptions.LongRunning);
task.Start();
}
private void DeleteExpiredUploads(TimeSpan trustInterval)
{
Configure();
if (_configErrors)
{
return;
}
public void DeleteExpiredUploadsAsync(TimeSpan trustInterval)
using var s3 = GetClient();
var nextKeyMarker = string.Empty;
var nextUploadIdMarker = string.Empty;
bool isTruncated;
do
{
var task = new Task(() =>
var request = new ListMultipartUploadsRequest { BucketName = _bucket };
if (!string.IsNullOrEmpty(nextKeyMarker))
{
DeleteExpiredUploads(trustInterval);
}, TaskCreationOptions.LongRunning);
task.Start();
}
private void DeleteExpiredUploads(TimeSpan trustInterval)
{
Configure();
if (configErrors)
{
return;
request.KeyMarker = nextKeyMarker;
}
using var s3 = GetClient();
var nextKeyMarker = string.Empty;
var nextUploadIdMarker = string.Empty;
bool isTruncated;
do
if (!string.IsNullOrEmpty(nextUploadIdMarker))
{
var request = new ListMultipartUploadsRequest { BucketName = bucket };
if (!string.IsNullOrEmpty(nextKeyMarker))
{
request.KeyMarker = nextKeyMarker;
}
if (!string.IsNullOrEmpty(nextUploadIdMarker))
{
request.UploadIdMarker = nextUploadIdMarker;
}
var response = s3.ListMultipartUploadsAsync(request).Result;
foreach (var u in response.MultipartUploads.Where(x => x.Initiated + trustInterval <= DateTime.UtcNow))
{
AbortMultipartUpload(u, s3);
}
isTruncated = response.IsTruncated;
nextKeyMarker = response.NextKeyMarker;
nextUploadIdMarker = response.NextUploadIdMarker;
request.UploadIdMarker = nextUploadIdMarker;
}
while (isTruncated);
}
private void AbortMultipartUpload(MultipartUpload u, AmazonS3Client client)
{
var request = new AbortMultipartUploadRequest
var response = s3.ListMultipartUploadsAsync(request).Result;
foreach (var u in response.MultipartUploads.Where(x => x.Initiated + trustInterval <= DateTime.UtcNow))
{
BucketName = bucket,
Key = u.Key,
UploadId = u.UploadId,
};
client.AbortMultipartUploadAsync(request).Wait();
}
private AmazonS3Client GetClient()
{
var s3Config = new AmazonS3Config { UseHttp = true, MaxErrorRetry = 3, RegionEndpoint = RegionEndpoint.GetBySystemName(region) };
return new AmazonS3Client(accessKey, secretAccessKey, s3Config);
}
private void Configure()
{
if (!configured)
{
var handler = Storage.GetHandler("s3");
if (handler != null)
{
var props = handler.GetProperties();
bucket = props["bucket"];
accessKey = props["acesskey"];
secretAccessKey = props["secretaccesskey"];
region = props["region"];
}
configErrors = string.IsNullOrEmpty(CoreSettings.BaseDomain) //localhost
|| string.IsNullOrEmpty(accessKey)
|| string.IsNullOrEmpty(secretAccessKey)
|| string.IsNullOrEmpty(bucket)
|| string.IsNullOrEmpty(region);
configured = true;
AbortMultipartUpload(u, s3);
}
isTruncated = response.IsTruncated;
nextKeyMarker = response.NextKeyMarker;
nextUploadIdMarker = response.NextUploadIdMarker;
}
while (isTruncated);
}
private void AbortMultipartUpload(MultipartUpload u, AmazonS3Client client)
{
var request = new AbortMultipartUploadRequest
{
BucketName = _bucket,
Key = u.Key,
UploadId = u.UploadId,
};
client.AbortMultipartUploadAsync(request).Wait();
}
private AmazonS3Client GetClient()
{
var s3Config = new AmazonS3Config { UseHttp = true, MaxErrorRetry = 3, RegionEndpoint = RegionEndpoint.GetBySystemName(_region) };
return new AmazonS3Client(_accessKey, _secretAccessKey, s3Config);
}
private void Configure()
{
if (!_configured)
{
var handler = Storage.GetHandler("s3");
if (handler != null)
{
var props = handler.GetProperties();
_bucket = props["bucket"];
_accessKey = props["acesskey"];
_secretAccessKey = props["secretaccesskey"];
_region = props["region"];
}
_configErrors = string.IsNullOrEmpty(_coreSettings.BaseDomain) //localhost
|| string.IsNullOrEmpty(_accessKey)
|| string.IsNullOrEmpty(_secretAccessKey)
|| string.IsNullOrEmpty(_bucket)
|| string.IsNullOrEmpty(_region);
_configured = true;
}
}
}

51
common/ASC.Data.Storage/S3/UnencodedUri.cs
View File

@ -23,38 +23,27 @@
*
*/
namespace ASC.Data.Storage.S3
namespace ASC.Data.Storage.S3;
public class UnencodedUri : Uri
{
public class UnencodedUri : Uri
public UnencodedUri(string uriString)
: base(uriString) { }
public UnencodedUri(string uriString, UriKind uriKind)
: base(uriString, uriKind) { }
public UnencodedUri(Uri baseUri, string relativeUri)
: base(baseUri, relativeUri) { }
public UnencodedUri(Uri baseUri, Uri relativeUri)
: base(baseUri, relativeUri) { }
protected UnencodedUri(SerializationInfo serializationInfo, StreamingContext streamingContext)
: base(serializationInfo, streamingContext) { }
public override string ToString()
{
public UnencodedUri(string uriString)
: base(uriString)
{
}
public UnencodedUri(string uriString, UriKind uriKind)
: base(uriString, uriKind)
{
}
public UnencodedUri(Uri baseUri, string relativeUri)
: base(baseUri, relativeUri)
{
}
public UnencodedUri(Uri baseUri, Uri relativeUri)
: base(baseUri, relativeUri)
{
}
protected UnencodedUri(SerializationInfo serializationInfo, StreamingContext streamingContext)
: base(serializationInfo, streamingContext)
{
}
public override string ToString()
{
return OriginalString;
}
return OriginalString;
}
}

26
common/ASC.Data.Storage/SecureHelper.cs
View File

@ -23,21 +23,21 @@
*
*/
namespace ASC.Data.Storage
namespace ASC.Data.Storage;
public static class SecureHelper
{
public static class SecureHelper
public static bool IsSecure(HttpContext httpContext, IOptionsMonitor<ILog> options)
{
public static bool IsSecure(HttpContext httpContext, IOptionsMonitor<ILog> options)
try
{
try
{
return httpContext != null && Uri.UriSchemeHttps.Equals(httpContext.Request.GetUrlRewriter().Scheme, StringComparison.OrdinalIgnoreCase);
}
catch (Exception err)
{
options.Get("ASC.Data.Storage.SecureHelper").Error(err);
return false;
}
return httpContext != null && Uri.UriSchemeHttps.Equals(httpContext.Request.GetUrlRewriter().Scheme, StringComparison.OrdinalIgnoreCase);
}
catch (Exception err)
{
options.Get("ASC.Data.Storage.SecureHelper").Error(err);
return false;
}
}
}
}

592
common/ASC.Data.Storage/StaticUploader.cs
View File

@ -23,292 +23,322 @@
*
*/
namespace ASC.Data.Storage
{
[Scope(Additional = typeof(StaticUploaderExtension))]
public class StaticUploader
{
private static readonly TaskScheduler Scheduler;
private static readonly CancellationTokenSource TokenSource;
private ICache Cache { get; set; }
private static readonly object Locker;
private IServiceProvider ServiceProvider { get; }
private TenantManager TenantManager { get; }
private SettingsManager SettingsManager { get; }
private StorageSettingsHelper StorageSettingsHelper { get; }
namespace ASC.Data.Storage;
protected readonly DistributedTaskQueue Queue;
static StaticUploader()
{
Scheduler = new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, 4).ConcurrentScheduler;
Locker = new object();
TokenSource = new CancellationTokenSource();
}
public StaticUploader(
IServiceProvider serviceProvider,
TenantManager tenantManager,
SettingsManager settingsManager,
StorageSettingsHelper storageSettingsHelper,
ICache cache,
DistributedTaskQueueOptionsManager options)
{
Cache = cache;
ServiceProvider = serviceProvider;
TenantManager = tenantManager;
SettingsManager = settingsManager;
StorageSettingsHelper = storageSettingsHelper;
Queue = options.Get<UploadOperationProgress>();
}
public string UploadFile(string relativePath, string mappedPath, Action<string> onComplete = null)
{
if (TokenSource.Token.IsCancellationRequested) return null;
if (!CanUpload()) return null;
if (!File.Exists(mappedPath)) return null;
var tenantId = TenantManager.GetCurrentTenant().TenantId;
UploadOperation uploadOperation;
var key = GetCacheKey(tenantId.ToString(), relativePath);
lock (Locker)
{
uploadOperation = Cache.Get<UploadOperation>(key);
if (uploadOperation != null)
{
return !string.IsNullOrEmpty(uploadOperation.Result) ? uploadOperation.Result : string.Empty;
}
uploadOperation = new UploadOperation(ServiceProvider, tenantId, relativePath, mappedPath);
Cache.Insert(key, uploadOperation, DateTime.MaxValue);
}
uploadOperation.DoJob();
onComplete?.Invoke(uploadOperation.Result);
return uploadOperation.Result;
}
public Task<string> UploadFileAsync(string relativePath, string mappedPath, Action<string> onComplete = null)
{
var tenantId = TenantManager.GetCurrentTenant().TenantId;
var task = new Task<string>(() =>
{
using var scope = ServiceProvider.CreateScope();
var scopeClass = scope.ServiceProvider.GetService<StaticUploaderScope>();
var (tenantManager, staticUploader, _, _, _) = scopeClass;
tenantManager.SetCurrentTenant(tenantId);
return staticUploader.UploadFile(relativePath, mappedPath, onComplete);
}, TaskCreationOptions.LongRunning);
task.ConfigureAwait(false);
task.Start(Scheduler);
return task;
}
public void UploadDir(string relativePath, string mappedPath)
{
if (!CanUpload()) return;
if (!Directory.Exists(mappedPath)) return;
var tenant = TenantManager.GetCurrentTenant();
var key = typeof(UploadOperationProgress).FullName + tenant.TenantId;
UploadOperationProgress uploadOperation;
lock (Locker)
{
uploadOperation = Queue.GetTask<UploadOperationProgress>(key);
if (uploadOperation != null) return;
uploadOperation = new UploadOperationProgress(ServiceProvider, key, tenant.TenantId, relativePath, mappedPath);
Queue.QueueTask(uploadOperation);
}
}
public bool CanUpload()
{
var current = StorageSettingsHelper.DataStoreConsumer(SettingsManager.Load<CdnStorageSettings>());
if (current == null || !current.IsSet || (string.IsNullOrEmpty(current["cnamessl"]) && string.IsNullOrEmpty(current["cname"])))
{
return false;
}
return true;
}
public static void Stop()
{
TokenSource.Cancel();
}
public UploadOperationProgress GetProgress(int tenantId)
{
lock (Locker)
{
var key = typeof(UploadOperationProgress).FullName + tenantId;
return Queue.GetTask<UploadOperationProgress>(key);
}
}
private static string GetCacheKey(string tenantId, string path)
{
return typeof(UploadOperation).FullName + tenantId + path;
}
}
public class UploadOperation
{
private readonly ILog Log;
private readonly int tenantId;
private readonly string path;
private readonly string mappedPath;
public string Result { get; private set; }
private IServiceProvider ServiceProvider { get; }
public UploadOperation(IServiceProvider serviceProvider, int tenantId, string path, string mappedPath)
{
ServiceProvider = serviceProvider;
Log = ServiceProvider.GetService<IOptionsMonitor<ILog>>().CurrentValue;
this.tenantId = tenantId;
this.path = path.TrimStart('/');
this.mappedPath = mappedPath;
Result = string.Empty;
}
public string DoJob()
{
try
{
using var scope = ServiceProvider.CreateScope();
var scopeClass = scope.ServiceProvider.GetService<StaticUploaderScope>();
var (tenantManager, _, securityContext, settingsManager, storageSettingsHelper) = scopeClass;
var tenant = tenantManager.GetTenant(tenantId);
tenantManager.SetCurrentTenant(tenant);
securityContext.AuthenticateMeWithoutCookie(tenant.OwnerId);
var dataStore = storageSettingsHelper.DataStore(settingsManager.Load<CdnStorageSettings>());
if (File.Exists(mappedPath))
{
if (!dataStore.IsFile(path))
{
using var stream = File.OpenRead(mappedPath);
dataStore.Save(path, stream);
}
Result = dataStore.GetInternalUri("", path, TimeSpan.Zero, null).AbsoluteUri.ToLower();
Log.DebugFormat("UploadFile {0}", Result);
return Result;
}
}
catch (Exception e)
{
Log.Error(e);
}
return null;
}
}
[Scope(Additional = typeof(StaticUploaderExtension))]
public class StaticUploader
{
protected readonly DistributedTaskQueue Queue;
private ICache _cache;
private static readonly TaskScheduler _scheduler;
private static readonly CancellationTokenSource _tokenSource;
private static readonly object _locker;
private readonly IServiceProvider _serviceProvider;
private readonly TenantManager _tenantManager;
private readonly SettingsManager _settingsManager;
private readonly StorageSettingsHelper _storageSettingsHelper;
[Transient]
public class UploadOperationProgress : DistributedTaskProgress
{
private readonly string relativePath;
private readonly string mappedPath;
private readonly IEnumerable<string> directoryFiles;
private IServiceProvider ServiceProvider { get; }
public int TenantId { get; }
static StaticUploader()
{
_scheduler = new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, 4).ConcurrentScheduler;
_locker = new object();
_tokenSource = new CancellationTokenSource();
}
public UploadOperationProgress(IServiceProvider serviceProvider, string key, int tenantId, string relativePath, string mappedPath)
public StaticUploader(
IServiceProvider serviceProvider,
TenantManager tenantManager,
SettingsManager settingsManager,
StorageSettingsHelper storageSettingsHelper,
ICache cache,
DistributedTaskQueueOptionsManager options)
{
_cache = cache;
_serviceProvider = serviceProvider;
_tenantManager = tenantManager;
_settingsManager = settingsManager;
_storageSettingsHelper = storageSettingsHelper;
Queue = options.Get<UploadOperationProgress>();
}
public string UploadFile(string relativePath, string mappedPath, Action<string> onComplete = null)
{
if (_tokenSource.Token.IsCancellationRequested)
{
ServiceProvider = serviceProvider;
Id = key;
Status = DistributedTaskStatus.Created;
TenantId = tenantId;
this.relativePath = relativePath;
this.mappedPath = mappedPath;
const string extensions = ".png|.jpeg|.jpg|.gif|.ico|.swf|.mp3|.ogg|.eot|.svg|.ttf|.woff|.woff2|.css|.less|.js";
var extensionsArray = extensions.Split('|');
directoryFiles = Directory.GetFiles(mappedPath, "*", SearchOption.AllDirectories)
.Where(r => extensionsArray.Contains(Path.GetExtension(r)))
.ToList();
StepCount = directoryFiles.Count();
}
protected override void DoJob()
{
using var scope = ServiceProvider.CreateScope();
var tenantManager = scope.ServiceProvider.GetService<TenantManager>();
var staticUploader = scope.ServiceProvider.GetService<StaticUploader>();
var tenant = tenantManager.GetTenant(TenantId);
tenantManager.SetCurrentTenant(tenant);
tenant.SetStatus(TenantStatus.Migrating);
tenantManager.SaveTenant(tenant);
PublishChanges();
foreach (var file in directoryFiles)
{
var filePath = file.Substring(mappedPath.TrimEnd('/').Length);
staticUploader.UploadFile(CrossPlatform.PathCombine(relativePath, filePath), file, (res) => StepDone());
}
tenant.SetStatus(Core.Tenants.TenantStatus.Active);
tenantManager.SaveTenant(tenant);
return null;
}
public object Clone()
{
return MemberwiseClone();
}
if (!CanUpload())
{
return null;
}
if (!File.Exists(mappedPath))
{
return null;
}
var tenantId = _tenantManager.GetCurrentTenant().TenantId;
UploadOperation uploadOperation;
var key = GetCacheKey(tenantId.ToString(), relativePath);
lock (_locker)
{
uploadOperation = _cache.Get<UploadOperation>(key);
if (uploadOperation != null)
{
return !string.IsNullOrEmpty(uploadOperation.Result) ? uploadOperation.Result : string.Empty;
}
uploadOperation = new UploadOperation(_serviceProvider, tenantId, relativePath, mappedPath);
_cache.Insert(key, uploadOperation, DateTime.MaxValue);
}
uploadOperation.DoJob();
onComplete?.Invoke(uploadOperation.Result);
return uploadOperation.Result;
}
public Task<string> UploadFileAsync(string relativePath, string mappedPath, Action<string> onComplete = null)
{
var tenantId = _tenantManager.GetCurrentTenant().TenantId;
var task = new Task<string>(() =>
{
using var scope = _serviceProvider.CreateScope();
var scopeClass = scope.ServiceProvider.GetService<StaticUploaderScope>();
var (tenantManager, staticUploader, _, _, _) = scopeClass;
tenantManager.SetCurrentTenant(tenantId);
return staticUploader.UploadFile(relativePath, mappedPath, onComplete);
}, TaskCreationOptions.LongRunning);
task.ConfigureAwait(false);
task.Start(_scheduler);
return task;
}
public void UploadDir(string relativePath, string mappedPath)
{
if (!CanUpload())
{
return;
}
if (!Directory.Exists(mappedPath))
{
return;
}
var tenant = _tenantManager.GetCurrentTenant();
var key = typeof(UploadOperationProgress).FullName + tenant.TenantId;
UploadOperationProgress uploadOperation;
lock (_locker)
{
uploadOperation = Queue.GetTask<UploadOperationProgress>(key);
if (uploadOperation != null)
{
return;
}
uploadOperation = new UploadOperationProgress(_serviceProvider, key, tenant.TenantId, relativePath, mappedPath);
Queue.QueueTask(uploadOperation);
}
}
public bool CanUpload()
{
var current = _storageSettingsHelper.DataStoreConsumer(_settingsManager.Load<CdnStorageSettings>());
if (current == null || !current.IsSet || (string.IsNullOrEmpty(current["cnamessl"]) && string.IsNullOrEmpty(current["cname"])))
{
return false;
}
return true;
}
public static void Stop()
{
_tokenSource.Cancel();
}
public UploadOperationProgress GetProgress(int tenantId)
{
lock (_locker)
{
var key = typeof(UploadOperationProgress).FullName + tenantId;
return Queue.GetTask<UploadOperationProgress>(key);
}
}
private static string GetCacheKey(string tenantId, string path)
{
return typeof(UploadOperation).FullName + tenantId + path;
}
}
public class UploadOperation
{
public string Result { get; private set; }
private readonly ILog _logger;
private readonly int _tenantId;
private readonly string _path;
private readonly string _mappedPath;
private readonly IServiceProvider _serviceProvider;
public UploadOperation(IServiceProvider serviceProvider, int tenantId, string path, string mappedPath)
{
_serviceProvider = serviceProvider;
_logger = _serviceProvider.GetService<IOptionsMonitor<ILog>>().CurrentValue;
_tenantId = tenantId;
_path = path.TrimStart('/');
_mappedPath = mappedPath;
Result = string.Empty;
}
public string DoJob()
{
try
{
using var scope = _serviceProvider.CreateScope();
var scopeClass = scope.ServiceProvider.GetService<StaticUploaderScope>();
var (tenantManager, _, securityContext, settingsManager, storageSettingsHelper) = scopeClass;
var tenant = tenantManager.GetTenant(_tenantId);
tenantManager.SetCurrentTenant(tenant);
securityContext.AuthenticateMeWithoutCookie(tenant.OwnerId);
var dataStore = storageSettingsHelper.DataStore(settingsManager.Load<CdnStorageSettings>());
if (File.Exists(_mappedPath))
{
if (!dataStore.IsFile(_path))
{
using var stream = File.OpenRead(_mappedPath);
dataStore.Save(_path, stream);
}
Result = dataStore.GetInternalUri("", _path, TimeSpan.Zero, null).AbsoluteUri.ToLower();
_logger.DebugFormat("UploadFile {0}", Result);
return Result;
}
}
catch (Exception e)
{
_logger.Error(e);
}
return null;
}
}
[Transient]
public class UploadOperationProgress : DistributedTaskProgress
{
public int TenantId { get; }
private readonly string _relativePath;
private readonly string _mappedPath;
private readonly IEnumerable<string> _directoryFiles;
private readonly IServiceProvider _serviceProvider;
public UploadOperationProgress(IServiceProvider serviceProvider, string key, int tenantId, string relativePath, string mappedPath)
{
_serviceProvider = serviceProvider;
Id = key;
Status = DistributedTaskStatus.Created;
TenantId = tenantId;
_relativePath = relativePath;
_mappedPath = mappedPath;
const string extensions = ".png|.jpeg|.jpg|.gif|.ico|.swf|.mp3|.ogg|.eot|.svg|.ttf|.woff|.woff2|.css|.less|.js";
var extensionsArray = extensions.Split('|');
_directoryFiles = Directory.GetFiles(mappedPath, "*", SearchOption.AllDirectories)
.Where(r => extensionsArray.Contains(Path.GetExtension(r)))
.ToList();
StepCount = _directoryFiles.Count();
}
protected override void DoJob()
{
using var scope = _serviceProvider.CreateScope();
var tenantManager = scope.ServiceProvider.GetService<TenantManager>();
var staticUploader = scope.ServiceProvider.GetService<StaticUploader>();
var tenant = tenantManager.GetTenant(TenantId);
tenantManager.SetCurrentTenant(tenant);
tenant.SetStatus(TenantStatus.Migrating);
tenantManager.SaveTenant(tenant);
PublishChanges();
foreach (var file in _directoryFiles)
{
var filePath = file.Substring(_mappedPath.TrimEnd('/').Length);
staticUploader.UploadFile(CrossPlatform.PathCombine(_relativePath, filePath), file, (res) => StepDone());
}
tenant.SetStatus(TenantStatus.Active);
tenantManager.SaveTenant(tenant);
}
public object Clone()
{
return MemberwiseClone();
}
}
[Scope]
public class StaticUploaderScope
{
private readonly TenantManager _tenantManager;
private readonly StaticUploader _staticUploader;
private readonly SecurityContext _securityContext;
private readonly SettingsManager _settingsManager;
private readonly StorageSettingsHelper _storageSettingsHelper;
public StaticUploaderScope(TenantManager tenantManager,
StaticUploader staticUploader,
SecurityContext securityContext,
SettingsManager settingsManager,
StorageSettingsHelper storageSettingsHelper)
{
_tenantManager = tenantManager;
_staticUploader = staticUploader;
_securityContext = securityContext;
_settingsManager = settingsManager;
_storageSettingsHelper = storageSettingsHelper;
}
public void Deconstruct(
out TenantManager tenantManager,
out StaticUploader staticUploader,
out SecurityContext securityContext,
out SettingsManager settingsManager,
out StorageSettingsHelper storageSettingsHelper)
{
tenantManager = _tenantManager;
staticUploader = _staticUploader;
securityContext = _securityContext;
settingsManager = _settingsManager;
storageSettingsHelper = _storageSettingsHelper;
}
}
public static class StaticUploaderExtension
{
public static void Register(DIHelper services)
{
services.TryAdd<StaticUploaderScope>();
services.AddDistributedTaskQueueService<UploadOperationProgress>(1);
}
[Scope]
public class StaticUploaderScope
{
private TenantManager TenantManager { get; }
private StaticUploader StaticUploader { get; }
private SecurityContext SecurityContext { get; }
private SettingsManager SettingsManager { get; }
private StorageSettingsHelper StorageSettingsHelper { get; }
public StaticUploaderScope(TenantManager tenantManager,
StaticUploader staticUploader,
SecurityContext securityContext,
SettingsManager settingsManager,
StorageSettingsHelper storageSettingsHelper)
{
TenantManager = tenantManager;
StaticUploader = staticUploader;
SecurityContext = securityContext;
SettingsManager = settingsManager;
StorageSettingsHelper = storageSettingsHelper;
}
public void Deconstruct(out TenantManager tenantManager, out StaticUploader staticUploader, out SecurityContext securityContext, out SettingsManager settingsManager, out StorageSettingsHelper storageSettingsHelper)
{
tenantManager = TenantManager;
staticUploader = StaticUploader;
securityContext = SecurityContext;
settingsManager = SettingsManager;
storageSettingsHelper = StorageSettingsHelper;
}
}
public static class StaticUploaderExtension
{
public static void Register(DIHelper services)
{
services.TryAdd<StaticUploaderScope>();
services.AddDistributedTaskQueueService<UploadOperationProgress>(1);
}
}
}

359
common/ASC.Data.Storage/StorageFactory.cs
View File

@ -23,232 +23,233 @@
*
*/
namespace ASC.Data.Storage
namespace ASC.Data.Storage;
[Singletone(Additional = typeof(StorageConfigExtension))]
public class StorageFactoryConfig
{
[Singletone(Additional = typeof(StorageConfigExtension))]
public class StorageFactoryConfig
public Configuration.Storage Section { get; }
public StorageFactoryConfig(IServiceProvider serviceProvider)
{
public Configuration.Storage Section { get; }
public StorageFactoryConfig(IServiceProvider serviceProvider)
{
Section = serviceProvider.GetService<Configuration.Storage>();
}
public IEnumerable<string> GetModuleList(string configpath, bool exceptDisabledMigration = false)
{
return Section.Module
.Where(x => x.Visible && (!exceptDisabledMigration || !x.DisableMigrate))
.Select(x => x.Name);
}
public IEnumerable<string> GetDomainList(string configpath, string modulename)
{
if (Section == null)
{
throw new ArgumentException("config section not found");
}
return
Section.Module
.Single(x => x.Name.Equals(modulename, StringComparison.OrdinalIgnoreCase))
.Domain
.Where(x => x.Visible)
.Select(x => x.Name);
}
Section = serviceProvider.GetService<Configuration.Storage>();
}
public static class StorageFactoryExtenstion
public IEnumerable<string> GetModuleList(string configpath, bool exceptDisabledMigration = false)
{
public static void InitializeHttpHandlers(this IEndpointRouteBuilder builder, string config = null)
return Section.Module
.Where(x => x.Visible && (!exceptDisabledMigration || !x.DisableMigrate))
.Select(x => x.Name);
}
public IEnumerable<string> GetDomainList(string configpath, string modulename)
{
if (Section == null)
{
//TODO:
//if (!HostingEnvironment.IsHosted)
//{
// throw new InvalidOperationException("Application not hosted.");
//}
throw new ArgumentException("config section not found");
}
var section = builder.ServiceProvider.GetService<Configuration.Storage>();
var pathUtils = builder.ServiceProvider.GetService<PathUtils>();
if (section != null)
return Section.Module
.Single(x => x.Name.Equals(modulename, StringComparison.OrdinalIgnoreCase))
.Domain
.Where(x => x.Visible)
.Select(x => x.Name);
}
}
public static class StorageFactoryExtenstion
{
public static void InitializeHttpHandlers(this IEndpointRouteBuilder builder, string config = null)
{
//TODO:
//if (!HostingEnvironment.IsHosted)
//{
// throw new InvalidOperationException("Application not hosted.");
//}
var section = builder.ServiceProvider.GetService<Configuration.Storage>();
var pathUtils = builder.ServiceProvider.GetService<PathUtils>();
if (section != null)
{
//old scheme
var discHandler = section.GetHandler("disc");
if (discHandler != null && section.Module != null)
{
//old scheme
var discHandler = section.GetHandler("disc");
if (discHandler != null && section.Module != null)
var props = discHandler.Property != null ? discHandler.Property.ToDictionary(r => r.Name, r => r.Value) : new Dictionary<string, string>();
foreach (var m in section.Module.Where(m => m.Type == "disc"))
{
var props = discHandler.Property != null ? discHandler.Property.ToDictionary(r => r.Name, r => r.Value) : new Dictionary<string, string>();
foreach (var m in section.Module.Where(m => m.Type == "disc"))
{
if (m.Path.Contains(Constants.STORAGE_ROOT_PARAM))
builder.RegisterDiscDataHandler(
pathUtils.ResolveVirtualPath(m.VirtualPath),
pathUtils.ResolvePhysicalPath(m.Path, props),
m.Public);
if (m.Path.Contains(Constants.StorageRootParam))
builder.RegisterDiscDataHandler(
pathUtils.ResolveVirtualPath(m.VirtualPath),
pathUtils.ResolvePhysicalPath(m.Path, props),
m.Public);
if (m.Domain != null)
if (m.Domain != null)
{
foreach (var d in m.Domain.Where(d => (d.Type == "disc" || string.IsNullOrEmpty(d.Type)) && d.Path.Contains(Constants.StorageRootParam)))
{
foreach (var d in m.Domain.Where(d => (d.Type == "disc" || string.IsNullOrEmpty(d.Type)) && d.Path.Contains(Constants.STORAGE_ROOT_PARAM)))
{
builder.RegisterDiscDataHandler(
pathUtils.ResolveVirtualPath(d.VirtualPath),
pathUtils.ResolvePhysicalPath(d.Path, props));
}
builder.RegisterDiscDataHandler(
pathUtils.ResolveVirtualPath(d.VirtualPath),
pathUtils.ResolvePhysicalPath(d.Path, props));
}
}
}
}
//new scheme
if (section.Module != null)
//new scheme
if (section.Module != null)
{
foreach (var m in section.Module)
{
foreach (var m in section.Module)
//todo: add path criterion
if (m.Type == "disc" || !m.Public || m.Path.Contains(Constants.StorageRootParam))
builder.RegisterStorageHandler(
m.Name,
string.Empty,
m.Public);
//todo: add path criterion
if (m.Domain != null)
{
//todo: add path criterion
if (m.Type == "disc" || !m.Public || m.Path.Contains(Constants.STORAGE_ROOT_PARAM))
foreach (var d in m.Domain.Where(d => d.Path.Contains(Constants.StorageRootParam)))
{
builder.RegisterStorageHandler(
m.Name,
string.Empty,
m.Public);
//todo: add path criterion
if (m.Domain != null)
{
foreach (var d in m.Domain.Where(d => d.Path.Contains(Constants.STORAGE_ROOT_PARAM)))
{
builder.RegisterStorageHandler(
m.Name,
d.Name,
d.Public);
}
d.Name,
d.Public);
}
}
}
}
}
}
}
[Scope(Additional = typeof(StorageFactoryExtension))]
public class StorageFactory
[Scope(Additional = typeof(StorageFactoryExtension))]
public class StorageFactory
{
private const string DefaultTenantName = "default";
private readonly StorageFactoryConfig _storageFactoryConfig;
private readonly SettingsManager _settingsManager;
private readonly StorageSettingsHelper _storageSettingsHelper;
private readonly TenantManager _tenantManager;
private readonly CoreBaseSettings _coreBaseSettings;
private readonly IServiceProvider _serviceProvider;
public StorageFactory(
IServiceProvider serviceProvider,
StorageFactoryConfig storageFactoryConfig,
SettingsManager settingsManager,
StorageSettingsHelper storageSettingsHelper,
TenantManager tenantManager,
CoreBaseSettings coreBaseSettings)
{
private const string DefaultTenantName = "default";
_serviceProvider = serviceProvider;
_storageFactoryConfig = storageFactoryConfig;
_settingsManager = settingsManager;
_storageSettingsHelper = storageSettingsHelper;
_tenantManager = tenantManager;
_coreBaseSettings = coreBaseSettings;
}
private StorageFactoryConfig StorageFactoryConfig { get; }
private SettingsManager SettingsManager { get; }
private StorageSettingsHelper StorageSettingsHelper { get; }
private TenantManager TenantManager { get; }
private CoreBaseSettings CoreBaseSettings { get; }
private IServiceProvider ServiceProvider { get; }
public IDataStore GetStorage(string tenant, string module)
{
return GetStorage(string.Empty, tenant, module);
}
public StorageFactory(
IServiceProvider serviceProvider,
StorageFactoryConfig storageFactoryConfig,
SettingsManager settingsManager,
StorageSettingsHelper storageSettingsHelper,
TenantManager tenantManager,
CoreBaseSettings coreBaseSettings)
public IDataStore GetStorage(string configpath, string tenant, string module)
{
int.TryParse(tenant, out var tenantId);
return GetStorage(configpath, tenant, module, new TenantQuotaController(tenantId, _tenantManager));
}
public IDataStore GetStorage(string configpath, string tenant, string module, IQuotaController controller)
{
var tenantId = -2;
if (string.IsNullOrEmpty(tenant))
{
ServiceProvider = serviceProvider;
StorageFactoryConfig = storageFactoryConfig;
SettingsManager = settingsManager;
StorageSettingsHelper = storageSettingsHelper;
TenantManager = tenantManager;
CoreBaseSettings = coreBaseSettings;
tenant = DefaultTenantName;
}
else
{
tenantId = Convert.ToInt32(tenant);
}
public IDataStore GetStorage(string tenant, string module)
//Make tennant path
tenant = TenantPath.CreatePath(tenant);
var section = _storageFactoryConfig.Section;
if (section == null)
{
return GetStorage(string.Empty, tenant, module);
throw new InvalidOperationException("config section not found");
}
public IDataStore GetStorage(string configpath, string tenant, string module)
var settings = _settingsManager.LoadForTenant<StorageSettings>(tenantId);
//TODO:GetStoreAndCache
return GetDataStore(tenant, module, _storageSettingsHelper.DataStoreConsumer(settings), controller);
}
public IDataStore GetStorageFromConsumer(string configpath, string tenant, string module, DataStoreConsumer consumer)
{
if (tenant == null) tenant = DefaultTenantName;
//Make tennant path
tenant = TenantPath.CreatePath(tenant);
var section = _storageFactoryConfig.Section;
if (section == null)
{
int.TryParse(tenant, out var tenantId);
return GetStorage(configpath, tenant, module, new TenantQuotaController(tenantId, TenantManager));
throw new InvalidOperationException("config section not found");
}
public IDataStore GetStorage(string configpath, string tenant, string module, IQuotaController controller)
int.TryParse(tenant, out var tenantId);
return GetDataStore(tenant, module, consumer, new TenantQuotaController(tenantId, _tenantManager));
}
private IDataStore GetDataStore(string tenant, string module, DataStoreConsumer consumer, IQuotaController controller)
{
var storage = _storageFactoryConfig.Section;
var moduleElement = storage.GetModuleElement(module);
if (moduleElement == null)
{
var tenantId = -2;
if (string.IsNullOrEmpty(tenant))
{
tenant = DefaultTenantName;
}
else
{
tenantId = Convert.ToInt32(tenant);
}
//Make tennant path
tenant = TenantPath.CreatePath(tenant);
var section = StorageFactoryConfig.Section;
if (section == null)
{
throw new InvalidOperationException("config section not found");
}
var settings = SettingsManager.LoadForTenant<StorageSettings>(tenantId);
//TODO:GetStoreAndCache
return GetDataStore(tenant, module, StorageSettingsHelper.DataStoreConsumer(settings), controller);
throw new ArgumentException("no such module", module);
}
public IDataStore GetStorageFromConsumer(string configpath, string tenant, string module, DataStoreConsumer consumer)
var handler = storage.GetHandler(moduleElement.Type);
Type instanceType;
IDictionary<string, string> props;
if (_coreBaseSettings.Standalone &&
!moduleElement.DisableMigrate &&
consumer.IsSet)
{
if (tenant == null) tenant = DefaultTenantName;
//Make tennant path
tenant = TenantPath.CreatePath(tenant);
var section = StorageFactoryConfig.Section;
if (section == null)
{
throw new InvalidOperationException("config section not found");
}
int.TryParse(tenant, out var tenantId);
return GetDataStore(tenant, module, consumer, new TenantQuotaController(tenantId, TenantManager));
instanceType = consumer.HandlerType;
props = consumer;
}
private IDataStore GetDataStore(string tenant, string module, DataStoreConsumer consumer, IQuotaController controller)
else
{
var storage = StorageFactoryConfig.Section;
var moduleElement = storage.GetModuleElement(module);
if (moduleElement == null)
{
throw new ArgumentException("no such module", module);
}
var handler = storage.GetHandler(moduleElement.Type);
Type instanceType;
IDictionary<string, string> props;
if (CoreBaseSettings.Standalone &&
!moduleElement.DisableMigrate &&
consumer.IsSet)
{
instanceType = consumer.HandlerType;
props = consumer;
}
else
{
instanceType = Type.GetType(handler.Type, true);
props = handler.Property.ToDictionary(r => r.Name, r => r.Value);
}
instanceType = Type.GetType(handler.Type, true);
props = handler.Property.ToDictionary(r => r.Name, r => r.Value);
}
return ((IDataStore)ActivatorUtilities.CreateInstance(ServiceProvider, instanceType))
.Configure(tenant, handler, moduleElement, props)
.SetQuotaController(moduleElement.Count ? controller : null
/*don't count quota if specified on module*/);
}
return ((IDataStore)ActivatorUtilities.CreateInstance(_serviceProvider, instanceType))
.Configure(tenant, handler, moduleElement, props)
.SetQuotaController(moduleElement.Count ? controller : null
/*don't count quota if specified on module*/);
}
}
public static class StorageFactoryExtension
public static class StorageFactoryExtension
{
public static void Register(DIHelper services)
{
public static void Register(DIHelper services)
{
services.TryAdd<DiscDataStore>();
services.TryAdd<GoogleCloudStorage>();
services.TryAdd<RackspaceCloudStorage>();
services.TryAdd<S3Storage>();
}
services.TryAdd<DiscDataStore>();
services.TryAdd<GoogleCloudStorage>();
services.TryAdd<RackspaceCloudStorage>();
services.TryAdd<S3Storage>();
}
}
}

328
common/ASC.Data.Storage/StorageHandler.cs
View File

@ -23,173 +23,177 @@
*
*/
namespace ASC.Data.Storage.DiscStorage
{
public class StorageHandler
{
private readonly string _path;
private readonly string _module;
private readonly string _domain;
private readonly bool _checkAuth;
public StorageHandler(IServiceProvider serviceProvider, string path, string module, string domain, bool checkAuth = true)
{
ServiceProvider = serviceProvider;
_path = path;
_module = module;
_domain = domain;
_checkAuth = checkAuth;
}
private IServiceProvider ServiceProvider { get; }
namespace ASC.Data.Storage.DiscStorage;
public Task Invoke(HttpContext context)
{
using var scope = ServiceProvider.CreateScope();
var scopeClass = scope.ServiceProvider.GetService<StorageHandlerScope>();
var (tenantManager, securityContext, storageFactory, emailValidationKeyProvider) = scopeClass;
if (_checkAuth && !securityContext.IsAuthenticated)
{
context.Response.StatusCode = (int)HttpStatusCode.Forbidden;
return Task.CompletedTask;
}
var storage = storageFactory.GetStorage(tenantManager.GetCurrentTenant().TenantId.ToString(CultureInfo.InvariantCulture), _module);
var path = CrossPlatform.PathCombine(_path, GetRouteValue("pathInfo", context).Replace('/', Path.DirectorySeparatorChar));
var header = context.Request.Query[Constants.QUERY_HEADER].FirstOrDefault() ?? "";
var auth = context.Request.Query[Constants.QUERY_AUTH].FirstOrDefault() ?? "";
var storageExpire = storage.GetExpire(_domain);
if (storageExpire != TimeSpan.Zero && storageExpire != TimeSpan.MinValue && storageExpire != TimeSpan.MaxValue || !string.IsNullOrEmpty(auth))
{
var expire = context.Request.Query[Constants.QUERY_EXPIRE];
if (string.IsNullOrEmpty(expire)) expire = storageExpire.TotalMinutes.ToString(CultureInfo.InvariantCulture);
var validateResult = emailValidationKeyProvider.ValidateEmailKey(path + "." + header + "." + expire, auth ?? "", TimeSpan.FromMinutes(Convert.ToDouble(expire)));
if (validateResult != EmailValidationKeyProvider.ValidationResult.Ok)
{
context.Response.StatusCode = (int)HttpStatusCode.Forbidden;
return Task.CompletedTask;
}
}
if (!storage.IsFile(_domain, path))
{
context.Response.StatusCode = (int)HttpStatusCode.NotFound;
return Task.CompletedTask;
}
var headers = header.Length > 0 ? header.Split('&').Select(HttpUtility.UrlDecode) : Array.Empty<string>();
if (storage.IsSupportInternalUri)
{
var uri = storage.GetInternalUri(_domain, path, TimeSpan.FromMinutes(15), headers);
//TODO
//context.Response.Cache.SetAllowResponseInBrowserHistory(false);
//context.Response.Cache.SetCacheability(HttpCacheability.NoCache);
context.Response.Redirect(uri.ToString());
return Task.CompletedTask;
}
public class StorageHandler
{
private readonly string _path;
private readonly string _module;
private readonly string _domain;
private readonly bool _checkAuth;
private readonly IServiceProvider _serviceProvider;
string encoding = null;
if (storage is DiscDataStore && storage.IsFile(_domain, path + ".gz"))
{
path += ".gz";
encoding = "gzip";
}
var headersToCopy = new List<string> { "Content-Disposition", "Cache-Control", "Content-Encoding", "Content-Language", "Content-Type", "Expires" };
foreach (var h in headers)
{
var toCopy = headersToCopy.Find(x => h.StartsWith(x));
if (string.IsNullOrEmpty(toCopy)) continue;
context.Response.Headers[toCopy] = h.Substring(toCopy.Length + 1);
}
try
{
context.Response.ContentType = MimeMapping.GetMimeMapping(path);
}
catch (Exception)
{
}
if (encoding != null)
context.Response.Headers["Content-Encoding"] = encoding;
return InternalInvoke(context, storage, path);
}
private async Task InternalInvoke(HttpContext context, IDataStore storage, string path)
{
using (var stream = storage.GetReadStream(_domain, path))
{
await stream.CopyToAsync(context.Response.Body);
}
await context.Response.Body.FlushAsync();
await context.Response.CompleteAsync();
}
private string GetRouteValue(string name, HttpContext context)
{
return (context.GetRouteValue(name) ?? "").ToString();
}
}
[Scope]
public class StorageHandlerScope
public StorageHandler(IServiceProvider serviceProvider, string path, string module, string domain, bool checkAuth = true)
{
private TenantManager TenantManager { get; }
private SecurityContext SecurityContext { get; }
private StorageFactory StorageFactory { get; }
private EmailValidationKeyProvider EmailValidationKeyProvider { get; }
_serviceProvider = serviceProvider;
_path = path;
_module = module;
_domain = domain;
_checkAuth = checkAuth;
}
public StorageHandlerScope(TenantManager tenantManager, SecurityContext securityContext, StorageFactory storageFactory, EmailValidationKeyProvider emailValidationKeyProvider)
public Task Invoke(HttpContext context)
{
using var scope = _serviceProvider.CreateScope();
var scopeClass = scope.ServiceProvider.GetService<StorageHandlerScope>();
var (tenantManager, securityContext, storageFactory, emailValidationKeyProvider) = scopeClass;
if (_checkAuth && !securityContext.IsAuthenticated)
{
TenantManager = tenantManager;
SecurityContext = securityContext;
StorageFactory = storageFactory;
EmailValidationKeyProvider = emailValidationKeyProvider;
context.Response.StatusCode = (int)HttpStatusCode.Forbidden;
return Task.CompletedTask;
}
public void Deconstruct(out TenantManager tenantManager, out SecurityContext securityContext, out StorageFactory storageFactory, out EmailValidationKeyProvider emailValidationKeyProvider)
var storage = storageFactory.GetStorage(tenantManager.GetCurrentTenant().TenantId.ToString(CultureInfo.InvariantCulture), _module);
var path = CrossPlatform.PathCombine(_path, GetRouteValue("pathInfo", context).Replace('/', Path.DirectorySeparatorChar));
var header = context.Request.Query[Constants.QueryHeader].FirstOrDefault() ?? "";
var auth = context.Request.Query[Constants.QueryAuth].FirstOrDefault() ?? "";
var storageExpire = storage.GetExpire(_domain);
if (storageExpire != TimeSpan.Zero && storageExpire != TimeSpan.MinValue && storageExpire != TimeSpan.MaxValue || !string.IsNullOrEmpty(auth))
{
tenantManager = TenantManager;
securityContext = SecurityContext;
storageFactory = StorageFactory;
emailValidationKeyProvider = EmailValidationKeyProvider;
var expire = context.Request.Query[Constants.QueryExpire];
if (string.IsNullOrEmpty(expire)) expire = storageExpire.TotalMinutes.ToString(CultureInfo.InvariantCulture);
var validateResult = emailValidationKeyProvider.ValidateEmailKey(path + "." + header + "." + expire, auth ?? "", TimeSpan.FromMinutes(Convert.ToDouble(expire)));
if (validateResult != EmailValidationKeyProvider.ValidationResult.Ok)
{
context.Response.StatusCode = (int)HttpStatusCode.Forbidden;
return Task.CompletedTask;
}
}
}
public static class StorageHandlerExtensions
{
public static IEndpointRouteBuilder RegisterStorageHandler(this IEndpointRouteBuilder builder, string module, string domain, bool publicRoute = false)
{
var pathUtils = builder.ServiceProvider.GetService<PathUtils>();
var virtPath = pathUtils.ResolveVirtualPath(module, domain);
virtPath = virtPath.TrimStart('/');
var handler = new StorageHandler(builder.ServiceProvider, string.Empty, module, domain, !publicRoute);
var url = virtPath + "{*pathInfo}";
if (!builder.DataSources.Any(r => r.Endpoints.Any(e => e.DisplayName == url)))
{
builder.Map(url, handler.Invoke);
var newUrl = url.Replace("{0}", "{t1}/{t2}/{t3}");
if (newUrl != url)
{
builder.Map(url, handler.Invoke);
}
}
return builder;
}
}
}
if (!storage.IsFile(_domain, path))
{
context.Response.StatusCode = (int)HttpStatusCode.NotFound;
return Task.CompletedTask;
}
var headers = header.Length > 0 ? header.Split('&').Select(HttpUtility.UrlDecode) : Array.Empty<string>();
if (storage.IsSupportInternalUri)
{
var uri = storage.GetInternalUri(_domain, path, TimeSpan.FromMinutes(15), headers);
//TODO
//context.Response.Cache.SetAllowResponseInBrowserHistory(false);
//context.Response.Cache.SetCacheability(HttpCacheability.NoCache);
context.Response.Redirect(uri.ToString());
return Task.CompletedTask;
}
string encoding = null;
if (storage is DiscDataStore && storage.IsFile(_domain, path + ".gz"))
{
path += ".gz";
encoding = "gzip";
}
var headersToCopy = new List<string> { "Content-Disposition", "Cache-Control", "Content-Encoding", "Content-Language", "Content-Type", "Expires" };
foreach (var h in headers)
{
var toCopy = headersToCopy.Find(x => h.StartsWith(x));
if (string.IsNullOrEmpty(toCopy))
{
continue;
}
context.Response.Headers[toCopy] = h.Substring(toCopy.Length + 1);
}
try
{
context.Response.ContentType = MimeMapping.GetMimeMapping(path);
}
catch (Exception)
{
}
if (encoding != null)
{
context.Response.Headers["Content-Encoding"] = encoding;
}
return InternalInvoke(context, storage, path);
}
private async Task InternalInvoke(HttpContext context, IDataStore storage, string path)
{
using (var stream = storage.GetReadStream(_domain, path))
{
await stream.CopyToAsync(context.Response.Body);
}
await context.Response.Body.FlushAsync();
await context.Response.CompleteAsync();
}
private string GetRouteValue(string name, HttpContext context)
{
return (context.GetRouteValue(name) ?? "").ToString();
}
}
[Scope]
public class StorageHandlerScope
{
private readonly TenantManager _tenantManager;
private readonly SecurityContext _securityContext;
private readonly StorageFactory _storageFactory;
private readonly EmailValidationKeyProvider _emailValidationKeyProvider;
public StorageHandlerScope(TenantManager tenantManager, SecurityContext securityContext, StorageFactory storageFactory, EmailValidationKeyProvider emailValidationKeyProvider)
{
_tenantManager = tenantManager;
_securityContext = securityContext;
_storageFactory = storageFactory;
_emailValidationKeyProvider = emailValidationKeyProvider;
}
public void Deconstruct(out TenantManager tenantManager, out SecurityContext securityContext, out StorageFactory storageFactory, out EmailValidationKeyProvider emailValidationKeyProvider)
{
tenantManager = _tenantManager;
securityContext = _securityContext;
storageFactory = _storageFactory;
emailValidationKeyProvider = _emailValidationKeyProvider;
}
}
public static class StorageHandlerExtensions
{
public static IEndpointRouteBuilder RegisterStorageHandler(this IEndpointRouteBuilder builder, string module, string domain, bool publicRoute = false)
{
var pathUtils = builder.ServiceProvider.GetService<PathUtils>();
var virtPath = pathUtils.ResolveVirtualPath(module, domain);
virtPath = virtPath.TrimStart('/');
var handler = new StorageHandler(builder.ServiceProvider, string.Empty, module, domain, !publicRoute);
var url = virtPath + "{*pathInfo}";
if (!builder.DataSources.Any(r => r.Endpoints.Any(e => e.DisplayName == url)))
{
builder.Map(url, handler.Invoke);
var newUrl = url.Replace("{0}", "{t1}/{t2}/{t3}");
if (newUrl != url)
{
builder.Map(url, handler.Invoke);
}
}
return builder;
}
}

444
common/ASC.Data.Storage/StorageUploader.cs
View File

@ -23,235 +23,235 @@
*
*/
namespace ASC.Data.Storage
namespace ASC.Data.Storage;
[Singletone]
public class StorageUploader
{
[Singletone]
public class StorageUploader
{
protected readonly DistributedTaskQueue Queue;
private static readonly object Locker;
private IServiceProvider ServiceProvider { get; }
private TempStream TempStream { get; }
private ICacheNotify<MigrationProgress> CacheMigrationNotify { get; }
static StorageUploader()
{
Locker = new object();
}
public StorageUploader(IServiceProvider serviceProvider, TempStream tempStream, ICacheNotify<MigrationProgress> cacheMigrationNotify, DistributedTaskQueueOptionsManager options)
{
ServiceProvider = serviceProvider;
TempStream = tempStream;
CacheMigrationNotify = cacheMigrationNotify;
Queue = options.Get(nameof(StorageUploader));
}
public void Start(int tenantId, StorageSettings newStorageSettings, StorageFactoryConfig storageFactoryConfig)
{
lock (Locker)
{
var id = GetCacheKey(tenantId);
var migrateOperation = Queue.GetTask<MigrateOperation>(id);
if (migrateOperation != null) return;
migrateOperation = new MigrateOperation(ServiceProvider, CacheMigrationNotify, id, tenantId, newStorageSettings, storageFactoryConfig, TempStream);
Queue.QueueTask(migrateOperation);
}
}
public MigrateOperation GetProgress(int tenantId)
{
lock (Locker)
{
return Queue.GetTask<MigrateOperation>(GetCacheKey(tenantId));
}
}
public void Stop()
{
foreach (var task in Queue.GetTasks<MigrateOperation>().Where(r => r.Status == DistributedTaskStatus.Running))
{
Queue.CancelTask(task.Id);
}
}
private static string GetCacheKey(int tenantId)
{
return typeof(MigrateOperation).FullName + tenantId;
}
}
[Transient]
public class MigrateOperation : DistributedTaskProgress
{
private readonly ILog Log;
private static readonly string ConfigPath;
private readonly IEnumerable<string> Modules;
private readonly StorageSettings settings;
private readonly int tenantId;
static MigrateOperation()
{
ConfigPath = "";
}
public MigrateOperation(
IServiceProvider serviceProvider,
ICacheNotify<MigrationProgress> cacheMigrationNotify,
string id,
int tenantId,
StorageSettings settings,
StorageFactoryConfig storageFactoryConfig,
TempStream tempStream)
{
Id = id;
Status = DistributedTaskStatus.Created;
ServiceProvider = serviceProvider;
CacheMigrationNotify = cacheMigrationNotify;
this.tenantId = tenantId;
this.settings = settings;
StorageFactoryConfig = storageFactoryConfig;
TempStream = tempStream;
Modules = storageFactoryConfig.GetModuleList(ConfigPath, true);
StepCount = Modules.Count();
Log = serviceProvider.GetService<IOptionsMonitor<ILog>>().CurrentValue;
}
private IServiceProvider ServiceProvider { get; }
private StorageFactoryConfig StorageFactoryConfig { get; }
private TempStream TempStream { get; }
private ICacheNotify<MigrationProgress> CacheMigrationNotify { get; }
protected override void DoJob()
{
try
{
Log.DebugFormat("Tenant: {0}", tenantId);
Status = DistributedTaskStatus.Running;
using var scope = ServiceProvider.CreateScope();
var tempPath = scope.ServiceProvider.GetService<TempPath>();
var scopeClass = scope.ServiceProvider.GetService<MigrateOperationScope>();
var (tenantManager, securityContext, storageFactory, options, storageSettingsHelper, settingsManager) = scopeClass;
var tenant = tenantManager.GetTenant(tenantId);
tenantManager.SetCurrentTenant(tenant);
securityContext.AuthenticateMeWithoutCookie(tenant.OwnerId);
foreach (var module in Modules)
{
var oldStore = storageFactory.GetStorage(ConfigPath, tenantId.ToString(), module);
var store = storageFactory.GetStorageFromConsumer(ConfigPath, tenantId.ToString(), module, storageSettingsHelper.DataStoreConsumer(settings));
var domains = StorageFactoryConfig.GetDomainList(ConfigPath, module).ToList();
var crossModuleTransferUtility = new CrossModuleTransferUtility(options, TempStream, tempPath, oldStore, store);
string[] files;
foreach (var domain in domains)
{
//Status = module + domain;
Log.DebugFormat("Domain: {0}", domain);
files = oldStore.ListFilesRelative(domain, "\\", "*.*", true);
foreach (var file in files)
{
Log.DebugFormat("File: {0}", file);
crossModuleTransferUtility.CopyFile(domain, file, domain, file);
}
}
Log.Debug("Domain:");
files = oldStore.ListFilesRelative(string.Empty, "\\", "*.*", true)
.Where(path => domains.All(domain => !path.Contains(domain + "/")))
.ToArray();
foreach (var file in files)
{
Log.DebugFormat("File: {0}", file);
crossModuleTransferUtility.CopyFile("", file, "", file);
}
StepDone();
MigrationPublish();
}
protected readonly DistributedTaskQueue Queue;
settingsManager.Save(settings);
tenant.SetStatus(TenantStatus.Active);
tenantManager.SaveTenant(tenant);
private static readonly object _locker;
private readonly IServiceProvider _serviceProvider;
private readonly TempStream _tempStream;
private readonly ICacheNotify<MigrationProgress> _cacheMigrationNotify;
Status = DistributedTaskStatus.Completed;
}
catch (Exception e)
{
Status = DistributedTaskStatus.Failted;
Exception = e;
Log.Error(e);
}
MigrationPublish();
}
public object Clone()
{
return MemberwiseClone();
}
private void MigrationPublish()
{
CacheMigrationNotify.Publish(new MigrationProgress
{
TenantId = tenantId,
Progress = Percentage,
Error = Exception.ToString(),
IsCompleted = IsCompleted
},
Common.Caching.CacheNotifyAction.Insert);
}
}
public class MigrateOperationScope
static StorageUploader()
{
private TenantManager TenantManager { get; }
private SecurityContext SecurityContext { get; }
private StorageFactory StorageFactory { get; }
private IOptionsMonitor<ILog> Options { get; }
private StorageSettingsHelper StorageSettingsHelper { get; }
private SettingsManager SettingsManager { get; }
_locker = new object();
}
public MigrateOperationScope(TenantManager tenantManager,
SecurityContext securityContext,
StorageFactory storageFactory,
IOptionsMonitor<ILog> options,
StorageSettingsHelper storageSettingsHelper,
SettingsManager settingsManager)
public StorageUploader(IServiceProvider serviceProvider, TempStream tempStream, ICacheNotify<MigrationProgress> cacheMigrationNotify, DistributedTaskQueueOptionsManager options)
{
_serviceProvider = serviceProvider;
_tempStream = tempStream;
_cacheMigrationNotify = cacheMigrationNotify;
Queue = options.Get(nameof(StorageUploader));
}
public void Start(int tenantId, StorageSettings newStorageSettings, StorageFactoryConfig storageFactoryConfig)
{
lock (_locker)
{
TenantManager = tenantManager;
SecurityContext = securityContext;
StorageFactory = storageFactory;
Options = options;
StorageSettingsHelper = storageSettingsHelper;
SettingsManager = settingsManager;
var id = GetCacheKey(tenantId);
var migrateOperation = Queue.GetTask<MigrateOperation>(id);
if (migrateOperation != null)
{
return;
}
migrateOperation = new MigrateOperation(_serviceProvider, _cacheMigrationNotify, id, tenantId, newStorageSettings, storageFactoryConfig, _tempStream);
Queue.QueueTask(migrateOperation);
}
}
public MigrateOperation GetProgress(int tenantId)
{
lock (_locker)
{
return Queue.GetTask<MigrateOperation>(GetCacheKey(tenantId));
}
}
public void Stop()
{
foreach (var task in Queue.GetTasks<MigrateOperation>().Where(r => r.Status == DistributedTaskStatus.Running))
{
Queue.CancelTask(task.Id);
}
}
private static string GetCacheKey(int tenantId)
{
return typeof(MigrateOperation).FullName + tenantId;
}
}
[Transient]
public class MigrateOperation : DistributedTaskProgress
{
private static readonly string _configPath;
private readonly ILog _logger;
private readonly IEnumerable<string> _modules;
private readonly StorageSettings _settings;
private readonly int _tenantId;
private readonly IServiceProvider _serviceProvider;
private readonly StorageFactoryConfig _storageFactoryConfig;
private readonly TempStream _tempStream;
private readonly ICacheNotify<MigrationProgress> _cacheMigrationNotify;
static MigrateOperation()
{
_configPath = string.Empty;
}
public MigrateOperation(
IServiceProvider serviceProvider,
ICacheNotify<MigrationProgress> cacheMigrationNotify,
string id,
int tenantId,
StorageSettings settings,
StorageFactoryConfig storageFactoryConfig,
TempStream tempStream)
{
Id = id;
Status = DistributedTaskStatus.Created;
_serviceProvider = serviceProvider;
_cacheMigrationNotify = cacheMigrationNotify;
_tenantId = tenantId;
_settings = settings;
_storageFactoryConfig = storageFactoryConfig;
_tempStream = tempStream;
_modules = storageFactoryConfig.GetModuleList(_configPath, true);
StepCount = _modules.Count();
_logger = serviceProvider.GetService<IOptionsMonitor<ILog>>().CurrentValue;
}
public object Clone()
{
return MemberwiseClone();
}
protected override void DoJob()
{
try
{
_logger.DebugFormat("Tenant: {0}", _tenantId);
Status = DistributedTaskStatus.Running;
using var scope = _serviceProvider.CreateScope();
var tempPath = scope.ServiceProvider.GetService<TempPath>();
var scopeClass = scope.ServiceProvider.GetService<MigrateOperationScope>();
var (tenantManager, securityContext, storageFactory, options, storageSettingsHelper, settingsManager) = scopeClass;
var tenant = tenantManager.GetTenant(_tenantId);
tenantManager.SetCurrentTenant(tenant);
securityContext.AuthenticateMeWithoutCookie(tenant.OwnerId);
foreach (var module in _modules)
{
var oldStore = storageFactory.GetStorage(_configPath, _tenantId.ToString(), module);
var store = storageFactory.GetStorageFromConsumer(_configPath, _tenantId.ToString(), module, storageSettingsHelper.DataStoreConsumer(_settings));
var domains = _storageFactoryConfig.GetDomainList(_configPath, module).ToList();
var crossModuleTransferUtility = new CrossModuleTransferUtility(options, _tempStream, tempPath, oldStore, store);
string[] files;
foreach (var domain in domains)
{
//Status = module + domain;
_logger.DebugFormat("Domain: {0}", domain);
files = oldStore.ListFilesRelative(domain, "\\", "*.*", true);
foreach (var file in files)
{
_logger.DebugFormat("File: {0}", file);
crossModuleTransferUtility.CopyFile(domain, file, domain, file);
}
}
_logger.Debug("Domain:");
files = oldStore.ListFilesRelative(string.Empty, "\\", "*.*", true)
.Where(path => domains.All(domain => !path.Contains(domain + "/")))
.ToArray();
foreach (var file in files)
{
_logger.DebugFormat("File: {0}", file);
crossModuleTransferUtility.CopyFile("", file, "", file);
}
StepDone();
MigrationPublish();
}
settingsManager.Save(_settings);
tenant.SetStatus(TenantStatus.Active);
tenantManager.SaveTenant(tenant);
Status = DistributedTaskStatus.Completed;
}
catch (Exception e)
{
Status = DistributedTaskStatus.Failted;
Exception = e;
_logger.Error(e);
}
public void Deconstruct(out TenantManager tenantManager,
out SecurityContext securityContext,
out StorageFactory storageFactory,
out IOptionsMonitor<ILog> options,
out StorageSettingsHelper storageSettingsHelper,
out SettingsManager settingsManager)
MigrationPublish();
}
private void MigrationPublish()
{
_cacheMigrationNotify.Publish(new MigrationProgress
{
tenantManager = TenantManager;
securityContext = SecurityContext;
storageFactory = StorageFactory;
options = Options;
storageSettingsHelper = StorageSettingsHelper;
settingsManager = SettingsManager;
}
}
TenantId = _tenantId,
Progress = Percentage,
Error = Exception.ToString(),
IsCompleted = IsCompleted
},
Common.Caching.CacheNotifyAction.Insert);
}
}
public class MigrateOperationScope
{
private readonly TenantManager _tenantManager;
private readonly SecurityContext _securityContext;
private readonly StorageFactory _storageFactory;
private readonly IOptionsMonitor<ILog> _options;
private readonly StorageSettingsHelper _storageSettingsHelper;
private readonly SettingsManager _settingsManager;
public MigrateOperationScope(TenantManager tenantManager,
SecurityContext securityContext,
StorageFactory storageFactory,
IOptionsMonitor<ILog> options,
StorageSettingsHelper storageSettingsHelper,
SettingsManager settingsManager)
{
_tenantManager = tenantManager;
_securityContext = securityContext;
_storageFactory = storageFactory;
_options = options;
_storageSettingsHelper = storageSettingsHelper;
_settingsManager = settingsManager;
}
public void Deconstruct(out TenantManager tenantManager,
out SecurityContext securityContext,
out StorageFactory storageFactory,
out IOptionsMonitor<ILog> options,
out StorageSettingsHelper storageSettingsHelper,
out SettingsManager settingsManager)
{
tenantManager = _tenantManager;
securityContext = _securityContext;
storageFactory = _storageFactory;
options = _options;
storageSettingsHelper = _storageSettingsHelper;
settingsManager = _settingsManager;
}
}

36
common/ASC.Data.Storage/TenantPath.cs
View File

@ -23,29 +23,31 @@
*
*/
namespace ASC.Data.Storage
namespace ASC.Data.Storage;
public static class TenantPath
{
public static class TenantPath
public static string CreatePath(string tenant)
{
public static string CreatePath(string tenant)
if (tenant == null)
{
if (tenant == null)
{
throw new ArgumentNullException(nameof(tenant));
}
if (long.TryParse(tenant, NumberStyles.Integer, CultureInfo.InvariantCulture, out var tenantId))
{
var culture = CultureInfo.InvariantCulture;
return tenantId == 0 ? tenantId.ToString(culture) : tenantId.ToString("00/00/00", culture);
}
return tenant;
}
public static bool TryGetTenant(string tenantPath, out int tenant)
if (long.TryParse(tenant, NumberStyles.Integer, CultureInfo.InvariantCulture, out var tenantId))
{
tenantPath = tenantPath.Replace("/", "");
return int.TryParse(tenantPath, out tenant);
var culture = CultureInfo.InvariantCulture;
return tenantId == 0 ? tenantId.ToString(culture) : tenantId.ToString("00/00/00", culture);
}
return tenant;
}
}
public static bool TryGetTenant(string tenantPath, out int tenant)
{
tenantPath = tenantPath.Replace("/", "");
return int.TryParse(tenantPath, out tenant);
}
}

207
common/ASC.Data.Storage/TenantQuotaController.cs
View File

@ -23,117 +23,116 @@
*
*/
namespace ASC.Data.Storage
namespace ASC.Data.Storage;
public class TenantQuotaController : IQuotaController
{
public class TenantQuotaController : IQuotaController
private long CurrentSize
{
private readonly int tenant;
private Lazy<long> lazyCurrentSize;
private long currentSize;
private long CurrentSize {
get
get
{
if (!_lazyCurrentSize.IsValueCreated)
{
if (!lazyCurrentSize.IsValueCreated)
{
return currentSize = lazyCurrentSize.Value;
}
return currentSize;
return _currentSize = _lazyCurrentSize.Value;
}
set
return _currentSize;
}
set => _currentSize = value;
}
private readonly int _tenant;
private readonly TenantManager _tenantManager;
private Lazy<long> _lazyCurrentSize;
private long _currentSize;
public TenantQuotaController(int tenant, TenantManager tenantManager)
{
_tenant = tenant;
_tenantManager = tenantManager;
_lazyCurrentSize = new Lazy<long>(() => _tenantManager.FindTenantQuotaRows(tenant)
.Where(r => UsedInQuota(r.Tag))
.Sum(r => r.Counter));
}
#region IQuotaController Members
public void QuotaUsedAdd(string module, string domain, string dataTag, long size, bool quotaCheckFileSize = true)
{
size = Math.Abs(size);
if (UsedInQuota(dataTag))
{
QuotaUsedCheck(size, quotaCheckFileSize);
CurrentSize += size;
}
SetTenantQuotaRow(module, domain, size, dataTag, true);
}
public void QuotaUsedDelete(string module, string domain, string dataTag, long size)
{
size = -Math.Abs(size);
if (UsedInQuota(dataTag))
{
CurrentSize += size;
}
SetTenantQuotaRow(module, domain, size, dataTag, true);
}
public void QuotaUsedSet(string module, string domain, string dataTag, long size)
{
size = Math.Max(0, size);
if (UsedInQuota(dataTag))
{
CurrentSize += size;
}
SetTenantQuotaRow(module, domain, size, dataTag, false);
}
public void QuotaUsedCheck(long size)
{
QuotaUsedCheck(size, true);
}
public void QuotaUsedCheck(long size, bool quotaCheckFileSize)
{
var quota = _tenantManager.GetTenantQuota(_tenant);
if (quota != null)
{
if (quotaCheckFileSize && quota.MaxFileSize != 0 && quota.MaxFileSize < size)
{
currentSize = value;
throw new TenantQuotaException(string.Format("Exceeds the maximum file size ({0}MB)", BytesToMegabytes(quota.MaxFileSize)));
}
}
private TenantManager TenantManager { get; }
public TenantQuotaController(int tenant, TenantManager tenantManager)
{
this.tenant = tenant;
TenantManager = tenantManager;
lazyCurrentSize = new Lazy<long>(() => TenantManager.FindTenantQuotaRows(tenant)
.Where(r => UsedInQuota(r.Tag))
.Sum(r => r.Counter));
}
#region IQuotaController Members
public void QuotaUsedAdd(string module, string domain, string dataTag, long size, bool quotaCheckFileSize = true)
{
size = Math.Abs(size);
if (UsedInQuota(dataTag))
if (quota.MaxTotalSize != 0 && quota.MaxTotalSize < CurrentSize + size)
{
QuotaUsedCheck(size, quotaCheckFileSize);
CurrentSize += size;
throw new TenantQuotaException(string.Format("Exceeded maximum amount of disk quota ({0}MB)", BytesToMegabytes(quota.MaxTotalSize)));
}
SetTenantQuotaRow(module, domain, size, dataTag, true);
}
public void QuotaUsedDelete(string module, string domain, string dataTag, long size)
{
size = -Math.Abs(size);
if (UsedInQuota(dataTag))
{
CurrentSize += size;
}
SetTenantQuotaRow(module, domain, size, dataTag, true);
}
public void QuotaUsedSet(string module, string domain, string dataTag, long size)
{
size = Math.Max(0, size);
if (UsedInQuota(dataTag))
{
CurrentSize += size;
}
SetTenantQuotaRow(module, domain, size, dataTag, false);
}
public void QuotaUsedCheck(long size)
{
QuotaUsedCheck(size, true);
}
public void QuotaUsedCheck(long size, bool quotaCheckFileSize)
{
var quota = TenantManager.GetTenantQuota(tenant);
if (quota != null)
{
if (quotaCheckFileSize && quota.MaxFileSize != 0 && quota.MaxFileSize < size)
{
throw new TenantQuotaException(string.Format("Exceeds the maximum file size ({0}MB)", BytesToMegabytes(quota.MaxFileSize)));
}
if (quota.MaxTotalSize != 0 && quota.MaxTotalSize < CurrentSize + size)
{
throw new TenantQuotaException(string.Format("Exceeded maximum amount of disk quota ({0}MB)", BytesToMegabytes(quota.MaxTotalSize)));
}
}
}
#endregion
public long QuotaCurrentGet()
{
return CurrentSize;
}
private void SetTenantQuotaRow(string module, string domain, long size, string dataTag, bool exchange)
{
TenantManager.SetTenantQuotaRow(
new TenantQuotaRow { Tenant = tenant, Path = $"/{module}/{domain}", Counter = size, Tag = dataTag },
exchange);
}
private bool UsedInQuota(string tag)
{
return !string.IsNullOrEmpty(tag) && new Guid(tag) != Guid.Empty;
}
private double BytesToMegabytes(long bytes)
{
return Math.Round(bytes / 1024d / 1024d, 1);
}
}
}
#endregion
public long QuotaCurrentGet()
{
return CurrentSize;
}
private void SetTenantQuotaRow(string module, string domain, long size, string dataTag, bool exchange)
{
_tenantManager.SetTenantQuotaRow(
new TenantQuotaRow { Tenant = _tenant, Path = $"/{module}/{domain}", Counter = size, Tag = dataTag },
exchange);
}
private bool UsedInQuota(string tag)
{
return !string.IsNullOrEmpty(tag) && new Guid(tag) != Guid.Empty;
}
private double BytesToMegabytes(long bytes)
{
return Math.Round(bytes / 1024d / 1024d, 1);
}
}

375
common/ASC.Data.Storage/WebPath.cs
View File

@ -23,216 +23,219 @@
*
*/
namespace ASC.Data.Storage
namespace ASC.Data.Storage;
[Singletone]
public class WebPathSettings
{
[Singletone]
public class WebPathSettings
private readonly IEnumerable<Appender> _appenders;
public WebPathSettings(Configuration.Storage storage)
{
private readonly IEnumerable<Appender> Appenders;
public WebPathSettings(Configuration.Storage storage)
var section = storage;
if (section != null)
{
var section = storage;
if (section != null)
{
Appenders = section.Appender;
}
}
public string GetRelativePath(HttpContext httpContext, IOptionsMonitor<ILog> options, string absolutePath)
{
if (!Uri.IsWellFormedUriString(absolutePath, UriKind.Absolute))
{
throw new ArgumentException($"bad path format {absolutePath} is not absolute");
}
var appender = Appenders.FirstOrDefault(x => absolutePath.Contains(x.Append) || (absolutePath.Contains(x.AppendSecure) && !string.IsNullOrEmpty(x.AppendSecure)));
if (appender == null)
{
return absolutePath;
}
return SecureHelper.IsSecure(httpContext, options) && !string.IsNullOrEmpty(appender.AppendSecure) ?
absolutePath.Remove(0, appender.AppendSecure.Length) :
absolutePath.Remove(0, appender.Append.Length);
}
public string GetPath(HttpContext httpContext, IOptionsMonitor<ILog> options, string relativePath)
{
if (!string.IsNullOrEmpty(relativePath) && relativePath.IndexOf('~') == 0)
{
throw new ArgumentException($"bad path format {relativePath} remove '~'", nameof(relativePath));
}
var result = relativePath;
var ext = Path.GetExtension(relativePath).ToLowerInvariant();
if (Appenders.Any())
{
var avaliableAppenders = Appenders.Where(x => x.Extensions != null && x.Extensions.Split('|').Contains(ext) || string.IsNullOrEmpty(ext)).ToList();
var avaliableAppendersCount = avaliableAppenders.Count;
Appender appender;
if (avaliableAppendersCount > 1)
{
appender = avaliableAppenders[relativePath.Length % avaliableAppendersCount];
}
else if (avaliableAppendersCount == 1)
{
appender = avaliableAppenders.First();
}
else
{
appender = Appenders.First();
}
if (appender.Append.IndexOf('~') == 0)
{
var query = string.Empty;
//Rel path
if (relativePath.IndexOfAny(new[] { '?', '=', '&' }) != -1)
{
//Cut it
query = relativePath.Substring(relativePath.IndexOf('?'));
relativePath = relativePath.Substring(0, relativePath.IndexOf('?'));
}
//if (HostingEnvironment.IsHosted)
//{
// result = VirtualPathUtility.ToAbsolute(string.Format("{0}/{1}{2}", appender.Append.TrimEnd('/'), relativePath.TrimStart('/'), query));
//}
//else
//{
result = $"{appender.Append.TrimEnd('/').TrimStart('~')}/{relativePath.TrimStart('/')}{query}";
//}
}
else
{
//TODO HostingEnvironment.IsHosted
if (SecureHelper.IsSecure(httpContext, options) && !string.IsNullOrEmpty(appender.AppendSecure))
{
result = $"{appender.AppendSecure.TrimEnd('/')}/{relativePath.TrimStart('/')}";
}
else
{
//Append directly
result = $"{appender.Append.TrimEnd('/')}/{relativePath.TrimStart('/')}";
}
}
}
return result;
_appenders = section.Appender;
}
}
[Scope]
public class WebPath
public string GetRelativePath(HttpContext httpContext, IOptionsMonitor<ILog> options, string absolutePath)
{
private static readonly IDictionary<string, bool> Existing = new ConcurrentDictionary<string, bool>();
private WebPathSettings WebPathSettings { get; }
public IServiceProvider ServiceProvider { get; }
private SettingsManager SettingsManager { get; }
private StorageSettingsHelper StorageSettingsHelper { get; }
private IHttpContextAccessor HttpContextAccessor { get; }
public IHostEnvironment HostEnvironment { get; }
private CoreBaseSettings CoreBaseSettings { get; }
private IOptionsMonitor<ILog> Options { get; }
private IHttpClientFactory ClientFactory { get; }
public WebPath(
WebPathSettings webPathSettings,
IServiceProvider serviceProvider,
SettingsManager settingsManager,
StorageSettingsHelper storageSettingsHelper,
IHostEnvironment hostEnvironment,
CoreBaseSettings coreBaseSettings,
IOptionsMonitor<ILog> options,
IHttpClientFactory clientFactory)
if (!Uri.IsWellFormedUriString(absolutePath, UriKind.Absolute))
{
WebPathSettings = webPathSettings;
ServiceProvider = serviceProvider;
SettingsManager = settingsManager;
StorageSettingsHelper = storageSettingsHelper;
HostEnvironment = hostEnvironment;
CoreBaseSettings = coreBaseSettings;
Options = options;
ClientFactory = clientFactory;
throw new ArgumentException($"bad path format {absolutePath} is not absolute");
}
public WebPath(
WebPathSettings webPathSettings,
IServiceProvider serviceProvider,
StaticUploader staticUploader,
SettingsManager settingsManager,
StorageSettingsHelper storageSettingsHelper,
IHttpContextAccessor httpContextAccessor,
IHostEnvironment hostEnvironment,
CoreBaseSettings coreBaseSettings,
var appender = _appenders.FirstOrDefault(x => absolutePath.Contains(x.Append) || (absolutePath.Contains(x.AppendSecure) && !string.IsNullOrEmpty(x.AppendSecure)));
if (appender == null)
{
return absolutePath;
}
return SecureHelper.IsSecure(httpContext, options) && !string.IsNullOrEmpty(appender.AppendSecure) ?
absolutePath.Remove(0, appender.AppendSecure.Length) :
absolutePath.Remove(0, appender.Append.Length);
}
public string GetPath(HttpContext httpContext, IOptionsMonitor<ILog> options, string relativePath)
{
if (!string.IsNullOrEmpty(relativePath) && relativePath.IndexOf('~') == 0)
{
throw new ArgumentException($"bad path format {relativePath} remove '~'", nameof(relativePath));
}
var result = relativePath;
var ext = Path.GetExtension(relativePath).ToLowerInvariant();
if (_appenders.Any())
{
var avaliableAppenders = _appenders.Where(x => x.Extensions != null && x.Extensions.Split('|').Contains(ext) || string.IsNullOrEmpty(ext)).ToList();
var avaliableAppendersCount = avaliableAppenders.Count;
Appender appender;
if (avaliableAppendersCount > 1)
{
appender = avaliableAppenders[relativePath.Length % avaliableAppendersCount];
}
else if (avaliableAppendersCount == 1)
{
appender = avaliableAppenders.First();
}
else
{
appender = _appenders.First();
}
if (appender.Append.IndexOf('~') == 0)
{
var query = string.Empty;
//Rel path
if (relativePath.IndexOfAny(new[] { '?', '=', '&' }) != -1)
{
//Cut it
query = relativePath.Substring(relativePath.IndexOf('?'));
relativePath = relativePath.Substring(0, relativePath.IndexOf('?'));
}
//if (HostingEnvironment.IsHosted)
//{
// result = VirtualPathUtility.ToAbsolute(string.Format("{0}/{1}{2}", appender.Append.TrimEnd('/'), relativePath.TrimStart('/'), query));
//}
//else
//{
result = $"{appender.Append.TrimEnd('/').TrimStart('~')}/{relativePath.TrimStart('/')}{query}";
//}
}
else
{
//TODO HostingEnvironment.IsHosted
if (SecureHelper.IsSecure(httpContext, options) && !string.IsNullOrEmpty(appender.AppendSecure))
{
result = $"{appender.AppendSecure.TrimEnd('/')}/{relativePath.TrimStart('/')}";
}
else
{
//Append directly
result = $"{appender.Append.TrimEnd('/')}/{relativePath.TrimStart('/')}";
}
}
}
return result;
}
}
[Scope]
public class WebPath
{
public IServiceProvider ServiceProvider { get; }
public IHostEnvironment HostEnvironment { get; }
private IHttpClientFactory ClientFactory { get; }
private static readonly IDictionary<string, bool> _existing = new ConcurrentDictionary<string, bool>();
private readonly WebPathSettings _webPathSettings;
private readonly SettingsManager _settingsManager;
private readonly StorageSettingsHelper _storageSettingsHelper;
private readonly IHttpContextAccessor _httpContextAccessor;
private readonly CoreBaseSettings _coreBaseSettings;
private readonly IOptionsMonitor<ILog> _options;
public WebPath(
WebPathSettings webPathSettings,
IServiceProvider serviceProvider,
SettingsManager settingsManager,
StorageSettingsHelper storageSettingsHelper,
IHostEnvironment hostEnvironment,
CoreBaseSettings coreBaseSettings,
IOptionsMonitor<ILog> options,
IHttpClientFactory clientFactory)
{
_webPathSettings = webPathSettings;
ServiceProvider = serviceProvider;
_settingsManager = settingsManager;
_storageSettingsHelper = storageSettingsHelper;
HostEnvironment = hostEnvironment;
_coreBaseSettings = coreBaseSettings;
_options = options;
ClientFactory = clientFactory;
}
public WebPath(
WebPathSettings webPathSettings,
IServiceProvider serviceProvider,
StaticUploader staticUploader,
SettingsManager settingsManager,
StorageSettingsHelper storageSettingsHelper,
IHttpContextAccessor httpContextAccessor,
IHostEnvironment hostEnvironment,
CoreBaseSettings coreBaseSettings,
IOptionsMonitor<ILog> options,
IHttpClientFactory clientFactory)
: this(webPathSettings, serviceProvider, settingsManager, storageSettingsHelper, hostEnvironment, coreBaseSettings, options, clientFactory)
{
HttpContextAccessor = httpContextAccessor;
}
{
_httpContextAccessor = httpContextAccessor;
}
public string GetPath(string relativePath)
public string GetPath(string relativePath)
{
if (!string.IsNullOrEmpty(relativePath) && relativePath.IndexOf('~') == 0)
{
if (!string.IsNullOrEmpty(relativePath) && relativePath.IndexOf('~') == 0)
{
throw new ArgumentException($"bad path format {relativePath} remove '~'", nameof(relativePath));
}
if (CoreBaseSettings.Standalone && ServiceProvider.GetService<StaticUploader>().CanUpload()) //hack for skip resolve DistributedTaskQueueOptionsManager
{
try
{
var result = StorageSettingsHelper.DataStore(SettingsManager.Load<CdnStorageSettings>()).GetInternalUri("", relativePath, TimeSpan.Zero, null).AbsoluteUri.ToLower();
if (!string.IsNullOrEmpty(result)) return result;
}
catch (Exception)
{
}
}
return WebPathSettings.GetPath(HttpContextAccessor?.HttpContext, Options, relativePath);
}
public bool Exists(string relativePath)
{
var path = GetPath(relativePath);
if (!Existing.ContainsKey(path))
{
if (Uri.IsWellFormedUriString(path, UriKind.Relative) && HttpContextAccessor?.HttpContext != null)
{
//Local
Existing[path] = File.Exists(CrossPlatform.PathCombine(HostEnvironment.ContentRootPath, path));
}
if (Uri.IsWellFormedUriString(path, UriKind.Absolute))
{
//Make request
Existing[path] = CheckWebPath(path);
}
}
return Existing[path];
}
private bool CheckWebPath(string path)
if (_coreBaseSettings.Standalone && ServiceProvider.GetService<StaticUploader>().CanUpload()) //hack for skip resolve DistributedTaskQueueOptionsManager
{
try
{
var request = new HttpRequestMessage();
request.RequestUri = new Uri(path);
request.Method = HttpMethod.Head;
var httpClient = ClientFactory.CreateClient();
using var response = httpClient.Send(request);
return response.StatusCode == HttpStatusCode.OK;
var result = _storageSettingsHelper.DataStore(_settingsManager.Load<CdnStorageSettings>()).GetInternalUri("", relativePath, TimeSpan.Zero, null).AbsoluteUri.ToLower();
if (!string.IsNullOrEmpty(result))
{
return result;
}
}
catch (Exception)
{
return false;
}
}
return _webPathSettings.GetPath(_httpContextAccessor?.HttpContext, _options, relativePath);
}
}
public bool Exists(string relativePath)
{
var path = GetPath(relativePath);
if (!_existing.ContainsKey(path))
{
if (Uri.IsWellFormedUriString(path, UriKind.Relative) && _httpContextAccessor?.HttpContext != null)
{
//Local
_existing[path] = File.Exists(CrossPlatform.PathCombine(HostEnvironment.ContentRootPath, path));
}
if (Uri.IsWellFormedUriString(path, UriKind.Absolute))
{
//Make request
_existing[path] = CheckWebPath(path);
}
}
return _existing[path];
}
private bool CheckWebPath(string path)
{
try
{
var request = new HttpRequestMessage();
request.RequestUri = new Uri(path);
request.Method = HttpMethod.Head;
var httpClient = ClientFactory.CreateClient();
using var response = httpClient.Send(request);
return response.StatusCode == HttpStatusCode.OK;
}
catch (Exception)
{
return false;
}
}
}

115
common/ASC.Data.Storage/Wildcard.cs
View File

@ -24,73 +24,72 @@
*/
namespace ASC.Data.Storage
{
public static class Wildcard
{
public static bool IsMatch(string pattern, string input)
{
return IsMatch(pattern, input, false);
}
namespace ASC.Data.Storage;
public static bool IsMatch(string pattern, string input, bool caseInsensitive)
{
var offsetInput = 0;
var isAsterix = false;
public static class Wildcard
{
public static bool IsMatch(string pattern, string input)
{
return IsMatch(pattern, input, false);
}
public static bool IsMatch(string pattern, string input, bool caseInsensitive)
{
var offsetInput = 0;
var isAsterix = false;
int i = 0;
while (i < pattern.Length)
{
switch (pattern[i])
{
switch (pattern[i])
{
case '?':
isAsterix = false;
offsetInput++;
break;
case '*':
isAsterix = true;
while (i < pattern.Length && pattern[i] == '*')
{
i++;
}
if (i >= pattern.Length)
{
return true;
}
continue;
default:
if (offsetInput >= input.Length)
case '?':
isAsterix = false;
offsetInput++;
break;
case '*':
isAsterix = true;
while (i < pattern.Length && pattern[i] == '*')
{
i++;
}
if (i >= pattern.Length)
{
return true;
}
continue;
default:
if (offsetInput >= input.Length)
{
return false;
}
if ((caseInsensitive ? char.ToLower(input[offsetInput]) : input[offsetInput]) != (caseInsensitive ? char.ToLower(pattern[i]) : pattern[i]))
{
if (!isAsterix)
{
return false;
}
if ((caseInsensitive ? char.ToLower(input[offsetInput]) : input[offsetInput]) != (caseInsensitive ? char.ToLower(pattern[i]) : pattern[i]))
{
if (!isAsterix)
{
return false;
}
offsetInput++;
continue;
}
offsetInput++;
break;
} // end switch
i++;
} // end for
continue;
}
offsetInput++;
break;
} // end switch
i++;
} // end for
// have we finished parsing our input?
if (i > input.Length)
{
return false;
}
// do we have any lingering asterixes we need to skip?
while (i < pattern.Length && pattern[i] == '*')
{
++i;
}
// final evaluation. The index should be pointing at the
// end of the string.
return offsetInput == input.Length;
// have we finished parsing our input?
if (i > input.Length)
{
return false;
}
// do we have any lingering asterixes we need to skip?
while (i < pattern.Length && pattern[i] == '*')
{
++i;
}
// final evaluation. The index should be pointing at the
// end of the string.
return offsetInput == input.Length;
}
}
}