DocSpace-client/common/ASC.Data.Storage/RackspaceCloud/RackspaceCloudStorage.cs
2022-02-16 15:57:37 +03:00

754 lines
27 KiB
C#

/*
*
* (c) Copyright Ascensio System Limited 2010-2018
*
* This program is freeware. You can redistribute it and/or modify it under the terms of the GNU
* General Public License (GPL) version 3 as published by the Free Software Foundation (https://www.gnu.org/copyleft/gpl.html).
* In accordance with Section 7(a) of the GNU GPL its Section 15 shall be amended to the effect that
* Ascensio System SIA expressly excludes the warranty of non-infringement of any third-party rights.
*
* THIS PROGRAM IS DISTRIBUTED WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR
* FITNESS FOR A PARTICULAR PURPOSE. For more details, see GNU GPL at https://www.gnu.org/copyleft/gpl.html
*
* You can contact Ascensio System SIA by email at sales@onlyoffice.com
*
* The interactive user interfaces in modified source and object code versions of ONLYOFFICE must display
* Appropriate Legal Notices, as required under Section 5 of the GNU GPL version 3.
*
* Pursuant to Section 7 § 3(b) of the GNU GPL you must retain the original ONLYOFFICE logo which contains
* relevant author attributions when distributing the software. If the display of the logo in its graphic
* form is not reasonably feasible for technical reasons, you must include the words "Powered by ONLYOFFICE"
* in every copy of the program you distribute.
* Pursuant to Section 7 § 3(e) we decline to grant you any rights under trademark law for use of our trademarks.
*
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using System.Web;
using ASC.Common;
using ASC.Common.Logging;
using ASC.Core;
using ASC.Data.Storage.Configuration;
using ASC.Security.Cryptography;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Options;
using net.openstack.Core.Domain;
using net.openstack.Providers.Rackspace;
using MimeMapping = ASC.Common.Web.MimeMapping;
namespace ASC.Data.Storage.RackspaceCloud
{
[Scope]
public class RackspaceCloudStorage : BaseStorage
{
private string _region;
private string _private_container;
private string _public_container;
private readonly List<string> _domains = new List<string>();
private Dictionary<string, ACL> _domainsAcl;
private ACL _moduleAcl;
private string _subDir;
private string _username;
private string _apiKey;
private bool _lowerCasing = true;
private Uri _cname;
private Uri _cnameSSL;
private readonly ILog _logger;
public RackspaceCloudStorage(
TempPath tempPath,
TempStream tempStream,
TenantManager tenantManager,
PathUtils pathUtils,
EmailValidationKeyProvider emailValidationKeyProvider,
IHttpContextAccessor httpContextAccessor,
IOptionsMonitor<ILog> options,
IHttpClientFactory httpClient)
: base(tempStream, tenantManager, pathUtils, emailValidationKeyProvider, httpContextAccessor, options, httpClient)
{
_logger = options.Get("ASC.Data.Storage.Rackspace.RackspaceCloudStorage");
TempPath = tempPath;
}
private string MakePath(string domain, string path)
{
string result;
path = path.TrimStart('\\', '/').TrimEnd('/').Replace('\\', '/');
if (!string.IsNullOrEmpty(_subDir))
{
if (_subDir.Length == 1 && (_subDir[0] == '/' || _subDir[0] == '\\'))
result = path;
else
result = $"{_subDir}/{path}"; // Ignory all, if _subDir is not null
}
else//Key combined from module+domain+filename
result = $"{_tenant}/{_modulename}/{domain}/{path}";
result = result.Replace("//", "/").TrimStart('/');
if (_lowerCasing)
{
result = result.ToLowerInvariant();
}
return result;
}
private CloudFilesProvider GetClient()
{
var cloudIdentity = new CloudIdentity()
{
Username = _username,
APIKey = _apiKey
};
return new CloudFilesProvider(cloudIdentity);
}
public override IDataStore Configure(string tenant, Handler handlerConfig, Module moduleConfig, IDictionary<string, string> props)
{
_tenant = tenant;
if (moduleConfig != null)
{
_modulename = moduleConfig.Name;
_dataList = new DataList(moduleConfig);
_domains.AddRange(moduleConfig.Domain.Select(x => $"{x.Name}/"));
_domainsExpires = moduleConfig.Domain.Where(x => x.Expires != TimeSpan.Zero).ToDictionary(x => x.Name, y => y.Expires);
_domainsExpires.Add(string.Empty, moduleConfig.Expires);
_domainsAcl = moduleConfig.Domain.ToDictionary(x => x.Name, y => y.Acl);
_moduleAcl = moduleConfig.Acl;
}
else
{
_modulename = string.Empty;
_dataList = null;
_domainsExpires = new Dictionary<string, TimeSpan> { { string.Empty, TimeSpan.Zero } };
_domainsAcl = new Dictionary<string, ACL>();
_moduleAcl = ACL.Auto;
}
_private_container = props["private_container"];
_region = props["region"];
_apiKey = props["apiKey"];
_username = props["username"];
if (props.TryGetValue("lower", out var value))
{
bool.TryParse(value, out _lowerCasing);
}
props.TryGetValue("subdir", out _subDir);
_public_container = props["public_container"];
if (string.IsNullOrEmpty(_public_container))
throw new ArgumentException("_public_container");
var client = GetClient();
var cdnHeaders = client.GetContainerCDNHeader(_public_container, _region);
_cname = props.ContainsKey("cname") && Uri.IsWellFormedUriString(props["cname"], UriKind.Absolute)
? new Uri(props["cname"], UriKind.Absolute)
: new Uri(cdnHeaders.CDNUri);
_cnameSSL = props.ContainsKey("cnamessl") &&
Uri.IsWellFormedUriString(props["cnamessl"], UriKind.Absolute)
? new Uri(props["cnamessl"], UriKind.Absolute)
: new Uri(cdnHeaders.CDNSslUri);
return this;
}
public override Task<Uri> GetInternalUriAsync(string domain, string path, TimeSpan expire, IEnumerable<string> headers)
{
if (expire == TimeSpan.Zero || expire == TimeSpan.MinValue || expire == TimeSpan.MaxValue)
{
expire = GetExpire(domain);
}
if (expire == TimeSpan.Zero || expire == TimeSpan.MinValue || expire == TimeSpan.MaxValue)
{
return Task.FromResult(GetUriShared(domain, path));
}
var client = GetClient();
var accounMetaData = client.GetAccountMetaData(_region);
string secretKey;
if (accounMetaData.TryGetValue("Temp-Url-Key", out secretKey))
{
}
else
{
secretKey = Common.Utils.RandomString.Generate(64);
accounMetaData.Add("Temp-Url-Key", secretKey);
client.UpdateAccountMetadata(accounMetaData, _region);
}
return Task.FromResult(client.CreateTemporaryPublicUri(
JSIStudios.SimpleRESTServices.Client.HttpMethod.GET,
_private_container,
MakePath(domain, path),
secretKey,
DateTime.UtcNow.Add(expire),
_region));
}
private Uri GetUriShared(string domain, string path)
{
return new Uri(string.Format("{0}{1}", SecureHelper.IsSecure(HttpContextAccessor?.HttpContext, Options) ? _cnameSSL : _cname, MakePath(domain, path)));
}
public override Task<Stream> GetReadStreamAsync(string domain, string path)
{
return GetReadStreamAsync(domain, path, 0);
}
public override Task<Stream> GetReadStreamAsync(string domain, string path, int offset)
{
return GetReadStreamAsync(domain, path, offset);
}
public override Task<Uri> SaveAsync(string domain, string path, Stream stream)
{
return SaveAsync(domain, path, stream, string.Empty, string.Empty);
}
public override Task<Uri> SaveAsync(string domain, string path, Stream stream, ACL acl)
{
return SaveAsync(domain, path, stream, null, null, acl);
}
protected override Task<Uri> SaveWithAutoAttachmentAsync(string domain, string path, Stream stream, string attachmentFileName)
{
var contentDisposition = $"attachment; filename={HttpUtility.UrlPathEncode(attachmentFileName)};";
if (attachmentFileName.Any(c => c >= 0 && c <= 127))
{
contentDisposition = $"attachment; filename*=utf-8''{HttpUtility.UrlPathEncode(attachmentFileName)};";
}
return SaveAsync(domain, path, stream, null, contentDisposition);
}
public override Task<Uri> SaveAsync(string domain, string path, Stream stream, string contentType, string contentDisposition)
{
return SaveAsync(domain, path, stream, contentType, contentDisposition, ACL.Auto);
}
public override Task<Uri> SaveAsync(string domain, string path, Stream stream, string contentEncoding, int cacheDays)
{
return SaveAsync(domain, path, stream, string.Empty, string.Empty, ACL.Auto, contentEncoding, cacheDays);
}
public Task<Uri> SaveAsync(string domain, string path, Stream stream, string contentType,
string contentDisposition, ACL acl, string contentEncoding = null, int cacheDays = 5,
DateTime? deleteAt = null, long? deleteAfter = null)
{
var buffered = TempStream.GetBuffered(stream);
if (QuotaController != null)
{
QuotaController.QuotaUsedCheck(buffered.Length);
}
var client = GetClient();
var mime = string.IsNullOrEmpty(contentType)
? MimeMapping.GetMimeMapping(Path.GetFileName(path))
: contentType;
var customHeaders = new Dictionary<string, string>();
if (cacheDays > 0)
{
customHeaders.Add("Cache-Control", string.Format("public, maxage={0}", (int)TimeSpan.FromDays(cacheDays).TotalSeconds));
customHeaders.Add("Expires", DateTime.UtcNow.Add(TimeSpan.FromDays(cacheDays)).ToString());
}
if (deleteAt.HasValue)
{
var ts = deleteAt.Value - new DateTime(1970, 1, 1, 0, 0, 0);
var unixTimestamp = (long)ts.TotalSeconds;
customHeaders.Add("X-Delete-At", unixTimestamp.ToString());
}
if (deleteAfter.HasValue)
{
customHeaders.Add("X-Delete-After", deleteAfter.ToString());
}
if (!string.IsNullOrEmpty(contentEncoding))
customHeaders.Add("Content-Encoding", contentEncoding);
var cannedACL = acl == ACL.Auto ? GetDomainACL(domain) : ACL.Read;
if (cannedACL == ACL.Read)
{
try
{
using (var emptyStream = TempStream.Create())
{
var headers = new Dictionary<string, string>
{
{ "X-Object-Manifest", $"{_private_container}/{MakePath(domain, path)}" }
};
// create symlink
client.CreateObject(_public_container,
emptyStream,
MakePath(domain, path),
mime,
4096,
headers,
_region
);
emptyStream.Close();
}
client.PurgeObjectFromCDN(_public_container, MakePath(domain, path));
}
catch (Exception exp)
{
_logger.InfoFormat("The invalidation {0} failed", _public_container + "/" + MakePath(domain, path));
_logger.Error(exp);
}
}
stream.Position = 0;
client.CreateObject(_private_container,
stream,
MakePath(domain, path),
mime,
4096,
customHeaders,
_region
);
QuotaUsedAdd(domain, buffered.Length);
return GetUriAsync(domain, path);
}
private ACL GetDomainACL(string domain)
{
if (GetExpire(domain) != TimeSpan.Zero)
{
return ACL.Auto;
}
if (_domainsAcl.TryGetValue(domain, out var value))
{
return value;
}
return _moduleAcl;
}
public override async Task DeleteAsync(string domain, string path)
{
var client = GetClient();
MakePath(domain, path);
var size = await GetFileSizeAsync(domain, path);
client.DeleteObject(_private_container, MakePath(domain, path));
QuotaUsedDelete(domain, size);
}
public override Task DeleteFilesAsync(string domain, string folderPath, string pattern, bool recursive)
{
var client = GetClient();
var files = client.ListObjects(_private_container, null, null, null, MakePath(domain, folderPath), _region)
.Where(x => Wildcard.IsMatch(pattern, Path.GetFileName(x.Name)));
if (!files.Any()) return Task.CompletedTask;
foreach(var file in files)
{
client.DeleteObject(_private_container, file.Name);
}
if (QuotaController != null)
{
QuotaUsedDelete(domain, files.Select(x => x.Bytes).Sum());
}
return Task.CompletedTask;
}
public override Task DeleteFilesAsync(string domain, List<string> paths)
{
if (paths.Count == 0) return Task.CompletedTask;
return InternalDeleteFilesAsync(domain, paths);
}
private async Task InternalDeleteFilesAsync(string domain, List<string> paths)
{
var keysToDel = new List<string>();
long quotaUsed = 0;
foreach (var path in paths)
{
try
{
var key = MakePath(domain, path);
if (QuotaController != null)
{
quotaUsed += await GetFileSizeAsync(domain, path);
}
keysToDel.Add(key);
}
catch (FileNotFoundException)
{
}
}
if (keysToDel.Count == 0) return;
var client = GetClient();
keysToDel.ForEach(x => client.DeleteObject(_private_container, x));
if (quotaUsed > 0)
{
QuotaUsedDelete(domain, quotaUsed);
}
}
public override Task DeleteFilesAsync(string domain, string folderPath, DateTime fromDate, DateTime toDate)
{
var client = GetClient();
var files = client.ListObjects(_private_container, null, null, null, MakePath(domain, folderPath), _region)
.Where(x => x.LastModified >= fromDate && x.LastModified <= toDate);
if (!files.Any()) return Task.CompletedTask;
foreach(var file in files)
{
client.DeleteObject(_private_container, file.Name);
}
if (QuotaController != null)
{
QuotaUsedDelete(domain, files.Select(x => x.Bytes).Sum());
}
return Task.CompletedTask;
}
public override Task MoveDirectoryAsync(string srcdomain, string srcdir, string newdomain, string newdir)
{
var client = GetClient();
var srckey = MakePath(srcdomain, srcdir);
var dstkey = MakePath(newdomain, newdir);
var paths = client.ListObjects(_private_container, null, null, srckey, _region).Select(x => x.Name);
foreach (var path in paths)
{
client.CopyObject(_private_container, path, _private_container, path.Replace(srckey, dstkey));
client.DeleteObject(_private_container, path);
}
return Task.CompletedTask;
}
public override async Task<Uri> MoveAsync(string srcdomain, string srcpath, string newdomain, string newpath, bool quotaCheckFileSize = true)
{
var srcKey = MakePath(srcdomain, srcpath);
var dstKey = MakePath(newdomain, newpath);
var size = await GetFileSizeAsync(srcdomain, srcpath);
var client = GetClient();
client.CopyObject(_private_container, srcKey, _private_container, dstKey);
await DeleteAsync(srcdomain, srcpath);
QuotaUsedDelete(srcdomain, size);
QuotaUsedAdd(newdomain, size, quotaCheckFileSize);
return await GetUriAsync(newdomain, newpath);
}
public override Task<Uri> SaveTempAsync(string domain, out string assignedPath, Stream stream)
{
assignedPath = Guid.NewGuid().ToString();
return SaveAsync(domain, assignedPath, stream);
}
public override IAsyncEnumerable<string> ListDirectoriesRelativeAsync(string domain, string path, bool recursive)
{
var client = GetClient();
return client.ListObjects(_private_container, null, null, null, MakePath(domain, path), _region)
.Select(x => x.Name.Substring(MakePath(domain, path + "/").Length)).ToAsyncEnumerable();
}
public override IAsyncEnumerable<string> ListFilesRelativeAsync(string domain, string path, string pattern, bool recursive)
{
var client = GetClient();
var paths = client.ListObjects(_private_container, null, null, null, MakePath(domain, path), _region).Select(x => x.Name);
return paths
.Where(x => Wildcard.IsMatch(pattern, Path.GetFileName(x)))
.Select(x => x.Substring(MakePath(domain, path + "/").Length).TrimStart('/')).ToAsyncEnumerable();
}
public override Task<bool> IsFileAsync(string domain, string path)
{
var client = GetClient();
var objects = client.ListObjects(_private_container, null, null, null, MakePath(domain, path), _region);
var result = objects.Any();
return Task.FromResult(result);
}
public override Task<bool> IsDirectoryAsync(string domain, string path)
{
return IsFileAsync(domain, path);
}
public override Task DeleteDirectoryAsync(string domain, string path)
{
var client = GetClient();
var objToDel = client.ListObjects(_private_container, null, null, null, MakePath(domain, path), _region);
foreach (var obj in objToDel)
{
client.DeleteObject(_private_container, obj.Name);
QuotaUsedDelete(domain, obj.Bytes);
}
return Task.CompletedTask;
}
public override Task<long> GetFileSizeAsync(string domain, string path)
{
var client = GetClient();
var obj = client
.ListObjects(_private_container, null, null, null, MakePath(domain, path));
if (obj.Any())
return Task.FromResult(obj.Single().Bytes);
return Task.FromResult<long>(0);
}
public override Task<long> GetDirectorySizeAsync(string domain, string path)
{
var client = GetClient();
var objToDel = client
.ListObjects(_private_container, null, null, null, MakePath(domain, path));
long result = 0;
foreach (var obj in objToDel)
{
result += obj.Bytes;
}
return Task.FromResult(result);
}
public override Task<long> ResetQuotaAsync(string domain)
{
var client = GetClient();
var objects = client
.ListObjects(_private_container, null, null, null, MakePath(domain, string.Empty), _region);
if (QuotaController != null)
{
long size = 0;
foreach (var obj in objects)
{
size += obj.Bytes;
}
QuotaController.QuotaUsedSet(_modulename, domain, _dataList.GetData(domain), size);
return Task.FromResult(size);
}
return Task.FromResult <long>(0);
}
public override Task<long> GetUsedQuotaAsync(string domain)
{
var client = GetClient();
var objects = client
.ListObjects(_private_container, null, null, null, MakePath(domain, string.Empty), _region);
long result = 0;
foreach (var obj in objects)
{
result += obj.Bytes;
}
return Task.FromResult(result);
}
public override async Task<Uri> CopyAsync(string srcdomain, string path, string newdomain, string newpath)
{
var srcKey = MakePath(srcdomain, path);
var dstKey = MakePath(newdomain, newpath);
var size = await GetFileSizeAsync(srcdomain, path);
var client = GetClient();
client.CopyObject(_private_container, srcKey, _private_container, dstKey);
QuotaUsedAdd(newdomain, size);
return await GetUriAsync(newdomain, newpath);
}
public override Task CopyDirectoryAsync(string srcdomain, string dir, string newdomain, string newdir)
{
var srckey = MakePath(srcdomain, dir);
var dstkey = MakePath(newdomain, newdir);
var client = GetClient();
var files = client.ListObjects(_private_container, null, null, null, srckey, _region);
foreach (var file in files)
{
client.CopyObject(_private_container, file.Name, _private_container, file.Name.Replace(srckey, dstkey));
QuotaUsedAdd(newdomain, file.Bytes);
}
return Task.CompletedTask;
}
public override async Task<string> SavePrivateAsync(string domain, string path, Stream stream, DateTime expires)
{
var uri = await SaveAsync(domain, path, stream, "application/octet-stream", "attachment", ACL.Auto, null, 5, expires);
return uri.ToString();
}
public override Task DeleteExpiredAsync(string domain, string path, TimeSpan oldThreshold)
{
return Task.CompletedTask;
// When the file is saved is specified life time
}
public override string GetUploadForm(string domain, string directoryPath, string redirectTo, long maxUploadSize, string contentType, string contentDisposition, string submitLabel)
{
throw new NotImplementedException();
}
public override Task<string> GetUploadedUrlAsync(string domain, string directoryPath)
{
throw new NotImplementedException();
}
public override string GetUploadUrl()
{
throw new NotImplementedException();
}
public override string GetPostParams(string domain, string directoryPath, long maxUploadSize, string contentType, string contentDisposition)
{
throw new NotImplementedException();
}
#region chunking
public override Task<string> InitiateChunkedUploadAsync(string domain, string path)
{
return Task.FromResult(TempPath.GetTempFileName());
}
public override async Task<string> UploadChunkAsync(string domain, string path, string filePath, Stream stream, long defaultChunkSize, int chunkNumber, long chunkLength)
{
const int BufferSize = 4096;
var mode = chunkNumber == 0 ? FileMode.Create : FileMode.Append;
using (var fs = new FileStream(filePath, mode))
{
var buffer = new byte[BufferSize];
int readed;
while ((readed = await stream.ReadAsync(buffer, 0, BufferSize)) != 0)
{
await fs.WriteAsync(buffer, 0, readed);
}
}
return string.Format("{0}_{1}", chunkNumber, filePath);
}
public override Task AbortChunkedUploadAsync(string domain, string path, string filePath)
{
if (File.Exists(filePath))
{
File.Delete(filePath);
}
return Task.CompletedTask;
}
public override async Task<Uri> FinalizeChunkedUploadAsync(string domain, string path, string filePath, Dictionary<int, string> eTags)
{
var client = GetClient();
client.CreateObjectFromFile(_private_container, filePath, MakePath(domain, path));
if (File.Exists(filePath))
{
File.Delete(filePath);
}
if (QuotaController != null)
{
var size = await GetFileSizeAsync(domain, path);
QuotaUsedAdd(domain, size);
}
return await GetUriAsync(domain, path);
}
public override bool IsSupportChunking { get { return true; } }
public TempPath TempPath { get; }
#endregion
}
}