DocSpace-client/common/ASC.Data.Storage/GoogleCloud/GoogleCloudStorage.cs

828 lines
27 KiB
C#
Raw Normal View History

2019-06-04 14:43:20 +00:00
/*
*
* (c) Copyright Ascensio System Limited 2010-2018
*
* This program is freeware. You can redistribute it and/or modify it under the terms of the GNU
* General Public License (GPL) version 3 as published by the Free Software Foundation (https://www.gnu.org/copyleft/gpl.html).
* In accordance with Section 7(a) of the GNU GPL its Section 15 shall be amended to the effect that
* Ascensio System SIA expressly excludes the warranty of non-infringement of any third-party rights.
*
* THIS PROGRAM IS DISTRIBUTED WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR
* FITNESS FOR A PARTICULAR PURPOSE. For more details, see GNU GPL at https://www.gnu.org/copyleft/gpl.html
*
* You can contact Ascensio System SIA by email at sales@onlyoffice.com
*
* The interactive user interfaces in modified source and object code versions of ONLYOFFICE must display
* Appropriate Legal Notices, as required under Section 5 of the GNU GPL version 3.
*
* Pursuant to Section 7 § 3(b) of the GNU GPL you must retain the original ONLYOFFICE logo which contains
* relevant author attributions when distributing the software. If the display of the logo in its graphic
* form is not reasonably feasible for technical reasons, you must include the words "Powered by ONLYOFFICE"
* in every copy of the program you distribute.
* Pursuant to Section 7 § 3(e) we decline to grant you any rights under trademark law for use of our trademarks.
*
*/
2022-02-10 11:24:16 +00:00
namespace ASC.Data.Storage.GoogleCloud;
[Scope]
public class GoogleCloudStorage : BaseStorage
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
public override bool IsSupportChunking => true;
private string _subDir = string.Empty;
private Dictionary<string, PredefinedObjectAcl> _domainsAcl;
private PredefinedObjectAcl _moduleAcl;
private string _bucket = string.Empty;
private string _json = string.Empty;
private Uri _bucketRoot;
private Uri _bucketSSlRoot;
private bool _lowerCasing = true;
public GoogleCloudStorage(
TempStream tempStream,
TenantManager tenantManager,
PathUtils pathUtils,
EmailValidationKeyProvider emailValidationKeyProvider,
IHttpContextAccessor httpContextAccessor,
2022-01-13 11:19:39 +00:00
IOptionsMonitor<ILog> options,
IHttpClientFactory clientFactory) : base(tempStream, tenantManager, pathUtils, emailValidationKeyProvider, httpContextAccessor, options, clientFactory)
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
}
2022-02-10 11:24:16 +00:00
public override IDataStore Configure(string tenant, Handler handlerConfig, Module moduleConfig, IDictionary<string, string> props)
{
2022-02-11 10:04:06 +00:00
Tenant = tenant;
2021-11-24 19:34:39 +00:00
2022-02-10 11:24:16 +00:00
if (moduleConfig != null)
2019-06-04 14:43:20 +00:00
{
2022-02-11 10:04:06 +00:00
Modulename = moduleConfig.Name;
DataList = new DataList(moduleConfig);
2019-06-04 14:43:20 +00:00
2022-02-11 10:04:06 +00:00
DomainsExpires = moduleConfig.Domain.Where(x => x.Expires != TimeSpan.Zero).ToDictionary(x => x.Name, y => y.Expires);
2019-06-04 14:43:20 +00:00
2022-02-11 10:04:06 +00:00
DomainsExpires.Add(string.Empty, moduleConfig.Expires);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
_domainsAcl = moduleConfig.Domain.ToDictionary(x => x.Name, y => GetGoogleCloudAcl(y.Acl));
_moduleAcl = GetGoogleCloudAcl(moduleConfig.Acl);
}
else
{
2022-02-11 10:04:06 +00:00
Modulename = string.Empty;
DataList = null;
2019-06-04 14:43:20 +00:00
2022-02-11 10:04:06 +00:00
DomainsExpires = new Dictionary<string, TimeSpan> { { string.Empty, TimeSpan.Zero } };
2022-02-10 11:24:16 +00:00
_domainsAcl = new Dictionary<string, PredefinedObjectAcl>();
_moduleAcl = PredefinedObjectAcl.PublicRead;
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
_bucket = props["bucket"];
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
_bucketRoot = props.ContainsKey("cname") && Uri.IsWellFormedUriString(props["cname"], UriKind.Absolute)
? new Uri(props["cname"], UriKind.Absolute)
2022-01-14 13:12:37 +00:00
: new Uri("https://storage.googleapis.com/" + _bucket + "/", UriKind.Absolute);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
_bucketSSlRoot = props.ContainsKey("cnamessl") &&
Uri.IsWellFormedUriString(props["cnamessl"], UriKind.Absolute)
? new Uri(props["cnamessl"], UriKind.Absolute)
2022-01-14 13:12:37 +00:00
: new Uri("https://storage.googleapis.com/" + _bucket + "/", UriKind.Absolute);
2019-06-04 14:43:20 +00:00
if (props.TryGetValue("lower", out var value))
2022-02-10 11:24:16 +00:00
{
bool.TryParse(value, out _lowerCasing);
2022-02-10 11:24:16 +00:00
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
_json = props["json"];
2019-08-15 12:04:42 +00:00
props.TryGetValue("subdir", out _subDir);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
return this;
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public static long DateToUnixTimestamp(DateTime date)
{
var ts = date - new DateTime(1970, 1, 1, 0, 0, 0);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
return (long)ts.TotalSeconds;
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override Uri GetInternalUri(string domain, string path, TimeSpan expire, IEnumerable<string> headers)
{
if (expire == TimeSpan.Zero || expire == TimeSpan.MinValue || expire == TimeSpan.MaxValue)
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
expire = GetExpire(domain);
2019-06-04 14:43:20 +00:00
}
2022-02-10 11:24:16 +00:00
if (expire == TimeSpan.Zero || expire == TimeSpan.MinValue || expire == TimeSpan.MaxValue)
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
return GetUriShared(domain, path);
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
using var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(_json ?? ""));
var preSignedURL = UrlSigner.FromServiceAccountData(stream).Sign(_bucket, MakePath(domain, path), expire, HttpMethod.Get);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
return MakeUri(preSignedURL);
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public Uri GetUriShared(string domain, string path)
{
2022-02-11 10:04:06 +00:00
return new Uri(SecureHelper.IsSecure(HttpContextAccessor.HttpContext, Options) ? _bucketSSlRoot : _bucketRoot, MakePath(domain, path));
2022-02-10 11:24:16 +00:00
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override Stream GetReadStream(string domain, string path)
{
return GetReadStream(domain, path, 0);
}
public override Stream GetReadStream(string domain, string path, int offset)
{
2022-02-11 10:04:06 +00:00
var tempStream = TempStream.Create();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
using var storage = GetStorage();
storage.DownloadObject(_bucket, MakePath(domain, path), tempStream, null, null);
if (offset > 0)
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
tempStream.Seek(offset, SeekOrigin.Begin);
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
tempStream.Position = 0;
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
return tempStream;
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override async Task<Stream> GetReadStreamAsync(string domain, string path, int offset)
{
2022-02-11 10:04:06 +00:00
var tempStream = TempStream.Create();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
await storage.DownloadObjectAsync(_bucket, MakePath(domain, path), tempStream);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
if (offset > 0)
2021-05-21 13:26:42 +00:00
{
2022-02-10 11:24:16 +00:00
tempStream.Seek(offset, SeekOrigin.Begin);
}
2021-05-21 13:26:42 +00:00
2022-02-10 11:24:16 +00:00
tempStream.Position = 0;
2021-05-21 13:26:42 +00:00
2022-02-10 11:24:16 +00:00
return tempStream;
}
2021-05-21 13:26:42 +00:00
2022-02-10 11:24:16 +00:00
public override Uri Save(string domain, string path, System.IO.Stream stream)
{
return Save(domain, path, stream, string.Empty, string.Empty);
}
2021-05-21 13:26:42 +00:00
2022-02-10 11:24:16 +00:00
public override Uri Save(string domain, string path, System.IO.Stream stream, Configuration.ACL acl)
{
return Save(domain, path, stream, null, null, acl);
}
2021-05-21 13:26:42 +00:00
2022-02-10 11:24:16 +00:00
public override Uri Save(string domain, string path, System.IO.Stream stream, string contentType, string contentDisposition)
{
return Save(domain, path, stream, contentType, contentDisposition, ACL.Auto);
}
2021-05-21 13:26:42 +00:00
2022-02-10 11:24:16 +00:00
public override Uri Save(string domain, string path, System.IO.Stream stream, string contentEncoding, int cacheDays)
{
return Save(domain, path, stream, string.Empty, string.Empty, ACL.Auto, contentEncoding, cacheDays);
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public Uri Save(string domain, string path, Stream stream, string contentType,
string contentDisposition, ACL acl, string contentEncoding = null, int cacheDays = 5)
{
2019-06-04 14:43:20 +00:00
2022-02-11 10:04:06 +00:00
var buffered = TempStream.GetBuffered(stream);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
if (QuotaController != null)
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
QuotaController.QuotaUsedCheck(buffered.Length);
2019-06-04 14:43:20 +00:00
}
2022-02-10 11:24:16 +00:00
var mime = string.IsNullOrEmpty(contentType)
? MimeMapping.GetMimeMapping(Path.GetFileName(path))
: contentType;
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
using var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var uploadObjectOptions = new UploadObjectOptions
{
PredefinedAcl = acl == ACL.Auto ? GetDomainACL(domain) : GetGoogleCloudAcl(acl)
};
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
buffered.Position = 0;
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var uploaded = storage.UploadObject(_bucket, MakePath(domain, path), mime, buffered, uploadObjectOptions, null);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
uploaded.ContentEncoding = contentEncoding;
uploaded.CacheControl = string.Format("public, maxage={0}", (int)TimeSpan.FromDays(cacheDays).TotalSeconds);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
if (uploaded.Metadata == null)
{
uploaded.Metadata = new Dictionary<string, string>();
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
uploaded.Metadata["Expires"] = DateTime.UtcNow.Add(TimeSpan.FromDays(cacheDays)).ToString("R");
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
if (!string.IsNullOrEmpty(contentDisposition))
{
uploaded.ContentDisposition = contentDisposition;
}
else if (mime == "application/octet-stream")
{
uploaded.ContentDisposition = "attachment";
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
storage.UpdateObject(uploaded);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
// InvalidateCloudFront(MakePath(domain, path));
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
QuotaUsedAdd(domain, buffered.Length);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
return GetUri(domain, path);
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override void Delete(string domain, string path)
{
using var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var key = MakePath(domain, path);
var size = GetFileSize(domain, path);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
storage.DeleteObject(_bucket, key);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
QuotaUsedDelete(domain, size);
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override void DeleteFiles(string domain, string folderPath, string pattern, bool recursive)
{
using var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
IEnumerable<Google.Apis.Storage.v1.Data.Object> objToDel;
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
if (recursive)
{
objToDel = storage
.ListObjects(_bucket, MakePath(domain, folderPath))
.Where(x => Wildcard.IsMatch(pattern, Path.GetFileName(x.Name)));
2019-06-04 14:43:20 +00:00
}
2022-02-10 11:24:16 +00:00
else
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
objToDel = new List<Google.Apis.Storage.v1.Data.Object>();
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
foreach (var obj in objToDel)
{
storage.DeleteObject(_bucket, obj.Name);
QuotaUsedDelete(domain, Convert.ToInt64(obj.Size));
2019-06-04 14:43:20 +00:00
}
2022-02-10 11:24:16 +00:00
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override void DeleteFiles(string domain, List<string> paths)
{
if (paths.Count == 0) return;
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var keysToDel = new List<string>();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
long quotaUsed = 0;
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
foreach (var path in paths)
{
try
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
var key = MakePath(domain, path);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
if (QuotaController != null)
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
quotaUsed += GetFileSize(domain, path);
2019-06-04 14:43:20 +00:00
}
2022-02-10 11:24:16 +00:00
keysToDel.Add(key);
2022-02-10 11:06:37 +00:00
}
2022-02-10 11:24:16 +00:00
catch (FileNotFoundException)
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
2019-06-04 14:43:20 +00:00
}
}
if (keysToDel.Count == 0) return;
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
using var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
keysToDel.ForEach(x => storage.DeleteObject(_bucket, x));
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
if (quotaUsed > 0)
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
QuotaUsedDelete(domain, quotaUsed);
}
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override void DeleteFiles(string domain, string folderPath, DateTime fromDate, DateTime toDate)
{
using var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var objToDel = GetObjects(domain, folderPath, true)
.Where(x => x.Updated >= fromDate && x.Updated <= toDate);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
foreach (var obj in objToDel)
{
storage.DeleteObject(_bucket, obj.Name);
QuotaUsedDelete(domain, Convert.ToInt64(obj.Size));
2019-06-04 14:43:20 +00:00
}
2022-02-10 11:24:16 +00:00
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override void MoveDirectory(string srcdomain, string srcdir, string newdomain, string newdir)
{
using var storage = GetStorage();
var srckey = MakePath(srcdomain, srcdir);
var dstkey = MakePath(newdomain, newdir);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var objects = storage.ListObjects(_bucket, srckey);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
foreach (var obj in objects)
{
storage.CopyObject(_bucket, srckey, _bucket, dstkey, new CopyObjectOptions
2019-06-04 14:43:20 +00:00
{
DestinationPredefinedAcl = GetDomainACL(newdomain)
});
2022-02-10 11:24:16 +00:00
storage.DeleteObject(_bucket, srckey);
2019-06-04 14:43:20 +00:00
}
2022-02-10 11:24:16 +00:00
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override Uri Move(string srcdomain, string srcpath, string newdomain, string newpath, bool quotaCheckFileSize = true)
{
using var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var srcKey = MakePath(srcdomain, srcpath);
var dstKey = MakePath(newdomain, newpath);
var size = GetFileSize(srcdomain, srcpath);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
storage.CopyObject(_bucket, srcKey, _bucket, dstKey, new CopyObjectOptions
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
DestinationPredefinedAcl = GetDomainACL(newdomain)
});
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
Delete(srcdomain, srcpath);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
QuotaUsedDelete(srcdomain, size);
QuotaUsedAdd(newdomain, size, quotaCheckFileSize);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
return GetUri(newdomain, newpath);
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override Uri SaveTemp(string domain, out string assignedPath, System.IO.Stream stream)
{
assignedPath = Guid.NewGuid().ToString();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
return Save(domain, assignedPath, stream);
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override string[] ListDirectoriesRelative(string domain, string path, bool recursive)
{
return GetObjects(domain, path, recursive)
.Select(x => x.Name.Substring(MakePath(domain, path + "/").Length))
.ToArray();
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
private IEnumerable<Google.Apis.Storage.v1.Data.Object> GetObjects(string domain, string path, bool recursive)
{
using var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var items = storage.ListObjects(_bucket, MakePath(domain, path));
2021-05-21 13:26:42 +00:00
2022-02-10 11:24:16 +00:00
if (recursive)
2021-05-21 13:26:42 +00:00
{
2022-02-10 11:24:16 +00:00
return items;
2019-06-04 14:43:20 +00:00
}
2022-02-10 11:24:16 +00:00
return items.Where(x => x.Name.IndexOf('/', MakePath(domain, path + "/").Length) == -1);
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override string[] ListFilesRelative(string domain, string path, string pattern, bool recursive)
{
return GetObjects(domain, path, recursive).Where(x => Wildcard.IsMatch(pattern, Path.GetFileName(x.Name)))
.Select(x => x.Name.Substring(MakePath(domain, path + "/").Length).TrimStart('/')).ToArray();
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override bool IsFile(string domain, string path)
{
using var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var objects = storage.ListObjects(_bucket, MakePath(domain, path), null);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
return objects.Any();
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override async Task<bool> IsFileAsync(string domain, string path)
{
var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var objects = await storage.ListObjectsAsync(_bucket, MakePath(domain, path)).ReadPageAsync(1);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
return objects.Any();
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override bool IsDirectory(string domain, string path)
{
return IsFile(domain, path);
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override void DeleteDirectory(string domain, string path)
{
using var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var objToDel = storage
.ListObjects(_bucket, MakePath(domain, path));
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
foreach (var obj in objToDel)
{
storage.DeleteObject(_bucket, obj.Name);
QuotaUsedDelete(domain, Convert.ToInt64(obj.Size));
2019-06-04 14:43:20 +00:00
}
2022-02-10 11:24:16 +00:00
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override long GetFileSize(string domain, string path)
{
using var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var obj = storage.GetObject(_bucket, MakePath(domain, path));
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
return obj.Size.HasValue ? Convert.ToInt64(obj.Size.Value) : 0;
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override long GetDirectorySize(string domain, string path)
{
using var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var objToDel = storage
.ListObjects(_bucket, MakePath(domain, path));
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
long result = 0;
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
foreach (var obj in objToDel)
{
if (obj.Size.HasValue)
{
result += Convert.ToInt64(obj.Size.Value);
}
2019-06-04 14:43:20 +00:00
}
2022-02-10 11:24:16 +00:00
return result;
}
public override long ResetQuota(string domain)
{
using var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var objects = storage
.ListObjects(_bucket, MakePath(domain, string.Empty));
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
if (QuotaController != null)
{
long size = 0;
2019-06-04 14:43:20 +00:00
foreach (var obj in objects)
{
if (obj.Size.HasValue)
2022-02-10 11:06:37 +00:00
{
2022-02-10 11:24:16 +00:00
size += Convert.ToInt64(obj.Size.Value);
2022-02-10 11:06:37 +00:00
}
2019-06-04 14:43:20 +00:00
}
2022-02-11 10:04:06 +00:00
QuotaController.QuotaUsedSet(Modulename, domain, DataList.GetData(domain), size);
2022-02-10 11:24:16 +00:00
return size;
2019-06-04 14:43:20 +00:00
}
2022-02-10 11:24:16 +00:00
return 0;
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override long GetUsedQuota(string domain)
{
using var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var objects = storage
.ListObjects(_bucket, MakePath(domain, string.Empty));
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
long result = 0;
foreach (var obj in objects)
{
if (obj.Size.HasValue)
2019-08-15 13:35:18 +00:00
{
2022-02-10 11:24:16 +00:00
result += Convert.ToInt64(obj.Size.Value);
}
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
return result;
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override Uri Copy(string srcdomain, string srcpath, string newdomain, string newpath)
{
using var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var size = GetFileSize(srcdomain, srcpath);
var options = new CopyObjectOptions
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
DestinationPredefinedAcl = GetDomainACL(newdomain)
};
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
storage.CopyObject(_bucket, MakePath(srcdomain, srcpath), _bucket, MakePath(newdomain, newpath), options);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
QuotaUsedAdd(newdomain, size);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
return GetUri(newdomain, newpath);
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override void CopyDirectory(string srcdomain, string srcdir, string newdomain, string newdir)
{
var srckey = MakePath(srcdomain, srcdir);
var dstkey = MakePath(newdomain, newdir);
//List files from src
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
using var storage = GetStorage();
var objects = storage.ListObjects(_bucket, srckey);
foreach (var obj in objects)
{
storage.CopyObject(_bucket, srckey, _bucket, dstkey, new CopyObjectOptions
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
DestinationPredefinedAcl = GetDomainACL(newdomain)
});
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
QuotaUsedAdd(newdomain, Convert.ToInt64(obj.Size));
2019-06-04 14:43:20 +00:00
}
2022-02-10 11:24:16 +00:00
}
public override string SavePrivate(string domain, string path, System.IO.Stream stream, DateTime expires)
{
using var storage = GetStorage();
2022-02-11 10:04:06 +00:00
var buffered = TempStream.GetBuffered(stream);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var uploadObjectOptions = new UploadObjectOptions
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
PredefinedAcl = PredefinedObjectAcl.BucketOwnerFullControl
};
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
buffered.Position = 0;
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var uploaded = storage.UploadObject(_bucket, MakePath(domain, path), "application/octet-stream", buffered, uploadObjectOptions, null);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
uploaded.CacheControl = string.Format("public, maxage={0}", (int)TimeSpan.FromDays(5).TotalSeconds);
uploaded.ContentDisposition = "attachment";
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
if (uploaded.Metadata == null)
{
uploaded.Metadata = new Dictionary<string, string>();
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
uploaded.Metadata["Expires"] = DateTime.UtcNow.Add(TimeSpan.FromDays(5)).ToString("R");
uploaded.Metadata.Add("private-expire", expires.ToFileTimeUtc().ToString(CultureInfo.InvariantCulture));
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
storage.UpdateObject(uploaded);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
using var mStream = new MemoryStream(Encoding.UTF8.GetBytes(_json ?? ""));
var preSignedURL = FromServiceAccountData(mStream).Sign(RequestTemplate.FromBucket(_bucket).WithObjectName(MakePath(domain, path)), UrlSigner.Options.FromExpiration(expires));
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
//TODO: CNAME!
return preSignedURL;
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override void DeleteExpired(string domain, string path, TimeSpan oldThreshold)
{
using var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var objects = storage.ListObjects(_bucket, MakePath(domain, path));
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
foreach (var obj in objects)
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
var objInfo = storage.GetObject(_bucket, MakePath(domain, path), null);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var privateExpireKey = objInfo.Metadata["private-expire"];
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
if (string.IsNullOrEmpty(privateExpireKey))
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
continue;
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
if (!long.TryParse(privateExpireKey, out var fileTime))
{
continue;
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
if (DateTime.UtcNow <= DateTime.FromFileTimeUtc(fileTime))
{
continue;
}
2022-02-10 11:06:37 +00:00
2022-02-10 11:24:16 +00:00
storage.DeleteObject(_bucket, MakePath(domain, path));
2022-02-10 11:06:37 +00:00
2022-02-10 11:24:16 +00:00
}
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
#region chunking
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override string InitiateChunkedUpload(string domain, string path)
{
using var storage = GetStorage();
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var tempUploader = storage.CreateObjectUploader(_bucket, MakePath(domain, path), null, new MemoryStream());
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var sessionUri = tempUploader.InitiateSessionAsync().Result;
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
return sessionUri.ToString();
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override string UploadChunk(string domain,
string path,
string uploadUri,
Stream stream,
long defaultChunkSize,
int chunkNumber,
long chunkLength)
{
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var bytesRangeStart = Convert.ToString((chunkNumber - 1) * defaultChunkSize);
var bytesRangeEnd = Convert.ToString((chunkNumber - 1) * defaultChunkSize + chunkLength - 1);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var totalBytes = "*";
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
if (chunkLength != defaultChunkSize)
{
totalBytes = Convert.ToString((chunkNumber - 1) * defaultChunkSize + chunkLength);
}
2019-06-04 14:43:20 +00:00
var contentRangeHeader = $"bytes {bytesRangeStart}-{bytesRangeEnd}/{totalBytes}";
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
var request = new HttpRequestMessage();
request.RequestUri = new Uri(uploadUri);
request.Method = HttpMethod.Put;
request.Headers.Add("Content-Range", contentRangeHeader);
request.Content = new StreamContent(stream);
2019-06-04 14:43:20 +00:00
const int MAX_RETRIES = 100;
2022-02-10 11:24:16 +00:00
int millisecondsTimeout;
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
for (var i = 0; i < MAX_RETRIES; i++)
{
millisecondsTimeout = Math.Min(Convert.ToInt32(Math.Pow(2, i)) + RandomNumberGenerator.GetInt32(1000), 32 * 1000);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
try
2019-06-04 14:43:20 +00:00
{
var httpClient = ClientFactory.CreateClient();
2022-02-10 11:24:16 +00:00
using var response = httpClient.Send(request);
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
break;
}
catch (HttpRequestException ex)
{
var status = (int)ex.StatusCode;
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
if (status == 408 || status == 500 || status == 502 || status == 503 || status == 504)
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
Thread.Sleep(millisecondsTimeout);
continue;
2019-06-04 14:43:20 +00:00
}
2022-02-10 11:24:16 +00:00
if (status != 308)
2019-06-04 14:43:20 +00:00
{
2020-11-17 10:47:17 +00:00
throw;
2019-06-04 14:43:20 +00:00
}
2022-02-10 11:24:16 +00:00
break;
}
catch
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
AbortChunkedUpload(domain, path, uploadUri);
throw;
2019-06-04 14:43:20 +00:00
}
}
2022-02-10 11:24:16 +00:00
return string.Empty;
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override Uri FinalizeChunkedUpload(string domain, string path, string uploadUri, Dictionary<int, string> eTags)
{
if (QuotaController != null)
2019-06-04 14:43:20 +00:00
{
2022-02-10 11:24:16 +00:00
var size = GetFileSize(domain, path);
QuotaUsedAdd(domain, size);
2019-06-04 14:43:20 +00:00
}
2022-02-10 11:24:16 +00:00
return GetUri(domain, path);
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override void AbortChunkedUpload(string domain, string path, string uploadUri) { }
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
#endregion
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
public override string GetUploadForm(string domain, string directoryPath, string redirectTo, long maxUploadSize, string contentType, string contentDisposition, string submitLabel)
{
throw new NotImplementedException();
}
2022-02-10 11:24:16 +00:00
public override string GetUploadedUrl(string domain, string directoryPath)
{
throw new NotImplementedException();
}
2022-02-10 11:24:16 +00:00
public override string GetUploadUrl()
{
throw new NotImplementedException();
}
2022-02-10 11:24:16 +00:00
public override string GetPostParams(string domain, string directoryPath, long maxUploadSize, string contentType, string contentDisposition)
{
throw new NotImplementedException();
}
2022-02-10 11:24:16 +00:00
protected override Uri SaveWithAutoAttachment(string domain, string path, System.IO.Stream stream, string attachmentFileName)
{
var contentDisposition = $"attachment; filename={HttpUtility.UrlPathEncode(attachmentFileName)};";
2022-02-10 11:24:16 +00:00
if (attachmentFileName.Any(c => c >= 0 && c <= 127))
{
contentDisposition = $"attachment; filename*=utf-8''{HttpUtility.UrlPathEncode(attachmentFileName)};";
2022-02-10 11:24:16 +00:00
}
return Save(domain, path, stream, null, contentDisposition);
}
2022-02-10 11:24:16 +00:00
private StorageClient GetStorage()
{
var credential = GoogleCredential.FromJson(_json);
return StorageClient.Create(credential);
}
2022-02-10 11:24:16 +00:00
private string MakePath(string domain, string path)
{
string result;
path = path.TrimStart('\\', '/').TrimEnd('/').Replace('\\', '/');
2022-02-10 11:24:16 +00:00
if (!string.IsNullOrEmpty(_subDir))
{
if (_subDir.Length == 1 && (_subDir[0] == '/' || _subDir[0] == '\\'))
result = path;
else
result = $"{_subDir}/{path}"; // Ignory all, if _subDir is not null
}
2022-02-10 11:24:16 +00:00
else//Key combined from module+domain+filename
result = $"{Tenant}/{Modulename}/{domain}/{path}";
2022-02-10 11:24:16 +00:00
result = result.Replace("//", "/").TrimStart('/');
if (_lowerCasing)
{
2022-02-10 11:24:16 +00:00
result = result.ToLowerInvariant();
}
2022-02-10 11:24:16 +00:00
return result;
}
private Uri MakeUri(string preSignedURL)
{
var uri = new Uri(preSignedURL);
var signedPart = uri.PathAndQuery.TrimStart('/');
return new Uri(uri.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase) ? _bucketSSlRoot : _bucketRoot, signedPart);
}
private void InvalidateCloudFront(params string[] paths)
{
throw new NotImplementedException();
}
private PredefinedObjectAcl GetGoogleCloudAcl(ACL acl)
{
return PredefinedObjectAcl.PublicRead;
//return acl switch
//{
// ACL.Read => PredefinedObjectAcl.PublicRead,
// _ => PredefinedObjectAcl.PublicRead,
//};
}
private PredefinedObjectAcl GetDomainACL(string domain)
{
if (GetExpire(domain) != TimeSpan.Zero)
{
2022-02-10 11:24:16 +00:00
return PredefinedObjectAcl.Private;
}
if (_domainsAcl.TryGetValue(domain, out var value))
{
return value;
}
2022-02-10 11:24:16 +00:00
return _moduleAcl;
}
2019-06-04 14:43:20 +00:00
2022-02-10 11:24:16 +00:00
}