Merge pull request #1465 from ONLYOFFICE/feature/cdn-for-dynamic-content

s3: added support presigned cdn url for Media/Image viewer
This commit is contained in:
Alexey Bannov 2023-06-10 09:41:51 +03:00 committed by GitHub
commit 083cf7e049
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 130 additions and 16 deletions

View File

@ -30,6 +30,7 @@ public abstract class BaseStorage : IDataStore
{ {
public IQuotaController QuotaController { get; set; } public IQuotaController QuotaController { get; set; }
public virtual bool IsSupportInternalUri => true; public virtual bool IsSupportInternalUri => true;
public virtual bool IsSupportCdnUri => false;
public virtual bool IsSupportedPreSignedUri => true; public virtual bool IsSupportedPreSignedUri => true;
public virtual bool IsSupportChunking => false; public virtual bool IsSupportChunking => false;
internal string Modulename { get; set; } internal string Modulename { get; set; }
@ -154,6 +155,11 @@ public abstract class BaseStorage : IDataStore
return null; return null;
} }
public virtual Task<Uri> GetCdnPreSignedUriAsync(string domain, string path, TimeSpan expire, IEnumerable<string> headers)
{
return null;
}
public abstract Task<Stream> GetReadStreamAsync(string domain, string path); public abstract Task<Stream> GetReadStreamAsync(string domain, string path);
public abstract Task<Stream> GetReadStreamAsync(string domain, string path, long offset); public abstract Task<Stream> GetReadStreamAsync(string domain, string path, long offset);

View File

@ -63,12 +63,20 @@ public interface IDataStore
/// <param name="headers"></param> /// <param name="headers"></param>
/// <returns></returns> /// <returns></returns>
Task<Uri> GetPreSignedUriAsync(string domain, string path, TimeSpan expire, IEnumerable<string> headers); Task<Uri> GetPreSignedUriAsync(string domain, string path, TimeSpan expire, IEnumerable<string> headers);
///<summary> ///<summary>
/// Supporting generate uri to the file /// Supporting generate uri to the file
///</summary> ///</summary>
///<returns></returns> ///<returns></returns>
bool IsSupportInternalUri { get; } bool IsSupportInternalUri { get; }
///<summary>
/// Supporting generate uri to the file
///</summary>
///<returns></returns>
bool IsSupportCdnUri { get; }
/// <summary> /// <summary>
/// Get absolute Uri for html links /// Get absolute Uri for html links
/// </summary> /// </summary>
@ -79,6 +87,16 @@ public interface IDataStore
/// <returns></returns> /// <returns></returns>
Task<Uri> GetInternalUriAsync(string domain, string path, TimeSpan expire, IEnumerable<string> headers); Task<Uri> GetInternalUriAsync(string domain, string path, TimeSpan expire, IEnumerable<string> headers);
/// <summary>
/// Get absolute Uri via CDN for html links
/// </summary>
/// <param name="domain"></param>
/// <param name="path"></param>
/// <param name="expire"></param>
/// <param name="headers"></param>
/// <returns></returns>
Task<Uri> GetCdnPreSignedUriAsync(string domain, string path, TimeSpan expire, IEnumerable<string> headers);
///<summary> ///<summary>
/// A stream of read-only. In the case of the C3 stream NetworkStream general, and with him we have to work /// A stream of read-only. In the case of the C3 stream NetworkStream general, and with him we have to work
/// Very carefully as a Jedi cutter groin lightsaber. /// Very carefully as a Jedi cutter groin lightsaber.

View File

@ -33,6 +33,7 @@ namespace ASC.Data.Storage.S3;
[Scope] [Scope]
public class S3Storage : BaseStorage public class S3Storage : BaseStorage
{ {
public override bool IsSupportCdnUri => true;
public override bool IsSupportChunking => true; public override bool IsSupportChunking => true;
private readonly List<string> _domains = new List<string>(); private readonly List<string> _domains = new List<string>();
@ -50,8 +51,10 @@ public class S3Storage : BaseStorage
private readonly ServerSideEncryptionMethod _sse = ServerSideEncryptionMethod.AES256; private readonly ServerSideEncryptionMethod _sse = ServerSideEncryptionMethod.AES256;
private bool _useHttp = true; private bool _useHttp = true;
private bool _lowerCasing = true; private bool _lowerCasing = true;
private bool _revalidateCloudFront; private bool _cdnEnabled;
private string _distributionId = ""; private string _cdnKeyPairId;
private string _cdnPrivateKeyPath;
private string _cdnDistributionDomain;
private string _subDir = ""; private string _subDir = "";
private EncryptionMethod _encryptionMethod = EncryptionMethod.None; private EncryptionMethod _encryptionMethod = EncryptionMethod.None;
@ -145,9 +148,80 @@ public class S3Storage : BaseStorage
} }
using var client = GetClient(); using var client = GetClient();
return Task.FromResult(MakeUri(client.GetPreSignedURL(pUrlRequest))); return Task.FromResult(MakeUri(client.GetPreSignedURL(pUrlRequest)));
} }
public override Task<Uri> GetCdnPreSignedUriAsync(string domain, string path, TimeSpan expire, IEnumerable<string> headers)
{
if (!_cdnEnabled) return GetInternalUriAsync(domain, path, expire, headers);
var proto = SecureHelper.IsSecure(_httpContextAccessor?.HttpContext, _options) ? "https" : "http";
var baseUrl = $"{proto}://{_cdnDistributionDomain}/{MakePath(domain, path)}";
var uriBuilder = new UriBuilder(baseUrl)
{
Port = -1
};
var queryParams = HttpUtility.ParseQueryString(uriBuilder.Query);
if (headers != null && headers.Any())
{
foreach (var h in headers)
{
if (h.StartsWith("Content-Disposition"))
{
queryParams["response-content-disposition"] = h.Substring("Content-Disposition".Length + 1);
}
else if (h.StartsWith("Cache-Control"))
{
queryParams["response-cache-control"] = h.Substring("Cache-Control".Length + 1);
}
else if (h.StartsWith("Content-Encoding"))
{
queryParams["response-content-encoding"] = h.Substring("Content-Encoding".Length + 1);
}
else if (h.StartsWith("Content-Language"))
{
queryParams["response-content-language"] = h.Substring("Content-Language".Length + 1);
}
else if (h.StartsWith("Content-Type"))
{
queryParams["response-content-type"] = h.Substring("Content-Type".Length + 1);
}
else if (h.StartsWith("Expires"))
{
queryParams["response-expires"] = h.Substring("Expires".Length + 1);
}
else if (h.StartsWith("Custom-Cache-Key"))
{
queryParams["custom-cache-key"] = h.Substring("Custom-Cache-Key".Length + 1);
}
else
{
throw new FormatException(string.Format("Invalid header: {0}", h));
}
}
}
uriBuilder.Query = queryParams.ToString();
var signedUrl = "";
using (TextReader textReader = File.OpenText(_cdnPrivateKeyPath))
{
signedUrl = AmazonCloudFrontUrlSigner.GetCannedSignedURL(
uriBuilder.ToString(),
textReader,
_cdnKeyPairId,
DateTime.UtcNow.Add(expire));
}
return Task.FromResult(new Uri(signedUrl));
}
public override Task<Stream> GetReadStreamAsync(string domain, string path) public override Task<Stream> GetReadStreamAsync(string domain, string path)
{ {
return GetReadStreamAsync(domain, path, 0); return GetReadStreamAsync(domain, path, 0);
@ -217,12 +291,7 @@ public class S3Storage : BaseStorage
ContentType = mime, ContentType = mime,
ServerSideEncryptionMethod = _sse, ServerSideEncryptionMethod = _sse,
InputStream = buffered, InputStream = buffered,
AutoCloseStream = false, AutoCloseStream = false
Headers =
{
CacheControl = string.Format("public, maxage={0}", (int)TimeSpan.FromDays(cacheDays).TotalSeconds),
ExpiresUtc = DateTime.UtcNow.Add(TimeSpan.FromDays(cacheDays))
}
}; };
if (!(client is IAmazonS3Encryption)) if (!(client is IAmazonS3Encryption))
@ -259,7 +328,7 @@ public class S3Storage : BaseStorage
await uploader.UploadAsync(request); await uploader.UploadAsync(request);
await InvalidateCloudFrontAsync(MakePath(domain, path)); //await InvalidateCloudFrontAsync(MakePath(domain, path));
await QuotaUsedAdd(domain, buffered.Length); await QuotaUsedAdd(domain, buffered.Length);
@ -348,7 +417,7 @@ public class S3Storage : BaseStorage
using (var s3 = GetClient()) using (var s3 = GetClient())
{ {
await s3.CompleteMultipartUploadAsync(request); await s3.CompleteMultipartUploadAsync(request);
await InvalidateCloudFrontAsync(MakePath(domain, path)); // await InvalidateCloudFrontAsync(MakePath(domain, path));
} }
if (QuotaController != null) if (QuotaController != null)
@ -969,12 +1038,17 @@ public class S3Storage : BaseStorage
{ {
bool.TryParse(lower, out _lowerCasing); bool.TryParse(lower, out _lowerCasing);
} }
if (props.TryGetValue("cloudfront", out var front))
if (props.TryGetValue("cdn_enabled", out var cdnEnabled))
{ {
bool.TryParse(front, out _revalidateCloudFront); if (bool.TryParse(cdnEnabled, out _cdnEnabled))
{
_cdnKeyPairId = props["cdn_keyPairId"];
_cdnPrivateKeyPath = props["cdn_privateKeyPath"];
_cdnDistributionDomain = props["cdn_distributionDomain"];
}
} }
props.TryGetValue("distribution", out _distributionId);
props.TryGetValue("subdir", out _subDir); props.TryGetValue("subdir", out _subDir);
return this; return this;
@ -1026,10 +1100,10 @@ public class S3Storage : BaseStorage
return new UnencodedUri(baseUri, signedPart); return new UnencodedUri(baseUri, signedPart);
} }
private Task InvalidateCloudFrontAsync(params string[] paths) private Task InvalidateCloudFrontAsync(params string[] paths)
{ {
if (!_revalidateCloudFront || string.IsNullOrEmpty(_distributionId)) if (!_cdnEnabled || string.IsNullOrEmpty(_cdnDistributionDomain))
{ {
return Task.CompletedTask; return Task.CompletedTask;
} }
@ -1042,7 +1116,7 @@ public class S3Storage : BaseStorage
using var cfClient = GetCloudFrontClient(); using var cfClient = GetCloudFrontClient();
var invalidationRequest = new CreateInvalidationRequest var invalidationRequest = new CreateInvalidationRequest
{ {
DistributionId = _distributionId, DistributionId = _cdnDistributionDomain,
InvalidationBatch = new InvalidationBatch InvalidationBatch = new InvalidationBatch
{ {
CallerReference = Guid.NewGuid().ToString(), CallerReference = Guid.NewGuid().ToString(),

View File

@ -48,11 +48,13 @@ internal class FileDao : AbstractDao, IFileDao<int>
private readonly IQuotaService _quotaService; private readonly IQuotaService _quotaService;
private readonly StorageFactory _storageFactory; private readonly StorageFactory _storageFactory;
private readonly TenantQuotaController _tenantQuotaController; private readonly TenantQuotaController _tenantQuotaController;
private readonly FileUtility _fileUtility;
public FileDao( public FileDao(
ILogger<FileDao> logger, ILogger<FileDao> logger,
FactoryIndexerFile factoryIndexer, FactoryIndexerFile factoryIndexer,
UserManager userManager, UserManager userManager,
FileUtility fileUtility,
IDbContextFactory<FilesDbContext> dbContextManager, IDbContextFactory<FilesDbContext> dbContextManager,
TenantManager tenantManager, TenantManager tenantManager,
TenantUtil tenantUtil, TenantUtil tenantUtil,
@ -108,6 +110,7 @@ internal class FileDao : AbstractDao, IFileDao<int>
_quotaService = quotaService; _quotaService = quotaService;
_storageFactory = storageFactory; _storageFactory = storageFactory;
_tenantQuotaController = tenantQuotaController; _tenantQuotaController = tenantQuotaController;
_fileUtility = fileUtility;
} }
public Task InvalidateCacheAsync(int fileId) public Task InvalidateCacheAsync(int fileId)
@ -377,6 +380,19 @@ internal class FileDao : AbstractDao, IFileDao<int>
public Task<Uri> GetPreSignedUriAsync(File<int> file, TimeSpan expires) public Task<Uri> GetPreSignedUriAsync(File<int> file, TimeSpan expires)
{ {
var storage = _globalStore.GetStore();
if (storage.IsSupportCdnUri && !_fileUtility.CanWebEdit(file.Title)
&& (_fileUtility.CanMediaView(file.Title) || _fileUtility.CanImageView(file.Title)))
{
return _globalStore.GetStore().GetCdnPreSignedUriAsync(string.Empty, GetUniqFilePath(file), expires,
new List<string>
{
$"Content-Disposition:{ContentDispositionUtil.GetHeaderValue(file.Title, withoutBase: true)}",
$"Custom-Cache-Key:{file.ModifiedOn.Ticks}"
});
}
return _globalStore.GetStore().GetPreSignedUriAsync(string.Empty, GetUniqFilePath(file), expires, return _globalStore.GetStore().GetPreSignedUriAsync(string.Empty, GetUniqFilePath(file), expires,
new List<string> new List<string>
{ {