diff --git a/common/ASC.Data.Backup.Core/GlobalUsings.cs b/common/ASC.Data.Backup.Core/GlobalUsings.cs index e515f7a4a8..2a544ba07e 100644 --- a/common/ASC.Data.Backup.Core/GlobalUsings.cs +++ b/common/ASC.Data.Backup.Core/GlobalUsings.cs @@ -36,20 +36,18 @@ global using System.Text; global using System.Text.RegularExpressions; global using System.Xml; global using System.Xml.Linq; -global using System.Xml.XPath; global using ASC.Api.Utils; global using ASC.Common; global using ASC.Common.Caching; global using ASC.Common.Log; -global using ASC.Common.Threading; +global using ASC.Common.Threading; global using ASC.Common.Utils; global using ASC.Core; global using ASC.Core.Billing; global using ASC.Core.ChunkedUploader; global using ASC.Core.Common.Configuration; global using ASC.Core.Common.EF; -global using ASC.Core.Common.EF.Context; global using ASC.Core.Common.EF.Model; global using ASC.Core.Tenants; global using ASC.Core.Users; @@ -70,6 +68,7 @@ global using ASC.Data.Backup.Utils; global using ASC.Data.Storage; global using ASC.Data.Storage.Configuration; global using ASC.Data.Storage.DiscStorage; +global using ASC.Data.Storage.S3; global using ASC.Data.Storage.ZipOperators; global using ASC.EventBus.Events; global using ASC.Files.Core; diff --git a/common/ASC.Data.Backup.Core/Service/BackupWorker.cs b/common/ASC.Data.Backup.Core/Service/BackupWorker.cs index 269fb08e8e..49a55939ce 100644 --- a/common/ASC.Data.Backup.Core/Service/BackupWorker.cs +++ b/common/ASC.Data.Backup.Core/Service/BackupWorker.cs @@ -221,7 +221,7 @@ public class BackupWorker } } - internal static string GetBackupHash(string path) + internal static string GetBackupHashSHA(string path) { using (var sha256 = SHA256.Create()) using (var fileStream = File.OpenRead(path)) @@ -230,6 +230,43 @@ public class BackupWorker var hash = sha256.ComputeHash(fileStream); return BitConverter.ToString(hash).Replace("-", string.Empty); } + } + + internal static string GetBackupHashMD5(string path, long chunkSize) + { + using (var md5 = MD5.Create()) + using (var fileStream = File.OpenRead(path)) + {var multipartSplitCount = 0; + var splitCount = fileStream.Length / chunkSize; + var mod = (int)(fileStream.Length - chunkSize * splitCount); + IEnumerable concatHash = new byte[] { }; + + for (var i = 0; i < splitCount; i++) + { + var offset = i == 0 ? 0 : chunkSize * i; + var chunk = GetChunk(fileStream, offset, (int)chunkSize); + var hash = md5.ComputeHash(chunk); + concatHash = concatHash.Concat(hash); + multipartSplitCount++; + } + if (mod != 0) + { + var chunk = GetChunk(fileStream, chunkSize * splitCount, mod); + var hash = md5.ComputeHash(chunk); + concatHash = concatHash.Concat(hash); + multipartSplitCount++; + } + var multipartHash = BitConverter.ToString(md5.ComputeHash(concatHash.ToArray())).Replace("-", string.Empty); + return multipartHash + "-" + multipartSplitCount; + } + } + + private static byte[] GetChunk(Stream sourceStream, long offset, int count) + { + var buffer = new byte[count]; + sourceStream.Position = offset; + sourceStream.Read(buffer, 0, count); + return buffer; } private BackupProgress ToBackupProgress(BaseBackupProgressItem progressItem) diff --git a/common/ASC.Data.Backup.Core/Service/ProgressItems/BackupProgressItem.cs b/common/ASC.Data.Backup.Core/Service/ProgressItems/BackupProgressItem.cs index 4a33730a37..d48a149f4d 100644 --- a/common/ASC.Data.Backup.Core/Service/ProgressItems/BackupProgressItem.cs +++ b/common/ASC.Data.Backup.Core/Service/ProgressItems/BackupProgressItem.cs @@ -31,9 +31,7 @@ namespace ASC.Data.Backup.Services; public class BackupProgressItem : BaseBackupProgressItem { public Dictionary StorageParams { get; set; } - public string TempFolder { get; set; } - - private const string ArchiveFormat = "tar"; + public string TempFolder { get; set; } private bool _isScheduled; private Guid _userId; @@ -97,16 +95,21 @@ public class BackupProgressItem : BaseBackupProgressItem _tempStream = scope.ServiceProvider.GetService(); var dateTime = _coreBaseSettings.Standalone ? DateTime.Now : DateTime.UtcNow; - var backupName = string.Format("{0}_{1:yyyy-MM-dd_HH-mm-ss}.{2}", (await _tenantManager.GetTenantAsync(TenantId)).Alias, dateTime, ArchiveFormat); - - var tempFile = CrossPlatform.PathCombine(TempFolder, backupName); - var storagePath = tempFile; string hash; + var tempFile = ""; + var storagePath = ""; try { var backupStorage = await _backupStorageFactory.GetBackupStorageAsync(_storageType, TenantId, StorageParams); - var writer = await ZipWriteOperatorFactory.GetWriteOperatorAsync(_tempStream, _storageBasePath, backupName, TempFolder, _userId, backupStorage as IGetterWriteOperator); + + var getter = backupStorage as IGetterWriteOperator; + var backupName = string.Format("{0}_{1:yyyy-MM-dd_HH-mm-ss}.{2}", (await _tenantManager.GetTenantAsync(TenantId)).Alias, dateTime, await getter.GetBackupExtensionAsync(_storageBasePath)); + + tempFile = CrossPlatform.PathCombine(TempFolder, backupName); + storagePath = tempFile; + + var writer = await DataOperatorFactory.GetWriteOperatorAsync(_tempStream, _storageBasePath, backupName, TempFolder, _userId, getter); _backupPortalTask.Init(TenantId, tempFile, _limit, writer); @@ -121,7 +124,7 @@ public class BackupProgressItem : BaseBackupProgressItem if (writer.NeedUpload) { storagePath = await backupStorage.UploadAsync(_storageBasePath, tempFile, _userId); - hash = BackupWorker.GetBackupHash(tempFile); + hash = BackupWorker.GetBackupHashSHA(tempFile); } else { diff --git a/common/ASC.Data.Backup.Core/Service/ProgressItems/RestoreProgressItem.cs b/common/ASC.Data.Backup.Core/Service/ProgressItems/RestoreProgressItem.cs index 6242745c4e..df97fcde55 100644 --- a/common/ASC.Data.Backup.Core/Service/ProgressItems/RestoreProgressItem.cs +++ b/common/ASC.Data.Backup.Core/Service/ProgressItems/RestoreProgressItem.cs @@ -47,8 +47,8 @@ * in every copy of the program you distribute. * Pursuant to Section 7 § 3(e) we decline to grant you any rights under trademark law for use of our trademarks. * -*/ - +*/ + namespace ASC.Data.Backup.Services; [Transient(Additional = typeof(RestoreProgressItemExtention))] @@ -62,7 +62,8 @@ public class RestoreProgressItem : BaseBackupProgressItem private readonly NotifyHelper _notifyHelper; private BackupRepository _backupRepository; private RestorePortalTask _restorePortalTask; - private readonly CoreBaseSettings _coreBaseSettings; + private readonly CoreBaseSettings _coreBaseSettings; + private readonly SetupInfo _setupInfo; private string _region; private string _upgradesPath; @@ -73,7 +74,8 @@ public class RestoreProgressItem : BaseBackupProgressItem ICache cache, IServiceScopeFactory serviceScopeFactory, NotifyHelper notifyHelper, - CoreBaseSettings coreBaseSettings) + CoreBaseSettings coreBaseSettings, + SetupInfo setupInfo) : base(logger, serviceScopeFactory) { _configuration = configuration; @@ -82,7 +84,8 @@ public class RestoreProgressItem : BaseBackupProgressItem _notifyHelper = notifyHelper; _coreBaseSettings = coreBaseSettings; - BackupProgressItemEnum = BackupProgressItemEnum.Restore; + BackupProgressItemEnum = BackupProgressItemEnum.Restore; + _setupInfo = setupInfo; } public BackupStorageType StorageType { get; set; } @@ -131,12 +134,17 @@ public class RestoreProgressItem : BaseBackupProgressItem if (!_coreBaseSettings.Standalone) { - var backupHash = BackupWorker.GetBackupHash(tempFile); - var record = await _backupRepository.GetBackupRecordAsync(backupHash, TenantId); + var shaHash = BackupWorker.GetBackupHashSHA(tempFile); + var record = await _backupRepository.GetBackupRecordAsync(shaHash, TenantId); if (record == null) - { - throw new Exception(BackupResource.BackupNotFound); + { + var md5Hash = BackupWorker.GetBackupHashMD5(tempFile, S3Storage.ChunkSize); + record = await _backupRepository.GetBackupRecordAsync(md5Hash, TenantId); + if (record == null) + { + throw new Exception(BackupResource.BackupNotFound); + } } } diff --git a/common/ASC.Data.Backup.Core/Storage/ConsumerBackupStorage.cs b/common/ASC.Data.Backup.Core/Storage/ConsumerBackupStorage.cs index 20d4ced4dc..3ce8784208 100644 --- a/common/ASC.Data.Backup.Core/Storage/ConsumerBackupStorage.cs +++ b/common/ASC.Data.Backup.Core/Storage/ConsumerBackupStorage.cs @@ -119,6 +119,11 @@ public class ConsumerBackupStorage : IBackupStorage, IGetterWriteOperator TempPath = title, UploadId = await _store.InitiateChunkedUploadAsync(Domain, title) }; - return _store.CreateDataWriteOperator(session, _sessionHolder); + return _store.CreateDataWriteOperator(session, _sessionHolder, true); + } + + public Task GetBackupExtensionAsync(string storageBasePath) + { + return Task.FromResult(_store.GetBackupExtension(true)); } } diff --git a/common/ASC.Data.Backup.Core/Storage/DocumentsBackupStorage.cs b/common/ASC.Data.Backup.Core/Storage/DocumentsBackupStorage.cs index a00e80920f..1c7bfe7410 100644 --- a/common/ASC.Data.Backup.Core/Storage/DocumentsBackupStorage.cs +++ b/common/ASC.Data.Backup.Core/Storage/DocumentsBackupStorage.cs @@ -192,7 +192,6 @@ public class DocumentsBackupStorage : IBackupStorage, IGetterWriteOperator var fileDao = await GetFileDaoAsync(); try { - var file = await fileDao.GetFileAsync(fileId); return file != null && file.RootFolderType != FolderType.TRASH; @@ -229,6 +228,21 @@ public class DocumentsBackupStorage : IBackupStorage, IGetterWriteOperator } } + public async Task GetBackupExtensionAsync(string storageBasePath) + { + await _tenantManager.SetCurrentTenantAsync(_tenantId); + if (int.TryParse(storageBasePath, out var fId)) + { + var folderDao = GetFolderDao(); + return await folderDao.GetBackupExtensionAsync(fId); + } + else + { + var folderDao = GetFolderDao(); + return await folderDao.GetBackupExtensionAsync(storageBasePath); + } + } + private async Task InitUploadChunkAsync(T folderId, string title) { var folderDao = GetFolderDao(); diff --git a/common/ASC.Data.Backup.Core/Storage/LocalBackupStorage.cs b/common/ASC.Data.Backup.Core/Storage/LocalBackupStorage.cs index 955883446c..38dfb092bc 100644 --- a/common/ASC.Data.Backup.Core/Storage/LocalBackupStorage.cs +++ b/common/ASC.Data.Backup.Core/Storage/LocalBackupStorage.cs @@ -71,4 +71,9 @@ public class LocalBackupStorage : IBackupStorage, IGetterWriteOperator { return Task.FromResult(null); } + + public Task GetBackupExtensionAsync(string storageBasePath) + { + return Task.FromResult("tar.gz"); + } } diff --git a/common/ASC.Data.Backup.Core/Tasks/RestorePortalTask.cs b/common/ASC.Data.Backup.Core/Tasks/RestorePortalTask.cs index d8a39f6854..de8d54fb82 100644 --- a/common/ASC.Data.Backup.Core/Tasks/RestorePortalTask.cs +++ b/common/ASC.Data.Backup.Core/Tasks/RestorePortalTask.cs @@ -90,7 +90,7 @@ public class RestorePortalTask : PortalTaskBase _options.DebugBeginRestoreData(); - using (var dataReader = new ZipReadOperator(BackupFilePath)) + using (var dataReader = DataOperatorFactory.GetReadOperator(BackupFilePath)) { await using (var entry = dataReader.GetEntry(KeyHelper.GetDumpKey())) { diff --git a/common/ASC.Data.Backup.Core/Tasks/TransferPortalTask.cs b/common/ASC.Data.Backup.Core/Tasks/TransferPortalTask.cs index 50c459086d..057ed03d68 100644 --- a/common/ASC.Data.Backup.Core/Tasks/TransferPortalTask.cs +++ b/common/ASC.Data.Backup.Core/Tasks/TransferPortalTask.cs @@ -96,7 +96,7 @@ public class TransferPortalTask : PortalTaskBase //save db data to temporary file var backupTask = _serviceProvider.GetService(); - backupTask.Init(TenantId, backupFilePath, Limit, ZipWriteOperatorFactory.GetDefaultWriteOperator(_tempStream, backupFilePath)); + backupTask.Init(TenantId, backupFilePath, Limit, DataOperatorFactory.GetDefaultWriteOperator(_tempStream, backupFilePath)); backupTask.ProcessStorage = false; backupTask.ProgressChanged += (sender, args) => SetCurrentStepProgress(args.Progress); foreach (var moduleName in _ignoredModules) diff --git a/common/ASC.Data.Storage/BaseStorage.cs b/common/ASC.Data.Storage/BaseStorage.cs index 355eb3637a..6de20d2cea 100644 --- a/common/ASC.Data.Storage/BaseStorage.cs +++ b/common/ASC.Data.Storage/BaseStorage.cs @@ -201,11 +201,16 @@ public abstract class BaseStorage : IDataStore public virtual IDataWriteOperator CreateDataWriteOperator( CommonChunkedUploadSession chunkedUploadSession, - CommonChunkedUploadSessionHolder sessionHolder) + CommonChunkedUploadSessionHolder sessionHolder, bool isConsumerStorage = false) { return new ChunkZipWriteOperator(_tempStream, chunkedUploadSession, sessionHolder); } + public virtual string GetBackupExtension(bool isConsumerStorage = false) + { + return "tar.gz"; + } + #endregion public abstract Task DeleteAsync(string domain, string path); diff --git a/common/ASC.Data.Storage/GlobalUsings.cs b/common/ASC.Data.Storage/GlobalUsings.cs index dc7f7a96b0..c1bb9efe1c 100644 --- a/common/ASC.Data.Storage/GlobalUsings.cs +++ b/common/ASC.Data.Storage/GlobalUsings.cs @@ -30,7 +30,7 @@ global using System.Net; global using System.Net.Http.Headers; global using System.Runtime.Serialization; global using System.Security.Cryptography; -global using System.ServiceModel; +global using System.ServiceModel; global using System.Text; global using System.Text.Json; global using System.Text.Json.Serialization; @@ -39,7 +39,10 @@ global using System.Web; global using Amazon; global using Amazon.CloudFront; global using Amazon.CloudFront.Model; +global using Amazon.Extensions.S3.Encryption; +global using Amazon.Extensions.S3.Encryption.Primitives; global using Amazon.S3; +global using Amazon.S3.Internal; global using Amazon.S3.Model; global using Amazon.S3.Transfer; global using Amazon.Util; @@ -66,6 +69,7 @@ global using ASC.Data.Storage.GoogleCloud; global using ASC.Data.Storage.Log; global using ASC.Data.Storage.RackspaceCloud; global using ASC.Data.Storage.S3; +global using ASC.Data.Storage.Tar; global using ASC.Data.Storage.ZipOperators; global using ASC.EventBus.Events; global using ASC.Notify.Messages; diff --git a/common/ASC.Data.Storage/IDataStore.cs b/common/ASC.Data.Storage/IDataStore.cs index d35c080af0..af0464aef3 100644 --- a/common/ASC.Data.Storage/IDataStore.cs +++ b/common/ASC.Data.Storage/IDataStore.cs @@ -33,7 +33,9 @@ public interface IDataStore { IDataWriteOperator CreateDataWriteOperator( CommonChunkedUploadSession chunkedUploadSession, - CommonChunkedUploadSessionHolder sessionHolder); + CommonChunkedUploadSessionHolder sessionHolder, + bool isConsumerStorage = false); + string GetBackupExtension(bool isConsumerStorage = false); IQuotaController QuotaController { get; set; } diff --git a/common/ASC.Data.Storage/S3/S3Storage.cs b/common/ASC.Data.Storage/S3/S3Storage.cs index 383e7df471..07272bc094 100644 --- a/common/ASC.Data.Storage/S3/S3Storage.cs +++ b/common/ASC.Data.Storage/S3/S3Storage.cs @@ -24,17 +24,12 @@ // content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0 // International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode -using Amazon.Extensions.S3.Encryption; -using Amazon.Extensions.S3.Encryption.Primitives; -using Amazon.S3.Internal; - -using ASC.Data.Storage.Tar; - namespace ASC.Data.Storage.S3; [Scope] public class S3Storage : BaseStorage { + public static long ChunkSize { get; } = 50 * 1024 * 1024; public override bool IsSupportChunking => true; private readonly List _domains = new List(); @@ -58,7 +53,6 @@ public class S3Storage : BaseStorage private EncryptionMethod _encryptionMethod = EncryptionMethod.None; private string _encryptionKey; - private readonly IConfiguration _configuration; private readonly CoreBaseSettings _coreBaseSettings; public S3Storage( @@ -70,13 +64,11 @@ public class S3Storage : BaseStorage ILoggerProvider factory, ILogger options, IHttpClientFactory clientFactory, - IConfiguration configuration, TenantQuotaFeatureStatHelper tenantQuotaFeatureStatHelper, QuotaSocketManager quotaSocketManager, CoreBaseSettings coreBaseSettings) : base(tempStream, tenantManager, pathUtils, emailValidationKeyProvider, httpContextAccessor, factory, options, clientFactory, tenantQuotaFeatureStatHelper, quotaSocketManager) { - _configuration = configuration; _coreBaseSettings = coreBaseSettings; } @@ -396,9 +388,9 @@ public class S3Storage : BaseStorage } public override IDataWriteOperator CreateDataWriteOperator(CommonChunkedUploadSession chunkedUploadSession, - CommonChunkedUploadSessionHolder sessionHolder) + CommonChunkedUploadSessionHolder sessionHolder, bool isConsumerStorage = false) { - if (_coreBaseSettings.Standalone) + if (_coreBaseSettings.Standalone || isConsumerStorage) { return new S3ZipWriteOperator(_tempStream, chunkedUploadSession, sessionHolder); } @@ -408,6 +400,18 @@ public class S3Storage : BaseStorage } } + public override string GetBackupExtension(bool isConsumerStorage = false) + { + if (_coreBaseSettings.Standalone || isConsumerStorage) + { + return "tar.gz"; + } + else + { + return "tar"; + } + } + #endregion public override async Task DeleteAsync(string domain, string path) @@ -1229,7 +1233,7 @@ public class S3Storage : BaseStorage var uploadId = initResponse.UploadId; - var partSize = GetChunkSize(); + var partSize = ChunkSize; long bytePosition = 0; for (var i = 1; bytePosition < objectSize; i++) @@ -1341,14 +1345,7 @@ public class S3Storage : BaseStorage UploadId = uploadId, PartETags = eTags }; - try - { - await s3.CompleteMultipartUploadAsync(completeRequest); - } - catch(Exception e) - { - - } + await s3.CompleteMultipartUploadAsync(completeRequest); } public async Task ConcatFileAsync(string pathFile, string tarKey, string destinationDomain, string destinationKey, string uploadId, List eTags, int partNumber) @@ -1459,7 +1456,24 @@ public class S3Storage : BaseStorage await s3.CompleteMultipartUploadAsync(completeRequest); } - public async Task<(string uploadId, List eTags, int partNumber)> InitiateConcatAsync(string domain, string key, bool removeEmptyHeader = false) + public async Task RemoveFirstBlockAsync(string domain, string key) + { + using var s3 = GetClient(); + var path = MakePath(domain, key); + + (var uploadId, var eTags, var partNumber) = await InitiateConcatAsync(domain, key, true, true); + var completeRequest = new CompleteMultipartUploadRequest + { + BucketName = _bucket, + Key = path, + UploadId = uploadId, + PartETags = eTags + }; + + await s3.CompleteMultipartUploadAsync(completeRequest); + } + + public async Task<(string uploadId, List eTags, int partNumber)> InitiateConcatAsync(string domain, string key, bool removeFirstBlock = false, bool lastInit = false) { using var s3 = GetClient(); @@ -1472,13 +1486,18 @@ public class S3Storage : BaseStorage }; var initResponse = await s3.InitiateMultipartUploadAsync(initiateRequest); + var eTags = new List(); try { - long bytePosition = removeEmptyHeader ? 5 * 1024 * 1024 : 0; + var mb5 = 5 * 1024 * 1024; + long bytePosition = removeFirstBlock ? mb5 : 0; + var obj = await s3.GetObjectMetadataAsync(_bucket, key); - var eTags = new List(); - var partSize = 5 * 1024 * 1024; - if (obj.ContentLength < partSize) + var objectSize = obj.ContentLength; + + var partSize = ChunkSize; + var partNumber = 1; + for (var i = 1; bytePosition < objectSize; i++) { var copyRequest = new CopyPartRequest { @@ -1487,46 +1506,42 @@ public class S3Storage : BaseStorage SourceBucket = _bucket, SourceKey = key, UploadId = initResponse.UploadId, - PartNumber = 1, FirstByte = bytePosition, - LastByte = obj.ContentLength - 1 + LastByte = bytePosition + partSize - 1 >= objectSize ? objectSize - 1 : bytePosition + partSize - 1, + PartNumber = i }; - eTags.Add(new PartETag(1, (await s3.CopyPartAsync(copyRequest)).ETag)); - return (initResponse.UploadId, eTags, 2); - } - else - { - var objectSize = obj.ContentLength; - var partNumber = 1; - for (var i = 1; bytePosition < objectSize; i++) - { - var copyRequest = new CopyPartRequest - { - DestinationBucket = _bucket, - DestinationKey = key, - SourceBucket = _bucket, - SourceKey = key, - UploadId = initResponse.UploadId, - FirstByte = bytePosition, - LastByte = bytePosition + partSize - 1 >= objectSize ? objectSize - 1 : bytePosition + partSize - 1, - PartNumber = i - }; - partNumber = i + 1; - bytePosition += partSize; - if (objectSize - bytePosition < 5 * 1024 * 1024) - { - copyRequest.LastByte = objectSize - 1; - bytePosition += partSize; - } - eTags.Add(new PartETag(i, (await s3.CopyPartAsync(copyRequest)).ETag)); + partNumber = i + 1; + bytePosition += partSize; + var x = objectSize - bytePosition; + if (!lastInit && x < mb5 && x > 0) + { + copyRequest.LastByte = objectSize - 1; + bytePosition += partSize; } - return (initResponse.UploadId, eTags, partNumber); + eTags.Add(new PartETag(i, (await s3.CopyPartAsync(copyRequest)).ETag)); + } + return (initResponse.UploadId, eTags, partNumber); } catch { - return (initResponse.UploadId, new List(), 1); + using var stream = new MemoryStream(); + var buffer = new byte[5 * 1024 * 1024]; + stream.Write(buffer); + stream.Position = 0; + + var uploadRequest = new UploadPartRequest + { + BucketName = _bucket, + Key = key, + UploadId = initResponse.UploadId, + PartNumber = 1, + InputStream = stream + }; + eTags.Add(new PartETag(1, (await s3.UploadPartAsync(uploadRequest)).ETag)); + + return (initResponse.UploadId, eTags, 2); } } @@ -1708,18 +1723,6 @@ public class S3Storage : BaseStorage return el.ETag; } - private long GetChunkSize() - { - var configSetting = _configuration["files:uploader:chunk-size"]; - if (!string.IsNullOrEmpty(configSetting)) - { - configSetting = configSetting.Trim(); - return long.Parse(configSetting); - } - long defaultValue = 10 * 1024 * 1024; - return defaultValue; - } - private enum EncryptionMethod { None, diff --git a/common/ASC.Data.Storage/ZipOperators/ZipWriteOperatorFactory.cs b/common/ASC.Data.Storage/ZipOperators/DataOperatorFactory.cs similarity index 87% rename from common/ASC.Data.Storage/ZipOperators/ZipWriteOperatorFactory.cs rename to common/ASC.Data.Storage/ZipOperators/DataOperatorFactory.cs index a0727f6787..ce5a5fe832 100644 --- a/common/ASC.Data.Storage/ZipOperators/ZipWriteOperatorFactory.cs +++ b/common/ASC.Data.Storage/ZipOperators/DataOperatorFactory.cs @@ -26,7 +26,7 @@ namespace ASC.Data.Storage.ZipOperators; -public static class ZipWriteOperatorFactory +public static class DataOperatorFactory { public static async Task GetWriteOperatorAsync(TempStream tempStream, string storageBasePath, string title, string tempFolder, Guid userId, IGetterWriteOperator getter) { @@ -39,5 +39,17 @@ public static class ZipWriteOperatorFactory { return new ZipWriteOperator(tempStream, backupFilePath); } + + public static IDataReadOperator GetReadOperator(string targetFile) + { + try + { + return new ZipReadOperator(targetFile); + } + catch + { + return new TarReadOperator(targetFile); + } + } } \ No newline at end of file diff --git a/common/ASC.Data.Storage/ZipOperators/IGetterWriteOperator.cs b/common/ASC.Data.Storage/ZipOperators/IGetterWriteOperator.cs index 0ecb2bc919..de46029128 100644 --- a/common/ASC.Data.Storage/ZipOperators/IGetterWriteOperator.cs +++ b/common/ASC.Data.Storage/ZipOperators/IGetterWriteOperator.cs @@ -29,4 +29,5 @@ namespace ASC.Data.Storage.ZipOperators; public interface IGetterWriteOperator { Task GetWriteOperatorAsync(string storageBasePath, string title, Guid userId); + Task GetBackupExtensionAsync(string storageBasePath); } diff --git a/common/ASC.Data.Storage/ZipOperators/S3/S3TarWriteOperator.cs b/common/ASC.Data.Storage/ZipOperators/S3/S3TarWriteOperator.cs index 20958e3389..ac640b0060 100644 --- a/common/ASC.Data.Storage/ZipOperators/S3/S3TarWriteOperator.cs +++ b/common/ASC.Data.Storage/ZipOperators/S3/S3TarWriteOperator.cs @@ -24,17 +24,12 @@ // content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0 // International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode -using System.IO; - -using Google.Apis.Storage.v1.Data; - namespace ASC.Data.Storage.ZipOperators; -internal class S3TarWriteOperator : IDataWriteOperator +public class S3TarWriteOperator : IDataWriteOperator { private readonly CommonChunkedUploadSession _chunkedUploadSession; private readonly CommonChunkedUploadSessionHolder _sessionHolder; private readonly S3Storage _store; - private bool _first = true; private readonly string _domain; private readonly string _key; @@ -50,12 +45,6 @@ internal class S3TarWriteOperator : IDataWriteOperator _key = _chunkedUploadSession.TempPath; _domain = string.IsNullOrEmpty(_sessionHolder.TempDomain) ? _sessionHolder.Domain : _sessionHolder.TempDomain; - - using var stream = new MemoryStream(); - var buffer = new byte[5 * 1024 * 1024]; - stream.Write(buffer); - stream.Position = 0; - _sessionHolder.UploadChunkAsync(_chunkedUploadSession, stream, stream.Length).Wait(); } public async Task WriteEntryAsync(string tarKey, string domain, string path, IDataStore store) @@ -65,7 +54,7 @@ internal class S3TarWriteOperator : IDataWriteOperator var s3Store = store as S3Storage; var fullPath = s3Store.MakePath(domain, path); - (var uploadId, var eTags, var chunkNumber) = await GetDataAsync(); + (var uploadId, var eTags, var chunkNumber) = await _store.InitiateConcatAsync(_domain, _key); await _store.ConcatFileAsync(fullPath, tarKey, _domain, _key, uploadId, eTags, chunkNumber); } else @@ -85,45 +74,26 @@ internal class S3TarWriteOperator : IDataWriteOperator public async Task WriteEntryAsync(string tarKey, Stream stream) { - (var uploadId, var eTags, var chunkNumber) = await GetDataAsync(); + (var uploadId, var eTags, var chunkNumber) = await _store.InitiateConcatAsync(_domain, _key); await _store.ConcatFileStreamAsync(stream, tarKey, _domain, _key, uploadId, eTags, chunkNumber); } - private async Task<(string uploadId, List eTags, int partNumber)> GetDataAsync() - { - List eTags = null; - var chunkNumber = 0; - string uploadId = null; - if (_first) - { - eTags = _chunkedUploadSession.GetItemOrDefault>("ETag").Select(x => new PartETag(x.Key, x.Value)).ToList(); - int.TryParse(_chunkedUploadSession.GetItemOrDefault("ChunksUploaded"), out chunkNumber); - chunkNumber++; - uploadId = _chunkedUploadSession.UploadId; - _first = false; - } - else - { - (uploadId, eTags, chunkNumber) = await _store.InitiateConcatAsync(_domain, _key); - } - return (uploadId, eTags, chunkNumber); - } - public async ValueTask DisposeAsync() { await _store.AddEndAsync(_domain ,_key); + await _store.RemoveFirstBlockAsync(_domain ,_key); var contentLength = await _store.GetFileSizeAsync(_domain, _key); + Hash = (await _store.GetFileEtagAsync(_domain, _key)).Trim('\"'); - (var uploadId, var eTags, var partNumber) = await _store.InitiateConcatAsync(_domain, _key, removeEmptyHeader: true); + (var uploadId, var eTags, var partNumber) = await _store.InitiateConcatAsync(_domain, _key, lastInit: true); _chunkedUploadSession.BytesUploaded = contentLength; _chunkedUploadSession.BytesTotal = contentLength; _chunkedUploadSession.UploadId = uploadId; _chunkedUploadSession.Items["ETag"] = eTags.ToDictionary(e => e.PartNumber, e => e.ETag); - _chunkedUploadSession.Items["ChunksUploaded"] = partNumber.ToString(); - StoragePath = await _sessionHolder.FinalizeAsync(_chunkedUploadSession); + _chunkedUploadSession.Items["ChunksUploaded"] = (partNumber - 1).ToString(); - Hash = ""; + StoragePath = await _sessionHolder.FinalizeAsync(_chunkedUploadSession); } } diff --git a/common/ASC.Data.Storage/ZipOperators/TarReadOperator.cs b/common/ASC.Data.Storage/ZipOperators/TarReadOperator.cs new file mode 100644 index 0000000000..d36ff3229b --- /dev/null +++ b/common/ASC.Data.Storage/ZipOperators/TarReadOperator.cs @@ -0,0 +1,72 @@ +// (c) Copyright Ascensio System SIA 2010-2022 +// +// This program is a free software product. +// You can redistribute it and/or modify it under the terms +// of the GNU Affero General Public License (AGPL) version 3 as published by the Free Software +// Foundation. In accordance with Section 7(a) of the GNU AGPL its Section 15 shall be amended +// to the effect that Ascensio System SIA expressly excludes the warranty of non-infringement of +// any third-party rights. +// +// This program is distributed WITHOUT ANY WARRANTY, without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For details, see +// the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html +// +// You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia, EU, LV-1021. +// +// The interactive user interfaces in modified source and object code versions of the Program must +// display Appropriate Legal Notices, as required under Section 5 of the GNU AGPL version 3. +// +// Pursuant to Section 7(b) of the License you must retain the original Product logo when +// distributing the program. Pursuant to Section 7(e) we decline to grant you any rights under +// trademark law for use of our trademarks. +// +// All the Product's GUI elements, including illustrations and icon sets, as well as technical writing +// content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0 +// International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode + +namespace ASC.Data.Storage.ZipOperators; +public class TarReadOperator: IDataReadOperator +{ + private readonly string tmpdir; + + public TarReadOperator(string targetFile) + { + tmpdir = Path.Combine(Path.GetDirectoryName(targetFile), Path.GetFileNameWithoutExtension(targetFile).Replace('>', '_').Replace(':', '_').Replace('?', '_')); + + using (var stream = File.OpenRead(targetFile)) + using (var tarOutputStream = TarArchive.CreateInputTarArchive(stream, Encoding.UTF8)) + { + tarOutputStream.ExtractContents(tmpdir); + } + + File.Delete(targetFile); + } + + public Stream GetEntry(string key) + { + var filePath = Path.Combine(tmpdir, key); + return File.Exists(filePath) ? File.Open(filePath, FileMode.Open, FileAccess.ReadWrite, FileShare.Read) : null; + } + + public IEnumerable GetEntries(string key) + { + var path = Path.Combine(tmpdir, key); + var files = Directory.EnumerateFiles(path); + return files; + } + + public IEnumerable GetDirectories(string key) + { + var path = Path.Combine(tmpdir, key); + var files = Directory.EnumerateDirectories(path); + return files; + } + + public void Dispose() + { + if (Directory.Exists(tmpdir)) + { + Directory.Delete(tmpdir, true); + } + } +} diff --git a/common/Tools/ASC.MigrationPersonalToDocspace/MigrationCreator.cs b/common/Tools/ASC.MigrationPersonalToDocspace/MigrationCreator.cs index e1b36fa4f0..73c2157480 100644 --- a/common/Tools/ASC.MigrationPersonalToDocspace/MigrationCreator.cs +++ b/common/Tools/ASC.MigrationPersonalToDocspace/MigrationCreator.cs @@ -378,7 +378,7 @@ public class MigrationCreator { var storage = await _storageFactory.GetStorageAsync(_fromTenantId, group.Key); var file1 = file; - await ActionInvoker.Try(async state => + await ActionInvoker.TryAsync(async state => { var f = (BackupFileInfo)state; using var fileStream = await storage.GetReadStreamAsync(f.Domain, f.Path); diff --git a/products/ASC.Files/Core/Core/Dao/Interfaces/IFolderDao.cs b/products/ASC.Files/Core/Core/Dao/Interfaces/IFolderDao.cs index bab168e70d..3d779b2e8a 100644 --- a/products/ASC.Files/Core/Core/Dao/Interfaces/IFolderDao.cs +++ b/products/ASC.Files/Core/Core/Dao/Interfaces/IFolderDao.cs @@ -216,6 +216,8 @@ public interface IFolderDao CommonChunkedUploadSession chunkedUploadSession, CommonChunkedUploadSessionHolder sessionHolder); + Task GetBackupExtensionAsync(T folderId); + #region Only for TMFolderDao /// diff --git a/products/ASC.Files/Core/Core/Dao/TeamlabDao/FolderDao.cs b/products/ASC.Files/Core/Core/Dao/TeamlabDao/FolderDao.cs index f7d96ab389..6c57597657 100644 --- a/products/ASC.Files/Core/Core/Dao/TeamlabDao/FolderDao.cs +++ b/products/ASC.Files/Core/Core/Dao/TeamlabDao/FolderDao.cs @@ -1543,57 +1543,9 @@ internal class FolderDao : AbstractDao, IFolderDao return (await _globalStore.GetStoreAsync()).CreateDataWriteOperator(chunkedUploadSession, sessionHolder); } - private string GetProjectTitle(object folderID) + public async Task GetBackupExtensionAsync(int folderId) { - return ""; - //if (!ApiServer.Available) - //{ - // return string.Empty; - //} - - //var cacheKey = "documents/folders/" + folderID.ToString(); - - //var projectTitle = Convert.ToString(cache.Get(cacheKey)); - - //if (!string.IsNullOrEmpty(projectTitle)) return projectTitle; - - //var bunchObjectID = GetBunchObjectID(folderID); - - //if (string.IsNullOrEmpty(bunchObjectID)) - // throw new Exception("Bunch Object id is null for " + folderID); - - //if (!bunchObjectID.StartsWith("projects/project/")) - // return string.Empty; - - //var bunchObjectIDParts = bunchObjectID.Split('/'); - - //if (bunchObjectIDParts.Length < 3) - // throw new Exception("Bunch object id is not supported format"); - - //var projectID = Convert.ToInt32(bunchObjectIDParts[bunchObjectIDParts.Length - 1]); - - //if (HttpContext.Current == null || !SecurityContext.IsAuthenticated) - // return string.Empty; - - //var apiServer = new ApiServer(); - - //var apiUrl = string.Format("{0}project/{1}.json?fields=id,title", SetupInfo.WebApiBaseUrl, projectID); - - //var responseApi = JObject.Parse(Encoding.UTF8.GetString(Convert.FromBase64String(apiServer.GetApiResponse(apiUrl, "GET"))))["response"]; - - //if (responseApi != null && responseApi.HasValues) - //{ - // projectTitle = Global.ReplaceInvalidCharsAndTruncate(responseApi["title"].Value()); - //} - //else - //{ - // return string.Empty; - //} - //if (!string.IsNullOrEmpty(projectTitle)) - //{ - // cache.Insert(cacheKey, projectTitle, TimeSpan.FromMinutes(15)); - //} - //return projectTitle; + return (await _globalStore.GetStoreAsync()).GetBackupExtension(); } } diff --git a/products/ASC.Files/Core/Core/Entries/ChunkedUploadSession.cs b/products/ASC.Files/Core/Core/Entries/ChunkedUploadSession.cs index 532e8e3ee8..31551d5853 100644 --- a/products/ASC.Files/Core/Core/Entries/ChunkedUploadSession.cs +++ b/products/ASC.Files/Core/Core/Entries/ChunkedUploadSession.cs @@ -67,7 +67,6 @@ public class ChunkedUploadSession : CommonChunkedUploadSession chunkedUploadSession.TransformItems(); return chunkedUploadSession; - } } diff --git a/products/ASC.Files/Core/Core/Thirdparty/ProviderDao/ProviderFolderDao.cs b/products/ASC.Files/Core/Core/Thirdparty/ProviderDao/ProviderFolderDao.cs index 864c120d69..920655fd54 100644 --- a/products/ASC.Files/Core/Core/Thirdparty/ProviderDao/ProviderFolderDao.cs +++ b/products/ASC.Files/Core/Core/Thirdparty/ProviderDao/ProviderFolderDao.cs @@ -466,6 +466,13 @@ internal class ProviderFolderDao : ProviderDaoBase, IFolderDao return await folderDao.CreateDataWriteOperatorAsync(folderId, chunkedUploadSession, sessionHolder); } + public async Task GetBackupExtensionAsync(string folderId) + { + var selector = _selectorFactory.GetSelector(folderId); + var folderDao = selector.GetFolderDao(folderId); + return await folderDao.GetBackupExtensionAsync(folderId); + } + private IAsyncEnumerable> FilterByProvider(IAsyncEnumerable> folders, ProviderFilter provider) { if (provider != ProviderFilter.kDrive && provider != ProviderFilter.WebDav && provider != ProviderFilter.Yandex) diff --git a/products/ASC.Files/Core/Core/Thirdparty/SharePoint/SharePointFolderDao.cs b/products/ASC.Files/Core/Core/Thirdparty/SharePoint/SharePointFolderDao.cs index 888bbe7c67..e06e96ef94 100644 --- a/products/ASC.Files/Core/Core/Thirdparty/SharePoint/SharePointFolderDao.cs +++ b/products/ASC.Files/Core/Core/Thirdparty/SharePoint/SharePointFolderDao.cs @@ -439,6 +439,11 @@ internal class SharePointFolderDao : SharePointDaoBase, IFolderDao { return Task.FromResult(null); } + + public Task GetBackupExtensionAsync(string folderId) + { + return Task.FromResult("tar.gz"); + } } static file class Queries diff --git a/products/ASC.Files/Core/Core/Thirdparty/Sharpbox/SharpBoxFolderDao.cs b/products/ASC.Files/Core/Core/Thirdparty/Sharpbox/SharpBoxFolderDao.cs index b8b6bfb450..b30008f707 100644 --- a/products/ASC.Files/Core/Core/Thirdparty/Sharpbox/SharpBoxFolderDao.cs +++ b/products/ASC.Files/Core/Core/Thirdparty/Sharpbox/SharpBoxFolderDao.cs @@ -504,6 +504,11 @@ internal class SharpBoxFolderDao : SharpBoxDaoBase, IFolderDao { return Task.FromResult(null); } + + public Task GetBackupExtensionAsync(string folderId) + { + return Task.FromResult("tar.gz"); + } } static file class Queries diff --git a/products/ASC.Files/Core/Core/Thirdparty/ThirdPartyFolderDao.cs b/products/ASC.Files/Core/Core/Thirdparty/ThirdPartyFolderDao.cs index 3391f375d2..5103bbebd3 100644 --- a/products/ASC.Files/Core/Core/Thirdparty/ThirdPartyFolderDao.cs +++ b/products/ASC.Files/Core/Core/Thirdparty/ThirdPartyFolderDao.cs @@ -510,6 +510,11 @@ internal class ThirdPartyFolderDao : BaseFolderDao, IFold return Task.FromResult(new ChunkZipWriteOperator(_tempStream, chunkedUploadSession, sessionHolder)); } + public Task GetBackupExtensionAsync(string folderId) + { + return Task.FromResult("tar.gz"); + } + public Task ReassignFoldersAsync(Guid oldOwnerId, Guid newOwnerId) { return Task.CompletedTask;