diff --git a/common/ASC.Common/Data/TypeExtension.cs b/common/ASC.Common/Data/TypeExtension.cs new file mode 100644 index 0000000000..8265627bc7 --- /dev/null +++ b/common/ASC.Common/Data/TypeExtension.cs @@ -0,0 +1,42 @@ +// (c) Copyright Ascensio System SIA 2010-2022 +// +// This program is a free software product. +// You can redistribute it and/or modify it under the terms +// of the GNU Affero General Public License (AGPL) version 3 as published by the Free Software +// Foundation. In accordance with Section 7(a) of the GNU AGPL its Section 15 shall be amended +// to the effect that Ascensio System SIA expressly excludes the warranty of non-infringement of +// any third-party rights. +// +// This program is distributed WITHOUT ANY WARRANTY, without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For details, see +// the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html +// +// You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia, EU, LV-1021. +// +// The interactive user interfaces in modified source and object code versions of the Program must +// display Appropriate Legal Notices, as required under Section 5 of the GNU AGPL version 3. +// +// Pursuant to Section 7(b) of the License you must retain the original Product logo when +// distributing the program. Pursuant to Section 7(e) we decline to grant you any rights under +// trademark law for use of our trademarks. +// +// All the Product's GUI elements, including illustrations and icon sets, as well as technical writing +// content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0 +// International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode + +namespace ASC.Common.Data; +public static class TypeExtension +{ + public static string GetFormattedName(this Type type) + { + if (type.IsGenericType) + { + var genericArguments = type.GetGenericArguments() + .Select(x => x.Name) + .Aggregate((x1, x2) => $"{x1}, {x2}"); + return $"{type.Name.Substring(0, type.Name.IndexOf("`"))}" + + $"<{genericArguments}>"; + } + return type.Name; + } +} diff --git a/common/ASC.Common/Log/DistributedTaskQueueLogger.cs b/common/ASC.Common/Log/DistributedTaskQueueLogger.cs index 8e656a2176..319341b200 100644 --- a/common/ASC.Common/Log/DistributedTaskQueueLogger.cs +++ b/common/ASC.Common/Log/DistributedTaskQueueLogger.cs @@ -27,7 +27,7 @@ namespace ASC.Common.Log; internal static partial class DistributedTaskQueueLogger { - [LoggerMessage(Level = LogLevel.Trace, Message = "EnqueueTask '{distributedTaskId}' by instanse id '{instanceId}'")] + [LoggerMessage(Level = LogLevel.Trace, Message = "EnqueueTask '{DistributedTaskId}' by instanse id '{instanceId}'")] public static partial void TraceEnqueueTask(this ILogger logger, string DistributedTaskId, int instanceId); [LoggerMessage(Level = LogLevel.Trace, Message = "Publication DistributedTask '{DistributedTaskId}' by instanse id '{instanceId}' ")] diff --git a/common/ASC.Common/Threading/DistributedTaskQueue.cs b/common/ASC.Common/Threading/DistributedTaskQueue.cs index e66f869b0d..edb3f6cb83 100644 --- a/common/ASC.Common/Threading/DistributedTaskQueue.cs +++ b/common/ASC.Common/Threading/DistributedTaskQueue.cs @@ -305,6 +305,13 @@ public class DistributedTaskQueue private void SaveToCache(IEnumerable queueTasks) { + if (!queueTasks.Any()) + { + _distributedCache.Remove(_name); + + return; + } + using var ms = new MemoryStream(); Serializer.Serialize(ms, queueTasks); @@ -332,21 +339,11 @@ public class DistributedTaskQueue private IEnumerable DeleteOrphanCacheItem(IEnumerable queueTasks) { - if (!queueTasks.Any()) - { - return queueTasks; - } + var listTasks = queueTasks.ToList(); - var orphans = queueTasks.Where(IsOrphanCacheItem); + listTasks.RemoveAll(IsOrphanCacheItem); - if (!orphans.Any()) - { - return queueTasks; - } - - queueTasks = queueTasks.Except(queueTasks); - - SaveToCache(queueTasks); + SaveToCache(listTasks); return queueTasks; } diff --git a/common/ASC.Core.Common/GlobalUsings.cs b/common/ASC.Core.Common/GlobalUsings.cs index f84cdede6c..3adac2331b 100644 --- a/common/ASC.Core.Common/GlobalUsings.cs +++ b/common/ASC.Core.Common/GlobalUsings.cs @@ -149,3 +149,4 @@ global using ProtoBuf; global using Telegram.Bot; global using static ASC.Security.Cryptography.EmailValidationKeyProvider; +global using ASC.Common.Data; \ No newline at end of file diff --git a/common/ASC.Core.Common/Hosting/RegisterInstanceDao.cs b/common/ASC.Core.Common/Hosting/RegisterInstanceDao.cs index f6526d21fe..ae1972b1be 100644 --- a/common/ASC.Core.Common/Hosting/RegisterInstanceDao.cs +++ b/common/ASC.Core.Common/Hosting/RegisterInstanceDao.cs @@ -79,7 +79,7 @@ public class RegisterInstanceDao : IRegisterInstanceDao where T : IHostedS public async Task> GetAll() { return await _instanceRegistrationContext.InstanceRegistrations - .Where(x => x.WorkerTypeName == typeof(T).Name) + .Where(x => x.WorkerTypeName == typeof(T).GetFormattedName()) .ToListAsync(); } diff --git a/common/ASC.Core.Common/Hosting/RegisterInstanceManager.cs b/common/ASC.Core.Common/Hosting/RegisterInstanceManager.cs index 13f96ee35a..0582776b24 100644 --- a/common/ASC.Core.Common/Hosting/RegisterInstanceManager.cs +++ b/common/ASC.Core.Common/Hosting/RegisterInstanceManager.cs @@ -51,7 +51,7 @@ public class RegisterInstanceManager : IRegisterInstanceManager where T : var instance = registeredInstance ?? new InstanceRegistration { InstanceRegistrationId = instanceId, - WorkerTypeName = typeof(T).Name + WorkerTypeName = typeof(T).GetFormattedName() }; instance.LastUpdated = DateTime.UtcNow; diff --git a/common/ASC.Core.Common/Hosting/RegisterInstanceWorkerService.cs b/common/ASC.Core.Common/Hosting/RegisterInstanceWorkerService.cs index 47f8e42713..1d84192fd4 100644 --- a/common/ASC.Core.Common/Hosting/RegisterInstanceWorkerService.cs +++ b/common/ASC.Core.Common/Hosting/RegisterInstanceWorkerService.cs @@ -24,7 +24,7 @@ // content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0 // International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode -namespace ASC.Core.Common.Hosting; +namespace ASC.Core.Common.Hosting; [Singletone] public class RegisterInstanceWorkerService : BackgroundService where T : IHostedService @@ -33,7 +33,7 @@ public class RegisterInstanceWorkerService : BackgroundService where T : IHos private readonly IHostApplicationLifetime _applicationLifetime; private readonly IServiceProvider _serviceProvider; public static readonly string InstanceId = - $"{typeof(T).Name}_{DateTime.UtcNow.Ticks}"; + $"{typeof(T).GetFormattedName()}_{DateTime.UtcNow.Ticks}"; public RegisterInstanceWorkerService( ILogger> logger, @@ -58,7 +58,7 @@ public class RegisterInstanceWorkerService : BackgroundService where T : IHos await registerInstanceService.Register(InstanceId); await registerInstanceService.DeleteOrphanInstances(); - _logger.InformationWorkingRunnging(DateTimeOffset.Now); + _logger.TraceWorkingRunnging(DateTimeOffset.Now); await Task.Delay(1000, stoppingToken); } @@ -87,5 +87,6 @@ public class RegisterInstanceWorkerService : BackgroundService where T : IHos } await base.StopAsync(cancellationToken); - } + } + } diff --git a/common/ASC.Core.Common/Log/RegisterInstanceWorkerServiceLogger.cs b/common/ASC.Core.Common/Log/RegisterInstanceWorkerServiceLogger.cs index 96357a8276..4ce795e0cb 100644 --- a/common/ASC.Core.Common/Log/RegisterInstanceWorkerServiceLogger.cs +++ b/common/ASC.Core.Common/Log/RegisterInstanceWorkerServiceLogger.cs @@ -31,8 +31,8 @@ internal static partial class RegisterInstanceWorkerServiceLogger [LoggerMessage(Level = LogLevel.Trace, Message = "DbUpdateConcurrencyException: then updating {instanceName} at {time} time.")] public static partial void TraceDbUpdateConcurrencyException(this ILogger logger, string instanceName, DateTimeOffset time); - [LoggerMessage(Level = LogLevel.Information, Message = "Worker running at: {time}")] - public static partial void InformationWorkingRunnging(this ILogger logger, DateTimeOffset time); + [LoggerMessage(Level = LogLevel.Trace, Message = "Worker running at: {time}")] + public static partial void TraceWorkingRunnging(this ILogger logger, DateTimeOffset time); [LoggerMessage(Level = LogLevel.Information, Message = "UnRegister Instance {instanceName} running at: {time}.")] public static partial void InformationUnRegister(this ILogger logger, string instanceName, DateTimeOffset time); diff --git a/products/ASC.Files/Core/Services/WCFService/FileOperations/FileOperationResult.cs b/products/ASC.Files/Core/Services/WCFService/FileOperations/FileOperationResult.cs index b559e3e674..7127f4f086 100644 --- a/products/ASC.Files/Core/Services/WCFService/FileOperations/FileOperationResult.cs +++ b/products/ASC.Files/Core/Services/WCFService/FileOperations/FileOperationResult.cs @@ -26,16 +26,32 @@ namespace ASC.Web.Files.Services.WCFService.FileOperations; +[ProtoContract] +[ProtoInclude(100, typeof(FileConverterOperationResult))] public class FileOperationResult { + [ProtoMember(1)] public string Id { get; set; } [JsonPropertyName("operation")] + [ProtoMember(2)] public FileOperationType OperationType { get; set; } + + [ProtoMember(3)] public int Progress { get; set; } + + [ProtoMember(4)] public string Source { get; set; } + + [ProtoMember(5)] public string Result { get; set; } + + [ProtoMember(6)] public string Error { get; set; } + + [ProtoMember(7)] public string Processed { get; set; } + + [ProtoMember(8)] public bool Finished { get; set; } } diff --git a/products/ASC.Files/Core/Utils/FileConverter.cs b/products/ASC.Files/Core/Utils/FileConverter.cs index 51bed8ccc7..b1357a5f50 100644 --- a/products/ASC.Files/Core/Utils/FileConverter.cs +++ b/products/ASC.Files/Core/Utils/FileConverter.cs @@ -24,44 +24,42 @@ // content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0 // International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode +using Microsoft.Extensions.Caching.Distributed; -using Timeout = System.Threading.Timeout; - namespace ASC.Web.Files.Utils; -[Singletone(Additional = typeof(FileConverterQueueExtension))] -internal class FileConverterQueue : IDisposable +[Singletone] +public class FileConverterQueue { - private readonly object _singleThread = new object(); - private readonly IDictionary, ConvertFileOperationResult> _conversionQueue; - private readonly Timer _timer; - private readonly object _locker; - private readonly ICache _cache; - private const int _timerPeriod = 500; + private readonly object _locker = new object(); + private readonly IDistributedCache _distributedCache; + private readonly string _cache_key_prefix = "asc_file_converter_queue_"; - private readonly IServiceScopeFactory _serviceScopeFactory; - - public FileConverterQueue(IServiceScopeFactory serviceScopeFactory, ICache cache) + public FileConverterQueue(IDistributedCache distributedCache) { - _conversionQueue = new Dictionary, ConvertFileOperationResult>(new FileComparer()); - _timer = new Timer(CheckConvertFilesStatus, null, 0, Timeout.Infinite); - _locker = new object(); - _serviceScopeFactory = serviceScopeFactory; - _cache = cache; + _distributedCache = distributedCache; } - public void Add(File file, string password, int tenantId, IAccount account, bool deleteAfter, string url, string serverRootPath) + public void Add(File file, + string password, + int tenantId, + IAccount account, + bool deleteAfter, + string url, + string serverRootPath) { lock (_locker) { - if (_conversionQueue.ContainsKey(file)) + var task = PeekTask(file); + + if (Contains(task)) { return; } - var queueResult = new ConvertFileOperationResult + var queueResult = new FileConverterOperationResult { - Source = string.Format("{{\"id\":\"{0}\", \"version\":\"{1}\"}}", file.Id, file.Version), + Source = System.Text.Json.JsonSerializer.Serialize(new { id = file.Id, version = file.Version }), OperationType = FileOperationType.Convert, Error = string.Empty, Progress = 0, @@ -69,316 +67,95 @@ internal class FileConverterQueue : IDisposable Processed = "", Id = string.Empty, TenantId = tenantId, - Account = account, + Account = account.ID, Delete = deleteAfter, StartDateTime = DateTime.Now, Url = url, Password = password, ServerRootPath = serverRootPath }; - _conversionQueue.Add(file, queueResult); - _cache.Insert(GetKey(file), queueResult, TimeSpan.FromMinutes(10)); - - _timer.Change(0, Timeout.Infinite); + + Enqueue(queueResult); } } - - public async Task GetStatusAsync(KeyValuePair, bool> pair, FileSecurity fileSecurity) + + + public void Enqueue(FileConverterOperationResult val) { - var file = pair.Key; - var key = GetKey(file); - var operation = _cache.Get(key); - if (operation != null && (pair.Value || await fileSecurity.CanReadAsync(file))) + var fromCache = LoadFromCache().ToList(); + + fromCache.Add(val); + + SaveToCache(fromCache); + } + + public void Dequeue(FileConverterOperationResult val) + { + var fromCache = LoadFromCache().ToList(); + + fromCache.Remove(val); + + SaveToCache(fromCache); + } + + public FileConverterOperationResult PeekTask(File file) + { + var exist = LoadFromCache(); + + return exist.FirstOrDefault(x => { - lock (_locker) - { - if (operation.Progress == 100) - { - _conversionQueue.Remove(file); - _cache.Remove(key); - } + var fileId = JsonDocument.Parse(x.Source).RootElement.GetProperty("id").Deserialize(); + var fileVersion = JsonDocument.Parse(x.Source).RootElement.GetProperty("version").Deserialize(); - return operation; - } - } - - return null; + return file.Id.ToString() == fileId.ToString() && file.Version == fileVersion; + }); } public bool IsConverting(File file) { - var result = _cache.Get(GetKey(file)); + var result = PeekTask(file); return result != null && result.Progress != 100 && string.IsNullOrEmpty(result.Error); } - private void CheckConvertFilesStatus(object _) + + public IEnumerable GetAllTask() { - if (Monitor.TryEnter(_singleThread)) + var queueTasks = LoadFromCache(); + + queueTasks = DeleteOrphanCacheItem(queueTasks); + + return queueTasks; + } + + public void SetAllTask(IEnumerable queueTasks) + { + SaveToCache(queueTasks); + } + + + public async Task GetStatusAsync(KeyValuePair, bool> pair, FileSecurity fileSecurity) + { + var file = pair.Key; + var operation = PeekTask(pair.Key); + + if (operation != null && (pair.Value || await fileSecurity.CanReadAsync(file))) { - using var scope = _serviceScopeFactory.CreateScope(); - TenantManager tenantManager; - UserManager userManager; - SecurityContext securityContext; - IDaoFactory daoFactory; - FileSecurity fileSecurity; - PathProvider pathProvider; - SetupInfo setupInfo; - FileUtility fileUtility; - DocumentServiceHelper documentServiceHelper; - DocumentServiceConnector documentServiceConnector; - EntryStatusManager entryManager; - FileConverter fileConverter; - - var logger = scope.ServiceProvider.GetService>>(); - - try + if (operation.Progress == 100) { - var filesIsConverting = new List>(); - lock (_locker) - { - _timer.Change(Timeout.Infinite, Timeout.Infinite); - - var queues = _conversionQueue.Where(x => !string.IsNullOrEmpty(x.Value.Processed) - && (x.Value.Progress == 100 && DateTime.UtcNow - x.Value.StopDateTime > TimeSpan.FromMinutes(1) || - DateTime.UtcNow - x.Value.StopDateTime > TimeSpan.FromMinutes(10))) - .ToList(); + var task = PeekTask(file); - foreach (var q in queues) - { - _conversionQueue.Remove(q); - _cache.Remove(GetKey(q.Key)); - } - - logger.DebugRunCheckConvertFilesStatus(_conversionQueue.Count); - - if (_conversionQueue.Count == 0) - { - return; - } - - filesIsConverting = _conversionQueue - .Where(x => string.IsNullOrEmpty(x.Value.Processed)) - .Select(x => x.Key) - .ToList(); - } - - string convertedFileUrl = null; - - foreach (var file in filesIsConverting) - { - var fileUri = file.Id.ToString(); - int operationResultProgress; - - try - { - int tenantId; - IAccount account; - string password; - string serverRootPath; - - lock (_locker) - { - if (!_conversionQueue.ContainsKey(file)) - { - continue; - } - - var operationResult = _conversionQueue[file]; - if (!string.IsNullOrEmpty(operationResult.Processed)) - { - continue; - } - - operationResult.Processed = "1"; - tenantId = operationResult.TenantId; - account = operationResult.Account; - password = operationResult.Password; - serverRootPath = operationResult.ServerRootPath; - - //if (HttpContext.Current == null && !WorkContext.IsMono) - //{ - // HttpContext.Current = new HttpContext( - // new HttpRequest("hack", operationResult.Url, string.Empty), - // new HttpResponse(new StringWriter())); - //} - - _cache.Insert(GetKey(file), operationResult, TimeSpan.FromMinutes(10)); - } - - var commonLinkUtilitySettings = scope.ServiceProvider.GetService(); - commonLinkUtilitySettings.ServerUri = serverRootPath; - - var scopeClass = scope.ServiceProvider.GetService(); - (_, tenantManager, userManager, securityContext, daoFactory, fileSecurity, pathProvider, setupInfo, fileUtility, documentServiceHelper, documentServiceConnector, entryManager, fileConverter) = scopeClass; - - tenantManager.SetCurrentTenant(tenantId); - - securityContext.AuthenticateMeWithoutCookie(account); - - var user = userManager.GetUsers(account.ID); - var culture = string.IsNullOrEmpty(user.CultureName) ? tenantManager.GetCurrentTenant().GetCulture() : CultureInfo.GetCultureInfo(user.CultureName); - Thread.CurrentThread.CurrentCulture = culture; - Thread.CurrentThread.CurrentUICulture = culture; - - if (!fileSecurity.CanReadAsync(file).Result && file.RootFolderType != FolderType.BUNCH) - { - //No rights in CRM after upload before attach - throw new SecurityException(FilesCommonResource.ErrorMassage_SecurityException_ReadFile); - } - if (file.ContentLength > setupInfo.AvailableFileSize) - { - throw new Exception(string.Format(FilesCommonResource.ErrorMassage_FileSizeConvert, FileSizeComment.FilesSizeToString(setupInfo.AvailableFileSize))); - } - - fileUri = pathProvider.GetFileStreamUrl(file); - - var toExtension = fileUtility.GetInternalExtension(file.Title); - var fileExtension = file.ConvertedExtension; - var docKey = documentServiceHelper.GetDocKey(file); - - fileUri = documentServiceConnector.ReplaceCommunityAdress(fileUri); - (operationResultProgress, convertedFileUrl) = documentServiceConnector.GetConvertedUriAsync(fileUri, fileExtension, toExtension, docKey, password, null, null, true).Result; - } - catch (Exception exception) - { - var password = exception.InnerException is DocumentServiceException documentServiceException - && documentServiceException.Code == DocumentServiceException.ErrorCode.ConvertPassword; - - logger.ErrorConvertFileWithUrl(file.Id.ToString(), fileUri, exception); - lock (_locker) - { - if (_conversionQueue.TryGetValue(file, out var operationResult)) - { - if (operationResult.Delete) - { - _conversionQueue.Remove(file); - _cache.Remove(GetKey(file)); - } - else - { - operationResult.Progress = 100; - operationResult.StopDateTime = DateTime.UtcNow; - operationResult.Error = exception.Message; - if (password) - { - operationResult.Result = "password"; - } - - _cache.Insert(GetKey(file), operationResult, TimeSpan.FromMinutes(10)); - } - } - } - - continue; - } - - operationResultProgress = Math.Min(operationResultProgress, 100); - if (operationResultProgress < 100) - { - lock (_locker) - { - if (_conversionQueue.TryGetValue(file, out var operationResult)) - { - if (DateTime.Now - operationResult.StartDateTime > TimeSpan.FromMinutes(10)) - { - operationResult.StopDateTime = DateTime.UtcNow; - operationResult.Error = FilesCommonResource.ErrorMassage_ConvertTimeout; - logger.ErrorCheckConvertFilesStatus(file.Id.ToString(), file.ContentLength); - } - else - { - operationResult.Processed = ""; - } - operationResult.Progress = operationResultProgress; - _cache.Insert(GetKey(file), operationResult, TimeSpan.FromMinutes(10)); - } - } - - logger.DebugCheckConvertFilesStatusIterationContinue(); - - continue; - } - - File newFile = null; - var operationResultError = string.Empty; - - try - { - newFile = fileConverter.SaveConvertedFileAsync(file, convertedFileUrl).Result; - } - catch (Exception e) - { - operationResultError = e.Message; - - logger.ErrorOperation(operationResultError, convertedFileUrl, fileUri, e); - - continue; - } - finally - { - lock (_locker) - { - if (_conversionQueue.TryGetValue(file, out var operationResult)) - { - if (operationResult.Delete) - { - _conversionQueue.Remove(file); - _cache.Remove(GetKey(file)); - } - else - { - if (newFile != null) - { - var folderDao = daoFactory.GetFolderDao(); - var folder = folderDao.GetFolderAsync(newFile.ParentId).Result; - var folderTitle = fileSecurity.CanReadAsync(folder).Result ? folder.Title : null; - operationResult.Result = FileJsonSerializerAsync(entryManager, newFile, folderTitle).Result; - } - - operationResult.Progress = 100; - operationResult.StopDateTime = DateTime.UtcNow; - operationResult.Processed = "1"; - if (!string.IsNullOrEmpty(operationResultError)) - { - operationResult.Error = operationResultError; - } - - _cache.Insert(GetKey(file), operationResult, TimeSpan.FromMinutes(10)); - } - } - } - } - - logger.DebugCheckConvertFilesStatusIterationEnd(); - } - - lock (_locker) - { - _timer.Change(_timerPeriod, _timerPeriod); - } - } - catch (Exception exception) - { - logger.ErrorWithException(exception); - lock (_locker) - { - _timer.Change(Timeout.Infinite, Timeout.Infinite); - } - } - finally - { - Monitor.Exit(_singleThread); + Dequeue(task); } + + return operation; } + + return null; } + - private string GetKey(File f) - { - return string.Format("fileConvertation-{0}", f.Id); - } - - internal async Task FileJsonSerializerAsync(EntryStatusManager EntryManager, File file, string folderTitle) + public async Task FileJsonSerializerAsync(EntryStatusManager EntryManager, File file, string folderTitle) { if (file == null) { @@ -406,94 +183,73 @@ internal class FileConverterQueue : IDisposable }, options); } - public void Dispose() + private bool Contains(FileConverterOperationResult val) { - if (_timer != null) + var queueTasks = LoadFromCache(); + + return queueTasks.Any(x => { - _timer.Dispose(); + return String.Compare(x.Source, val.Source) == 0; + }); + } + + private bool IsOrphanCacheItem(FileConverterOperationResult x) + { + return !string.IsNullOrEmpty(x.Processed) + && (x.Progress == 100 && DateTime.UtcNow - x.StopDateTime > TimeSpan.FromMinutes(1) || + DateTime.UtcNow - x.StopDateTime > TimeSpan.FromMinutes(10)); + } + + private IEnumerable DeleteOrphanCacheItem(IEnumerable queueTasks) + { + var listTasks = queueTasks.ToList(); + + listTasks.RemoveAll(IsOrphanCacheItem); + + SaveToCache(listTasks); + + return queueTasks; + } + + private void SaveToCache(IEnumerable queueTasks) + { + if (!queueTasks.Any()) + { + _distributedCache.Remove(GetCacheKey()); + + return; } + + using var ms = new MemoryStream(); + + ProtoBuf.Serializer.Serialize(ms, queueTasks); + + _distributedCache.Set(GetCacheKey(), ms.ToArray(), new DistributedCacheEntryOptions + { + SlidingExpiration = TimeSpan.FromMinutes(15) + }); + } + + private string GetCacheKey() + { + return $"{_cache_key_prefix}_{typeof(T).Name}".ToLowerInvariant(); + } + + private IEnumerable LoadFromCache() + { + var serializedObject = _distributedCache.Get(GetCacheKey()); + + if (serializedObject == null) + { + return new List(); + } + + using var ms = new MemoryStream(serializedObject); + + return ProtoBuf.Serializer.Deserialize>(ms); } } -[Scope] -public class FileConverterQueueScope -{ - private readonly ILogger _options; - private readonly TenantManager _tenantManager; - private readonly UserManager _userManager; - private readonly SecurityContext _securityContext; - private readonly IDaoFactory _daoFactory; - private readonly FileSecurity _fileSecurity; - private readonly PathProvider _pathProvider; - private readonly SetupInfo _setupInfo; - private readonly FileUtility _fileUtility; - private readonly DocumentServiceHelper _documentServiceHelper; - private readonly DocumentServiceConnector _documentServiceConnector; - private readonly EntryStatusManager _entryManager; - private readonly FileConverter _fileConverter; - - public FileConverterQueueScope( - ILogger options, - TenantManager tenantManager, - UserManager userManager, - SecurityContext securityContext, - IDaoFactory daoFactory, - FileSecurity fileSecurity, - PathProvider pathProvider, - SetupInfo setupInfo, - FileUtility fileUtility, - DocumentServiceHelper documentServiceHelper, - DocumentServiceConnector documentServiceConnector, - EntryStatusManager entryManager, - FileConverter fileConverter) - { - _options = options; - _tenantManager = tenantManager; - _userManager = userManager; - _securityContext = securityContext; - _daoFactory = daoFactory; - _fileSecurity = fileSecurity; - _pathProvider = pathProvider; - _setupInfo = setupInfo; - _fileUtility = fileUtility; - _documentServiceHelper = documentServiceHelper; - _documentServiceConnector = documentServiceConnector; - _entryManager = entryManager; - _fileConverter = fileConverter; - } - - - public void Deconstruct(out ILogger optionsMonitor, - out TenantManager tenantManager, - out UserManager userManager, - out SecurityContext securityContext, - out IDaoFactory daoFactory, - out FileSecurity fileSecurity, - out PathProvider pathProvider, - out SetupInfo setupInfo, - out FileUtility fileUtility, - out DocumentServiceHelper documentServiceHelper, - out DocumentServiceConnector documentServiceConnector, - out EntryStatusManager entryManager, - out FileConverter fileConverter) - { - optionsMonitor = _options; - tenantManager = _tenantManager; - userManager = _userManager; - securityContext = _securityContext; - daoFactory = _daoFactory; - fileSecurity = _fileSecurity; - pathProvider = _pathProvider; - setupInfo = _setupInfo; - fileUtility = _fileUtility; - documentServiceHelper = _documentServiceHelper; - documentServiceConnector = _documentServiceConnector; - entryManager = _entryManager; - fileConverter = _fileConverter; - } - -} - public class FileJsonSerializerData { public T Id { get; set; } @@ -710,9 +466,9 @@ public class FileConverter var uriTuple = await _documentServiceConnector.GetConvertedUriAsync(fileUri, fileExtension, toExtension, docKey, null, null, null, false); var convertUri = uriTuple.ConvertedDocumentUri; - var operationResult = new ConvertFileOperationResult + var operationResult = new FileConverterOperationResult { - Source = string.Format("{{\"id\":\"{0}\", \"version\":\"{1}\"}}", file.Id, file.Version), + Source = System.Text.Json.JsonSerializer.Serialize(new { id = file.Id, version = file.Version }), OperationType = FileOperationType.Convert, Error = string.Empty, Progress = 0, @@ -720,7 +476,7 @@ public class FileConverter Processed = "", Id = string.Empty, TenantId = _tenantManager.GetCurrentTenant().Id, - Account = _authContext.CurrentAccount, + Account = _authContext.CurrentAccount.ID, Delete = false, StartDateTime = DateTime.Now, Url = _httpContextAccesor?.HttpContext != null ? _httpContextAccesor.HttpContext.Request.GetUrlRewriter().ToString() : null, @@ -918,27 +674,6 @@ internal class FileComparer : IEqualityComparer> } } -internal class ConvertFileOperationResult : FileOperationResult -{ - public DateTime StartDateTime { get; set; } - public DateTime StopDateTime { get; set; } - public int TenantId { get; set; } - public IAccount Account { get; set; } - public bool Delete { get; set; } - public string Url { get; set; } - public string Password { get; set; } - - //hack for download - public string ServerRootPath { get; set; } -} - -public static class FileConverterQueueExtension -{ - public static void Register(DIHelper services) - { - services.TryAdd(); - } -} public static class FileConverterExtension { diff --git a/products/ASC.Files/Core/Utils/FileConverterOperationResult.cs b/products/ASC.Files/Core/Utils/FileConverterOperationResult.cs new file mode 100644 index 0000000000..ee08a6a811 --- /dev/null +++ b/products/ASC.Files/Core/Utils/FileConverterOperationResult.cs @@ -0,0 +1,56 @@ +// (c) Copyright Ascensio System SIA 2010-2022 +// +// This program is a free software product. +// You can redistribute it and/or modify it under the terms +// of the GNU Affero General Public License (AGPL) version 3 as published by the Free Software +// Foundation. In accordance with Section 7(a) of the GNU AGPL its Section 15 shall be amended +// to the effect that Ascensio System SIA expressly excludes the warranty of non-infringement of +// any third-party rights. +// +// This program is distributed WITHOUT ANY WARRANTY, without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For details, see +// the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html +// +// You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia, EU, LV-1021. +// +// The interactive user interfaces in modified source and object code versions of the Program must +// display Appropriate Legal Notices, as required under Section 5 of the GNU AGPL version 3. +// +// Pursuant to Section 7(b) of the License you must retain the original Product logo when +// distributing the program. Pursuant to Section 7(e) we decline to grant you any rights under +// trademark law for use of our trademarks. +// +// All the Product's GUI elements, including illustrations and icon sets, as well as technical writing +// content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0 +// International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode + +namespace ASC.Web.Files.Utils; + +[ProtoContract] +public class FileConverterOperationResult : FileOperationResult +{ + [ProtoMember(1)] + public DateTime StartDateTime { get; set; } + + [ProtoMember(2)] + public DateTime StopDateTime { get; set; } + + [ProtoMember(3)] + public int TenantId { get; set; } + + [ProtoMember(4)] + public Guid Account { get; set; } + + [ProtoMember(5)] + public bool Delete { get; set; } + + [ProtoMember(6)] + public string Url { get; set; } + + [ProtoMember(7)] + public string Password { get; set; } + + [ProtoMember(8)] + //hack for download + public string ServerRootPath { get; set; } +} diff --git a/products/ASC.Files/Service/FileConverterService.cs b/products/ASC.Files/Service/FileConverterService.cs new file mode 100644 index 0000000000..2852b9862f --- /dev/null +++ b/products/ASC.Files/Service/FileConverterService.cs @@ -0,0 +1,362 @@ +// (c) Copyright Ascensio System SIA 2010-2022 +// +// This program is a free software product. +// You can redistribute it and/or modify it under the terms +// of the GNU Affero General Public License (AGPL) version 3 as published by the Free Software +// Foundation. In accordance with Section 7(a) of the GNU AGPL its Section 15 shall be amended +// to the effect that Ascensio System SIA expressly excludes the warranty of non-infringement of +// any third-party rights. +// +// This program is distributed WITHOUT ANY WARRANTY, without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For details, see +// the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html +// +// You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia, EU, LV-1021. +// +// The interactive user interfaces in modified source and object code versions of the Program must +// display Appropriate Legal Notices, as required under Section 5 of the GNU AGPL version 3. +// +// Pursuant to Section 7(b) of the License you must retain the original Product logo when +// distributing the program. Pursuant to Section 7(e) we decline to grant you any rights under +// trademark law for use of our trademarks. +// +// All the Product's GUI elements, including illustrations and icon sets, as well as technical writing +// content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0 +// International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode + +using System.Globalization; +using System.Text.Json; + +using ASC.Common.Log; +using ASC.Core.Common.Hosting; +using ASC.Core.Common.Hosting.Interfaces; +using ASC.Files.Core.Log; +using ASC.Files.Core.Resources; +using ASC.Web.Files.Utils; +using ASC.Web.Studio.Core; + +using static ASC.Web.Core.Files.DocumentService; + +namespace ASC.Files.ThumbnailBuilder; + +[Singletone(Additional = typeof(FileConverterQueueExtension))] +internal class FileConverterService : BackgroundService +{ + private readonly IServiceScopeFactory _serviceScopeFactory; + private readonly int _timerDelay = 1000; + private readonly ILogger> _logger; + + public FileConverterService( + IServiceScopeFactory serviceScopeFactory, + ILogger> logger) + { + _logger = logger; + _serviceScopeFactory = serviceScopeFactory; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.DebugFileConverterServiceRuning(); + + stoppingToken.Register(() => _logger.DebugFileConverterServiceStopping()); + + while (!stoppingToken.IsCancellationRequested) + { + using var serviceScope = _serviceScopeFactory.CreateScope(); + + var registerInstanceService = serviceScope.ServiceProvider.GetService>>(); + + if (!await registerInstanceService.IsActive(RegisterInstanceWorkerService>.InstanceId)) + { + await Task.Delay(1000, stoppingToken); + + continue; + } + + await ExecuteCheckFileConverterStatus(serviceScope); + + await Task.Delay(_timerDelay, stoppingToken); + } + } + + private async Task ExecuteCheckFileConverterStatus(IServiceScope scope) + { + TenantManager tenantManager; + UserManager userManager; + SecurityContext securityContext; + IDaoFactory daoFactory; + FileSecurity fileSecurity; + PathProvider pathProvider; + SetupInfo setupInfo; + FileUtility fileUtility; + DocumentServiceHelper documentServiceHelper; + DocumentServiceConnector documentServiceConnector; + EntryStatusManager entryManager; + FileConverter fileConverter; + FileConverterQueue fileConverterQueue; + + var logger = scope.ServiceProvider.GetService>>(); + + try + { + var scopeClass = scope.ServiceProvider.GetService(); + (_, tenantManager, userManager, securityContext, daoFactory, fileSecurity, pathProvider, setupInfo, fileUtility, documentServiceHelper, documentServiceConnector, entryManager, fileConverter) = scopeClass; + + fileConverterQueue = scope.ServiceProvider.GetService>(); + + var _conversionQueue = fileConverterQueue.GetAllTask().ToList(); + + logger.DebugRunCheckConvertFilesStatus(_conversionQueue.Count); + + var filesIsConverting = _conversionQueue + .Where(x => string.IsNullOrEmpty(x.Processed)) + .ToList(); + + foreach (var converter in filesIsConverting) + { + var fileId = JsonDocument.Parse(converter.Source).RootElement.GetProperty("id").Deserialize(); + var fileVersion = JsonDocument.Parse(converter.Source).RootElement.GetProperty("version").Deserialize(); + + int operationResultProgress; + var password = converter.Password; + + var commonLinkUtilitySettings = scope.ServiceProvider.GetService(); + commonLinkUtilitySettings.ServerUri = converter.ServerRootPath; + + tenantManager.SetCurrentTenant(converter.TenantId); + + securityContext.AuthenticateMeWithoutCookie(converter.Account); + + var file = await daoFactory.GetFileDao().GetFileAsync(fileId, fileVersion); + var fileUri = file.Id.ToString(); + + string convertedFileUrl; + + try + { + + var user = userManager.GetUsers(converter.Account); + + var culture = string.IsNullOrEmpty(user.CultureName) ? tenantManager.GetCurrentTenant().GetCulture() : CultureInfo.GetCultureInfo(user.CultureName); + + Thread.CurrentThread.CurrentCulture = culture; + Thread.CurrentThread.CurrentUICulture = culture; + + if (!fileSecurity.CanReadAsync(file).Result && file.RootFolderType != FolderType.BUNCH) + { + //No rights in CRM after upload before attach + throw new System.Security.SecurityException(FilesCommonResource.ErrorMassage_SecurityException_ReadFile); + } + + if (file.ContentLength > setupInfo.AvailableFileSize) + { + throw new Exception(string.Format(FilesCommonResource.ErrorMassage_FileSizeConvert, FileSizeComment.FilesSizeToString(setupInfo.AvailableFileSize))); + } + + fileUri = pathProvider.GetFileStreamUrl(file); + + var toExtension = fileUtility.GetInternalExtension(file.Title); + var fileExtension = file.ConvertedExtension; + var docKey = documentServiceHelper.GetDocKey(file); + + fileUri = documentServiceConnector.ReplaceCommunityAdress(fileUri); + (operationResultProgress, convertedFileUrl) = documentServiceConnector.GetConvertedUriAsync(fileUri, fileExtension, toExtension, docKey, password, null, null, true).Result; + } + catch (Exception exception) + { + var password1 = exception.InnerException is DocumentServiceException documentServiceException + && documentServiceException.Code == DocumentServiceException.ErrorCode.ConvertPassword; + + logger.ErrorConvertFileWithUrl(file.Id.ToString(), fileUri, exception); + + var operationResult = converter; + + if (operationResult.Delete) + { + _conversionQueue.Remove(operationResult); + } + else + { + operationResult.Progress = 100; + operationResult.StopDateTime = DateTime.UtcNow; + operationResult.Error = exception.Message; + + if (password1) + { + operationResult.Result = "password"; + } + } + + continue; + } + + operationResultProgress = Math.Min(operationResultProgress, 100); + + if (operationResultProgress < 100) + { + var operationResult = converter; + + if (DateTime.Now - operationResult.StartDateTime > TimeSpan.FromMinutes(10)) + { + operationResult.StopDateTime = DateTime.UtcNow; + operationResult.Error = FilesCommonResource.ErrorMassage_ConvertTimeout; + + logger.ErrorCheckConvertFilesStatus(file.Id.ToString(), file.ContentLength); + } + else + { + operationResult.Processed = ""; + } + + operationResult.Progress = operationResultProgress; + + logger.DebugCheckConvertFilesStatusIterationContinue(); + + continue; + } + + File newFile = null; + + var operationResultError = string.Empty; + + try + { + newFile = fileConverter.SaveConvertedFileAsync(file, convertedFileUrl).Result; + } + catch (Exception e) + { + operationResultError = e.Message; + + logger.ErrorOperation(operationResultError, convertedFileUrl, fileUri, e); + + continue; + } + finally + { + var operationResult = converter; + + if (operationResult.Delete) + { + _conversionQueue.Remove(operationResult); + } + else + { + if (newFile != null) + { + var folderDao = daoFactory.GetFolderDao(); + var folder = folderDao.GetFolderAsync(newFile.ParentId).Result; + var folderTitle = fileSecurity.CanReadAsync(folder).Result ? folder.Title : null; + + operationResult.Result = fileConverterQueue.FileJsonSerializerAsync(entryManager, newFile, folderTitle).Result; + } + + operationResult.Progress = 100; + operationResult.StopDateTime = DateTime.UtcNow; + operationResult.Processed = "1"; + + if (!string.IsNullOrEmpty(operationResultError)) + { + operationResult.Error = operationResultError; + } + } + } + + logger.DebugCheckConvertFilesStatusIterationEnd(); + } + + fileConverterQueue.SetAllTask(_conversionQueue); + + } + catch (Exception exception) + { + logger.ErrorWithException(exception); + } + } +} + +public static class FileConverterQueueExtension +{ + public static void Register(DIHelper services) + { + services.TryAdd(); + } +} + +[Scope] +public class FileConverterQueueScope +{ + private readonly ILogger _options; + private readonly TenantManager _tenantManager; + private readonly UserManager _userManager; + private readonly SecurityContext _securityContext; + private readonly IDaoFactory _daoFactory; + private readonly FileSecurity _fileSecurity; + private readonly PathProvider _pathProvider; + private readonly SetupInfo _setupInfo; + private readonly FileUtility _fileUtility; + private readonly DocumentServiceHelper _documentServiceHelper; + private readonly DocumentServiceConnector _documentServiceConnector; + private readonly EntryStatusManager _entryManager; + private readonly FileConverter _fileConverter; + + public FileConverterQueueScope( + ILogger options, + TenantManager tenantManager, + UserManager userManager, + SecurityContext securityContext, + IDaoFactory daoFactory, + FileSecurity fileSecurity, + PathProvider pathProvider, + SetupInfo setupInfo, + FileUtility fileUtility, + DocumentServiceHelper documentServiceHelper, + DocumentServiceConnector documentServiceConnector, + EntryStatusManager entryManager, + FileConverter fileConverter) + { + _options = options; + _tenantManager = tenantManager; + _userManager = userManager; + _securityContext = securityContext; + _daoFactory = daoFactory; + _fileSecurity = fileSecurity; + _pathProvider = pathProvider; + _setupInfo = setupInfo; + _fileUtility = fileUtility; + _documentServiceHelper = documentServiceHelper; + _documentServiceConnector = documentServiceConnector; + _entryManager = entryManager; + _fileConverter = fileConverter; + } + + + public void Deconstruct(out ILogger optionsMonitor, + out TenantManager tenantManager, + out UserManager userManager, + out SecurityContext securityContext, + out IDaoFactory daoFactory, + out FileSecurity fileSecurity, + out PathProvider pathProvider, + out SetupInfo setupInfo, + out FileUtility fileUtility, + out DocumentServiceHelper documentServiceHelper, + out DocumentServiceConnector documentServiceConnector, + out EntryStatusManager entryManager, + out FileConverter fileConverter) + { + optionsMonitor = _options; + tenantManager = _tenantManager; + userManager = _userManager; + securityContext = _securityContext; + daoFactory = _daoFactory; + fileSecurity = _fileSecurity; + pathProvider = _pathProvider; + setupInfo = _setupInfo; + fileUtility = _fileUtility; + documentServiceHelper = _documentServiceHelper; + documentServiceConnector = _documentServiceConnector; + entryManager = _entryManager; + fileConverter = _fileConverter; + } + +} diff --git a/products/ASC.Files/Core/Log/FileConverterLogger.cs b/products/ASC.Files/Service/Log/FileConverterLogger.cs similarity index 87% rename from products/ASC.Files/Core/Log/FileConverterLogger.cs rename to products/ASC.Files/Service/Log/FileConverterLogger.cs index 5e733d3669..03da98ae77 100644 --- a/products/ASC.Files/Core/Log/FileConverterLogger.cs +++ b/products/ASC.Files/Service/Log/FileConverterLogger.cs @@ -25,7 +25,7 @@ // International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode namespace ASC.Files.Core.Log; -internal static partial class FileConverterLogger +public static partial class FileConverterLogger { [LoggerMessage(Level = LogLevel.Debug, Message = "Run CheckConvertFilesStatus: count {count}")] public static partial void DebugRunCheckConvertFilesStatus(this ILogger logger, int count); @@ -44,4 +44,11 @@ internal static partial class FileConverterLogger [LoggerMessage(Level = LogLevel.Error, Message = "CheckConvertFilesStatus timeout: {fileId} ({contentLengthString})")] public static partial void ErrorCheckConvertFilesStatus(this ILogger logger, string fileId, long contentLengthString); + + [LoggerMessage(Level = LogLevel.Debug, Message = "FileConverterService is starting.")] + public static partial void DebugFileConverterServiceRuning(this ILogger logger); + + [LoggerMessage(Level = LogLevel.Debug, Message = "FileConverterService is stopping")] + public static partial void DebugFileConverterServiceStopping(this ILogger logger); + } diff --git a/products/ASC.Files/Service/Program.cs b/products/ASC.Files/Service/Program.cs index f651491261..a3bbcc8af6 100644 --- a/products/ASC.Files/Service/Program.cs +++ b/products/ASC.Files/Service/Program.cs @@ -68,6 +68,13 @@ builder.Host.ConfigureDefault(args, (hostContext, config, env, path) => diHelper.TryAdd(); + services.AddActivePassiveHostedService>(); + diHelper.TryAdd>(); + + services.AddActivePassiveHostedService>(); + diHelper.TryAdd>(); + + services.AddHostedService(); diHelper.TryAdd();