IlyaSobolev/DocSpace-buildtools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #691 from ONLYOFFICE/feature/refactoring-file-converter

Browse Source 
fileconverter: moved timer to separete worker service
This commit is contained in:
Alexey Bannov 2022-06-15 13:32:59 +04:00 committed by GitHub
commit 20b85665e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 658 additions and 434 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
common/ASC.Common/Data/TypeExtension.cs Normal file
View File

@ -0,0 +1,42 @@
// (c) Copyright Ascensio System SIA 2010-2022
//
// This program is a free software product.
// You can redistribute it and/or modify it under the terms
// of the GNU Affero General Public License (AGPL) version 3 as published by the Free Software
// Foundation. In accordance with Section 7(a) of the GNU AGPL its Section 15 shall be amended
// to the effect that Ascensio System SIA expressly excludes the warranty of non-infringement of
// any third-party rights.
//
// This program is distributed WITHOUT ANY WARRANTY, without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For details, see
// the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
//
// You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia, EU, LV-1021.
//
// The interactive user interfaces in modified source and object code versions of the Program must
// display Appropriate Legal Notices, as required under Section 5 of the GNU AGPL version 3.
//
// Pursuant to Section 7(b) of the License you must retain the original Product logo when
// distributing the program. Pursuant to Section 7(e) we decline to grant you any rights under
// trademark law for use of our trademarks.
//
// All the Product's GUI elements, including illustrations and icon sets, as well as technical writing
// content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0
// International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
namespace ASC.Common.Data;
public static class TypeExtension
{
public static string GetFormattedName(this Type type)
{
if (type.IsGenericType)
{
var genericArguments = type.GetGenericArguments()
.Select(x => x.Name)
.Aggregate((x1, x2) => $"{x1}, {x2}");
return $"{type.Name.Substring(0, type.Name.IndexOf("`"))}"
+ $"<{genericArguments}>";
}
return type.Name;
}
}

2
common/ASC.Common/Log/DistributedTaskQueueLogger.cs
View File

@ -27,7 +27,7 @@
namespace ASC.Common.Log;
internal static partial class DistributedTaskQueueLogger
{
[LoggerMessage(Level = LogLevel.Trace, Message = "EnqueueTask '{distributedTaskId}' by instanse id '{instanceId}'")]
[LoggerMessage(Level = LogLevel.Trace, Message = "EnqueueTask '{DistributedTaskId}' by instanse id '{instanceId}'")]
public static partial void TraceEnqueueTask(this ILogger<DistributedTaskQueue> logger, string DistributedTaskId, int instanceId);
[LoggerMessage(Level = LogLevel.Trace, Message = "Publication DistributedTask '{DistributedTaskId}' by instanse id '{instanceId}' ")]

23
common/ASC.Common/Threading/DistributedTaskQueue.cs
View File

@ -305,6 +305,13 @@ public class DistributedTaskQueue
private void SaveToCache(IEnumerable<DistributedTask> queueTasks)
{
if (!queueTasks.Any())
{
_distributedCache.Remove(_name);
return;
}
using var ms = new MemoryStream();
Serializer.Serialize(ms, queueTasks);
@ -332,21 +339,11 @@ public class DistributedTaskQueue
private IEnumerable<DistributedTask> DeleteOrphanCacheItem(IEnumerable<DistributedTask> queueTasks)
{
if (!queueTasks.Any())
{
return queueTasks;
}
var listTasks = queueTasks.ToList();
var orphans = queueTasks.Where(IsOrphanCacheItem);
listTasks.RemoveAll(IsOrphanCacheItem);
if (!orphans.Any())
{
return queueTasks;
}
queueTasks = queueTasks.Except(queueTasks);
SaveToCache(queueTasks);
SaveToCache(listTasks);
return queueTasks;
}

1
common/ASC.Core.Common/GlobalUsings.cs
View File

@ -149,3 +149,4 @@ global using ProtoBuf;
global using Telegram.Bot;
global using static ASC.Security.Cryptography.EmailValidationKeyProvider;
global using ASC.Common.Data;

2
common/ASC.Core.Common/Hosting/RegisterInstanceDao.cs
View File

@ -79,7 +79,7 @@ public class RegisterInstanceDao<T> : IRegisterInstanceDao<T> where T : IHostedS
public async Task<IEnumerable<InstanceRegistration>> GetAll()
{
return await _instanceRegistrationContext.InstanceRegistrations
.Where(x => x.WorkerTypeName == typeof(T).Name)
.Where(x => x.WorkerTypeName == typeof(T).GetFormattedName())
.ToListAsync();
}

2
common/ASC.Core.Common/Hosting/RegisterInstanceManager.cs
View File

@ -51,7 +51,7 @@ public class RegisterInstanceManager<T> : IRegisterInstanceManager<T> where T :
var instance = registeredInstance ?? new InstanceRegistration
{
InstanceRegistrationId = instanceId,
WorkerTypeName = typeof(T).Name
WorkerTypeName = typeof(T).GetFormattedName()
};
instance.LastUpdated = DateTime.UtcNow;

9
common/ASC.Core.Common/Hosting/RegisterInstanceWorkerService.cs
View File

@ -24,7 +24,7 @@
// content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0
// International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
namespace ASC.Core.Common.Hosting;
namespace ASC.Core.Common.Hosting;
[Singletone]
public class RegisterInstanceWorkerService<T> : BackgroundService where T : IHostedService
@ -33,7 +33,7 @@ public class RegisterInstanceWorkerService<T> : BackgroundService where T : IHos
private readonly IHostApplicationLifetime _applicationLifetime;
private readonly IServiceProvider _serviceProvider;
public static readonly string InstanceId =
$"{typeof(T).Name}_{DateTime.UtcNow.Ticks}";
$"{typeof(T).GetFormattedName()}_{DateTime.UtcNow.Ticks}";
public RegisterInstanceWorkerService(
ILogger<RegisterInstanceWorkerService<T>> logger,
@ -58,7 +58,7 @@ public class RegisterInstanceWorkerService<T> : BackgroundService where T : IHos
await registerInstanceService.Register(InstanceId);
await registerInstanceService.DeleteOrphanInstances();
_logger.InformationWorkingRunnging(DateTimeOffset.Now);
_logger.TraceWorkingRunnging(DateTimeOffset.Now);
await Task.Delay(1000, stoppingToken);
}
@ -87,5 +87,6 @@ public class RegisterInstanceWorkerService<T> : BackgroundService where T : IHos
}
await base.StopAsync(cancellationToken);
}
}
}

4
common/ASC.Core.Common/Log/RegisterInstanceWorkerServiceLogger.cs
View File

@ -31,8 +31,8 @@ internal static partial class RegisterInstanceWorkerServiceLogger
[LoggerMessage(Level = LogLevel.Trace, Message = "DbUpdateConcurrencyException: then updating {instanceName} at {time} time.")]
public static partial void TraceDbUpdateConcurrencyException(this ILogger logger, string instanceName, DateTimeOffset time);
[LoggerMessage(Level = LogLevel.Information, Message = "Worker running at: {time}")]
public static partial void InformationWorkingRunnging(this ILogger logger, DateTimeOffset time);
[LoggerMessage(Level = LogLevel.Trace, Message = "Worker running at: {time}")]
public static partial void TraceWorkingRunnging(this ILogger logger, DateTimeOffset time);
[LoggerMessage(Level = LogLevel.Information, Message = "UnRegister Instance {instanceName} running at: {time}.")]
public static partial void InformationUnRegister(this ILogger logger, string instanceName, DateTimeOffset time);

16
products/ASC.Files/Core/Services/WCFService/FileOperations/FileOperationResult.cs
View File

@ -26,16 +26,32 @@
namespace ASC.Web.Files.Services.WCFService.FileOperations;
[ProtoContract]
[ProtoInclude(100, typeof(FileConverterOperationResult))]
public class FileOperationResult
{
[ProtoMember(1)]
public string Id { get; set; }
[JsonPropertyName("operation")]
[ProtoMember(2)]
public FileOperationType OperationType { get; set; }
[ProtoMember(3)]
public int Progress { get; set; }
[ProtoMember(4)]
public string Source { get; set; }
[ProtoMember(5)]
public string Result { get; set; }
[ProtoMember(6)]
public string Error { get; set; }
[ProtoMember(7)]
public string Processed { get; set; }
[ProtoMember(8)]
public bool Finished { get; set; }
}

557
products/ASC.Files/Core/Utils/FileConverter.cs
View File

@ -24,44 +24,42 @@
// content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0
// International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
using Microsoft.Extensions.Caching.Distributed;
using Timeout = System.Threading.Timeout;
namespace ASC.Web.Files.Utils;
[Singletone(Additional = typeof(FileConverterQueueExtension))]
internal class FileConverterQueue<T> : IDisposable
[Singletone]
public class FileConverterQueue<T>
{
private readonly object _singleThread = new object();
private readonly IDictionary<File<T>, ConvertFileOperationResult> _conversionQueue;
private readonly Timer _timer;
private readonly object _locker;
private readonly ICache _cache;
private const int _timerPeriod = 500;
private readonly object _locker = new object();
private readonly IDistributedCache _distributedCache;
private readonly string _cache_key_prefix = "asc_file_converter_queue_";
private readonly IServiceScopeFactory _serviceScopeFactory;
public FileConverterQueue(IServiceScopeFactory serviceScopeFactory, ICache cache)
public FileConverterQueue(IDistributedCache distributedCache)
{
_conversionQueue = new Dictionary<File<T>, ConvertFileOperationResult>(new FileComparer<T>());
_timer = new Timer(CheckConvertFilesStatus, null, 0, Timeout.Infinite);
_locker = new object();
_serviceScopeFactory = serviceScopeFactory;
_cache = cache;
_distributedCache = distributedCache;
}
public void Add(File<T> file, string password, int tenantId, IAccount account, bool deleteAfter, string url, string serverRootPath)
public void Add(File<T> file,
string password,
int tenantId,
IAccount account,
bool deleteAfter,
string url,
string serverRootPath)
{
lock (_locker)
{
if (_conversionQueue.ContainsKey(file))
var task = PeekTask(file);
if (Contains(task))
{
return;
}
var queueResult = new ConvertFileOperationResult
var queueResult = new FileConverterOperationResult
{
Source = string.Format("{{\"id\":\"{0}\", \"version\":\"{1}\"}}", file.Id, file.Version),
Source = System.Text.Json.JsonSerializer.Serialize(new { id = file.Id, version = file.Version }),
OperationType = FileOperationType.Convert,
Error = string.Empty,
Progress = 0,
@ -69,316 +67,95 @@ internal class FileConverterQueue<T> : IDisposable
Processed = "",
Id = string.Empty,
TenantId = tenantId,
Account = account,
Account = account.ID,
Delete = deleteAfter,
StartDateTime = DateTime.Now,
Url = url,
Password = password,
ServerRootPath = serverRootPath
};
_conversionQueue.Add(file, queueResult);
_cache.Insert(GetKey(file), queueResult, TimeSpan.FromMinutes(10));
_timer.Change(0, Timeout.Infinite);
Enqueue(queueResult);
}
}
public async Task<ConvertFileOperationResult> GetStatusAsync(KeyValuePair<File<T>, bool> pair, FileSecurity fileSecurity)
public void Enqueue(FileConverterOperationResult val)
{
var file = pair.Key;
var key = GetKey(file);
var operation = _cache.Get<ConvertFileOperationResult>(key);
if (operation != null && (pair.Value || await fileSecurity.CanReadAsync(file)))
var fromCache = LoadFromCache().ToList();
fromCache.Add(val);
SaveToCache(fromCache);
}
public void Dequeue(FileConverterOperationResult val)
{
var fromCache = LoadFromCache().ToList();
fromCache.Remove(val);
SaveToCache(fromCache);
}
public FileConverterOperationResult PeekTask(File<T> file)
{
var exist = LoadFromCache();
return exist.FirstOrDefault(x =>
{
lock (_locker)
{
if (operation.Progress == 100)
{
_conversionQueue.Remove(file);
_cache.Remove(key);
}
var fileId = JsonDocument.Parse(x.Source).RootElement.GetProperty("id").Deserialize<T>();
var fileVersion = JsonDocument.Parse(x.Source).RootElement.GetProperty("version").Deserialize<int>();
return operation;
}
}
return null;
return file.Id.ToString() == fileId.ToString() && file.Version == fileVersion;
});
}
public bool IsConverting(File<T> file)
{
var result = _cache.Get<ConvertFileOperationResult>(GetKey(file));
var result = PeekTask(file);
return result != null && result.Progress != 100 && string.IsNullOrEmpty(result.Error);
}
private void CheckConvertFilesStatus(object _)
public IEnumerable<FileConverterOperationResult> GetAllTask()
{
if (Monitor.TryEnter(_singleThread))
var queueTasks = LoadFromCache();
queueTasks = DeleteOrphanCacheItem(queueTasks);
return queueTasks;
}
public void SetAllTask(IEnumerable<FileConverterOperationResult> queueTasks)
{
SaveToCache(queueTasks);
}
public async Task<FileConverterOperationResult> GetStatusAsync(KeyValuePair<File<T>, bool> pair, FileSecurity fileSecurity)
{
var file = pair.Key;
var operation = PeekTask(pair.Key);
if (operation != null && (pair.Value || await fileSecurity.CanReadAsync(file)))
{
using var scope = _serviceScopeFactory.CreateScope();
TenantManager tenantManager;
UserManager userManager;
SecurityContext securityContext;
IDaoFactory daoFactory;
FileSecurity fileSecurity;
PathProvider pathProvider;
SetupInfo setupInfo;
FileUtility fileUtility;
DocumentServiceHelper documentServiceHelper;
DocumentServiceConnector documentServiceConnector;
EntryStatusManager entryManager;
FileConverter fileConverter;
var logger = scope.ServiceProvider.GetService<ILogger<FileConverterQueue<T>>>();
try
if (operation.Progress == 100)
{
var filesIsConverting = new List<File<T>>();
lock (_locker)
{
_timer.Change(Timeout.Infinite, Timeout.Infinite);
var queues = _conversionQueue.Where(x => !string.IsNullOrEmpty(x.Value.Processed)
&& (x.Value.Progress == 100 && DateTime.UtcNow - x.Value.StopDateTime > TimeSpan.FromMinutes(1) ||
DateTime.UtcNow - x.Value.StopDateTime > TimeSpan.FromMinutes(10)))
.ToList();
var task = PeekTask(file);
foreach (var q in queues)
{
_conversionQueue.Remove(q);
_cache.Remove(GetKey(q.Key));
}
logger.DebugRunCheckConvertFilesStatus(_conversionQueue.Count);
if (_conversionQueue.Count == 0)
{
return;
}
filesIsConverting = _conversionQueue
.Where(x => string.IsNullOrEmpty(x.Value.Processed))
.Select(x => x.Key)
.ToList();
}
string convertedFileUrl = null;
foreach (var file in filesIsConverting)
{
var fileUri = file.Id.ToString();
int operationResultProgress;
try
{
int tenantId;
IAccount account;
string password;
string serverRootPath;
lock (_locker)
{
if (!_conversionQueue.ContainsKey(file))
{
continue;
}
var operationResult = _conversionQueue[file];
if (!string.IsNullOrEmpty(operationResult.Processed))
{
continue;
}
operationResult.Processed = "1";
tenantId = operationResult.TenantId;
account = operationResult.Account;
password = operationResult.Password;
serverRootPath = operationResult.ServerRootPath;
//if (HttpContext.Current == null && !WorkContext.IsMono)
//{
// HttpContext.Current = new HttpContext(
// new HttpRequest("hack", operationResult.Url, string.Empty),
// new HttpResponse(new StringWriter()));
//}
_cache.Insert(GetKey(file), operationResult, TimeSpan.FromMinutes(10));
}
var commonLinkUtilitySettings = scope.ServiceProvider.GetService<CommonLinkUtilitySettings>();
commonLinkUtilitySettings.ServerUri = serverRootPath;
var scopeClass = scope.ServiceProvider.GetService<FileConverterQueueScope>();
(_, tenantManager, userManager, securityContext, daoFactory, fileSecurity, pathProvider, setupInfo, fileUtility, documentServiceHelper, documentServiceConnector, entryManager, fileConverter) = scopeClass;
tenantManager.SetCurrentTenant(tenantId);
securityContext.AuthenticateMeWithoutCookie(account);
var user = userManager.GetUsers(account.ID);
var culture = string.IsNullOrEmpty(user.CultureName) ? tenantManager.GetCurrentTenant().GetCulture() : CultureInfo.GetCultureInfo(user.CultureName);
Thread.CurrentThread.CurrentCulture = culture;
Thread.CurrentThread.CurrentUICulture = culture;
if (!fileSecurity.CanReadAsync(file).Result && file.RootFolderType != FolderType.BUNCH)
{
//No rights in CRM after upload before attach
throw new SecurityException(FilesCommonResource.ErrorMassage_SecurityException_ReadFile);
}
if (file.ContentLength > setupInfo.AvailableFileSize)
{
throw new Exception(string.Format(FilesCommonResource.ErrorMassage_FileSizeConvert, FileSizeComment.FilesSizeToString(setupInfo.AvailableFileSize)));
}
fileUri = pathProvider.GetFileStreamUrl(file);
var toExtension = fileUtility.GetInternalExtension(file.Title);
var fileExtension = file.ConvertedExtension;
var docKey = documentServiceHelper.GetDocKey(file);
fileUri = documentServiceConnector.ReplaceCommunityAdress(fileUri);
(operationResultProgress, convertedFileUrl) = documentServiceConnector.GetConvertedUriAsync(fileUri, fileExtension, toExtension, docKey, password, null, null, true).Result;
}
catch (Exception exception)
{
var password = exception.InnerException is DocumentServiceException documentServiceException
&& documentServiceException.Code == DocumentServiceException.ErrorCode.ConvertPassword;
logger.ErrorConvertFileWithUrl(file.Id.ToString(), fileUri, exception);
lock (_locker)
{
if (_conversionQueue.TryGetValue(file, out var operationResult))
{
if (operationResult.Delete)
{
_conversionQueue.Remove(file);
_cache.Remove(GetKey(file));
}
else
{
operationResult.Progress = 100;
operationResult.StopDateTime = DateTime.UtcNow;
operationResult.Error = exception.Message;
if (password)
{
operationResult.Result = "password";
}
_cache.Insert(GetKey(file), operationResult, TimeSpan.FromMinutes(10));
}
}
}
continue;
}
operationResultProgress = Math.Min(operationResultProgress, 100);
if (operationResultProgress < 100)
{
lock (_locker)
{
if (_conversionQueue.TryGetValue(file, out var operationResult))
{
if (DateTime.Now - operationResult.StartDateTime > TimeSpan.FromMinutes(10))
{
operationResult.StopDateTime = DateTime.UtcNow;
operationResult.Error = FilesCommonResource.ErrorMassage_ConvertTimeout;
logger.ErrorCheckConvertFilesStatus(file.Id.ToString(), file.ContentLength);
}
else
{
operationResult.Processed = "";
}
operationResult.Progress = operationResultProgress;
_cache.Insert(GetKey(file), operationResult, TimeSpan.FromMinutes(10));
}
}
logger.DebugCheckConvertFilesStatusIterationContinue();
continue;
}
File<T> newFile = null;
var operationResultError = string.Empty;
try
{
newFile = fileConverter.SaveConvertedFileAsync(file, convertedFileUrl).Result;
}
catch (Exception e)
{
operationResultError = e.Message;
logger.ErrorOperation(operationResultError, convertedFileUrl, fileUri, e);
continue;
}
finally
{
lock (_locker)
{
if (_conversionQueue.TryGetValue(file, out var operationResult))
{
if (operationResult.Delete)
{
_conversionQueue.Remove(file);
_cache.Remove(GetKey(file));
}
else
{
if (newFile != null)
{
var folderDao = daoFactory.GetFolderDao<T>();
var folder = folderDao.GetFolderAsync(newFile.ParentId).Result;
var folderTitle = fileSecurity.CanReadAsync(folder).Result ? folder.Title : null;
operationResult.Result = FileJsonSerializerAsync(entryManager, newFile, folderTitle).Result;
}
operationResult.Progress = 100;
operationResult.StopDateTime = DateTime.UtcNow;
operationResult.Processed = "1";
if (!string.IsNullOrEmpty(operationResultError))
{
operationResult.Error = operationResultError;
}
_cache.Insert(GetKey(file), operationResult, TimeSpan.FromMinutes(10));
}
}
}
}
logger.DebugCheckConvertFilesStatusIterationEnd();
}
lock (_locker)
{
_timer.Change(_timerPeriod, _timerPeriod);
}
}
catch (Exception exception)
{
logger.ErrorWithException(exception);
lock (_locker)
{
_timer.Change(Timeout.Infinite, Timeout.Infinite);
}
}
finally
{
Monitor.Exit(_singleThread);
Dequeue(task);
}
return operation;
}
return null;
}
private string GetKey(File<T> f)
{
return string.Format("fileConvertation-{0}", f.Id);
}
internal async Task<string> FileJsonSerializerAsync(EntryStatusManager EntryManager, File<T> file, string folderTitle)
public async Task<string> FileJsonSerializerAsync(EntryStatusManager EntryManager, File<T> file, string folderTitle)
{
if (file == null)
{
@ -406,94 +183,73 @@ internal class FileConverterQueue<T> : IDisposable
}, options);
}
public void Dispose()
private bool Contains(FileConverterOperationResult val)
{
if (_timer != null)
var queueTasks = LoadFromCache();
return queueTasks.Any(x =>
{
_timer.Dispose();
return String.Compare(x.Source, val.Source) == 0;
});
}
private bool IsOrphanCacheItem(FileConverterOperationResult x)
{
return !string.IsNullOrEmpty(x.Processed)
&& (x.Progress == 100 && DateTime.UtcNow - x.StopDateTime > TimeSpan.FromMinutes(1) ||
DateTime.UtcNow - x.StopDateTime > TimeSpan.FromMinutes(10));
}
private IEnumerable<FileConverterOperationResult> DeleteOrphanCacheItem(IEnumerable<FileConverterOperationResult> queueTasks)
{
var listTasks = queueTasks.ToList();
listTasks.RemoveAll(IsOrphanCacheItem);
SaveToCache(listTasks);
return queueTasks;
}
private void SaveToCache(IEnumerable<FileConverterOperationResult> queueTasks)
{
if (!queueTasks.Any())
{
_distributedCache.Remove(GetCacheKey());
return;
}
using var ms = new MemoryStream();
ProtoBuf.Serializer.Serialize(ms, queueTasks);
_distributedCache.Set(GetCacheKey(), ms.ToArray(), new DistributedCacheEntryOptions
{
SlidingExpiration = TimeSpan.FromMinutes(15)
});
}
private string GetCacheKey()
{
return $"{_cache_key_prefix}_{typeof(T).Name}".ToLowerInvariant();
}
private IEnumerable<FileConverterOperationResult> LoadFromCache()
{
var serializedObject = _distributedCache.Get(GetCacheKey());
if (serializedObject == null)
{
return new List<FileConverterOperationResult>();
}
using var ms = new MemoryStream(serializedObject);
return ProtoBuf.Serializer.Deserialize<List<FileConverterOperationResult>>(ms);
}
}
[Scope]
public class FileConverterQueueScope
{
private readonly ILogger _options;
private readonly TenantManager _tenantManager;
private readonly UserManager _userManager;
private readonly SecurityContext _securityContext;
private readonly IDaoFactory _daoFactory;
private readonly FileSecurity _fileSecurity;
private readonly PathProvider _pathProvider;
private readonly SetupInfo _setupInfo;
private readonly FileUtility _fileUtility;
private readonly DocumentServiceHelper _documentServiceHelper;
private readonly DocumentServiceConnector _documentServiceConnector;
private readonly EntryStatusManager _entryManager;
private readonly FileConverter _fileConverter;
public FileConverterQueueScope(
ILogger<FileConverterQueueScope> options,
TenantManager tenantManager,
UserManager userManager,
SecurityContext securityContext,
IDaoFactory daoFactory,
FileSecurity fileSecurity,
PathProvider pathProvider,
SetupInfo setupInfo,
FileUtility fileUtility,
DocumentServiceHelper documentServiceHelper,
DocumentServiceConnector documentServiceConnector,
EntryStatusManager entryManager,
FileConverter fileConverter)
{
_options = options;
_tenantManager = tenantManager;
_userManager = userManager;
_securityContext = securityContext;
_daoFactory = daoFactory;
_fileSecurity = fileSecurity;
_pathProvider = pathProvider;
_setupInfo = setupInfo;
_fileUtility = fileUtility;
_documentServiceHelper = documentServiceHelper;
_documentServiceConnector = documentServiceConnector;
_entryManager = entryManager;
_fileConverter = fileConverter;
}
public void Deconstruct(out ILogger optionsMonitor,
out TenantManager tenantManager,
out UserManager userManager,
out SecurityContext securityContext,
out IDaoFactory daoFactory,
out FileSecurity fileSecurity,
out PathProvider pathProvider,
out SetupInfo setupInfo,
out FileUtility fileUtility,
out DocumentServiceHelper documentServiceHelper,
out DocumentServiceConnector documentServiceConnector,
out EntryStatusManager entryManager,
out FileConverter fileConverter)
{
optionsMonitor = _options;
tenantManager = _tenantManager;
userManager = _userManager;
securityContext = _securityContext;
daoFactory = _daoFactory;
fileSecurity = _fileSecurity;
pathProvider = _pathProvider;
setupInfo = _setupInfo;
fileUtility = _fileUtility;
documentServiceHelper = _documentServiceHelper;
documentServiceConnector = _documentServiceConnector;
entryManager = _entryManager;
fileConverter = _fileConverter;
}
}
public class FileJsonSerializerData<T>
{
public T Id { get; set; }
@ -710,9 +466,9 @@ public class FileConverter
var uriTuple = await _documentServiceConnector.GetConvertedUriAsync(fileUri, fileExtension, toExtension, docKey, null, null, null, false);
var convertUri = uriTuple.ConvertedDocumentUri;
var operationResult = new ConvertFileOperationResult
var operationResult = new FileConverterOperationResult
{
Source = string.Format("{{\"id\":\"{0}\", \"version\":\"{1}\"}}", file.Id, file.Version),
Source = System.Text.Json.JsonSerializer.Serialize(new { id = file.Id, version = file.Version }),
OperationType = FileOperationType.Convert,
Error = string.Empty,
Progress = 0,
@ -720,7 +476,7 @@ public class FileConverter
Processed = "",
Id = string.Empty,
TenantId = _tenantManager.GetCurrentTenant().Id,
Account = _authContext.CurrentAccount,
Account = _authContext.CurrentAccount.ID,
Delete = false,
StartDateTime = DateTime.Now,
Url = _httpContextAccesor?.HttpContext != null ? _httpContextAccesor.HttpContext.Request.GetUrlRewriter().ToString() : null,
@ -918,27 +674,6 @@ internal class FileComparer<T> : IEqualityComparer<File<T>>
}
}
internal class ConvertFileOperationResult : FileOperationResult
{
public DateTime StartDateTime { get; set; }
public DateTime StopDateTime { get; set; }
public int TenantId { get; set; }
public IAccount Account { get; set; }
public bool Delete { get; set; }
public string Url { get; set; }
public string Password { get; set; }
//hack for download
public string ServerRootPath { get; set; }
}
public static class FileConverterQueueExtension
{
public static void Register(DIHelper services)
{
services.TryAdd<FileConverterQueueScope>();
}
}
public static class FileConverterExtension
{

56
products/ASC.Files/Core/Utils/FileConverterOperationResult.cs Normal file
View File

@ -0,0 +1,56 @@
// (c) Copyright Ascensio System SIA 2010-2022
//
// This program is a free software product.
// You can redistribute it and/or modify it under the terms
// of the GNU Affero General Public License (AGPL) version 3 as published by the Free Software
// Foundation. In accordance with Section 7(a) of the GNU AGPL its Section 15 shall be amended
// to the effect that Ascensio System SIA expressly excludes the warranty of non-infringement of
// any third-party rights.
//
// This program is distributed WITHOUT ANY WARRANTY, without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For details, see
// the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
//
// You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia, EU, LV-1021.
//
// The interactive user interfaces in modified source and object code versions of the Program must
// display Appropriate Legal Notices, as required under Section 5 of the GNU AGPL version 3.
//
// Pursuant to Section 7(b) of the License you must retain the original Product logo when
// distributing the program. Pursuant to Section 7(e) we decline to grant you any rights under
// trademark law for use of our trademarks.
//
// All the Product's GUI elements, including illustrations and icon sets, as well as technical writing
// content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0
// International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
namespace ASC.Web.Files.Utils;
[ProtoContract]
public class FileConverterOperationResult : FileOperationResult
{
[ProtoMember(1)]
public DateTime StartDateTime { get; set; }
[ProtoMember(2)]
public DateTime StopDateTime { get; set; }
[ProtoMember(3)]
public int TenantId { get; set; }
[ProtoMember(4)]
public Guid Account { get; set; }
[ProtoMember(5)]
public bool Delete { get; set; }
[ProtoMember(6)]
public string Url { get; set; }
[ProtoMember(7)]
public string Password { get; set; }
[ProtoMember(8)]
//hack for download
public string ServerRootPath { get; set; }
}

362
products/ASC.Files/Service/FileConverterService.cs Normal file
View File

@ -0,0 +1,362 @@
// (c) Copyright Ascensio System SIA 2010-2022
//
// This program is a free software product.
// You can redistribute it and/or modify it under the terms
// of the GNU Affero General Public License (AGPL) version 3 as published by the Free Software
// Foundation. In accordance with Section 7(a) of the GNU AGPL its Section 15 shall be amended
// to the effect that Ascensio System SIA expressly excludes the warranty of non-infringement of
// any third-party rights.
//
// This program is distributed WITHOUT ANY WARRANTY, without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For details, see
// the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
//
// You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia, EU, LV-1021.
//
// The interactive user interfaces in modified source and object code versions of the Program must
// display Appropriate Legal Notices, as required under Section 5 of the GNU AGPL version 3.
//
// Pursuant to Section 7(b) of the License you must retain the original Product logo when
// distributing the program. Pursuant to Section 7(e) we decline to grant you any rights under
// trademark law for use of our trademarks.
//
// All the Product's GUI elements, including illustrations and icon sets, as well as technical writing
// content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0
// International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
using System.Globalization;
using System.Text.Json;
using ASC.Common.Log;
using ASC.Core.Common.Hosting;
using ASC.Core.Common.Hosting.Interfaces;
using ASC.Files.Core.Log;
using ASC.Files.Core.Resources;
using ASC.Web.Files.Utils;
using ASC.Web.Studio.Core;
using static ASC.Web.Core.Files.DocumentService;
namespace ASC.Files.ThumbnailBuilder;
[Singletone(Additional = typeof(FileConverterQueueExtension))]
internal class FileConverterService<T> : BackgroundService
{
private readonly IServiceScopeFactory _serviceScopeFactory;
private readonly int _timerDelay = 1000;
private readonly ILogger<FileConverterService<T>> _logger;
public FileConverterService(
IServiceScopeFactory serviceScopeFactory,
ILogger<FileConverterService<T>> logger)
{
_logger = logger;
_serviceScopeFactory = serviceScopeFactory;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.DebugFileConverterServiceRuning();
stoppingToken.Register(() => _logger.DebugFileConverterServiceStopping());
while (!stoppingToken.IsCancellationRequested)
{
using var serviceScope = _serviceScopeFactory.CreateScope();
var registerInstanceService = serviceScope.ServiceProvider.GetService<IRegisterInstanceManager<FileConverterService<T>>>();
if (!await registerInstanceService.IsActive(RegisterInstanceWorkerService<FileConverterService<T>>.InstanceId))
{
await Task.Delay(1000, stoppingToken);
continue;
}
await ExecuteCheckFileConverterStatus(serviceScope);
await Task.Delay(_timerDelay, stoppingToken);
}
}
private async Task ExecuteCheckFileConverterStatus(IServiceScope scope)
{
TenantManager tenantManager;
UserManager userManager;
SecurityContext securityContext;
IDaoFactory daoFactory;
FileSecurity fileSecurity;
PathProvider pathProvider;
SetupInfo setupInfo;
FileUtility fileUtility;
DocumentServiceHelper documentServiceHelper;
DocumentServiceConnector documentServiceConnector;
EntryStatusManager entryManager;
FileConverter fileConverter;
FileConverterQueue<T> fileConverterQueue;
var logger = scope.ServiceProvider.GetService<ILogger<FileConverterQueue<T>>>();
try
{
var scopeClass = scope.ServiceProvider.GetService<FileConverterQueueScope>();
(_, tenantManager, userManager, securityContext, daoFactory, fileSecurity, pathProvider, setupInfo, fileUtility, documentServiceHelper, documentServiceConnector, entryManager, fileConverter) = scopeClass;
fileConverterQueue = scope.ServiceProvider.GetService<FileConverterQueue<T>>();
var _conversionQueue = fileConverterQueue.GetAllTask().ToList();
logger.DebugRunCheckConvertFilesStatus(_conversionQueue.Count);
var filesIsConverting = _conversionQueue
.Where(x => string.IsNullOrEmpty(x.Processed))
.ToList();
foreach (var converter in filesIsConverting)
{
var fileId = JsonDocument.Parse(converter.Source).RootElement.GetProperty("id").Deserialize<T>();
var fileVersion = JsonDocument.Parse(converter.Source).RootElement.GetProperty("version").Deserialize<int>();
int operationResultProgress;
var password = converter.Password;
var commonLinkUtilitySettings = scope.ServiceProvider.GetService<CommonLinkUtilitySettings>();
commonLinkUtilitySettings.ServerUri = converter.ServerRootPath;
tenantManager.SetCurrentTenant(converter.TenantId);
securityContext.AuthenticateMeWithoutCookie(converter.Account);
var file = await daoFactory.GetFileDao<T>().GetFileAsync(fileId, fileVersion);
var fileUri = file.Id.ToString();
string convertedFileUrl;
try
{
var user = userManager.GetUsers(converter.Account);
var culture = string.IsNullOrEmpty(user.CultureName) ? tenantManager.GetCurrentTenant().GetCulture() : CultureInfo.GetCultureInfo(user.CultureName);
Thread.CurrentThread.CurrentCulture = culture;
Thread.CurrentThread.CurrentUICulture = culture;
if (!fileSecurity.CanReadAsync(file).Result && file.RootFolderType != FolderType.BUNCH)
{
//No rights in CRM after upload before attach
throw new System.Security.SecurityException(FilesCommonResource.ErrorMassage_SecurityException_ReadFile);
}
if (file.ContentLength > setupInfo.AvailableFileSize)
{
throw new Exception(string.Format(FilesCommonResource.ErrorMassage_FileSizeConvert, FileSizeComment.FilesSizeToString(setupInfo.AvailableFileSize)));
}
fileUri = pathProvider.GetFileStreamUrl(file);
var toExtension = fileUtility.GetInternalExtension(file.Title);
var fileExtension = file.ConvertedExtension;
var docKey = documentServiceHelper.GetDocKey(file);
fileUri = documentServiceConnector.ReplaceCommunityAdress(fileUri);
(operationResultProgress, convertedFileUrl) = documentServiceConnector.GetConvertedUriAsync(fileUri, fileExtension, toExtension, docKey, password, null, null, true).Result;
}
catch (Exception exception)
{
var password1 = exception.InnerException is DocumentServiceException documentServiceException
&& documentServiceException.Code == DocumentServiceException.ErrorCode.ConvertPassword;
logger.ErrorConvertFileWithUrl(file.Id.ToString(), fileUri, exception);
var operationResult = converter;
if (operationResult.Delete)
{
_conversionQueue.Remove(operationResult);
}
else
{
operationResult.Progress = 100;
operationResult.StopDateTime = DateTime.UtcNow;
operationResult.Error = exception.Message;
if (password1)
{
operationResult.Result = "password";
}
}
continue;
}
operationResultProgress = Math.Min(operationResultProgress, 100);
if (operationResultProgress < 100)
{
var operationResult = converter;
if (DateTime.Now - operationResult.StartDateTime > TimeSpan.FromMinutes(10))
{
operationResult.StopDateTime = DateTime.UtcNow;
operationResult.Error = FilesCommonResource.ErrorMassage_ConvertTimeout;
logger.ErrorCheckConvertFilesStatus(file.Id.ToString(), file.ContentLength);
}
else
{
operationResult.Processed = "";
}
operationResult.Progress = operationResultProgress;
logger.DebugCheckConvertFilesStatusIterationContinue();
continue;
}
File<T> newFile = null;
var operationResultError = string.Empty;
try
{
newFile = fileConverter.SaveConvertedFileAsync(file, convertedFileUrl).Result;
}
catch (Exception e)
{
operationResultError = e.Message;
logger.ErrorOperation(operationResultError, convertedFileUrl, fileUri, e);
continue;
}
finally
{
var operationResult = converter;
if (operationResult.Delete)
{
_conversionQueue.Remove(operationResult);
}
else
{
if (newFile != null)
{
var folderDao = daoFactory.GetFolderDao<T>();
var folder = folderDao.GetFolderAsync(newFile.ParentId).Result;
var folderTitle = fileSecurity.CanReadAsync(folder).Result ? folder.Title : null;
operationResult.Result = fileConverterQueue.FileJsonSerializerAsync(entryManager, newFile, folderTitle).Result;
}
operationResult.Progress = 100;
operationResult.StopDateTime = DateTime.UtcNow;
operationResult.Processed = "1";
if (!string.IsNullOrEmpty(operationResultError))
{
operationResult.Error = operationResultError;
}
}
}
logger.DebugCheckConvertFilesStatusIterationEnd();
}
fileConverterQueue.SetAllTask(_conversionQueue);
}
catch (Exception exception)
{
logger.ErrorWithException(exception);
}
}
}
public static class FileConverterQueueExtension
{
public static void Register(DIHelper services)
{
services.TryAdd<FileConverterQueueScope>();
}
}
[Scope]
public class FileConverterQueueScope
{
private readonly ILogger _options;
private readonly TenantManager _tenantManager;
private readonly UserManager _userManager;
private readonly SecurityContext _securityContext;
private readonly IDaoFactory _daoFactory;
private readonly FileSecurity _fileSecurity;
private readonly PathProvider _pathProvider;
private readonly SetupInfo _setupInfo;
private readonly FileUtility _fileUtility;
private readonly DocumentServiceHelper _documentServiceHelper;
private readonly DocumentServiceConnector _documentServiceConnector;
private readonly EntryStatusManager _entryManager;
private readonly FileConverter _fileConverter;
public FileConverterQueueScope(
ILogger<FileConverterQueueScope> options,
TenantManager tenantManager,
UserManager userManager,
SecurityContext securityContext,
IDaoFactory daoFactory,
FileSecurity fileSecurity,
PathProvider pathProvider,
SetupInfo setupInfo,
FileUtility fileUtility,
DocumentServiceHelper documentServiceHelper,
DocumentServiceConnector documentServiceConnector,
EntryStatusManager entryManager,
FileConverter fileConverter)
{
_options = options;
_tenantManager = tenantManager;
_userManager = userManager;
_securityContext = securityContext;
_daoFactory = daoFactory;
_fileSecurity = fileSecurity;
_pathProvider = pathProvider;
_setupInfo = setupInfo;
_fileUtility = fileUtility;
_documentServiceHelper = documentServiceHelper;
_documentServiceConnector = documentServiceConnector;
_entryManager = entryManager;
_fileConverter = fileConverter;
}
public void Deconstruct(out ILogger optionsMonitor,
out TenantManager tenantManager,
out UserManager userManager,
out SecurityContext securityContext,
out IDaoFactory daoFactory,
out FileSecurity fileSecurity,
out PathProvider pathProvider,
out SetupInfo setupInfo,
out FileUtility fileUtility,
out DocumentServiceHelper documentServiceHelper,
out DocumentServiceConnector documentServiceConnector,
out EntryStatusManager entryManager,
out FileConverter fileConverter)
{
optionsMonitor = _options;
tenantManager = _tenantManager;
userManager = _userManager;
securityContext = _securityContext;
daoFactory = _daoFactory;
fileSecurity = _fileSecurity;
pathProvider = _pathProvider;
setupInfo = _setupInfo;
fileUtility = _fileUtility;
documentServiceHelper = _documentServiceHelper;
documentServiceConnector = _documentServiceConnector;
entryManager = _entryManager;
fileConverter = _fileConverter;
}
}

9
products/ASC.Files/Core/Log/FileConverterLogger.cs → products/ASC.Files/Service/Log/FileConverterLogger.cs
View File

@ -25,7 +25,7 @@
// International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
namespace ASC.Files.Core.Log;
internal static partial class FileConverterLogger
public static partial class FileConverterLogger
{
[LoggerMessage(Level = LogLevel.Debug, Message = "Run CheckConvertFilesStatus: count {count}")]
public static partial void DebugRunCheckConvertFilesStatus(this ILogger logger, int count);
@ -44,4 +44,11 @@ internal static partial class FileConverterLogger
[LoggerMessage(Level = LogLevel.Error, Message = "CheckConvertFilesStatus timeout: {fileId} ({contentLengthString})")]
public static partial void ErrorCheckConvertFilesStatus(this ILogger logger, string fileId, long contentLengthString);
[LoggerMessage(Level = LogLevel.Debug, Message = "FileConverterService is starting.")]
public static partial void DebugFileConverterServiceRuning(this ILogger logger);
[LoggerMessage(Level = LogLevel.Debug, Message = "FileConverterService is stopping")]
public static partial void DebugFileConverterServiceStopping(this ILogger logger);
}

7
products/ASC.Files/Service/Program.cs
View File

@ -68,6 +68,13 @@ builder.Host.ConfigureDefault(args, (hostContext, config, env, path) =>
diHelper.TryAdd<FileDataQueue>();
services.AddActivePassiveHostedService<FileConverterService<string>>();
diHelper.TryAdd<FileConverterService<string>>();
services.AddActivePassiveHostedService<FileConverterService<int>>();
diHelper.TryAdd<FileConverterService<int>>();
services.AddHostedService<ThumbnailBuilderService>();
diHelper.TryAdd<ThumbnailBuilderService>();