From 5650bc2a932109c1c7e07296c6992903310ec8b8 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Mon, 13 Jul 2020 18:26:21 +0300 Subject: [PATCH 001/278] DistributedTaskQueue: configuring --- .../Threading/DistributedTaskQueue.cs | 88 ++++++++++++------- .../DocbuilderReportsUtility.cs | 4 +- .../FileOperations/FileOperationsManager.cs | 16 +++- 3 files changed, 70 insertions(+), 38 deletions(-) diff --git a/common/ASC.Common/Threading/DistributedTaskQueue.cs b/common/ASC.Common/Threading/DistributedTaskQueue.cs index 462392002a..01fbd59c38 100644 --- a/common/ASC.Common/Threading/DistributedTaskQueue.cs +++ b/common/ASC.Common/Threading/DistributedTaskQueue.cs @@ -32,8 +32,10 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using ASC.Common.Caching; - +using ASC.Common.Caching; + +using Microsoft.Extensions.Options; + namespace ASC.Common.Threading { public class DistributedTaskCacheNotify @@ -87,17 +89,57 @@ namespace ASC.Common.Threading notifyCache.Publish(new DistributedTaskCache() { Id = id, Key = key }, CacheNotifyAction.Remove); } } + + public class DistributedTaskQueueOptionsManager : OptionsManager + { + public DistributedTaskQueueOptionsManager(IOptionsFactory factory) : base(factory) + { + } + } + + public class ConfigureDistributedTaskQueue : IConfigureNamedOptions + { + public DistributedTaskCacheNotify DistributedTaskCacheNotify { get; } + + public ConfigureDistributedTaskQueue(DistributedTaskCacheNotify distributedTaskCacheNotify) + { + DistributedTaskCacheNotify = distributedTaskCacheNotify; + } + + public void Configure(DistributedTaskQueue queue) + { + queue.DistributedTaskCacheNotify = DistributedTaskCacheNotify; + } + + public void Configure(string name, DistributedTaskQueue options) + { + Configure(options); + options.Name = name; + } + } public class DistributedTaskQueue { public static readonly string InstanceId; - private readonly string key; - private readonly ICache cache; - private readonly TaskScheduler scheduler; - private readonly ConcurrentDictionary cancelations; + private string key; + private TaskScheduler Scheduler { get; set; } = TaskScheduler.Default; - public DistributedTaskCacheNotify DistributedTaskCacheNotify { get; } + public string Name { set { key = value + GetType().Name; } } + private ICache Cache { get => DistributedTaskCacheNotify.Cache; } + private ConcurrentDictionary Cancelations { get => DistributedTaskCacheNotify.Cancelations; } + + public int MaxThreadsCount + { + set + { + Scheduler = value <= 0 + ? TaskScheduler.Default + : new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, 4).ConcurrentScheduler; + } + } + + public DistributedTaskCacheNotify DistributedTaskCacheNotify { get; set; } static DistributedTaskQueue() { @@ -105,28 +147,6 @@ namespace ASC.Common.Threading } - /// - /// Constructor - /// - /// Name of queue - /// limit of threads count; Default: -1 - no limit - public DistributedTaskQueue(DistributedTaskCacheNotify distributedTaskCacheNotify, string name, int maxThreadsCount = -1) - { - if (string.IsNullOrEmpty(name)) - { - throw new ArgumentNullException(nameof(name)); - } - - key = name + GetType().Name; - scheduler = maxThreadsCount <= 0 - ? TaskScheduler.Default - : new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, 4).ConcurrentScheduler; - DistributedTaskCacheNotify = distributedTaskCacheNotify; - cancelations = DistributedTaskCacheNotify.Cancelations; - cache = DistributedTaskCacheNotify.Cache; - } - - public void QueueTask(Action action, DistributedTask distributedTask = null) { if (distributedTask == null) @@ -138,7 +158,7 @@ namespace ASC.Common.Threading var cancelation = new CancellationTokenSource(); var token = cancelation.Token; - cancelations[distributedTask.Id] = cancelation; + Cancelations[distributedTask.Id] = cancelation; var task = new Task(() => action(distributedTask, token), token, TaskCreationOptions.LongRunning); task @@ -154,12 +174,12 @@ namespace ASC.Common.Threading } distributedTask.PublishChanges(); - task.Start(scheduler); + task.Start(Scheduler); } public IEnumerable GetTasks() { - var tasks = cache.HashGetAll(key).Values.Select(r => new DistributedTask(r)).ToList(); + var tasks = Cache.HashGetAll(key).Values.Select(r => new DistributedTask(r)).ToList(); tasks.ForEach(t => { if (t.Publication == null) @@ -172,7 +192,7 @@ namespace ASC.Common.Threading public DistributedTask GetTask(string id) { - var task = new DistributedTask(cache.HashGet(key, id)); + var task = new DistributedTask(Cache.HashGet(key, id)); if (task != null && task.Publication == null) { task.Publication = GetPublication(); @@ -211,7 +231,7 @@ namespace ASC.Common.Threading distributedTask.Status = DistributedTaskStatus.Canceled; } - cancelations.TryRemove(id, out _); + Cancelations.TryRemove(id, out _); distributedTask.PublishChanges(); } diff --git a/products/ASC.Files/Server/Services/DocumentService/DocbuilderReportsUtility.cs b/products/ASC.Files/Server/Services/DocumentService/DocbuilderReportsUtility.cs index f6e56021c6..238817d850 100644 --- a/products/ASC.Files/Server/Services/DocumentService/DocbuilderReportsUtility.cs +++ b/products/ASC.Files/Server/Services/DocumentService/DocbuilderReportsUtility.cs @@ -267,9 +267,9 @@ namespace ASC.Web.Files.Services.DocumentService } } - public DocbuilderReportsUtility(DistributedTaskCacheNotify distributedTaskCacheNotify) + public DocbuilderReportsUtility(DistributedTaskQueueOptionsManager distributedTaskQueueOptionsManager) { - tasks = new DistributedTaskQueue(distributedTaskCacheNotify, "DocbuilderReportsUtility", 10); + tasks = distributedTaskQueueOptionsManager.Get("DocbuilderReportsUtility"); Locker = new object(); } diff --git a/products/ASC.Files/Server/Services/WCFService/FileOperations/FileOperationsManager.cs b/products/ASC.Files/Server/Services/WCFService/FileOperations/FileOperationsManager.cs index 1c51e66333..36bc6456e0 100644 --- a/products/ASC.Files/Server/Services/WCFService/FileOperations/FileOperationsManager.cs +++ b/products/ASC.Files/Server/Services/WCFService/FileOperations/FileOperationsManager.cs @@ -35,6 +35,7 @@ using ASC.Common.Threading; using ASC.Core; using ASC.Files.Resources; +using Microsoft.Extensions.Options; using Microsoft.Extensions.Primitives; namespace ASC.Web.Files.Services.WCFService.FileOperations @@ -45,9 +46,9 @@ namespace ASC.Web.Files.Services.WCFService.FileOperations public IServiceProvider ServiceProvider { get; } - public FileOperationsManager(DistributedTaskCacheNotify distributedTaskCacheNotify, IServiceProvider serviceProvider) + public FileOperationsManager(DistributedTaskQueueOptionsManager distributedTaskQueueOptionsManager, IServiceProvider serviceProvider) { - tasks = new DistributedTaskQueue(distributedTaskCacheNotify, "fileOperations", 10); + tasks = distributedTaskQueueOptionsManager.Get("fileOperations"); ServiceProvider = serviceProvider; } @@ -238,6 +239,17 @@ namespace ASC.Web.Files.Services.WCFService.FileOperations services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddScoped(); + + services + .TryAddSingleton() + .TryAddSingleton() + .AddSingleton, ConfigureDistributedTaskQueue>(); + + _ = services.Configure("fileOperations", r => + { + r.MaxThreadsCount = 10; + //r.errorCount = 1; + }); return services .AddAuthContextService() From df5925ffc1ca19328e3c0badaf25ebcd41b11b54 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Thu, 6 Aug 2020 13:53:34 +0300 Subject: [PATCH 002/278] Files: FileMarker. Replace WorkerQueue with DistributedTaskQueue --- common/ASC.Common/DIHelper.cs | 37 ------------ .../Threading/DistributedTaskQueue.cs | 22 ++++++- .../Threading/Workers/WorkerQueue.cs | 35 ++++++++++- .../FileOperations/FileOperationsManager.cs | 28 +++------ products/ASC.Files/Core/Utils/FileMarker.cs | 60 ++++++++----------- 5 files changed, 87 insertions(+), 95 deletions(-) diff --git a/common/ASC.Common/DIHelper.cs b/common/ASC.Common/DIHelper.cs index a97e5bd73b..fd1b1fc0de 100644 --- a/common/ASC.Common/DIHelper.cs +++ b/common/ASC.Common/DIHelper.cs @@ -1,9 +1,6 @@ using System; using System.Collections.Generic; -using ASC.Common.Threading.Progress; -using ASC.Common.Threading.Workers; - using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; @@ -159,40 +156,6 @@ namespace ASC.Common return this; } - private void AddToConfigured(string type, Action action) where TOptions : class - { - if (!Configured.Contains(type)) - { - Configured.Add(type); - ServiceCollection.Configure(action); - } - } - - public DIHelper AddWorkerQueue(int workerCount, int waitInterval, bool stopAfterFinsih, int errorCount) - { - Action> action = (a) => - { - a.workerCount = workerCount; - a.waitInterval = waitInterval; - a.stopAfterFinsih = stopAfterFinsih; - a.errorCount = errorCount; - }; - AddToConfigured($"{typeof(WorkerQueue)}", action); - return this; - } - public DIHelper AddProgressQueue(int workerCount, int waitInterval, bool removeAfterCompleted, bool stopAfterFinsih, int errorCount) where T1 : class, IProgressItem - { - Action> action = (a) => - { - a.workerCount = workerCount; - a.waitInterval = waitInterval; - a.stopAfterFinsih = stopAfterFinsih; - a.errorCount = errorCount; - a.removeAfterCompleted = removeAfterCompleted; - }; - AddToConfigured($"{typeof(ProgressQueue)}", action); - return this; - } public DIHelper Configure(string name, Action configureOptions) where TOptions : class { var serviceName = $"{typeof(TOptions)}{name}"; diff --git a/common/ASC.Common/Threading/DistributedTaskQueue.cs b/common/ASC.Common/Threading/DistributedTaskQueue.cs index 01fbd59c38..c58a199f6e 100644 --- a/common/ASC.Common/Threading/DistributedTaskQueue.cs +++ b/common/ASC.Common/Threading/DistributedTaskQueue.cs @@ -248,5 +248,25 @@ namespace ASC.Common.Threading DistributedTaskCacheNotify.SetTask(t); }; } - } + } + + public static class DistributedTaskQueueExtention + { + public static DIHelper AddDistributedTaskQueueService(this DIHelper services, string name, int maxThreadsCount) + { + services.TryAddSingleton(); + services + .TryAddSingleton() + .TryAddSingleton() + .AddSingleton, ConfigureDistributedTaskQueue>(); + + _ = services.Configure(name, r => + { + r.MaxThreadsCount = maxThreadsCount; + //r.errorCount = 1; + }); + + return services; + } + } } diff --git a/common/ASC.Common/Threading/Workers/WorkerQueue.cs b/common/ASC.Common/Threading/Workers/WorkerQueue.cs index 3621a5ac1b..368ea85822 100644 --- a/common/ASC.Common/Threading/Workers/WorkerQueue.cs +++ b/common/ASC.Common/Threading/Workers/WorkerQueue.cs @@ -27,8 +27,11 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Threading; +using System.Threading; + using ASC.Common.Logging; +using ASC.Common.Threading.Progress; + using Microsoft.Extensions.Options; namespace ASC.Common.Threading.Workers @@ -345,5 +348,35 @@ namespace ASC.Common.Threading.Workers log.Error(err); } } + } + + public static class WorkerQueueExtention + { + public static DIHelper AddWorkerQueue(this DIHelper services, int workerCount, int waitInterval, bool stopAfterFinsih, int errorCount) + { + void action(WorkerQueue a) + { + a.workerCount = workerCount; + a.waitInterval = waitInterval; + a.stopAfterFinsih = stopAfterFinsih; + a.errorCount = errorCount; + } + + return services.Configure>(action); + } + + public static DIHelper AddProgressQueue(this DIHelper services, int workerCount, int waitInterval, bool removeAfterCompleted, bool stopAfterFinsih, int errorCount) where T1 : class, IProgressItem + { + void action(ProgressQueue a) + { + a.workerCount = workerCount; + a.waitInterval = waitInterval; + a.stopAfterFinsih = stopAfterFinsih; + a.errorCount = errorCount; + a.removeAfterCompleted = removeAfterCompleted; + } + + return services.Configure>(action); + } } } \ No newline at end of file diff --git a/products/ASC.Files/Core/Services/WCFService/FileOperations/FileOperationsManager.cs b/products/ASC.Files/Core/Services/WCFService/FileOperations/FileOperationsManager.cs index 068409f2a6..f0281a33d9 100644 --- a/products/ASC.Files/Core/Services/WCFService/FileOperations/FileOperationsManager.cs +++ b/products/ASC.Files/Core/Services/WCFService/FileOperations/FileOperationsManager.cs @@ -33,9 +33,8 @@ using System.Text.Json; using ASC.Common; using ASC.Common.Threading; using ASC.Core; -using ASC.Files.Core.Resources; +using ASC.Files.Core.Resources; -using Microsoft.Extensions.Options; using Microsoft.Extensions.Primitives; namespace ASC.Web.Files.Services.WCFService.FileOperations @@ -238,26 +237,15 @@ namespace ASC.Web.Files.Services.WCFService.FileOperations { if (services.TryAddScoped()) { - services.TryAddSingleton(); services.TryAddSingleton(); + services.TryAddSingleton(); + services.AddDistributedTaskQueueService("fileOperations", 10); - services - .TryAddSingleton() - .TryAddSingleton() - .AddSingleton, ConfigureDistributedTaskQueue>(); - - _ = services.Configure("fileOperations", r => - { - r.MaxThreadsCount = 10; - //r.errorCount = 1; - }); - - return services - .AddAuthContextService() - .AddTenantManagerService() - ; - } - return services; + return services + .AddAuthContextService() + .AddTenantManagerService(); + } + return services; } } } \ No newline at end of file diff --git a/products/ASC.Files/Core/Utils/FileMarker.cs b/products/ASC.Files/Core/Utils/FileMarker.cs index e01b7686a0..f75ceb4881 100644 --- a/products/ASC.Files/Core/Utils/FileMarker.cs +++ b/products/ASC.Files/Core/Utils/FileMarker.cs @@ -32,7 +32,7 @@ using System.Security; using ASC.Common; using ASC.Common.Caching; using ASC.Common.Logging; -using ASC.Common.Threading.Workers; +using ASC.Common.Threading; using ASC.Core; using ASC.Core.Users; using ASC.Files.Core; @@ -51,33 +51,21 @@ namespace ASC.Web.Files.Utils { private IServiceProvider ServiceProvider { get; } public ILog Log { get; } - - public object Locker { get; set; } - public WorkerQueue> Tasks { get; set; } + public DistributedTaskQueue Tasks { get; set; } public FileMarkerHelper( IServiceProvider serviceProvider, - IOptionsMonitor optionsMonitor, - WorkerQueueOptionsManager> workerQueueOptionsManager) + IOptionsMonitor optionsMonitor, + DistributedTaskQueueOptionsManager distributedTaskQueueOptionsManager) { ServiceProvider = serviceProvider; Log = optionsMonitor.CurrentValue; - Locker = new object(); - Tasks = workerQueueOptionsManager.Value; + Tasks = distributedTaskQueueOptionsManager.Get("FileMarkerHelper"); } internal void Add(AsyncTaskData taskData) { - - lock (Locker) - { - Tasks.Add(taskData); - - if (!Tasks.IsStarted) - { - Tasks.Start(ExecMarkFileAsNew); - } - } + Tasks.QueueTask((d, c) => ExecMarkFileAsNew(taskData), taskData); } private void ExecMarkFileAsNew(AsyncTaskData obj) @@ -694,9 +682,9 @@ namespace ASC.Web.Files.Utils } } - public class AsyncTaskData + public class AsyncTaskData : DistributedTask { - public AsyncTaskData(TenantManager tenantManager, AuthContext authContext) + public AsyncTaskData(TenantManager tenantManager, AuthContext authContext) : base() { TenantID = tenantManager.GetCurrentTenant().TenantId; CurrentAccountId = authContext.CurrentAccount.ID; @@ -722,24 +710,24 @@ namespace ASC.Web.Files.Utils } public static DIHelper AddFileMarkerService(this DIHelper services) - { + { services.TryAddTransient>(); - services.TryAddScoped(); services.TryAddSingleton>(); - services.TryAddSingleton>>(); - services.TryAddSingleton>>(); - services.AddSingleton>>, ConfigureWorkerQueue>>(); - - _ = services.AddWorkerQueue>(1, (int)TimeSpan.FromSeconds(60).TotalMilliseconds, false, 1); - - return services - .AddTenantManagerService() - .AddUserManagerService() - .AddDaoFactoryService() - .AddGlobalFolderService() - .AddFileSecurityService() - .AddCoreBaseSettingsService() - .AddAuthContextService(); + + if (services.TryAddScoped()) + { + return services + .AddDistributedTaskQueueService("FileMarkerHelper", 1) + .AddTenantManagerService() + .AddUserManagerService() + .AddDaoFactoryService() + .AddGlobalFolderService() + .AddFileSecurityService() + .AddCoreBaseSettingsService() + .AddAuthContextService(); + } + + return services; } } } \ No newline at end of file From b0679912561e7560de440f0a2355b376aa8ffc54 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Thu, 6 Aug 2020 14:35:22 +0300 Subject: [PATCH 003/278] UserPhotoManager: replace WorkerQueue with DistributedTaskQueue --- web/ASC.Web.Core/Users/UserPhotoManager.cs | 53 ++++++++-------------- 1 file changed, 18 insertions(+), 35 deletions(-) diff --git a/web/ASC.Web.Core/Users/UserPhotoManager.cs b/web/ASC.Web.Core/Users/UserPhotoManager.cs index 77b498702d..646132cf66 100644 --- a/web/ASC.Web.Core/Users/UserPhotoManager.cs +++ b/web/ASC.Web.Core/Users/UserPhotoManager.cs @@ -37,7 +37,7 @@ using System.Text.RegularExpressions; using ASC.Common; using ASC.Common.Caching; using ASC.Common.Logging; -using ASC.Common.Threading.Workers; +using ASC.Common.Threading; using ASC.Core; using ASC.Core.Common.Settings; using ASC.Core.Tenants; @@ -48,9 +48,9 @@ using Microsoft.Extensions.Options; namespace ASC.Web.Core.Users { - public class ResizeWorkerItem + public class ResizeWorkerItem : DistributedTask { - public ResizeWorkerItem(Guid userId, byte[] data, long maxFileSize, Size size, IDataStore dataStore, UserPhotoThumbnailSettings settings) + public ResizeWorkerItem(Guid userId, byte[] data, long maxFileSize, Size size, IDataStore dataStore, UserPhotoThumbnailSettings settings) : base() { UserId = userId; Data = data; @@ -193,8 +193,6 @@ namespace ASC.Web.Core.Users @"(?'user'\{{0,1}([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}\}{0,1}){1}" + @"_(?'kind'orig|size){1}_(?'size'(?'width'[0-9]{1,5})-{1}(?'height'[0-9]{1,5})){0,1}\..*", RegexOptions.Compiled); - private static readonly ConcurrentDictionary> Photofiles = new ConcurrentDictionary>(); - public UserManager UserManager { get; } public WebImageSupplier WebImageSupplier { get; } public TenantManager TenantManager { get; } @@ -205,10 +203,10 @@ namespace ASC.Web.Core.Users public ILog Log { get; } private Tenant tenant; - public Tenant Tenant { get { return tenant ?? (tenant = TenantManager.GetCurrentTenant()); } } + public Tenant Tenant { get { return tenant ??= TenantManager.GetCurrentTenant(); } } //note: using auto stop queue - private readonly WorkerQueue ResizeQueue;//TODO: configure + private readonly DistributedTaskQueue ResizeQueue;//TODO: configure public UserPhotoManager( UserManager userManager, @@ -217,11 +215,11 @@ namespace ASC.Web.Core.Users StorageFactory storageFactory, UserPhotoManagerCache userPhotoManagerCache, IOptionsMonitor options, - WorkerQueueOptionsManager optionsQueue, + DistributedTaskQueueOptionsManager optionsQueue, SettingsManager settingsManager, IServiceProvider serviceProvider) { - ResizeQueue = optionsQueue.Value; + ResizeQueue = optionsQueue.Get("ResizeWorker"); UserManager = userManager; WebImageSupplier = webImageSupplier; TenantManager = tenantManager; @@ -235,7 +233,7 @@ namespace ASC.Web.Core.Users public string defaultAbsoluteWebPath; public string GetDefaultPhotoAbsoluteWebPath() { - return defaultAbsoluteWebPath ?? (defaultAbsoluteWebPath = WebImageSupplier.GetAbsoluteWebPath(_defaultAvatar)); + return defaultAbsoluteWebPath ??= WebImageSupplier.GetAbsoluteWebPath(_defaultAvatar); } public string GetRetinaPhotoURL(Guid userID) @@ -298,31 +296,31 @@ namespace ASC.Web.Core.Users public string defaultSmallPhotoURL; public string GetDefaultSmallPhotoURL() { - return defaultSmallPhotoURL ?? (defaultSmallPhotoURL = GetDefaultPhotoAbsoluteWebPath(SmallFotoSize)); + return defaultSmallPhotoURL ??= GetDefaultPhotoAbsoluteWebPath(SmallFotoSize); } public string defaultMediumPhotoURL; public string GetDefaultMediumPhotoURL() { - return defaultMediumPhotoURL ?? (defaultMediumPhotoURL = GetDefaultPhotoAbsoluteWebPath(MediumFotoSize)); + return defaultMediumPhotoURL ??= GetDefaultPhotoAbsoluteWebPath(MediumFotoSize); } public string defaultBigPhotoURL; public string GetDefaultBigPhotoURL() { - return defaultBigPhotoURL ?? (defaultBigPhotoURL = GetDefaultPhotoAbsoluteWebPath(BigFotoSize)); + return defaultBigPhotoURL ??= GetDefaultPhotoAbsoluteWebPath(BigFotoSize); } public string defaultMaxPhotoURL; public string GetDefaultMaxPhotoURL() { - return defaultMaxPhotoURL ?? (defaultMaxPhotoURL = GetDefaultPhotoAbsoluteWebPath(MaxFotoSize)); + return defaultMaxPhotoURL ??= GetDefaultPhotoAbsoluteWebPath(MaxFotoSize); } public string defaultRetinaPhotoURL; public string GetDefaultRetinaPhotoURL() { - return defaultRetinaPhotoURL ?? (defaultRetinaPhotoURL = GetDefaultPhotoAbsoluteWebPath(RetinaFotoSize)); + return defaultRetinaPhotoURL ??= GetDefaultPhotoAbsoluteWebPath(RetinaFotoSize); } @@ -666,6 +664,8 @@ namespace ASC.Web.Core.Users if (maxFileSize != -1 && data.Length > maxFileSize) throw new ImageWeightLimitException(); var resizeTask = new ResizeWorkerItem(userID, data, maxFileSize, size, GetDataStore(), SettingsManager.LoadForUser(userID)); + var key = $"{userID}{size}"; + resizeTask.SetProperty("key", key); if (now) { @@ -675,14 +675,10 @@ namespace ASC.Web.Core.Users } else { - if (!ResizeQueue.GetItems().Contains(resizeTask)) + if (!ResizeQueue.GetTasks().OfType().Any(r => r.GetProperty("key") == key)) { //Add - ResizeQueue.Add(resizeTask); - if (!ResizeQueue.IsStarted) - { - ResizeQueue.Start(ResizeImage); - } + ResizeQueue.QueueTask((a, b) => ResizeImage(resizeTask), resizeTask); } return GetDefaultPhotoAbsoluteWebPath(size); //NOTE: return default photo here. Since task will update cache @@ -1002,19 +998,6 @@ namespace ASC.Web.Core.Users (w, h) = (size.Width, size.Height); } - public static class ResizeWorkerItemExtension - { - public static DIHelper AddResizeWorkerItemService(this DIHelper services) - { - services.TryAddSingleton>(); - services.TryAddSingleton>(); - services.AddSingleton>, ConfigureWorkerQueue>(); - - services.AddWorkerQueue(2, (int)TimeSpan.FromSeconds(30).TotalMilliseconds, true, 1); - return services; - } - } - public static class UserPhotoManagerExtension { public static DIHelper AddUserPhotoManagerService(this DIHelper services) @@ -1029,7 +1012,7 @@ namespace ASC.Web.Core.Users .AddWebImageSupplierService() .AddUserManagerService() .AddTenantManagerService() - .AddResizeWorkerItemService(); + .AddDistributedTaskQueueService("ResizeWorker", 2); } return services; From ffc756f32bcce253f160680242ee62c6773e49a0 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Thu, 6 Aug 2020 21:07:27 +0300 Subject: [PATCH 004/278] Backup: DistributedTaskQueue --- .../ASC.Common/Threading/DistributedTask.cs | 2 +- .../Threading/DistributedTaskQueue.cs | 30 +++- .../Threading/Progress/IProgressItem.cs | 4 +- .../Threading/Progress/ProgressBase.cs | 16 +-- .../ASC.Data.Reassigns/RemoveProgressItem.cs | 13 +- common/ASC.Data.Storage/StorageUploader.cs | 2 +- .../ASC.Data.Backup/Service/BackupWorker.cs | 132 ++++++++++++------ common/services/ASC.Data.Backup/Startup.cs | 3 +- products/ASC.People/Server/Startup.cs | 7 +- 9 files changed, 141 insertions(+), 68 deletions(-) diff --git a/common/ASC.Common/Threading/DistributedTask.cs b/common/ASC.Common/Threading/DistributedTask.cs index fb6f5f8bcd..f18e988b0e 100644 --- a/common/ASC.Common/Threading/DistributedTask.cs +++ b/common/ASC.Common/Threading/DistributedTask.cs @@ -66,7 +66,7 @@ namespace ASC.Common.Threading { return Enum.Parse(DistributedTaskCache.Status); } - internal set + set { DistributedTaskCache.Status = value.ToString(); } diff --git a/common/ASC.Common/Threading/DistributedTaskQueue.cs b/common/ASC.Common/Threading/DistributedTaskQueue.cs index c58a199f6e..e796f8e7d5 100644 --- a/common/ASC.Common/Threading/DistributedTaskQueue.cs +++ b/common/ASC.Common/Threading/DistributedTaskQueue.cs @@ -34,6 +34,7 @@ using System.Threading.Tasks; using ASC.Common.Caching; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; namespace ASC.Common.Threading @@ -100,15 +101,18 @@ namespace ASC.Common.Threading public class ConfigureDistributedTaskQueue : IConfigureNamedOptions { public DistributedTaskCacheNotify DistributedTaskCacheNotify { get; } - - public ConfigureDistributedTaskQueue(DistributedTaskCacheNotify distributedTaskCacheNotify) + public IServiceProvider ServiceProvider { get; } + + public ConfigureDistributedTaskQueue(DistributedTaskCacheNotify distributedTaskCacheNotify, IServiceProvider serviceProvider) { DistributedTaskCacheNotify = distributedTaskCacheNotify; + ServiceProvider = serviceProvider; } public void Configure(DistributedTaskQueue queue) { - queue.DistributedTaskCacheNotify = DistributedTaskCacheNotify; + queue.DistributedTaskCacheNotify = DistributedTaskCacheNotify; + queue.ServiceProvider = ServiceProvider; } public void Configure(string name, DistributedTaskQueue options) @@ -125,6 +129,7 @@ namespace ASC.Common.Threading private string key; private TaskScheduler Scheduler { get; set; } = TaskScheduler.Default; + public IServiceProvider ServiceProvider { get; set; } public string Name { set { key = value + GetType().Name; } } private ICache Cache { get => DistributedTaskCacheNotify.Cache; } private ConcurrentDictionary Cancelations { get => DistributedTaskCacheNotify.Cancelations; } @@ -180,6 +185,25 @@ namespace ASC.Common.Threading public IEnumerable GetTasks() { var tasks = Cache.HashGetAll(key).Values.Select(r => new DistributedTask(r)).ToList(); + tasks.ForEach(t => + { + if (t.Publication == null) + { + t.Publication = GetPublication(); + } + }); + return tasks; + } + + public IEnumerable GetTasks() where T : DistributedTask + { + var tasks = Cache.HashGetAll(key).Values.Select(r => + { + var result = ServiceProvider.GetService(); + result.DistributedTaskCache = r; + return result; + }).ToList(); + tasks.ForEach(t => { if (t.Publication == null) diff --git a/common/ASC.Common/Threading/Progress/IProgressItem.cs b/common/ASC.Common/Threading/Progress/IProgressItem.cs index 70b2d42c6c..66e01f5d63 100644 --- a/common/ASC.Common/Threading/Progress/IProgressItem.cs +++ b/common/ASC.Common/Threading/Progress/IProgressItem.cs @@ -30,8 +30,8 @@ namespace ASC.Common.Threading.Progress { public interface IProgressItem : ICloneable { - object Id { get; set; } - object Status { get; set; } + string Id { get; } + DistributedTaskStatus Status { get; set; } object Error { get; set; } double Percentage { get; set; } bool IsCompleted { get; set; } diff --git a/common/ASC.Common/Threading/Progress/ProgressBase.cs b/common/ASC.Common/Threading/Progress/ProgressBase.cs index b220d98b7b..aa4b51b5f1 100644 --- a/common/ASC.Common/Threading/Progress/ProgressBase.cs +++ b/common/ASC.Common/Threading/Progress/ProgressBase.cs @@ -35,13 +35,13 @@ namespace ASC.Common.Threading.Progress protected int StepCount { get; set; } - public object Id { get; set; } - - public object Status { get; set; } - - public object Error { get; set; } - - + public string Id { get; set; } + + public DistributedTaskStatus Status { get; set; } + + public object Error { get; set; } + + public double Percentage { get { return Math.Min(100.0, Math.Max(0, _percentage)); } @@ -89,7 +89,7 @@ namespace ASC.Common.Threading.Progress protected ProgressBase() { - Id = Guid.NewGuid(); // random id + Id = Guid.NewGuid().ToString(); // random id } protected void ProgressAdd(double value) diff --git a/common/ASC.Data.Reassigns/RemoveProgressItem.cs b/common/ASC.Data.Reassigns/RemoveProgressItem.cs index 1bf3bb19fe..419fc598d9 100644 --- a/common/ASC.Data.Reassigns/RemoveProgressItem.cs +++ b/common/ASC.Data.Reassigns/RemoveProgressItem.cs @@ -32,6 +32,7 @@ using System.Text; using ASC.Common; using ASC.Common.Logging; +using ASC.Common.Threading; using ASC.Common.Threading.Progress; using ASC.Core; using ASC.Core.Users; @@ -61,8 +62,8 @@ namespace ASC.Data.Reassigns //private readonly IFileStorageService _docService; //private readonly MailGarbageEngine _mailEraser; - public object Id { get; set; } - public object Status { get; set; } + public string Id { get; } + public DistributedTaskStatus Status { get; set; } public object Error { get; set; } public double Percentage { get; set; } public bool IsCompleted { get; set; } @@ -88,7 +89,7 @@ namespace ASC.Data.Reassigns //_mailEraser = new MailGarbageEngine(); Id = queueWorkerRemove.GetProgressItemId(tenantId, FromUser); - Status = ProgressStatus.Queued; + Status = DistributedTaskStatus.Created; Error = null; Percentage = 0; IsCompleted = false; @@ -114,7 +115,7 @@ namespace ASC.Data.Reassigns try { Percentage = 0; - Status = ProgressStatus.Started; + Status = DistributedTaskStatus.Running; securityContext.AuthenticateMe(_currentUserId); @@ -158,12 +159,12 @@ namespace ASC.Data.Reassigns SendSuccessNotify(studioNotifyService, messageService, messageTarget, userName, docsSpace, crmSpace, mailSpace, talkSpace); Percentage = 100; - Status = ProgressStatus.Done; + Status = DistributedTaskStatus.Completed; } catch (Exception ex) { logger.Error(ex); - Status = ProgressStatus.Failed; + Status = DistributedTaskStatus.Failted; Error = ex.Message; SendErrorNotify(studioNotifyService, ex.Message, userName); } diff --git a/common/ASC.Data.Storage/StorageUploader.cs b/common/ASC.Data.Storage/StorageUploader.cs index fc875ec0f1..0f560271f1 100644 --- a/common/ASC.Data.Storage/StorageUploader.cs +++ b/common/ASC.Data.Storage/StorageUploader.cs @@ -171,7 +171,7 @@ namespace ASC.Data.Storage string[] files; foreach (var domain in domains) { - Status = module + domain; + //Status = module + domain; Log.DebugFormat("Domain: {0}", domain); files = oldStore.ListFilesRelative(domain, "\\", "*.*", true); diff --git a/common/services/ASC.Data.Backup/Service/BackupWorker.cs b/common/services/ASC.Data.Backup/Service/BackupWorker.cs index 8efe6f5ffd..8ca5a89e3c 100644 --- a/common/services/ASC.Data.Backup/Service/BackupWorker.cs +++ b/common/services/ASC.Data.Backup/Service/BackupWorker.cs @@ -33,6 +33,7 @@ using System.Threading; using ASC.Common; using ASC.Common.Caching; using ASC.Common.Logging; +using ASC.Common.Threading; using ASC.Common.Threading.Progress; using ASC.Core; using ASC.Core.Billing; @@ -55,7 +56,7 @@ namespace ASC.Data.Backup.Service public class BackupWorker { private ILog Log { get; set; } - private ProgressQueue ProgressQueue { get; set; } + private DistributedTaskQueue ProgressQueue { get; set; } internal string TempFolder { get; set; } private string CurrentRegion { get; set; } private Dictionary ConfigPaths { get; set; } @@ -64,14 +65,16 @@ namespace ASC.Data.Backup.Service private ICacheNotify CacheBackupProgress { get; } private FactoryProgressItem FactoryProgressItem { get; set; } + private readonly object SynchRoot = new object(); + public BackupWorker( IOptionsMonitor options, ICacheNotify cacheBackupProgress, - ProgressQueueOptionsManager progressQueue, + DistributedTaskQueueOptionsManager progressQueue, FactoryProgressItem factoryProgressItem) { Log = options.CurrentValue; - ProgressQueue = progressQueue.Value; + ProgressQueue = progressQueue.Get("backup"); CacheBackupProgress = cacheBackupProgress; FactoryProgressItem = factoryProgressItem; } @@ -101,25 +104,30 @@ namespace ASC.Data.Backup.Service { if (ProgressQueue != null) { - ProgressQueue.Terminate(); + var tasks = ProgressQueue.GetTasks(); + foreach (var t in tasks) + { + ProgressQueue.CancelTask(t.Id); + } + ProgressQueue = null; } } public BackupProgress StartBackup(StartBackupRequest request) { - lock (ProgressQueue.SynchRoot) + lock (SynchRoot) { - var item = ProgressQueue.GetItems().OfType().FirstOrDefault(t => t.TenantId == request.TenantId); + var item = ProgressQueue.GetTasks().FirstOrDefault(t => t.TenantId == request.TenantId); if (item != null && item.IsCompleted) { - ProgressQueue.Remove(item); + ProgressQueue.RemoveTask(item.Id); item = null; } if (item == null) { item = FactoryProgressItem.CreateBackupProgressItem(request, false, TempFolder, Limit, CurrentRegion, ConfigPaths); - ProgressQueue.Add(item); + ProgressQueue.QueueTask((a, b) => item.RunJob(), item); } var progress = ToBackupProgress(item); @@ -132,35 +140,35 @@ namespace ASC.Data.Backup.Service public void StartScheduledBackup(BackupSchedule schedule) { - lock (ProgressQueue.SynchRoot) + lock (SynchRoot) { - var item = ProgressQueue.GetItems().OfType().FirstOrDefault(t => t.TenantId == schedule.TenantId); + var item = ProgressQueue.GetTasks().FirstOrDefault(t => t.TenantId == schedule.TenantId); if (item != null && item.IsCompleted) { - ProgressQueue.Remove(item); + ProgressQueue.RemoveTask(item.Id); item = null; } if (item == null) { item = FactoryProgressItem.CreateBackupProgressItem(schedule, false, TempFolder, Limit, CurrentRegion, ConfigPaths); - ProgressQueue.Add(item); + ProgressQueue.QueueTask((a, b) => item.RunJob(), item); } } } public BackupProgress GetBackupProgress(int tenantId) { - lock (ProgressQueue.SynchRoot) + lock (SynchRoot) { - return ToBackupProgress(ProgressQueue.GetItems().OfType().FirstOrDefault(t => t.TenantId == tenantId)); + return ToBackupProgress(ProgressQueue.GetTasks().FirstOrDefault(t => t.TenantId == tenantId)); } } public void ResetBackupError(int tenantId) { - lock (ProgressQueue.SynchRoot) + lock (SynchRoot) { - var progress = ProgressQueue.GetItems().OfType().FirstOrDefault(t => t.TenantId == tenantId); + var progress = ProgressQueue.GetTasks().FirstOrDefault(t => t.TenantId == tenantId); if (progress != null) { progress.Error = null; @@ -170,9 +178,9 @@ namespace ASC.Data.Backup.Service public void ResetRestoreError(int tenantId) { - lock (ProgressQueue.SynchRoot) + lock (SynchRoot) { - var progress = ProgressQueue.GetItems().OfType().FirstOrDefault(t => t.TenantId == tenantId); + var progress = ProgressQueue.GetTasks().FirstOrDefault(t => t.TenantId == tenantId); if (progress != null) { progress.Error = null; @@ -182,18 +190,18 @@ namespace ASC.Data.Backup.Service public BackupProgress StartRestore(StartRestoreRequest request) { - lock (ProgressQueue.SynchRoot) + lock (SynchRoot) { - var item = ProgressQueue.GetItems().OfType().FirstOrDefault(t => t.TenantId == request.TenantId); + var item = ProgressQueue.GetTasks().FirstOrDefault(t => t.TenantId == request.TenantId); if (item != null && item.IsCompleted) { - ProgressQueue.Remove(item); + ProgressQueue.RemoveTask(item.Id); item = null; } if (item == null) { item = FactoryProgressItem.CreateRestoreProgressItem(request, TempFolder, UpgradesPath, CurrentRegion, ConfigPaths); - ProgressQueue.Add(item); + ProgressQueue.QueueTask((a, b) => item.RunJob(), item); } return ToBackupProgress(item); } @@ -201,19 +209,19 @@ namespace ASC.Data.Backup.Service public BackupProgress StartTransfer(int tenantId, string targetRegion, bool transferMail, bool notify) { - lock (ProgressQueue.SynchRoot) + lock (SynchRoot) { - var item = ProgressQueue.GetItems().OfType().FirstOrDefault(t => t.TenantId == tenantId); + var item = ProgressQueue.GetTasks().FirstOrDefault(t => t.TenantId == tenantId); if (item != null && item.IsCompleted) { - ProgressQueue.Remove(item); + ProgressQueue.RemoveTask(item.Id); item = null; } if (item == null) { item = FactoryProgressItem.CreateTransferProgressItem(targetRegion, transferMail, tenantId, TempFolder, Limit, notify, CurrentRegion, ConfigPaths); - ProgressQueue.Add(item); + ProgressQueue.QueueTask((a, b) => item.RunJob(), item); } return ToBackupProgress(item); @@ -253,6 +261,7 @@ namespace ASC.Data.Backup.Service internal void PublishProgress(BaseBackupProgressItem progress) { + progress.PublishChanges(); PublishProgress(ToBackupProgress(progress)); } @@ -281,19 +290,63 @@ namespace ASC.Data.Backup.Service }; } - public abstract class BaseBackupProgressItem : IProgressItem + public abstract class BaseBackupProgressItem : DistributedTask, IProgressItem { - public object Id { get; set; } + public object error; + public object Error + { + get + { + return error ?? GetProperty(nameof(error)); + } + set + { + error = value; + SetProperty(nameof(error), value); + } + } - public object Status { get; set; } + public double? percentage; + public double Percentage + { + get + { + return percentage ?? GetProperty(nameof(percentage)); + } + set + { + percentage = value; + SetProperty(nameof(percentage), value); + } + } - public object Error { get; set; } + public bool? isCompleted; + public bool IsCompleted + { + get + { + return isCompleted ?? GetProperty(nameof(isCompleted)); + } + set + { + isCompleted = value; + SetProperty(nameof(isCompleted), value); + } + } - public double Percentage { get; set; } - - public bool IsCompleted { get; set; } - - public int TenantId { get; set; } + private int? tenantId; + public int TenantId + { + get + { + return tenantId ?? GetProperty(nameof(tenantId)); + } + set + { + tenantId = value; + SetProperty(nameof(tenantId), value); + } + } public abstract BackupProgressItemEnum BackupProgressItemEnum { get; } @@ -324,11 +377,8 @@ namespace ASC.Data.Backup.Service public BackupProgressItem(IServiceProvider serviceProvider, IOptionsMonitor options) { - Id = Guid.NewGuid(); - Log = options.CurrentValue; ServiceProvider = serviceProvider; - } public void Init(BackupSchedule schedule, bool isScheduled, string tempFolder, int limit, string currentRegion, Dictionary configPaths) @@ -409,7 +459,7 @@ namespace ASC.Data.Backup.Service repo.SaveBackupRecord( new BackupRecord { - Id = (Guid)Id, + Id = Guid.Parse(Id), TenantId = TenantId, IsScheduled = IsScheduled, Name = Path.GetFileName(tempFile), @@ -491,7 +541,6 @@ namespace ASC.Data.Backup.Service } public void Init(StartRestoreRequest request, string tempFolder, string upgradesPath, string currentRegion, Dictionary configPaths) { - Id = Guid.NewGuid(); TenantId = request.TenantId; Notify = request.NotifyAfterCompletion; StoragePath = request.FilePathOrId; @@ -525,7 +574,7 @@ namespace ASC.Data.Backup.Service tenantManager.SaveTenant(tenant); var columnMapper = new ColumnMapper(); - columnMapper.SetMapping("tenants_tenants", "alias", tenant.TenantAlias, ((Guid)Id).ToString("N")); + columnMapper.SetMapping("tenants_tenants", "alias", tenant.TenantAlias, (Guid.Parse(Id)).ToString("N")); columnMapper.Commit(); var restoreTask = scope.ServiceProvider.GetService(); @@ -652,7 +701,6 @@ namespace ASC.Data.Backup.Service string currentRegion, Dictionary configPaths) { - Id = Guid.NewGuid(); TenantId = tenantId; TargetRegion = targetRegion; TransferMail = transferMail; diff --git a/common/services/ASC.Data.Backup/Startup.cs b/common/services/ASC.Data.Backup/Startup.cs index 6fc48f9887..eabad8b5b4 100644 --- a/common/services/ASC.Data.Backup/Startup.cs +++ b/common/services/ASC.Data.Backup/Startup.cs @@ -1,7 +1,8 @@ using System; using ASC.Api.Core; -using ASC.Common; +using ASC.Common; +using ASC.Common.Threading.Workers; using ASC.Data.Backup.Controllers; using ASC.Data.Backup.Service; diff --git a/products/ASC.People/Server/Startup.cs b/products/ASC.People/Server/Startup.cs index d5022b5997..477f38c050 100644 --- a/products/ASC.People/Server/Startup.cs +++ b/products/ASC.People/Server/Startup.cs @@ -2,10 +2,10 @@ using System; using ASC.Api.Core; -using ASC.Common; +using ASC.Common; +using ASC.Common.Threading.Workers; using ASC.Data.Reassigns; using ASC.Employee.Core.Controllers; -using ASC.Web.Core.Users; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -27,8 +27,7 @@ namespace ASC.People var diHelper = new DIHelper(services); diHelper.AddProgressQueue(1, (int)TimeSpan.FromMinutes(5).TotalMilliseconds, true, false, 0); - diHelper.AddProgressQueue(1, (int)TimeSpan.FromMinutes(5).TotalMilliseconds, true, false, 0); - diHelper.AddWorkerQueue(2, (int)TimeSpan.FromMinutes(30).TotalMilliseconds, true, 1); + diHelper.AddProgressQueue(1, (int)TimeSpan.FromMinutes(5).TotalMilliseconds, true, false, 0); diHelper .AddPeopleController() From 0b31269c8eeaac05a57bc887b1ac2549b1db535f Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Thu, 6 Aug 2020 22:20:37 +0300 Subject: [PATCH 005/278] Backup: removed BackupServiceNotifier --- .../ASC.Data.Backup/BackupServiceLauncher.cs | 5 +- .../ASC.Data.Backup/Service/BackupService.cs | 48 ++----------------- .../ASC.Data.Backup/Service/BackupWorker.cs | 29 +++++++---- web/ASC.Web.Core/Users/UserPhotoManager.cs | 2 +- 4 files changed, 25 insertions(+), 59 deletions(-) diff --git a/common/services/ASC.Data.Backup/BackupServiceLauncher.cs b/common/services/ASC.Data.Backup/BackupServiceLauncher.cs index bf8dd87b3d..8f363671f4 100644 --- a/common/services/ASC.Data.Backup/BackupServiceLauncher.cs +++ b/common/services/ASC.Data.Backup/BackupServiceLauncher.cs @@ -44,22 +44,19 @@ namespace ASC.Data.Backup.Service private BackupSchedulerService SchedulerService { get; set; } private BackupWorker BackupWorker { get; set; } private IConfiguration Configuration { get; set; } - public BackupServiceNotifier BackupServiceNotifier { get; } public BackupServiceLauncher( IServiceProvider serviceProvider, BackupCleanerService cleanerService, BackupSchedulerService schedulerService, BackupWorker backupWorker, - IConfiguration configuration, - BackupServiceNotifier backupServiceNotifier) + IConfiguration configuration) { ServiceProvider = serviceProvider; CleanerService = cleanerService; SchedulerService = schedulerService; BackupWorker = backupWorker; Configuration = configuration; - BackupServiceNotifier = backupServiceNotifier; } public Task StartAsync(CancellationToken cancellationToken) diff --git a/common/services/ASC.Data.Backup/Service/BackupService.cs b/common/services/ASC.Data.Backup/Service/BackupService.cs index 022041e3d0..a1fe331149 100644 --- a/common/services/ASC.Data.Backup/Service/BackupService.cs +++ b/common/services/ASC.Data.Backup/Service/BackupService.cs @@ -31,7 +31,6 @@ using System.Linq; using System.ServiceModel; using ASC.Common; -using ASC.Common.Caching; using ASC.Common.Logging; using ASC.Common.Utils; using ASC.Data.Backup.Contracts; @@ -46,63 +45,25 @@ using Newtonsoft.Json; namespace ASC.Data.Backup.Service { - public class BackupServiceNotifier - { - public ICacheNotify СacheBackupProgress { get; } - public ICache Cache { get; } - - public BackupServiceNotifier(ICacheNotify сacheBackupProgress) - { - Cache = AscCache.Memory; - СacheBackupProgress = сacheBackupProgress; - - СacheBackupProgress.Subscribe((a) => - { - Cache.Insert(GetCacheKey(a.TenantId, a.BackupProgressEnum), a, DateTime.UtcNow.AddDays(1)); - }, - CacheNotifyAction.InsertOrUpdate); - } - - public BackupProgress GetBackupProgress(int tenantId) - { - return Cache.Get(GetCacheKey(tenantId, BackupProgressEnum.Backup)); - } - - public BackupProgress GetTransferProgress(int tenantID) - { - return Cache.Get(GetCacheKey(tenantID, BackupProgressEnum.Transfer)); - } - - public BackupProgress GetRestoreProgress(int tenantId) - { - return Cache.Get(GetCacheKey(tenantId, BackupProgressEnum.Restore)); - } - - private string GetCacheKey(int tenantId, BackupProgressEnum backupProgressEnum) => $"{backupProgressEnum}backup{tenantId}"; - } - public class BackupService : IBackupService { private ILog Log { get; set; } private BackupStorageFactory BackupStorageFactory { get; set; } private BackupWorker BackupWorker { get; set; } private BackupRepository BackupRepository { get; } - private BackupServiceNotifier BackupServiceNotifier { get; } private IConfiguration Configuration { get; } public BackupService( IOptionsMonitor options, BackupStorageFactory backupStorageFactory, BackupWorker backupWorker, - BackupRepository backupRepository, - BackupServiceNotifier backupServiceNotifier, + BackupRepository backupRepository, IConfiguration configuration) { Log = options.CurrentValue; BackupStorageFactory = backupStorageFactory; BackupWorker = backupWorker; BackupRepository = backupRepository; - BackupServiceNotifier = backupServiceNotifier; Configuration = configuration; } @@ -210,17 +171,17 @@ namespace ASC.Data.Backup.Service public BackupProgress GetBackupProgress(int tenantId) { - return BackupServiceNotifier.GetBackupProgress(tenantId); + return BackupWorker.GetBackupProgress(tenantId); } public BackupProgress GetTransferProgress(int tenantId) { - return BackupServiceNotifier.GetTransferProgress(tenantId); + return BackupWorker.GetTransferProgress(tenantId); } public BackupProgress GetRestoreProgress(int tenantId) { - return BackupServiceNotifier.GetRestoreProgress(tenantId); + return BackupWorker.GetRestoreProgress(tenantId); } public string GetTmpFolder() @@ -294,7 +255,6 @@ namespace ASC.Data.Backup.Service { if (services.TryAddScoped()) { - services.TryAddSingleton(); return services .AddBackupWorkerService(); } diff --git a/common/services/ASC.Data.Backup/Service/BackupWorker.cs b/common/services/ASC.Data.Backup/Service/BackupWorker.cs index 8ca5a89e3c..94b4987d6e 100644 --- a/common/services/ASC.Data.Backup/Service/BackupWorker.cs +++ b/common/services/ASC.Data.Backup/Service/BackupWorker.cs @@ -130,11 +130,9 @@ namespace ASC.Data.Backup.Service ProgressQueue.QueueTask((a, b) => item.RunJob(), item); } - var progress = ToBackupProgress(item); + PublishProgress(item); - PublishProgress(progress); - - return progress; + return ToBackupProgress(item); } } @@ -164,6 +162,22 @@ namespace ASC.Data.Backup.Service } } + public BackupProgress GetTransferProgress(int tenantId) + { + lock (SynchRoot) + { + return ToBackupProgress(ProgressQueue.GetTasks().FirstOrDefault(t => t.TenantId == tenantId)); + } + } + + public BackupProgress GetRestoreProgress(int tenantId) + { + lock (SynchRoot) + { + return ToBackupProgress(ProgressQueue.GetTasks().FirstOrDefault(t => t.TenantId == tenantId)); + } + } + public void ResetBackupError(int tenantId) { lock (SynchRoot) @@ -228,6 +242,7 @@ namespace ASC.Data.Backup.Service } } + private BackupProgress ToBackupProgress(BaseBackupProgressItem progressItem) { if (progressItem == null) @@ -262,12 +277,6 @@ namespace ASC.Data.Backup.Service internal void PublishProgress(BaseBackupProgressItem progress) { progress.PublishChanges(); - PublishProgress(ToBackupProgress(progress)); - } - - internal void PublishProgress(BackupProgress progress) - { - CacheBackupProgress.Publish(progress, CacheNotifyAction.InsertOrUpdate); } } diff --git a/web/ASC.Web.Core/Users/UserPhotoManager.cs b/web/ASC.Web.Core/Users/UserPhotoManager.cs index 646132cf66..c1faaf1768 100644 --- a/web/ASC.Web.Core/Users/UserPhotoManager.cs +++ b/web/ASC.Web.Core/Users/UserPhotoManager.cs @@ -675,7 +675,7 @@ namespace ASC.Web.Core.Users } else { - if (!ResizeQueue.GetTasks().OfType().Any(r => r.GetProperty("key") == key)) + if (!ResizeQueue.GetTasks().Any(r => r.GetProperty("key") == key)) { //Add ResizeQueue.QueueTask((a, b) => ResizeImage(resizeTask), resizeTask); From 72426588c56c342c0196f821b9e4a31ba19037e3 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Thu, 6 Aug 2020 22:53:50 +0300 Subject: [PATCH 006/278] Backup: configure --- common/services/ASC.Data.Backup/Service/BackupWorker.cs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/common/services/ASC.Data.Backup/Service/BackupWorker.cs b/common/services/ASC.Data.Backup/Service/BackupWorker.cs index 94b4987d6e..e984dfdbb7 100644 --- a/common/services/ASC.Data.Backup/Service/BackupWorker.cs +++ b/common/services/ASC.Data.Backup/Service/BackupWorker.cs @@ -866,11 +866,6 @@ namespace ASC.Data.Backup.Service services.TryAddTransient(); services.TryAddTransient(); - - services.TryAddSingleton>(); - services.TryAddSingleton>(); - services.AddSingleton>, ConfigureProgressQueue>(); - return services .AddTenantManagerService() .AddCoreBaseSettingsService() @@ -880,7 +875,8 @@ namespace ASC.Data.Backup.Service .AddNotifyHelperService() .AddBackupPortalTaskService() .AddDbFactoryService() - .AddRestorePortalTaskService(); + .AddRestorePortalTaskService() + .AddDistributedTaskQueueService("backup", 5); } } } From 992ab07b87b23964ad337bceb5e16c17f3a1fa76 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Fri, 7 Aug 2020 18:42:58 +0300 Subject: [PATCH 007/278] Common: Reassign. DistributedTaskQueue --- .../ASC.Common/Threading/DistributedTask.cs | 6 +- .../Threading/DistributedTaskQueue.cs | 38 ++++++++-- common/ASC.Data.Reassigns/QueueWorker.cs | 72 +++++++++++-------- .../ReassignProgressItem.cs | 60 ++++++++-------- .../ASC.Data.Reassigns/RemoveProgressItem.cs | 59 ++++++++------- .../Server/Controllers/PeopleController.cs | 6 +- 6 files changed, 140 insertions(+), 101 deletions(-) diff --git a/common/ASC.Common/Threading/DistributedTask.cs b/common/ASC.Common/Threading/DistributedTask.cs index f18e988b0e..5ff59cc200 100644 --- a/common/ASC.Common/Threading/DistributedTask.cs +++ b/common/ASC.Common/Threading/DistributedTask.cs @@ -33,9 +33,9 @@ namespace ASC.Common.Threading { public class DistributedTask { - public Action Publication { get; set; } + protected internal Action Publication { get; set; } - public DistributedTaskCache DistributedTaskCache { get; internal set; } + protected internal DistributedTaskCache DistributedTaskCache { get; internal set; } public string InstanceId { @@ -54,7 +54,7 @@ namespace ASC.Common.Threading { return DistributedTaskCache.Id; } - private set + protected set { DistributedTaskCache.Id = value?.ToString() ?? ""; } diff --git a/common/ASC.Common/Threading/DistributedTaskQueue.cs b/common/ASC.Common/Threading/DistributedTaskQueue.cs index e796f8e7d5..12195b5130 100644 --- a/common/ASC.Common/Threading/DistributedTaskQueue.cs +++ b/common/ASC.Common/Threading/DistributedTaskQueue.cs @@ -214,14 +214,38 @@ namespace ASC.Common.Threading return tasks; } + public T GetTask(string id) where T : DistributedTask + { + var cache = Cache.HashGet(key, id); + if (cache != null) + { + using var scope = ServiceProvider.CreateScope(); + var task = scope.ServiceProvider.GetService(); + task.DistributedTaskCache = cache; + if (task != null && task.Publication == null) + { + task.Publication = GetPublication(); + } + return task; + } + + return null; + } + public DistributedTask GetTask(string id) - { - var task = new DistributedTask(Cache.HashGet(key, id)); - if (task != null && task.Publication == null) - { - task.Publication = GetPublication(); - } - return task; + { + var cache = Cache.HashGet(key, id); + if (cache != null) + { + var task = new DistributedTask(); + if (task != null && task.Publication == null) + { + task.Publication = GetPublication(); + } + return task; + } + + return null; } public void SetTask(DistributedTask task) diff --git a/common/ASC.Data.Reassigns/QueueWorker.cs b/common/ASC.Data.Reassigns/QueueWorker.cs index 3f2c3f925e..ed3a5d1540 100644 --- a/common/ASC.Data.Reassigns/QueueWorker.cs +++ b/common/ASC.Data.Reassigns/QueueWorker.cs @@ -28,11 +28,12 @@ using System; using System.Collections.Generic; using ASC.Common; +using ASC.Common.Threading; using ASC.Common.Threading.Progress; using ASC.Core.Users; using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Options; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Primitives; namespace ASC.Data.Reassigns @@ -44,32 +45,34 @@ namespace ASC.Data.Reassigns return httpRequest?.Headers; } } - public class QueueWorker where T : class, IProgressItem + public class QueueWorker where T : DistributedTask, IProgressItem { - protected readonly ProgressQueue Queue; + protected readonly DistributedTaskQueue Queue; public IHttpContextAccessor HttpContextAccessor { get; } public IServiceProvider ServiceProvider { get; } + public object SynchRoot = new object(); + public QueueWorker( IHttpContextAccessor httpContextAccessor, IServiceProvider serviceProvider, - ProgressQueueOptionsManager optionsQueue) + DistributedTaskQueueOptionsManager options) { HttpContextAccessor = httpContextAccessor; ServiceProvider = serviceProvider; - Queue = optionsQueue.Value; + Queue = options.Get(nameof(T)); } - public string GetProgressItemId(int tenantId, Guid userId) + public static string GetProgressItemId(int tenantId, Guid userId, Type type) { - return string.Format("{0}_{1}_{2}", tenantId, userId, typeof(T).Name); + return string.Format("{0}_{1}_{2}", tenantId, userId, type.Name); } public T GetProgressItemStatus(int tenantId, Guid userId) { - var id = GetProgressItemId(tenantId, userId); - return Queue.GetStatus(id) as T; + var id = GetProgressItemId(tenantId, userId, typeof(T)); + return Queue.GetTask(id); } public void Terminate(int tenantId, Guid userId) @@ -77,30 +80,29 @@ namespace ASC.Data.Reassigns var item = GetProgressItemStatus(tenantId, userId); if (item != null) - Queue.Remove(item); + { + Queue.CancelTask(item.Id); + } } protected IProgressItem Start(int tenantId, Guid userId, Func constructor) { - lock (Queue.SynchRoot) + lock (SynchRoot) { var task = GetProgressItemStatus(tenantId, userId); if (task != null && task.IsCompleted) { - Queue.Remove(task); + Queue.RemoveTask(task.Id); task = null; } if (task == null) { task = constructor(); - Queue.Add(task); + Queue.QueueTask((a, b) => task.RunJob(), task); } - if (!Queue.IsStarted) - Queue.Start(x => x.RunJob()); - return task; } } @@ -113,30 +115,41 @@ namespace ASC.Data.Reassigns IHttpContextAccessor httpContextAccessor, IServiceProvider serviceProvider, QueueWorkerRemove queueWorkerRemove, - ProgressQueueOptionsManager optionsQueue) : - base(httpContextAccessor, serviceProvider, optionsQueue) + DistributedTaskQueueOptionsManager options) : + base(httpContextAccessor, serviceProvider, options) { QueueWorkerRemove = queueWorkerRemove; } public ReassignProgressItem Start(int tenantId, Guid fromUserId, Guid toUserId, Guid currentUserId, bool deleteProfile) { - return Start(tenantId, fromUserId, () => new ReassignProgressItem(ServiceProvider, HttpContextAccessor.HttpContext, this, QueueWorkerRemove, tenantId, fromUserId, toUserId, currentUserId, deleteProfile)) as ReassignProgressItem; + return Start(tenantId, fromUserId, () => + { + var result = ServiceProvider.GetService(); + result.Init(tenantId, fromUserId, toUserId, currentUserId, deleteProfile); + return result; + }) as ReassignProgressItem; } } + public class QueueWorkerRemove : QueueWorker { public QueueWorkerRemove( IHttpContextAccessor httpContextAccessor, IServiceProvider serviceProvider, - ProgressQueueOptionsManager optionsQueue) : - base(httpContextAccessor, serviceProvider, optionsQueue) + DistributedTaskQueueOptionsManager options) : + base(httpContextAccessor, serviceProvider, options) { } public RemoveProgressItem Start(int tenantId, UserInfo user, Guid currentUserId, bool notify) { - return Start(tenantId, user.ID, () => new RemoveProgressItem(ServiceProvider, HttpContextAccessor.HttpContext, this, tenantId, user, currentUserId, notify)) as RemoveProgressItem; + return Start(tenantId, user.ID, () => + { + var result = ServiceProvider.GetService(); + result.Init(tenantId, user, currentUserId, notify); + return result; + }) as RemoveProgressItem; } } @@ -146,23 +159,22 @@ namespace ASC.Data.Reassigns { if (services.TryAddScoped()) { - services.TryAddSingleton>(); - services.TryAddSingleton>(); - services.AddSingleton>, ConfigureProgressQueue>(); + return services + .AddRemoveProgressItemService() + .AddDistributedTaskQueueService(nameof(RemoveProgressItem), 1); } return services; } + public static DIHelper AddQueueWorkerReassignService(this DIHelper services) { if (services.TryAddScoped()) { - services.TryAddSingleton>(); - services.TryAddSingleton>(); - services.AddSingleton>, ConfigureProgressQueue>(); - return services - .AddQueueWorkerRemoveService(); + .AddReassignProgressItemService() + .AddQueueWorkerRemoveService() + .AddDistributedTaskQueueService(nameof(ReassignProgressItem), 1); } return services; diff --git a/common/ASC.Data.Reassigns/ReassignProgressItem.cs b/common/ASC.Data.Reassigns/ReassignProgressItem.cs index 67d18435f2..5673ef6130 100644 --- a/common/ASC.Data.Reassigns/ReassignProgressItem.cs +++ b/common/ASC.Data.Reassigns/ReassignProgressItem.cs @@ -29,7 +29,8 @@ using System.Collections.Generic; using ASC.Common; using ASC.Common.Logging; -//using System.Web; +using ASC.Common.Threading; +//using System.Web; using ASC.Common.Threading.Progress; using ASC.Core; using ASC.Core.Users; @@ -48,40 +49,45 @@ using Microsoft.Extensions.Primitives; namespace ASC.Data.Reassigns { - public class ReassignProgressItem : IProgressItem + public class ReassignProgressItem : DistributedTask, IProgressItem { - private readonly HttpContext _context; private readonly IDictionary _httpHeaders; - private readonly int _tenantId; - private readonly Guid _currentUserId; - private readonly bool _deleteProfile; + private int _tenantId; + private Guid _currentUserId; + private bool _deleteProfile; //private readonly IFileStorageService _docService; - //private readonly ProjectsReassign _projectsReassign; + //private readonly ProjectsReassign _projectsReassign; - public object Id { get; set; } - public object Status { get; set; } public object Error { get; set; } public double Percentage { get; set; } public bool IsCompleted { get; set; } - public Guid FromUser { get; } - public Guid ToUser { get; } - public IServiceProvider ServiceProvider { get; } - public QueueWorkerRemove QueueWorkerRemove { get; } + public Guid FromUser { get; private set; } + public Guid ToUser { get; private set; } + private IServiceProvider ServiceProvider { get; } + private QueueWorkerRemove QueueWorkerRemove { get; } public ReassignProgressItem( IServiceProvider serviceProvider, - HttpContext context, - QueueWorkerReassign queueWorkerReassign, - QueueWorkerRemove queueWorkerRemove, - int tenantId, Guid fromUserId, Guid toUserId, Guid currentUserId, bool deleteProfile) + IHttpContextAccessor httpContextAccessor, + QueueWorkerRemove queueWorkerRemove) { ServiceProvider = serviceProvider; - _context = context; QueueWorkerRemove = queueWorkerRemove; - _httpHeaders = QueueWorker.GetHttpHeaders(context.Request); + _httpHeaders = QueueWorker.GetHttpHeaders(httpContextAccessor.HttpContext.Request); + //_docService = Web.Files.Classes.Global.FileStorageService; + //_projectsReassign = new ProjectsReassign(); + + Status = DistributedTaskStatus.Created; + Error = null; + Percentage = 0; + IsCompleted = false; + } + + public void Init(int tenantId, Guid fromUserId, Guid toUserId, Guid currentUserId, bool deleteProfile) + { _tenantId = tenantId; FromUser = fromUserId; ToUser = toUserId; @@ -91,8 +97,8 @@ namespace ASC.Data.Reassigns //_docService = Web.Files.Classes.Global.FileStorageService; //_projectsReassign = new ProjectsReassign(); - Id = queueWorkerReassign.GetProgressItemId(tenantId, fromUserId); - Status = ProgressStatus.Queued; + Id = QueueWorkerReassign.GetProgressItemId(tenantId, fromUserId, typeof(ReassignProgressItem)); + Status = DistributedTaskStatus.Created; Error = null; Percentage = 0; IsCompleted = false; @@ -118,7 +124,7 @@ namespace ASC.Data.Reassigns try { Percentage = 0; - Status = ProgressStatus.Started; + Status = DistributedTaskStatus.Running; securityContext.AuthenticateMe(_currentUserId); @@ -152,7 +158,7 @@ namespace ASC.Data.Reassigns SendSuccessNotify(userManager, studioNotifyService, messageService, messageTarget, displayUserSettingsHelper); Percentage = 100; - Status = ProgressStatus.Done; + Status = DistributedTaskStatus.Completed; if (_deleteProfile) { @@ -162,7 +168,7 @@ namespace ASC.Data.Reassigns catch (Exception ex) { logger.Error(ex); - Status = ProgressStatus.Failed; + Status = DistributedTaskStatus.Failted; Error = ex.Message; SendErrorNotify(userManager, studioNotifyService, ex.Message); } @@ -221,10 +227,8 @@ namespace ASC.Data.Reassigns public static class ReassignProgressItemExtension { public static DIHelper AddReassignProgressItemService(this DIHelper services) - { - services.TryAddSingleton>(); - services.TryAddSingleton>(); - services.AddSingleton>, ConfigureProgressQueue>(); + { + services.TryAddScoped(); return services; } } diff --git a/common/ASC.Data.Reassigns/RemoveProgressItem.cs b/common/ASC.Data.Reassigns/RemoveProgressItem.cs index 419fc598d9..8e8995a77f 100644 --- a/common/ASC.Data.Reassigns/RemoveProgressItem.cs +++ b/common/ASC.Data.Reassigns/RemoveProgressItem.cs @@ -51,49 +51,51 @@ using Microsoft.Extensions.Primitives; namespace ASC.Data.Reassigns { - public class RemoveProgressItem : IProgressItem + public class RemoveProgressItem : DistributedTask, IProgressItem { private readonly IDictionary _httpHeaders; - private readonly int _tenantId; - private readonly Guid _currentUserId; - private readonly bool _notify; + private int _tenantId; + private Guid _currentUserId; + private bool _notify; //private readonly IFileStorageService _docService; //private readonly MailGarbageEngine _mailEraser; - public string Id { get; } - public DistributedTaskStatus Status { get; set; } public object Error { get; set; } public double Percentage { get; set; } public bool IsCompleted { get; set; } - public Guid FromUser { get; } - public IServiceProvider ServiceProvider { get; } - public UserInfo User { get; } + public Guid FromUser { get; private set; } + private IServiceProvider ServiceProvider { get; } + private UserInfo User { get; set; } public RemoveProgressItem( IServiceProvider serviceProvider, - HttpContext context, - QueueWorkerRemove queueWorkerRemove, - int tenantId, UserInfo user, Guid currentUserId, bool notify) + IHttpContextAccessor httpContextAccessor) { - _httpHeaders = QueueWorker.GetHttpHeaders(context.Request); - ServiceProvider = serviceProvider; - _tenantId = tenantId; - User = user; - FromUser = user.ID; - _currentUserId = currentUserId; - _notify = notify; - - //_docService = Web.Files.Classes.Global.FileStorageService; - //_mailEraser = new MailGarbageEngine(); - - Id = queueWorkerRemove.GetProgressItemId(tenantId, FromUser); + _httpHeaders = QueueWorker.GetHttpHeaders(httpContextAccessor.HttpContext.Request); + ServiceProvider = serviceProvider; + + + //_docService = Web.Files.Classes.Global.FileStorageService; + //_mailEraser = new MailGarbageEngine(); + + Status = DistributedTaskStatus.Created; Error = null; Percentage = 0; IsCompleted = false; } + + public void Init(int tenantId, UserInfo user, Guid currentUserId, bool notify) + { + _tenantId = tenantId; + User = user; + FromUser = user.ID; + _currentUserId = currentUserId; + _notify = notify; + Id = QueueWorkerRemove.GetProgressItemId(tenantId, FromUser, typeof(RemoveProgressItem)); + } public void RunJob() { @@ -252,17 +254,14 @@ namespace ASC.Data.Reassigns if (!_notify) return; studioNotifyService.SendMsgRemoveUserDataFailed(_currentUserId, User, userName, errorMessage); - } + } } public static class RemoveProgressItemExtension { public static DIHelper AddRemoveProgressItemService(this DIHelper services) - { - - services.TryAddSingleton>(); - services.TryAddSingleton>(); - services.AddSingleton>, ConfigureProgressQueue>(); + { + services.TryAddScoped(); return services; } } diff --git a/products/ASC.People/Server/Controllers/PeopleController.cs b/products/ASC.People/Server/Controllers/PeopleController.cs index 1dae29bc56..f0b5586bd2 100644 --- a/products/ASC.People/Server/Controllers/PeopleController.cs +++ b/products/ASC.People/Server/Controllers/PeopleController.cs @@ -186,7 +186,7 @@ namespace ASC.Employee.Core.Controllers } [Read("email")] - public EmployeeWraperFull GetByEmail([FromQuery]string email) + public EmployeeWraperFull GetByEmail([FromQuery] string email) { if (CoreBaseSettings.Personal && !UserManager.GetUsers(SecurityContext.CurrentAccount.ID).IsOwner(Tenant)) throw new MethodAccessException("Method not available"); @@ -246,13 +246,13 @@ namespace ASC.Employee.Core.Controllers } [Read("search")] - public IEnumerable GetPeopleSearch([FromQuery]string query) + public IEnumerable GetPeopleSearch([FromQuery] string query) { return GetSearch(query); } [Read("status/{status}/search")] - public IEnumerable GetAdvanced(EmployeeStatus status, [FromQuery]string query) + public IEnumerable GetAdvanced(EmployeeStatus status, [FromQuery] string query) { if (CoreBaseSettings.Personal) throw new MethodAccessException("Method not available"); try From e4acc15773a6e2fca969a517abb7d968c5b771b2 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Tue, 29 Sep 2020 16:02:47 +0300 Subject: [PATCH 008/278] removed WorkerQueue, ProgressQueue --- .../ASC.Common/Threading/DistributedTask.cs | 2 +- .../Threading/Progress/ProgressBase.cs | 122 ------ .../Threading/Progress/ProgressQueue.cs | 139 ------- .../ASC.Common/Threading/Workers/WorkItem.cs | 76 ---- .../Threading/Workers/WorkerQueue.cs | 382 ------------------ common/ASC.Data.Storage/StaticUploader.cs | 84 ++-- common/ASC.Data.Storage/StorageUploader.cs | 104 ++--- 7 files changed, 112 insertions(+), 797 deletions(-) delete mode 100644 common/ASC.Common/Threading/Progress/ProgressBase.cs delete mode 100644 common/ASC.Common/Threading/Progress/ProgressQueue.cs delete mode 100644 common/ASC.Common/Threading/Workers/WorkItem.cs delete mode 100644 common/ASC.Common/Threading/Workers/WorkerQueue.cs diff --git a/common/ASC.Common/Threading/DistributedTask.cs b/common/ASC.Common/Threading/DistributedTask.cs index 5ff59cc200..1e846e4df3 100644 --- a/common/ASC.Common/Threading/DistributedTask.cs +++ b/common/ASC.Common/Threading/DistributedTask.cs @@ -33,7 +33,7 @@ namespace ASC.Common.Threading { public class DistributedTask { - protected internal Action Publication { get; set; } + public Action Publication { get; set; } protected internal DistributedTaskCache DistributedTaskCache { get; internal set; } diff --git a/common/ASC.Common/Threading/Progress/ProgressBase.cs b/common/ASC.Common/Threading/Progress/ProgressBase.cs deleted file mode 100644 index aa4b51b5f1..0000000000 --- a/common/ASC.Common/Threading/Progress/ProgressBase.cs +++ /dev/null @@ -1,122 +0,0 @@ -/* - * - * (c) Copyright Ascensio System Limited 2010-2018 - * - * This program is freeware. You can redistribute it and/or modify it under the terms of the GNU - * General Public License (GPL) version 3 as published by the Free Software Foundation (https://www.gnu.org/copyleft/gpl.html). - * In accordance with Section 7(a) of the GNU GPL its Section 15 shall be amended to the effect that - * Ascensio System SIA expressly excludes the warranty of non-infringement of any third-party rights. - * - * THIS PROGRAM IS DISTRIBUTED WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR - * FITNESS FOR A PARTICULAR PURPOSE. For more details, see GNU GPL at https://www.gnu.org/copyleft/gpl.html - * - * You can contact Ascensio System SIA by email at sales@onlyoffice.com - * - * The interactive user interfaces in modified source and object code versions of ONLYOFFICE must display - * Appropriate Legal Notices, as required under Section 5 of the GNU GPL version 3. - * - * Pursuant to Section 7 § 3(b) of the GNU GPL you must retain the original ONLYOFFICE logo which contains - * relevant author attributions when distributing the software. If the display of the logo in its graphic - * form is not reasonably feasible for technical reasons, you must include the words "Powered by ONLYOFFICE" - * in every copy of the program you distribute. - * Pursuant to Section 7 § 3(e) we decline to grant you any rights under trademark law for use of our trademarks. - * -*/ - - -using System; -using System.Threading.Tasks; - -namespace ASC.Common.Threading.Progress -{ - public abstract class ProgressBase : IProgressItem - { - private double _percentage; - - protected int StepCount { get; set; } - - public string Id { get; set; } - - public DistributedTaskStatus Status { get; set; } - - public object Error { get; set; } - - - public double Percentage - { - get { return Math.Min(100.0, Math.Max(0, _percentage)); } - set { _percentage = value; } - } - - public virtual bool IsCompleted { get; set; } - - - public void RunJob() - { - try - { - Percentage = 0; - DoJob(); - } - catch (Exception e) - { - Error = e; - } - finally - { - Percentage = 100; - IsCompleted = true; - } - } - - public async Task RunJobAsync() - { - try - { - Percentage = 0; - await DoJobAsync(); - } - catch (Exception e) - { - Error = e; - } - finally - { - Percentage = 100; - IsCompleted = true; - } - } - - protected ProgressBase() - { - Id = Guid.NewGuid().ToString(); // random id - } - - protected void ProgressAdd(double value) - { - Percentage += value; - } - - protected void StepDone() - { - if (StepCount > 0) - { - Percentage += 100.0 / StepCount; - } - } - - - protected abstract void DoJob(); - - protected virtual Task DoJobAsync() - { - return Task.Run(() => { }); - } - - - object ICloneable.Clone() - { - return MemberwiseClone(); - } - } -} \ No newline at end of file diff --git a/common/ASC.Common/Threading/Progress/ProgressQueue.cs b/common/ASC.Common/Threading/Progress/ProgressQueue.cs deleted file mode 100644 index deb65fd694..0000000000 --- a/common/ASC.Common/Threading/Progress/ProgressQueue.cs +++ /dev/null @@ -1,139 +0,0 @@ -/* - * - * (c) Copyright Ascensio System Limited 2010-2018 - * - * This program is freeware. You can redistribute it and/or modify it under the terms of the GNU - * General Public License (GPL) version 3 as published by the Free Software Foundation (https://www.gnu.org/copyleft/gpl.html). - * In accordance with Section 7(a) of the GNU GPL its Section 15 shall be amended to the effect that - * Ascensio System SIA expressly excludes the warranty of non-infringement of any third-party rights. - * - * THIS PROGRAM IS DISTRIBUTED WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR - * FITNESS FOR A PARTICULAR PURPOSE. For more details, see GNU GPL at https://www.gnu.org/copyleft/gpl.html - * - * You can contact Ascensio System SIA by email at sales@onlyoffice.com - * - * The interactive user interfaces in modified source and object code versions of ONLYOFFICE must display - * Appropriate Legal Notices, as required under Section 5 of the GNU GPL version 3. - * - * Pursuant to Section 7 § 3(b) of the GNU GPL you must retain the original ONLYOFFICE logo which contains - * relevant author attributions when distributing the software. If the display of the logo in its graphic - * form is not reasonably feasible for technical reasons, you must include the words "Powered by ONLYOFFICE" - * in every copy of the program you distribute. - * Pursuant to Section 7 § 3(e) we decline to grant you any rights under trademark law for use of our trademarks. - * -*/ - - -using System; -using System.Linq; - -using ASC.Common.Logging; -using ASC.Common.Threading.Workers; - -using Microsoft.Extensions.Options; - -namespace ASC.Common.Threading.Progress -{ - public class ProgressQueueOptionsManager : OptionsManager> where T : class, IProgressItem - { - public ProgressQueueOptionsManager(IOptionsFactory> factory) : base(factory) - { - } - } - - public class ConfigureProgressQueue : IPostConfigureOptions> where T : class, IProgressItem - { - public ConfigureProgressQueue(IOptionsMonitor log) - { - Log = log; - } - - public IOptionsMonitor Log { get; } - - public void PostConfigure(string name, ProgressQueue queue) - { - queue.log = Log.Get("ASC.WorkerQueue"); - queue.Start(x => x.RunJob()); - } - } - - public class ProgressQueue : WorkerQueue where T : class, IProgressItem - { - public bool removeAfterCompleted; - - public ProgressQueue() - { - - } - - public override void Add(T item) - { - if (GetStatus(item.Id) == null) - { - base.Add(item); - } - } - - public T GetStatus(object id) - { - T item; - lock (SynchRoot) - { - item = GetItems().Where(x => Equals(x.Id, id)).SingleOrDefault(); - if (item != null) - { - if (removeAfterCompleted && item.IsCompleted) - { - Remove(item); - } - return (T)item.Clone(); - } - } - return item; - } - - public void PostComplete(object id) - { - lock (SynchRoot) - { - var item = GetItems().Where(x => Equals(x.Id, id)).SingleOrDefault(); - - if (item != null) - { - item.IsCompleted = true; - - if (removeAfterCompleted) - { - Remove(item); - } - } - } - } - - protected override WorkItem Selector() - { - return Items - .Where(x => !x.IsProcessed && !x.IsCompleted) - .OrderBy(x => x.Added) - .FirstOrDefault(); - } - - protected override void PostComplete(WorkItem item) - { - item.IsCompleted = true; - } - - protected override void ErrorLimit(WorkItem item) - { - PostComplete(item); - } - - protected override void Error(WorkItem workItem, Exception exception) - { - workItem.Item.Error = exception; - workItem.Item.IsCompleted = true; - - base.Error(workItem, exception); - } - } -} \ No newline at end of file diff --git a/common/ASC.Common/Threading/Workers/WorkItem.cs b/common/ASC.Common/Threading/Workers/WorkItem.cs deleted file mode 100644 index 65f3c7fe76..0000000000 --- a/common/ASC.Common/Threading/Workers/WorkItem.cs +++ /dev/null @@ -1,76 +0,0 @@ -/* - * - * (c) Copyright Ascensio System Limited 2010-2018 - * - * This program is freeware. You can redistribute it and/or modify it under the terms of the GNU - * General Public License (GPL) version 3 as published by the Free Software Foundation (https://www.gnu.org/copyleft/gpl.html). - * In accordance with Section 7(a) of the GNU GPL its Section 15 shall be amended to the effect that - * Ascensio System SIA expressly excludes the warranty of non-infringement of any third-party rights. - * - * THIS PROGRAM IS DISTRIBUTED WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR - * FITNESS FOR A PARTICULAR PURPOSE. For more details, see GNU GPL at https://www.gnu.org/copyleft/gpl.html - * - * You can contact Ascensio System SIA by email at sales@onlyoffice.com - * - * The interactive user interfaces in modified source and object code versions of ONLYOFFICE must display - * Appropriate Legal Notices, as required under Section 5 of the GNU GPL version 3. - * - * Pursuant to Section 7 § 3(b) of the GNU GPL you must retain the original ONLYOFFICE logo which contains - * relevant author attributions when distributing the software. If the display of the logo in its graphic - * form is not reasonably feasible for technical reasons, you must include the words "Powered by ONLYOFFICE" - * in every copy of the program you distribute. - * Pursuant to Section 7 § 3(e) we decline to grant you any rights under trademark law for use of our trademarks. - * -*/ - - -using System; - -namespace ASC.Common.Threading.Workers -{ - public class WorkItem : IDisposable - { - private bool disposed; - - internal DateTime Added { get; set; } - internal int ErrorCount { get; set; } - internal bool IsProcessed { get; set; } - - public T Item { get; set; } - public bool IsCompleted { get; set; } - - - public WorkItem(T item) - { - Item = item; - Added = DateTime.Now; - IsProcessed = false; - } - - ~WorkItem() - { - Dispose(false); - } - - public void Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } - - private void Dispose(bool disposing) - { - if (!disposed) - { - if (disposing) - { - if (Item is IDisposable disposable) - { - disposable.Dispose(); - } - } - disposed = true; - } - } - } -} \ No newline at end of file diff --git a/common/ASC.Common/Threading/Workers/WorkerQueue.cs b/common/ASC.Common/Threading/Workers/WorkerQueue.cs deleted file mode 100644 index 368ea85822..0000000000 --- a/common/ASC.Common/Threading/Workers/WorkerQueue.cs +++ /dev/null @@ -1,382 +0,0 @@ -/* - * - * (c) Copyright Ascensio System Limited 2010-2018 - * - * This program is freeware. You can redistribute it and/or modify it under the terms of the GNU - * General Public License (GPL) version 3 as published by the Free Software Foundation (https://www.gnu.org/copyleft/gpl.html). - * In accordance with Section 7(a) of the GNU GPL its Section 15 shall be amended to the effect that - * Ascensio System SIA expressly excludes the warranty of non-infringement of any third-party rights. - * - * THIS PROGRAM IS DISTRIBUTED WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR - * FITNESS FOR A PARTICULAR PURPOSE. For more details, see GNU GPL at https://www.gnu.org/copyleft/gpl.html - * - * You can contact Ascensio System SIA by email at sales@onlyoffice.com - * - * The interactive user interfaces in modified source and object code versions of ONLYOFFICE must display - * Appropriate Legal Notices, as required under Section 5 of the GNU GPL version 3. - * - * Pursuant to Section 7 § 3(b) of the GNU GPL you must retain the original ONLYOFFICE logo which contains - * relevant author attributions when distributing the software. If the display of the logo in its graphic - * form is not reasonably feasible for technical reasons, you must include the words "Powered by ONLYOFFICE" - * in every copy of the program you distribute. - * Pursuant to Section 7 § 3(e) we decline to grant you any rights under trademark law for use of our trademarks. - * -*/ - - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; - -using ASC.Common.Logging; -using ASC.Common.Threading.Progress; - -using Microsoft.Extensions.Options; - -namespace ASC.Common.Threading.Workers -{ - public class WorkerQueueOptionsManager : OptionsManager> - { - public WorkerQueueOptionsManager(IOptionsFactory> factory) : base(factory) - { - } - } - - public class ConfigureWorkerQueue : IConfigureOptions> - { - public ConfigureWorkerQueue(IOptionsMonitor log) - { - Log = log; - } - - public IOptionsMonitor Log { get; } - - public void Configure(Workers.WorkerQueue queue) - { - queue.log = Log.Get("ASC.WorkerQueue"); - } - } - - public class WorkerQueue - { - internal ILog log; - - private readonly ICollection> items = new List>(); - private readonly List threads = new List(); - - private readonly AutoResetEvent waitEvent = new AutoResetEvent(false); - private readonly ManualResetEvent stopEvent = new ManualResetEvent(false); - - public int workerCount; - public bool stopAfterFinsih; - public int errorCount; - public int waitInterval; - - private Action action; - private volatile bool started; - - public object SynchRoot { get { return Items; } } - - protected virtual ICollection> Items { get { return items; } } - - public bool IsStarted { get { return started; } } - - public WorkerQueue() - { - - } - - - public void Start(Action starter) - { - Start(starter, true); - } - - public IEnumerable GetItems() - { - lock (Items) - { - return Items.Select(x => x.Item).ToList(); - } - } - - public virtual void AddRange(IEnumerable items) - { - lock (Items) - { - foreach (var item in items) - { - Items.Add(new WorkItem(item)); - } - } - waitEvent.Set(); - ReviveThreads(); - } - - public virtual void Add(T item) - { - lock (Items) - { - Items.Add(new WorkItem(item)); - } - waitEvent.Set(); - ReviveThreads(); - } - - public void Remove(T item) - { - lock (Items) - { - var existing = Items.Where(x => Equals(x.Item, item)).SingleOrDefault(); - RemoveInternal(existing); - } - } - - public void Clear() - { - lock (Items) - { - foreach (var workItem in Items) - { - workItem.Dispose(); - } - Items.Clear(); - } - } - - public void Stop() - { - if (started) - { - started = false; - - stopEvent.Set(); - waitEvent.Set(); - - log.Debug("Stoping queue. Joining threads"); - foreach (var workerThread in threads) - { - workerThread.Join(); - } - threads.Clear(); - log.Debug("Queue stoped. Threads cleared"); - } - } - - public void Terminate() - { - if (started) - { - started = false; - - stopEvent.Set(); - waitEvent.Set(); - - log.Debug("Stoping queue. Terminating threads"); - foreach (var worker in threads.Where(t => t != Thread.CurrentThread)) - { - worker.Abort(); - } - if (threads.Contains(Thread.CurrentThread)) - { - threads.Clear(); - log.Debug("Terminate called from current worker thread. Terminating"); - Thread.CurrentThread.Abort(); - } - threads.Clear(); - log.Debug("Queue stoped. Threads cleared"); - } - } - - - protected virtual WorkItem Selector() - { - return Items.Where(x => !x.IsProcessed).OrderBy(x => x.Added).FirstOrDefault(); - } - - protected virtual void PostComplete(WorkItem item) - { - RemoveInternal(item); - } - - protected void RemoveInternal(WorkItem item) - { - if (item != null) - { - Items.Remove(item); - item.Dispose(); - } - } - - protected virtual void ErrorLimit(WorkItem item) - { - RemoveInternal(item); - } - - protected virtual void Error(WorkItem item, Exception exception) - { - log.Error(item, exception); - - item.IsProcessed = false; - item.Added = DateTime.Now; - } - - - private WaitHandle[] WaitObjects() - { - return new WaitHandle[] { stopEvent, waitEvent }; - } - - private void ReviveThreads() - { - if (threads.Count != 0) - { - var haveLiveThread = threads.Count(x => x.IsAlive) > 0; - if (!haveLiveThread) - { - Stop(); - Start(action); - } - } - } - - private void Start(Action starter, bool backgroundThreads) - { - if (!started) - { - started = true; - action = starter; - - stopEvent.Reset(); - waitEvent.Reset(); - - log.Debug("Creating threads"); - for (var i = 0; i < workerCount; i++) - { - threads.Add(new Thread(DoWork) { IsBackground = backgroundThreads }); - } - - log.Debug("Starting threads"); - foreach (var thread in threads) - { - thread.Start(stopAfterFinsih); - } - } - } - - private void DoWork(object state) - { - try - { - var stopAfterFinsih = false; - if (state != null && state is bool) - { - stopAfterFinsih = (bool)state; - } - do - { - WorkItem item; - Action localAction; - lock (Items) - { - localAction = action; - item = Selector(); - if (item != null) - { - item.IsProcessed = true; - } - } - if (localAction == null) - break;//Exit if action is null - - if (item != null) - { - try - { - localAction(item.Item); - var fallSleep = false; - lock (Items) - { - PostComplete(item); - if (Items.Count == 0) - { - fallSleep = true; - } - } - if (fallSleep) - { - if (stopAfterFinsih || WaitHandle.WaitAny(WaitObjects(), Timeout.Infinite, false) == 0) - { - break; - } - } - } - catch (ThreadAbortException) - { - return; - } - catch (Exception e) - { - lock (Items) - { - - Error(item, e); - item.ErrorCount++; - if (item.ErrorCount > errorCount) - { - ErrorLimit(item); - } - } - } - } - else - { - if (stopAfterFinsih || WaitHandle.WaitAny(WaitObjects(), waitInterval, false) == 0) - { - break; - } - } - } while (true); - } - catch (ThreadAbortException) - { - return; - } - catch (Exception err) - { - log.Error(err); - } - } - } - - public static class WorkerQueueExtention - { - public static DIHelper AddWorkerQueue(this DIHelper services, int workerCount, int waitInterval, bool stopAfterFinsih, int errorCount) - { - void action(WorkerQueue a) - { - a.workerCount = workerCount; - a.waitInterval = waitInterval; - a.stopAfterFinsih = stopAfterFinsih; - a.errorCount = errorCount; - } - - return services.Configure>(action); - } - - public static DIHelper AddProgressQueue(this DIHelper services, int workerCount, int waitInterval, bool removeAfterCompleted, bool stopAfterFinsih, int errorCount) where T1 : class, IProgressItem - { - void action(ProgressQueue a) - { - a.workerCount = workerCount; - a.waitInterval = waitInterval; - a.stopAfterFinsih = stopAfterFinsih; - a.errorCount = errorCount; - a.removeAfterCompleted = removeAfterCompleted; - } - - return services.Configure>(action); - } - } -} \ No newline at end of file diff --git a/common/ASC.Data.Storage/StaticUploader.cs b/common/ASC.Data.Storage/StaticUploader.cs index a95d894ee9..ee58f8b79f 100644 --- a/common/ASC.Data.Storage/StaticUploader.cs +++ b/common/ASC.Data.Storage/StaticUploader.cs @@ -34,7 +34,8 @@ using System.Threading.Tasks; using ASC.Common; using ASC.Common.Caching; using ASC.Common.Logging; -using ASC.Common.Threading.Progress; +using ASC.Common.Threading; +using ASC.Common.Threading.Progress; using ASC.Core; using ASC.Core.Common.Settings; using ASC.Core.Tenants; @@ -57,7 +58,8 @@ namespace ASC.Data.Storage public TenantManager TenantManager { get; } public SettingsManager SettingsManager { get; } public StorageSettingsHelper StorageSettingsHelper { get; } - + + protected readonly DistributedTaskQueue Queue; static StaticUploader() { Scheduler = new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, 4).ConcurrentScheduler; @@ -70,12 +72,14 @@ namespace ASC.Data.Storage IServiceProvider serviceProvider, TenantManager tenantManager, SettingsManager settingsManager, - StorageSettingsHelper storageSettingsHelper) + StorageSettingsHelper storageSettingsHelper, + DistributedTaskQueueOptionsManager options) { ServiceProvider = serviceProvider; TenantManager = tenantManager; SettingsManager = settingsManager; - StorageSettingsHelper = storageSettingsHelper; + StorageSettingsHelper = storageSettingsHelper; + Queue = options.Get(nameof(StaticUploader)); } public string UploadFile(string relativePath, string mappedPath, Action onComplete = null) @@ -125,7 +129,7 @@ namespace ASC.Data.Storage return task; } - public async void UploadDir(string relativePath, string mappedPath) + public void UploadDir(string relativePath, string mappedPath) { if (!CanUpload()) return; if (!Directory.Exists(mappedPath)) return; @@ -136,21 +140,12 @@ namespace ASC.Data.Storage lock (Locker) { - uploadOperation = Cache.Get(key); + uploadOperation = Queue.GetTask(key); if (uploadOperation != null) return; - uploadOperation = new UploadOperationProgress(this, relativePath, mappedPath); - Cache.Insert(key, uploadOperation, DateTime.MaxValue); + uploadOperation = new UploadOperationProgress(this, key, tenant.TenantId, relativePath, mappedPath); + Queue.QueueTask((a, b) => uploadOperation.RunJob(), uploadOperation); } - - - tenant.SetStatus(TenantStatus.Migrating); - TenantManager.SaveTenant(tenant); - - await uploadOperation.RunJobAsync(); - - tenant.SetStatus(Core.Tenants.TenantStatus.Active); - TenantManager.SaveTenant(tenant); } public bool CanUpload() @@ -240,18 +235,29 @@ namespace ASC.Data.Storage } } - public class UploadOperationProgress : ProgressBase + public class UploadOperationProgress : DistributedTask, IProgressItem { private readonly string relativePath; private readonly string mappedPath; private readonly IEnumerable directoryFiles; + private readonly int StepCount; public IServiceProvider ServiceProvider { get; } - public StaticUploader StaticUploader { get; } + public StaticUploader StaticUploader { get; } + public int TenantId { get; } + public object Error { get; set; } + + public double Percentage { get; set; } + + public bool IsCompleted { get; set; } + + public UploadOperationProgress(StaticUploader staticUploader, string key, int tenantId, string relativePath, string mappedPath) + { + Id = key; + Status = DistributedTaskStatus.Created; - public UploadOperationProgress(StaticUploader staticUploader, string relativePath, string mappedPath) - { - StaticUploader = staticUploader; + StaticUploader = staticUploader; + TenantId = tenantId; this.relativePath = relativePath; this.mappedPath = mappedPath; @@ -265,7 +271,7 @@ namespace ASC.Data.Storage StepCount = directoryFiles.Count(); } - protected override async Task DoJobAsync() + protected async Task RunJobAsync() { var tasks = new List(); foreach (var file in directoryFiles) @@ -277,14 +283,38 @@ namespace ASC.Data.Storage await Task.WhenAll(tasks); } - protected override void DoJob() - { + public void RunJob() + { + using var scope = ServiceProvider.CreateScope(); + var tenantManager = scope.ServiceProvider.GetService(); + var tenant = tenantManager.GetTenant(TenantId); + tenantManager.SetCurrentTenant(tenant); + + tenant.SetStatus(TenantStatus.Migrating); + tenantManager.SaveTenant(tenant); + foreach (var file in directoryFiles) { var filePath = file.Substring(mappedPath.TrimEnd('/').Length); StaticUploader.UploadFileAsync(Path.Combine(relativePath, filePath), file, (res) => StepDone()).Wait(); - } - } + } + + tenant.SetStatus(Core.Tenants.TenantStatus.Active); + tenantManager.SaveTenant(tenant); + } + + protected void StepDone() + { + if (StepCount > 0) + { + Percentage += 100.0 / StepCount; + } + } + + public object Clone() + { + return MemberwiseClone(); + } } public static class StaticUploaderExtension diff --git a/common/ASC.Data.Storage/StorageUploader.cs b/common/ASC.Data.Storage/StorageUploader.cs index 0f560271f1..0ddc63e9c4 100644 --- a/common/ASC.Data.Storage/StorageUploader.cs +++ b/common/ASC.Data.Storage/StorageUploader.cs @@ -26,12 +26,10 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; +using System.Linq; -using ASC.Common.Caching; using ASC.Common.Logging; +using ASC.Common.Threading; using ASC.Common.Threading.Progress; using ASC.Core; using ASC.Core.Common.Settings; @@ -44,69 +42,48 @@ using Microsoft.Extensions.Options; namespace ASC.Data.Storage { public class StorageUploader - { - private static readonly TaskScheduler Scheduler; - private static readonly CancellationTokenSource TokenSource; + { + protected readonly DistributedTaskQueue Queue; - private static readonly ICache Cache; private static readonly object Locker; public IServiceProvider ServiceProvider { get; } static StorageUploader() { - Scheduler = new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, 4).ConcurrentScheduler; - TokenSource = new CancellationTokenSource(); - Cache = AscCache.Memory; Locker = new object(); } - public StorageUploader(IServiceProvider serviceProvider) + public StorageUploader(IServiceProvider serviceProvider, DistributedTaskQueueOptionsManager options) { - ServiceProvider = serviceProvider; + ServiceProvider = serviceProvider; + Queue = options.Get(nameof(StorageUploader)); } public void Start(int tenantId, StorageSettings newStorageSettings, StorageFactoryConfig storageFactoryConfig) { - if (TokenSource.Token.IsCancellationRequested) return; - - MigrateOperation migrateOperation; - lock (Locker) - { - migrateOperation = Cache.Get(GetCacheKey(tenantId)); + { + var id = GetCacheKey(tenantId); + var migrateOperation = Queue.GetTask(id); if (migrateOperation != null) return; - migrateOperation = new MigrateOperation(ServiceProvider, tenantId, newStorageSettings, storageFactoryConfig); - Cache.Insert(GetCacheKey(tenantId), migrateOperation, DateTime.MaxValue); + migrateOperation = new MigrateOperation(ServiceProvider, id, tenantId, newStorageSettings, storageFactoryConfig); + Queue.QueueTask((a, b) => migrateOperation.RunJob(), migrateOperation); } - - var task = new Task(migrateOperation.RunJob, TokenSource.Token, TaskCreationOptions.LongRunning); - - task.ConfigureAwait(false) - .GetAwaiter() - .OnCompleted(() => - { - lock (Locker) - { - Cache.Remove(GetCacheKey(tenantId)); - } - }); - - task.Start(Scheduler); } - public static MigrateOperation GetProgress(int tenantId) + public MigrateOperation GetProgress(int tenantId) { lock (Locker) { - return Cache.Get(GetCacheKey(tenantId)); + return Queue.GetTask(GetCacheKey(tenantId)); } } - public static void Stop() + public void Stop(int tenantId) { - TokenSource.Cancel(); + Queue.CancelTask(GetCacheKey(tenantId)); } private static string GetCacheKey(int tenantId) @@ -115,21 +92,25 @@ namespace ASC.Data.Storage } } - public class MigrateOperation : ProgressBase + public class MigrateOperation : DistributedTask, IProgressItem { private readonly ILog Log; private static readonly string ConfigPath; private readonly IEnumerable Modules; private readonly StorageSettings settings; - private readonly int tenantId; + private readonly int tenantId; + private readonly int StepCount; static MigrateOperation() { ConfigPath = ""; } - public MigrateOperation(IServiceProvider serviceProvider, int tenantId, StorageSettings settings, StorageFactoryConfig storageFactoryConfig) - { + public MigrateOperation(IServiceProvider serviceProvider, string id, int tenantId, StorageSettings settings, StorageFactoryConfig storageFactoryConfig) + { + Id = id; + Status = DistributedTaskStatus.Created; + ServiceProvider = serviceProvider; this.tenantId = tenantId; this.settings = settings; @@ -140,13 +121,21 @@ namespace ASC.Data.Storage } public IServiceProvider ServiceProvider { get; } - public StorageFactoryConfig StorageFactoryConfig { get; } - - protected override void DoJob() + public StorageFactoryConfig StorageFactoryConfig { get; } + + public object Error { get; set; } + + public double Percentage { get; set; } + + public bool IsCompleted { get; set; } + + public void RunJob() { try { - Log.DebugFormat("Tenant: {0}", tenantId); + Log.DebugFormat("Tenant: {0}", tenantId); + Status = DistributedTaskStatus.Running; + using var scope = ServiceProvider.CreateScope(); var tenantManager = scope.ServiceProvider.GetService(); var tenant = tenantManager.GetTenant(tenantId); @@ -199,13 +188,28 @@ namespace ASC.Data.Storage settingsManager.Save(settings); tenant.SetStatus(TenantStatus.Active); - tenantManager.SaveTenant(tenant); + tenantManager.SaveTenant(tenant); + Status = DistributedTaskStatus.Completed; } catch (Exception e) - { + { + Status = DistributedTaskStatus.Failted; Error = e; Log.Error(e); } - } + } + + public object Clone() + { + return MemberwiseClone(); + } + + protected void StepDone() + { + if (StepCount > 0) + { + Percentage += 100.0 / StepCount; + } + } } } From 90e61ba0827f7f4cbbdafd44559d249abbf305dc Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Wed, 30 Sep 2020 13:54:49 +0300 Subject: [PATCH 009/278] Added DistributedTaskProgress --- .../ASC.Common/Threading/DistributedTask.cs | 6 +- .../Threading/DistributedTaskProgress.cs | 75 ++++++++++++++++++ .../Threading/DistributedTaskQueue.cs | 6 +- .../protos/DistributedTaskCache.proto | 3 + common/ASC.Data.Reassigns/QueueWorker.cs | 13 ++-- .../ReassignProgressItem.cs | 76 +++++++++---------- .../ASC.Data.Reassigns/RemoveProgressItem.cs | 69 ++++++++--------- .../Migration/MigrateServiceClient.cs | 6 +- common/ASC.Data.Storage/StorageUploader.cs | 38 ++++++---- .../ASC.Data.Backup/Service/BackupService.cs | 51 +++++++++++-- .../ASC.Data.Backup/Service/BackupWorker.cs | 7 +- .../FileOperations/FileOperationsManager.cs | 4 +- products/ASC.Files/Core/Utils/FileMarker.cs | 4 +- products/ASC.People/Server/Startup.cs | 9 +-- web/ASC.Web.Core/Users/UserPhotoManager.cs | 4 +- 15 files changed, 243 insertions(+), 128 deletions(-) create mode 100644 common/ASC.Common/Threading/DistributedTaskProgress.cs diff --git a/common/ASC.Common/Threading/DistributedTask.cs b/common/ASC.Common/Threading/DistributedTask.cs index 1e846e4df3..15911ceb6e 100644 --- a/common/ASC.Common/Threading/DistributedTask.cs +++ b/common/ASC.Common/Threading/DistributedTask.cs @@ -72,13 +72,13 @@ namespace ASC.Common.Threading } } - public AggregateException Exception + public Exception Exception { get { - return new AggregateException(DistributedTaskCache.Exception); + return new Exception(DistributedTaskCache.Exception); } - internal set + set { DistributedTaskCache.Exception = value?.ToString() ?? ""; } diff --git a/common/ASC.Common/Threading/DistributedTaskProgress.cs b/common/ASC.Common/Threading/DistributedTaskProgress.cs new file mode 100644 index 0000000000..e250764124 --- /dev/null +++ b/common/ASC.Common/Threading/DistributedTaskProgress.cs @@ -0,0 +1,75 @@ +using System; + +namespace ASC.Common.Threading +{ + public class DistributedTaskProgress : DistributedTask + { + public double Percentage + { + get + { + return Math.Min(100.0, Math.Max(0, DistributedTaskCache.Percentage)); + } + set + { + DistributedTaskCache.Percentage = value; + } + } + + public bool IsCompleted + { + get + { + return DistributedTaskCache.IsCompleted; + } + set + { + DistributedTaskCache.IsCompleted = value; + } + } + + protected int StepCount + { + get + { + return DistributedTaskCache.StepCount; + } + set + { + DistributedTaskCache.StepCount = value; + } + } + + protected void StepDone() + { + if (StepCount > 0) + { + Percentage += 100.0 / StepCount; + } + } + + public void RunJob() + { + try + { + Percentage = 0; + DoJob(); + } + catch (AggregateException e) + { + Exception = e; + } + finally + { + Percentage = 100; + IsCompleted = true; + PublishChanges(); + } + } + + protected virtual void DoJob() + { + + } + } +} diff --git a/common/ASC.Common/Threading/DistributedTaskQueue.cs b/common/ASC.Common/Threading/DistributedTaskQueue.cs index 6479d1b22b..6bcc787949 100644 --- a/common/ASC.Common/Threading/DistributedTaskQueue.cs +++ b/common/ASC.Common/Threading/DistributedTaskQueue.cs @@ -300,15 +300,15 @@ namespace ASC.Common.Threading public static class DistributedTaskQueueExtention { - public static DIHelper AddDistributedTaskQueueService(this DIHelper services, string name, int maxThreadsCount) + public static DIHelper AddDistributedTaskQueueService(this DIHelper services, int maxThreadsCount) { services.TryAddSingleton(); services .TryAddSingleton() - .TryAddSingleton() + .TryAddSingleton() //check .AddSingleton, ConfigureDistributedTaskQueue>(); - _ = services.Configure(name, r => + _ = services.Configure(typeof(T).Name, r => { r.MaxThreadsCount = maxThreadsCount; //r.errorCount = 1; diff --git a/common/ASC.Common/protos/DistributedTaskCache.proto b/common/ASC.Common/protos/DistributedTaskCache.proto index 0942f1476a..06d318ae44 100644 --- a/common/ASC.Common/protos/DistributedTaskCache.proto +++ b/common/ASC.Common/protos/DistributedTaskCache.proto @@ -9,6 +9,9 @@ message DistributedTaskCache { string Exception = 4; repeated DistributedTaskCacheProp Props = 5; string Key = 6; + double Percentage = 7; + bool IsCompleted = 8; + int32 StepCount = 9; message DistributedTaskCacheProp { diff --git a/common/ASC.Data.Reassigns/QueueWorker.cs b/common/ASC.Data.Reassigns/QueueWorker.cs index a6ca82c8ed..5218ebdc83 100644 --- a/common/ASC.Data.Reassigns/QueueWorker.cs +++ b/common/ASC.Data.Reassigns/QueueWorker.cs @@ -45,7 +45,7 @@ namespace ASC.Data.Reassigns return httpRequest?.Headers; } } - public class QueueWorker where T : DistributedTask, IProgressItem + public class QueueWorker where T : DistributedTaskProgress { protected readonly DistributedTaskQueue Queue; @@ -61,7 +61,7 @@ namespace ASC.Data.Reassigns { HttpContextAccessor = httpContextAccessor; ServiceProvider = serviceProvider; - Queue = options.Get(nameof(T)); + Queue = options.Get(typeof(T).Name); } public static string GetProgressItemId(int tenantId, Guid userId, Type type) @@ -85,7 +85,7 @@ namespace ASC.Data.Reassigns } } - protected IProgressItem Start(int tenantId, Guid userId, Func constructor) + protected DistributedTaskProgress Start(int tenantId, Guid userId, Func constructor) { lock (SynchRoot) { @@ -110,15 +110,12 @@ namespace ASC.Data.Reassigns public class QueueWorkerReassign : QueueWorker { - private QueueWorkerRemove QueueWorkerRemove { get; } public QueueWorkerReassign( IHttpContextAccessor httpContextAccessor, IServiceProvider serviceProvider, - QueueWorkerRemove queueWorkerRemove, DistributedTaskQueueOptionsManager options) : base(httpContextAccessor, serviceProvider, options) { - QueueWorkerRemove = queueWorkerRemove; } public ReassignProgressItem Start(int tenantId, Guid fromUserId, Guid toUserId, Guid currentUserId, bool deleteProfile) @@ -161,7 +158,7 @@ namespace ASC.Data.Reassigns { return services .AddRemoveProgressItemService() - .AddDistributedTaskQueueService(nameof(RemoveProgressItem), 1); + .AddDistributedTaskQueueService(1); } return services; @@ -174,7 +171,7 @@ namespace ASC.Data.Reassigns return services .AddReassignProgressItemService() .AddQueueWorkerRemoveService() - .AddDistributedTaskQueueService(nameof(ReassignProgressItem), 1); + .AddDistributedTaskQueueService(1); } return services; diff --git a/common/ASC.Data.Reassigns/ReassignProgressItem.cs b/common/ASC.Data.Reassigns/ReassignProgressItem.cs index 49e76acd75..6b526eb8f9 100644 --- a/common/ASC.Data.Reassigns/ReassignProgressItem.cs +++ b/common/ASC.Data.Reassigns/ReassignProgressItem.cs @@ -49,7 +49,7 @@ using Microsoft.Extensions.Primitives; namespace ASC.Data.Reassigns { - public class ReassignProgressItem : DistributedTask, IProgressItem + public class ReassignProgressItem : DistributedTaskProgress { private readonly IDictionary _httpHeaders; @@ -60,9 +60,6 @@ namespace ASC.Data.Reassigns //private readonly IFileStorageService _docService; //private readonly ProjectsReassign _projectsReassign; - public object Error { get; set; } - public double Percentage { get; set; } - public bool IsCompleted { get; set; } public Guid FromUser { get; private set; } public Guid ToUser { get; private set; } private IServiceProvider ServiceProvider { get; } @@ -79,11 +76,6 @@ namespace ASC.Data.Reassigns //_docService = Web.Files.Classes.Global.FileStorageService; //_projectsReassign = new ProjectsReassign(); - - Status = DistributedTaskStatus.Created; - Error = null; - Percentage = 0; - IsCompleted = false; } public void Init(int tenantId, Guid fromUserId, Guid toUserId, Guid currentUserId, bool deleteProfile) @@ -99,12 +91,12 @@ namespace ASC.Data.Reassigns Id = QueueWorkerReassign.GetProgressItemId(tenantId, fromUserId, typeof(ReassignProgressItem)); Status = DistributedTaskStatus.Created; - Error = null; + Exception = null; Percentage = 0; IsCompleted = false; } - public void RunJob() + protected override void DoJob() { using var scope = ServiceProvider.CreateScope(); var scopeClass = scope.ServiceProvider.GetService(); @@ -121,29 +113,33 @@ namespace ASC.Data.Reassigns logger.InfoFormat("reassignment of data from {0} to {1}", FromUser, ToUser); - logger.Info("reassignment of data from documents"); + logger.Info("reassignment of data from documents"); + + + //_docService.ReassignStorage(_fromUserId, _toUserId); + Percentage = 33; + PublishChanges(); - Percentage = 33; - //_docService.ReassignStorage(_fromUserId, _toUserId); - - logger.Info("reassignment of data from projects"); - - Percentage = 66; - //_projectsReassign.Reassign(_fromUserId, _toUserId); + logger.Info("reassignment of data from projects"); + + //_projectsReassign.Reassign(_fromUserId, _toUserId); + Percentage = 66; + PublishChanges(); if (!coreBaseSettings.CustomMode) { - logger.Info("reassignment of data from crm"); - - Percentage = 99; - //using (var scope = DIHelper.Resolve(_tenantId)) - //{ - // var crmDaoFactory = scope.Resolve(); - // crmDaoFactory.ContactDao.ReassignContactsResponsible(_fromUserId, _toUserId); - // crmDaoFactory.DealDao.ReassignDealsResponsible(_fromUserId, _toUserId); - // crmDaoFactory.TaskDao.ReassignTasksResponsible(_fromUserId, _toUserId); - // crmDaoFactory.CasesDao.ReassignCasesResponsible(_fromUserId, _toUserId); - //} + logger.Info("reassignment of data from crm"); + + //using (var scope = DIHelper.Resolve(_tenantId)) + //{ + // var crmDaoFactory = scope.Resolve(); + // crmDaoFactory.ContactDao.ReassignContactsResponsible(_fromUserId, _toUserId); + // crmDaoFactory.DealDao.ReassignDealsResponsible(_fromUserId, _toUserId); + // crmDaoFactory.TaskDao.ReassignTasksResponsible(_fromUserId, _toUserId); + // crmDaoFactory.CasesDao.ReassignCasesResponsible(_fromUserId, _toUserId); + //} + Percentage = 99; + PublishChanges(); } SendSuccessNotify(userManager, studioNotifyService, messageService, messageTarget, displayUserSettingsHelper); @@ -160,14 +156,15 @@ namespace ASC.Data.Reassigns { logger.Error(ex); Status = DistributedTaskStatus.Failted; - Error = ex.Message; + Exception = ex; SendErrorNotify(userManager, studioNotifyService, ex.Message); } finally { logger.Info("data reassignment is complete"); IsCompleted = true; - } + } + PublishChanges(); } public object Clone() @@ -251,16 +248,16 @@ namespace ASC.Data.Reassigns Options = options; } - public void Deconstruct(out TenantManager tenantManager, + public void Deconstruct(out TenantManager tenantManager, out CoreBaseSettings coreBaseSettings, out MessageService messageService, out StudioNotifyService studioNotifyService, out SecurityContext securityContext, - out UserManager userManager, + out UserManager userManager, out UserPhotoManager userPhotoManager, - out DisplayUserSettingsHelper displayUserSettingsHelper, - out MessageTarget messageTarget, - out IOptionsMonitor optionsMonitor ) + out DisplayUserSettingsHelper displayUserSettingsHelper, + out MessageTarget messageTarget, + out IOptionsMonitor optionsMonitor) { tenantManager = TenantManager; coreBaseSettings = CoreBaseSettings; @@ -279,10 +276,7 @@ namespace ASC.Data.Reassigns { public static DIHelper AddReassignProgressItemService(this DIHelper services) { - services.TryAddSingleton>(); - services.TryAddSingleton>(); - services.TryAddScoped(); - services.AddSingleton>, ConfigureProgressQueue>(); + services.TryAddScoped(); return services; } } diff --git a/common/ASC.Data.Reassigns/RemoveProgressItem.cs b/common/ASC.Data.Reassigns/RemoveProgressItem.cs index c3662efe84..8329c2645a 100644 --- a/common/ASC.Data.Reassigns/RemoveProgressItem.cs +++ b/common/ASC.Data.Reassigns/RemoveProgressItem.cs @@ -51,7 +51,7 @@ using Microsoft.Extensions.Primitives; namespace ASC.Data.Reassigns { - public class RemoveProgressItem : DistributedTask, IProgressItem + public class RemoveProgressItem : DistributedTaskProgress { private readonly IDictionary _httpHeaders; @@ -61,13 +61,9 @@ namespace ASC.Data.Reassigns //private readonly IFileStorageService _docService; //private readonly MailGarbageEngine _mailEraser; - - public object Error { get; set; } - public double Percentage { get; set; } - public bool IsCompleted { get; set; } - public Guid FromUser { get; } + public Guid FromUser { get; private set; } private IServiceProvider ServiceProvider { get; } - public UserInfo User { get; } + public UserInfo User { get; private set; } public RemoveProgressItem( IServiceProvider serviceProvider, @@ -78,13 +74,7 @@ namespace ASC.Data.Reassigns //_docService = Web.Files.Classes.Global.FileStorageService; - //_mailEraser = new MailGarbageEngine(); - - - Status = DistributedTaskStatus.Created; - Error = null; - Percentage = 0; - IsCompleted = false; + //_mailEraser = new MailGarbageEngine(); } public void Init(int tenantId, UserInfo user, Guid currentUserId, bool notify) @@ -94,12 +84,17 @@ namespace ASC.Data.Reassigns FromUser = user.ID; _currentUserId = currentUserId; _notify = notify; + Id = QueueWorkerRemove.GetProgressItemId(tenantId, FromUser, typeof(RemoveProgressItem)); + Status = DistributedTaskStatus.Created; + Exception = null; + Percentage = 0; + IsCompleted = false; } - public void RunJob() + protected override void DoJob() { - using var scope = ServiceProvider.CreateScope(); + using var scope = ServiceProvider.CreateScope(); var scopeClass = scope.ServiceProvider.GetService(); var (tenantManager, coreBaseSettings, messageService, studioNotifyService, securityContext, userManager, messageTarget, webItemManagerSecurity, storageFactory, userFormatter, options) = scopeClass; var logger = options.Get("ASC.Web"); @@ -120,35 +115,40 @@ namespace ASC.Data.Reassigns logger.Info("deleting of data from documents"); - Percentage = 25; //_docService.DeleteStorage(_userId); + Percentage = 25; + PublishChanges(); if (!coreBaseSettings.CustomMode) { logger.Info("deleting of data from crm"); - Percentage = 50; + //using (var scope = DIHelper.Resolve(_tenantId)) //{ // var crmDaoFactory = scope.Resolve(); - crmSpace = 0;// crmDaoFactory.ReportDao.GetFiles(_userId).Sum(file => file.ContentLength); - // crmDaoFactory.ReportDao.DeleteFiles(_userId); - //} + crmSpace = 0;// crmDaoFactory.ReportDao.GetFiles(_userId).Sum(file => file.ContentLength); + // crmDaoFactory.ReportDao.DeleteFiles(_userId); + //} + Percentage = 50; } else { crmSpace = 0; - } + } + + PublishChanges(); logger.Info("deleting of data from mail"); - Percentage = 75; //_mailEraser.ClearUserMail(_userId); + Percentage = 75; + PublishChanges(); logger.Info("deleting of data from talk"); - - Percentage = 99; DeleteTalkStorage(storageFactory); + Percentage = 99; + PublishChanges(); SendSuccessNotify(studioNotifyService, messageService, messageTarget, userName, docsSpace, crmSpace, mailSpace, talkSpace); @@ -159,13 +159,14 @@ namespace ASC.Data.Reassigns { logger.Error(ex); Status = DistributedTaskStatus.Failted; - Error = ex.Message; + Exception = ex; SendErrorNotify(studioNotifyService, ex.Message, userName); } finally { logger.Info("data deletion is complete"); - IsCompleted = true; + IsCompleted = true; + PublishChanges(); } } @@ -290,15 +291,15 @@ namespace ASC.Data.Reassigns public void Deconstruct(out TenantManager tenantManager, out CoreBaseSettings coreBaseSettings, - out MessageService messageService, - out StudioNotifyService studioNotifyService, - out SecurityContext securityContext, - out UserManager userManager, - out MessageTarget messageTarget, + out MessageService messageService, + out StudioNotifyService studioNotifyService, + out SecurityContext securityContext, + out UserManager userManager, + out MessageTarget messageTarget, out WebItemManagerSecurity webItemManagerSecurity, - out StorageFactory storageFactory, + out StorageFactory storageFactory, out UserFormatter userFormatter, - out IOptionsMonitor optionsMonitor ) + out IOptionsMonitor optionsMonitor) { tenantManager = TenantManager; coreBaseSettings = CoreBaseSettings; diff --git a/common/ASC.Data.Storage/Migration/MigrateServiceClient.cs b/common/ASC.Data.Storage/Migration/MigrateServiceClient.cs index e356fad400..edb5844517 100644 --- a/common/ASC.Data.Storage/Migration/MigrateServiceClient.cs +++ b/common/ASC.Data.Storage/Migration/MigrateServiceClient.cs @@ -59,11 +59,13 @@ namespace ASC.Data.Storage.Migration { ProgressMigrationNotify.Subscribe(n => { - var migrationProgress = new MigrationProgress { + var migrationProgress = new MigrationProgress + { TenantId = n.TenantId, Progress = n.Progress, IsCompleted = n.IsCompleted, - Error = n.Error }; + Error = n.Error + }; Cache.Insert(GetCacheKey(n.TenantId), migrationProgress, DateTime.MaxValue); }, diff --git a/common/ASC.Data.Storage/StorageUploader.cs b/common/ASC.Data.Storage/StorageUploader.cs index 76e1c81cfe..8d55bb7d88 100644 --- a/common/ASC.Data.Storage/StorageUploader.cs +++ b/common/ASC.Data.Storage/StorageUploader.cs @@ -28,6 +28,7 @@ using System; using System.Collections.Generic; using System.Linq; +using ASC.Common.Caching; using ASC.Common.Logging; using ASC.Common.Threading; using ASC.Common.Threading.Progress; @@ -84,9 +85,12 @@ namespace ASC.Data.Storage } } - public void Stop(int tenantId) - { - Queue.CancelTask(GetCacheKey(tenantId)); + public void Stop() + { + foreach (var task in Queue.GetTasks().Where(r => r.Status == DistributedTaskStatus.Running)) + { + Queue.CancelTask(task.Id); + } } private static string GetCacheKey(int tenantId) @@ -155,7 +159,7 @@ namespace ASC.Data.Storage var store = storageFactory.GetStorageFromConsumer(ConfigPath, tenantId.ToString(), module, storageSettingsHelper.DataStoreConsumer(settings)); var domains = StorageFactoryConfig.GetDomainList(ConfigPath, module).ToList(); - var crossModuleTransferUtility = new CrossModuleTransferUtility (options, oldStore, store); + var crossModuleTransferUtility = new CrossModuleTransferUtility(options, oldStore, store); string[] files; foreach (var domain in domains) @@ -201,12 +205,12 @@ namespace ASC.Data.Storage } MigrationPublish(); - } - - public object Clone() + } + + public object Clone() { - return MemberwiseClone(); - } + return MemberwiseClone(); + } private void MigrationPublish() { @@ -218,6 +222,14 @@ namespace ASC.Data.Storage IsCompleted = IsCompleted }, CacheNotifyAction.Insert); + } + + protected void StepDone() + { + if (StepCount > 0) + { + Percentage += 100.0 / StepCount; + } } } @@ -232,8 +244,8 @@ namespace ASC.Data.Storage public MigrateOperationScope(TenantManager tenantManager, SecurityContext securityContext, - StorageFactory storageFactory, - IOptionsMonitor options, + StorageFactory storageFactory, + IOptionsMonitor options, StorageSettingsHelper storageSettingsHelper, SettingsManager settingsManager) { @@ -247,10 +259,10 @@ namespace ASC.Data.Storage public void Deconstruct(out TenantManager tenantManager, out SecurityContext securityContext, - out StorageFactory storageFactory, + out StorageFactory storageFactory, out IOptionsMonitor options, out StorageSettingsHelper storageSettingsHelper, - out SettingsManager settingsManager ) + out SettingsManager settingsManager) { tenantManager = TenantManager; securityContext = SecurityContext; diff --git a/common/services/ASC.Data.Backup/Service/BackupService.cs b/common/services/ASC.Data.Backup/Service/BackupService.cs index 2286eb45b0..c6316da67c 100644 --- a/common/services/ASC.Data.Backup/Service/BackupService.cs +++ b/common/services/ASC.Data.Backup/Service/BackupService.cs @@ -31,6 +31,7 @@ using System.Linq; using System.ServiceModel; using ASC.Common; +using ASC.Common.Caching; using ASC.Common.Logging; using ASC.Common.Utils; using ASC.Data.Backup.Contracts; @@ -44,26 +45,64 @@ using Microsoft.Extensions.Options; using Newtonsoft.Json; namespace ASC.Data.Backup.Service -{ +{ + public class BackupServiceNotifier + { + private ICacheNotify СacheBackupProgress { get; } + public ICache Cache { get; } + + public BackupServiceNotifier(ICacheNotify сacheBackupProgress) + { + Cache = AscCache.Memory; + СacheBackupProgress = сacheBackupProgress; + + СacheBackupProgress.Subscribe((a) => + { + Cache.Insert(GetCacheKey(a.TenantId, a.BackupProgressEnum), a, DateTime.UtcNow.AddDays(1)); + }, + CacheNotifyAction.InsertOrUpdate); + } + + public BackupProgress GetBackupProgress(int tenantId) + { + return Cache.Get(GetCacheKey(tenantId, BackupProgressEnum.Backup)); + } + + public BackupProgress GetTransferProgress(int tenantID) + { + return Cache.Get(GetCacheKey(tenantID, BackupProgressEnum.Transfer)); + } + + public BackupProgress GetRestoreProgress(int tenantId) + { + return Cache.Get(GetCacheKey(tenantId, BackupProgressEnum.Restore)); + } + + private string GetCacheKey(int tenantId, BackupProgressEnum backupProgressEnum) => $"{backupProgressEnum}backup{tenantId}"; + } + public class BackupService : IBackupService { private ILog Log { get; set; } private BackupStorageFactory BackupStorageFactory { get; set; } private BackupWorker BackupWorker { get; set; } private BackupRepository BackupRepository { get; } + public BackupServiceNotifier BackupServiceNotifier { get; } private IConfiguration Configuration { get; } public BackupService( IOptionsMonitor options, BackupStorageFactory backupStorageFactory, BackupWorker backupWorker, - BackupRepository backupRepository, + BackupRepository backupRepository, + BackupServiceNotifier backupServiceNotifier, IConfiguration configuration) { Log = options.CurrentValue; BackupStorageFactory = backupStorageFactory; BackupWorker = backupWorker; BackupRepository = backupRepository; + BackupServiceNotifier = backupServiceNotifier; Configuration = configuration; } @@ -171,17 +210,17 @@ namespace ASC.Data.Backup.Service public BackupProgress GetBackupProgress(int tenantId) { - return BackupWorker.GetBackupProgress(tenantId); + return BackupServiceNotifier.GetBackupProgress(tenantId); } public BackupProgress GetTransferProgress(int tenantId) { - return BackupWorker.GetTransferProgress(tenantId); + return BackupServiceNotifier.GetTransferProgress(tenantId); } public BackupProgress GetRestoreProgress(int tenantId) { - return BackupWorker.GetRestoreProgress(tenantId); + return BackupServiceNotifier.GetRestoreProgress(tenantId); } public string GetTmpFolder() @@ -255,6 +294,8 @@ namespace ASC.Data.Backup.Service { if (services.TryAddScoped()) { + services.TryAddSingleton(); + return services .AddBackupWorkerService() .AddBackupStorageFactory() diff --git a/common/services/ASC.Data.Backup/Service/BackupWorker.cs b/common/services/ASC.Data.Backup/Service/BackupWorker.cs index 988a1c2bb6..2630f2b59d 100644 --- a/common/services/ASC.Data.Backup/Service/BackupWorker.cs +++ b/common/services/ASC.Data.Backup/Service/BackupWorker.cs @@ -74,7 +74,7 @@ namespace ASC.Data.Backup.Service FactoryProgressItem factoryProgressItem) { Log = options.CurrentValue; - ProgressQueue = progressQueue.Get("backup"); + ProgressQueue = progressQueue.Get(nameof(BackupProgressItem)); CacheBackupProgress = cacheBackupProgress; FactoryProgressItem = factoryProgressItem; } @@ -914,9 +914,6 @@ namespace ASC.Data.Backup.Service services.TryAddTransient(); services.TryAddTransient(); services.TryAddScoped(); - services.TryAddSingleton>(); - services.TryAddSingleton>(); - services.AddSingleton>, ConfigureProgressQueue>(); return services .AddTenantManagerService() @@ -928,7 +925,7 @@ namespace ASC.Data.Backup.Service .AddBackupPortalTaskService() .AddDbFactoryService() .AddRestorePortalTaskService() - .AddDistributedTaskQueueService("backup", 5) + .AddDistributedTaskQueueService(5) .AddTransferPortalTaskService(); } } diff --git a/products/ASC.Files/Core/Services/WCFService/FileOperations/FileOperationsManager.cs b/products/ASC.Files/Core/Services/WCFService/FileOperations/FileOperationsManager.cs index aeb520680c..27359b2128 100644 --- a/products/ASC.Files/Core/Services/WCFService/FileOperations/FileOperationsManager.cs +++ b/products/ASC.Files/Core/Services/WCFService/FileOperations/FileOperationsManager.cs @@ -47,7 +47,7 @@ namespace ASC.Web.Files.Services.WCFService.FileOperations public FileOperationsManager(DistributedTaskQueueOptionsManager distributedTaskQueueOptionsManager, IServiceProvider serviceProvider) { - tasks = distributedTaskQueueOptionsManager.Get("fileOperations"); + tasks = distributedTaskQueueOptionsManager.Get(nameof(FileOperationsManager)); ServiceProvider = serviceProvider; } @@ -239,7 +239,7 @@ namespace ASC.Web.Files.Services.WCFService.FileOperations { services.TryAddSingleton(); services.TryAddSingleton(); - services.AddDistributedTaskQueueService("fileOperations", 10); + services.AddDistributedTaskQueueService(10); services.TryAddScoped(); services.TryAddScoped(); services.TryAddScoped(); diff --git a/products/ASC.Files/Core/Utils/FileMarker.cs b/products/ASC.Files/Core/Utils/FileMarker.cs index e01f00dcdf..767123bf10 100644 --- a/products/ASC.Files/Core/Utils/FileMarker.cs +++ b/products/ASC.Files/Core/Utils/FileMarker.cs @@ -60,7 +60,7 @@ namespace ASC.Web.Files.Utils { ServiceProvider = serviceProvider; Log = optionsMonitor.CurrentValue; - Tasks = distributedTaskQueueOptionsManager.Get("FileMarkerHelper"); + Tasks = distributedTaskQueueOptionsManager.Get(nameof(FileMarker)); } internal void Add(AsyncTaskData taskData) @@ -745,7 +745,7 @@ namespace ASC.Web.Files.Utils if (services.TryAddScoped()) { return services - .AddDistributedTaskQueueService("FileMarkerHelper", 1) + .AddDistributedTaskQueueService(1) .AddTenantManagerService() .AddUserManagerService() .AddDaoFactoryService() diff --git a/products/ASC.People/Server/Startup.cs b/products/ASC.People/Server/Startup.cs index 477f38c050..41d3a303a8 100644 --- a/products/ASC.People/Server/Startup.cs +++ b/products/ASC.People/Server/Startup.cs @@ -1,10 +1,6 @@  -using System; - using ASC.Api.Core; using ASC.Common; -using ASC.Common.Threading.Workers; -using ASC.Data.Reassigns; using ASC.Employee.Core.Controllers; using Microsoft.Extensions.Configuration; @@ -24,10 +20,7 @@ namespace ASC.People public override void ConfigureServices(IServiceCollection services) { - var diHelper = new DIHelper(services); - - diHelper.AddProgressQueue(1, (int)TimeSpan.FromMinutes(5).TotalMilliseconds, true, false, 0); - diHelper.AddProgressQueue(1, (int)TimeSpan.FromMinutes(5).TotalMilliseconds, true, false, 0); + var diHelper = new DIHelper(services); diHelper .AddPeopleController() diff --git a/web/ASC.Web.Core/Users/UserPhotoManager.cs b/web/ASC.Web.Core/Users/UserPhotoManager.cs index f9c2e152e6..0fbba32033 100644 --- a/web/ASC.Web.Core/Users/UserPhotoManager.cs +++ b/web/ASC.Web.Core/Users/UserPhotoManager.cs @@ -219,7 +219,7 @@ namespace ASC.Web.Core.Users SettingsManager settingsManager, IServiceProvider serviceProvider) { - ResizeQueue = optionsQueue.Get("ResizeWorker"); + ResizeQueue = optionsQueue.Get(nameof(ResizeWorkerItem)); UserManager = userManager; WebImageSupplier = webImageSupplier; TenantManager = tenantManager; @@ -1012,7 +1012,7 @@ namespace ASC.Web.Core.Users .AddWebImageSupplierService() .AddUserManagerService() .AddTenantManagerService() - .AddDistributedTaskQueueService("ResizeWorker", 2); + .AddDistributedTaskQueueService(2); } return services; From 3741f3a5d1991784e53bef5d9c9b8d09478f1843 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Wed, 30 Sep 2020 15:23:35 +0300 Subject: [PATCH 010/278] Encryption: DistributedQueue --- common/ASC.Data.Reassigns/QueueWorker.cs | 1 - .../ReassignProgressItem.cs | 1 - .../ASC.Data.Reassigns/RemoveProgressItem.cs | 1 - common/services/ASC.Data.Backup/Startup.cs | 31 +++++++++-- .../EncryptionOperation.cs | 49 +++++++++++------ .../EncryptionWorker.cs | 52 +++++++------------ 6 files changed, 78 insertions(+), 57 deletions(-) diff --git a/common/ASC.Data.Reassigns/QueueWorker.cs b/common/ASC.Data.Reassigns/QueueWorker.cs index 5218ebdc83..8b3e813229 100644 --- a/common/ASC.Data.Reassigns/QueueWorker.cs +++ b/common/ASC.Data.Reassigns/QueueWorker.cs @@ -29,7 +29,6 @@ using System.Collections.Generic; using ASC.Common; using ASC.Common.Threading; -using ASC.Common.Threading.Progress; using ASC.Core.Users; using Microsoft.AspNetCore.Http; diff --git a/common/ASC.Data.Reassigns/ReassignProgressItem.cs b/common/ASC.Data.Reassigns/ReassignProgressItem.cs index 6b526eb8f9..021c4a2902 100644 --- a/common/ASC.Data.Reassigns/ReassignProgressItem.cs +++ b/common/ASC.Data.Reassigns/ReassignProgressItem.cs @@ -31,7 +31,6 @@ using ASC.Common; using ASC.Common.Logging; using ASC.Common.Threading; //using System.Web; -using ASC.Common.Threading.Progress; using ASC.Core; using ASC.Core.Users; using ASC.MessagingSystem; diff --git a/common/ASC.Data.Reassigns/RemoveProgressItem.cs b/common/ASC.Data.Reassigns/RemoveProgressItem.cs index 8329c2645a..0942690b95 100644 --- a/common/ASC.Data.Reassigns/RemoveProgressItem.cs +++ b/common/ASC.Data.Reassigns/RemoveProgressItem.cs @@ -33,7 +33,6 @@ using System.Text; using ASC.Common; using ASC.Common.Logging; using ASC.Common.Threading; -using ASC.Common.Threading.Progress; using ASC.Core; using ASC.Core.Users; using ASC.Data.Storage; diff --git a/common/services/ASC.Data.Backup/Startup.cs b/common/services/ASC.Data.Backup/Startup.cs index eabad8b5b4..adb247d781 100644 --- a/common/services/ASC.Data.Backup/Startup.cs +++ b/common/services/ASC.Data.Backup/Startup.cs @@ -1,8 +1,31 @@ -using System; - +/* + * + * (c) Copyright Ascensio System Limited 2010-2018 + * + * This program is freeware. You can redistribute it and/or modify it under the terms of the GNU + * General Public License (GPL) version 3 as published by the Free Software Foundation (https://www.gnu.org/copyleft/gpl.html). + * In accordance with Section 7(a) of the GNU GPL its Section 15 shall be amended to the effect that + * Ascensio System SIA expressly excludes the warranty of non-infringement of any third-party rights. + * + * THIS PROGRAM IS DISTRIBUTED WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR + * FITNESS FOR A PARTICULAR PURPOSE. For more details, see GNU GPL at https://www.gnu.org/copyleft/gpl.html + * + * You can contact Ascensio System SIA by email at sales@onlyoffice.com + * + * The interactive user interfaces in modified source and object code versions of ONLYOFFICE must display + * Appropriate Legal Notices, as required under Section 5 of the GNU GPL version 3. + * + * Pursuant to Section 7 § 3(b) of the GNU GPL you must retain the original ONLYOFFICE logo which contains + * relevant author attributions when distributing the software. If the display of the logo in its graphic + * form is not reasonably feasible for technical reasons, you must include the words "Powered by ONLYOFFICE" + * in every copy of the program you distribute. + * Pursuant to Section 7 § 3(e) we decline to grant you any rights under trademark law for use of our trademarks. + * +*/ + using ASC.Api.Core; using ASC.Common; -using ASC.Common.Threading.Workers; +using ASC.Common.Threading; using ASC.Data.Backup.Controllers; using ASC.Data.Backup.Service; @@ -28,7 +51,7 @@ namespace ASC.Data.Backup diHelper .AddBackupServiceLauncher() .AddBackupController() - .AddProgressQueue(1, (int)TimeSpan.FromMinutes(5).TotalMilliseconds, true, false, 0); + .AddDistributedTaskQueueService(1); services.AddHostedService(); base.ConfigureServices(services); diff --git a/common/services/ASC.Data.Storage.Encryption/EncryptionOperation.cs b/common/services/ASC.Data.Storage.Encryption/EncryptionOperation.cs index 79ddd9213c..989967754a 100644 --- a/common/services/ASC.Data.Storage.Encryption/EncryptionOperation.cs +++ b/common/services/ASC.Data.Storage.Encryption/EncryptionOperation.cs @@ -32,7 +32,7 @@ using System.Threading.Tasks; using ASC.Common; using ASC.Common.Caching; using ASC.Common.Logging; -using ASC.Common.Threading.Progress; +using ASC.Common.Threading; using ASC.Core; using ASC.Core.Tenants; using ASC.Data.Storage.DiscStorage; @@ -43,7 +43,7 @@ using Microsoft.Extensions.Options; namespace ASC.Data.Storage.Encryption { - public class EncryptionOperation : ProgressBase + public class EncryptionOperation : DistributedTaskProgress { private const string ConfigPath = ""; private bool HasErrors = false; @@ -61,8 +61,9 @@ namespace ASC.Data.Storage.Encryption ServiceProvider = serviceProvider; } - public void Init(EncryptionSettingsProto encryptionSettingsProto) - { + public void Init(EncryptionSettingsProto encryptionSettingsProto, string id) + { + Id = id; EncryptionSettings = new EncryptionSettings(encryptionSettingsProto); IsEncryption = EncryptionSettings.Status == EncryprtionStatus.EncryptionStarted; ServerRootPath = encryptionSettingsProto.ServerRootPath; @@ -79,7 +80,9 @@ namespace ASC.Data.Storage.Encryption UseProgressFile = Convert.ToBoolean(configuration["storage:encryption:progressfile"] ?? "true"); Percentage = 10; - GetProgress(progressEncryption); + PublishProgress(progressEncryption); + PublishChanges(); + try { if (!coreBaseSettings.Standalone) @@ -91,12 +94,15 @@ namespace ASC.Data.Storage.Encryption { log.Debug("Storage already " + EncryptionSettings.Status); return; - } + } + Percentage = 30; - GetProgress(progressEncryption); + PublishProgress(progressEncryption); + PublishChanges(); + foreach (var tenant in Tenants) { - Dictionary dictionary = new Dictionary(); + var dictionary = new Dictionary(); foreach (var module in Modules) { dictionary.Add(module, (DiscDataStore)storageFactory.GetStorage(ConfigPath, tenant.TenantId.ToString(), module)); @@ -105,23 +111,31 @@ namespace ASC.Data.Storage.Encryption { EncryptStore(tenant, elem.Key, elem.Value, storageFactoryConfig, log); }); - } + } + Percentage = 70; - GetProgress(progressEncryption); + PublishProgress(progressEncryption); + PublishChanges(); + if (!HasErrors) { DeleteProgressFiles(storageFactory); SaveNewSettings(encryptionSettingsHelper, log); - } + } + Percentage = 90; - GetProgress(progressEncryption); - ActivateTenants(tenantManager, log, notifyHelper); + PublishProgress(progressEncryption); + PublishChanges(); + + ActivateTenants(tenantManager, log, notifyHelper); + Percentage = 100; - GetProgress(progressEncryption); + PublishProgress(progressEncryption); + PublishChanges(); } catch (Exception e) { - Error = e; + Exception = e; log.Error(e); } } @@ -181,12 +195,13 @@ namespace ASC.Data.Storage.Encryption return encryptedFiles; } - public void GetProgress(ICacheNotify progress) + public void PublishProgress(ICacheNotify progress) { var progressEncryption = new ProgressEncryption() { Progress = Percentage - }; + }; + progress.Publish(progressEncryption, CacheNotifyAction.Insert); } diff --git a/common/services/ASC.Data.Storage.Encryption/EncryptionWorker.cs b/common/services/ASC.Data.Storage.Encryption/EncryptionWorker.cs index 5c69bd1229..f3fc36f341 100644 --- a/common/services/ASC.Data.Storage.Encryption/EncryptionWorker.cs +++ b/common/services/ASC.Data.Storage.Encryption/EncryptionWorker.cs @@ -25,28 +25,26 @@ using System; -using System.Threading; -using System.Threading.Tasks; using ASC.Common; -using ASC.Common.Caching; - +using ASC.Common.Threading; + using Microsoft.Extensions.DependencyInjection; namespace ASC.Data.Storage.Encryption { public class EncryptionWorker { - private CancellationTokenSource TokenSource { get; set; } - private ICache Cache { get; } private object Locker { get; } - private FactoryOperation FactoryOperation { get; } + private FactoryOperation FactoryOperation { get; } + + private DistributedTaskQueue Queue { get; } - public EncryptionWorker(FactoryOperation factoryOperation) + public EncryptionWorker(FactoryOperation factoryOperation, DistributedTaskQueueOptionsManager options) { - Cache = AscCache.Memory; Locker = new object(); - FactoryOperation = factoryOperation; + FactoryOperation = factoryOperation; + Queue = options.Get(nameof(EncryptionWorker)); } public void Start(EncryptionSettingsProto encryptionSettings) @@ -54,30 +52,15 @@ namespace ASC.Data.Storage.Encryption EncryptionOperation encryptionOperation; lock (Locker) { - if (Cache.Get(GetCacheKey()) != null) return; - TokenSource = new CancellationTokenSource(); - encryptionOperation = FactoryOperation.CreateOperation(encryptionSettings); - Cache.Insert(GetCacheKey(), encryptionOperation, DateTime.MaxValue); + if (Queue.GetTask(GetCacheKey()) != null) return; + encryptionOperation = FactoryOperation.CreateOperation(encryptionSettings, GetCacheKey()); + Queue.QueueTask((a, b) => encryptionOperation.RunJob(), encryptionOperation); } - - var task = new Task(encryptionOperation.RunJob, TokenSource.Token, TaskCreationOptions.LongRunning); - - task.ConfigureAwait(false) - .GetAwaiter() - .OnCompleted(() => - { - lock (Locker) - { - Cache.Remove(GetCacheKey()); - } - }); - - task.Start(); } public void Stop() { - TokenSource.Cancel(); + Queue.CancelTask(GetCacheKey()); } public string GetCacheKey() @@ -95,10 +78,10 @@ namespace ASC.Data.Storage.Encryption ServiceProvider = serviceProvider; } - public EncryptionOperation CreateOperation(EncryptionSettingsProto encryptionSettings) + public EncryptionOperation CreateOperation(EncryptionSettingsProto encryptionSettings, string id) { var item = ServiceProvider.GetService(); - item.Init(encryptionSettings); + item.Init(encryptionSettings, id); return item; } } @@ -106,10 +89,13 @@ namespace ASC.Data.Storage.Encryption public static class EncryptionWorkerExtension { public static DIHelper AddEncryptionWorkerService(this DIHelper services) - { + { + services.TryAddTransient(); services.TryAddSingleton(); services.TryAddSingleton(); - return services.AddEncryptionOperationService(); + return services + .AddEncryptionOperationService() + .AddDistributedTaskQueueService(1); } } } From 88d7a0d2675833340fba0b4ec45e252259c665cd Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Thu, 1 Oct 2020 10:22:08 +0300 Subject: [PATCH 011/278] DistributedTaskQueue: fix --- common/ASC.Common/Threading/DistributedTaskQueue.cs | 2 +- common/ASC.Data.Reassigns/QueueWorker.cs | 7 ++++--- common/ASC.Data.Reassigns/ReassignProgressItem.cs | 2 +- common/ASC.Data.Reassigns/RemoveProgressItem.cs | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/common/ASC.Common/Threading/DistributedTaskQueue.cs b/common/ASC.Common/Threading/DistributedTaskQueue.cs index 6bcc787949..9d1147b79a 100644 --- a/common/ASC.Common/Threading/DistributedTaskQueue.cs +++ b/common/ASC.Common/Threading/DistributedTaskQueue.cs @@ -305,7 +305,7 @@ namespace ASC.Common.Threading services.TryAddSingleton(); services .TryAddSingleton() - .TryAddSingleton() //check + .TryAddSingleton() .AddSingleton, ConfigureDistributedTaskQueue>(); _ = services.Configure(typeof(T).Name, r => diff --git a/common/ASC.Data.Reassigns/QueueWorker.cs b/common/ASC.Data.Reassigns/QueueWorker.cs index 8b3e813229..10a512127c 100644 --- a/common/ASC.Data.Reassigns/QueueWorker.cs +++ b/common/ASC.Data.Reassigns/QueueWorker.cs @@ -44,6 +44,7 @@ namespace ASC.Data.Reassigns return httpRequest?.Headers; } } + public class QueueWorker where T : DistributedTaskProgress { protected readonly DistributedTaskQueue Queue; @@ -63,14 +64,14 @@ namespace ASC.Data.Reassigns Queue = options.Get(typeof(T).Name); } - public static string GetProgressItemId(int tenantId, Guid userId, Type type) + public static string GetProgressItemId(int tenantId, Guid userId) { - return string.Format("{0}_{1}_{2}", tenantId, userId, type.Name); + return string.Format("{0}_{1}_{2}", tenantId, userId, typeof(T).Name); } public T GetProgressItemStatus(int tenantId, Guid userId) { - var id = GetProgressItemId(tenantId, userId, typeof(T)); + var id = GetProgressItemId(tenantId, userId); return Queue.GetTask(id); } diff --git a/common/ASC.Data.Reassigns/ReassignProgressItem.cs b/common/ASC.Data.Reassigns/ReassignProgressItem.cs index 021c4a2902..8c7657bdce 100644 --- a/common/ASC.Data.Reassigns/ReassignProgressItem.cs +++ b/common/ASC.Data.Reassigns/ReassignProgressItem.cs @@ -88,7 +88,7 @@ namespace ASC.Data.Reassigns //_docService = Web.Files.Classes.Global.FileStorageService; //_projectsReassign = new ProjectsReassign(); - Id = QueueWorkerReassign.GetProgressItemId(tenantId, fromUserId, typeof(ReassignProgressItem)); + Id = QueueWorkerReassign.GetProgressItemId(tenantId, fromUserId); Status = DistributedTaskStatus.Created; Exception = null; Percentage = 0; diff --git a/common/ASC.Data.Reassigns/RemoveProgressItem.cs b/common/ASC.Data.Reassigns/RemoveProgressItem.cs index 0942690b95..1b3a5f3c59 100644 --- a/common/ASC.Data.Reassigns/RemoveProgressItem.cs +++ b/common/ASC.Data.Reassigns/RemoveProgressItem.cs @@ -84,7 +84,7 @@ namespace ASC.Data.Reassigns _currentUserId = currentUserId; _notify = notify; - Id = QueueWorkerRemove.GetProgressItemId(tenantId, FromUser, typeof(RemoveProgressItem)); + Id = QueueWorkerRemove.GetProgressItemId(tenantId, FromUser); Status = DistributedTaskStatus.Created; Exception = null; Percentage = 0; From 3f5b8b77c427f9e211996563a831e0db299a79ad Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Fri, 2 Oct 2020 10:19:02 +0300 Subject: [PATCH 012/278] DistributedTaskQueue: refactoring --- .../Threading/DistributedTaskProgress.cs | 5 + .../Threading/DistributedTaskQueue.cs | 5 + common/ASC.Data.Reassigns/QueueWorker.cs | 2 +- common/ASC.Data.Storage/StaticUploader.cs | 51 +++------- common/ASC.Data.Storage/StorageUploader.cs | 30 ++---- .../ASC.Data.Backup/Service/BackupWorker.cs | 99 +++++-------------- .../EncryptionWorker.cs | 2 +- 7 files changed, 58 insertions(+), 136 deletions(-) diff --git a/common/ASC.Common/Threading/DistributedTaskProgress.cs b/common/ASC.Common/Threading/DistributedTaskProgress.cs index e250764124..8d6378146e 100644 --- a/common/ASC.Common/Threading/DistributedTaskProgress.cs +++ b/common/ASC.Common/Threading/DistributedTaskProgress.cs @@ -46,6 +46,8 @@ namespace ASC.Common.Threading { Percentage += 100.0 / StepCount; } + + PublishChanges(); } public void RunJob() @@ -53,16 +55,19 @@ namespace ASC.Common.Threading try { Percentage = 0; + Status = DistributedTaskStatus.Running; DoJob(); } catch (AggregateException e) { + Status = DistributedTaskStatus.Failted; Exception = e; } finally { Percentage = 100; IsCompleted = true; + Status = DistributedTaskStatus.Completed; PublishChanges(); } } diff --git a/common/ASC.Common/Threading/DistributedTaskQueue.cs b/common/ASC.Common/Threading/DistributedTaskQueue.cs index 9d1147b79a..9b5cb8289b 100644 --- a/common/ASC.Common/Threading/DistributedTaskQueue.cs +++ b/common/ASC.Common/Threading/DistributedTaskQueue.cs @@ -152,6 +152,11 @@ namespace ASC.Common.Threading } + public void QueueTask(DistributedTaskProgress taskProgress) + { + QueueTask(() => taskProgress.RunJob(), taskProgress); + } + public void QueueTask(Action action, DistributedTask distributedTask = null) { if (distributedTask == null) diff --git a/common/ASC.Data.Reassigns/QueueWorker.cs b/common/ASC.Data.Reassigns/QueueWorker.cs index 10a512127c..df8f240e09 100644 --- a/common/ASC.Data.Reassigns/QueueWorker.cs +++ b/common/ASC.Data.Reassigns/QueueWorker.cs @@ -100,7 +100,7 @@ namespace ASC.Data.Reassigns if (task == null) { task = constructor(); - Queue.QueueTask((a, b) => task.RunJob(), task); + Queue.QueueTask(task); } return task; diff --git a/common/ASC.Data.Storage/StaticUploader.cs b/common/ASC.Data.Storage/StaticUploader.cs index 8b646444ff..6a84e97819 100644 --- a/common/ASC.Data.Storage/StaticUploader.cs +++ b/common/ASC.Data.Storage/StaticUploader.cs @@ -117,7 +117,7 @@ namespace ASC.Data.Storage { using var scope = ServiceProvider.CreateScope(); var scopeClass = scope.ServiceProvider.GetService(); - var(tenantManager, staticUploader, _, _, _) = scopeClass; + var (tenantManager, staticUploader, _, _, _) = scopeClass; tenantManager.SetCurrentTenant(tenantId); return staticUploader.UploadFile(relativePath, mappedPath, onComplete); }, TaskCreationOptions.LongRunning); @@ -143,8 +143,8 @@ namespace ASC.Data.Storage uploadOperation = Queue.GetTask(key); if (uploadOperation != null) return; - uploadOperation = new UploadOperationProgress(this, key, tenant.TenantId, relativePath, mappedPath); - Queue.QueueTask((a, b) => uploadOperation.RunJob(), uploadOperation); + uploadOperation = new UploadOperationProgress(key, tenant.TenantId, relativePath, mappedPath); + Queue.QueueTask(uploadOperation); } } @@ -164,12 +164,12 @@ namespace ASC.Data.Storage TokenSource.Cancel(); } - public static UploadOperationProgress GetProgress(int tenantId) + public UploadOperationProgress GetProgress(int tenantId) { lock (Locker) { var key = typeof(UploadOperationProgress).FullName + tenantId; - return Cache.Get(key); + return Queue.GetTask(key); } } @@ -232,28 +232,20 @@ namespace ASC.Data.Storage } } - public class UploadOperationProgress : DistributedTask, IProgressItem + public class UploadOperationProgress : DistributedTaskProgress { private readonly string relativePath; private readonly string mappedPath; - private readonly IEnumerable directoryFiles; - private readonly int StepCount; + private readonly IEnumerable directoryFiles; private IServiceProvider ServiceProvider { get; } - private StaticUploader StaticUploader { get; } public int TenantId { get; } - public object Error { get; set; } - public double Percentage { get; set; } - - public bool IsCompleted { get; set; } - - public UploadOperationProgress(StaticUploader staticUploader, string key, int tenantId, string relativePath, string mappedPath) + public UploadOperationProgress(string key, int tenantId, string relativePath, string mappedPath) { Id = key; Status = DistributedTaskStatus.Created; - - StaticUploader = staticUploader; + TenantId = tenantId; this.relativePath = relativePath; this.mappedPath = mappedPath; @@ -268,45 +260,28 @@ namespace ASC.Data.Storage StepCount = directoryFiles.Count(); } - protected async Task RunJobAsync() - { - var tasks = new List(); - foreach (var file in directoryFiles) - { - var filePath = file.Substring(mappedPath.TrimEnd('/').Length); - tasks.Add(StaticUploader.UploadFileAsync(Path.Combine(relativePath, filePath), file, (res) => StepDone())); - } - - await Task.WhenAll(tasks); - } - - public void RunJob() + protected override void DoJob() { using var scope = ServiceProvider.CreateScope(); var tenantManager = scope.ServiceProvider.GetService(); + var staticUploader = scope.ServiceProvider.GetService(); var tenant = tenantManager.GetTenant(TenantId); tenantManager.SetCurrentTenant(tenant); tenant.SetStatus(TenantStatus.Migrating); tenantManager.SaveTenant(tenant); + PublishChanges(); foreach (var file in directoryFiles) { var filePath = file.Substring(mappedPath.TrimEnd('/').Length); - StaticUploader.UploadFileAsync(Path.Combine(relativePath, filePath), file, (res) => StepDone()).Wait(); + staticUploader.UploadFile(Path.Combine(relativePath, filePath), file, (res) => StepDone()); } tenant.SetStatus(Core.Tenants.TenantStatus.Active); tenantManager.SaveTenant(tenant); } - protected void StepDone() - { - if (StepCount > 0) - { - Percentage += 100.0 / StepCount; - } - } public object Clone() { return MemberwiseClone(); diff --git a/common/ASC.Data.Storage/StorageUploader.cs b/common/ASC.Data.Storage/StorageUploader.cs index 8d55bb7d88..6b4478d9cf 100644 --- a/common/ASC.Data.Storage/StorageUploader.cs +++ b/common/ASC.Data.Storage/StorageUploader.cs @@ -73,7 +73,7 @@ namespace ASC.Data.Storage if (migrateOperation != null) return; migrateOperation = new MigrateOperation(ServiceProvider, CacheMigrationNotify, id, tenantId, newStorageSettings, storageFactoryConfig); - Queue.QueueTask((a, b) => migrateOperation.RunJob(), migrateOperation); + Queue.QueueTask(migrateOperation); } } @@ -99,14 +99,13 @@ namespace ASC.Data.Storage } } - public class MigrateOperation : DistributedTask, IProgressItem + public class MigrateOperation : DistributedTaskProgress { private readonly ILog Log; private static readonly string ConfigPath; private readonly IEnumerable Modules; private readonly StorageSettings settings; - private readonly int tenantId; - private readonly int StepCount; + private readonly int tenantId; static MigrateOperation() { @@ -130,15 +129,9 @@ namespace ASC.Data.Storage private IServiceProvider ServiceProvider { get; } private StorageFactoryConfig StorageFactoryConfig { get; } - private ICacheNotify CacheMigrationNotify { get; } + private ICacheNotify CacheMigrationNotify { get; } - public object Error { get; set; } - - public double Percentage { get; set; } - - public bool IsCompleted { get; set; } - - public void RunJob() + protected override void DoJob() { try { @@ -195,12 +188,13 @@ namespace ASC.Data.Storage settingsManager.Save(settings); tenant.SetStatus(TenantStatus.Active); tenantManager.SaveTenant(tenant); + Status = DistributedTaskStatus.Completed; } catch (Exception e) { Status = DistributedTaskStatus.Failted; - Error = e; + Exception = e; Log.Error(e); } @@ -218,18 +212,10 @@ namespace ASC.Data.Storage { TenantId = tenantId, Progress = Percentage, - Error = Error.ToString(), + Error = Exception.ToString(), IsCompleted = IsCompleted }, CacheNotifyAction.Insert); - } - - protected void StepDone() - { - if (StepCount > 0) - { - Percentage += 100.0 / StepCount; - } } } diff --git a/common/services/ASC.Data.Backup/Service/BackupWorker.cs b/common/services/ASC.Data.Backup/Service/BackupWorker.cs index 2630f2b59d..e9fe9a5644 100644 --- a/common/services/ASC.Data.Backup/Service/BackupWorker.cs +++ b/common/services/ASC.Data.Backup/Service/BackupWorker.cs @@ -127,10 +127,10 @@ namespace ASC.Data.Backup.Service if (item == null) { item = FactoryProgressItem.CreateBackupProgressItem(request, false, TempFolder, Limit, CurrentRegion, ConfigPaths); - ProgressQueue.QueueTask((a, b) => item.RunJob(), item); + ProgressQueue.QueueTask(item); } - PublishProgress(item); + item.PublishChanges(); return ToBackupProgress(item); } @@ -149,7 +149,7 @@ namespace ASC.Data.Backup.Service if (item == null) { item = FactoryProgressItem.CreateBackupProgressItem(schedule, false, TempFolder, Limit, CurrentRegion, ConfigPaths); - ProgressQueue.QueueTask((a, b) => item.RunJob(), item); + ProgressQueue.QueueTask(item); } } } @@ -185,7 +185,7 @@ namespace ASC.Data.Backup.Service var progress = ProgressQueue.GetTasks().FirstOrDefault(t => t.TenantId == tenantId); if (progress != null) { - progress.Error = null; + progress.Exception = null; } } } @@ -197,7 +197,7 @@ namespace ASC.Data.Backup.Service var progress = ProgressQueue.GetTasks().FirstOrDefault(t => t.TenantId == tenantId); if (progress != null) { - progress.Error = null; + progress.Exception = null; } } } @@ -215,7 +215,7 @@ namespace ASC.Data.Backup.Service if (item == null) { item = FactoryProgressItem.CreateRestoreProgressItem(request, TempFolder, UpgradesPath, CurrentRegion, ConfigPaths); - ProgressQueue.QueueTask((a, b) => item.RunJob(), item); + ProgressQueue.QueueTask(item); } return ToBackupProgress(item); } @@ -235,7 +235,7 @@ namespace ASC.Data.Backup.Service if (item == null) { item = FactoryProgressItem.CreateTransferProgressItem(targetRegion, transferMail, tenantId, TempFolder, Limit, notify, CurrentRegion, ConfigPaths); - ProgressQueue.QueueTask((a, b) => item.RunJob(), item); + ProgressQueue.QueueTask(item); } return ToBackupProgress(item); @@ -253,7 +253,7 @@ namespace ASC.Data.Backup.Service { IsCompleted = progressItem.IsCompleted, Progress = (int)progressItem.Percentage, - Error = progressItem.Error != null ? ((Exception)progressItem.Error).Message : "", + Error = progressItem.Exception != null ? ((Exception)progressItem.Exception).Message : "", TenantId = progressItem.TenantId, BackupProgressEnum = progressItem.BackupProgressItemEnum.Convert() }; @@ -273,11 +273,6 @@ namespace ASC.Data.Backup.Service return progress; } - - internal void PublishProgress(BaseBackupProgressItem progress) - { - progress.PublishChanges(); - } } public enum BackupProgressItemEnum @@ -299,50 +294,8 @@ namespace ASC.Data.Backup.Service }; } - public abstract class BaseBackupProgressItem : DistributedTask, IProgressItem + public abstract class BaseBackupProgressItem : DistributedTaskProgress { - public object error; - public object Error - { - get - { - return error ?? GetProperty(nameof(error)); - } - set - { - error = value; - SetProperty(nameof(error), value); - } - } - - public double? percentage; - public double Percentage - { - get - { - return percentage ?? GetProperty(nameof(percentage)); - } - set - { - percentage = value; - SetProperty(nameof(percentage), value); - } - } - - public bool? isCompleted; - public bool IsCompleted - { - get - { - return isCompleted ?? GetProperty(nameof(isCompleted)); - } - set - { - isCompleted = value; - SetProperty(nameof(isCompleted), value); - } - } - private int? tenantId; public int TenantId { @@ -360,8 +313,6 @@ namespace ASC.Data.Backup.Service public abstract BackupProgressItemEnum BackupProgressItemEnum { get; } public abstract object Clone(); - - public abstract void RunJob(); } public class BackupProgressItem : BaseBackupProgressItem @@ -420,7 +371,7 @@ namespace ASC.Data.Backup.Service ConfigPaths = configPaths; } - public override void RunJob() + protected override void DoJob() { if (ThreadPriority.BelowNormal < Thread.CurrentThread.Priority) { @@ -450,7 +401,7 @@ namespace ASC.Data.Backup.Service backupTask.ProgressChanged += (sender, args) => { Percentage = 0.9 * args.Progress; - backupWorker.PublishProgress(this); + PublishChanges(); }; backupTask.RunJob(); @@ -486,19 +437,19 @@ namespace ASC.Data.Backup.Service } IsCompleted = true; - backupWorker.PublishProgress(this); + PublishChanges(); } catch (Exception error) { Log.ErrorFormat("RunJob - Params: {0}, Error = {1}", new { Id, Tenant = TenantId, File = tempFile, BasePath = StorageBasePath, }, error); - Error = error; + Exception = error; IsCompleted = true; } finally { try { - backupWorker.PublishProgress(this); + PublishChanges(); } catch (Exception error) { @@ -558,7 +509,7 @@ namespace ASC.Data.Backup.Service ConfigPaths = configPaths; } - public override void RunJob() + protected override void DoJob() { using var scope = ServiceProvider.CreateScope(); var scopeClass = scope.ServiceProvider.GetService(); @@ -586,7 +537,7 @@ namespace ASC.Data.Backup.Service restoreTask.ProgressChanged += (sender, args) => { Percentage = Percentage = (10d + 0.65 * args.Progress); - backupWorker.PublishProgress(this); + PublishChanges(); }; restoreTask.RunJob(); @@ -626,17 +577,17 @@ namespace ASC.Data.Backup.Service Percentage = 75; - backupWorker.PublishProgress(this); + PublishChanges(); File.Delete(tempFile); Percentage = 100; - backupWorker.PublishProgress(this); + PublishChanges(); } catch (Exception error) { Log.Error(error); - Error = error; + Exception = error; if (tenant != null) { @@ -648,7 +599,7 @@ namespace ASC.Data.Backup.Service { try { - backupWorker.PublishProgress(this); + PublishChanges(); } catch (Exception error) { @@ -716,7 +667,7 @@ namespace ASC.Data.Backup.Service } - public override void RunJob() + protected override void DoJob() { using var scope = ServiceProvider.CreateScope(); var scopeClass = scope.ServiceProvider.GetService(); @@ -733,7 +684,7 @@ namespace ASC.Data.Backup.Service transferProgressItem.ProgressChanged += (sender, args) => { Percentage = args.Progress; - backupWorker.PublishProgress(this); + PublishChanges(); }; if (!TransferMail) { @@ -743,12 +694,12 @@ namespace ASC.Data.Backup.Service Link = GetLink(alias, false); notifyHelper.SendAboutTransferComplete(tenant, TargetRegion, Link, !Notify); - backupWorker.PublishProgress(this); + PublishChanges(); } catch (Exception error) { Log.Error(error); - Error = error; + Exception = error; Link = GetLink(alias, true); notifyHelper.SendAboutTransferError(tenant, TargetRegion, Link, !Notify); @@ -757,7 +708,7 @@ namespace ASC.Data.Backup.Service { try { - backupWorker.PublishProgress(this); + PublishChanges(); } catch (Exception error) { diff --git a/common/services/ASC.Data.Storage.Encryption/EncryptionWorker.cs b/common/services/ASC.Data.Storage.Encryption/EncryptionWorker.cs index f3fc36f341..43603d74d3 100644 --- a/common/services/ASC.Data.Storage.Encryption/EncryptionWorker.cs +++ b/common/services/ASC.Data.Storage.Encryption/EncryptionWorker.cs @@ -54,7 +54,7 @@ namespace ASC.Data.Storage.Encryption { if (Queue.GetTask(GetCacheKey()) != null) return; encryptionOperation = FactoryOperation.CreateOperation(encryptionSettings, GetCacheKey()); - Queue.QueueTask((a, b) => encryptionOperation.RunJob(), encryptionOperation); + Queue.QueueTask(encryptionOperation); } } From 38b204ca5a01475bb713db226748019f951d7570 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Fri, 2 Oct 2020 10:34:29 +0300 Subject: [PATCH 013/278] DistributedTaskQueue: refactoring --- common/ASC.Common/Threading/DistributedTaskQueue.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/ASC.Common/Threading/DistributedTaskQueue.cs b/common/ASC.Common/Threading/DistributedTaskQueue.cs index 9b5cb8289b..ba30ee0a51 100644 --- a/common/ASC.Common/Threading/DistributedTaskQueue.cs +++ b/common/ASC.Common/Threading/DistributedTaskQueue.cs @@ -154,7 +154,7 @@ namespace ASC.Common.Threading public void QueueTask(DistributedTaskProgress taskProgress) { - QueueTask(() => taskProgress.RunJob(), taskProgress); + QueueTask((a, b) => taskProgress.RunJob(), taskProgress); } public void QueueTask(Action action, DistributedTask distributedTask = null) From 2e4f612f935e22d024473b0fa8675d18ab6a78b0 Mon Sep 17 00:00:00 2001 From: SuhorukovAnton <62381554+SuhorukovAnton@users.noreply.github.com> Date: Tue, 13 Oct 2020 17:28:10 +0300 Subject: [PATCH 014/278] remove-wq: fix encryption --- .../Threading/DistributedTaskProgress.cs | 21 ++------- .../Threading/DistributedTaskQueue.cs | 3 +- .../Encryption}/EncryptionOperation.cs | 46 +++++++++---------- .../Encryption/EncryptionServiceNotifier.cs | 41 ----------------- .../Encryption}/EncryptionWorker.cs | 34 +++++++++----- .../Encryption}/NotifyHelper.cs | 0 .../Controllers/SettingsController.cs | 23 ++-------- 7 files changed, 55 insertions(+), 113 deletions(-) rename common/{services/ASC.Data.Storage.Encryption => ASC.Data.Storage/Encryption}/EncryptionOperation.cs (99%) delete mode 100644 common/ASC.Data.Storage/Encryption/EncryptionServiceNotifier.cs rename common/{services/ASC.Data.Storage.Encryption => ASC.Data.Storage/Encryption}/EncryptionWorker.cs (87%) rename common/{services/ASC.Data.Storage.Encryption => ASC.Data.Storage/Encryption}/NotifyHelper.cs (100%) diff --git a/common/ASC.Common/Threading/DistributedTaskProgress.cs b/common/ASC.Common/Threading/DistributedTaskProgress.cs index 8d6378146e..eeace99d40 100644 --- a/common/ASC.Common/Threading/DistributedTaskProgress.cs +++ b/common/ASC.Common/Threading/DistributedTaskProgress.cs @@ -52,24 +52,9 @@ namespace ASC.Common.Threading public void RunJob() { - try - { - Percentage = 0; - Status = DistributedTaskStatus.Running; - DoJob(); - } - catch (AggregateException e) - { - Status = DistributedTaskStatus.Failted; - Exception = e; - } - finally - { - Percentage = 100; - IsCompleted = true; - Status = DistributedTaskStatus.Completed; - PublishChanges(); - } + Percentage = 0; + Status = DistributedTaskStatus.Running; + DoJob(); } protected virtual void DoJob() diff --git a/common/ASC.Common/Threading/DistributedTaskQueue.cs b/common/ASC.Common/Threading/DistributedTaskQueue.cs index ba30ee0a51..5a89f05d1b 100644 --- a/common/ASC.Common/Threading/DistributedTaskQueue.cs +++ b/common/ASC.Common/Threading/DistributedTaskQueue.cs @@ -130,7 +130,7 @@ namespace ASC.Common.Threading private TaskScheduler Scheduler { get; set; } = TaskScheduler.Default; public IServiceProvider ServiceProvider { get; set; } - public string Name { set { key = value + GetType().Name; } } + public string Name { get { return key; } set { key = value + GetType().Name; } } private ICache Cache { get => DistributedTaskCacheNotify.Cache; } private ConcurrentDictionary Cancelations { get => DistributedTaskCacheNotify.Cancelations; } @@ -243,6 +243,7 @@ namespace ASC.Common.Threading if (cache != null) { var task = new DistributedTask(); + task.DistributedTaskCache = cache; if (task != null && task.Publication == null) { task.Publication = GetPublication(); diff --git a/common/services/ASC.Data.Storage.Encryption/EncryptionOperation.cs b/common/ASC.Data.Storage/Encryption/EncryptionOperation.cs similarity index 99% rename from common/services/ASC.Data.Storage.Encryption/EncryptionOperation.cs rename to common/ASC.Data.Storage/Encryption/EncryptionOperation.cs index 989967754a..fa1b37f7b3 100644 --- a/common/services/ASC.Data.Storage.Encryption/EncryptionOperation.cs +++ b/common/ASC.Data.Storage/Encryption/EncryptionOperation.cs @@ -28,15 +28,15 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; - + using ASC.Common; using ASC.Common.Caching; using ASC.Common.Logging; -using ASC.Common.Threading; +using ASC.Common.Threading; using ASC.Core; using ASC.Core.Tenants; using ASC.Data.Storage.DiscStorage; - + using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; @@ -62,7 +62,7 @@ namespace ASC.Data.Storage.Encryption } public void Init(EncryptionSettingsProto encryptionSettingsProto, string id) - { + { Id = id; EncryptionSettings = new EncryptionSettings(encryptionSettingsProto); IsEncryption = EncryptionSettings.Status == EncryprtionStatus.EncryptionStarted; @@ -80,8 +80,8 @@ namespace ASC.Data.Storage.Encryption UseProgressFile = Convert.ToBoolean(configuration["storage:encryption:progressfile"] ?? "true"); Percentage = 10; - PublishProgress(progressEncryption); - PublishChanges(); + PublishProgress(progressEncryption); + PublishChanges(); try { @@ -94,11 +94,11 @@ namespace ASC.Data.Storage.Encryption { log.Debug("Storage already " + EncryptionSettings.Status); return; - } - + } + Percentage = 30; - PublishProgress(progressEncryption); - PublishChanges(); + PublishProgress(progressEncryption); + PublishChanges(); foreach (var tenant in Tenants) { @@ -111,26 +111,26 @@ namespace ASC.Data.Storage.Encryption { EncryptStore(tenant, elem.Key, elem.Value, storageFactoryConfig, log); }); - } + } Percentage = 70; - PublishProgress(progressEncryption); - PublishChanges(); + PublishProgress(progressEncryption); + PublishChanges(); if (!HasErrors) { DeleteProgressFiles(storageFactory); SaveNewSettings(encryptionSettingsHelper, log); - } + } Percentage = 90; - PublishProgress(progressEncryption); - PublishChanges(); + PublishProgress(progressEncryption); + PublishChanges(); - ActivateTenants(tenantManager, log, notifyHelper); + ActivateTenants(tenantManager, log, notifyHelper); Percentage = 100; - PublishProgress(progressEncryption); + PublishProgress(progressEncryption); PublishChanges(); } catch (Exception e) @@ -198,9 +198,9 @@ namespace ASC.Data.Storage.Encryption public void PublishProgress(ICacheNotify progress) { var progressEncryption = new ProgressEncryption() - { + { Progress = Percentage - }; + }; progress.Publish(progressEncryption, CacheNotifyAction.Insert); } @@ -325,7 +325,7 @@ namespace ASC.Data.Storage.Encryption log.DebugFormat("Tenant {0} SetStatus Active", tenant.TenantAlias); if (!HasErrors) - { + { if (EncryptionSettings.NotifyUsers) { if (IsEncryption) @@ -356,7 +356,7 @@ namespace ASC.Data.Storage.Encryption } } - public class EncryptionOperationScope + public class EncryptionOperationScope { private ILog Log { get; set; } private EncryptionSettingsHelper EncryptionSettingsHelper { get; set; } @@ -399,7 +399,7 @@ namespace ASC.Data.Storage.Encryption storageFactoryConfig = StorageFactoryConfig; storageFactory = StorageFactory; progressEncryption = ProgressEncryption; - configuration = Configuration; + configuration = Configuration; } } diff --git a/common/ASC.Data.Storage/Encryption/EncryptionServiceNotifier.cs b/common/ASC.Data.Storage/Encryption/EncryptionServiceNotifier.cs deleted file mode 100644 index aa680b6699..0000000000 --- a/common/ASC.Data.Storage/Encryption/EncryptionServiceNotifier.cs +++ /dev/null @@ -1,41 +0,0 @@ -using System; - -using ASC.Common; -using ASC.Common.Caching; - -namespace ASC.Data.Storage.Encryption -{ - public class EncryptionServiceNotifier - { - private ICacheNotify СacheBackupProgress { get; } - private ICache Cache { get; } - - public EncryptionServiceNotifier(ICacheNotify сacheBackupProgress) - { - Cache = AscCache.Memory; - СacheBackupProgress = сacheBackupProgress; - - СacheBackupProgress.Subscribe((a) => - { - Cache.Insert(GetCacheKey(a.TenantId), a, DateTime.UtcNow.AddDays(1)); - }, - CacheNotifyAction.InsertOrUpdate); - } - - public ProgressEncryption GetEncryptionProgress(int tenantId) - { - return Cache.Get(GetCacheKey(tenantId)); - } - - private string GetCacheKey(int tenantId) => $"encryption{tenantId}"; - } - - public static class EncryptionServiceNotifierExtension - { - public static DIHelper AddEncryptionServiceNotifierService(this DIHelper services) - { - services.TryAddSingleton(); - return services; - } - } -} diff --git a/common/services/ASC.Data.Storage.Encryption/EncryptionWorker.cs b/common/ASC.Data.Storage/Encryption/EncryptionWorker.cs similarity index 87% rename from common/services/ASC.Data.Storage.Encryption/EncryptionWorker.cs rename to common/ASC.Data.Storage/Encryption/EncryptionWorker.cs index 43603d74d3..c19c6fc4bc 100644 --- a/common/services/ASC.Data.Storage.Encryption/EncryptionWorker.cs +++ b/common/ASC.Data.Storage/Encryption/EncryptionWorker.cs @@ -27,8 +27,9 @@ using System; using ASC.Common; -using ASC.Common.Threading; - +using ASC.Common.Caching; +using ASC.Common.Threading; + using Microsoft.Extensions.DependencyInjection; namespace ASC.Data.Storage.Encryption @@ -36,14 +37,17 @@ namespace ASC.Data.Storage.Encryption public class EncryptionWorker { private object Locker { get; } - private FactoryOperation FactoryOperation { get; } - + private FactoryOperation FactoryOperation { get; } + private ICache Cache { get; } + + private DistributedTaskQueue Queue { get; } public EncryptionWorker(FactoryOperation factoryOperation, DistributedTaskQueueOptionsManager options) { + Cache = AscCache.Memory; Locker = new object(); - FactoryOperation = factoryOperation; + FactoryOperation = factoryOperation; Queue = options.Get(nameof(EncryptionWorker)); } @@ -52,21 +56,27 @@ namespace ASC.Data.Storage.Encryption EncryptionOperation encryptionOperation; lock (Locker) { - if (Queue.GetTask(GetCacheKey()) != null) return; - encryptionOperation = FactoryOperation.CreateOperation(encryptionSettings, GetCacheKey()); + if (Queue.GetTask(GetCacheId()) != null) return; + encryptionOperation = FactoryOperation.CreateOperation(encryptionSettings, GetCacheId()); Queue.QueueTask(encryptionOperation); } } public void Stop() { - Queue.CancelTask(GetCacheKey()); + Queue.CancelTask(GetCacheId()); } - public string GetCacheKey() + public string GetCacheId() { return typeof(EncryptionOperation).FullName; } + + public double? GetEncryptionProgress() + { + var dic = Cache.HashGet(Queue.Name, GetCacheId()); + return dic.Percentage; + } } public class FactoryOperation @@ -89,12 +99,12 @@ namespace ASC.Data.Storage.Encryption public static class EncryptionWorkerExtension { public static DIHelper AddEncryptionWorkerService(this DIHelper services) - { + { services.TryAddTransient(); services.TryAddSingleton(); services.TryAddSingleton(); - return services - .AddEncryptionOperationService() + return services + .AddEncryptionOperationService() .AddDistributedTaskQueueService(1); } } diff --git a/common/services/ASC.Data.Storage.Encryption/NotifyHelper.cs b/common/ASC.Data.Storage/Encryption/NotifyHelper.cs similarity index 100% rename from common/services/ASC.Data.Storage.Encryption/NotifyHelper.cs rename to common/ASC.Data.Storage/Encryption/NotifyHelper.cs diff --git a/web/ASC.Web.Api/Controllers/SettingsController.cs b/web/ASC.Web.Api/Controllers/SettingsController.cs index 9cd4d77138..d1920f3166 100644 --- a/web/ASC.Web.Api/Controllers/SettingsController.cs +++ b/web/ASC.Web.Api/Controllers/SettingsController.cs @@ -161,7 +161,7 @@ namespace ASC.Api.Settings private EncryptionSettingsHelper EncryptionSettingsHelper { get; } private BackupServiceNotifier BackupServiceNotifier { get; } private ICacheNotify CacheDeleteSchedule { get; } - private EncryptionServiceNotifier EncryptionServiceNotifier { get; } + private EncryptionWorker EncryptionWorker { get; } private PasswordHasher PasswordHasher { get; } private ILog Log { get; set; } private TelegramHelper TelegramHelper { get; } @@ -225,7 +225,7 @@ namespace ASC.Api.Settings EncryptionSettingsHelper encryptionSettingsHelper, BackupServiceNotifier backupServiceNotifier, ICacheNotify cacheDeleteSchedule, - EncryptionServiceNotifier encryptionServiceNotifier, + EncryptionWorker encryptionWorker, PasswordHasher passwordHasher) { Log = option.Get("ASC.Api"); @@ -283,7 +283,7 @@ namespace ASC.Api.Settings EncryptionSettingsHelper = encryptionSettingsHelper; BackupServiceNotifier = backupServiceNotifier; CacheDeleteSchedule = cacheDeleteSchedule; - EncryptionServiceNotifier = encryptionServiceNotifier; + EncryptionWorker = encryptionWorker; PasswordHasher = passwordHasher; StorageFactory = storageFactory; UrlShortener = urlShortener; @@ -1758,19 +1758,6 @@ namespace ASC.Api.Settings return ServiceClient.GetProgress(Tenant.TenantId); } - [Read("encryption")] - public void StartEncryption(EncryptionSettingsModel settings) - { - EncryptionSettingsProto encryptionSettingsProto = new EncryptionSettingsProto - { - NotifyUsers = settings.NotifyUsers, - Password = settings.Password, - Status = settings.Status, - ServerRootPath = settings.ServerRootPath - }; - EncryptionServiceClient.Start(encryptionSettingsProto); - } - public readonly object Locker = new object(); [Create("encryption/start")] @@ -1948,7 +1935,7 @@ namespace ASC.Api.Settings throw new BillingException(Resource.ErrorNotAllowedOption, "DiscEncryption"); } - return EncryptionServiceNotifier.GetEncryptionProgress(Tenant.TenantId)?.Progress; + return EncryptionWorker.GetEncryptionProgress(); } [Update("storage")] @@ -2440,7 +2427,7 @@ namespace ASC.Api.Settings .AddEncryptionSettingsHelperService() .AddStorageFactoryService() .AddBackupService() - .AddEncryptionServiceNotifierService() + .AddEncryptionWorkerService() .AddTelegramLoginProviderService() .AddTelegramHelperSerivce() .AddPasswordHasherService(); From ab84f48a86f7ffddf051a7a6cfd4f511c6dc964a Mon Sep 17 00:00:00 2001 From: SuhorukovAnton <62381554+SuhorukovAnton@users.noreply.github.com> Date: Wed, 14 Oct 2020 17:26:18 +0300 Subject: [PATCH 015/278] remove-wq: refactoring --- .../Encryption/EncryptionOperation.cs | 26 ++-------- .../Encryption/EncryptionWorker.cs | 5 +- common/ASC.Data.Storage/StaticUploader.cs | 1 - common/ASC.Data.Storage/StorageUploader.cs | 1 - .../ASC.Data.Backup/BackupAjaxHandler.cs | 10 +++- .../ASC.Data.Backup/Service/BackupService.cs | 50 ++----------------- .../Controllers/SettingsController.cs | 11 ++-- 7 files changed, 26 insertions(+), 78 deletions(-) diff --git a/common/ASC.Data.Storage/Encryption/EncryptionOperation.cs b/common/ASC.Data.Storage/Encryption/EncryptionOperation.cs index fa1b37f7b3..928ec701d9 100644 --- a/common/ASC.Data.Storage/Encryption/EncryptionOperation.cs +++ b/common/ASC.Data.Storage/Encryption/EncryptionOperation.cs @@ -30,7 +30,6 @@ using System.Linq; using System.Threading.Tasks; using ASC.Common; -using ASC.Common.Caching; using ASC.Common.Logging; using ASC.Common.Threading; using ASC.Core; @@ -73,14 +72,13 @@ namespace ASC.Data.Storage.Encryption { using var scope = ServiceProvider.CreateScope(); var scopeClass = scope.ServiceProvider.GetService(); - var (log, encryptionSettingsHelper, tenantManager, notifyHelper, coreBaseSettings, storageFactoryConfig, storageFactory, progressEncryption, configuration) = scopeClass; + var (log, encryptionSettingsHelper, tenantManager, notifyHelper, coreBaseSettings, storageFactoryConfig, storageFactory, configuration) = scopeClass; notifyHelper.Init(ServerRootPath); Tenants = tenantManager.GetTenants(false); Modules = storageFactoryConfig.GetModuleList(ConfigPath, true); UseProgressFile = Convert.ToBoolean(configuration["storage:encryption:progressfile"] ?? "true"); Percentage = 10; - PublishProgress(progressEncryption); PublishChanges(); try @@ -97,7 +95,6 @@ namespace ASC.Data.Storage.Encryption } Percentage = 30; - PublishProgress(progressEncryption); PublishChanges(); foreach (var tenant in Tenants) @@ -114,7 +111,6 @@ namespace ASC.Data.Storage.Encryption } Percentage = 70; - PublishProgress(progressEncryption); PublishChanges(); if (!HasErrors) @@ -124,13 +120,11 @@ namespace ASC.Data.Storage.Encryption } Percentage = 90; - PublishProgress(progressEncryption); PublishChanges(); ActivateTenants(tenantManager, log, notifyHelper); Percentage = 100; - PublishProgress(progressEncryption); PublishChanges(); } catch (Exception e) @@ -195,16 +189,6 @@ namespace ASC.Data.Storage.Encryption return encryptedFiles; } - public void PublishProgress(ICacheNotify progress) - { - var progressEncryption = new ProgressEncryption() - { - Progress = Percentage - }; - - progress.Publish(progressEncryption, CacheNotifyAction.Insert); - } - private IEnumerable GetFiles(List domains, List progress, DiscDataStore targetStore, string targetDomain) { IEnumerable files = targetStore.ListFilesRelative(targetDomain, "\\", "*.*", true); @@ -365,7 +349,6 @@ namespace ASC.Data.Storage.Encryption private CoreBaseSettings CoreBaseSettings { get; set; } private StorageFactoryConfig StorageFactoryConfig { get; set; } private StorageFactory StorageFactory { get; set; } - private ICacheNotify ProgressEncryption { get; } private IConfiguration Configuration { get; } public EncryptionOperationScope(IOptionsMonitor options, @@ -375,8 +358,7 @@ namespace ASC.Data.Storage.Encryption CoreBaseSettings coreBaseSettings, NotifyHelper notifyHelper, EncryptionSettingsHelper encryptionSettingsHelper, - IConfiguration configuration, - ICacheNotify progressEncryption) + IConfiguration configuration) { Log = options.CurrentValue; StorageFactoryConfig = storageFactoryConfig; @@ -385,11 +367,10 @@ namespace ASC.Data.Storage.Encryption CoreBaseSettings = coreBaseSettings; NotifyHelper = notifyHelper; EncryptionSettingsHelper = encryptionSettingsHelper; - ProgressEncryption = progressEncryption; Configuration = configuration; } - public void Deconstruct(out ILog log, out EncryptionSettingsHelper encryptionSettingsHelper, out TenantManager tenantManager, out NotifyHelper notifyHelper, out CoreBaseSettings coreBaseSettings, out StorageFactoryConfig storageFactoryConfig, out StorageFactory storageFactory, out ICacheNotify progressEncryption, out IConfiguration configuration) + public void Deconstruct(out ILog log, out EncryptionSettingsHelper encryptionSettingsHelper, out TenantManager tenantManager, out NotifyHelper notifyHelper, out CoreBaseSettings coreBaseSettings, out StorageFactoryConfig storageFactoryConfig, out StorageFactory storageFactory, out IConfiguration configuration) { log = Log; encryptionSettingsHelper = EncryptionSettingsHelper; @@ -398,7 +379,6 @@ namespace ASC.Data.Storage.Encryption coreBaseSettings = CoreBaseSettings; storageFactoryConfig = StorageFactoryConfig; storageFactory = StorageFactory; - progressEncryption = ProgressEncryption; configuration = Configuration; } } diff --git a/common/ASC.Data.Storage/Encryption/EncryptionWorker.cs b/common/ASC.Data.Storage/Encryption/EncryptionWorker.cs index c19c6fc4bc..f6ca5f58ee 100644 --- a/common/ASC.Data.Storage/Encryption/EncryptionWorker.cs +++ b/common/ASC.Data.Storage/Encryption/EncryptionWorker.cs @@ -25,6 +25,7 @@ using System; +using System.Linq; using ASC.Common; using ASC.Common.Caching; @@ -74,8 +75,8 @@ namespace ASC.Data.Storage.Encryption public double? GetEncryptionProgress() { - var dic = Cache.HashGet(Queue.Name, GetCacheId()); - return dic.Percentage; + var progress = Queue.GetTasks().FirstOrDefault(); + return progress.Percentage; } } diff --git a/common/ASC.Data.Storage/StaticUploader.cs b/common/ASC.Data.Storage/StaticUploader.cs index 6a84e97819..a66639fbff 100644 --- a/common/ASC.Data.Storage/StaticUploader.cs +++ b/common/ASC.Data.Storage/StaticUploader.cs @@ -35,7 +35,6 @@ using ASC.Common; using ASC.Common.Caching; using ASC.Common.Logging; using ASC.Common.Threading; -using ASC.Common.Threading.Progress; using ASC.Core; using ASC.Core.Common.Settings; using ASC.Core.Tenants; diff --git a/common/ASC.Data.Storage/StorageUploader.cs b/common/ASC.Data.Storage/StorageUploader.cs index 6b4478d9cf..a94caf1aea 100644 --- a/common/ASC.Data.Storage/StorageUploader.cs +++ b/common/ASC.Data.Storage/StorageUploader.cs @@ -31,7 +31,6 @@ using System.Linq; using ASC.Common.Caching; using ASC.Common.Logging; using ASC.Common.Threading; -using ASC.Common.Threading.Progress; using ASC.Core; using ASC.Core.Common.Settings; using ASC.Core.Tenants; diff --git a/common/services/ASC.Data.Backup/BackupAjaxHandler.cs b/common/services/ASC.Data.Backup/BackupAjaxHandler.cs index b73a411c97..bb2a4d6366 100644 --- a/common/services/ASC.Data.Backup/BackupAjaxHandler.cs +++ b/common/services/ASC.Data.Backup/BackupAjaxHandler.cs @@ -100,6 +100,13 @@ namespace ASC.Data.Backup return BackupService.GetBackupProgress(GetCurrentTenantId()); } + public BackupProgress GetBackupProgress(int tenantId) + { + DemandPermissionsBackup(); + + return BackupService.GetBackupProgress(tenantId); + } + public void DeleteBackup(Guid id) { DemandPermissionsBackup(); @@ -415,7 +422,8 @@ namespace ASC.Data.Backup .AddUserManagerService() .AddTenantExtraService() .AddConsumerFactoryService() - .AddBackupHelperService(); + .AddBackupHelperService() + .AddBackupService(); } return services; diff --git a/common/services/ASC.Data.Backup/Service/BackupService.cs b/common/services/ASC.Data.Backup/Service/BackupService.cs index c6316da67c..7bf2efab10 100644 --- a/common/services/ASC.Data.Backup/Service/BackupService.cs +++ b/common/services/ASC.Data.Backup/Service/BackupService.cs @@ -31,7 +31,6 @@ using System.Linq; using System.ServiceModel; using ASC.Common; -using ASC.Common.Caching; using ASC.Common.Logging; using ASC.Common.Utils; using ASC.Data.Backup.Contracts; @@ -45,64 +44,26 @@ using Microsoft.Extensions.Options; using Newtonsoft.Json; namespace ASC.Data.Backup.Service -{ - public class BackupServiceNotifier - { - private ICacheNotify СacheBackupProgress { get; } - public ICache Cache { get; } - - public BackupServiceNotifier(ICacheNotify сacheBackupProgress) - { - Cache = AscCache.Memory; - СacheBackupProgress = сacheBackupProgress; - - СacheBackupProgress.Subscribe((a) => - { - Cache.Insert(GetCacheKey(a.TenantId, a.BackupProgressEnum), a, DateTime.UtcNow.AddDays(1)); - }, - CacheNotifyAction.InsertOrUpdate); - } - - public BackupProgress GetBackupProgress(int tenantId) - { - return Cache.Get(GetCacheKey(tenantId, BackupProgressEnum.Backup)); - } - - public BackupProgress GetTransferProgress(int tenantID) - { - return Cache.Get(GetCacheKey(tenantID, BackupProgressEnum.Transfer)); - } - - public BackupProgress GetRestoreProgress(int tenantId) - { - return Cache.Get(GetCacheKey(tenantId, BackupProgressEnum.Restore)); - } - - private string GetCacheKey(int tenantId, BackupProgressEnum backupProgressEnum) => $"{backupProgressEnum}backup{tenantId}"; - } - +{ public class BackupService : IBackupService { private ILog Log { get; set; } private BackupStorageFactory BackupStorageFactory { get; set; } private BackupWorker BackupWorker { get; set; } private BackupRepository BackupRepository { get; } - public BackupServiceNotifier BackupServiceNotifier { get; } private IConfiguration Configuration { get; } public BackupService( IOptionsMonitor options, BackupStorageFactory backupStorageFactory, BackupWorker backupWorker, - BackupRepository backupRepository, - BackupServiceNotifier backupServiceNotifier, + BackupRepository backupRepository, IConfiguration configuration) { Log = options.CurrentValue; BackupStorageFactory = backupStorageFactory; BackupWorker = backupWorker; BackupRepository = backupRepository; - BackupServiceNotifier = backupServiceNotifier; Configuration = configuration; } @@ -210,17 +171,17 @@ namespace ASC.Data.Backup.Service public BackupProgress GetBackupProgress(int tenantId) { - return BackupServiceNotifier.GetBackupProgress(tenantId); + return BackupWorker.GetBackupProgress(tenantId); } public BackupProgress GetTransferProgress(int tenantId) { - return BackupServiceNotifier.GetTransferProgress(tenantId); + return BackupWorker.GetTransferProgress(tenantId); } public BackupProgress GetRestoreProgress(int tenantId) { - return BackupServiceNotifier.GetRestoreProgress(tenantId); + return BackupWorker.GetRestoreProgress(tenantId); } public string GetTmpFolder() @@ -294,7 +255,6 @@ namespace ASC.Data.Backup.Service { if (services.TryAddScoped()) { - services.TryAddSingleton(); return services .AddBackupWorkerService() diff --git a/web/ASC.Web.Api/Controllers/SettingsController.cs b/web/ASC.Web.Api/Controllers/SettingsController.cs index d1920f3166..89ca40cc40 100644 --- a/web/ASC.Web.Api/Controllers/SettingsController.cs +++ b/web/ASC.Web.Api/Controllers/SettingsController.cs @@ -51,6 +51,7 @@ using ASC.Core.Common.Notify; using ASC.Core.Common.Settings; using ASC.Core.Tenants; using ASC.Core.Users; +using ASC.Data.Backup; using ASC.Data.Backup.Contracts; using ASC.Data.Backup.Service; using ASC.Data.Storage; @@ -159,7 +160,7 @@ namespace ASC.Api.Settings private UrlShortener UrlShortener { get; } private EncryptionServiceClient EncryptionServiceClient { get; } private EncryptionSettingsHelper EncryptionSettingsHelper { get; } - private BackupServiceNotifier BackupServiceNotifier { get; } + private BackupAjaxHandler BackupAjaxHandler { get; } private ICacheNotify CacheDeleteSchedule { get; } private EncryptionWorker EncryptionWorker { get; } private PasswordHasher PasswordHasher { get; } @@ -223,7 +224,7 @@ namespace ASC.Api.Settings UrlShortener urlShortener, EncryptionServiceClient encryptionServiceClient, EncryptionSettingsHelper encryptionSettingsHelper, - BackupServiceNotifier backupServiceNotifier, + BackupAjaxHandler backupAjaxHandler, ICacheNotify cacheDeleteSchedule, EncryptionWorker encryptionWorker, PasswordHasher passwordHasher) @@ -281,7 +282,7 @@ namespace ASC.Api.Settings ServiceClient = serviceClient; EncryptionServiceClient = encryptionServiceClient; EncryptionSettingsHelper = encryptionSettingsHelper; - BackupServiceNotifier = backupServiceNotifier; + BackupAjaxHandler = backupAjaxHandler; CacheDeleteSchedule = cacheDeleteSchedule; EncryptionWorker = encryptionWorker; PasswordHasher = passwordHasher; @@ -1813,7 +1814,7 @@ namespace ASC.Api.Settings foreach (var tenant in tenants) { - var progress = BackupServiceNotifier.GetBackupProgress(tenant.TenantId); + var progress = BackupAjaxHandler.GetBackupProgress(tenant.TenantId); if (progress != null && progress.IsCompleted == false) { throw new Exception(); @@ -2426,7 +2427,7 @@ namespace ASC.Api.Settings .AddEncryptionServiceClient() .AddEncryptionSettingsHelperService() .AddStorageFactoryService() - .AddBackupService() + .AddBackupAjaxHandler() .AddEncryptionWorkerService() .AddTelegramLoginProviderService() .AddTelegramHelperSerivce() From e1df253b22730e6a13c737335372a0b4db3c404c Mon Sep 17 00:00:00 2001 From: Alexey Kostenko Date: Tue, 23 Mar 2021 18:29:29 +0300 Subject: [PATCH 016/278] Web: People/Components: Added login settings buttons to edit profile page, social button - style fix --- .../asc-web-components/social-button/index.js | 5 +- .../social-button/styled-social-button.js | 6 +- packages/asc-web-components/themes/base.js | 7 +- .../Client/public/images/share.google.svg | 6 ++ .../Client/public/images/share.linkedin.svg | 4 + .../Section/Body/updateUserForm.js | 90 +++++++++++++++++++ 6 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 products/ASC.People/Client/public/images/share.google.svg create mode 100644 products/ASC.People/Client/public/images/share.linkedin.svg diff --git a/packages/asc-web-components/social-button/index.js b/packages/asc-web-components/social-button/index.js index 6c90f923e9..05e9ba9a2d 100644 --- a/packages/asc-web-components/social-button/index.js +++ b/packages/asc-web-components/social-button/index.js @@ -13,7 +13,7 @@ class SocialButton extends React.Component { } render() { - const { label, iconName } = this.props; + const { label, iconName, iconOptions } = this.props; return ( @@ -42,6 +42,8 @@ SocialButton.propTypes = { id: PropTypes.string, /** Accepts css style */ style: PropTypes.oneOfType([PropTypes.object, PropTypes.array]), + onClick: PropTypes.func, + iconOptions: PropTypes.object, }; SocialButton.defaultProps = { @@ -49,6 +51,7 @@ SocialButton.defaultProps = { iconName: "SocialButtonGoogleIcon", tabIndex: -1, isDisabled: false, + iconOptions: {}, }; export default SocialButton; diff --git a/packages/asc-web-components/social-button/styled-social-button.js b/packages/asc-web-components/social-button/styled-social-button.js index 7b17c22285..be27782345 100644 --- a/packages/asc-web-components/social-button/styled-social-button.js +++ b/packages/asc-web-components/social-button/styled-social-button.js @@ -1,3 +1,4 @@ +import React from "react"; import styled, { css } from "styled-components"; import Base from "../themes/base"; import PropTypes from "prop-types"; @@ -20,7 +21,8 @@ const StyledSocialButton = styled(ButtonWrapper).attrs((props) => ({ }))` font-family: ${(props) => props.theme.fontFamily}; border: none; - display: inline-block; + display: flex; + align-items: center; font-weight: ${(props) => props.theme.socialButton.fontWeight}; text-decoration: ${(props) => props.theme.socialButton.textDecoration}; @@ -76,7 +78,7 @@ const StyledSocialButton = styled(ButtonWrapper).attrs((props) => ({ `}; .social_button_text { - position: absolute; + position: relative; width: ${(props) => props.theme.socialButton.text.width}; height: ${(props) => props.theme.socialButton.text.height}; diff --git a/packages/asc-web-components/themes/base.js b/packages/asc-web-components/themes/base.js index 94348077a8..c847bcab18 100644 --- a/packages/asc-web-components/themes/base.js +++ b/packages/asc-web-components/themes/base.js @@ -213,10 +213,8 @@ const Base = { socialButton: { fontWeight: "600", textDecoration: "none", - margin: "20px 0 0 20px", padding: "0", borderRadius: "2px", - width: "201px", height: "40px", textAlign: "left", stroke: " none", @@ -238,14 +236,15 @@ const Base = { text: { width: "142px", height: "16px", - margin: "12px 9px 12px 10px", - fontWeight: "500", + margin: "0 11px", + fontWeight: "600", fontSize: "14px", lineHeight: "16px", letterSpacing: "0.21875px", overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap", + color: "#757575", }, svg: { diff --git a/products/ASC.People/Client/public/images/share.google.svg b/products/ASC.People/Client/public/images/share.google.svg new file mode 100644 index 0000000000..8a07d68b28 --- /dev/null +++ b/products/ASC.People/Client/public/images/share.google.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/products/ASC.People/Client/public/images/share.linkedin.svg b/products/ASC.People/Client/public/images/share.linkedin.svg new file mode 100644 index 0000000000..65c1ed4211 --- /dev/null +++ b/products/ASC.People/Client/public/images/share.linkedin.svg @@ -0,0 +1,4 @@ + + + + diff --git a/products/ASC.People/Client/src/pages/ProfileAction/Section/Body/updateUserForm.js b/products/ASC.People/Client/src/pages/ProfileAction/Section/Body/updateUserForm.js index e565dddcaa..cd74e433fa 100644 --- a/products/ASC.People/Client/src/pages/ProfileAction/Section/Body/updateUserForm.js +++ b/products/ASC.People/Client/src/pages/ProfileAction/Section/Body/updateUserForm.js @@ -6,6 +6,7 @@ import Button from "@appserver/components/button"; import Textarea from "@appserver/components/textarea"; import Text from "@appserver/components/text"; import AvatarEditor from "@appserver/components/avatar-editor"; +import SocialButton from "@appserver/components/social-button"; import Link from "@appserver/components/link"; import { isTablet } from "@appserver/components/utils/device"; @@ -51,6 +52,9 @@ import config from "../../../../../package.json"; import { combineUrl } from "@appserver/common/utils"; import { AppServerConfig } from "@appserver/common/constants"; +const facebookIconOptions = { isfill: true, color: "#4469B0" }; +const twitterIconOptions = { isfill: true, color: "#2AA3EF" }; + const dialogsDataset = { changeEmail: "changeEmail", changePassword: "changePassword", @@ -69,6 +73,13 @@ const Th = styled.th` const Td = styled.td``; +const StyledWrapper = styled.div` + align-items: center; + display: grid; + grid-template-columns: auto 1fr; + grid-gap: 16px 22px; +`; + class UpdateUserForm extends React.Component { constructor(props) { super(props); @@ -533,6 +544,19 @@ class UpdateUserForm extends React.Component { this.setIsEdit(); } + onClickGoogleConnect = () => { + console.log("Connect to Google"); + }; + onClickFacebookConnect = () => { + console.log("Connect to Facebook"); + }; + onClickTwitterConnect = () => { + console.log("Connect to Twitter"); + }; + onClickLinkedInConnect = () => { + console.log("Connect to LinkedIn"); + }; + render() { const { isLoading, @@ -826,6 +850,72 @@ class UpdateUserForm extends React.Component { /> + + + + + + {t("Connect")} + +
+ +
+ + + {t("Connect")} + +
+ +
+ + {t("Connect")} + +
+ +
+ + {t("Connect")} + + +