From 44f84c80569f90eca2de84d1bdcbd97eba968616 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Thu, 23 Apr 2020 12:38:50 +0300 Subject: [PATCH 01/20] Files: Service. first version --- ASC.Web.sln | 6 + common/ASC.Common/Caching/KafkaCache.cs | 2 - common/ASC.Common/Logging/EFLoggerFactory.cs | 3 - common/ASC.Common/Logging/Log.cs | 1 - common/ASC.Common/Utils/Signature.cs | 2 - .../Caching/CachedQuotaService.cs | 1 - .../Caching/CachedTenantService.cs | 2 - .../Caching/CachedUserService.cs | 2 - .../EF/Context/DbContextManager.cs | 2 - .../Notify/RecipientProviderImpl.cs | 2 - .../Security/Authorizing/AzManager.cs | 2 - .../Authorizing/PermissionProvider.cs | 2 - .../Authorizing/PermissionResolver.cs | 2 - .../Security/Authorizing/RoleProvider.cs | 1 - common/ASC.Data.Reassigns/QueueWorker.cs | 1 - .../Configuration/Appender.cs | 1 - .../ASC.ElasticSearch/Engine/BaseIndexer.cs | 12 +- .../ASC.ElasticSearch/Engine/IIndexer.cs | 4 +- .../ASC.Files.Service.csproj | 15 ++ .../Service/ASC.Files.Service/Program.cs | 67 ++++++ .../Properties/launchSettings.json | 27 +++ .../ASC.Files.Service/ServiceLauncher.cs | 210 ++++++++++++++++++ .../Common/IO/PathHelper.cs | 3 +- .../Common/Net/HttpException.cs | 1 - .../Net/oAuth/Context/OAuthConsumerContext.cs | 3 +- .../Net/oAuth/Context/OAuthServiceContext.cs | 4 +- .../Net/oAuth/Impl/OAuthStreamParser.cs | 3 +- .../Common/Net/oAuth/Token/OAuthToken.cs | 4 +- .../ICloudDirectoryEntry.cs | 3 +- .../StorageProvider/API/GenericHelper.cs | 4 +- .../API/IStorageProviderService.cs | 3 +- .../BaseObjects/BaseFileEntryDataTransfer.cs | 3 +- .../Logic/BoxNetStorageProviderService.cs | 3 +- .../DropBox/DropBoxAccountInfo.cs | 3 +- .../DropBox/DropBoxBaseTokenInformation.cs | 4 +- .../DropBox/DropBoxResourceIDHelpers.cs | 4 +- .../StorageProvider/DropBox/DropBoxToken.cs | 1 - .../Logic/DropBoxStorageProviderService.cs | 1 - .../Logic/DropBoxStorageProviderSession.cs | 3 +- .../GoogleDocs/GoogleDocsResourceHelper.cs | 3 +- .../GoogleDocs/GoogleDocsToken.cs | 3 +- .../SkyDrive/SkyDriveConstants.cs | 3 +- .../SkyDrive/SkyDriveHelpers.cs | 3 +- web/ASC.Web.Core/Notify/TagValues.cs | 3 - web/ASC.Web.Core/Users/UserPhotoManager.cs | 2 - 45 files changed, 352 insertions(+), 82 deletions(-) create mode 100644 products/ASC.Files/Service/ASC.Files.Service/ASC.Files.Service.csproj create mode 100644 products/ASC.Files/Service/ASC.Files.Service/Program.cs create mode 100644 products/ASC.Files/Service/ASC.Files.Service/Properties/launchSettings.json create mode 100644 products/ASC.Files/Service/ASC.Files.Service/ServiceLauncher.cs diff --git a/ASC.Web.sln b/ASC.Web.sln index 4b1b1fe835..503eabd225 100644 --- a/ASC.Web.sln +++ b/ASC.Web.sln @@ -50,6 +50,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ASC.Files", "products\ASC.F EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AppLimit.CloudComputing.SharpBox", "thirdparty\AppLimit.CloudComputing.SharpBox\AppLimit.CloudComputing.SharpBox.csproj", "{5B53855C-4347-4402-B750-76C6295A35D3}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ASC.Files.Service", "products\ASC.Files\Service\ASC.Files.Service\ASC.Files.Service.csproj", "{725944D5-FDDC-45A9-802C-E2724DBD29F8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -144,6 +146,10 @@ Global {5B53855C-4347-4402-B750-76C6295A35D3}.Debug|Any CPU.Build.0 = Debug|Any CPU {5B53855C-4347-4402-B750-76C6295A35D3}.Release|Any CPU.ActiveCfg = Release|Any CPU {5B53855C-4347-4402-B750-76C6295A35D3}.Release|Any CPU.Build.0 = Release|Any CPU + {725944D5-FDDC-45A9-802C-E2724DBD29F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {725944D5-FDDC-45A9-802C-E2724DBD29F8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {725944D5-FDDC-45A9-802C-E2724DBD29F8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {725944D5-FDDC-45A9-802C-E2724DBD29F8}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/common/ASC.Common/Caching/KafkaCache.cs b/common/ASC.Common/Caching/KafkaCache.cs index 6ed0062528..f3ac4cacc3 100644 --- a/common/ASC.Common/Caching/KafkaCache.cs +++ b/common/ASC.Common/Caching/KafkaCache.cs @@ -12,8 +12,6 @@ using Confluent.Kafka; using Google.Protobuf; using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; namespace ASC.Common.Caching diff --git a/common/ASC.Common/Logging/EFLoggerFactory.cs b/common/ASC.Common/Logging/EFLoggerFactory.cs index 7cee860a0e..58ef292a39 100644 --- a/common/ASC.Common/Logging/EFLoggerFactory.cs +++ b/common/ASC.Common/Logging/EFLoggerFactory.cs @@ -1,8 +1,5 @@ using System; using System.Collections.Generic; - -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; diff --git a/common/ASC.Common/Logging/Log.cs b/common/ASC.Common/Logging/Log.cs index eb5278c264..378d9a0028 100644 --- a/common/ASC.Common/Logging/Log.cs +++ b/common/ASC.Common/Logging/Log.cs @@ -35,7 +35,6 @@ using log4net.Config; using log4net.Core; using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; using NLog; diff --git a/common/ASC.Common/Utils/Signature.cs b/common/ASC.Common/Utils/Signature.cs index aa1161afb5..a8f1f0dbce 100644 --- a/common/ASC.Common/Utils/Signature.cs +++ b/common/ASC.Common/Utils/Signature.cs @@ -30,8 +30,6 @@ using System.Text; using ASC.Security.Cryptography; using Microsoft.AspNetCore.WebUtilities; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; using Newtonsoft.Json; namespace ASC.Common.Utils diff --git a/common/ASC.Core.Common/Caching/CachedQuotaService.cs b/common/ASC.Core.Common/Caching/CachedQuotaService.cs index 4a27b0b98e..80bf321920 100644 --- a/common/ASC.Core.Common/Caching/CachedQuotaService.cs +++ b/common/ASC.Core.Common/Caching/CachedQuotaService.cs @@ -35,7 +35,6 @@ using ASC.Core.Data; using ASC.Core.Tenants; using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; namespace ASC.Core.Caching diff --git a/common/ASC.Core.Common/Caching/CachedTenantService.cs b/common/ASC.Core.Common/Caching/CachedTenantService.cs index 0eac8a9ca7..a40c57455d 100644 --- a/common/ASC.Core.Common/Caching/CachedTenantService.cs +++ b/common/ASC.Core.Common/Caching/CachedTenantService.cs @@ -33,8 +33,6 @@ using ASC.Common.Utils; using ASC.Core.Common.EF.Context; using ASC.Core.Data; using ASC.Core.Tenants; - -using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; namespace ASC.Core.Caching diff --git a/common/ASC.Core.Common/Caching/CachedUserService.cs b/common/ASC.Core.Common/Caching/CachedUserService.cs index f55d236c38..2acdebd78d 100644 --- a/common/ASC.Core.Common/Caching/CachedUserService.cs +++ b/common/ASC.Core.Common/Caching/CachedUserService.cs @@ -36,8 +36,6 @@ using ASC.Core.Common.EF; using ASC.Core.Data; using ASC.Core.Tenants; using ASC.Core.Users; - -using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; namespace ASC.Core.Caching diff --git a/common/ASC.Core.Common/EF/Context/DbContextManager.cs b/common/ASC.Core.Common/EF/Context/DbContextManager.cs index 73257798dc..bbfb652b68 100644 --- a/common/ASC.Core.Common/EF/Context/DbContextManager.cs +++ b/common/ASC.Core.Common/EF/Context/DbContextManager.cs @@ -2,8 +2,6 @@ using System.Collections.Generic; using ASC.Common; - -using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; namespace ASC.Core.Common.EF diff --git a/common/ASC.Core.Common/Notify/RecipientProviderImpl.cs b/common/ASC.Core.Common/Notify/RecipientProviderImpl.cs index 5240962022..7b9b92f6c0 100644 --- a/common/ASC.Core.Common/Notify/RecipientProviderImpl.cs +++ b/common/ASC.Core.Common/Notify/RecipientProviderImpl.cs @@ -31,8 +31,6 @@ using System.Linq; using ASC.Common; using ASC.Core.Users; using ASC.Notify.Recipients; - -using Microsoft.Extensions.DependencyInjection.Extensions; namespace ASC.Core.Notify { diff --git a/common/ASC.Core.Common/Security/Authorizing/AzManager.cs b/common/ASC.Core.Common/Security/Authorizing/AzManager.cs index ba4bce27b8..e7b700f062 100644 --- a/common/ASC.Core.Common/Security/Authorizing/AzManager.cs +++ b/common/ASC.Core.Common/Security/Authorizing/AzManager.cs @@ -27,8 +27,6 @@ using System; using System.Collections.Generic; using ASC.Core.Security.Authorizing; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; namespace ASC.Common.Security.Authorizing { diff --git a/common/ASC.Core.Common/Security/Authorizing/PermissionProvider.cs b/common/ASC.Core.Common/Security/Authorizing/PermissionProvider.cs index 5d01b7365b..c00e884998 100644 --- a/common/ASC.Core.Common/Security/Authorizing/PermissionProvider.cs +++ b/common/ASC.Core.Common/Security/Authorizing/PermissionProvider.cs @@ -31,8 +31,6 @@ using System.Linq; using ASC.Common; using ASC.Common.Security; using ASC.Common.Security.Authorizing; - -using Microsoft.Extensions.DependencyInjection.Extensions; namespace ASC.Core.Security.Authorizing { diff --git a/common/ASC.Core.Common/Security/Authorizing/PermissionResolver.cs b/common/ASC.Core.Common/Security/Authorizing/PermissionResolver.cs index 80614af77b..704de4da8b 100644 --- a/common/ASC.Core.Common/Security/Authorizing/PermissionResolver.cs +++ b/common/ASC.Core.Common/Security/Authorizing/PermissionResolver.cs @@ -33,8 +33,6 @@ using ASC.Common.Security; using ASC.Common.Security.Authentication; using ASC.Common.Security.Authorizing; -using Microsoft.Extensions.DependencyInjection.Extensions; - using Constants = ASC.Core.Configuration.Constants; namespace ASC.Core.Security.Authorizing diff --git a/common/ASC.Core.Common/Security/Authorizing/RoleProvider.cs b/common/ASC.Core.Common/Security/Authorizing/RoleProvider.cs index d5337a13f8..67455fe923 100644 --- a/common/ASC.Core.Common/Security/Authorizing/RoleProvider.cs +++ b/common/ASC.Core.Common/Security/Authorizing/RoleProvider.cs @@ -35,7 +35,6 @@ using ASC.Common.Security.Authorizing; using ASC.Core.Users; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; namespace ASC.Core.Security.Authorizing { diff --git a/common/ASC.Data.Reassigns/QueueWorker.cs b/common/ASC.Data.Reassigns/QueueWorker.cs index 9ae74fff63..552f48a54b 100644 --- a/common/ASC.Data.Reassigns/QueueWorker.cs +++ b/common/ASC.Data.Reassigns/QueueWorker.cs @@ -33,7 +33,6 @@ using ASC.Common.Threading.Progress; using ASC.Core.Users; using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; namespace ASC.Data.Reassigns diff --git a/common/ASC.Data.Storage/Configuration/Appender.cs b/common/ASC.Data.Storage/Configuration/Appender.cs index 754ca625a2..1cb51c2aae 100644 --- a/common/ASC.Data.Storage/Configuration/Appender.cs +++ b/common/ASC.Data.Storage/Configuration/Appender.cs @@ -7,7 +7,6 @@ using ASC.Common.Utils; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; namespace ASC.Data.Storage.Configuration { diff --git a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs index bac26c2901..3c56ad2618 100644 --- a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs @@ -32,7 +32,7 @@ using System.Linq; using System.Linq.Expressions; using System.Reflection; using System.Text; -using System.Threading.Tasks; +using System.Threading.Tasks; using ASC.Common; using ASC.Common.Caching; @@ -75,7 +75,7 @@ namespace ASC.ElasticSearch } } - public class BaseIndexer : IIndexer where T : Wrapper + public abstract class BaseIndexer : IIndexer where T : Wrapper { private static readonly object Locker = new object(); @@ -271,7 +271,7 @@ namespace ASC.ElasticSearch return false; } - void IIndexer.Check() + public void Check() { var data = ServiceProvider.GetService(); if (!CheckExist(data)) return; @@ -332,7 +332,7 @@ namespace ASC.ElasticSearch } } - async Task IIndexer.ReIndex() + public async Task ReIndex() { Clear(); //((IIndexer) this).IndexAll(); @@ -707,7 +707,9 @@ namespace ASC.ElasticSearch var selector = ServiceProvider.GetService>(); var descriptor = func(selector).Where(r => r.TenantId, tenantId); return descriptor.GetDescriptorForUpdate(this, GetScriptForUpdate(data, action, fields), immediately); - } + } + + public abstract void IndexAll(); } static class CamelCaseExtension diff --git a/common/services/ASC.ElasticSearch/Engine/IIndexer.cs b/common/services/ASC.ElasticSearch/Engine/IIndexer.cs index b915eac522..907bcfc93e 100644 --- a/common/services/ASC.ElasticSearch/Engine/IIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/IIndexer.cs @@ -23,14 +23,14 @@ * */ - using System.Threading.Tasks; namespace ASC.ElasticSearch { - internal interface IIndexer + public interface IIndexer { string IndexName { get; } + void IndexAll(); void Check(); diff --git a/products/ASC.Files/Service/ASC.Files.Service/ASC.Files.Service.csproj b/products/ASC.Files/Service/ASC.Files.Service/ASC.Files.Service.csproj new file mode 100644 index 0000000000..80d8cc46a3 --- /dev/null +++ b/products/ASC.Files/Service/ASC.Files.Service/ASC.Files.Service.csproj @@ -0,0 +1,15 @@ + + + + Exe + netcoreapp3.1 + + + + + + + + + + diff --git a/products/ASC.Files/Service/ASC.Files.Service/Program.cs b/products/ASC.Files/Service/ASC.Files.Service/Program.cs new file mode 100644 index 0000000000..9052324aaf --- /dev/null +++ b/products/ASC.Files/Service/ASC.Files.Service/Program.cs @@ -0,0 +1,67 @@ +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; + +using ASC.Common; +using ASC.Common.DependencyInjection; +using ASC.Common.Logging; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace ASC.Files.Service +{ + public class Program + { + public static async Task Main(string[] args) + { + var host = Host.CreateDefaultBuilder(args) + .ConfigureAppConfiguration((hostContext, config) => + { + var buided = config.Build(); + var path = buided["pathToConf"]; + if (!Path.IsPathRooted(path)) + { + path = Path.GetFullPath(Path.Combine(hostContext.HostingEnvironment.ContentRootPath, path)); + } + config.SetBasePath(path); + var env = hostContext.Configuration.GetValue("ENVIRONMENT", "Production"); + config + .AddInMemoryCollection(new Dictionary + { + {"pathToConf", path } + } + ) + .AddJsonFile("appsettings.json") + .AddJsonFile($"appsettings.{env}.json", true) + .AddJsonFile($"appsettings.services.json", true) + .AddJsonFile("storage.json") + .AddJsonFile("notify.json") + .AddJsonFile("kafka.json") + .AddJsonFile($"kafka.{env}.json", true) + .AddEnvironmentVariables(); + }) + .ConfigureServices((hostContext, services) => + { + var diHelper = new DIHelper(services); + diHelper.AddNLogManager("ASC.Files"); + services.AddHostedService(); + diHelper.AddServiceLauncher(); + + services.AddAutofac(hostContext.Configuration, hostContext.HostingEnvironment.ContentRootPath); + }) + .UseConsoleLifetime() + .Build(); + + using (host) + { + // Start the host + await host.StartAsync(); + + // Wait for the host to shutdown + await host.WaitForShutdownAsync(); + } + } + } +} diff --git a/products/ASC.Files/Service/ASC.Files.Service/Properties/launchSettings.json b/products/ASC.Files/Service/ASC.Files.Service/Properties/launchSettings.json new file mode 100644 index 0000000000..c0ed800a13 --- /dev/null +++ b/products/ASC.Files/Service/ASC.Files.Service/Properties/launchSettings.json @@ -0,0 +1,27 @@ +{ + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:61034/", + "sslPort": 44308 + } + }, + "profiles": { + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "ASC.Files.Service": { + "commandName": "Project", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + }, + "applicationUrl": "https://localhost:5001;http://localhost:5000" + } + } +} \ No newline at end of file diff --git a/products/ASC.Files/Service/ASC.Files.Service/ServiceLauncher.cs b/products/ASC.Files/Service/ASC.Files.Service/ServiceLauncher.cs new file mode 100644 index 0000000000..43428c070b --- /dev/null +++ b/products/ASC.Files/Service/ASC.Files.Service/ServiceLauncher.cs @@ -0,0 +1,210 @@ +/* + * + * (c) Copyright Ascensio System Limited 2010-2018 + * + * This program is freeware. You can redistribute it and/or modify it under the terms of the GNU + * General Public License (GPL) version 3 as published by the Free Software Foundation (https://www.gnu.org/copyleft/gpl.html). + * In accordance with Section 7(a) of the GNU GPL its Section 15 shall be amended to the effect that + * Ascensio System SIA expressly excludes the warranty of non-infringement of any third-party rights. + * + * THIS PROGRAM IS DISTRIBUTED WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR + * FITNESS FOR A PARTICULAR PURPOSE. For more details, see GNU GPL at https://www.gnu.org/copyleft/gpl.html + * + * You can contact Ascensio System SIA by email at sales@onlyoffice.com + * + * The interactive user interfaces in modified source and object code versions of ONLYOFFICE must display + * Appropriate Legal Notices, as required under Section 5 of the GNU GPL version 3. + * + * Pursuant to Section 7 § 3(b) of the GNU GPL you must retain the original ONLYOFFICE logo which contains + * relevant author attributions when distributing the software. If the display of the logo in its graphic + * form is not reasonably feasible for technical reasons, you must include the words "Powered by ONLYOFFICE" + * in every copy of the program you distribute. + * Pursuant to Section 7 § 3(e) we decline to grant you any rights under trademark law for use of our trademarks. + * +*/ + + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +using ASC.Common; +using ASC.Common.Caching; +using ASC.Common.Logging; +using ASC.ElasticSearch; +using ASC.Web.Files.Core.Search; + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; + +namespace ASC.Files.Service +{ + public class ServiceLauncher : IHostedService + { + private ILog Log { get; } + private ICacheNotify Notify { get; } + public IServiceProvider ServiceProvider { get; } + private bool IsStarted { get; set; } + private string Indexing { get; set; } + private CancellationTokenSource CancellationTokenSource { get; set; } + private Timer Timer { get; set; } + private DateTime? LastIndexed { get; set; } + private TimeSpan Period { get { return TimeSpan.FromMinutes(1); } }//Settings.Default.Period + + public ServiceLauncher(IOptionsMonitor options, ICacheNotify notify, IServiceProvider serviceProvider) + { + Log = options.Get("ASC.Indexer"); + Notify = notify; + ServiceProvider = serviceProvider; + } + + public Task StartAsync(CancellationToken cancellationToken) + { + try + { + Notify.Subscribe(async (item) => + { + while (IsStarted) + { + await Task.Delay(10000); + } + IndexAll(true); + }, CacheNotifyAction.Any); + } + catch (Exception e) + { + Log.Error("Subscribe on start", e); + } + + var task = new Task(() => + { + using var scope = ServiceProvider.CreateScope(); + var factoryIndexer = scope.ServiceProvider.GetService(); + + while (!factoryIndexer.CheckState(false)) + { + if (CancellationTokenSource.IsCancellationRequested) + { + return; + } + Thread.Sleep(10000); + } + + CheckIfChange(); + }, CancellationTokenSource.Token, TaskCreationOptions.LongRunning); + + task.Start(); + + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + IsStarted = false; + + if (Timer != null) + { + Timer.Dispose(); + } + + CancellationTokenSource.Cancel(); + + return Task.CompletedTask; + } + + private void CheckIfChange() + { + using var scope = ServiceProvider.CreateScope(); + var filesWrapper = scope.ServiceProvider.GetService>(); + IsStarted = true; + + var products = new[] { filesWrapper }.ToList(); + + products.ForEach(product => + { + try + { + if (!IsStarted) return; + + Log.DebugFormat("Product check {0}", product.Indexer.IndexName); + Indexing = product.Indexer.IndexName; + product.Indexer.Check(); + } + catch (Exception e) + { + Log.Error(e); + Log.ErrorFormat("Product check {0}", product.Indexer.IndexName); + } + }); + + IsStarted = false; + Indexing = null; + + Timer = new Timer(_ => IndexAll(), null, TimeSpan.Zero, TimeSpan.Zero); + } + + private void IndexAll(bool reindex = false) + { + Timer.Change(-1, -1); + IsStarted = true; + + using var scope = ServiceProvider.CreateScope(); + var filesWrapper = scope.ServiceProvider.GetService>(); + var foldersWrapper = scope.ServiceProvider.GetService>(); + var products = new[] { (IIndexer)filesWrapper, (IIndexer)foldersWrapper }.ToList(); + + if (reindex) + { + products.ForEach(product => + { + try + { + if (!IsStarted) return; + + Log.DebugFormat("Product reindex {0}", product.IndexName); + product.ReIndex(); + } + catch (Exception e) + { + Log.Error(e); + Log.ErrorFormat("Product reindex {0}", product.IndexName); + } + }); + } + + products.ForEach(product => + { + try + { + if (!IsStarted) return; + + Log.DebugFormat("Product {0}", product.IndexName); + Indexing = product.IndexName; + product.IndexAll(); + } + catch (Exception e) + { + Log.Error(e); + Log.ErrorFormat("Product {0}", product.IndexName); + } + }); + + Timer.Change(Period, Period); + LastIndexed = DateTime.UtcNow; + IsStarted = false; + Indexing = null; + } + } + + public static class ServiceLauncherExtension + { + public static DIHelper AddServiceLauncher(this DIHelper services) + { + services.TryAddSingleton(); + + return services; + } + } +} diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/Common/IO/PathHelper.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/Common/IO/PathHelper.cs index b955aacbbd..dc3c27e506 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/Common/IO/PathHelper.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/Common/IO/PathHelper.cs @@ -1,5 +1,4 @@ -using System; -using System.IO; +using System.IO; namespace AppLimit.CloudComputing.SharpBox.Common.IO { diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/HttpException.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/HttpException.cs index c7ab3ba31d..7a84a51ce7 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/HttpException.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/HttpException.cs @@ -1,5 +1,4 @@ using System; -using System.Net; namespace AppLimit.CloudComputing.SharpBox.Common.Net { diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/oAuth/Context/OAuthConsumerContext.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/oAuth/Context/OAuthConsumerContext.cs index f5ea051a68..994cc05948 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/oAuth/Context/OAuthConsumerContext.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/oAuth/Context/OAuthConsumerContext.cs @@ -1,5 +1,4 @@ -using System; - + using AppLimit.CloudComputing.SharpBox.Common.Net.oAuth.Impl; namespace AppLimit.CloudComputing.SharpBox.Common.Net.oAuth.Context diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/oAuth/Context/OAuthServiceContext.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/oAuth/Context/OAuthServiceContext.cs index a6cf6f35e8..038dd1b1e1 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/oAuth/Context/OAuthServiceContext.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/oAuth/Context/OAuthServiceContext.cs @@ -1,6 +1,4 @@ -using System; - -namespace AppLimit.CloudComputing.SharpBox.Common.Net.oAuth.Context +namespace AppLimit.CloudComputing.SharpBox.Common.Net.oAuth.Context { internal class OAuthServiceContext { diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/oAuth/Impl/OAuthStreamParser.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/oAuth/Impl/OAuthStreamParser.cs index cfda834a06..9ca1ee0388 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/oAuth/Impl/OAuthStreamParser.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/oAuth/Impl/OAuthStreamParser.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.IO; using System.Linq; using AppLimit.CloudComputing.SharpBox.Common.Net.oAuth.Token; diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/oAuth/Token/OAuthToken.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/oAuth/Token/OAuthToken.cs index 7a52a24796..adea0618b7 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/oAuth/Token/OAuthToken.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/Common/Net/oAuth/Token/OAuthToken.cs @@ -1,6 +1,4 @@ -using System; - -namespace AppLimit.CloudComputing.SharpBox.Common.Net.oAuth.Token +namespace AppLimit.CloudComputing.SharpBox.Common.Net.oAuth.Token { /// /// The OAuthToken will be generated during the login process in a oauth compliant diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/ICloudDirectoryEntry.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/ICloudDirectoryEntry.cs index 5a9588577d..b2260c322b 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/ICloudDirectoryEntry.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/ICloudDirectoryEntry.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; namespace AppLimit.CloudComputing.SharpBox { diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/API/GenericHelper.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/API/GenericHelper.cs index 4dac26c273..ca2421e64e 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/API/GenericHelper.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/API/GenericHelper.cs @@ -1,6 +1,4 @@ -using System; - -namespace AppLimit.CloudComputing.SharpBox.StorageProvider.API +namespace AppLimit.CloudComputing.SharpBox.StorageProvider.API { internal class GenericHelper { diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/API/IStorageProviderService.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/API/IStorageProviderService.cs index bd1fb3a6cf..4a9057b0ab 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/API/IStorageProviderService.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/API/IStorageProviderService.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.IO; namespace AppLimit.CloudComputing.SharpBox.StorageProvider.API diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/BaseObjects/BaseFileEntryDataTransfer.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/BaseObjects/BaseFileEntryDataTransfer.cs index 22cf510703..f1623b4602 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/BaseObjects/BaseFileEntryDataTransfer.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/BaseObjects/BaseFileEntryDataTransfer.cs @@ -1,5 +1,4 @@ -using System; -using System.IO; +using System.IO; using AppLimit.CloudComputing.SharpBox.Exceptions; using AppLimit.CloudComputing.SharpBox.StorageProvider.API; using System.Threading; diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/BoxNet/Logic/BoxNetStorageProviderService.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/BoxNet/Logic/BoxNetStorageProviderService.cs index c432b9c50a..a97e3cca34 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/BoxNet/Logic/BoxNetStorageProviderService.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/BoxNet/Logic/BoxNetStorageProviderService.cs @@ -1,5 +1,4 @@ -using System; -using AppLimit.CloudComputing.SharpBox.StorageProvider.WebDav.Logic; +using AppLimit.CloudComputing.SharpBox.StorageProvider.WebDav.Logic; using AppLimit.CloudComputing.SharpBox.StorageProvider.API; namespace AppLimit.CloudComputing.SharpBox.StorageProvider.BoxNet.Logic diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/DropBoxAccountInfo.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/DropBoxAccountInfo.cs index c9d6e008b9..9406747352 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/DropBoxAccountInfo.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/DropBoxAccountInfo.cs @@ -1,5 +1,4 @@ -using System; -using AppLimit.CloudComputing.SharpBox.Common.Net.Json; +using AppLimit.CloudComputing.SharpBox.Common.Net.Json; namespace AppLimit.CloudComputing.SharpBox.StorageProvider.DropBox { diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/DropBoxBaseTokenInformation.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/DropBoxBaseTokenInformation.cs index 47faf419f2..d5a74e3dc7 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/DropBoxBaseTokenInformation.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/DropBoxBaseTokenInformation.cs @@ -1,6 +1,4 @@ -using System; - -namespace AppLimit.CloudComputing.SharpBox.StorageProvider.DropBox +namespace AppLimit.CloudComputing.SharpBox.StorageProvider.DropBox { /// /// The base class of all credentials diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/DropBoxResourceIDHelpers.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/DropBoxResourceIDHelpers.cs index fa74dc30e1..494e29852c 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/DropBoxResourceIDHelpers.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/DropBoxResourceIDHelpers.cs @@ -1,6 +1,4 @@ -using System; - -namespace AppLimit.CloudComputing.SharpBox.StorageProvider.DropBox +namespace AppLimit.CloudComputing.SharpBox.StorageProvider.DropBox { internal static class DropBoxResourceIDHelpers { diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/DropBoxToken.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/DropBoxToken.cs index 328dec8124..b5d3a0f434 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/DropBoxToken.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/DropBoxToken.cs @@ -1,4 +1,3 @@ -using System; using AppLimit.CloudComputing.SharpBox.Common.Net.oAuth.Token; using AppLimit.CloudComputing.SharpBox.Common.Net.Json; diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/Logic/DropBoxStorageProviderService.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/Logic/DropBoxStorageProviderService.cs index ce5ac871d8..f19437bde6 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/Logic/DropBoxStorageProviderService.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/Logic/DropBoxStorageProviderService.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.IO; using System.Net; -using System.Web; using AppLimit.CloudComputing.SharpBox.Common.Extensions; using AppLimit.CloudComputing.SharpBox.Common.IO; using AppLimit.CloudComputing.SharpBox.Common.Net; diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/Logic/DropBoxStorageProviderSession.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/Logic/DropBoxStorageProviderSession.cs index f65fa3b475..6627bf791b 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/Logic/DropBoxStorageProviderSession.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/DropBox/Logic/DropBoxStorageProviderSession.cs @@ -1,5 +1,4 @@ -using System; -using AppLimit.CloudComputing.SharpBox.StorageProvider.API; +using AppLimit.CloudComputing.SharpBox.StorageProvider.API; using AppLimit.CloudComputing.SharpBox.Common.Net.oAuth.Context; namespace AppLimit.CloudComputing.SharpBox.StorageProvider.DropBox.Logic diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/GoogleDocs/GoogleDocsResourceHelper.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/GoogleDocs/GoogleDocsResourceHelper.cs index c9fbba2c25..479c15aa37 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/GoogleDocs/GoogleDocsResourceHelper.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/GoogleDocs/GoogleDocsResourceHelper.cs @@ -1,5 +1,4 @@ -using System; -using System.Linq; +using System.Linq; using System.Text.RegularExpressions; using AppLimit.CloudComputing.SharpBox.StorageProvider.API; using AppLimit.CloudComputing.SharpBox.StorageProvider.GoogleDocs.Logic; diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/GoogleDocs/GoogleDocsToken.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/GoogleDocs/GoogleDocsToken.cs index 539fcc8225..85fd901c27 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/GoogleDocs/GoogleDocsToken.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/GoogleDocs/GoogleDocsToken.cs @@ -1,5 +1,4 @@ -using System; -using AppLimit.CloudComputing.SharpBox.Common.Net.oAuth.Token; +using AppLimit.CloudComputing.SharpBox.Common.Net.oAuth.Token; namespace AppLimit.CloudComputing.SharpBox.StorageProvider.GoogleDocs { diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/SkyDrive/SkyDriveConstants.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/SkyDrive/SkyDriveConstants.cs index 38025706e4..b22d0a61dc 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/SkyDrive/SkyDriveConstants.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/SkyDrive/SkyDriveConstants.cs @@ -1,5 +1,4 @@ -using System; -using System.Text.RegularExpressions; +using System.Text.RegularExpressions; namespace AppLimit.CloudComputing.SharpBox.StorageProvider.SkyDrive { diff --git a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/SkyDrive/SkyDriveHelpers.cs b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/SkyDrive/SkyDriveHelpers.cs index 9fabab078b..83cc7b7369 100644 --- a/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/SkyDrive/SkyDriveHelpers.cs +++ b/thirdparty/AppLimit.CloudComputing.SharpBox/StorageProvider/SkyDrive/SkyDriveHelpers.cs @@ -1,5 +1,4 @@ -using System; -using AppLimit.CloudComputing.SharpBox.StorageProvider.BaseObjects; +using AppLimit.CloudComputing.SharpBox.StorageProvider.BaseObjects; namespace AppLimit.CloudComputing.SharpBox.StorageProvider.SkyDrive { diff --git a/web/ASC.Web.Core/Notify/TagValues.cs b/web/ASC.Web.Core/Notify/TagValues.cs index 03078d0dfd..0397dac8ab 100644 --- a/web/ASC.Web.Core/Notify/TagValues.cs +++ b/web/ASC.Web.Core/Notify/TagValues.cs @@ -25,10 +25,7 @@ using System; -using ASC.Common.Security.Authentication; -using ASC.Core; using ASC.Notify.Patterns; -using ASC.Web.Core.Users; namespace ASC.Web.Studio.Core.Notify { diff --git a/web/ASC.Web.Core/Users/UserPhotoManager.cs b/web/ASC.Web.Core/Users/UserPhotoManager.cs index 1f94da5b10..0b02f167e1 100644 --- a/web/ASC.Web.Core/Users/UserPhotoManager.cs +++ b/web/ASC.Web.Core/Users/UserPhotoManager.cs @@ -43,8 +43,6 @@ using ASC.Core.Common.Settings; using ASC.Core.Tenants; using ASC.Data.Storage; using ASC.Web.Core.Utility.Skins; - -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; namespace ASC.Web.Core.Users From 4db07a6010584b76baaed0ed583212a14c03e7c4 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Thu, 23 Apr 2020 20:24:56 +0300 Subject: [PATCH 02/20] ElasticSearch: updated to 7.6.1 --- .../ASC.ElasticSearch.csproj | 4 ++-- .../ASC.ElasticSearch/Engine/BaseIndexer.cs | 20 +++++++++---------- .../ASC.ElasticSearch/Engine/Client.cs | 2 +- .../Engine/FactoryIndexer.cs | 6 +++--- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/common/services/ASC.ElasticSearch/ASC.ElasticSearch.csproj b/common/services/ASC.ElasticSearch/ASC.ElasticSearch.csproj index aee918a9d8..964f7b444b 100644 --- a/common/services/ASC.ElasticSearch/ASC.ElasticSearch.csproj +++ b/common/services/ASC.ElasticSearch/ASC.ElasticSearch.csproj @@ -22,12 +22,12 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs index 3c56ad2618..67a8c367aa 100644 --- a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs @@ -238,26 +238,26 @@ namespace ASC.ElasticSearch public void Flush() { - Client.Instance.Flush(new FlushRequest(IndexName)); + Client.Instance.Indices.Flush(new FlushRequest(IndexName)); } public void Refresh() { - Client.Instance.Refresh(new RefreshRequest(IndexName)); + Client.Instance.Indices.Refresh(new RefreshRequest(IndexName)); } internal bool CheckExist(T data) { try { - var isExist = BaseIndexerHelper.IsExist.GetOrAdd(data.IndexName, (k) => Client.Instance.IndexExists(k).Exists); + var isExist = BaseIndexerHelper.IsExist.GetOrAdd(data.IndexName, (k) => Client.Instance.Indices.Exists(k).Exists); if (isExist) return true; lock (Locker) { if (isExist) return true; - isExist = Client.Instance.IndexExists(data.IndexName).Exists; + isExist = Client.Instance.Indices.Exists(data.IndexName).Exists; _ = BaseIndexerHelper.IsExist.TryUpdate(data.IndexName, IsExist, false); @@ -277,11 +277,11 @@ namespace ASC.ElasticSearch if (!CheckExist(data)) return; var result = false; - var currentMappings = Client.Instance.GetMapping(r => r.Index(data.IndexName)); + var currentMappings = Client.Instance.Indices.GetMapping(r => r.Index(data.IndexName)); var newMappings = GetMappings(data).Invoke(new CreateIndexDescriptor(data.IndexName)); var newMappingDict = new Dictionary(); - var props = newMappings.Mappings.SelectMany(r => r.Value.Properties).ToList(); + var props = newMappings.Mappings.Properties.ToList(); foreach (var prop in props.Where(r => r.Key.Property != null && r.Key.Property.Name != "Document")) { var propKey = prop.Key.Property.Name.ToLowerCamelCase(); @@ -303,7 +303,7 @@ namespace ASC.ElasticSearch foreach (var ind in currentMappings.Indices) { - foreach (var prop in ind.Value.Mappings.SelectMany(r => r.Value.Properties).Where(r => r.Key.Name != "document")) + foreach (var prop in ind.Value.Mappings.Properties.Where(r => r.Key.Name != "document")) { var key = ind.Key.Name + "." + prop.Key.Name.ToLowerCamelCase(); @@ -350,7 +350,7 @@ namespace ASC.ElasticSearch WebstudioDbContext.SaveChanges(); Log.DebugFormat("Delete {0}", Wrapper.IndexName); - Client.Instance.DeleteIndex(Wrapper.IndexName); + Client.Instance.Indices.Delete(Wrapper.IndexName); BaseIndexerHelper.Clear(Wrapper); CreateIfNotExist(ServiceProvider.GetService()); } @@ -396,11 +396,11 @@ namespace ASC.ElasticSearch if (!columns.Any() && !nestedColumns.Any()) { - Client.Instance.CreateIndex(data.IndexName); + Client.Instance.Indices.Create(data.IndexName); } else { - Client.Instance.CreateIndex(data.IndexName, GetMappings(data)); + Client.Instance.Indices.Create(data.IndexName, GetMappings(data)); } IsExist = true; diff --git a/common/services/ASC.ElasticSearch/Engine/Client.cs b/common/services/ASC.ElasticSearch/Engine/Client.cs index b47e1cd305..8bb8295ba2 100644 --- a/common/services/ASC.ElasticSearch/Engine/Client.cs +++ b/common/services/ASC.ElasticSearch/Engine/Client.cs @@ -107,7 +107,7 @@ namespace ASC.ElasticSearch if (result.IsValid) { - client.PutPipeline("attachments", p => p + client.Ingest.PutPipeline("attachments", p => p .Processors(pp => pp .Attachment(a => a.Field("document.data").TargetField("document.attachment")) )); diff --git a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs index 565b691daa..5c1869d715 100644 --- a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs @@ -507,12 +507,12 @@ namespace ASC.ElasticSearch public object GetState(TenantUtil tenantUtil) { var indices = CoreBaseSettings.Standalone ? - Client.Instance.CatIndices(new CatIndicesRequest { SortByColumns = new[] { "index" } }).Records.Select(r => new + Client.Instance.Cat.Indices(new CatIndicesRequest { SortByColumns = new[] { "index" } }).Records.Select(r => new { r.Index, r.DocsCount, r.StoreSize - }) : + }) : null; State state = null; @@ -534,7 +534,7 @@ namespace ASC.ElasticSearch return new { state, - indices, + //indices, status = CheckState() }; } From 57f4c3788ecfb48d543e41586ed210e771033a57 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Mon, 27 Apr 2020 14:27:58 +0300 Subject: [PATCH 03/20] ElasticSearch: refactoring. Wrapper replaced with ISearchItem --- .../ASC.ElasticSearch/Core/SearchSettings.cs | 2 +- .../ASC.ElasticSearch/Core/Selector.cs | 6 +- .../ASC.ElasticSearch/Core/Wrapper.cs | 24 +- .../ASC.ElasticSearch/Engine/BaseIndexer.cs | 259 +++++------------- .../Engine/FactoryIndexer.cs | 13 +- .../ASC.ElasticSearch/Engine/IIndexer.cs | 3 +- .../Server/Configuration/ProductEntryPoint.cs | 8 +- .../Server/Core/Dao/TeamlabDao/FileDao.cs | 92 +++++-- .../Server/Core/Dao/TeamlabDao/FolderDao.cs | 25 +- products/ASC.Files/Server/Core/EF/DbFile.cs | 32 ++- products/ASC.Files/Server/Core/EF/DbFolder.cs | 15 +- .../ASC.Files/Server/Core/EF/DbFolderTree.cs | 3 + .../Server/Core/FileStorageService.cs | 25 +- .../Server/Core/Search/FilesWrapper.cs | 5 +- .../Server/Core/Search/FoldersWrapper.cs | 5 +- .../ASC.Files.Service/ServiceLauncher.cs | 10 +- 16 files changed, 259 insertions(+), 268 deletions(-) diff --git a/common/services/ASC.ElasticSearch/Core/SearchSettings.cs b/common/services/ASC.ElasticSearch/Core/SearchSettings.cs index 4d5c00c3f2..47232d9270 100644 --- a/common/services/ASC.ElasticSearch/Core/SearchSettings.cs +++ b/common/services/ASC.ElasticSearch/Core/SearchSettings.cs @@ -145,7 +145,7 @@ namespace ASC.ElasticSearch.Core //} } - public bool CanSearchByContent(int tenantId) where T : Wrapper + public bool CanSearchByContent(int tenantId) where T : class, ISearchItem { if (!SearchByContentEnabled) return false; diff --git a/common/services/ASC.ElasticSearch/Core/Selector.cs b/common/services/ASC.ElasticSearch/Core/Selector.cs index 298c23b6a6..c11d0e6de3 100644 --- a/common/services/ASC.ElasticSearch/Core/Selector.cs +++ b/common/services/ASC.ElasticSearch/Core/Selector.cs @@ -29,13 +29,11 @@ using System.Globalization; using System.Linq; using System.Linq.Expressions; -using Microsoft.Extensions.DependencyInjection; - using Nest; namespace ASC.ElasticSearch { - public class Selector where T : Wrapper + public class Selector where T : class, ISearchItem { private readonly QueryContainerDescriptor queryContainerDescriptor = new QueryContainerDescriptor(); private SortDescriptor sortContainerDescriptor = new SortDescriptor(); @@ -199,7 +197,7 @@ namespace ASC.ElasticSearch public Selector MatchAll(string value) { - Match(() => ServiceProvider.GetService().GetContentProperties(), value); + //Match(() => ServiceProvider.GetService().GetContentProperties(), value); TODO: return this; } diff --git a/common/services/ASC.ElasticSearch/Core/Wrapper.cs b/common/services/ASC.ElasticSearch/Core/Wrapper.cs index bfd786a042..9c05d9cb2b 100644 --- a/common/services/ASC.ElasticSearch/Core/Wrapper.cs +++ b/common/services/ASC.ElasticSearch/Core/Wrapper.cs @@ -29,12 +29,26 @@ using System.Collections; using System.Collections.Generic; using System.Linq; using System.Reflection; + using ASC.ElasticSearch.Core; + using Nest; + using Newtonsoft.Json.Linq; namespace ASC.ElasticSearch { + public interface ISearchItem + { + public int Id { get; set; } + public int TenantId { get; set; } + public string IndexName { get; } + } + public interface ISearchItemDocument : ISearchItem + { + public Document Document { get; set; } + } + public abstract class Wrapper { protected internal abstract string Table { get; } @@ -129,7 +143,7 @@ namespace ASC.ElasticSearch .Where(r => r.GetCustomAttribute() != null) .ToList(); - if (!joins.Any()) return (Wrapper) result; + if (!joins.Any()) return (Wrapper)result; data = data.Skip(i).ToArray(); @@ -159,7 +173,7 @@ namespace ASC.ElasticSearch } else { - newArray = new List {data}; + newArray = new List { data }; } var newArrayValue = newArray.ConvertAll(joinWrapper.GetDataConverter(joinSub)); @@ -190,7 +204,7 @@ namespace ASC.ElasticSearch data = data.Skip(skipCount).ToArray(); } - return (Wrapper) result; + return (Wrapper)result; }; } @@ -285,7 +299,7 @@ namespace ASC.ElasticSearch var conditions = Columns.OfType(); - foreach (var con in conditions.Where(r=> r.Value != null)) + foreach (var con in conditions.Where(r => r.Value != null)) { result.Add(alias + "." + con.ColumnName, con.Value); } @@ -298,7 +312,7 @@ namespace ASC.ElasticSearch return char.ToLowerInvariant(value[0]) + value.Substring(1); } - internal Func, IPromise> GetProperties() where T : Wrapper + internal Func, IPromise> GetProperties() where T : class, ISearchItem { var analyzers = GetAnalyzers(); diff --git a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs index 67a8c367aa..8b3d543058 100644 --- a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs @@ -30,7 +30,6 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; -using System.Reflection; using System.Text; using System.Threading.Tasks; @@ -45,8 +44,6 @@ using ASC.ElasticSearch.Service; using Autofac; -using Elasticsearch.Net; - using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; @@ -69,13 +66,13 @@ namespace ASC.ElasticSearch }, CacheNotifyAction.Any); } - public void Clear(T t) where T : Wrapper + public void Clear(T t) where T : class, ISearchItem { Notify.Publish(new SearchItem() { Id = t.IndexName }, CacheNotifyAction.Any); } } - public abstract class BaseIndexer : IIndexer where T : Wrapper + public class BaseIndexer : IIndexer where T : class, ISearchItem { private static readonly object Locker = new object(); @@ -90,8 +87,8 @@ namespace ASC.ElasticSearch public SearchSettingsHelper SearchSettingsHelper { get; } public BaseIndexerHelper BaseIndexerHelper { get; } public IServiceProvider ServiceProvider { get; } - public WebstudioDbContext WebstudioDbContext { get; } - + public WebstudioDbContext WebstudioDbContext { get; } + public BaseIndexer( Client client, IOptionsMonitor log, @@ -114,12 +111,13 @@ namespace ASC.ElasticSearch { BeforeIndex(data); - Client.Instance.Index(data, idx => GetMeta(idx, data, immediately)); + var r = Client.Instance.Index(data, idx => GetMeta(idx, data, immediately)); + var d = 0; } internal void Index(List data, bool immediately = true) { - CreateIfNotExist(data[0]); + //CreateIfNotExist(data[0]); if (typeof(T).IsSubclassOf(typeof(WrapperWithDoc))) { @@ -204,25 +202,22 @@ namespace ASC.ElasticSearch internal void Update(T data, bool immediately = true, params Expression>[] fields) { - CreateIfNotExist(data); Client.Instance.Update(DocumentPath.Id(data), r => GetMetaForUpdate(r, data, immediately, fields)); } internal void Update(T data, UpdateAction action, Expression> fields, bool immediately = true) { - CreateIfNotExist(data); Client.Instance.Update(DocumentPath.Id(data), r => GetMetaForUpdate(r, data, action, fields, immediately)); } internal void Update(T data, Expression, Selector>> expression, int tenantId, bool immediately = true, params Expression>[] fields) { - CreateIfNotExist(data); + //CreateIfNotExist(data); Client.Instance.UpdateByQuery(GetDescriptorForUpdate(data, expression, tenantId, immediately, fields)); } internal void Update(T data, Expression, Selector>> expression, int tenantId, UpdateAction action, Expression> fields, bool immediately = true) { - CreateIfNotExist(data); Client.Instance.UpdateByQuery(GetDescriptorForUpdate(data, expression, tenantId, action, fields, immediately)); } @@ -271,67 +266,6 @@ namespace ASC.ElasticSearch return false; } - public void Check() - { - var data = ServiceProvider.GetService(); - if (!CheckExist(data)) return; - - var result = false; - var currentMappings = Client.Instance.Indices.GetMapping(r => r.Index(data.IndexName)); - var newMappings = GetMappings(data).Invoke(new CreateIndexDescriptor(data.IndexName)); - - var newMappingDict = new Dictionary(); - var props = newMappings.Mappings.Properties.ToList(); - foreach (var prop in props.Where(r => r.Key.Property != null && r.Key.Property.Name != "Document")) - { - var propKey = prop.Key.Property.Name.ToLowerCamelCase(); - var key = newMappings.Index.Name + "." + propKey; - if (prop.Key.Property.CustomAttributes.Any()) - { - newMappingDict.Add(key, props.Any(r => r.Key == propKey && r.Value is INestedProperty) ? FieldType.Nested.GetStringValue() : prop.Value.Type); - } - - - if (prop.Value is ObjectProperty obj) - { - foreach (var objProp in obj.Properties) - { - newMappingDict.Add(key + "." + objProp.Key.Property.Name.ToLowerCamelCase(), objProp.Value.Type); - } - } - } - - foreach (var ind in currentMappings.Indices) - { - foreach (var prop in ind.Value.Mappings.Properties.Where(r => r.Key.Name != "document")) - { - var key = ind.Key.Name + "." + prop.Key.Name.ToLowerCamelCase(); - - if (!newMappingDict.Contains(new KeyValuePair(key, prop.Value.Type))) - { - result = true; - break; - } - - var nested = prop.Value as NestedProperty ?? prop.Value as ObjectProperty; - - if (nested != null) - { - if (nested.Properties.Any(nProp => !newMappingDict.Contains(new KeyValuePair(key + "." + nProp.Key.Name.ToLowerCamelCase(), nProp.Value.Type)))) - { - result = true; - } - } - } - } - - - if (result) - { - Clear(); - } - } - public async Task ReIndex() { Clear(); @@ -352,7 +286,6 @@ namespace ASC.ElasticSearch Log.DebugFormat("Delete {0}", Wrapper.IndexName); Client.Instance.Indices.Delete(Wrapper.IndexName); BaseIndexerHelper.Clear(Wrapper); - CreateIfNotExist(ServiceProvider.GetService()); } internal IReadOnlyCollection Select(Expression, Selector>> expression, bool onlyId = false) @@ -375,128 +308,61 @@ namespace ASC.ElasticSearch private void BeforeIndex(T data) { - CreateIfNotExist(data); - if (data is WrapperWithDoc wrapperWithDoc) { wrapperWithDoc.InitDocument(SearchSettingsHelper.CanSearchByContent(data.TenantId), Log); } - } - - private void CreateIfNotExist(T data) - { - try - { - if (CheckExist(data)) return; - - lock (Locker) - { - var columns = data.GetAnalyzers(); - var nestedColumns = data.GetNested(); - - if (!columns.Any() && !nestedColumns.Any()) - { - Client.Instance.Indices.Create(data.IndexName); - } - else - { - Client.Instance.Indices.Create(data.IndexName, GetMappings(data)); - } - - IsExist = true; - } - } - catch (Exception e) - { - Log.Error("CreateIfNotExist", e); - } - } - - public Func GetMappings(T data) - { - var columns = data.GetAnalyzers(); - var nestedColumns = data.GetNested(); - - Func> analyzers = b => - { - foreach (var c in Enum.GetNames(typeof(Analyzer))) - { - var c1 = c; - b.Custom(c1 + "custom", ca => ca.Tokenizer(c1).Filters(Filter.lowercase.ToString()).CharFilters(CharFilter.io.ToString())); - } - - foreach (var c in columns) - { - if (c.Value.CharFilter == CharFilter.io) continue; - var charFilters = new List(); - foreach (var r in Enum.GetValues(typeof(CharFilter))) - { - if ((c.Value.CharFilter & (CharFilter)r) == (CharFilter)r) charFilters.Add(r.ToString()); - } - - var c1 = c; - b.Custom(c1.Key, ca => ca.Tokenizer(c1.Value.Analyzer.ToString()).Filters(c1.Value.Filter.ToString()).CharFilters(charFilters)); - } - - if (data is WrapperWithDoc) - { - b.Custom("document", ca => ca.Tokenizer(Analyzer.whitespace.ToString()).Filters(Filter.lowercase.ToString()).CharFilters(CharFilter.io.ToString())); - } - - return b; - }; - - Func, IPromise> nestedSelector = p => - { - foreach (var c in nestedColumns) - { - var isNested = c.Key.IsGenericType; - Type prop; - MethodInfo nested; - Type typeDescriptor; - - if (isNested) - { - prop = c.Key.GenericTypeArguments[0]; - nested = p.GetType().GetMethod("Nested"); - typeDescriptor = typeof(NestedPropertyDescriptor<,>); - } - else - { - prop = c.Key; - nested = p.GetType().GetMethod("Object"); - typeDescriptor = typeof(ObjectTypeDescriptor<,>); - } - - var desc = typeDescriptor.MakeGenericType(typeof(T), prop); - - var methods = desc.GetMethods(); - var name = methods.FirstOrDefault(r => r.Name == "Name" && r.GetParameters().FirstOrDefault(q => q.ParameterType == typeof(PropertyName)) != null); - var autoMap = methods.FirstOrDefault(r => r.Name == "AutoMap" && r.GetParameters().Length == 2); - var props = methods.FirstOrDefault(r => r.Name == "Properties"); - if (name == null || autoMap == null || props == null) continue; - - var param = Expression.Parameter(desc, "a"); - var nameFunc = Expression.Call(param, name, Expression.Constant(new PropertyName(c.Value.ToLowerCamelCase()))); //a.Name(value(Nest.PropertyName)) - var autoMapFunc = Expression.Call(param, autoMap, Expression.Constant(null, typeof(IPropertyVisitor)), Expression.Constant(0)); //a.AutoMap() - - var inst = (Wrapper)Activator.CreateInstance(prop); - var instMethods = prop.GetMethods(BindingFlags.Instance | BindingFlags.NonPublic); - var getProperties = instMethods.First(r => r.Name == "GetProperties").MakeGenericMethod(prop); - var propsFunc = Expression.Call(param, props, Expression.Constant(getProperties.Invoke(inst, null))); //a.AutoMap() - - var nestedFunc = Expression.Lambda(Expression.Block(nameFunc, autoMapFunc, propsFunc), param).Compile(); - var fooRef = nested.MakeGenericMethod(prop); - fooRef.Invoke(p, new object[] { nestedFunc });//p.Nested(r=> r.Name(c.Value.ToLowerCamelCase()).AutoMap().Properties(getProperties())) - } - - return p; - }; - - return c => - c.Settings(r => r.Analysis(a => a.Analyzers(analyzers).CharFilters(d => d.HtmlStrip(CharFilter.html.ToString()).Mapping(CharFilter.io.ToString(), m => m.Mappings("ё => е", "Ё => Е"))))) - .Mappings(r => r.Map(m => m.AutoMap().Properties(data.GetProperties()).Properties(nestedSelector))); - } + } + + public void CreateIfNotExist(T data) + { + try + { + if (CheckExist(data)) return; + + lock (Locker) + { + Func> analyzers = b => + { + foreach (var c in Enum.GetNames(typeof(Analyzer))) + { + var c1 = c; + b.Custom(c1 + "custom", ca => ca.Tokenizer(c1).Filters(Filter.lowercase.ToString()).CharFilters(CharFilter.io.ToString())); + } + + foreach (var c in Enum.GetNames(typeof(CharFilter))) + { + if (c == CharFilter.io.ToString()) continue; + + var charFilters = new List() { CharFilter.io.ToString(), c }; + var c1 = c; + b.Custom(c1 + "custom", ca => ca.Tokenizer(Analyzer.whitespace.ToString()).Filters(Filter.lowercase.ToString()).CharFilters(charFilters)); + } + + if (data is WrapperWithDoc) + { + b.Custom("document", ca => ca.Tokenizer(Analyzer.whitespace.ToString()).Filters(Filter.lowercase.ToString()).CharFilters(CharFilter.io.ToString())); + } + + return b; + }; + + var createIndexResponse = Client.Instance.Indices.Create(data.IndexName, + c => + c.Map(m => m.AutoMap()) + .Settings(r => r.Analysis(a => + a.Analyzers(analyzers) + .CharFilters(d => d.HtmlStrip(CharFilter.html.ToString()) + .Mapping(CharFilter.io.ToString(), m => m.Mappings("ё => е", "Ё => Е")))))); + + IsExist = true; + } + } + catch (Exception e) + { + Log.Error("CreateIfNotExist", e); + } + } private IIndexRequest GetMeta(IndexDescriptor request, T data, bool immediately = true) { @@ -709,7 +575,10 @@ namespace ASC.ElasticSearch return descriptor.GetDescriptorForUpdate(this, GetScriptForUpdate(data, action, fields), immediately); } - public abstract void IndexAll(); + public void IndexAll() + { + + } } static class CamelCaseExtension @@ -736,7 +605,7 @@ namespace ASC.ElasticSearch return services.AddKafkaService(); } - public static DIHelper AddBaseIndexerService(this DIHelper services) where T : Wrapper + public static DIHelper AddBaseIndexerService(this DIHelper services) where T : class, ISearchItem { services.TryAddScoped>(); diff --git a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs index 5c1869d715..7af0d5a209 100644 --- a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs @@ -64,7 +64,7 @@ namespace ASC.ElasticSearch FactoryIndexer = factoryIndexer; } - public bool Support(T t) where T : Wrapper + public bool Support(T t) where T : class, ISearchItem { if (!FactoryIndexer.CheckState()) return false; @@ -97,7 +97,7 @@ namespace ASC.ElasticSearch } - public class FactoryIndexer where T : Wrapper + public class FactoryIndexer where T : class, ISearchItem { private static readonly TaskScheduler Scheduler = new LimitedConcurrencyLevelTaskScheduler(10); @@ -128,7 +128,9 @@ namespace ASC.ElasticSearch FactoryIndexerCommon = factoryIndexer; Indexer = baseIndexer; Client = client; - ServiceProvider = serviceProvider; + ServiceProvider = serviceProvider; + + Indexer.CreateIfNotExist(ServiceProvider.GetService()); } public bool TrySelect(Expression, Selector>> expression, out IReadOnlyCollection result) @@ -572,9 +574,10 @@ namespace ASC.ElasticSearch .AddFactoryIndexerService(); } - public static DIHelper AddFactoryIndexerService(this DIHelper services) where T : Wrapper + public static DIHelper AddFactoryIndexerService(this DIHelper services) where T : class, ISearchItem { - services.TryAddScoped>(); + services.TryAddScoped>(); + services.TryAddScoped>(); return services .AddFactoryIndexerHelperService() diff --git a/common/services/ASC.ElasticSearch/Engine/IIndexer.cs b/common/services/ASC.ElasticSearch/Engine/IIndexer.cs index 907bcfc93e..4d35b01c06 100644 --- a/common/services/ASC.ElasticSearch/Engine/IIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/IIndexer.cs @@ -30,9 +30,8 @@ namespace ASC.ElasticSearch public interface IIndexer { string IndexName { get; } - void IndexAll(); - void Check(); + void IndexAll(); Task ReIndex(); } diff --git a/products/ASC.Files/Server/Configuration/ProductEntryPoint.cs b/products/ASC.Files/Server/Configuration/ProductEntryPoint.cs index c04aefcb2d..17f631a9e9 100644 --- a/products/ASC.Files/Server/Configuration/ProductEntryPoint.cs +++ b/products/ASC.Files/Server/Configuration/ProductEntryPoint.cs @@ -31,6 +31,8 @@ using System.Reflection; using ASC.Common; using ASC.Core; +using ASC.ElasticSearch; +using ASC.Files.Core.EF; using ASC.Files.Resources; using ASC.Web.Core; using ASC.Web.Core.PublicResources; @@ -46,6 +48,8 @@ namespace ASC.Web.Files.Configuration public CoreBaseSettings CoreBaseSettings { get; } public AuthContext AuthContext { get; } public UserManager UserManager { get; } + public IServiceProvider ServiceProvider { get; } + //public SubscriptionManager SubscriptionManager { get; } public ProductEntryPoint() @@ -97,6 +101,7 @@ namespace ASC.Web.Files.Configuration UserOpportunities = userOpportunities, CanNotBeDisabled = true, }; + //SearchHandlerManager.Registry(new SearchHandler()); } @@ -177,7 +182,8 @@ namespace ASC.Web.Files.Configuration .AddAuthContextService() .AddUserManagerService() .AddGlobalService() - .AddFilesSubscriptionManagerService(); + .AddFilesSubscriptionManagerService() + .AddFactoryIndexerService(); } } } \ No newline at end of file diff --git a/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs b/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs index 9e6c070831..07afdbaa33 100644 --- a/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs +++ b/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs @@ -55,8 +55,10 @@ namespace ASC.Files.Core.Data { internal class FileDao : AbstractDao, IFileDao { + public const long MaxContentLength = 2 * 1024 * 1024 * 1024L; + private static readonly object syncRoot = new object(); - public FactoryIndexer FactoryIndexer { get; } + public FactoryIndexer FactoryIndexer { get; } public GlobalStore GlobalStore { get; } public GlobalSpace GlobalSpace { get; } public GlobalFolder GlobalFolder { get; } @@ -66,7 +68,7 @@ namespace ASC.Files.Core.Data public CrossDao CrossDao { get; } public FileDao( - FactoryIndexer factoryIndexer, + FactoryIndexer factoryIndexer, UserManager userManager, DbContextManager dbContextManager, TenantManager tenantManager, @@ -252,7 +254,7 @@ namespace ASC.Files.Core.Data { var func = GetFuncForSearch(parentId, orderBy, filterType, subjectGroup, subjectID, searchText, searchInContent, withSubfolders); - Expression, Selector>> expression = s => func(s); + Expression, Selector>> expression = s => func(s); if (FactoryIndexer.TrySelectIds(expression, out var searchIds)) { @@ -368,6 +370,8 @@ namespace ASC.Files.Core.Data var isNew = false; List parentFoldersIds; + DbFile toInsert = null; + lock (syncRoot) { using var tx = FilesDbContext.Database.BeginTransaction(); @@ -400,7 +404,7 @@ namespace ASC.Files.Core.Data FilesDbContext.SaveChanges(); } - var toInsert = new DbFile + toInsert = new DbFile { Id = file.ID, Version = file.Version, @@ -428,14 +432,15 @@ namespace ASC.Files.Core.Data file.PureTitle = file.Title; - parentFoldersIds = + var parentFolders = FilesDbContext.Tree .Where(r => r.FolderId == file.FolderID) .OrderByDescending(r => r.Level) - .Select(r => r.ParentId) .ToList(); - if (parentFoldersIds.Count > 0) + parentFoldersIds = parentFolders.Select(r => r.ParentId).ToList(); + + if (parentFoldersIds.Any()) { var folderToUpdate = FilesDbContext.Folders .Where(r => parentFoldersIds.Any(a => a == r.Id)); @@ -449,6 +454,8 @@ namespace ASC.Files.Core.Data FilesDbContext.SaveChanges(); } + toInsert.Folders = parentFolders; + if (isNew) { RecalculateFilesCount(file.FolderID); @@ -476,7 +483,8 @@ namespace ASC.Files.Core.Data } } - FactoryIndexer.IndexAsync(FilesWrapper.GetFilesWrapper(ServiceProvider, file, parentFoldersIds)); + FactoryIndexer.IndexAsync(InitDocument(toInsert)); + //FactoryIndexer.IndexAsync(FilesWrapper.GetFilesWrapper(ServiceProvider, file, parentFoldersIds)); return GetFile(file.ID); } @@ -541,13 +549,14 @@ namespace ASC.Files.Core.Data file.PureTitle = file.Title; - parentFoldersIds = FilesDbContext.Tree + var parentFolders = FilesDbContext.Tree .Where(r => r.FolderId == file.FolderID) .OrderByDescending(r => r.Level) - .Select(r => r.ParentId) .ToList(); - if (parentFoldersIds.Count > 0) + parentFoldersIds = parentFolders.Select(r => r.ParentId).ToList(); + + if (parentFoldersIds.Any()) { var folderToUpdate = FilesDbContext.Folders .Where(r => parentFoldersIds.Any(a => a == r.Id)); @@ -560,6 +569,9 @@ namespace ASC.Files.Core.Data FilesDbContext.SaveChanges(); } + + toUpdate.Folders = parentFolders; + FactoryIndexer.IndexAsync(InitDocument(toUpdate)); } if (fileStream != null) @@ -579,7 +591,7 @@ namespace ASC.Files.Core.Data } } - FactoryIndexer.IndexAsync(FilesWrapper.GetFilesWrapper(ServiceProvider, file, parentFoldersIds)); + //FactoryIndexer.IndexAsync(FilesWrapper.GetFilesWrapper(ServiceProvider, file, parentFoldersIds)); return GetFile(file.ID); } @@ -635,6 +647,11 @@ namespace ASC.Files.Core.Data var toDeleteFiles = Query(FilesDbContext.Files).Where(r => r.Id == fileId); FilesDbContext.RemoveRange(toDeleteFiles); + foreach (var d in toDeleteFiles) + { + FactoryIndexer.DeleteAsync(d); + } + var toDeleteLinks = Query(FilesDbContext.TagLink).Where(r => r.EntryId == fileId.ToString()).Where(r => r.EntryType == FileEntryType.File); FilesDbContext.RemoveRange(toDeleteFiles); @@ -657,9 +674,9 @@ namespace ASC.Files.Core.Data if (deleteFolder) DeleteFolder(fileId); - var wrapper = ServiceProvider.GetService(); - wrapper.Id = fileId; - FactoryIndexer.DeleteAsync(wrapper); + //var wrapper = ServiceProvider.GetService(); + //wrapper.Id = fileId; + // FactoryIndexer.DeleteAsync(wrapper); } public bool IsExist(string title, object folderId) @@ -716,6 +733,10 @@ namespace ASC.Files.Core.Data f.ModifiedBy = AuthContext.CurrentAccount.ID; f.ModifiedOn = DateTime.UtcNow; } + + FactoryIndexer.Update(f, + UpdateAction.Replace, + w => w.Folders); } FilesDbContext.SaveChanges(); @@ -736,9 +757,9 @@ namespace ASC.Files.Core.Data wrapper.Id = fileId; wrapper.Folders = parentFoldersIds.Select(r => new FilesFoldersWrapper() { FolderId = r.ToString() }).ToList(); - FactoryIndexer.Update(wrapper, - UpdateAction.Replace, - w => w.Folders); + /* FactoryIndexer.Update(wrapper, + UpdateAction.Replace, + w => w.Folders); */ return fileId; } @@ -821,6 +842,8 @@ namespace ASC.Files.Core.Data FilesDbContext.SaveChanges(); + FactoryIndexer.UpdateAsync(toUpdate, true, r => r.Title, r => r.ModifiedBy, r => r.ModifiedOn); + return file.ID; } @@ -1171,7 +1194,7 @@ namespace ASC.Files.Core.Data #endregion - private Func, Selector> GetFuncForSearch(object parentId, OrderBy orderBy, FilterType filterType, bool subjectGroup, Guid subjectID, string searchText, bool searchInContent, bool withSubfolders = false) + private Func, Selector> GetFuncForSearch(object parentId, OrderBy orderBy, FilterType filterType, bool subjectGroup, Guid subjectID, string searchText, bool searchInContent, bool withSubfolders = false) { return s => { @@ -1183,11 +1206,11 @@ namespace ASC.Files.Core.Data { if (withSubfolders) { - result.In(a => a.Folders.Select(r => r.FolderId), new[] { parentId.ToString() }); + result.In(a => a.Folders.Select(r => r.ParentId), new[] { parentId }); } else { - result.InAll(a => a.Folders.Select(r => r.FolderId), new[] { parentId.ToString() }); + result.InAll(a => a.Folders.Select(r => r.ParentId), new[] { parentId }); } } @@ -1205,7 +1228,7 @@ namespace ASC.Files.Core.Data // result.Sort(r => r.Title, orderBy.IsAsc); // break; case SortedByType.DateAndTime: - result.Sort(r => r.LastModifiedOn, orderBy.IsAsc); + result.Sort(r => r.ModifiedOn, orderBy.IsAsc); break; case SortedByType.DateAndTimeCreation: result.Sort(r => r.CreateOn, orderBy.IsAsc); @@ -1310,6 +1333,29 @@ namespace ASC.Files.Core.Data file.Forcesave = r.file.Forcesave; return file; } + + private DbFile InitDocument(DbFile dbFile) + { + //if (FactoryIndexer.CanSearchByContent()) return dbFile; + + var file = ServiceProvider.GetService>(); + file.ID = dbFile.Id; + file.Title = dbFile.Title; + file.Version = dbFile.Version; + file.ContentLength = dbFile.ContentLength; + + if (!IsExistOnStorage(file) || file.ContentLength > MaxContentLength) return dbFile; + + using var stream = GetFileStream(file); + if (stream == null) return dbFile; + + dbFile.Document = new Document + { + Data = Convert.ToBase64String(stream.GetCorrectBuffer()) + }; + + return dbFile; + } } public class DbFileQuery @@ -1340,7 +1386,7 @@ namespace ASC.Files.Core.Data .AddAuthContextService() .AddGlobalStoreService() .AddGlobalSpaceService() - .AddFactoryIndexerService() + .AddFactoryIndexerService() .AddGlobalFolderService() .AddChunkedUploadSessionHolderService() .AddFolderDaoService(); diff --git a/products/ASC.Files/Server/Core/Dao/TeamlabDao/FolderDao.cs b/products/ASC.Files/Server/Core/Dao/TeamlabDao/FolderDao.cs index 0053a261dd..72c1eb2d86 100644 --- a/products/ASC.Files/Server/Core/Dao/TeamlabDao/FolderDao.cs +++ b/products/ASC.Files/Server/Core/Dao/TeamlabDao/FolderDao.cs @@ -41,7 +41,6 @@ using ASC.Files.Core.Thirdparty; using ASC.Files.Resources; using ASC.Files.Thirdparty.ProviderDao; using ASC.Web.Files.Classes; -using ASC.Web.Files.Core.Search; using ASC.Web.Studio.Core; using ASC.Web.Studio.UserControls.Statistics; using ASC.Web.Studio.Utility; @@ -58,14 +57,14 @@ namespace ASC.Files.Core.Data private const string trash = "trash"; private const string projects = "projects"; - public FactoryIndexer FactoryIndexer { get; } + public FactoryIndexer FactoryIndexer { get; } public GlobalSpace GlobalSpace { get; } public IDaoFactory DaoFactory { get; } public ProviderFolderDao ProviderFolderDao { get; } public CrossDao CrossDao { get; } public FolderDao( - FactoryIndexer factoryIndexer, + FactoryIndexer factoryIndexer, UserManager userManager, DbContextManager dbContextManager, TenantManager tenantManager, @@ -312,6 +311,8 @@ namespace ASC.Files.Core.Data toUpdate.ModifiedBy = folder.ModifiedBy; FilesDbContext.SaveChanges(); + + FactoryIndexer.IndexAsync(toUpdate); } else { @@ -331,6 +332,7 @@ namespace ASC.Files.Core.Data newFolder = FilesDbContext.Folders.Add(newFolder).Entity; FilesDbContext.SaveChanges(); + FactoryIndexer.IndexAsync(newFolder); folder.ID = newFolder.Id; //itself link @@ -371,7 +373,7 @@ namespace ASC.Files.Core.Data RecalculateFoldersCount(folder.ID); } - FactoryIndexer.IndexAsync(FoldersWrapper.GetFolderWrapper(ServiceProvider, folder)); + //FactoryIndexer.IndexAsync(FoldersWrapper.GetFolderWrapper(ServiceProvider, folder)); return folder.ID; } @@ -404,9 +406,14 @@ namespace ASC.Files.Core.Data var folderToDelete = Query(FilesDbContext.Folders).Where(r => subfolders.Any(a => r.Id == a)); FilesDbContext.Folders.RemoveRange(folderToDelete); + foreach (var f in folderToDelete) + { + FactoryIndexer.DeleteAsync(f); + } + var treeToDelete = FilesDbContext.Tree.Where(r => subfolders.Any(a => r.FolderId == a)); FilesDbContext.Tree.RemoveRange(treeToDelete); - + var subfoldersStrings = subfolders.Select(r => r.ToString()).ToList(); var linkToDelete = Query(FilesDbContext.TagLink) .Where(r => subfoldersStrings.Any(a => r.EntryId == a)) @@ -436,7 +443,7 @@ namespace ASC.Files.Core.Data RecalculateFoldersCount(parent); } - FactoryIndexer.DeleteAsync(new FoldersWrapper { Id = id }); + //FactoryIndexer.DeleteAsync(new FoldersWrapper { Id = id }); } public TTo MoveFolder(int folderId, TTo toFolderId, CancellationToken? cancellationToken) @@ -565,7 +572,7 @@ namespace ASC.Files.Core.Data copy = GetFolder(SaveFolder(copy)); - FactoryIndexer.IndexAsync(FoldersWrapper.GetFolderWrapper(ServiceProvider, copy)); + //FactoryIndexer.IndexAsync(FoldersWrapper.GetFolderWrapper(ServiceProvider, copy)); return copy; } @@ -664,6 +671,8 @@ namespace ASC.Files.Core.Data FilesDbContext.SaveChanges(); + FactoryIndexer.IndexAsync(toUpdate); + return folder.ID; } @@ -1143,7 +1152,7 @@ namespace ASC.Files.Core.Data services.TryAddTransient>(); return services - .AddFactoryIndexerService() + .AddFactoryIndexerService() .AddTenantManagerService() .AddUserManagerService() .AddFilesDbContextService() diff --git a/products/ASC.Files/Server/Core/EF/DbFile.cs b/products/ASC.Files/Server/Core/EF/DbFile.cs index 8169178086..4299f9c656 100644 --- a/products/ASC.Files/Server/Core/EF/DbFile.cs +++ b/products/ASC.Files/Server/Core/EF/DbFile.cs @@ -1,14 +1,28 @@ using System; +using System.Collections.Generic; using System.ComponentModel.DataAnnotations.Schema; using ASC.Core.Common.EF; +using ASC.ElasticSearch; using Microsoft.EntityFrameworkCore; +using Nest; + +using ColumnAttribute = System.ComponentModel.DataAnnotations.Schema.ColumnAttribute; + namespace ASC.Files.Core.EF { + public static class Tables + { + public const string File = "file"; + public const string Tree = "tree"; + public const string Folder = "folder"; + } + + [ElasticsearchType(RelationName = Tables.File)] [Table("files_file")] - public class DbFile : BaseEntity, IDbFile, IDbSearch + public class DbFile : BaseEntity, IDbFile, IDbSearch, ISearchItemDocument { public int Id { get; set; } public int Version { get; set; } @@ -22,6 +36,7 @@ namespace ASC.Files.Core.EF [Column("folder_id")] public int FolderId { get; set; } + [Text(Analyzer = "whitespacecustom")] public string Title { get; set; } [Column("content_length")] @@ -50,12 +65,25 @@ namespace ASC.Files.Core.EF [Column("converted_type")] public string ConvertedType { get; set; } - public string Comment { get; set; } public string Changes { get; set; } public bool Encrypted { get; set; } public ForcesaveType Forcesave { get; set; } + + [Nested] + [NotMapped] + public List Folders { get; set; } + + [NotMapped] + public string IndexName + { + get => Tables.File; + } + + [NotMapped] + public Document Document { get; set; } + public override object[] GetKeys() { return new object[] { TenantId, Id, Version }; diff --git a/products/ASC.Files/Server/Core/EF/DbFolder.cs b/products/ASC.Files/Server/Core/EF/DbFolder.cs index dd5563d64d..2826f217ab 100644 --- a/products/ASC.Files/Server/Core/EF/DbFolder.cs +++ b/products/ASC.Files/Server/Core/EF/DbFolder.cs @@ -1,10 +1,17 @@ using System; using System.ComponentModel.DataAnnotations.Schema; +using ASC.ElasticSearch; + +using Nest; + +using ColumnAttribute = System.ComponentModel.DataAnnotations.Schema.ColumnAttribute; + namespace ASC.Files.Core.EF { + [ElasticsearchType(RelationName = Tables.Folder)] [Table("files_folder")] - public class DbFolder : IDbFile, IDbSearch + public class DbFolder : IDbFile, IDbSearch, ISearchItem { public int Id { get; set; } @@ -32,5 +39,11 @@ namespace ASC.Files.Core.EF public int TenantId { get; set; } public int FoldersCount { get; set; } public int FilesCount { get; set; } + + [NotMapped] + public string IndexName + { + get => Tables.Folder; + } } } diff --git a/products/ASC.Files/Server/Core/EF/DbFolderTree.cs b/products/ASC.Files/Server/Core/EF/DbFolderTree.cs index b1c4b43f4b..8ec2738ae2 100644 --- a/products/ASC.Files/Server/Core/EF/DbFolderTree.cs +++ b/products/ASC.Files/Server/Core/EF/DbFolderTree.cs @@ -4,9 +4,12 @@ using ASC.Core.Common.EF; using Microsoft.EntityFrameworkCore; +using Nest; + namespace ASC.Files.Core.EF { + [ElasticsearchType(RelationName = Tables.Tree)] [Table("files_folder_tree")] public class DbFolderTree : BaseEntity { diff --git a/products/ASC.Files/Server/Core/FileStorageService.cs b/products/ASC.Files/Server/Core/FileStorageService.cs index 6c7c921a21..4ff7be6c9d 100644 --- a/products/ASC.Files/Server/Core/FileStorageService.cs +++ b/products/ASC.Files/Server/Core/FileStorageService.cs @@ -48,6 +48,7 @@ using ASC.ElasticSearch; using ASC.FederatedLogin.LoginProviders; using ASC.Files.Core; using ASC.Files.Core.Data; +using ASC.Files.Core.EF; using ASC.Files.Core.Security; using ASC.Files.Resources; using ASC.MessagingSystem; @@ -81,8 +82,8 @@ namespace ASC.Web.Files.Services.WCFService public FilesSettingsHelper FilesSettingsHelper { get; } public AuthContext AuthContext { get; } public UserManager UserManager { get; } - public FactoryIndexer FoldersIndexer { get; } - public FactoryIndexer FilesIndexer { get; } + public FactoryIndexer FoldersIndexer { get; } + public FactoryIndexer FilesIndexer { get; } public FileUtility FileUtility { get; } public FilesLinkUtility FilesLinkUtility { get; } public BaseCommonLinkUtility BaseCommonLinkUtility { get; } @@ -123,8 +124,8 @@ namespace ASC.Web.Files.Services.WCFService FilesSettingsHelper filesSettingsHelper, AuthContext authContext, UserManager userManager, - FactoryIndexer foldersIndexer, - FactoryIndexer filesIndexer, + FactoryIndexer foldersIndexer, + FactoryIndexer filesIndexer, FileUtility fileUtility, FilesLinkUtility filesLinkUtility, BaseCommonLinkUtility baseCommonLinkUtility, @@ -415,10 +416,10 @@ namespace ASC.Web.Files.Services.WCFService FilesMessageService.Send(folder, GetHttpHeaders(), MessageAction.FolderRenamed, folder.Title); - if (!folder.ProviderEntry) - { - FoldersIndexer.IndexAsync(FoldersWrapper.GetFolderWrapper(ServiceProvider, folder)); - } + //if (!folder.ProviderEntry) + //{ + // FoldersIndexer.IndexAsync(FoldersWrapper.GetFolderWrapper(ServiceProvider, folder)); + //} } var tag = tagDao.GetNewTags(AuthContext.CurrentAccount.ID, folder).FirstOrDefault(); @@ -745,10 +746,10 @@ namespace ASC.Web.Files.Services.WCFService { FilesMessageService.Send(file, GetHttpHeaders(), MessageAction.FileRenamed, file.Title); - if (!file.ProviderEntry) - { - FilesIndexer.UpdateAsync(FilesWrapper.GetFilesWrapper(ServiceProvider, file), true, r => r.Title); - } + //if (!file.ProviderEntry) + //{ + // FilesIndexer.UpdateAsync(FilesWrapper.GetFilesWrapper(ServiceProvider, file), true, r => r.Title); + //} } if (file.RootFolderType == FolderType.USER diff --git a/products/ASC.Files/Server/Core/Search/FilesWrapper.cs b/products/ASC.Files/Server/Core/Search/FilesWrapper.cs index 33abaea157..372c37822a 100644 --- a/products/ASC.Files/Server/Core/Search/FilesWrapper.cs +++ b/products/ASC.Files/Server/Core/Search/FilesWrapper.cs @@ -35,6 +35,7 @@ using ASC.ElasticSearch; using ASC.ElasticSearch.Core; using ASC.Files.Core; using ASC.Files.Core.Data; +using ASC.Files.Core.EF; using ASC.Files.Resources; using ASC.Web.Core.Files; @@ -163,12 +164,12 @@ namespace ASC.Web.Files.Core.Search { public static DIHelper AddFilesWrapperService(this DIHelper services) { - services.TryAddTransient(); + services.TryAddTransient(); return services .AddTenantManagerService() .AddFileUtilityService() .AddDaoFactoryService() - .AddFactoryIndexerService(); + .AddFactoryIndexerService(); } } } \ No newline at end of file diff --git a/products/ASC.Files/Server/Core/Search/FoldersWrapper.cs b/products/ASC.Files/Server/Core/Search/FoldersWrapper.cs index 5535c718bd..418d532d63 100644 --- a/products/ASC.Files/Server/Core/Search/FoldersWrapper.cs +++ b/products/ASC.Files/Server/Core/Search/FoldersWrapper.cs @@ -30,6 +30,7 @@ using ASC.Common; using ASC.Core; using ASC.ElasticSearch; using ASC.Files.Core; +using ASC.Files.Core.EF; using Microsoft.Extensions.DependencyInjection; @@ -61,9 +62,9 @@ namespace ASC.Web.Files.Core.Search { public static DIHelper AddFoldersWrapperService(this DIHelper services) { - services.TryAddTransient(); + services.TryAddTransient(); return services - .AddFactoryIndexerService(); + .AddFactoryIndexerService(); } } } \ No newline at end of file diff --git a/products/ASC.Files/Service/ASC.Files.Service/ServiceLauncher.cs b/products/ASC.Files/Service/ASC.Files.Service/ServiceLauncher.cs index 43428c070b..d87bbdf440 100644 --- a/products/ASC.Files/Service/ASC.Files.Service/ServiceLauncher.cs +++ b/products/ASC.Files/Service/ASC.Files.Service/ServiceLauncher.cs @@ -33,7 +33,7 @@ using ASC.Common; using ASC.Common.Caching; using ASC.Common.Logging; using ASC.ElasticSearch; -using ASC.Web.Files.Core.Search; +using ASC.Files.Core.EF; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -117,7 +117,7 @@ namespace ASC.Files.Service private void CheckIfChange() { using var scope = ServiceProvider.CreateScope(); - var filesWrapper = scope.ServiceProvider.GetService>(); + var filesWrapper = scope.ServiceProvider.GetService>(); IsStarted = true; var products = new[] { filesWrapper }.ToList(); @@ -130,7 +130,7 @@ namespace ASC.Files.Service Log.DebugFormat("Product check {0}", product.Indexer.IndexName); Indexing = product.Indexer.IndexName; - product.Indexer.Check(); + //product.Indexer.Check(); } catch (Exception e) { @@ -151,8 +151,8 @@ namespace ASC.Files.Service IsStarted = true; using var scope = ServiceProvider.CreateScope(); - var filesWrapper = scope.ServiceProvider.GetService>(); - var foldersWrapper = scope.ServiceProvider.GetService>(); + var filesWrapper = scope.ServiceProvider.GetService>(); + var foldersWrapper = scope.ServiceProvider.GetService>(); var products = new[] { (IIndexer)filesWrapper, (IIndexer)foldersWrapper }.ToList(); if (reindex) From b5671cc4603066820fb95c3563aaf94ab9146fef Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Mon, 27 Apr 2020 19:59:52 +0300 Subject: [PATCH 04/20] Files: Service --- .../EF/Context/TenantDbContext.cs | 6 +- .../EF/Model/DbWebstudioIndex.cs | 4 +- .../EF/Model/Tenant/DbTenant.cs | 15 +++ .../ASC.ElasticSearch/Core/Wrapper.cs | 3 +- .../ASC.ElasticSearch/Engine/BaseIndexer.cs | 47 +++++++- .../Engine/FactoryIndexer.cs | 17 ++- .../Server/Core/Dao/TeamlabDao/AbstractDao.cs | 2 +- .../Server/Core/Dao/TeamlabDao/FileDao.cs | 12 +- .../Server/Core/Dao/TeamlabDao/FolderDao.cs | 2 +- products/ASC.Files/Server/Core/EF/DbFolder.cs | 1 + .../Server/Core/EF/FilesDbContext.cs | 5 +- .../Server/Core/Search/FilesWrapper.cs | 74 +++++++++++- .../Server/Core/Search/FoldersWrapper.cs | 73 +++++++++++- .../Properties/launchSettings.json | 22 ++-- .../ASC.Files.Service/ServiceLauncher.cs | 106 ++++++------------ .../ASC.Files.Service/appsettings.json | 3 + 16 files changed, 283 insertions(+), 109 deletions(-) create mode 100644 products/ASC.Files/Service/ASC.Files.Service/appsettings.json diff --git a/common/ASC.Core.Common/EF/Context/TenantDbContext.cs b/common/ASC.Core.Common/EF/Context/TenantDbContext.cs index 8de2321661..fb6ede7cf4 100644 --- a/common/ASC.Core.Common/EF/Context/TenantDbContext.cs +++ b/common/ASC.Core.Common/EF/Context/TenantDbContext.cs @@ -28,11 +28,7 @@ namespace ASC.Core.Common.EF.Context modelBuilder.AddCoreSettings(); - modelBuilder.Entity() - .HasOne(r => r.Partner) - .WithOne(r => r.Tenant) - .HasForeignKey(r => new { r.TenantId }) - .HasPrincipalKey(r => new { r.Id }); + modelBuilder.AddDbTenant(); } } diff --git a/common/ASC.Core.Common/EF/Model/DbWebstudioIndex.cs b/common/ASC.Core.Common/EF/Model/DbWebstudioIndex.cs index 57bace117e..add86d763a 100644 --- a/common/ASC.Core.Common/EF/Model/DbWebstudioIndex.cs +++ b/common/ASC.Core.Common/EF/Model/DbWebstudioIndex.cs @@ -5,7 +5,7 @@ using System.ComponentModel.DataAnnotations.Schema; namespace ASC.Core.Common.EF.Model { [Table("webstudio_index")] - public class DbWebstudioIndex + public class DbWebstudioIndex : BaseEntity { [Key] [Column("index_name")] @@ -13,5 +13,7 @@ namespace ASC.Core.Common.EF.Model [Column("last_modified")] public DateTime LastModified { get; set; } + + public override object[] GetKeys() => new[] { IndexName }; } } diff --git a/common/ASC.Core.Common/EF/Model/Tenant/DbTenant.cs b/common/ASC.Core.Common/EF/Model/Tenant/DbTenant.cs index 06e07fedbf..53b25b73cf 100644 --- a/common/ASC.Core.Common/EF/Model/Tenant/DbTenant.cs +++ b/common/ASC.Core.Common/EF/Model/Tenant/DbTenant.cs @@ -1,7 +1,10 @@ using System; using System.ComponentModel.DataAnnotations.Schema; + using ASC.Core.Tenants; +using Microsoft.EntityFrameworkCore; + namespace ASC.Core.Common.EF.Model { [Table("tenants_tenants")] @@ -39,4 +42,16 @@ namespace ASC.Core.Common.EF.Model public DbTenantPartner Partner { get; set; } } + + public static class DbTenantExtension + { + public static void AddDbTenant(this ModelBuilder modelBuilder) + { + modelBuilder.Entity() + .HasOne(r => r.Partner) + .WithOne(r => r.Tenant) + .HasForeignKey(r => new { r.TenantId }) + .HasPrincipalKey(r => new { r.Id }); + } + } } diff --git a/common/services/ASC.ElasticSearch/Core/Wrapper.cs b/common/services/ASC.ElasticSearch/Core/Wrapper.cs index 9c05d9cb2b..78010d7965 100644 --- a/common/services/ASC.ElasticSearch/Core/Wrapper.cs +++ b/common/services/ASC.ElasticSearch/Core/Wrapper.cs @@ -43,7 +43,8 @@ namespace ASC.ElasticSearch public int Id { get; set; } public int TenantId { get; set; } public string IndexName { get; } - } + } + public interface ISearchItemDocument : ISearchItem { public Document Document { get; set; } diff --git a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs index 8b3d543058..ebac9e9cde 100644 --- a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs @@ -39,6 +39,7 @@ using ASC.Common.Logging; using ASC.Core; using ASC.Core.Common.EF; using ASC.Core.Common.EF.Context; +using ASC.Core.Common.EF.Model; using ASC.ElasticSearch.Core; using ASC.ElasticSearch.Service; @@ -72,13 +73,15 @@ namespace ASC.ElasticSearch } } - public class BaseIndexer : IIndexer where T : class, ISearchItem + public class BaseIndexer where T : class, ISearchItem { private static readonly object Locker = new object(); protected internal T Wrapper { get { return ServiceProvider.GetService(); } } public string IndexName { get { return Wrapper.IndexName; } } + + private const int QueryLimit = 1000; public bool IsExist { get; set; } public Client Client { get; } @@ -111,8 +114,7 @@ namespace ASC.ElasticSearch { BeforeIndex(data); - var r = Client.Instance.Index(data, idx => GetMeta(idx, data, immediately)); - var d = 0; + Client.Instance.Index(data, idx => GetMeta(idx, data, immediately)); } internal void Index(List data, bool immediately = true) @@ -575,9 +577,46 @@ namespace ASC.ElasticSearch return descriptor.GetDescriptorForUpdate(this, GetScriptForUpdate(data, action, fields), immediately); } - public void IndexAll() + public IEnumerable> IndexAll(Func getCount, Func> getData) { + var lastIndexed = WebstudioDbContext.WebstudioIndex + .Where(r => r.IndexName == Wrapper.IndexName) + .Select(r => r.LastModified) + .FirstOrDefault(); + var (count, max, min) = getCount(lastIndexed); + Log.Debug($"Index: {IndexName}, Count {count}, Max: {max}, Min: {min}"); + + if (count != 0) + { + var step = (max - min + 1) / count; + + if (step == 0) + { + step = 1; + } + + if (step < QueryLimit) + { + step = QueryLimit; + } + + for (var i = min; i <= max; i += step) + { + yield return getData(i, step, lastIndexed); + //FactoryIndexer.Index(data.Cast().ToList()); + } + } + + + WebstudioDbContext.AddOrUpdate(r => r.WebstudioIndex, new DbWebstudioIndex() + { + IndexName = Wrapper.IndexName + }); + + WebstudioDbContext.SaveChanges(); + + Log.Debug($"index completed {Wrapper.IndexName}"); } } diff --git a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs index 7af0d5a209..7c9451c59c 100644 --- a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs @@ -110,6 +110,8 @@ namespace ASC.ElasticSearch public BaseIndexer Indexer { get; } public Client Client { get; } public IServiceProvider ServiceProvider { get; } + public string IndexName { get => Indexer.IndexName; } + public FactoryIndexer( IOptionsMonitor options, @@ -425,6 +427,11 @@ namespace ASC.ElasticSearch task.ConfigureAwait(false); task.Start(Scheduler); return task; + } + + public virtual void IndexAll() + { + return; } } @@ -574,9 +581,13 @@ namespace ASC.ElasticSearch .AddFactoryIndexerService(); } - public static DIHelper AddFactoryIndexerService(this DIHelper services) where T : class, ISearchItem - { - services.TryAddScoped>(); + public static DIHelper AddFactoryIndexerService(this DIHelper services, bool addBase = true) where T : class, ISearchItem + { + if (addBase) + { + services.TryAddScoped>(); + } + services.TryAddScoped>(); return services diff --git a/products/ASC.Files/Server/Core/Dao/TeamlabDao/AbstractDao.cs b/products/ASC.Files/Server/Core/Dao/TeamlabDao/AbstractDao.cs index 8879e37d58..8def672f74 100644 --- a/products/ASC.Files/Server/Core/Dao/TeamlabDao/AbstractDao.cs +++ b/products/ASC.Files/Server/Core/Dao/TeamlabDao/AbstractDao.cs @@ -100,7 +100,7 @@ namespace ASC.Files.Core.Data return set.Where(r => r.TenantId == TenantID); } - protected IQueryable GetFileQuery(Expression> where) + protected internal IQueryable GetFileQuery(Expression> where) { return Query(FilesDbContext.Files) .Where(where); diff --git a/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs b/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs index 07afdbaa33..b4ee6be869 100644 --- a/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs +++ b/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs @@ -484,7 +484,6 @@ namespace ASC.Files.Core.Data } FactoryIndexer.IndexAsync(InitDocument(toInsert)); - //FactoryIndexer.IndexAsync(FilesWrapper.GetFilesWrapper(ServiceProvider, file, parentFoldersIds)); return GetFile(file.ID); } @@ -510,6 +509,8 @@ namespace ASC.Files.Core.Data } } + DbFile toUpdate = null; + List parentFoldersIds; lock (syncRoot) { @@ -524,7 +525,7 @@ namespace ASC.Files.Core.Data if (file.CreateBy == default) file.CreateBy = AuthContext.CurrentAccount.ID; if (file.CreateOn == default) file.CreateOn = TenantUtil.DateTimeNow(); - var toUpdate = FilesDbContext.Files + toUpdate = FilesDbContext.Files .Where(r => r.Id == file.ID && r.Version == file.Version) .FirstOrDefault(); @@ -571,7 +572,6 @@ namespace ASC.Files.Core.Data } toUpdate.Folders = parentFolders; - FactoryIndexer.IndexAsync(InitDocument(toUpdate)); } if (fileStream != null) @@ -591,7 +591,7 @@ namespace ASC.Files.Core.Data } } - //FactoryIndexer.IndexAsync(FilesWrapper.GetFilesWrapper(ServiceProvider, file, parentFoldersIds)); + FactoryIndexer.IndexAsync(InitDocument(toUpdate)); return GetFile(file.ID); } @@ -674,7 +674,7 @@ namespace ASC.Files.Core.Data if (deleteFolder) DeleteFolder(fileId); - //var wrapper = ServiceProvider.GetService(); + var wrapper = ServiceProvider.GetService(); //wrapper.Id = fileId; // FactoryIndexer.DeleteAsync(wrapper); } @@ -1336,7 +1336,7 @@ namespace ASC.Files.Core.Data private DbFile InitDocument(DbFile dbFile) { - //if (FactoryIndexer.CanSearchByContent()) return dbFile; + if (!FactoryIndexer.CanSearchByContent()) return dbFile; var file = ServiceProvider.GetService>(); file.ID = dbFile.Id; diff --git a/products/ASC.Files/Server/Core/Dao/TeamlabDao/FolderDao.cs b/products/ASC.Files/Server/Core/Dao/TeamlabDao/FolderDao.cs index 72c1eb2d86..870060aa3c 100644 --- a/products/ASC.Files/Server/Core/Dao/TeamlabDao/FolderDao.cs +++ b/products/ASC.Files/Server/Core/Dao/TeamlabDao/FolderDao.cs @@ -956,7 +956,7 @@ namespace ASC.Files.Core.Data #endregion - protected IQueryable GetFolderQuery(Expression> where = null) + protected internal IQueryable GetFolderQuery(Expression> where = null) { var q = Query(FilesDbContext.Folders); if (where != null) diff --git a/products/ASC.Files/Server/Core/EF/DbFolder.cs b/products/ASC.Files/Server/Core/EF/DbFolder.cs index 2826f217ab..3eaec58529 100644 --- a/products/ASC.Files/Server/Core/EF/DbFolder.cs +++ b/products/ASC.Files/Server/Core/EF/DbFolder.cs @@ -18,6 +18,7 @@ namespace ASC.Files.Core.EF [Column("parent_id")] public int ParentId { get; set; } + [Text(Analyzer = "whitespacecustom")] public string Title { get; set; } [Column("folder_type")] diff --git a/products/ASC.Files/Server/Core/EF/FilesDbContext.cs b/products/ASC.Files/Server/Core/EF/FilesDbContext.cs index 14dfb10f6c..4cc1e2b3f4 100644 --- a/products/ASC.Files/Server/Core/EF/FilesDbContext.cs +++ b/products/ASC.Files/Server/Core/EF/FilesDbContext.cs @@ -1,5 +1,6 @@ using ASC.Common; using ASC.Core.Common.EF; +using ASC.Core.Common.EF.Model; using Microsoft.EntityFrameworkCore; @@ -18,6 +19,7 @@ namespace ASC.Files.Core.EF public DbSet Tag { get; set; } public DbSet ThirdpartyApp { get; set; } public DbSet EncryptedData { get; set; } + public DbSet Tenants { get; set; } protected override void OnModelCreating(ModelBuilder modelBuilder) { @@ -29,7 +31,8 @@ namespace ASC.Files.Core.EF .AddDbFilesThirdpartyIdMapping() .AddDbFilesTagLink() .AddDbFilesThirdpartyApp() - .AddDbEncryptedData(); + .AddDbEncryptedData() + .AddDbTenant(); } } diff --git a/products/ASC.Files/Server/Core/Search/FilesWrapper.cs b/products/ASC.Files/Server/Core/Search/FilesWrapper.cs index 372c37822a..6bf0bacd82 100644 --- a/products/ASC.Files/Server/Core/Search/FilesWrapper.cs +++ b/products/ASC.Files/Server/Core/Search/FilesWrapper.cs @@ -30,6 +30,7 @@ using System.IO; using System.Linq; using ASC.Common; +using ASC.Common.Logging; using ASC.Core; using ASC.ElasticSearch; using ASC.ElasticSearch.Core; @@ -40,9 +41,71 @@ using ASC.Files.Resources; using ASC.Web.Core.Files; using Microsoft.Extensions.DependencyInjection; - +using Microsoft.Extensions.Options; + namespace ASC.Web.Files.Core.Search -{ +{ + public class FactoryIndexerFile : FactoryIndexer + { + public IDaoFactory DaoFactory { get; } + + public FactoryIndexerFile( + IOptionsMonitor options, + FactoryIndexerHelper factoryIndexerSupport, + TenantManager tenantManager, + SearchSettingsHelper searchSettingsHelper, + FactoryIndexer factoryIndexer, + BaseIndexer baseIndexer, + Client client, + IServiceProvider serviceProvider, + IDaoFactory daoFactory) + : base(options, factoryIndexerSupport, tenantManager, searchSettingsHelper, factoryIndexer, baseIndexer, client, serviceProvider) + { + DaoFactory = daoFactory; + } + + public override void IndexAll() + { + var fileDao = DaoFactory.GetFileDao() as FileDao; + + (int, int, int) getCount(DateTime lastIndexed) + { + var q = fileDao.GetFileQuery(r => r.ModifiedOn >= lastIndexed) + .Where(r => r.CurrentVersion) + .Join(fileDao.FilesDbContext.Tenants, r => r.TenantId, r => r.Id, (f, t) => new { f, t }) + .Where(r => r.t.Status == ASC.Core.Tenants.TenantStatus.Active); + + var count = q.GroupBy(a => a.f.Id).Count(); + var min = q.Min(r => r.f.Id); + var max = q.Max(r => r.f.Id); + + return (count, max, min); + } + + List getData(long i, long step, DateTime lastIndexed) => + fileDao.GetFileQuery(r => r.ModifiedOn >= lastIndexed) + .Where(r => r.CurrentVersion) + .Where(r => r.Id >= i && r.Id <= i + step) + .Join(fileDao.FilesDbContext.Tenants, r => r.TenantId, r => r.Id, (f, t) => new { f, t }) + .Where(r => r.t.Status == ASC.Core.Tenants.TenantStatus.Active) + .Select(r => r.f) + .ToList(); + + try + { + foreach (var data in Indexer.IndexAll(getCount, getData)) + { + Index(data); + } + } + catch (Exception e) + { + Logger.Error(e); + throw; + } + } + } + public sealed class FilesWrapper : WrapperWithDoc { [Column("title", 1)] @@ -165,11 +228,10 @@ namespace ASC.Web.Files.Core.Search public static DIHelper AddFilesWrapperService(this DIHelper services) { services.TryAddTransient(); + services.TryAddScoped, FactoryIndexerFile>(); + return services - .AddTenantManagerService() - .AddFileUtilityService() - .AddDaoFactoryService() - .AddFactoryIndexerService(); + .AddFactoryIndexerService(false); } } } \ No newline at end of file diff --git a/products/ASC.Files/Server/Core/Search/FoldersWrapper.cs b/products/ASC.Files/Server/Core/Search/FoldersWrapper.cs index 418d532d63..aab1e9042b 100644 --- a/products/ASC.Files/Server/Core/Search/FoldersWrapper.cs +++ b/products/ASC.Files/Server/Core/Search/FoldersWrapper.cs @@ -25,17 +25,82 @@ using System; +using System.Collections.Generic; +using System.Linq; using ASC.Common; +using ASC.Common.Logging; using ASC.Core; using ASC.ElasticSearch; +using ASC.ElasticSearch.Core; using ASC.Files.Core; +using ASC.Files.Core.Data; using ASC.Files.Core.EF; using Microsoft.Extensions.DependencyInjection; - +using Microsoft.Extensions.Options; + namespace ASC.Web.Files.Core.Search -{ +{ + public class FactoryIndexerFolder : FactoryIndexer + { + public IDaoFactory DaoFactory { get; } + + public FactoryIndexerFolder( + IOptionsMonitor options, + FactoryIndexerHelper factoryIndexerSupport, + TenantManager tenantManager, + SearchSettingsHelper searchSettingsHelper, + FactoryIndexer factoryIndexer, + BaseIndexer baseIndexer, + Client client, + IServiceProvider serviceProvider, + IDaoFactory daoFactory) + : base(options, factoryIndexerSupport, tenantManager, searchSettingsHelper, factoryIndexer, baseIndexer, client, serviceProvider) + { + DaoFactory = daoFactory; + } + + public override void IndexAll() + { + var folderDao = DaoFactory.GetFileDao() as FolderDao; + + (int, int, int) getCount(DateTime lastIndexed) + { + var q = folderDao.GetFolderQuery(r => r.ModifiedOn >= lastIndexed) + .Join(folderDao.FilesDbContext.Tenants, r => r.TenantId, r => r.Id, (f, t) => new { f, t }) + .Where(r => r.t.Status == ASC.Core.Tenants.TenantStatus.Active); + + var count = q.GroupBy(a => a.f.Id).Count(); + var min = q.Min(r => r.f.Id); + var max = q.Max(r => r.f.Id); + + return (count, max, min); + } + + List getData(long i, long step, DateTime lastIndexed) => + folderDao.GetFolderQuery(r => r.ModifiedOn >= lastIndexed) + .Where(r => r.Id >= i && r.Id <= i + step) + .Join(folderDao.FilesDbContext.Tenants, r => r.TenantId, r => r.Id, (f, t) => new { f, t }) + .Where(r => r.t.Status == ASC.Core.Tenants.TenantStatus.Active) + .Select(r => r.f) + .ToList(); + + try + { + foreach (var data in Indexer.IndexAll(getCount, getData)) + { + Index(data); + } + } + catch (Exception e) + { + Logger.Error(e); + throw; + } + } + } + public sealed class FoldersWrapper : Wrapper { [Column("title", 1)] @@ -63,8 +128,10 @@ namespace ASC.Web.Files.Core.Search public static DIHelper AddFoldersWrapperService(this DIHelper services) { services.TryAddTransient(); + services.TryAddScoped, FactoryIndexerFolder>(); + return services - .AddFactoryIndexerService(); + .AddFactoryIndexerService(false); } } } \ No newline at end of file diff --git a/products/ASC.Files/Service/ASC.Files.Service/Properties/launchSettings.json b/products/ASC.Files/Service/ASC.Files.Service/Properties/launchSettings.json index c0ed800a13..d85bbca027 100644 --- a/products/ASC.Files/Service/ASC.Files.Service/Properties/launchSettings.json +++ b/products/ASC.Files/Service/ASC.Files.Service/Properties/launchSettings.json @@ -3,25 +3,31 @@ "windowsAuthentication": false, "anonymousAuthentication": true, "iisExpress": { - "applicationUrl": "http://localhost:61034/", - "sslPort": 44308 + "applicationUrl": "http://localhost:5010", + "sslPort": 0 } }, "profiles": { "IIS Express": { "commandName": "IISExpress", - "launchBrowser": true, + "launchBrowser": false, "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" + "ASPNETCORE_ENVIRONMENT": "Development", + "$STORAGE_ROOT": "../../../Data", + "log__name": "files.service", + "log__dir": "../../../Logs" } }, "ASC.Files.Service": { "commandName": "Project", - "launchBrowser": true, + "launchBrowser": false, + "applicationUrl": "http://localhost:5010", "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - }, - "applicationUrl": "https://localhost:5001;http://localhost:5000" + "ASPNETCORE_ENVIRONMENT": "Development", + "$STORAGE_ROOT": "../../../Data", + "log__name": "files.service", + "log__dir": "../../../Logs" + } } } } \ No newline at end of file diff --git a/products/ASC.Files/Service/ASC.Files.Service/ServiceLauncher.cs b/products/ASC.Files/Service/ASC.Files.Service/ServiceLauncher.cs index d87bbdf440..5fc9d5cabc 100644 --- a/products/ASC.Files/Service/ASC.Files.Service/ServiceLauncher.cs +++ b/products/ASC.Files/Service/ASC.Files.Service/ServiceLauncher.cs @@ -25,7 +25,6 @@ using System; -using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -92,7 +91,8 @@ namespace ASC.Files.Service Thread.Sleep(10000); } - CheckIfChange(); + Timer = new Timer(_ => IndexAll(), null, TimeSpan.Zero, TimeSpan.Zero); + }, CancellationTokenSource.Token, TaskCreationOptions.LongRunning); task.Start(); @@ -114,37 +114,6 @@ namespace ASC.Files.Service return Task.CompletedTask; } - private void CheckIfChange() - { - using var scope = ServiceProvider.CreateScope(); - var filesWrapper = scope.ServiceProvider.GetService>(); - IsStarted = true; - - var products = new[] { filesWrapper }.ToList(); - - products.ForEach(product => - { - try - { - if (!IsStarted) return; - - Log.DebugFormat("Product check {0}", product.Indexer.IndexName); - Indexing = product.Indexer.IndexName; - //product.Indexer.Check(); - } - catch (Exception e) - { - Log.Error(e); - Log.ErrorFormat("Product check {0}", product.Indexer.IndexName); - } - }); - - IsStarted = false; - Indexing = null; - - Timer = new Timer(_ => IndexAll(), null, TimeSpan.Zero, TimeSpan.Zero); - } - private void IndexAll(bool reindex = false) { Timer.Change(-1, -1); @@ -153,49 +122,48 @@ namespace ASC.Files.Service using var scope = ServiceProvider.CreateScope(); var filesWrapper = scope.ServiceProvider.GetService>(); var foldersWrapper = scope.ServiceProvider.GetService>(); - var products = new[] { (IIndexer)filesWrapper, (IIndexer)foldersWrapper }.ToList(); - if (reindex) - { - products.ForEach(product => - { - try - { - if (!IsStarted) return; - - Log.DebugFormat("Product reindex {0}", product.IndexName); - product.ReIndex(); - } - catch (Exception e) - { - Log.Error(e); - Log.ErrorFormat("Product reindex {0}", product.IndexName); - } - }); - } - - products.ForEach(product => - { - try - { - if (!IsStarted) return; - - Log.DebugFormat("Product {0}", product.IndexName); - Indexing = product.IndexName; - product.IndexAll(); - } - catch (Exception e) - { - Log.Error(e); - Log.ErrorFormat("Product {0}", product.IndexName); - } - }); + IndexProduct(filesWrapper, reindex); + IndexProduct(foldersWrapper, reindex); Timer.Change(Period, Period); LastIndexed = DateTime.UtcNow; IsStarted = false; Indexing = null; } + + public void IndexProduct(FactoryIndexer product, bool reindex) where T : class, ISearchItem + { + if (reindex) + { + try + { + if (!IsStarted) return; + + Log.DebugFormat("Product reindex {0}", product.IndexName); + product.Indexer.ReIndex(); + } + catch (Exception e) + { + Log.Error(e); + Log.ErrorFormat("Product reindex {0}", product.IndexName); + } + } + + try + { + if (!IsStarted) return; + + Log.DebugFormat("Product {0}", product.IndexName); + Indexing = product.IndexName; + product.IndexAll(); + } + catch (Exception e) + { + Log.Error(e); + Log.ErrorFormat("Product {0}", product.IndexName); + } + } } public static class ServiceLauncherExtension diff --git a/products/ASC.Files/Service/ASC.Files.Service/appsettings.json b/products/ASC.Files/Service/ASC.Files.Service/appsettings.json new file mode 100644 index 0000000000..9bf702e310 --- /dev/null +++ b/products/ASC.Files/Service/ASC.Files.Service/appsettings.json @@ -0,0 +1,3 @@ +{ + "pathToConf": "..\\..\\..\\config" +} From c51b40247a2f0c8179d3f1ab499933538c71ace5 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Mon, 27 Apr 2020 20:02:15 +0300 Subject: [PATCH 05/20] Files: service replaced --- ASC.Web.sln | 10 +++++----- .../{ASC.Files.Service => }/ASC.Files.Service.csproj | 0 .../Service/{ASC.Files.Service => }/Program.cs | 0 .../Properties/launchSettings.json | 0 .../Service/{ASC.Files.Service => }/ServiceLauncher.cs | 0 .../Service/{ASC.Files.Service => }/appsettings.json | 0 6 files changed, 5 insertions(+), 5 deletions(-) rename products/ASC.Files/Service/{ASC.Files.Service => }/ASC.Files.Service.csproj (100%) rename products/ASC.Files/Service/{ASC.Files.Service => }/Program.cs (100%) rename products/ASC.Files/Service/{ASC.Files.Service => }/Properties/launchSettings.json (100%) rename products/ASC.Files/Service/{ASC.Files.Service => }/ServiceLauncher.cs (100%) rename products/ASC.Files/Service/{ASC.Files.Service => }/appsettings.json (100%) diff --git a/ASC.Web.sln b/ASC.Web.sln index 503eabd225..a619fa0a47 100644 --- a/ASC.Web.sln +++ b/ASC.Web.sln @@ -50,7 +50,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ASC.Files", "products\ASC.F EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AppLimit.CloudComputing.SharpBox", "thirdparty\AppLimit.CloudComputing.SharpBox\AppLimit.CloudComputing.SharpBox.csproj", "{5B53855C-4347-4402-B750-76C6295A35D3}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ASC.Files.Service", "products\ASC.Files\Service\ASC.Files.Service\ASC.Files.Service.csproj", "{725944D5-FDDC-45A9-802C-E2724DBD29F8}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ASC.Files.Service", "products\ASC.Files\Service\ASC.Files.Service.csproj", "{5D41FFFF-816C-40B2-95CD-E2DDDCB83784}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -146,10 +146,10 @@ Global {5B53855C-4347-4402-B750-76C6295A35D3}.Debug|Any CPU.Build.0 = Debug|Any CPU {5B53855C-4347-4402-B750-76C6295A35D3}.Release|Any CPU.ActiveCfg = Release|Any CPU {5B53855C-4347-4402-B750-76C6295A35D3}.Release|Any CPU.Build.0 = Release|Any CPU - {725944D5-FDDC-45A9-802C-E2724DBD29F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {725944D5-FDDC-45A9-802C-E2724DBD29F8}.Debug|Any CPU.Build.0 = Debug|Any CPU - {725944D5-FDDC-45A9-802C-E2724DBD29F8}.Release|Any CPU.ActiveCfg = Release|Any CPU - {725944D5-FDDC-45A9-802C-E2724DBD29F8}.Release|Any CPU.Build.0 = Release|Any CPU + {5D41FFFF-816C-40B2-95CD-E2DDDCB83784}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5D41FFFF-816C-40B2-95CD-E2DDDCB83784}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5D41FFFF-816C-40B2-95CD-E2DDDCB83784}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5D41FFFF-816C-40B2-95CD-E2DDDCB83784}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/products/ASC.Files/Service/ASC.Files.Service/ASC.Files.Service.csproj b/products/ASC.Files/Service/ASC.Files.Service.csproj similarity index 100% rename from products/ASC.Files/Service/ASC.Files.Service/ASC.Files.Service.csproj rename to products/ASC.Files/Service/ASC.Files.Service.csproj diff --git a/products/ASC.Files/Service/ASC.Files.Service/Program.cs b/products/ASC.Files/Service/Program.cs similarity index 100% rename from products/ASC.Files/Service/ASC.Files.Service/Program.cs rename to products/ASC.Files/Service/Program.cs diff --git a/products/ASC.Files/Service/ASC.Files.Service/Properties/launchSettings.json b/products/ASC.Files/Service/Properties/launchSettings.json similarity index 100% rename from products/ASC.Files/Service/ASC.Files.Service/Properties/launchSettings.json rename to products/ASC.Files/Service/Properties/launchSettings.json diff --git a/products/ASC.Files/Service/ASC.Files.Service/ServiceLauncher.cs b/products/ASC.Files/Service/ServiceLauncher.cs similarity index 100% rename from products/ASC.Files/Service/ASC.Files.Service/ServiceLauncher.cs rename to products/ASC.Files/Service/ServiceLauncher.cs diff --git a/products/ASC.Files/Service/ASC.Files.Service/appsettings.json b/products/ASC.Files/Service/appsettings.json similarity index 100% rename from products/ASC.Files/Service/ASC.Files.Service/appsettings.json rename to products/ASC.Files/Service/appsettings.json From adab38548a3276b5904f0f2183a818d216746fa3 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Tue, 28 Apr 2020 18:11:10 +0300 Subject: [PATCH 06/20] Files: Index Service --- .../Context/Impl/TenantManager.cs | 11 ++++++++- .../Context/SecurityContext.cs | 15 +++++++++++- .../EF/Context/DbContextManager.cs | 4 +++- .../Security/Authentication/CookieStorage.cs | 12 ++++++++-- .../Settings/SettingsManager.cs | 4 +++- .../Configuration/StorageSettings.cs | 17 ++++++++++++-- common/ASC.Data.Storage/WebPath.cs | 18 ++++++++++++--- common/ASC.MessagingSystem/MessageService.cs | 14 +++++++++-- .../ASC.ElasticSearch/Engine/BaseIndexer.cs | 23 +++---------------- .../Engine/FactoryIndexer.cs | 16 +++++++++++-- .../Server/Configuration/ProductEntryPoint.cs | 5 ++-- .../Server/Core/Dao/TeamlabDao/AbstractDao.cs | 6 +++-- .../Server/Core/Dao/TeamlabDao/FileDao.cs | 4 ++-- .../Server/Core/Search/FilesWrapper.cs | 12 ++++++++-- .../Server/Core/Search/FoldersWrapper.cs | 10 +++++--- .../Core/Thirdparty/ProviderAccountDao.cs | 6 +++-- .../Thirdparty/ProviderDao/ProviderDaoBase.cs | 6 +++-- .../Service/ASC.Files.Service.csproj | 8 +++---- products/ASC.Files/Service/ServiceLauncher.cs | 17 ++++++++++---- 19 files changed, 148 insertions(+), 60 deletions(-) diff --git a/common/ASC.Core.Common/Context/Impl/TenantManager.cs b/common/ASC.Core.Common/Context/Impl/TenantManager.cs index 700cc35b1b..72de85770d 100644 --- a/common/ASC.Core.Common/Context/Impl/TenantManager.cs +++ b/common/ASC.Core.Common/Context/Impl/TenantManager.cs @@ -128,7 +128,6 @@ namespace ASC.Core ITenantService tenantService, IQuotaService quotaService, ITariffService tariffService, - IHttpContextAccessor httpContextAccessor, CoreBaseSettings coreBaseSettings, CoreSettings coreSettings) { @@ -137,6 +136,16 @@ namespace ASC.Core TariffService = tariffService; CoreBaseSettings = coreBaseSettings; CoreSettings = coreSettings; + } + + public TenantManager( + ITenantService tenantService, + IQuotaService quotaService, + ITariffService tariffService, + IHttpContextAccessor httpContextAccessor, + CoreBaseSettings coreBaseSettings, + CoreSettings coreSettings) : this(tenantService, quotaService, tariffService, coreBaseSettings, coreSettings) + { HttpContext = httpContextAccessor?.HttpContext; } diff --git a/common/ASC.Core.Common/Context/SecurityContext.cs b/common/ASC.Core.Common/Context/SecurityContext.cs index 55f6108890..4690593eab 100644 --- a/common/ASC.Core.Common/Context/SecurityContext.cs +++ b/common/ASC.Core.Common/Context/SecurityContext.cs @@ -76,7 +76,6 @@ namespace ASC.Core private IHttpContextAccessor HttpContextAccessor { get; } public SecurityContext( - IHttpContextAccessor httpContextAccessor, UserManager userManager, AuthManager authentication, AuthContext authContext, @@ -95,6 +94,20 @@ namespace ASC.Core UserFormatter = userFormatter; CookieStorage = cookieStorage; TenantCookieSettingsHelper = tenantCookieSettingsHelper; + } + + public SecurityContext( + IHttpContextAccessor httpContextAccessor, + UserManager userManager, + AuthManager authentication, + AuthContext authContext, + TenantManager tenantManager, + UserFormatter userFormatter, + CookieStorage cookieStorage, + TenantCookieSettingsHelper tenantCookieSettingsHelper, + IOptionsMonitor options + ) : this(userManager, authentication, authContext, tenantManager, userFormatter, cookieStorage, tenantCookieSettingsHelper, options) + { HttpContextAccessor = httpContextAccessor; } diff --git a/common/ASC.Core.Common/EF/Context/DbContextManager.cs b/common/ASC.Core.Common/EF/Context/DbContextManager.cs index bbfb652b68..b57e91fda7 100644 --- a/common/ASC.Core.Common/EF/Context/DbContextManager.cs +++ b/common/ASC.Core.Common/EF/Context/DbContextManager.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using ASC.Common; +using ASC.Common.Logging; + using Microsoft.Extensions.Options; namespace ASC.Core.Common.EF @@ -77,7 +79,7 @@ namespace ASC.Core.Common.EF services.TryAddScoped>, ConfigureMultiRegionalDbContext>(); //services.TryAddScoped(); - return services; + return services.AddLoggerService(); } } } diff --git a/common/ASC.Core.Common/Security/Authentication/CookieStorage.cs b/common/ASC.Core.Common/Security/Authentication/CookieStorage.cs index 60f60c52c6..cf8a14d90d 100644 --- a/common/ASC.Core.Common/Security/Authentication/CookieStorage.cs +++ b/common/ASC.Core.Common/Security/Authentication/CookieStorage.cs @@ -48,17 +48,25 @@ namespace ASC.Core.Security.Authentication private ILog Log { get; } public CookieStorage( - IHttpContextAccessor httpContextAccessor, InstanceCrypto instanceCrypto, TenantCookieSettingsHelper tenantCookieSettingsHelper, IOptionsMonitor options) { InstanceCrypto = instanceCrypto; TenantCookieSettingsHelper = tenantCookieSettingsHelper; - HttpContext = httpContextAccessor.HttpContext; Log = options.CurrentValue; } + public CookieStorage( + IHttpContextAccessor httpContextAccessor, + InstanceCrypto instanceCrypto, + TenantCookieSettingsHelper tenantCookieSettingsHelper, + IOptionsMonitor options) + : this(instanceCrypto, tenantCookieSettingsHelper, options) + { + HttpContext = httpContextAccessor.HttpContext; + } + public bool DecryptCookie(string cookie, out int tenant, out Guid userid, out string login, out string password, out int indexTenant, out DateTime expire, out int indexUser) { tenant = Tenant.DEFAULT_TENANT; diff --git a/common/ASC.Core.Common/Settings/SettingsManager.cs b/common/ASC.Core.Common/Settings/SettingsManager.cs index f512752268..05eea6a6f1 100644 --- a/common/ASC.Core.Common/Settings/SettingsManager.cs +++ b/common/ASC.Core.Common/Settings/SettingsManager.cs @@ -57,7 +57,9 @@ namespace ASC.Core.Common.Settings { services.TryAddScoped(); - return services.AddDbSettingsManagerService(); + return services + .AddAuthContextService() + .AddDbSettingsManagerService(); } } } \ No newline at end of file diff --git a/common/ASC.Data.Storage/Configuration/StorageSettings.cs b/common/ASC.Data.Storage/Configuration/StorageSettings.cs index f3c150718d..d63422c502 100644 --- a/common/ASC.Data.Storage/Configuration/StorageSettings.cs +++ b/common/ASC.Data.Storage/Configuration/StorageSettings.cs @@ -133,7 +133,6 @@ namespace ASC.Data.Storage.Configuration IOptionsMonitor options, TenantManager tenantManager, SettingsManager settingsManager, - IHttpContextAccessor httpContextAccessor, ConsumerFactory consumerFactory) { BaseStorageSettingsListener = baseStorageSettingsListener; @@ -144,9 +143,23 @@ namespace ASC.Data.Storage.Configuration Options = options; TenantManager = tenantManager; SettingsManager = settingsManager; - HttpContextAccessor = httpContextAccessor; ConsumerFactory = consumerFactory; } + public StorageSettingsHelper( + BaseStorageSettingsListener baseStorageSettingsListener, + StorageFactoryConfig storageFactoryConfig, + PathUtils pathUtils, + EmailValidationKeyProvider emailValidationKeyProvider, + ICacheNotify cache, + IOptionsMonitor options, + TenantManager tenantManager, + SettingsManager settingsManager, + IHttpContextAccessor httpContextAccessor, + ConsumerFactory consumerFactory) + : this(baseStorageSettingsListener, storageFactoryConfig, pathUtils, emailValidationKeyProvider, cache, options, tenantManager, settingsManager, consumerFactory) + { + HttpContextAccessor = httpContextAccessor; + } public bool Save(BaseStorageSettings baseStorageSettings) where T : class, ISettings, new() { diff --git a/common/ASC.Data.Storage/WebPath.cs b/common/ASC.Data.Storage/WebPath.cs index c08d5be376..a2d9d9c599 100644 --- a/common/ASC.Data.Storage/WebPath.cs +++ b/common/ASC.Data.Storage/WebPath.cs @@ -159,7 +159,6 @@ namespace ASC.Data.Storage StaticUploader staticUploader, SettingsManager settingsManager, StorageSettingsHelper storageSettingsHelper, - IHttpContextAccessor httpContextAccessor, IHostEnvironment hostEnvironment, CoreBaseSettings coreBaseSettings, IOptionsMonitor options) @@ -168,12 +167,25 @@ namespace ASC.Data.Storage StaticUploader = staticUploader; SettingsManager = settingsManager; StorageSettingsHelper = storageSettingsHelper; - HttpContextAccessor = httpContextAccessor; HostEnvironment = hostEnvironment; CoreBaseSettings = coreBaseSettings; Options = options; } + public WebPath( + WebPathSettings webPathSettings, + StaticUploader staticUploader, + SettingsManager settingsManager, + StorageSettingsHelper storageSettingsHelper, + IHttpContextAccessor httpContextAccessor, + IHostEnvironment hostEnvironment, + CoreBaseSettings coreBaseSettings, + IOptionsMonitor options) + : this(webPathSettings, staticUploader, settingsManager, storageSettingsHelper, hostEnvironment, coreBaseSettings, options) + { + HttpContextAccessor = httpContextAccessor; + } + public string GetPath(string relativePath) { if (!string.IsNullOrEmpty(relativePath) && relativePath.IndexOf('~') == 0) @@ -194,7 +206,7 @@ namespace ASC.Data.Storage } } - return WebPathSettings.GetPath(HttpContextAccessor.HttpContext, Options, relativePath); + return WebPathSettings.GetPath(HttpContextAccessor?.HttpContext, Options, relativePath); } public bool Exists(string relativePath) diff --git a/common/ASC.MessagingSystem/MessageService.cs b/common/ASC.MessagingSystem/MessageService.cs index f9935769ab..c756848ec4 100644 --- a/common/ASC.MessagingSystem/MessageService.cs +++ b/common/ASC.MessagingSystem/MessageService.cs @@ -48,7 +48,6 @@ namespace ASC.MessagingSystem public MessageService( IConfiguration configuration, - IHttpContextAccessor httpContextAccessor, MessageFactory messageFactory, DbMessageSender sender, MessagePolicy messagePolicy, @@ -61,11 +60,22 @@ namespace ASC.MessagingSystem this.sender = sender; MessagePolicy = messagePolicy; - request = httpContextAccessor?.HttpContext?.Request; MessageFactory = messageFactory; log = options.CurrentValue; } + public MessageService( + IConfiguration configuration, + IHttpContextAccessor httpContextAccessor, + MessageFactory messageFactory, + DbMessageSender sender, + MessagePolicy messagePolicy, + IOptionsMonitor options) + : this(configuration, messageFactory, sender, messagePolicy, options) + { + request = httpContextAccessor?.HttpContext?.Request; + } + #region HttpRequest public void Send(MessageAction action) diff --git a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs index ebac9e9cde..6e674746cf 100644 --- a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs @@ -112,16 +112,15 @@ namespace ASC.ElasticSearch internal void Index(T data, bool immediately = true) { - BeforeIndex(data); - Client.Instance.Index(data, idx => GetMeta(idx, data, immediately)); } internal void Index(List data, bool immediately = true) { //CreateIfNotExist(data[0]); + if (!data.Any()) return; - if (typeof(T).IsSubclassOf(typeof(WrapperWithDoc))) + if (data[0] is ISearchItemDocument) { var currentLength = 0L; var portion = new List(); @@ -132,9 +131,6 @@ namespace ASC.ElasticSearch var t = data[i]; var runBulk = i == data.Count - 1; - BeforeIndex(t); - - if (!(t is WrapperWithDoc wwd) || wwd.Document == null || string.IsNullOrEmpty(wwd.Document.Data)) { portion.Add(t); @@ -177,7 +173,7 @@ namespace ASC.ElasticSearch Client.Instance.Bulk(r => r.IndexMany(portion1, GetMeta)); for (var j = portionStart; j < i; j++) { - if (data[j] is WrapperWithDoc doc && doc.Document != null) + if (data[j] is ISearchItemDocument doc && doc.Document != null) { doc.Document.Data = null; doc.Document = null; @@ -193,11 +189,6 @@ namespace ASC.ElasticSearch } else { - foreach (var item in data) - { - BeforeIndex(item); - } - Client.Instance.Bulk(r => r.IndexMany(data, GetMeta)); } } @@ -306,14 +297,6 @@ namespace ASC.ElasticSearch var result = Client.Instance.Search(descriptor.GetDescriptor(this, onlyId)); total = result.Total; return result.Documents; - } - - private void BeforeIndex(T data) - { - if (data is WrapperWithDoc wrapperWithDoc) - { - wrapperWithDoc.InitDocument(SearchSettingsHelper.CanSearchByContent(data.TenantId), Log); - } } public void CreateIfNotExist(T data) diff --git a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs index 7c9451c59c..987b21da59 100644 --- a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs @@ -96,8 +96,15 @@ namespace ASC.ElasticSearch } } + + public interface IFactoryIndexer + { + void IndexAll(); + string IndexName { get; } + void ReIndex(); + } - public class FactoryIndexer where T : class, ISearchItem + public class FactoryIndexer : IFactoryIndexer where T : class, ISearchItem { private static readonly TaskScheduler Scheduler = new LimitedConcurrencyLevelTaskScheduler(10); @@ -432,7 +439,12 @@ namespace ASC.ElasticSearch public virtual void IndexAll() { return; - } + } + + public void ReIndex() + { + Indexer.ReIndex(); + } } public class FactoryIndexer diff --git a/products/ASC.Files/Server/Configuration/ProductEntryPoint.cs b/products/ASC.Files/Server/Configuration/ProductEntryPoint.cs index 17f631a9e9..682c83877e 100644 --- a/products/ASC.Files/Server/Configuration/ProductEntryPoint.cs +++ b/products/ASC.Files/Server/Configuration/ProductEntryPoint.cs @@ -31,12 +31,11 @@ using System.Reflection; using ASC.Common; using ASC.Core; -using ASC.ElasticSearch; -using ASC.Files.Core.EF; using ASC.Files.Resources; using ASC.Web.Core; using ASC.Web.Core.PublicResources; using ASC.Web.Files.Classes; +using ASC.Web.Files.Core.Search; namespace ASC.Web.Files.Configuration { @@ -183,7 +182,7 @@ namespace ASC.Web.Files.Configuration .AddUserManagerService() .AddGlobalService() .AddFilesSubscriptionManagerService() - .AddFactoryIndexerService(); + .AddFilesWrapperService(); } } } \ No newline at end of file diff --git a/products/ASC.Files/Server/Core/Dao/TeamlabDao/AbstractDao.cs b/products/ASC.Files/Server/Core/Dao/TeamlabDao/AbstractDao.cs index 8def672f74..9aec72a785 100644 --- a/products/ASC.Files/Server/Core/Dao/TeamlabDao/AbstractDao.cs +++ b/products/ASC.Files/Server/Core/Dao/TeamlabDao/AbstractDao.cs @@ -53,8 +53,10 @@ namespace ASC.Files.Core.Data public FilesDbContext FilesDbContext { get; } - protected internal int TenantID { get; } + private int tenantID; + protected internal int TenantID { get => tenantID != 0 ? tenantID : (tenantID = TenantManager.GetCurrentTenant().TenantId); } public UserManager UserManager { get; } + public TenantManager TenantManager { get; } public TenantUtil TenantUtil { get; } public SetupInfo SetupInfo { get; } public TenantExtra TenantExtra { get; } @@ -81,8 +83,8 @@ namespace ASC.Files.Core.Data { cache = AscCache.Memory; FilesDbContext = dbContextManager.Get(FileConstant.DatabaseId); - TenantID = tenantManager.GetCurrentTenant().TenantId; UserManager = userManager; + TenantManager = tenantManager; TenantUtil = tenantUtil; SetupInfo = setupInfo; TenantExtra = tenantExtra; diff --git a/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs b/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs index b4ee6be869..c30785d2a2 100644 --- a/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs +++ b/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs @@ -1334,7 +1334,7 @@ namespace ASC.Files.Core.Data return file; } - private DbFile InitDocument(DbFile dbFile) + internal protected DbFile InitDocument(DbFile dbFile) { if (!FactoryIndexer.CanSearchByContent()) return dbFile; @@ -1386,7 +1386,7 @@ namespace ASC.Files.Core.Data .AddAuthContextService() .AddGlobalStoreService() .AddGlobalSpaceService() - .AddFactoryIndexerService() + .AddFilesWrapperService() .AddGlobalFolderService() .AddChunkedUploadSessionHolderService() .AddFolderDaoService(); diff --git a/products/ASC.Files/Server/Core/Search/FilesWrapper.cs b/products/ASC.Files/Server/Core/Search/FilesWrapper.cs index 6bf0bacd82..c9ef208ece 100644 --- a/products/ASC.Files/Server/Core/Search/FilesWrapper.cs +++ b/products/ASC.Files/Server/Core/Search/FilesWrapper.cs @@ -70,7 +70,8 @@ namespace ASC.Web.Files.Core.Search (int, int, int) getCount(DateTime lastIndexed) { - var q = fileDao.GetFileQuery(r => r.ModifiedOn >= lastIndexed) + var q = fileDao.FilesDbContext.Files + .Where(r => r.ModifiedOn >= lastIndexed) .Where(r => r.CurrentVersion) .Join(fileDao.FilesDbContext.Tenants, r => r.TenantId, r => r.Id, (f, t) => new { f, t }) .Where(r => r.t.Status == ASC.Core.Tenants.TenantStatus.Active); @@ -83,7 +84,8 @@ namespace ASC.Web.Files.Core.Search } List getData(long i, long step, DateTime lastIndexed) => - fileDao.GetFileQuery(r => r.ModifiedOn >= lastIndexed) + fileDao.FilesDbContext.Files + .Where(r => r.ModifiedOn >= lastIndexed) .Where(r => r.CurrentVersion) .Where(r => r.Id >= i && r.Id <= i + step) .Join(fileDao.FilesDbContext.Tenants, r => r.TenantId, r => r.Id, (f, t) => new { f, t }) @@ -95,6 +97,12 @@ namespace ASC.Web.Files.Core.Search { foreach (var data in Indexer.IndexAll(getCount, getData)) { + data.ForEach(r => + { + TenantManager.SetCurrentTenant(r.TenantId); + fileDao.InitDocument(r); + TenantManager.CurrentTenant = null; + }); Index(data); } } diff --git a/products/ASC.Files/Server/Core/Search/FoldersWrapper.cs b/products/ASC.Files/Server/Core/Search/FoldersWrapper.cs index aab1e9042b..754e2b4bc8 100644 --- a/products/ASC.Files/Server/Core/Search/FoldersWrapper.cs +++ b/products/ASC.Files/Server/Core/Search/FoldersWrapper.cs @@ -67,7 +67,9 @@ namespace ASC.Web.Files.Core.Search (int, int, int) getCount(DateTime lastIndexed) { - var q = folderDao.GetFolderQuery(r => r.ModifiedOn >= lastIndexed) + var q = + folderDao.FilesDbContext.Folders + .Where(r => r.ModifiedOn >= lastIndexed) .Join(folderDao.FilesDbContext.Tenants, r => r.TenantId, r => r.Id, (f, t) => new { f, t }) .Where(r => r.t.Status == ASC.Core.Tenants.TenantStatus.Active); @@ -79,7 +81,8 @@ namespace ASC.Web.Files.Core.Search } List getData(long i, long step, DateTime lastIndexed) => - folderDao.GetFolderQuery(r => r.ModifiedOn >= lastIndexed) + folderDao.FilesDbContext.Folders + .Where(r => r.ModifiedOn >= lastIndexed) .Where(r => r.Id >= i && r.Id <= i + step) .Join(folderDao.FilesDbContext.Tenants, r => r.TenantId, r => r.Id, (f, t) => new { f, t }) .Where(r => r.t.Status == ASC.Core.Tenants.TenantStatus.Active) @@ -131,7 +134,8 @@ namespace ASC.Web.Files.Core.Search services.TryAddScoped, FactoryIndexerFolder>(); return services - .AddFactoryIndexerService(false); + .AddFactoryIndexerService(false) + .AddDaoFactoryService(); } } } \ No newline at end of file diff --git a/products/ASC.Files/Server/Core/Thirdparty/ProviderAccountDao.cs b/products/ASC.Files/Server/Core/Thirdparty/ProviderAccountDao.cs index 6d4000623c..70bed48c03 100644 --- a/products/ASC.Files/Server/Core/Thirdparty/ProviderAccountDao.cs +++ b/products/ASC.Files/Server/Core/Thirdparty/ProviderAccountDao.cs @@ -73,11 +73,13 @@ namespace ASC.Files.Thirdparty internal class ProviderAccountDao : IProviderDao { - protected int TenantID { get; private set; } + private int tenantID; + protected int TenantID { get => tenantID != 0 ? tenantID : (tenantID = TenantManager.GetCurrentTenant().TenantId); } public FilesDbContext FilesDbContext { get; } public ILog Logger { get; } public IServiceProvider ServiceProvider { get; } public TenantUtil TenantUtil { get; } + public TenantManager TenantManager { get; } public InstanceCrypto InstanceCrypto { get; } public SecurityContext SecurityContext { get; } public ConsumerFactory ConsumerFactory { get; } @@ -92,11 +94,11 @@ namespace ASC.Files.Thirdparty DbContextManager dbContextManager, IOptionsMonitor options) { - TenantID = tenantManager.GetCurrentTenant().TenantId; FilesDbContext = dbContextManager.Get(FileConstant.DatabaseId); Logger = options.Get("ASC.Files"); ServiceProvider = serviceProvider; TenantUtil = tenantUtil; + TenantManager = tenantManager; InstanceCrypto = instanceCrypto; SecurityContext = securityContext; ConsumerFactory = consumerFactory; diff --git a/products/ASC.Files/Server/Core/Thirdparty/ProviderDao/ProviderDaoBase.cs b/products/ASC.Files/Server/Core/Thirdparty/ProviderDao/ProviderDaoBase.cs index a2a4d38993..189bfc3a2b 100644 --- a/products/ASC.Files/Server/Core/Thirdparty/ProviderDao/ProviderDaoBase.cs +++ b/products/ASC.Files/Server/Core/Thirdparty/ProviderDao/ProviderDaoBase.cs @@ -49,7 +49,8 @@ namespace ASC.Files.Thirdparty.ProviderDao { private readonly List Selectors; - private int TenantID { get; set; } + private int tenantID; + private int TenantID { get => tenantID != 0 ? tenantID : (tenantID = TenantManager.GetCurrentTenant().TenantId); } public ProviderDaoBase( IServiceProvider serviceProvider, @@ -59,10 +60,10 @@ namespace ASC.Files.Thirdparty.ProviderDao CrossDao crossDao) { ServiceProvider = serviceProvider; + TenantManager = tenantManager; SecurityDao = securityDao; TagDao = tagDao; CrossDao = crossDao; - TenantID = tenantManager.GetCurrentTenant().TenantId; Selectors = new List { @@ -77,6 +78,7 @@ namespace ASC.Files.Thirdparty.ProviderDao } public IServiceProvider ServiceProvider { get; } + public TenantManager TenantManager { get; } public SecurityDao SecurityDao { get; } public TagDao TagDao { get; } public CrossDao CrossDao { get; } diff --git a/products/ASC.Files/Service/ASC.Files.Service.csproj b/products/ASC.Files/Service/ASC.Files.Service.csproj index 80d8cc46a3..e91e1d89a6 100644 --- a/products/ASC.Files/Service/ASC.Files.Service.csproj +++ b/products/ASC.Files/Service/ASC.Files.Service.csproj @@ -6,10 +6,10 @@ - - - - + + + + diff --git a/products/ASC.Files/Service/ServiceLauncher.cs b/products/ASC.Files/Service/ServiceLauncher.cs index 5fc9d5cabc..fbc5467da9 100644 --- a/products/ASC.Files/Service/ServiceLauncher.cs +++ b/products/ASC.Files/Service/ServiceLauncher.cs @@ -33,6 +33,8 @@ using ASC.Common.Caching; using ASC.Common.Logging; using ASC.ElasticSearch; using ASC.Files.Core.EF; +using ASC.Web.Files.Core.Search; +using ASC.Web.Files.Utils; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -46,10 +48,10 @@ namespace ASC.Files.Service private ICacheNotify Notify { get; } public IServiceProvider ServiceProvider { get; } private bool IsStarted { get; set; } - private string Indexing { get; set; } + //private string Indexing { get; set; } private CancellationTokenSource CancellationTokenSource { get; set; } private Timer Timer { get; set; } - private DateTime? LastIndexed { get; set; } + //private DateTime? LastIndexed { get; set; } private TimeSpan Period { get { return TimeSpan.FromMinutes(1); } }//Settings.Default.Period public ServiceLauncher(IOptionsMonitor options, ICacheNotify notify, IServiceProvider serviceProvider) @@ -57,6 +59,7 @@ namespace ASC.Files.Service Log = options.Get("ASC.Indexer"); Notify = notify; ServiceProvider = serviceProvider; + CancellationTokenSource = new CancellationTokenSource(); } public Task StartAsync(CancellationToken cancellationToken) @@ -132,7 +135,7 @@ namespace ASC.Files.Service Indexing = null; } - public void IndexProduct(FactoryIndexer product, bool reindex) where T : class, ISearchItem + public void IndexProduct(IFactoryIndexer product, bool reindex) { if (reindex) { @@ -141,7 +144,7 @@ namespace ASC.Files.Service if (!IsStarted) return; Log.DebugFormat("Product reindex {0}", product.IndexName); - product.Indexer.ReIndex(); + product.ReIndex(); } catch (Exception e) { @@ -172,7 +175,11 @@ namespace ASC.Files.Service { services.TryAddSingleton(); - return services; + return services + .AddFileConverterService() + .AddKafkaService() + .AddFilesWrapperService() + .AddFoldersWrapperService(); } } } From b9a432cb897121a1eb5d399f39196097e16b15c8 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Tue, 28 Apr 2020 20:32:51 +0300 Subject: [PATCH 07/20] Files: removed FilesWrapper, FolderWrapper --- .../Server/Core/Dao/TeamlabDao/FileDao.cs | 32 ++- ...{FilesWrapper.cs => FactoryIndexerFile.cs} | 187 +++--------------- ...dersWrapper.cs => FactoryIndexerFolder.cs} | 93 ++++----- products/ASC.Files/Service/ServiceLauncher.cs | 4 +- 4 files changed, 84 insertions(+), 232 deletions(-) rename products/ASC.Files/Server/Core/Search/{FilesWrapper.cs => FactoryIndexerFile.cs} (54%) rename products/ASC.Files/Server/Core/Search/{FoldersWrapper.cs => FactoryIndexerFolder.cs} (83%) diff --git a/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs b/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs index c30785d2a2..7e73a27673 100644 --- a/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs +++ b/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs @@ -674,9 +674,11 @@ namespace ASC.Files.Core.Data if (deleteFolder) DeleteFolder(fileId); - var wrapper = ServiceProvider.GetService(); - //wrapper.Id = fileId; - // FactoryIndexer.DeleteAsync(wrapper); + var toDeleteFile = toDeleteFiles.FirstOrDefault(r => r.CurrentVersion); + if (toDeleteFile != null) + { + FactoryIndexer.DeleteAsync(toDeleteFile); + } } public bool IsExist(string title, object folderId) @@ -712,6 +714,8 @@ namespace ASC.Files.Core.Data { if (fileId == default) return default; + List toUpdate; + using (var tx = FilesDbContext.Database.BeginTransaction()) { var fromFolders = Query(FilesDbContext.Files) @@ -720,7 +724,7 @@ namespace ASC.Files.Core.Data .Distinct() .ToList(); - var toUpdate = Query(FilesDbContext.Files) + toUpdate = Query(FilesDbContext.Files) .Where(r => r.Id == fileId) .ToList(); @@ -733,10 +737,6 @@ namespace ASC.Files.Core.Data f.ModifiedBy = AuthContext.CurrentAccount.ID; f.ModifiedOn = DateTime.UtcNow; } - - FactoryIndexer.Update(f, - UpdateAction.Replace, - w => w.Folders); } FilesDbContext.SaveChanges(); @@ -746,20 +746,19 @@ namespace ASC.Files.Core.Data RecalculateFilesCount(toFolderId); } - var parentFoldersIds = + var parentFolders = FilesDbContext.Tree .Where(r => r.FolderId == toFolderId) .OrderByDescending(r => r.Level) - .Select(r => r.ParentId) .ToList(); - var wrapper = ServiceProvider.GetService(); - wrapper.Id = fileId; - wrapper.Folders = parentFoldersIds.Select(r => new FilesFoldersWrapper() { FolderId = r.ToString() }).ToList(); + var toUpdateFile = toUpdate.FirstOrDefault(r => r.CurrentVersion); - /* FactoryIndexer.Update(wrapper, - UpdateAction.Replace, - w => w.Folders); */ + if (toUpdateFile != null) + { + toUpdateFile.Folders = parentFolders; + FactoryIndexer.Update(toUpdateFile, UpdateAction.Replace, w => w.Folders); + } return fileId; } @@ -829,7 +828,6 @@ namespace ASC.Files.Core.Data public int FileRename(File file, string newTitle) { - var fileIdString = file.ID.ToString(); newTitle = Global.ReplaceInvalidCharsAndTruncate(newTitle); var toUpdate = Query(FilesDbContext.Files) .Where(r => r.Id == file.ID) diff --git a/products/ASC.Files/Server/Core/Search/FilesWrapper.cs b/products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs similarity index 54% rename from products/ASC.Files/Server/Core/Search/FilesWrapper.cs rename to products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs index c9ef208ece..e17093731b 100644 --- a/products/ASC.Files/Server/Core/Search/FilesWrapper.cs +++ b/products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs @@ -1,49 +1,44 @@ -/* - * - * (c) Copyright Ascensio System Limited 2010-2018 - * - * This program is freeware. You can redistribute it and/or modify it under the terms of the GNU - * General Public License (GPL) version 3 as published by the Free Software Foundation (https://www.gnu.org/copyleft/gpl.html). - * In accordance with Section 7(a) of the GNU GPL its Section 15 shall be amended to the effect that - * Ascensio System SIA expressly excludes the warranty of non-infringement of any third-party rights. - * - * THIS PROGRAM IS DISTRIBUTED WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR - * FITNESS FOR A PARTICULAR PURPOSE. For more details, see GNU GPL at https://www.gnu.org/copyleft/gpl.html - * - * You can contact Ascensio System SIA by email at sales@onlyoffice.com - * - * The interactive user interfaces in modified source and object code versions of ONLYOFFICE must display - * Appropriate Legal Notices, as required under Section 5 of the GNU GPL version 3. - * - * Pursuant to Section 7 § 3(b) of the GNU GPL you must retain the original ONLYOFFICE logo which contains - * relevant author attributions when distributing the software. If the display of the logo in its graphic - * form is not reasonably feasible for technical reasons, you must include the words "Powered by ONLYOFFICE" - * in every copy of the program you distribute. - * Pursuant to Section 7 § 3(e) we decline to grant you any rights under trademark law for use of our trademarks. - * -*/ - - -using System; +/* + * + * (c) Copyright Ascensio System Limited 2010-2018 + * + * This program is freeware. You can redistribute it and/or modify it under the terms of the GNU + * General Public License (GPL) version 3 as published by the Free Software Foundation (https://www.gnu.org/copyleft/gpl.html). + * In accordance with Section 7(a) of the GNU GPL its Section 15 shall be amended to the effect that + * Ascensio System SIA expressly excludes the warranty of non-infringement of any third-party rights. + * + * THIS PROGRAM IS DISTRIBUTED WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR + * FITNESS FOR A PARTICULAR PURPOSE. For more details, see GNU GPL at https://www.gnu.org/copyleft/gpl.html + * + * You can contact Ascensio System SIA by email at sales@onlyoffice.com + * + * The interactive user interfaces in modified source and object code versions of ONLYOFFICE must display + * Appropriate Legal Notices, as required under Section 5 of the GNU GPL version 3. + * + * Pursuant to Section 7 § 3(b) of the GNU GPL you must retain the original ONLYOFFICE logo which contains + * relevant author attributions when distributing the software. If the display of the logo in its graphic + * form is not reasonably feasible for technical reasons, you must include the words "Powered by ONLYOFFICE" + * in every copy of the program you distribute. + * Pursuant to Section 7 § 3(e) we decline to grant you any rights under trademark law for use of our trademarks. + * +*/ + +using System; using System.Collections.Generic; -using System.IO; using System.Linq; using ASC.Common; using ASC.Common.Logging; using ASC.Core; -using ASC.ElasticSearch; -using ASC.ElasticSearch.Core; +using ASC.ElasticSearch; +using ASC.ElasticSearch.Core; using ASC.Files.Core; using ASC.Files.Core.Data; using ASC.Files.Core.EF; -using ASC.Files.Resources; -using ASC.Web.Core.Files; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -namespace ASC.Web.Files.Core.Search +namespace ASC.Web.Files.Core.Search { public class FactoryIndexerFile : FactoryIndexer { @@ -113,127 +108,9 @@ namespace ASC.Web.Files.Core.Search } } } - - public sealed class FilesWrapper : WrapperWithDoc - { - [Column("title", 1)] - public string Title { get; set; } - - [ColumnLastModified("modified_on")] - public override DateTime LastModifiedOn { get; set; } - - [ColumnMeta("version", 2)] - public int Version { get; set; } - - [ColumnCondition("current_version", 3, true)] - public bool Current { get; set; } - - [ColumnMeta("encrypted", 4)] - public bool Encrypted { get; set; } - - [ColumnMeta("content_length", 5)] - public long ContentLength { get; set; } - - [ColumnMeta("create_by", 6)] - public Guid CreateBy { get; set; } - - [ColumnMeta("create_on", 7)] - public DateTime CreateOn { get; set; } - - [ColumnMeta("category", 8)] - public int Category { get; set; } - - - [Join(JoinTypeEnum.Sub, "folder_id:folder_id")] - public List Folders { get; set; } - - protected override string Table { get { return "files_file"; } } - - - public FilesWrapper() - { - - } - - public FilesWrapper(IServiceProvider serviceProvider, TenantManager tenantManager, FileUtility fileUtility, IDaoFactory daoFactory) - { - ServiceProvider = serviceProvider; - TenantManager = tenantManager; - FileUtility = fileUtility; - DaoFactory = daoFactory; - } - - public static FilesWrapper GetFilesWrapper(IServiceProvider serviceProvider, File d, List parentFolders = null) - { - var wrapper = serviceProvider.GetService(); - var tenantManager = serviceProvider.GetService(); - - wrapper.Id = Convert.ToInt32(d.ID); - wrapper.Title = d.Title; - wrapper.Version = d.Version; - wrapper.Encrypted = d.Encrypted; - wrapper.ContentLength = d.ContentLength; - wrapper.LastModifiedOn = d.ModifiedOn; - wrapper.TenantId = tenantManager.GetCurrentTenant().TenantId; - - if (parentFolders != null) - { - wrapper.Folders = parentFolders.Select(r => new FilesFoldersWrapper { FolderId = r.ToString() }).ToList(); - } - return wrapper; - } - - protected override Stream GetDocumentStream() - { - TenantManager.SetCurrentTenant(TenantId); - - if (Encrypted) return null; - if (!FileUtility.CanIndex(Title)) return null; - - var fileDao = DaoFactory.GetFileDao(); - var file = ServiceProvider.GetService>(); - file.ID = Id; - file.Title = Title; - file.Version = Version; - file.ContentLength = ContentLength; - - if (!fileDao.IsExistOnStorage(file)) return null; - if (file.ContentLength > MaxContentLength) return null; - - return fileDao.GetFileStream(file); - } - - public override string SettingsTitle - { - get { return FilesCommonResource.IndexTitle; } - } - - private IServiceProvider ServiceProvider { get; } - private TenantManager TenantManager { get; } - private FileUtility FileUtility { get; } - private IDaoFactory DaoFactory { get; } - } - - public sealed class FilesFoldersWrapper : Wrapper + public static class FilesWrapperExtention { - [Column("parent_id", 1)] - public string FolderId { get; set; } - - [ColumnId("")] - public override int Id { get; set; } - - [ColumnTenantId("")] - public override int TenantId { get; set; } - - [ColumnLastModified("")] - public override DateTime LastModifiedOn { get; set; } - - protected override string Table { get { return "files_folder_tree"; } } - } - - public static class FilesWrapperExtention - { - public static DIHelper AddFilesWrapperService(this DIHelper services) + public static DIHelper AddFilesWrapperService(this DIHelper services) { services.TryAddTransient(); services.TryAddScoped, FactoryIndexerFile>(); @@ -242,4 +119,4 @@ namespace ASC.Web.Files.Core.Search .AddFactoryIndexerService(false); } } -} \ No newline at end of file +} diff --git a/products/ASC.Files/Server/Core/Search/FoldersWrapper.cs b/products/ASC.Files/Server/Core/Search/FactoryIndexerFolder.cs similarity index 83% rename from products/ASC.Files/Server/Core/Search/FoldersWrapper.cs rename to products/ASC.Files/Server/Core/Search/FactoryIndexerFolder.cs index 754e2b4bc8..9f0473e0af 100644 --- a/products/ASC.Files/Server/Core/Search/FoldersWrapper.cs +++ b/products/ASC.Files/Server/Core/Search/FactoryIndexerFolder.cs @@ -1,46 +1,45 @@ -/* - * - * (c) Copyright Ascensio System Limited 2010-2018 - * - * This program is freeware. You can redistribute it and/or modify it under the terms of the GNU - * General Public License (GPL) version 3 as published by the Free Software Foundation (https://www.gnu.org/copyleft/gpl.html). - * In accordance with Section 7(a) of the GNU GPL its Section 15 shall be amended to the effect that - * Ascensio System SIA expressly excludes the warranty of non-infringement of any third-party rights. - * - * THIS PROGRAM IS DISTRIBUTED WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR - * FITNESS FOR A PARTICULAR PURPOSE. For more details, see GNU GPL at https://www.gnu.org/copyleft/gpl.html - * - * You can contact Ascensio System SIA by email at sales@onlyoffice.com - * - * The interactive user interfaces in modified source and object code versions of ONLYOFFICE must display - * Appropriate Legal Notices, as required under Section 5 of the GNU GPL version 3. - * - * Pursuant to Section 7 § 3(b) of the GNU GPL you must retain the original ONLYOFFICE logo which contains - * relevant author attributions when distributing the software. If the display of the logo in its graphic - * form is not reasonably feasible for technical reasons, you must include the words "Powered by ONLYOFFICE" - * in every copy of the program you distribute. - * Pursuant to Section 7 § 3(e) we decline to grant you any rights under trademark law for use of our trademarks. - * -*/ - - -using System; +/* + * + * (c) Copyright Ascensio System Limited 2010-2018 + * + * This program is freeware. You can redistribute it and/or modify it under the terms of the GNU + * General Public License (GPL) version 3 as published by the Free Software Foundation (https://www.gnu.org/copyleft/gpl.html). + * In accordance with Section 7(a) of the GNU GPL its Section 15 shall be amended to the effect that + * Ascensio System SIA expressly excludes the warranty of non-infringement of any third-party rights. + * + * THIS PROGRAM IS DISTRIBUTED WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR + * FITNESS FOR A PARTICULAR PURPOSE. For more details, see GNU GPL at https://www.gnu.org/copyleft/gpl.html + * + * You can contact Ascensio System SIA by email at sales@onlyoffice.com + * + * The interactive user interfaces in modified source and object code versions of ONLYOFFICE must display + * Appropriate Legal Notices, as required under Section 5 of the GNU GPL version 3. + * + * Pursuant to Section 7 § 3(b) of the GNU GPL you must retain the original ONLYOFFICE logo which contains + * relevant author attributions when distributing the software. If the display of the logo in its graphic + * form is not reasonably feasible for technical reasons, you must include the words "Powered by ONLYOFFICE" + * in every copy of the program you distribute. + * Pursuant to Section 7 § 3(e) we decline to grant you any rights under trademark law for use of our trademarks. + * +*/ + + +using System; using System.Collections.Generic; using System.Linq; using ASC.Common; using ASC.Common.Logging; using ASC.Core; -using ASC.ElasticSearch; +using ASC.ElasticSearch; using ASC.ElasticSearch.Core; -using ASC.Files.Core; +using ASC.Files.Core; using ASC.Files.Core.Data; using ASC.Files.Core.EF; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -namespace ASC.Web.Files.Core.Search +namespace ASC.Web.Files.Core.Search { public class FactoryIndexerFolder : FactoryIndexer { @@ -103,32 +102,10 @@ namespace ASC.Web.Files.Core.Search } } } - - public sealed class FoldersWrapper : Wrapper - { - [Column("title", 1)] - public string Title { get; set; } - - [ColumnLastModified("modified_on")] - public override DateTime LastModifiedOn { get; set; } - - protected override string Table { get { return "files_folder"; } } - public static FoldersWrapper GetFolderWrapper(IServiceProvider serviceProvider, Folder d) - { - var tenantManager = serviceProvider.GetService(); - - return new FoldersWrapper - { - Id = Convert.ToInt32(d.ID), - Title = d.Title, - TenantId = tenantManager.GetCurrentTenant().TenantId - }; - } - } - public static class FoldersWrapperExtention - { - public static DIHelper AddFoldersWrapperService(this DIHelper services) + public static class FoldersWrapperExtention + { + public static DIHelper AddFoldersWrapperService(this DIHelper services) { services.TryAddTransient(); services.TryAddScoped, FactoryIndexerFolder>(); @@ -137,5 +114,5 @@ namespace ASC.Web.Files.Core.Search .AddFactoryIndexerService(false) .AddDaoFactoryService(); } - } -} \ No newline at end of file + } +} diff --git a/products/ASC.Files/Service/ServiceLauncher.cs b/products/ASC.Files/Service/ServiceLauncher.cs index fbc5467da9..b7b0a93ce6 100644 --- a/products/ASC.Files/Service/ServiceLauncher.cs +++ b/products/ASC.Files/Service/ServiceLauncher.cs @@ -48,10 +48,10 @@ namespace ASC.Files.Service private ICacheNotify Notify { get; } public IServiceProvider ServiceProvider { get; } private bool IsStarted { get; set; } - //private string Indexing { get; set; } + private string Indexing { get; set; } private CancellationTokenSource CancellationTokenSource { get; set; } private Timer Timer { get; set; } - //private DateTime? LastIndexed { get; set; } + private DateTime? LastIndexed { get; set; } private TimeSpan Period { get { return TimeSpan.FromMinutes(1); } }//Settings.Default.Period public ServiceLauncher(IOptionsMonitor options, ICacheNotify notify, IServiceProvider serviceProvider) From bcfbac05c6c97fd16b1a26f8f5535df142ceebce Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Wed, 29 Apr 2020 11:04:19 +0300 Subject: [PATCH 08/20] ElastiSearch: removed Wrapper, WrapperWithDoc --- .../ASC.ElasticSearch/Core/ISearchItem.cs | 21 + .../ASC.ElasticSearch/Core/SearchSettings.cs | 12 +- .../ASC.ElasticSearch/Core/Wrapper.cs | 385 ------------------ .../ASC.ElasticSearch/Core/WrapperWithDoc.cs | 81 ---- .../ASC.ElasticSearch/Engine/BaseIndexer.cs | 12 +- .../Engine/FactoryIndexer.cs | 2 +- products/ASC.Files/Server/Core/EF/DbFile.cs | 6 + 7 files changed, 40 insertions(+), 479 deletions(-) create mode 100644 common/services/ASC.ElasticSearch/Core/ISearchItem.cs delete mode 100644 common/services/ASC.ElasticSearch/Core/Wrapper.cs delete mode 100644 common/services/ASC.ElasticSearch/Core/WrapperWithDoc.cs diff --git a/common/services/ASC.ElasticSearch/Core/ISearchItem.cs b/common/services/ASC.ElasticSearch/Core/ISearchItem.cs new file mode 100644 index 0000000000..75902c1654 --- /dev/null +++ b/common/services/ASC.ElasticSearch/Core/ISearchItem.cs @@ -0,0 +1,21 @@ +using Nest; + +using Newtonsoft.Json; + +namespace ASC.ElasticSearch +{ + public interface ISearchItem + { + public int Id { get; set; } + public int TenantId { get; set; } + public string IndexName { get; } + } + + public interface ISearchItemDocument : ISearchItem + { + public Document Document { get; set; } + + [Ignore, JsonIgnore] + public abstract string SettingsTitle { get; } + } +} diff --git a/common/services/ASC.ElasticSearch/Core/SearchSettings.cs b/common/services/ASC.ElasticSearch/Core/SearchSettings.cs index 47232d9270..a6875ce8b8 100644 --- a/common/services/ASC.ElasticSearch/Core/SearchSettings.cs +++ b/common/services/ASC.ElasticSearch/Core/SearchSettings.cs @@ -114,13 +114,13 @@ namespace ASC.ElasticSearch.Core }).ToList(); } - private List allItems; - internal List AllItems + private List allItems; + internal List AllItems { get { - return allItems ?? (allItems = FactoryIndexer.Builder.Resolve>() - .OfType() + return allItems ?? (allItems = FactoryIndexer.Builder.Resolve>() + .OfType() .ToList()); } } @@ -148,8 +148,8 @@ namespace ASC.ElasticSearch.Core public bool CanSearchByContent(int tenantId) where T : class, ISearchItem { if (!SearchByContentEnabled) return false; - - if (!typeof(T).IsSubclassOf(typeof(WrapperWithDoc))) + + if (typeof(ISearchItemDocument).IsAssignableFrom(typeof(T))) { return false; } diff --git a/common/services/ASC.ElasticSearch/Core/Wrapper.cs b/common/services/ASC.ElasticSearch/Core/Wrapper.cs deleted file mode 100644 index 78010d7965..0000000000 --- a/common/services/ASC.ElasticSearch/Core/Wrapper.cs +++ /dev/null @@ -1,385 +0,0 @@ -/* - * - * (c) Copyright Ascensio System Limited 2010-2018 - * - * This program is freeware. You can redistribute it and/or modify it under the terms of the GNU - * General Public License (GPL) version 3 as published by the Free Software Foundation (https://www.gnu.org/copyleft/gpl.html). - * In accordance with Section 7(a) of the GNU GPL its Section 15 shall be amended to the effect that - * Ascensio System SIA expressly excludes the warranty of non-infringement of any third-party rights. - * - * THIS PROGRAM IS DISTRIBUTED WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR - * FITNESS FOR A PARTICULAR PURPOSE. For more details, see GNU GPL at https://www.gnu.org/copyleft/gpl.html - * - * You can contact Ascensio System SIA by email at sales@onlyoffice.com - * - * The interactive user interfaces in modified source and object code versions of ONLYOFFICE must display - * Appropriate Legal Notices, as required under Section 5 of the GNU GPL version 3. - * - * Pursuant to Section 7 § 3(b) of the GNU GPL you must retain the original ONLYOFFICE logo which contains - * relevant author attributions when distributing the software. If the display of the logo in its graphic - * form is not reasonably feasible for technical reasons, you must include the words "Powered by ONLYOFFICE" - * in every copy of the program you distribute. - * Pursuant to Section 7 § 3(e) we decline to grant you any rights under trademark law for use of our trademarks. - * -*/ - - -using System; -using System.Collections; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; - -using ASC.ElasticSearch.Core; - -using Nest; - -using Newtonsoft.Json.Linq; - -namespace ASC.ElasticSearch -{ - public interface ISearchItem - { - public int Id { get; set; } - public int TenantId { get; set; } - public string IndexName { get; } - } - - public interface ISearchItemDocument : ISearchItem - { - public Document Document { get; set; } - } - - public abstract class Wrapper - { - protected internal abstract string Table { get; } - - protected internal virtual string IndexName - { - get { return Table; } - } - - [ColumnId("id")] - public virtual int Id { get; set; } - - [ColumnTenantId("tenant_id")] - public virtual int TenantId { get; set; } - - [ColumnLastModified("last_modified_on"), Date] - public virtual DateTime LastModifiedOn { get; set; } - - private List orderedProperties; - - private List OrderedProperties - { - get - { - return orderedProperties ?? (orderedProperties = GetType() - .GetProperties() - .Where(r => - { - var column = r.GetCustomAttribute(); - return column != null && !string.IsNullOrEmpty(column.ColumnName); - }) - .OrderBy(r => r.GetCustomAttribute().Order) - .ToList()); - } - } - - private List columns; - - private List Columns - { - get - { - return columns ?? (columns = OrderedProperties - .Select(r => r.GetCustomAttribute()) - .ToList()); - } - } - - internal Converter GetDataConverter(bool sub = false) - { - return data => - { - var result = Activator.CreateInstance(GetType()); - - var i = 0; - var props = OrderedProperties; - for (; i < props.Count; i++) - { - if (data[i] == null) continue; - object newValue; - if (props[i].PropertyType == typeof(Guid)) - { - newValue = Guid.Parse(data[i].ToString()); - } - else if (props[i].PropertyType == typeof(bool)) - { - try - { - newValue = Convert.ToBoolean(data[i]); - } - catch (Exception) - { - newValue = data[i] != null; - } - } - else - { - try - { - newValue = Convert.ChangeType(data[i], props[i].PropertyType); - } - catch (Exception) - { - newValue = Activator.CreateInstance(props[i].PropertyType); - } - } - props[i].SetValue(result, newValue); - } - - var joins = GetType() - .GetProperties() - .Where(r => r.GetCustomAttribute() != null) - .ToList(); - - if (!joins.Any()) return (Wrapper)result; - - data = data.Skip(i).ToArray(); - - foreach (var join in joins) - { - Wrapper joinWrapper; - - if (join.PropertyType.IsGenericType) - { - joinWrapper = Activator.CreateInstance(join.PropertyType.GenericTypeArguments[0]) as Wrapper; - } - else - { - joinWrapper = Activator.CreateInstance(join.PropertyType) as Wrapper; - } - - if (joinWrapper == null) continue; - - var joinAttr = join.GetCustomAttribute(); - var joinSub = sub || joinAttr.JoinType == JoinTypeEnum.Sub; - List newArray; - if (joinSub && join.PropertyType.IsGenericType) - { - newArray = data[0] == null - ? new List() - : JArray.Parse(data[0].ToString()).Select(r => ((JArray)r).Select(q => (object)q).ToArray()).ToList(); - } - else - { - newArray = new List { data }; - } - - var newArrayValue = newArray.ConvertAll(joinWrapper.GetDataConverter(joinSub)); - - object newValue; - - if (joinSub && join.PropertyType.IsGenericType) - { - var list = (IList)Activator.CreateInstance(join.PropertyType); - foreach (var item in newArrayValue) - { - list.Add(item); - } - - newValue = list; - } - else - { - newValue = Convert.ChangeType(newArrayValue.First(), join.PropertyType); - } - - join.SetValue(result, newValue); - - var skipCount = joinSub - ? 1 - : joinWrapper.OrderedProperties.Count; - - data = data.Skip(skipCount).ToArray(); - } - - return (Wrapper)result; - }; - } - - internal string GetColumnName(ColumnTypeEnum columnType, string alias) - { - var column = Columns.FirstOrDefault(r => r.ColumnType == columnType); - if (column == null || string.IsNullOrEmpty(column.ColumnName)) return null; - return alias + "." + column.ColumnName; - } - - internal bool TryGetColumnName(ColumnTypeEnum columnType, string alias, out string columnName) - { - columnName = GetColumnName(columnType, alias); - return columnName != null; - } - - internal bool TryGetColumnNames(List columnType, string alias, out List columnName) - { - columnName = new List(); - foreach (var cType in columnType) - { - var type = cType; - var column = Columns.Where(r => r.ColumnType == type); - columnName.AddRange(column.Select(r => alias + "." + r.ColumnName)); - } - - return columnName.Any(); - } - - internal string[] GetColumnNames(string alias) - { - return Columns - .Select(r => alias + "." + r.ColumnName) - .ToArray(); - } - - internal string[] GetContentProperties() - { - var result = OrderedProperties - .Where(r => r.GetCustomAttribute().ColumnType == ColumnTypeEnum.Content) - .Select(r => ToLowerCamelCase(r.Name)) - .ToList(); - - if (this is WrapperWithDoc) - { - result.Add("document.attachment.content"); - } - - foreach (var join in GetJoins()) - { - var joinType = join.Value.Name; - var joinAttr = join.Value.GetCustomAttribute().JoinType; - result.AddRange(join.Key.GetContentProperties() - .Select(r => (joinAttr == JoinTypeEnum.Sub ? joinAttr + ":" : "") + ToLowerCamelCase(joinType) + "." + r)); - } - - return result.ToArray(); - } - - internal Dictionary GetJoins() - { - var result = new Dictionary(); - - var joins = GetType() - .GetProperties() - .Where(r => r.GetCustomAttribute() != null) - .ToList(); - - foreach (var join in joins) - { - Wrapper joinWrapper; - if (join.PropertyType.IsGenericType) - { - joinWrapper = Activator.CreateInstance(join.PropertyType.GenericTypeArguments[0]) as Wrapper; - } - else - { - joinWrapper = Activator.CreateInstance(join.PropertyType) as Wrapper; - } - - if (joinWrapper == null) continue; - - result.Add(joinWrapper, join); - } - - return result; - } - - internal Dictionary GetConditions(string alias) - { - var result = new Dictionary(); - - var conditions = Columns.OfType(); - - foreach (var con in conditions.Where(r => r.Value != null)) - { - result.Add(alias + "." + con.ColumnName, con.Value); - } - - return result; - } - - internal string ToLowerCamelCase(string value) - { - return char.ToLowerInvariant(value[0]) + value.Substring(1); - } - - internal Func, IPromise> GetProperties() where T : class, ISearchItem - { - var analyzers = GetAnalyzers(); - - return p => - { - foreach (var c in analyzers) - { - var c1 = c; - string analyzer; - - if (c.Value.CharFilter != CharFilter.io) - { - analyzer = c.Key; - } - else - { - analyzer = c1.Value.Analyzer + "custom"; - } - - p.Text(s => s.Name(c1.Key).Analyzer(analyzer)); - } - if (this is WrapperWithDoc) - { - p.Object( - r => r.Name("document").Properties( - q => q.Object( - a => a.Name("attachment").Properties( - w => w.Text( - t => t.Name("content").Analyzer("document")))))); - } - return p; - }; - } - - internal Dictionary GetAnalyzers() - { - var result = new Dictionary(); - - foreach (var prop in OrderedProperties) - { - var column = prop.GetCustomAttribute(); - - if (column == null || column.ColumnType != ColumnTypeEnum.Content || column.Analyzer == Analyzer.standard) continue; - - result.Add(prop.Name.ToLowerInvariant()[0] + prop.Name.Substring(1), column); - } - - return result; - } - - internal Dictionary GetNested() - { - var result = new Dictionary(); - - var joins = GetType().GetProperties().Where(r => - { - var attr = r.GetCustomAttribute(); - return attr != null; - }); - - foreach (var prop in joins) - { - result.Add(prop.PropertyType, prop.Name); - } - - return result; - } - } -} \ No newline at end of file diff --git a/common/services/ASC.ElasticSearch/Core/WrapperWithDoc.cs b/common/services/ASC.ElasticSearch/Core/WrapperWithDoc.cs deleted file mode 100644 index c75881afda..0000000000 --- a/common/services/ASC.ElasticSearch/Core/WrapperWithDoc.cs +++ /dev/null @@ -1,81 +0,0 @@ -/* - * - * (c) Copyright Ascensio System Limited 2010-2018 - * - * This program is freeware. You can redistribute it and/or modify it under the terms of the GNU - * General Public License (GPL) version 3 as published by the Free Software Foundation (https://www.gnu.org/copyleft/gpl.html). - * In accordance with Section 7(a) of the GNU GPL its Section 15 shall be amended to the effect that - * Ascensio System SIA expressly excludes the warranty of non-infringement of any third-party rights. - * - * THIS PROGRAM IS DISTRIBUTED WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR - * FITNESS FOR A PARTICULAR PURPOSE. For more details, see GNU GPL at https://www.gnu.org/copyleft/gpl.html - * - * You can contact Ascensio System SIA by email at sales@onlyoffice.com - * - * The interactive user interfaces in modified source and object code versions of ONLYOFFICE must display - * Appropriate Legal Notices, as required under Section 5 of the GNU GPL version 3. - * - * Pursuant to Section 7 § 3(b) of the GNU GPL you must retain the original ONLYOFFICE logo which contains - * relevant author attributions when distributing the software. If the display of the logo in its graphic - * form is not reasonably feasible for technical reasons, you must include the words "Powered by ONLYOFFICE" - * in every copy of the program you distribute. - * Pursuant to Section 7 § 3(e) we decline to grant you any rights under trademark law for use of our trademarks. - * -*/ - - -using System; -using System.IO; -using System.Text; - -using ASC.Common.Logging; - -using Nest; - -using Newtonsoft.Json; - -namespace ASC.ElasticSearch.Core -{ - public abstract class WrapperWithDoc : Wrapper - { - public Document Document { get; set; } - - public const long MaxContentLength = 2 * 1024 * 1024 * 1024L; - - protected abstract Stream GetDocumentStream(); - - [Ignore, JsonIgnore] - public abstract string SettingsTitle { get; } - - internal void InitDocument(bool index, ILog log) - { - Document = new Document - { - Data = Convert.ToBase64String(Encoding.UTF8.GetBytes("")) - }; - - try - { - if (!index) return; - - using (var stream = GetDocumentStream()) - { - if (stream == null) return; - - Document = new Document - { - Data = Convert.ToBase64String(stream.GetCorrectBuffer()) - }; - } - } - catch (FileNotFoundException e) - { - log.Error("InitDocument FileNotFoundException", e); - } - catch (Exception e) - { - log.Error("InitDocument", e); - } - } - } -} diff --git a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs index 6e674746cf..a4e9c5f712 100644 --- a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs @@ -131,7 +131,7 @@ namespace ASC.ElasticSearch var t = data[i]; var runBulk = i == data.Count - 1; - if (!(t is WrapperWithDoc wwd) || wwd.Document == null || string.IsNullOrEmpty(wwd.Document.Data)) + if (!(t is ISearchItemDocument wwd) || wwd.Document == null || string.IsNullOrEmpty(wwd.Document.Data)) { portion.Add(t); } @@ -307,7 +307,7 @@ namespace ASC.ElasticSearch lock (Locker) { - Func> analyzers = b => + IPromise analyzers(AnalyzersDescriptor b) { foreach (var c in Enum.GetNames(typeof(Analyzer))) { @@ -324,13 +324,13 @@ namespace ASC.ElasticSearch b.Custom(c1 + "custom", ca => ca.Tokenizer(Analyzer.whitespace.ToString()).Filters(Filter.lowercase.ToString()).CharFilters(charFilters)); } - if (data is WrapperWithDoc) + if (data is ISearchItemDocument) { b.Custom("document", ca => ca.Tokenizer(Analyzer.whitespace.ToString()).Filters(Filter.lowercase.ToString()).CharFilters(CharFilter.io.ToString())); } return b; - }; + } var createIndexResponse = Client.Instance.Indices.Create(data.IndexName, c => @@ -358,7 +358,7 @@ namespace ASC.ElasticSearch result.Refresh(Elasticsearch.Net.Refresh.True); } - if (data is WrapperWithDoc) + if (data is ISearchItemDocument) { result.Pipeline("attachments"); } @@ -369,7 +369,7 @@ namespace ASC.ElasticSearch { var result = desc.Index(IndexName).Id(data.Id); - if (data is WrapperWithDoc) + if (data is ISearchItemDocument) { result.Pipeline("attachments"); } diff --git a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs index 987b21da59..7a26dd6e98 100644 --- a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs @@ -565,7 +565,7 @@ namespace ASC.ElasticSearch if (!CoreBaseSettings.Standalone) return; var generic = typeof(BaseIndexer<>); - var indexers = Builder.Resolve>() + var indexers = Builder.Resolve>() .Where(r => string.IsNullOrEmpty(name) || r.IndexName == name) .Select(r => (IIndexer)Activator.CreateInstance(generic.MakeGenericType(r.GetType()), r)); diff --git a/products/ASC.Files/Server/Core/EF/DbFile.cs b/products/ASC.Files/Server/Core/EF/DbFile.cs index 4299f9c656..168e3b74e8 100644 --- a/products/ASC.Files/Server/Core/EF/DbFile.cs +++ b/products/ASC.Files/Server/Core/EF/DbFile.cs @@ -4,6 +4,7 @@ using System.ComponentModel.DataAnnotations.Schema; using ASC.Core.Common.EF; using ASC.ElasticSearch; +using ASC.Files.Resources; using Microsoft.EntityFrameworkCore; @@ -84,6 +85,11 @@ namespace ASC.Files.Core.EF [NotMapped] public Document Document { get; set; } + public string SettingsTitle + { + get { return FilesCommonResource.IndexTitle; } + } + public override object[] GetKeys() { return new object[] { TenantId, Id, Version }; From 37b126e5cdaa1b91d55eb4e66a51776469bbf7c4 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Wed, 29 Apr 2020 17:11:31 +0300 Subject: [PATCH 09/20] Files: search --- .../DependencyInjection/AutofacExtension.cs | 25 ++++++++++---- .../ASC.Core.Common/Configuration/Consumer.cs | 5 +++ .../Context/Impl/CoreConfiguration.cs | 3 +- .../ASC.ElasticSearch/Core/SearchSettings.cs | 2 +- .../ASC.ElasticSearch/Engine/BaseIndexer.cs | 3 +- .../Engine/FactoryIndexer.cs | 17 +++++++--- .../ASC.ElasticSearch/Service/Launcher.cs | 33 ++++++++++--------- .../ASC.ElasticSearch/Service/Service.cs | 4 +-- .../Server/Core/Dao/TeamlabDao/FileDao.cs | 10 +++++- .../Server/Core/Search/FactoryIndexerFile.cs | 4 +-- .../Core/Search/FactoryIndexerFolder.cs | 6 ++-- products/ASC.Files/Service/Program.cs | 14 ++++++-- products/ASC.Files/Service/search.json | 22 +++++++++++++ web/ASC.Web.Core/WebItemManager.cs | 11 ++++++- 14 files changed, 120 insertions(+), 39 deletions(-) rename products/ASC.Files/Service/ServiceLauncher.cs => common/services/ASC.ElasticSearch/Service/Launcher.cs (85%) create mode 100644 products/ASC.Files/Service/search.json diff --git a/common/ASC.Common/DependencyInjection/AutofacExtension.cs b/common/ASC.Common/DependencyInjection/AutofacExtension.cs index f2cc70e240..c8613173ff 100644 --- a/common/ASC.Common/DependencyInjection/AutofacExtension.cs +++ b/common/ASC.Common/DependencyInjection/AutofacExtension.cs @@ -26,7 +26,7 @@ namespace ASC.Common.DependencyInjection public static class AutofacExtension { - public static IContainer AddAutofac(this IServiceCollection services, IConfiguration configuration, string currentDir) + public static IContainer AddAutofac(this IServiceCollection services, IConfiguration configuration, string currentDir, params string[] intern) { var folder = configuration["core:products:folder"]; var subfolder = configuration["core:products:subfolder"]; @@ -42,19 +42,32 @@ namespace ASC.Common.DependencyInjection } var builder = new ContainerBuilder(); - var modules = new string[] { "autofac.json", "autofac.products.json", "autofac.consumers.json" }; + var modules = new List<(bool, string)> + { + (true, "autofac.json"), + (true, "autofac.products.json"), + (true, "autofac.consumers.json") + }; + + if (intern != null) + { + modules.AddRange(intern.Select(r => (false, r))); + } foreach (var p in modules) { - var config = new ConfigurationBuilder() - .SetBasePath(configuration["pathToConf"]) - .AddJsonFile(p); + var config = new ConfigurationBuilder(); + if (p.Item1) + { + config.SetBasePath(configuration["pathToConf"]); + } + config.AddJsonFile(p.Item2); var root = config.Build(); var module = new ConfigurationModule(root); builder.RegisterModule(module); - if (p == "autofac.products.json") + if (p.Item2 == "autofac.products.json") { FindAndLoad(root.GetSection("components")); } diff --git a/common/ASC.Core.Common/Configuration/Consumer.cs b/common/ASC.Core.Common/Configuration/Consumer.cs index fba266ea1d..bee03c63bf 100644 --- a/common/ASC.Core.Common/Configuration/Consumer.cs +++ b/common/ASC.Core.Common/Configuration/Consumer.cs @@ -377,6 +377,11 @@ namespace ASC.Core.Common.Configuration public ConsumerFactory(IContainer builder) { Builder = builder.BeginLifetimeScope(); + } + + public ConsumerFactory(ILifetimeScope builder) + { + Builder = builder; } public Consumer GetByKey(string key) diff --git a/common/ASC.Core.Common/Context/Impl/CoreConfiguration.cs b/common/ASC.Core.Common/Context/Impl/CoreConfiguration.cs index 508dd2be11..87aef07695 100644 --- a/common/ASC.Core.Common/Context/Impl/CoreConfiguration.cs +++ b/common/ASC.Core.Common/Context/Impl/CoreConfiguration.cs @@ -389,7 +389,8 @@ namespace ASC.Core public static DIHelper AddCoreConfigurationService(this DIHelper services) { services.TryAddScoped(); - return services + return services + .AddTenantManagerService() .AddCoreSettingsService(); } } diff --git a/common/services/ASC.ElasticSearch/Core/SearchSettings.cs b/common/services/ASC.ElasticSearch/Core/SearchSettings.cs index a6875ce8b8..9288e626a8 100644 --- a/common/services/ASC.ElasticSearch/Core/SearchSettings.cs +++ b/common/services/ASC.ElasticSearch/Core/SearchSettings.cs @@ -119,7 +119,7 @@ namespace ASC.ElasticSearch.Core { get { - return allItems ?? (allItems = FactoryIndexer.Builder.Resolve>() + return allItems ?? (allItems = FactoryIndexer.Builder.Resolve>() .OfType() .ToList()); } diff --git a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs index a4e9c5f712..5b34325d5f 100644 --- a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs @@ -594,7 +594,8 @@ namespace ASC.ElasticSearch WebstudioDbContext.AddOrUpdate(r => r.WebstudioIndex, new DbWebstudioIndex() { - IndexName = Wrapper.IndexName + IndexName = Wrapper.IndexName, + LastModified = DateTime.UtcNow }); WebstudioDbContext.SaveChanges(); diff --git a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs index 7a26dd6e98..36f0b321fd 100644 --- a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs @@ -450,11 +450,20 @@ namespace ASC.ElasticSearch public class FactoryIndexer { private static ICache cache = AscCache.Memory; - internal IContainer Builder { get; set; } + internal ILifetimeScope Builder { get; set; } internal static bool Init { get; set; } public ILog Log { get; } public Client Client { get; } - public CoreBaseSettings CoreBaseSettings { get; } + public CoreBaseSettings CoreBaseSettings { get; } + + public FactoryIndexer( + ILifetimeScope container, + Client client, + IOptionsMonitor options, + CoreBaseSettings coreBaseSettings) : this(null, client, options, coreBaseSettings) + { + Builder = container; + } public FactoryIndexer( IContainer container, @@ -565,9 +574,9 @@ namespace ASC.ElasticSearch if (!CoreBaseSettings.Standalone) return; var generic = typeof(BaseIndexer<>); - var indexers = Builder.Resolve>() + var indexers = Builder.Resolve>() .Where(r => string.IsNullOrEmpty(name) || r.IndexName == name) - .Select(r => (IIndexer)Activator.CreateInstance(generic.MakeGenericType(r.GetType()), r)); + .Select(r => (IFactoryIndexer)Activator.CreateInstance(generic.MakeGenericType(r.GetType()), r)); foreach (var indexer in indexers) { diff --git a/products/ASC.Files/Service/ServiceLauncher.cs b/common/services/ASC.ElasticSearch/Service/Launcher.cs similarity index 85% rename from products/ASC.Files/Service/ServiceLauncher.cs rename to common/services/ASC.ElasticSearch/Service/Launcher.cs index b7b0a93ce6..cad2f47855 100644 --- a/products/ASC.Files/Service/ServiceLauncher.cs +++ b/common/services/ASC.ElasticSearch/Service/Launcher.cs @@ -25,28 +25,28 @@ using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using ASC.Common; using ASC.Common.Caching; using ASC.Common.Logging; -using ASC.ElasticSearch; -using ASC.Files.Core.EF; -using ASC.Web.Files.Core.Search; -using ASC.Web.Files.Utils; + +using Autofac; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Options; -namespace ASC.Files.Service +namespace ASC.ElasticSearch { public class ServiceLauncher : IHostedService { private ILog Log { get; } private ICacheNotify Notify { get; } public IServiceProvider ServiceProvider { get; } + public IContainer Container { get; } private bool IsStarted { get; set; } private string Indexing { get; set; } private CancellationTokenSource CancellationTokenSource { get; set; } @@ -54,11 +54,16 @@ namespace ASC.Files.Service private DateTime? LastIndexed { get; set; } private TimeSpan Period { get { return TimeSpan.FromMinutes(1); } }//Settings.Default.Period - public ServiceLauncher(IOptionsMonitor options, ICacheNotify notify, IServiceProvider serviceProvider) + public ServiceLauncher( + IOptionsMonitor options, + ICacheNotify notify, + IServiceProvider serviceProvider, + IContainer container) { Log = options.Get("ASC.Indexer"); Notify = notify; ServiceProvider = serviceProvider; + Container = container; CancellationTokenSource = new CancellationTokenSource(); } @@ -122,12 +127,13 @@ namespace ASC.Files.Service Timer.Change(-1, -1); IsStarted = true; - using var scope = ServiceProvider.CreateScope(); - var filesWrapper = scope.ServiceProvider.GetService>(); - var foldersWrapper = scope.ServiceProvider.GetService>(); + using var scope = Container.BeginLifetimeScope(); + var wrappers = scope.Resolve>(); - IndexProduct(filesWrapper, reindex); - IndexProduct(foldersWrapper, reindex); + foreach (var w in wrappers) + { + IndexProduct(w, reindex); + } Timer.Change(Period, Period); LastIndexed = DateTime.UtcNow; @@ -176,10 +182,7 @@ namespace ASC.Files.Service services.TryAddSingleton(); return services - .AddFileConverterService() - .AddKafkaService() - .AddFilesWrapperService() - .AddFoldersWrapperService(); + .AddFactoryIndexerService(); } } } diff --git a/common/services/ASC.ElasticSearch/Service/Service.cs b/common/services/ASC.ElasticSearch/Service/Service.cs index d298605516..3908dbcfce 100644 --- a/common/services/ASC.ElasticSearch/Service/Service.cs +++ b/common/services/ASC.ElasticSearch/Service/Service.cs @@ -52,12 +52,12 @@ namespace ASC.ElasticSearch.Service public bool Support(string table) { - return FactoryIndexer.Builder.Resolve>().Any(r => r.IndexName == table); + return FactoryIndexer.Builder.Resolve>().Any(r => r.IndexName == table); } public void ReIndex(List toReIndex, int tenant) { - var allItems = FactoryIndexer.Builder.Resolve>().ToList(); + var allItems = FactoryIndexer.Builder.Resolve>().ToList(); var tasks = new List(toReIndex.Count); foreach (var item in toReIndex) diff --git a/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs b/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs index 7e73a27673..39bec09820 100644 --- a/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs +++ b/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs @@ -29,6 +29,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Linq.Expressions; +using System.Text; using ASC.Common; using ASC.Core; @@ -1334,7 +1335,14 @@ namespace ASC.Files.Core.Data internal protected DbFile InitDocument(DbFile dbFile) { - if (!FactoryIndexer.CanSearchByContent()) return dbFile; + if (!FactoryIndexer.CanSearchByContent()) + { + dbFile.Document = new Document + { + Data = Convert.ToBase64String(Encoding.UTF8.GetBytes("")) + }; + return dbFile; + } var file = ServiceProvider.GetService>(); file.ID = dbFile.Id; diff --git a/products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs b/products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs index e17093731b..cfaa18a406 100644 --- a/products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs +++ b/products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs @@ -72,8 +72,8 @@ namespace ASC.Web.Files.Core.Search .Where(r => r.t.Status == ASC.Core.Tenants.TenantStatus.Active); var count = q.GroupBy(a => a.f.Id).Count(); - var min = q.Min(r => r.f.Id); - var max = q.Max(r => r.f.Id); + var min = count > 0 ? q.Min(r => r.f.Id) : 0; + var max = count > 0 ? q.Max(r => r.f.Id) : 0; return (count, max, min); } diff --git a/products/ASC.Files/Server/Core/Search/FactoryIndexerFolder.cs b/products/ASC.Files/Server/Core/Search/FactoryIndexerFolder.cs index 9f0473e0af..93a64b9367 100644 --- a/products/ASC.Files/Server/Core/Search/FactoryIndexerFolder.cs +++ b/products/ASC.Files/Server/Core/Search/FactoryIndexerFolder.cs @@ -62,7 +62,7 @@ namespace ASC.Web.Files.Core.Search public override void IndexAll() { - var folderDao = DaoFactory.GetFileDao() as FolderDao; + var folderDao = DaoFactory.GetFolderDao() as FolderDao; (int, int, int) getCount(DateTime lastIndexed) { @@ -73,8 +73,8 @@ namespace ASC.Web.Files.Core.Search .Where(r => r.t.Status == ASC.Core.Tenants.TenantStatus.Active); var count = q.GroupBy(a => a.f.Id).Count(); - var min = q.Min(r => r.f.Id); - var max = q.Max(r => r.f.Id); + var min = count > 0 ? q.Min(r => r.f.Id) : 0; + var max = count > 0 ? q.Max(r => r.f.Id) : 0; return (count, max, min); } diff --git a/products/ASC.Files/Service/Program.cs b/products/ASC.Files/Service/Program.cs index 9052324aaf..57fb290fd2 100644 --- a/products/ASC.Files/Service/Program.cs +++ b/products/ASC.Files/Service/Program.cs @@ -3,8 +3,12 @@ using System.IO; using System.Threading.Tasks; using ASC.Common; +using ASC.Common.Caching; using ASC.Common.DependencyInjection; using ASC.Common.Logging; +using ASC.ElasticSearch; +using ASC.Web.Files.Core.Search; +using ASC.Web.Files.Utils; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -47,9 +51,15 @@ namespace ASC.Files.Service var diHelper = new DIHelper(services); diHelper.AddNLogManager("ASC.Files"); services.AddHostedService(); - diHelper.AddServiceLauncher(); + diHelper + .AddServiceLauncher() + .AddFileConverterService() + .AddKafkaService() + .AddFilesWrapperService() + .AddFoldersWrapperService(); - services.AddAutofac(hostContext.Configuration, hostContext.HostingEnvironment.ContentRootPath); + var a = typeof(FactoryIndexer).ToString(); + services.AddAutofac(hostContext.Configuration, hostContext.HostingEnvironment.ContentRootPath, "search.json"); }) .UseConsoleLifetime() .Build(); diff --git a/products/ASC.Files/Service/search.json b/products/ASC.Files/Service/search.json new file mode 100644 index 0000000000..5954c530c7 --- /dev/null +++ b/products/ASC.Files/Service/search.json @@ -0,0 +1,22 @@ +{ + "components": [ + { + "type": "ASC.Web.Files.Core.Search.FactoryIndexerFolder, ASC.Files", + "services": [ + { + "type": "ASC.ElasticSearch.IFactoryIndexer, ASC.ElasticSearch" + } + ], + "instanceScope": "perlifetimescope" + }, + { + "type": "ASC.Web.Files.Core.Search.FactoryIndexerFile, ASC.Files", + "services": [ + { + "type": "ASC.ElasticSearch.IFactoryIndexer, ASC.ElasticSearch" + } + ], + "instanceScope": "perlifetimescope" + } + ] +} \ No newline at end of file diff --git a/web/ASC.Web.Core/WebItemManager.cs b/web/ASC.Web.Core/WebItemManager.cs index 76cbd9b282..5735f191ae 100644 --- a/web/ASC.Web.Core/WebItemManager.cs +++ b/web/ASC.Web.Core/WebItemManager.cs @@ -105,7 +105,7 @@ namespace ASC.Web.Core get { return new Guid("{46CFA73A-F320-46CF-8D5B-CD82E1D67F26}"); } } - public IContainer Container { get; } + public ILifetimeScope Container { get; } public IConfiguration Configuration { get; } public IWebItem this[Guid id] @@ -126,8 +126,17 @@ namespace ASC.Web.Core LoadItems(); } + public WebItemManager(ILifetimeScope container, IConfiguration configuration, IOptionsMonitor options) + : this(null, configuration, options) + { + Container = container; + LoadItems(); + } + public void LoadItems() { + if (Container == null) return; + foreach (var webitem in Container.Resolve>()) { var file = webitem.ID.ToString(); From 7ffd2c702f273aaba78c7b338a68514cf856ecf5 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Wed, 29 Apr 2020 19:01:38 +0300 Subject: [PATCH 10/20] Files: search --- common/services/ASC.ElasticSearch/Core/ISearchItem.cs | 9 +-------- common/services/ASC.ElasticSearch/Core/SearchSettings.cs | 8 +++----- .../services/ASC.ElasticSearch/Engine/FactoryIndexer.cs | 2 ++ products/ASC.Files/Server/Core/EF/DbFile.cs | 6 ------ .../ASC.Files/Server/Core/Search/FactoryIndexerFile.cs | 7 +++++++ 5 files changed, 13 insertions(+), 19 deletions(-) diff --git a/common/services/ASC.ElasticSearch/Core/ISearchItem.cs b/common/services/ASC.ElasticSearch/Core/ISearchItem.cs index 75902c1654..5519fba136 100644 --- a/common/services/ASC.ElasticSearch/Core/ISearchItem.cs +++ b/common/services/ASC.ElasticSearch/Core/ISearchItem.cs @@ -1,8 +1,4 @@ -using Nest; - -using Newtonsoft.Json; - -namespace ASC.ElasticSearch +namespace ASC.ElasticSearch { public interface ISearchItem { @@ -14,8 +10,5 @@ namespace ASC.ElasticSearch public interface ISearchItemDocument : ISearchItem { public Document Document { get; set; } - - [Ignore, JsonIgnore] - public abstract string SettingsTitle { get; } } } diff --git a/common/services/ASC.ElasticSearch/Core/SearchSettings.cs b/common/services/ASC.ElasticSearch/Core/SearchSettings.cs index 9288e626a8..ce286a1417 100644 --- a/common/services/ASC.ElasticSearch/Core/SearchSettings.cs +++ b/common/services/ASC.ElasticSearch/Core/SearchSettings.cs @@ -114,14 +114,12 @@ namespace ASC.ElasticSearch.Core }).ToList(); } - private List allItems; - internal List AllItems + private List allItems; + internal List AllItems { get { - return allItems ?? (allItems = FactoryIndexer.Builder.Resolve>() - .OfType() - .ToList()); + return allItems ?? (allItems = FactoryIndexer.Builder.Resolve>().ToList()); } } diff --git a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs index 36f0b321fd..7735bcfd32 100644 --- a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs @@ -102,6 +102,7 @@ namespace ASC.ElasticSearch void IndexAll(); string IndexName { get; } void ReIndex(); + string SettingsTitle { get; } } public class FactoryIndexer : IFactoryIndexer where T : class, ISearchItem @@ -119,6 +120,7 @@ namespace ASC.ElasticSearch public IServiceProvider ServiceProvider { get; } public string IndexName { get => Indexer.IndexName; } + public virtual string SettingsTitle { get => ""; } public FactoryIndexer( IOptionsMonitor options, diff --git a/products/ASC.Files/Server/Core/EF/DbFile.cs b/products/ASC.Files/Server/Core/EF/DbFile.cs index 168e3b74e8..4299f9c656 100644 --- a/products/ASC.Files/Server/Core/EF/DbFile.cs +++ b/products/ASC.Files/Server/Core/EF/DbFile.cs @@ -4,7 +4,6 @@ using System.ComponentModel.DataAnnotations.Schema; using ASC.Core.Common.EF; using ASC.ElasticSearch; -using ASC.Files.Resources; using Microsoft.EntityFrameworkCore; @@ -85,11 +84,6 @@ namespace ASC.Files.Core.EF [NotMapped] public Document Document { get; set; } - public string SettingsTitle - { - get { return FilesCommonResource.IndexTitle; } - } - public override object[] GetKeys() { return new object[] { TenantId, Id, Version }; diff --git a/products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs b/products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs index cfaa18a406..9aacc540fb 100644 --- a/products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs +++ b/products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs @@ -35,6 +35,7 @@ using ASC.ElasticSearch.Core; using ASC.Files.Core; using ASC.Files.Core.Data; using ASC.Files.Core.EF; +using ASC.Files.Resources; using Microsoft.Extensions.Options; @@ -107,6 +108,12 @@ namespace ASC.Web.Files.Core.Search throw; } } + + + public override string SettingsTitle + { + get { return FilesCommonResource.IndexTitle; } + } } public static class FilesWrapperExtention { From af7fb87e73b11a24ab974ec5349724aa124c9792 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Sun, 3 May 2020 18:50:40 +0300 Subject: [PATCH 11/20] Elastic: settings --- .../ASC.Common/Utils/ConfigurationManager.cs | 1 + .../Config/ElasticSection.cs | 64 ------------------- .../ASC.ElasticSearch/Engine/BaseIndexer.cs | 13 ++-- .../ASC.ElasticSearch/Engine/Client.cs | 16 +++-- .../ASC.ElasticSearch/Service/Launcher.cs | 8 ++- .../ASC.ElasticSearch/Service/Settings.cs | 63 ++++++++---------- 6 files changed, 52 insertions(+), 113 deletions(-) delete mode 100644 common/services/ASC.ElasticSearch/Config/ElasticSection.cs diff --git a/common/ASC.Common/Utils/ConfigurationManager.cs b/common/ASC.Common/Utils/ConfigurationManager.cs index f5f04216d6..9f3c5fe05c 100644 --- a/common/ASC.Common/Utils/ConfigurationManager.cs +++ b/common/ASC.Common/Utils/ConfigurationManager.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Configuration; using System.Linq; + using Microsoft.Extensions.Configuration; namespace ASC.Common.Utils diff --git a/common/services/ASC.ElasticSearch/Config/ElasticSection.cs b/common/services/ASC.ElasticSearch/Config/ElasticSection.cs deleted file mode 100644 index f3adbf88ba..0000000000 --- a/common/services/ASC.ElasticSearch/Config/ElasticSection.cs +++ /dev/null @@ -1,64 +0,0 @@ -/* - * - * (c) Copyright Ascensio System Limited 2010-2018 - * - * This program is freeware. You can redistribute it and/or modify it under the terms of the GNU - * General Public License (GPL) version 3 as published by the Free Software Foundation (https://www.gnu.org/copyleft/gpl.html). - * In accordance with Section 7(a) of the GNU GPL its Section 15 shall be amended to the effect that - * Ascensio System SIA expressly excludes the warranty of non-infringement of any third-party rights. - * - * THIS PROGRAM IS DISTRIBUTED WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR - * FITNESS FOR A PARTICULAR PURPOSE. For more details, see GNU GPL at https://www.gnu.org/copyleft/gpl.html - * - * You can contact Ascensio System SIA by email at sales@onlyoffice.com - * - * The interactive user interfaces in modified source and object code versions of ONLYOFFICE must display - * Appropriate Legal Notices, as required under Section 5 of the GNU GPL version 3. - * - * Pursuant to Section 7 § 3(b) of the GNU GPL you must retain the original ONLYOFFICE logo which contains - * relevant author attributions when distributing the software. If the display of the logo in its graphic - * form is not reasonably feasible for technical reasons, you must include the words "Powered by ONLYOFFICE" - * in every copy of the program you distribute. - * Pursuant to Section 7 § 3(e) we decline to grant you any rights under trademark law for use of our trademarks. - * -*/ - - -using System; -using System.Configuration; - -namespace ASC.ElasticSearch.Config -{ - class ElasticSection : ConfigurationSection - { - [ConfigurationProperty("host", IsRequired = true, DefaultValue = "localhost")] - public string Host - { - get { return (string)this["host"]; } - } - - [ConfigurationProperty("port", IsRequired = false, DefaultValue = "9200")] - public int Port - { - get { return Convert.ToInt32(this["port"]); } - } - - [ConfigurationProperty("scheme", IsRequired = false, DefaultValue = "http")] - public string Scheme - { - get { return (string)this["scheme"]; } - } - - [ConfigurationProperty("period", IsRequired = false, DefaultValue = 1)] - public int Period - { - get { return Convert.ToInt32(this["period"]); } - } - - [ConfigurationProperty("memoryLimit", IsRequired = false, DefaultValue = 10 * 1024 * 1024L)] - public long MemoryLimit - { - get { return Convert.ToInt64(this["memoryLimit"]); } - } - } -} diff --git a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs index 5b34325d5f..aca17a750d 100644 --- a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs @@ -88,7 +88,8 @@ namespace ASC.ElasticSearch public ILog Log { get; } public TenantManager TenantManager { get; } public SearchSettingsHelper SearchSettingsHelper { get; } - public BaseIndexerHelper BaseIndexerHelper { get; } + public BaseIndexerHelper BaseIndexerHelper { get; } + public Settings Settings { get; } public IServiceProvider ServiceProvider { get; } public WebstudioDbContext WebstudioDbContext { get; } @@ -98,14 +99,16 @@ namespace ASC.ElasticSearch DbContextManager dbContextManager, TenantManager tenantManager, SearchSettingsHelper searchSettingsHelper, - BaseIndexerHelper baseIndexerHelper, + BaseIndexerHelper baseIndexerHelper, + Settings settings, IServiceProvider serviceProvider) { Client = client; Log = log.CurrentValue; TenantManager = tenantManager; SearchSettingsHelper = searchSettingsHelper; - BaseIndexerHelper = baseIndexerHelper; + BaseIndexerHelper = baseIndexerHelper; + Settings = settings; ServiceProvider = serviceProvider; WebstudioDbContext = dbContextManager.Value; } @@ -138,7 +141,7 @@ namespace ASC.ElasticSearch else { var dLength = wwd.Document.Data.Length; - if (dLength >= Settings.Default.MemoryLimit) + if (dLength >= Settings.MemoryLimit) { try { @@ -155,7 +158,7 @@ namespace ASC.ElasticSearch continue; } - if (currentLength + dLength < Settings.Default.MemoryLimit) + if (currentLength + dLength < Settings.MemoryLimit) { portion.Add(t); currentLength += dLength; diff --git a/common/services/ASC.ElasticSearch/Engine/Client.cs b/common/services/ASC.ElasticSearch/Engine/Client.cs index 8bb8295ba2..f7b9340803 100644 --- a/common/services/ASC.ElasticSearch/Engine/Client.cs +++ b/common/services/ASC.ElasticSearch/Engine/Client.cs @@ -49,12 +49,14 @@ namespace ASC.ElasticSearch public ILog Log { get; } - public CoreConfiguration CoreConfiguration { get; } - - public Client(IOptionsMonitor option, CoreConfiguration coreConfiguration) + public CoreConfiguration CoreConfiguration { get; } + public Settings Settings { get; } + + public Client(IOptionsMonitor option, CoreConfiguration coreConfiguration, Settings settings) { Log = option.Get("ASC.Indexer"); - CoreConfiguration = coreConfiguration; + CoreConfiguration = coreConfiguration; + Settings = settings; } public ElasticClient Instance @@ -67,8 +69,7 @@ namespace ASC.ElasticSearch { if (client != null) return client; - var launchSettings = CoreConfiguration.GetSection(Tenant.DEFAULT_TENANT) ?? - Settings.Default; + var launchSettings = CoreConfiguration.GetSection(Tenant.DEFAULT_TENANT) ?? Settings; var uri = new Uri(string.Format("{0}://{1}:{2}", launchSettings.Scheme, launchSettings.Host, launchSettings.Port)); var settings = new ConnectionSettings(new SingleNodeConnectionPool(uri)) @@ -132,7 +133,8 @@ namespace ASC.ElasticSearch public static DIHelper AddClientService(this DIHelper services) { services.TryAddScoped(); - return services + return services + .AddSettingsService() .AddCoreConfigurationService(); } } diff --git a/common/services/ASC.ElasticSearch/Service/Launcher.cs b/common/services/ASC.ElasticSearch/Service/Launcher.cs index cad2f47855..9eb4c6e6e9 100644 --- a/common/services/ASC.ElasticSearch/Service/Launcher.cs +++ b/common/services/ASC.ElasticSearch/Service/Launcher.cs @@ -32,6 +32,7 @@ using System.Threading.Tasks; using ASC.Common; using ASC.Common.Caching; using ASC.Common.Logging; +using ASC.ElasticSearch.Service; using Autofac; @@ -52,19 +53,21 @@ namespace ASC.ElasticSearch private CancellationTokenSource CancellationTokenSource { get; set; } private Timer Timer { get; set; } private DateTime? LastIndexed { get; set; } - private TimeSpan Period { get { return TimeSpan.FromMinutes(1); } }//Settings.Default.Period + private TimeSpan Period { get; set; } public ServiceLauncher( IOptionsMonitor options, ICacheNotify notify, IServiceProvider serviceProvider, - IContainer container) + IContainer container, + Settings settings) { Log = options.Get("ASC.Indexer"); Notify = notify; ServiceProvider = serviceProvider; Container = container; CancellationTokenSource = new CancellationTokenSource(); + Period = TimeSpan.FromMinutes(settings.Period.Value); } public Task StartAsync(CancellationToken cancellationToken) @@ -182,6 +185,7 @@ namespace ASC.ElasticSearch services.TryAddSingleton(); return services + .AddSettingsService() .AddFactoryIndexerService(); } } diff --git a/common/services/ASC.ElasticSearch/Service/Settings.cs b/common/services/ASC.ElasticSearch/Service/Settings.cs index 9b4d3cc8da..60e83daf77 100644 --- a/common/services/ASC.ElasticSearch/Service/Settings.cs +++ b/common/services/ASC.ElasticSearch/Service/Settings.cs @@ -24,54 +24,47 @@ */ -using System.Configuration; -using ASC.ElasticSearch.Config; - +using ASC.Common; +using ASC.Common.Utils; + +using Microsoft.Extensions.Configuration; + namespace ASC.ElasticSearch.Service { public class Settings { - private static readonly Settings DefaultSettings; - - static Settings() + public Settings() + { + + } + + public Settings(IConfiguration configuration) { - DefaultSettings = new Settings - { - Scheme = "http", - Host = "localhost", - Port = 9200, - Period = 1, - MemoryLimit = 10 * 1024 * 1024L - }; - - - var cfg = ConfigurationManager.GetSection("elastic") as ElasticSection; - if (cfg == null) return; - - DefaultSettings = new Settings - { - Scheme = cfg.Scheme, - Host = cfg.Host, - Port = cfg.Port, - Period = cfg.Period, - MemoryLimit = cfg.MemoryLimit - }; - + var cfg = configuration.GetSetting("elastic"); + Scheme = cfg.Scheme ?? "http"; + Host = cfg.Host ?? "localhost"; + Port = cfg.Port ?? 9200; + Period = cfg.Period ?? 1; + MemoryLimit = cfg.MemoryLimit ?? 10 * 1024 * 1024L; } public string Host { get; set; } - public int Port { get; set; } + public int? Port { get; set; } public string Scheme { get; set; } - public int Period { get; set; } + public int? Period { get; set; } - public long MemoryLimit { get; set; } - - public static Settings Default - { - get { return DefaultSettings; } + public long? MemoryLimit { get; set; } + } + + public static class SettingsExtention + { + public static DIHelper AddSettingsService(this DIHelper services) + { + services.TryAddSingleton(); + return services; } } } \ No newline at end of file From f8812bdc05ec40eab8eff569f456414bf2921a64 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Tue, 5 May 2020 20:40:24 +0300 Subject: [PATCH 12/20] ElasticSearch: reindex --- .../ASC.ElasticSearch/Core/SearchSettings.cs | 33 +++++++++++-------- .../ASC.ElasticSearch/Engine/BaseIndexer.cs | 8 ++--- .../ASC.ElasticSearch/Service/Launcher.cs | 8 +++-- .../ASC.ElasticSearch/Service/Service.cs | 25 ++++++++++---- .../ASC.ElasticSearch/protos/SearchItem.proto | 7 +++- 5 files changed, 55 insertions(+), 26 deletions(-) diff --git a/common/services/ASC.ElasticSearch/Core/SearchSettings.cs b/common/services/ASC.ElasticSearch/Core/SearchSettings.cs index ce286a1417..69d7bd7b11 100644 --- a/common/services/ASC.ElasticSearch/Core/SearchSettings.cs +++ b/common/services/ASC.ElasticSearch/Core/SearchSettings.cs @@ -30,6 +30,7 @@ using System.Linq; using System.Runtime.Serialization; using ASC.Common; +using ASC.Common.Caching; using ASC.Core; using ASC.Core.Common.Settings; @@ -82,21 +83,27 @@ namespace ASC.ElasticSearch.Core } public class SearchSettingsHelper - { + { + public TenantManager TenantManager { get; } public SettingsManager SettingsManager { get; } public CoreBaseSettings CoreBaseSettings { get; } - public FactoryIndexer FactoryIndexer { get; } + public FactoryIndexer FactoryIndexer { get; } + public ICacheNotify CacheNotify { get; } public IServiceProvider ServiceProvider { get; } - public SearchSettingsHelper( + public SearchSettingsHelper( + TenantManager tenantManager, SettingsManager settingsManager, CoreBaseSettings coreBaseSettings, - FactoryIndexer factoryIndexer, + FactoryIndexer factoryIndexer, + ICacheNotify cacheNotify, IServiceProvider serviceProvider) - { + { + TenantManager = tenantManager; SettingsManager = settingsManager; CoreBaseSettings = coreBaseSettings; - FactoryIndexer = factoryIndexer; + FactoryIndexer = factoryIndexer; + CacheNotify = cacheNotify; ServiceProvider = serviceProvider; } @@ -135,12 +142,11 @@ namespace ASC.ElasticSearch.Core settings.Items = items; settings.Data = JsonConvert.SerializeObject(items); SettingsManager.Save(settings); - - //TODO: - //using (var service = new ServiceClient()) - //{ - // service.ReIndex(toReIndex.Select(r => r.ID).ToList(), TenantManager.GetCurrentTenant().TenantId); - //} + + var action = new ReIndexAction() { Tenant = TenantManager.GetCurrentTenant().TenantId }; + action.Names.AddRange(toReIndex.Select(r => r.ID).ToList()); + + CacheNotify.Publish(action, CacheNotifyAction.Any); } public bool CanSearchByContent(int tenantId) where T : class, ISearchItem @@ -188,7 +194,8 @@ namespace ASC.ElasticSearch.Core return services .AddSettingsManagerService() .AddCoreBaseSettingsService() - .AddFactoryIndexerService(); + .AddFactoryIndexerService() + .AddTenantManagerService(); } } } diff --git a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs index aca17a750d..e25df16b03 100644 --- a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs @@ -55,21 +55,21 @@ namespace ASC.ElasticSearch public class BaseIndexerHelper { public ConcurrentDictionary IsExist { get; set; } - private readonly ICacheNotify Notify; + private readonly ICacheNotify Notify; - public BaseIndexerHelper(ICacheNotify cacheNotify) + public BaseIndexerHelper(ICacheNotify cacheNotify) { IsExist = new ConcurrentDictionary(); Notify = cacheNotify; Notify.Subscribe((a) => - { + { IsExist.AddOrUpdate(a.Id, false, (q, w) => false); }, CacheNotifyAction.Any); } public void Clear(T t) where T : class, ISearchItem { - Notify.Publish(new SearchItem() { Id = t.IndexName }, CacheNotifyAction.Any); + Notify.Publish(new ClearIndexAction() { Id = t.IndexName }, CacheNotifyAction.Any); } } diff --git a/common/services/ASC.ElasticSearch/Service/Launcher.cs b/common/services/ASC.ElasticSearch/Service/Launcher.cs index 9eb4c6e6e9..792151da5f 100644 --- a/common/services/ASC.ElasticSearch/Service/Launcher.cs +++ b/common/services/ASC.ElasticSearch/Service/Launcher.cs @@ -88,10 +88,11 @@ namespace ASC.ElasticSearch Log.Error("Subscribe on start", e); } - var task = new Task(() => + var task = new Task(async () => { using var scope = ServiceProvider.CreateScope(); var factoryIndexer = scope.ServiceProvider.GetService(); + var service = scope.ServiceProvider.GetService(); while (!factoryIndexer.CheckState(false)) { @@ -99,9 +100,11 @@ namespace ASC.ElasticSearch { return; } - Thread.Sleep(10000); + + await Task.Delay(10000); } + service.Subscribe(); Timer = new Timer(_ => IndexAll(), null, TimeSpan.Zero, TimeSpan.Zero); }, CancellationTokenSource.Token, TaskCreationOptions.LongRunning); @@ -183,6 +186,7 @@ namespace ASC.ElasticSearch public static DIHelper AddServiceLauncher(this DIHelper services) { services.TryAddSingleton(); + services.TryAddSingleton(); return services .AddSettingsService() diff --git a/common/services/ASC.ElasticSearch/Service/Service.cs b/common/services/ASC.ElasticSearch/Service/Service.cs index 3908dbcfce..01f3d8ea5b 100644 --- a/common/services/ASC.ElasticSearch/Service/Service.cs +++ b/common/services/ASC.ElasticSearch/Service/Service.cs @@ -29,6 +29,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using ASC.Common.Caching; using ASC.Core; using ASC.Core.Common.Settings; using ASC.ElasticSearch.Core; @@ -38,26 +39,36 @@ using Autofac; using Microsoft.Extensions.DependencyInjection; namespace ASC.ElasticSearch.Service -{ +{ public class Service { - public FactoryIndexer FactoryIndexer { get; } + public IContainer Container { get; } public IServiceProvider ServiceProvider { get; } + public ICacheNotify CacheNotify { get; } - public Service(FactoryIndexer factoryIndexer, IServiceProvider serviceProvider) + public Service(IContainer container, IServiceProvider serviceProvider, ICacheNotify cacheNotify) { - FactoryIndexer = factoryIndexer; + Container = container; ServiceProvider = serviceProvider; + CacheNotify = cacheNotify; + } + + public void Subscribe() + { + CacheNotify.Subscribe((a) => + { + ReIndex(a.Names.ToList(), a.Tenant); + }, CacheNotifyAction.Any); } public bool Support(string table) { - return FactoryIndexer.Builder.Resolve>().Any(r => r.IndexName == table); + return Container.Resolve>().Any(r => r.IndexName == table); } public void ReIndex(List toReIndex, int tenant) { - var allItems = FactoryIndexer.Builder.Resolve>().ToList(); + var allItems = Container.Resolve>().ToList(); var tasks = new List(toReIndex.Count); foreach (var item in toReIndex) @@ -69,6 +80,8 @@ namespace ASC.ElasticSearch.Service var instance = (IIndexer)Activator.CreateInstance(generic.MakeGenericType(index.GetType()), index); tasks.Add(instance.ReIndex()); } + + if (!tasks.Any()) return; Task.WhenAll(tasks).ContinueWith(r => { diff --git a/common/services/ASC.ElasticSearch/protos/SearchItem.proto b/common/services/ASC.ElasticSearch/protos/SearchItem.proto index cc8dda6cf9..a3c0ecaec1 100644 --- a/common/services/ASC.ElasticSearch/protos/SearchItem.proto +++ b/common/services/ASC.ElasticSearch/protos/SearchItem.proto @@ -2,6 +2,11 @@ package ASC.ElasticSearch; -message SearchItem { +message ClearIndexAction { string Id = 1; +} + +message ReIndexAction { + int32 Tenant = 1; + repeated string Names = 2; } \ No newline at end of file From 7cc06c2322508043962da2350dd961194f4d797e Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Wed, 6 May 2020 14:32:26 +0300 Subject: [PATCH 13/20] ElasticSearch: state --- .../Engine/FactoryIndexer.cs | 167 +++++++++--------- .../ASC.ElasticSearch/Service/Launcher.cs | 10 +- .../ASC.ElasticSearch/protos/SearchItem.proto | 5 + .../Server/Core/Search/FactoryIndexerFile.cs | 4 +- .../Core/Search/FactoryIndexerFolder.cs | 4 +- 5 files changed, 94 insertions(+), 96 deletions(-) diff --git a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs index 7735bcfd32..d116f93e9a 100644 --- a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs @@ -52,49 +52,21 @@ using Nest; namespace ASC.ElasticSearch { public class FactoryIndexerHelper - { - public ICache Cache { get; } - public ILog Logger { get; } - public FactoryIndexer FactoryIndexer { get; } + { + public DateTime LastIndexed { get; set; } + public string Indexing { get; set; } - public FactoryIndexerHelper(IOptionsMonitor options, FactoryIndexer factoryIndexer) - { - Cache = AscCache.Memory; - Logger = options.Get("ASC.Indexer"); - FactoryIndexer = factoryIndexer; + public FactoryIndexerHelper(ICacheNotify cacheNotify) + { + cacheNotify.Subscribe((a) => + { + if (a.LastIndexed != 0) + { + LastIndexed = new DateTime(a.LastIndexed); + } + Indexing = a.Indexing; + }, CacheNotifyAction.Any); } - - public bool Support(T t) where T : class, ISearchItem - { - if (!FactoryIndexer.CheckState()) return false; - - var cacheTime = DateTime.UtcNow.AddMinutes(15); - var key = "elasticsearch " + t.IndexName; - try - { - var cacheValue = Cache.Get(key); - if (!string.IsNullOrEmpty(cacheValue)) - { - return Convert.ToBoolean(cacheValue); - } - - //TODO: - //var service = new Service.Service(); - - //var result = service.Support(t.IndexName); - - //Cache.Insert(key, result.ToString(CultureInfo.InvariantCulture).ToLower(), cacheTime); - - return true; - } - catch (Exception e) - { - Cache.Insert(key, "false", cacheTime); - Logger.Error("FactoryIndexer CheckState", e); - return false; - } - } - } public interface IFactoryIndexer @@ -111,34 +83,30 @@ namespace ASC.ElasticSearch public ILog Logger { get; } - public FactoryIndexerHelper FactoryIndexerHelper { get; } public TenantManager TenantManager { get; } public SearchSettingsHelper SearchSettingsHelper { get; } public FactoryIndexer FactoryIndexerCommon { get; } public BaseIndexer Indexer { get; } - public Client Client { get; } public IServiceProvider ServiceProvider { get; } public string IndexName { get => Indexer.IndexName; } + public ICache Cache { get; } public virtual string SettingsTitle { get => ""; } public FactoryIndexer( IOptionsMonitor options, - FactoryIndexerHelper factoryIndexerSupport, TenantManager tenantManager, SearchSettingsHelper searchSettingsHelper, FactoryIndexer factoryIndexer, BaseIndexer baseIndexer, - Client client, IServiceProvider serviceProvider) - { + { + Cache = AscCache.Memory; Logger = options.Get("ASC.Indexer"); - FactoryIndexerHelper = factoryIndexerSupport; TenantManager = tenantManager; SearchSettingsHelper = searchSettingsHelper; FactoryIndexerCommon = factoryIndexer; Indexer = baseIndexer; - Client = client; ServiceProvider = serviceProvider; Indexer.CreateIfNotExist(ServiceProvider.GetService()); @@ -147,7 +115,7 @@ namespace ASC.ElasticSearch public bool TrySelect(Expression, Selector>> expression, out IReadOnlyCollection result) { var t = ServiceProvider.GetService(); - if (!FactoryIndexerHelper.Support(t) || !Indexer.CheckExist(t)) + if (!Support(t) || !Indexer.CheckExist(t)) { result = new List(); return false; @@ -169,7 +137,7 @@ namespace ASC.ElasticSearch public bool TrySelectIds(Expression, Selector>> expression, out List result) { var t = ServiceProvider.GetService(); - if (!FactoryIndexerHelper.Support(t) || !Indexer.CheckExist(t)) + if (!Support(t) || !Indexer.CheckExist(t)) { result = new List(); return false; @@ -192,7 +160,7 @@ namespace ASC.ElasticSearch public bool TrySelectIds(Expression, Selector>> expression, out List result, out long total) { var t = ServiceProvider.GetService(); - if (!FactoryIndexerHelper.Support(t) || !Indexer.CheckExist(t)) + if (!Support(t) || !Indexer.CheckExist(t)) { result = new List(); total = 0; @@ -222,7 +190,7 @@ namespace ASC.ElasticSearch public bool Index(T data, bool immediately = true) { var t = ServiceProvider.GetService(); - if (!FactoryIndexerHelper.Support(t)) return false; + if (!Support(t)) return false; try { @@ -239,7 +207,7 @@ namespace ASC.ElasticSearch public void Index(List data, bool immediately = true) { var t = ServiceProvider.GetService(); - if (!FactoryIndexerHelper.Support(t) || !data.Any()) return; + if (!Support(t) || !data.Any()) return; try { @@ -271,7 +239,7 @@ namespace ASC.ElasticSearch public void Update(T data, bool immediately = true, params Expression>[] fields) { var t = ServiceProvider.GetService(); - if (!FactoryIndexerHelper.Support(t)) return; + if (!Support(t)) return; try { @@ -286,7 +254,7 @@ namespace ASC.ElasticSearch public void Update(T data, UpdateAction action, Expression> field, bool immediately = true) { var t = ServiceProvider.GetService(); - if (!FactoryIndexerHelper.Support(t)) return; + if (!Support(t)) return; try { @@ -301,7 +269,7 @@ namespace ASC.ElasticSearch public void Update(T data, Expression, Selector>> expression, bool immediately = true, params Expression>[] fields) { var t = ServiceProvider.GetService(); - if (!FactoryIndexerHelper.Support(t)) return; + if (!Support(t)) return; try { @@ -317,7 +285,7 @@ namespace ASC.ElasticSearch public void Update(T data, Expression, Selector>> expression, UpdateAction action, Expression> fields, bool immediately = true) { var t = ServiceProvider.GetService(); - if (!FactoryIndexerHelper.Support(t)) return; + if (!Support(t)) return; try { @@ -333,7 +301,7 @@ namespace ASC.ElasticSearch public void Delete(T data, bool immediately = true) { var t = ServiceProvider.GetService(); - if (!FactoryIndexerHelper.Support(t)) return; + if (!Support(t)) return; try { @@ -348,7 +316,7 @@ namespace ASC.ElasticSearch public void Delete(Expression, Selector>> expression, bool immediately = true) { var t = ServiceProvider.GetService(); - if (!FactoryIndexerHelper.Support(t)) return; + if (!Support(t)) return; var tenant = TenantManager.GetCurrentTenant().TenantId; @@ -365,35 +333,35 @@ namespace ASC.ElasticSearch public Task IndexAsync(T data, bool immediately = true) { var t = ServiceProvider.GetService(); - if (!FactoryIndexerHelper.Support(t)) return Task.FromResult(false); + if (!Support(t)) return Task.FromResult(false); return Queue(() => Indexer.Index(data, immediately)); } public Task IndexAsync(List data, bool immediately = true) { var t = ServiceProvider.GetService(); - if (!FactoryIndexerHelper.Support(t)) return Task.FromResult(false); + if (!Support(t)) return Task.FromResult(false); return Queue(() => Indexer.Index(data, immediately)); } public Task UpdateAsync(T data, bool immediately = true, params Expression>[] fields) { var t = ServiceProvider.GetService(); - if (!FactoryIndexerHelper.Support(t)) return Task.FromResult(false); + if (!Support(t)) return Task.FromResult(false); return Queue(() => Indexer.Update(data, immediately, fields)); } public Task DeleteAsync(T data, bool immediately = true) { var t = ServiceProvider.GetService(); - if (!FactoryIndexerHelper.Support(t)) return Task.FromResult(false); + if (!Support(t)) return Task.FromResult(false); return Queue(() => Indexer.Delete(data, immediately)); } public Task DeleteAsync(Expression, Selector>> expression, bool immediately = true) { var t = ServiceProvider.GetService(); - if (!FactoryIndexerHelper.Support(t)) return Task.FromResult(false); + if (!Support(t)) return Task.FromResult(false); var tenant = TenantManager.GetCurrentTenant().TenantId; return Queue(() => Indexer.Delete(expression, tenant, immediately)); } @@ -402,14 +370,14 @@ namespace ASC.ElasticSearch public void Flush() { var t = ServiceProvider.GetService(); - if (!FactoryIndexerHelper.Support(t)) return; + if (!Support(t)) return; Indexer.Flush(); } public void Refresh() { var t = ServiceProvider.GetService(); - if (!FactoryIndexerHelper.Support(t)) return; + if (!Support(t)) return; Indexer.Refresh(); } @@ -447,11 +415,44 @@ namespace ASC.ElasticSearch { Indexer.ReIndex(); } + + public bool Support(T t) + { + if (!FactoryIndexerCommon.CheckState()) return false; + + var cacheTime = DateTime.UtcNow.AddMinutes(15); + var key = "elasticsearch " + t.IndexName; + try + { + var cacheValue = Cache.Get(key); + if (!string.IsNullOrEmpty(cacheValue)) + { + return Convert.ToBoolean(cacheValue); + } + + //TODO: + //var service = new Service.Service(); + + //var result = service.Support(t.IndexName); + + //Cache.Insert(key, result.ToString(CultureInfo.InvariantCulture).ToLower(), cacheTime); + + return true; + } + catch (Exception e) + { + Cache.Insert(key, "false", cacheTime); + Logger.Error("FactoryIndexer CheckState", e); + return false; + } + } } public class FactoryIndexer { - private static ICache cache = AscCache.Memory; + private static ICache cache = AscCache.Memory; + + public FactoryIndexerHelper FactoryIndexerHelper { get; } internal ILifetimeScope Builder { get; set; } internal static bool Init { get; set; } public ILog Log { get; } @@ -459,20 +460,23 @@ namespace ASC.ElasticSearch public CoreBaseSettings CoreBaseSettings { get; } public FactoryIndexer( - ILifetimeScope container, + ILifetimeScope container, + FactoryIndexerHelper factoryIndexerHelper, Client client, IOptionsMonitor options, - CoreBaseSettings coreBaseSettings) : this(null, client, options, coreBaseSettings) + CoreBaseSettings coreBaseSettings) : this(null, factoryIndexerHelper, client, options, coreBaseSettings) { Builder = container; } public FactoryIndexer( - IContainer container, + IContainer container, + FactoryIndexerHelper factoryIndexerHelper, Client client, IOptionsMonitor options, CoreBaseSettings coreBaseSettings) { + FactoryIndexerHelper = factoryIndexerHelper; Client = client; CoreBaseSettings = coreBaseSettings; @@ -550,12 +554,12 @@ namespace ASC.ElasticSearch State state = null; if (CoreBaseSettings.Standalone) - { - //TODO - //using (var service = new ServiceClient()) - //{ - // state = service.GetState(); - //} + { + state = new State + { + Indexing = FactoryIndexerHelper.Indexing, + LastIndexed = FactoryIndexerHelper.LastIndexed != DateTime.MinValue ? FactoryIndexerHelper.LastIndexed : default(DateTime?) + }; if (state.LastIndexed.HasValue) { @@ -566,7 +570,7 @@ namespace ASC.ElasticSearch return new { state, - //indices, + indices, status = CheckState() }; } @@ -591,19 +595,13 @@ namespace ASC.ElasticSearch { public static DIHelper AddFactoryIndexerService(this DIHelper services) { + services.TryAddSingleton(); services.TryAddScoped(); return services .AddClientService() .AddCoreBaseSettingsService(); } - public static DIHelper AddFactoryIndexerHelperService(this DIHelper services) - { - services.TryAddScoped(); - return services - .AddFactoryIndexerService(); - } - public static DIHelper AddFactoryIndexerService(this DIHelper services, bool addBase = true) where T : class, ISearchItem { if (addBase) @@ -614,7 +612,6 @@ namespace ASC.ElasticSearch services.TryAddScoped>(); return services - .AddFactoryIndexerHelperService() .AddTenantManagerService() .AddFactoryIndexerService() .AddClientService() diff --git a/common/services/ASC.ElasticSearch/Service/Launcher.cs b/common/services/ASC.ElasticSearch/Service/Launcher.cs index 792151da5f..c498898430 100644 --- a/common/services/ASC.ElasticSearch/Service/Launcher.cs +++ b/common/services/ASC.ElasticSearch/Service/Launcher.cs @@ -46,24 +46,25 @@ namespace ASC.ElasticSearch { private ILog Log { get; } private ICacheNotify Notify { get; } + public ICacheNotify IndexNotify { get; } public IServiceProvider ServiceProvider { get; } public IContainer Container { get; } private bool IsStarted { get; set; } - private string Indexing { get; set; } private CancellationTokenSource CancellationTokenSource { get; set; } private Timer Timer { get; set; } - private DateTime? LastIndexed { get; set; } private TimeSpan Period { get; set; } public ServiceLauncher( IOptionsMonitor options, ICacheNotify notify, + ICacheNotify indexNotify, IServiceProvider serviceProvider, IContainer container, Settings settings) { Log = options.Get("ASC.Indexer"); Notify = notify; + IndexNotify = indexNotify; ServiceProvider = serviceProvider; Container = container; CancellationTokenSource = new CancellationTokenSource(); @@ -142,9 +143,8 @@ namespace ASC.ElasticSearch } Timer.Change(Period, Period); - LastIndexed = DateTime.UtcNow; + IndexNotify.Publish(new IndexAction() { Indexing = "", LastIndexed = DateTime.Now.Ticks }, CacheNotifyAction.Any); IsStarted = false; - Indexing = null; } public void IndexProduct(IFactoryIndexer product, bool reindex) @@ -170,7 +170,7 @@ namespace ASC.ElasticSearch if (!IsStarted) return; Log.DebugFormat("Product {0}", product.IndexName); - Indexing = product.IndexName; + IndexNotify.Publish(new IndexAction() { Indexing = product.IndexName, LastIndexed = 0 }, CacheNotifyAction.Any); product.IndexAll(); } catch (Exception e) diff --git a/common/services/ASC.ElasticSearch/protos/SearchItem.proto b/common/services/ASC.ElasticSearch/protos/SearchItem.proto index a3c0ecaec1..486eff724e 100644 --- a/common/services/ASC.ElasticSearch/protos/SearchItem.proto +++ b/common/services/ASC.ElasticSearch/protos/SearchItem.proto @@ -9,4 +9,9 @@ message ClearIndexAction { message ReIndexAction { int32 Tenant = 1; repeated string Names = 2; +} + +message IndexAction { + string Indexing = 1; + int64 LastIndexed = 2; } \ No newline at end of file diff --git a/products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs b/products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs index 9aacc540fb..2aa4bbeb4f 100644 --- a/products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs +++ b/products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs @@ -47,15 +47,13 @@ namespace ASC.Web.Files.Core.Search public FactoryIndexerFile( IOptionsMonitor options, - FactoryIndexerHelper factoryIndexerSupport, TenantManager tenantManager, SearchSettingsHelper searchSettingsHelper, FactoryIndexer factoryIndexer, BaseIndexer baseIndexer, - Client client, IServiceProvider serviceProvider, IDaoFactory daoFactory) - : base(options, factoryIndexerSupport, tenantManager, searchSettingsHelper, factoryIndexer, baseIndexer, client, serviceProvider) + : base(options, tenantManager, searchSettingsHelper, factoryIndexer, baseIndexer, serviceProvider) { DaoFactory = daoFactory; } diff --git a/products/ASC.Files/Server/Core/Search/FactoryIndexerFolder.cs b/products/ASC.Files/Server/Core/Search/FactoryIndexerFolder.cs index 93a64b9367..9977607f30 100644 --- a/products/ASC.Files/Server/Core/Search/FactoryIndexerFolder.cs +++ b/products/ASC.Files/Server/Core/Search/FactoryIndexerFolder.cs @@ -47,15 +47,13 @@ namespace ASC.Web.Files.Core.Search public FactoryIndexerFolder( IOptionsMonitor options, - FactoryIndexerHelper factoryIndexerSupport, TenantManager tenantManager, SearchSettingsHelper searchSettingsHelper, FactoryIndexer factoryIndexer, BaseIndexer baseIndexer, - Client client, IServiceProvider serviceProvider, IDaoFactory daoFactory) - : base(options, factoryIndexerSupport, tenantManager, searchSettingsHelper, factoryIndexer, baseIndexer, client, serviceProvider) + : base(options, tenantManager, searchSettingsHelper, factoryIndexer, baseIndexer, serviceProvider) { DaoFactory = daoFactory; } From 90d93db14cdb5548365f34bb155d47f109d8007a Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Fri, 8 May 2020 11:35:04 +0300 Subject: [PATCH 14/20] FileService: bat and launchSettings --- build/run/FileService.bat | 2 ++ products/ASC.Files/Service/Properties/launchSettings.json | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 build/run/FileService.bat diff --git a/build/run/FileService.bat b/build/run/FileService.bat new file mode 100644 index 0000000000..ab02e8eb4a --- /dev/null +++ b/build/run/FileService.bat @@ -0,0 +1,2 @@ +echo "RUN ASC.Files" +call dotnet run --project ..\..\products\ASC.Files\Service\ASC.Files.Service.csproj --no-build --$STORAGE_ROOT=..\..\..\Data --log__dir=..\..\..\Logs --log__name=files \ No newline at end of file diff --git a/products/ASC.Files/Service/Properties/launchSettings.json b/products/ASC.Files/Service/Properties/launchSettings.json index d85bbca027..987ba7f654 100644 --- a/products/ASC.Files/Service/Properties/launchSettings.json +++ b/products/ASC.Files/Service/Properties/launchSettings.json @@ -3,7 +3,7 @@ "windowsAuthentication": false, "anonymousAuthentication": true, "iisExpress": { - "applicationUrl": "http://localhost:5010", + "applicationUrl": "http://localhost:5009", "sslPort": 0 } }, @@ -21,7 +21,7 @@ "ASC.Files.Service": { "commandName": "Project", "launchBrowser": false, - "applicationUrl": "http://localhost:5010", + "applicationUrl": "http://localhost:5009", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development", "$STORAGE_ROOT": "../../../Data", From cd70aebb0c9543bbc6acf8d9333bc92b0717141d Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Fri, 8 May 2020 17:04:05 +0300 Subject: [PATCH 15/20] Elastic: removed unused classes --- .../Attributes/ColumnAttribute.cs | 84 ------------------- .../Attributes/JoinAttribute.cs | 58 ------------- .../ASC.ElasticSearch/Core/Selector.cs | 13 ++- 3 files changed, 6 insertions(+), 149 deletions(-) delete mode 100644 common/services/ASC.ElasticSearch/Attributes/JoinAttribute.cs diff --git a/common/services/ASC.ElasticSearch/Attributes/ColumnAttribute.cs b/common/services/ASC.ElasticSearch/Attributes/ColumnAttribute.cs index 046fa4f93f..819ed3526b 100644 --- a/common/services/ASC.ElasticSearch/Attributes/ColumnAttribute.cs +++ b/common/services/ASC.ElasticSearch/Attributes/ColumnAttribute.cs @@ -28,90 +28,6 @@ using System; namespace ASC.ElasticSearch { - [AttributeUsage(AttributeTargets.Property)] - public class ColumnAttribute : Attribute - { - public string ColumnName { get; private set; } - - public ColumnTypeEnum ColumnType { get; private set; } - - public int Order { get; private set; } - - public Analyzer Analyzer { get; set; } - - public Filter Filter { get; set; } - - public CharFilter CharFilter { get; set; } - - public ColumnAttribute(string columnName, int order, ColumnTypeEnum columnType = ColumnTypeEnum.Content, Filter filter = Filter.lowercase, CharFilter charFilter = CharFilter.io) - { - ColumnName = columnName; - ColumnType = columnType; - Order = order; - Analyzer = Analyzer.whitespace; - Filter = filter; - CharFilter = charFilter; - - } - } - - [AttributeUsage(AttributeTargets.Property)] - public class ColumnIdAttribute : ColumnAttribute - { - public ColumnIdAttribute(string columnName) - : base(columnName, -3, ColumnTypeEnum.Id) - { - } - } - - [AttributeUsage(AttributeTargets.Property)] - public class ColumnTenantIdAttribute : ColumnAttribute - { - public ColumnTenantIdAttribute(string columnName) - : base(columnName, -2, ColumnTypeEnum.TenantId) - { - } - } - - [AttributeUsage(AttributeTargets.Property)] - public class ColumnLastModifiedAttribute : ColumnAttribute - { - public ColumnLastModifiedAttribute(string columnName) - : base(columnName, -1, ColumnTypeEnum.LastModified) - { - } - } - - [AttributeUsage(AttributeTargets.Property)] - public class ColumnConditionAttribute : ColumnAttribute - { - internal object Value { get; private set; } - public ColumnConditionAttribute(string columnName, int order, object value = null) - : base(columnName, order, ColumnTypeEnum.Condition) - { - Value = value; - } - } - - [AttributeUsage(AttributeTargets.Property)] - public class ColumnMeta : ColumnAttribute - { - public ColumnMeta(string columnName, int order) - : base(columnName, order, ColumnTypeEnum.Meta) - { - } - } - - public enum ColumnTypeEnum - { - Id, - TenantId, - LastModified, - Content, - Condition, - Meta - } - public enum Analyzer { standard, diff --git a/common/services/ASC.ElasticSearch/Attributes/JoinAttribute.cs b/common/services/ASC.ElasticSearch/Attributes/JoinAttribute.cs deleted file mode 100644 index b09e347286..0000000000 --- a/common/services/ASC.ElasticSearch/Attributes/JoinAttribute.cs +++ /dev/null @@ -1,58 +0,0 @@ -/* - * - * (c) Copyright Ascensio System Limited 2010-2018 - * - * This program is freeware. You can redistribute it and/or modify it under the terms of the GNU - * General Public License (GPL) version 3 as published by the Free Software Foundation (https://www.gnu.org/copyleft/gpl.html). - * In accordance with Section 7(a) of the GNU GPL its Section 15 shall be amended to the effect that - * Ascensio System SIA expressly excludes the warranty of non-infringement of any third-party rights. - * - * THIS PROGRAM IS DISTRIBUTED WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR - * FITNESS FOR A PARTICULAR PURPOSE. For more details, see GNU GPL at https://www.gnu.org/copyleft/gpl.html - * - * You can contact Ascensio System SIA by email at sales@onlyoffice.com - * - * The interactive user interfaces in modified source and object code versions of ONLYOFFICE must display - * Appropriate Legal Notices, as required under Section 5 of the GNU GPL version 3. - * - * Pursuant to Section 7 § 3(b) of the GNU GPL you must retain the original ONLYOFFICE logo which contains - * relevant author attributions when distributing the software. If the display of the logo in its graphic - * form is not reasonably feasible for technical reasons, you must include the words "Powered by ONLYOFFICE" - * in every copy of the program you distribute. - * Pursuant to Section 7 § 3(e) we decline to grant you any rights under trademark law for use of our trademarks. - * -*/ - - -using System; - -namespace ASC.ElasticSearch -{ - [AttributeUsage(AttributeTargets.Property, AllowMultiple = true)] - public class JoinAttribute : Attribute - { - public JoinTypeEnum JoinType { get; private set; } - public string[] ColumnsFrom { get; private set; } - public string[] ColumnsTo { get; private set; } - - public JoinAttribute(JoinTypeEnum joinType, params string[] columns) - { - JoinType = joinType; - ColumnsFrom = new string[columns.Length]; - ColumnsTo = new string[columns.Length]; - - for (var i = 0; i< columns.Length; i++) - { - var column = columns[i].Split(':'); - ColumnsFrom[i] = column[0]; - ColumnsTo[i] = column[1]; - } - } - } - - public enum JoinTypeEnum - { - Inner, - Sub - } -} diff --git a/common/services/ASC.ElasticSearch/Core/Selector.cs b/common/services/ASC.ElasticSearch/Core/Selector.cs index c11d0e6de3..4dca9ac94a 100644 --- a/common/services/ASC.ElasticSearch/Core/Selector.cs +++ b/common/services/ASC.ElasticSearch/Core/Selector.cs @@ -118,11 +118,11 @@ namespace ASC.ElasticSearch if (IsExactlyPhrase(value)) { - queryContainer = queryContainer & Wrap(selector, (a, w) => w.MatchPhrase(r => r.Field(a).Query(value.TrimQuotes()))); + queryContainer &= Wrap(selector, (a, w) => w.MatchPhrase(r => r.Field(a).Query(value.TrimQuotes()))); } else if (value.HasOtherLetter() || IsExactly(value)) { - queryContainer = queryContainer & Wrap(selector, (a, w) => w.Match(r => r.Field(a).Query(value.TrimQuotes()))); + queryContainer &= Wrap(selector, (a, w) => w.Match(r => r.Field(a).Query(value.TrimQuotes()))); } else { @@ -132,19 +132,19 @@ namespace ASC.ElasticSearch foreach (var p in phrase) { var p1 = p; - queryContainer = queryContainer & Wrap(selector, (a, w) => w.Wildcard(r => r.Field(a).Value(p1.WrapAsterisk()))); + queryContainer &= Wrap(selector, (a, w) => w.Wildcard(r => r.Field(a).Value(p1.WrapAsterisk()))); } } else { - queryContainer = queryContainer & Wrap(selector, (a, w) => w.Wildcard(r => r.Field(a).Value(value.WrapAsterisk()))); + queryContainer &= Wrap(selector, (a, w) => w.Wildcard(r => r.Field(a).Value(value.WrapAsterisk()))); } } if (IsExactly(value)) { - queryContainer = queryContainer | Wrap(selector, (a, w) => w.MatchPhrase(r => r.Field(a).Query(value))); + queryContainer |= Wrap(selector, (a, w) => w.MatchPhrase(r => r.Field(a).Query(value))); } return this; @@ -197,7 +197,7 @@ namespace ASC.ElasticSearch public Selector MatchAll(string value) { - //Match(() => ServiceProvider.GetService().GetContentProperties(), value); TODO: + //Match(() => ServiceProvider.GetService().GetContentProperties(), value); return this; } @@ -329,7 +329,6 @@ namespace ASC.ElasticSearch if (string.IsNullOrEmpty(path) && !string.IsNullOrEmpty(fieldSelector.Name) && - fieldSelector.Name.StartsWith(JoinTypeEnum.Sub + ":") && fieldSelector.Name.IndexOf(".", StringComparison.InvariantCulture) > 0) { var splitted = fieldSelector.Name.Split(':')[1]; From 74e851e01b138481bada3fc08da10c4bb338ff15 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Fri, 8 May 2020 18:10:53 +0300 Subject: [PATCH 16/20] ElasticSearch: fixed MatchAll --- common/services/ASC.ElasticSearch/Core/ISearchItem.cs | 7 ++++++- common/services/ASC.ElasticSearch/Core/Selector.cs | 6 ++++-- products/ASC.Files/Server/Core/EF/DbFile.cs | 6 ++++++ products/ASC.Files/Server/Core/EF/DbFolder.cs | 9 +++++++++ 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/common/services/ASC.ElasticSearch/Core/ISearchItem.cs b/common/services/ASC.ElasticSearch/Core/ISearchItem.cs index 5519fba136..af18c598f0 100644 --- a/common/services/ASC.ElasticSearch/Core/ISearchItem.cs +++ b/common/services/ASC.ElasticSearch/Core/ISearchItem.cs @@ -1,10 +1,15 @@ -namespace ASC.ElasticSearch +using System; +using System.Linq.Expressions; + +namespace ASC.ElasticSearch { public interface ISearchItem { public int Id { get; set; } public int TenantId { get; set; } public string IndexName { get; } + + public Expression> SearchContentFields { get; } } public interface ISearchItemDocument : ISearchItem diff --git a/common/services/ASC.ElasticSearch/Core/Selector.cs b/common/services/ASC.ElasticSearch/Core/Selector.cs index 4dca9ac94a..5f0e52b025 100644 --- a/common/services/ASC.ElasticSearch/Core/Selector.cs +++ b/common/services/ASC.ElasticSearch/Core/Selector.cs @@ -29,7 +29,9 @@ using System.Globalization; using System.Linq; using System.Linq.Expressions; -using Nest; +using Microsoft.Extensions.DependencyInjection; + +using Nest; namespace ASC.ElasticSearch { @@ -197,7 +199,7 @@ namespace ASC.ElasticSearch public Selector MatchAll(string value) { - //Match(() => ServiceProvider.GetService().GetContentProperties(), value); + Match(() => ((NewArrayExpression)(ServiceProvider.GetService().SearchContentFields).Body).Expressions.ToArray(), value); return this; } diff --git a/products/ASC.Files/Server/Core/EF/DbFile.cs b/products/ASC.Files/Server/Core/EF/DbFile.cs index 4299f9c656..ec81eeefd7 100644 --- a/products/ASC.Files/Server/Core/EF/DbFile.cs +++ b/products/ASC.Files/Server/Core/EF/DbFile.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.ComponentModel.DataAnnotations.Schema; +using System.Linq.Expressions; using ASC.Core.Common.EF; using ASC.ElasticSearch; @@ -84,6 +85,11 @@ namespace ASC.Files.Core.EF [NotMapped] public Document Document { get; set; } + public Expression> SearchContentFields + { + get => (a) => new[] { Title, Comment, Changes, Document.Attachment.Content }; + } + public override object[] GetKeys() { return new object[] { TenantId, Id, Version }; diff --git a/products/ASC.Files/Server/Core/EF/DbFolder.cs b/products/ASC.Files/Server/Core/EF/DbFolder.cs index 3eaec58529..9e147fe5eb 100644 --- a/products/ASC.Files/Server/Core/EF/DbFolder.cs +++ b/products/ASC.Files/Server/Core/EF/DbFolder.cs @@ -1,5 +1,6 @@ using System; using System.ComponentModel.DataAnnotations.Schema; +using System.Linq.Expressions; using ASC.ElasticSearch; @@ -46,5 +47,13 @@ namespace ASC.Files.Core.EF { get => Tables.Folder; } + + public Expression> SearchContentFields + { + get + { + return (a) => new[] { Title }; + } + } } } From 6bf191b161d3b1a13e0856ca8a8d7887891fa36c Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Fri, 8 May 2020 18:15:08 +0300 Subject: [PATCH 17/20] AddFilesWrapperService -> AddFactoryIndexerFileService, AddFolderWrapperService -> AddFactoryIndexerFolderService, --- products/ASC.Files/Server/Configuration/ProductEntryPoint.cs | 2 +- products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs | 2 +- products/ASC.Files/Server/Core/FileStorageService.cs | 4 ++-- products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs | 4 ++-- products/ASC.Files/Server/Core/Search/FactoryIndexerFolder.cs | 4 ++-- products/ASC.Files/Service/Program.cs | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/products/ASC.Files/Server/Configuration/ProductEntryPoint.cs b/products/ASC.Files/Server/Configuration/ProductEntryPoint.cs index 682c83877e..05bb3c9cfa 100644 --- a/products/ASC.Files/Server/Configuration/ProductEntryPoint.cs +++ b/products/ASC.Files/Server/Configuration/ProductEntryPoint.cs @@ -182,7 +182,7 @@ namespace ASC.Web.Files.Configuration .AddUserManagerService() .AddGlobalService() .AddFilesSubscriptionManagerService() - .AddFilesWrapperService(); + .AddFactoryIndexerFileService(); } } } \ No newline at end of file diff --git a/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs b/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs index 30e0d710c7..70c586f755 100644 --- a/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs +++ b/products/ASC.Files/Server/Core/Dao/TeamlabDao/FileDao.cs @@ -1396,7 +1396,7 @@ namespace ASC.Files.Core.Data .AddAuthContextService() .AddGlobalStoreService() .AddGlobalSpaceService() - .AddFilesWrapperService() + .AddFactoryIndexerFileService() .AddGlobalFolderService() .AddChunkedUploadSessionHolderService() .AddFolderDaoService(); diff --git a/products/ASC.Files/Server/Core/FileStorageService.cs b/products/ASC.Files/Server/Core/FileStorageService.cs index 5b957d7815..5501a85232 100644 --- a/products/ASC.Files/Server/Core/FileStorageService.cs +++ b/products/ASC.Files/Server/Core/FileStorageService.cs @@ -2062,8 +2062,8 @@ namespace ASC.Web.Files.Services.WCFService .AddGlobalFolderHelperService() .AddAuthContextService() .AddUserManagerService() - .AddFoldersWrapperService() - .AddFilesWrapperService() + .AddFactoryIndexerFolderService() + .AddFactoryIndexerFileService() .AddFilesLinkUtilityService() .AddBaseCommonLinkUtilityService() .AddCoreBaseSettingsService() diff --git a/products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs b/products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs index 2aa4bbeb4f..14dfb6deda 100644 --- a/products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs +++ b/products/ASC.Files/Server/Core/Search/FactoryIndexerFile.cs @@ -113,9 +113,9 @@ namespace ASC.Web.Files.Core.Search get { return FilesCommonResource.IndexTitle; } } } - public static class FilesWrapperExtention + public static class FactoryIndexerFileExtention { - public static DIHelper AddFilesWrapperService(this DIHelper services) + public static DIHelper AddFactoryIndexerFileService(this DIHelper services) { services.TryAddTransient(); services.TryAddScoped, FactoryIndexerFile>(); diff --git a/products/ASC.Files/Server/Core/Search/FactoryIndexerFolder.cs b/products/ASC.Files/Server/Core/Search/FactoryIndexerFolder.cs index 9977607f30..403dfcf353 100644 --- a/products/ASC.Files/Server/Core/Search/FactoryIndexerFolder.cs +++ b/products/ASC.Files/Server/Core/Search/FactoryIndexerFolder.cs @@ -101,9 +101,9 @@ namespace ASC.Web.Files.Core.Search } } - public static class FoldersWrapperExtention + public static class FactoryIndexerFolderExtention { - public static DIHelper AddFoldersWrapperService(this DIHelper services) + public static DIHelper AddFactoryIndexerFolderService(this DIHelper services) { services.TryAddTransient(); services.TryAddScoped, FactoryIndexerFolder>(); diff --git a/products/ASC.Files/Service/Program.cs b/products/ASC.Files/Service/Program.cs index 57fb290fd2..229a3f3b0a 100644 --- a/products/ASC.Files/Service/Program.cs +++ b/products/ASC.Files/Service/Program.cs @@ -55,8 +55,8 @@ namespace ASC.Files.Service .AddServiceLauncher() .AddFileConverterService() .AddKafkaService() - .AddFilesWrapperService() - .AddFoldersWrapperService(); + .AddFactoryIndexerFileService() + .AddFactoryIndexerFolderService(); var a = typeof(FactoryIndexer).ToString(); services.AddAutofac(hostContext.Configuration, hostContext.HostingEnvironment.ContentRootPath, "search.json"); From 17a88df5ac1323ac0690b50858433de018560c76 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Thu, 14 May 2020 13:47:53 +0300 Subject: [PATCH 18/20] Files: fix download --- products/ASC.Files/Server/Model/BatchModel.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/products/ASC.Files/Server/Model/BatchModel.cs b/products/ASC.Files/Server/Model/BatchModel.cs index c4b93e2493..90918cb295 100644 --- a/products/ASC.Files/Server/Model/BatchModel.cs +++ b/products/ASC.Files/Server/Model/BatchModel.cs @@ -18,10 +18,10 @@ namespace ASC.Files.Model public class DownloadModel : BaseBatchModel { - public IEnumerable> FileConvertIds { get; set; } + public IEnumerable> FileConvertIds { get; set; } public DownloadModel() : base() { - FileConvertIds = new List>(); + FileConvertIds = new List>(); } } From 28c09d37f2d6dcb7a3fcd6882e674e5c037e2932 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Thu, 14 May 2020 15:43:56 +0300 Subject: [PATCH 19/20] Files: fixed DbFilesSecurity keys --- products/ASC.Files/Server/Core/EF/DbFilesSecurity.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/products/ASC.Files/Server/Core/EF/DbFilesSecurity.cs b/products/ASC.Files/Server/Core/EF/DbFilesSecurity.cs index 9caf5ed386..e3bfd7e401 100644 --- a/products/ASC.Files/Server/Core/EF/DbFilesSecurity.cs +++ b/products/ASC.Files/Server/Core/EF/DbFilesSecurity.cs @@ -27,7 +27,7 @@ namespace ASC.Files.Core.EF public override object[] GetKeys() { - return new object[] { TenantId, EntryType, EntryId, Owner }; + return new object[] { TenantId, EntryId, EntryType, Subject }; } } @@ -36,7 +36,7 @@ namespace ASC.Files.Core.EF public static ModelBuilder AddDbFilesSecurity(this ModelBuilder modelBuilder) { modelBuilder.Entity() - .HasKey(c => new { c.TenantId, c.EntryType, c.EntryId, c.Owner }); + .HasKey(c => new { c.TenantId, c.EntryId, c.EntryType, c.Subject }); return modelBuilder; } From 8aaf528f2ac2a67e8710782982ffaf88b4a728ce Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Thu, 14 May 2020 17:03:24 +0300 Subject: [PATCH 20/20] Elastic: CreateIfNotExist --- .../ASC.ElasticSearch/Engine/BaseIndexer.cs | 21 ++++++++++++------- .../Engine/FactoryIndexer.cs | 2 -- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs index e25df16b03..1d499974a1 100644 --- a/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/BaseIndexer.cs @@ -114,15 +114,16 @@ namespace ASC.ElasticSearch } internal void Index(T data, bool immediately = true) - { + { + CreateIfNotExist(data); Client.Instance.Index(data, idx => GetMeta(idx, data, immediately)); } internal void Index(List data, bool immediately = true) { - //CreateIfNotExist(data[0]); if (!data.Any()) return; - + + CreateIfNotExist(data[0]); if (data[0] is ISearchItemDocument) { var currentLength = 0L; @@ -197,23 +198,26 @@ namespace ASC.ElasticSearch } internal void Update(T data, bool immediately = true, params Expression>[] fields) - { + { + CreateIfNotExist(data); Client.Instance.Update(DocumentPath.Id(data), r => GetMetaForUpdate(r, data, immediately, fields)); } internal void Update(T data, UpdateAction action, Expression> fields, bool immediately = true) - { + { + CreateIfNotExist(data); Client.Instance.Update(DocumentPath.Id(data), r => GetMetaForUpdate(r, data, action, fields, immediately)); } internal void Update(T data, Expression, Selector>> expression, int tenantId, bool immediately = true, params Expression>[] fields) { - //CreateIfNotExist(data); + CreateIfNotExist(data); Client.Instance.UpdateByQuery(GetDescriptorForUpdate(data, expression, tenantId, immediately, fields)); } internal void Update(T data, Expression, Selector>> expression, int tenantId, UpdateAction action, Expression> fields, bool immediately = true) - { + { + CreateIfNotExist(data); Client.Instance.UpdateByQuery(GetDescriptorForUpdate(data, expression, tenantId, action, fields, immediately)); } @@ -281,7 +285,8 @@ namespace ASC.ElasticSearch Log.DebugFormat("Delete {0}", Wrapper.IndexName); Client.Instance.Indices.Delete(Wrapper.IndexName); - BaseIndexerHelper.Clear(Wrapper); + BaseIndexerHelper.Clear(Wrapper); + CreateIfNotExist(Wrapper); } internal IReadOnlyCollection Select(Expression, Selector>> expression, bool onlyId = false) diff --git a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs index d116f93e9a..b1f1e3dc9d 100644 --- a/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs +++ b/common/services/ASC.ElasticSearch/Engine/FactoryIndexer.cs @@ -108,8 +108,6 @@ namespace ASC.ElasticSearch FactoryIndexerCommon = factoryIndexer; Indexer = baseIndexer; ServiceProvider = serviceProvider; - - Indexer.CreateIfNotExist(ServiceProvider.GetService()); } public bool TrySelect(Expression, Selector>> expression, out IReadOnlyCollection result)