/* * * (c) Copyright Ascensio System Limited 2010-2018 * * This program is freeware. You can redistribute it and/or modify it under the terms of the GNU * General Public License (GPL) version 3 as published by the Free Software Foundation (https://www.gnu.org/copyleft/gpl.html). * In accordance with Section 7(a) of the GNU GPL its Section 15 shall be amended to the effect that * Ascensio System SIA expressly excludes the warranty of non-infringement of any third-party rights. * * THIS PROGRAM IS DISTRIBUTED WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR * FITNESS FOR A PARTICULAR PURPOSE. For more details, see GNU GPL at https://www.gnu.org/copyleft/gpl.html * * You can contact Ascensio System SIA by email at sales@onlyoffice.com * * The interactive user interfaces in modified source and object code versions of ONLYOFFICE must display * Appropriate Legal Notices, as required under Section 5 of the GNU GPL version 3. * * Pursuant to Section 7 § 3(b) of the GNU GPL you must retain the original ONLYOFFICE logo which contains * relevant author attributions when distributing the software. If the display of the logo in its graphic * form is not reasonably feasible for technical reasons, you must include the words "Powered by ONLYOFFICE" * in every copy of the program you distribute. * Pursuant to Section 7 § 3(e) we decline to grant you any rights under trademark law for use of our trademarks. * */ using System; using System.Collections; using System.Collections.Generic; using System.Globalization; using System.Linq; using System.Linq.Expressions; using System.Threading; using System.Threading.Tasks; using ASC.Common; using ASC.Common.Caching; using ASC.Common.Logging; using ASC.Core; using ASC.Core.Tenants; using ASC.ElasticSearch.Core; using Autofac; using Elasticsearch.Net; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; using Nest; namespace ASC.ElasticSearch { [Singletone] public class FactoryIndexerHelper { public DateTime LastIndexed { get; set; } public string Indexing { get; set; } public FactoryIndexerHelper(ICacheNotify cacheNotify) { cacheNotify.Subscribe((a) => { if (a.LastIndexed != 0) { LastIndexed = new DateTime(a.LastIndexed); } Indexing = a.Indexing; }, CacheNotifyAction.Any); } } public interface IFactoryIndexer { void IndexAll(); string IndexName { get; } void ReIndex(); string SettingsTitle { get; } } [Scope] public class FactoryIndexer : IFactoryIndexer where T : class, ISearchItem { private static readonly TaskScheduler Scheduler = new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, 10).ConcurrentScheduler; public ILog Logger { get; } protected TenantManager TenantManager { get; } private SearchSettingsHelper SearchSettingsHelper { get; } private FactoryIndexer FactoryIndexerCommon { get; } protected BaseIndexer Indexer { get; } private IServiceProvider ServiceProvider { get; } public string IndexName { get => Indexer.IndexName; } private ICache Cache { get; } public virtual string SettingsTitle { get => ""; } public FactoryIndexer( IOptionsMonitor options, TenantManager tenantManager, SearchSettingsHelper searchSettingsHelper, FactoryIndexer factoryIndexer, BaseIndexer baseIndexer, IServiceProvider serviceProvider, ICache cache) { Cache = cache; Logger = options.Get("ASC.Indexer"); TenantManager = tenantManager; SearchSettingsHelper = searchSettingsHelper; FactoryIndexerCommon = factoryIndexer; Indexer = baseIndexer; ServiceProvider = serviceProvider; } public bool TrySelect(Expression, Selector>> expression, out IReadOnlyCollection result) { var t = ServiceProvider.GetService(); if (!Support(t) || !Indexer.CheckExist(t)) { result = new List(); return false; } try { result = Indexer.Select(expression); } catch (Exception e) { Logger.Error("Select", e); result = new List(); return false; } return true; } public bool TrySelectIds(Expression, Selector>> expression, out List result) { var t = ServiceProvider.GetService(); if (!Support(t) || !Indexer.CheckExist(t)) { result = new List(); return false; } try { result = Indexer.Select(expression, true).Select(r => r.Id).ToList(); } catch (Exception e) { Logger.Error("Select", e); result = new List(); return false; } return true; } public bool TrySelectIds(Expression, Selector>> expression, out List result, out long total) { var t = ServiceProvider.GetService(); if (!Support(t) || !Indexer.CheckExist(t)) { result = new List(); total = 0; return false; } try { result = Indexer.Select(expression, true, out total).Select(r => r.Id).ToList(); } catch (Exception e) { Logger.Error("Select", e); total = 0; result = new List(); return false; } return true; } public bool CanIndexByContent(T t) { return Support(t) && SearchSettingsHelper.CanIndexByContent(TenantManager.GetCurrentTenant().TenantId); } public bool Index(T data, bool immediately = true) { var t = ServiceProvider.GetService(); if (!Support(t)) return false; try { Indexer.Index(data, immediately); return true; } catch (Exception e) { Logger.Error("Index", e); } return false; } public void Index(List data, bool immediately = true, int retry = 0) { var t = ServiceProvider.GetService(); if (!Support(t) || data.Count == 0) return; try { Indexer.Index(data, immediately); } catch (ElasticsearchClientException e) { Logger.Error(e); if (e.Response != null) { Logger.Error(e.Response.HttpStatusCode); if (e.Response.HttpStatusCode == 413 || e.Response.HttpStatusCode == 403 || e.Response.HttpStatusCode == 408) { data.ForEach(r => Index(r, immediately)); } else if (e.Response.HttpStatusCode == 429) { Thread.Sleep(60000); if (retry < 5) { Index(data, immediately, retry + 1); return; } throw; } } } catch (AggregateException e) //ElasticsearchClientException { if (e.InnerExceptions.Count == 0) throw; var inner = e.InnerExceptions.OfType().FirstOrDefault(); if (inner != null) { Logger.Error(inner); if (inner.Response.HttpStatusCode == 413 || inner.Response.HttpStatusCode == 403) { Logger.Error(inner.Response.HttpStatusCode); data.ForEach(r => Index(r, immediately)); } else if (inner.Response.HttpStatusCode == 429) { Thread.Sleep(60000); if (retry < 5) { Index(data, immediately, retry + 1); return; } throw; } } else { throw; } } } public Task IndexAsync(List data, bool immediately = true, int retry = 0) { var t = ServiceProvider.GetService(); if (!Support(t) || data.Count == 0) return Task.CompletedTask; return InternalIndexAsync(data, immediately, retry); } private async Task InternalIndexAsync(List data, bool immediately, int retry) { try { await Indexer.IndexAsync(data, immediately).ConfigureAwait(false); } catch (ElasticsearchClientException e) { Logger.Error(e); if (e.Response != null) { Logger.Error(e.Response.HttpStatusCode); if (e.Response.HttpStatusCode == 413 || e.Response.HttpStatusCode == 403 || e.Response.HttpStatusCode == 408) { data.ForEach(r => Index(r, immediately)); } else if (e.Response.HttpStatusCode == 429) { await Task.Delay(60000); if (retry < 5) { await IndexAsync(data, immediately, retry + 1); return; } throw; } } } catch (AggregateException e) //ElasticsearchClientException { if (e.InnerExceptions.Count == 0) throw; var inner = e.InnerExceptions.OfType().FirstOrDefault(); if (inner != null) { Logger.Error(inner); if (inner.Response.HttpStatusCode == 413 || inner.Response.HttpStatusCode == 403) { Logger.Error(inner.Response.HttpStatusCode); data.ForEach(r => Index(r, immediately)); } else if (inner.Response.HttpStatusCode == 429) { await Task.Delay(60000); if (retry < 5) { await IndexAsync(data, immediately, retry + 1); return; } throw; } } else { throw; } } } public void Update(T data, bool immediately = true, params Expression>[] fields) { var t = ServiceProvider.GetService(); if (!Support(t)) return; try { Indexer.Update(data, immediately, fields); } catch (Exception e) { Logger.Error("Update", e); } } public void Update(T data, UpdateAction action, Expression> field, bool immediately = true) { var t = ServiceProvider.GetService(); if (!Support(t)) return; try { Indexer.Update(data, action, field, immediately); } catch (Exception e) { Logger.Error("Update", e); } } public void Update(T data, Expression, Selector>> expression, bool immediately = true, params Expression>[] fields) { var t = ServiceProvider.GetService(); if (!Support(t)) return; try { var tenant = TenantManager.GetCurrentTenant().TenantId; Indexer.Update(data, expression, tenant, immediately, fields); } catch (Exception e) { Logger.Error("Update", e); } } public void Update(T data, Expression, Selector>> expression, UpdateAction action, Expression> fields, bool immediately = true) { var t = ServiceProvider.GetService(); if (!Support(t)) return; try { var tenant = TenantManager.GetCurrentTenant().TenantId; Indexer.Update(data, expression, tenant, action, fields, immediately); } catch (Exception e) { Logger.Error("Update", e); } } public void Delete(T data, bool immediately = true) { var t = ServiceProvider.GetService(); if (!Support(t)) return; try { Indexer.Delete(data, immediately); } catch (Exception e) { Logger.Error("Delete", e); } } public void Delete(Expression, Selector>> expression, bool immediately = true) { var t = ServiceProvider.GetService(); if (!Support(t)) return; var tenant = TenantManager.GetCurrentTenant().TenantId; try { Indexer.Delete(expression, tenant, immediately); } catch (Exception e) { Logger.Error("Index", e); } } public Task IndexAsync(T data, bool immediately = true) { var t = ServiceProvider.GetService(); if (!Support(t)) return Task.FromResult(false); return Queue(() => Indexer.Index(data, immediately)); } public Task UpdateAsync(T data, bool immediately = true, params Expression>[] fields) { var t = ServiceProvider.GetService(); if (!Support(t)) return Task.FromResult(false); return Queue(() => Indexer.Update(data, immediately, fields)); } public Task DeleteAsync(T data, bool immediately = true) { var t = ServiceProvider.GetService(); if (!Support(t)) return Task.FromResult(false); return Queue(() => Indexer.Delete(data, immediately)); } public async Task DeleteAsync(Expression, Selector>> expression, bool immediately = true) { var t = ServiceProvider.GetService(); if (!await SupportAsync(t)) return false; var tenant = TenantManager.GetCurrentTenant().TenantId; return await Queue(() => Indexer.Delete(expression, tenant, immediately)); } public void Flush() { var t = ServiceProvider.GetService(); if (!Support(t)) return; Indexer.Flush(); } public void Refresh() { var t = ServiceProvider.GetService(); if (!Support(t)) return; Indexer.Refresh(); } private Task Queue(Action actionData) { var task = new Task(() => { try { actionData(); return true; } catch (AggregateException agg) { foreach (var e in agg.InnerExceptions) { Logger.Error(e); } throw; } }, TaskCreationOptions.LongRunning); task.ConfigureAwait(false); task.Start(Scheduler); return task; } public virtual void IndexAll() { return; } public void ReIndex() { Indexer.ReIndex(); } public bool Support(T t) { if (!FactoryIndexerCommon.CheckState()) return false; var cacheTime = DateTime.UtcNow.AddMinutes(15); var key = "elasticsearch " + t.IndexName; try { var cacheValue = Cache.Get(key); if (!string.IsNullOrEmpty(cacheValue)) { return Convert.ToBoolean(cacheValue); } //TODO: //var service = new Service.Service(); //var result = service.Support(t.IndexName); //Cache.Insert(key, result.ToString(CultureInfo.InvariantCulture).ToLower(), cacheTime); return true; } catch (Exception e) { Cache.Insert(key, "false", cacheTime); Logger.Error("FactoryIndexer CheckState", e); return false; } } public Task SupportAsync(T t) { return FactoryIndexerCommon.CheckStateAsync(); } } [Scope] public class FactoryIndexer { private readonly ICache cache; private IServiceProvider ServiceProvider { get; } private FactoryIndexerHelper FactoryIndexerHelper { get; } public ILog Log { get; } private Client Client { get; } private CoreBaseSettings CoreBaseSettings { get; } public FactoryIndexer( IServiceProvider serviceProvider, FactoryIndexerHelper factoryIndexerHelper, Client client, IOptionsMonitor options, CoreBaseSettings coreBaseSettings, ICache cache) { this.cache = cache; ServiceProvider = serviceProvider; FactoryIndexerHelper = factoryIndexerHelper; Client = client; CoreBaseSettings = coreBaseSettings; try { Log = options.Get("ASC.Indexer"); } catch (Exception e) { Log.Fatal("FactoryIndexer", e); } } public bool CheckState(bool cacheState = true) { const string key = "elasticsearch"; if (cacheState) { var cacheValue = cache.Get(key); if (!string.IsNullOrEmpty(cacheValue)) { return Convert.ToBoolean(cacheValue); } } var cacheTime = DateTime.UtcNow.AddMinutes(15); try { var isValid = Client.Ping(); if (cacheState) { cache.Insert(key, isValid.ToString(CultureInfo.InvariantCulture).ToLower(), cacheTime); } return isValid; } catch (Exception e) { if (cacheState) { cache.Insert(key, "false", cacheTime); } Log.Error("Ping false", e); return false; } } public Task CheckStateAsync(bool cacheState = true) { const string key = "elasticsearch"; if (cacheState) { var cacheValue = cache.Get(key); if (!string.IsNullOrEmpty(cacheValue)) { return Task.FromResult(Convert.ToBoolean(cacheValue)); } } return InternalCheckStateAsync(cacheState, key); } private async Task InternalCheckStateAsync(bool cacheState, string key) { var cacheTime = DateTime.UtcNow.AddMinutes(15); try { var result = await Client.Instance.PingAsync(new PingRequest()); var isValid = result.IsValid; Log.DebugFormat("CheckState ping {0}", result.DebugInformation); if (cacheState) { cache.Insert(key, isValid.ToString(CultureInfo.InvariantCulture).ToLower(), cacheTime); } return isValid; } catch (Exception e) { if (cacheState) { cache.Insert(key, "false", cacheTime); } Log.Error("Ping false", e); return false; } } public object GetState(TenantUtil tenantUtil) { State state = null; IEnumerable indices = null; Dictionary count = null; if (!CoreBaseSettings.Standalone) { return new { state, indices, status = CheckState() }; } state = new State { Indexing = FactoryIndexerHelper.Indexing, LastIndexed = FactoryIndexerHelper.LastIndexed != DateTime.MinValue ? FactoryIndexerHelper.LastIndexed : default(DateTime?) }; if (state.LastIndexed.HasValue) { state.LastIndexed = tenantUtil.DateTimeFromUtc(state.LastIndexed.Value); } indices = Client.Instance.Cat.Indices(new CatIndicesRequest { SortByColumns = new[] { "index" } }).Records .Select(r => new { r.Index, Count = count.ContainsKey(r.Index) ? count[r.Index] : 0, DocsCount = Client.Instance.Count(new CountRequest(r.Index)).Count, r.StoreSize }) .Where(r => { return r.Count > 0; }); return new { state, indices, status = CheckState() }; } public void Reindex(string name) { if (!CoreBaseSettings.Standalone) return; var generic = typeof(BaseIndexer<>); var indexers = ServiceProvider.GetService>() .Where(r => string.IsNullOrEmpty(name) || r.IndexName == name) .Select(r => (IFactoryIndexer)Activator.CreateInstance(generic.MakeGenericType(r.GetType()), r)); foreach (var indexer in indexers) { indexer.ReIndex(); } } } }