pavelbannov
6925a83060
# Conflicts: # common/ASC.Common/Caching/ICacheNotify.cs # common/ASC.Common/Caching/KafkaCache.cs # common/ASC.Common/Caching/MemoryCacheNotify.cs # common/ASC.Common/Threading/DistributedTaskProgress.cs # common/ASC.Common/Threading/DistributedTaskQueue.cs # common/ASC.Core.Common/Notify/Signalr/SignalrServiceClient.cs # common/ASC.Data.Backup.Core/Core/FileBackupProvider.cs # common/ASC.Data.Backup.Core/Storage/BackupRepository.cs # common/ASC.Data.Backup.Core/Storage/ConsumerBackupStorage.cs # common/ASC.Data.Backup.Core/Storage/DataStoreBackupStorage.cs # common/ASC.Data.Backup.Core/Storage/DocumentsBackupStorage.cs # common/ASC.Data.Backup.Core/Tasks/BackupPortalTask.cs # common/ASC.Data.Backup.Core/Tasks/DeletePortalTask.cs # common/ASC.Data.Backup.Core/Tasks/PortalTaskBase.cs # common/ASC.Data.Backup.Core/Tasks/RestorePortalTask.cs # common/ASC.Data.Backup.Core/Tasks/TransferPortalTask.cs # common/ASC.Data.Reassigns/RemoveProgressItem.cs # common/ASC.Data.Storage/BaseStorage.cs # common/ASC.Data.Storage/ChunkedUploader/CommonChunkedUploadSessionHolder.cs # common/ASC.Data.Storage/CrossModuleTransferUtility.cs # common/ASC.Data.Storage/DiscStorage/DiscDataStore.cs # common/ASC.Data.Storage/Encryption/EncryptionOperation.cs # common/ASC.Data.Storage/Extensions.cs # common/ASC.Data.Storage/GoogleCloud/GoogleCloudStorage.cs # common/ASC.Data.Storage/IDataStore.cs # common/ASC.Data.Storage/RackspaceCloud/RackspaceCloudStorage.cs # common/ASC.Data.Storage/S3/S3Storage.cs # common/ASC.Data.Storage/S3/S3UploadGuard.cs # common/ASC.Data.Storage/StaticUploader.cs # common/ASC.Data.Storage/StorageHandler.cs # common/ASC.Data.Storage/StorageUploader.cs # common/ASC.Data.Storage/WebPath.cs # common/services/ASC.ApiSystem/Controllers/PortalController.cs # common/services/ASC.AuditTrail/AuditEventsRepository.cs # common/services/ASC.AuditTrail/AuditReportCreator.cs # common/services/ASC.AuditTrail/LoginEventsRepository.cs # products/ASC.Files/Core/Configuration/FilesSpaceUsageStatManager.cs # products/ASC.Files/Core/Core/Dao/Interfaces/IFolderDao.cs # products/ASC.Files/Core/Core/Dao/Interfaces/ILinkDao.cs # products/ASC.Files/Core/Core/Dao/Interfaces/IProviderDao.cs # products/ASC.Files/Core/Core/Dao/Interfaces/IProviderInfo.cs # products/ASC.Files/Core/Core/Dao/Interfaces/ITagDao.cs # products/ASC.Files/Core/Core/Dao/TeamlabDao/AbstractDao.cs # products/ASC.Files/Core/Core/Dao/TeamlabDao/FolderDao.cs # products/ASC.Files/Core/Core/Dao/TeamlabDao/LinkDao.cs # products/ASC.Files/Core/Core/Dao/TeamlabDao/SecurityDao.cs # products/ASC.Files/Core/Core/Dao/TeamlabDao/TagDao.cs # products/ASC.Files/Core/Core/Entries/ChunkedUploadSession.cs # products/ASC.Files/Core/Core/Entries/EncryptionKeyPair.cs # products/ASC.Files/Core/Core/FileStorageService.cs # products/ASC.Files/Core/Core/FilesIntegration.cs # products/ASC.Files/Core/Core/Security/FileSecurity.cs # products/ASC.Files/Core/Core/Security/IFileSecurity.cs # products/ASC.Files/Core/Core/Security/ISecurityDao.cs # products/ASC.Files/Core/Core/Thirdparty/Box/BoxDaoBase.cs # products/ASC.Files/Core/Core/Thirdparty/Box/BoxFileDao.cs # products/ASC.Files/Core/Core/Thirdparty/Box/BoxFolderDao.cs # products/ASC.Files/Core/Core/Thirdparty/Box/BoxProviderInfo.cs # products/ASC.Files/Core/Core/Thirdparty/Box/BoxStorage.cs # products/ASC.Files/Core/Core/Thirdparty/CrossDao.cs # products/ASC.Files/Core/Core/Thirdparty/Dropbox/DropboxDaoBase.cs # products/ASC.Files/Core/Core/Thirdparty/Dropbox/DropboxFileDao.cs # products/ASC.Files/Core/Core/Thirdparty/Dropbox/DropboxFolderDao.cs # products/ASC.Files/Core/Core/Thirdparty/Dropbox/DropboxProviderInfo.cs # products/ASC.Files/Core/Core/Thirdparty/Dropbox/DropboxStorage.cs # products/ASC.Files/Core/Core/Thirdparty/GoogleDrive/GoogleDriveDaoBase.cs # products/ASC.Files/Core/Core/Thirdparty/GoogleDrive/GoogleDriveFileDao.cs # products/ASC.Files/Core/Core/Thirdparty/GoogleDrive/GoogleDriveFolderDao.cs # products/ASC.Files/Core/Core/Thirdparty/GoogleDrive/GoogleDriveProviderInfo.cs # products/ASC.Files/Core/Core/Thirdparty/GoogleDrive/GoogleDriveStorage.cs # products/ASC.Files/Core/Core/Thirdparty/OneDrive/OneDriveDaoBase.cs # products/ASC.Files/Core/Core/Thirdparty/OneDrive/OneDriveFileDao.cs # products/ASC.Files/Core/Core/Thirdparty/OneDrive/OneDriveFolderDao.cs # products/ASC.Files/Core/Core/Thirdparty/OneDrive/OneDriveProviderInfo.cs # products/ASC.Files/Core/Core/Thirdparty/OneDrive/OneDriveStorage.cs # products/ASC.Files/Core/Core/Thirdparty/ProviderAccountDao.cs # products/ASC.Files/Core/Core/Thirdparty/ProviderDao/ProviderDaoBase.cs # products/ASC.Files/Core/Core/Thirdparty/ProviderDao/ProviderFileDao.cs # products/ASC.Files/Core/Core/Thirdparty/ProviderDao/ProviderFolderDao.cs # products/ASC.Files/Core/Core/Thirdparty/ProviderDao/ProviderSecutiryDao.cs # products/ASC.Files/Core/Core/Thirdparty/ProviderDao/ProviderTagDao.cs # products/ASC.Files/Core/Core/Thirdparty/RegexDaoSelectorBase.cs # products/ASC.Files/Core/Core/Thirdparty/SharePoint/SharePointDaoBase.cs # products/ASC.Files/Core/Core/Thirdparty/SharePoint/SharePointFileDao.cs # products/ASC.Files/Core/Core/Thirdparty/SharePoint/SharePointFolderDao.cs # products/ASC.Files/Core/Core/Thirdparty/SharePoint/SharePointProviderInfo.cs # products/ASC.Files/Core/Core/Thirdparty/Sharpbox/SharpBoxDaoBase.cs # products/ASC.Files/Core/Core/Thirdparty/Sharpbox/SharpBoxFileDao.cs # products/ASC.Files/Core/Core/Thirdparty/Sharpbox/SharpBoxFolderDao.cs # products/ASC.Files/Core/Core/Thirdparty/Sharpbox/SharpBoxProviderInfo.cs # products/ASC.Files/Core/Helpers/DocuSignHelper.cs # products/ASC.Files/Core/Helpers/Global.cs # products/ASC.Files/Core/Helpers/PathProvider.cs # products/ASC.Files/Core/HttpHandlers/FileHandler.ashx.cs # products/ASC.Files/Core/HttpHandlers/SearchHandler.cs # products/ASC.Files/Core/Model/FileEntryWrapper.cs # products/ASC.Files/Core/Model/FileOperationWraper.cs # products/ASC.Files/Core/Model/FileWrapper.cs # products/ASC.Files/Core/Model/FolderContentWrapper.cs # products/ASC.Files/Core/Model/FolderWrapper.cs # products/ASC.Files/Core/Services/DocumentService/Configuration.cs # products/ASC.Files/Core/Services/DocumentService/DocumentServiceConnector.cs # products/ASC.Files/Core/Services/DocumentService/DocumentServiceHelper.cs # products/ASC.Files/Core/Services/DocumentService/DocumentServiceTracker.cs # products/ASC.Files/Core/Services/NotifyService/NotifyClient.cs # products/ASC.Files/Core/Services/WCFService/FileOperations/FileDeleteOperation.cs # products/ASC.Files/Core/Services/WCFService/FileOperations/FileDownloadOperation.cs # products/ASC.Files/Core/Services/WCFService/FileOperations/FileMarkAsReadOperation.cs # products/ASC.Files/Core/Services/WCFService/FileOperations/FileMoveCopyOperation.cs # products/ASC.Files/Core/Services/WCFService/FileOperations/FileOperationsManager.cs # products/ASC.Files/Core/ThirdPartyApp/BoxApp.cs # products/ASC.Files/Core/ThirdPartyApp/GoogleDriveApp.cs # products/ASC.Files/Core/ThirdPartyApp/IThirdPartyApp.cs # products/ASC.Files/Core/Utils/ChunkedUploadSessionHolder.cs # products/ASC.Files/Core/Utils/EntryManager.cs # products/ASC.Files/Core/Utils/FileConverter.cs # products/ASC.Files/Core/Utils/FileMarker.cs # products/ASC.Files/Core/Utils/FileShareLink.cs # products/ASC.Files/Core/Utils/FileSharing.cs # products/ASC.Files/Core/Utils/FileUploader.cs # products/ASC.Files/Core/Utils/MailMergeTask.cs # products/ASC.Files/Core/Utils/SocketManager.cs # products/ASC.Files/Server/Controllers/FilesController.cs # products/ASC.Files/Server/Controllers/PrivacyRoomController.cs # products/ASC.Files/Server/Helpers/FilesControllerHelper.cs # products/ASC.People/Server/Controllers/PeopleController.cs # web/ASC.Web.Api/Controllers/AuthenticationController.cs # web/ASC.Web.Api/Controllers/PortalController.cs # web/ASC.Web.Api/Controllers/SettingsController.cs # web/ASC.Web.Api/Models/BuildVersion.cs # web/ASC.Web.Core/Files/DocumentService.cs # web/ASC.Web.Core/Files/DocumentServiceLicense.cs # web/ASC.Web.Core/Helpers/ApiSystemHelper.cs # web/ASC.Web.Core/Notify/StudioNotifyServiceSender.cs # web/ASC.Web.Core/Notify/StudioPeriodicNotify.cs # web/ASC.Web.Core/Recaptcha.cs # web/ASC.Web.Core/Sms/SmsManager.cs # web/ASC.Web.Core/Sms/SmsProvider.cs # web/ASC.Web.Core/Sms/SmsSender.cs # web/ASC.Web.Core/SpaceUsageStatManager.cs # web/ASC.Web.Core/Utility/UrlShortener.cs
237 lines
7.4 KiB
C#
237 lines
7.4 KiB
C#
namespace ASC.Common.Caching;
|
|
|
|
[Singletone]
|
|
public class KafkaCacheNotify<T> : IDisposable, ICacheNotify<T> where T : IMessage<T>, new()
|
|
{
|
|
private IProducer<AscCacheItem, T> _producer;
|
|
|
|
private bool _disposedValue = false; // To detect redundant calls
|
|
private readonly ClientConfig _clientConfig;
|
|
private readonly AdminClientConfig _adminClientConfig;
|
|
private readonly ILog _logger;
|
|
private readonly ConcurrentDictionary<string, CancellationTokenSource> _cancelationToken;
|
|
private readonly ConcurrentDictionary<string, Action<T>> _actions;
|
|
private readonly ProtobufSerializer<T> _valueSerializer = new ProtobufSerializer<T>();
|
|
private readonly ProtobufDeserializer<T> _valueDeserializer = new ProtobufDeserializer<T>();
|
|
private readonly ProtobufSerializer<AscCacheItem> _keySerializer = new ProtobufSerializer<AscCacheItem>();
|
|
private readonly ProtobufDeserializer<AscCacheItem> _keyDeserializer = new ProtobufDeserializer<AscCacheItem>();
|
|
private readonly Guid _key;
|
|
|
|
public KafkaCacheNotify(ConfigurationExtension configuration, IOptionsMonitor<ILog> options)
|
|
{
|
|
_logger = options.CurrentValue;
|
|
_cancelationToken = new ConcurrentDictionary<string, CancellationTokenSource>();
|
|
_actions = new ConcurrentDictionary<string, Action<T>>();
|
|
_key = Guid.NewGuid();
|
|
|
|
var settings = configuration.GetSetting<KafkaSettings>("kafka");
|
|
|
|
_clientConfig = new ClientConfig { BootstrapServers = settings.BootstrapServers };
|
|
_adminClientConfig = new AdminClientConfig { BootstrapServers = settings.BootstrapServers };
|
|
}
|
|
|
|
public void Publish(T obj, CacheNotifyAction notifyAction)
|
|
{
|
|
try
|
|
{
|
|
if (_producer == null)
|
|
{
|
|
_producer = new ProducerBuilder<AscCacheItem, T>(new ProducerConfig(_clientConfig))
|
|
.SetErrorHandler((_, e) => _logger.Error(e))
|
|
.SetKeySerializer(_keySerializer)
|
|
.SetValueSerializer(_valueSerializer)
|
|
.Build();
|
|
}
|
|
|
|
var channelName = GetChannelName(notifyAction);
|
|
|
|
if (_actions.TryGetValue(channelName, out var onchange))
|
|
{
|
|
onchange(obj);
|
|
}
|
|
|
|
var message = new Message<AscCacheItem, T>
|
|
{
|
|
Value = obj,
|
|
Key = new AscCacheItem
|
|
{
|
|
Id = _key.ToString()
|
|
}
|
|
};
|
|
|
|
_producer.ProduceAsync(channelName, message);
|
|
}
|
|
catch (ProduceException<Null, string> e)
|
|
{
|
|
_logger.Error(e);
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
_logger.Error(e);
|
|
}
|
|
}
|
|
|
|
public async Task PublishAsync(T obj, CacheNotifyAction cacheNotifyAction)
|
|
{
|
|
try
|
|
{
|
|
if (_producer == null)
|
|
{
|
|
_producer = new ProducerBuilder<AscCacheItem, T>(new ProducerConfig(_clientConfig))
|
|
.SetErrorHandler((_, e) => _logger.Error(e))
|
|
.SetKeySerializer(_keySerializer)
|
|
.SetValueSerializer(_valueSerializer)
|
|
.Build();
|
|
}
|
|
|
|
var channelName = GetChannelName(cacheNotifyAction);
|
|
|
|
if (_actions.TryGetValue(channelName, out var onchange))
|
|
{
|
|
onchange(obj);
|
|
}
|
|
|
|
var message = new Message<AscCacheItem, T>
|
|
{
|
|
Value = obj,
|
|
Key = new AscCacheItem
|
|
{
|
|
Id = _key.ToString()
|
|
}
|
|
};
|
|
|
|
await _producer.ProduceAsync(channelName, message);
|
|
}
|
|
catch (ProduceException<Null, string> e)
|
|
{
|
|
_logger.Error(e);
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
_logger.Error(e);
|
|
}
|
|
}
|
|
|
|
public void Subscribe(Action<T> onchange, CacheNotifyAction notifyAction)
|
|
{
|
|
var channelName = GetChannelName(notifyAction);
|
|
|
|
_cancelationToken[channelName] = new CancellationTokenSource();
|
|
_actions[channelName] = onchange;
|
|
|
|
void action()
|
|
{
|
|
var conf = new ConsumerConfig(_clientConfig)
|
|
{
|
|
GroupId = Guid.NewGuid().ToString()
|
|
};
|
|
|
|
|
|
using (var adminClient = new AdminClientBuilder(_adminClientConfig)
|
|
.SetErrorHandler((_, e) => _logger.Error(e))
|
|
.Build())
|
|
{
|
|
try
|
|
{
|
|
//TODO: must add checking exist
|
|
adminClient.CreateTopicsAsync(
|
|
new TopicSpecification[]
|
|
{
|
|
new TopicSpecification
|
|
{
|
|
Name = channelName,
|
|
NumPartitions = 1,
|
|
ReplicationFactor = 1
|
|
}
|
|
}).Wait();
|
|
}
|
|
catch (AggregateException) { }
|
|
}
|
|
|
|
using var c = new ConsumerBuilder<AscCacheItem, T>(conf)
|
|
.SetErrorHandler((_, e) => _logger.Error(e))
|
|
.SetKeyDeserializer(_keyDeserializer)
|
|
.SetValueDeserializer(_valueDeserializer)
|
|
.Build();
|
|
|
|
c.Assign(new TopicPartition(channelName, new Partition()));
|
|
|
|
try
|
|
{
|
|
while (true)
|
|
{
|
|
try
|
|
{
|
|
var cr = c.Consume(_cancelationToken[channelName].Token);
|
|
if (cr != null && cr.Message != null && cr.Message.Value != null && !(new Guid(cr.Message.Key.Id)).Equals(_key) && _actions.TryGetValue(channelName, out var act))
|
|
{
|
|
try
|
|
{
|
|
act(cr.Message.Value);
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
_logger.Error("Kafka onmessage", e);
|
|
}
|
|
}
|
|
}
|
|
catch (ConsumeException e)
|
|
{
|
|
_logger.Error(e);
|
|
}
|
|
}
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
c.Close();
|
|
}
|
|
}
|
|
|
|
var task = new Task(action, TaskCreationOptions.LongRunning);
|
|
task.Start();
|
|
}
|
|
|
|
public void Unsubscribe(CacheNotifyAction notifyAction)
|
|
{
|
|
_cancelationToken.TryGetValue(GetChannelName(notifyAction), out var source);
|
|
|
|
if (source != null)
|
|
{
|
|
source.Cancel();
|
|
}
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
Dispose(true);
|
|
GC.SuppressFinalize(this);
|
|
}
|
|
|
|
protected virtual void Dispose(bool disposing)
|
|
{
|
|
if (!_disposedValue)
|
|
{
|
|
if (disposing && _producer != null)
|
|
{
|
|
_producer.Dispose();
|
|
}
|
|
|
|
_disposedValue = true;
|
|
}
|
|
}
|
|
|
|
~KafkaCacheNotify()
|
|
{
|
|
Dispose(false);
|
|
}
|
|
|
|
private string GetChannelName(CacheNotifyAction notifyAction)
|
|
{
|
|
return $"ascchannel{notifyAction}{typeof(T).FullName}".ToLower();
|
|
}
|
|
}
|
|
|
|
public class KafkaSettings
|
|
{
|
|
public string BootstrapServers { get; set; }
|
|
} |