PersonalToDocspace: fix
This commit is contained in:
parent
bbb30f0554
commit
119722f308
@ -1,11 +1,38 @@
|
||||
global using System.Data;
|
||||
// (c) Copyright Ascensio System SIA 2010-2022
|
||||
//
|
||||
// This program is a free software product.
|
||||
// You can redistribute it and/or modify it under the terms
|
||||
// of the GNU Affero General Public License (AGPL) version 3 as published by the Free Software
|
||||
// Foundation. In accordance with Section 7(a) of the GNU AGPL its Section 15 shall be amended
|
||||
// to the effect that Ascensio System SIA expressly excludes the warranty of non-infringement of
|
||||
// any third-party rights.
|
||||
//
|
||||
// This program is distributed WITHOUT ANY WARRANTY, without even the implied warranty
|
||||
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For details, see
|
||||
// the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
|
||||
//
|
||||
// You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia, EU, LV-1021.
|
||||
//
|
||||
// The interactive user interfaces in modified source and object code versions of the Program must
|
||||
// display Appropriate Legal Notices, as required under Section 5 of the GNU AGPL version 3.
|
||||
//
|
||||
// Pursuant to Section 7(b) of the License you must retain the original Product logo when
|
||||
// distributing the program. Pursuant to Section 7(e) we decline to grant you any rights under
|
||||
// trademark law for use of our trademarks.
|
||||
//
|
||||
// All the Product's GUI elements, including illustrations and icon sets, as well as technical writing
|
||||
// content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0
|
||||
// International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
|
||||
|
||||
global using System.Data;
|
||||
global using System.Data.Common;
|
||||
global using System.Xml.Linq;
|
||||
|
||||
|
||||
global using ASC.Api.Core;
|
||||
global using ASC.Api.Core.Extensions;
|
||||
global using ASC.Common;
|
||||
global using ASC.Common.Logging;
|
||||
global using ASC.Common.Utils;
|
||||
global using ASC.Core.Common.EF;
|
||||
global using ASC.Core.Common.EF.Context;
|
||||
global using ASC.Core.Common.Hosting;
|
||||
@ -32,6 +59,8 @@ global using ASC.Migration.PersonalToDocspace;
|
||||
global using ASC.Migration.PersonalToDocspace.Creator;
|
||||
global using ASC.Migration.PersonalToDocspace.Runner;
|
||||
global using ASC.Webhooks.Core.EF.Context;
|
||||
|
||||
|
||||
global using Autofac.Extensions.DependencyInjection;
|
||||
|
||||
global using Microsoft.EntityFrameworkCore;
|
||||
global using Microsoft.Extensions.Hosting.WindowsServices;
|
||||
global using Microsoft.Extensions.Hosting.WindowsServices;
|
||||
|
||||
@ -28,7 +28,7 @@ namespace ASC.Migration.PersonalToDocspace.Creator;
|
||||
|
||||
public class MigrationCreator
|
||||
{
|
||||
private readonly IDbContextFactory<UserDbContext> _userDbContext;
|
||||
private readonly IDbContextFactory<UserDbContext> _userDbContext;
|
||||
private readonly IDbContextFactory<BackupsContext> _backupsContext;
|
||||
private readonly IDbContextFactory<FilesDbContext> _filesDbContext;
|
||||
private readonly IHostEnvironment _hostEnvironment;
|
||||
@ -37,12 +37,14 @@ public class MigrationCreator
|
||||
private readonly TempStream _tempStream;
|
||||
private readonly DbFactory _dbFactory;
|
||||
private readonly StorageFactory _storageFactory;
|
||||
private readonly StorageFactoryConfig _storageFactoryConfig;
|
||||
private readonly List<IModuleSpecifics> _modules;
|
||||
private readonly string _pathToSave;
|
||||
private readonly string _userName;
|
||||
private readonly string _toRegion;
|
||||
private readonly int _tenant;
|
||||
private readonly StorageFactoryConfig _storageFactoryConfig;
|
||||
private readonly ModuleProvider _moduleProvider;
|
||||
|
||||
private List<IModuleSpecifics> _modules;
|
||||
private string _pathToSave;
|
||||
private string _userName;
|
||||
private string _toRegion;
|
||||
private int _tenant;
|
||||
private readonly int _limit = 1000;
|
||||
private readonly List<ModuleName> _namesModules = new List<ModuleName>()
|
||||
{
|
||||
@ -51,24 +53,53 @@ public class MigrationCreator
|
||||
ModuleName.Files2,
|
||||
ModuleName.Tenants,
|
||||
ModuleName.WebStudio
|
||||
};
|
||||
};
|
||||
|
||||
public MigrationCreator(
|
||||
IDbContextFactory<UserDbContext> userDbContext,
|
||||
IDbContextFactory<BackupsContext> backupsContext,
|
||||
IDbContextFactory<FilesDbContext> filesDbContext,
|
||||
IHostEnvironment hostEnvironment,
|
||||
IConfiguration configuration,
|
||||
TenantDomainValidator tenantDomainValidator,
|
||||
TempStream tempStream,
|
||||
DbFactory dbFactory,
|
||||
StorageFactory storageFactory,
|
||||
StorageFactoryConfig storageFactoryConfig,
|
||||
ModuleProvider moduleProvider)
|
||||
{
|
||||
_userDbContext = userDbContext;
|
||||
_backupsContext = backupsContext;
|
||||
_filesDbContext = filesDbContext;
|
||||
_hostEnvironment = hostEnvironment;
|
||||
_configuration = configuration;
|
||||
_tenantDomainValidator = tenantDomainValidator;
|
||||
_tempStream = tempStream;
|
||||
_dbFactory = dbFactory;
|
||||
_storageFactory = storageFactory;
|
||||
_storageFactoryConfig = storageFactoryConfig;
|
||||
_moduleProvider = moduleProvider;
|
||||
}
|
||||
|
||||
public MigrationCreator(IServiceProvider serviceProvider, int tenant, string userName, string toRegion)
|
||||
|
||||
|
||||
public async Task Create(int tenant, string userName, string toRegion)
|
||||
{
|
||||
Init(tenant, userName, toRegion);
|
||||
|
||||
var id = GetId();
|
||||
|
||||
var path = Path.Combine(_pathToSave, _userName + ".tar.gz");
|
||||
using (var writer = new ZipWriteOperator(_tempStream, path))
|
||||
{
|
||||
DoMigrationDb(id, writer);
|
||||
await DoMigrationStorage(id, writer);
|
||||
}
|
||||
}
|
||||
|
||||
private void Init(int tenant, string userName, string toRegion)
|
||||
{
|
||||
_userDbContext = serviceProvider.GetService<IDbContextFactory<UserDbContext>>();
|
||||
_backupsContext = serviceProvider.GetService<IDbContextFactory<BackupsContext>>();
|
||||
_filesDbContext = serviceProvider.GetService<IDbContextFactory<FilesDbContext>>();
|
||||
_tempStream = serviceProvider.GetService<TempStream>();
|
||||
_dbFactory = serviceProvider.GetService<DbFactory>();
|
||||
_storageFactory = serviceProvider.GetService<StorageFactory>();
|
||||
_storageFactoryConfig = serviceProvider.GetService<StorageFactoryConfig>();
|
||||
_hostEnvironment = serviceProvider.GetService<IHostEnvironment>();
|
||||
_configuration = serviceProvider.GetService<IConfiguration>();
|
||||
_tenantDomainValidator = serviceProvider.GetService<TenantDomainValidator>();
|
||||
|
||||
|
||||
var moduleProvider = serviceProvider.GetService<ModuleProvider>();
|
||||
_modules = moduleProvider.AllModules.Where(m => _namesModules.Contains(m.ModuleName)).ToList();
|
||||
_modules = _moduleProvider.AllModules.Where(m => _namesModules.Contains(m.ModuleName)).ToList();
|
||||
|
||||
_pathToSave = "";
|
||||
_toRegion = toRegion;
|
||||
@ -80,7 +111,7 @@ public class MigrationCreator
|
||||
|
||||
private void CheckExistDataStorage()
|
||||
{
|
||||
var store = _storageFactory.GetStorage(_tenant.ToString(), "files");
|
||||
var store = _storageFactory.GetStorage(_tenant, "files");
|
||||
if (store is DiscDataStore)
|
||||
{
|
||||
var path = Path.Combine(_hostEnvironment.ContentRootPath, _configuration[Data.Storage.Constants.StorageRootParam]);
|
||||
@ -89,27 +120,16 @@ public class MigrationCreator
|
||||
throw new Exception("Wrong $STORAGE_ROOT, change $STORAGE_ROOT in appsettings.json");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Create()
|
||||
{
|
||||
var id = GetId();
|
||||
|
||||
var path = Path.Combine(_pathToSave, _userName + ".tar.gz");
|
||||
using (var writer = new ZipWriteOperator(_tempStream, path))
|
||||
{
|
||||
DoMigrationDb(id, writer);
|
||||
DoMigrationStorage(id, writer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Guid GetId()
|
||||
{
|
||||
try
|
||||
{
|
||||
return _userDbContext.CreateDbContext().Users.FirstOrDefault(q => q.Tenant == _tenant && q.Status == EmployeeStatus.Active && q.UserName == _userName).Id;
|
||||
{
|
||||
using var dbContext = _userDbContext.CreateDbContext();
|
||||
return dbContext.Users.FirstOrDefault(q => q.Tenant == _tenant && q.Status == EmployeeStatus.Active && q.UserName == _userName).Id;
|
||||
}
|
||||
catch (Exception ex)
|
||||
catch (Exception)
|
||||
{
|
||||
throw new Exception("username was not found");
|
||||
}
|
||||
@ -154,7 +174,7 @@ public class MigrationCreator
|
||||
|
||||
module.PrepareData(data);
|
||||
|
||||
if(data.TableName == "tenants_tenants")
|
||||
if (data.TableName == "tenants_tenants")
|
||||
{
|
||||
ChangeAlias(data);
|
||||
}
|
||||
@ -169,8 +189,8 @@ public class MigrationCreator
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private void ChangeAlias(DataTable data)
|
||||
@ -221,14 +241,14 @@ public class MigrationCreator
|
||||
}
|
||||
}
|
||||
|
||||
private void DoMigrationStorage(Guid id, IDataWriteOperator writer)
|
||||
private async Task DoMigrationStorage(Guid id, IDataWriteOperator writer)
|
||||
{
|
||||
var fileGroups = GetFilesGroup(id);
|
||||
var fileGroups = await GetFilesGroup(id);
|
||||
foreach (var group in fileGroups)
|
||||
{
|
||||
foreach (var file in group)
|
||||
{
|
||||
var storage = _storageFactory.GetStorage(_tenant.ToString(), group.Key);
|
||||
var storage = _storageFactory.GetStorage(_tenant, group.Key);
|
||||
var file1 = file;
|
||||
ActionInvoker.Try(state =>
|
||||
{
|
||||
@ -253,36 +273,38 @@ public class MigrationCreator
|
||||
}
|
||||
}
|
||||
|
||||
private List<IGrouping<string, BackupFileInfo>> GetFilesGroup(Guid id)
|
||||
private async Task<List<IGrouping<string, BackupFileInfo>>> GetFilesGroup(Guid id)
|
||||
{
|
||||
var files = GetFilesToProcess(id).ToList();
|
||||
|
||||
var exclude = _backupsContext.CreateDbContext().Backups.AsQueryable().Where(b => b.TenantId == _tenant && b.StorageType == 0 && b.StoragePath != null).ToList();
|
||||
var files = (await GetFilesToProcess(id)).ToList();
|
||||
|
||||
using var dbManager = _backupsContext.CreateDbContext();
|
||||
var exclude = dbManager.Backups.AsQueryable().Where(b => b.TenantId == _tenant && b.StorageType == 0 && b.StoragePath != null).ToList();
|
||||
files = files.Where(f => !exclude.Any(e => f.Path.Replace('\\', '/').Contains($"/file_{e.StoragePath}/"))).ToList();
|
||||
|
||||
return files.GroupBy(file => file.Module).ToList();
|
||||
}
|
||||
|
||||
protected IEnumerable<BackupFileInfo> GetFilesToProcess(Guid id)
|
||||
protected async Task<IEnumerable<BackupFileInfo>> GetFilesToProcess(Guid id)
|
||||
{
|
||||
var files = new List<BackupFileInfo>();
|
||||
var files = new List<BackupFileInfo>();
|
||||
|
||||
foreach (var module in _storageFactoryConfig.GetModuleList().Where(m => m == "files"))
|
||||
{
|
||||
var store = _storageFactory.GetStorage(_tenant.ToString(), module);
|
||||
var store = _storageFactory.GetStorage(_tenant, module);
|
||||
var domains = _storageFactoryConfig.GetDomainList(module).ToArray();
|
||||
|
||||
foreach (var domain in domains)
|
||||
{
|
||||
files.AddRange(
|
||||
store.ListFilesRelativeAsync(domain, "\\", "*.*", true).ToArrayAsync().Result
|
||||
.Select(path => new BackupFileInfo(domain, module, path, _tenant)));
|
||||
files.AddRange(await store.ListFilesRelativeAsync(domain, "\\", "*.*", true).Select(path => new BackupFileInfo(domain, module, path, _tenant)).ToListAsync());
|
||||
}
|
||||
|
||||
files.AddRange(
|
||||
store.ListFilesRelativeAsync(string.Empty, "\\", "*.*", true).ToArrayAsync().Result
|
||||
.Where(path => domains.All(domain => !path.Contains(domain + "/")))
|
||||
.Select(path => new BackupFileInfo(string.Empty, module, path, _tenant)));
|
||||
}
|
||||
files.AddRange(await
|
||||
store.ListFilesRelativeAsync(string.Empty, "\\", "*.*", true)
|
||||
.Where(path => domains.All(domain => !path.Contains(domain + "/")))
|
||||
.Select(path => new BackupFileInfo(string.Empty, module, path, _tenant))
|
||||
.ToListAsync());
|
||||
}
|
||||
|
||||
using var filesRecordContext = _filesDbContext.CreateDbContext();
|
||||
files = files.Where(f => UserIsFileOwner(id, f, filesRecordContext)).ToList();
|
||||
return files.Distinct();
|
||||
|
||||
@ -28,15 +28,15 @@ namespace ASC.Migration.PersonalToDocspace.Runner;
|
||||
|
||||
public class MigrationRunner
|
||||
{
|
||||
private readonly DbFactory _dbFactory;
|
||||
private readonly DbFactory _dbFactory;
|
||||
private readonly StorageFactory _storageFactory;
|
||||
private readonly StorageFactoryConfig _storageFactoryConfig;
|
||||
private readonly ModuleProvider _moduleProvider;
|
||||
private readonly ILogger<RestoreDbModuleTask> _logger;
|
||||
|
||||
private readonly string _backupFile;
|
||||
private readonly string _region;
|
||||
private readonly List<IModuleSpecifics> _modules;
|
||||
private string _backupFile;
|
||||
private string _region;
|
||||
private List<IModuleSpecifics> _modules;
|
||||
private readonly List<ModuleName> _namesModules = new List<ModuleName>()
|
||||
{
|
||||
ModuleName.Core,
|
||||
@ -44,23 +44,27 @@ public class MigrationRunner
|
||||
ModuleName.Files2,
|
||||
ModuleName.Tenants,
|
||||
ModuleName.WebStudio
|
||||
};
|
||||
|
||||
public MigrationRunner(IServiceProvider serviceProvider, string backupFile, string region)
|
||||
{
|
||||
_moduleProvider = serviceProvider.GetService<ModuleProvider>();
|
||||
_dbFactory = serviceProvider.GetService<DbFactory>();
|
||||
_storageFactory = serviceProvider.GetService<StorageFactory>();
|
||||
_storageFactoryConfig = serviceProvider.GetService<StorageFactoryConfig>();
|
||||
_logger = serviceProvider.GetService<ILogger<RestoreDbModuleTask>>();
|
||||
};
|
||||
|
||||
public MigrationRunner(
|
||||
DbFactory dbFactory,
|
||||
StorageFactory storageFactory,
|
||||
StorageFactoryConfig storageFactoryConfig,
|
||||
ModuleProvider moduleProvider,
|
||||
ILogger<RestoreDbModuleTask> logger)
|
||||
{
|
||||
_dbFactory = dbFactory;
|
||||
_storageFactory = storageFactory;
|
||||
_storageFactoryConfig = storageFactoryConfig;
|
||||
_moduleProvider = moduleProvider;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task Run(string backupFile, string region)
|
||||
{
|
||||
_region = region;
|
||||
_modules = _moduleProvider.AllModules.Where(m => _namesModules.Contains(m.ModuleName)).ToList();
|
||||
_backupFile = backupFile;
|
||||
}
|
||||
|
||||
public void Run()
|
||||
{
|
||||
_backupFile = backupFile;
|
||||
var columnMapper = new ColumnMapper();
|
||||
using (var dataReader = new ZipReadOperator(_backupFile))
|
||||
{
|
||||
@ -71,13 +75,13 @@ public class MigrationRunner
|
||||
restoreTask.RunJob();
|
||||
}
|
||||
|
||||
DoRestoreStorage(dataReader, columnMapper);
|
||||
await DoRestoreStorage(dataReader, columnMapper);
|
||||
|
||||
SetTenantActive(columnMapper.GetTenantMapping());
|
||||
}
|
||||
}
|
||||
|
||||
private void DoRestoreStorage(IDataReadOperator dataReader, ColumnMapper columnMapper)
|
||||
private async Task DoRestoreStorage(IDataReadOperator dataReader, ColumnMapper columnMapper)
|
||||
{
|
||||
var fileGroups = GetFilesToProcess(dataReader).GroupBy(file => file.Module).ToList();
|
||||
|
||||
@ -85,7 +89,7 @@ public class MigrationRunner
|
||||
{
|
||||
foreach (var file in group)
|
||||
{
|
||||
var storage = _storageFactory.GetStorage(columnMapper.GetTenantMapping().ToString(), group.Key, _region);
|
||||
var storage = _storageFactory.GetStorage(columnMapper.GetTenantMapping(), group.Key, _region);
|
||||
var quotaController = storage.QuotaController;
|
||||
storage.SetQuotaController(null);
|
||||
|
||||
@ -98,7 +102,7 @@ public class MigrationRunner
|
||||
var key = file.GetZipKey();
|
||||
using var stream = dataReader.GetEntry(key);
|
||||
|
||||
storage.SaveAsync(file.Domain, adjustedPath, module != null ? module.PrepareData(key, stream, columnMapper) : stream).Wait();
|
||||
await storage.SaveAsync(file.Domain, adjustedPath, module != null ? module.PrepareData(key, stream, columnMapper) : stream);
|
||||
}
|
||||
}
|
||||
finally
|
||||
|
||||
@ -24,9 +24,6 @@
|
||||
// content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0
|
||||
// International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
|
||||
|
||||
using ASC.Common.Utils;
|
||||
|
||||
using Autofac.Extensions.DependencyInjection;
|
||||
|
||||
var options = new WebApplicationOptions
|
||||
{
|
||||
@ -107,11 +104,11 @@ var tenant = Int32.Parse(args[0]);
|
||||
var userName = args[1];
|
||||
var region = args[2];
|
||||
|
||||
var migrationCreator = new MigrationCreator(app.Services, tenant, userName, region);
|
||||
migrationCreator.Create();
|
||||
var migrationCreator = app.Services.GetService<MigrationCreator>();
|
||||
await migrationCreator.Create(tenant, userName, region);
|
||||
|
||||
var migrationRunner = new MigrationRunner(app.Services.CreateScope().ServiceProvider, userName + ".tar.gz", region);
|
||||
migrationRunner.Run();
|
||||
var migrationRunner = app.Services.GetService<MigrationRunner>();
|
||||
await migrationRunner.Run(userName + ".tar.gz", region);
|
||||
|
||||
Directory.GetFiles(AppContext.BaseDirectory).Where(f => f.Contains(".tar")).ToList().ForEach(File.Delete);
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user