557 lines
21 KiB
C#
557 lines
21 KiB
C#
// (c) Copyright Ascensio System SIA 2010-2022
|
||
//
|
||
// This program is a free software product.
|
||
// You can redistribute it and/or modify it under the terms
|
||
// of the GNU Affero General Public License (AGPL) version 3 as published by the Free Software
|
||
// Foundation. In accordance with Section 7(a) of the GNU AGPL its Section 15 shall be amended
|
||
// to the effect that Ascensio System SIA expressly excludes the warranty of non-infringement of
|
||
// any third-party rights.
|
||
//
|
||
// This program is distributed WITHOUT ANY WARRANTY, without even the implied warranty
|
||
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For details, see
|
||
// the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
|
||
//
|
||
// You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia, EU, LV-1021.
|
||
//
|
||
// The interactive user interfaces in modified source and object code versions of the Program must
|
||
// display Appropriate Legal Notices, as required under Section 5 of the GNU AGPL version 3.
|
||
//
|
||
// Pursuant to Section 7(b) of the License you must retain the original Product logo when
|
||
// distributing the program. Pursuant to Section 7(e) we decline to grant you any rights under
|
||
// trademark law for use of our trademarks.
|
||
//
|
||
// All the Product's GUI elements, including illustrations and icon sets, as well as technical writing
|
||
// content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0
|
||
// International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
|
||
|
||
using ASC.Migration.OwnCloud.Models;
|
||
|
||
namespace ASC.Migration.OwnCloud;
|
||
|
||
[ApiMigrator("OwncloudMigrate")]
|
||
public class OwnCloudMigration : AbstractMigration<OCMigrationInfo, OCMigratingUser, OCMigratingContacts, OCMigratingCalendar, OCMigratingFiles, OCMigratingMail>
|
||
{
|
||
private string _takeouts;
|
||
public string[] TempParse;
|
||
private string _tmpFolder;
|
||
private readonly GlobalFolderHelper _globalFolderHelper;
|
||
private readonly IDaoFactory _daoFactory;
|
||
private readonly FileSecurity _fileSecurity;
|
||
private readonly FileStorageService<int> _fileStorageService;
|
||
private readonly SecurityContext _securityContext;
|
||
private readonly TenantManager _tenantManager;
|
||
private readonly UserManager _userManager;
|
||
|
||
public OwnCloudMigration(
|
||
GlobalFolderHelper globalFolderHelper,
|
||
IDaoFactory daoFactory,
|
||
FileSecurity fileSecurity,
|
||
FileStorageService<int> fileStorageService,
|
||
SecurityContext securityContext,
|
||
TenantManager tenantManager,
|
||
UserManager userManager,
|
||
MigrationLogger migrationLogger) : base(migrationLogger)
|
||
{
|
||
_globalFolderHelper = globalFolderHelper;
|
||
_daoFactory = daoFactory;
|
||
_fileSecurity = fileSecurity;
|
||
_fileStorageService = fileStorageService;
|
||
_securityContext = securityContext;
|
||
_tenantManager = tenantManager;
|
||
_userManager = userManager;
|
||
}
|
||
|
||
public override void Init(string path, CancellationToken cancellationToken)
|
||
{
|
||
_cancellationToken = cancellationToken;
|
||
var files = Directory.GetFiles(path);
|
||
if (!files.Any() || !files.Any(f => f.EndsWith(".zip")))
|
||
{
|
||
throw new Exception("Folder must not be empty and should contain only .zip files.");
|
||
}
|
||
for (var i = 0; i < files.Length; i++)
|
||
{
|
||
if (files[i].EndsWith(".zip"))
|
||
{
|
||
var creationTime = File.GetCreationTimeUtc(files[i]);
|
||
_takeouts = files[i];
|
||
}
|
||
}
|
||
|
||
_migrationInfo = new OCMigrationInfo();
|
||
_migrationInfo.MigratorName = GetType().CustomAttributes.First().ConstructorArguments.First().Value.ToString();
|
||
_tmpFolder = path;
|
||
}
|
||
public override Task<MigrationApiInfo> Parse()
|
||
{
|
||
ReportProgress(0, MigrationResource.Unzipping);
|
||
try
|
||
{
|
||
try
|
||
{
|
||
ZipFile.ExtractToDirectory(_takeouts, _tmpFolder);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log($"Couldn't to unzip {_takeouts}", ex);
|
||
}
|
||
if (_cancellationToken.IsCancellationRequested) { ReportProgress(100, MigrationResource.MigrationCanceled); return null; }
|
||
ReportProgress(30, MigrationResource.UnzippingFinished);
|
||
var bdFile = "";
|
||
try
|
||
{
|
||
bdFile = Directory.GetFiles(Directory.GetDirectories(_tmpFolder)[0], "*.bak")[0];
|
||
if (bdFile == null)
|
||
{
|
||
throw new Exception();
|
||
}
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
_migrationInfo.failedArchives.Add(Path.GetFileName(_takeouts));
|
||
Log("Archive must not be empty and should contain .bak files.", ex);
|
||
}
|
||
|
||
ReportProgress(40, MigrationResource.DumpParse);
|
||
var users = DBExtractUser(bdFile);
|
||
var progress = 40;
|
||
foreach (var item in users)
|
||
{
|
||
if (_cancellationToken.IsCancellationRequested) { ReportProgress(100, MigrationResource.MigrationCanceled); return null; }
|
||
ReportProgress(progress, MigrationResource.DataProcessing);
|
||
progress += 30 / users.Count;
|
||
if (item.Data.DisplayName != null)
|
||
{
|
||
try
|
||
{
|
||
var userName = item.Data.DisplayName.Split(' ');
|
||
item.Data.DisplayName = userName.Length > 1 ? string.Format("{0} {1}", userName[0], userName[1]).Trim() : userName[0].Trim();
|
||
var user = new OCMigratingUser(_globalFolderHelper, _daoFactory, _fileSecurity, _fileStorageService, _tenantManager, _userManager, item.Uid, item, Directory.GetDirectories(_tmpFolder)[0], Log);
|
||
user.Parse();
|
||
foreach (var element in user.ModulesList)
|
||
{
|
||
if (!_migrationInfo.Modules.Exists(x => x.MigrationModule == element.MigrationModule))
|
||
{
|
||
_migrationInfo.Modules.Add(new MigrationModules(element.MigrationModule, element.Module));
|
||
}
|
||
}
|
||
_migrationInfo.Users.Add(item.Uid, user);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log($"Couldn't parse user {item.Data.DisplayName}", ex);
|
||
}
|
||
}
|
||
}
|
||
|
||
var groups = DBExtractGroup(bdFile);
|
||
progress = 80;
|
||
foreach (var item in groups)
|
||
{
|
||
ReportProgress(progress, MigrationResource.DataProcessing);
|
||
progress += 10 / groups.Count;
|
||
var group = new OCMigratingGroups(_userManager, item, Log);
|
||
group.Parse();
|
||
if (group.Module.MigrationModule != null)
|
||
{
|
||
_migrationInfo.Groups.Add(group);
|
||
if (!_migrationInfo.Modules.Exists(x => x.MigrationModule == group.Module.MigrationModule))
|
||
{
|
||
_migrationInfo.Modules.Add(new MigrationModules(group.Module.MigrationModule, group.Module.Module));
|
||
}
|
||
}
|
||
}
|
||
ReportProgress(90, MigrationResource.ClearingTemporaryData);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
_migrationInfo.failedArchives.Add(Path.GetFileName(_takeouts));
|
||
Log($"Couldn't parse users from {Path.GetFileNameWithoutExtension(_takeouts)} archive", ex);
|
||
}
|
||
ReportProgress(100, MigrationResource.DataProcessingCompleted);
|
||
return Task.FromResult(_migrationInfo.ToApiInfo());
|
||
}
|
||
|
||
public List<OCGroup> DBExtractGroup(string dbFile)
|
||
{
|
||
var groups = new List<OCGroup>();
|
||
|
||
var sqlFile = File.ReadAllText(dbFile);
|
||
|
||
var groupList = GetDumpChunk("oc_groups", sqlFile);
|
||
if (groupList == null)
|
||
{
|
||
return groups;
|
||
}
|
||
|
||
foreach (var group in groupList)
|
||
{
|
||
groups.Add(new OCGroup
|
||
{
|
||
GroupGid = group.Trim('\''),
|
||
UsersUid = new List<string>()
|
||
});
|
||
}
|
||
|
||
var usersInGroups = GetDumpChunk("oc_group_user", sqlFile);
|
||
foreach (var user in usersInGroups)
|
||
{
|
||
var userGroupGid = user.Split(',').First().Trim('\'');
|
||
var userUid = user.Split(',').Last().Trim('\'');
|
||
groups.Find(ggid => userGroupGid == ggid.GroupGid).UsersUid.Add(userUid);
|
||
}
|
||
|
||
return groups;
|
||
}
|
||
|
||
public List<OCUser> DBExtractUser(string dbFile)
|
||
{
|
||
var userDataList = new Dictionary<string, OCUser>();
|
||
|
||
var sqlFile = File.ReadAllText(dbFile);
|
||
|
||
var accountsData = GetDumpChunk("oc_accounts_data", sqlFile);
|
||
if (accountsData != null)
|
||
{
|
||
throw new Exception();
|
||
}
|
||
|
||
var accounts = GetDumpChunk("oc_accounts", sqlFile);
|
||
if (accounts == null)
|
||
{
|
||
return userDataList.Values.ToList();
|
||
}
|
||
|
||
foreach (var account in accounts)
|
||
{
|
||
|
||
var userId = account.Split(',')[2].Trim('\'');
|
||
|
||
userDataList.Add(userId, new OCUser
|
||
{
|
||
Uid = account.Split(',')[2].Trim('\''),
|
||
Data = new OCUserData
|
||
{
|
||
DisplayName = account.Split(',')[4].Trim('\''),
|
||
Email = account.Split(',')[1].Trim('\'')
|
||
},
|
||
Addressbooks = null,
|
||
Calendars = new List<OCCalendars>(),
|
||
Storages = new OCStorages()
|
||
});
|
||
}
|
||
|
||
var calendarsData = GetDumpChunk("oc_calendars", sqlFile);
|
||
if (calendarsData != null)
|
||
{
|
||
foreach (var calendarData in calendarsData)
|
||
{
|
||
var values = calendarData.Split(',')
|
||
.Select(s => s.Trim('\'')).ToArray();
|
||
var uid = values[1].Split('/').Last();
|
||
userDataList.TryGetValue(uid, out var user);
|
||
if (user == null)
|
||
{
|
||
continue;
|
||
}
|
||
|
||
user.Calendars.Add(new OCCalendars()
|
||
{
|
||
Id = int.Parse(values[0]),
|
||
CalendarObject = new List<OCCalendarObjects>(),
|
||
DisplayName = values[2]
|
||
});
|
||
}
|
||
}
|
||
|
||
var calendars = userDataList.Values
|
||
.SelectMany(u => u.Calendars)
|
||
.ToDictionary(c => c.Id, c => c);
|
||
var calendarObjects = GetDumpChunk("oc_calendarobjects", sqlFile);
|
||
if (calendarObjects != null)
|
||
{
|
||
foreach (var calendarObject in calendarObjects)
|
||
{
|
||
var values = calendarObject.Split(',')
|
||
.Select(s => s.Trim('\'')).ToArray();
|
||
var calId = int.Parse(values[3]);
|
||
calendars.TryGetValue(calId, out var cal);
|
||
if (cal == null)
|
||
{
|
||
continue;
|
||
}
|
||
|
||
cal.CalendarObject.Add(new OCCalendarObjects()
|
||
{
|
||
Id = int.Parse(values[0]),
|
||
CalendarData = Encoding.UTF8.GetBytes(values[1]
|
||
.Replace("\\r", "")
|
||
.Replace("\\n", "\n")),
|
||
});
|
||
}
|
||
}
|
||
|
||
var addressBooks = GetDumpChunk("oc_addressbooks", sqlFile);
|
||
if (addressBooks != null)
|
||
{
|
||
foreach (var addressBook in addressBooks)
|
||
{
|
||
var values = addressBook.Split(',')
|
||
.Select(s => s.Trim('\'')).ToArray();
|
||
var uid = values[1].Split('/').Last();
|
||
userDataList.TryGetValue(uid, out var user);
|
||
if (user == null)
|
||
{
|
||
continue;
|
||
}
|
||
|
||
user.Addressbooks = new OCAddressbooks();
|
||
user.Addressbooks.Id = int.Parse(values[0]);
|
||
user.Addressbooks.Cards = new List<OCCards>();
|
||
}
|
||
}
|
||
|
||
var addressBooksDict = userDataList.Values
|
||
.Select(u => u.Addressbooks)
|
||
.Where(x => x != null)
|
||
.ToDictionary(b => b.Id, b => b);
|
||
var cards = GetDumpChunk("oc_cards", sqlFile);
|
||
if (cards != null)
|
||
{
|
||
foreach (var card in cards)
|
||
{
|
||
var values = card.Split(',')
|
||
.Select(s => s.Trim('\'')).ToArray();
|
||
var bookId = int.Parse(values[1]);
|
||
addressBooksDict.TryGetValue(bookId, out var book);
|
||
if (book == null)
|
||
{
|
||
continue;
|
||
}
|
||
|
||
book.Cards.Add(new OCCards()
|
||
{
|
||
Id = int.Parse(values[0]),
|
||
CardData = Encoding.UTF8.GetBytes(values[2]
|
||
.Replace("\\r", "")
|
||
.Replace("\\n", "\n")),
|
||
});
|
||
}
|
||
}
|
||
|
||
var storages = GetDumpChunk("oc_storages", sqlFile);
|
||
if (storages != null)
|
||
{
|
||
foreach (var storage in storages)
|
||
{
|
||
var values = storage.Split(',')
|
||
.Select(s => s.Trim('\'')).ToArray();
|
||
var uid = values[0].Split(':').Last();
|
||
userDataList.TryGetValue(uid, out var user);
|
||
if (user == null)
|
||
{
|
||
continue;
|
||
}
|
||
|
||
user.Storages.NumericId = int.Parse(values[1]);
|
||
user.Storages.Id = values[0];
|
||
user.Storages.FileCache = new List<OCFileCache>();
|
||
}
|
||
}
|
||
|
||
var storagesDict = userDataList.Values
|
||
.Select(u => u.Storages)
|
||
.ToDictionary(s => s.NumericId, s => s);
|
||
var fileCaches = GetDumpChunk("oc_filecache", sqlFile);
|
||
if (fileCaches != null)
|
||
{
|
||
foreach (var cache in fileCaches)
|
||
{
|
||
var values = cache.Split(',')
|
||
.Select(s => s.Trim('\'')).ToArray();
|
||
var storageId = int.Parse(values[1]);
|
||
storagesDict.TryGetValue(storageId, out var storage);
|
||
if (storage == null)
|
||
{
|
||
continue;
|
||
}
|
||
|
||
storage.FileCache.Add(new OCFileCache()
|
||
{
|
||
FileId = int.Parse(values[0]),
|
||
Path = values[2],
|
||
Share = new List<OCShare>()
|
||
});
|
||
}
|
||
}
|
||
|
||
var files = userDataList.Values
|
||
.SelectMany(u => u.Storages.FileCache)
|
||
.ToDictionary(f => f.FileId, f => f);
|
||
var shares = GetDumpChunk("oc_share", sqlFile);
|
||
if (shares != null)
|
||
{
|
||
foreach (var share in shares)
|
||
{
|
||
var values = share.Split(',')
|
||
.Select(s => s.Trim('\'')).ToArray();
|
||
var fileId = int.Parse(values[9]);
|
||
files.TryGetValue(fileId, out var file);
|
||
if (file == null)
|
||
{
|
||
continue;
|
||
}
|
||
|
||
file.Share.Add(new OCShare()
|
||
{
|
||
Id = int.Parse(values[0]),
|
||
ShareWith = values[2],
|
||
Premissions = int.Parse(values[11])
|
||
});
|
||
}
|
||
}
|
||
|
||
return userDataList.Values.ToList();
|
||
}
|
||
|
||
private IEnumerable<string> GetDumpChunk(string tableName, string dump)
|
||
{
|
||
var regex = new Regex($"INSERT INTO `{tableName}` VALUES (.*);");
|
||
var match = regex.Match(dump);
|
||
if (!match.Success)
|
||
{
|
||
return null;
|
||
}
|
||
|
||
var entryRegex = new Regex(@"(\(.*?\))[,;]");
|
||
var accountDataMatches = entryRegex.Matches(match.Groups[1].Value + ";");
|
||
return accountDataMatches.Cast<Match>()
|
||
.Select(m => m.Groups[1].Value.Trim(new[] { '(', ')' }));
|
||
}
|
||
|
||
public override Task Migrate(MigrationApiInfo migrationApiInfo)
|
||
{
|
||
ReportProgress(0, MigrationResource.PreparingForMigration);
|
||
_migrationInfo.Merge(migrationApiInfo);
|
||
|
||
var usersForImport = _migrationInfo.Users
|
||
.Where(u => u.Value.ShouldImport)
|
||
.Select(u => u.Value);
|
||
|
||
_importedUsers = new List<Guid>();
|
||
var failedUsers = new List<OCMigratingUser>();
|
||
var usersCount = usersForImport.Count();
|
||
var progressStep = 25 / usersCount;
|
||
var i = 1;
|
||
foreach (var user in usersForImport)
|
||
{
|
||
if (_cancellationToken.IsCancellationRequested) { ReportProgress(100, MigrationResource.MigrationCanceled); return null; }
|
||
ReportProgress(GetProgress() + progressStep, string.Format(MigrationResource.UserMigration, user.DisplayName, i++, usersCount));
|
||
try
|
||
{
|
||
user.dataСhange(migrationApiInfo.Users.Find(element => element.Key == user.Key));
|
||
user.Migrate();
|
||
_importedUsers.Add(user.Guid);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
failedUsers.Add(user);
|
||
Log($"Couldn't migrate user {user.DisplayName} ({user.Email})", ex);
|
||
}
|
||
}
|
||
|
||
var groupsForImport = _migrationInfo.Groups
|
||
.Where(g => g.ShouldImport)
|
||
.Select(g => g);
|
||
var groupsCount = groupsForImport.Count();
|
||
if (groupsCount != 0)
|
||
{
|
||
progressStep = 25 / groupsForImport.Count();
|
||
//Create all groups
|
||
i = 1;
|
||
foreach (var group in groupsForImport)
|
||
{
|
||
if (_cancellationToken.IsCancellationRequested) { ReportProgress(100, MigrationResource.MigrationCanceled); return null; }
|
||
ReportProgress(GetProgress() + progressStep, string.Format(MigrationResource.GroupMigration, group.GroupName, i++, groupsCount));
|
||
try
|
||
{
|
||
group.UsersGuidList = _migrationInfo.Users
|
||
.Where(user => group.UserUidList.Exists(u => user.Key == u))
|
||
.Select(u => u)
|
||
.ToDictionary(k => k.Key, v => v.Value.Guid);
|
||
group.Migrate();
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log($"Couldn't migrate group {group.GroupName} ", ex);
|
||
}
|
||
}
|
||
}
|
||
|
||
i = 1;
|
||
foreach (var user in usersForImport)
|
||
{
|
||
if (_cancellationToken.IsCancellationRequested) { ReportProgress(100, MigrationResource.MigrationCanceled); return null; }
|
||
if (failedUsers.Contains(user))
|
||
{
|
||
ReportProgress(GetProgress() + progressStep, string.Format(MigrationResource.UserSkipped, user.DisplayName, i, usersCount));
|
||
continue;
|
||
}
|
||
|
||
var smallStep = progressStep / 3;
|
||
|
||
try
|
||
{
|
||
user.MigratingContacts.Migrate();
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log($"Couldn't migrate user {user.DisplayName} ({user.Email}) contacts", ex);
|
||
}
|
||
finally
|
||
{
|
||
ReportProgress(GetProgress() + smallStep, string.Format(MigrationResource.MigratingUserContacts, user.DisplayName, i, usersCount));
|
||
}
|
||
|
||
/*try
|
||
{
|
||
user.MigratingCalendar.Migrate();
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log($"Couldn't migrate user {user.DisplayName} ({user.Email}) calendar", ex);
|
||
}
|
||
finally
|
||
{
|
||
ReportProgress(GetProgress() + smallStep, String.Format(MigrationResource.UserCalendarMigration, user.DisplayName, i, usersCount));
|
||
}*/
|
||
|
||
try
|
||
{
|
||
var currentUser = _securityContext.CurrentAccount;
|
||
_securityContext.AuthenticateMe(user.Guid);
|
||
user.MigratingFiles.SetUsersDict(usersForImport.Except(failedUsers));
|
||
user.MigratingFiles.SetGroupsDict(groupsForImport);
|
||
user.MigratingFiles.Migrate();
|
||
_securityContext.AuthenticateMe(currentUser.ID);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Log($"Couldn't migrate user {user.DisplayName} ({user.Email}) files", ex);
|
||
}
|
||
finally
|
||
{
|
||
ReportProgress(GetProgress() + smallStep, string.Format(MigrationResource.MigratingUserFiles, user.DisplayName, i, usersCount));
|
||
}
|
||
i++;
|
||
}
|
||
|
||
if (Directory.Exists(_tmpFolder))
|
||
{
|
||
Directory.Delete(_tmpFolder, true);
|
||
}
|
||
ReportProgress(100, MigrationResource.MigrationCompleted);
|
||
return Task.CompletedTask;
|
||
}
|
||
}
|