2022-03-15 18:00:53 +00:00
// (c) Copyright Ascensio System SIA 2010-2022
//
// This program is a free software product.
// You can redistribute it and/or modify it under the terms
// of the GNU Affero General Public License (AGPL) version 3 as published by the Free Software
// Foundation. In accordance with Section 7(a) of the GNU AGPL its Section 15 shall be amended
// to the effect that Ascensio System SIA expressly excludes the warranty of non-infringement of
// any third-party rights.
//
// This program is distributed WITHOUT ANY WARRANTY, without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For details, see
// the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
//
// You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia, EU, LV-1021.
//
// The interactive user interfaces in modified source and object code versions of the Program must
// display Appropriate Legal Notices, as required under Section 5 of the GNU AGPL version 3.
//
// Pursuant to Section 7(b) of the License you must retain the original Product logo when
// distributing the program. Pursuant to Section 7(e) we decline to grant you any rights under
// trademark law for use of our trademarks.
//
// All the Product's GUI elements, including illustrations and icon sets, as well as technical writing
// content are licensed under the terms of the Creative Commons Attribution-ShareAlike 4.0
// International. See the License terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
2022-02-09 18:49:52 +00:00
namespace ASC.Data.Backup.Tasks ;
[Scope]
public class BackupPortalTask : PortalTaskBase
2021-08-31 09:40:28 +00:00
{
2022-02-09 18:49:52 +00:00
public string BackupFilePath { get ; private set ; }
public int Limit { get ; private set ; }
private BackupsContext BackupRecordContext = > _lazyBackupsContext . Value ;
2022-03-25 16:26:06 +00:00
private const int MaxLength = 250 ;
private const int BatchLimit = 5000 ;
2022-02-09 18:49:52 +00:00
2022-05-26 09:01:54 +00:00
private readonly bool _dump ;
private readonly ILogger < BackupPortalTask > _logger ;
2022-02-09 18:49:52 +00:00
private readonly TenantManager _tenantManager ;
private readonly TempStream _tempStream ;
private readonly Lazy < BackupsContext > _lazyBackupsContext ;
2022-04-26 17:53:48 +00:00
public BackupPortalTask (
DbFactory dbFactory ,
DbContextManager < BackupsContext > dbContextManager ,
2022-05-26 09:01:54 +00:00
ILogger < BackupPortalTask > logger ,
2022-04-26 17:53:48 +00:00
TenantManager tenantManager ,
CoreBaseSettings coreBaseSettings ,
StorageFactory storageFactory ,
StorageFactoryConfig storageFactoryConfig ,
ModuleProvider moduleProvider ,
TempStream tempStream )
2022-05-26 09:01:54 +00:00
: base ( dbFactory , logger , storageFactory , storageFactoryConfig , moduleProvider )
2021-08-31 09:40:28 +00:00
{
2022-05-26 09:01:54 +00:00
_dump = coreBaseSettings . Standalone ;
_logger = logger ;
2022-02-09 18:49:52 +00:00
_tenantManager = tenantManager ;
_tempStream = tempStream ;
2022-06-02 12:42:45 +00:00
_lazyBackupsContext = new Lazy < BackupsContext > ( ( ) = > dbContextManager . Value ) ;
2022-02-09 18:49:52 +00:00
}
2020-06-16 11:34:44 +00:00
2022-02-09 18:49:52 +00:00
public void Init ( int tenantId , string fromConfigPath , string toFilePath , int limit )
2022-03-09 17:15:51 +00:00
{
ArgumentNullOrEmptyException . ThrowIfNullOrEmpty ( toFilePath ) ;
2022-02-09 18:49:52 +00:00
BackupFilePath = toFilePath ;
Limit = limit ;
Init ( tenantId , fromConfigPath ) ;
2020-06-10 12:35:10 +00:00
2022-02-09 18:49:52 +00:00
}
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
public override void RunJob ( )
{
2022-05-26 09:01:54 +00:00
_logger . DebugBeginBackup ( TenantId ) ;
2022-02-09 18:49:52 +00:00
_tenantManager . SetCurrentTenant ( TenantId ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:33:50 +00:00
2022-02-09 18:49:52 +00:00
using ( var writer = new ZipWriteOperator ( _tempStream , BackupFilePath ) )
2020-05-20 15:14:44 +00:00
{
2022-02-09 18:49:52 +00:00
if ( _dump )
{
DoDump ( writer ) ;
}
else
2021-08-31 09:40:28 +00:00
{
2022-02-09 18:49:52 +00:00
var modulesToProcess = GetModulesToProcess ( ) . ToList ( ) ;
var fileGroups = GetFilesGroup ( ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
var stepscount = ProcessStorage ? fileGroups . Count : 0 ;
SetStepsCount ( modulesToProcess . Count + stepscount ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
foreach ( var module in modulesToProcess )
2020-05-20 15:14:44 +00:00
{
2022-02-09 18:49:52 +00:00
DoBackupModule ( writer , module ) ;
2020-05-20 15:14:44 +00:00
}
2022-02-09 18:49:52 +00:00
if ( ProcessStorage )
2021-08-31 09:40:28 +00:00
{
2022-02-09 18:49:52 +00:00
DoBackupStorage ( writer , fileGroups ) ;
2020-05-20 15:14:44 +00:00
}
}
}
2022-05-26 09:01:54 +00:00
_logger . DebugEndBackup ( TenantId ) ;
2022-02-09 18:49:52 +00:00
}
public List < object [ ] > ExecuteList ( DbCommand command )
{
var list = new List < object [ ] > ( ) ;
using ( var result = command . ExecuteReader ( ) )
2021-08-31 09:40:28 +00:00
{
2022-02-09 18:49:52 +00:00
while ( result . Read ( ) )
2021-08-31 09:40:28 +00:00
{
2022-02-09 18:49:52 +00:00
var objects = new object [ result . FieldCount ] ;
result . GetValues ( objects ) ;
list . Add ( objects ) ;
2021-05-17 18:27:39 +00:00
}
2022-02-09 18:49:52 +00:00
}
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
return list ;
}
2022-02-09 18:33:50 +00:00
2022-02-09 18:49:52 +00:00
private void DoDump ( IDataWriteOperator writer )
2022-06-02 12:42:45 +00:00
{
var databases = new Dictionary < Tuple < string , string > , List < string > > ( ) ;
using ( var connection = DbFactory . OpenConnection ( ) )
2022-02-09 18:49:52 +00:00
{
2022-06-02 12:42:45 +00:00
var command = connection . CreateCommand ( ) ;
command . CommandText = "select id, connection_string from mail_server_server" ;
ExecuteList ( command ) . ForEach ( r = >
{
var connectionString = GetConnectionString ( ( int ) r [ 0 ] , JsonConvert . DeserializeObject < Dictionary < string , object > > ( Convert . ToString ( r [ 1 ] ) ) [ "DbConnection" ] . ToString ( ) ) ;
var command = connection . CreateCommand ( ) ;
command . CommandText = "show tables" ;
var tables = ExecuteList ( command ) . Select ( r = > Convert . ToString ( r [ 0 ] ) ) . ToList ( ) ;
databases . Add ( new Tuple < string , string > ( connectionString . Name , connectionString . ConnectionString ) , tables ) ;
} ) ;
}
2022-02-09 18:49:52 +00:00
using ( var connection = DbFactory . OpenConnection ( ) )
2021-08-31 09:40:28 +00:00
{
2022-02-09 18:49:52 +00:00
var command = connection . CreateCommand ( ) ;
command . CommandText = "show tables" ;
2022-06-02 12:42:45 +00:00
var tables = ExecuteList ( command ) . Select ( r = > Convert . ToString ( r [ 0 ] ) ) . ToList ( ) ;
databases . Add ( new Tuple < string , string > ( "default" , DbFactory . ConnectionStringSettings ( ) ) , tables ) ;
}
using ( var stream = new MemoryStream ( Encoding . UTF8 . GetBytes ( true . ToString ( ) ) ) )
{
writer . WriteEntry ( KeyHelper . GetDumpKey ( ) , stream ) ;
}
var files = new List < BackupFileInfo > ( ) ;
var stepscount = 0 ;
foreach ( var db in databases )
{
stepscount + = db . Value . Count * 4 ; // (schema + data) * (dump + zip)
2022-03-09 17:15:51 +00:00
}
if ( ProcessStorage )
{
var tenants = _tenantManager . GetTenants ( false ) . Select ( r = > r . Id ) ;
foreach ( var t in tenants )
{
files . AddRange ( GetFiles ( t ) ) ;
}
stepscount + = files . Count * 2 + 1 ;
2022-05-26 09:01:54 +00:00
_logger . DebugFilesCount ( files . Count ) ;
2022-03-09 17:15:51 +00:00
}
2020-05-20 15:14:44 +00:00
2022-06-02 12:42:45 +00:00
SetStepsCount ( stepscount ) ;
foreach ( var db in databases )
{
DoDump ( writer , db . Key . Item1 , db . Key . Item2 , db . Value ) ;
}
var dir = Path . GetDirectoryName ( BackupFilePath ) ;
var subDir = Path . Combine ( dir , Path . GetFileNameWithoutExtension ( BackupFilePath ) ) ;
_logger . DebugDirRemoveStart ( subDir ) ;
Directory . Delete ( subDir , true ) ;
_logger . DebugDirRemoveEnd ( subDir ) ;
if ( ProcessStorage )
{
DoDumpStorage ( writer , files ) ;
}
}
private void DoDump ( IDataWriteOperator writer , string dbName , string connectionString , List < string > tables )
{
2022-04-14 19:23:57 +00:00
var excluded = ModuleProvider . AllModules . Where ( r = > _ignoredModules . Contains ( r . ModuleName ) ) . SelectMany ( r = > r . Tables ) . Select ( r = > r . Name ) . ToList ( ) ;
excluded . AddRange ( _ignoredTables ) ;
2022-02-09 18:49:52 +00:00
excluded . Add ( "res_" ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
var dir = Path . GetDirectoryName ( BackupFilePath ) ;
2022-06-02 12:42:45 +00:00
var subDir = CrossPlatform . PathCombine ( dir , Path . GetFileNameWithoutExtension ( BackupFilePath ) ) ;
var schemeDir = "" ;
var dataDir = "" ;
if ( dbName = = "default" )
{
schemeDir = Path . Combine ( subDir , KeyHelper . GetDatabaseSchema ( ) ) ;
dataDir = Path . Combine ( subDir , KeyHelper . GetDatabaseData ( ) ) ;
}
else
{
schemeDir = Path . Combine ( subDir , dbName , KeyHelper . GetDatabaseSchema ( ) ) ;
dataDir = Path . Combine ( subDir , dbName , KeyHelper . GetDatabaseData ( ) ) ;
}
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
if ( ! Directory . Exists ( schemeDir ) )
{
Directory . CreateDirectory ( schemeDir ) ;
}
if ( ! Directory . Exists ( dataDir ) )
{
Directory . CreateDirectory ( dataDir ) ;
2022-06-02 12:42:45 +00:00
}
var dict = new Dictionary < string , int > ( ) ;
foreach ( var table in tables )
{
dict . Add ( table , SelectCount ( table , connectionString ) ) ;
2022-02-09 18:49:52 +00:00
}
tables . Sort ( ( pair1 , pair2 ) = > dict [ pair1 ] . CompareTo ( dict [ pair2 ] ) ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
for ( var i = 0 ; i < tables . Count ; i + = TasksLimit )
{
var tasks = new List < Task > ( TasksLimit * 2 ) ;
for ( var j = 0 ; j < TasksLimit & & i + j < tables . Count ; j + + )
2020-05-20 15:14:44 +00:00
{
2022-02-09 18:49:52 +00:00
var t = tables [ i + j ] ;
2022-06-02 12:42:45 +00:00
tasks . Add ( Task . Run ( ( ) = > DumpTableScheme ( t , schemeDir , connectionString ) ) ) ;
2022-02-09 18:49:52 +00:00
if ( ! excluded . Any ( t . StartsWith ) )
2020-05-20 15:14:44 +00:00
{
2022-06-02 12:42:45 +00:00
tasks . Add ( Task . Run ( ( ) = > DumpTableData ( t , dataDir , dict [ t ] , connectionString ) ) ) ;
2022-02-09 18:49:52 +00:00
}
else
{
SetStepCompleted ( 2 ) ;
2020-05-20 15:14:44 +00:00
}
}
2022-02-09 18:49:52 +00:00
Task . WaitAll ( tasks . ToArray ( ) ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
ArchiveDir ( writer , subDir ) ;
2020-05-20 15:14:44 +00:00
}
2022-02-09 18:49:52 +00:00
}
private IEnumerable < BackupFileInfo > GetFiles ( int tenantId )
{
2022-03-09 17:15:51 +00:00
var files = GetFilesToProcess ( tenantId ) . ToList ( ) ;
var exclude = BackupRecordContext . Backups . AsQueryable ( ) . Where ( b = > b . TenantId = = tenantId & & b . StorageType = = 0 & & b . StoragePath ! = null ) . ToList ( ) ;
files = files . Where ( f = > ! exclude . Any ( e = > f . Path . Replace ( '\\' , '/' ) . Contains ( $"/file_{e.StoragePath}/" ) ) ) . ToList ( ) ;
2022-02-09 18:49:52 +00:00
return files ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
}
2020-05-20 15:14:44 +00:00
2022-06-02 12:42:45 +00:00
private void DumpTableScheme ( string t , string dir , string connectionString )
2022-02-09 18:49:52 +00:00
{
try
2020-05-20 15:14:44 +00:00
{
2022-05-26 09:01:54 +00:00
_logger . DebugDumpTableSchemeStart ( t ) ;
2022-06-02 12:42:45 +00:00
using ( var connection = DbFactory . OpenConnection ( connectionString : connectionString ) )
2020-05-20 15:14:44 +00:00
{
2022-03-09 17:15:51 +00:00
var command = connection . CreateCommand ( ) ;
command . CommandText = $"SHOW CREATE TABLE `{t}`" ;
2022-02-09 18:49:52 +00:00
var createScheme = ExecuteList ( command ) ;
2022-03-09 17:15:51 +00:00
var creates = new StringBuilder ( ) ;
creates . Append ( $"DROP TABLE IF EXISTS `{t}`;" ) ;
2022-02-09 18:49:52 +00:00
creates . AppendLine ( ) ;
creates . Append ( createScheme
. Select ( r = > Convert . ToString ( r [ 1 ] ) )
2022-03-09 17:15:51 +00:00
. FirstOrDefault ( ) ) ;
creates . Append ( ';' ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
var path = CrossPlatform . PathCombine ( dir , t ) ;
using ( var stream = File . OpenWrite ( path ) )
{
var bytes = Encoding . UTF8 . GetBytes ( creates . ToString ( ) ) ;
stream . Write ( bytes , 0 , bytes . Length ) ;
2021-08-31 09:40:28 +00:00
}
2022-02-09 18:49:52 +00:00
SetStepCompleted ( ) ;
2020-05-20 15:14:44 +00:00
}
2022-05-26 09:01:54 +00:00
_logger . DebugDumpTableSchemeStop ( t ) ;
2020-05-20 15:14:44 +00:00
}
2022-02-09 18:49:52 +00:00
catch ( Exception e )
2020-05-20 15:14:44 +00:00
{
2022-05-26 09:01:54 +00:00
_logger . ErrorDumpTableScheme ( e ) ;
2022-02-09 18:49:52 +00:00
}
2022-02-09 18:33:50 +00:00
2022-02-09 18:49:52 +00:00
}
2020-05-20 15:14:44 +00:00
2022-06-02 12:42:45 +00:00
private int SelectCount ( string t , string dbName )
2022-02-09 18:49:52 +00:00
{
try
2020-05-20 15:14:44 +00:00
{
2022-06-02 12:42:45 +00:00
using var connection = DbFactory . OpenConnection ( connectionString : dbName ) ;
2022-02-09 18:49:52 +00:00
using var analyzeCommand = connection . CreateCommand ( ) ;
analyzeCommand . CommandText = $"analyze table {t}" ;
analyzeCommand . ExecuteNonQuery ( ) ;
using var command = connection . CreateCommand ( ) ;
command . CommandText = $"select TABLE_ROWS from INFORMATION_SCHEMA.TABLES where TABLE_NAME = '{t}' and TABLE_SCHEMA = '{connection.Database}'" ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
return int . Parse ( command . ExecuteScalar ( ) . ToString ( ) ) ;
2020-05-20 15:14:44 +00:00
}
2022-02-09 18:49:52 +00:00
catch ( Exception e )
{
2022-05-26 09:01:54 +00:00
_logger . ErrorSelectCount ( e ) ;
2022-02-09 18:49:52 +00:00
throw ;
2020-05-20 15:14:44 +00:00
}
2022-02-09 18:49:52 +00:00
}
2020-05-20 15:14:44 +00:00
2022-06-02 12:42:45 +00:00
private void DumpTableData ( string t , string dir , int count , string connectionString )
2022-02-09 18:49:52 +00:00
{
try
2020-05-20 15:14:44 +00:00
{
2022-02-09 18:49:52 +00:00
if ( count = = 0 )
2020-05-20 15:14:44 +00:00
{
2022-05-26 09:01:54 +00:00
_logger . DebugDumpTableDataStop ( t ) ;
2022-02-09 18:49:52 +00:00
SetStepCompleted ( 2 ) ;
2022-02-09 18:33:50 +00:00
2022-02-09 18:49:52 +00:00
return ;
}
2020-05-20 15:14:44 +00:00
2022-05-26 09:01:54 +00:00
_logger . DebugDumpTableDataStart ( t ) ;
2022-03-09 17:15:51 +00:00
bool searchWithPrimary ;
2022-02-09 18:49:52 +00:00
string primaryIndex ;
var primaryIndexStep = 0 ;
var primaryIndexStart = 0 ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
List < string > columns ;
2022-02-09 18:33:50 +00:00
2022-06-02 12:42:45 +00:00
using ( var connection = DbFactory . OpenConnection ( connectionString : connectionString ) )
2022-02-09 18:49:52 +00:00
{
2022-03-09 17:15:51 +00:00
var command = connection . CreateCommand ( ) ;
command . CommandText = string . Format ( $"SHOW COLUMNS FROM `{t}`" ) ;
2022-02-09 18:49:52 +00:00
columns = ExecuteList ( command ) . Select ( r = > "`" + Convert . ToString ( r [ 0 ] ) + "`" ) . ToList ( ) ;
if ( command . CommandText . Contains ( "tenants_quota" ) | | command . CommandText . Contains ( "webstudio_settings" ) )
2020-05-20 15:14:44 +00:00
{
2020-06-10 12:35:10 +00:00
2020-05-26 08:52:47 +00:00
}
2022-02-09 18:49:52 +00:00
}
2020-06-10 12:35:10 +00:00
2022-02-09 18:49:52 +00:00
using ( var connection = DbFactory . OpenConnection ( ) )
{
2022-03-09 17:15:51 +00:00
var command = connection . CreateCommand ( ) ;
command . CommandText = $"select COLUMN_NAME from information_schema.`COLUMNS` where TABLE_SCHEMA = '{connection.Database}' and TABLE_NAME = '{t}' and COLUMN_KEY = 'PRI' and DATA_TYPE = 'int'" ;
2022-02-09 18:49:52 +00:00
primaryIndex = ExecuteList ( command ) . ConvertAll ( r = > Convert . ToString ( r [ 0 ] ) ) . FirstOrDefault ( ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
}
2022-02-09 18:33:50 +00:00
2022-02-09 18:49:52 +00:00
using ( var connection = DbFactory . OpenConnection ( ) )
{
2022-03-09 17:15:51 +00:00
var command = connection . CreateCommand ( ) ;
command . CommandText = $"SHOW INDEXES FROM {t} WHERE COLUMN_NAME='{primaryIndex}' AND seq_in_index=1" ;
2022-02-09 18:49:52 +00:00
var isLeft = ExecuteList ( command ) ;
searchWithPrimary = isLeft . Count = = 1 ;
}
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
if ( searchWithPrimary )
{
using var connection = DbFactory . OpenConnection ( ) ;
2022-03-09 17:15:51 +00:00
var command = connection . CreateCommand ( ) ;
command . CommandText = $"select max({primaryIndex}), min({primaryIndex}) from {t}" ;
2022-02-09 18:49:52 +00:00
var minMax = ExecuteList ( command ) . ConvertAll ( r = > new Tuple < int , int > ( Convert . ToInt32 ( r [ 0 ] ) , Convert . ToInt32 ( r [ 1 ] ) ) ) . FirstOrDefault ( ) ;
primaryIndexStart = minMax . Item2 ;
primaryIndexStep = ( minMax . Item1 - minMax . Item2 ) / count ;
2021-08-31 09:40:28 +00:00
2022-02-09 18:49:52 +00:00
if ( primaryIndexStep < Limit )
2020-05-26 08:52:47 +00:00
{
2022-02-09 18:49:52 +00:00
primaryIndexStep = Limit ;
2020-05-20 15:14:44 +00:00
}
2022-02-09 18:49:52 +00:00
}
2021-08-31 09:40:28 +00:00
2022-02-09 18:49:52 +00:00
var path = CrossPlatform . PathCombine ( dir , t ) ;
2021-08-31 09:40:28 +00:00
2022-02-09 18:49:52 +00:00
var offset = 0 ;
2021-08-31 09:40:28 +00:00
2022-02-09 18:49:52 +00:00
do
{
List < object [ ] > result ;
2021-08-31 09:40:28 +00:00
2022-02-09 18:49:52 +00:00
if ( searchWithPrimary )
{
2022-06-02 12:42:45 +00:00
result = GetDataWithPrimary ( t , columns , primaryIndex , primaryIndexStart , primaryIndexStep , connectionString ) ;
2022-02-09 18:49:52 +00:00
primaryIndexStart + = primaryIndexStep ;
}
else
{
2022-06-02 12:42:45 +00:00
result = GetData ( t , columns , offset , connectionString ) ;
2022-02-09 18:49:52 +00:00
}
2021-08-31 09:40:28 +00:00
2022-02-09 18:49:52 +00:00
offset + = Limit ;
2021-08-31 09:40:28 +00:00
2022-02-09 18:49:52 +00:00
var resultCount = result . Count ;
2021-08-31 09:40:28 +00:00
2022-02-09 18:49:52 +00:00
if ( resultCount = = 0 )
{
break ;
}
2021-08-31 09:40:28 +00:00
2022-02-09 18:49:52 +00:00
SaveToFile ( path , t , columns , result ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
} while ( true ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
SetStepCompleted ( ) ;
2022-05-26 09:01:54 +00:00
_logger . DebugDumpTableDataStop ( t ) ;
2020-05-20 15:14:44 +00:00
}
2022-02-09 18:49:52 +00:00
catch ( Exception e )
2020-05-20 15:14:44 +00:00
{
2022-05-26 09:01:54 +00:00
_logger . ErrorDumpTableData ( e ) ;
2022-02-09 18:49:52 +00:00
throw ;
2021-08-31 09:40:28 +00:00
}
2022-02-09 18:49:52 +00:00
}
2021-08-31 09:40:28 +00:00
2022-06-02 12:42:45 +00:00
private List < object [ ] > GetData ( string t , List < string > columns , int offset , string connectionString )
2022-02-09 18:49:52 +00:00
{
2022-06-02 12:42:45 +00:00
using var connection = DbFactory . OpenConnection ( connectionString : connectionString ) ;
2022-02-09 18:49:52 +00:00
var command = connection . CreateCommand ( ) ;
var selects = string . Join ( ',' , columns ) ;
command . CommandText = $"select {selects} from {t} LIMIT {offset}, {Limit}" ;
2022-02-09 18:33:50 +00:00
2022-02-09 18:49:52 +00:00
return ExecuteList ( command ) ;
}
2021-08-31 09:40:28 +00:00
2022-06-02 12:42:45 +00:00
private List < object [ ] > GetDataWithPrimary ( string t , List < string > columns , string primary , int start , int step , string connectionString )
2022-02-09 18:49:52 +00:00
{
2022-06-02 12:42:45 +00:00
using var connection = DbFactory . OpenConnection ( connectionString : connectionString ) ;
2022-02-09 18:49:52 +00:00
var command = connection . CreateCommand ( ) ;
var selects = string . Join ( ',' , columns ) ;
command . CommandText = $"select {selects} from {t} where {primary} BETWEEN {start} and {start + step} " ;
return ExecuteList ( command ) ;
2022-06-02 12:42:45 +00:00
}
private ConnectionStringSettings GetConnectionString ( int id , string connectionString )
{
connectionString = connectionString + ";convert zero datetime=True" ;
return new ConnectionStringSettings ( "mailservice-" + id , connectionString , "MySql.Data.MySqlClient" ) ;
}
2022-02-09 18:49:52 +00:00
private void SaveToFile ( string path , string t , IReadOnlyCollection < string > columns , List < object [ ] > data )
{
2022-05-26 09:01:54 +00:00
_logger . DebugSaveTable ( t ) ;
2022-03-09 17:15:51 +00:00
List < object [ ] > portion ;
2022-03-25 16:26:06 +00:00
while ( ( portion = data . Take ( BatchLimit ) . ToList ( ) ) . Count > 0 )
2020-05-20 15:14:44 +00:00
{
2022-02-09 18:49:52 +00:00
using ( var sw = new StreamWriter ( path , true ) )
using ( var writer = new JsonTextWriter ( sw ) )
2021-08-31 09:40:28 +00:00
{
2022-02-09 18:49:52 +00:00
writer . QuoteChar = '\'' ;
writer . DateFormatString = "yyyy-MM-dd HH:mm:ss" ;
sw . Write ( "REPLACE INTO `{0}` ({1}) VALUES " , t , string . Join ( "," , columns ) ) ;
sw . WriteLine ( ) ;
for ( var j = 0 ; j < portion . Count ; j + + )
2020-05-20 15:14:44 +00:00
{
2022-02-09 18:49:52 +00:00
var obj = portion [ j ] ;
sw . Write ( "(" ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
for ( var i = 0 ; i < obj . Length ; i + + )
2022-03-09 17:15:51 +00:00
{
var value = obj [ i ] ;
2022-06-02 12:42:45 +00:00
if ( value is byte [ ] byteArray & & byteArray . Length ! = 0 )
2020-05-20 15:14:44 +00:00
{
2022-02-09 18:49:52 +00:00
sw . Write ( "0x" ) ;
foreach ( var b in byteArray )
2020-05-20 15:14:44 +00:00
{
2022-02-09 18:49:52 +00:00
sw . Write ( "{0:x2}" , b ) ;
2020-05-20 15:14:44 +00:00
}
}
2022-02-09 18:49:52 +00:00
else
2022-06-02 12:42:45 +00:00
{
var s = obj [ i ] as string ;
if ( s ! = null )
{
sw . Write ( "'" + s . Replace ( "\r" , "\\r" ) . Replace ( "\n" , "\\n" ) + "'" ) ;
}
else
{
var ser = new JsonSerializer ( ) ;
ser . Serialize ( writer , value ) ;
}
2020-05-20 15:14:44 +00:00
}
2022-02-09 18:49:52 +00:00
if ( i ! = obj . Length - 1 )
2020-05-20 15:14:44 +00:00
{
2022-02-09 18:49:52 +00:00
sw . Write ( "," ) ;
2020-05-20 15:14:44 +00:00
}
}
2022-02-09 18:49:52 +00:00
sw . Write ( ")" ) ;
2022-02-09 18:33:50 +00:00
2022-02-09 18:49:52 +00:00
if ( j ! = portion . Count - 1 )
{
sw . Write ( "," ) ;
}
else
{
sw . Write ( ";" ) ;
2020-05-20 15:14:44 +00:00
}
2022-02-09 18:33:50 +00:00
2022-02-09 18:49:52 +00:00
sw . WriteLine ( ) ;
2020-05-20 15:14:44 +00:00
}
}
2022-02-09 18:49:52 +00:00
2022-03-25 16:26:06 +00:00
data = data . Skip ( BatchLimit ) . ToList ( ) ;
2020-05-20 15:14:44 +00:00
}
2022-02-09 18:49:52 +00:00
}
private void DoDumpStorage ( IDataWriteOperator writer , IReadOnlyList < BackupFileInfo > files )
{
2022-05-26 09:01:54 +00:00
_logger . DebugBeginBackupStorage ( ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
var dir = Path . GetDirectoryName ( BackupFilePath ) ;
var subDir = CrossPlatform . PathCombine ( dir , Path . GetFileNameWithoutExtension ( BackupFilePath ) ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
for ( var i = 0 ; i < files . Count ; i + = TasksLimit )
2020-05-20 15:14:44 +00:00
{
2022-02-09 18:49:52 +00:00
var storageDir = CrossPlatform . PathCombine ( subDir , KeyHelper . GetStorage ( ) ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
if ( ! Directory . Exists ( storageDir ) )
{
Directory . CreateDirectory ( storageDir ) ;
}
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
var tasks = new List < Task > ( TasksLimit ) ;
for ( var j = 0 ; j < TasksLimit & & i + j < files . Count ; j + + )
2020-05-20 15:14:44 +00:00
{
2022-02-09 18:49:52 +00:00
var t = files [ i + j ] ;
tasks . Add ( Task . Run ( ( ) = > DoDumpFile ( t , storageDir ) ) ) ;
}
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
Task . WaitAll ( tasks . ToArray ( ) ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
ArchiveDir ( writer , subDir ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
Directory . Delete ( storageDir , true ) ;
}
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
var restoreInfoXml = new XElement ( "storage_restore" , files . Select ( file = > ( object ) file . ToXElement ( ) ) . ToArray ( ) ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
var tmpPath = CrossPlatform . PathCombine ( subDir , KeyHelper . GetStorageRestoreInfoZipKey ( ) ) ;
Directory . CreateDirectory ( Path . GetDirectoryName ( tmpPath ) ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
using ( var tmpFile = new FileStream ( tmpPath , FileMode . OpenOrCreate , FileAccess . ReadWrite , FileShare . Read , 4096 , FileOptions . DeleteOnClose ) )
{
restoreInfoXml . WriteTo ( tmpFile ) ;
writer . WriteEntry ( KeyHelper . GetStorageRestoreInfoZipKey ( ) , tmpFile ) ;
}
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
SetStepCompleted ( ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
Directory . Delete ( subDir , true ) ;
2020-05-20 15:14:44 +00:00
2022-05-26 09:01:54 +00:00
_logger . DebugEndBackupStorage ( ) ;
2022-02-09 18:49:52 +00:00
}
private async Task DoDumpFile ( BackupFileInfo file , string dir )
{
var storage = StorageFactory . GetStorage ( ConfigPath , file . Tenant . ToString ( ) , file . Module ) ;
var filePath = CrossPlatform . PathCombine ( dir , file . GetZipKey ( ) ) ;
var dirName = Path . GetDirectoryName ( filePath ) ;
2020-05-20 15:14:44 +00:00
2022-05-26 09:01:54 +00:00
_logger . DebugBackupFile ( filePath ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
if ( ! Directory . Exists ( dirName ) & & ! string . IsNullOrEmpty ( dirName ) )
{
Directory . CreateDirectory ( dirName ) ;
2020-05-20 15:14:44 +00:00
}
2022-03-25 16:26:06 +00:00
if ( ! WorkContext . IsMono & & filePath . Length > MaxLength )
2020-05-20 15:14:44 +00:00
{
2022-02-09 18:49:52 +00:00
filePath = @"\\?\" + filePath ;
2022-03-09 17:15:51 +00:00
}
using ( var fileStream = await storage . GetReadStreamAsync ( file . Domain , file . Path ) )
2022-02-09 18:49:52 +00:00
using ( var tmpFile = File . OpenWrite ( filePath ) )
{
await fileStream . CopyToAsync ( tmpFile ) ;
}
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
SetStepCompleted ( ) ;
}
2021-08-31 09:40:28 +00:00
2022-02-09 18:49:52 +00:00
private void ArchiveDir ( IDataWriteOperator writer , string subDir )
{
2022-05-26 09:01:54 +00:00
_logger . DebugArchiveDirStart ( subDir ) ;
2022-02-09 18:49:52 +00:00
foreach ( var enumerateFile in Directory . EnumerateFiles ( subDir , "*" , SearchOption . AllDirectories ) )
{
var f = enumerateFile ;
2022-03-25 16:26:06 +00:00
if ( ! WorkContext . IsMono & & enumerateFile . Length > MaxLength )
2021-08-31 09:40:28 +00:00
{
2022-02-09 18:49:52 +00:00
f = @"\\?\" + f ;
2021-08-31 09:40:28 +00:00
}
2020-09-11 14:12:35 +00:00
2022-02-09 18:49:52 +00:00
using ( var tmpFile = new FileStream ( f , FileMode . OpenOrCreate , FileAccess . ReadWrite , FileShare . Read , 4096 , FileOptions . DeleteOnClose ) )
2020-05-20 15:14:44 +00:00
{
2022-02-09 18:49:52 +00:00
writer . WriteEntry ( enumerateFile . Substring ( subDir . Length ) , tmpFile ) ;
2020-05-20 15:14:44 +00:00
}
SetStepCompleted ( ) ;
}
2022-05-26 09:01:54 +00:00
_logger . DebugArchiveDirEnd ( subDir ) ;
2022-02-09 18:49:52 +00:00
}
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
private List < IGrouping < string , BackupFileInfo > > GetFilesGroup ( )
{
2022-03-09 17:15:51 +00:00
var files = GetFilesToProcess ( TenantId ) . ToList ( ) ;
var exclude = BackupRecordContext . Backups . AsQueryable ( ) . Where ( b = > b . TenantId = = TenantId & & b . StorageType = = 0 & & b . StoragePath ! = null ) . ToList ( ) ;
files = files . Where ( f = > ! exclude . Any ( e = > f . Path . Replace ( '\\' , '/' ) . Contains ( $"/file_{e.StoragePath}/" ) ) ) . ToList ( ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
return files . GroupBy ( file = > file . Module ) . ToList ( ) ;
}
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
private void DoBackupModule ( IDataWriteOperator writer , IModuleSpecifics module )
{
2022-05-26 09:01:54 +00:00
_logger . DebugBeginSavingDataForModule ( module . ModuleName ) ;
2022-04-14 19:23:57 +00:00
var tablesToProcess = module . Tables . Where ( t = > ! _ignoredTables . Contains ( t . Name ) & & t . InsertMethod ! = InsertMethod . None ) . ToList ( ) ;
2022-02-09 18:49:52 +00:00
var tablesCount = tablesToProcess . Count ;
var tablesProcessed = 0 ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
using ( var connection = DbFactory . OpenConnection ( ) )
2020-05-20 15:14:44 +00:00
{
2022-02-09 18:49:52 +00:00
foreach ( var table in tablesToProcess )
2020-05-20 15:14:44 +00:00
{
2022-05-26 09:01:54 +00:00
_logger . DebugBeginLoadTable ( table . Name ) ;
2022-02-09 18:49:52 +00:00
using ( var data = new DataTable ( table . Name ) )
2020-05-20 15:14:44 +00:00
{
2022-02-09 18:49:52 +00:00
ActionInvoker . Try (
state = >
2020-05-20 15:14:44 +00:00
{
2022-02-09 18:49:52 +00:00
data . Clear ( ) ;
int counts ;
var offset = 0 ;
do
{
var t = ( TableInfo ) state ;
var dataAdapter = DbFactory . CreateDataAdapter ( ) ;
dataAdapter . SelectCommand = module . CreateSelectCommand ( connection . Fix ( ) , TenantId , t , Limit , offset ) . WithTimeout ( 600 ) ;
counts = ( ( DbDataAdapter ) dataAdapter ) . Fill ( data ) ;
offset + = Limit ;
} while ( counts = = Limit ) ;
} ,
table ,
maxAttempts : 5 ,
onFailure : error = > { throw ThrowHelper . CantBackupTable ( table . Name , error ) ; } ,
2022-05-26 09:01:54 +00:00
onAttemptFailure : error = > _logger . WarningBackupAttemptFailure ( error ) ) ;
2022-02-09 18:49:52 +00:00
foreach ( var col in data . Columns . Cast < DataColumn > ( ) . Where ( col = > col . DataType = = typeof ( DateTime ) ) )
{
col . DateTimeMode = DataSetDateTime . Unspecified ;
}
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
module . PrepareData ( data ) ;
2020-05-20 15:14:44 +00:00
2022-05-26 09:01:54 +00:00
_logger . DebugEndLoadTable ( table . Name ) ;
2020-05-20 15:14:44 +00:00
2022-05-26 09:01:54 +00:00
_logger . DebugBeginSavingTable ( table . Name ) ;
2021-08-31 09:40:28 +00:00
2022-02-09 18:49:52 +00:00
using ( var file = _tempStream . Create ( ) )
{
data . WriteXml ( file , XmlWriteMode . WriteSchema ) ;
data . Clear ( ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
writer . WriteEntry ( KeyHelper . GetTableZipKey ( module , data . TableName ) , file ) ;
2020-05-20 15:14:44 +00:00
}
2022-05-26 09:01:54 +00:00
_logger . DebugEndSavingTable ( table . Name ) ;
2022-03-09 17:15:51 +00:00
}
SetCurrentStepProgress ( ( int ) ( + + tablesProcessed * 100 / ( double ) tablesCount ) ) ;
2020-05-20 15:14:44 +00:00
}
}
2022-05-26 09:01:54 +00:00
_logger . DebugEndSavingDataForModule ( module . ModuleName ) ;
2022-02-09 18:49:52 +00:00
}
private void DoBackupStorage ( IDataWriteOperator writer , List < IGrouping < string , BackupFileInfo > > fileGroups )
{
2022-05-26 09:01:54 +00:00
_logger . DebugBeginBackupStorage ( ) ;
2022-02-09 18:49:52 +00:00
foreach ( var group in fileGroups )
2020-05-20 15:14:44 +00:00
{
2022-02-09 18:49:52 +00:00
var filesProcessed = 0 ;
var filesCount = group . Count ( ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
foreach ( var file in group )
2020-05-20 15:14:44 +00:00
{
2022-02-09 18:49:52 +00:00
var storage = StorageFactory . GetStorage ( ConfigPath , TenantId . ToString ( ) , group . Key ) ;
var file1 = file ;
ActionInvoker . Try ( state = >
2020-05-20 15:14:44 +00:00
{
2022-03-09 17:15:51 +00:00
var f = ( BackupFileInfo ) state ;
using var fileStream = storage . GetReadStreamAsync ( f . Domain , f . Path ) . Result ;
2022-02-09 18:49:52 +00:00
writer . WriteEntry ( file1 . GetZipKey ( ) , fileStream ) ;
2022-05-26 09:01:54 +00:00
} , file , 5 , error = > _logger . WarningCanNotBackupFile ( file1 . Module , file1 . Path , error ) ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
SetCurrentStepProgress ( ( int ) ( + + filesProcessed * 100 / ( double ) filesCount ) ) ;
2020-05-20 15:14:44 +00:00
}
2022-02-09 18:49:52 +00:00
}
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
var restoreInfoXml = new XElement (
"storage_restore" ,
fileGroups
. SelectMany ( group = > group . Select ( file = > ( object ) file . ToXElement ( ) ) )
. ToArray ( ) ) ;
2020-05-20 15:14:44 +00:00
2022-02-09 18:49:52 +00:00
using ( var tmpFile = _tempStream . Create ( ) )
2020-05-26 08:52:47 +00:00
{
2022-02-09 18:49:52 +00:00
restoreInfoXml . WriteTo ( tmpFile ) ;
writer . WriteEntry ( KeyHelper . GetStorageRestoreInfoZipKey ( ) , tmpFile ) ;
2020-05-26 08:52:47 +00:00
}
2022-02-09 18:49:52 +00:00
2022-05-26 09:01:54 +00:00
_logger . DebugEndBackupStorage ( ) ;
2020-05-20 15:14:44 +00:00
}
2022-02-09 18:49:52 +00:00
2020-05-20 15:14:44 +00:00
}