From 84ad8502b3e3854289b31c7e2034aea7d2fffc19 Mon Sep 17 00:00:00 2001 From: Alexey Safronov Date: Mon, 16 Oct 2023 15:38:33 +0400 Subject: [PATCH 01/47] Fix running client translations tests --- run.translations.tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.translations.tests.sh b/run.translations.tests.sh index 82c630b53e..ced8d7652e 100755 --- a/run.translations.tests.sh +++ b/run.translations.tests.sh @@ -5,4 +5,4 @@ dir=$(builtin cd $rd/../; pwd) echo "Root directory:" $dir -dotnet test $dir/client/common/Tests/Frontend.Translations.Tests/Frontend.Translations.Tests.csproj --filter "TestCategory=Locales" -l:html --results-directory "$dir/TestsResults" --environment "BASE_DIR=$dir" \ No newline at end of file +dotnet test $dir/client/common/Tests/Frontend.Translations.Tests/Frontend.Translations.Tests.csproj --filter "TestCategory=Locales" -l:html --results-directory "$dir/TestsResults" --environment "BASE_DIR=$dir/client" \ No newline at end of file From 04d28e7b7357a757f3f178f91d35aa6a0c4d335b Mon Sep 17 00:00:00 2001 From: Alexey Golubev Date: Mon, 16 Oct 2023 17:57:50 +0300 Subject: [PATCH 02/47] Feature/doc space enterprise (#17) * Add Enterprise build and bundled prerequisites * Small fixes * Small fixes * Fix issue with Licenses * Revert changes with License_Redist.rtf --------- Co-authored-by: Nasrullo Nurullaev --- install/win/DocSpace.aip | 144 +++++++++++++++++++++++++++--------- install/win/build-batch.bat | 15 ++-- 2 files changed, 115 insertions(+), 44 deletions(-) diff --git a/install/win/DocSpace.aip b/install/win/DocSpace.aip index 5ff1cfcfac..0aaaa72022 100644 --- a/install/win/DocSpace.aip +++ b/install/win/DocSpace.aip @@ -50,7 +50,9 @@ + + @@ -58,7 +60,7 @@ - + @@ -210,8 +212,10 @@ + + @@ -264,6 +268,7 @@ + @@ -310,8 +315,8 @@ - - + + @@ -415,11 +420,13 @@ + + @@ -427,6 +434,7 @@ + @@ -474,8 +482,8 @@ - - + + @@ -501,6 +509,7 @@ + @@ -538,6 +547,7 @@ + @@ -1075,6 +1085,7 @@ + @@ -1188,6 +1199,8 @@ + + @@ -1207,16 +1220,16 @@ - - - - - - - - - - + + + + + + + + + + @@ -1241,7 +1254,7 @@ - + @@ -1271,6 +1284,7 @@ + @@ -1304,7 +1318,7 @@ - + @@ -1392,6 +1406,7 @@ + @@ -1402,26 +1417,49 @@ + + + + + + + + + + + + + + + + + + + + + + + @@ -1431,6 +1469,9 @@ + + + @@ -1524,47 +1565,80 @@ - - - - - - + + + + + - - - + + + + + + + + + + + + + + + + + - - - - - - - + + + + + + + + + + - + - + + + + + + + + + + + + + + + + + + diff --git a/install/win/build-batch.bat b/install/win/build-batch.bat index 631538a0b6..a6fc2dd2e8 100644 --- a/install/win/build-batch.bat +++ b/install/win/build-batch.bat @@ -101,14 +101,11 @@ IF "%SignBuild%"=="true" ( %AdvancedInstaller% /edit buildtools\install\win\DocSpace.aip /SetDigitalCertificateFile -file %onlyoffice_codesign_path% -password "%onlyoffice_codesign_password%" ) -%AdvancedInstaller% /rebuild buildtools\install\win\DocSpace.aip +:: Build DocSpace Community +%AdvancedInstaller% /rebuild buildtools\install\win\DocSpace.aip -buildslist DOCSPACE_COMMUNITY -REM echo ######## Build DocSpace Enterprise package ######## -%AdvancedInstaller% /edit buildtools\install\win\DocSpace.Enterprise.aip /SetVersion %BUILD_VERSION%.%BUILD_NUMBER% +:: Build DocSpace Enterprise +copy "buildtools\install\win\Resources\License_Enterprise.rtf" "buildtools\install\win\Resources\License.rtf" /y +copy "buildtools\install\win\Resources\License_Enterprise_Redist.rtf" "buildtools\install\win\Resources\License_Redist.rtf" /y -IF "%SignBuild%"=="true" ( -%AdvancedInstaller% /edit buildtools\install\win\DocSpace.Enterprise.aip /SetSig -%AdvancedInstaller% /edit buildtools\install\win\DocSpace.Enterprise.aip /SetDigitalCertificateFile -file %onlyoffice_codesign_path% -password "%onlyoffice_codesign_password%" -) - -%AdvancedInstaller% /rebuild buildtools\install\win\DocSpace.Enterprise.aip +%AdvancedInstaller% /rebuild buildtools\install\win\DocSpace.aip -buildslist DOCSPACE_ENTERPRISE From 5971f66ff6c1c0f1ed7dd651115e928a60c8b9f5 Mon Sep 17 00:00:00 2001 From: Nasrullo Nurullaev <61620246+nasrullonurullaev@users.noreply.github.com> Date: Tue, 17 Oct 2023 13:02:12 +0500 Subject: [PATCH 03/47] Fix issue with machinekey when updating DocSpace (#18) --- install/win/DocSpace.aip | 4 ++-- install/win/utils.vbs | 29 ++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/install/win/DocSpace.aip b/install/win/DocSpace.aip index 0aaaa72022..23cf0463a3 100644 --- a/install/win/DocSpace.aip +++ b/install/win/DocSpace.aip @@ -1079,7 +1079,7 @@ - + @@ -1254,7 +1254,6 @@ - @@ -1309,6 +1308,7 @@ + diff --git a/install/win/utils.vbs b/install/win/utils.vbs index d23642b75e..da8985eb0f 100644 --- a/install/win/utils.vbs +++ b/install/win/utils.vbs @@ -121,7 +121,34 @@ End Function Function SetMACHINEKEY On Error Resume Next - Session.Property("MACHINE_KEY") = RandomString( 12 ) + Dim strFilePath, strJSON, strPattern, objRegExp, objMatches + + ' Specify the path to your JSON file + strFilePath = Session.Property("APPDIR") & "config\appsettings.production.json" + + ' Read the JSON content + Set objFSO = CreateObject("Scripting.FileSystemObject") + Set objFile = objFSO.OpenTextFile(strFilePath, 1) + strJSON = objFile.ReadAll + objFile.Close + + ' Define the regular expression pattern to match the "machinekey" value + strPattern = """machinekey"": ""([^""]+)""" + + ' Create a regular expression object and execute the pattern on the JSON string + Set objRegExp = New RegExp + objRegExp.Global = False + objRegExp.IgnoreCase = True + objRegExp.Pattern = strPattern + + Set objMatches = objRegExp.Execute(strJSON) + + ' Check if a match was found + If objMatches.Count > 0 Then + Session.Property("MACHINE_KEY") = objMatches(0).Submatches(0) + Else + Session.Property("MACHINE_KEY") = RandomString(12) + End If End Function From 8773eeeb29ed3a8924c315902b95c56af7f9a770 Mon Sep 17 00:00:00 2001 From: Nasrullo Nurullaev <61620246+nasrullonurullaev@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:17:46 +0500 Subject: [PATCH 04/47] fix Bug 64516 - Fix issue with infinite logs in EventViewer (#19) --- install/win/build-batch.bat | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/install/win/build-batch.bat b/install/win/build-batch.bat index a6fc2dd2e8..b7a8b0f6c4 100644 --- a/install/win/build-batch.bat +++ b/install/win/build-batch.bat @@ -68,6 +68,10 @@ REM echo ######## Remove AWSTarget from nlog.config ######## del /f /q buildtools\install\win\Files\nginx\conf\onlyoffice-login.conf del /f /q buildtools\install\win\Files\nginx\conf\onlyoffice-story.conf +::Remove unused services from HealthCheck | Bug 64516 +%sed% -i "/\"Name\": \"ASC.ApiCache\"/,/{/d" buildtools\install\win\Files\services\ASC.Web.HealthChecks.UI\service\appsettings.json +%sed% -i "/\"Name\": \"ASC.ApiSystem\"/,/{/d" buildtools\install\win\Files\services\ASC.Web.HealthChecks.UI\service\appsettings.json + ::configure nuget.config copy "server\NuGet.config" . /y %sed% -i "s/\.nuget\\packages/server\\.nuget\\packages/g" NuGet.config From b2f2890fbcdd5463eb7fae427cafbb35bfc9901a Mon Sep 17 00:00:00 2001 From: Nasrullo Nurullaev <61620246+nasrullonurullaev@users.noreply.github.com> Date: Tue, 17 Oct 2023 16:41:21 +0500 Subject: [PATCH 05/47] Increase the generation of MACHINE_KEY length (#21) --- install/win/utils.vbs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/win/utils.vbs b/install/win/utils.vbs index da8985eb0f..7a02458a5c 100644 --- a/install/win/utils.vbs +++ b/install/win/utils.vbs @@ -147,7 +147,7 @@ Function SetMACHINEKEY If objMatches.Count > 0 Then Session.Property("MACHINE_KEY") = objMatches(0).Submatches(0) Else - Session.Property("MACHINE_KEY") = RandomString(12) + Session.Property("MACHINE_KEY") = RandomString(16) End If End Function From 0075961d5a0e163f4933425f43b386ca031c9952 Mon Sep 17 00:00:00 2001 From: Alexey Safronov Date: Wed, 18 Oct 2023 18:07:33 +0400 Subject: [PATCH 06/47] Added new python scripts --- build.backend.docker.py | 200 +++++++++++++++++++++++++++++++ install/common/build-services.py | 68 +++++++++++ requirements.txt | 22 ++++ 3 files changed, 290 insertions(+) create mode 100755 build.backend.docker.py create mode 100755 install/common/build-services.py create mode 100644 requirements.txt diff --git a/build.backend.docker.py b/build.backend.docker.py new file mode 100755 index 0000000000..f1bd5cec38 --- /dev/null +++ b/build.backend.docker.py @@ -0,0 +1,200 @@ +#!/usr/bin/python3 + +import os +import subprocess +import sys, getopt +import shutil +import netifaces as ni + +def help(): + # Display Help + print("Build and run backend and working environment. (Use 'yarn start' to run client -> https://github.com/ONLYOFFICE/DocSpace-client)") + print() + print("Syntax: available params [-h|f|s|c|d|]") + print("options:") + print("h Print this Help.") + print("f Force rebuild base images.") + print("s Run as SAAS otherwise as STANDALONE.") + print("c Run as COMMUNITY otherwise ENTERPRISE.") + print("d Run dnsmasq.") + print() + +rd = os.path.dirname(os.path.abspath(__file__)) +dir = os.path.abspath(os.path.join(rd, "..")) +dockerDir = os.path.join(dir, "buildtools", "install", "docker") +# local_ip = subprocess.check_output(["ipconfig", "getifaddr", "en0"], text=True) + +local_ip = ni.ifaddresses('en0')[ni.AF_INET][0]['addr'] + +doceditor = local_ip + ":5013" +login = f"{local_ip}:5011" +client = f"{local_ip}:5001" +portal_url = f"http://{local_ip}" + +force = False +dns = False +standalone = True +community = False + +migration_type = "STANDALONE" # SAAS +installation_type = "ENTERPRISE" +document_server_image_name = "onlyoffice/documentserver-de:latest" + +# Get the options +opts, args = getopt.getopt(sys.argv[1:], "hfscd") +for opt, arg in opts: + if opt == "-h": + help() + sys.exit() + elif opt == "-f": + force = arg if arg else True + elif opt == "-s": + standalone = arg if arg else False + elif opt == "-c": + community = arg if arg else True + elif opt == "-d": + dns = arg if arg else True + else: + print("Error: Invalid '-" + opt + "' option") + sys.exit() + +print("Run script directory:", dir) +print("Root directory:", dir) +print("Docker files root directory:", dockerDir) + +print() +print(f"SERVICE_DOCEDITOR: {doceditor}") +print(f"SERVICE_LOGIN: {login}") +print(f"SERVICE_CLIENT: {client}") +print(f"DOCSPACE_APP_URL: {portal_url}") + +print() +print("FORCE REBUILD BASE IMAGES:", force) +print("Run dnsmasq:", dns) + +if standalone == False: + migration_type = "SAAS" + +if community == True: + installation_type = "COMMUNITY" + document_server_image_name = "onlyoffice/documentserver:latest" + +print() +print("MIGRATION TYPE:", migration_type) +print("INSTALLATION TYPE:", installation_type) +print("DS image:", document_server_image_name) +print() + +# Stop all backend services +subprocess.call([os.path.join(dir, "buildtools", "start", "stop.backend.docker.sh")]) + +print("Run MySQL") + +arch_name = os.uname().machine + +existsnetwork = subprocess.check_output(["docker", "network", "ls"]).decode("utf-8").splitlines() +existsnetwork = [line.split()[1] for line in existsnetwork] + +if "onlyoffice" not in existsnetwork: + subprocess.call(["docker", "network", "create", "--driver", "bridge", "onlyoffice"]) + +if arch_name == "x86_64": + print("CPU Type: x86_64 -> run db.yml") + subprocess.call(["docker", "compose", "-f", os.path.join(dockerDir, "db.yml"), "up", "-d"]) +elif arch_name == "arm64": + print("CPU Type: arm64 -> run db.yml with arm64v8 image") + os.environ["MYSQL_IMAGE"] = "arm64v8/mysql:8.0.32-oracle" + subprocess.call(["docker", "compose", "-f", os.path.join(dockerDir, "db.yml"), "up", "-d"]) +else: + print("Error: Unknown CPU Type:", arch_name) + sys.exit(1) + +if dns == True: + print("Run local dns server") + os.environ["ROOT_DIR"] = dir + subprocess.call(["docker", "compose", "-f", os.path.join(dockerDir, "dnsmasq.yml"), "up", "-d"]) + +print("Clear publish folder") +shutil.rmtree(os.path.join(dir, "publish/services"), True) + +print("Build backend services (to 'publish/' folder)") +subprocess.call([os.path.join(dir, "buildtools", "install", "common", "build-services.py")]) + +def check_image(image_name): + return subprocess.check_output(f'docker images --format "{{.Repository}}:{{.Tag}}"', shell=True, text=True).__contains__(image_name) + +dotnet_image_name = "onlyoffice/4testing-docspace-dotnet-runtime" +dotnet_version = "dev" +dotnet_image = f"{dotnet_image_name}:{dotnet_version}" + +exists = check_image(dotnet_image) + +if not exists or force == True: + print("Build dotnet base image from source (apply new dotnet config)") + subprocess.call(["docker", "build", "-t", dotnet_image, "-f", os.path.join(dockerDir, "Dockerfile.runtime"), "--target", "dotnetrun", "."]) +else: + print(f"SKIP build {dotnet_image} (already exists)") + +node_image_name = "onlyoffice/4testing-docspace-nodejs-runtime" +node_version = "dev" +node_image = f"{node_image_name}:{node_version}" + +exists = check_image(node_image) + +if not exists or force == True: + print("Build nodejs base image from source") + subprocess.call(["docker", "build", "-t", node_image, "-f", os.path.join(dockerDir, "Dockerfile.runtime"), "--target", "noderun", "."]) +else: + print(f"SKIP build {node_image} (already exists)") + +proxy_image_name = "onlyoffice/4testing-docspace-proxy-runtime" +proxy_version = "dev" +proxy_image = f"{proxy_image_name}:{proxy_version}" + +exists = check_image(proxy_image) + +if not exists or force == True: + print("Build proxy base image from source (apply new nginx config)") + subprocess.call(["docker", "build", "-t", proxy_image, "-f", os.path.join(dockerDir, "Dockerfile.runtime"), "--target", "router", "."]) +else: + print(f"SKIP build {proxy_image} (already exists)") + +print("Run migration and services") + +os.environ["ENV_EXTENSION"] = "dev" +os.environ["INSTALLATION_TYPE"] = installation_type +os.environ["Baseimage_Dotnet_Run"] = "onlyoffice/4testing-docspace-dotnet-runtime:" + dotnet_version +os.environ["Baseimage_Nodejs_Run"] = "onlyoffice/4testing-docspace-nodejs-runtime:" + node_version +os.environ["Baseimage_Proxy_Run"] = "onlyoffice/4testing-docspace-proxy-runtime:" + proxy_version +os.environ["DOCUMENT_SERVER_IMAGE_NAME"] = document_server_image_name +os.environ["SERVICE_DOCEDITOR"] = doceditor +os.environ["SERVICE_LOGIN"] = login +os.environ["SERVICE_CLIENT"] = client +os.environ["ROOT_DIR"] = dir +os.environ["BUILD_PATH"] = "/var/www" +os.environ["SRC_PATH"] = os.path.join(dir, "publish/services") +os.environ["DATA_DIR"] = os.path.join(dir, "data") +os.environ["APP_URL_PORTAL"] = portal_url +os.environ["MIGRATION_TYPE"] = migration_type +subprocess.call(["docker-compose", "-f", os.path.join(dockerDir, "docspace.profiles.yml"), "-f", os.path.join(dockerDir, "docspace.overcome.yml"), "--profile", "migration-runner", "--profile", "backend-local", "up", "-d"]) + +print() +print("Run script directory:", dir) +print("Root directory:", dir) +print("Docker files root directory:", dockerDir) + +print() +print(f"SERVICE_DOCEDITOR: {doceditor}") +print(f"SERVICE_LOGIN: {login}") +print(f"SERVICE_CLIENT: {client}") +print(f"DOCSPACE_APP_URL: {portal_url}") + +print() +print("FORCE REBUILD BASE IMAGES:", force) +print("Run dnsmasq:", dns) + +print() +print("MIGRATION TYPE:", migration_type) +print("INSTALLATION TYPE:", installation_type) +print("DS image:", document_server_image_name) +print() diff --git a/install/common/build-services.py b/install/common/build-services.py new file mode 100755 index 0000000000..9e3d69c7e3 --- /dev/null +++ b/install/common/build-services.py @@ -0,0 +1,68 @@ +#!/usr/bin/python3 + +import os +import stat +import subprocess +import shutil + +SRC_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) +BUILD_PATH = os.path.join(SRC_PATH, "publish") + +print(f"SRC_PATH = {SRC_PATH}") +print(f"BUILD_PATH = {BUILD_PATH}") + +BACKEND_NODEJS_SERVICES = ["ASC.Socket.IO", "ASC.SsoAuth"] +BACKEND_DOTNETCORE_SERVICES = ["ASC.Files", "ASC.People", "ASC.Data.Backup", "ASC.Files.Service", "ASC.Notify", "ASC.Studio.Notify", "ASC.Web.Api", "ASC.Web.Studio", "ASC.Data.Backup.BackgroundTasks", "ASC.ClearEvents", "ASC.ApiSystem", "ASC.Web.HealthChecks.UI"] +SELF_CONTAINED = "false" +PUBLISH_CNF = "Debug" + +FRONTEND_BUILD_ARGS = "build" +FRONTEND_DEPLOY_ARGS = "deploy" +DEBUG_INFO_CHECK = "" +MIGRATION_CHECK = "true" +DOCKER_ENTRYPOINT="docker-entrypoint.py" +DOCKER_ENTRYPOINT_PATH = os.path.join(SRC_PATH, "buildtools", "install", "docker", DOCKER_ENTRYPOINT) + +if os.path.exists(os.path.join(BUILD_PATH, "services")): + print("== Clean up services ==") + shutil.rmtree(os.path.join(BUILD_PATH, "services")) + +print("== Build ASC.Web.slnf ==") +subprocess.run(["dotnet", "build", os.path.join(SRC_PATH, "server", "ASC.Web.slnf")]) + +print("== Build ASC.Migrations.sln ==") +subprocess.run(["dotnet", "build", os.path.join(SRC_PATH, "server", "ASC.Migrations.sln"), "-o", os.path.join(BUILD_PATH, "services", "ASC.Migration.Runner", "service")]) + +print("== Add docker-migration-entrypoint.sh to ASC.Migration.Runner ==") +file_path = os.path.join(BUILD_PATH, "services", "ASC.Migration.Runner", "service", "docker-migration-entrypoint.sh") + +with open(os.path.join(SRC_PATH, "buildtools", "install", "docker", "docker-migration-entrypoint.sh"), "r") as file: + content = file.read().replace("\r", "") + with open(file_path, "w") as new_file: + new_file.write(content) + +st = os.stat(file_path) +os.chmod(file_path, st.st_mode | stat.S_IEXEC) + +for service in BACKEND_NODEJS_SERVICES: + print(f"== Build {service} project ==") + src = os.path.join(SRC_PATH, "server", "common", service) + subprocess.run(["yarn", "install", "--cwd", src]) + + dst = os.path.join(BUILD_PATH, "services", service, "service") + if not os.path.exists(dst): + os.makedirs(dst, exist_ok=True) + + print(f"== Copy service data to {dst}") + shutil.copytree(src, dst, dirs_exist_ok=True) + + print(f"== Add docker-entrypoint.py to {service}") + shutil.copyfile(DOCKER_ENTRYPOINT_PATH, os.path.join(dst, DOCKER_ENTRYPOINT)) + +print("== Publish ASC.Web.slnf ==") +subprocess.run(["dotnet", "publish", os.path.join(SRC_PATH, "server", "ASC.Web.slnf"), "-p", "PublishProfile=FolderProfile"]) + +for service in BACKEND_DOTNETCORE_SERVICES: + print(f"== Add {DOCKER_ENTRYPOINT} to {service}") + dst = os.path.join(BUILD_PATH, "services", service, "service") + shutil.copyfile(DOCKER_ENTRYPOINT_PATH, os.path.join(dst, DOCKER_ENTRYPOINT)) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..2a04f9a7f5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,22 @@ +aiohttp==3.8.4 +aiosignal==1.3.1 +APScheduler==3.6.3 +async-timeout==4.0.2 +attrs==23.1.0 +cachetools==4.2.2 +certifi==2023.5.7 +charset-normalizer==3.2.0 +frozenlist==1.3.3 +idna==3.4 +multidict==6.0.4 +netifaces==0.11.0 +openai==0.27.8 +python-telegram-bot==13.6 +pytz==2023.3 +requests==2.31.0 +six==1.16.0 +tornado==6.3.2 +tqdm==4.65.0 +tzlocal==5.0.1 +urllib3==2.0.3 +yarl==1.9.2 From b820deabc9cff91d7b52a04587424fd7b13fe4c5 Mon Sep 17 00:00:00 2001 From: Alexey Safronov Date: Thu, 19 Oct 2023 12:08:35 +0400 Subject: [PATCH 07/47] Replaced netifaces --- build.backend.docker.py | 4 ++-- requirements.txt | 22 ---------------------- 2 files changed, 2 insertions(+), 24 deletions(-) delete mode 100644 requirements.txt diff --git a/build.backend.docker.py b/build.backend.docker.py index f1bd5cec38..ab88f8a665 100755 --- a/build.backend.docker.py +++ b/build.backend.docker.py @@ -1,10 +1,10 @@ #!/usr/bin/python3 import os +import socket import subprocess import sys, getopt import shutil -import netifaces as ni def help(): # Display Help @@ -24,7 +24,7 @@ dir = os.path.abspath(os.path.join(rd, "..")) dockerDir = os.path.join(dir, "buildtools", "install", "docker") # local_ip = subprocess.check_output(["ipconfig", "getifaddr", "en0"], text=True) -local_ip = ni.ifaddresses('en0')[ni.AF_INET][0]['addr'] +local_ip = socket.gethostbyname_ex(socket.gethostname())[-1][-1] doceditor = local_ip + ":5013" login = f"{local_ip}:5011" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 2a04f9a7f5..0000000000 --- a/requirements.txt +++ /dev/null @@ -1,22 +0,0 @@ -aiohttp==3.8.4 -aiosignal==1.3.1 -APScheduler==3.6.3 -async-timeout==4.0.2 -attrs==23.1.0 -cachetools==4.2.2 -certifi==2023.5.7 -charset-normalizer==3.2.0 -frozenlist==1.3.3 -idna==3.4 -multidict==6.0.4 -netifaces==0.11.0 -openai==0.27.8 -python-telegram-bot==13.6 -pytz==2023.3 -requests==2.31.0 -six==1.16.0 -tornado==6.3.2 -tqdm==4.65.0 -tzlocal==5.0.1 -urllib3==2.0.3 -yarl==1.9.2 From fc7561d4e76890f6f490d88919c37350691dd769 Mon Sep 17 00:00:00 2001 From: Alexey Safronov Date: Thu, 19 Oct 2023 12:55:30 +0400 Subject: [PATCH 08/47] Added stop dev docker python script --- build.backend.docker.py | 2 +- start/stop.backend.docker.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100755 start/stop.backend.docker.py diff --git a/build.backend.docker.py b/build.backend.docker.py index ab88f8a665..90a59a63e0 100755 --- a/build.backend.docker.py +++ b/build.backend.docker.py @@ -86,7 +86,7 @@ print("DS image:", document_server_image_name) print() # Stop all backend services -subprocess.call([os.path.join(dir, "buildtools", "start", "stop.backend.docker.sh")]) +subprocess.call([os.path.join(dir, "buildtools", "start", "stop.backend.docker.py")]) print("Run MySQL") diff --git a/start/stop.backend.docker.py b/start/stop.backend.docker.py new file mode 100755 index 0000000000..85759dab84 --- /dev/null +++ b/start/stop.backend.docker.py @@ -0,0 +1,24 @@ +#!/usr/bin/python3 + +import subprocess + +# Execute command to find and filter containers +containers = subprocess.check_output(['docker', 'ps', '-a', '-f', 'label=com.docker.compose.project=docker', '--format={{.ID}}'], text=True).rstrip().split('\n') + +print(containers) + +#containers = [line.split(' ')[0] for line in output.split('\n') if line and not any(keyword in line for keyword in ["mysql", "rabbitmq", "redis", "elasticsearch", "documentserver"])] + +if not containers: + print("No containers to stop") + exit() + +print("Stop all backend services (containers)") +for c in containers: + if not c: + continue + + subprocess.run(['docker', 'stop', c]) + +#print("Stop all backend services (containers)") +#subprocess.run(['docker', 'stop', containers]) From 41cb7ab9a73df5b3a404870fa846b5985aa0ce3f Mon Sep 17 00:00:00 2001 From: Alexey Safronov Date: Thu, 19 Oct 2023 14:07:50 +0400 Subject: [PATCH 09/47] Change getting arch_name --- build.backend.docker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.backend.docker.py b/build.backend.docker.py index 90a59a63e0..76a2e98462 100755 --- a/build.backend.docker.py +++ b/build.backend.docker.py @@ -5,6 +5,7 @@ import socket import subprocess import sys, getopt import shutil +import platform def help(): # Display Help @@ -90,7 +91,9 @@ subprocess.call([os.path.join(dir, "buildtools", "start", "stop.backend.docker.p print("Run MySQL") -arch_name = os.uname().machine +arch_name = platform.uname().machine + +print(f"PLATFORM {arch_name}") existsnetwork = subprocess.check_output(["docker", "network", "ls"]).decode("utf-8").splitlines() existsnetwork = [line.split()[1] for line in existsnetwork] From 80f1ecea84b8ed41e642fbc2c22658480c030f12 Mon Sep 17 00:00:00 2001 From: Ilya Oleshko Date: Thu, 19 Oct 2023 14:27:45 +0300 Subject: [PATCH 10/47] Added start/stop/restart docker containers runner on python --- start/restart.backend.docker.py | 16 ++++++++++++++++ start/start.backend.docker.py | 15 +++++++++++++++ start/stop.backend.docker.py | 27 ++++++++++----------------- 3 files changed, 41 insertions(+), 17 deletions(-) create mode 100644 start/restart.backend.docker.py create mode 100644 start/start.backend.docker.py diff --git a/start/restart.backend.docker.py b/start/restart.backend.docker.py new file mode 100644 index 0000000000..534fc24a60 --- /dev/null +++ b/start/restart.backend.docker.py @@ -0,0 +1,16 @@ +#!/usr/bin/python3 + +import subprocess +import time +import os + +rd = os.path.dirname(__file__) + +start = time.time() + +print("Restart all backend services (containers)") +subprocess.run(["python", os.path.join(rd, "stop.backend.docker.py")]) +subprocess.run(["python", os.path.join(rd, "start.backend.docker.py")]) + +end = time.time() +print("\nElapsed time", end - start) \ No newline at end of file diff --git a/start/start.backend.docker.py b/start/start.backend.docker.py new file mode 100644 index 0000000000..6e994af59f --- /dev/null +++ b/start/start.backend.docker.py @@ -0,0 +1,15 @@ +#!/usr/bin/python3 + +import subprocess +import time + +start = time.time() +container_ids = subprocess.check_output("docker ps -aq -f label=com.docker.compose.project=docker", encoding='utf-8') +containers = container_ids.strip().split() + +if containers: + print("Start all backend services (containers)") + subprocess.run(['docker', 'start'] + containers, check=True) + +end = time.time() +print("\nElapsed time", end - start) \ No newline at end of file diff --git a/start/stop.backend.docker.py b/start/stop.backend.docker.py index 85759dab84..4666b59c67 100755 --- a/start/stop.backend.docker.py +++ b/start/stop.backend.docker.py @@ -1,24 +1,17 @@ #!/usr/bin/python3 import subprocess +import time -# Execute command to find and filter containers -containers = subprocess.check_output(['docker', 'ps', '-a', '-f', 'label=com.docker.compose.project=docker', '--format={{.ID}}'], text=True).rstrip().split('\n') +start = time.time() +container_ids = subprocess.check_output("docker ps -q -f label=com.docker.compose.project=docker", encoding='utf-8') +containers = container_ids.strip().split() -print(containers) - -#containers = [line.split(' ')[0] for line in output.split('\n') if line and not any(keyword in line for keyword in ["mysql", "rabbitmq", "redis", "elasticsearch", "documentserver"])] - -if not containers: +if containers: + print("Stop all backend services (containers)") + subprocess.run(['docker', 'stop'] + containers, check=True) +else: print("No containers to stop") - exit() -print("Stop all backend services (containers)") -for c in containers: - if not c: - continue - - subprocess.run(['docker', 'stop', c]) - -#print("Stop all backend services (containers)") -#subprocess.run(['docker', 'stop', containers]) +end = time.time() +print("\nElapsed time", end - start) From ca504f945200ec4116fecf7f5ff34ceda3ae80a4 Mon Sep 17 00:00:00 2001 From: Alexey Safronov Date: Thu, 19 Oct 2023 17:11:37 +0400 Subject: [PATCH 11/47] Fix running on macOS --- start/start.backend.docker.py | 2 +- start/stop.backend.docker.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/start/start.backend.docker.py b/start/start.backend.docker.py index 6e994af59f..7ba4b52ac8 100644 --- a/start/start.backend.docker.py +++ b/start/start.backend.docker.py @@ -4,7 +4,7 @@ import subprocess import time start = time.time() -container_ids = subprocess.check_output("docker ps -aq -f label=com.docker.compose.project=docker", encoding='utf-8') +container_ids = subprocess.check_output(["docker", "ps", "-aq", "-f", "label=com.docker.compose.project=docker"], encoding='utf-8') containers = container_ids.strip().split() if containers: diff --git a/start/stop.backend.docker.py b/start/stop.backend.docker.py index 4666b59c67..574c0746a5 100755 --- a/start/stop.backend.docker.py +++ b/start/stop.backend.docker.py @@ -4,7 +4,7 @@ import subprocess import time start = time.time() -container_ids = subprocess.check_output("docker ps -q -f label=com.docker.compose.project=docker", encoding='utf-8') +container_ids = subprocess.check_output(["docker", "ps", "-q", "-f", "label=com.docker.compose.project=docker"], encoding='utf-8') containers = container_ids.strip().split() if containers: From 70ca81008ba075d6413466f37e512cc5dd700f6f Mon Sep 17 00:00:00 2001 From: Alexey Safronov Date: Thu, 19 Oct 2023 17:12:38 +0400 Subject: [PATCH 12/47] Added clear.backend.docker.py --- clear.backend.docker.py | 52 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100755 clear.backend.docker.py diff --git a/clear.backend.docker.py b/clear.backend.docker.py new file mode 100755 index 0000000000..0f6efa0abf --- /dev/null +++ b/clear.backend.docker.py @@ -0,0 +1,52 @@ +#!/usr/bin/python3 + +import os, sys +import subprocess + +rd = os.path.dirname(os.path.abspath(__file__)) +root_dir = os.path.abspath(os.path.join(rd, "..")) +docker_dir = os.path.join(root_dir, "buildtools", "install", "docker") + +containers = subprocess.check_output(["docker", "ps", "-aq", "-f", "name=^onlyoffice"], encoding='utf-8').strip().split() +images = subprocess.check_output(["docker", "images", "onlyoffice/4testing-docspace*", "-q"], encoding='utf-8').strip().split() + +if containers or images: + print("Clean up containers, volumes or networks") + + print("Remove all backend containers") + + env_vars = { + "DOCUMENT_SERVER_IMAGE_NAME": "onlyoffice/documentserver-de:latest", + "Baseimage_Dotnet_Run": "onlyoffice/4testing-docspace-dotnet-runtime:dev", + "Baseimage_Nodejs_Run": "onlyoffice/4testing-docspace-nodejs-runtime:dev", + "Baseimage_Proxy_Run": "onlyoffice/4testing-docspace-proxy-runtime:dev", + "SERVICE_CLIENT": "localhost:5001", + "BUILD_PATH": "/var/www", + "SRC_PATH": os.path.join(root_dir, "publish", "services"), + "ROOT_DIR": root_dir, + "DATA_DIR": os.path.join(root_dir, "data") + } + + compose_command = f"docker compose -f {os.path.join(docker_dir, "docspace.profiles.yml")} -f {os.path.join(docker_dir, "docspace.overcome.yml")} --profile \"migration-runner\" --profile \"backend-local\" down --volumes" + subprocess.run(compose_command, shell=True, env=env_vars) + + print("Remove docker contatiners 'mysql'") + db_command = f"docker compose -f {os.path.join(docker_dir, 'db.yml')} down --volumes" + subprocess.run(db_command, shell=True) + + print("Remove docker volumes") + volumes_command = f"docker volume prune -fa" + subprocess.run(volumes_command, shell=True) + + print("Remove docker base images (onlyoffice/4testing-docspace)") + subprocess.run(['docker', 'rmi', '-f'] + images, check=True) + + print("Remove docker networks") + network_command = f"docker network prune -f" + subprocess.run(network_command, shell=True) + + print("Remove docker build cache") + cache_command = f"docker buildx prune -f" + subprocess.run(cache_command, shell=True) +else: + print("No containers or images to clean up") From cc8c1c72cea36254ba2df0f0f1a55fdacdf9e5a8 Mon Sep 17 00:00:00 2001 From: Alexey Safronov Date: Thu, 19 Oct 2023 17:27:43 +0400 Subject: [PATCH 13/47] Replaced call to run --- build.backend.docker.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/build.backend.docker.py b/build.backend.docker.py index 76a2e98462..38349e6bad 100755 --- a/build.backend.docker.py +++ b/build.backend.docker.py @@ -23,11 +23,9 @@ def help(): rd = os.path.dirname(os.path.abspath(__file__)) dir = os.path.abspath(os.path.join(rd, "..")) dockerDir = os.path.join(dir, "buildtools", "install", "docker") -# local_ip = subprocess.check_output(["ipconfig", "getifaddr", "en0"], text=True) - local_ip = socket.gethostbyname_ex(socket.gethostname())[-1][-1] -doceditor = local_ip + ":5013" +doceditor = f"{local_ip}:5013" login = f"{local_ip}:5011" client = f"{local_ip}:5001" portal_url = f"http://{local_ip}" @@ -87,7 +85,7 @@ print("DS image:", document_server_image_name) print() # Stop all backend services -subprocess.call([os.path.join(dir, "buildtools", "start", "stop.backend.docker.py")]) +subprocess.run([os.path.join(dir, "buildtools", "start", "stop.backend.docker.py")]) print("Run MySQL") @@ -99,15 +97,15 @@ existsnetwork = subprocess.check_output(["docker", "network", "ls"]).decode("utf existsnetwork = [line.split()[1] for line in existsnetwork] if "onlyoffice" not in existsnetwork: - subprocess.call(["docker", "network", "create", "--driver", "bridge", "onlyoffice"]) + subprocess.run(["docker", "network", "create", "--driver", "bridge", "onlyoffice"]) if arch_name == "x86_64": print("CPU Type: x86_64 -> run db.yml") - subprocess.call(["docker", "compose", "-f", os.path.join(dockerDir, "db.yml"), "up", "-d"]) + subprocess.run(["docker", "compose", "-f", os.path.join(dockerDir, "db.yml"), "up", "-d"]) elif arch_name == "arm64": print("CPU Type: arm64 -> run db.yml with arm64v8 image") os.environ["MYSQL_IMAGE"] = "arm64v8/mysql:8.0.32-oracle" - subprocess.call(["docker", "compose", "-f", os.path.join(dockerDir, "db.yml"), "up", "-d"]) + subprocess.run(["docker", "compose", "-f", os.path.join(dockerDir, "db.yml"), "up", "-d"]) else: print("Error: Unknown CPU Type:", arch_name) sys.exit(1) @@ -115,13 +113,13 @@ else: if dns == True: print("Run local dns server") os.environ["ROOT_DIR"] = dir - subprocess.call(["docker", "compose", "-f", os.path.join(dockerDir, "dnsmasq.yml"), "up", "-d"]) + subprocess.run(["docker", "compose", "-f", os.path.join(dockerDir, "dnsmasq.yml"), "up", "-d"]) print("Clear publish folder") shutil.rmtree(os.path.join(dir, "publish/services"), True) print("Build backend services (to 'publish/' folder)") -subprocess.call([os.path.join(dir, "buildtools", "install", "common", "build-services.py")]) +subprocess.run([os.path.join(dir, "buildtools", "install", "common", "build-services.py")]) def check_image(image_name): return subprocess.check_output(f'docker images --format "{{.Repository}}:{{.Tag}}"', shell=True, text=True).__contains__(image_name) @@ -134,7 +132,7 @@ exists = check_image(dotnet_image) if not exists or force == True: print("Build dotnet base image from source (apply new dotnet config)") - subprocess.call(["docker", "build", "-t", dotnet_image, "-f", os.path.join(dockerDir, "Dockerfile.runtime"), "--target", "dotnetrun", "."]) + subprocess.run(["docker", "build", "-t", dotnet_image, "-f", os.path.join(dockerDir, "Dockerfile.runtime"), "--target", "dotnetrun", "."]) else: print(f"SKIP build {dotnet_image} (already exists)") @@ -146,7 +144,7 @@ exists = check_image(node_image) if not exists or force == True: print("Build nodejs base image from source") - subprocess.call(["docker", "build", "-t", node_image, "-f", os.path.join(dockerDir, "Dockerfile.runtime"), "--target", "noderun", "."]) + subprocess.run(["docker", "build", "-t", node_image, "-f", os.path.join(dockerDir, "Dockerfile.runtime"), "--target", "noderun", "."]) else: print(f"SKIP build {node_image} (already exists)") @@ -158,7 +156,7 @@ exists = check_image(proxy_image) if not exists or force == True: print("Build proxy base image from source (apply new nginx config)") - subprocess.call(["docker", "build", "-t", proxy_image, "-f", os.path.join(dockerDir, "Dockerfile.runtime"), "--target", "router", "."]) + subprocess.run(["docker", "build", "-t", proxy_image, "-f", os.path.join(dockerDir, "Dockerfile.runtime"), "--target", "router", "."]) else: print(f"SKIP build {proxy_image} (already exists)") @@ -179,7 +177,7 @@ os.environ["SRC_PATH"] = os.path.join(dir, "publish/services") os.environ["DATA_DIR"] = os.path.join(dir, "data") os.environ["APP_URL_PORTAL"] = portal_url os.environ["MIGRATION_TYPE"] = migration_type -subprocess.call(["docker-compose", "-f", os.path.join(dockerDir, "docspace.profiles.yml"), "-f", os.path.join(dockerDir, "docspace.overcome.yml"), "--profile", "migration-runner", "--profile", "backend-local", "up", "-d"]) +subprocess.run(["docker-compose", "-f", os.path.join(dockerDir, "docspace.profiles.yml"), "-f", os.path.join(dockerDir, "docspace.overcome.yml"), "--profile", "migration-runner", "--profile", "backend-local", "up", "-d"]) print() print("Run script directory:", dir) From 5cb03f3ba58dd0e666601d77eea8fa580d39d8b0 Mon Sep 17 00:00:00 2001 From: Alexey Safronov Date: Thu, 19 Oct 2023 17:28:24 +0400 Subject: [PATCH 14/47] Fix run on Windows --- clear.backend.docker.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/clear.backend.docker.py b/clear.backend.docker.py index 0f6efa0abf..b4a0d92fca 100755 --- a/clear.backend.docker.py +++ b/clear.backend.docker.py @@ -15,20 +15,16 @@ if containers or images: print("Remove all backend containers") - env_vars = { - "DOCUMENT_SERVER_IMAGE_NAME": "onlyoffice/documentserver-de:latest", - "Baseimage_Dotnet_Run": "onlyoffice/4testing-docspace-dotnet-runtime:dev", - "Baseimage_Nodejs_Run": "onlyoffice/4testing-docspace-nodejs-runtime:dev", - "Baseimage_Proxy_Run": "onlyoffice/4testing-docspace-proxy-runtime:dev", - "SERVICE_CLIENT": "localhost:5001", - "BUILD_PATH": "/var/www", - "SRC_PATH": os.path.join(root_dir, "publish", "services"), - "ROOT_DIR": root_dir, - "DATA_DIR": os.path.join(root_dir, "data") - } - - compose_command = f"docker compose -f {os.path.join(docker_dir, "docspace.profiles.yml")} -f {os.path.join(docker_dir, "docspace.overcome.yml")} --profile \"migration-runner\" --profile \"backend-local\" down --volumes" - subprocess.run(compose_command, shell=True, env=env_vars) + os.environ["Baseimage_Dotnet_Run"] = "onlyoffice/4testing-docspace-dotnet-runtime:dev" + os.environ["Baseimage_Nodejs_Run"] = "onlyoffice/4testing-docspace-nodejs-runtime:dev" + os.environ["Baseimage_Proxy_Run"] = "onlyoffice/4testing-docspace-proxy-runtime:dev" + os.environ["DOCUMENT_SERVER_IMAGE_NAME"] = "onlyoffice/documentserver-de:latest" + os.environ["SERVICE_CLIENT"] = "localhost:5001" + os.environ["ROOT_DIR"] = root_dir + os.environ["BUILD_PATH"] = "/var/www" + os.environ["SRC_PATH"] = os.path.join(root_dir, "publish/services") + os.environ["DATA_DIR"] = os.path.join(root_dir, "data") + subprocess.run(["docker-compose", "-f", os.path.join(docker_dir, "docspace.profiles.yml"), "-f", os.path.join(docker_dir, "docspace.overcome.yml"), "--profile", "migration-runner", "--profile", "backend-local", "down", "--volumes"]) print("Remove docker contatiners 'mysql'") db_command = f"docker compose -f {os.path.join(docker_dir, 'db.yml')} down --volumes" From 38dd71539bd1c435652a85666d8c64f4cc4ee4f2 Mon Sep 17 00:00:00 2001 From: Andrey Savihin Date: Fri, 20 Oct 2023 14:47:57 +0300 Subject: [PATCH 15/47] Cfg: added UserNameRegex --- config/appsettings.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/config/appsettings.json b/config/appsettings.json index 1096f8727c..3549e9b8ea 100644 --- a/config/appsettings.json +++ b/config/appsettings.json @@ -34,7 +34,10 @@ "oidc": { "authority": "" }, - "server-root": "" + "server-root": "", + "username": { + "regex": "^[\p{L}\p{M}' \-]+$" + } }, "license": { "file": { From bc9f3f5c07f1a1f9358503118101c5c19cd5a80d Mon Sep 17 00:00:00 2001 From: Andrey Savihin Date: Fri, 20 Oct 2023 16:31:19 +0300 Subject: [PATCH 16/47] Cfg: fixed UserNameRegex --- config/appsettings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/appsettings.json b/config/appsettings.json index 3549e9b8ea..175514a086 100644 --- a/config/appsettings.json +++ b/config/appsettings.json @@ -36,7 +36,7 @@ }, "server-root": "", "username": { - "regex": "^[\p{L}\p{M}' \-]+$" + "regex": "^[\\p{L}\\p{M}' \\-]+$" } }, "license": { From 2b112d934473228abc0c208a86e417d54d18f0a7 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Fri, 20 Oct 2023 17:25:21 +0300 Subject: [PATCH 17/47] Store: added IDataStoreValidator --- config/storage.json | 1 + 1 file changed, 1 insertion(+) diff --git a/config/storage.json b/config/storage.json index 6b9faa8ba6..62245ec012 100644 --- a/config/storage.json +++ b/config/storage.json @@ -41,6 +41,7 @@ "data": "00000000-0000-0000-0000-000000000000", "type": "disc", "path": "$STORAGE_ROOT\\Products\\Files\\logos\\{0}", + "validatorType": "ASC.Files.Core.VirtualRooms.RoomLogoValidator, ASC.Files.Core", "domain": [ { "name": "logos_temp", From c6fb634e699db38150669f958fac77a036f92390 Mon Sep 17 00:00:00 2001 From: Nasrullo Nurullaev <61620246+nasrullonurullaev@users.noreply.github.com> Date: Mon, 23 Oct 2023 19:02:20 +0500 Subject: [PATCH 18/47] fix Bug 64405 - Fix issue with starting services (#27) * fix Bug 64405 - Fix issue with starting services * Small fixes --- install/win/tools/DocEditor.xml | 1 + install/win/tools/Login.xml | 1 + install/win/tools/Socket.IO.xml | 1 + install/win/tools/SsoAuth.xml | 1 + 4 files changed, 4 insertions(+) diff --git a/install/win/tools/DocEditor.xml b/install/win/tools/DocEditor.xml index 99b8557128..ea5eff2e8f 100644 --- a/install/win/tools/DocEditor.xml +++ b/install/win/tools/DocEditor.xml @@ -4,6 +4,7 @@ ONLYOFFICE DocSpace DocEditor RealTime Automatic + false node "{APPDIR}products\ASC.Files\editor\server.js" diff --git a/install/win/tools/Login.xml b/install/win/tools/Login.xml index 82299d38e0..379a6961c2 100644 --- a/install/win/tools/Login.xml +++ b/install/win/tools/Login.xml @@ -4,6 +4,7 @@ ONLYOFFICE DocSpace Login RealTime Automatic + false node "{APPDIR}products\ASC.Login\login\server.js" diff --git a/install/win/tools/Socket.IO.xml b/install/win/tools/Socket.IO.xml index be40e5d953..8377604e7b 100644 --- a/install/win/tools/Socket.IO.xml +++ b/install/win/tools/Socket.IO.xml @@ -4,6 +4,7 @@ ONLYOFFICE DocSpace Socket.IO RealTime Automatic + false node "{APPDIR}services\ASC.Socket.IO\service\server.js" diff --git a/install/win/tools/SsoAuth.xml b/install/win/tools/SsoAuth.xml index 6d46ff54d3..c14532ae75 100644 --- a/install/win/tools/SsoAuth.xml +++ b/install/win/tools/SsoAuth.xml @@ -4,6 +4,7 @@ ONLYOFFICE DocSpace ASC.SsoAuth.Svc RealTime Automatic + false node "{APPDIR}services\ASC.SsoAuth\service\app.js" From 103e93dbab846e5e69716c9e9071faef4a67fee7 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Tue, 24 Oct 2023 18:58:28 +0300 Subject: [PATCH 19/47] fix Bug 64808 --- config/appsettings.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/appsettings.json b/config/appsettings.json index 175514a086..930211cf6b 100644 --- a/config/appsettings.json +++ b/config/appsettings.json @@ -418,7 +418,8 @@ "script": ["'self'", "'unsafe-inline'", "'unsafe-eval'"], "style": ["'self'", "'unsafe-inline'"], "img": ["'self'", "data:", "blob:"], - "frame": ["'self'"] + "frame": ["'self'"], + "fonts": ["'self'", "data:"] }, "zendesk": { "def": ["*.zdassets.com", "*.zopim.com", "*.zendesk.com", "wss:"], From b8cdde4db0f8ae1b0c0ef90fc2db605536732360 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Tue, 24 Oct 2023 20:01:16 +0300 Subject: [PATCH 20/47] fix Bug 64787 --- config/appsettings.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/appsettings.json b/config/appsettings.json index 930211cf6b..78d87d805f 100644 --- a/config/appsettings.json +++ b/config/appsettings.json @@ -431,8 +431,8 @@ "def": ["*.googleapis.com"] }, "oform": { - "img": ["*.onlyoffice.com"], - "def": ["*.onlyoffice.com"] + "img": ["static-oforms.teamlab.info"], + "def": ["cmsoforms.teamlab.info"] } }, "logocolors": [ From 330cffc88b0bac2a0b9009d794bc46d238523a6d Mon Sep 17 00:00:00 2001 From: Danil Titarenko <77471369+danilapog@users.noreply.github.com> Date: Fri, 27 Oct 2023 12:09:56 +0300 Subject: [PATCH 21/47] Fix docker daemon sed modifier (#26) --- install/OneClickInstall/install-Docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/OneClickInstall/install-Docker.sh b/install/OneClickInstall/install-Docker.sh index 519d082620..1a6289bbd6 100644 --- a/install/OneClickInstall/install-Docker.sh +++ b/install/OneClickInstall/install-Docker.sh @@ -947,7 +947,7 @@ domain_check () { echo "Select 'N' to cancel ${PACKAGE_SYSNAME^^} ${PRODUCT_NAME} installation." if read_continue_installation; then if [[ -f "$DOCKER_DAEMON_FILE" ]]; then - sed -i '/{/a\ "dns": ["8.8.8.8", "8.8.4.4"],' "$DOCKER_DAEMON_FILE" + sed -i 's!{!& "dns": ["8.8.8.8", "8.8.4.4"],!' "$DOCKER_DAEMON_FILE" else echo "{\"dns\": [\"8.8.8.8\", \"8.8.4.4\"]}" | tee "$DOCKER_DAEMON_FILE" >/dev/null fi From cca908fe0f29456f06be64f11bb0df89e90715f2 Mon Sep 17 00:00:00 2001 From: Nasrullo Nurullaev <61620246+nasrullonurullaev@users.noreply.github.com> Date: Fri, 27 Oct 2023 14:31:22 +0500 Subject: [PATCH 22/47] fix Bug 64516 - Fix issue with healthcheck logs (#29) --- install/docker/docker-healthchecks-entrypoint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install/docker/docker-healthchecks-entrypoint.sh b/install/docker/docker-healthchecks-entrypoint.sh index fecc4a3d88..04dd04ee20 100755 --- a/install/docker/docker-healthchecks-entrypoint.sh +++ b/install/docker/docker-healthchecks-entrypoint.sh @@ -31,6 +31,7 @@ STUDIO_NOTIFY_HOST=${STUDIO_NOTIFY_HOST:-"${CONTAINER_PREFIX}studio-notify:${SER API_HOST=${API_HOST:-"${CONTAINER_PREFIX}api:${SERVICE_PORT}"} STUDIO_HOST=${STUDIO_HOST:-"${CONTAINER_PREFIX}studio:${SERVICE_PORT}"} +sed -i "/\"Name\": \"ASC.ApiCache\"/,/{/d" ${PATH_TO_CONF}/appsettings.json sed -i "s!localhost:5010!${API_SYSTEM_HOST}!g" ${PATH_TO_CONF}/appsettings.json sed -i "s!localhost:5012!${BACKUP_HOST}!g" ${PATH_TO_CONF}/appsettings.json sed -i "s!localhost:5032!${BACKUP_BACKGRUOND_TASKS_HOST}!g" ${PATH_TO_CONF}/appsettings.json From 39e235633ef87ba32d6e33b177954abfe46988a5 Mon Sep 17 00:00:00 2001 From: Alexey Safronov Date: Sat, 28 Oct 2023 12:26:00 +0400 Subject: [PATCH 23/47] Create debuginfo.py --- debuginfo.py | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100755 debuginfo.py diff --git a/debuginfo.py b/debuginfo.py new file mode 100755 index 0000000000..fabeb90e30 --- /dev/null +++ b/debuginfo.py @@ -0,0 +1,97 @@ +#!/usr/bin/python3 + +import os +from git import Repo +from datetime import datetime + +rd = os.path.dirname(os.path.abspath(__file__)) +root_dir = os.path.abspath(os.path.join(rd, "..")) + +CLIENT = "client" +SERVER = "server" +BUILDTOOLS = "buildtools" +REPO_CLIENT_URL = f"https://github.com/ONLYOFFICE/DocSpace-client" +REPO_SERVER_URL = f"https://github.com/ONLYOFFICE/DocSpace-server" +REPO_BUILDTOOLS_URL = f"https://github.com/ONLYOFFICE/DocSpace-buildtools" +LIMIT_DAYS = 30 +MESSAGE_SEPARATOR = '__MESSAGE_SEPARATOR__' +SEP = '§' + +# https://git-scm.com/docs/pretty-formats +format = f"%H{SEP}%as{SEP}%an{SEP}%s %b{MESSAGE_SEPARATOR}" + +data = {} + + +def fetchCommits(url, type): + path = os.path.join(root_dir, type) + #print(path, os.path.exists(path)) + if os.path.exists(path) == False: + print("Error folder does not exists", path) + return + + repo = Repo(path) + + info = f"|{type.upper()}|{repo.active_branch.name} |{repo.head.commit}|{os.linesep}" + + commits_str = repo.git.log(f"--pretty=format: {format}", "--no-merges", f"--since={LIMIT_DAYS}.days") + #print(commits_str) + + commits = commits_str.strip().split(MESSAGE_SEPARATOR) + #print(commits) + + for item in commits: + elements = item.replace('\n', '').split(SEP) + + if len(elements) != 4: + continue + + hash = elements[0].strip() + date = datetime.strptime(elements[1].strip(), "%Y-%m-%d") + name = elements[2].strip() + text = elements[3].strip().capitalize() + + if date not in data: + data[date] = {} + + if name not in data[date]: + data[date][name] = [] + + data[date][name].append(f"- [{type}]: {text} {hash[0:7]}") + + return info + +result = f"## Changelog{os.linesep}" + +result += f"| Repo | Branch | Last Commit |{os.linesep}" +result += f"| :--- | :--- | :--- |{os.linesep}" + +result += fetchCommits(REPO_CLIENT_URL, CLIENT) +result += fetchCommits(REPO_SERVER_URL, SERVER) +result += fetchCommits(REPO_BUILDTOOLS_URL, BUILDTOOLS) + +# Create debuginfo.md content +for date in sorted(data, reverse=True): + niceDate = date.strftime("%d %B %Y") + result += f"### {niceDate}{os.linesep}" + for name in sorted(data[date]): + result += f"#### {name}{os.linesep}" + for commit in data[date][name]: + result += f"{commit}{os.linesep}" + +print(result) + +pathMD = os.path.join(root_dir, CLIENT, "public/debuginfo.md") +# Open text file in write mode +text_file = open(pathMD, "w") + +# Write content to file +n = text_file.write(result) + +if n == len(result): + print("Success! String written to text file.") +else: + print("Failure! String not written to text file.") + +# Close file +text_file.close() \ No newline at end of file From 94a91bfdb73146c29ce685c9082f51bc39a4f357 Mon Sep 17 00:00:00 2001 From: Alexey Safronov Date: Sat, 28 Oct 2023 12:33:35 +0400 Subject: [PATCH 24/47] Create requirements.txt --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..edc51ee8c8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +gitdb==4.0.11 +GitPython==3.1.40 +smmap==5.0.1 From 6b7222443a4efc541778c71c8d0f3a5cab953997 Mon Sep 17 00:00:00 2001 From: Alexey Safronov Date: Sat, 28 Oct 2023 14:50:57 +0400 Subject: [PATCH 25/47] Added highlight --- debuginfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debuginfo.py b/debuginfo.py index fabeb90e30..9991d8948b 100755 --- a/debuginfo.py +++ b/debuginfo.py @@ -57,7 +57,7 @@ def fetchCommits(url, type): if name not in data[date]: data[date][name] = [] - data[date][name].append(f"- [{type}]: {text} {hash[0:7]}") + data[date][name].append(f"- [{type}]: {text} `{hash[0:7]}`") return info From de2d9d12544c360c7a55979e3339a59ee3a2da84 Mon Sep 17 00:00:00 2001 From: Alexey Safronov Date: Sat, 28 Oct 2023 15:47:00 +0400 Subject: [PATCH 26/47] Refactoring table and links --- debuginfo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debuginfo.py b/debuginfo.py index 9991d8948b..a6fc574dcd 100755 --- a/debuginfo.py +++ b/debuginfo.py @@ -32,7 +32,7 @@ def fetchCommits(url, type): repo = Repo(path) - info = f"|{type.upper()}|{repo.active_branch.name} |{repo.head.commit}|{os.linesep}" + info = f"| [DocSpace-{type}]({url}) | [{repo.active_branch.name}]({url}/tree/{repo.active_branch.name}) | [{repo.head.commit}]({url}/commit/{repo.head.commit}) |{os.linesep}" commits_str = repo.git.log(f"--pretty=format: {format}", "--no-merges", f"--since={LIMIT_DAYS}.days") #print(commits_str) @@ -57,14 +57,14 @@ def fetchCommits(url, type): if name not in data[date]: data[date][name] = [] - data[date][name].append(f"- [{type}]: {text} `{hash[0:7]}`") + data[date][name].append(f"- [{type}]: {text} [`{hash[0:7]}`]({url}/commit/{hash})") return info result = f"## Changelog{os.linesep}" result += f"| Repo | Branch | Last Commit |{os.linesep}" -result += f"| :--- | :--- | :--- |{os.linesep}" +result += f"| :- | :- | :- |{os.linesep}" result += fetchCommits(REPO_CLIENT_URL, CLIENT) result += fetchCommits(REPO_SERVER_URL, SERVER) From 6230df5c40350b28ffad5e7140f28b3414597337 Mon Sep 17 00:00:00 2001 From: pavelbannov Date: Mon, 30 Oct 2023 20:33:01 +0300 Subject: [PATCH 27/47] fix Bug 64869 --- config/appsettings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/appsettings.json b/config/appsettings.json index 78d87d805f..178cee0333 100644 --- a/config/appsettings.json +++ b/config/appsettings.json @@ -432,7 +432,7 @@ }, "oform": { "img": ["static-oforms.teamlab.info"], - "def": ["cmsoforms.teamlab.info"] + "def": ["cmsoforms.teamlab.info", "oforms.teamlab.info"] } }, "logocolors": [ From b5242f5b5346f3e94bb691d63fe9a11ab9bb7320 Mon Sep 17 00:00:00 2001 From: Sergey Kirichenko Date: Tue, 31 Oct 2023 15:08:54 +0300 Subject: [PATCH 28/47] Add debuginfo into docker image --- install/common/build-frontend.sh | 6 ++++-- install/docker/Dockerfile.app | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/install/common/build-frontend.sh b/install/common/build-frontend.sh index 9304bae8f2..a5f74e9910 100644 --- a/install/common/build-frontend.sh +++ b/install/common/build-frontend.sh @@ -57,10 +57,12 @@ done echo "== FRONT-END-BUILD ==" cd ${SRC_PATH} -yarn install # debug config if [ "$DEBUG_INFO" = true ]; then - yarn debug-info + pip install -r ${SRC_PATH}/buildtools/requirements.txt + python3 ${SRC_PATH}/buildtools/debuginfo.py fi +cd ${SRC_PATH}/client +yarn install yarn ${BUILD_ARGS} yarn ${DEPLOY_ARGS} diff --git a/install/docker/Dockerfile.app b/install/docker/Dockerfile.app index 0a2869c6e4..5ee20f103f 100644 --- a/install/docker/Dockerfile.app +++ b/install/docker/Dockerfile.app @@ -28,6 +28,7 @@ RUN apt-get -y update && \ sudo \ locales \ git \ + python3-pip \ npm && \ locale-gen en_US.UTF-8 && \ npm install --global yarn && \ @@ -54,7 +55,7 @@ RUN cd ${SRC_PATH} && \ sed -i "s/\"number\".*,/\"number\": \"${PRODUCT_VERSION}.${BUILD_NUMBER}\",/g" /app/onlyoffice/config/appsettings.json && \ sed -e 's/#//' -i /etc/nginx/conf.d/onlyoffice.conf && \ cd ${SRC_PATH}/buildtools/install/common/ && \ - bash build-frontend.sh -sp "${SRC_PATH}/client" -ba "${BUILD_ARGS}" -da "${DEPLOY_ARGS}" -di "${DEBUG_INFO}" && \ + bash build-frontend.sh -sp "${SRC_PATH}" -ba "${BUILD_ARGS}" -da "${DEPLOY_ARGS}" -di "${DEBUG_INFO}" && \ bash build-backend.sh -sp "${SRC_PATH}/server" && \ bash publish-backend.sh -pc "${PUBLISH_CNF}" -sp "${SRC_PATH}/server" -bp "${BUILD_PATH}" && \ cp -rf ${SRC_PATH}/server/products/ASC.Files/Server/DocStore ${BUILD_PATH}/products/ASC.Files/server/ && \ From 7e05d72b015dc9d723161afb26a4e1c310f9c625 Mon Sep 17 00:00:00 2001 From: Nasrullo Nurullaev Date: Wed, 1 Nov 2023 11:02:05 +0500 Subject: [PATCH 29/47] Fix issue with installing vcredist --- install/win/DocSpace.aip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/win/DocSpace.aip b/install/win/DocSpace.aip index 23cf0463a3..55aa9d4cf8 100644 --- a/install/win/DocSpace.aip +++ b/install/win/DocSpace.aip @@ -1583,7 +1583,7 @@ - + From 0fc70ed192b6e8d014c3d9ff5504d08bafbdc579 Mon Sep 17 00:00:00 2001 From: Evgeniy Antonyuk Date: Wed, 1 Nov 2023 14:05:30 +0700 Subject: [PATCH 30/47] Fix default value in redis\elastic port description --- install/OneClickInstall/install-Docker.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/OneClickInstall/install-Docker.sh b/install/OneClickInstall/install-Docker.sh index 1a6289bbd6..06159901e1 100644 --- a/install/OneClickInstall/install-Docker.sh +++ b/install/OneClickInstall/install-Docker.sh @@ -491,9 +491,9 @@ while [ "$1" != "" ]; do echo " -ies, --installelastic install or update elasticsearch (true|false)" echo " -espr, --elasticprotocol the protocol for the connection to elasticsearch (default value http)" echo " -esh, --elastichost the IP address or hostname of the elasticsearch" - echo " -esp, --elasticport elasticsearch port number (default value 6379)" + echo " -esp, --elasticport elasticsearch port number (default value 9200)" echo " -rdsh, --redishost the IP address or hostname of the redis server" - echo " -rdsp, --redisport redis server port number (default value 9200)" + echo " -rdsp, --redisport redis server port number (default value 6379)" echo " -rdsu, --redisusername redis user name" echo " -rdspass, --redispassword password set for redis account" echo " -rbth, --rabbitmqhost the IP address or hostname of the rabbitmq server" From 7eeb155b4eea37bdfaf8bfad1f07d52d965f8ff1 Mon Sep 17 00:00:00 2001 From: Evgeniy Antonyuk Date: Wed, 1 Nov 2023 16:46:54 +0700 Subject: [PATCH 31/47] fix Bug 64628 - Fix log level for error `failed to get redis key` --- config/nginx/onlyoffice.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/nginx/onlyoffice.conf b/config/nginx/onlyoffice.conf index 9a0b7c30d7..8041d92980 100644 --- a/config/nginx/onlyoffice.conf +++ b/config/nginx/onlyoffice.conf @@ -113,7 +113,7 @@ server { if redis_pass ~= "" then local res, err = red:auth(redis_pass) if not res then - ngx.log(ngx.ERR, "failed to authenticate: ", err) + ngx.log(ngx.INFO, "failed to authenticate: ", err) return end end From 06b5facf1fcc47dd3e28b1d6f8003f1d4cd61a94 Mon Sep 17 00:00:00 2001 From: Evgeniy Antonyuk Date: Wed, 1 Nov 2023 16:52:05 +0700 Subject: [PATCH 32/47] fix Bug 64628 - Fix error `cycle while evaluating variable public_root` --- install/deb/debian/rules | 1 + install/rpm/SPECS/build.spec | 1 + 2 files changed, 2 insertions(+) diff --git a/install/deb/debian/rules b/install/deb/debian/rules index d86143429a..dacb7ecb8f 100755 --- a/install/deb/debian/rules +++ b/install/deb/debian/rules @@ -50,6 +50,7 @@ override_dh_auto_build: sed 's_\(minlevel=\)"[^"]*"_\1"Warn"_g' -i ${SRC_PATH}/config/nlog.config sed 's_etc/nginx_etc/openresty_g' -i ${SRC_PATH}/config/nginx/*.conf + sed -i 's_$public_root_/var/www/%{product}/public/_' config/nginx/onlyoffice.conf sed 's/teamlab.info/onlyoffice.com/g' -i ${SRC_PATH}/config/autofac.consumers.json json -I -f ${SRC_PATH}/client/public/scripts/config.json -e "this.wrongPortalNameUrl=\"\"" sed -e 's/$$router_host/127.0.0.1/g' -e 's/the_host/host/g' -e 's/the_scheme/scheme/g' -e 's_includes_/etc/openresty/includes_g' -i ${SRC_PATH}/buildtools/install/docker/config/nginx/onlyoffice-proxy*.conf diff --git a/install/rpm/SPECS/build.spec b/install/rpm/SPECS/build.spec index 630636104b..de9bcbf3c3 100644 --- a/install/rpm/SPECS/build.spec +++ b/install/rpm/SPECS/build.spec @@ -26,6 +26,7 @@ sed 's_\(minlevel=\)"[^"]*"_\1"Warn"_g' -i config/nlog.config sed 's/teamlab.info/onlyoffice.com/g' -i config/autofac.consumers.json sed 's_etc/nginx_etc/openresty_g' -i config/nginx/*.conf +sed -i 's_$public_root_/var/www/%{product}/public/_' config/nginx/onlyoffice.conf sed -e 's/$router_host/127.0.0.1/g' -e 's/the_host/host/g' -e 's/the_scheme/scheme/g' -e 's_includes_/etc/openresty/includes_g' -i install/docker/config/nginx/onlyoffice-proxy*.conf sed -e '/.pid/d' -e '/temp_path/d' -e 's_etc/nginx_etc/openresty_g' -e 's/\.log/-openresty.log/g' -i install/docker/config/nginx/templates/nginx.conf.template sed -i "s_\(.*root\).*;_\1 \"/var/www/%{product}\";_g" -i install/docker/config/nginx/letsencrypt.conf From 1e00e705564430fef07a4c6b18a33ed79475406f Mon Sep 17 00:00:00 2001 From: "anton.sukhorukov" Date: Wed, 1 Nov 2023 15:48:14 +0300 Subject: [PATCH 33/47] fix backup.json --- config/backup.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/config/backup.json b/config/backup.json index c3dc289bd2..6e3acf6dc2 100644 --- a/config/backup.json +++ b/config/backup.json @@ -15,8 +15,6 @@ "Cleaner": { "Period":"00:15:00" - }, - "ChunkSize": 524288000, - "MaxLocalSize": 1048576000 + } } } \ No newline at end of file From dce1a62cf98c253bf578c59e260f4304af584634 Mon Sep 17 00:00:00 2001 From: Ilya Oleshko Date: Thu, 2 Nov 2023 18:11:21 +0300 Subject: [PATCH 34/47] Fixed commands for win --- build.backend.docker.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build.backend.docker.py b/build.backend.docker.py index 38349e6bad..7692da1530 100755 --- a/build.backend.docker.py +++ b/build.backend.docker.py @@ -85,7 +85,7 @@ print("DS image:", document_server_image_name) print() # Stop all backend services -subprocess.run([os.path.join(dir, "buildtools", "start", "stop.backend.docker.py")]) +subprocess.run(["python", os.path.join(dir, "buildtools", "start", "stop.backend.docker.py")]) print("Run MySQL") @@ -99,7 +99,7 @@ existsnetwork = [line.split()[1] for line in existsnetwork] if "onlyoffice" not in existsnetwork: subprocess.run(["docker", "network", "create", "--driver", "bridge", "onlyoffice"]) -if arch_name == "x86_64": +if arch_name == "x86_64" or arch_name == "AMD64": print("CPU Type: x86_64 -> run db.yml") subprocess.run(["docker", "compose", "-f", os.path.join(dockerDir, "db.yml"), "up", "-d"]) elif arch_name == "arm64": @@ -119,10 +119,10 @@ print("Clear publish folder") shutil.rmtree(os.path.join(dir, "publish/services"), True) print("Build backend services (to 'publish/' folder)") -subprocess.run([os.path.join(dir, "buildtools", "install", "common", "build-services.py")]) +subprocess.run(["python", os.path.join(dir, "buildtools", "install", "common", "build-services.py")]) def check_image(image_name): - return subprocess.check_output(f'docker images --format "{{.Repository}}:{{.Tag}}"', shell=True, text=True).__contains__(image_name) + return subprocess.check_output(["docker", "images", "--format", "'{{.Repository}}:{{.Tag}}'"], shell=True, text=True).__contains__(image_name) dotnet_image_name = "onlyoffice/4testing-docspace-dotnet-runtime" dotnet_version = "dev" From 0f159f8568f488de687216f40d97c84abc72f00c Mon Sep 17 00:00:00 2001 From: Nasrullo Nurullaev Date: Fri, 3 Nov 2023 09:46:27 +0500 Subject: [PATCH 35/47] Add license path fo DS --- install/win/DocSpace.aip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/win/DocSpace.aip b/install/win/DocSpace.aip index 55aa9d4cf8..dd09d89220 100644 --- a/install/win/DocSpace.aip +++ b/install/win/DocSpace.aip @@ -1571,7 +1571,7 @@ - + From 7c744d9b2a0209c1188a8c014823acc757af947c Mon Sep 17 00:00:00 2001 From: Alexey Safronov Date: Fri, 3 Nov 2023 21:30:29 +0400 Subject: [PATCH 36/47] Speed up build deploy nodejs services --- install/common/build-services.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/install/common/build-services.py b/install/common/build-services.py index 9e3d69c7e3..fcf56725fa 100755 --- a/install/common/build-services.py +++ b/install/common/build-services.py @@ -44,17 +44,28 @@ with open(os.path.join(SRC_PATH, "buildtools", "install", "docker", "docker-migr st = os.stat(file_path) os.chmod(file_path, st.st_mode | stat.S_IEXEC) +format = "zip" + for service in BACKEND_NODEJS_SERVICES: print(f"== Build {service} project ==") src = os.path.join(SRC_PATH, "server", "common", service) - subprocess.run(["yarn", "install", "--cwd", src]) + subprocess.run(["yarn", "install"], cwd=src) dst = os.path.join(BUILD_PATH, "services", service, "service") if not os.path.exists(dst): os.makedirs(dst, exist_ok=True) - print(f"== Copy service data to {dst}") - shutil.copytree(src, dst, dirs_exist_ok=True) + archive = os.path.join(BUILD_PATH, "services", service, f"service.{format}") + + print("Make service archive", archive) + + shutil.make_archive(root_dir=src, format=format, base_name=dst) + + print("Unpack service archive", archive) + shutil.unpack_archive(archive, dst) + + print("Remove service archive", archive) + os.remove(archive) print(f"== Add docker-entrypoint.py to {service}") shutil.copyfile(DOCKER_ENTRYPOINT_PATH, os.path.join(dst, DOCKER_ENTRYPOINT)) From eabddf8efb16ada84f72d5eff6b8701af7c2fb32 Mon Sep 17 00:00:00 2001 From: Alexey Bannov Date: Sun, 5 Nov 2023 18:36:21 +0300 Subject: [PATCH 37/47] nlog: ignore /health/ requests --- config/nlog.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/nlog.config b/config/nlog.config index f4522c0b2d..e6dfd6ee1a 100644 --- a/config/nlog.config +++ b/config/nlog.config @@ -46,12 +46,12 @@ - + - + From da9dcc8139c158e8c60211d795ed873202804a8b Mon Sep 17 00:00:00 2001 From: Nasrullo Nurullaev <61620246+nasrullonurullaev@users.noreply.github.com> Date: Tue, 7 Nov 2023 12:56:46 +0500 Subject: [PATCH 38/47] Change prerequisites selection for installing (#40) --- install/win/DocSpace.aip | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/install/win/DocSpace.aip b/install/win/DocSpace.aip index dd09d89220..d108500bd7 100644 --- a/install/win/DocSpace.aip +++ b/install/win/DocSpace.aip @@ -1566,20 +1566,20 @@ - + - + - - + + - - - + + + @@ -1589,7 +1589,7 @@ - + @@ -1597,7 +1597,7 @@ - + From cce220f3fda85283726563fe931b66506e9581c2 Mon Sep 17 00:00:00 2001 From: Ilya Oleshko Date: Tue, 7 Nov 2023 12:05:23 +0300 Subject: [PATCH 39/47] Fixed windows shell output at build-services.py FileNotFoundError: [WinError 2] --- install/common/build-services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/common/build-services.py b/install/common/build-services.py index fcf56725fa..1460a2960f 100755 --- a/install/common/build-services.py +++ b/install/common/build-services.py @@ -49,7 +49,7 @@ format = "zip" for service in BACKEND_NODEJS_SERVICES: print(f"== Build {service} project ==") src = os.path.join(SRC_PATH, "server", "common", service) - subprocess.run(["yarn", "install"], cwd=src) + subprocess.run(["yarn", "install"], cwd=src, shell=True) dst = os.path.join(BUILD_PATH, "services", service, "service") if not os.path.exists(dst): From c85d82d4d2bf5d9d680f05adc821252a39b0edc4 Mon Sep 17 00:00:00 2001 From: Alexey Bannov Date: Tue, 7 Nov 2023 13:09:44 +0300 Subject: [PATCH 40/47] added AWS DynamoDB connection string --- config/appsettings.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/appsettings.json b/config/appsettings.json index 178cee0333..1ab6f09de4 100644 --- a/config/appsettings.json +++ b/config/appsettings.json @@ -466,6 +466,11 @@ "region": "us-east-1", "logGroupName": "/asc/docspace/cluster/cluster_name/general", "logStreamName": "${hostname} - ${applicationContext} - ${date} - ${guid}" + }, + "dynamoDB": { + "accessKeyId": "", + "secretAccessKey": "", + "region": "us-east-1" } } } From 7f798a934c04d32cf60dc9cad8f8fac6702b9957 Mon Sep 17 00:00:00 2001 From: Ilya Oleshko Date: Tue, 7 Nov 2023 14:06:12 +0300 Subject: [PATCH 41/47] Fixed issue with end of line on windows --- install/common/build-services.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/install/common/build-services.py b/install/common/build-services.py index 1460a2960f..4ac75c2a72 100755 --- a/install/common/build-services.py +++ b/install/common/build-services.py @@ -4,6 +4,7 @@ import os import stat import subprocess import shutil +import time SRC_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) BUILD_PATH = os.path.join(SRC_PATH, "publish") @@ -35,11 +36,18 @@ subprocess.run(["dotnet", "build", os.path.join(SRC_PATH, "server", "ASC.Migrati print("== Add docker-migration-entrypoint.sh to ASC.Migration.Runner ==") file_path = os.path.join(BUILD_PATH, "services", "ASC.Migration.Runner", "service", "docker-migration-entrypoint.sh") +src_file_path = os.path.join(SRC_PATH, "buildtools", "install", "docker", "docker-migration-entrypoint.sh") -with open(os.path.join(SRC_PATH, "buildtools", "install", "docker", "docker-migration-entrypoint.sh"), "r") as file: - content = file.read().replace("\r", "") - with open(file_path, "w") as new_file: - new_file.write(content) +WINDOWS_LINE_ENDING = b'\r\n' +UNIX_LINE_ENDING = b'\n' + +with open(src_file_path, 'rb') as open_file: + content = open_file.read() + +content = content.replace(WINDOWS_LINE_ENDING, UNIX_LINE_ENDING) + +with open(file_path, 'wb') as open_file: + open_file.write(content) st = os.stat(file_path) os.chmod(file_path, st.st_mode | stat.S_IEXEC) @@ -58,11 +66,16 @@ for service in BACKEND_NODEJS_SERVICES: archive = os.path.join(BUILD_PATH, "services", service, f"service.{format}") print("Make service archive", archive) - + start = time.time() shutil.make_archive(root_dir=src, format=format, base_name=dst) + end = time.time() + print(f"Took {(end-start)*1000.0} ms") print("Unpack service archive", archive) + start = time.time() shutil.unpack_archive(archive, dst) + end = time.time() + print(f"Took {(end-start)*1000.0} ms") print("Remove service archive", archive) os.remove(archive) From fa0f105b2b1802e4d9dd6a1e992281606f4c851c Mon Sep 17 00:00:00 2001 From: Nasrullo Nurullaev <61620246+nasrullonurullaev@users.noreply.github.com> Date: Tue, 7 Nov 2023 18:08:25 +0500 Subject: [PATCH 42/47] Add versions to prerequisites name (#41) --- install/win/DocSpace.aip | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/install/win/DocSpace.aip b/install/win/DocSpace.aip index d108500bd7..da79c53627 100644 --- a/install/win/DocSpace.aip +++ b/install/win/DocSpace.aip @@ -1569,7 +1569,7 @@ - + @@ -1577,8 +1577,8 @@ - - + + @@ -1589,15 +1589,15 @@ - - + + - + From 50442a67ae41b6a3253e5e356e501d88e5ad45e7 Mon Sep 17 00:00:00 2001 From: Sergey Kirichenko Date: Tue, 7 Nov 2023 17:26:26 +0300 Subject: [PATCH 43/47] Modify build yarn version (#42) * Fix yarn build * Add for Web.Studio eventsBus --- install/common/build-backend.sh | 5 +++-- install/docker/Dockerfile.app | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/install/common/build-backend.sh b/install/common/build-backend.sh index 87fdcbcd97..88d7715d73 100644 --- a/install/common/build-backend.sh +++ b/install/common/build-backend.sh @@ -42,10 +42,11 @@ done echo "== BACK-END-BUILD ==" -cd ${SRC_PATH} +cd ${SRC_PATH}/server dotnet build ASC.Web.slnf ${ARGS} dotnet build ASC.Migrations.sln -o ${SRC_PATH}/ASC.Migration.Runner/service/ +cd ${SRC_PATH}/client # Array of names backend services in directory common (Nodejs) services_name_backend_nodejs=() services_name_backend_nodejs+=(ASC.Socket.IO) @@ -54,5 +55,5 @@ services_name_backend_nodejs+=(ASC.SsoAuth) # Build backend services (Nodejs) for i in ${!services_name_backend_nodejs[@]}; do echo "== Build ${services_name_backend_nodejs[$i]} project ==" - yarn install --cwd common/${services_name_backend_nodejs[$i]} --frozen-lockfile + yarn install --cwd ${SRC_PATH}/server/common/${services_name_backend_nodejs[$i]} --frozen-lockfile done diff --git a/install/docker/Dockerfile.app b/install/docker/Dockerfile.app index 5ee20f103f..21d7eaa815 100644 --- a/install/docker/Dockerfile.app +++ b/install/docker/Dockerfile.app @@ -56,7 +56,7 @@ RUN cd ${SRC_PATH} && \ sed -e 's/#//' -i /etc/nginx/conf.d/onlyoffice.conf && \ cd ${SRC_PATH}/buildtools/install/common/ && \ bash build-frontend.sh -sp "${SRC_PATH}" -ba "${BUILD_ARGS}" -da "${DEPLOY_ARGS}" -di "${DEBUG_INFO}" && \ - bash build-backend.sh -sp "${SRC_PATH}/server" && \ + bash build-backend.sh -sp "${SRC_PATH}" && \ bash publish-backend.sh -pc "${PUBLISH_CNF}" -sp "${SRC_PATH}/server" -bp "${BUILD_PATH}" && \ cp -rf ${SRC_PATH}/server/products/ASC.Files/Server/DocStore ${BUILD_PATH}/products/ASC.Files/server/ && \ rm -rf ${SRC_PATH}/server/common/* && \ @@ -312,7 +312,7 @@ WORKDIR ${BUILD_PATH}/studio/ASC.Web.Studio/ COPY --chown=onlyoffice:onlyoffice docker-entrypoint.py ./docker-entrypoint.py COPY --from=base --chown=onlyoffice:onlyoffice ${BUILD_PATH}/services/ASC.Web.Studio/service/ . -CMD ["ASC.Web.Studio.dll", "ASC.Web.Studio"] +CMD ["ASC.Web.Studio.dll", "ASC.Web.Studio", "core:eventBus:subscriptionClientName=asc_event_bus_webstudio_queue"] ## ASC.Web.HealthChecks.UI ## FROM dotnetrun AS healthchecks From 4af0dc486d36884192b41a97e28b3470e6b386f1 Mon Sep 17 00:00:00 2001 From: Sergey Kirichenko Date: Tue, 7 Nov 2023 19:39:48 +0300 Subject: [PATCH 44/47] Add fix path for migration runner service build (#43) * Fix yarn build * Fix build migration path --- install/common/build-backend.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/common/build-backend.sh b/install/common/build-backend.sh index 88d7715d73..c748b3081a 100644 --- a/install/common/build-backend.sh +++ b/install/common/build-backend.sh @@ -44,7 +44,7 @@ echo "== BACK-END-BUILD ==" cd ${SRC_PATH}/server dotnet build ASC.Web.slnf ${ARGS} -dotnet build ASC.Migrations.sln -o ${SRC_PATH}/ASC.Migration.Runner/service/ +dotnet build ASC.Migrations.sln -o ${SRC_PATH}/server/ASC.Migration.Runner/service/ cd ${SRC_PATH}/client # Array of names backend services in directory common (Nodejs) From bb10d8e3c6c845a0477a84b16e08ca0e31978f6e Mon Sep 17 00:00:00 2001 From: Alexey Safronov Date: Wed, 8 Nov 2023 12:12:38 +0400 Subject: [PATCH 45/47] Update source archive path --- install/common/build-services.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install/common/build-services.py b/install/common/build-services.py index 4ac75c2a72..9d4c93399a 100755 --- a/install/common/build-services.py +++ b/install/common/build-services.py @@ -63,9 +63,10 @@ for service in BACKEND_NODEJS_SERVICES: if not os.path.exists(dst): os.makedirs(dst, exist_ok=True) + archive_src = os.path.join(SRC_PATH, "server", "common", service, f"service.{format}") archive = os.path.join(BUILD_PATH, "services", service, f"service.{format}") - print("Make service archive", archive) + print("Make service archive", archive_src) start = time.time() shutil.make_archive(root_dir=src, format=format, base_name=dst) end = time.time() From a24bbec130f45ff7e76a6a7f87ce34d8d5134e23 Mon Sep 17 00:00:00 2001 From: Alexey Safronov Date: Wed, 8 Nov 2023 18:26:04 +0400 Subject: [PATCH 46/47] =?UTF-8?q?Removed=20old=20scripts=20for=20d=D1=89?= =?UTF-8?q?=D1=81=D0=BB=D1=83=D0=BA=20dev?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.backend.docker.bat | 7 -- build.backend.docker.ps1 | 133 -------------------- build.backend.docker.sh | 203 ------------------------------ clear.backend.docker.bat | 7 -- clear.backend.docker.ps1 | 37 ------ clear.backend.docker.sh | 38 ------ install/common/build-services.ps1 | 57 --------- start/restart.backend.docker.bat | 7 -- start/restart.backend.docker.ps1 | 5 - start/restart.backend.docker.sh | 8 -- start/start.backend.docker.bat | 7 -- start/start.backend.docker.ps1 | 32 ----- start/start.backend.docker.sh | 41 ------ start/stop.backend.docker.bat | 7 -- start/stop.backend.docker.ps1 | 17 --- start/stop.backend.docker.sh | 4 - 16 files changed, 610 deletions(-) delete mode 100644 build.backend.docker.bat delete mode 100644 build.backend.docker.ps1 delete mode 100755 build.backend.docker.sh delete mode 100644 clear.backend.docker.bat delete mode 100644 clear.backend.docker.ps1 delete mode 100755 clear.backend.docker.sh delete mode 100644 install/common/build-services.ps1 delete mode 100644 start/restart.backend.docker.bat delete mode 100644 start/restart.backend.docker.ps1 delete mode 100755 start/restart.backend.docker.sh delete mode 100644 start/start.backend.docker.bat delete mode 100644 start/start.backend.docker.ps1 delete mode 100755 start/start.backend.docker.sh delete mode 100644 start/stop.backend.docker.bat delete mode 100644 start/stop.backend.docker.ps1 delete mode 100755 start/stop.backend.docker.sh diff --git a/build.backend.docker.bat b/build.backend.docker.bat deleted file mode 100644 index e538a525ed..0000000000 --- a/build.backend.docker.bat +++ /dev/null @@ -1,7 +0,0 @@ -@echo off - -pwsh %~dp0/build.backend.docker.ps1 %* - -echo. - -pause \ No newline at end of file diff --git a/build.backend.docker.ps1 b/build.backend.docker.ps1 deleted file mode 100644 index 15fe253899..0000000000 --- a/build.backend.docker.ps1 +++ /dev/null @@ -1,133 +0,0 @@ -param( - [switch] $h = $false, - [switch] $f = $false, - [switch] $s = $true, - [switch] $c = $false, - [switch] $d = $false -) - -if ($h) { - Write-Host "Build and run backend and working environment. (Use 'yarn start' to run client -> https://github.com/ONLYOFFICE/DocSpace-client)" - Write-Host - Write-Host "Syntax: available params [-h|f|s|c|d|]" - Write-Host "Options:" - Write-Host "h Print this Help." - Write-Host "f Force rebuild base images." - Write-Host "s Run as SAAS otherwise as STANDALONE." - Write-Host "c Run as COMMUNITY otherwise ENTERPRISE." - Write-Host "d Run dnsmasq." - Write-Host - exit -} - -$PSversionMajor = $PSVersionTable.PSVersion | sort-object major | ForEach-Object { $_.major } -$PSversionMinor = $PSVersionTable.PSVersion | sort-object minor | ForEach-Object { $_.minor } - -if ($PSversionMajor -lt 7 -or $PSversionMinor -lt 2) { - Write-Error "Powershell version must be greater than or equal to 7.2." - exit -} - -$RootDir = Split-Path -Parent $PSScriptRoot -$DockerDir = "$RootDir\buildtools\install\docker" -$LocalIp = (Get-CimInstance -ClassName Win32_NetworkAdapterConfiguration | Where-Object { $_.DHCPEnabled -ne $null -and $_.DefaultIPGateway -ne $null }).IPAddress | Select-Object -First 1 - -$Doceditor = ($LocalIp + ":5013") -$Login = ($LocalIp + ":5011") -$Client = ($LocalIp + ":5001") -$PortalUrl = ("http://" + $LocalIp) -$ProxyVersion="v1.0.0" - -# Stop all backend services" -& "$PSScriptRoot\start\stop.backend.docker.ps1" - -$Env:COMPOSE_IGNORE_ORPHANS = "True" - -$ExistsNetwork= docker network ls --format '{{.Name}}' | findstr "onlyoffice" - -if (-not $ExistsNetwork) { - docker network create --driver bridge onlyoffice -} - -Write-Host "Run MySQL" -ForegroundColor Green -docker compose -f "$DockerDir\db.yml" up -d - -if ($d) { - Write-Host "Run local dns server" -ForegroundColor Green - $Env:ROOT_DIR=$RootDir - docker compose -f "$DockerDir\dnsmasq.yml" up -d -} - -Write-Host "Build backend services (to `publish/` folder)" -ForegroundColor Green -& "$PSScriptRoot\install\common\build-services.ps1" - -$Env:DOCUMENT_SERVER_IMAGE_NAME = "onlyoffice/documentserver-de:latest" -$Env:INSTALLATION_TYPE = "ENTERPRISE" -$Env:MIGRATION_TYPE = "STANDALONE" - -if ($c) { - $Env:DOCUMENT_SERVER_IMAGE_NAME = "onlyoffice/documentserver:latest" - $Env:INSTALLATION_TYPE = "COMMUNITY" -} - -if (-not $s) { - $Env:MIGRATION_TYPE = "SAAS" -} - -Set-Location -Path $RootDir - -$DotnetVersion = "dev" -$NodeVersion = "dev" -$ProxyVersion = "dev" - -$ExistsDotnet= docker images --format "{{.Repository}}:{{.Tag}}" | findstr "onlyoffice/4testing-docspace-dotnet-runtime:$DotnetVersion" -$ExistsNode= docker images --format "{{.Repository}}:{{.Tag}}" | findstr "onlyoffice/4testing-docspace-nodejs-runtime:$NodeVersion" -$ExistsProxy= docker images --format "{{.Repository}}:{{.Tag}}" | findstr "onlyoffice/4testing-docspace-proxy-runtime:$ProxyVersion" - -if (!$ExistsDotnet -or $f) { - Write-Host "Build dotnet base image from source (apply new dotnet config)" -ForegroundColor Green - docker build -t "onlyoffice/4testing-docspace-dotnet-runtime:$DotnetVersion" -f "$DockerDir\Dockerfile.runtime" --target dotnetrun . -} else { - Write-Host "SKIP build dotnet base image (already exists)" -ForegroundColor Blue -} - -if (!$ExistsNode -or $f) { - Write-Host "Build node base image from source" -ForegroundColor Green - docker build -t "onlyoffice/4testing-docspace-nodejs-runtime:$NodeVersion" -f "$DockerDir\Dockerfile.runtime" --target noderun . -} else { - Write-Host "SKIP build node base image (already exists)" -ForegroundColor Blue -} - -if (!$ExistsProxy -or $f) { - Write-Host "Build proxy base image from source (apply new nginx config)" -ForegroundColor Green - docker build -t "onlyoffice/4testing-docspace-proxy-runtime:$ProxyVersion" -f "$DockerDir\Dockerfile.runtime" --target router . -} else { - Write-Host "SKIP build proxy base image (already exists)" -ForegroundColor Blue -} - -Write-Host "Run migration and services" -ForegroundColor Green -$Env:ENV_EXTENSION="dev" -$Env:Baseimage_Dotnet_Run="onlyoffice/4testing-docspace-dotnet-runtime:$DotnetVersion" -$Env:Baseimage_Nodejs_Run="onlyoffice/4testing-docspace-nodejs-runtime:$NodeVersion" -$Env:Baseimage_Proxy_Run="onlyoffice/4testing-docspace-proxy-runtime:$ProxyVersion" -$Env:SERVICE_DOCEDITOR=$Doceditor -$Env:SERVICE_LOGIN=$Login -$Env:SERVICE_CLIENT=$Client -$Env:ROOT_DIR=$RootDir -$Env:BUILD_PATH="/var/www" -$Env:SRC_PATH="$RootDir\publish\services" -$Env:DATA_DIR="$RootDir\data" -$Env:APP_URL_PORTAL=$PortalUrl -docker compose -f "$DockerDir\docspace.profiles.yml" -f "$DockerDir\docspace.overcome.yml" --profile migration-runner --profile backend-local up -d - -Write-Host "== Build params ==" -ForegroundColor Green -Write-Host "APP_URL_PORTAL: $PortalUrl" -ForegroundColor Blue -Write-Host "LOCAL IP: $LocalIp" -ForegroundColor Blue -Write-Host "SERVICE_DOCEDITOR: $Env:SERVICE_DOCEDITOR" -ForegroundColor Blue -Write-Host "SERVICE_LOGIN: $Env:SERVICE_LOGIN" -ForegroundColor Blue -Write-Host "SERVICE_CLIENT: $Env:SERVICE_CLIENT" -ForegroundColor Blue -Write-Host "INSTALLATION_TYPE: $Env:INSTALLATION_TYPE" -ForegroundColor Blue -Write-Host "MIGRATION TYPE: $Env:MIGRATION_TYPE" -ForegroundColor Blue -Write-Host "DS IMAGE: $Env:DOCUMENT_SERVER_IMAGE_NAME" -ForegroundColor Blue - -Set-Location -Path $PSScriptRoot \ No newline at end of file diff --git a/build.backend.docker.sh b/build.backend.docker.sh deleted file mode 100755 index 7cbc9b17bc..0000000000 --- a/build.backend.docker.sh +++ /dev/null @@ -1,203 +0,0 @@ -#!/bin/bash - -############################################################ -# Help # -############################################################ -Help() -{ - # Display Help - echo "Build and run backend and working environment. (Use 'yarn start' to run client -> https://github.com/ONLYOFFICE/DocSpace-client)" - echo - echo "Syntax: available params [-h|f|s|c|d|]" - echo "options:" - echo "h Print this Help." - echo "f Force rebuild base images." - echo "s Run as SAAS otherwise as STANDALONE." - echo "c Run as COMMUNITY otherwise ENTERPRISE." - echo "d Run dnsmasq." - echo -} - -rd="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -dir=$(builtin cd $rd/../; pwd) -dockerDir="$dir/buildtools/install/docker" -local_ip=$(ipconfig getifaddr en0) - -doceditor=${local_ip}:5013 -login=${local_ip}:5011 -client=${local_ip}:5001 -portal_url="http://$local_ip" - -force=false -dns=false -standalone=true -community=false - -migration_type="STANDALONE" # SAAS -installation_type=ENTERPRISE -document_server_image_name=onlyoffice/documentserver-de:latest - -# Get the options -while getopts "h:f:s:c:d:" opt; do - echo "argument -${opt} called with parameter $OPTARG" >&2 - case $opt in - h) # Display this Help - Help - exit - ;; - f) # Force rebuild base images - force=${OPTARG:-true} - ;; - s) # Run as STANDALONE (otherwise SAAS) - standalone=${OPTARG:-true} - ;; - c) # Run as COMMUNITY (otherwise ENTERPRISE) - community=${OPTARG:-true} - ;; - d) # Run dnsmasq - dns=${OPTARG:-true} - ;; - \?) # Invalid option - echo "Error: Invalid '-$OPTARG' option" - exit - ;; - esac -done - -echo "Run script directory:" $dir -echo "Root directory:" $dir -echo "Docker files root directory:" $dockerDir - -echo -echo "SERVICE_DOCEDITOR: $doceditor" -echo "SERVICE_LOGIN: $login" -echo "SERVICE_CLIENT: $client" -echo "DOCSPACE_APP_URL: $portal_url" - -echo -echo "FORCE REBUILD BASE IMAGES: $force" -echo "Run dnsmasq: $dns" - -if [ "$standalone" = false ]; then - migration_type="SAAS" -fi - -if [ "$community" = true ]; then - installation_type="COMMUNITY" - document_server_image_name=onlyoffice/documentserver:latest -fi - -echo -echo "MIGRATION TYPE: $migration_type" -echo "INSTALLATION TYPE: $installation_type" -echo "DS image: $document_server_image_name" -echo - -# Stop all backend services" -$dir/buildtools/start/stop.backend.docker.sh - -echo "Run MySQL" - -arch_name="$(uname -m)" - -existsnetwork=$(docker network ls | awk '{print $2;}' | { grep -x onlyoffice || true; }); - -if [[ -z ${existsnetwork} ]]; then - docker network create --driver bridge onlyoffice -fi - -if [ "${arch_name}" = "x86_64" ]; then - echo "CPU Type: x86_64 -> run db.yml" - docker compose -f $dockerDir/db.yml up -d -elif [ "${arch_name}" = "arm64" ]; then - echo "CPU Type: arm64 -> run db.yml with arm64v8 image" - MYSQL_IMAGE=arm64v8/mysql:8.0.32-oracle \ - docker compose -f $dockerDir/db.yml up -d -else - echo "Error: Unknown CPU Type: ${arch_name}." - exit 1 -fi - -if [ "$dns" = true ]; then - echo "Run local dns server" - ROOT_DIR=$dir \ - docker compose -f $dockerDir/dnsmasq.yml up -d -fi - -echo "Clear publish folder" -rm -rf $dir/publish/services - -echo "Build backend services (to "publish/" folder)" -bash $dir/buildtools/install/common/build-services.sh -pb backend-publish -pc Debug -de "$dockerDir/docker-entrypoint.py" - -dotnet_version=dev - -exists=$(docker images | egrep "onlyoffice/4testing-docspace-dotnet-runtime" | egrep "$dotnet_version" | awk 'NR>0 {print $1 ":" $2}') - -if [ "${exists}" = "" ] || [ "$force" = true ]; then - echo "Build dotnet base image from source (apply new dotnet config)" - docker build -t onlyoffice/4testing-docspace-dotnet-runtime:$dotnet_version -f $dockerDir/Dockerfile.runtime --target dotnetrun . -else - echo "SKIP build dotnet base image (already exists)" -fi - -node_version=dev - -exists=$(docker images | egrep "onlyoffice/4testing-docspace-nodejs-runtime" | egrep "$node_version" | awk 'NR>0 {print $1 ":" $2}') - -if [ "${exists}" = "" ] || [ "$force" = true ]; then - echo "Build nodejs base image from source" - docker build -t onlyoffice/4testing-docspace-nodejs-runtime:$node_version -f $dockerDir/Dockerfile.runtime --target noderun . -else - echo "SKIP build nodejs base image (already exists)" -fi - -proxy_version=dev - -exists=$(docker images | egrep "onlyoffice/4testing-docspace-proxy-runtime" | egrep "$proxy_version" | awk 'NR>0 {print $1 ":" $2}') - -if [ "${exists}" = "" ] || [ "$force" = true ]; then - echo "Build proxy base image from source (apply new nginx config)" - docker build -t onlyoffice/4testing-docspace-proxy-runtime:$proxy_version -f $dockerDir/Dockerfile.runtime --target router . -else - echo "SKIP build proxy base image (already exists)" -fi - -echo "Run migration and services" -ENV_EXTENSION="dev" \ -INSTALLATION_TYPE=$installation_type \ -Baseimage_Dotnet_Run="onlyoffice/4testing-docspace-dotnet-runtime:$dotnet_version" \ -Baseimage_Nodejs_Run="onlyoffice/4testing-docspace-nodejs-runtime:$node_version" \ -Baseimage_Proxy_Run="onlyoffice/4testing-docspace-proxy-runtime:$proxy_version" \ -DOCUMENT_SERVER_IMAGE_NAME=$document_server_image_name \ -SERVICE_DOCEDITOR=$doceditor \ -SERVICE_LOGIN=$login \ -SERVICE_CLIENT=$client \ -ROOT_DIR=$dir \ -BUILD_PATH="/var/www" \ -SRC_PATH="$dir/publish/services" \ -DATA_DIR="$dir/data" \ -APP_URL_PORTAL=$portal_url \ -MIGRATION_TYPE=$migration_type \ -docker-compose -f $dockerDir/docspace.profiles.yml -f $dockerDir/docspace.overcome.yml --profile migration-runner --profile backend-local up -d - -echo -echo "Run script directory:" $dir -echo "Root directory:" $dir -echo "Docker files root directory:" $dockerDir - -echo -echo "SERVICE_DOCEDITOR: $doceditor" -echo "SERVICE_LOGIN: $login" -echo "SERVICE_CLIENT: $client" -echo "DOCSPACE_APP_URL: $portal_url" - -echo -echo "FORCE REBUILD BASE IMAGES: $force" -echo "Run dnsmasq: $dns" - -echo -echo "MIGRATION TYPE: $migration_type" -echo "INSTALLATION TYPE: $installation_type" -echo "DS image: $document_server_image_name" -echo diff --git a/clear.backend.docker.bat b/clear.backend.docker.bat deleted file mode 100644 index 7d215cfc09..0000000000 --- a/clear.backend.docker.bat +++ /dev/null @@ -1,7 +0,0 @@ -@echo off - -pwsh %~dp0/clear.backend.docker.ps1 - -echo. - -pause \ No newline at end of file diff --git a/clear.backend.docker.ps1 b/clear.backend.docker.ps1 deleted file mode 100644 index de6f9a2dfd..0000000000 --- a/clear.backend.docker.ps1 +++ /dev/null @@ -1,37 +0,0 @@ -$Containers = docker ps -aqf "name=^onlyoffice" -$Images = docker images onlyoffice/4testing-docspace* -q -$RootDir = Split-Path -Parent $PSScriptRoot -$DockerDir = ($RootDir + "\buildtools\install\docker") - -Write-Host "Clean up containers, volumes or networks" -ForegroundColor Green - -if ($Containers -or $Images) { - Write-Host "Remove all backend containers" -ForegroundColor Blue - - $Env:DOCUMENT_SERVER_IMAGE_NAME="onlyoffice/documentserver-de:latest" - $Env:Baseimage_Dotnet_Run="onlyoffice/4testing-docspace-dotnet-runtime:dev" - $Env:Baseimage_Nodejs_Run="onlyoffice/4testing-docspace-nodejs-runtime:dev" - $Env:Baseimage_Proxy_Run="onlyoffice/4testing-docspace-proxy-runtime:dev" - $Env:SERVICE_CLIENT="localhost:5001" - $Env:BUILD_PATH="/var/www" - $Env:SRC_PATH="$RootDir\publish\services" - $Env:ROOT_DIR=$RootDir - $Env:DATA_DIR="$RootDir\data" - - docker compose -f "$DockerDir\docspace.profiles.yml" -f "$DockerDir\docspace.overcome.yml" --profile "migration-runner" --profile "backend-local" down --volumes - - Write-Host "Remove docker contatiners 'mysql'" -ForegroundColor Blue - docker compose -f "$DockerDir\db.yml" down --volumes - - Write-Host "Remove docker volumes" -ForegroundColor Blue - docker volume prune -f -a - - Write-Host "Remove docker base images (onlyoffice/4testing-docspace)" -ForegroundColor Blue - docker rmi -f $Images - - Write-Host "Remove docker networks" -ForegroundColor Blue - docker network prune -f -} -else { - Write-Host "No containers, images, volumes or networks to clean up" -ForegroundColor Green -} \ No newline at end of file diff --git a/clear.backend.docker.sh b/clear.backend.docker.sh deleted file mode 100755 index c92bbe5bfd..0000000000 --- a/clear.backend.docker.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -Containers=$(docker ps -a | egrep "onlyoffice" | awk 'NR>0 {print $1}') -RunDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -RootDir=$(builtin cd $RunDir/../; pwd) -DockerDir="${RootDir}/buildtools/install/docker" - -echo "Clean up containers, volumes or networks" - -if [[ $Containers != "" ]] -then - echo "Remove all backend containers" - - DOCUMENT_SERVER_IMAGE_NAME=onlyoffice/documentserver-de:latest \ - Baseimage_Dotnet_Run="onlyoffice/4testing-docspace-dotnet-runtime:dev" \ - Baseimage_Nodejs_Run="onlyoffice/4testing-docspace-nodejs-runtime:dev" \ - Baseimage_Proxy_Run="onlyoffice/4testing-docspace-proxy-runtime:dev" \ - SERVICE_CLIENT="localhost:5001" \ - BUILD_PATH="/var/www" \ - SRC_PATH="${RootDir}/publish/services" \ - ROOT_DIR=$RootDir \ - DATA_DIR="${RootDir}/data" \ - docker-compose -f "${DockerDir}/docspace.profiles.yml" -f "${DockerDir}/docspace.overcome.yml" --profile migration-runner --profile backend-local down --volumes - - echo "Remove docker contatiners 'mysql'" - docker compose -f "${DockerDir}/db.yml" down --volumes - - echo "Remove docker volumes" - docker volume prune -f -a - - echo "Remove docker base images (onlyoffice/4testing-docspace)" - docker rmi -f $(docker images -a | egrep "onlyoffice/4testing-docspace" | awk 'NR>0 {print $3}') - - echo "Remove unused networks." - docker network prune -f -else - echo "No containers, images, volumes or networks to clean up" -fi diff --git a/install/common/build-services.ps1 b/install/common/build-services.ps1 deleted file mode 100644 index 30dc6f2a39..0000000000 --- a/install/common/build-services.ps1 +++ /dev/null @@ -1,57 +0,0 @@ -$SRC_PATH=(get-item $PSScriptRoot ).parent.parent.parent.FullName -$BUILD_PATH="$SRC_PATH\publish" - -$BACKEND_NODEJS_SERVICES="ASC.Socket.IO","ASC.SsoAuth" -$BACKEND_DOTNETCORE_SERVICES="ASC.Files", "ASC.People", "ASC.Data.Backup", "ASC.Files.Service", "ASC.Notify", "ASC.Studio.Notify", "ASC.Web.Api", "ASC.Web.Studio", "ASC.Data.Backup.BackgroundTasks", "ASC.ClearEvents", "ASC.ApiSystem", "ASC.Web.HealthChecks.UI" -$SELF_CONTAINED="false" -$PUBLISH_CNF="Debug" - -$FRONTEND_BUILD_ARGS="build" -$FRONTEND_DEPLOY_ARGS="deploy" -$DEBUG_INFO_CHECK="" -$MIGRATION_CHECK="true" -$DOCKER_ENTRYPOINT="$SRC_PATH\buildtools\install\docker\docker-entrypoint.py" - -if(Test-Path -Path "$BUILD_PATH\services" ){ - Write-Host "== Clean up services ==" -ForegroundColor Green - Remove-Item "$BUILD_PATH\services" -Recurse -} - -Write-Host "== Build ASC.Web.slnf ==" -ForegroundColor Green -dotnet build "$SRC_PATH\server\ASC.Web.slnf" - -Write-Host "== Build ASC.Migrations.sln ==" -ForegroundColor Green -dotnet build "$SRC_PATH\server\ASC.Migrations.sln" -o "$BUILD_PATH\services\ASC.Migration.Runner\service\" - -Write-Host "== Add docker-migration-entrypoint.sh to ASC.Migration.Runner ==" -ForegroundColor Green -$FilePath = "$BUILD_PATH\services\ASC.Migration.Runner\service\docker-migration-entrypoint.sh" -Get-Content "$SRC_PATH\buildtools\install\docker\docker-migration-entrypoint.sh" -raw | % {$_ -replace "`r", ""} | Set-Content -NoNewline $FilePath - -foreach ($SERVICE in $BACKEND_NODEJS_SERVICES) -{ - Write-Host "== Build $SERVICE project ==" -ForegroundColor Green - yarn install --cwd "$SRC_PATH\server\common\$SERVICE" --frozen-lockfile - - $DST = "$BUILD_PATH\services\$SERVICE\service\" - - if(!(Test-Path -Path $DST )){ - New-Item -ItemType "directory" -Path $DST - } - - Write-Host "== Copy service data to `publish\services\${SERVICE}\service` ==" -ForegroundColor Green - Copy-Item -Path "$SRC_PATH\server\common\$SERVICE\*" -Destination $DST -Recurse - Write-Host "== Add docker-entrypoint.py to $SERVICE ==" -ForegroundColor Green - Copy-Item $DOCKER_ENTRYPOINT -Destination $DST -} - -Write-Host "== Publish ASC.Web.slnf ==" -ForegroundColor Green -dotnet publish "$SRC_PATH\server\ASC.Web.slnf" -p "PublishProfile=FolderProfile" - -Set-Location -Path $PSScriptRoot - -foreach ($SERVICE in $BACKEND_DOTNETCORE_SERVICES) -{ - Write-Host "== Add docker-entrypoint.py to $SERVICE ==" -ForegroundColor Green - $DST = "$BUILD_PATH\services\$SERVICE\service\" - Copy-Item $DOCKER_ENTRYPOINT -Destination $DST -} diff --git a/start/restart.backend.docker.bat b/start/restart.backend.docker.bat deleted file mode 100644 index 464988e1fd..0000000000 --- a/start/restart.backend.docker.bat +++ /dev/null @@ -1,7 +0,0 @@ -@echo off - -pwsh %~dp0/restart.backend.docker.ps1 %* - -echo. - -pause \ No newline at end of file diff --git a/start/restart.backend.docker.ps1 b/start/restart.backend.docker.ps1 deleted file mode 100644 index c42a57a008..0000000000 --- a/start/restart.backend.docker.ps1 +++ /dev/null @@ -1,5 +0,0 @@ -# Stop all backend services" -& "$PSScriptRoot\stop.backend.docker.ps1" - -# Start all backend services" -& "$PSScriptRoot\start.backend.docker.ps1" \ No newline at end of file diff --git a/start/restart.backend.docker.sh b/start/restart.backend.docker.sh deleted file mode 100755 index 4edfe6df43..0000000000 --- a/start/restart.backend.docker.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -echo "Root directory:" $dir - -$dir/stop.backend.docker.sh - -$dir/start.backend.docker.sh \ No newline at end of file diff --git a/start/start.backend.docker.bat b/start/start.backend.docker.bat deleted file mode 100644 index 090dcdb495..0000000000 --- a/start/start.backend.docker.bat +++ /dev/null @@ -1,7 +0,0 @@ -@echo off - -pwsh %~dp0/start.backend.docker.ps1 %* - -echo. - -pause \ No newline at end of file diff --git a/start/start.backend.docker.ps1 b/start/start.backend.docker.ps1 deleted file mode 100644 index fdf9bda954..0000000000 --- a/start/start.backend.docker.ps1 +++ /dev/null @@ -1,32 +0,0 @@ -$PSversionMajor = $PSVersionTable.PSVersion | sort-object major | ForEach-Object { $_.major } -$PSversionMinor = $PSVersionTable.PSVersion | sort-object minor | ForEach-Object { $_.minor } - -if ($PSversionMajor -lt 7 -or $PSversionMinor -lt 2) { - Write-Error "Powershell version must be greater than or equal to 7.2." - exit -} - -$RootDir = Split-Path (Split-Path -Parent $PSScriptRoot) -Parent -$DockerDir = ($RootDir + "\buildtools\install\docker") -$LocalIp = (Get-CimInstance -ClassName Win32_NetworkAdapterConfiguration | Where-Object { $_.DHCPEnabled -ne $null -and $_.DefaultIPGateway -ne $null }).IPAddress | Select-Object -First 1 - -$Doceditor = ($LocalIp + ":5013") -$Login = ($LocalIp + ":5011") -$Client = ($LocalIp + ":5001") - -Set-Location -Path $DockerDir - -Write-Host "Start all services (containers)" -ForegroundColor Green -$Env:ENV_EXTENSION="dev" -$Env:Baseimage_Dotnet_Run="onlyoffice/4testing-docspace-dotnet-runtime:v1.0.0" -$Env:Baseimage_Nodejs_Run="onlyoffice/4testing-docspace-nodejs-runtime:v1.0.0" -$Env:Baseimage_Proxy_Run="onlyoffice/4testing-docspace-proxy-runtime:v1.0.0" -$Env:DOCUMENT_SERVER_IMAGE_NAME="onlyoffice/documentserver-de:latest" -$Env:SERVICE_DOCEDITOR=$Doceditor -$Env:SERVICE_LOGIN=$Login -$Env:SERVICE_CLIENT=$Client -$Env:ROOT_DIR=$RootDir -$Env:BUILD_PATH="/var/www" -$Env:SRC_PATH="$RootDir\publish\services" -$Env:DATA_DIR="$RootDir\data" -docker compose -f docspace.profiles.yml -f docspace.overcome.yml --profile migration-runner --profile backend-local start \ No newline at end of file diff --git a/start/start.backend.docker.sh b/start/start.backend.docker.sh deleted file mode 100755 index dd78fdda2b..0000000000 --- a/start/start.backend.docker.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - - -rd="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -echo "Run script directory:" $dir - -dir=$(builtin cd $rd/../../; pwd) - -echo "Root directory:" $dir - -cd $dir/buildtools/install/docker/ - -docker_dir="$( pwd )" - -echo "Docker directory:" $docker_dir - -local_ip=$(ipconfig getifaddr en0) - -echo "LOCAL IP: $local_ip" - -doceditor=${local_ip}:5013 -login=${local_ip}:5011 -client=${local_ip}:5001 - -echo "SERVICE_DOCEDITOR: $doceditor" -echo "SERVICE_LOGIN: $login" -echo "SERVICE_CLIENT: $client" - -Baseimage_Dotnet_Run="onlyoffice/4testing-docspace-dotnet-runtime:v1.0.0" \ -Baseimage_Nodejs_Run="onlyoffice/4testing-docspace-nodejs-runtime:v1.0.0" \ -Baseimage_Proxy_Run="onlyoffice/4testing-docspace-proxy-runtime:v1.0.0" \ -BUILD_PATH="/var/www" \ -SRC_PATH="$dir/publish/services" \ -SERVICE_DOCEDITOR=$doceditor \ -SERVICE_LOGIN=$login \ -SERVICE_CLIENT=$client \ -ROOT_DIR=$dir \ -DATA_DIR="$dir/data" \ -ENV_EXTENSION="dev" \ -DOCUMENT_SERVER_IMAGE_NAME=onlyoffice/documentserver-de:latest \ -docker-compose -f docspace.profiles.yml -f docspace.overcome.yml --profile backend-local start diff --git a/start/stop.backend.docker.bat b/start/stop.backend.docker.bat deleted file mode 100644 index c68a50e400..0000000000 --- a/start/stop.backend.docker.bat +++ /dev/null @@ -1,7 +0,0 @@ -@echo off - -pwsh %~dp0/stop.backend.docker.ps1 - -echo. - -pause \ No newline at end of file diff --git a/start/stop.backend.docker.ps1 b/start/stop.backend.docker.ps1 deleted file mode 100644 index b62cc6f7ea..0000000000 --- a/start/stop.backend.docker.ps1 +++ /dev/null @@ -1,17 +0,0 @@ -$PSversionMajor = $PSVersionTable.PSVersion | sort-object major | ForEach-Object { $_.major } -$PSversionMinor = $PSVersionTable.PSVersion | sort-object minor | ForEach-Object { $_.minor } - -if ($PSversionMajor -lt 7 -or $PSversionMinor -lt 2) { - Write-Error "Powershell version must be greater than or equal to 7.2." - exit -} - -$Containers = docker ps -a -f "name=^onlyoffice" --format="{{.ID}} {{.Names}}" | Select-String -Pattern ("mysql|rabbitmq|redis|elasticsearch|documentserver") -NotMatch | ConvertFrom-String | ForEach-Object P1 - -if (-not $Containers) { - Write-Host "No containers to stop" -ForegroundColor Blue - exit -} - -Write-Host "Stop all backend services (containers)" -ForegroundColor Green -docker stop $Containers \ No newline at end of file diff --git a/start/stop.backend.docker.sh b/start/stop.backend.docker.sh deleted file mode 100755 index 6a6154bf38..0000000000 --- a/start/stop.backend.docker.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -echo "Stop all backend services (containers)" -docker stop $(docker ps -a | egrep "onlyoffice" | egrep -v "mysql|rabbitmq|redis|elasticsearch|documentserver" | awk 'NR>0 {print $1}') \ No newline at end of file From 1fac1b41e89e0f944c4a821d27ab6f177dc4a62a Mon Sep 17 00:00:00 2001 From: Alexey Safronov Date: Wed, 8 Nov 2023 18:33:29 +0400 Subject: [PATCH 47/47] Removed useless params --- install/common/build-services.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/install/common/build-services.py b/install/common/build-services.py index 9d4c93399a..aec3d154cb 100755 --- a/install/common/build-services.py +++ b/install/common/build-services.py @@ -14,13 +14,7 @@ print(f"BUILD_PATH = {BUILD_PATH}") BACKEND_NODEJS_SERVICES = ["ASC.Socket.IO", "ASC.SsoAuth"] BACKEND_DOTNETCORE_SERVICES = ["ASC.Files", "ASC.People", "ASC.Data.Backup", "ASC.Files.Service", "ASC.Notify", "ASC.Studio.Notify", "ASC.Web.Api", "ASC.Web.Studio", "ASC.Data.Backup.BackgroundTasks", "ASC.ClearEvents", "ASC.ApiSystem", "ASC.Web.HealthChecks.UI"] -SELF_CONTAINED = "false" -PUBLISH_CNF = "Debug" -FRONTEND_BUILD_ARGS = "build" -FRONTEND_DEPLOY_ARGS = "deploy" -DEBUG_INFO_CHECK = "" -MIGRATION_CHECK = "true" DOCKER_ENTRYPOINT="docker-entrypoint.py" DOCKER_ENTRYPOINT_PATH = os.path.join(SRC_PATH, "buildtools", "install", "docker", DOCKER_ENTRYPOINT)