Migration to opensearch in docker (#167)

* Migration to opensearch

* Implement indexes update on version change

* Optimize indexes update on version change

* Migration to opensearch in docspace.profiles.yml

config/elastic.test.json

@@ -1,7 +1,7 @@
 {
     "elastic": {
         "Scheme": "http",
-        "Host": "onlyoffice-elasticsearch",
+        "Host": "onlyoffice-opensearch",
         "Port": "9200",
         "Threads": "1"
     }

install/OneClickInstall/install-Docker.sh

@@ -1109,6 +1109,7 @@ set_docspace_params() {
 	APP_CORE_BASE_DOMAIN=${APP_CORE_BASE_DOMAIN:-$(get_env_parameter "APP_CORE_BASE_DOMAIN" "${CONTAINER_NAME}")};
 	EXTERNAL_PORT=${EXTERNAL_PORT:-$(get_env_parameter "EXTERNAL_PORT" "${CONTAINER_NAME}")};
 
+	PREVIOUS_ELK_VERSION=$(get_env_parameter "ELK_VERSION");
 	ELK_SHEME=${ELK_SHEME:-$(get_env_parameter "ELK_SHEME" "${CONTAINER_NAME}")};
 	ELK_HOST=${ELK_HOST:-$(get_env_parameter "ELK_HOST" "${CONTAINER_NAME}")};
 	ELK_PORT=${ELK_PORT:-$(get_env_parameter "ELK_PORT" "${CONTAINER_NAME}")};
@@ -1248,14 +1249,14 @@ install_redis () {
 install_elasticsearch () {
 	if [[ -z ${ELK_HOST} ]] && [ "$INSTALL_ELASTICSEARCH" == "true" ]; then
 		if [ $(free --mega | grep -oP '\d+' | head -n 1) -gt "12000" ]; then #RAM ~12Gb
-			sed -i 's/Xms[0-9]g/Xms4g/g; s/Xmx[0-9]g/Xmx4g/g' $BASE_DIR/elasticsearch.yml
+			sed -i 's/Xms[0-9]g/Xms4g/g; s/Xmx[0-9]g/Xmx4g/g' $BASE_DIR/opensearch.yml
 		else
-			sed -i 's/Xms[0-9]g/Xms1g/g; s/Xmx[0-9]g/Xmx1g/g' $BASE_DIR/elasticsearch.yml
+			sed -i 's/Xms[0-9]g/Xms1g/g; s/Xmx[0-9]g/Xmx1g/g' $BASE_DIR/opensearch.yml
 		fi
 		reconfigure ELK_VERSION ${ELK_VERSION}
-		docker-compose -f $BASE_DIR/elasticsearch.yml up -d
+		docker-compose -f $BASE_DIR/opensearch.yml up -d
 	elif [ ! -z "$ELK_HOST" ]; then
-		establish_conn ${ELK_HOST} "${ELK_PORT:-"9200"}" "Elasticsearch"
+		establish_conn ${ELK_HOST} "${ELK_PORT:-"9200"}" "search engine"
 		reconfigure ELK_SHEME "${ELK_SHEME:-"http"}"	
 		reconfigure ELK_HOST ${ELK_HOST}
 		reconfigure ELK_PORT "${ELK_PORT:-"9200"}"	
@@ -1286,6 +1287,12 @@ install_product () {
 	docker-compose -f $BASE_DIR/notify.yml up -d
 	docker-compose -f $BASE_DIR/healthchecks.yml up -d
 
+	if [[ -n "${PREVIOUS_ELK_VERSION}" && "$(get_env_parameter "ELK_VERSION")" != "${PREVIOUS_ELK_VERSION}" ]]; then
+		MYSQL_TAG=$(docker images --format "{{.Tag}}" mysql | head -n1)
+		MYSQL_CONTAINER_NAME=$(get_env_parameter "MYSQL_CONTAINER_NAME" | sed "s/\${CONTAINER_PREFIX}/${PACKAGE_SYSNAME}-/g")
+		docker run --rm --network="$(get_env_parameter "NETWORK_NAME")" mysql:${MYSQL_TAG:-latest} mysql -h "${MYSQL_HOST:-${MYSQL_CONTAINER_NAME}}" -P "${MYSQL_PORT:-3306}" -u "${MYSQL_USER}" -p"${MYSQL_PASSWORD}" "${MYSQL_DATABASE}" -e "TRUNCATE webstudio_index;"
+	fi
+
 	if [ ! -z "${CERTIFICATE_PATH}" ] && [ ! -z "${CERTIFICATE_KEY_PATH}" ] && [[ ! -z "${APP_DOMAIN_PORTAL}" ]]; then
 		bash $BASE_DIR/config/${PRODUCT}-ssl-setup -f "${APP_DOMAIN_PORTAL}" "${CERTIFICATE_PATH}" "${CERTIFICATE_KEY_PATH}"
 	elif [ ! -z "${LETS_ENCRYPT_DOMAIN}" ] && [ ! -z "${LETS_ENCRYPT_MAIL}" ]; then

install/docker/.env

@@ -8,7 +8,7 @@
     CONTAINER_PREFIX=${PRODUCT}-
     MYSQL_VERSION=8.3.0
     MYSQL_IMAGE=mysql:${MYSQL_VERSION}
-    ELK_VERSION=7.16.3
+    ELK_VERSION=2.11.1
     SERVICE_PORT=5050
     DOCUMENT_SERVER_IMAGE_NAME=onlyoffice/4testing-documentserver-ee:latest
     DOCKERFILE=Dockerfile.app
@@ -31,7 +31,7 @@
     KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
 
 # elasticsearch #
-    ELK_CONTAINER_NAME=${CONTAINER_PREFIX}elasticsearch
+    ELK_CONTAINER_NAME=${CONTAINER_PREFIX}opensearch
     ELK_SHEME=http
     ELK_HOST=""
     ELK_PORT=9200

install/docker/config/docspace-logs

@@ -14,7 +14,7 @@ else
   echo "Error: yml files not found." && exit 1
 fi
 
-FILES=("${PRODUCT}" "notify" "healthchecks" "proxy" "ds" "rabbitmq" "redis" "elasticsearch" "db")
+FILES=("${PRODUCT}" "notify" "healthchecks" "proxy" "ds" "rabbitmq" "redis" "opensearch" "db")
 
 LOG_DIR="${DOCKERCOMPOSE}/logs"
 mkdir -p ${LOG_DIR}

install/docker/docker-entrypoint.py

@@ -50,7 +50,7 @@ DOCUMENT_SERVER_URL_EXTERNAL = os.environ["DOCUMENT_SERVER_URL_EXTERNAL"] if env
 DOCUMENT_SERVER_URL_PUBLIC = DOCUMENT_SERVER_URL_EXTERNAL if DOCUMENT_SERVER_URL_EXTERNAL else os.environ["DOCUMENT_SERVER_URL_PUBLIC"] if environ.get("DOCUMENT_SERVER_URL_PUBLIC") else "/ds-vpath/"
 DOCUMENT_SERVER_CONNECTION_HOST = DOCUMENT_SERVER_URL_EXTERNAL if DOCUMENT_SERVER_URL_EXTERNAL else DOCUMENT_SERVER_URL_INTERNAL
 
-ELK_CONTAINER_NAME = os.environ["ELK_CONTAINER_NAME"] if environ.get("ELK_CONTAINER_NAME") else "onlyoffice-elasticsearch"
+ELK_CONTAINER_NAME = os.environ["ELK_CONTAINER_NAME"] if environ.get("ELK_CONTAINER_NAME") else "onlyoffice-opensearch"
 ELK_SHEME = os.environ["ELK_SHEME"] if environ.get("ELK_SHEME") else "http"
 ELK_HOST = os.environ["ELK_HOST"] if environ.get("ELK_HOST") else None
 ELK_PORT = os.environ["ELK_PORT"] if environ.get("ELK_PORT") else "9200"

install/docker/docker-entrypoint.sh

@@ -50,7 +50,7 @@ DOCUMENT_SERVER_URL_PUBLIC=${DOCUMENT_SERVER_URL_PUBLIC:-"/ds-vpath/"}
 DOCUMENT_SERVER_URL_INTERNAL=${DOCUMENT_SERVER_URL_INTERNAL:-"${SHEME}://${PRODUCT}-document-server/"}
 
 ELK_SHEME=${ELK_SHEME:-"http"}
-ELK_HOST=${ELK_HOST:-"${PRODUCT}-elasticsearch"}
+ELK_HOST=${ELK_HOST:-"${PRODUCT}-opensearch"}
 ELK_PORT=${ELK_PORT:-"9200"}
 ELK_THREADS=${ELK_THREADS:-"1"}
 

install/docker/docspace.profiles.yml

@@ -64,30 +64,31 @@ x-service: &x-service-base
     - people_data:/var/www/products/ASC.People/server/
 
 services:
-  onlyoffice-elasticsearch:
+  onlyoffice-opensearch:
     <<: [*x-profiles-extra-services]
-    image: onlyoffice/elasticsearch:${ELK_VERSION}
-    container_name: ${ELK_HOST}
+    image: onlyoffice/opensearch:${ELK_VERSION}
+    container_name: ${ELK_CONTAINER_NAME}
     restart: always
     environment:
       - discovery.type=single-node
       - bootstrap.memory_lock=true
-      - "ES_JAVA_OPTS=-Xms4g -Xmx4g -Dlog4j2.formatMsgNoLookups=true"
+      - "OPENSEARCH_JAVA_OPTS=-Xms4g -Xmx4g -Dlog4j2.formatMsgNoLookups=true"
       - "indices.fielddata.cache.size=30%"
       - "indices.memory.index_buffer_size=30%"
-      - "ingest.geoip.downloader.enabled=false"
+      - "DISABLE_INSTALL_DEMO_CONFIG=true"
+      - "DISABLE_SECURITY_PLUGIN=true"
     ulimits:
       memlock:
         soft: -1
         hard: -1
       nofile:
-        soft: 65535
-        hard: 65535
+        soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
+        hard: 65536
     volumes:
-      - es_data:/usr/share/elasticsearch/data
+      - os_data:/usr/share/opensearch/data
     expose:
       - "9200"
-      - "9300"
+      - "9600" # required for Performance Analyzer
 
   onlyoffice-rabbitmq:
     <<: [*x-profiles-extra-services]

install/docker/elasticsearch.yml

@@ -1,32 +0,0 @@
-version: "3"
-services:
-  onlyoffice-elasticsearch:
-    image: onlyoffice/elasticsearch:${ELK_VERSION}
-    container_name: ${ELK_CONTAINER_NAME}
-    restart: always
-    environment:
-      - discovery.type=single-node
-      - bootstrap.memory_lock=true
-      - "ES_JAVA_OPTS=-Xms4g -Xmx4g -Dlog4j2.formatMsgNoLookups=true"
-      - "indices.fielddata.cache.size=30%"
-      - "indices.memory.index_buffer_size=30%" 
-      - "ingest.geoip.downloader.enabled=false"
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
-      nofile:
-        soft: 65535
-        hard: 65535
-    volumes:
-      - es_data:/usr/share/elasticsearch/data
-    expose:
-      - "9200"
-      - "9300"
-networks:
-  default:
-    name: ${NETWORK_NAME}
-    external: true
-
-volumes:
-  es_data:

install/docker/opensearch.yml

@@ -0,0 +1,33 @@
+version: "3"
+services:
+  onlyoffice-opensearch:
+    image: onlyoffice/opensearch:${ELK_VERSION}
+    container_name: ${ELK_CONTAINER_NAME}
+    restart: always
+    environment:
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+      - "OPENSEARCH_JAVA_OPTS=-Xms4g -Xmx4g -Dlog4j2.formatMsgNoLookups=true"
+      - "indices.fielddata.cache.size=30%"
+      - "indices.memory.index_buffer_size=30%"
+      - "DISABLE_INSTALL_DEMO_CONFIG=true"
+      - "DISABLE_SECURITY_PLUGIN=true"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
+        hard: 65536
+    volumes:
+      - os_data:/usr/share/opensearch/data
+    expose:
+      - "9200"
+      - "9600" # required for Performance Analyzer
+networks:
+  default:
+    name: ${NETWORK_NAME}
+    external: true
+
+volumes:
+  os_data: