Modify running fluent-bit through docker and reading log files (#234)

install/OneClickInstall/install-Docker.sh

@@ -1324,41 +1324,32 @@ install_elasticsearch () {
 
 install_fluent_bit () {
 	if [ "$INSTALL_FLUENT_BIT" == "true" ]; then
-		curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh
+		if ! command_exists crontab; then
-		systemctl enable fluent-bit
+			if command_exists apt-get; then
-
+				install_service crontab cron
-		if systemctl list-unit-files --type=service | grep -q "fluent-bit.service"; then
+			elif command_exists yum; then
-			sed -i "s/OPENSEARCH_SCHEME/$(get_env_parameter "ELK_SHEME")/g" "${BASE_DIR}/config/fluent-bit.conf"
+				install_service crontab cronie
-			sed -i "s/OPENSEARCH_HOST/${ELK_HOST:-127.0.0.1}/g" "${BASE_DIR}/config/fluent-bit.conf"
-			sed -i "s/OPENSEARCH_PORT/$(get_env_parameter "ELK_PORT")/g" ${BASE_DIR}/config/fluent-bit.conf
-			sed -i "s/OPENSEARCH_INDEX/${OPENSEARCH_INDEX:-"${PACKAGE_SYSNAME}-fluent-bit"}/g" ${BASE_DIR}/config/fluent-bit.conf
-			[ ! -z "${ELK_HOST}" ] && sed -i "s/ELK_CONTAINER_NAME/ELK_HOST/g" ${BASE_DIR}/dashboards.yml
-			cp -rf ${BASE_DIR}/config/fluent-bit.conf /etc/fluent-bit/fluent-bit.conf
-			systemctl restart fluent-bit
-
-			DOCKER_SYSTEMD_DIR="/etc/systemd/system/docker.service.d"
-			if [ ! -f "${DOCKER_SYSTEMD_DIR}/fluent-after.conf" ]; then
-				mkdir -p ${DOCKER_SYSTEMD_DIR}
-				echo -e "[Unit]\n$(grep After= $(systemctl show -p FragmentPath docker.service | awk -F= '{print $2}')) fluent-bit.service" > "${DOCKER_SYSTEMD_DIR}/fluent-after.conf"
-				systemctl daemon-reload
 			fi
-
-			DOCKER_DAEMON_FILE="/etc/docker/daemon.json"
-			if [[ ! -f "${DOCKER_DAEMON_FILE}" ]]; then
-				echo "{\"log-driver\": \"fluentd\", \"log-opts\": { \"fluentd-address\": \"127.0.0.1:24224\" }}" > "${DOCKER_DAEMON_FILE}"
-				systemctl restart docker
-			elif ! grep -q "log-driver" ${DOCKER_DAEMON_FILE}; then
-				sed -i 's!{!& "log-driver": "fluentd", "log-opts": { "fluentd-address": "127.0.0.1:24224" },!' "${DOCKER_DAEMON_FILE}"
-				systemctl restart docker
-			fi
-
-			reconfigure DASHBOARDS_USERNAME "${DASHBOARDS_USERNAME:-"onlyoffice"}"
-			reconfigure DASHBOARDS_PASSWORD "${DASHBOARDS_PASSWORD:-$(get_random_str 20)}"
-
-			docker-compose -f ${BASE_DIR}/dashboards.yml up -d
-		else
-			echo "The installation of the fluent-bit service was unsuccessful."
 		fi
+
+		[ ! -z "$ELK_HOST" ] && sed -i "s/ELK_CONTAINER_NAME/ELK_HOST/g" $BASE_DIR/fluent.yml ${BASE_DIR}/dashboards.yml
+
+		OPENSEARCH_INDEX="${OPENSEARCH_INDEX:-"${PACKAGE_SYSNAME}-fluent-bit"}"
+		if crontab -l | grep -q "${OPENSEARCH_INDEX}"; then
+			crontab < <(crontab -l | grep -v "${OPENSEARCH_INDEX}")
+		fi
+		(crontab -l 2>/dev/null; echo "0 0 */1 * * curl -s -X POST "$(get_env_parameter 'ELK_SHEME')"://${ELK_HOST:-127.0.0.1}:$(get_env_parameter 'ELK_PORT')/${OPENSEARCH_INDEX}/_delete_by_query -H 'Content-Type: application/json' -d '{\"query\": {\"range\": {\"@timestamp\": {\"lt\": \"now-30d\"}}}}'") | crontab -		
+
+		sed -i "s/OPENSEARCH_HOST/${ELK_HOST:-"${PACKAGE_SYSNAME}-opensearch"}/g" "${BASE_DIR}/config/fluent-bit.conf" 
+		sed -i "s/OPENSEARCH_PORT/$(get_env_parameter "ELK_PORT")/g" ${BASE_DIR}/config/fluent-bit.conf
+		sed -i "s/OPENSEARCH_INDEX/${OPENSEARCH_INDEX}/g" ${BASE_DIR}/config/fluent-bit.conf
+
+		reconfigure DASHBOARDS_USERNAME "${DASHBOARDS_USERNAME:-"${PACKAGE_SYSNAME}"}"
+		reconfigure DASHBOARDS_PASSWORD "${DASHBOARDS_PASSWORD:-$(get_random_str 20)}"
+
+		docker-compose -f ${BASE_DIR}/fluent.yml -f ${BASE_DIR}/dashboards.yml up -d
+	elif [ "$INSTALL_FLUENT_BIT" == "pull" ]; then
+		docker-compose -f ${BASE_DIR}/fluent.yml -f ${BASE_DIR}/dashboards.yml pull
 	fi
 }
 

install/deb/debian/rules

@@ -84,8 +84,12 @@ override_dh_auto_build: check_archives
 	sed -e '/.pid/d' -e '/temp_path/d' -e 's_etc/nginx_etc/openresty_g' -e 's/\.log/-openresty.log/g' -i ${BUILDTOOLS_PATH}/install/docker/config/nginx/templates/nginx.conf.template
 	mv -f ${BUILDTOOLS_PATH}/install/docker/config/nginx/onlyoffice-proxy-ssl.conf ${BUILDTOOLS_PATH}/install/docker/config/nginx/onlyoffice-proxy-ssl.conf.template
 	cp -rf ${BUILDTOOLS_PATH}/install/docker/config/nginx/onlyoffice-proxy.conf ${BUILDTOOLS_PATH}/install/docker/config/nginx/onlyoffice-proxy.conf.template
-	sed -i '/^\s*Name\s\+forward\s*/d; /^\s*Listen\s\+127\.0\.0\.1\s*/d; /^\s*Port\s\+24224\s*/d' ${BUILDTOOLS_PATH}/install/docker/config/fluent-bit.conf
+	sed -i "s#\(/var/log/onlyoffice/\)#\1${PRODUCT}/#" ${BUILDTOOLS_PATH}/install/docker/config/fluent-bit.conf 
-	sed -i '0,/\[INPUT\]/ s/\(\[INPUT\]\)/\1\n    Name tail\n    Path \/var\/log\/onlyoffice\/${PRODUCT}\/*.log\n    Path_Key filename/' ${BUILDTOOLS_PATH}/install/docker/config/fluent-bit.conf
+	sed -i '/^\[OUTPUT\]/i\[INPUT]' ${BUILDTOOLS_PATH}/install/docker/config/fluent-bit.conf 
+	sed -i '/^\[OUTPUT\]/i\    Name                exec' ${BUILDTOOLS_PATH}/install/docker/config/fluent-bit.conf 
+	sed -i '/^\[OUTPUT\]/i\    Interval_Sec        86400' ${BUILDTOOLS_PATH}/install/docker/config/fluent-bit.conf 
+	sed -i '/^\[OUTPUT\]/i\    Command             curl -s -X POST OPENSEARCH_SCHEME://OPENSEARCH_HOST:OPENSEARCH_PORT/OPENSEARCH_INDEX/_delete_by_query -H '\''Content-Type: application/json'\'' -d '\''{"query": {"range": {"@timestamp": {"lt": "now-30d"}}}}'\'' ${BUILDTOOLS_PATH}/install/docker/config/fluent-bit.conf 
+	sed -i '/^\[OUTPUT\]/i\\' ${BUILDTOOLS_PATH}/install/docker/config/fluent-bit.conf 
 
 	for i in ${PRODUCT} $$(ls ${CURDIR}/debian/*.install | grep -oP 'debian/\K.*' | grep -o '^[^.]*'); do \
 		cp ${CURDIR}/debian/source/lintian-overrides ${CURDIR}/debian/$$i.lintian-overrides; \

install/docker/.env

@@ -24,6 +24,8 @@
     DASHBOARDS_CONTAINER_NAME=${CONTAINER_PREFIX}opensearch-dashboards
     DASHBOARDS_USERNAME=onlyoffice
     DASHBOARDS_PASSWORD=onlyoffice
+    FLUENT_BIT_VERSION=3.0.2
+    FLUENT_BIT_CONTAINER_NAME=${CONTAINER_PREFIX}fluent-bit
 
 # app service environment #
     ENV_EXTENSION=none

install/docker/config/docspace-logs

@@ -1,33 +0,0 @@
-#!/bin/bash
-
-set -e
-
-PRODUCT="docspace"
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-DOCKERCOMPOSE=$(dirname "$DIR")
-
-if [ -f "${DOCKERCOMPOSE}/docspace.yml" ]; then 
-  :
-elif [ -f "/app/onlyoffice/${PRODUCT}.yml" ]; then
-  DOCKERCOMPOSE="/app/onlyoffice"
-else
-  echo "Error: yml files not found." && exit 1
-fi
-
-FILES=("${PRODUCT}" "notify" "healthchecks" "proxy" "ds" "rabbitmq" "redis" "opensearch" "dashboards" "db")
-
-LOG_DIR="${DOCKERCOMPOSE}/logs"
-mkdir -p ${LOG_DIR}
-
-echo "Creating ${PRODUCT} logs to a directory ${LOG_DIR}..."
-for FILE in "${FILES[@]}"; do
-  SERVICE_NAMES=($(docker-compose -f ${DOCKERCOMPOSE}/${FILE}.yml config --services))
-  for SERVICE_NAME in "${SERVICE_NAMES[@]}"; do
-    if [[ $(docker-compose -f ${DOCKERCOMPOSE}/${FILE}.yml ps -q ${SERVICE_NAME} | wc -l) -eq 1 ]]; then
-      docker-compose -f ${DOCKERCOMPOSE}/${FILE}.yml logs ${SERVICE_NAME} > ${LOG_DIR}/${SERVICE_NAME}.log
-    else
-      echo "The ${SERVICE_NAME} service is not running"
-    fi
-  done
-done
-echo "OK"

install/docker/config/fluent-bit.conf

@@ -4,14 +4,14 @@
     Daemon              off
 
 [INPUT]
-    Name                forward
+    Name                tail
-    Listen              127.0.0.1
+    Path                /var/log/onlyoffice/*.log, /var/log/onlyoffice/**/**/*.log
-    Port                24224
+    Exclude_Path        /var/log/onlyoffice/*.sql.log
-
+    Path_Key            filename
-[INPUT]
+    Mem_Buf_Limit       500MB
-    Name                exec
+    Refresh_Interval    60
-    Interval_Sec        86400
+    Ignore_Older        30d
-    Command             curl -s -X POST 'OPENSEARCH_SCHEME://OPENSEARCH_HOST:OPENSEARCH_PORT/OPENSEARCH_INDEX/_delete_by_query' -H 'Content-Type: application/json' -d "{\"query\": {\"range\": {\"@timestamp\": {\"lt\": \"$(date -u -d '30 days ago' '+%Y-%m-%dT%H:%M:%S')\"}}}}"
+    Skip_Empty_Lines    true
 
 [OUTPUT]
     Name                opensearch
@@ -20,6 +20,7 @@
     Port                OPENSEARCH_PORT
     Replace_Dots        On
     Suppress_Type_Name  On
+    Compress            gzip
     Time_Key            @timestamp  
     Type                _doc
     Index               OPENSEARCH_INDEX

install/docker/config/nginx/onlyoffice-proxy-ssl.conf

@@ -7,6 +7,9 @@ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 proxy_hide_header 'Server';
 proxy_hide_header 'X-Powered-By';
 
+access_log /var/log/nginx/access-proxy.log;
+error_log /var/log/nginx/error-proxy.log;
+
 ## HTTP host
 server {
 	listen 0.0.0.0:80;

install/docker/config/nginx/onlyoffice-proxy.conf

@@ -7,6 +7,9 @@ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 proxy_hide_header 'Server';
 proxy_hide_header 'X-Powered-By';
 
+access_log /var/log/nginx/access-proxy.log;
+error_log /var/log/nginx/error-proxy.log;
+
 server {
     listen 0.0.0.0:80;
     listen [::]:80 default_server;

install/docker/docker-entrypoint.py

@@ -200,6 +200,7 @@ writeJsonFile(filePath, jsonData)
 
 filePath = "/app/onlyoffice/config/appsettings.services.json"
 jsonData = openJsonFile(filePath)
+updateJsonData(jsonData,"$.logPath", LOG_DIR)
 updateJsonData(jsonData,"$.logLevel", LOG_LEVEL)
 writeJsonFile(filePath, jsonData)
 

install/docker/docspace.yml

@@ -53,6 +53,7 @@ x-service: &x-service-base
     CERTIFICATE_PATH: ${CERTIFICATE_PATH}
   volumes:
     #- /app/onlyoffice/CommunityServer/data:/app/onlyoffice/data
+    - log_data:/var/log/onlyoffice
     - app_data:/app/onlyoffice/data
     - files_data:/var/www/products/ASC.Files/server/
     - people_data:/var/www/products/ASC.People/server/
@@ -227,7 +228,7 @@ services:
       - DASHBOARDS_USERNAME=${DASHBOARDS_USERNAME}
       - DASHBOARDS_PASSWORD=${DASHBOARDS_PASSWORD}
     volumes:
-      - router_log:/var/log/nginx
+      - log_data:/var/log/nginx
 
 networks:
   default:
@@ -235,7 +236,7 @@ networks:
     external: true
 
 volumes:
-  router_log:
+  log_data:
   app_data:
   files_data:
   people_data:

install/docker/ds.yml

@@ -9,6 +9,7 @@ services:
       - JWT_HEADER=${DOCUMENT_SERVER_JWT_HEADER}
       - JWT_IN_BODY=true
     volumes:
+      - log_data:/var/log/onlyoffice
       - app_data:/var/www/onlyoffice/Data
     expose:
       - '80'
@@ -22,4 +23,5 @@ networks:
     external: true
 
 volumes:
+  log_data:
   app_data:

install/docker/fluent.yml

@@ -0,0 +1,19 @@
+services:
+  fluent-bit:
+    image: fluent/fluent-bit:${FLUENT_BIT_VERSION}
+    container_name: ${FLUENT_BIT_CONTAINER_NAME}
+    restart: always
+    environment:
+      - HOST=${ELK_CONTAINER_NAME}
+      - PORT=${ELK_PORT}
+    volumes:
+      - log_data:/var/log/onlyoffice
+      - ./config/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
+
+networks:
+  default:
+    name: ${NETWORK_NAME}
+    external: true
+
+volumes:
+  log_data:

install/docker/notify.yml

@@ -53,6 +53,7 @@ x-service:
       DEBUG_INFO: ${DEBUG_INFO}
     volumes:
     #- /app/onlyoffice/CommunityServer/data:/app/onlyoffice/data
+    - log_data:/var/log/onlyoffice
     - app_data:/app/onlyoffice/data
     - files_data:/var/www/products/ASC.Files/server/
     - people_data:/var/www/products/ASC.People/server/
@@ -72,6 +73,7 @@ networks:
     external: true
 
 volumes:
+  log_data:
   app_data:
   files_data:
   people_data:

install/docker/proxy-ssl.yml

@@ -23,7 +23,7 @@ services:
       - EXTERNAL_PORT=${EXTERNAL_PORT}
     volumes:
       - webroot_path:/letsencrypt
-      - proxy_log:/var/log/nginx
+      - log_data:/var/log/nginx
       - ./config/nginx/templates/nginx.conf.template:/etc/nginx/nginx.conf
       - ./config/nginx/letsencrypt.conf:/etc/nginx/includes/letsencrypt.conf
       - ./config/nginx/templates/proxy.upstream.conf.template:/etc/nginx/templates/proxy.upstream.conf.template:ro
@@ -38,5 +38,5 @@ networks:
     external: true
 
 volumes:
-  proxy_log:
+  log_data:
   webroot_path:

install/docker/proxy.yml

@@ -21,7 +21,7 @@ services:
       - EXTERNAL_PORT=${EXTERNAL_PORT}
     volumes:
       - webroot_path:/letsencrypt
-      - proxy_log:/var/log/nginx
+      - log_data:/var/log/nginx
       - ./config/nginx/templates/nginx.conf.template:/etc/nginx/nginx.conf
       - ./config/nginx/letsencrypt.conf:/etc/nginx/includes/letsencrypt.conf
       - ./config/nginx/templates/proxy.upstream.conf.template:/etc/nginx/templates/proxy.upstream.conf.template:ro
@@ -33,5 +33,5 @@ networks:
     external: true
 
 volumes:
-  proxy_log:
+  log_data:
   webroot_path:

install/rpm/SPECS/build.spec

@@ -30,8 +30,12 @@ sed -E 's_(http://)[^:]+(:5601)_\1localhost\2_g' -i config/nginx/onlyoffice.conf
 sed -e 's/$router_host/127.0.0.1/g' -e 's/this_host\|proxy_x_forwarded_host/host/g' -e 's/proxy_x_forwarded_proto/scheme/g' -e 's/proxy_x_forwarded_port/server_port/g' -e 's_includes_/etc/openresty/includes_g' -e '/quic\|alt-svc/Id' -i install/docker/config/nginx/onlyoffice-proxy*.conf
 sed -e '/.pid/d' -e '/temp_path/d' -e 's_etc/nginx_etc/openresty_g' -e 's/\.log/-openresty.log/g' -i install/docker/config/nginx/templates/nginx.conf.template
 sed -i "s_\(.*root\).*;_\1 \"/var/www/%{product}\";_g" -i install/docker/config/nginx/letsencrypt.conf
-sed -i '/^\s*Name\s\+forward\s*$/d; /^\s*Listen\s\+127\.0\.0\.1\s*$/d; /^\s*Port\s\+24224\s*$/d' -i install/docker/config/fluent-bit.conf
+sed -i "s#\(/var/log/onlyoffice/\)#\1%{product}#" install/docker/config/fluent-bit.conf
-sed -i "0,/\[INPUT\]/ s/\(\[INPUT\]\)/\1\n    Name tail\n    Path \/var\/log\/onlyoffice\/%{product}\/*.log\n    Path_Key filename/" -i install/docker/config/fluent-bit.conf
+sed -i '/^\[OUTPUT\]/i\[INPUT]' install/docker/config/fluent-bit.conf
+sed -i '/^\[OUTPUT\]/i\    Name                exec' install/docker/config/fluent-bit.conf
+sed -i '/^\[OUTPUT\]/i\    Interval_Sec        86400' install/docker/config/fluent-bit.conf
+sed -i '/^\[OUTPUT\]/i\    Command             curl -s -X POST OPENSEARCH_SCHEME://OPENSEARCH_HOST:OPENSEARCH_PORT/OPENSEARCH_INDEX/_delete_by_query -H '\''Content-Type: application/json'\'' -d '\''{"query": {"range": {"@timestamp": {"lt": "now-30d"}}}}'\'' ${BUILDTOOLS_PATH}/install/docker/config/fluent-bit.conf 
+sed -i '/^\[OUTPUT\]/i\\' install/docker/config/fluent-bit.conf
 
 find %{_builddir}/server/publish/ \
      %{_builddir}/server/ASC.Migration.Runner \

install/win/build-batch.bat

@@ -65,10 +65,12 @@ REM echo ######## SSL configs ########
 %sed% -i "s/\/etc\/nginx\/\.htpasswd_dashboards/\.htpasswd_dashboards/g" buildtools\install\win\Files\nginx\conf\onlyoffice.conf
 
 REM echo ######## Configure fluent-bit config for windows ########
-%sed% -i "s/forward/tail/" buildtools\install\win\Files\config\fluent-bit.conf
+%sed% -i -e "s|/var/log/onlyoffice/|{APPDIR}Logs\\|g" -e "s|\*\*/|\*\*\\|g" buildtools\install\win\Files\config\fluent-bit.conf
-%sed% -i "s/Port/Path/" buildtools\install\win\Files\config\fluent-bit.conf
+%sed% -i "/^\[OUTPUT\]/i\[INPUT]" buildtools\install\win\Files\config\fluent-bit.conf
-%sed% -i "s/24224/{APPDIR}Logs\*.log/" buildtools\install\win\Files\config\fluent-bit.conf
+%sed% -i "/^\[OUTPUT\]/i\    Name                exec" buildtools\install\win\Files\config\fluent-bit.conf
-%sed% -i "/Listen\s*127\.0\.0\.1/d" buildtools\install\win\Files\config\fluent-bit.conf
+%sed% -i "/^\[OUTPUT\]/i\    Interval_Sec        86400" buildtools\install\win\Files\config\fluent-bit.conf
+%sed% -i "/^\[OUTPUT\]/i\    Command             curl -s -X POST OPENSEARCH_SCHEME://OPENSEARCH_HOST:OPENSEARCH_PORT/OPENSEARCH_INDEX/_delete_by_query -H 'Content-Type: application/json' -d '{\"query\": {\"range\": {\"@timestamp\": {\"lt\": \"now-30d\"}}}}'" buildtools\install\win\Files\config\fluent-bit.conf
+%sed% -i "/^\[OUTPUT\]/i\ " buildtools\install\win\Files\config\fluent-bit.conf
 
 REM echo ######## Delete test and dev configs ########
 del /f /q buildtools\install\win\Files\config\*.test.json