IlyaSobolev/DocSpace-buildtools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add opensearch-dashboards and fluent-bit to retrieve docker logs (#170)

Browse Source 
* Migration to opensearch

* Implement indexes update on version change

* Add opensearch-dashboard and logstash to retrieve docker logs

* Replace logstash with fluent-bit to retrieve docker logs

* Add running fluent-bit logging to OCI

* Change fluent-bit supply to package delivery

* Implement index cleanup every 30 days via lua script

* Implement index cleanup via exec on timer

* Optimize indexes update on version change

* Add a check that fluent-bit has installed successfully

* Add the dashboard location and authorization for it

* Move dashboards location to router

* Migration to opensearch in docspace.profiles.yml

* Replace the naming with dashboards

* Add password generation for /dashboards/
This commit is contained in:
Evgeniy Antonyuk 2024-04-17 19:08:49 +07:00 committed by GitHub
parent 6913c108ea
commit fde212c05c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 143 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
config/nginx/onlyoffice.conf
View File

@ -157,6 +157,19 @@ server {
}
location ^~ /dashboards/ {
auth_basic "Restricted Access";
auth_basic_user_file /etc/nginx/.htpasswd_dashboards;
rewrite ^/dashboards(/.*)$ $1 break;
proxy_pass http://127.0.0.1:5601;
proxy_redirect off;
proxy_buffering off;
proxy_set_header Connection "Keep-Alive";
proxy_set_header Proxy-Connection "Keep-Alive";
}
location / {
proxy_pass http://127.0.0.1:5001;
proxy_redirect off;

68
install/OneClickInstall/install-Docker.sh
View File

@ -60,6 +60,7 @@ INSTALL_RABBITMQ="true";
INSTALL_MYSQL_SERVER="true";
INSTALL_DOCUMENT_SERVER="true";
INSTALL_ELASTICSEARCH="true";
INSTALL_FLUENT_BIT="true";
INSTALL_PRODUCT="true";
UPDATE="false";
@ -372,6 +373,13 @@ while [ "$1" != "" ]; do
fi
;;
-ifb | --installfluentbit )
if [ "$2" != "" ]; then
INSTALL_FLUENT_BIT=$2
shift
fi
;;
-rdsh | --redishost )
if [ "$2" != "" ]; then
REDIS_HOST=$2
@ -463,6 +471,20 @@ while [ "$1" != "" ]; do
fi
;;
-du | --dashboadrsusername )
if [ "$2" != "" ]; then
DASHBOARDS_USERNAME=$2
shift
fi
;;
-dp | --dashboadrspassword )
if [ "$2" != "" ]; then
DASHBOARDS_PASSWORD=$2
shift
fi
;;
-noni | --noninteractive )
if [ "$2" != "" ]; then
NON_INTERACTIVE=$2
@ -496,6 +518,9 @@ while [ "$1" != "" ]; do
echo " -irds, --installredis install or update redis (true|false)"
echo " -imysql, --installmysql install or update mysql (true|false)"
echo " -ies, --installelastic install or update elasticsearch (true|false)"
echo " -ifb, --installfluentbit install or update fluent-bit (true|false)"
echo " -du, --dashboadrsusername login for authorization in /dashboards/"
echo " -dp, --dashboadrspassword password for authorization in /dashboards/"
echo " -espr, --elasticprotocol the protocol for the connection to elasticsearch (default value http)"
echo " -esh, --elastichost the IP address or hostname of the elasticsearch"
echo " -esp, --elasticport elasticsearch port number (default value 9200)"
@ -1137,6 +1162,9 @@ set_docspace_params() {
RABBIT_PASSWORD=${RABBIT_PASSWORD:-$(get_env_parameter "RABBIT_PASSWORD" "${CONTAINER_NAME}")};
RABBIT_VIRTUAL_HOST=${RABBIT_VIRTUAL_HOST:-$(get_env_parameter "RABBIT_VIRTUAL_HOST" "${CONTAINER_NAME}")};
DASHBOARDS_USERNAME=${DASHBOARDS_USERNAME:-$(get_env_parameter "DASHBOARDS_USERNAME" "${CONTAINER_NAME}")};
DASHBOARDS_PASSWORD=${DASHBOARDS_PASSWORD:-$(get_env_parameter "DASHBOARDS_PASSWORD" "${CONTAINER_NAME}")};
CERTIFICATE_PATH=${CERTIFICATE_PATH:-$(get_env_parameter "CERTIFICATE_PATH")};
CERTIFICATE_KEY_PATH=${CERTIFICATE_KEY_PATH:-$(get_env_parameter "CERTIFICATE_KEY_PATH")};
DHPARAM_PATH=${DHPARAM_PATH:-$(get_env_parameter "DHPARAM_PATH")};
@ -1285,6 +1313,38 @@ install_elasticsearch () {
fi
}
install_fluent_bit () {
if [ "$INSTALL_FLUENT_BIT" == "true" ]; then
curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh
if systemctl list-unit-files --type=service | grep -q "fluent-bit.service"; then
sed -i "s/OPENSEARCH_SCHEME/$(get_env_parameter "ELK_SHEME")/g" "${BASE_DIR}/config/fluent-bit.conf"
sed -i "s/OPENSEARCH_HOST/${ELK_HOST:-127.0.0.1}/g" "${BASE_DIR}/config/fluent-bit.conf"
sed -i "s/OPENSEARCH_PORT/$(get_env_parameter "ELK_PORT")/g" ${BASE_DIR}/config/fluent-bit.conf
sed -i "s/OPENSEARCH_INDEX/${OPENSEARCH_INDEX:-"${PACKAGE_SYSNAME}-fluent-bit"}/g" ${BASE_DIR}/config/fluent-bit.conf
[ ! -z "${ELK_HOST}" ] && sed -i "s/ELK_CONTAINER_NAME/ELK_HOST/g" ${BASE_DIR}/dashboards.yml
cp -rf ${BASE_DIR}/config/fluent-bit.conf /etc/fluent-bit/fluent-bit.conf
systemctl restart fluent-bit
DOCKER_DAEMON_FILE="/etc/docker/daemon.json"
if [[ ! -f "${DOCKER_DAEMON_FILE}" ]]; then
echo "{\"log-driver\": \"fluentd\", \"log-opts\": { \"fluentd-address\": \"127.0.0.1:24224\" }}" > "${DOCKER_DAEMON_FILE}"
systemctl restart docker
elif ! grep -q "log-driver" ${DOCKER_DAEMON_FILE}; then
sed -i 's!{!& "log-driver": "fluentd", "log-opts": { "fluentd-address": "127.0.0.1:24224" },!' "${DOCKER_DAEMON_FILE}"
systemctl restart docker
fi
reconfigure DASHBOARDS_USERNAME "${DASHBOARDS_USERNAME:-"onlyoffice"}"
reconfigure DASHBOARDS_PASSWORD "${DASHBOARDS_PASSWORD:-$(get_random_str 20)}"
docker-compose -f ${BASE_DIR}/dashboards.yml up -d
else
echo "The installation of the fluent-bit service was unsuccessful."
fi
fi
}
install_product () {
DOCKER_TAG="${DOCKER_TAG:-$(get_available_version ${IMAGE_NAME})}"
reconfigure DOCKER_TAG ${DOCKER_TAG}
@ -1402,15 +1462,17 @@ start_installation () {
download_files
install_mysql_server
install_elasticsearch
install_document_server
install_fluent_bit
install_mysql_server
install_rabbitmq
install_redis
install_elasticsearch
install_document_server
install_product

8
install/docker/.env
View File

@ -8,18 +8,22 @@
CONTAINER_PREFIX=${PRODUCT}-
MYSQL_VERSION=8.3.0
MYSQL_IMAGE=mysql:${MYSQL_VERSION}
ELK_VERSION=2.11.1
SERVICE_PORT=5050
DOCUMENT_SERVER_IMAGE_NAME=onlyoffice/4testing-documentserver-ee:latest
DOCKERFILE=Dockerfile.app
APP_DOTNET_ENV=""
EXTERNAL_PORT="80"
# elasticsearch #
# opensearch stack #
ELK_VERSION=2.11.1
ELK_CONTAINER_NAME=${CONTAINER_PREFIX}opensearch
ELK_SHEME=http
ELK_HOST=""
ELK_PORT=9200
DASHBOARDS_VERSION=2.11.1
DASHBOARDS_CONTAINER_NAME=${CONTAINER_PREFIX}opensearch-dashboards
DASHBOARDS_USERNAME=onlyoffice
DASHBOARDS_PASSWORD=onlyoffice
# app service environment #
ENV_EXTENSION=none

1
install/docker/Dockerfile.app
View File

@ -173,6 +173,7 @@ RUN sed -i 's/127.0.0.1:5010/$service_api_system/' /etc/nginx/conf.d/onlyoffice.
if [[ -z "${SERVICE_CLIENT}" ]] ; then sed -i 's/127.0.0.1:5001/$service_client/' /etc/nginx/conf.d/onlyoffice.conf; fi && \
if [[ -z "${SERVICE_MANAGEMENT}" ]] ; then sed -i 's/127.0.0.1:5015/$service_management/' /etc/nginx/conf.d/onlyoffice.conf; fi && \
sed -i 's/127.0.0.1:5033/$service_healthchecks/' /etc/nginx/conf.d/onlyoffice.conf && \
sed -i 's/127.0.0.1:5601/$dashboards_host:5601/' /etc/nginx/conf.d/onlyoffice.conf && \
sed -i 's/$public_root/\/var\/www\/public\//' /etc/nginx/conf.d/onlyoffice.conf && \
sed -i 's/http:\/\/172.*/$document_server;/' /etc/nginx/conf.d/onlyoffice.conf && \
sed -i '/client_body_temp_path/ i \ \ \ \ $MAP_HASH_BUCKET_SIZE' /etc/nginx/nginx.conf.template && \

2
install/docker/config/docspace-logs
View File

@ -14,7 +14,7 @@ else
echo "Error: yml files not found." && exit 1
fi
FILES=("${PRODUCT}" "notify" "healthchecks" "proxy" "ds" "rabbitmq" "redis" "opensearch" "db")
FILES=("${PRODUCT}" "notify" "healthchecks" "proxy" "ds" "rabbitmq" "redis" "opensearch" "dashboards" "db")
LOG_DIR="${DOCKERCOMPOSE}/logs"
mkdir -p ${LOG_DIR}

25
install/docker/config/fluent-bit.conf Normal file
View File

@ -0,0 +1,25 @@
[SERVICE]
Flush 1
Log_Level info
Daemon off
[INPUT]
Name forward
Listen 127.0.0.1
Port 24224
[INPUT]
Name exec
Interval_Sec 86400
Command curl -s -X POST 'OPENSEARCH_SCHEME://OPENSEARCH_HOST:OPENSEARCH_PORT/OPENSEARCH_INDEX/_delete_by_query' -H 'Content-Type: application/json' -d "{\"query\": {\"range\": {\"@timestamp\": {\"lt\": \"$(date -u -d '30 days ago' '+%Y-%m-%dT%H:%M:%S')\"}}}}"
[OUTPUT]
Name opensearch
Match *
Host OPENSEARCH_HOST
Port OPENSEARCH_PORT
Replace_Dots On
Suppress_Type_Name On
Time_Key @timestamp
Type _doc
Index OPENSEARCH_INDEX

6
install/docker/config/nginx/templates/upstream.conf.template
View File

@ -83,3 +83,9 @@ map $SERVICE_CLIENT $service_client {
"" 127.0.0.1:5001;
default $SERVICE_CLIENT;
}
map $DASHBOARDS_CONTAINER_NAME $dashboards_host {
volatile;
default onlyoffice-opensearch-dashboards;
~^(.*)$ $1;
}

17
install/docker/dashboards.yml Normal file
View File

@ -0,0 +1,17 @@
version: "3"
services:
onlyoffice-opensearch-dashboards:
image: opensearchproject/opensearch-dashboards:${DASHBOARDS_VERSION}
container_name: ${DASHBOARDS_CONTAINER_NAME}
restart: always
environment:
- OPENSEARCH_HOSTS=${ELK_SHEME}://${ELK_CONTAINER_NAME}:${ELK_PORT}
- "DISABLE_SECURITY_DASHBOARDS_PLUGIN=true"
- "SERVER_BASEPATH=/dashboards"
expose:
- "5601"
networks:
default:
name: ${NETWORK_NAME}
external: true

1
install/docker/docspace.profiles.yml
View File

@ -279,6 +279,7 @@ services:
- REDIS_HOST=${REDIS_HOST}
- REDIS_PORT=${REDIS_PORT}
- SERVICE_PORT=${SERVICE_PORT}
- DASHBOARDS_CONTAINER_NAME=${DASHBOARDS_CONTAINER_NAME}
volumes:
- router_log:/var/log/nginx

3
install/docker/docspace.yml
View File

@ -223,6 +223,9 @@ services:
- REDIS_PORT=${REDIS_PORT}
- REDIS_PASSWORD=${REDIS_PASSWORD}
- SERVICE_PORT=${SERVICE_PORT}
- DASHBOARDS_CONTAINER_NAME=${DASHBOARDS_CONTAINER_NAME}
- DASHBOARDS_USERNAME=${DASHBOARDS_USERNAME}
- DASHBOARDS_PASSWORD=${DASHBOARDS_PASSWORD}
volumes:
- router_log:/var/log/nginx

3
install/docker/opensearch.yml
View File

@ -23,6 +23,9 @@ services:
expose:
- "9200"
- "9600" # required for Performance Analyzer
ports:
- 127.0.0.1:9200:9200
networks:
default:
name: ${NETWORK_NAME}

3
install/docker/prepare-nginx-router.sh
View File

@ -1,4 +1,4 @@
#!/bin/sh
#!/bin/bash
WRONG_PORTAL_NAME_URL=${WRONG_PORTAL_NAME_URL:-""}
REDIS_HOST=${REDIS_HOST:-"${REDIS_CONTAINER_NAME}"}
REDIS_PORT=${REDIS_PORT:-"6379"}
@ -9,3 +9,4 @@ sed -i "s~\(redis_host =\).*~\1 \"$REDIS_HOST\"~" /etc/nginx/conf.d/onlyoffice.c
sed -i "s~\(redis_port =\).*~\1 $REDIS_PORT~" /etc/nginx/conf.d/onlyoffice.conf
sed -i "s~\(redis_pass =\).*~\1 \"$REDIS_PASSWORD\"~" /etc/nginx/conf.d/onlyoffice.conf
sed -i "s~\(\"wrongPortalNameUrl\":\).*,~\1 \"${WRONG_PORTAL_NAME_URL}\",~g" /var/www/public/scripts/config.json
echo "${DASHBOARDS_USERNAME:-onlyoffice}:$(openssl passwd -6 -stdin <<< "${DASHBOARDS_PASSWORD:-onlyoffice}")" > /etc/nginx/.htpasswd_dashboards