Add opensearch-dashboards and fluent-bit to retrieve docker logs (#170)
* Migration to opensearch * Implement indexes update on version change * Add opensearch-dashboard and logstash to retrieve docker logs * Replace logstash with fluent-bit to retrieve docker logs * Add running fluent-bit logging to OCI * Change fluent-bit supply to package delivery * Implement index cleanup every 30 days via lua script * Implement index cleanup via exec on timer * Optimize indexes update on version change * Add a check that fluent-bit has installed successfully * Add the dashboard location and authorization for it * Move dashboards location to router * Migration to opensearch in docspace.profiles.yml * Replace the naming with dashboards * Add password generation for /dashboards/
This commit is contained in:
parent
6913c108ea
commit
fde212c05c
@ -157,6 +157,19 @@ server {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
location ^~ /dashboards/ {
|
||||||
|
auth_basic "Restricted Access";
|
||||||
|
auth_basic_user_file /etc/nginx/.htpasswd_dashboards;
|
||||||
|
|
||||||
|
rewrite ^/dashboards(/.*)$ $1 break;
|
||||||
|
proxy_pass http://127.0.0.1:5601;
|
||||||
|
proxy_redirect off;
|
||||||
|
proxy_buffering off;
|
||||||
|
|
||||||
|
proxy_set_header Connection "Keep-Alive";
|
||||||
|
proxy_set_header Proxy-Connection "Keep-Alive";
|
||||||
|
}
|
||||||
|
|
||||||
location / {
|
location / {
|
||||||
proxy_pass http://127.0.0.1:5001;
|
proxy_pass http://127.0.0.1:5001;
|
||||||
proxy_redirect off;
|
proxy_redirect off;
|
||||||
|
@ -60,6 +60,7 @@ INSTALL_RABBITMQ="true";
|
|||||||
INSTALL_MYSQL_SERVER="true";
|
INSTALL_MYSQL_SERVER="true";
|
||||||
INSTALL_DOCUMENT_SERVER="true";
|
INSTALL_DOCUMENT_SERVER="true";
|
||||||
INSTALL_ELASTICSEARCH="true";
|
INSTALL_ELASTICSEARCH="true";
|
||||||
|
INSTALL_FLUENT_BIT="true";
|
||||||
INSTALL_PRODUCT="true";
|
INSTALL_PRODUCT="true";
|
||||||
UPDATE="false";
|
UPDATE="false";
|
||||||
|
|
||||||
@ -372,6 +373,13 @@ while [ "$1" != "" ]; do
|
|||||||
fi
|
fi
|
||||||
;;
|
;;
|
||||||
|
|
||||||
|
-ifb | --installfluentbit )
|
||||||
|
if [ "$2" != "" ]; then
|
||||||
|
INSTALL_FLUENT_BIT=$2
|
||||||
|
shift
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
|
||||||
-rdsh | --redishost )
|
-rdsh | --redishost )
|
||||||
if [ "$2" != "" ]; then
|
if [ "$2" != "" ]; then
|
||||||
REDIS_HOST=$2
|
REDIS_HOST=$2
|
||||||
@ -463,6 +471,20 @@ while [ "$1" != "" ]; do
|
|||||||
fi
|
fi
|
||||||
;;
|
;;
|
||||||
|
|
||||||
|
-du | --dashboadrsusername )
|
||||||
|
if [ "$2" != "" ]; then
|
||||||
|
DASHBOARDS_USERNAME=$2
|
||||||
|
shift
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
|
||||||
|
-dp | --dashboadrspassword )
|
||||||
|
if [ "$2" != "" ]; then
|
||||||
|
DASHBOARDS_PASSWORD=$2
|
||||||
|
shift
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
|
||||||
-noni | --noninteractive )
|
-noni | --noninteractive )
|
||||||
if [ "$2" != "" ]; then
|
if [ "$2" != "" ]; then
|
||||||
NON_INTERACTIVE=$2
|
NON_INTERACTIVE=$2
|
||||||
@ -496,6 +518,9 @@ while [ "$1" != "" ]; do
|
|||||||
echo " -irds, --installredis install or update redis (true|false)"
|
echo " -irds, --installredis install or update redis (true|false)"
|
||||||
echo " -imysql, --installmysql install or update mysql (true|false)"
|
echo " -imysql, --installmysql install or update mysql (true|false)"
|
||||||
echo " -ies, --installelastic install or update elasticsearch (true|false)"
|
echo " -ies, --installelastic install or update elasticsearch (true|false)"
|
||||||
|
echo " -ifb, --installfluentbit install or update fluent-bit (true|false)"
|
||||||
|
echo " -du, --dashboadrsusername login for authorization in /dashboards/"
|
||||||
|
echo " -dp, --dashboadrspassword password for authorization in /dashboards/"
|
||||||
echo " -espr, --elasticprotocol the protocol for the connection to elasticsearch (default value http)"
|
echo " -espr, --elasticprotocol the protocol for the connection to elasticsearch (default value http)"
|
||||||
echo " -esh, --elastichost the IP address or hostname of the elasticsearch"
|
echo " -esh, --elastichost the IP address or hostname of the elasticsearch"
|
||||||
echo " -esp, --elasticport elasticsearch port number (default value 9200)"
|
echo " -esp, --elasticport elasticsearch port number (default value 9200)"
|
||||||
@ -1137,6 +1162,9 @@ set_docspace_params() {
|
|||||||
RABBIT_PASSWORD=${RABBIT_PASSWORD:-$(get_env_parameter "RABBIT_PASSWORD" "${CONTAINER_NAME}")};
|
RABBIT_PASSWORD=${RABBIT_PASSWORD:-$(get_env_parameter "RABBIT_PASSWORD" "${CONTAINER_NAME}")};
|
||||||
RABBIT_VIRTUAL_HOST=${RABBIT_VIRTUAL_HOST:-$(get_env_parameter "RABBIT_VIRTUAL_HOST" "${CONTAINER_NAME}")};
|
RABBIT_VIRTUAL_HOST=${RABBIT_VIRTUAL_HOST:-$(get_env_parameter "RABBIT_VIRTUAL_HOST" "${CONTAINER_NAME}")};
|
||||||
|
|
||||||
|
DASHBOARDS_USERNAME=${DASHBOARDS_USERNAME:-$(get_env_parameter "DASHBOARDS_USERNAME" "${CONTAINER_NAME}")};
|
||||||
|
DASHBOARDS_PASSWORD=${DASHBOARDS_PASSWORD:-$(get_env_parameter "DASHBOARDS_PASSWORD" "${CONTAINER_NAME}")};
|
||||||
|
|
||||||
CERTIFICATE_PATH=${CERTIFICATE_PATH:-$(get_env_parameter "CERTIFICATE_PATH")};
|
CERTIFICATE_PATH=${CERTIFICATE_PATH:-$(get_env_parameter "CERTIFICATE_PATH")};
|
||||||
CERTIFICATE_KEY_PATH=${CERTIFICATE_KEY_PATH:-$(get_env_parameter "CERTIFICATE_KEY_PATH")};
|
CERTIFICATE_KEY_PATH=${CERTIFICATE_KEY_PATH:-$(get_env_parameter "CERTIFICATE_KEY_PATH")};
|
||||||
DHPARAM_PATH=${DHPARAM_PATH:-$(get_env_parameter "DHPARAM_PATH")};
|
DHPARAM_PATH=${DHPARAM_PATH:-$(get_env_parameter "DHPARAM_PATH")};
|
||||||
@ -1285,6 +1313,38 @@ install_elasticsearch () {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
install_fluent_bit () {
|
||||||
|
if [ "$INSTALL_FLUENT_BIT" == "true" ]; then
|
||||||
|
curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh
|
||||||
|
|
||||||
|
if systemctl list-unit-files --type=service | grep -q "fluent-bit.service"; then
|
||||||
|
sed -i "s/OPENSEARCH_SCHEME/$(get_env_parameter "ELK_SHEME")/g" "${BASE_DIR}/config/fluent-bit.conf"
|
||||||
|
sed -i "s/OPENSEARCH_HOST/${ELK_HOST:-127.0.0.1}/g" "${BASE_DIR}/config/fluent-bit.conf"
|
||||||
|
sed -i "s/OPENSEARCH_PORT/$(get_env_parameter "ELK_PORT")/g" ${BASE_DIR}/config/fluent-bit.conf
|
||||||
|
sed -i "s/OPENSEARCH_INDEX/${OPENSEARCH_INDEX:-"${PACKAGE_SYSNAME}-fluent-bit"}/g" ${BASE_DIR}/config/fluent-bit.conf
|
||||||
|
[ ! -z "${ELK_HOST}" ] && sed -i "s/ELK_CONTAINER_NAME/ELK_HOST/g" ${BASE_DIR}/dashboards.yml
|
||||||
|
cp -rf ${BASE_DIR}/config/fluent-bit.conf /etc/fluent-bit/fluent-bit.conf
|
||||||
|
systemctl restart fluent-bit
|
||||||
|
|
||||||
|
DOCKER_DAEMON_FILE="/etc/docker/daemon.json"
|
||||||
|
if [[ ! -f "${DOCKER_DAEMON_FILE}" ]]; then
|
||||||
|
echo "{\"log-driver\": \"fluentd\", \"log-opts\": { \"fluentd-address\": \"127.0.0.1:24224\" }}" > "${DOCKER_DAEMON_FILE}"
|
||||||
|
systemctl restart docker
|
||||||
|
elif ! grep -q "log-driver" ${DOCKER_DAEMON_FILE}; then
|
||||||
|
sed -i 's!{!& "log-driver": "fluentd", "log-opts": { "fluentd-address": "127.0.0.1:24224" },!' "${DOCKER_DAEMON_FILE}"
|
||||||
|
systemctl restart docker
|
||||||
|
fi
|
||||||
|
|
||||||
|
reconfigure DASHBOARDS_USERNAME "${DASHBOARDS_USERNAME:-"onlyoffice"}"
|
||||||
|
reconfigure DASHBOARDS_PASSWORD "${DASHBOARDS_PASSWORD:-$(get_random_str 20)}"
|
||||||
|
|
||||||
|
docker-compose -f ${BASE_DIR}/dashboards.yml up -d
|
||||||
|
else
|
||||||
|
echo "The installation of the fluent-bit service was unsuccessful."
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
install_product () {
|
install_product () {
|
||||||
DOCKER_TAG="${DOCKER_TAG:-$(get_available_version ${IMAGE_NAME})}"
|
DOCKER_TAG="${DOCKER_TAG:-$(get_available_version ${IMAGE_NAME})}"
|
||||||
reconfigure DOCKER_TAG ${DOCKER_TAG}
|
reconfigure DOCKER_TAG ${DOCKER_TAG}
|
||||||
@ -1402,15 +1462,17 @@ start_installation () {
|
|||||||
|
|
||||||
download_files
|
download_files
|
||||||
|
|
||||||
|
install_elasticsearch
|
||||||
|
|
||||||
|
install_fluent_bit
|
||||||
|
|
||||||
install_mysql_server
|
install_mysql_server
|
||||||
|
|
||||||
install_document_server
|
|
||||||
|
|
||||||
install_rabbitmq
|
install_rabbitmq
|
||||||
|
|
||||||
install_redis
|
install_redis
|
||||||
|
|
||||||
install_elasticsearch
|
install_document_server
|
||||||
|
|
||||||
install_product
|
install_product
|
||||||
|
|
||||||
|
@ -8,18 +8,22 @@
|
|||||||
CONTAINER_PREFIX=${PRODUCT}-
|
CONTAINER_PREFIX=${PRODUCT}-
|
||||||
MYSQL_VERSION=8.3.0
|
MYSQL_VERSION=8.3.0
|
||||||
MYSQL_IMAGE=mysql:${MYSQL_VERSION}
|
MYSQL_IMAGE=mysql:${MYSQL_VERSION}
|
||||||
ELK_VERSION=2.11.1
|
|
||||||
SERVICE_PORT=5050
|
SERVICE_PORT=5050
|
||||||
DOCUMENT_SERVER_IMAGE_NAME=onlyoffice/4testing-documentserver-ee:latest
|
DOCUMENT_SERVER_IMAGE_NAME=onlyoffice/4testing-documentserver-ee:latest
|
||||||
DOCKERFILE=Dockerfile.app
|
DOCKERFILE=Dockerfile.app
|
||||||
APP_DOTNET_ENV=""
|
APP_DOTNET_ENV=""
|
||||||
EXTERNAL_PORT="80"
|
EXTERNAL_PORT="80"
|
||||||
|
|
||||||
# elasticsearch #
|
# opensearch stack #
|
||||||
|
ELK_VERSION=2.11.1
|
||||||
ELK_CONTAINER_NAME=${CONTAINER_PREFIX}opensearch
|
ELK_CONTAINER_NAME=${CONTAINER_PREFIX}opensearch
|
||||||
ELK_SHEME=http
|
ELK_SHEME=http
|
||||||
ELK_HOST=""
|
ELK_HOST=""
|
||||||
ELK_PORT=9200
|
ELK_PORT=9200
|
||||||
|
DASHBOARDS_VERSION=2.11.1
|
||||||
|
DASHBOARDS_CONTAINER_NAME=${CONTAINER_PREFIX}opensearch-dashboards
|
||||||
|
DASHBOARDS_USERNAME=onlyoffice
|
||||||
|
DASHBOARDS_PASSWORD=onlyoffice
|
||||||
|
|
||||||
# app service environment #
|
# app service environment #
|
||||||
ENV_EXTENSION=none
|
ENV_EXTENSION=none
|
||||||
|
@ -173,6 +173,7 @@ RUN sed -i 's/127.0.0.1:5010/$service_api_system/' /etc/nginx/conf.d/onlyoffice.
|
|||||||
if [[ -z "${SERVICE_CLIENT}" ]] ; then sed -i 's/127.0.0.1:5001/$service_client/' /etc/nginx/conf.d/onlyoffice.conf; fi && \
|
if [[ -z "${SERVICE_CLIENT}" ]] ; then sed -i 's/127.0.0.1:5001/$service_client/' /etc/nginx/conf.d/onlyoffice.conf; fi && \
|
||||||
if [[ -z "${SERVICE_MANAGEMENT}" ]] ; then sed -i 's/127.0.0.1:5015/$service_management/' /etc/nginx/conf.d/onlyoffice.conf; fi && \
|
if [[ -z "${SERVICE_MANAGEMENT}" ]] ; then sed -i 's/127.0.0.1:5015/$service_management/' /etc/nginx/conf.d/onlyoffice.conf; fi && \
|
||||||
sed -i 's/127.0.0.1:5033/$service_healthchecks/' /etc/nginx/conf.d/onlyoffice.conf && \
|
sed -i 's/127.0.0.1:5033/$service_healthchecks/' /etc/nginx/conf.d/onlyoffice.conf && \
|
||||||
|
sed -i 's/127.0.0.1:5601/$dashboards_host:5601/' /etc/nginx/conf.d/onlyoffice.conf && \
|
||||||
sed -i 's/$public_root/\/var\/www\/public\//' /etc/nginx/conf.d/onlyoffice.conf && \
|
sed -i 's/$public_root/\/var\/www\/public\//' /etc/nginx/conf.d/onlyoffice.conf && \
|
||||||
sed -i 's/http:\/\/172.*/$document_server;/' /etc/nginx/conf.d/onlyoffice.conf && \
|
sed -i 's/http:\/\/172.*/$document_server;/' /etc/nginx/conf.d/onlyoffice.conf && \
|
||||||
sed -i '/client_body_temp_path/ i \ \ \ \ $MAP_HASH_BUCKET_SIZE' /etc/nginx/nginx.conf.template && \
|
sed -i '/client_body_temp_path/ i \ \ \ \ $MAP_HASH_BUCKET_SIZE' /etc/nginx/nginx.conf.template && \
|
||||||
|
@ -14,7 +14,7 @@ else
|
|||||||
echo "Error: yml files not found." && exit 1
|
echo "Error: yml files not found." && exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
FILES=("${PRODUCT}" "notify" "healthchecks" "proxy" "ds" "rabbitmq" "redis" "opensearch" "db")
|
FILES=("${PRODUCT}" "notify" "healthchecks" "proxy" "ds" "rabbitmq" "redis" "opensearch" "dashboards" "db")
|
||||||
|
|
||||||
LOG_DIR="${DOCKERCOMPOSE}/logs"
|
LOG_DIR="${DOCKERCOMPOSE}/logs"
|
||||||
mkdir -p ${LOG_DIR}
|
mkdir -p ${LOG_DIR}
|
||||||
|
25
install/docker/config/fluent-bit.conf
Normal file
25
install/docker/config/fluent-bit.conf
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
[SERVICE]
|
||||||
|
Flush 1
|
||||||
|
Log_Level info
|
||||||
|
Daemon off
|
||||||
|
|
||||||
|
[INPUT]
|
||||||
|
Name forward
|
||||||
|
Listen 127.0.0.1
|
||||||
|
Port 24224
|
||||||
|
|
||||||
|
[INPUT]
|
||||||
|
Name exec
|
||||||
|
Interval_Sec 86400
|
||||||
|
Command curl -s -X POST 'OPENSEARCH_SCHEME://OPENSEARCH_HOST:OPENSEARCH_PORT/OPENSEARCH_INDEX/_delete_by_query' -H 'Content-Type: application/json' -d "{\"query\": {\"range\": {\"@timestamp\": {\"lt\": \"$(date -u -d '30 days ago' '+%Y-%m-%dT%H:%M:%S')\"}}}}"
|
||||||
|
|
||||||
|
[OUTPUT]
|
||||||
|
Name opensearch
|
||||||
|
Match *
|
||||||
|
Host OPENSEARCH_HOST
|
||||||
|
Port OPENSEARCH_PORT
|
||||||
|
Replace_Dots On
|
||||||
|
Suppress_Type_Name On
|
||||||
|
Time_Key @timestamp
|
||||||
|
Type _doc
|
||||||
|
Index OPENSEARCH_INDEX
|
@ -83,3 +83,9 @@ map $SERVICE_CLIENT $service_client {
|
|||||||
"" 127.0.0.1:5001;
|
"" 127.0.0.1:5001;
|
||||||
default $SERVICE_CLIENT;
|
default $SERVICE_CLIENT;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
map $DASHBOARDS_CONTAINER_NAME $dashboards_host {
|
||||||
|
volatile;
|
||||||
|
default onlyoffice-opensearch-dashboards;
|
||||||
|
~^(.*)$ $1;
|
||||||
|
}
|
||||||
|
17
install/docker/dashboards.yml
Normal file
17
install/docker/dashboards.yml
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
version: "3"
|
||||||
|
services:
|
||||||
|
onlyoffice-opensearch-dashboards:
|
||||||
|
image: opensearchproject/opensearch-dashboards:${DASHBOARDS_VERSION}
|
||||||
|
container_name: ${DASHBOARDS_CONTAINER_NAME}
|
||||||
|
restart: always
|
||||||
|
environment:
|
||||||
|
- OPENSEARCH_HOSTS=${ELK_SHEME}://${ELK_CONTAINER_NAME}:${ELK_PORT}
|
||||||
|
- "DISABLE_SECURITY_DASHBOARDS_PLUGIN=true"
|
||||||
|
- "SERVER_BASEPATH=/dashboards"
|
||||||
|
expose:
|
||||||
|
- "5601"
|
||||||
|
|
||||||
|
networks:
|
||||||
|
default:
|
||||||
|
name: ${NETWORK_NAME}
|
||||||
|
external: true
|
@ -279,6 +279,7 @@ services:
|
|||||||
- REDIS_HOST=${REDIS_HOST}
|
- REDIS_HOST=${REDIS_HOST}
|
||||||
- REDIS_PORT=${REDIS_PORT}
|
- REDIS_PORT=${REDIS_PORT}
|
||||||
- SERVICE_PORT=${SERVICE_PORT}
|
- SERVICE_PORT=${SERVICE_PORT}
|
||||||
|
- DASHBOARDS_CONTAINER_NAME=${DASHBOARDS_CONTAINER_NAME}
|
||||||
volumes:
|
volumes:
|
||||||
- router_log:/var/log/nginx
|
- router_log:/var/log/nginx
|
||||||
|
|
||||||
|
@ -223,6 +223,9 @@ services:
|
|||||||
- REDIS_PORT=${REDIS_PORT}
|
- REDIS_PORT=${REDIS_PORT}
|
||||||
- REDIS_PASSWORD=${REDIS_PASSWORD}
|
- REDIS_PASSWORD=${REDIS_PASSWORD}
|
||||||
- SERVICE_PORT=${SERVICE_PORT}
|
- SERVICE_PORT=${SERVICE_PORT}
|
||||||
|
- DASHBOARDS_CONTAINER_NAME=${DASHBOARDS_CONTAINER_NAME}
|
||||||
|
- DASHBOARDS_USERNAME=${DASHBOARDS_USERNAME}
|
||||||
|
- DASHBOARDS_PASSWORD=${DASHBOARDS_PASSWORD}
|
||||||
volumes:
|
volumes:
|
||||||
- router_log:/var/log/nginx
|
- router_log:/var/log/nginx
|
||||||
|
|
||||||
|
@ -23,6 +23,9 @@ services:
|
|||||||
expose:
|
expose:
|
||||||
- "9200"
|
- "9200"
|
||||||
- "9600" # required for Performance Analyzer
|
- "9600" # required for Performance Analyzer
|
||||||
|
ports:
|
||||||
|
- 127.0.0.1:9200:9200
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
default:
|
default:
|
||||||
name: ${NETWORK_NAME}
|
name: ${NETWORK_NAME}
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
#!/bin/sh
|
#!/bin/bash
|
||||||
WRONG_PORTAL_NAME_URL=${WRONG_PORTAL_NAME_URL:-""}
|
WRONG_PORTAL_NAME_URL=${WRONG_PORTAL_NAME_URL:-""}
|
||||||
REDIS_HOST=${REDIS_HOST:-"${REDIS_CONTAINER_NAME}"}
|
REDIS_HOST=${REDIS_HOST:-"${REDIS_CONTAINER_NAME}"}
|
||||||
REDIS_PORT=${REDIS_PORT:-"6379"}
|
REDIS_PORT=${REDIS_PORT:-"6379"}
|
||||||
@ -9,3 +9,4 @@ sed -i "s~\(redis_host =\).*~\1 \"$REDIS_HOST\"~" /etc/nginx/conf.d/onlyoffice.c
|
|||||||
sed -i "s~\(redis_port =\).*~\1 $REDIS_PORT~" /etc/nginx/conf.d/onlyoffice.conf
|
sed -i "s~\(redis_port =\).*~\1 $REDIS_PORT~" /etc/nginx/conf.d/onlyoffice.conf
|
||||||
sed -i "s~\(redis_pass =\).*~\1 \"$REDIS_PASSWORD\"~" /etc/nginx/conf.d/onlyoffice.conf
|
sed -i "s~\(redis_pass =\).*~\1 \"$REDIS_PASSWORD\"~" /etc/nginx/conf.d/onlyoffice.conf
|
||||||
sed -i "s~\(\"wrongPortalNameUrl\":\).*,~\1 \"${WRONG_PORTAL_NAME_URL}\",~g" /var/www/public/scripts/config.json
|
sed -i "s~\(\"wrongPortalNameUrl\":\).*,~\1 \"${WRONG_PORTAL_NAME_URL}\",~g" /var/www/public/scripts/config.json
|
||||||
|
echo "${DASHBOARDS_USERNAME:-onlyoffice}:$(openssl passwd -6 -stdin <<< "${DASHBOARDS_PASSWORD:-onlyoffice}")" > /etc/nginx/.htpasswd_dashboards
|
||||||
|
Loading…
Reference in New Issue
Block a user