Add opensearch-dashboards and fluent-bit to retrieve docker logs (#170)
* Migration to opensearch * Implement indexes update on version change * Add opensearch-dashboard and logstash to retrieve docker logs * Replace logstash with fluent-bit to retrieve docker logs * Add running fluent-bit logging to OCI * Change fluent-bit supply to package delivery * Implement index cleanup every 30 days via lua script * Implement index cleanup via exec on timer * Optimize indexes update on version change * Add a check that fluent-bit has installed successfully * Add the dashboard location and authorization for it * Move dashboards location to router * Migration to opensearch in docspace.profiles.yml * Replace the naming with dashboards * Add password generation for /dashboards/
This commit is contained in:
parent
6913c108ea
commit
fde212c05c
@ -157,6 +157,19 @@ server {
|
||||
|
||||
}
|
||||
|
||||
location ^~ /dashboards/ {
|
||||
auth_basic "Restricted Access";
|
||||
auth_basic_user_file /etc/nginx/.htpasswd_dashboards;
|
||||
|
||||
rewrite ^/dashboards(/.*)$ $1 break;
|
||||
proxy_pass http://127.0.0.1:5601;
|
||||
proxy_redirect off;
|
||||
proxy_buffering off;
|
||||
|
||||
proxy_set_header Connection "Keep-Alive";
|
||||
proxy_set_header Proxy-Connection "Keep-Alive";
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:5001;
|
||||
proxy_redirect off;
|
||||
|
@ -60,6 +60,7 @@ INSTALL_RABBITMQ="true";
|
||||
INSTALL_MYSQL_SERVER="true";
|
||||
INSTALL_DOCUMENT_SERVER="true";
|
||||
INSTALL_ELASTICSEARCH="true";
|
||||
INSTALL_FLUENT_BIT="true";
|
||||
INSTALL_PRODUCT="true";
|
||||
UPDATE="false";
|
||||
|
||||
@ -372,6 +373,13 @@ while [ "$1" != "" ]; do
|
||||
fi
|
||||
;;
|
||||
|
||||
-ifb | --installfluentbit )
|
||||
if [ "$2" != "" ]; then
|
||||
INSTALL_FLUENT_BIT=$2
|
||||
shift
|
||||
fi
|
||||
;;
|
||||
|
||||
-rdsh | --redishost )
|
||||
if [ "$2" != "" ]; then
|
||||
REDIS_HOST=$2
|
||||
@ -463,6 +471,20 @@ while [ "$1" != "" ]; do
|
||||
fi
|
||||
;;
|
||||
|
||||
-du | --dashboadrsusername )
|
||||
if [ "$2" != "" ]; then
|
||||
DASHBOARDS_USERNAME=$2
|
||||
shift
|
||||
fi
|
||||
;;
|
||||
|
||||
-dp | --dashboadrspassword )
|
||||
if [ "$2" != "" ]; then
|
||||
DASHBOARDS_PASSWORD=$2
|
||||
shift
|
||||
fi
|
||||
;;
|
||||
|
||||
-noni | --noninteractive )
|
||||
if [ "$2" != "" ]; then
|
||||
NON_INTERACTIVE=$2
|
||||
@ -496,6 +518,9 @@ while [ "$1" != "" ]; do
|
||||
echo " -irds, --installredis install or update redis (true|false)"
|
||||
echo " -imysql, --installmysql install or update mysql (true|false)"
|
||||
echo " -ies, --installelastic install or update elasticsearch (true|false)"
|
||||
echo " -ifb, --installfluentbit install or update fluent-bit (true|false)"
|
||||
echo " -du, --dashboadrsusername login for authorization in /dashboards/"
|
||||
echo " -dp, --dashboadrspassword password for authorization in /dashboards/"
|
||||
echo " -espr, --elasticprotocol the protocol for the connection to elasticsearch (default value http)"
|
||||
echo " -esh, --elastichost the IP address or hostname of the elasticsearch"
|
||||
echo " -esp, --elasticport elasticsearch port number (default value 9200)"
|
||||
@ -1137,6 +1162,9 @@ set_docspace_params() {
|
||||
RABBIT_PASSWORD=${RABBIT_PASSWORD:-$(get_env_parameter "RABBIT_PASSWORD" "${CONTAINER_NAME}")};
|
||||
RABBIT_VIRTUAL_HOST=${RABBIT_VIRTUAL_HOST:-$(get_env_parameter "RABBIT_VIRTUAL_HOST" "${CONTAINER_NAME}")};
|
||||
|
||||
DASHBOARDS_USERNAME=${DASHBOARDS_USERNAME:-$(get_env_parameter "DASHBOARDS_USERNAME" "${CONTAINER_NAME}")};
|
||||
DASHBOARDS_PASSWORD=${DASHBOARDS_PASSWORD:-$(get_env_parameter "DASHBOARDS_PASSWORD" "${CONTAINER_NAME}")};
|
||||
|
||||
CERTIFICATE_PATH=${CERTIFICATE_PATH:-$(get_env_parameter "CERTIFICATE_PATH")};
|
||||
CERTIFICATE_KEY_PATH=${CERTIFICATE_KEY_PATH:-$(get_env_parameter "CERTIFICATE_KEY_PATH")};
|
||||
DHPARAM_PATH=${DHPARAM_PATH:-$(get_env_parameter "DHPARAM_PATH")};
|
||||
@ -1285,6 +1313,38 @@ install_elasticsearch () {
|
||||
fi
|
||||
}
|
||||
|
||||
install_fluent_bit () {
|
||||
if [ "$INSTALL_FLUENT_BIT" == "true" ]; then
|
||||
curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh
|
||||
|
||||
if systemctl list-unit-files --type=service | grep -q "fluent-bit.service"; then
|
||||
sed -i "s/OPENSEARCH_SCHEME/$(get_env_parameter "ELK_SHEME")/g" "${BASE_DIR}/config/fluent-bit.conf"
|
||||
sed -i "s/OPENSEARCH_HOST/${ELK_HOST:-127.0.0.1}/g" "${BASE_DIR}/config/fluent-bit.conf"
|
||||
sed -i "s/OPENSEARCH_PORT/$(get_env_parameter "ELK_PORT")/g" ${BASE_DIR}/config/fluent-bit.conf
|
||||
sed -i "s/OPENSEARCH_INDEX/${OPENSEARCH_INDEX:-"${PACKAGE_SYSNAME}-fluent-bit"}/g" ${BASE_DIR}/config/fluent-bit.conf
|
||||
[ ! -z "${ELK_HOST}" ] && sed -i "s/ELK_CONTAINER_NAME/ELK_HOST/g" ${BASE_DIR}/dashboards.yml
|
||||
cp -rf ${BASE_DIR}/config/fluent-bit.conf /etc/fluent-bit/fluent-bit.conf
|
||||
systemctl restart fluent-bit
|
||||
|
||||
DOCKER_DAEMON_FILE="/etc/docker/daemon.json"
|
||||
if [[ ! -f "${DOCKER_DAEMON_FILE}" ]]; then
|
||||
echo "{\"log-driver\": \"fluentd\", \"log-opts\": { \"fluentd-address\": \"127.0.0.1:24224\" }}" > "${DOCKER_DAEMON_FILE}"
|
||||
systemctl restart docker
|
||||
elif ! grep -q "log-driver" ${DOCKER_DAEMON_FILE}; then
|
||||
sed -i 's!{!& "log-driver": "fluentd", "log-opts": { "fluentd-address": "127.0.0.1:24224" },!' "${DOCKER_DAEMON_FILE}"
|
||||
systemctl restart docker
|
||||
fi
|
||||
|
||||
reconfigure DASHBOARDS_USERNAME "${DASHBOARDS_USERNAME:-"onlyoffice"}"
|
||||
reconfigure DASHBOARDS_PASSWORD "${DASHBOARDS_PASSWORD:-$(get_random_str 20)}"
|
||||
|
||||
docker-compose -f ${BASE_DIR}/dashboards.yml up -d
|
||||
else
|
||||
echo "The installation of the fluent-bit service was unsuccessful."
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
install_product () {
|
||||
DOCKER_TAG="${DOCKER_TAG:-$(get_available_version ${IMAGE_NAME})}"
|
||||
reconfigure DOCKER_TAG ${DOCKER_TAG}
|
||||
@ -1402,15 +1462,17 @@ start_installation () {
|
||||
|
||||
download_files
|
||||
|
||||
install_elasticsearch
|
||||
|
||||
install_fluent_bit
|
||||
|
||||
install_mysql_server
|
||||
|
||||
install_document_server
|
||||
|
||||
install_rabbitmq
|
||||
|
||||
install_redis
|
||||
|
||||
install_elasticsearch
|
||||
install_document_server
|
||||
|
||||
install_product
|
||||
|
||||
|
@ -8,18 +8,22 @@
|
||||
CONTAINER_PREFIX=${PRODUCT}-
|
||||
MYSQL_VERSION=8.3.0
|
||||
MYSQL_IMAGE=mysql:${MYSQL_VERSION}
|
||||
ELK_VERSION=2.11.1
|
||||
SERVICE_PORT=5050
|
||||
DOCUMENT_SERVER_IMAGE_NAME=onlyoffice/4testing-documentserver-ee:latest
|
||||
DOCKERFILE=Dockerfile.app
|
||||
APP_DOTNET_ENV=""
|
||||
EXTERNAL_PORT="80"
|
||||
|
||||
# elasticsearch #
|
||||
# opensearch stack #
|
||||
ELK_VERSION=2.11.1
|
||||
ELK_CONTAINER_NAME=${CONTAINER_PREFIX}opensearch
|
||||
ELK_SHEME=http
|
||||
ELK_HOST=""
|
||||
ELK_PORT=9200
|
||||
DASHBOARDS_VERSION=2.11.1
|
||||
DASHBOARDS_CONTAINER_NAME=${CONTAINER_PREFIX}opensearch-dashboards
|
||||
DASHBOARDS_USERNAME=onlyoffice
|
||||
DASHBOARDS_PASSWORD=onlyoffice
|
||||
|
||||
# app service environment #
|
||||
ENV_EXTENSION=none
|
||||
|
@ -173,6 +173,7 @@ RUN sed -i 's/127.0.0.1:5010/$service_api_system/' /etc/nginx/conf.d/onlyoffice.
|
||||
if [[ -z "${SERVICE_CLIENT}" ]] ; then sed -i 's/127.0.0.1:5001/$service_client/' /etc/nginx/conf.d/onlyoffice.conf; fi && \
|
||||
if [[ -z "${SERVICE_MANAGEMENT}" ]] ; then sed -i 's/127.0.0.1:5015/$service_management/' /etc/nginx/conf.d/onlyoffice.conf; fi && \
|
||||
sed -i 's/127.0.0.1:5033/$service_healthchecks/' /etc/nginx/conf.d/onlyoffice.conf && \
|
||||
sed -i 's/127.0.0.1:5601/$dashboards_host:5601/' /etc/nginx/conf.d/onlyoffice.conf && \
|
||||
sed -i 's/$public_root/\/var\/www\/public\//' /etc/nginx/conf.d/onlyoffice.conf && \
|
||||
sed -i 's/http:\/\/172.*/$document_server;/' /etc/nginx/conf.d/onlyoffice.conf && \
|
||||
sed -i '/client_body_temp_path/ i \ \ \ \ $MAP_HASH_BUCKET_SIZE' /etc/nginx/nginx.conf.template && \
|
||||
|
@ -14,7 +14,7 @@ else
|
||||
echo "Error: yml files not found." && exit 1
|
||||
fi
|
||||
|
||||
FILES=("${PRODUCT}" "notify" "healthchecks" "proxy" "ds" "rabbitmq" "redis" "opensearch" "db")
|
||||
FILES=("${PRODUCT}" "notify" "healthchecks" "proxy" "ds" "rabbitmq" "redis" "opensearch" "dashboards" "db")
|
||||
|
||||
LOG_DIR="${DOCKERCOMPOSE}/logs"
|
||||
mkdir -p ${LOG_DIR}
|
||||
|
25
install/docker/config/fluent-bit.conf
Normal file
25
install/docker/config/fluent-bit.conf
Normal file
@ -0,0 +1,25 @@
|
||||
[SERVICE]
|
||||
Flush 1
|
||||
Log_Level info
|
||||
Daemon off
|
||||
|
||||
[INPUT]
|
||||
Name forward
|
||||
Listen 127.0.0.1
|
||||
Port 24224
|
||||
|
||||
[INPUT]
|
||||
Name exec
|
||||
Interval_Sec 86400
|
||||
Command curl -s -X POST 'OPENSEARCH_SCHEME://OPENSEARCH_HOST:OPENSEARCH_PORT/OPENSEARCH_INDEX/_delete_by_query' -H 'Content-Type: application/json' -d "{\"query\": {\"range\": {\"@timestamp\": {\"lt\": \"$(date -u -d '30 days ago' '+%Y-%m-%dT%H:%M:%S')\"}}}}"
|
||||
|
||||
[OUTPUT]
|
||||
Name opensearch
|
||||
Match *
|
||||
Host OPENSEARCH_HOST
|
||||
Port OPENSEARCH_PORT
|
||||
Replace_Dots On
|
||||
Suppress_Type_Name On
|
||||
Time_Key @timestamp
|
||||
Type _doc
|
||||
Index OPENSEARCH_INDEX
|
@ -83,3 +83,9 @@ map $SERVICE_CLIENT $service_client {
|
||||
"" 127.0.0.1:5001;
|
||||
default $SERVICE_CLIENT;
|
||||
}
|
||||
|
||||
map $DASHBOARDS_CONTAINER_NAME $dashboards_host {
|
||||
volatile;
|
||||
default onlyoffice-opensearch-dashboards;
|
||||
~^(.*)$ $1;
|
||||
}
|
||||
|
17
install/docker/dashboards.yml
Normal file
17
install/docker/dashboards.yml
Normal file
@ -0,0 +1,17 @@
|
||||
version: "3"
|
||||
services:
|
||||
onlyoffice-opensearch-dashboards:
|
||||
image: opensearchproject/opensearch-dashboards:${DASHBOARDS_VERSION}
|
||||
container_name: ${DASHBOARDS_CONTAINER_NAME}
|
||||
restart: always
|
||||
environment:
|
||||
- OPENSEARCH_HOSTS=${ELK_SHEME}://${ELK_CONTAINER_NAME}:${ELK_PORT}
|
||||
- "DISABLE_SECURITY_DASHBOARDS_PLUGIN=true"
|
||||
- "SERVER_BASEPATH=/dashboards"
|
||||
expose:
|
||||
- "5601"
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: ${NETWORK_NAME}
|
||||
external: true
|
@ -279,6 +279,7 @@ services:
|
||||
- REDIS_HOST=${REDIS_HOST}
|
||||
- REDIS_PORT=${REDIS_PORT}
|
||||
- SERVICE_PORT=${SERVICE_PORT}
|
||||
- DASHBOARDS_CONTAINER_NAME=${DASHBOARDS_CONTAINER_NAME}
|
||||
volumes:
|
||||
- router_log:/var/log/nginx
|
||||
|
||||
|
@ -223,6 +223,9 @@ services:
|
||||
- REDIS_PORT=${REDIS_PORT}
|
||||
- REDIS_PASSWORD=${REDIS_PASSWORD}
|
||||
- SERVICE_PORT=${SERVICE_PORT}
|
||||
- DASHBOARDS_CONTAINER_NAME=${DASHBOARDS_CONTAINER_NAME}
|
||||
- DASHBOARDS_USERNAME=${DASHBOARDS_USERNAME}
|
||||
- DASHBOARDS_PASSWORD=${DASHBOARDS_PASSWORD}
|
||||
volumes:
|
||||
- router_log:/var/log/nginx
|
||||
|
||||
|
@ -23,6 +23,9 @@ services:
|
||||
expose:
|
||||
- "9200"
|
||||
- "9600" # required for Performance Analyzer
|
||||
ports:
|
||||
- 127.0.0.1:9200:9200
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: ${NETWORK_NAME}
|
||||
|
@ -1,4 +1,4 @@
|
||||
#!/bin/sh
|
||||
#!/bin/bash
|
||||
WRONG_PORTAL_NAME_URL=${WRONG_PORTAL_NAME_URL:-""}
|
||||
REDIS_HOST=${REDIS_HOST:-"${REDIS_CONTAINER_NAME}"}
|
||||
REDIS_PORT=${REDIS_PORT:-"6379"}
|
||||
@ -9,3 +9,4 @@ sed -i "s~\(redis_host =\).*~\1 \"$REDIS_HOST\"~" /etc/nginx/conf.d/onlyoffice.c
|
||||
sed -i "s~\(redis_port =\).*~\1 $REDIS_PORT~" /etc/nginx/conf.d/onlyoffice.conf
|
||||
sed -i "s~\(redis_pass =\).*~\1 \"$REDIS_PASSWORD\"~" /etc/nginx/conf.d/onlyoffice.conf
|
||||
sed -i "s~\(\"wrongPortalNameUrl\":\).*,~\1 \"${WRONG_PORTAL_NAME_URL}\",~g" /var/www/public/scripts/config.json
|
||||
echo "${DASHBOARDS_USERNAME:-onlyoffice}:$(openssl passwd -6 -stdin <<< "${DASHBOARDS_PASSWORD:-onlyoffice}")" > /etc/nginx/.htpasswd_dashboards
|
||||
|
Loading…
Reference in New Issue
Block a user