Newer
Older
services:
# Summary
# =======
# brief: create certificates
# dir: init
# see also: init/Dockerfile
#
# Description
# ===========
# This service run init/init.sh, which create and install
# ssl certificates for minio and sftp
init:
build:
args:
UID: "${UID}"
GID: "${GID}"
user: "${UID}:${GID}"
volumes:
- ./nginx/html/:/home/tsm/html
- ./cron:/tmp/cron
# Summary
# =======
# brief: central database
# dir: data/postgres mosquitto frontend dispatcher
# see also:
# - data/postgres/Dockerfile,
# - scripts mentioned in the 'volumes' section below
# admin login:
# - TODO: replace with correct $ENVVARS
# - `psql postgresql:// POSTGRES_USER : POSTGRES_PASSWORD @ HOST?? : PORT?? / DATABASE??`
# - `psql postgresql://postgres:postgres@localhost:5432/postgres`
#
# Description
# ===========
# This service provide database(s) for other services, namely
# 1. a database for each group/project to store the things and the observations
# 2. 'mqtt_auth' database for the service 'mqtt-broker'. It holds the mqtt-user,
# which are allowed to send data to the broker
# 3. 's3map_db' database, for some worker-services. It maps group specific
# object-storage buckets to things and corresponding databases (see 1.)
# 4. the 'frontenddb' database for the service 'frontend', which use django
# 5. SMS: TODO
# 6. SMS_CV: TODO
# To see how and where the creation of each db is defined see in the volumes section
# for the scripts, which gets mounted and executed on startup.
database:
restart: "${RESTART}"
# image: timescale/timescaledb:${DATABASE_IMAGE_TAG-pg14}
build:
args:
UID: "${UID}"
DATABASE_IMAGE_TAG: "${DATABASE_IMAGE_TAG}"
ports:
- "${POSTGRES_PORT}:5432"
environment:
POSTGRES_USER: "${POSTGRES_USER}"
POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}"
PGDATA: /var/lib/postgresql/data/pgdata
MQTT_AUTH_POSTGRES_USER: "${MQTT_AUTH_POSTGRES_USER}"
MQTT_AUTH_POSTGRES_PASS: "${MQTT_AUTH_POSTGRES_PASS}"
S3MAP_POSTGRES_USER: "${S3MAP_POSTGRES_USER}"
S3MAP_POSTGRES_PASS: "${S3MAP_POSTGRES_PASS}"
FRONTEND_POSTGRES_USER: "${FRONTEND_POSTGRES_USER}"
FRONTEND_POSTGRES_PASS: "${FRONTEND_POSTGRES_PASS}"
SMS_DB_USER: "${SMS_DB_USER}"
SMS_DB_PASSWORD: "${SMS_DB_PASSWORD}"
SMS_DB_PORT: "${SMS_DB_PORT}"
SMS_DB_DB: "${SMS_DB_DB}"
SMS_DB_HOST: "${SMS_DB_HOST}"
SMS_ACCESS_TYPE: "${SMS_ACCESS_TYPE}"
CV_DB_USER: "${CV_DB_USER}"
CV_DB_PASSWORD: "${CV_DB_PASSWORD}"
CV_DB_PORT: "${CV_DB_PORT}"
CV_DB_DB: "${CV_DB_DB}"
CV_DB_HOST: "${CV_DB_HOST}"
CV_ACCESS_TYPE: "${CV_ACCESS_TYPE}"
volumes:
- ./postgres/sms_db/sms_ddl.sql:/sql/sms/sms_ddl.sql
- ./postgres/sms_db/sms_foreign_tables.sql:/sql/sms/sms_foreign_tables.sql
- ./postgres/sms_db/sms_init_tables.sh:/docker-entrypoint-initdb.d/01_sms_init_tables.sh
- ./postgres/sms_cv_db/sms_cv_ddl.sql:/sql/sms_cv/sms_cv_ddl.sql
- ./postgres/sms_cv_db/sms_cv_foreign_tables.sql:/sql/sms_cv/sms_cv_foreign_tables.sql
- ./postgres/sms_cv_db/sms_cv_init_tables.sh:/docker-entrypoint-initdb.d/02_sms_cv_init_tables.sh
- ./postgres/postgres-force-ssl.sh:/docker-entrypoint-initdb.d/postgres-force-ssl.sh
- ./postgres/postgis.sh:/docker-entrypoint-initdb.d/03_postgis.sh
- ./mosquitto/mosquitto-go-auth.sh:/docker-entrypoint-initdb.d/mosquitto-go-auth.sh:ro
- ./frontend/frontend-database.sh:/docker-entrypoint-initdb.d/frontend-database.sh:ro
- ./dispatcher/s3_to_db.sh:/docker-entrypoint-initdb.d/s3_to_db.sh:ro
- ./data/postgres/data:/var/lib/postgresql/data
- "${POSTGRES_TLS_CERT_PATH}:/var/lib/postgresql/server.crt"
- "${POSTGRES_TLS_KEY_PATH}:/var/lib/postgresql/server.key"
user: "${UID}:${GID}"
command: "${POSTGRES_EXTRA_PARAMS}"
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $POSTGRES_USER"]
interval: "${POSTGRES_HEALTHCHECK_INTERVAL}"
timeout: 5s
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
# Summary
# =======
# brief: raw storage for observation files
# dir: none
# see also: none
# mqtt-topics: "object_storage_notification" (send)
# admin login:
# -> http://localhost/object-storage/login
# -> user, pass: $MINIO_ROOT_USER, $MINIO_ROOT_PASSWORD (default: minioadmin)
#
# Description
# ===========
# The 'object-storage' aka. 'raw-data-storage' (or simply 'the minio' or 'S3') is
# a file based storage for the users to upload raw data files (e.g. some
# observations as a CSV).
# The upload can be done via (S)FTP or via a client or via a web frontend. The storage
# sends mqtt messages to the broker on file upload (and other events).
object-storage:
image: "minio/minio:${OBJECT_STORAGE_IMAGE_TAG}"
restart: "${RESTART}"
ports:
- "${MINIO_SFTP_PORT}:22"
- "${MINIO_FTP_PORT}:21"
- "${MINIO_FTP_PASV_PORTS}"
depends_on:
mqtt-broker:
condition: service_completed_successfully
environment:
MINIO_ROOT_USER: "${MINIO_ROOT_USER}"
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD}"
MINIO_BROWSER_REDIRECT_URL: "${MINIO_BROWSER_REDIRECT_URL}"
MINIO_NOTIFY_MQTT_ENABLE_LOCAL_BROKER: on
MINIO_NOTIFY_MQTT_BROKER_LOCAL_BROKER: tcp://mqtt-broker:1883
MINIO_NOTIFY_MQTT_TOPIC_LOCAL_BROKER: object_storage_notification
MINIO_NOTIFY_MQTT_USERNAME_LOCAL_BROKER: "${MQTT_USER}"
MINIO_NOTIFY_MQTT_PASSWORD_LOCAL_BROKER: "${MQTT_PASSWORD}"
MINIO_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL_LOCAL_BROKER: 60s
MINIO_NOTIFY_MQTT_QOS_LOCAL_BROKER: "${MQTT_QOS}"
MINIO_NOTIFY_MQTT_RECONNECT_INTERVAL_LOCAL_BROKER: 60s
# MINIO_NOTIFY_MQTT_QUEUE_DIR_LOCAL_BROKER: "<string>"
# MINIO_NOTIFY_MQTT_QUEUE_LIMIT_LOCAL_BROKER: "<string>"
# MINIO_NOTIFY_MQTT_COMMENT_LOCAL_BROKER: "<string>"
MINIO_SERVER_URL: "${MINIO_SERVER_URL}"
volumes:
# FTP server tls key and cert
- "${MINIO_FTP_TLS_CRT}:/certs/minio-ftp.crt:ro"
- "${MINIO_FTP_TLS_KEY}:/certs/minio-ftp.key:ro"
# SSH Server key to provide constant ssh host key
- "${MINIO_SFTP_HOSTKEY}:/certs/id_ed25519:ro"
user: "${UID}:${GID}"
command: >
server
--console-address :9001
--ftp address=:21
--ftp passive-port-range=30000-30010
--sftp address=:22
--sftp ssh-private-key=/certs/id_ed25519
--ftp tls-private-key=/certs/minio-ftp.key
--ftp tls-public-cert=/certs/minio-ftp.crt
--json
/vol0
healthcheck:
test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/9000' || exit 1
interval: "${MINIO_HEALTHCHECK_INTERVAL}"
timeout: 5s
retries: 15
# Summary
# =======
# brief: central mqtt message bus / broker
# dir: mosquitto
# see also: mosquitto/docker-entrypoint.sh
#
# Description
# ===========
# This is the main message bus, all mqtt messages are send here. Services can
# subscribe to topics and will receive messages that was sent to them.
mqtt-broker:
restart: "${RESTART}"
image: "iegomez/mosquitto-go-auth:${MQTT_BROKER_IMAGE_TAG}"
depends_on:
database:
command: >
/usr/sbin/mosquitto
-c /var/lib/mosquitto/mosquitto.conf
user: "${UID}:${GID}"
environment:
MQTT_USER: "${MQTT_USER}"
MQTT_PASSWORD: "${MQTT_PASSWORD}"
MQTT_INGEST_USER: "${MQTT_INGEST_USER}"
MQTT_INGEST_PASSWORD: "${MQTT_INGEST_PASSWORD}"
MQTT_AUTH_POSTGRES_HOST: "${MQTT_AUTH_POSTGRES_HOST}"
MQTT_AUTH_POSTGRES_PORT: "${MQTT_AUTH_POSTGRES_PORT}"
MQTT_AUTH_POSTGRES_USER: "${MQTT_AUTH_POSTGRES_USER}"
MQTT_AUTH_POSTGRES_PASS: "${MQTT_AUTH_POSTGRES_PASS}"
MQTT_AUTH_POSTGRES_DB: "${MQTT_AUTH_POSTGRES_DB}"
MQTT_AUTH_PG_TLSMODE: "${MQTT_AUTH_PG_TLSMODE}"
FRONTEND_MQTT_USER: "${FRONTEND_MQTT_USER}"
FRONTEND_MQTT_PASS: "${FRONTEND_MQTT_PASS}"
ports:
- "${MOSQUITTO_PORT}:1883"
- "${MOSQUITTO_PORT_SECURE}:8883"
volumes:
- "${MOSQUITTO_CONFIG}:/etc/mosquitto/config/mosquitto.conf:ro"
- "${MOSQUITTO_TLS_CONFIG}:/etc/mosquitto/config/tls/mosquitto.tls.conf:ro"
- ./data/mosquitto/auth:/mosquitto-auth/
- ./data/mosquitto/data:/mosquitto-data/
- "${MOSQUITTO_TLS_CERT_PATH}:/mosquitto/config/certs/server.crt:ro"
- "${MOSQUITTO_TLS_KEY_PATH}:/mosquitto/config/certs/server.key:ro"
- "${MOSQUITTO_TLS_CA_PATH}:/mosquitto/config/certs/ca.crt:ro"
- ./mosquitto/docker-entrypoint.sh:/docker-entrypoint.sh
- /var/lib/mosquitto/:uid=${UID}
test: >
mosquitto_sub
-C 1
-t '$$SYS/broker/version'
-u "${MQTT_USER}"
-P "${MQTT_PASSWORD}"
--id docker-compose-healthcheck
interval: "${MQTT_BROKER_HEALTHCHECK_INTERVAL}"
logging:
options:
max-size: "${MQTT_BROKER_LOG_SIZE}"
max-file: "${MQTT_BROKER_FILE_COUNT}"
# Summary
# =======
# brief: visualisation of observations stored in the -> database
# dir: grafana
# see also: none
# admin login:
# -> http://localhost/visualization/login
# -> user, pass: $GRAFANA_USER, $GRAFANA_PASSWORD (default: grafana)
#
# Description
# ===========
# This visualize observations of a group or project that are stored in the database.
visualization:
restart: "${RESTART}"
image: "grafana/grafana:${GRAFANA_IMAGE_TAG}"
user: "${UID}:${GID}"
environment:
GF_SECURITY_ADMIN_USER: "${GRAFANA_USER}"
GF_SECURITY_ADMIN_PASSWORD: "${GRAFANA_PASSWORD}"
GF_SERVER_ROOT_URL: "${GRAFANA_PROXY_URL}"
GF_SERVER_SERVE_FROM_SUB_PATH: true
GF_AUTH_GENERIC_OAUTH_ENABLED: true
GF_AUTH_GENERIC_OAUTH_NAME: "Helmholtz AAI"
GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP: true
GF_AUTH_GENERIC_OAUTH_CLIENT_ID: "${DJANGO_HELMHOLTZ_CLIENT_ID}"
GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: "${DJANGO_HELMHOLTZ_CLIENT_SECRET}"
GF_AUTH_GENERIC_OAUTH_SCOPES: "email profile eduperson_principal_name"
GF_AUTH_GENERIC_OAUTH_AUTH_URL: "${GF_AUTH_GENERIC_OAUTH_AUTH_URL}"
GF_AUTH_GENERIC_OAUTH_TOKEN_URL: "${GF_AUTH_GENERIC_OAUTH_TOKEN_URL}"
GF_AUTH_GENERIC_OAUTH_API_URL: "${GF_AUTH_GENERIC_OAUTH_API_URL}"
GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH: eduperson_principal_name
GF_AUTH_OAUTH_SKIP_ORG_ROLE_UPDATE_SYNC: true
- ./data/grafana/:/var/lib/grafana
# Summary
# =======
# brief:
# dir:
# see also:
#
# Description
# ===========
# TODO
tsmdl:
image: "registry.hzdr.de/hub-terra/tsmdl-time-series-management-decoupling-layer/tsmdl-api:${TSM_DL_IMAGE_TAG}"
entrypoint: "/app/start.${TSM_DL_ENVIRONMENT}.sh"
${CREATEDB_POSTGRES_USER}:\
${CREATEDB_POSTGRES_PASSWORD}@\
${CREATEDB_POSTGRES_HOST}/\
${CREATEDB_POSTGRES_DATABASE}"
STA_ENDPOINT_URL: "${TOMCAT_PROXY_URL}"
UVICORN_ARGS: "${UVICORN_ARGS}"
ROOT_PATH: /tsmdl
ORGANISATION: ufz
depends_on:
database:
condition: service_healthy
# Summary
# =======
# brief:
# dir:
# see also:
#
# Description
# ===========
# TODO
timeio-db-api:
image: "registry.hzdr.de/ufz-tsm/timeio-db-api/timeio-db-api:${DB_API_IMAGE_TAG}"
restart: "${RESTART}"
entrypoint: "sh /app/start.sh"
environment:
DB_URL: "postgresql://\
${CREATEDB_POSTGRES_USER}:\
${CREATEDB_POSTGRES_PASSWORD}@\
${CREATEDB_POSTGRES_HOST}/\
${CREATEDB_POSTGRES_DATABASE}"
UVICORN_ARGS: "${UVICORN_ARGS} --root-path /db_api"
depends_on:
database:
condition: service_healthy
healthcheck:
test: >
wget -q http://0.0.0.0:8001/dbhealth -O /dev/null || exit 1
interval: 10s
timeout: 10s
retries: 3
# Summary
# =======
# brief:
# dir:
# see also:
#
# Description
# ===========
# TODO
frost:
build:
args:
UID: "${UID}"
GID: "${GID}"
TOMCAT_IMAGE_TAG: "${TOMCAT_IMAGE_TAG}"
restart: "${RESTART}"
- ./data/tomcat/context:/usr/local/tomcat/conf/Catalina/localhost:ro
- ./tomcat/index.jsp:/usr/local/tomcat/webapps/ROOT/index.jsp
# Summary
# =======
# brief:
# dir:
# see also:
#
# Description
# ===========
# TODO
flyway:
image: flyway/flyway:${FLYWAY_IMAGE_TAG}
command: -configFiles=/flyway/conf/flyway.conf -skipCheckForUpdate migrate
volumes:
- ./flyway/migrations:/flyway/sql
- ./flyway/flyway.conf:/flyway/conf/flyway.conf
environment:
FLYWAY_URL: "jdbc:postgresql://${CREATEDB_POSTGRES_HOST}/${CREATEDB_POSTGRES_DATABASE}"
FLYWAY_USER: "${CREATEDB_POSTGRES_USER}"
FLYWAY_PASSWORD: "${CREATEDB_POSTGRES_PASSWORD}"
FLYWAY_BASELINE_VERSION: "${FLYWAY_BASELINE_VERSION}"
FLYWAY_PLACEHOLDERS_CONFIGDB_USER: "${CONFIGDB_USER}"
FLYWAY_PLACEHOLDERS_CONFIGDB_PASSWORD: "${CONFIGDB_PASSWORD}"
depends_on:
database:
condition: service_healthy
# Summary
# =======
# brief: A user frontend to add things groups parser etc.
# dir:
# see also:
#
# Description
# ===========
# TODO
frontend:
image: "registry.hzdr.de/ufz-tsm/tsm-frontend/tsm-frontend:${FRONTEND_IMAGE_TAG}"
restart: "${RESTART}"
command: >
bash -c "python3 manage.py migrate
&& python3 manage.py loaddata admin_interface_theme_foundation.json
&& python3 manage.py loaddata ufz_theme.json
&& python3 manage.py loaddata nm_station.json
&& python3 manage.py createsuperuser --noinput
|| echo 'Superuser already created'
&& python3 manage.py collectstatic --noinput
&& gunicorn main.wsgi:application --bind 0.0.0.0:8000 -w 6"
- frontend-statics:/home/appuser/app/static
- sftp-private-keys:/home/appuser/app/sftp-private-keys
- ./frontend/user.json:/home/appuser/app/tsm/fixtures/user.json
- ./frontend/thing.json:/home/appuser/app/tsm/fixtures/thing.json
- ./frontend/nm_station.json:/home/appuser/app/tsm/fixtures/nm_station.json
entrypoint: ""
# ports:
# - "127.0.0.1:8000:8000"
environment:
POSTGRES_HOST: "${FRONTEND_POSTGRES_HOST}"
POSTGRES_NAME: "${FRONTEND_POSTGRES_DB}"
POSTGRES_USER: "${FRONTEND_POSTGRES_USER}"
POSTGRES_PASSWORD: "${FRONTEND_POSTGRES_PASS}"
PUBLISH_THING_TO_BROKER: True
CREATEDB_POSTGRES_HOST: "${CREATEDB_POSTGRES_HOST}"
CREATEDB_POSTGRES_DATABASE: "${CREATEDB_POSTGRES_DATABASE}"
MQTT_BROKER_HOST: "${FRONTEND_MQTT_HOST}"
MQTT_USER: "${FRONTEND_MQTT_USER}"
MQTT_PASSWORD: "${FRONTEND_MQTT_PASS}"
DJANGO_SUPERUSER_USERNAME: "${DJANGO_SUPERUSER_USERNAME}"
DJANGO_SUPERUSER_PASSWORD: "${DJANGO_SUPERUSER_PASSWORD}"
DJANGO_SUPERUSER_EMAIL: "${DJANGO_SUPERUSER_EMAIL}"
DJANGO_SECRET_KEY: "${DJANGO_SECRET_KEY}"
DJANGO_DEBUG: "${DJANGO_DEBUG}"
DJANGO_TRUSTED_ORIGINS: "${DJANGO_TRUSTED_ORIGINS}"
DJANGO_BASE_PATH: "${DJANGO_BASE_PATH}"
DJANGO_ALLOWED_HOSTS: "${DJANGO_ALLOWED_HOSTS}"
POSTGRES_SSLMODE: "${DJANGO_POSTGRES_SSLMODE}"
DJANGO_HELMHOLTZ_CLIENT_ID: "${DJANGO_HELMHOLTZ_CLIENT_ID}"
DJANGO_HELMHOLTZ_CLIENT_SECRET: "${DJANGO_HELMHOLTZ_CLIENT_SECRET}"
DJANGO_HELMHOLTZ_AAI_CONF_URL: "${DJANGO_HELMHOLTZ_AAI_CONF_URL}"
MINIO_FTP_PORT: "${MINIO_FTP_PORT}"
MINIO_SFTP_PORT: "${MINIO_SFTP_PORT}"
PROXY_URL: "${PROXY_URL}"
ALLOWED_VOS: "${ALLOWED_VOS}"
STA_ROOT_URL: "${STA_ROOT_URL}"
QAQC_SETTING_SAQC_VERSION: "${QAQC_SETTING_SAQC_VERSION}"
FERNET_ENCRYPTION_SECRET: "${FERNET_ENCRYPTION_SECRET}"
depends_on:
database:
condition: service_healthy
mqtt-broker:
condition: service_healthy
worker-configdb-updater:
condition: service_started
# Summary
# =======
# brief: Make services accessible on single address (eg. tsm.ufz.de)
# dir: nginx
# see also: ./nginx/*.conf
#
# Description
# ===========
# This maps service endpoints to a path on a main URL, mainly
# - the service frontend to URL/frontend
# - the service object-storage to URL/object-storage
# - the service visualization to URL/visualization
# - the service frost to URL/sta
# - the service tsmdl to URL/tsmdl
proxy:
image: "nginxinc/nginx-unprivileged:${NGINX_IMAGE_TAG}"
user: "${UID}:${GID}"
ports:
- "${PROXY_PLAIN_PORT}"
- "${PROXY_TLS_PORT}"
- "${PROXY_MINIO_PORT}"
volumes:
- frontend-statics:/home/appuser/app/static
- ./nginx/html:/usr/share/nginx/html
- "./nginx/${PROXY_SITE_CONFIG_FILE}:/etc/nginx/conf.d/default.conf:ro"
- ./nginx/locations:/etc/nginx/locations:ro
- "${PROXY_TLS_CERT_PATH}:/etc/ssl/public.crt:ro"
- "${PROXY_TLS_KEY_PATH}:/etc/ssl/private.key:ro"
depends_on:
frontend:
condition: service_completed_successfully
# =================================================================
# SETUP worker (topic: thing_creation)
# =================================================================
# fills and updates the config-db from setup mqtt-messages
# (e.g. topic: frontend-thing-update)
worker-configdb-updater:
image: "registry.hzdr.de/ufz-tsm/timeio-configdb-updater/configdb-updater:${CONFIGDB_UPDATER_IMAGE_TAG}"
restart: "${RESTART}"
depends_on:
database:
Bert Palm
committed
LOG_LEVEL: "${LOG_LEVEL}"
MQTT_BROKER_HOST: mqtt-broker
MQTT_BROKER_PORT: 1883
MQTT_USER: "${MQTT_USER}"
MQTT_PASSWORD: "${MQTT_PASSWORD}"
MQTT_CLIENT_ID: configdb-updater
MQTT_CLEAN_SESSION: "${MQTT_CLEAN_SESSION}"
MQTT_SUBSCRIBE_QOS: "${MQTT_QOS}"
MQTT_PUBLISH_TOPIC: configdb_update
MQTT_PUBLISH_QOS: "${MQTT_QOS}"
CONFIGDB_CONNECTION_INITIAL_TIMEOUT: 10
CONFIGDB_DSN: "postgresql://\
Bert Palm
committed
${CREATEDB_POSTGRES_USER}:\
${CREATEDB_POSTGRES_PASSWORD}@\
${CREATEDB_POSTGRES_HOST}/\
${CREATEDB_POSTGRES_DATABASE}"
# brief: create a new bucket for a Thing in S3 storage
worker-object-storage-setup:
image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG}"
restart: "${RESTART}"
depends_on:
mqtt-broker:
timeio-db-api:
condition: service_healthy
environment:
LOG_LEVEL: "${LOG_LEVEL}"
TOPIC: thing_creation
MQTT_BROKER: mqtt-broker:1883
MQTT_USER: "${MQTT_USER}"
MQTT_PASSWORD: "${MQTT_PASSWORD}"
MQTT_CLIENT_ID: object-storage-setup
MQTT_CLEAN_SESSION: "${MQTT_CLEAN_SESSION}"
MQTT_QOS: "${MQTT_QOS}"
MINIO_SECURE: "${MINIO_SECURE}"
MINIO_URL: "${MINIO_HOST}"
MINIO_ACCESS_KEY: "${MINIO_ROOT_USER}"
MINIO_SECURE_KEY: "${MINIO_ROOT_PASSWORD}"
JOURNALING: "${JOURNALING}"
entrypoint: ["python3", "minio_setup.py"]
# Summary
# =======
# brief: create things in project-DB
#
# Description
# ===========
# - create a new DB for a project, if not it not already exist
# - create/update a thing in DB
# - create/update frost view for the thing
worker-db-setup:
image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG}"
restart: "${RESTART}"
depends_on:
mqtt-broker:
timeio-db-api:
condition: service_healthy
environment:
LOG_LEVEL: "${LOG_LEVEL}"
TOPIC: thing_creation
MQTT_BROKER: mqtt-broker:1883
MQTT_USER: "${MQTT_USER}"
MQTT_PASSWORD: "${MQTT_PASSWORD}"
MQTT_CLEAN_SESSION: "${MQTT_CLEAN_SESSION}"
MQTT_QOS: "${MQTT_QOS}"
DATABASE_URL: "postgresql://\
${CREATEDB_POSTGRES_USER}:\
${CREATEDB_POSTGRES_PASSWORD}@\
${CREATEDB_POSTGRES_HOST}/\
${CREATEDB_POSTGRES_DATABASE}"
SMS_URL: "${SMS_URL}"
CV_URL: "${CV_URL}"
JOURNALING: "${JOURNALING}"
FERNET_ENCRYPTION_SECRET: "${FERNET_ENCRYPTION_SECRET}"
worker-frost-setup:
image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG}"
restart: "${RESTART}"
depends_on:
mqtt-broker:
timeio-db-api:
condition: service_healthy
- ./data/tomcat/context:/home/appuser/app/src/CreateNewFrostInstanceAction/tomcat/context_files:rw
environment:
LOG_LEVEL: "${LOG_LEVEL}"
TOPIC: thing_creation
MQTT_BROKER: mqtt-broker:1883
MQTT_USER: "${MQTT_USER}"
MQTT_PASSWORD: "${MQTT_PASSWORD}"
MQTT_CLEAN_SESSION: "${MQTT_CLEAN_SESSION}"
MQTT_QOS: "${MQTT_QOS}"
TOMCAT_PROXY_URL: "${TOMCAT_PROXY_URL}"
JOURNALING: "${JOURNALING}"
FERNET_ENCRYPTION_SECRET: "${FERNET_ENCRYPTION_SECRET}"
entrypoint: ["python3", "frost_setup.py"]
# brief: Add user to mqtt_auth DB, so they are allowed to use mqtt
worker-mqtt-user-creation:
image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG}"
depends_on:
mqtt-broker:
timeio-db-api:
condition: service_healthy
environment:
LOG_LEVEL: "${LOG_LEVEL}"
TOPIC: thing_creation
MQTT_BROKER: mqtt-broker:1883
MQTT_USER: "${MQTT_USER}"
MQTT_PASSWORD: "${MQTT_PASSWORD}"
MQTT_CLIENT_ID: mqtt-user-creation
MQTT_CLEAN_SESSION: "${MQTT_CLEAN_SESSION}"
MQTT_QOS: "${MQTT_QOS}"
DATABASE_URL: "postgresql://\
${CREATEDB_POSTGRES_USER}:\
${CREATEDB_POSTGRES_PASSWORD}@\
${CREATEDB_POSTGRES_HOST}/\
${CREATEDB_POSTGRES_DATABASE}"
JOURNALING: "${JOURNALING}"
entrypoint: ["python3", "mqtt_user_setup.py"]
worker-grafana-dashboard:
image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG}"
restart: "${RESTART}"
depends_on:
mqtt-broker:
timeio-db-api:
condition: service_healthy
environment:
LOG_LEVEL: "${LOG_LEVEL}"
TOPIC: thing_creation
MQTT_BROKER: mqtt-broker:1883
MQTT_USER: "${MQTT_USER}"
MQTT_PASSWORD: "${MQTT_PASSWORD}"
MQTT_CLIENT_ID: grafana-dashboard
MQTT_CLEAN_SESSION: "${MQTT_CLEAN_SESSION}"
MQTT_QOS: "${MQTT_QOS}"
GRAFANA_URL: "${GRAFANA_URL}"
GRAFANA_USER: "${GRAFANA_USER}"
GRAFANA_PASSWORD: "${GRAFANA_PASSWORD}"
GRAFANA_DEFAULT_DATASOURCE_SSLMODE: "${GRAFANA_DEFAULT_DATASOURCE_SSLMODE}"
JOURNALING: "${JOURNALING}"
FERNET_ENCRYPTION_SECRET: "${FERNET_ENCRYPTION_SECRET}"
entrypoint: ["python3", "grafana_dashboard_setup.py"]
# =================================================================
# Ingest worker (incoming observations)
# =================================================================
image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG}"
depends_on:
mqtt-broker:
timeio-db-api:
condition: service_healthy
environment:
LOG_LEVEL: "${LOG_LEVEL}"
TOPIC: object_storage_notification
MQTT_BROKER: mqtt-broker:1883
MQTT_USER: "${MQTT_USER}"
MQTT_PASSWORD: "${MQTT_PASSWORD}"
MQTT_CLEAN_SESSION: "${MQTT_CLEAN_SESSION}"
MQTT_QOS: "${MQTT_QOS}"
TOPIC_DATA_PARSED: "${TOPIC_DATA_PARSED}"
MINIO_SECURE: "${MINIO_SECURE}"
MINIO_URL: "${MINIO_HOST}"
MINIO_ACCESS_KEY: "${MINIO_ROOT_USER}"
MINIO_SECURE_KEY: "${MINIO_ROOT_PASSWORD}"
CONFIGDB_DSN: "postgresql://\
${CONFIGDB_USER}:\
${CONFIGDB_PASSWORD}@\
${CONFIGDB_HOST}:\
${CONFIGDB_PORT}/\
JOURNALING: "${JOURNALING}"
worker-run-qaqc:
image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG}"
depends_on:
mqtt-broker:
environment:
LOG_LEVEL: "${LOG_LEVEL}"
TOPIC: "${TOPIC_DATA_PARSED}"
MQTT_BROKER: mqtt-broker:1883
MQTT_USER: "${MQTT_USER}"
MQTT_PASSWORD: "${MQTT_PASSWORD}"
MQTT_CLEAN_SESSION: "${MQTT_CLEAN_SESSION}"
MQTT_QOS: "${MQTT_QOS}"
TOPIC_QC_DONE: qaqc_done
TOPIC_QC_DONE_QOS: "${MQTT_QOS}"
DATABASE_DSN: "postgresql://\
${CREATEDB_POSTGRES_USER}:\
${CREATEDB_POSTGRES_PASSWORD}@\
${CREATEDB_POSTGRES_HOST}/\
${CREATEDB_POSTGRES_DATABASE}"
JOURNALING: "${JOURNALING}"
worker-mqtt-ingest:
image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG}"
depends_on:
mqtt-broker:
timeio-db-api:
condition: service_healthy
environment:
LOG_LEVEL: "${LOG_LEVEL}"
TOPIC: mqtt_ingest/#
MQTT_BROKER: mqtt-broker:1883
MQTT_USER: "${MQTT_INGEST_USER}"
MQTT_PASSWORD: "${MQTT_INGEST_PASSWORD}"
MQTT_CLEAN_SESSION: "${MQTT_CLEAN_SESSION}"
MQTT_QOS: "${MQTT_QOS}"
CONFIGDB_DSN: "postgresql://\
${CONFIGDB_USER}:\
${CONFIGDB_PASSWORD}@\
${CONFIGDB_HOST}:\
${CONFIGDB_PORT}/\
${CONFIGDB_DB}"
DB_API_BASE_URL: "${DB_API_BASE_URL}"
JOURNALING: "${JOURNALING}"
worker-crontab-setup:
image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG}"
restart: "${RESTART}"
depends_on:
mqtt-broker:
condition: service_completed_successfully
timeio-db-api:
condition: service_healthy
TOPIC: thing_creation
MQTT_BROKER: mqtt-broker:1883
MQTT_USER: "${MQTT_USER}"
MQTT_PASSWORD: "${MQTT_PASSWORD}"
MQTT_CLIENT_ID: crontab-setup
MQTT_CLEAN_SESSION: "${MQTT_CLEAN_SESSION}"
MQTT_QOS: "${MQTT_QOS}"
JOURNALING: "${JOURNALING}"
entrypoint: ["python3", "crontab_setup.py"]
- ./cron/crontab.txt:/tmp/cron/crontab.txt
# =================================================================
# other worker
# =================================================================
worker-grafana-user-orgs:
image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG}"
restart: "${RESTART}"
depends_on:
mqtt-broker:
timeio-db-api:
condition: service_healthy
environment:
LOG_LEVEL: "${LOG_LEVEL}"
TOPIC: user_login
MQTT_BROKER: mqtt-broker:1883
MQTT_USER: "${MQTT_USER}"
MQTT_PASSWORD: "${MQTT_PASSWORD}"
MQTT_CLIENT_ID: grafana-user-orgs
MQTT_CLEAN_SESSION: "${MQTT_CLEAN_SESSION}"
MQTT_QOS: "${MQTT_QOS}"
GRAFANA_URL: "${GRAFANA_URL}"
GRAFANA_USER: "${GRAFANA_USER}"
GRAFANA_PASSWORD: "${GRAFANA_PASSWORD}"
ALLOWED_VOS: "${ALLOWED_VOS}"
JOURNALING: "${JOURNALING}"
entrypoint: ["python3", "grafana_user_setup.py"]
cron-scheduler:
build:
args:
UID: "${UID}"
DEBIAN_IMAGE_TAG: "${CRON_DEBIAN_IMAGE_TAG}"
restart: "${RESTART}"
# Prevent blocking/slowing-down of other services if many scheduler jobs occur at
# the same time. Cap the processing power to equivalent of one CPU of the system.
cpus: 0.5
environment:
SETUP_SERVICE: "${CRON_SETUP_SERVICE}"
CREATEDB_POSTGRES_USER: "${CRON_CREATEDB_POSTGRES_USER}"
CREATEDB_POSTGRES_PASSWORD: "${CRON_CREATEDB_POSTGRES_PASSWORD}"
CREATEDB_POSTGRES_HOST: "${CRON_CREATEDB_POSTGRES_HOST}"
CREATEDB_POSTGRES_DATABASE: "${CRON_CREATEDB_POSTGRES_DATABASE}"
SMS_ACCESS_TYPE: "${CRON_SMS_ACCESS_TYPE}"
SMS_API_URL: "${CRON_SMS_API_URL}"
SMS_API_TOKEN: "${CRON_SMS_API_TOKEN}"
CV_ACCESS_TYPE: "${CRON_CV_ACCESS_TYPE}"
CV_API_URL: "${CRON_CV_API_URL}"
# The following is needed for sftp sync jobs
LOG_LEVEL: "${LOG_LEVEL}"
Bert Palm
committed
MINIO_USER: "${MINIO_ROOT_USER}"
MINIO_PASSWORD: "${MINIO_ROOT_PASSWORD}"
MINIO_SECURE: "${MINIO_SECURE}"
CONFIGDB_DSN: "postgresql://\
${CONFIGDB_USER}:\
${CONFIGDB_PASSWORD}@\
${CONFIGDB_HOST}:\
${CONFIGDB_PORT}/\
${CONFIGDB_DB}"
MQTT_BROKER: mqtt-broker:1883
MQTT_USER: "${MQTT_USER}"
MQTT_PASSWORD: "${MQTT_PASSWORD}"
MQTT_CLIENT_ID: cron-scheduler
MQTT_CLEAN_SESSION: "${MQTT_CLEAN_SESSION}"
MQTT_QOS: "${MQTT_QOS}"
depends_on:
init:
condition: service_completed_successfully
test: /supercronic-health.sh || exit 1
interval: 5s
timeout: 3s
retries: 5
volumes:
- ./cron/crontab.txt:/tmp/cron/crontab.txt:ro
- ./cron/scripts:/scripts:ro
- sftp-private-keys:/sftp-private-keys:ro
restart: "${RESTART}"
image: "eclipse-mosquitto:${MQTT_CAT_IMAGE_TAG}"
logging:
options:
max-size: "${MQTT_CAT_LOG_SIZE}"
max-file: "${MQTT_CAT_FILE_COUNT}"
command: >
mosquitto_sub --pretty -v
-h mqtt-broker
-t "#"
-u "${MQTT_USER}"
-P "${MQTT_PASSWORD}"
--id "mqtt-cat"
image: "gcr.io/cadvisor/cadvisor:${MONITORING_IMAGE_TAG}"
restart: "${RESTART}"
- /:/rootfs:ro
- /var/run:/var/run:ro
- /sys:/sys:ro
- /var/lib/docker/:/var/lib/docker:ro