split compose file in backend, worker and frontend compose files. added...

split compose file in backend, worker and frontend compose files. added descriptions for backend services

docker-compose-backend.yml

+---
+version: "3.9"
+services:
+
+  # Summary
+  # =======
+  # brief: create certificates
+  # dir: init
+  # see also: init/Dockerfile
+  #
+  # Description
+  # ===========
+  # This service run init/init.sh, which create and install
+  # ssl certificates for minio and sftp
+  init:
+    build:
+      context: "init"
+      args:
+        UID: "${UID}"
+        GID: "${GID}"
+    restart: "no"
+    user: "${UID}:${GID}"
+    volumes:
+      - "./data/minio/certs/:/tmp/certs"
+      - "./nginx/html/:/home/tsm/html"
+
+
+  # Summary
+  # =======
+  # brief: central database
+  # dir: data/postgres mosquitto frontend dispatcher
+  # see also:
+  #     - data/postgres/Dockerfile,
+  #     - scripts mentioned in the 'volumes' section below
+  # admin login:
+  #     - TODO: replace with correct $ENVVARS
+  #     - `psql postgresql:// USER : PASSWORD @ HOST : PORT / DATABASE`
+  #     - `psql postgresql://postgres:postgres@localhost:5432/postgres`
+  #
+  # Description
+  # ===========
+  # This service provide database(s) for other services, namely
+  # 1. a database for each group/project to store the things and the observations
+  # 2. 'mqtt_auth' database for the service 'mqtt-broker'. It holds the mqtt-user,
+  #   which are allowed to send data to the broker
+  # 3. 's3map_db' database, for some worker-services. It maps group specific
+  #   object-storage buckets to things and corresponding databases (see 1.)
+  # 4. the 'frontenddb' database for the service 'frontend', which use django
+  # 5. SMS: TODO
+  # 6. SMS_CV: TODO
+  # To see how and where the creation of each db is defined see in the volumes section
+  # for the scripts, which gets mounted and executed on startup.
+  database:
+    restart: "${RESTART:-on-failure}"
+    # image: timescale/timescaledb:${DATABASE_IMAGE_TAG:-latest-pg14}
+    build:
+      context: "data/postgres"
+      args:
+        UID: "${UID}"
+    ports:
+      - "${POSTGRES_PORT:-127.0.0.1:5432}:5432"
+    environment:
+      POSTGRES_USER: "${POSTGRES_USER:\
+                     ?Please define an postgres user!}"
+      POSTGRES_PASSWORD: "${POSTGRES_PASSWORD:\
+                         ?Please define an postgres password!}"
+      PGDATA: "/var/lib/postgresql/data/pgdata"
+      MQTT_AUTH_POSTGRES_USER: "${MQTT_AUTH_POSTGRES_USER:?\
+                               Please define a mqtt auth database user!}"
+      MQTT_AUTH_POSTGRES_PASS: "${MQTT_AUTH_POSTGRES_PASS:?\
+                               Please define a mqtt auth database password!}"
+      S3MAP_POSTGRES_USER: "${S3MAP_POSTGRES_USER:?\
+                               Please define a s3-map database user!}"
+      S3MAP_POSTGRES_PASS: "${S3MAP_POSTGRES_PASS:?\
+                               Please define a s3-map database password!}"
+      FRONTEND_POSTGRES_USER: "${FRONTEND_POSTGRES_USER:?\
+                               Please define a frontend database user!}"
+      FRONTEND_POSTGRES_PASS: "${FRONTEND_POSTGRES_PASS:?\
+                               Please define a frontend database password!}"
+      SMS_DB_USER: "${SMS_DB_USER:\
+                           ?Please define a SMS postgres user!}"
+      SMS_DB_PASSWORD: "${SMS_DB_PASSWORD:\
+                           ?Please define a SMS postgres password!}"
+      SMS_DB_PORT: "${SMS_DB_PORT:\
+                           ?Please define a SMS postgres port!}"
+      SMS_DB_DB: "${SMS_DB_DB:\
+                           ?Please define a SMS postgres DB name!}"
+      SMS_DB_HOST: "${SMS_DB_HOST:\
+                           ?Please define a SMS postgres host!}"
+      CV_DB_USER: "${CV_DB_USER:\
+                           ?Please define a SMS_CV postgres user!}"
+      CV_DB_PASSWORD: "${CV_DB_PASSWORD:\
+                           ?Please define a SMS_CV postgres password!}"
+      CV_DB_PORT: "${CV_DB_PORT:\
+                           ?Please define a SMS_CV postgres port!}"
+      CV_DB_DB: "${CV_DB_DB:\
+                           ?Please define a SMS_CV postgres DB name!}"
+      CV_DB_HOST: "${CV_DB_HOST:\
+                           ?Please define a SMS_CV postgres host!}"
+    volumes:
+      - "./data/postgres/sms_db/sms_ddl.sql:/docker-entrypoint-initdb.d/01_sms_ddl.sql"
+      - "./data/postgres/sms_db/sms_fdw.sh:/docker-entrypoint-initdb.d/02_sms_fdw.sh"
+      - "./data/postgres/sms_db/sms_foreign_tables.sql:/docker-entrypoint-initdb.d/03_sms_foreign_tables.sql"
+      - "./data/postgres/postgis.sh:/docker-entrypoint-initdb.d/04_postgis.sh"
+      - "./data/postgres/sms_cv_db/sms_cv_ddl.sql:/docker-entrypoint-initdb.d/05_sms_cv_ddl.sql"
+      - "./data/postgres/sms_cv_db/sms_cv_fdw.sh:/docker-entrypoint-initdb.d/06_sms_cv_fdw.sh"
+      - "./data/postgres/sms_cv_db/sms_cv_foreign_tables.sql:/docker-entrypoint-initdb.d/07_sms_cv_foreign_tables.sql"
+      - "./mosquitto/mosquitto-go-auth.sh:/docker-entrypoint-initdb.d/mosquitto-go-auth.sh:ro"
+      - "./frontend/frontend-database.sh:/docker-entrypoint-initdb.d/frontend-database.sh:ro"
+      - "./dispatcher/s3_to_db.sh:/docker-entrypoint-initdb.d/s3_to_db.sh:ro"
+      - "./data/postgres/data:/var/lib/postgresql/data"
+      - "./data/postgres/postgres-force-ssl.sh:\
+      /docker-entrypoint-initdb.d/postgres-force-ssl.sh"
+      - "${POSTGRES_TLS_CERT_PATH:-/tmp/c8cf2d92-\
+      73cd-11ec-b035-54e1ad7c5c19}:/var/lib/postgresql/server.crt"
+      - "${POSTGRES_TLS_KEY_PATH:-/tmp/c8cf2d92-\
+      73cd-11ec-b035-54e1ad7c5c19}:/var/lib/postgresql/server.key"
+    user: "${UID}:${GID}"
+    command: "${POSTGRES_EXTRA_PARAMS:-}"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $POSTGRES_USER"]
+      interval: "${POSTGRES_HEALTHCHECK_INTERVAL:-10s}"
+      timeout: 5s
+      retries: 5
+
+  # Summary
+  # =======
+  # brief: raw storage for observation files
+  # dir: none
+  # see also: none
+  # mqtt-topics: "object_storage_notification" (send)
+  # admin login:
+  #     -> http://localhost/object-storage/login
+  #     -> user, pass: $MINIO_ROOT_USER, $MINIO_ROOT_PASSWORD  (default: minioadmin)
+  #
+  # Description
+  # ===========
+  # The 'object-storage' aka. 'raw-data-storage' (or simply 'the minio' or 'S3') is
+  # a file based storage for the users to upload raw data files (e.g. some
+  # observations as a CSV).
+  # The upload can be done via (S)FTP or via a client or via a web frontend. The storage
+  # sends mqtt messages to the broker on file upload (and other events).
+  object-storage:
+    image: "minio/minio:${OBJECT_STORAGE_IMAGE_TAG:-latest}"
+    restart: "${RESTART:-on-failure}"
+    ports:
+      - "${MINIO_SFTP_PORT:-127.0.0.1:40022}:22"
+      - "${MINIO_FTP_PORT:-127.0.0.1:40021}:21"
+      - "${MINIO_FTP_PASV_PORTS:-127.0.0.1:30000-30010:30000-30010}"
+    depends_on:
+      mqtt-broker:
+        condition: "service_started"
+      init:
+        condition: "service_completed_successfully"
+    environment:
+      MINIO_ROOT_USER: "${MINIO_ROOT_USER:?Please define an minio root user!}"
+      MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:?\
+      Please define an minio root user password!}"
+      MINIO_BROWSER_REDIRECT_URL: "${MINIO_BROWSER_REDIRECT_URL}"
+      MINIO_NOTIFY_MQTT_ENABLE_LOCAL_BROKER: "on"
+      MINIO_NOTIFY_MQTT_BROKER_LOCAL_BROKER: "tcp://mqtt-broker:1883"
+      MINIO_NOTIFY_MQTT_TOPIC_LOCAL_BROKER: "object_storage_notification"
+      MINIO_NOTIFY_MQTT_USERNAME_LOCAL_BROKER: "${MQTT_USER:\
+      ?Please define a mqtt user!}"
+      MINIO_NOTIFY_MQTT_PASSWORD_LOCAL_BROKER: "${MQTT_PASSWORD:\
+      ?Please define a mqtt password!}"
+      MINIO_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL_LOCAL_BROKER: "60s"
+      #      MINIO_NOTIFY_MQTT_QOS_LOCAL_BROKER: "<string>"
+      MINIO_NOTIFY_MQTT_RECONNECT_INTERVAL_LOCAL_BROKER: "60s"
+      #      MINIO_NOTIFY_MQTT_QUEUE_DIR_LOCAL_BROKER: "<string>"
+      #      MINIO_NOTIFY_MQTT_QUEUE_LIMIT_LOCAL_BROKER: "<string>"
+      #      MINIO_NOTIFY_MQTT_COMMENT_LOCAL_BROKER: "<string>"
+      MINIO_SERVER_URL: "${MINIO_SERVER_URL:-}"
+    volumes:
+      - "./data/minio/vol0:/vol0"
+      # FTP server tls key and cert
+      - "${MINIO_FTP_TLS_CRT:-./data/minio/certs/minio-ftp.crt}:/certs/minio-ftp.crt:ro"
+      - "${MINIO_FTP_TLS_KEY:-./data/minio/certs/minio-ftp.key}:/certs/minio-ftp.key:ro"
+      # SSH Server key to provide constant ssh host key
+      - "${MINIO_SFTP_HOSTKEY:-./data/minio/certs/id_ed25519}:/certs/id_ed25519:ro"
+    user: "${UID}:${GID}"
+    command:
+      - "server"
+      - "--console-address"
+      - ":9001"
+      - "--ftp"
+      - "address=:21"
+      - "--ftp"
+      - "passive-port-range=30000-30010"
+      - "--sftp"
+      - "address=:22"
+      - "--sftp"
+      - "ssh-private-key=/certs/id_ed25519"
+      - "--ftp"
+      - "tls-private-key=/certs/minio-ftp.key"
+      - "--ftp"
+      - "tls-public-cert=/certs/minio-ftp.crt"
+      - "--json"
+      - "/vol0"
+    healthcheck:
+      test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/9000' || exit 1
+      interval: "${MINIO_HEALTHCHECK_INTERVAL:-2s}"
+      timeout: 5s
+      retries: 15
+
+
+  # Summary
+  # =======
+  # brief: central mqtt message bus / broker
+  # dir: mosquitto
+  # see also: mosquitto/docker-entrypoint.sh
+  #
+  # Description
+  # ===========
+  # This is the main message bus, all mqtt messages are send here. Services can
+  # subscribe to topics and will receive messages that was sent to them.
+  mqtt-broker:
+    restart: "${RESTART:-on-failure}"
+    image: "iegomez/mosquitto-go-auth:${MQTT_BROKER_IMAGE_TAG:-latest}"
+    depends_on:
+      database:
+        condition: "service_healthy"
+    command:
+      - "/usr/sbin/mosquitto"
+      - "-c"
+      - "/var/lib/mosquitto/mosquitto.conf"
+    entrypoint:
+      - "/docker-entrypoint.sh"
+    user: "${UID}:${GID}"
+    environment:
+      MQTT_USER: "${MQTT_USER:?Please define a mqtt user!}"
+      MQTT_PASSWORD: "${MQTT_PASSWORD:?Please define a mqtt password!}"
+      MQTT_INGEST_USER: "${MQTT_INGEST_USER:?Please define a mqtt user!}"
+      MQTT_INGEST_PASSWORD: "${MQTT_INGEST_PASSWORD:?\
+                            Please define a mqtt password!}"
+      MQTT_AUTH_POSTGRES_HOST: "${MQTT_AUTH_POSTGRES_HOST:?\
+                               Please define a mqtt auth database host!}"
+      MQTT_AUTH_POSTGRES_PORT: "${MQTT_AUTH_POSTGRES_PORT:?\
+                               Please define a mqtt auth database portt!}"
+      MQTT_AUTH_POSTGRES_USER: "${MQTT_AUTH_POSTGRES_USER:?\
+                               Please define a mqtt auth database user!}"
+      MQTT_AUTH_POSTGRES_PASS: "${MQTT_AUTH_POSTGRES_PASS:?\
+                               Please define a mqtt auth database password!}"
+      MQTT_AUTH_POSTGRES_DB: "${MQTT_AUTH_POSTGRES_DB\
+                               :?Please define a mqtt auth database name!}"
+      MQTT_AUTH_PG_TLSMODE: "${MQTT_AUTH_PG_TLSMODE\
+                              :?Please define a mqtt auth database connection\
+                               tls mode!}"
+      FRONTEND_MQTT_USER: "${FRONTEND_MQTT_USER\
+                            :?Please define a frontend mqtt user!}"
+      FRONTEND_MQTT_PASS: "${FRONTEND_MQTT_PASS\
+                            :?Please define a frontend mqtt password!}"
+    ports:
+      - "${MOSQUITTO_PORT:-127.0.0.1:1883}:1883"
+      - "${MOSQUITTO_PORT_SECURE:-127.0.0.1:8883}:8883"
+    volumes:
+      - "${MOSQUITTO_CONFIG:-./mosquitto/mosquitto.dev.conf}\
+      :/etc/mosquitto/config/mosquitto.conf:ro"
+      - "${MOSQUITTO_TLS_CONFIG:-./mosquitto/mosquitto.tls.conf}\
+      :/etc/mosquitto/config/tls/mosquitto.tls.conf:ro"
+      - "./data/mosquitto/auth:/mosquitto-auth/"
+      - "${MOSQUITTO_TLS_CERT_PATH:-/tmp/c8cf2d92\
+      -73cd-11ec-b035-54e1ad7c5c19}:\
+      /mosquitto/config/certs/server.crt:ro"
+      - "${MOSQUITTO_TLS_KEY_PATH:-/tmp/c8cf2d92\
+      -73cd-11ec-b035-54e1ad7c5c19}:\
+      /mosquitto/config/certs/server.key:ro"
+      - "${MOSQUITTO_TLS_CA_PATH:-/tmp/c8cf2d92\
+      -73cd-11ec-b035-54e1ad7c5c19}:/mosquitto/config/certs/ca.crt:ro"
+      - "./mosquitto/docker-entrypoint.sh:/docker-entrypoint.sh"
+    tmpfs:
+      - "/var/lib/mosquitto/:uid=${UID}"
+    healthcheck:
+      test: ["CMD-SHELL", "mosquitto_sub -t \
+      '$$SYS/broker/version' -C 1 -u $${MQTT_USER} \
+      -P $${MQTT_PASSWORD} --id docker-compose-healthcheck"]
+      interval: "${MQTT_BROKER_HEALTHCHECK_INTERVAL:-10s}"
+      timeout: 10s
+      retries: 6
+    logging:
+      options:
+        max-size: "${MQTT_BROKER_LOG_SIZE:-100K}"
+        max-file: "${MQTT_BROKER_FILE_COUNT:-10}"
+
+
+
+  # Summary
+  # =======
+  # brief: visualisation of observations stored in the -> database
+  # dir: grafana
+  # see also: none
+  # admin login:
+  #     -> http://localhost/visualization/login
+  #     -> user, pass: $GRAFANA_USER, $GRAFANA_PASSWORD  (default: grafana)
+  #
+  # Description
+  # ===========
+  # This visualize observations of a group or project that are stored in the database.
+  visualization:
+    restart: "${RESTART:-on-failure}"
+    image: "grafana/grafana:${GRAFANA_IMAGE_TAG:-latest}"
+    user: "${UID}:${GID}"
+    environment:
+      - "GF_SECURITY_ADMIN_USER=\
+      ${GRAFANA_USER:?Please define a grafana user}"
+      - "GF_SECURITY_ADMIN_PASSWORD=\
+      ${GRAFANA_PASSWORD:?Please define a grafana password}"
+      - "GF_SERVER_ROOT_URL=${GRAFANA_PROXY_URL:-http://localhost/visualization/}"
+      - "GF_SERVER_SERVE_FROM_SUB_PATH=true"
+      - "GF_AUTH_GENERIC_OAUTH_ENABLED=true"
+      - "GF_AUTH_GENERIC_OAUTH_NAME=Helmholtz AAI"
+      - "GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true"
+      - "GF_AUTH_GENERIC_OAUTH_CLIENT_ID=${DJANGO_HELMHOLTZ_CLIENT_ID}"
+      - "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${DJANGO_HELMHOLTZ_CLIENT_SECRET}"
+      - "GF_AUTH_GENERIC_OAUTH_SCOPES=email profile eduperson_principal_name"
+      - "GF_AUTH_GENERIC_OAUTH_AUTH_URL=${GF_AUTH_GENERIC_OAUTH_AUTH_URL}"
+      - "GF_AUTH_GENERIC_OAUTH_TOKEN_URL=${GF_AUTH_GENERIC_OAUTH_TOKEN_URL}"
+      - "GF_AUTH_GENERIC_OAUTH_API_URL=${GF_AUTH_GENERIC_OAUTH_API_URL}"
+      - "GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH=eduperson_principal_name"
+      - "GF_AUTH_OAUTH_SKIP_ORG_ROLE_UPDATE_SYNC=true"
+    volumes:
+      - "./grafana/provisioning:/etc/grafana/provisioning/"
+      - "./data/grafana/:/var/lib/grafana"
+
+
+  # Summary
+  # =======
+  # brief:
+  # dir:
+  # see also:
+  #
+  # Description
+  # ===========
+  # TODO
+  tsmdl:
+    image: "registry.hzdr.de/ufz-tsm/tsm-ufz-tsmdl/tsm-ufz-tsmdl:${TSM_DL_IMAGE_TAG:-latest}"
+    restart: "${RESTART:-on-failure}"
+    entrypoint: "/app/start.sh"
+    environment:
+      DB_URL: "postgresql://\
+      ${CREATEDB_POSTGRES_USER:?Please define an postgres user!}:\
+      ${CREATEDB_POSTGRES_PASSWORD:?Please define an postgres password!}@\
+      ${CREATEDB_POSTGRES_HOST:-database}/\
+      ${CREATEDB_POSTGRES_DATABASE:-postgres}"
+      WEB_CONCURRENCY: 10
+      UVICORN_ARGS: "${UVICORN_ARGS:-} --root-path /tsmdl"
+    depends_on:
+      database:
+        condition: service_healthy
+
+
+  # Summary
+  # =======
+  # brief:
+  # dir:
+  # see also:
+  #
+  # Description
+  # ===========
+  # TODO
+  frost:
+    build:
+      context: "data/tomcat"
+      args:
+        UID: "${UID:-1000}"
+        GID: "${GID:-1000}"
+    restart: "${RESTART:-on-failure}"
+    volumes:
+      - "./data/tomcat/context:/usr/local/tomcat/conf/Catalina/localhost:ro"
+

docker-compose-frontend.yml

+---
+version: "3.9"
+services:
+
+  frontend:
+    image: "registry.hzdr.de/ufz-tsm/tsm-frontend/tsm-frontend:${FRONTEND_IMAGE_TAG:-latest}"
+    restart: "${RESTART:-on-failure}"
+    command: >
+      bash -c "python3 manage.py migrate
+      && python3 manage.py loaddata admin_interface_theme_foundation.json
+      && python3 manage.py loaddata ufz_theme.json
+      && python3 manage.py createsuperuser --noinput
+      || echo 'Superuser already created'
+      && python3 manage.py collectstatic --noinput
+      && gunicorn main.wsgi:application --bind 0.0.0.0:8000 -w 6"
+    user: "appuser"
+    volumes:
+      - "frontend-statics:/home/appuser/app/static"
+      - "./frontend/user.json:/home/appuser/app/tsm/fixtures/user.json"
+      - "./frontend/thing.json:/home/appuser/app/tsm/fixtures/thing.json"
+    entrypoint: ""
+    # ports:
+    #   - "127.0.0.1:8000:8000"
+    environment:
+      - "POSTGRES_HOST=${FRONTEND_POSTGRES_HOST}"
+      - "POSTGRES_NAME=${FRONTEND_POSTGRES_DB:-postgres}"
+      - "POSTGRES_USER=${FRONTEND_POSTGRES_USER}"
+      - "POSTGRES_PASSWORD=${FRONTEND_POSTGRES_PASS}"
+      - "PUBLISH_THING_TO_BROKER=True"
+      - "CREATEDB_POSTGRES_HOST=${CREATEDB_POSTGRES_HOST}"
+      - "CREATEDB_POSTGRES_DATABASE=${CREATEDB_POSTGRES_DATABASE}"
+      - "MQTT_BROKER_HOST=${FRONTEND_MQTT_HOST}"
+      - "MQTT_USER=${FRONTEND_MQTT_USER}"
+      - "MQTT_PASSWORD=${FRONTEND_MQTT_PASS}"
+      - "DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME}"
+      - "DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD}"
+      - "DJANGO_SUPERUSER_EMAIL=${DJANGO_SUPERUSER_EMAIL:-example@example.com}"
+      - "DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY:?Please define a unique and\
+        secure Django secret key!}"
+      - "DJANGO_DEBUG=${DJANGO_DEBUG:-0}"
+      - "DJANGO_TRUSTED_ORIGINS=${DJANGO_TRUSTED_ORIGINS:-http://localhost:80}"
+      - "DJANGO_BASE_PATH=${DJANGO_BASE_PATH:-/frontend/}"
+      - "DJANGO_ALLOWED_HOSTS=${DJANGO_ALLOWED_HOSTS:-}"
+      - "POSTGRES_SSLMODE=${DJANGO_POSTGRES_SSLMODE:-verify-full}"
+      - "DJANGO_HELMHOLTZ_CLIENT_ID=${DJANGO_HELMHOLTZ_CLIENT_ID:-ufz-tsm}"
+      - "DJANGO_HELMHOLTZ_CLIENT_SECRET=${DJANGO_HELMHOLTZ_CLIENT_SECRET:-00000000000000000000}"
+      - "DJANGO_HELMHOLTZ_AAI_CONF_URL=${DJANGO_HELMHOLTZ_AAI_CONF_URL:\
+            -https://login-dev.helmholtz.de/oauth2/.well-known/openid-configuration}"
+      - "MINIO_FTP_PORT=${MINIO_FTP_PORT:-127.0.0.1:40021}"
+      - "MINIO_SFTP_PORT=${MINIO_SFTP_PORT:-127.0.0.1:40022}"
+      - "PROXY_URL=${PROXY_URL:-http://localhost:80}"
+    depends_on:
+      database:
+        condition: service_healthy
+      mqtt-broker:
+        condition: service_healthy
+
+
+  proxy:
+    image: "nginx:${NGINX_IMAGE_TAG:-latest}"
+    restart: "${RESTART:-on-failure}"
+    ports:
+      - "${PROXY_PLAIN_PORT:-127.0.0.1:80:80}"
+      - "${PROXY_TLS_PORT:-127.0.0.1:443:443}"
+      - "${PROXY_MINIO_PORT:-127.0.0.1:9000:9000}"
+    volumes:
+      - "frontend-statics:/home/appuser/app/static"
+      - "./nginx/html:/usr/share/nginx/html"
+      - "./nginx/${PROXY_SITE_CONFIG_FILE:-tsm.dev.conf}:/etc/nginx/conf.d/default.conf:ro"
+      - "./nginx/locations:/etc/nginx/locations:ro"
+      - "${PROXY_TLS_CERT_PATH:-/tmp/c8cf2d92-73cd-11ec-b035-54e1ad7c5c19}:/etc/ssl/public.crt:ro"
+      - "${PROXY_TLS_KEY_PATH:-/tmp/c8cf2d92-73cd-11ec-b035-54e1ad7c5c19}:/etc/ssl/private.key:ro"
+    depends_on:
+      frontend:
+        condition: "service_started"
+      visualization:
+        condition: "service_started"
+      object-storage:
+        condition: "service_started"
+      tsmdl:
+        condition: "service_started"
+      frost:
+        condition: "service_started"
+      init:
+        condition: "service_completed_successfully"
+
+
+volumes:
+  frontend-statics:

docker-compose-worker.yml

+---
+version: "3.9"
+services:
+
+  # =================================================================
+  # SETUP worker (topic: thing_creation)
+  # =================================================================
+
+  worker-object-storage-setup:
+    image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG:-latest}"
+    restart: "${RESTART:-on-failure}"
+    depends_on:
+      mqtt-broker:
+        condition: "service_healthy"
+      object-storage:
+        condition: "service_healthy"
+    environment:
+      LOG_LEVEL: "${LOG_LEVEL:-INFO}"
+      TOPIC: "thing_creation"
+      MQTT_BROKER: "mqtt-broker:1883"
+      MQTT_USER: "${MQTT_USER:?Please define a mqtt user!}"
+      MQTT_PASSWORD: "${MQTT_PASSWORD:?Please define a mqtt password!}"
+      MINIO_SECURE: "${MINIO_SECURE:-False}"
+      MINIO_URL: "${MINIO_HOST:-minio:9000}"
+      MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:?Please define an minio root user!}"
+      MINIO_SECURE_KEY: "${MINIO_ROOT_PASSWORD:?Please define an minio root user password!}"
+      S3MAP_DB_URL: "postgresql://\
+        ${S3MAP_POSTGRES_USER:?Please define an postgres user!}:\
+        ${S3MAP_POSTGRES_PASS:?Please define an postgres password!}@\
+        ${S3MAP_POSTGRES_HOST:-database}/\
+        ${S3MAP_POSTGRES_DB:-postgres}"
+    command:
+      - "run-create-thing-on-minio-action-service"
+
+
+  worker-db-setup:
+    image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG:-latest}"
+    restart: "${RESTART:-on-failure}"
+    depends_on:
+      mqtt-broker:
+        condition: "service_healthy"
+      database:
+        condition: "service_healthy"
+    environment:
+      LOG_LEVEL: "${LOG_LEVEL:-INFO}"
+      TOPIC: "thing_creation"
+      MQTT_BROKER: "mqtt-broker:1883"
+      MQTT_USER: "${MQTT_USER:?Please define a mqtt user!}"
+      MQTT_PASSWORD: "${MQTT_PASSWORD:?Please define a mqtt password!}"
+      DATABASE_URL: "postgresql://\
+        ${CREATEDB_POSTGRES_USER:?Please define an postgres user!}:\
+        ${CREATEDB_POSTGRES_PASSWORD:?Please define an postgres password!}@\
+        ${CREATEDB_POSTGRES_HOST:-database}/\
+        ${CREATEDB_POSTGRES_DATABASE:-postgres}"
+      SMS_URL: "${SMS_URL}"
+      CV_URL: "${CV_URL}"
+    command:
+      - "run-create-database-schema-action-service"
+
+
+  worker-frost-setup:
+    image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG:-latest}"
+    restart: "${RESTART:-on-failure}"
+    depends_on:
+      mqtt-broker:
+        condition: "service_healthy"
+      database:
+        condition: "service_healthy"
+    volumes:
+      - "./data/tomcat/context:/home/appuser/app/src/CreateNewFrostInstanceAction/tomcat/context_files:rw"
+    environment:
+      LOG_LEVEL: "${LOG_LEVEL:-INFO}"
+      TOPIC: "thing_creation"
+      MQTT_BROKER: "mqtt-broker:1883"
+      MQTT_USER: "${MQTT_USER:?Please define a mqtt user!}"
+      MQTT_PASSWORD: "${MQTT_PASSWORD:?Please define a mqtt password!}"
+      TOMCAT_PROXY_URL: "${TOMCAT_PROXY_URL:-http://localhost/sta/}"
+    command:
+      - "run-create-new-frost-server-service"
+
+
+  worker-mqtt-user-creation:
+    image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG:-latest}"
+    restart: on-failure
+    depends_on:
+      mqtt-broker:
+        condition: "service_healthy"
+      database:
+        condition: "service_healthy"
+    environment:
+      LOG_LEVEL: "${LOG_LEVEL:-INFO}"
+      TOPIC: "thing_creation"
+      MQTT_BROKER: "mqtt-broker:1883"
+      MQTT_USER: "${MQTT_USER:?Please define a mqtt user!}"
+      MQTT_PASSWORD: "${MQTT_PASSWORD:?Please define a mqtt password!}"
+      DATABASE_URL: "postgresql://\
+        ${CREATEDB_POSTGRES_USER:?Please define an postgres user!}:\
+        ${CREATEDB_POSTGRES_PASSWORD:?Please define an postgres password!}@\
+        ${CREATEDB_POSTGRES_HOST:-database}/\
+        ${CREATEDB_POSTGRES_DATABASE:-postgres}"
+    command:
+      - run-create-mqtt-user-action-service
+
+
+  worker-grafana-dashboard:
+    image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG:-latest}"
+    restart: "${RESTART:-on-failure}"
+    depends_on:
+      mqtt-broker:
+        condition: "service_healthy"
+      visualization:
+        condition: "service_started"
+    environment:
+      LOG_LEVEL: "${LOG_LEVEL:-INFO}"
+      TOPIC: "thing_creation"
+      MQTT_BROKER: "mqtt-broker:1883"
+      MQTT_USER: "${MQTT_USER:?Please define a mqtt user!}"
+      MQTT_PASSWORD: "${MQTT_PASSWORD:?Please define a mqtt password!}"
+      GRAFANA_URL: "${GRAFANA_URL:-http://visualization:3000}"
+      GRAFANA_USER: "${GRAFANA_USER:?Please define a grafana user!}"
+      GRAFANA_PASSWORD: "${GRAFANA_PASSWORD:?Please define a grafana password!}"
+      GRAFANA_DEFAULT_DATASOURCE_SSLMODE: "${GRAFANA_DEFAULT_DATASOURCE_SSLMODE:-disable}"
+    command:
+      - "run-create-grafana-dashboard-service"
+
+
+  # =================================================================
+  # Ingest worker (incoming observations)
+  # =================================================================
+
+  worker-file-ingest:
+    image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG:-latest}"
+    restart: "on-failure"
+    depends_on:
+      mqtt-broker:
+        condition: "service_healthy"
+      object-storage:
+        condition: "service_healthy"
+      basic-demo-scheduler:
+        condition: "service_healthy"
+    environment:
+      LOG_LEVEL: "${LOG_LEVEL:-INFO}"
+      TOPIC: "object_storage_notification"
+      MQTT_BROKER: "mqtt-broker:1883"
+      MQTT_USER: "${MQTT_USER:?Please define a mqtt user!}"
+      MQTT_PASSWORD: "${MQTT_PASSWORD:?Please define a mqtt password!}"
+      MINIO_SECURE: "${MINIO_SECURE:-False}"
+      MINIO_URL: "${MINIO_HOST:-minio:9000}"
+      MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:?Please define an minio root user!}"
+      MINIO_SECURE_KEY: "${MINIO_ROOT_PASSWORD:?Please define an minio root user password!}"
+      S3MAP_DB_URL: "postgresql://\
+        ${S3MAP_POSTGRES_USER:?Please define an postgres user!}:\
+        ${S3MAP_POSTGRES_PASS:?Please define an postgres password!}@\
+        ${S3MAP_POSTGRES_HOST:-database}/\
+        ${S3MAP_POSTGRES_DB:-postgres}"
+      SCHEDULER: "${SCHEDULER:-BasicScheduler}"
+      SCHED_HOST: "${SCHED_HOST:-http://basic-demo-scheduler:5000}"
+      SCHED_MQTT_BROKER: "${SCHED_MQTT_BROKER:-None}"
+      SCHED_MQTT_USER: "${SCHED_MQTT_USER:-None}"
+      SCHED_MQTT_PASSWORD: "${SCHED_MQTT_PASSWORD:-None}"
+      SCHED_JOB_LOG_LEVEL: "${SCHED_JOB_LOG_LEVEL:-INFO}"
+      SCHED_SLURM_RESTAPI_VERSION: "${SCHED_SLURM_RESTAPI_VERSION:-None}"
+      SCHED_SLURM_USER: "${SCHED_SLURM_USER:-None}"
+      SCHED_SLURM_JWT_TOKEN: "${SCHED_SLURM_JWT_TOKEN:-None}"
+      # replaced by SCHED_HOST; remove after dispatcher/MR73 is merged
+      SCHED_ENDPOINT_URL: "http://basic-demo-scheduler:5000/extractor/run"
+    command:
+      - "run-process-new-file-service"
+
+
+  worker-run-qaqc:
+    image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG:-latest}"
+    restart: "on-failure"
+    depends_on:
+      mqtt-broker:
+        condition: "service_healthy"
+      basic-demo-scheduler:
+        condition: "service_healthy"
+    environment:
+      LOG_LEVEL: "${LOG_LEVEL:-INFO}"
+      TOPIC: "data_parsed"
+      MQTT_BROKER: "mqtt-broker:1883"
+      MQTT_USER: "${MQTT_USER:?Please define a mqtt user!}"
+      MQTT_PASSWORD: "${MQTT_PASSWORD:?Please define a mqtt password!}"
+      SCHEDULER: "${SCHEDULER:-BasicScheduler}"
+      SCHED_HOST: "${SCHED_HOST:-http://basic-demo-scheduler:5000}"
+      SCHED_MQTT_BROKER: "${SCHED_MQTT_BROKER:-None}"
+      SCHED_MQTT_USER: "${SCHED_MQTT_USER:-None}"
+      SCHED_MQTT_PASSWORD: "${SCHED_MQTT_PASSWORD:-None}"
+      SCHED_JOB_LOG_LEVEL: "${SCHED_JOB_LOG_LEVEL:-INFO}"
+      SCHED_SLURM_RESTAPI_VERSION: "${SCHED_SLURM_RESTAPI_VERSION:-None}"
+      SCHED_SLURM_USER: "${SCHED_SLURM_USER:-None}"
+      SCHED_SLURM_JWT_TOKEN: "${SCHED_SLURM_JWT_TOKEN:-None}"
+      # replaced by SCHED_HOST; remove after dispatcher/MR73 is merged
+      SCHED_ENDPOINT_URL: "http://basic-demo-scheduler:5000/qaqc/run"
+    command:
+      - "run-qaqc"
+
+
+  worker-mqtt-ingest:
+    image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG:-latest}"
+    restart: on-failure
+    depends_on:
+      mqtt-broker:
+        condition: "service_healthy"
+      database:
+        condition: "service_healthy"
+    environment:
+      LOG_LEVEL: "${LOG_LEVEL:-INFO}"
+      TOPIC: "mqtt_ingest/#"
+      MQTT_BROKER: "mqtt-broker:1883"
+      MQTT_USER: "${MQTT_INGEST_USER:?Please define a mqtt user!}"
+      MQTT_PASSWORD: "${MQTT_INGEST_PASSWORD:?Please define a mqtt password!}"
+    command:
+      - parse-data
+      - --target-uri
+      - "postgresql://\
+        ${CREATEDB_POSTGRES_USER:?Please define an postgres user!}:\
+        ${CREATEDB_POSTGRES_PASSWORD:?Please define an postgres password!}@\
+        ${CREATEDB_POSTGRES_HOST:-database}/\
+        ${CREATEDB_POSTGRES_DATABASE:-postgres}"
+
+
+  # =================================================================
+  # other worker
+  # =================================================================
+
+  worker-grafana-user-orgs:
+    image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG:-latest}"
+    restart: "${RESTART:-on-failure}"
+    depends_on:
+      mqtt-broker:
+        condition: "service_healthy"
+      visualization:
+        condition: "service_started"
+    environment:
+      LOG_LEVEL: "${LOG_LEVEL:-INFO}"
+      TOPIC: "user_login"
+      MQTT_BROKER: "mqtt-broker:1883"
+      MQTT_USER: "${MQTT_USER:?Please define a mqtt user!}"
+      MQTT_PASSWORD: "${MQTT_PASSWORD:?Please define a mqtt password!}"
+      GRAFANA_URL: "${GRAFANA_URL:-http://visualization:3000}"
+      GRAFANA_USER: "${GRAFANA_USER:?Please define a grafana user!}"
+      GRAFANA_PASSWORD: "${GRAFANA_PASSWORD:?Please define a grafana password!}"
+    command:
+      - "run-create-grafana-user-orgs-service"
+
+
+  basic-demo-scheduler:
+    image: "registry.hzdr.de/ufz-tsm/tsm-basic-demo-scheduler/basic_demo_scheduler:${SCHEDULER_IMAGE_TAG:-latest}"
+    restart: "${RESTART:-on-failure}"
+    # prevent blocking/slowing-down of other services
+    # if many scheduler jobs occur at the same time
+    cpu_count: ${SCHEDULER_CPU_COUNT:-1}
+    healthcheck:
+      test: [ "CMD-SHELL", "curl --fail http://localhost:5000/health || exit 1" ]
+      interval: 10s
+      timeout: 2s
+      retries: 5
+    environment:
+      MQTT_BROKER: "mqtt-broker:1883"
+      MQTT_USER: "${MQTT_USER:?Please define a mqtt user!}"
+      MQTT_PASSWORD: "${MQTT_PASSWORD:?Please define a mqtt password!}"
+    command:
+      - "--verbose"