Merge branch 'qc_as_a_service' into 'main'

Qc as a service

See merge request !215

.env.example

@@ -223,33 +223,12 @@ CONFIGDB_UPDATER_IMAGE_TAG=latest
 #  soon we will change that.
 ############################################################
 DISPATCHER_IMAGE_TAG=latest
-
-SCHEDULER=BasicScheduler
-SCHED_HOST=http://basic-demo-scheduler:5000
-
-# use only on stage/production
-#SCHEDULER=SlurmScheduler
-#SCHED_HOST=https://head3.eve.ufz.de/slurmrest
-SCHED_SLURM_RESTAPI_VERSION=v0.0.38
-SCHED_MQTT_BROKER=tsm.intranet.ufz.de:8883
-SCHED_MQTT_USER=$MQTT_USER
-SCHED_MQTT_PASSWORD=$MQTT_PASSWORD
-SCHED_JOB_LOG_LEVEL=DEBUG
-SCHED_SLURM_USER=
-SCHED_SLURM_JWT_TOKEN=
-
 TOMCAT_PROXY_URL=${PROXY_URL}/sta/
 
 ############################################################
-# basic-demo-scheduler (python, flask)
-############################################################
-
-############################################################
-# basic-demo-scheduler (python, flask)
+# QC worker (python)
 ############################################################
-SCHEDULER_IMAGE_TAG=latest
-# This is used in the Dockerfile of basic-demo-scheduler
-EXTRACTOR_IMAGE_TAG=latest
+TIMEIO_QAQC_IMAGE_TAG=latest
 
 ############################################################
 # cron-scheduler (debian, python)

docker-compose-dev.example.yml

@@ -23,7 +23,8 @@ services:
 
   worker-run-qaqc:
     volumes:
-      - "${DEV_DISPATCHER_SRC_DIR:-../tsm-dispatcher/src}:/home/appuser/app/src"
+      - "${DEV_QAQC_SRC_DIR:-../timeio-qaqc/timeio_qaqc}:/app/timeio_qaqc"
+      - "${DEV_QAQC_SRC_DIR:-../timeio-qaqc/app.py}:/app/app.py"
 
   worker-mqtt-ingest:
     volumes:
@@ -45,22 +46,6 @@ services:
     volumes:
       - "${DEV_DISPATCHER_SRC_DIR:-../tsm-dispatcher/src}:/home/appuser/app/src"
 
-  tsm-extractor:
-    image: "registry.hzdr.de/ufz-tsm/tsm-extractor/extractor:${EXTRACTOR_IMAGE_TAG}"
-    build:
-      context: "../tsm-extractor"
-
-  basic-demo-scheduler:
-    volumes:
-      - "${DEV_EXTRACTOR_SRC_DIR:-../tsm-extractor/src}:/home/appuser/app/src"
-      - "${DEV_SCHEDULER_SRC_DIR:-../tsm-basic-demo-scheduler/src}:/home/appuser/app/basic_demo_scheduler"
-    ports:
-      - "127.0.0.1:5000:5000"
-    build:
-      context: "../tsm-basic-demo-scheduler"
-      args:
-        EXTRACTOR_IMAGE_TAG: "${EXTRACTOR_IMAGE_TAG}"
-
   frontend:
     environment:
       - "DJANGO_HELMHOLTZ_CLIENT_ID=${DJANGO_HELMHOLTZ_CLIENT_ID:-timeIO-client}"

docker-compose.yml

@@ -693,8 +693,6 @@ services:
         condition: service_healthy
       object-storage:
         condition: service_healthy
-      basic-demo-scheduler:
-        condition: service_healthy
     environment:
       LOG_LEVEL: "${LOG_LEVEL}"
       TOPIC: object_storage_notification
@@ -713,12 +711,6 @@ services:
         ${S3MAP_POSTGRES_PASS}@\
         ${S3MAP_POSTGRES_HOST}/\
         ${S3MAP_POSTGRES_DB}"
-      SCHEDULER: "${SCHEDULER}"
-      SCHED_HOST: "${SCHED_HOST}"
-      SCHED_MQTT_BROKER: "${SCHED_MQTT_BROKER}"
-      SCHED_MQTT_USER: "${SCHED_MQTT_USER}"
-      SCHED_MQTT_PASSWORD: "${SCHED_MQTT_PASSWORD}"
-      SCHED_JOB_LOG_LEVEL: "${SCHED_JOB_LOG_LEVEL}"
       CONFIGDB_DSN: "postgresql://\
         ${CONFIGDB_USER}:\
         ${CONFIGDB_PASSWORD}@\
@@ -734,33 +726,34 @@ services:
 
 
   worker-run-qaqc:
-    image: "registry.hzdr.de/ufz-tsm/tsm-dispatcher/dispatcher:${DISPATCHER_IMAGE_TAG}"
+    image: "registry.hzdr.de/ufz-tsm/timeio-qaqc/timeio-qaqc:${TIMEIO_QAQC_IMAGE_TAG}"
     restart: "${RESTART}"
     depends_on:
       mqtt-broker:
         condition: service_healthy
-      basic-demo-scheduler:
+      database:
+        condition: service_healthy
+      timeio-db-api:
         condition: service_healthy
     environment:
       LOG_LEVEL: "${LOG_LEVEL}"
-      TOPIC: data_parsed
-      MQTT_BROKER: mqtt-broker:1883
+      MQTT_BROKER_HOST: mqtt-broker
+      MQTT_BROKER_PORT: 1883
       MQTT_USER: "${MQTT_USER}"
       MQTT_PASSWORD: "${MQTT_PASSWORD}"
-      MQTT_CLIENT_ID: run-qaqc
       MQTT_CLEAN_SESSION: "${MQTT_CLEAN_SESSION}"
-      MQTT_QOS: "${MQTT_QOS}"
-      SCHEDULER: "${SCHEDULER}"
-      SCHED_HOST: "${SCHED_HOST}"
-      SCHED_MQTT_BROKER: "${SCHED_MQTT_BROKER}"
-      SCHED_MQTT_USER: "${SCHED_MQTT_USER}"
-      SCHED_MQTT_PASSWORD: "${SCHED_MQTT_PASSWORD}"
-      SCHED_JOB_LOG_LEVEL: "${SCHED_JOB_LOG_LEVEL}"
-      SCHED_SLURM_RESTAPI_VERSION: "${SCHED_SLURM_RESTAPI_VERSION}"
-      SCHED_SLURM_USER: "${SCHED_SLURM_USER}"
-      SCHED_SLURM_JWT_TOKEN: "${SCHED_SLURM_JWT_TOKEN}"
-    command:
-      - schedule-qaqc-job
+      MQTT_CLIENT_ID: worker-qaqc
+      MQTT_SUBSCRIBE_TOPIC: data_parsed
+      MQTT_SUBSCRIBE_QOS: "${MQTT_QOS}"
+      MQTT_PUBLISH_TOPIC: qaqc_done
+      MQTT_PUBLISH_QOS: "${MQTT_QOS}"
+      CONFIGDB_DSN: "postgresql://\
+        ${CREATEDB_POSTGRES_USER}:\
+        ${CREATEDB_POSTGRES_PASSWORD}@\
+        ${CREATEDB_POSTGRES_HOST}/\
+        ${CREATEDB_POSTGRES_DATABASE}"
+      DB_API_BASE_URL: "${DB_API_BASE_URL}"
+      CONFIGDB_CONNECTION_INITIAL_TIMEOUT: 10
 
 
   worker-mqtt-ingest:
@@ -841,37 +834,6 @@ services:
       - create-grafana-user
 
 
-  basic-demo-scheduler:
-    image: "registry.hzdr.de/ufz-tsm/tsm-basic-demo-scheduler/basic_demo_scheduler:${SCHEDULER_IMAGE_TAG}"
-    restart: "${RESTART}"
-    # Prevent blocking/slowing-down of other services if many scheduler jobs occur at
-    # the same time. Cap the processing power to equivalent of one CPU of the system.
-    cpus: 0.5
-    mem_limit: 4g
-    healthcheck:
-      test: [ "CMD-SHELL", "curl --fail http://localhost:5000/health || exit 1" ]
-      interval: 10s
-      timeout: 2s
-      retries: 5
-    environment:
-      MQTT_BROKER: mqtt-broker:1883
-      MQTT_USER: "${MQTT_USER}"
-      MQTT_PASSWORD: "${MQTT_PASSWORD}"
-      MQTT_CLIENT_ID: basic-demo-scheduler
-      MQTT_QOS: "${MQTT_QOS}"
-      CONFIGDB_DSN: "postgresql://\
-        ${CONFIGDB_USER}:\
-        ${CONFIGDB_PASSWORD}@\
-        ${CONFIGDB_HOST}:\
-        ${CONFIGDB_PORT}/\
-        ${CONFIGDB_DB}"
-      DB_API_BASE_URL: "${DB_API_BASE_URL}"
-      DATABASE_HOST: "${CREATEDB_POSTGRES_HOST}"
-      DATABASE_NAME: "${CREATEDB_POSTGRES_DATABASE}"
-    command:
-      - --verbose
-
-
   cron-scheduler:
     build:
       context: cron