From 7c1bd897c17dfbfac41d84fab5ed50dabeb2b38d Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Fri, 21 Feb 2025 10:06:10 +0100
Subject: [PATCH 1/9] thing abstraction for crontab setup
---
docker-compose.yml | 8 +++++++-
src/setup_crontab.py | 8 +++++---
2 files changed, 12 insertions(+), 4 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index 59c9a3de..9a59a911 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -828,7 +828,7 @@ services:
condition: service_healthy
environment:
LOG_LEVEL: "${LOG_LEVEL}"
- TOPIC: thing_creation
+ TOPIC: configdb_update
MQTT_BROKER: mqtt-broker:1883
MQTT_USER: "${MQTT_USER}"
MQTT_PASSWORD: "${MQTT_PASSWORD}"
@@ -837,6 +837,12 @@ services:
MQTT_QOS: "${MQTT_QOS}"
DB_API_BASE_URL: "${DB_API_BASE_URL}"
JOURNALING: "${JOURNALING}"
+ CONFIGDB_DSN: "postgresql://\
+ ${CONFIGDB_USER}:\
+ ${CONFIGDB_PASSWORD}@\
+ ${CONFIGDB_HOST}:\
+ ${CONFIGDB_PORT}/\
+ ${CONFIGDB_DB}"
entrypoint: ["python3", "setup_crontab.py"]
volumes:
- ./cron/crontab.txt:/tmp/cron/crontab.txt
diff --git a/src/setup_crontab.py b/src/setup_crontab.py
index faf92099..102f8fed 100755
--- a/src/setup_crontab.py
+++ b/src/setup_crontab.py
@@ -7,9 +7,10 @@ from random import randint
from crontab import CronItem, CronTab
from timeio.mqtt import AbstractHandler, MQTTMessage
-from timeio.thing import Thing
+from timeio.feta import Thing
from timeio.common import get_envvar, setup_logging
from timeio.journaling import Journal
+from timeio.typehints import MqttPayload
logger = logging.getLogger("crontab-setup")
journal = Journal("Cron")
@@ -27,9 +28,10 @@ class CreateThingInCrontabHandler(AbstractHandler):
mqtt_clean_session=get_envvar("MQTT_CLEAN_SESSION", cast_to=bool),
)
self.tabfile = "/tmp/cron/crontab.txt"
+ self.configdb_dsn = get_envvar("CONFIGDB_DSN")
- def act(self, content: dict, message: MQTTMessage):
- thing = Thing.get_instance(content)
+ def act(self, content: MqttPayload.ConfigDBUpdate, message: MQTTMessage):
+ thing = Thing.from_uuid(content["thing"], dsn=self.configdb_dsn)
with CronTab(tabfile=self.tabfile) as crontab:
for job in crontab:
if self.job_belongs_to_thing(job, thing):
--
GitLab
From e8f76851be7ac6b1eb89a8d13e53af82bdcf791a Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Fri, 21 Feb 2025 11:14:58 +0100
Subject: [PATCH 2/9] add thing abstraction to frost_setup
---
docker-compose.yml | 8 +++++++-
src/setup_frost.py | 22 +++++++++++++++++-----
2 files changed, 24 insertions(+), 6 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index 9a59a911..1d733ef1 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -624,7 +624,7 @@ services:
- tomcat-context:/home/appuser/app/src/frost_context_files:rw
environment:
LOG_LEVEL: "${LOG_LEVEL}"
- TOPIC: thing_creation
+ TOPIC: configdb_update
MQTT_BROKER: mqtt-broker:1883
MQTT_USER: "${MQTT_USER}"
MQTT_PASSWORD: "${MQTT_PASSWORD}"
@@ -635,6 +635,12 @@ services:
DB_API_BASE_URL: "${DB_API_BASE_URL}"
JOURNALING: "${JOURNALING}"
FERNET_ENCRYPTION_SECRET: "${FERNET_ENCRYPTION_SECRET}"
+ CONFIGDB_DSN: "postgresql://\
+ ${CONFIGDB_USER}:\
+ ${CONFIGDB_PASSWORD}@\
+ ${CONFIGDB_HOST}:\
+ ${CONFIGDB_PORT}/\
+ ${CONFIGDB_DB}"
entrypoint: ["python3", "setup_frost.py"]
diff --git a/src/setup_frost.py b/src/setup_frost.py
index 80bd108a..d2747e1b 100755
--- a/src/setup_frost.py
+++ b/src/setup_frost.py
@@ -1,8 +1,10 @@
import logging
from timeio.mqtt import AbstractHandler, MQTTMessage
-from timeio.thing import Thing
+from timeio.feta import Thing
from timeio.common import get_envvar, setup_logging
+from timeio.typehints import MqttPayload
+from timeio import frost
logger = logging.getLogger("frost-setup")
@@ -20,10 +22,20 @@ class CreateFrostInstanceHandler(AbstractHandler):
mqtt_clean_session=get_envvar("MQTT_CLEAN_SESSION", cast_to=bool),
)
self.tomcat_proxy_url = get_envvar("TOMCAT_PROXY_URL")
-
- def act(self, content: dict, message: MQTTMessage):
- thing = Thing.get_instance(content)
- thing.setup_frost(self.tomcat_proxy_url)
+ self.configdb_dsn = get_envvar("CONFIGDB_DSN")
+
+ def act(self, content: MqttPayload.ConfigDBUpdate, message: MQTTMessage):
+ thing = Thing.from_uuid(content["thing"], dsn=self.configdb_dsn)
+ self.setup_frost(thing)
+
+ def setup_frost(self, thing):
+ frost.write_context_file(
+ schema=thing.database.schema,
+ user=f"sta_{thing.database.ro_username.lower()}",
+ password=thing.database.ro_password,
+ db_url=thing.database.db_url,
+ tomcat_proxy_url=self.tomcat_proxy_url,
+ )
if __name__ == "__main__":
--
GitLab
From 6507f189def35e45b75fa19516d66acbad1d6c94 Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Fri, 21 Feb 2025 11:22:27 +0100
Subject: [PATCH 3/9] update crontab_setup
---
src/setup_crontab.py | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/src/setup_crontab.py b/src/setup_crontab.py
index 102f8fed..eedbf982 100755
--- a/src/setup_crontab.py
+++ b/src/setup_crontab.py
@@ -57,28 +57,28 @@ class CreateThingInCrontabHandler(AbstractHandler):
info = ""
comment = cls.mk_comment(thing)
uuid = thing.uuid
- if thing.external_sftp is not None:
- interval = int(thing.external_sftp.sync_interval)
+ if thing.ext_sftp is not None:
+ interval = int(thing.ext_sftp.sync_interval)
schedule = cls.get_schedule(interval)
script = "/scripts/sync_sftp.py"
- keyfile = thing.external_sftp.private_key_path
+ keyfile = thing.ext_sftp.ssh_priv_key
command = f"{script} {uuid} {keyfile} > $STDOUT 2> $STDERR"
- job.enable(enabled=thing.external_sftp.enabled)
+ job.enable(enabled=thing.ext_sftp.sync_enabled)
job.set_comment(comment, pre_comment=True)
job.setall(schedule)
job.set_command(command)
- info = f"sFTP {thing.external_sftp.uri} @ {interval}s"
- if thing.external_api is not None:
- interval = int(thing.external_api.sync_interval)
+ info = f"sFTP {thing.ext_sftp.uri} @ {interval}s"
+ if thing.ext_api is not None:
+ interval = int(thing.ext_api.sync_interval)
schedule = cls.get_schedule(interval)
- script = f"/scripts/sync_{thing.external_api.api_type_name}_api.py"
+ script = f"/scripts/sync_{thing.ext_api.api_type_name}_api.py"
target_uri = thing.database.url
- command = f"""{script} {uuid} "{thing.external_api.settings}" {target_uri} > $STDOUT 2> $STDERR"""
- job.enable(enabled=thing.external_api.enabled)
+ command = f"""{script} {uuid} "{thing.ext_api.settings}" {target_uri} > $STDOUT 2> $STDERR"""
+ job.enable(enabled=thing.ext_api.sync_enabled)
job.set_comment(comment, pre_comment=True)
job.setall(schedule)
job.set_command(command)
- info = f"{thing.external_api.api_type_name}-API @ {interval}s"
+ info = f"{thing.ext_api.api_type_name}-API @ {interval}s"
return info
# alias
--
GitLab
From 7710777f18d546eecb48ef4cf092e8071be9c428 Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Fri, 21 Feb 2025 11:31:13 +0100
Subject: [PATCH 4/9] add abstarction to grafana_setup
---
docker-compose.yml | 8 +++++++-
src/setup_grafana_dashboard.py | 8 +++++---
2 files changed, 12 insertions(+), 4 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index 1d733ef1..c8713180 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -692,7 +692,7 @@ services:
condition: service_healthy
environment:
LOG_LEVEL: "${LOG_LEVEL}"
- TOPIC: thing_creation
+ TOPIC: configdb_update
MQTT_BROKER: mqtt-broker:1883
MQTT_USER: "${MQTT_USER}"
MQTT_PASSWORD: "${MQTT_PASSWORD}"
@@ -706,6 +706,12 @@ services:
DB_API_BASE_URL: "${DB_API_BASE_URL}"
JOURNALING: "${JOURNALING}"
FERNET_ENCRYPTION_SECRET: "${FERNET_ENCRYPTION_SECRET}"
+ CONFIGDB_DSN: "postgresql://\
+ ${CONFIGDB_USER}:\
+ ${CONFIGDB_PASSWORD}@\
+ ${CONFIGDB_HOST}:\
+ ${CONFIGDB_PORT}/\
+ ${CONFIGDB_DB}"
entrypoint: ["python3", "setup_grafana_dashboard.py"]
diff --git a/src/setup_grafana_dashboard.py b/src/setup_grafana_dashboard.py
index b3df9be5..1b447e75 100755
--- a/src/setup_grafana_dashboard.py
+++ b/src/setup_grafana_dashboard.py
@@ -5,9 +5,10 @@ from grafana_client import GrafanaApi
from grafana_client.client import GrafanaException
from timeio.mqtt import AbstractHandler, MQTTMessage
-from timeio.thing import Thing
+from timeio.feta import Thing
from timeio.common import get_envvar, setup_logging
from timeio.crypto import decrypt, get_crypt_key
+from timeio.typehints import MqttPayload
logger = logging.getLogger("grafana-dashboard-setup")
@@ -33,9 +34,10 @@ class CreateThingInGrafanaHandler(AbstractHandler):
)
# needed when defining new datasource
self.sslmode = get_envvar("GRAFANA_DEFAULT_DATASOURCE_SSLMODE")
+ self.configdb_dsn = get_envvar("CONFIGDB_DSN")
- def act(self, content: dict, message: MQTTMessage):
- thing = Thing.get_instance(content)
+ def act(self, content: MqttPayload.ConfigDBUpdate, message: MQTTMessage):
+ thing = Thing.from_uuid(content["thing"], dsn=self.configdb_dsn)
self.create_organization(thing)
# create datasource, folder, dashboard in project org
--
GitLab
From 077471dc44f79622d703773431512ca56b0e4afd Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Fri, 21 Feb 2025 11:37:45 +0100
Subject: [PATCH 5/9] add thing abstraction to minio_setup
---
docker-compose.yml | 8 +++++++-
src/setup_minio.py | 8 +++++---
2 files changed, 12 insertions(+), 4 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index c8713180..5107772b 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -546,7 +546,7 @@ services:
condition: service_healthy
environment:
LOG_LEVEL: "${LOG_LEVEL}"
- TOPIC: thing_creation
+ TOPIC: configdb_update
MQTT_BROKER: mqtt-broker:1883
MQTT_USER: "${MQTT_USER}"
MQTT_PASSWORD: "${MQTT_PASSWORD}"
@@ -560,6 +560,12 @@ services:
DB_API_BASE_URL: "${DB_API_BASE_URL}"
JOURNALING: "${JOURNALING}"
FERNET_ENCRYPTION_SECRET: "${FERNET_ENCRYPTION_SECRET}"
+ CONFIGDB_DSN: "postgresql://\
+ ${CONFIGDB_USER}:\
+ ${CONFIGDB_PASSWORD}@\
+ ${CONFIGDB_HOST}:\
+ ${CONFIGDB_PORT}/\
+ ${CONFIGDB_DB}"
entrypoint: ["python3", "setup_minio.py"]
diff --git a/src/setup_minio.py b/src/setup_minio.py
index f13c338b..6ef3d565 100755
--- a/src/setup_minio.py
+++ b/src/setup_minio.py
@@ -5,9 +5,10 @@ import logging
from minio_cli_wrapper.mc import Mc
from timeio.mqtt import AbstractHandler, MQTTMessage
-from timeio.thing import Thing
+from timeio.feta import Thing
from timeio.common import get_envvar, setup_logging
from timeio.crypto import decrypt, get_crypt_key
+from timeio.typehints import MqttPayload
logger = logging.getLogger("minio-setup")
@@ -30,9 +31,10 @@ class CreateThingInMinioHandler(AbstractHandler):
secret_key=get_envvar("MINIO_SECURE_KEY"),
secure=get_envvar("MINIO_SECURE", default=True, cast_to=bool),
)
+ self.configdb_dsn = get_envvar("CONFIGDB_DSN")
- def act(self, content: dict, message: MQTTMessage):
- thing = Thing.get_instance(content)
+ def act(self, content: MqttPayload.ConfigDBUpdate, message: MQTTMessage):
+ thing = Thing.from_uuid(content["thing"], dsn=self.configdb_dsn)
user = thing.raw_data_storage.username
passw = decrypt(thing.raw_data_storage.password, get_crypt_key())
bucket = thing.raw_data_storage.bucket_name
--
GitLab
From 25f427753ee5b39765f514e04114dd4ff8b3da59 Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Fri, 21 Feb 2025 11:53:12 +0100
Subject: [PATCH 6/9] add thing abstarction to setup_mqtt_user
---
docker-compose.yml | 8 +++++++-
src/setup_mqtt_user.py | 30 ++++++++++++++----------------
2 files changed, 21 insertions(+), 17 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index 5107772b..77628dd1 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -666,7 +666,7 @@ services:
condition: service_healthy
environment:
LOG_LEVEL: "${LOG_LEVEL}"
- TOPIC: thing_creation
+ TOPIC: configdb_update
MQTT_BROKER: mqtt-broker:1883
MQTT_USER: "${MQTT_USER}"
MQTT_PASSWORD: "${MQTT_PASSWORD}"
@@ -680,6 +680,12 @@ services:
${CREATEDB_POSTGRES_DATABASE}"
DB_API_BASE_URL: "${DB_API_BASE_URL}"
JOURNALING: "${JOURNALING}"
+ CONFIGDB_DSN: "postgresql://\
+ ${CONFIGDB_USER}:\
+ ${CONFIGDB_PASSWORD}@\
+ ${CONFIGDB_HOST}:\
+ ${CONFIGDB_PORT}/\
+ ${CONFIGDB_DB}"
entrypoint: ["python3", "setup_mqtt_user.py"]
diff --git a/src/setup_mqtt_user.py b/src/setup_mqtt_user.py
index 113f8088..100bc475 100755
--- a/src/setup_mqtt_user.py
+++ b/src/setup_mqtt_user.py
@@ -6,9 +6,10 @@ import logging
import psycopg2
from timeio.mqtt import AbstractHandler, MQTTMessage
-from timeio.thing import Thing
+from timeio.feta import Thing
from timeio.common import get_envvar, setup_logging
from timeio.journaling import Journal
+from timeio.typehints import MqttPayload
logger = logging.getLogger("mqtt-user-setup")
journal = Journal("System")
@@ -26,33 +27,31 @@ class CreateMqttUserHandler(AbstractHandler):
mqtt_clean_session=get_envvar("MQTT_CLEAN_SESSION", cast_to=bool),
)
self.db = psycopg2.connect(get_envvar("DATABASE_URL"))
+ self.configdb_dsn = get_envvar("CONFIGDB_DSN")
- def act(self, content: dict, message: MQTTMessage):
- thing = Thing.get_instance(content)
- if content["mqtt_authentication_credentials"]:
- user = content["mqtt_authentication_credentials"]["username"]
- pw = content["mqtt_authentication_credentials"]["password_hash"]
+ def act(self, content: MqttPayload.ConfigDBUpdate, message: MQTTMessage):
+ thing = Thing.from_uuid(content["thing"], dsn=self.configdb_dsn)
+ user = thing.mqtt.user
+ pw = thing.mqtt.password_hashed
+
+ logger.info(f"create user. {user=}")
+ created = self.create_user(thing, user, pw)
+ action = "Created" if created else "Updated"
+ journal.info(f"{action} MQTT user {user}", thing.uuid)
- logger.info(f"create user. {user=}")
- created = self.create_user(thing, user, pw)
- action = "Created" if created else "Updated"
- journal.info(f"{action} MQTT user {user}", thing.uuid)
- else:
- logger.warning(f"no 'mqtt_authentication_credentials' present")
def create_user(self, thing, user, pw) -> bool:
"""Returns True for insert and False for update"""
sql = (
"INSERT INTO mqtt_auth.mqtt_user (project_uuid, thing_uuid, username, "
- "password, description,properties, db_schema) "
- "VALUES (%s, %s, %s, %s ,%s ,%s, %s) "
+ "password, description, db_schema) "
+ "VALUES (%s, %s, %s, %s ,%s ,%s) "
"ON CONFLICT (thing_uuid) "
"DO UPDATE SET"
" project_uuid = EXCLUDED.project_uuid,"
" username = EXCLUDED.username,"
" password=EXCLUDED.password,"
" description = EXCLUDED.description,"
- " properties = EXCLUDED.properties,"
" db_schema = EXCLUDED.db_schema "
"RETURNING (xmax = 0)"
)
@@ -66,7 +65,6 @@ class CreateMqttUserHandler(AbstractHandler):
user,
pw,
thing.description,
- json.dumps(thing.properties),
thing.database.username,
),
)
--
GitLab
From f6618a37664b2515349ba3f8273a01d6c939a1c4 Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Fri, 21 Feb 2025 12:01:59 +0100
Subject: [PATCH 7/9] add thing abstraction to setup_database
---
docker-compose.yml | 10 ++++++++--
src/setup_user_database.py | 6 ++++--
2 files changed, 12 insertions(+), 4 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index 77628dd1..5754926a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -497,7 +497,7 @@ services:
start_period: 10s
# =================================================================
- # SETUP worker (topic: thing_creation)
+ # SETUP worker (topic: configdb_update)
# =================================================================
# fills and updates the config-db from setup mqtt-messages
@@ -593,7 +593,7 @@ services:
condition: service_healthy
environment:
LOG_LEVEL: "${LOG_LEVEL}"
- TOPIC: thing_creation
+ TOPIC: configdb_update
MQTT_BROKER: mqtt-broker:1883
MQTT_USER: "${MQTT_USER}"
MQTT_PASSWORD: "${MQTT_PASSWORD}"
@@ -610,6 +610,12 @@ services:
DB_API_BASE_URL: "${DB_API_BASE_URL}"
JOURNALING: "${JOURNALING}"
FERNET_ENCRYPTION_SECRET: "${FERNET_ENCRYPTION_SECRET}"
+ CONFIGDB_DSN: "postgresql://\
+ ${CONFIGDB_USER}:\
+ ${CONFIGDB_PASSWORD}@\
+ ${CONFIGDB_HOST}:\
+ ${CONFIGDB_PORT}/\
+ ${CONFIGDB_DB}"
entrypoint: ["python3", "setup_user_database.py"]
diff --git a/src/setup_user_database.py b/src/setup_user_database.py
index 744c11d2..c025f366 100755
--- a/src/setup_user_database.py
+++ b/src/setup_user_database.py
@@ -9,10 +9,11 @@ import psycopg
from timeio.mqtt import AbstractHandler, MQTTMessage
from timeio.databases import ReentrantConnection
-from timeio.thing import Thing
+from timeio.feta import Thing
from timeio.common import get_envvar, setup_logging
from timeio.journaling import Journal
from timeio.crypto import decrypt, get_crypt_key
+from timeio.typehints import MqttPayload
logger = logging.getLogger("db-setup")
journal = Journal("System")
@@ -31,10 +32,11 @@ class CreateThingInPostgresHandler(AbstractHandler):
)
self.db_conn = ReentrantConnection(get_envvar("DATABASE_URL"))
self.db = self.db_conn.connect()
+ self.configdb_dsn = get_envvar("CONFIGDB_DSN")
def act(self, content: dict, message: MQTTMessage):
self.db = self.db_conn.reconnect()
- thing = Thing.get_instance(content)
+ thing = Thing.from_uuid(content["thing"], dsn=self.configdb_dsn)
logger.info(f"start processing. {thing.name=}, {thing.uuid=}")
STA_PREFIX = "sta_"
GRF_PREFIX = "grf_"
--
GitLab
From 4fc7d872b30b7a29db3595a36209b67a7b8e26c0 Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Fri, 21 Feb 2025 12:04:23 +0100
Subject: [PATCH 8/9] black
---
src/setup_mqtt_user.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/src/setup_mqtt_user.py b/src/setup_mqtt_user.py
index 100bc475..a05f7ebb 100755
--- a/src/setup_mqtt_user.py
+++ b/src/setup_mqtt_user.py
@@ -39,7 +39,6 @@ class CreateMqttUserHandler(AbstractHandler):
action = "Created" if created else "Updated"
journal.info(f"{action} MQTT user {user}", thing.uuid)
-
def create_user(self, thing, user, pw) -> bool:
"""Returns True for insert and False for update"""
sql = (
--
GitLab
From 92ae449397ee942e19983478c60c0e8dc50fd7c2 Mon Sep 17 00:00:00 2001
From: Florian Gransee <florian.gransee@ufz.de>
Date: Mon, 24 Feb 2025 15:18:08 +0100
Subject: [PATCH 9/9] fix db_url and simplify
---
src/setup_frost.py | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/src/setup_frost.py b/src/setup_frost.py
index d2747e1b..300cc894 100755
--- a/src/setup_frost.py
+++ b/src/setup_frost.py
@@ -26,14 +26,11 @@ class CreateFrostInstanceHandler(AbstractHandler):
def act(self, content: MqttPayload.ConfigDBUpdate, message: MQTTMessage):
thing = Thing.from_uuid(content["thing"], dsn=self.configdb_dsn)
- self.setup_frost(thing)
-
- def setup_frost(self, thing):
frost.write_context_file(
schema=thing.database.schema,
user=f"sta_{thing.database.ro_username.lower()}",
password=thing.database.ro_password,
- db_url=thing.database.db_url,
+ db_url=thing.database.ro_url,
tomcat_proxy_url=self.tomcat_proxy_url,
)
--
GitLab