removed the need for frontendDB, instead we get the credentials from the configDB.

We now also use a privileged minio user to do the sync.
With that change we dont have to store the client credentials in the configDB anymore.

cron/scripts/sftp_sync/sftp_sync.py

@@ -9,35 +9,32 @@ from remote_fs import MinioFS, FtpFS, sync
 from paramiko import WarningPolicy
 
 
-def get_minio_credentials(conn, thing_id) -> tuple[str, str, str, str]:
-    """Returns (uri, access_key, secret_key, bucket_name)"""
+def get_minio_bucket_name(conn, thing_id) -> str:
+    """Returns bucket_name from configdb."""
     with conn.cursor() as cur:
         res = cur.execute(
-            "SELECT r.access_key, r.secret_key, r.bucket "
-            "FROM tsm_thing t JOIN tsm_rawdatastorage r ON t.id = r.thing_id "
-            "WHERE t.thing_id = %s",
+            "select s3.bucket from config_db.s3_store s3 "
+            "join config_db.thing t on s3.id = t.s3_store_id "
+            "where t.uuid = %s",
             [thing_id],
         ).fetchone()
         if res is None or not res[0]:
-            raise RuntimeError(
-                "No object storage credentials found in frontend database"
-            )
-        a, s, b = res
-        return os.environ["MINIO_URL"], a, s, b
+            raise RuntimeError(f"No S3 bucket found for thing {thing_id!r}")
+        return res[0]
 
 
 def get_external_ftp_credentials(conn, thing_id) -> tuple[str, str, str, str]:
     """Returns (uri, username, password, path)"""
     with conn.cursor() as cur:
         res = cur.execute(
-            "SELECT ext_sftp_uri, ext_sftp_username, ext_sftp_password, ext_sftp_path "
-            "FROM tsm_thing WHERE thing_id = %s",
+            'select ftp.uri, ftp."user", ftp.password, ftp.path '
+            "from config_db.ext_sftp ftp join config_db.thing t "
+            "on ftp.id = t.ext_sftp_id "
+            "where t.uuid = %s",
             [thing_id],
         ).fetchone()
         if res is None or res[0] in ["", None] or res[1] in ["", None]:
-            raise RuntimeError(
-                "No external sftp credentials present in frontend database"
-            )
+            raise RuntimeError(f"No Ext-sFTP credentials found for thing {thing_id!r}")
         return res
 
 
@@ -49,11 +46,15 @@ Arguments
   THING_UUID        UUID of the thing.
   KEYFILE           SSH private key file to authenticate at the sftp server.
 
-Additional set the following environment variables:
+Additional set the following nvironment variables:
 
+  MINIO_URL         Minio URL to sync to.
+  MINIO_USER        Minio user with r/w privileges 
+  MINIO_PASSWORD    Password for minio user above.
   MINIO_SECURE      Use minio secure connection; [true, false, 1, 0] 
-  CONFIGDB_DSN      DB which store the credentials for the internal 
-                    and external sftp server. See also DSN format below. 
+  CONFIGDB_DSN      DB which stores the credentials for the external sftp server 
+                    (source of sync) and also the (existing) bucket-name for the 
+                    target S3 storage. See also DSN format below. 
   LOG_LEVEL         Set the verbosity, defaults to INFO.
                     [DEBUG, INFO, WARNING, ERROR, CRITICAL]
 
@@ -82,9 +83,15 @@ if __name__ == "__main__":
 
     with psycopg.connect(dsn) as conn:
         ftp_ext = get_external_ftp_credentials(conn, thing_id)
-        storage = get_minio_credentials(conn, thing_id)
-
-    target = MinioFS.from_credentials(*storage, secure=minio_secure)
+        bucket = get_minio_bucket_name(conn, thing_id)
+
+    target = MinioFS.from_credentials(
+        endpoint=os.environ["MINIO_URL"],
+        access_key=os.environ["MINIO_USER"],
+        secret_key=os.environ["MINIO_PASSWORD"],
+        bucket_name=bucket,
+        secure=minio_secure,
+    )
     source = FtpFS.from_credentials(
         *ftp_ext, keyfile_path=ssh_priv_key, missing_host_key_policy=WarningPolicy()
     )

docker-compose.yml

@@ -871,6 +871,8 @@ services:
       # The following is needed for sftp sync jobs
       LOG_LEVEL: "${LOG_LEVEL}"
       MINIO_URL: "${MINIO_HOST}"
+      MINIO_USER: "${MINIO_ROOT_USER}"
+      MINIO_PASSWORD: "${MINIO_ROOT_PASSWORD}"
       MINIO_SECURE: "${MINIO_SECURE}"
       CONFIGDB_DSN: "postgresql://\
         ${CONFIGDB_USER}:\