From 241f2843f230df3e7d8fe3b2d62fb77811632d41 Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Wed, 22 Jan 2025 15:05:51 +0100
Subject: [PATCH 01/12] add implementaion for measure endpoint
---
cron/scripts/ext_api_sync/uba_api_sync.py | 158 ++++++++++++++++++++++
1 file changed, 158 insertions(+)
create mode 100644 cron/scripts/ext_api_sync/uba_api_sync.py
diff --git a/cron/scripts/ext_api_sync/uba_api_sync.py b/cron/scripts/ext_api_sync/uba_api_sync.py
new file mode 100644
index 00000000..c7bfd153
--- /dev/null
+++ b/cron/scripts/ext_api_sync/uba_api_sync.py
@@ -0,0 +1,158 @@
+#! /usr/bin/env python3
+
+import requests
+import os
+import logging
+import json
+import click
+
+
+api_base_url = os.environ.get("DB_API_BASE_URL")
+
+
+def get_components_and_scopes():
+ """Get components (i.e measured quantites) and scopes (aggregation infos) for later mapping"""
+ response_components = requests.get(
+ "https://www.umweltbundesamt.de/api/air_data/v3/components/json"
+ )
+ response_scopes = requests.get(
+ "https://www.umweltbundesamt.de/api/air_data/v3/scopes/json"
+ )
+ if response_components.status_code == 200 and response_scopes.status_code == 200:
+ components = {
+ int(v[0]): v[1]
+ for k, v in response_components.json().items()
+ if k not in ["count", "indices"]
+ }
+ scopes = {
+ int(v[0]): v[1]
+ for k, v in response_scopes.json().items()
+ if k not in ["count", "indices"]
+ }
+ return components, scopes
+
+
+def get_station_info(station_id: str) -> list:
+ """Get all available components and scope combinations of a given station"""
+ station_info = list()
+ response = requests.get(
+ "https://www.umweltbundesamt.de/api/air_data/v3/measures/limits"
+ )
+ if response.status_code == 200:
+ response_json = response.json()["data"]
+ for k, v in response_json.items():
+ if v[2] == station_id:
+ station_info.append({"scope": int(v[0]), "component": int(v[1])})
+ return station_info
+
+
+def request_uba_api(
+ station_id: str,
+ component_id: int,
+ scope_id: int,
+ date_from: str,
+ date_to: str,
+ time_from: int,
+ time_to: int,
+) -> dict:
+ """Request uba api measure endpoint for a given component and scope and a given time range"""
+ params = {
+ "date_from": date_from,
+ "date_to": date_to,
+ "time_from": time_from,
+ "time_to": time_to,
+ "station": station_id,
+ "component": component_id,
+ "scope": scope_id,
+ }
+ response = requests.get(
+ url="https://www.umweltbundesamt.de/api/air_data/v3/measures/json",
+ params=params,
+ )
+ if response.status_code == 200:
+ response_json = response.json()["data"][station_id]
+ if response_json:
+ return response_json
+
+
+def combine_uba_responses(
+ station_id: str,
+ date_from: str,
+ date_to: str,
+ time_from: int,
+ time_to: int,
+) -> list:
+ """Combine uba respones for all component/scope combinations into one object"""
+ uba_data = list()
+ station_info = get_station_info(station_id)
+ components, scopes = get_components_and_scopes()
+ for entry in station_info:
+ response = request_uba_api(
+ station_id,
+ entry["component"],
+ entry["scope"],
+ date_from,
+ date_to,
+ time_from,
+ time_to,
+ )
+ for k, v in response.items():
+ uba_data.append(
+ {
+ "timestamp": v[3],
+ "value": v[2],
+ "parameter": f"{components[entry['component']]} {scopes[entry['scope']]}",
+ }
+ )
+ return uba_data
+
+
+def parse_uba_data(uba_data: list, station_id: str) -> dict:
+ """Creates POST body from combined uba data"""
+ bodies = []
+ source = {"uba_station_id": station_id}
+ for entry in uba_data:
+ if entry["timestamp"][11:13] == "24":
+ entry["timestamp"] = (
+ entry["timestamp"][:11] + "00" + entry["timestamp"][13:]
+ )
+ if entry["value"]:
+ body = {
+ "result_time": entry["timestamp"],
+ "result_type": 0,
+ "result_number": entry["value"],
+ "datastream_pos": entry["parameter"],
+ "parameters": json.dumps(
+ {"origin": "uba_data", "column_header": source}
+ ),
+ }
+ bodies.append(body)
+ return {"observations": bodies}
+
+
+@click.command()
+@click.argument("thing_uuid")
+@click.argument("parameters")
+@click.argument("target_uri")
+def main(thing_uuid, parameters, target_uri):
+ logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO").upper())
+
+ params = json.loads(parameters.replace("'", '"'))
+ uba_data = combine_uba_responses(params["station_id"])
+ parsed_observations = parse_uba_data(uba_data, params["station_id"])
+ req = requests.post(
+ f"{api_base_url}/observations/upsert/{thing_uuid}",
+ json=parsed_observations,
+ headers={"Content-type": "application/json"},
+ )
+ if req.status_code == 201:
+ logging.info(
+ f"Successfully inserted {len(parsed_observations['observations'])} "
+ f"observations for thing {thing_uuid} from UBA API into TimeIO DB"
+ )
+ else:
+ logging.error(f"{req.text}")
+
+
+if __name__ == "__main__":
+ main()
--
GitLab
From 0a19ca4a898284f6b53285a083aa20a8f88be44b Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Thu, 23 Jan 2025 13:53:12 +0100
Subject: [PATCH 02/12] add airquality data
---
cron/scripts/ext_api_sync/uba_api_sync.py | 96 +++++++++++++++++++----
1 file changed, 82 insertions(+), 14 deletions(-)
diff --git a/cron/scripts/ext_api_sync/uba_api_sync.py b/cron/scripts/ext_api_sync/uba_api_sync.py
index c7bfd153..8508df9e 100644
--- a/cron/scripts/ext_api_sync/uba_api_sync.py
+++ b/cron/scripts/ext_api_sync/uba_api_sync.py
@@ -46,7 +46,7 @@ def get_station_info(station_id: str) -> list:
return station_info
-def request_uba_api(
+def request_measure_endpoint(
station_id: str,
component_id: int,
scope_id: int,
@@ -75,19 +75,20 @@ def request_uba_api(
return response_json
-def combine_uba_responses(
+def combine_measure_responses(
station_id: str,
date_from: str,
date_to: str,
time_from: int,
time_to: int,
+ components: dict,
+ scopes: dict,
) -> list:
"""Combine uba respones for all component/scope combinations into one object"""
- uba_data = list()
+ measure_data = list()
station_info = get_station_info(station_id)
- components, scopes = get_components_and_scopes()
for entry in station_info:
- response = request_uba_api(
+ response = request_measure_endpoint(
station_id,
entry["component"],
entry["scope"],
@@ -97,21 +98,21 @@ def combine_uba_responses(
time_to,
)
for k, v in response.items():
- uba_data.append(
+ measure_data.append(
{
"timestamp": v[3],
"value": v[2],
- "parameter": f"{components[entry['component']]} {scopes[entry['scope']]}",
+ "measure": f"{components[entry['component']]} {scopes[entry['scope']]}",
}
)
- return uba_data
+ return measure_data
-def parse_uba_data(uba_data: list, station_id: str) -> dict:
+def parse_measure_data(measure_data: list, station_id: str) -> list:
"""Creates POST body from combined uba data"""
bodies = []
source = {"uba_station_id": station_id}
- for entry in uba_data:
+ for entry in measure_data:
if entry["timestamp"][11:13] == "24":
entry["timestamp"] = (
entry["timestamp"][:11] + "00" + entry["timestamp"][13:]
@@ -121,13 +122,72 @@ def parse_uba_data(uba_data: list, station_id: str) -> dict:
"result_time": entry["timestamp"],
"result_type": 0,
"result_number": entry["value"],
- "datastream_pos": entry["parameter"],
+ "datastream_pos": entry["measure"],
"parameters": json.dumps(
{"origin": "uba_data", "column_header": source}
),
}
bodies.append(body)
- return {"observations": bodies}
+ return bodies
+
+
+def get_airquality_data(
+ station_id: str,
+ date_from: str,
+ date_to: str,
+ time_from: int,
+ time_to: int,
+ components: dict,
+) -> list:
+ params = {
+ "date_from": date_from,
+ "date_to": date_to,
+ "time_from": time_from,
+ "time_to": time_to,
+ "station": station_id,
+ }
+ response = requests.get(
+ "https://www.umweltbundesamt.de/api/air_data/v3/airquality/json", params=params
+ )
+ response_data = response.json()["data"][station_id]
+ aqi_data = list()
+ for k, v in response_data.items():
+ pollutant_info = list()
+ for i in range(3, len(v)):
+ entry = {"component": components[v[i][0]], "airquality_index": v[i][2]}
+ pollutant_info.append(entry)
+ aqi_data.append(
+ {
+ "timestamp": v[0],
+ "airquality_index": v[1],
+ "data_complete": v[2],
+ "pollutant_info": pollutant_info,
+ }
+ )
+ return aqi_data
+
+
+def parse_aqi_data(aqi_data: list, station_id: str) -> list:
+ bodies = []
+ for entry in aqi_data:
+ source = {
+ "uba_station_id": station_id,
+ "endpoint": "/airquality",
+ "pollutant_info": entry["pollutant_info"],
+ }
+ if entry["timestamp"][11:13] == "24":
+ entry["timestamp"] = (
+ entry["timestamp"][:11] + "00" + entry["timestamp"][13:]
+ )
+ body = {
+ "result_time": entry["timestamp"],
+ "result_type": 0,
+ "result_number": entry["airquality_index"],
+ "datastream_pos": "AQI",
+ "parameters": json.dumps({"origin": "uba_data", "column_header": source}),
+ }
+ bodies.append(body)
+ return bodies
@click.command()
@@ -138,8 +198,16 @@ def main(thing_uuid, parameters, target_uri):
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO").upper())
params = json.loads(parameters.replace("'", '"'))
- uba_data = combine_uba_responses(params["station_id"])
- parsed_observations = parse_uba_data(uba_data, params["station_id"])
+ components, scopes = get_components_and_scopes()
+ measure_data = combine_measure_responses(
+ params["station_id"], date_from, date_to, time_from, time_to, components, scopes
+ )
+ aqi_data = get_airquality_data(
+ params["station_id"], date_from, date_to, time_from, time_to, components
+ )
+ parsed_measure_data = parse_measure_data(measure_data, params["station_id"])
+ parsed_aqi_data = parse_aqi_data(aqi_data, params["station_id"])
+ parsed_observations = {"observations": parsed_measure_data + parsed_aqi_data}
req = requests.post(
f"{api_base_url}/observations/upsert/{thing_uuid}",
json=parsed_observations,
--
GitLab
From 5562998861e6f875fdd10d105325d9b7ca2e6c3e Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Mon, 27 Jan 2025 11:39:01 +0100
Subject: [PATCH 03/12] add datetime parameters and mapping to uba datetime
logic
---
cron/scripts/ext_api_sync/uba_api_sync.py | 44 +++++++++++++++++++----
1 file changed, 38 insertions(+), 6 deletions(-)
diff --git a/cron/scripts/ext_api_sync/uba_api_sync.py b/cron/scripts/ext_api_sync/uba_api_sync.py
index 8508df9e..eb94a33b 100644
--- a/cron/scripts/ext_api_sync/uba_api_sync.py
+++ b/cron/scripts/ext_api_sync/uba_api_sync.py
@@ -6,10 +6,41 @@ import logging
import json
import click
+from datetime import datetime, timedelta
+
api_base_url = os.environ.get("DB_API_BASE_URL")
+def adjust_datetime(datetime_str: str) -> str:
+ """UBA API returns datetime format with hours from 1 to 24 so it has to be parsed for timeIO DB API"""
+ date = datetime.strptime(datetime_str[0:10], "%Y-%m-%d")
+ date_adjusted = date + timedelta(days=1)
+
+ return date_adjusted.strftime("%Y-%m-%d %H:%M:%S")
+
+
+def get_timerange_parameters():
+ """UBA API expects time_from/time_to in the range of 1 to 24"""
+ datetime_now = datetime.now()
+ datetime_from = datetime_now - timedelta(hours=1)
+ if datetime_now.hour == 0:
+ time_to = 24
+ date_to = (datetime_now - timedelta(days=1)).strftime("%Y-%m-%d")
+ else:
+ time_to = datetime_now.hour
+ date_to = datetime_now.strftime("%Y-%m-%d")
+
+ if datetime_from.hour == 0:
+ time_from = 24
+ date_from = (datetime_from - timedelta(days=1)).strftime("%Y-%m-%d")
+ else:
+ time_from = datetime_from.hour
+ date_from = datetime_from.strftime("%Y-%m-%d")
+
+ return date_from, time_from, date_to, time_to
+
+
def get_components_and_scopes():
"""Get components (i.e measured quantites) and scopes (aggregation infos) for later mapping"""
response_components = requests.get(
@@ -70,9 +101,11 @@ def request_measure_endpoint(
params=params,
)
if response.status_code == 200:
- response_json = response.json()["data"][station_id]
- if response_json:
- return response_json
+ response_json = response.json()
+ if response_json["data"]:
+ return response_json["data"][station_id]
+ else:
+ return response_json["data"]
def combine_measure_responses(
@@ -114,9 +147,7 @@ def parse_measure_data(measure_data: list, station_id: str) -> list:
source = {"uba_station_id": station_id}
for entry in measure_data:
if entry["timestamp"][11:13] == "24":
- entry["timestamp"] = (
- entry["timestamp"][:11] + "00" + entry["timestamp"][13:]
- )
+ entry["timestamp"] = adjust_datetime(entry["timestamp"])
if entry["value"]:
body = {
"result_time": entry["timestamp"],
@@ -198,6 +229,7 @@ def main(thing_uuid, parameters, target_uri):
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO").upper())
params = json.loads(parameters.replace("'", '"'))
+ date_from, time_from, date_to, time_to = get_timerange_parameters()
components, scopes = get_components_and_scopes()
measure_data = combine_measure_responses(
params["station_id"], date_from, date_to, time_from, time_to, components, scopes
--
GitLab
From b01d46933af090eb52df1c39276fd1be0ea66739 Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Mon, 27 Jan 2025 11:49:27 +0100
Subject: [PATCH 04/12] extend source info
---
cron/scripts/ext_api_sync/uba_api_sync.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/cron/scripts/ext_api_sync/uba_api_sync.py b/cron/scripts/ext_api_sync/uba_api_sync.py
index eb94a33b..410afd96 100644
--- a/cron/scripts/ext_api_sync/uba_api_sync.py
+++ b/cron/scripts/ext_api_sync/uba_api_sync.py
@@ -144,7 +144,10 @@ def combine_measure_responses(
def parse_measure_data(measure_data: list, station_id: str) -> list:
"""Creates POST body from combined uba data"""
bodies = []
- source = {"uba_station_id": station_id}
+ source = {
+ "uba_station_id": station_id,
+ "endpoint": "/measures",
+ }
for entry in measure_data:
if entry["timestamp"][11:13] == "24":
entry["timestamp"] = adjust_datetime(entry["timestamp"])
--
GitLab
From 2d9cbf7682764cfe020feaa16b5f67763090d60b Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Mon, 27 Jan 2025 14:36:42 +0100
Subject: [PATCH 05/12] add api doc links for uba and dwd
---
cron/scripts/ext_api_sync/dwd_api_sync.py | 2 ++
cron/scripts/ext_api_sync/uba_api_sync.py | 2 ++
2 files changed, 4 insertions(+)
diff --git a/cron/scripts/ext_api_sync/dwd_api_sync.py b/cron/scripts/ext_api_sync/dwd_api_sync.py
index 900084af..427d579b 100755
--- a/cron/scripts/ext_api_sync/dwd_api_sync.py
+++ b/cron/scripts/ext_api_sync/dwd_api_sync.py
@@ -1,5 +1,7 @@
#! /usr/bin/env python3
+# DWD Brightsky API docs: https://brightsky.dev/docs/#/operations/getWeather
+
import requests
import os
import logging
diff --git a/cron/scripts/ext_api_sync/uba_api_sync.py b/cron/scripts/ext_api_sync/uba_api_sync.py
index 410afd96..aae108ac 100644
--- a/cron/scripts/ext_api_sync/uba_api_sync.py
+++ b/cron/scripts/ext_api_sync/uba_api_sync.py
@@ -1,5 +1,7 @@
#! /usr/bin/env python3
+# UBA API docs: https://www.umweltbundesamt.de/daten/luft/luftdaten/doc
+
import requests
import os
import logging
--
GitLab
From 3ae3b04402496448e66cc62c1547ca84bf679814 Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Mon, 27 Jan 2025 14:43:10 +0100
Subject: [PATCH 06/12] adjust datetime for airquality parsing
---
cron/scripts/ext_api_sync/uba_api_sync.py | 25 ++++++++++++-----------
1 file changed, 13 insertions(+), 12 deletions(-)
diff --git a/cron/scripts/ext_api_sync/uba_api_sync.py b/cron/scripts/ext_api_sync/uba_api_sync.py
index aae108ac..db7f8328 100644
--- a/cron/scripts/ext_api_sync/uba_api_sync.py
+++ b/cron/scripts/ext_api_sync/uba_api_sync.py
@@ -144,7 +144,7 @@ def combine_measure_responses(
def parse_measure_data(measure_data: list, station_id: str) -> list:
- """Creates POST body from combined uba data"""
+ """Creates POST body from combined uba measures data"""
bodies = []
source = {
"uba_station_id": station_id,
@@ -175,6 +175,7 @@ def get_airquality_data(
time_to: int,
components: dict,
) -> list:
+ """Request uba api airquality endpoint for a given station_id and time range"""
params = {
"date_from": date_from,
"date_to": date_to,
@@ -204,6 +205,7 @@ def get_airquality_data(
def parse_aqi_data(aqi_data: list, station_id: str) -> list:
+ """Creates POST body from uba air quality data"""
bodies = []
for entry in aqi_data:
source = {
@@ -212,17 +214,16 @@ def parse_aqi_data(aqi_data: list, station_id: str) -> list:
"pollutant_info": entry["pollutant_info"],
}
if entry["timestamp"][11:13] == "24":
- entry["timestamp"] = (
- entry["timestamp"][:11] + "00" + entry["timestamp"][13:]
- )
- body = {
- "result_time": entry["timestamp"],
- "result_type": 0,
- "result_number": entry["airquality_index"],
- "datastream_pos": "AQI",
- "parameters": json.dumps({"origin": "uba_data", "column_header": source}),
- }
- bodies.append(body)
+ entry["timestamp"] = adjust_datetime(entry["timestamp"])
+ if entry["airquality_index"]:
+ body = {
+ "result_time": entry["timestamp"],
+ "result_type": 0,
+ "result_number": entry["airquality_index"],
+ "datastream_pos": "AQI",
+ "parameters": json.dumps({"origin": "uba_data", "column_header": source}),
+ }
+ bodies.append(body)
return bodies
--
GitLab
From e8e2c57ee5b296a2f738b0c5d1e02a8f726574ae Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Mon, 27 Jan 2025 14:43:54 +0100
Subject: [PATCH 07/12] black
---
cron/scripts/ext_api_sync/uba_api_sync.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/cron/scripts/ext_api_sync/uba_api_sync.py b/cron/scripts/ext_api_sync/uba_api_sync.py
index db7f8328..fcbe3a45 100644
--- a/cron/scripts/ext_api_sync/uba_api_sync.py
+++ b/cron/scripts/ext_api_sync/uba_api_sync.py
@@ -221,7 +221,9 @@ def parse_aqi_data(aqi_data: list, station_id: str) -> list:
"result_type": 0,
"result_number": entry["airquality_index"],
"datastream_pos": "AQI",
- "parameters": json.dumps({"origin": "uba_data", "column_header": source}),
+ "parameters": json.dumps(
+ {"origin": "uba_data", "column_header": source}
+ ),
}
bodies.append(body)
return bodies
--
GitLab
From 974e37515925c7121be64bf9705c968bbbcb868e Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Tue, 28 Jan 2025 14:28:31 +0100
Subject: [PATCH 08/12] add send_mqtt_info
---
cron/scripts/ext_api_sync/uba_api_sync.py | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
diff --git a/cron/scripts/ext_api_sync/uba_api_sync.py b/cron/scripts/ext_api_sync/uba_api_sync.py
index fcbe3a45..55b40770 100644
--- a/cron/scripts/ext_api_sync/uba_api_sync.py
+++ b/cron/scripts/ext_api_sync/uba_api_sync.py
@@ -7,6 +7,7 @@ import os
import logging
import json
import click
+import mqtt
from datetime import datetime, timedelta
@@ -248,19 +249,21 @@ def main(thing_uuid, parameters, target_uri):
parsed_measure_data = parse_measure_data(measure_data, params["station_id"])
parsed_aqi_data = parse_aqi_data(aqi_data, params["station_id"])
parsed_observations = {"observations": parsed_measure_data + parsed_aqi_data}
- req = requests.post(
+ resp = requests.post(
f"{api_base_url}/observations/upsert/{thing_uuid}",
json=parsed_observations,
headers={"Content-type": "application/json"},
)
- if req.status_code == 201:
- logging.info(
- f"Successfully inserted {len(parsed_observations['observations'])} "
- f"observations for thing {thing_uuid} from UBA API into TimeIO DB"
- )
- else:
- logging.error(f"{req.text}")
+ if resp.status_code != 201:
+ logging.error(f"{resp.text}")
+ resp.raise_for_status()
+ # exit
+ logging.info(
+ f"Successfully inserted {len(parsed_observations['observations'])} "
+ f"observations for thing {thing_uuid} from UBA API into TimeIO DB"
+ )
+ mqtt.send_mqtt_info("data_parsed", json.dumps({"thing_uuid": thing_uuid}))
if __name__ == "__main__":
main()
--
GitLab
From 74fac208061a542b550c323331a28758cf850617 Mon Sep 17 00:00:00 2001
From: Florian Gransee <florian.gransee@ufz.de>
Date: Tue, 28 Jan 2025 14:36:53 +0100
Subject: [PATCH 09/12] change list definition
---
cron/scripts/ext_api_sync/uba_api_sync.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/cron/scripts/ext_api_sync/uba_api_sync.py b/cron/scripts/ext_api_sync/uba_api_sync.py
index 55b40770..9f89885b 100644
--- a/cron/scripts/ext_api_sync/uba_api_sync.py
+++ b/cron/scripts/ext_api_sync/uba_api_sync.py
@@ -121,7 +121,7 @@ def combine_measure_responses(
scopes: dict,
) -> list:
"""Combine uba respones for all component/scope combinations into one object"""
- measure_data = list()
+ measure_data = []
station_info = get_station_info(station_id)
for entry in station_info:
response = request_measure_endpoint(
--
GitLab
From fd4ca2a629112e3953557310bd3af8dbab2164ef Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Tue, 28 Jan 2025 14:35:52 +0100
Subject: [PATCH 10/12] adjust docstrings
---
cron/scripts/ext_api_sync/uba_api_sync.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/cron/scripts/ext_api_sync/uba_api_sync.py b/cron/scripts/ext_api_sync/uba_api_sync.py
index 9f89885b..293adbd2 100644
--- a/cron/scripts/ext_api_sync/uba_api_sync.py
+++ b/cron/scripts/ext_api_sync/uba_api_sync.py
@@ -16,7 +16,9 @@ api_base_url = os.environ.get("DB_API_BASE_URL")
def adjust_datetime(datetime_str: str) -> str:
- """UBA API returns datetime format with hours from 1 to 24 so it has to be parsed for timeIO DB API"""
+ """UBA API returns datetime format with hours from 1 to 24 so it
+ has to be parsed for timeIO DB API
+ """
date = datetime.strptime(datetime_str[0:10], "%Y-%m-%d")
date_adjusted = date + timedelta(days=1)
@@ -121,7 +123,7 @@ def combine_measure_responses(
scopes: dict,
) -> list:
"""Combine uba respones for all component/scope combinations into one object"""
- measure_data = []
+ measure_data = list()
station_info = get_station_info(station_id)
for entry in station_info:
response = request_measure_endpoint(
--
GitLab
From 8a3bd539c83b0749c02fd03c0e110c2e72fc1030 Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Tue, 28 Jan 2025 14:40:31 +0100
Subject: [PATCH 11/12] adjust docstrings
---
cron/scripts/ext_api_sync/uba_api_sync.py | 20 +++++++++++++++-----
1 file changed, 15 insertions(+), 5 deletions(-)
diff --git a/cron/scripts/ext_api_sync/uba_api_sync.py b/cron/scripts/ext_api_sync/uba_api_sync.py
index 293adbd2..46443fc7 100644
--- a/cron/scripts/ext_api_sync/uba_api_sync.py
+++ b/cron/scripts/ext_api_sync/uba_api_sync.py
@@ -47,7 +47,9 @@ def get_timerange_parameters():
def get_components_and_scopes():
- """Get components (i.e measured quantites) and scopes (aggregation infos) for later mapping"""
+ """Get components (i.e measured quantites) and scopes
+ (aggregation infos) for later mapping
+ """
response_components = requests.get(
"https://www.umweltbundesamt.de/api/air_data/v3/components/json"
)
@@ -69,7 +71,9 @@ def get_components_and_scopes():
def get_station_info(station_id: str) -> list:
- """Get all available components and scope combinations of a given station"""
+ """Get all available components and scope combinations of a given
+ station
+ """
station_info = list()
response = requests.get(
"https://www.umweltbundesamt.de/api/air_data/v3/measures/limits"
@@ -91,7 +95,9 @@ def request_measure_endpoint(
time_from: int,
time_to: int,
) -> dict:
- """Request uba api measure endpoint for a given component and scope and a given time range"""
+ """Request uba api measure endpoint for a given component and scope
+ and a given time range
+ """
params = {
"date_from": date_from,
"date_to": date_to,
@@ -122,7 +128,9 @@ def combine_measure_responses(
components: dict,
scopes: dict,
) -> list:
- """Combine uba respones for all component/scope combinations into one object"""
+ """Combine uba respones for all component/scope combinations into
+ one object
+ """
measure_data = list()
station_info = get_station_info(station_id)
for entry in station_info:
@@ -178,7 +186,9 @@ def get_airquality_data(
time_to: int,
components: dict,
) -> list:
- """Request uba api airquality endpoint for a given station_id and time range"""
+ """Request uba api airquality endpoint for a given station_id and
+ time range
+ """
params = {
"date_from": date_from,
"date_to": date_to,
--
GitLab
From a770fa2e7a2334a5b4f1c92ac76edf91c8e27d0f Mon Sep 17 00:00:00 2001
From: granseef <florian.gransee@ufz.de>
Date: Tue, 28 Jan 2025 14:52:38 +0100
Subject: [PATCH 12/12] make use of raise_for_status
---
cron/scripts/ext_api_sync/uba_api_sync.py | 50 ++++++++++++-----------
1 file changed, 26 insertions(+), 24 deletions(-)
diff --git a/cron/scripts/ext_api_sync/uba_api_sync.py b/cron/scripts/ext_api_sync/uba_api_sync.py
index 46443fc7..0934cb80 100644
--- a/cron/scripts/ext_api_sync/uba_api_sync.py
+++ b/cron/scripts/ext_api_sync/uba_api_sync.py
@@ -53,21 +53,22 @@ def get_components_and_scopes():
response_components = requests.get(
"https://www.umweltbundesamt.de/api/air_data/v3/components/json"
)
+ response_components.raise_for_status()
response_scopes = requests.get(
"https://www.umweltbundesamt.de/api/air_data/v3/scopes/json"
)
- if response_components.status_code == 200 and response_scopes.status_code == 200:
- components = {
- int(v[0]): v[1]
- for k, v in response_components.json().items()
- if k not in ["count", "indices"]
- }
- scopes = {
- int(v[0]): v[1]
- for k, v in response_scopes.json().items()
- if k not in ["count", "indices"]
- }
- return components, scopes
+ response_scopes.raise_for_status()
+ components = {
+ int(v[0]): v[1]
+ for k, v in response_components.json().items()
+ if k not in ["count", "indices"]
+ }
+ scopes = {
+ int(v[0]): v[1]
+ for k, v in response_scopes.json().items()
+ if k not in ["count", "indices"]
+ }
+ return components, scopes
def get_station_info(station_id: str) -> list:
@@ -78,12 +79,12 @@ def get_station_info(station_id: str) -> list:
response = requests.get(
"https://www.umweltbundesamt.de/api/air_data/v3/measures/limits"
)
- if response.status_code == 200:
- response_json = response.json()["data"]
- for k, v in response_json.items():
- if v[2] == station_id:
- station_info.append({"scope": int(v[0]), "component": int(v[1])})
- return station_info
+ response.raise_for_status()
+ response_json = response.json()["data"]
+ for k, v in response_json.items():
+ if v[2] == station_id:
+ station_info.append({"scope": int(v[0]), "component": int(v[1])})
+ return station_info
def request_measure_endpoint(
@@ -111,12 +112,12 @@ def request_measure_endpoint(
url="https://www.umweltbundesamt.de/api/air_data/v3/measures/json",
params=params,
)
- if response.status_code == 200:
- response_json = response.json()
- if response_json["data"]:
- return response_json["data"][station_id]
- else:
- return response_json["data"]
+ response.raise_for_status()
+ response_json = response.json()
+ if response_json["data"]:
+ return response_json["data"][station_id]
+ else:
+ return response_json["data"]
def combine_measure_responses(
@@ -199,6 +200,7 @@ def get_airquality_data(
response = requests.get(
"https://www.umweltbundesamt.de/api/air_data/v3/airquality/json", params=params
)
+ response.raise_for_status()
response_data = response.json()["data"][station_id]
aqi_data = list()
for k, v in response_data.items():
--
GitLab