From ea45a0265f8b59fd5cc8a5c772024d8412c45b79 Mon Sep 17 00:00:00 2001
From: Nicola Stoira <nicola.stoira@accenture.com>
Date: Wed, 26 Jun 2024 13:02:44 +0200
Subject: [PATCH 1/2] Update /Get_Airflow_logs
---
api/app/main.py | 5 +++--
api/tests/test_main.py | 16 ++++++++++++++--
2 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/api/app/main.py b/api/app/main.py
index b4f77674..fe2a5502 100644
--- a/api/app/main.py
+++ b/api/app/main.py
@@ -715,8 +715,9 @@ async def get_init_logs(project: str = Query(..., description="Name of the proje
@app.get("/Get_Airflow_logs", summary="Get latest logs from Airflow", description="In the event of an abrupt failure of an Airflow process, this endpoint returns the logs of the latest executed dags and can give some additional information about the failure ", tags=["Monitoring"],
response_class=MultipartResponse, responses=get_responses(Endpoint.GET_AIRFLOW_LOGS))
-async def get_airflow_logs(project: str = Query(..., description="Name of the project"), DAG: str = Query("all", enum = ['run_shacler', 'run_sparqler', 'run_database_extraction', 'run_batch_ingestion', 'run_whole_pipeline', 'run_pre_check_and_de_identification', 'run_integration', 'run_validation',
- 'run_statistics', 'run_connector_backup', 'run_minio_download', 'run_real_time_count'], description="Airflow DAG"),
+async def get_airflow_logs(project: str = Query(..., description="Name of the project"), DAG: str = Query("all", enum = ['all', 'run_batch_ingestion', 'run_connector_backup', 'run_database_extraction', 'run_grafana_count', 'run_integration', 'run_minio_download',
+ 'run_notify_einstein', 'run_pre_check_and_de_identification', 'run_real_time_count', 'run_shacler', 'run_sparqler', 'run_statistics',
+ 'run_validation', 'run_whole_pipeline'], description="Airflow DAG"),
current_user: User = Depends(authentication.get_standard_level_permissions)):
"""
Get Airflow logs extract
diff --git a/api/tests/test_main.py b/api/tests/test_main.py
index b346657b..ac517d98 100644
--- a/api/tests/test_main.py
+++ b/api/tests/test_main.py
@@ -1231,8 +1231,20 @@ class IngestProcessTest(object):
# Extract Ariflow logs
response = client.get("/Get_Airflow_logs", params={"project": project_name}, headers=headers)
assert response.status_code == 200
- dags_msgs = ["Logs for dag_id 'run_shacler'", "Logs for dag_id 'run_database_extraction'", "Logs for dag_id 'run_whole_pipeline'",
- "Logs for dag_id 'run_pre_check_and_de_identification'", "Logs for dag_id 'run_integration'", "Logs for dag_id 'run_validation'"]
+ dags_msgs = ["Logs for dag_id 'run_batch_ingestion'",
+ "Logs for dag_id 'run_connector_backup'",
+ "Logs for dag_id 'run_database_extraction'",
+ "Logs for dag_id 'run_grafana_count'",
+ "Logs for dag_id 'run_integration'",
+ "Logs for dag_id 'run_minio_download'",
+ "Logs for dag_id 'run_notify_einstein'",
+ "Logs for dag_id 'run_pre_check_and_de_identification'",
+ "Logs for dag_id 'run_real_time_count'",
+ "Logs for dag_id 'run_shacler'",
+ "Logs for dag_id 'run_sparqler'",
+ "Logs for dag_id 'run_statistics'",
+ "Logs for dag_id 'run_validation'",
+ "Logs for dag_id 'run_whole_pipeline'"]
for dag_msg in dags_msgs:
assert dag_msg in response.text
--
GitLab
From 9d7d07eb68a6bb779887fe73aececbbfa60a1bb7 Mon Sep 17 00:00:00 2001
From: Nicola Stoira <nicola.stoira@accenture.com>
Date: Wed, 26 Jun 2024 13:22:31 +0200
Subject: [PATCH 2/2] Update library
---
lib/airflow_lib.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lib/airflow_lib.py b/lib/airflow_lib.py
index a7981b77..2125e4fa 100644
--- a/lib/airflow_lib.py
+++ b/lib/airflow_lib.py
@@ -494,7 +494,9 @@ class Airflow(object):
Response: log file
"""
if dag == AirflowDag.ALL:
- dag_ids = ['run_shacler', 'run_sparqler', 'run_database_extraction', 'run_whole_pipeline', 'run_pre_check_and_de_identification', 'run_integration', 'run_validation', 'run_statistics']
+ dag_ids = ['run_batch_ingestion', 'run_connector_backup', 'run_database_extraction', 'run_grafana_count', 'run_integration', 'run_minio_download',
+ 'run_notify_einstein', 'run_pre_check_and_de_identification', 'run_real_time_count', 'run_shacler', 'run_sparqler', 'run_statistics',
+ 'run_validation', 'run_whole_pipeline']
else:
dag_ids = [dag.value]
airflow_logs = []
--
GitLab