From 327bb51647d696c6980be7f4dbaff61595f047f1 Mon Sep 17 00:00:00 2001
From: Nicola Stoira <nicola.stoira@accenture.com>
Date: Mon, 27 Jan 2025 10:22:38 +0100
Subject: [PATCH 1/3] Improve info message if unversioned code has been
replaced
---
api/tests/test_main.py | 2 +-
lib/pre_checks.py | 10 +++++++++-
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/api/tests/test_main.py b/api/tests/test_main.py
index 80a26f19..aaebde1c 100644
--- a/api/tests/test_main.py
+++ b/api/tests/test_main.py
@@ -956,7 +956,7 @@ class IngestProcessTest(object):
"Pre-Check 'defaultValidationCheck' of type 'validationCheck' PASSED",
"Pre-Check 'testRegexCheck' of type 'regexCheck' FAILED with severity level 'warn'",
"Pre-Check 'jsonPreProcessing' of type 'jsonPreProcessing' PASSED",
- "Pre-Check 'replaceUnversionedCodes' of type 'replaceUnversionedCodes' PASSED"]
+ "Pre-Check 'replaceUnversionedCodes' of type 'replaceUnversionedCodes' PASSED. At least one one unversioned code has been found where versioned codes can be used, the codes have been reassigned to the versioned codes"]
assert patient_2_pre_checks == ["Pre-Check 'testDataTypeCheck' of type 'dataTypeCheck' FAILED with severity level 'warn'", "Values not matching data type 'xsd:dateTime': ['2022-04-24']"]
assert patient_1_de_id == ["De-Identification rule 'testScrambleField' of type 'scrambleField' EXECUTED", "De-Identification rule 'scrambleAllergenID' of type 'scrambleField' EXECUTED",
"De-Identification rule 'scrambleAllergenCode' of type 'scrambleField' EXECUTED", "De-Identification rule 'testDateShift' of type 'dateShift' EXECUTED",
diff --git a/lib/pre_checks.py b/lib/pre_checks.py
index b12c4ff5..a4c116cb 100644
--- a/lib/pre_checks.py
+++ b/lib/pre_checks.py
@@ -573,11 +573,19 @@ class ReplaceUnversionedCodes(PreCheck):
PreCheckResult: object with results of execution
"""
try:
+ original_patient_data = deepcopy(patient_data)
start_time = datetime.now()
filtered_keys = [key for key in keys if re.sub(r"(\/[0-9]*\/)", "/", key) in code_term_paths and key.endswith('/termid')]
filtered_keys_1 = [key.replace('/termid', '/iri') for key in filtered_keys if key in keys]
self.replace_unversioned_codes(data=patient_data, key_paths=filtered_keys_1)
- return PreCheckResult(status=PreCheckStatus.SUCCESS, check=self, start_time=start_time, execution_time=(datetime.now() - start_time).seconds)
+ execution_time = (datetime.now() - start_time).seconds
+ if original_patient_data != patient_data:
+ message = f"Pre-Check '{self.check_name}' of type '{self.type.value}' PASSED. Start time: {start_time}. Execution time: {execution_time}. \
+ At least one one unversioned code has been found where versioned codes can be used, the codes have been reassigned to the versioned codes"
+ else:
+ message = f"Pre-Check '{self.check_name}' of type '{self.type.value}' PASSED. Start time: {start_time}. Execution time: {execution_time}. \
+ No unversioned codes have been reassigned"
+ return PreCheckResult(status=PreCheckStatus.SUCCESS, check=self, start_time=start_time, execution_time=execution_time, message=message)
except Exception:
return PreCheckResult(status=PreCheckStatus.FAILED, check=self, start_time=start_time, execution_time=(datetime.now() - start_time).seconds, message=traceback.format_exc())
--
GitLab
From 5df65e968b8ec3713f5e0b5d9974bcbf2874a3b2 Mon Sep 17 00:00:00 2001
From: Nicola Stoira <nicola.stoira@accenture.com>
Date: Mon, 27 Jan 2025 10:39:54 +0100
Subject: [PATCH 2/3] Add message to pre.check when SUCCESS
---
lib/pre_checks.py | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/lib/pre_checks.py b/lib/pre_checks.py
index a4c116cb..7cb7beb2 100644
--- a/lib/pre_checks.py
+++ b/lib/pre_checks.py
@@ -175,7 +175,10 @@ class PreCheckResult(object):
self.status = status
self.level = check.level
if self.status == PreCheckStatus.SUCCESS:
- self.message = f"Pre-Check '{check.check_name}' of type '{check.type.value}' PASSED. Start time: {start_time}. Execution time: {execution_time}"
+ if message is not None:
+ self.message = f"Pre-Check '{check.check_name}' of type '{check.type.value}' PASSED. Start time: {start_time}. Execution time: {execution_time}. {message}"
+ else:
+ self.message = f"Pre-Check '{check.check_name}' of type '{check.type.value}' PASSED. Start time: {start_time}. Execution time: {execution_time}"
else:
self.message = f"Pre-Check '{check.check_name}' of type '{check.type.value}' FAILED with severity level '{self.level.value.lower()}'. Start time: {start_time}. Execution time: {execution_time}. Error message: {message}"
@@ -578,14 +581,11 @@ class ReplaceUnversionedCodes(PreCheck):
filtered_keys = [key for key in keys if re.sub(r"(\/[0-9]*\/)", "/", key) in code_term_paths and key.endswith('/termid')]
filtered_keys_1 = [key.replace('/termid', '/iri') for key in filtered_keys if key in keys]
self.replace_unversioned_codes(data=patient_data, key_paths=filtered_keys_1)
- execution_time = (datetime.now() - start_time).seconds
if original_patient_data != patient_data:
- message = f"Pre-Check '{self.check_name}' of type '{self.type.value}' PASSED. Start time: {start_time}. Execution time: {execution_time}. \
- At least one one unversioned code has been found where versioned codes can be used, the codes have been reassigned to the versioned codes"
+ message = f"At least one one unversioned code has been found where versioned codes can be used, the codes have been reassigned to the versioned codes"
else:
- message = f"Pre-Check '{self.check_name}' of type '{self.type.value}' PASSED. Start time: {start_time}. Execution time: {execution_time}. \
- No unversioned codes have been reassigned"
- return PreCheckResult(status=PreCheckStatus.SUCCESS, check=self, start_time=start_time, execution_time=execution_time, message=message)
+ message = "No unversioned codes have been reassigned"
+ return PreCheckResult(status=PreCheckStatus.SUCCESS, check=self, start_time=start_time, execution_time=(datetime.now() - start_time).seconds, message=message)
except Exception:
return PreCheckResult(status=PreCheckStatus.FAILED, check=self, start_time=start_time, execution_time=(datetime.now() - start_time).seconds, message=traceback.format_exc())
--
GitLab
From 100db08daf862b1b59acb3fc21091fd23a0b05ce Mon Sep 17 00:00:00 2001
From: Nicola Stoira <nicola.stoira@accenture.com>
Date: Mon, 27 Jan 2025 11:47:15 +0100
Subject: [PATCH 3/3] Revert changes on unit test
---
api/tests/test_main.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/api/tests/test_main.py b/api/tests/test_main.py
index aaebde1c..80a26f19 100644
--- a/api/tests/test_main.py
+++ b/api/tests/test_main.py
@@ -956,7 +956,7 @@ class IngestProcessTest(object):
"Pre-Check 'defaultValidationCheck' of type 'validationCheck' PASSED",
"Pre-Check 'testRegexCheck' of type 'regexCheck' FAILED with severity level 'warn'",
"Pre-Check 'jsonPreProcessing' of type 'jsonPreProcessing' PASSED",
- "Pre-Check 'replaceUnversionedCodes' of type 'replaceUnversionedCodes' PASSED. At least one one unversioned code has been found where versioned codes can be used, the codes have been reassigned to the versioned codes"]
+ "Pre-Check 'replaceUnversionedCodes' of type 'replaceUnversionedCodes' PASSED"]
assert patient_2_pre_checks == ["Pre-Check 'testDataTypeCheck' of type 'dataTypeCheck' FAILED with severity level 'warn'", "Values not matching data type 'xsd:dateTime': ['2022-04-24']"]
assert patient_1_de_id == ["De-Identification rule 'testScrambleField' of type 'scrambleField' EXECUTED", "De-Identification rule 'scrambleAllergenID' of type 'scrambleField' EXECUTED",
"De-Identification rule 'scrambleAllergenCode' of type 'scrambleField' EXECUTED", "De-Identification rule 'testDateShift' of type 'dateShift' EXECUTED",
--
GitLab