Coverage for lib/util/lib_audit_prediag.py: 58%
107 statements
« prev ^ index » next coverage.py v7.9.1, created at 2026-01-26 23:58 +0100
« prev ^ index » next coverage.py v7.9.1, created at 2026-01-26 23:58 +0100
1# 12/25 Library to audit consistency of results to identify different type of error
2# and also compute prediagnostic
5map_data_func_error_case = {
6 "missing_pages" : {
7 "min_version_datou" : "tav",
8 "max_version_datou" : "tav",
9 "datou_id" : 40,
10 "key" : "io_exec/5/output/list_page_per_doc",
11 "func" : "check_consistency_page",
12 "file" : "lib.util.lib_audit_prediag",
13 "list_results_correct" : ["COMPLETE"]
14 },
15 "incomplete_treatment" : {
16 "key" : "",
17 "func" : "check_complete_treatment",
18 "list_results_correct" : ["COMPLETE"]
19 },
20 "file_content" : {
21 "key" : "io_exec/0/output",
22 "func" : "check_no_input_data",
23 "list_results_incorrect" : ["EXCEPTION_FILE_NONE", "NO_FILE_CONTENT"]
24 },
25 "correct_columns" : {
26 "key" : "io_exec/9/output/df_complet_as_json",
27 "func" : "check_columns_df",
28 "list_results_correct" : ["PB_ORDER_COLUMNS", "COLUMNS_AS_EXPECTED"]
29 },
30 "check_api_quota" : {
31 "key" : "io_exec/2/output/result",
32 "func" : "check_quota_api",
33 "list_results_correct" : ["OK"]
34 }
35}
37map_cost_strat = {"type_prediag" : {"MAUVAIS" : 2, "MANUSCRIT" : 2, "TABLEAU": 2},
38 "tolerance" : 1.2}
40def check_quota_api(data):
41 if data == 'ERROR You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.':
42 return "MISSING_QUOTA"
43 else:
44 return "OK"
46def check_columns_df(data):
47 import json, pandas as pd
48 df_complet_as_json = json.loads(data) if data else {}
50 df_complet = pd.DataFrame(df_complet_as_json)
52 list_references = ['Titre', 'Nombre de pages', 'Liste des pages', 'Commentaires', 'document_type', 'prediag', 'date', 'medecin_nom', 'medecin_prenom', 'medecin_specialite', 'nom_hopital', 'genre_service_hopital', 'indication_examen', 'date_entree_hospitalisation', 'date_sortie_hospitalisation', 'motif_hospitalisation', 'date_fin_arret_travail', 'TitreMeta', 'datet', 'date_parsed_or_forced', 'date_fin_arret_travailt', 'date_entree_hospitalisationt', 'date_sortie_hospitalisationt', 'id', 'cr_back', 'intro_back']
53 list_current = list(df_complet.columns)
55 if len(list_references) != len(list_current):
56 return "MISSING_COLUMNS"
57 elif list_references != list_current:
58 return "PB_ORDER_COLUMNS"
59 else:
60 return "COLUMNS_AS_EXPECTED"
64def check_no_input_data(data):
65 if "exception" in data and data["exception"] == 'stat: path should be string, bytes, os.PathLike or integer, not NoneType':
66 return "EXCEPTION_FILE_NONE"
67 elif "list_page_content_text" in data:
68 nb_page = len(data["list_page_content_text"])
69 total_size = sum(list(map(len, data["list_page_content_text"])))
70 return "nb_page_" + str(nb_page) + "_total_char_" + str(total_size)
71 else :
72 return "NO_FILE_CONTENT"
74def check_complete_treatment(data, nb_step = 10):
76 io_exec = data["io_exec"] if "io_exec" in data else {}
77 for i in range(nb_step):
78 if str(i) not in io_exec:
79 return "MISSING_STEP_" + str(i)
81 return "COMPLETE"
83def check_consistency_page(data):
85 from lib.lib_util import from_list_page_per_doc_ccsv_to_list_of_list_of_page
86 list_of_pages_as_sccsv = data
87 list_of_list_of_page, nb_page, max_page = from_list_page_per_doc_ccsv_to_list_of_list_of_page(list_of_pages_as_sccsv)
89 if nb_page == max_page:
90 return "COMPLETE"
91 else :
92 list_of_page = []
93 for l in list_of_list_of_page:
94 list_of_page.extend(l)
95 diff = list(set(range(0, max_page)) - set(list_of_page))
96 return "MISSING_PAGE_RESULT_" + str(max_page - nb_page) + "_:_" + str(diff)
99def compute_pre_diagnostic(info_date):
100 from lib.sandbox.migration.append_prediag_df_complet_as_json import build_map_from_prediag_id_page_c_csv
101 data_prediag = info_date["prediag_csv"] if "prediag_csv" in info_date else ""
102 if ":" in data_prediag:
103 map_id_page_prediag = build_map_from_prediag_id_page_c_csv(data_prediag)
104 elif data_prediag == "":
105 map_id_page_prediag = {}
106 else:
107 map_id_page_prediag = {str(i): data_prediag.split(",")[i] for i in range(len(data_prediag.split(",")))}
109 if map_id_page_prediag != {}:
110 total_cost = 0
111 nb_page = 0
113 for page in map_id_page_prediag:
114 prediag_this_page = map_id_page_prediag[page]
115 cost_this_page = 1
116 if prediag_this_page in map_cost_strat["type_prediag"]:
117 cost_this_page = map_cost_strat["type_prediag"][prediag_this_page]
118 nb_page += 1
119 total_cost += cost_this_page
121 if nb_page > 0:
122 ratio = float(total_cost) / float(nb_page)
123 if ratio > map_cost_strat["tolerance"]:
124 return "HIGH_COST_" + str(ratio)
125 else:
126 return "CORRECT_COST_" + str(ratio)
127 else:
128 return "MISSING_DATA"
130 return "MISSING_DATA"
132def parse_error_case(audit_info, map_data_func_error_case_input = {},
133 during_exec = False,
134 id_step = None):
135 if map_data_func_error_case_input == {}:
136 map_data_func_error_case_input = map_data_func_error_case
138 map_flag_error_case = {}
140 correct_treatment = True
142 for case in map_data_func_error_case_input:
143 this_audit_data = map_data_func_error_case_input[case]
144 key = this_audit_data["key"] if "key" in this_audit_data else ""
145 func_name = this_audit_data["func"] if "func" in this_audit_data else ""
146 file = this_audit_data["file"] if "file" in this_audit_data else "lib.util.lib_audit_prediag"
148 min_version_datou = this_audit_data["min_version_datou"] if "min_version_datou" in this_audit_data else ""
149 max_version_datou = this_audit_data["max_version_datou"] if "max_version_datou" in this_audit_data else ""
150 datou_id = this_audit_data["datou_id"] if "datou_id" in this_audit_data else ""
151 list_results_correct = this_audit_data["list_results_correct"] if "list_results_correct" in this_audit_data else []
152 list_results_incorrect = this_audit_data["list_results_incorrect"] if "list_results_incorrect" in this_audit_data else []
153 # TODO : check consistency of version (need to know this alphabet)
155 # - --- -... --- : load data and detect error
156 from lib.manaudit.lib_datou_audit import load_sub_json
157 if during_exec:
158 if id_step == None or (type(id_step) != int and (type(id_step) != str or not id_step.isdigit())):
159 continue
160 prefix_current_step = "io_exec/" + str(id_step) + "/output"
161 if not key.startswith(prefix_current_step):
162 continue
163 key_used = key.lstrip(prefix_current_step)
164 else:
165 key_used = key
166 data = load_sub_json(audit_info, key_used)
167 from lib.batch.lib_batch import load_module
169 try:
170 func = load_module(file, func_name)
171 res = func(data)
172 if len(list_results_incorrect) > 0 and res in list_results_incorrect:
173 correct_treatment = False
174 if len(list_results_correct) > 0 and res not in list_results_correct:
175 correct_treatment = False
176 except Exception as e:
177 print(str(e))
178 res = "MISSING_INFO"
179 #if res == False:
180 map_flag_error_case[case] = res
182 if not during_exec:
183 map_flag_error_case["TOTAL_CORRECTNESS"] = correct_treatment
185 return map_flag_error_case