Coverage for lib/util/lib_audit_prediag.py: 58%

107 statements  

« prev     ^ index     » next       coverage.py v7.9.1, created at 2026-01-26 23:58 +0100

1# 12/25 Library to audit consistency of results to identify different type of error 

2# and also compute prediagnostic 

3 

4 

5map_data_func_error_case = { 

6 "missing_pages" : { 

7 "min_version_datou" : "tav", 

8 "max_version_datou" : "tav", 

9 "datou_id" : 40, 

10 "key" : "io_exec/5/output/list_page_per_doc", 

11 "func" : "check_consistency_page", 

12 "file" : "lib.util.lib_audit_prediag", 

13 "list_results_correct" : ["COMPLETE"] 

14 }, 

15 "incomplete_treatment" : { 

16 "key" : "", 

17 "func" : "check_complete_treatment", 

18 "list_results_correct" : ["COMPLETE"] 

19 }, 

20 "file_content" : { 

21 "key" : "io_exec/0/output", 

22 "func" : "check_no_input_data", 

23 "list_results_incorrect" : ["EXCEPTION_FILE_NONE", "NO_FILE_CONTENT"] 

24 }, 

25 "correct_columns" : { 

26 "key" : "io_exec/9/output/df_complet_as_json", 

27 "func" : "check_columns_df", 

28 "list_results_correct" : ["PB_ORDER_COLUMNS", "COLUMNS_AS_EXPECTED"] 

29 }, 

30 "check_api_quota" : { 

31 "key" : "io_exec/2/output/result", 

32 "func" : "check_quota_api", 

33 "list_results_correct" : ["OK"] 

34 } 

35} 

36 

37map_cost_strat = {"type_prediag" : {"MAUVAIS" : 2, "MANUSCRIT" : 2, "TABLEAU": 2}, 

38 "tolerance" : 1.2} 

39 

40def check_quota_api(data): 

41 if data == 'ERROR You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.': 

42 return "MISSING_QUOTA" 

43 else: 

44 return "OK" 

45 

46def check_columns_df(data): 

47 import json, pandas as pd 

48 df_complet_as_json = json.loads(data) if data else {} 

49 

50 df_complet = pd.DataFrame(df_complet_as_json) 

51 

52 list_references = ['Titre', 'Nombre de pages', 'Liste des pages', 'Commentaires', 'document_type', 'prediag', 'date', 'medecin_nom', 'medecin_prenom', 'medecin_specialite', 'nom_hopital', 'genre_service_hopital', 'indication_examen', 'date_entree_hospitalisation', 'date_sortie_hospitalisation', 'motif_hospitalisation', 'date_fin_arret_travail', 'TitreMeta', 'datet', 'date_parsed_or_forced', 'date_fin_arret_travailt', 'date_entree_hospitalisationt', 'date_sortie_hospitalisationt', 'id', 'cr_back', 'intro_back'] 

53 list_current = list(df_complet.columns) 

54 

55 if len(list_references) != len(list_current): 

56 return "MISSING_COLUMNS" 

57 elif list_references != list_current: 

58 return "PB_ORDER_COLUMNS" 

59 else: 

60 return "COLUMNS_AS_EXPECTED" 

61 

62 

63 

64def check_no_input_data(data): 

65 if "exception" in data and data["exception"] == 'stat: path should be string, bytes, os.PathLike or integer, not NoneType': 

66 return "EXCEPTION_FILE_NONE" 

67 elif "list_page_content_text" in data: 

68 nb_page = len(data["list_page_content_text"]) 

69 total_size = sum(list(map(len, data["list_page_content_text"]))) 

70 return "nb_page_" + str(nb_page) + "_total_char_" + str(total_size) 

71 else : 

72 return "NO_FILE_CONTENT" 

73 

74def check_complete_treatment(data, nb_step = 10): 

75 

76 io_exec = data["io_exec"] if "io_exec" in data else {} 

77 for i in range(nb_step): 

78 if str(i) not in io_exec: 

79 return "MISSING_STEP_" + str(i) 

80 

81 return "COMPLETE" 

82 

83def check_consistency_page(data): 

84 

85 from lib.lib_util import from_list_page_per_doc_ccsv_to_list_of_list_of_page 

86 list_of_pages_as_sccsv = data 

87 list_of_list_of_page, nb_page, max_page = from_list_page_per_doc_ccsv_to_list_of_list_of_page(list_of_pages_as_sccsv) 

88 

89 if nb_page == max_page: 

90 return "COMPLETE" 

91 else : 

92 list_of_page = [] 

93 for l in list_of_list_of_page: 

94 list_of_page.extend(l) 

95 diff = list(set(range(0, max_page)) - set(list_of_page)) 

96 return "MISSING_PAGE_RESULT_" + str(max_page - nb_page) + "_:_" + str(diff) 

97 

98 

99def compute_pre_diagnostic(info_date): 

100 from lib.sandbox.migration.append_prediag_df_complet_as_json import build_map_from_prediag_id_page_c_csv 

101 data_prediag = info_date["prediag_csv"] if "prediag_csv" in info_date else "" 

102 if ":" in data_prediag: 

103 map_id_page_prediag = build_map_from_prediag_id_page_c_csv(data_prediag) 

104 elif data_prediag == "": 

105 map_id_page_prediag = {} 

106 else: 

107 map_id_page_prediag = {str(i): data_prediag.split(",")[i] for i in range(len(data_prediag.split(",")))} 

108 

109 if map_id_page_prediag != {}: 

110 total_cost = 0 

111 nb_page = 0 

112 

113 for page in map_id_page_prediag: 

114 prediag_this_page = map_id_page_prediag[page] 

115 cost_this_page = 1 

116 if prediag_this_page in map_cost_strat["type_prediag"]: 

117 cost_this_page = map_cost_strat["type_prediag"][prediag_this_page] 

118 nb_page += 1 

119 total_cost += cost_this_page 

120 

121 if nb_page > 0: 

122 ratio = float(total_cost) / float(nb_page) 

123 if ratio > map_cost_strat["tolerance"]: 

124 return "HIGH_COST_" + str(ratio) 

125 else: 

126 return "CORRECT_COST_" + str(ratio) 

127 else: 

128 return "MISSING_DATA" 

129 

130 return "MISSING_DATA" 

131 

132def parse_error_case(audit_info, map_data_func_error_case_input = {}, 

133 during_exec = False, 

134 id_step = None): 

135 if map_data_func_error_case_input == {}: 

136 map_data_func_error_case_input = map_data_func_error_case 

137 

138 map_flag_error_case = {} 

139 

140 correct_treatment = True 

141 

142 for case in map_data_func_error_case_input: 

143 this_audit_data = map_data_func_error_case_input[case] 

144 key = this_audit_data["key"] if "key" in this_audit_data else "" 

145 func_name = this_audit_data["func"] if "func" in this_audit_data else "" 

146 file = this_audit_data["file"] if "file" in this_audit_data else "lib.util.lib_audit_prediag" 

147 

148 min_version_datou = this_audit_data["min_version_datou"] if "min_version_datou" in this_audit_data else "" 

149 max_version_datou = this_audit_data["max_version_datou"] if "max_version_datou" in this_audit_data else "" 

150 datou_id = this_audit_data["datou_id"] if "datou_id" in this_audit_data else "" 

151 list_results_correct = this_audit_data["list_results_correct"] if "list_results_correct" in this_audit_data else [] 

152 list_results_incorrect = this_audit_data["list_results_incorrect"] if "list_results_incorrect" in this_audit_data else [] 

153 # TODO : check consistency of version (need to know this alphabet) 

154 

155 # - --- -... --- : load data and detect error 

156 from lib.manaudit.lib_datou_audit import load_sub_json 

157 if during_exec: 

158 if id_step == None or (type(id_step) != int and (type(id_step) != str or not id_step.isdigit())): 

159 continue 

160 prefix_current_step = "io_exec/" + str(id_step) + "/output" 

161 if not key.startswith(prefix_current_step): 

162 continue 

163 key_used = key.lstrip(prefix_current_step) 

164 else: 

165 key_used = key 

166 data = load_sub_json(audit_info, key_used) 

167 from lib.batch.lib_batch import load_module 

168 

169 try: 

170 func = load_module(file, func_name) 

171 res = func(data) 

172 if len(list_results_incorrect) > 0 and res in list_results_incorrect: 

173 correct_treatment = False 

174 if len(list_results_correct) > 0 and res not in list_results_correct: 

175 correct_treatment = False 

176 except Exception as e: 

177 print(str(e)) 

178 res = "MISSING_INFO" 

179 #if res == False: 

180 map_flag_error_case[case] = res 

181 

182 if not during_exec: 

183 map_flag_error_case["TOTAL_CORRECTNESS"] = correct_treatment 

184 

185 return map_flag_error_case