Coverage for lib/util/lib_formal_conf.py: 73%

257 statements  

« prev     ^ index     » next       coverage.py v7.9.1, created at 2026-02-10 01:10 +0100

1# TODO l'incipit 

2# Fonctionnalité pour rassembler le code de configuration des contextes d'utilisateur, d'application, de projet puis lors de l'execution tout en cherchant à dédupliquer les configurations 

3# NRT test/use_case/test_saxia.py::test_lib_formal_conf -vvv --log-cli-level=10 

4 

5def formal_conf_prepare(conf_as_json, lss, project_id = None): 

6 return formal_conf_switch_context(conf_as_json, lss, type_context = "prepare", project_id = project_id) 

7 

8def formal_conf_exec(conf_as_json, io_datou_exec, hit = None): 

9 return formal_conf_switch_context(conf_as_json, io_datou_exec = io_datou_exec, type_context = "exec", hit = hit) 

10 

11def formal_conf_all(conf_as_json, lss, io_datou_exec ={}): 

12 return formal_conf_switch_context(conf_as_json, lss = lss, io_datou_exec = io_datou_exec, type_context = "all") 

13 

14def formal_conf_switch_context(conf_as_json, lss = None, io_datou_exec = {}, 

15 type_context = "all", hit = None, project_id = None): 

16 """ 

17 This function is a wrapper for the formal configuration of the library. 

18 It is meant to be called by the user to set the configuration of the library. 

19 """ 

20 try : 

21 if type_context == "exec": 

22 list_key_word = ["safia.datou.io", "exec.func"] 

23 if lss != None: 

24 list_key_word.append("safia.audit.col") 

25 conf_as_json = formal_conf_aux(conf_as_json, lss=None, io_datou_exec=io_datou_exec, 

26 list_key_word = list_key_word, hit = hit) 

27 

28 # En fait ce n'est peut-etre pas nécessaire, car get_referenced_config dans le cas format.string va appeler pour chaque variable 

29 conf_as_json = formal_conf_aux(conf_as_json, lss=None, io_datou_exec={}, 

30 list_key_word=["format.string", "template.jinja"]) 

31 return conf_as_json 

32 elif type_context == "prepare": 

33 return formal_conf_aux(conf_as_json, lss=lss, io_datou_exec = {}, 

34 list_key_word = ["safia.audit.col_REFACTO_NOW_DONE_LATER", "safia.datou.param", "saxia.config.user", "saxia.config.project"], # , "exec.func", "format.string" 

35 project_id = project_id) 

36 elif type_context == "all": 

37 return formal_conf_aux(conf_as_json, lss=lss, io_datou_exec=io_datou_exec, project_id = project_id) 

38 else : 

39 print("type_context not supported : " + str(type_context) + ", config return like it was !") 

40 return conf_as_json 

41 except Exception as e: 

42 print(str(e)) 

43 print("Error in formal_conf_switch_context : Worst case : sme mail should be horrible !") 

44 return conf_as_json 

45 

46# TODO VR 28/0/24 en fait on aurait bien envie de faire les format string après avoir instancier ! 

47def formal_conf_aux(conf_as_json, lss = None, io_datou_exec = {}, 

48 list_key_word = ["safia.audit.col", "safia.datou.io", "safia.datou.param", "saxia.config.user", "saxia.config.project", "format.string", "exec.func", "template.jinja"], 

49 nb_level_instantiation = 16, 

50 hit = None, 

51 project_id = None): # 10, c'est déjà beaucoup 

52 """ 

53 This function is a wrapper for the formal configuration of the library. 

54 It is meant to be called by the user to set the configuration of the library. 

55 """ 

56 conf_instanciated = conf_as_json 

57 for i in range(nb_level_instantiation): 

58 list_data_referenced_and_key = collect_data_to_reference_key_word(conf_as_json, list_key_word = list_key_word) 

59 count_collected_data_not_format_string_type = sum(map(lambda x : 0 if "type" in x["data"] and x["data"]["type"] == "format.string" else 1, list_data_referenced_and_key)) 

60 # VR 25-12-24 : ce code n'est vraiment pas adapté, on ne permet que les format string au debug, je ne sais plus pourquoi j'ai fais cela, à priori c'est rempolacer par le code plus intelligent qui récupère les feuilles d'abord 

61 # Par contre on ne sait pas ce qu'il se passe lorsqu'a l'étape prepare les feuilles sont ignoré 

62 #if count_collected_data_not_format_string_type == 0 and i > 0 or (len(list_data_referenced_and_key) == 0 and i == 0): # Ce n'est pas exactement, il faudrait faire les string puis les autres 

63# En fait j'ai enlevé les commentaires au dessus dans la V0.5.85 en décembre 2024 mais je ne sais pas ce que je voulais faire, est-ce qu'il y a d'autres problèmes avec ce test un peu complexe ? 

64 # Je ne crois vraiment pas car les autres types d'instanciations rajouter en novembre 2024 ne sont pas considéré ici 

65 if count_collected_data_not_format_string_type == 0 and i > 0 or (len(list_data_referenced_and_key) == 0 and i == 0): # Ce n'est pas exactement, il faudrait faire les string puis les autres 

66# if count_collected_data_not_format_string_type == 0: 

67 print("Number level instanciation when it was finished : " + str(i)) 

68 break 

69 for data_referenced_and_key in list_data_referenced_and_key: 

70 conf_with_reference_to_add_context = data_referenced_and_key["data"] 

71 sub_conf_instanciated, key_input = get_referenced_conf(conf_with_reference_to_add_context, data_referenced_and_key["key"], io_datou_exec = io_datou_exec, lss = lss, hit = hit, project_id = project_id) 

72 from lib.manaudit.lib_datou_audit import load_sub_json 

73 key_output = data_referenced_and_key["key"] 

74# sub_conf_in = load_sub_json(sub_conf_instanciated, key_input) 

75 sub_conf_out = load_sub_json(conf_instanciated, key_output) 

76 #if type(sub_conf_instanciated) == dict: 

77 # from pydantic.utils import deep_update 

78 # sub_conf_out = deep_update(sub_conf_instanciated, sub_conf_out) 

79 #else: 

80 from lib.manaudit.lib_datou_audit import set_sub_json 

81 if key_output == "": 

82 conf_instanciated = sub_conf_instanciated 

83 else: 

84 set_sub_json(conf_instanciated, key_output, sub_conf_instanciated) 

85 if i == nb_level_instantiation - 1: 

86 print("WARNING : Number level instanciation reached the limit : " + str(nb_level_instantiation) + " maybe instanciation wasn't done correctly !") 

87 return conf_instanciated 

88 

89# VR TODO DOC : différencier les noms des variables key selon qu'elle designe des champ à récupérer ou la clef concatener du champ courant qu'on essaie d'instancier 

90# VR 28/0/24 Je ne comprends pas comment les array sont gérer avec les clef definissant leur position et donc ou on doit cherche les informations 

91# Sans doute tout n'est pas fonctionnel dans lib_conf_system 

92# Mais a mon avis on ne veut pas gérer le fait qu'une configuration référencé ne le soit que dans un champ d'un array 

93# Et du coup, j'aurai dit que le deeupdate suffit pour le reste et qu'il n'est pas nécessaire de gérer les array, mais du coup je ne comprends pas ! 

94# TODO OPTIMISATION REFACTO : pour ne récupérer qu'une fois (par niveau de référencement) les données d'un projet ou des datous par exemple, il faudrait avoir des index pour chacun de leur type, on ne va pas le faire immédiatement et assumer cette inefficience 

95def collect_data_to_reference_key_word(conf_as_json, 

96 list_key_word = ["safia.audit.col", "safia.datou.io", "safia.datou.param", "saxia.config.user", "saxia.config.project", "format.string", "exec.func"], 

97 current_sub_key = ""): 

98 """ 

99 Find the keywords and param 

100 """ 

101 list_data_referenced_and_key = [] 

102 # TODO 

103 found_key_word = False 

104 if type(conf_as_json) not in [list, dict]: 

105 return [] 

106 if type(conf_as_json) == list: 

107 # load_sub_json manage int so it would work 

108 instance_json_arr = [] 

109 for i, sub_json in enumerate(conf_as_json): 

110 list_data_referenced_and_key_aux = collect_data_to_reference_key_word(sub_json, list_key_word, current_sub_key + "/" + str(i)) 

111 list_data_referenced_and_key.extend(list_data_referenced_and_key_aux) 

112 

113 data_referenced_with_key_current = None 

114 if "type" in conf_as_json and conf_as_json["type"] in list_key_word: 

115 found_key_word = True 

116 data_referenced_with_key_current = {"key" : current_sub_key, "data" : conf_as_json} 

117# list_data_referenced_and_key.append(data_referenced_with_key_current) 

118 

119 # For Migration, will be deprecated, managing old type of config, until vav expected 15/12/24, where key is the type of reference and instanciation 

120 found_key_word_deprecated = False 

121 if not found_key_word: 

122 for k in conf_as_json: 

123 if k in list_key_word: 

124 print("DEPRECATED until vav expected 15/12/24 : MIGRATION SOFT TRASNITION TO BE DOCUMENTED ") 

125# conf_as_json = conf_as_json[k] 

126 conf_as_json[k]["type"] = k 

127 # This makes management in instantiation in later function useless 

128 # key input and output are the same if not specified in input 

129 # if key is specified in sub_json, it will be used in instantiation instead of the current key 

130 if "key" not in conf_as_json[k]: 

131 conf_as_json[k]["key"] = current_sub_key 

132 # This next key is the output key 

133 data_referenced_with_key_current = {"key" : current_sub_key, "data" : conf_as_json[k]} 

134 # VR 1-12 : we add it here because inner element of dict isn't the correct one to use => it is not clear what should be the behavior then, but we will test 

135 list_data_referenced_and_key.append(data_referenced_with_key_current) 

136 found_key_word_deprecated = True 

137 

138 found_key_word = True 

139 # Multiple type would make all of this fail ! 

140 break 

141 

142 # if not found_key_word: 

143# for k in conf_as_json: 

144 if not found_key_word_deprecated: 

145 for k, sub_json_or_key in enumerate(conf_as_json): 

146 if type(conf_as_json) == list: 

147 key = k 

148 data = sub_json_or_key 

149 else : # should be dict by now 

150 key = sub_json_or_key 

151 data = conf_as_json[sub_json_or_key] 

152 list_data_referenced_and_key_aux = collect_data_to_reference_key_word(data, list_key_word, current_sub_key + "/" + str(key)) 

153 list_data_referenced_and_key.extend(list_data_referenced_and_key_aux) 

154 

155 if data_referenced_with_key_current != None : 

156 # We should add this condition for exec.func, format.string (when not using specific loop on variable) 

157 if len(list_data_referenced_and_key) == 0: 

158 list_data_referenced_and_key.append(data_referenced_with_key_current) 

159 else : 

160 print("WARNING TREATED AS ERROR ON 30-11-24 LFC ORDER EXECUTION => if problem we could revert back by removing the condition to instanciate, but it should be harmless and useful to have this order of execution !") 

161 if "type" in conf_as_json: 

162 print(" WARNING TREATED : " + str(conf_as_json["type"])) 

163 

164 return list_data_referenced_and_key 

165 

166def format_string(format, map_var_val): 

167 for var in map_var_val: 

168 val = map_var_val[var] 

169 format = format.replace("{" + var + "}", str(val)) 

170 return format 

171 

172# DOC : prend en entrée un seul json correspondant à une configuration à instancié à partir des elements référencés 

173def get_referenced_conf(conf_with_reference_to_add_context, 

174 sub_keys = None, 

175 io_datou_exec = {}, 

176 lss = None, 

177 hit = None, 

178 project_id = None, 

179 verbose = False): 

180 """ 

181 Get the referenced configuration 

182 """ 

183 # maybe duplicate with stuff above ! 

184 # manage compatibility while refacto migration listed as TODO 

185 if "type" not in conf_with_reference_to_add_context: 

186 print("DEPRECATED conf, where the type is the only key if more we are in trouble") 

187 if len(conf_with_reference_to_add_context.keys()) > 1: 

188 print("WARNING : more than one key in the conf and no type specified, we haven't choose a default behaovior, we are in trouble, it should exit later without correct configuration !") 

189 if len(conf_with_reference_to_add_context.keys()) == 0: 

190 print("ERROR : no key in the conf, we are in trouble, it could exit later without correct configuration") 

191 return conf_with_reference_to_add_context 

192 conf_with_reference_to_add_context["type"] = list(conf_with_reference_to_add_context.keys())[0] 

193 type_formal_config = conf_with_reference_to_add_context["type"] if "type" in conf_with_reference_to_add_context else list(conf_with_reference_to_add_context.keys())[0] 

194 

195 conf_instanciated = None 

196 if "key" in conf_with_reference_to_add_context: 

197 sub_keys = conf_with_reference_to_add_context["key"] 

198 

199 if type_formal_config == "saxia.config.project": 

200 project_id = conf_with_reference_to_add_context["project_id"] if "project_id" in conf_with_reference_to_add_context else project_id 

201 if project_id == None: 

202 list_projects = lss.get_project() 

203 if len(list_projects) > 0 and "id" in list_projects[0]: 

204 project_id = list_projects[0]["id"] 

205 from server.safia import lpgss_singleton 

206 project_configuration_referenced = lpgss_singleton.load_conf_project(project_id) 

207 from lib.manaudit.lib_datou_audit import load_sub_json 

208 if sub_keys != None: 

209 conf_instanciated = load_sub_json(project_configuration_referenced, sub_keys.lstrip("/")) 

210 else : 

211 conf_instanciated = project_configuration_referenced 

212 

213# TODO use this or delete these comment 

214# from pydantic.utils import deep_update 

215# conf_instanciated = deep_update(conf_with_reference_to_add_context, project_configuration_referenced_sub_key) 

216 

217 elif type_formal_config == "saxia.config.user": 

218 

219 print("TODO in LFC l 184") 

220 

221 user_info = lss.get_user_info() 

222 if "key" in conf_with_reference_to_add_context: 

223 key = conf_with_reference_to_add_context["key"] 

224 if "/" in key: 

225 from lib.manaudit.lib_datou_audit import load_sub_json 

226 conf_instanciated = load_sub_json(user_info, key.lstrip("/")) 

227 elif key in user_info: 

228 conf_instanciated = user_info[key] 

229 else: 

230 conf_instanciated = {} 

231 else : 

232 conf_instanciated = {} 

233 elif type_formal_config == "safia.audit.col": 

234 

235# sub_keys = None 

236 print("test in LFC") 

237 if "col_source" not in conf_with_reference_to_add_context: 

238 print("Missing col for type safia.audit.col, avoiding the instanciation of the configuration") 

239 return None, None 

240 col_in = conf_with_reference_to_add_context["col_source"] 

241 if hit == None: 

242 print("Missing hash_id_treatment for type safia.audit.col, avoiding the instanciation of the configuration") 

243 return None, None 

244 list_col_as_csv = str(col_in) 

245 data_audit = lss.lib_user_data_internal.load_data_audit(hash_id_treatment=hit, col_csv=list_col_as_csv, limit=1) 

246 

247 conf_instanciated = None 

248 if data_audit != None and col_in in data_audit: 

249 if ("key" in conf_with_reference_to_add_context and 

250 type(data_audit[col_in]) == dict and 

251 conf_with_reference_to_add_context["key"] in data_audit[col_in]): 

252 conf_instanciated = data_audit[col_in][conf_with_reference_to_add_context["key"]] 

253 else: 

254 conf_instanciated = data_audit[col_in] 

255 

256 if conf_instanciated != None and sub_keys != None and type(conf_instanciated) == dict: 

257 from lib.manaudit.lib_datou_audit import load_sub_json 

258 conf_instanciated = load_sub_json(conf_instanciated, sub_keys) 

259 

260 elif type_formal_config == "safia.datou.io": 

261 if "key" in conf_with_reference_to_add_context : 

262 key = conf_with_reference_to_add_context["key"] 

263 if key in io_datou_exec: 

264 conf_instanciated = io_datou_exec[key] 

265 else : 

266 conf_instanciated = {} 

267 else : 

268 conf_instanciated = {} 

269 elif type_formal_config == "safia.datou.param": 

270 print("TODO in LFC l 228") 

271 conf_referenced = lss.get_datou_param(conf_with_reference_to_add_context["datou_param_id"]) 

272 elif type_formal_config == "format.string": 

273 print("TODO in LFC l 231") 

274 map_variables = conf_with_reference_to_add_context["variables"] if "variables" in conf_with_reference_to_add_context else [] 

275 map_var_val = {} 

276 format = conf_with_reference_to_add_context["format"] if "format" in conf_with_reference_to_add_context else "" 

277 

278 if type(map_variables) == list: 

279 print("WILL FAIL !") 

280 

281 for variable in map_variables: 

282 var = variable #["name"] if "name" in variable else "default_variable_name" 

283 val = map_variables[variable]["val"] if map_variables[variable] != None and "val" in map_variables[variable] else map_variables[variable] 

284 type_var = map_variables[variable]["type"] if map_variables[variable] != None and type(map_variables[variable]) == dict and "type" in map_variables[variable] else None 

285 if type_var != None: 

286 print("DEPRECATE LFC 271") 

287 val, _ = get_referenced_conf(map_variables[variable], 

288 sub_keys = None, 

289 io_datou_exec = io_datou_exec, 

290 lss = lss) 

291 

292 map_var_val[var] = val 

293 ret = format_string(format, map_var_val) 

294 return ret, None 

295 elif type_formal_config == "exec.func": 

296 sub_keys = None 

297 func = conf_with_reference_to_add_context["func"] if "func" in conf_with_reference_to_add_context else None 

298 variables = conf_with_reference_to_add_context["variables"] if "variables" in conf_with_reference_to_add_context else conf_with_reference_to_add_context 

299 

300 if func == "load.df.fromjsonasstring": 

301 print("One variable needed : df_as_string !") 

302 if "df_as_json" in variables: 

303 df_as_json = variables["df_as_json"] 

304 import pandas as pd 

305 if type(df_as_json) == type(None): 

306 print("ERROR : df_as_json is None, we are in trouble, it could exit later without correct configuration") 

307 conf_instanciated = "None" 

308 conf_instanciated = pd.read_json(df_as_json) 

309 if func == "extract.df.col": 

310 print("Two variable needed : df, col !") 

311 if "df" in variables and "col" in variables: 

312 df = variables["df"] 

313 col = variables["col"] 

314 import pandas as pd 

315 if type(df) != pd.DataFrame or not hasattr(df, "columns"): 

316 print("ERROR : df is not a pandas DataFrame with columns, we are in trouble, it could exit later without correct configuration") 

317 return conf_with_reference_to_add_context, sub_keys 

318 if col in df.columns: 

319 conf_instanciated_tuple = df[col] 

320 conf_instanciated = str(conf_instanciated_tuple) 

321 col_as_txt = "" 

322 for index, row in df.iterrows(): 

323 col_as_txt += "Doc id : " + str(index) + " : " + str(row[col]) + "\n" 

324 conf_instanciated = col_as_txt 

325 else : 

326 print(" col : " + str(col) + " not in df.columns : " + str(df.columns)) 

327 conf_instanciated_tuple = None 

328 conf_instanciated = str(conf_instanciated_tuple) 

329 if func == "var": 

330 sub_type = conf_with_reference_to_add_context["sub_type"] if "sub_type" in conf_with_reference_to_add_context else None 

331 

332 key_ext = conf_with_reference_to_add_context["key_ext"] if "key_ext" in conf_with_reference_to_add_context else None 

333 key_int = conf_with_reference_to_add_context["key_int"] if "key_int" in conf_with_reference_to_add_context else None 

334 key_or_data = conf_with_reference_to_add_context["key_or_data"] if "key_or_data" in conf_with_reference_to_add_context else None 

335 

336 input = conf_with_reference_to_add_context["input"] if "input" in conf_with_reference_to_add_context else None 

337 

338 if key_ext != None: 

339 from lib.manaudit.lib_datou_audit import load_sub_json 

340 input = load_sub_json(input, key_ext) 

341 

342 if sub_type == "arr": 

343 conf_instanciated = [] 

344 print("TODO arr func exec.fun VAR") 

345 for k in input : 

346 data = input[k] 

347 if key_int != None and key_int != '': # pas possible avec key_or_data 

348 from lib.manaudit.lib_datou_audit import load_sub_json 

349 data = load_sub_json(data, key_int) 

350 if key_or_data: 

351 data = k 

352 conf_instanciated.append(data) 

353 

354 elif sub_type == "dict": 

355 print("TODO dict func exec.fun VAR") 

356 else : 

357 print("ERROR : sub_type not managed, we are in trouble, it could exit later without correct configuration") 

358 if func == "htmlencode": 

359 print("ERROR : func not specified, we are in trouble, it could exit later without correct configuration") 

360 import urllib 

361 if "input_encode" in variables and type(variables["input_encode"]) == str: 

362 input_encode = variables["input_encode"] 

363 conf_instanciated = urllib.parse.quote(input_encode) 

364# if type(conf_with_reference_to_add_context) == str: 

365# conf_instanciated = urllib.parse.quote_plus(conf_with_reference_to_add_context) 

366 else: 

367 print("ERROR : func htmlencode need a string in input in input_encode, we are in trouble, it could exit later without correct configuration : " + str(variables)) 

368 return conf_with_reference_to_add_context, sub_keys 

369 return conf_instanciated, sub_keys 

370 

371 elif type_formal_config == "template.jinja": 

372 

373 try : 

374 template_jinja_file = conf_with_reference_to_add_context["template_jinja_file"] if "template_jinja_file" in conf_with_reference_to_add_context else "templates/email/admin_end_auto_saxia.html" 

375 email_data = conf_with_reference_to_add_context["email_data"] if "email_data" in conf_with_reference_to_add_context else {"text_content": "Empty content !"} 

376 

377 # Read the Jinja2 email template 

378 with open(template_jinja_file, "r") as file: 

379 template_str = file.read() 

380 

381 from jinja2 import Template 

382 jinja_template = Template(template_str) 

383 email_content = jinja_template.render({"email_data" : email_data}) 

384 conf_instanciated = email_content 

385 except Exception as e: 

386 print(str(e)) 

387 print("ERror in templating jinja in LFC") 

388 else: 

389 print("ERROR : type not managed, we are in trouble, it could exit later without correct configuration") 

390 return conf_with_reference_to_add_context, sub_keys 

391 

392 return conf_instanciated, sub_keys 

393 

394