Coverage for lib/util/lib_formal_conf.py: 73%
257 statements
« prev ^ index » next coverage.py v7.9.1, created at 2026-02-10 01:10 +0100
« prev ^ index » next coverage.py v7.9.1, created at 2026-02-10 01:10 +0100
1# TODO l'incipit
2# Fonctionnalité pour rassembler le code de configuration des contextes d'utilisateur, d'application, de projet puis lors de l'execution tout en cherchant à dédupliquer les configurations
3# NRT test/use_case/test_saxia.py::test_lib_formal_conf -vvv --log-cli-level=10
5def formal_conf_prepare(conf_as_json, lss, project_id = None):
6 return formal_conf_switch_context(conf_as_json, lss, type_context = "prepare", project_id = project_id)
8def formal_conf_exec(conf_as_json, io_datou_exec, hit = None):
9 return formal_conf_switch_context(conf_as_json, io_datou_exec = io_datou_exec, type_context = "exec", hit = hit)
11def formal_conf_all(conf_as_json, lss, io_datou_exec ={}):
12 return formal_conf_switch_context(conf_as_json, lss = lss, io_datou_exec = io_datou_exec, type_context = "all")
14def formal_conf_switch_context(conf_as_json, lss = None, io_datou_exec = {},
15 type_context = "all", hit = None, project_id = None):
16 """
17 This function is a wrapper for the formal configuration of the library.
18 It is meant to be called by the user to set the configuration of the library.
19 """
20 try :
21 if type_context == "exec":
22 list_key_word = ["safia.datou.io", "exec.func"]
23 if lss != None:
24 list_key_word.append("safia.audit.col")
25 conf_as_json = formal_conf_aux(conf_as_json, lss=None, io_datou_exec=io_datou_exec,
26 list_key_word = list_key_word, hit = hit)
28 # En fait ce n'est peut-etre pas nécessaire, car get_referenced_config dans le cas format.string va appeler pour chaque variable
29 conf_as_json = formal_conf_aux(conf_as_json, lss=None, io_datou_exec={},
30 list_key_word=["format.string", "template.jinja"])
31 return conf_as_json
32 elif type_context == "prepare":
33 return formal_conf_aux(conf_as_json, lss=lss, io_datou_exec = {},
34 list_key_word = ["safia.audit.col_REFACTO_NOW_DONE_LATER", "safia.datou.param", "saxia.config.user", "saxia.config.project"], # , "exec.func", "format.string"
35 project_id = project_id)
36 elif type_context == "all":
37 return formal_conf_aux(conf_as_json, lss=lss, io_datou_exec=io_datou_exec, project_id = project_id)
38 else :
39 print("type_context not supported : " + str(type_context) + ", config return like it was !")
40 return conf_as_json
41 except Exception as e:
42 print(str(e))
43 print("Error in formal_conf_switch_context : Worst case : sme mail should be horrible !")
44 return conf_as_json
46# TODO VR 28/0/24 en fait on aurait bien envie de faire les format string après avoir instancier !
47def formal_conf_aux(conf_as_json, lss = None, io_datou_exec = {},
48 list_key_word = ["safia.audit.col", "safia.datou.io", "safia.datou.param", "saxia.config.user", "saxia.config.project", "format.string", "exec.func", "template.jinja"],
49 nb_level_instantiation = 16,
50 hit = None,
51 project_id = None): # 10, c'est déjà beaucoup
52 """
53 This function is a wrapper for the formal configuration of the library.
54 It is meant to be called by the user to set the configuration of the library.
55 """
56 conf_instanciated = conf_as_json
57 for i in range(nb_level_instantiation):
58 list_data_referenced_and_key = collect_data_to_reference_key_word(conf_as_json, list_key_word = list_key_word)
59 count_collected_data_not_format_string_type = sum(map(lambda x : 0 if "type" in x["data"] and x["data"]["type"] == "format.string" else 1, list_data_referenced_and_key))
60 # VR 25-12-24 : ce code n'est vraiment pas adapté, on ne permet que les format string au debug, je ne sais plus pourquoi j'ai fais cela, à priori c'est rempolacer par le code plus intelligent qui récupère les feuilles d'abord
61 # Par contre on ne sait pas ce qu'il se passe lorsqu'a l'étape prepare les feuilles sont ignoré
62 #if count_collected_data_not_format_string_type == 0 and i > 0 or (len(list_data_referenced_and_key) == 0 and i == 0): # Ce n'est pas exactement, il faudrait faire les string puis les autres
63# En fait j'ai enlevé les commentaires au dessus dans la V0.5.85 en décembre 2024 mais je ne sais pas ce que je voulais faire, est-ce qu'il y a d'autres problèmes avec ce test un peu complexe ?
64 # Je ne crois vraiment pas car les autres types d'instanciations rajouter en novembre 2024 ne sont pas considéré ici
65 if count_collected_data_not_format_string_type == 0 and i > 0 or (len(list_data_referenced_and_key) == 0 and i == 0): # Ce n'est pas exactement, il faudrait faire les string puis les autres
66# if count_collected_data_not_format_string_type == 0:
67 print("Number level instanciation when it was finished : " + str(i))
68 break
69 for data_referenced_and_key in list_data_referenced_and_key:
70 conf_with_reference_to_add_context = data_referenced_and_key["data"]
71 sub_conf_instanciated, key_input = get_referenced_conf(conf_with_reference_to_add_context, data_referenced_and_key["key"], io_datou_exec = io_datou_exec, lss = lss, hit = hit, project_id = project_id)
72 from lib.manaudit.lib_datou_audit import load_sub_json
73 key_output = data_referenced_and_key["key"]
74# sub_conf_in = load_sub_json(sub_conf_instanciated, key_input)
75 sub_conf_out = load_sub_json(conf_instanciated, key_output)
76 #if type(sub_conf_instanciated) == dict:
77 # from pydantic.utils import deep_update
78 # sub_conf_out = deep_update(sub_conf_instanciated, sub_conf_out)
79 #else:
80 from lib.manaudit.lib_datou_audit import set_sub_json
81 if key_output == "":
82 conf_instanciated = sub_conf_instanciated
83 else:
84 set_sub_json(conf_instanciated, key_output, sub_conf_instanciated)
85 if i == nb_level_instantiation - 1:
86 print("WARNING : Number level instanciation reached the limit : " + str(nb_level_instantiation) + " maybe instanciation wasn't done correctly !")
87 return conf_instanciated
89# VR TODO DOC : différencier les noms des variables key selon qu'elle designe des champ à récupérer ou la clef concatener du champ courant qu'on essaie d'instancier
90# VR 28/0/24 Je ne comprends pas comment les array sont gérer avec les clef definissant leur position et donc ou on doit cherche les informations
91# Sans doute tout n'est pas fonctionnel dans lib_conf_system
92# Mais a mon avis on ne veut pas gérer le fait qu'une configuration référencé ne le soit que dans un champ d'un array
93# Et du coup, j'aurai dit que le deeupdate suffit pour le reste et qu'il n'est pas nécessaire de gérer les array, mais du coup je ne comprends pas !
94# TODO OPTIMISATION REFACTO : pour ne récupérer qu'une fois (par niveau de référencement) les données d'un projet ou des datous par exemple, il faudrait avoir des index pour chacun de leur type, on ne va pas le faire immédiatement et assumer cette inefficience
95def collect_data_to_reference_key_word(conf_as_json,
96 list_key_word = ["safia.audit.col", "safia.datou.io", "safia.datou.param", "saxia.config.user", "saxia.config.project", "format.string", "exec.func"],
97 current_sub_key = ""):
98 """
99 Find the keywords and param
100 """
101 list_data_referenced_and_key = []
102 # TODO
103 found_key_word = False
104 if type(conf_as_json) not in [list, dict]:
105 return []
106 if type(conf_as_json) == list:
107 # load_sub_json manage int so it would work
108 instance_json_arr = []
109 for i, sub_json in enumerate(conf_as_json):
110 list_data_referenced_and_key_aux = collect_data_to_reference_key_word(sub_json, list_key_word, current_sub_key + "/" + str(i))
111 list_data_referenced_and_key.extend(list_data_referenced_and_key_aux)
113 data_referenced_with_key_current = None
114 if "type" in conf_as_json and conf_as_json["type"] in list_key_word:
115 found_key_word = True
116 data_referenced_with_key_current = {"key" : current_sub_key, "data" : conf_as_json}
117# list_data_referenced_and_key.append(data_referenced_with_key_current)
119 # For Migration, will be deprecated, managing old type of config, until vav expected 15/12/24, where key is the type of reference and instanciation
120 found_key_word_deprecated = False
121 if not found_key_word:
122 for k in conf_as_json:
123 if k in list_key_word:
124 print("DEPRECATED until vav expected 15/12/24 : MIGRATION SOFT TRASNITION TO BE DOCUMENTED ")
125# conf_as_json = conf_as_json[k]
126 conf_as_json[k]["type"] = k
127 # This makes management in instantiation in later function useless
128 # key input and output are the same if not specified in input
129 # if key is specified in sub_json, it will be used in instantiation instead of the current key
130 if "key" not in conf_as_json[k]:
131 conf_as_json[k]["key"] = current_sub_key
132 # This next key is the output key
133 data_referenced_with_key_current = {"key" : current_sub_key, "data" : conf_as_json[k]}
134 # VR 1-12 : we add it here because inner element of dict isn't the correct one to use => it is not clear what should be the behavior then, but we will test
135 list_data_referenced_and_key.append(data_referenced_with_key_current)
136 found_key_word_deprecated = True
138 found_key_word = True
139 # Multiple type would make all of this fail !
140 break
142 # if not found_key_word:
143# for k in conf_as_json:
144 if not found_key_word_deprecated:
145 for k, sub_json_or_key in enumerate(conf_as_json):
146 if type(conf_as_json) == list:
147 key = k
148 data = sub_json_or_key
149 else : # should be dict by now
150 key = sub_json_or_key
151 data = conf_as_json[sub_json_or_key]
152 list_data_referenced_and_key_aux = collect_data_to_reference_key_word(data, list_key_word, current_sub_key + "/" + str(key))
153 list_data_referenced_and_key.extend(list_data_referenced_and_key_aux)
155 if data_referenced_with_key_current != None :
156 # We should add this condition for exec.func, format.string (when not using specific loop on variable)
157 if len(list_data_referenced_and_key) == 0:
158 list_data_referenced_and_key.append(data_referenced_with_key_current)
159 else :
160 print("WARNING TREATED AS ERROR ON 30-11-24 LFC ORDER EXECUTION => if problem we could revert back by removing the condition to instanciate, but it should be harmless and useful to have this order of execution !")
161 if "type" in conf_as_json:
162 print(" WARNING TREATED : " + str(conf_as_json["type"]))
164 return list_data_referenced_and_key
166def format_string(format, map_var_val):
167 for var in map_var_val:
168 val = map_var_val[var]
169 format = format.replace("{" + var + "}", str(val))
170 return format
172# DOC : prend en entrée un seul json correspondant à une configuration à instancié à partir des elements référencés
173def get_referenced_conf(conf_with_reference_to_add_context,
174 sub_keys = None,
175 io_datou_exec = {},
176 lss = None,
177 hit = None,
178 project_id = None,
179 verbose = False):
180 """
181 Get the referenced configuration
182 """
183 # maybe duplicate with stuff above !
184 # manage compatibility while refacto migration listed as TODO
185 if "type" not in conf_with_reference_to_add_context:
186 print("DEPRECATED conf, where the type is the only key if more we are in trouble")
187 if len(conf_with_reference_to_add_context.keys()) > 1:
188 print("WARNING : more than one key in the conf and no type specified, we haven't choose a default behaovior, we are in trouble, it should exit later without correct configuration !")
189 if len(conf_with_reference_to_add_context.keys()) == 0:
190 print("ERROR : no key in the conf, we are in trouble, it could exit later without correct configuration")
191 return conf_with_reference_to_add_context
192 conf_with_reference_to_add_context["type"] = list(conf_with_reference_to_add_context.keys())[0]
193 type_formal_config = conf_with_reference_to_add_context["type"] if "type" in conf_with_reference_to_add_context else list(conf_with_reference_to_add_context.keys())[0]
195 conf_instanciated = None
196 if "key" in conf_with_reference_to_add_context:
197 sub_keys = conf_with_reference_to_add_context["key"]
199 if type_formal_config == "saxia.config.project":
200 project_id = conf_with_reference_to_add_context["project_id"] if "project_id" in conf_with_reference_to_add_context else project_id
201 if project_id == None:
202 list_projects = lss.get_project()
203 if len(list_projects) > 0 and "id" in list_projects[0]:
204 project_id = list_projects[0]["id"]
205 from server.safia import lpgss_singleton
206 project_configuration_referenced = lpgss_singleton.load_conf_project(project_id)
207 from lib.manaudit.lib_datou_audit import load_sub_json
208 if sub_keys != None:
209 conf_instanciated = load_sub_json(project_configuration_referenced, sub_keys.lstrip("/"))
210 else :
211 conf_instanciated = project_configuration_referenced
213# TODO use this or delete these comment
214# from pydantic.utils import deep_update
215# conf_instanciated = deep_update(conf_with_reference_to_add_context, project_configuration_referenced_sub_key)
217 elif type_formal_config == "saxia.config.user":
219 print("TODO in LFC l 184")
221 user_info = lss.get_user_info()
222 if "key" in conf_with_reference_to_add_context:
223 key = conf_with_reference_to_add_context["key"]
224 if "/" in key:
225 from lib.manaudit.lib_datou_audit import load_sub_json
226 conf_instanciated = load_sub_json(user_info, key.lstrip("/"))
227 elif key in user_info:
228 conf_instanciated = user_info[key]
229 else:
230 conf_instanciated = {}
231 else :
232 conf_instanciated = {}
233 elif type_formal_config == "safia.audit.col":
235# sub_keys = None
236 print("test in LFC")
237 if "col_source" not in conf_with_reference_to_add_context:
238 print("Missing col for type safia.audit.col, avoiding the instanciation of the configuration")
239 return None, None
240 col_in = conf_with_reference_to_add_context["col_source"]
241 if hit == None:
242 print("Missing hash_id_treatment for type safia.audit.col, avoiding the instanciation of the configuration")
243 return None, None
244 list_col_as_csv = str(col_in)
245 data_audit = lss.lib_user_data_internal.load_data_audit(hash_id_treatment=hit, col_csv=list_col_as_csv, limit=1)
247 conf_instanciated = None
248 if data_audit != None and col_in in data_audit:
249 if ("key" in conf_with_reference_to_add_context and
250 type(data_audit[col_in]) == dict and
251 conf_with_reference_to_add_context["key"] in data_audit[col_in]):
252 conf_instanciated = data_audit[col_in][conf_with_reference_to_add_context["key"]]
253 else:
254 conf_instanciated = data_audit[col_in]
256 if conf_instanciated != None and sub_keys != None and type(conf_instanciated) == dict:
257 from lib.manaudit.lib_datou_audit import load_sub_json
258 conf_instanciated = load_sub_json(conf_instanciated, sub_keys)
260 elif type_formal_config == "safia.datou.io":
261 if "key" in conf_with_reference_to_add_context :
262 key = conf_with_reference_to_add_context["key"]
263 if key in io_datou_exec:
264 conf_instanciated = io_datou_exec[key]
265 else :
266 conf_instanciated = {}
267 else :
268 conf_instanciated = {}
269 elif type_formal_config == "safia.datou.param":
270 print("TODO in LFC l 228")
271 conf_referenced = lss.get_datou_param(conf_with_reference_to_add_context["datou_param_id"])
272 elif type_formal_config == "format.string":
273 print("TODO in LFC l 231")
274 map_variables = conf_with_reference_to_add_context["variables"] if "variables" in conf_with_reference_to_add_context else []
275 map_var_val = {}
276 format = conf_with_reference_to_add_context["format"] if "format" in conf_with_reference_to_add_context else ""
278 if type(map_variables) == list:
279 print("WILL FAIL !")
281 for variable in map_variables:
282 var = variable #["name"] if "name" in variable else "default_variable_name"
283 val = map_variables[variable]["val"] if map_variables[variable] != None and "val" in map_variables[variable] else map_variables[variable]
284 type_var = map_variables[variable]["type"] if map_variables[variable] != None and type(map_variables[variable]) == dict and "type" in map_variables[variable] else None
285 if type_var != None:
286 print("DEPRECATE LFC 271")
287 val, _ = get_referenced_conf(map_variables[variable],
288 sub_keys = None,
289 io_datou_exec = io_datou_exec,
290 lss = lss)
292 map_var_val[var] = val
293 ret = format_string(format, map_var_val)
294 return ret, None
295 elif type_formal_config == "exec.func":
296 sub_keys = None
297 func = conf_with_reference_to_add_context["func"] if "func" in conf_with_reference_to_add_context else None
298 variables = conf_with_reference_to_add_context["variables"] if "variables" in conf_with_reference_to_add_context else conf_with_reference_to_add_context
300 if func == "load.df.fromjsonasstring":
301 print("One variable needed : df_as_string !")
302 if "df_as_json" in variables:
303 df_as_json = variables["df_as_json"]
304 import pandas as pd
305 if type(df_as_json) == type(None):
306 print("ERROR : df_as_json is None, we are in trouble, it could exit later without correct configuration")
307 conf_instanciated = "None"
308 conf_instanciated = pd.read_json(df_as_json)
309 if func == "extract.df.col":
310 print("Two variable needed : df, col !")
311 if "df" in variables and "col" in variables:
312 df = variables["df"]
313 col = variables["col"]
314 import pandas as pd
315 if type(df) != pd.DataFrame or not hasattr(df, "columns"):
316 print("ERROR : df is not a pandas DataFrame with columns, we are in trouble, it could exit later without correct configuration")
317 return conf_with_reference_to_add_context, sub_keys
318 if col in df.columns:
319 conf_instanciated_tuple = df[col]
320 conf_instanciated = str(conf_instanciated_tuple)
321 col_as_txt = ""
322 for index, row in df.iterrows():
323 col_as_txt += "Doc id : " + str(index) + " : " + str(row[col]) + "\n"
324 conf_instanciated = col_as_txt
325 else :
326 print(" col : " + str(col) + " not in df.columns : " + str(df.columns))
327 conf_instanciated_tuple = None
328 conf_instanciated = str(conf_instanciated_tuple)
329 if func == "var":
330 sub_type = conf_with_reference_to_add_context["sub_type"] if "sub_type" in conf_with_reference_to_add_context else None
332 key_ext = conf_with_reference_to_add_context["key_ext"] if "key_ext" in conf_with_reference_to_add_context else None
333 key_int = conf_with_reference_to_add_context["key_int"] if "key_int" in conf_with_reference_to_add_context else None
334 key_or_data = conf_with_reference_to_add_context["key_or_data"] if "key_or_data" in conf_with_reference_to_add_context else None
336 input = conf_with_reference_to_add_context["input"] if "input" in conf_with_reference_to_add_context else None
338 if key_ext != None:
339 from lib.manaudit.lib_datou_audit import load_sub_json
340 input = load_sub_json(input, key_ext)
342 if sub_type == "arr":
343 conf_instanciated = []
344 print("TODO arr func exec.fun VAR")
345 for k in input :
346 data = input[k]
347 if key_int != None and key_int != '': # pas possible avec key_or_data
348 from lib.manaudit.lib_datou_audit import load_sub_json
349 data = load_sub_json(data, key_int)
350 if key_or_data:
351 data = k
352 conf_instanciated.append(data)
354 elif sub_type == "dict":
355 print("TODO dict func exec.fun VAR")
356 else :
357 print("ERROR : sub_type not managed, we are in trouble, it could exit later without correct configuration")
358 if func == "htmlencode":
359 print("ERROR : func not specified, we are in trouble, it could exit later without correct configuration")
360 import urllib
361 if "input_encode" in variables and type(variables["input_encode"]) == str:
362 input_encode = variables["input_encode"]
363 conf_instanciated = urllib.parse.quote(input_encode)
364# if type(conf_with_reference_to_add_context) == str:
365# conf_instanciated = urllib.parse.quote_plus(conf_with_reference_to_add_context)
366 else:
367 print("ERROR : func htmlencode need a string in input in input_encode, we are in trouble, it could exit later without correct configuration : " + str(variables))
368 return conf_with_reference_to_add_context, sub_keys
369 return conf_instanciated, sub_keys
371 elif type_formal_config == "template.jinja":
373 try :
374 template_jinja_file = conf_with_reference_to_add_context["template_jinja_file"] if "template_jinja_file" in conf_with_reference_to_add_context else "templates/email/admin_end_auto_saxia.html"
375 email_data = conf_with_reference_to_add_context["email_data"] if "email_data" in conf_with_reference_to_add_context else {"text_content": "Empty content !"}
377 # Read the Jinja2 email template
378 with open(template_jinja_file, "r") as file:
379 template_str = file.read()
381 from jinja2 import Template
382 jinja_template = Template(template_str)
383 email_content = jinja_template.render({"email_data" : email_data})
384 conf_instanciated = email_content
385 except Exception as e:
386 print(str(e))
387 print("ERror in templating jinja in LFC")
388 else:
389 print("ERROR : type not managed, we are in trouble, it could exit later without correct configuration")
390 return conf_with_reference_to_add_context, sub_keys
392 return conf_instanciated, sub_keys