Coverage for lib/lib_safia.py: 33%
133 statements
« prev ^ index » next coverage.py v7.9.1, created at 2026-02-10 01:10 +0100
« prev ^ index » next coverage.py v7.9.1, created at 2026-02-10 01:10 +0100
1# Functionnal function to use safia
2import json
3import os
4import shutil
7# plop
9def parse_json_log_queries(res):
10 res_output = []
11 for r in res:
12 try :
13 misc_info_str = r["detailed_time"]
14 misc_info_str = misc_info_str.replace("ImmutableMultiDict(", "")
15 misc_info_str = misc_info_str.replace(")]),", ")],")
16 misc_info_str = misc_info_str.replace('(', '[').replace(')', ']')
17 misc_info_str = misc_info_str.replace("False", "\"false\"")
18 misc_info_str = misc_info_str.replace("True", "\"true\"")
19 misc_info_json = json.loads(misc_info_str, object_pairs_hook=None) # {"ImmutableMultiDict" : None})
20 res_output.append(misc_info_json)
21 except Exception as e:
22 print(str(e))
23 print(misc_info_str)
25 return res_output
26def parse_tsne_for_latex(res):
27 res = parse_json_log_queries(res)
29 group_by = 10
30 count = 0
31 dir_latex = "graph/tsne_030923"
33 latex_code = ""
34 list_file_names = []
36 for r in res :
37 inputs = r["inputs"]
38 filename = r["filename"]
40 local_filename = os.path.basename(filename)
41 print(local_filename)
42 list_file_names.append(local_filename)
44 if count % 10 == 0:
45 if count > 0:
46 print("\end{figure}")
47 latex_code += "\begin{figure}[!htb]"
48 latex_code += '\centering'
50 latex_code += "\begin{subfigure}[b]{0.3\textwidth}"
52 list_args_for_http = [str(r) + "=" + str(inputs[r]) for r in inputs]
53 input_as_arg = ",".join(list_args_for_http)
55 latex_code += "\caption{\\url{https://safia.app/kmean?" +str(input_as_arg)+ "}}"
56 latex_code += "\includegraphics[width=\textwidth]{graph/tsne_030923/" + local_filename + "}"
57 latex_code += "\end{subfigure}"
58 count = count + 1
60# if count % 10 == 0:
61# print("\end{figure}")
62 latex_code += "\end{figure}"
64 return {"latex" : latex_code, "list" : list_file_names}
65# print(latex_code)
67def safia_desc_export(project_id,
68 lpgss=None,
69 verbose = False,
70 out_folder="temp",
71 photo_desc_type = 1888,
72 offset_media_id = 1000000,
73 offset_data_file_id = 0,
74 type_data = "float",
75 limit = 0,
76 dim_input = 0,
77 batch_size_insert_fvs = 0):
78 offset_media_id_input_value_for_list_assoc = offset_media_id
79 from lib.lib_safia_system import LibSafiaSystem
81 lss = LibSafiaSystem(lib_user_data_internal=lpgss)
83 list_id_local_desc_to_project_id_doc=[]
85 map_descs = []
86 if batch_size_insert_fvs == 0:
87 print(" batch_size_insert_fvs == 0")
88 map_descs = lss.load_descs(project_id=project_id, limit = limit)
89 else :
90 print(" batch_size_insert_fvs : " + str(batch_size_insert_fvs))
91 offset = 0
92 while offset < limit:
93 if offset + batch_size_insert_fvs > limit:
94 limit_aux_used = limit - offset
95 else :
96 limit_aux_used = batch_size_insert_fvs
97 map_descs_aux = lss.load_descs(project_id=project_id, limit = limit_aux_used, offset = offset)
98 print(" batch_size_insert_fvs with offset : " + str(offset) + " and limit_aux_used : " + str(limit_aux_used))
99 offset = offset + batch_size_insert_fvs
100 map_descs = map_descs + map_descs_aux
102 for d in map_descs:
103 list_id_local_desc_to_project_id_doc.append((project_id, d["id"], d["content"], offset_media_id_input_value_for_list_assoc))
104 offset_media_id_input_value_for_list_assoc += 1
106 from lib.lib_util import write_as_float#, write_as_tiny_int
107 filename_prefix = os.path.join(out_folder, "data_fvs/binary" + str(photo_desc_type) + "/")
108 # - [ ] TODO VR 17-9 move this code outside this function
109 # - [ ] TODO VR 17-9 comment the two following line and study this strange bug by calling multiple time this : http://localhost:4999/kmean?pids=8,2&limit=2000&dim=32&launch_fvs=1&build_fvs_desc=1&verbose=1&save_cluster=desc_32_loc_500_step&load_cluster=&nb_step=1
110# if os.path.exists(filename_prefix):
111# shutil.rmtree(filename_prefix)
112 if not os.path.exists(filename_prefix):
113 os.makedirs(filename_prefix)
114 number_dimension = 0
115 if type_data == "float" or type_data == "tinyint":
116 number_dimension_out, offset_media_id, offset_data_file_id = write_as_float(list_data_map=map_descs, filename_prefix=filename_prefix,
117 offset_media_id = offset_media_id,
118 offset_data_file_id = offset_data_file_id,
119 photo_desc_type=photo_desc_type,
120 type_data=type_data,
121 dim_input = dim_input)
122 if number_dimension == 0:
123 number_dimension = number_dimension_out
124 elif number_dimension != number_dimension_out:
125 print(" Unexpected different dimension, ignoring ! number_dimension : " + str(number_dimension) + " " + str(number_dimension_out))
126 else :
127 print("type_data " + str(type_data) + " not supported, only float and tinyint, ERROR NOTHING EXPORTED, NO DESC WRITTEN")
128 return offset_media_id, offset_data_file_id, 0, []
130 print("Tentative conf_file : ")
131 print(f"0,0,{number_dimension},0,0")
133 print(" write_as_float exported !")
134 return offset_media_id, offset_data_file_id, number_dimension, list_id_local_desc_to_project_id_doc
136# TODO manage recursive zip ? GRRR
137def safia_import(in_file,
138 temp_folder = "temp",
139 managed_extension = [".pages", ".pptx", ".mp3", ".ogg", ".amr", ".mp4", ".m4a", ".wav", ".jpeg", ".jpg", ".png", ".pdf", ".txt", ".md", ".py", ".cuda", ".cpp", ".docx", ".json", ".zip", ".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tbz"],
140 verbose = False):
142 from lib.import_util.lib_path_to_vec import list_files, load_files, write_json_index_for_import, unzip_files
144 index, list = list_files(in_file, managed_zip_extension_to_avoid = [])
146 if verbose :
147 print(" index : " + str(index))
148 print(" list : " + str(list))
150 managed_zip_extension = [".zip", ".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tbz"]
151 list_files_zip = []
152 for l in list:
153 is_zip = False
154 if l["extension"].lower() in managed_zip_extension:
155 list_files_zip.append(l)
157 if len(list_files_zip) > 0:
158 if len(list_files_zip) > 1:
159 print("Internal error !")
160 unzip_files(list_files_zip, managed_zip_extension)
161 # now relist by avoiding zip
162 # where we expect unzip to have work
163 dirname_in_file = os.path.dirname(in_file)
164 index, list = list_files(dirname_in_file, managed_zip_extension_to_avoid = managed_zip_extension)
167 import logging
168 logger = logging.getLogger()
169 logger.info(" before load_files : list : " + str(list))
170 # datou_step
171 data_for_retrieval, list_detailed_time, sorted_dict_unmanaged_extension = load_files(list, managed_extension, verbose = verbose)
173 # os.path.basename(in_file) ou in_file.replace("/", "_") ??
175 out_file_json = os.path.join(temp_folder, os.path.basename(in_file) + ".json")
176 if not os.path.exists(os.path.dirname(out_file_json)):
177 os.makedirs(os.path.dirname(out_file_json))
179 write_json_index_for_import(data_for_retrieval,
180 out_file_json,
181 verbose = False)
183# list_detailed_time
184 print("out_file_json : " + str(out_file_json))
185 return out_file_json, list_detailed_time, sorted_dict_unmanaged_extension
187def insert_query_log(lss, endpoint, request, user_id, project_id, elapsed_time):
188 if project_id == -1:
189 return -1
190 import logging
191 logger = logging.getLogger()
193 logger.info("prepare all the needed info from the request")
194 print("TODO : prepare all the needed info from the request ! ")
195 logger.info(str(request))
197 ip_address = request.remote_addr
198 request_method = request.method
199 if request_method == "GET":
200 logger.info("GET")
201 logger.info(str(request.args))
202 request_body = request.args["input"] if "input" in request.args else str(request.args)
203 logger.info(str(request_body))
204 else:
205 print("Need to check what to do with POST request")
206 request_body = str(request.args) + str(request.files)
208 logger.info("Before lss.insert_query_log request_body :" + str(request_body))
209 query_id = lss.insert_query_log(user_id = user_id, project_id = project_id, endpoint = endpoint,
210 request_method = request_method, request_body = request_body, ip_address = ip_address,
211 elapsed_time = elapsed_time)
213 return query_id