Coverage for lib/lib_safia.py: 33%

133 statements  

« prev     ^ index     » next       coverage.py v7.9.1, created at 2026-02-10 01:10 +0100

1# Functionnal function to use safia 

2import json 

3import os 

4import shutil 

5 

6 

7# plop 

8 

9def parse_json_log_queries(res): 

10 res_output = [] 

11 for r in res: 

12 try : 

13 misc_info_str = r["detailed_time"] 

14 misc_info_str = misc_info_str.replace("ImmutableMultiDict(", "") 

15 misc_info_str = misc_info_str.replace(")]),", ")],") 

16 misc_info_str = misc_info_str.replace('(', '[').replace(')', ']') 

17 misc_info_str = misc_info_str.replace("False", "\"false\"") 

18 misc_info_str = misc_info_str.replace("True", "\"true\"") 

19 misc_info_json = json.loads(misc_info_str, object_pairs_hook=None) # {"ImmutableMultiDict" : None}) 

20 res_output.append(misc_info_json) 

21 except Exception as e: 

22 print(str(e)) 

23 print(misc_info_str) 

24 

25 return res_output 

26def parse_tsne_for_latex(res): 

27 res = parse_json_log_queries(res) 

28 

29 group_by = 10 

30 count = 0 

31 dir_latex = "graph/tsne_030923" 

32 

33 latex_code = "" 

34 list_file_names = [] 

35 

36 for r in res : 

37 inputs = r["inputs"] 

38 filename = r["filename"] 

39 

40 local_filename = os.path.basename(filename) 

41 print(local_filename) 

42 list_file_names.append(local_filename) 

43 

44 if count % 10 == 0: 

45 if count > 0: 

46 print("\end{figure}") 

47 latex_code += "\begin{figure}[!htb]" 

48 latex_code += '\centering' 

49 

50 latex_code += "\begin{subfigure}[b]{0.3\textwidth}" 

51 

52 list_args_for_http = [str(r) + "=" + str(inputs[r]) for r in inputs] 

53 input_as_arg = ",".join(list_args_for_http) 

54 

55 latex_code += "\caption{\\url{https://safia.app/kmean?" +str(input_as_arg)+ "}}" 

56 latex_code += "\includegraphics[width=\textwidth]{graph/tsne_030923/" + local_filename + "}" 

57 latex_code += "\end{subfigure}" 

58 count = count + 1 

59 

60# if count % 10 == 0: 

61# print("\end{figure}") 

62 latex_code += "\end{figure}" 

63 

64 return {"latex" : latex_code, "list" : list_file_names} 

65# print(latex_code) 

66 

67def safia_desc_export(project_id, 

68 lpgss=None, 

69 verbose = False, 

70 out_folder="temp", 

71 photo_desc_type = 1888, 

72 offset_media_id = 1000000, 

73 offset_data_file_id = 0, 

74 type_data = "float", 

75 limit = 0, 

76 dim_input = 0, 

77 batch_size_insert_fvs = 0): 

78 offset_media_id_input_value_for_list_assoc = offset_media_id 

79 from lib.lib_safia_system import LibSafiaSystem 

80 

81 lss = LibSafiaSystem(lib_user_data_internal=lpgss) 

82 

83 list_id_local_desc_to_project_id_doc=[] 

84 

85 map_descs = [] 

86 if batch_size_insert_fvs == 0: 

87 print(" batch_size_insert_fvs == 0") 

88 map_descs = lss.load_descs(project_id=project_id, limit = limit) 

89 else : 

90 print(" batch_size_insert_fvs : " + str(batch_size_insert_fvs)) 

91 offset = 0 

92 while offset < limit: 

93 if offset + batch_size_insert_fvs > limit: 

94 limit_aux_used = limit - offset 

95 else : 

96 limit_aux_used = batch_size_insert_fvs 

97 map_descs_aux = lss.load_descs(project_id=project_id, limit = limit_aux_used, offset = offset) 

98 print(" batch_size_insert_fvs with offset : " + str(offset) + " and limit_aux_used : " + str(limit_aux_used)) 

99 offset = offset + batch_size_insert_fvs 

100 map_descs = map_descs + map_descs_aux 

101 

102 for d in map_descs: 

103 list_id_local_desc_to_project_id_doc.append((project_id, d["id"], d["content"], offset_media_id_input_value_for_list_assoc)) 

104 offset_media_id_input_value_for_list_assoc += 1 

105 

106 from lib.lib_util import write_as_float#, write_as_tiny_int 

107 filename_prefix = os.path.join(out_folder, "data_fvs/binary" + str(photo_desc_type) + "/") 

108 # - [ ] TODO VR 17-9 move this code outside this function 

109 # - [ ] TODO VR 17-9 comment the two following line and study this strange bug by calling multiple time this : http://localhost:4999/kmean?pids=8,2&limit=2000&dim=32&launch_fvs=1&build_fvs_desc=1&verbose=1&save_cluster=desc_32_loc_500_step&load_cluster=&nb_step=1 

110# if os.path.exists(filename_prefix): 

111# shutil.rmtree(filename_prefix) 

112 if not os.path.exists(filename_prefix): 

113 os.makedirs(filename_prefix) 

114 number_dimension = 0 

115 if type_data == "float" or type_data == "tinyint": 

116 number_dimension_out, offset_media_id, offset_data_file_id = write_as_float(list_data_map=map_descs, filename_prefix=filename_prefix, 

117 offset_media_id = offset_media_id, 

118 offset_data_file_id = offset_data_file_id, 

119 photo_desc_type=photo_desc_type, 

120 type_data=type_data, 

121 dim_input = dim_input) 

122 if number_dimension == 0: 

123 number_dimension = number_dimension_out 

124 elif number_dimension != number_dimension_out: 

125 print(" Unexpected different dimension, ignoring ! number_dimension : " + str(number_dimension) + " " + str(number_dimension_out)) 

126 else : 

127 print("type_data " + str(type_data) + " not supported, only float and tinyint, ERROR NOTHING EXPORTED, NO DESC WRITTEN") 

128 return offset_media_id, offset_data_file_id, 0, [] 

129 

130 print("Tentative conf_file : ") 

131 print(f"0,0,{number_dimension},0,0") 

132 

133 print(" write_as_float exported !") 

134 return offset_media_id, offset_data_file_id, number_dimension, list_id_local_desc_to_project_id_doc 

135 

136# TODO manage recursive zip ? GRRR 

137def safia_import(in_file, 

138 temp_folder = "temp", 

139 managed_extension = [".pages", ".pptx", ".mp3", ".ogg", ".amr", ".mp4", ".m4a", ".wav", ".jpeg", ".jpg", ".png", ".pdf", ".txt", ".md", ".py", ".cuda", ".cpp", ".docx", ".json", ".zip", ".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tbz"], 

140 verbose = False): 

141 

142 from lib.import_util.lib_path_to_vec import list_files, load_files, write_json_index_for_import, unzip_files 

143 

144 index, list = list_files(in_file, managed_zip_extension_to_avoid = []) 

145 

146 if verbose : 

147 print(" index : " + str(index)) 

148 print(" list : " + str(list)) 

149 

150 managed_zip_extension = [".zip", ".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tbz"] 

151 list_files_zip = [] 

152 for l in list: 

153 is_zip = False 

154 if l["extension"].lower() in managed_zip_extension: 

155 list_files_zip.append(l) 

156 

157 if len(list_files_zip) > 0: 

158 if len(list_files_zip) > 1: 

159 print("Internal error !") 

160 unzip_files(list_files_zip, managed_zip_extension) 

161 # now relist by avoiding zip 

162 # where we expect unzip to have work 

163 dirname_in_file = os.path.dirname(in_file) 

164 index, list = list_files(dirname_in_file, managed_zip_extension_to_avoid = managed_zip_extension) 

165 

166 

167 import logging 

168 logger = logging.getLogger() 

169 logger.info(" before load_files : list : " + str(list)) 

170 # datou_step 

171 data_for_retrieval, list_detailed_time, sorted_dict_unmanaged_extension = load_files(list, managed_extension, verbose = verbose) 

172 

173 # os.path.basename(in_file) ou in_file.replace("/", "_") ?? 

174 

175 out_file_json = os.path.join(temp_folder, os.path.basename(in_file) + ".json") 

176 if not os.path.exists(os.path.dirname(out_file_json)): 

177 os.makedirs(os.path.dirname(out_file_json)) 

178 

179 write_json_index_for_import(data_for_retrieval, 

180 out_file_json, 

181 verbose = False) 

182 

183# list_detailed_time 

184 print("out_file_json : " + str(out_file_json)) 

185 return out_file_json, list_detailed_time, sorted_dict_unmanaged_extension 

186 

187def insert_query_log(lss, endpoint, request, user_id, project_id, elapsed_time): 

188 if project_id == -1: 

189 return -1 

190 import logging 

191 logger = logging.getLogger() 

192 

193 logger.info("prepare all the needed info from the request") 

194 print("TODO : prepare all the needed info from the request ! ") 

195 logger.info(str(request)) 

196 

197 ip_address = request.remote_addr 

198 request_method = request.method 

199 if request_method == "GET": 

200 logger.info("GET") 

201 logger.info(str(request.args)) 

202 request_body = request.args["input"] if "input" in request.args else str(request.args) 

203 logger.info(str(request_body)) 

204 else: 

205 print("Need to check what to do with POST request") 

206 request_body = str(request.args) + str(request.files) 

207 

208 logger.info("Before lss.insert_query_log request_body :" + str(request_body)) 

209 query_id = lss.insert_query_log(user_id = user_id, project_id = project_id, endpoint = endpoint, 

210 request_method = request_method, request_body = request_body, ip_address = ip_address, 

211 elapsed_time = elapsed_time) 

212 

213 return query_id 

214 

215 

216 

217