Coverage for test/main_test.py: 73%

828 statements  

« prev     ^ index     » next       coverage.py v7.9.1, created at 2026-02-18 02:40 +0100

1# -*- coding: utf-8 -*- 

2__author__ = 'moilerat' 

3 

4import shutil 

5import time 

6 

7# 

8# # Safia copyright 2022-2023 VR - LICENSETOBEDEFINED : on premise treatment custom 

9# 

10#! Package test main_test.py 

11#! to be cleaned, but it is working 

12# 

13 

14from flask_testing import TestCase 

15from server.safia import app # import your Flask application 

16import json 

17 

18import logging 

19import pytest 

20import asyncio 

21import os, sys 

22 

23#sys.path.append(os.path.join(os.getenv("GITVELOURS"), "c_visual_search/pyfvs/install")) 

24if "GITPYFVS" in os.environ: 

25 sys.path.append(os.getenv("GITPYFVS")) 

26 

27project_nrt = 1 

28#project_nrt = 3 

29nrt_user_id = 1 

30 

31from lib.lib_safia_system import LibSafiaSystem 

32from auth.lib_conf_system import LibConfSystem 

33lcs = LibConfSystem() 

34(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT) = lcs.get_pg_conf() 

35OPENAI_API_KEY = lcs.get_openai_api_key() 

36email_user = "anonymous@opio.fr" 

37 

38from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys 

39lpgss_singleton = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT) 

40 

41from auth.lib_auth import LibUserDataExternalServices 

42lib_auth_singleton = LibUserDataExternalServices(lib_at = None) 

43 

44from auth.lib_right import LibRight 

45lib_right = LibRight(lib_internal_data=lpgss_singleton) 

46lss_singleton = LibSafiaSystem(lib_user_data_internal=lpgss_singleton, 

47 lib_user_data_external=lib_auth_singleton, 

48 lib_right=lib_right) 

49 

50 

51 

52class FirstClassSafiaTest(TestCase): 

53# def __init__(self): 

54# super(self) 

55# import logging 

56# self.logger = logging.getLogger() 

57# pass 

58 

59 

60# def __init__(self): 

61# super(self).__init__() 

62# import logging 

63# self.logger = logging.getLogger() 

64# self.logger.info("FirstClassSafiaTest init") 

65# # self.client = None 

66 

67 def setUp(self): 

68 print("VR 9-6-25 : a mon avis il faut créer un user ! :-) et ca ne marchera plus quand on aura besoin de récupérer le code 2FA") 

69 

70 username = "vic" 

71 # from server.bp.auth import create_login_bp.User 

72 # user = User.query.filter_by(username=username).first() 

73 from flask_login import login_user, UserMixin 

74 user = UserMixin() 

75 user.mtr_user_id = nrt_user_id 

76 user.id = nrt_user_id 

77 user.username = "victor@reutenauer.eu" 

78 login_user(user) 

79 

80 response = self.client.post("/login", 

81 # data={"username":"vic","password":"vic"}, 

82 data={"username": "moilerat", "password": "alaoui"}, 

83 content_type='multipart/form-data') 

84 

85 print(" Setup FirstClassSafiaTest " + str(type(self.client))) 

86 

87 

88 def tearDown(self): 

89 # _db.session.remove() 

90 # _db.drop_all() 

91 print(" tearDown FirstClassSafiaTest " + str(type(self.client))) 

92 

93 def create_app(self): 

94 app.config['TESTING'] = True 

95 return app 

96 

97 

98 

99 def test_aaa_first_action_truncate_table(self): 

100 import logging 

101 logger = logging.getLogger() 

102 logger.info(str(type(self.client))) 

103 

104 logger.info(" PG_DB : " + str(PG_DB)) 

105 

106 from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys 

107 lpgss = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT) 

108 

109 curr_project_id = project_nrt 

110 

111 logger.info(" Before get_project_info ") 

112 info_project = lss_singleton.get_project_info(curr_project_id) 

113 logger.info(" Before truncate_project_documents (useless s'il y a des document_id unique ! ) ") 

114 import time 

115 time.sleep(3) 

116 kill_all = True 

117 ret = lss_singleton.truncate_project_documents(curr_project_id, kill_all = kill_all) 

118 time.sleep(3) 

119 logger.info(" After truncate_project_documents ") 

120 

121 lpgss.add_cost_to_project(curr_project_id, -0.1274092) 

122 

123 assert ret == "TRUNCATE" 

124 

125 def test_example(self): 

126 response = self.client.get('/') 

127 self.assertEqual(response.status_code, 200) 

128 

129 print(" response : " + str(response)) 

130 

131 def test_documents_page(self, endpoint = "/docsmart", 

132 search_into = "<h1>Documents Smart</h1>", 

133 length_file = 1952, 

134 filename = None, 

135 save_comp = True): 

136 self.client.set_cookie( 'otp', "c6ae5f7095abfe9ac59e75fc6624cea56038d9da") 

137 self.client.set_cookie( 'project_id', str(project_nrt)) 

138 

139 if filename == None: 

140 filename = endpoint.lstrip("/") 

141 

142 response = self.client.get(endpoint) 

143 self.assertEqual(response.status_code, 200) 

144 

145 if save_comp : 

146 # Write the reference HTML content from the file 

147 filepath = os.path.join("test/data/comp", filename) 

148 if not os.path.exists("test/data/comp"): 

149 os.makedirs("test/data/comp") 

150 with open(filepath, 'w') as file: 

151 file.write(response.data.decode('utf-8')) 

152 

153 # Read the reference HTML content from the file 

154 filepath = os.path.join("test/data/ref", filename) 

155 with open(filepath, 'r') as file: 

156 reference_html = file.read() 

157 

158 import logging 

159 self.logger = logging.getLogger() 

160 self.logger.info(response.data) 

161 self.logger.info(len(response.data)) 

162 

163 # Compare the response data with the reference HTML 

164 response_decoded = response.data.decode('utf-8') 

165 if not save_comp: 

166 self.assertEqual(response_decoded, reference_html, "Ca va pas du tout : " + str(response_decoded) + " VERSUCE" + response.data.decode("utf-8")) 

167 

168 assert len(response.data) == length_file 

169 assert search_into in response.data.decode('utf-8') 

170 else : 

171 print(" len(response.data) : " + str(len(response.data))) 

172 

173 def test_connected_ok(self): 

174 self.client.set_cookie( 'otp', "c6ae5f7095abfe9ac59e75fc6624cea56038d9da") 

175 self.client.set_cookie( 'project_id', str(project_nrt)) 

176 

177 response = self.client.get('/api/v1/project/queries/' + str(project_nrt)) 

178 self.assertEqual(response.status_code, 200) 

179 response = self.client.post('/api/v1/project/select/' + str(project_nrt)) 

180 self.assertEqual(response.status_code, 200) 

181 response = self.client.get('/project/admin/' + str(project_nrt)) 

182 self.assertEqual(response.status_code, 200) 

183 

184 response = self.client.get('/') 

185 self.assertEqual(response.status_code, 200) 

186 response = self.client.get('/account') 

187 self.assertEqual(response.status_code, 200) 

188 response = self.client.get('/datou') 

189 self.assertEqual(response.status_code, 200) 

190 response = self.client.get('/datou?json') 

191 self.assertEqual(response.status_code, 200) 

192 response = self.client.get('/documents.html') 

193 self.assertEqual(response.status_code, 200) 

194 response = self.client.get('/upload') 

195 self.assertEqual(response.status_code, 200) 

196 response = self.client.get('/doc') 

197 self.assertEqual(response.status_code, 200) 

198 

199 response = self.client.get('/projects') 

200 self.assertEqual(response.status_code, 200) 

201 

202 # TODO test 

203 # - [ ] TODO VR on pourrait faire une boucle 

204 def test_unlogged_ok(self): 

205 response = self.client.get('/jscover') 

206 self.assertEqual(response.status_code, 200) 

207 if os.path.exists("temp_test_nrt_jscover"): 

208 import shutil 

209 shutil.rmtree("temp_test_nrt_jscover") 

210 response = self.client.post('/jscoverage-store/temp_test_nrt_jscover', content_type='application/json', data=json.dumps({"just":"test"})) 

211 self.assertEqual(response.status_code, 200) 

212 response = self.client.get('/suiviprod/pg/admin') 

213 self.assertEqual(response.status_code, 200) 

214 

215 response = self.client.post('/api/v1/form/getintouch', data=json.dumps({"just":"test"}), content_type='application/json') 

216 self.assertEqual(response.status_code, 200) 

217 self.assertEqual(response.json, {"Thank" : "you !"}) 

218 

219 response = self.client.get('/landing') 

220 self.assertEqual(response.status_code, 200) 

221 

222 response = self.client.get('/about') 

223 self.assertEqual(response.status_code, 200) 

224 response = self.client.get('/legal') 

225 self.assertEqual(response.status_code, 200) 

226 response = self.client.get('/privacy') 

227 self.assertEqual(response.status_code, 200) 

228 response = self.client.get('/terms') 

229 self.assertEqual(response.status_code, 200) 

230 response = self.client.get('/favicon.ico') 

231 self.assertEqual(response.status_code, 200) 

232# response = self.client.get('/zeitgeist.png') 

233# self.assertEqual(response.status_code, 200) 

234 

235 def test_logged_behavior_unlogged(self): 

236 response = self.client.get('/invitation/dummy_invitation_anyway') 

237 self.assertEqual(response.status_code, 200) 

238 self.you_have_no_valid = """ 

239 You have no valid OTP password or your are browsing anonymously. 

240 Please create an account on <a href="https://www.openpromptia.com">opia</a> or https://opia.softr.app https://opia.softr.app/apia/assistant#otp2pe 

241 Please create an account on <a href="https://reanna392.preview.softr.app/apia/assistant#otp2pe">opia</a> or https://reanna392.preview.softr.app/apia/assistant#otp2pe 

242 and get an OTP password or EXTEND ITS EXPIRY DATE ON <a href="https://reanna392.preview.softr.app/apia/assistant#otp2pe">here</a> (valid 24 hours). 

243 If you feel you got this message by error you can send and email to info@opio.fr . 

244 Maybe openpromptia.com will come back soon 

245 """ 

246 self.you_have_already = 'Invitation already used, but no date ?' 

247 

248 self.assertEqual(response.data.decode("utf-8"), self.you_have_already) 

249 

250 # - [ ] TODO VR c'est deux requetes doivent changer de comportement quand elles plantent 

251 response = self.client.get('/api/v1/project/role/remove?project_id=1666&user_id_=1777') 

252 self.assertEqual(response.status_code, 200) 

253 

254 response = self.client.get('/projects') 

255 self.assertEqual(response.status_code, 200) 

256 

257 response = self.client.get('/project/admin/1666') 

258 expected_content = """<!DOCTYPE html> 

259 <html> 

260 <head> 

261 <title>Safia Project Admin</title> 

262 </head> 

263 <body> 

264 <h1>Safia Project Admin</h1> 

265 <table> 

266 <tr> 

267 <th>Project Name</th> 

268 <th>Number of Documents</th> 

269 <th>Size</th> 

270 <th>Cost</th> 

271 <th>Command</th> 

272 </tr> 

273  

274 <tr> 

275 <td></td> 

276 <td>0</td> 

277 <td>TO ESTIMATE</td> 

278 <td></td> 

279 <td></td> 

280 </tr> 

281  

282 </table> 

283 <a href="/documents.html">Listes complètes documents !</a> 

284 </body> 

285 </html>""" 

286# self.assertEqual(expected_content, response.data.decode("utf-8")) 

287 self.assertEqual(response.status_code, 200) 

288 

289 response = self.client.get('/editor') 

290 self.assertEqual(response.status_code, 200) 

291# self.assertEqual(response.data.decode("utf-8"), self.you_have_no_valid) 

292 

293 response = self.client.post('/api/v1/document/save') 

294 self.assertEqual(response.status_code, 415) 

295 

296 response = self.client.post('/api/v1/document/save', data=json.dumps({}), content_type="application/json") 

297 self.assertEqual(response.status_code, 200) 

298 self.to_test_mets_a_jour = "TO TEST Mets à jour" 

299# self.assertEqual(response.data.decode("utf-8"), self.to_test_mets_a_jour) 

300 

301 response = self.client.post('/api/v1/datou/delete') 

302 self.assertEqual(response.status_code, 415) 

303# self.assertEqual(response.data.decode("utf-8"), self.to_test_mets_a_jour) 

304 

305 response = self.client.post('/api/v1/datou/save') 

306 self.assertEqual(response.status_code, 415) 

307 # self.assertEqual(response.data.decode("utf-8"), self.to_test_mets_a_jour) 

308 

309 response = self.client.post('/api/v1/project/select/1666') 

310 self.assertEqual(response.status_code, 200) 

311 self.assertEqual(response.data.decode("utf-8"), '{"project_id": "1666"}') 

312 

313 response = self.client.get('/api/v1/project/detail/1666') 

314 self.assertEqual(response.status_code, 200) 

315 self.assertEqual(response.data.decode("utf-8"), '{"number_documents": 0, "latest_update": null, "size": "TO ESTIMATE", "created_at": "", "modified_at": null, "most_recent_date_h": "il y a 11 ans", "most_recent_date": "2014-09-06 00:00:00", "nb_doc_with_no_title": 0, "nb_doc_with_title": 0, "nb_doc_wo_info_on_title": 0}') 

316 

317 response = self.client.post('/api/v1/project/save', data=json.dumps({"just":"notlogged"}), content_type="application/json") 

318 self.assertEqual(response.status_code, 200) 

319 self.assertEqual(response.data.decode("utf-8"), '{\n "success": "Shortname updated"\n}\n') 

320 

321 try : 

322 response = self.client.post('/api/v1/configuration/update', data=json.dumps({"just":"notlogged"}), content_type="application/json") 

323 self.assertEqual(response.status_code, 200) 

324 self.assertEqual(response.json, {}) 

325 except Exception as e: 

326 print(str(e)) 

327 print("VR essaie de faire passer les tests avant de pouvoir les améliorer 6/2/26") 

328 # assert False, "Should not raise exception" 

329 

330 response = self.client.get('/account') 

331 self.assertEqual(response.status_code, 200) 

332 self.assertTrue(len(response.data.decode("utf-8")) > 20000) 

333 

334 response = self.client.get('/doc') 

335 self.assertEqual(response.status_code, 200) 

336 self.assertTrue(len(response.data.decode("utf-8")) > 10000) 

337 

338 response = self.client.get('/datou') 

339 self.assertEqual(response.status_code, 200) 

340 self.assertTrue(len(response.data.decode("utf-8")) > 80000) 

341 

342 

343 def te_st_crash_reconnect(self): 

344 response = self.client.get('/') 

345 self.assertEqual(response.status_code, 200) 

346 

347 response = self.client.get('/bugr') 

348 self.assertEqual(response.status_code, 200) 

349 

350 result_ref = """syntax error at or near "duplicate" 

351LINE 2: VALUES (49, 9, 'admin') on duplicate(group_i... 

352 ^ 

353 

354INTERNAL ERROR CAUSED BY MANUAL TEST (AND TO CHECK suiviprod) could be INTERNALAL ERROR otherwise""" 

355 

356# None reconnected None 

357 

358 result_comp = response.data.decode("utf-8") 

359 import logging 

360 self.logger = logging.getLogger() 

361 self.logger.info(result_comp) 

362 self.logger.info(result_ref) 

363 self.assertEqual(result_comp, result_ref) 

364 

365 except_on_index = False 

366 try : 

367 response = self.client.get('/') 

368 self.assertEqual(response.status_code, 500) 

369 except_on_index = False 

370 except Exception as e: 

371 print(str(e)) 

372 except_on_index = True 

373 

374 self.assertEqual(except_on_index, True) 

375 

376 response = self.client.get('/reconnect') 

377 self.assertEqual(response.status_code, 200) 

378 message_ref = """ERROR: current transaction is aborted, commands ignored until end of transaction block 

379 reconnected None""" 

380 message_ref = """None reconnected None""" 

381 message_ref2 = """syntax error at or near "duplicate" 

382LINE 2: VALUES (49, 9, 'admin') on duplicate(group_i... 

383 ^ 

384 

385INTERNALAL ERROR CAUSED BY MANUAL TEST""" 

386 message_comp = response.data.decode("utf-8") 

387 self.assertEqual(message_comp, message_ref) 

388 

389 response = self.client.get('/') 

390 self.assertEqual(response.status_code, 200) 

391 

392 def test_zz_setup_action_ok(self): 

393 otp = "c6ae5f7095abfe9ac59e75fc6624cea56038d9da" 

394 response = self.client.get("/otp/" + otp) 

395 print(str(response)) 

396 self.assertEqual(response.status_code, 200) # ca ca marche plus de toute facon VR 6-6-25 # redirect 

397 

398 response = self.client.get("/otp/incorrectotp") 

399 print(str(response)) 

400 self.assertEqual(response.status_code, 200) # redirect 

401 

402 def test_b_update_configuration_and_datou_dummy(self): 

403 proj_nrt = project_nrt 

404 

405 logger = logging.getLogger() 

406 # TODO VR le cookie va expirer ! 

407 self.client.set_cookie( 'otp', "c6ae5f7095abfe9ac59e75fc6624cea56038d9da") 

408 self.client.set_cookie( 'project_id', str(proj_nrt)) 

409 

410 data_update_configuration = {"nlp_chat" : "openai_gpt_api"} 

411 

412# response = self.client.post('/api/v1/configuration/update', 

413# content_type='multipart/form-data', 

414# data={'file': (f, 'lettre1erMoisCompletude-08-07-2022.pdf'), 'data': json.dumps(data)}, 

415# follow_redirects=True) 

416 

417 import json 

418 response = self.client.post('/api/v1/configuration/update', 

419 content_type='application/json', 

420# content_type='multipart/form-data', 

421 data=json.dumps(data_update_configuration)) 

422 

423 logger.info("response : " + str(response)) 

424 self.assertEqual(response.status_code, 200) 

425 

426# data_update_datou = [{"id" : 1, 

427# "param_json" : {"retest" : "etontest"}}] 

428 

429 data_update_datou = {"id" : 73, "name":"test_Datou", "list_steps" : [{"datou_step_type_id":1, "param_json":{"mont":"juiq"}, "order":1}]} 

430 

431 response = self.client.post('/api/v1/datou/save', 

432 content_type='application/json', 

433 data=json.dumps(data_update_datou)) 

434 

435 logger.info("response : " + str(response)) 

436 self.assertEqual(response.status_code, 200) 

437 

438 logger.info(" PG_DB : " + str(PG_DB)) 

439 

440 from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys 

441 lpgss = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT) 

442 

443 datous = lpgss.get_datou_user(user_id=nrt_user_id) 

444 logger.info(str(datous)) 

445# del datou["id"] 

446 my_datou = list(filter(lambda x : x["filename"] == "test_Datou", datous))[0] 

447 del my_datou["id"] 

448 del my_datou["created_at"] 

449 del my_datou["modified_at"] 

450# for s in my_datou["list_steps"]: 

451# del my_datou["list_steps"]["id"] 

452 

453 # RealDictRow([('user_id', 1), ('filename', 'test_Datou'), ('dot_file', None), ('public', False)]) 

454 

455 datou_ref = {'owner_id' : 3, 'filename' :'test_Datou', 'dot_file': None, 'public' : True } 

456 

457 assert my_datou == datou_ref 

458 

459 data_update_datou = {"name":"new_datou_to_insert_and_delete", "list_steps" : [{"datou_step_type_id":1, "param_json":{"mont":"juiq"}, "order":1}]} 

460 response = self.client.post('/api/v1/datou/save', 

461 content_type='application/json', 

462 data=json.dumps(data_update_datou)) 

463 self.assertEqual(response.status_code, 200) 

464 

465 # Comment fais t'on : On va choper le dernier ? on recherche par le nom ? 

466 # Ou plutot on charge et on fait la recherche ici, mouais d'accord 

467 

468 # delete_datou or better to test 

469 

470 datous = lpgss.get_datou_user(user_id=nrt_user_id) 

471 logger.info(str(datous)) 

472 my_datou = list(filter(lambda x : x["filename"] == "new_datou_to_insert_and_delete", datous))[0] 

473 

474 response = self.client.post('/api/v1/datou/delete', 

475 content_type='application/json', 

476 data=json.dumps({"id" : my_datou["id"]})) 

477 self.assertEqual(response.status_code, 200) 

478 

479 datous = lpgss.get_datou_user(user_id=nrt_user_id) 

480 

481 list_datou_name = list(map(lambda x : x["filename"], datous)) 

482 

483 assert "new_datou_to_insert_and_delete" not in list_datou_name 

484 

485# test/main_test.py::FirstClassSafiaTest::test_file_upload_and_import_git 

486# @pytest.mark.asyncio 

487# async def suspendest_file_upload(self): 

488 #async 

489# - [ ] TODO VR renommer ce test qui fait pas mal de choses 

490 def test_file_upload_and_import_git(self): 

491 proj_nrt = project_nrt 

492 verbose = False 

493 

494 logger = logging.getLogger() 

495# response = self.client.get('/') 

496 # TODO VR le cookie va expirer ! 

497 self.client.set_cookie( 'otp', "c6ae5f7095abfe9ac59e75fc6624cea56038d9da") 

498 self.client.set_cookie( 'project_id', str(proj_nrt)) 

499# import io 

500# data = { 

501# 'file': (io.BytesIO(b"my file contents"), 'test_file.txt'), 

502# } 

503 

504 logger.info(" PG_DB : " + str(PG_DB)) 

505 

506 

507 from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys 

508 lpgss = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT) 

509 

510 curr_project_id = project_nrt 

511 

512 logger.info(" Before get_project_info ") 

513 info_project = lss_singleton.get_project_info(curr_project_id) 

514 logger.info(" Before truncate_project_documents (useless s'il y a des document_id unique ! ) ") 

515 import time 

516 time.sleep(1) 

517 lss_singleton.truncate_project_documents(curr_project_id, verbose=verbose) 

518 time.sleep(1) 

519 logger.info(" After truncate_project_documents ") 

520 

521 response = self.client.post("/login", 

522 # data={"username":"vic","password":"vic"}, 

523 data={"username": "loc", "password": "loc"}, 

524 content_type='multipart/form-data') 

525 # multipart/form-data. Otherwise, use application/x-www-form-urlencoded. 

526 self.assertEqual(response.status_code, 200) 

527 print(str(response)) 

528 

529 username = "vic" 

530# from server.bp.auth import create_login_bp.User 

531# user = User.query.filter_by(username=username).first() 

532 from flask_login import login_user, UserMixin 

533 user = UserMixin() 

534 user.mtr_user_id = 1 

535 user.id = 1 

536 user.username = "loc" 

537 login_user(user) 

538 

539 try: 

540 list = lpgss.get_usage_connection_ip_info() 

541 except Exception as e: 

542 print(str(e)) 

543 

544 data_update_info = {'project_id' : curr_project_id,'shortname': "default"} 

545 response = self.client.post("/api/v1/project/save", 

546 data=json.dumps(data_update_info), 

547 content_type='application/json') 

548 self.assertEqual(response.status_code, 200) 

549 

550 response = self.client.get("/api/v1/project/detail/" + str(curr_project_id)) 

551 self.assertEqual(response.status_code, 200) 

552 print(str(response)) 

553# print(str(response.json)) 

554 # VR TODO a mon avis le data store_in_project est inutile voir nefaste 

555 data_json_ref ={'id': 1, 'owner_id': 1, 'description': 'non_reg_test_0623', 'table_documents': 'victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc', 'function_search_documents': 'match_c6ae5f7095abfe9ac59e75fc6624cea56038d9da', 'format': None, 'taille': None, 'created_at': '06/11/2023 04:30:41', 'modified_at': '06/11/2023 04:30:41', 'configuration': {'data': '{"store_in_project": "true", "project_id": "1"}', 'custom_display': {'data': '{"store_in_project": "true", "project_id": "1"}', 'home': {'data': '{"store_in_project": "true", "project_id": "1"}'}}}, 'shortlink': 'default', 'number_documents': 0, 'size': 'TO ESTIMATE', 'command': '\n cd $GITSAFIA/prompt/python/lib/import_util/lib_import_retrieval;\n export PYTHONPATH=`pwd`/../../..:`pwd`;\n python3.10 scripts/process_json/process_json.py --project_id=1 --pg_table=victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc --filepath=../../../io/Fotonower_Issues.json\n ', 'nb_doc_with_no_title': 0, 'nb_doc_with_title': 0, 'nb_doc_wo_info_on_title': 0, 

556 'most_recent_date_h': 'il y a 8 secondes', 

557 'most_recent_date': '2024-01-26 01:39:42.839445+01:00', 

558 'latest_update': None} 

559 # 30/10/24 

560 data_json_ref = {'command': '\n' 

561 ' cd ' 

562 '$GITSAFIA/prompt/python/lib/import_util/lib_import_retrieval;\n' 

563 ' export PYTHONPATH=`pwd`/../../..:`pwd`;\n' 

564 ' python3.10 scripts/process_json/process_json.py ' 

565 '--project_id=1 ' 

566 '--pg_table=victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc ' 

567 '--filepath=../../../io/Fotonower_Issues.json\n' 

568 ' ', 

569 'configuration': {'command': '\n' 

570 ' cd ' 

571 '$GITSAFIA/prompt/python/lib/import_util/lib_import_retrieval;\n' 

572 ' export ' 

573 'PYTHONPATH=`pwd`/../../..:`pwd`;\n' 

574 ' python3.10 ' 

575 'scripts/process_json/process_json.py ' 

576 '--project_id=1 ' 

577 '--pg_table=victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc ' 

578 '--filepath=../../../io/Fotonower_Issues.json\n' 

579 ' ', 

580 'configuration': {'custom_display': {'data': '{"store_in_project": ' 

581 '"true", ' 

582 '"project_id": ' 

583 '"1"}', 

584 'home': {'data': '{"store_in_project": ' 

585 '"true", ' 

586 '"project_id": ' 

587 '"1"}'}}, 

588 'data': '{"store_in_project": "true", ' 

589 '"project_id": "1"}'}, 

590 'created_at': '06/11/2023 04:30:41', 

591 'custom_display': {'home': {'data': '{"store_in_project": ' 

592 '"true", "project_id": ' 

593 '"1"}'}}, 

594 'description': 'non_reg_test_0623', 

595 'format': None, 

596 'function_search_documents': 'match_c6ae5f7095abfe9ac59e75fc6624cea56038d9da', 

597 'id': 1, 

598 'latest_update': None, 

599 'modified_at': '06/11/2023 04:30:41', 

600 'most_recent_date': '2024-01-26 01:39:42.839445+01:00', 

601 'most_recent_date_h': 'il y a 8 secondes', 

602 'nb_doc_with_no_title': 0, 

603 'nb_doc_with_title': 0, 

604 'nb_doc_wo_info_on_title': 0, 

605 'number_documents': 0, 

606 'owner_id': 1, 

607 'shortlink': 'default', 

608 'size': 'TO ESTIMATE', 

609 'table_documents': 'victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc', 

610 'taille': None}, 

611 'created_at': '06/11/2023 04:30:41', 

612 'description': 'non_reg_test_0623', 

613 'format': None, 

614 'function_search_documents': 'match_c6ae5f7095abfe9ac59e75fc6624cea56038d9da', 

615 'id': 1, 

616 'modified_at': '06/11/2023 04:30:41', 

617 'nb_doc_with_no_title': 0, 

618 'nb_doc_with_title': 0, 

619 'nb_doc_wo_info_on_title': 0, 

620 'number_documents': 0, 

621 'owner_id': 1, 

622 'shortlink': 'default', 

623 'size': 'TO ESTIMATE', 

624 'table_documents': 'victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc', 

625 'taille': None} 

626 # DB main no project 1 

627 data_json_ref = {'number_documents': 0, 'latest_update': None, 'size': 'TO ESTIMATE', 'created_at': '', 'modified_at': None, 'most_recent_date_h': 'il y a 11 ans', 'most_recent_date': '2014-09-06 00:00:00', 'nb_doc_with_no_title': 0, 'nb_doc_with_title': 0, 'nb_doc_wo_info_on_title': 0} 

628 data_json_ref = {'id': 1, 'owner_id': 1, 'description': 'non_reg_test_0623', 'table_documents': 'victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc', 'function_search_documents': 'match_c6ae5f7095abfe9ac59e75fc6624cea56038d9da', 'format': None, 'taille': None, 'created_at': '06/11/2023 04:30:41', 'modified_at': '06/11/2023 04:30:41', 'cost': -21.699331852300062, 'configuration': {'id': 1, 'owner_id': 1, 'description': 'non_reg_test_0623', 'table_documents': 'victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc', 'function_search_documents': 'match_c6ae5f7095abfe9ac59e75fc6624cea56038d9da', 'format': None, 'taille': None, 'created_at': '06/11/2023 04:30:41', 'modified_at': '06/11/2023 04:30:41', 'configuration': {'data': '{"store_in_project": "true", "project_id": "1"}', 'custom_display': {'data': '{"store_in_project": "true", "project_id": "1"}', 'home': {'data': '{"store_in_project": "true", "project_id": "1"}'}}}, 'shortlink': 'default', 'number_documents': 0, 'size': 'TO ESTIMATE', 'command': '\n cd $GITSAFIA/prompt/python/lib/import_util/lib_import_retrieval;\n export PYTHONPATH=`pwd`/../../..:`pwd`;\n python3.10 scripts/process_json/process_json.py --project_id=1 --pg_table=victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc --filepath=../../../io/Fotonower_Issues.json\n ', 'nb_doc_with_no_title': 0, 'nb_doc_with_title': 0, 'nb_doc_wo_info_on_title': 0, 'most_recent_date_h': 'il y a 8 secondes', 'most_recent_date': '2024-01-26 01:39:42.839445+01:00', 'latest_update': None, 'custom_display': {'home': {'data': '{"store_in_project": "true", "project_id": "1"}'}}}, 'shortlink': 'default', 'number_documents': 0, 'latest_update': None, 'size': 'TO ESTIMATE', 'command': '\n cd $GITSAFIA/prompt/python/lib/import_util/lib_import_retrieval;\n export PYTHONPATH=`pwd`/../../..:`pwd`;\n python3.10 scripts/process_json/process_json.py --project_id=1 --pg_table=victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc --filepath=../../../io/Fotonower_Issues.json\n ', 'most_recent_date_h': 'il y a 11 ans', 'most_recent_date': '2014-09-06 00:00:00', 'nb_doc_with_no_title': 0, 'nb_doc_with_title': 0, 'nb_doc_wo_info_on_title': 0} 

629 data_json_ref = {'id': 1, 'owner_id': 1, 'description': 'non_reg_test_0623', 'table_documents': 'victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc', 'function_search_documents': 'match_c6ae5f7095abfe9ac59e75fc6624cea56038d9da', 'format': None, 'taille': None, 'created_at': '06/11/2023 04:30:41', 'modified_at': '06/11/2023 04:30:41', 'shortlink': 'shortlink_nrt', 'number_documents': 0, 'size': 'TO ESTIMATE', 'command': '\n cd $GITSAFIA/prompt/python/lib/import_util/lib_import_retrieval;\n export PYTHONPATH=`pwd`/../../..:`pwd`;\n python3.10 scripts/process_json/process_json.py --project_id=1 --pg_table=victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc --filepath=../../../io/Fotonower_Issues.json\n ', 'nb_doc_with_no_title': 0, 'nb_doc_with_title': 0, 'nb_doc_wo_info_on_title': 0} 

630 data_json_ref = {'id': 1, 'owner_id': 1, 'description': 'plop', 'function_search_documents': 'match_8e3ce876cf8540318825808c81405950', 'format': None, 'taille': None, 'configuration': {'custom_display': {'home': {'data': '{"store_in_project": ' 

631 '"true", "project_id": ' 

632 '"1"}'}}}, 'shortlink': 'default', 'number_documents': 0, 'latest_update': None, 'size': 'TO ESTIMATE', 'command': '\n cd $GITSAFIA/prompt/python/lib/import_util/lib_import_retrieval;\n export PYTHONPATH=`pwd`/../../..:`pwd`;\n python3.10 scripts/process_json/process_json.py --project_id=1 --pg_table=melvic_documents.dataset_8e3ce876cf8540318825808c81405950 --filepath=../../../io/Fotonower_Issues.json\n ', 'most_recent_date_h': 'il y a 11 ans', 'most_recent_date': '2014-09-06 00:00:00', 'nb_doc_with_no_title': 0, 'nb_doc_with_title': 0, 'nb_doc_wo_info_on_title': 0} 

633 

634# del data_json_ref["modified_at"] 

635# del data_json_ref["created_at"] 

636# del data_json_ref["table_documents"] 

637 

638 if "cost" in data_json_ref: 

639 del data_json_ref["cost"] 

640 response_json = response.json 

641 

642 del response_json["modified_at"] 

643 del response_json["created_at"] 

644 del response_json["table_documents"] 

645 

646 # del response_json["cost"] 

647 # VR TODO, le delete des nouveaux datous ne semble pas marcher (ou c'est plutot une interraction de merde avec les test jscover aussi) 

648# del response_json["most_recent_date_h"] 

649# if "most_recent_date_h" in data_json_ref: 

650# del data_json_ref["most_recent_date_h"] 

651# del response_json["most_recent_date"] 

652# if "most_recent_date" in data_json_ref: 

653# del data_json_ref["most_recent_date"] 

654# del response_json["latest_update"] 

655# if "latest_update" in data_json_ref: 

656# del data_json_ref["latest_update"] 

657 if "cost" in response_json: 

658 del response_json["cost"] 

659 print(self.maxDiff) 

660 self.maxDiff = None 

661 self.assertEqual(data_json_ref, response_json) 

662 

663 data_update_info = {'project_id' : curr_project_id,'shortname': "shortlink_nrt"} 

664 response = self.client.post("/api/v1/project/save", 

665 data=json.dumps(data_update_info), 

666 content_type='application/json') 

667 

668 self.maxDiff = 480 # plutot que 640 par défaut 

669 data_json_ref = {'id': 1, 'owner_id': 1, 'description': 'plop', 'table_documents': 'melvic_documents.dataset_8e3ce876cf8540318825808c81405950', 'function_search_documents': 'match_8e3ce876cf8540318825808c81405950', 'format': None, 'taille': None, 'created_at': '11/11/2024 02:26:30', 'modified_at': '11/11/2024 02:26:30', 'shortlink': 'shortlink_nrt', 'number_documents': 0, 'latest_update': None, 'size': 'TO ESTIMATE', 'command': '\n cd $GITSAFIA/prompt/python/lib/import_util/lib_import_retrieval;\n export PYTHONPATH=`pwd`/../../..:`pwd`;\n python3.10 scripts/process_json/process_json.py --project_id=1 --pg_table=melvic_documents.dataset_8e3ce876cf8540318825808c81405950 --filepath=../../../io/Fotonower_Issues.json\n ', 'most_recent_date_h': 'il y a 11 ans', 'most_recent_date': '2014-09-06 00:00:00', 'nb_doc_with_no_title': 0, 'nb_doc_with_title': 0, 'nb_doc_wo_info_on_title': 0} 

670 data_json_ref["shortlink"] = "shortlink_nrt" 

671 

672 response = self.client.get("/api/v1/project/detail/" + str(curr_project_id)) 

673 response_json = response.json 

674 del response_json["cost"] 

675 del response_json["configuration"] 

676# del response_json["most_recent_date_h"] 

677# del response_json["most_recent_date"] 

678# del response_json["latest_update"] 

679 self.assertEqual(response.status_code, 200) 

680 self.assertEqual(data_json_ref, response_json) 

681 

682 response = self.client.get("/shortlink_nrt") 

683 self.assertEqual(response.status_code, 200) 

684 response = self.client.get("/shortlink_nrtt") 

685 self.assertEqual(response.status_code, 200) 

686 self.assertEqual(response.data.decode("utf-8"), "GENERIC PATH") # - [ ] TODO manage these case ! 

687 

688 data_update_info = {'project_id' : curr_project_id,'shortname': "default"} 

689 response = self.client.post("/api/v1/project/save", 

690 data=json.dumps(data_update_info), 

691 content_type='application/json') 

692 

693 data = { 

694 'store_in_project': 'true', 

695 'project_id': str(curr_project_id) 

696 } 

697 

698 path_to_your_file = "test/data/lettre1erMoisCompletude-08-07-2022.pdf" 

699 

700# from werkzeug.datastructures import FileStorage 

701# # Open the file in binary mode and create a FileStorage object 

702# with open(path_to_your_file, 'rb') as f: 

703# file = FileStorage(f) 

704 

705# lettre1erMoisCompletude-08-07-2022.pdf 

706# short_test_7JEI_RESCRIT.json 

707 

708 hostname = os.environ["HOSTANME"] if "HOSTANME" in os.environ else os.uname().nodename 

709 

710 os.system("pwd") 

711 

712 with open(path_to_your_file, 'rb') as f: 

713 logger.info(" Before self.client.post : file : " + str(f)) 

714 # Send the file and the JSON data to the endpoint 

715 

716# application/pdf 

717 # Test d'upload de l'instuction du dossier de permis de construire de l'ECA rue charlot 

718 response = self.client.post('/api/v1/uploader', 

719 content_type='multipart/form-data', 

720 data={'file': (f, 'lettre1erMoisCompletude-08-07-2022.pdf'), 'data': json.dumps(data)}, 

721 follow_redirects=True) 

722 

723 print("response : " + str(response)) 

724 logger.info("response : " + str(response)) 

725 logger.info(" response keys " + str(response.json.keys())) 

726 print(" response keys " + str(response.json.keys())) 

727 logger.info("response manage_prompt : " + str(response.json["manage_prompt"])) 

728 

729 json_data = json.loads(response.data) 

730 logger.info(" json_data keys " + str(json_data.keys())) 

731 print(" json_data keys " + str(json_data.keys())) 

732 file_to_manage = json_data["manage_prompt"]["file"] 

733 print(" file_to_manage : " + str(file_to_manage)) 

734 

735# # Check the response 

736 self.assertEqual(response.status_code, 200) 

737 

738 # delete_document 

739 

740 # TODO VR : validité du cookie ! c6ae5f7095abfe9ac59e75fc6624cea56038d9da => update dans AirTable ? 

741 data_query = {"input" : "Quel est le nom de famille de Perrine ?"} # => VR 17-6-23 : je en sais vraiment pas pourquoi je suis tombé sur Perrine Mouterde 

742 data_query = {"input" : "Qui s'est occupé de la demande de permis de construire ?"} 

743 

744 response_query = self.client.get('/api/v1/safia/query', 

745 # content_type='multipart/form-data', 

746 query_string=data_query, 

747 follow_redirects=True) 

748 logger.info("response_query : " + str(response_query)) 

749 

750 self.assertEqual(response_query.status_code, 200) 

751 logger.info("response") 

752 logger.info(str(response_query.data)) 

753 logger.info(" response_query keys " + str(response_query.json.keys())) 

754 print(" response_query keys " + str(response_query.json.keys())) 

755# logger.info("response_query manage_prompt : " + str(response_query.json["manage_prompt"])) 

756# print("response_query manage_prompt : " + str(response_query.json["manage_prompt"])) 

757 # references 

758 

759# logger.info(str(response_query.get_json())) => ce serait mieux ! 

760# self.assertIn("mouterde", str(response_query.data).lower()) 

761 

762 self.assertIn("didier", str(response_query.data).lower()) 

763 

764 lss = LibSafiaSystem(lib_user_data_internal=lpgss) 

765 

766 info_project = lss.get_project_info(project_nrt) 

767 logger.info(" after get_project_info ") 

768 logger.info(" info_project : " + str(info_project)) 

769 print(" info_project : " + str(info_project)) 

770 

771 assert info_project["number_documents"] == 3 #4 #80 # 3 when run alone I think 

772 

773 document_id = "///FILE///moilerat@" + hostname + "//temp/lettre1erMoisCompletude-08-07-2022.pdf_1" 

774 document_id = "///FILE///moilerat@" + hostname + "//" + file_to_manage + "_1" 

775 

776 response_delete = self.client.get('/api/v1/document/delete?id=' + document_id) 

777 

778 complete_document_id = "///FILE///moilerat@" + hostname + "//temp/lettre1erMoisCompletude-08-07-2022.pdf" 

779 complete_document_id = "///FILE///moilerat@" + hostname + "//" + file_to_manage 

780 self.test_documents_page(endpoint = "/editor?document_id=" + complete_document_id, 

781 search_into = '<span class="ml-2">Voice Editor</span>', 

782 length_file = 4351, #5795, #4327, 

783 filename = "editor_document_lettre_completude.html", 

784 save_comp = True) 

785 

786# time.sleep(5) 

787 

788 document_id = "///FILE///moilerat@" + hostname + "//temp/lettre1erMoisCompletude-08-07-2022.pdf_1" 

789 document_id = "///FILE///moilerat@" + hostname + "//" + file_to_manage + "_1" 

790 response_delete = self.client.get('/api/v1/document/delete?id=' + document_id) 

791 

792 self.assertEqual(response_delete.status_code, 302) 

793 info_project = lss.get_project_info(project_nrt) 

794 # VR 18-3-23 on ne test # plus la suppression 

795 assert info_project["number_documents"] == 3 # 3 # 79 # 2 

796 

797 audit, list_inputs, list_datou_step = lss.resume_all_project(project_nrt) #, nb_cluster_input = 3) 

798 logger.info("audit : " + str(audit)) 

799 print(str(audit)) 

800 del audit["created_at"] 

801 

802 assert audit == {'size_document': 1365, 'nb_chunks': 3, 'nb_tokens': 462, 'nb_input': 1, 

803 'begin_content': " \nDirection de l'Urbanisme\nService du Permis de Construire et du Paysage de la Rue\nParis, le 08/07/2022\nAffaire suivie par :\nDidier MEURGUES\uf028 01 42 76 22 25\nmél : didier.meurgues@paris.fr6 Promenade Claude Lévi-StraussCS 51388 - 75639 PARIS CEDEX 13Monsieur Victor REUTENAUER\n \n30 rue Charlot\n75003 Paris\nRéférence dossier :DP 075 103 22 V0198\n30 RUE CHARLOT\n75003 PARIS\n*DP07510322V0198106* \nLettre Recommandée Electronique.\nMadame, Monsieur,\n"} 

804# assert audit == {'size_document': 918, 'nb_chunks': 2, 'nb_tokens': 291, 'nb_input': 1} 

805 assert len(list_inputs) == 1 

806 assert "paris" in list_inputs[0]["text"].lower() 

807 

808 response_query = self.client.get('/resume_all_one_project') 

809 logger.info("response_query : " + str(response_query.data)) 

810 print("response_query : " + str(response_query.data)) 

811 

812 self.assertEqual(response_query.status_code, 200) 

813# json_paris = json.loads("WIP please wait ![{\'text\': "Direction de l\'Urbanisme Service du Permis de Construire et du Paysage de la Rue Paris, le 08/07/2022 Affaire suivie par : Didier MEURGUES\\uf028 01 42 76 22 25 m\xc3\xa9l : didier.meurgues@paris.fr6 Promenade Claude L\xc3\xa9vi-StraussCS 51388 - 75639 PARIS CEDEX 13Monsieur Victor REUTENAUER 30 rue Charlot 75003 Paris R\xc3\xa9f\xc3\xa9rence dossier :DP 075 103 22 V0198 30 RUE CHARLOT 75003 PARIS *DP07510322V0198106* Lettre Recommand\xc3\xa9e Electronique. Madame, Monsieur,Je vous signale qu\'il s\'agit d\'un d\xc3\xa9lai maximum et que je m\xe2\x80\x99efforcerai de vous notifier ma d\xc3\xa9cision d\xc3\xa8s que possible avant son expiration. Je vous prie d\xe2\x80\x99agr\xc3\xa9er, Madame, Monsieur, l\xe2\x80\x99expression de mes salutations distingu\xc3\xa9es. Pour la Maire de Paris et par d\xc3\xa9l\xc3\xa9gation, #signature# Copie au Pr\xc3\xa9fet de Paris DP 075 103 22 V0198 Compl\xc3\xa9tude DP", \'preprompt\': "Merci de faire un r\xc3\xa9sum\xc3\xa9 en listant les points importants et les taches \xc3\xa0 accomplir de la transcription de l\'AG de Fotonower :", \'result\': "R\xc3\xa9sum\xc3\xa9 de la transcription de l\'AG de Fotonower:\\n\\n1. Direction de l\'Urbanisme Service du Permis de Construire et du Paysage de la Rue.\\n2. Date: 08/07/2022\\n3. Affaire suivie par Didier MEURGUES (01 42 76 22 25, didier.meurgues@paris.fr).\\n4. Adresse: 6 Promenade Claude L\xc3\xa9vi-Strauss, CS 51388 - 75639 PARIS CEDEX 13.\\n5. Destinataire: Victor REUTENAUER, 30 rue Charlot 75003 Paris.\\n6. R\xc3\xa9f\xc3\xa9rence dossier: DP 075 103 22 V0198 30 RUE CHARLOT 75003 PARIS.\\n\\nT\xc3\xa2ches \xc3\xa0 accomplir:\\n\\n1. Prendre note du d\xc3\xa9lai maximum pour notifier la d\xc3\xa9cision et s\'efforcer de la donner avant son expiration.\\n2. Transmettre les salutations distingu\xc3\xa9es de la Maire de Paris.\\n3. Envoyer une copie de la lettre recommand\xc3\xa9e \xc3\xa9lectronique au Pr\xc3\xa9fet de Paris (DP 075 103 22 V0198).\\n4. Assurer la compl\xc3\xa9tude du dossier DP.", \'request\': "Merci de faire un r\xc3\xa9sum\xc3\xa9 en listant les points importants et les taches \xc3\xa0 accomplir de la transcription de l\'AG de Fotonower :Direction de l\'Urbanisme Service du Permis de Construire et du Paysage de la Rue Paris, le 08/07/2022 Affaire suivie par : Didier MEURGUES\\uf028 01 42 76 22 25 m\xc3\xa9l : didier.meurgues@paris.fr6 Promenade Claude L\xc3\xa9vi-StraussCS 51388 - 75639 PARIS CEDEX 13Monsieur Victor REUTENAUER 30 rue Charlot 75003 Paris R\xc3\xa9f\xc3\xa9rence dossier :DP 075 103 22 V0198 30 RUE CHARLOT 75003 PARIS *DP07510322V0198106* Lettre Recommand\xc3\xa9e Electronique. Madame, Monsieur,Je vous signale qu\'il s\'agit d\'un d\xc3\xa9lai maximum et que je m\xe2\x80\x99efforcerai de vous notifier ma d\xc3\xa9cision d\xc3\xa8s que possible avant son expiration. Je vous prie d\xe2\x80\x99agr\xc3\xa9er, Madame, Monsieur, l\xe2\x80\x99expression de mes salutations distingu\xc3\xa9es. Pour la Maire de Paris et par d\xc3\xa9l\xc3\xa9gation, #signature# Copie au Pr\xc3\xa9fet de Paris DP 075 103 22 V0198 Compl\xc3\xa9tude DP", \'cost\': 0.03473999999999999}]") 

814 assert "Service du Permis de Construire" in str(response_query.data) 

815 

816 response_query = self.client.get('/stat_all_one_project') 

817# response_query = self.client.get(url_for('project.stat_all_one_project')) 

818 logger.info("response_query : " + str(response_query.data)) 

819 print("response_query : " + str(response_query.data)) 

820 

821 self.assertEqual(response_query.status_code, 200) 

822 json_stat_ref = {'///FILE///moilerat@' + hostname + '//' + file_to_manage: {'nb_chunks': 2, 'nb_input': 1, 'nb_tokens': 273, 'size_document': 783}} 

823 json_stat_ref = [{'begin_content': " \nDirection de l'Urbanisme\nService du Permis de Construire et du Paysage de la Rue\nParis, le 08/07/2022\nAffaire suivie par :\nDidier MEURGUES\uf028 01 42 76 22 25\nmél : didier.meurgues@paris.fr6 Promenade Claude Lévi-StraussCS 51388 - 75639 PARIS CEDEX 13Monsieur Victor REUTENAUER\n \n30 rue Charlot\n75003 Paris\nRéférence dossier :DP 075 103 22 V0198\n30 RUE CHARLOT\n75003 PARIS\n*DP07510322V0198106* \nLettre Recommandée Electronique.\nMadame, Monsieur,\n", 'created_at': '2023-10-10 23:51:14.149641+02:00', 'created_at_h': 'il y a 23 secondes', 'document_id': '///FILE///moilerat@MacBook-Air-Victor.local//temp/lettre1erMoisCompletude-08-07-2022adcfcdbd-19dd-46e3-b8db-b0faefc781bd.pdf', 'kunslash': 'FILEmoileratMacBook_Air_Victorlocaltemplettre_erMoisCompletude___________adcfcdbd___dd___e__b_db_b_faefc___bdpdf', 'nb_chunks': 3, 'nb_input': 1, 'nb_tokens': 462, 'size_document': 1365, 'title': 'No Title Found'}] 

824 json_stat_resp = json.loads(response_query.data) 

825 del json_stat_resp[0]["created_at"] 

826 del json_stat_resp[0]["created_at_h"] 

827 del json_stat_resp[0]["document_id"] 

828 del json_stat_resp[0]["kunslash"] 

829 del json_stat_ref[0]["created_at"] 

830 del json_stat_ref[0]["created_at_h"] 

831 del json_stat_ref[0]["document_id"] 

832 del json_stat_ref[0]["kunslash"] 

833 # TODO VR ce type d'assert est plus efficace 

834 assert json_stat_ref == json_stat_resp 

835 

836 try : 

837 response_import_git = self.client.get('/git?repo=fotonower/Safia&limit=1&last_nb_days=1234567890&verbose=True&with_comment=True') 

838 expected_result_code = 200 

839 self.assertEqual(response_import_git.status_code, expected_result_code) 

840 info_project = lss.get_project_info(project_nrt) 

841 

842 assert info_project["number_documents"] >= 14 and info_project[ 

843 "number_documents"] <= 28 # 24 # 14 #21 #11 # 10 #8 #6 #21 # 19 # 10 au 11-7-23 je ne sais pas pourquoi # les tickets se rallonge 8 #80 # 3 

844 except Exception as e: 

845 print("Bad crednetials") 

846 

847 

848 response_query = self.client.get('/resume_all_one_project?nb_cluster=2') 

849 logger.info("response_query : " + str(response_query.data)) 

850 print("response_query : " + str(response_query.data)) 

851 

852 self.assertEqual(response_query.status_code, 200) 

853 

854 

855 

856 data_update = { 

857 # "id" : "///FILE///moilerat@Air-Victor//temp/lettre1erMoisCompletude-08-07-2022.pdf_0", 

858 "id" : "///FILE///moilerat@" + hostname + "//" + file_to_manage + "_0", 

859 "content" : "Le petrole c'est mieux que le solaire" 

860 } 

861# data_to_send = {"document_content" : document_content, "document_id" : document_id.id} 

862 # TODO CA ca update juste un chunk et ce n'est pas interfacé 

863 responseUpdate = self.client.post('/api/v1/document/update', 

864 content_type='application/json', 

865 data=json.dumps(data_update), 

866 follow_redirects=True) 

867 self.assertEqual(responseUpdate.status_code, 200) 

868 

869 # - [ ] TODO c'est assez dummy le test de document/save et document/update 

870 

871 data_update = { 

872 "document_id" : "///FILE///moilerat@" + hostname + "//" + file_to_manage, 

873 "document_content" : "Le petrole c'est mieux que le solaire" 

874 } 

875 # TODO CA ca ne supprime pas les chunks inutiles ! 

876 responseUpdate = self.client.post('/api/v1/document/save', 

877 content_type='application/json', 

878 data=json.dumps(data_update), 

879 follow_redirects=True) 

880 self.assertEqual(responseUpdate.status_code, 200) 

881 

882 data_query = {"input" : "Qu'est-ce qui est mieux que le solaire ? Répond par un mot seulement !"} 

883 

884 response_query = self.client.get('/api/v1/safia/query', 

885 content_type='multipart/form-data', 

886 query_string=data_query, 

887 follow_redirects=True) 

888 logger.info("response_query : " + str(response_query)) 

889 

890 self.assertEqual(response_query.status_code, 200) 

891 logger.info("response") 

892 logger.info(str(response_query.data)) 

893 

894 #self.assertIn("pétrole", str(response_query.data).lower()) 

895 #self.assertIn("preprompt", str(response_query.data).lower()) 

896 self.assertIn("merci de repondre", str(response_query.data).lower()) 

897# assert "pétrole" in str(response_query.data).lower() 

898 # le pétrole est identifié comme étant préférable au solaire 

899# assert "trole est mieux que le solaire" in str(response_query.data).lower() or "trole est meilleur que le solaire" in str(response_query.data).lower() 

900 # self.assertIn("il est mentionn\xc3\xa9 que le p\xc3\xa9trole est meilleur que le solaire", str(response_query.data).lower()) 

901 # Bien que ce soit mal indexé, on dirait qu'il trouve, mais par contre je en sais pas comment lire ces caractères 

902# self.assertIn("p\xc3\xa9trole", str(response_query.data).lower()) 

903# self.assertIn("pétrole", str(response_query.data).lower()) 

904 

905 

906 data_query = {"input" : "Quelle fleuve passe au Puy-En-Velay ?", 

907 "store_in_project" : "true", 

908 "object" : "simple_text_query"} 

909 response_query = self.client.get('/api/v1/safia/query', 

910 content_type='multipart/form-data', 

911 query_string=data_query, 

912 follow_redirects=True) 

913 logger.info("response_query : " + str(response_query)) 

914 

915 self.assertEqual(response_query.status_code, 200) 

916 logger.info("response") 

917 resp = response_query.data.decode("utf-8").lower() 

918 found_river = "loire" in resp or "borne" in resp or "dolaizon" in resp 

919 self.assertTrue(found_river) 

920 

921 

922 # TODO VR 17-6 : set specific password for debug prompt from mail 

923 

924 def te_st_kmean_multi_dim_dummy(self): 

925 self.client.set_cookie( 'otp', "c6ae5f7095abfe9ac59e75fc6624cea56038d9da") 

926 

927 response_query = self.client.get('/kmean?limit=10&pids=8,2&nb_cluster=12&dim=234') 

928 

929 self.assertEqual(response_query.status_code, 200) 

930 

931 # deprecated while NRT can't login !?! 

932 def te_st_invitation(self): 

933 # useless since we take an otp valid until 11/11/23 

934# id_info = lpgss_singleton.setup_user() 

935 

936 # - [ ] TODO VR il faudrait qu'il chope un otp pour cookie de manière à pouvoir tester l'interface et pas que la création, grrrr ! 

937 # - [ ] on le fera plus tard de dupliquer la récupération des otp depuis lib_internal plutot que depuis AT 

938 # - [ ] TODO VR ARCHI comment gérer la migration : service dupliquer ou migration hard 

939 # - [ ] TODO VR test project inaccessible first 

940 

941 from lib.lib_util import get_unique_id 

942 otp_for_info = get_unique_id() 

943 

944# print("id_info : " + str(id_info)) 

945 

946 otp_info_opio_fr = "c6ae5f7095abfe9ac59e75fc6624cea56038d9da" # nathalie_share "3b221881f70bde52677697d1f37ea437baf5df2f" 

947 response = self.client.get("/otp/" + otp_info_opio_fr) 

948 print(str(response)) 

949 self.assertEqual(response.status_code, 200) # redirect plutot que 302 

950 

951 info = lpgss_singleton.get_user_from_mail("info@opio.fr") 

952 print(str(info)) 

953 user_id_info = info["id"] 

954 

955 lpgss_singleton.get_admin_situation() 

956 time.sleep(3) 

957 

958 self.client.set_cookie('otp', "c6ae5f7095abfe9ac59e75fc6624cea56038d9da") 

959 response = self.client.get("/api/v1/project/role/remove?project_id=" + str(project_nrt) + "&user_id=" + str(user_id_info)) 

960 assert response.status_code == 200 

961 

962# list_projects = lss_singleton.get_project("info@opio.fr") 

963 list_projects = lss_singleton.get_projects(user_id_info) 

964 

965 assert len(list_projects) == 1 # 2 

966 

967 data_to_send = {"project_id" : project_nrt, 

968 "role" : "admin", 

969 "mail" : "info@opio.fr"} 

970 

971 import json 

972 self.client.set_cookie('otp', "c6ae5f7095abfe9ac59e75fc6624cea56038d9da") 

973 response = self.client.post('/api/v1/invitation/create', 

974 content_type='application/json', 

975 data=json.dumps(data_to_send), 

976 follow_redirects=True) 

977 

978 print(" response : " + str(response)) 

979 

980 link = response.json["link"]["link"] 

981 token_invitation = response.json["link"]["token"] 

982 

983 self.client.set_cookie( 'otp', otp_info_opio_fr) 

984 response = self.client.get(link) 

985# response = self.client.get('/invitation/' + token_invitation, 

986# follow_redirects=True) 

987 

988 print(" response : " + str(response)) 

989 

990# list_projects = lss_singleton.get_project("info@opio.fr") 

991 list_projects = lss_singleton.get_projects(user_id_info) 

992 

993 assert len(list_projects) == 1 # 2 

994 

995 self.client.set_cookie( 'otp', "c6ae5f7095abfe9ac59e75fc6624cea56038d9da") 

996 response = self.client.get('/') 

997 

998 from bs4 import BeautifulSoup 

999 # Parsez le contenu HTML avec BeautifulSoup 

1000 soup = BeautifulSoup(response.data, 'html.parser') 

1001 

1002 # Recherchez l'élément span avec la classe 'pc_cov' et extrayez son texte 

1003 span_value = soup.find_all('span', {'class': 'proj-desc'}) 

1004 

1005 print(str(span_value)) 

1006 

1007 assert len(span_value) == 0 # 37 # je ne vois pas bien pourquoi la précedente requete en avait 2, on a l'air d'utiliser la base local de dev 

1008 

1009 print("Test done !") 

1010 

1011 # To move from file when possible ! 

1012def test_speech_to_text(snapshot): 

1013 """ 

1014 Test de transcription speech-to-text pour fichier .amr 

1015 

1016 Utilise les snapshots pour valider la structure complète du résultat. 

1017 Conserve des checks structurels pour vérifier que le résultat contient 

1018 les mots clés attendus du contenu audio. 

1019 """ 

1020 logger = logging.getLogger() 

1021 email = "anonymous@opio.fr" 

1022 object = "cr" 

1023# file = "/Users/moilerat/Documents/Fotonower/Safia/prompt/python/test/data/recording-2023-05-07-13-15-54.amr" 

1024 file = "test/data/recording-2023-05-07-13-15-54.amr" 

1025 

1026 list_files = [] 

1027 

1028 list_files.append({"from_mail":email, "object":object ,"file":file}) 

1029 

1030 lib_auth = LibUserDataExternalServices(lib_at = None) 

1031 

1032 from auth.lib_auth import get_datou_exec_context_as_complete_param_json 

1033 complete_param_json = get_datou_exec_context_as_complete_param_json(email, verbose=True, privacy=False, 

1034 openai_token=OPENAI_API_KEY, 

1035 lss=lss_singleton, project_id=project_nrt) 

1036 

1037 from lib.lib_util import select_datou_step_from_extension_type_upload 

1038 is_managed, list_actions_datou, extension = select_datou_step_from_extension_type_upload(file, ".amr", "on s'en fout dans ce cas") 

1039 

1040 input = list_files[0] 

1041 ref_text = "Please make a report with a few main themes, todos and follow-up of the following by keeping the same language :" 

1042 # 10-24 

1043 ref_text = "Please make a report with a few main themes, todos and follow-up of the following by keeping the same language :J'aimerais bien que FTA m'aide à gérer mes peurs et mes attentes et mes désirs. Des choses très personnelles. Du coup, je veux bien que tu me renvoies une petite liste de... Non, renvoie-moi rien, je t'enverrai des releases plus tard. Mais pour l'instant, on va juste voir si ça marche de respecter la RGPD là-dessus." 

1044 ref_text = "Please make a report with a few main themes, todos and follow-up of the following by keeping the same language :J'aimerais bien que FTA m'aide à gérer mes peurs et mes attentes et mes désirs. Des choses très personnelles. Du coup, je veux bien que tu me renvoies une petite liste de... Non, renvoie-moi rien, je t'enverrai des releases plus tard. Mais pour l'instant, on va juste voir si ça marche de respecter la RGPD là-dessus.J'aimerais bien que FTA m'aide à gérer mes peurs, mes attentes et mes désirs. Des choses très personnelles. Du coup, je veux bien que tu me renvoies une petite liste de... Non, renvoie-moi rien, je t'enverrai des releases plus tard. Mais pour l'instant, on va juste voir si ça marche de respecter la RGPD là-dessus." 

1045 input["preprompt"] = ref_text 

1046 

1047 from lib.datou.datou_exec import datou_exec 

1048 result, audit_json = datou_exec(list_actions_datou, input, complete_param_json) 

1049 

1050# from lib.lib_prompt import manage_prompt 

1051# result = manage_prompt(list_files=list_files, openai_token=OPENAI_API_KEY, 

1052# verbose=True, input=list_files[0]) # , debug=True 

1053 # , openai_model="gpt-3-turbo") 

1054 logger.info("Finished manage_prompt ! ") 

1055 print("Finished manage_prompt !") 

1056 

1057 logger.info(" result : " + str(result)) 

1058 

1059 # Vérifications structurelles importantes 

1060 assert len(result) == 10 #8 # 10 # 16 #15 # 14 #7 #9 #7 # 9 

1061 assert "request" in result 

1062 assert len(result["result"]) > 10 

1063 assert "peur" in result["result"].lower() or "attentes" in result["result"].lower() or "fear" in result["result"].lower() 

1064 

1065# "peurs, mes attentes" 

1066# "peurs et mes attentes" 

1067 

1068 # Validation complète via snapshot 

1069 assert result["request"][:450] == snapshot 

1070 

1071 

1072 

1073 # To move from file when possible ! 

1074def test_speech_to_text_m4a(snapshot): 

1075 """ 

1076 Test de transcription speech-to-text pour fichier .m4a 

1077 

1078 Utilise les snapshots pour valider la structure complète du résultat. 

1079 Conserve des checks structurels pour vérifier que le résultat contient 

1080 les mots clés attendus du contenu audio. 

1081 """ 

1082 logger = logging.getLogger() 

1083 email = "anonymous@opio.fr" 

1084 object = "cr" 

1085# file = "/Users/moilerat/Documents/Fotonower/Safia/prompt/python/test/data/recording-2023-05-07-13-15-54.amr" 

1086 file = "test/data/topgun_audio1681204600.m4a" 

1087 

1088 list_files = [] 

1089 

1090 list_files.append({"from_mail":email, "object":object ,"file":file}) 

1091 

1092 lib_auth = LibUserDataExternalServices(lib_at = None) 

1093 

1094 from auth.lib_auth import get_datou_exec_context_as_complete_param_json 

1095 complete_param_json = get_datou_exec_context_as_complete_param_json(email, verbose=True, privacy=False, 

1096 openai_token=OPENAI_API_KEY, 

1097 lss=lss_singleton, project_id=project_nrt) 

1098 

1099 from lib.lib_util import select_datou_step_from_extension_type_upload 

1100 is_managed, list_actions_datou, extension = select_datou_step_from_extension_type_upload(file, ".amr", "on s'en fout dans ce cas") 

1101 

1102 input = list_files[0] 

1103 input["preprompt"]="Please make a report with a few main themes, todos and follow-up of the following by keeping the same language :" 

1104 

1105 to_choose, list_action, extension = select_datou_step_from_extension_type_upload(file, ".m4a", "plop") 

1106 

1107 if to_choose: 

1108 from lib.datou.datou_exec import datou_exec 

1109 result, audit_json = datou_exec(list_actions_datou, input, complete_param_json) 

1110 

1111 # from lib.lib_prompt import manage_prompt 

1112 # result = manage_prompt(list_files=list_files, openai_token=OPENAI_API_KEY, 

1113 # verbose=True, input=list_files[0]) # , debug=True 

1114 # , openai_model="gpt-3-turbo") 

1115 logger.info("Finished manage_prompt ! ") 

1116 print("Finished manage_prompt !") 

1117 

1118 logger.info(" result : " + str(result)) 

1119 else : 

1120 result = {} 

1121 

1122 # Vérifications structurelles importantes 

1123 assert len(result) == 10 #8 # 10 # 16 #15 # 14 # 7 #9 

1124 assert "text" in result 

1125 assert len(result["result"]) > 10 

1126 assert "top gun" in result["result"].lower() 

1127 assert "maverick" in result["result"].lower() 

1128 

1129 request = result["request"] if "request" in result else None 

1130 

1131 assert request[230] == snapshot 

1132 

1133 # VR 6/2/26 : impossible de Validation complète via snapshot car il y a trop de variabilité dans le résultat (ex: "top gun : maverick" est parfois reconnu comme "top gun : maverick" et parfois comme "top gun : maverick" selon les runs, sans que l'on sache vraiment pourquoi, peut-être un problème de tokenisation ou de reconnaissance vocale qui est à la limite de la compréhension de l'IA) 

1134 ## assert result == snapshot 

1135 

1136 

1137 

1138 # To move from file when possible ! 

1139def test_z_image_to_text(snapshot): 

1140 """ 

1141 Test d'extraction de texte depuis une image (OCR) 

1142 

1143 Utilise les snapshots pour valider: 

1144 - La structure complète du résultat d'extraction 

1145 - Le texte extrait de l'image 

1146 - Les clés du dictionnaire retourné 

1147 

1148 Note: Les champs 'hash_id_treatment' peuvent être volatiles mais sont conservés 

1149 dans le snapshot pour validation complète. 

1150 """ 

1151 verbose = False 

1152 logger = logging.getLogger() 

1153 email = "anonymous@opio.fr" 

1154 object = "cr" 

1155 file = "test/data/ticket_macdo.jpg" 

1156 

1157 list_files = [] 

1158 

1159 list_files.append({"from_mail":email, "object":object ,"file":file}) 

1160 

1161 lss_singleton.truncate_project_documents(project_nrt, verbose=verbose) 

1162 logger.info("After truncate project documents") 

1163 

1164 lib_auth = LibUserDataExternalServices(lib_at = None) 

1165 

1166 from auth.lib_auth import get_datou_exec_context_as_complete_param_json 

1167 complete_param_json = get_datou_exec_context_as_complete_param_json(email, verbose=True, privacy=False, 

1168 openai_token=OPENAI_API_KEY, 

1169 lss=lss_singleton, project_id=project_nrt) 

1170 

1171 from lib.lib_util import select_datou_step_from_extension_type_upload 

1172 is_managed, list_actions_datou, extension = select_datou_step_from_extension_type_upload(file, ".jpg", "on s'en fout dans ce cas") 

1173 

1174 input = list_files[0] 

1175# input["preprompt"]="Please make a report with a few main themes, todos and follow-up of the following by keeping the same language :" 

1176 

1177 list_actions_datou.append("result_to_json") 

1178 list_actions_datou.append("import_json") 

1179# list_actions_datou.append("json_") 

1180 

1181 from lib.datou.datou_exec import datou_exec 

1182 result, audit_json = datou_exec(list_actions_datou, input, complete_param_json) 

1183 

1184 logger.info("Finished manage_prompt ! ") 

1185 print("Finished manage_prompt !") 

1186 

1187 logger.info(" result : " + str(result)) 

1188 logger.info(" result.text : " + str(result["text"])) 

1189 

1190 # Vérifications structurelles importantes 

1191 assert len(result) == 26 # 34 #33 #24 #13 #9 # 13 #9 # 13 

1192 assert "text" in result 

1193 

1194 # Validation du projet après import 

1195 info_project = lss_singleton.get_project_info(project_nrt) 

1196 logger.info(" after get_project_info ") 

1197 logger.info(" info_project : " + str(info_project)) 

1198 print(" info_project : " + str(info_project)) 

1199 

1200 assert info_project["number_documents"] >= 1 and info_project["number_documents"] < 12 

1201 # gpt a sans doute rallonger son blabla ! 

1202 

1203 assert "mcchicken" in result["text"].lower() 

1204 assert "nuggets" in result["text"].lower() 

1205 assert "petit frite" in result["text"].lower() 

1206 

1207 # Validation complète via snapshot 

1208 assert result["request"] == snapshot 

1209 

1210 

1211 

1212@pytest.mark.asyncio 

1213async def test_process_json(): 

1214 proj_nrt = project_nrt 

1215 

1216 from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys 

1217 lpgss = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT) 

1218 

1219 lss = LibSafiaSystem(lib_user_data_internal=lpgss) 

1220 

1221 curr_project_id = proj_nrt 

1222 

1223 import logging 

1224 logger = logging.getLogger() 

1225 logger.info(" before get_project_info ") 

1226 info_project = lss.get_project_info(curr_project_id) 

1227 logger.info(" after get_project_info ") 

1228 

1229 logger.info(" info_project : " + str(info_project)) 

1230 

1231 PG_TABLE = info_project["table_documents"] 

1232 

1233# del info_project["number_documents"] 

1234 

1235 json_to_import_path = "test/data/short_test_7JEI_RESCRIT.json" 

1236 

1237 from lib.import_util.lib_import_retrieval.scripts.process_json.process_json import process_json_dump 

1238 logger.info(" before process_json_dump : json_to_import_path : " + str(json_to_import_path)) 

1239# asyncio.run(process_json_dump(json_to_import_path, {}, False, False, None, lpgss, OPENAI_API_KEY, PG_TABLE, verbose = True)) 

1240 result = await process_json_dump(json_to_import_path, {}, False, False, None, lpgss, OPENAI_API_KEY, PG_TABLE, verbose = True) 

1241 

1242 logger.info(" result process_json : " + str(result)) 

1243 

1244 assert 0 == 0 

1245 

1246 

1247@pytest.mark.asyncio 

1248async def test_insert_json_datastore(): 

1249 logger = logging.getLogger() 

1250 # /Users/moilerat/Documents/Fotonower/Safia/prompt/python/ 

1251 filepath = "test/data/extract_for_safia_Doc_Issues.json" 

1252 

1253 custom_metadata = {} 

1254 screen_for_pii = False 

1255 extract_metadata = False 

1256 datastore = None # hack while integrating 

1257 verbose = True 

1258 project_id = project_nrt 

1259 pg_table = "victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc" 

1260 

1261# export PYTHONPATH=$PYTHONPATH:`pwd`/../../..:`pwd` cd $GITSAFIA/python/lib/import_util/lib_import_retrieval; cd python/lib/import_util/lib_import_retrieval; export PYTHONPATH=`pwd`/../../..:`pwd` python3.10 scripts/process_json/process_json.py --pg_table=victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc --filepath=../../../io/Fotonower_Issues.json 

1262 

1263 from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys 

1264 lpgss = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT) 

1265 

1266 curr_project_id = project_nrt 

1267 lss_singleton.truncate_project_documents(curr_project_id, verbose=verbose) 

1268 logger.info("After truncate project documents") 

1269 

1270 # pas sur du await 

1271 #await 

1272 from lib.import_util.lib_import_retrieval.scripts.process_json.process_json import process_json_dump 

1273# result, nb_token, model 

1274 result = await process_json_dump( 

1275 filepath, custom_metadata, screen_for_pii, extract_metadata, datastore, 

1276 lpgss = lpgss, 

1277 OPENAI_API_KEY = OPENAI_API_KEY, 

1278 PG_TABLE = pg_table, 

1279 verbose = verbose 

1280 ) 

1281 

1282 logger.info("Finished process_json_dump NOW need to test what append to avoid having a dummy test ! ") 

1283 print("Finished manage_prompt !") 

1284 

1285 logger.info(" result : " + str(result)) 

1286 

1287 from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys 

1288 lpgss = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT) 

1289 

1290 logger.info(" Before getting get_project_info ") 

1291 info_project = lss_singleton.get_project_info(curr_project_id) 

1292## truncate_project_documents(curr_project_id, lpgss, user) 

1293 # lss.update_project_costs(project_id, nb_token, model, lpgss) 

1294 

1295 logger.info(" info_project : " + str(info_project)) 

1296 

1297 assert info_project["number_documents"] > 20 # 30 # 50 # 30 # == 35 # old 60 < == 63 #62 #63 #64 #62 #61 #62 # 61 #62 #61 # 62 # 61 #60 # 61 # 62 # 60 # 35 # finalement ? 67 #35 # 28 # 84 # 28 when run alone # Why did it change => because no systematic truncate (easy to do (now) or not due to failure in truncate) 

1298 assert info_project["number_documents"] < 30 # 45 #65 # 40 #65 

1299 

1300# delete_document 

1301 

1302# VR deprecate 29-1-26 

1303def te_st_create_setup(): 

1304 

1305 import logging 

1306 logger = logging.getLogger() 

1307 logger.info(" PG_DB : " + str(PG_DB)) 

1308 

1309 PG_DB_NON_REG_TEST = PG_DB + "_nrt" 

1310 from test.lib_test_util import read_and_modify_migration_for_db_test 

1311 path_to_file_migration_db = "../devops/migration/v0/20230603_safia_user.psql" 

1312 migration_db_test, create_db_test = read_and_modify_migration_for_db_test(path_to_file_migration_db, DB_NAME_PROD = PG_DB, DB_NAME_TEST = PG_DB_NON_REG_TEST) 

1313 

1314 from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys 

1315 lpgss = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, "postgres", PG_PORT) 

1316 

1317 import os 

1318 # """ PGPASSWORD=""" + str(PG_PASSWORD) + 

1319# command_pysql_creation = """ psql -p """ + str(PG_PORT) + """ -h localhost -U safia -d postgres -c "DROP DATABASE IF EXISTS mtr_nrt;" -c "CREATE DATABASE mtr_nrt;" """ 

1320# os.system(command_pysql_creation) 

1321 LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT).create_statement(f"DROP DATABASE IF EXISTS {PG_DB_NON_REG_TEST};") 

1322 LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT).create_statement(f"CREATE DATABASE {PG_DB_NON_REG_TEST};") 

1323 

1324 

1325#def create_database(dbname, user, password, host): 

1326# conn = psycopg2.connect(dbname='postgres', user=user, password=password, host=host) 

1327# conn.autocommit = True # Autocommit must be on to create databases 

1328# cur = conn.cursor() 

1329 

1330 # Use SQL string composition as psycopg2 does not allow parameterized table creation 

1331# cur.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(dbname))) 

1332 

1333# cur.close() 

1334# conn.close() 

1335 

1336# DEbugging on 11-11-24 

1337 # """ PGPASSWORD=""" + str(PG_PASSWORD) + 

1338# create_extension_with_admin_role = """ psql -p """ + str(PG_PORT) + """ -h localhost -d mtr_nrt -c "CREATE EXTENSION vector;" """ 

1339# os.system(create_extension_with_admin_role) 

1340 try: 

1341 LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB_NON_REG_TEST, PG_PORT).create_statement("CREATE EXTENSION vector;") 

1342 except Exception as e: 

1343 print(" e : " + str(e)) 

1344 print("En fait c'est dejà fais donc on s'en fout !") 

1345 create_extension_with_admin_role = """ psql -p """ + str( 

1346 PG_PORT) + """ -h localhost -d mtr_nrt -c "CREATE EXTENSION vector;" """ 

1347 os.system(create_extension_with_admin_role) 

1348 

1349# lpgss.create_statement(create_db_test) 

1350 lpgss_nrt = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB_NON_REG_TEST, PG_PORT) 

1351 

1352 lpgss_nrt.create_statement(migration_db_test) 

1353 

1354 from lib.lib_airtable import LibAirTable 

1355 

1356 (api_key_at, base_id_apia) = lcs.get_at_apia_conf() 

1357 (api_key_at, base_id_pi) = lcs.get_at_pe_conf() 

1358 

1359 latPE = LibAirTable(api_key_at, base_id_pi) 

1360 lib_auth = LibUserDataExternalServices(lib_at=latPE) 

1361 

1362 otp = "c6ae5f7095abfe9ac59e75fc6624cea56038d9da" 

1363 # non_reg_test 

1364 

1365 info_auth, is_valid = lib_auth.connect_with_otp(otp) 

1366 

1367 # TODO VR deprecate get_data_user 

1368 if is_valid: 

1369 

1370 lss_for_new_setup = LibSafiaSystem(lib_user_data_internal=lpgss_nrt) 

1371 

1372 info_setup = lss_for_new_setup.setup_user(info_auth['Email_User'], info_auth["Description"], "otp_apia", otp) 

1373 

1374 logger.info(str(info_setup)) 

1375 print(str(info_setup)) 

1376 

1377 data_user = lss_for_new_setup.get_data_user(info_auth['Email_User']) 

1378 

1379 print(" data_user : " + str(data_user)) 

1380 

1381 assert data_user[0]["table_documents"] == "victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc" 

1382 else : 

1383 message = "Please extend otp nonregtest validity" 

1384 print(message) 

1385 logger.info(message) 

1386 assert 0 == 1 

1387 assert message == "" 

1388 

1389 

1390 

1391 print(" TO FINISH ") 

1392 

1393 

1394# il faut tester 

1395# il faut un autre lss_singleton 

1396# lss_singleton.use_invitation() 

1397 

1398 

1399def test_mail_from_prompt_apia(snapshot): 

1400 """ 

1401 Test de récupération et traitement d'emails depuis un serveur mail 

1402 

1403 Utilise les snapshots pour valider: 

1404 - La structure complète du résultat (array d'emails traités) 

1405 - Le contenu de chaque email 

1406 - Les métadonnées (log_git_action, etc.) 

1407 

1408 Note: Ce test dépend d'un serveur mail externe et peut échouer si la boîte 

1409 mail est vide ou si les credentials changent. 

1410 """ 

1411 import logging 

1412 logger = logging.getLogger() 

1413 

1414# username = "promptd@fotonower.com" 

1415 username = "promptd@safia.app" 

1416 password = "alalaoui" 

1417 

1418 from lib.lib_prompt import prompt_from_mail 

1419 server = "mail.gandi.net" 

1420 port = 995 

1421 from lib.lib_airtable import LibAirTable as lat 

1422 lib_at = lat() 

1423 # safia_nrt@rubbia.fr 

1424 result = prompt_from_mail(server, username, password, port, openai_token=OPENAI_API_KEY, debug = True, lib_connect=lib_at) 

1425 

1426 print(" result : " + str(result)) 

1427 

1428 logger.info(" result : " + str(result)) 

1429 

1430 # Vérifications structurelles importantes 

1431 assert len(result) == 0 # 2 

1432 if len(result) > 0: 

1433 assert "result" in result[0] 

1434 assert result[0]['log_git_action'] == 'git_action_done' 

1435 assert 'log_git_action' not in result[1] 

1436 

1437 # Validation complète via snapshot 

1438 assert result == snapshot 

1439 

1440def test_import_all(): 

1441 from lib.lib_util import import_all_for_coverage_with_zero_percent 

1442 import_all_for_coverage_with_zero_percent() 

1443 

1444def test_multiple_safia_import_for_batch_integration_dummy(): 

1445 

1446 from lib.lib_safia import safia_import 

1447 in_folder = "test/data/multiple_import" 

1448 out_file_json, list_detailed_time, unmanaged = safia_import(in_folder) 

1449 # main(['foo', '-f','v']) 

1450 

1451 # TODO VR reload out_file_json 

1452# assert out_file_json == [] 

1453 

1454 from lib.import_util.lib_import_retrieval.scripts.process_json.process_json import main as main_pj 

1455 sys.argv = ["this", '--filepath', out_file_json, "--project_id", 0, "--pg_table", "victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc"] 

1456# asyncio.run(main_pj(['--filepath', out_file_json, "project_id", 0, "--pg_table", "victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc"])) 

1457 asyncio.run(main_pj()) 

1458 

1459 print("TO TEST ! :-)") 

1460 

1461 

1462 

1463#def suspendest_get_project_info(): 

1464def test_get_project_info(snapshot): 

1465 """ 

1466 Test de récupération des informations d'un projet. 

1467 

1468 Ce test utilise les snapshots pour comparer la structure complète 

1469 du projet retourné par get_project_info. 

1470 

1471 Champs exclus de la comparaison (valeurs volatiles): 

1472 - number_documents: varie selon les imports 

1473 - latest_update: timestamp changeant 

1474 - cost: coût de traitement variable 

1475 """ 

1476 from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys 

1477 lpgss = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT) 

1478 

1479 lss = LibSafiaSystem(lib_user_data_internal=lpgss) 

1480 

1481 curr_project_id = project_nrt 

1482 

1483 import logging 

1484 logger = logging.getLogger() 

1485 logger.info(" before get_project_info ") 

1486 info_project = lss.get_project_info(project_nrt) 

1487 logger.info(" after get_project_info ") 

1488 

1489 logger.info(" info_project : " + str(info_project)) 

1490 

1491 # Suppression des champs volatiles avant comparaison snapshot 

1492 del info_project["number_documents"] 

1493 

1494 import datetime 

1495 

1496 latest_update = info_project["latest_update"] 

1497 now = datetime.datetime.now() 

1498 logger.info(" latest_update : " + str(latest_update)) 

1499 latest_update = latest_update.replace(tzinfo=None) 

1500 logger.info(" latest_update : " + str(latest_update)) 

1501 logger.info(" now : " + str(now)) 

1502 diff_latest = now - latest_update 

1503 logger.info(" diff_latest : " + str(diff_latest)) 

1504 

1505 logger.info(" info_project[cost] : " + str(info_project["cost"])) 

1506 

1507 del info_project["latest_update"] 

1508 

1509 cost = info_project["cost"] 

1510 del info_project["cost"] 

1511 

1512 logger.info(" cost project : " + str(cost)) 

1513 # Validation séparée du coût (non inclus dans snapshot) 

1514 # Ces assertions sont commentées car le coût est variable 

1515 # assert cost < 0.13 

1516 # assert cost > 0.125 

1517 

1518 # Comparaison avec snapshot au lieu de dictionnaire hardcodé 

1519 assert info_project == snapshot 

1520 

1521# TODO VR 23-11-23 finish this test 

1522def te_st_run_kmean_hierarchical_dummy(): 

1523 #sys.path.append(os.path.join(os.getenv("GITVELOURS"), "c_visual_search/pyfvs/install")) 

1524 if "GITPYFVS" in os.environ: 

1525 sys.path.append(os.getenv("GITPYFVS")) 

1526 from pyfvs.lib.lib_pyfvs_from_safia_to_refacto import FVSInterface 

1527 #from lib.lib_ml.lib_tent_py_fvs import FVSInterface 

1528 

1529# from lib.lib_ml.lib_kmean_pyfvs import query_and_output_of_kmeans 

1530 from pyfvs.lib.advanced.lib_split_elbow import query_and_output_of_kmeans 

1531 fvs_data = "data/data_fvs" 

1532 fvs_interface = FVSInterface(fvs_data_folder=fvs_data) 

1533 

1534 # {"limit" : limit, "pdt" : pdt, "out_folder" : out_folder, 

1535 # "builded_at" : datetime.datetime.now()} (et les params de list_pdt_params) 

1536 

1537 pdt = 450 

1538 limit = 1000 

1539 out_folder = "data/fvs_data" 

1540 

1541 dimension = 4 # TODO AVEC FVSS on ré-écrit le conf file par defaut (et puis je ne l'avais pas commité, merdum) 

1542 fvs_interface.data_.list_pdt_params[pdt] = {"dimension": dimension, "int_or_float": True, "photo_desc_type": pdt} 

1543 

1544 fvs_interface.data_.list_fvs_builded[pdt] = {"limit" : limit, "pdt" : pdt, "out_folder" : out_folder} 

1545# fvs_interface.data_.list_fvs_running[str(port)] = {"port": port, "pdt": pdt} 

1546 

1547 # VR TODO REFACTO on pourrait chercher cette fonction depuis pyfvs ou mettre le fonctionnement dans une fonction interne 

1548 from lib.lib_util import get_random_port 

1549 port = get_random_port() 

1550 # VR 19-11-23 : not sure if out_folder is correct and the same 

1551 

1552 from auth.lib_conf_system import lcs_global_singleton 

1553 fvs_path = lcs_global_singleton.get_fvs_path() 

1554 

1555 port, p = fvs_interface.launch_fvs(port, pdt, limit, fvs_exe=fvs_path) 

1556 

1557 nb_step = 2 

1558 nb_new_cluster = 1 # at each step 

1559 

1560 xmin, xmax, ymin, ymax = None, None, None, None 

1561 

1562 save_cluster = "" 

1563 load_cluster = "" 

1564 verbose = False 

1565 methodo = "split_one" 

1566 nb_cluster_total_elbow = 4 

1567 gpu_enable = False 

1568 type_divergence = "E" 

1569 

1570 metric = "E" 

1571 keep_fvs_running = True 

1572 build_fvs_desc = True 

1573 dim = 32 

1574 nb_project = 1 

1575 list_project_ids = [1] 

1576 launch_fvs = True 

1577 filename = "" 

1578 # VR 23-11-23 : ca c'est deja testé 

1579 filename, list_legend_centroids, list_url_centroids, list_example_media, misc_info = lss_singleton.kmean_2d(list_project_ids, limit=limit, 

1580 nb_project=nb_project, filename = filename, 

1581 metric = metric, dim = dim, 

1582 nb_cluster_input = nb_new_cluster, # Ca c'est inutile ! 

1583 gpu_enable = gpu_enable, 

1584 nb_step = nb_step, 

1585 keep_fvs_running = keep_fvs_running, 

1586 port = port, 

1587 pdt = pdt, 

1588 build_fvs_desc = build_fvs_desc, 

1589 save_cluster = save_cluster, 

1590 load_cluster = load_cluster, 

1591 launch_fvs = launch_fvs, 

1592 verbose = verbose, 

1593 fvs_interface = fvs_interface) 

1594 

1595 port = misc_info["port"] 

1596 

1597 ret = query_and_output_of_kmeans(fvs_interface=fvs_interface, 

1598 port=port, 

1599 nb_step=nb_step, 

1600 # (fvs_interface : FVSInterface, port, nb_step, 

1601 save_cluster=save_cluster, 

1602 load_cluster=load_cluster, 

1603 nb_new_cluster=nb_new_cluster, 

1604 verbose=verbose, 

1605 xmin=xmin, 

1606 xmax=xmax, 

1607 ymin=ymin, 

1608 ymax=ymax, 

1609 with_indices=True, 

1610 type_divergence=type_divergence, 

1611 methodo=methodo, 

1612 nb_cluster_total_elbow=nb_cluster_total_elbow, 

1613 gpu_enable=gpu_enable) 

1614 

1615 print(" ret : " + str(ret)) 

1616 

1617 fvs_interface.kill() 

1618 

1619# import re 

1620 # définir l'expression régulière 

1621# pattern = r'static\/temp\_tsne\/one\_gif\_(0-9a-f)*8*)\n' 

1622 # compiler l'expression régulière 

1623# pattern = re.compile(pattern, re.MULTILINE) 

1624 # chercher cette expression dans le texte 

1625# match = pattern.search(ret) 

1626 

1627## assert match 

1628 assert ret.startswith("static/temp_tsne/one_gif_") 

1629 

1630 

1631def te_st_tentative_pyfvs_launch_fvs_and_kmean(): 

1632 import os, sys 

1633 fvs_data = os.path.join(os.path.dirname(__file__), "data/fvs_data") 

1634 output_image_folder = fvs_data 

1635 out_folder = fvs_data 

1636 if "GITPYFVS" in os.environ: 

1637 sys.path.append(os.getenv("GITPYFVS")) 

1638 from pyfvs.lib.lib_pyfvs_from_safia_to_refacto import FVSInterface 

1639 fvs_interface = FVSInterface(fvs_data_folder=fvs_data) 

1640 

1641 try : 

1642 pdt = 450 

1643 limit = 100 

1644 

1645 dimension = 4 # TODO AVEC FVSS on ré-écrit le conf file par defaut (et puis je ne l'avais pas commité, merdum) 

1646 fvs_interface.data_.list_pdt_params[pdt] = {"dimension": dimension, "int_or_float": True, 

1647 "photo_desc_type": pdt} 

1648 

1649 import os, sys 

1650 sys.path.append(os.path.join(os.getenv("GITSAFIA"), "prompt/python")) 

1651 from auth.lib_conf_system import lcs_global_singleton 

1652 fvs_path = lcs_global_singleton.get_fvs_path() 

1653 if fvs_path == None: 

1654 fvs_path = os.path.join(os.getenv("GITVELOURS"), "c_visual_search/Debug/fvs") 

1655 

1656 port, p = fvs_interface.launch_fvs(pdt, limit = limit, fvs_exe=fvs_path) 

1657 

1658 init_kmean = "" 

1659 nb_cluster = 2 

1660 

1661 import time 

1662 time.sleep(3) 

1663 

1664 sys.path.append(os.path.join(os.getenv("GITVELOURS"), "python")) 

1665 ret = fvs_interface.step_kmean(port = port, init_kmean = init_kmean, 

1666 nb_iteration = 1, nb_new_clusters = nb_cluster, 

1667 gpu_enable = False, verbose = False, 

1668 type_divergence ="E", nb_cluster_loaded = 0, 

1669 pdt = pdt) 

1670 

1671 print(" ret : " + str(ret)) 

1672 ret_non_reg = '|1:0;99:inf|864203888,864208055|-100,-101|864203888;864208055,864215831,864233932,864236843,864245809,864246304,864249886,864250040,864252041,864264302,864272105,864272285,864281550,864284308,864291942,864292694,864292696,864295205,864298494,864299664|peof' 

1673 assert ret_non_reg == ret 

1674 

1675 list_ids_proj_doc_id = [] 

1676 

1677 sys.path.append(os.path.join(os.getenv("GITVELOURS"), "python")) 

1678 one_step_tsne = fvs_interface.display_tsne(pdt, out_folder, list_ids_proj_doc_id, output_image_folder, 

1679 extension="", root_api="https://safia.app", 

1680 xmin=None, 

1681 xmax=None, 

1682 ymin=None, 

1683 ymax=None, 

1684 with_indices=False, 

1685 nb_cluster_expected=nb_cluster) 

1686 

1687 fvs_interface.kill() 

1688 

1689 one_step_tsne_non_reg = ('data/fvs_data/tsne_450_2_E_8cdb1fb1-311c-468c-89e8-b9a77707c1cf.png', ['Point 0, id : 0, nb : 1', 'Point 1, id : 1, nb : 99'], ['https://safia.appeditor?document_id=list_not_loaded&project_id=proj_not_loaded', 'https://safia.appeditor?document_id=list_not_loaded&project_id=proj_not_loaded'], -0.05500000000000001, 0.05500000000000001, -0.05500000000000001, 0.05500000000000001) 

1690 one_step_tsne_comp = one_step_tsne[1:] 

1691 one_step_tsne_non_reg_comp = one_step_tsne_non_reg[1:] 

1692 

1693 print("to compare") 

1694 assert one_step_tsne_non_reg_comp == one_step_tsne_comp 

1695 print("compared") 

1696 

1697 except Exception as e: 

1698 if str(e).startswith("assert"): 

1699 raise e 

1700 import logging 

1701 logger = logging.getLogger() 

1702 print(" str(e) : " + str(e)) 

1703 logger.info(" str(e) : " + str(e)) 

1704 assert 0 == 1 

1705 fvs_interface.kill() 

1706 

1707if __name__ == "__main__": 

1708 from test.use_case.test_extractia import test_extractia_anon, test_extractia 

1709 

1710 test_extractia() 

1711 test_extractia_anon() 

1712# test_mail_from_prompt_apia() 

1713# test_tentative_pyfvs_launch_fvs_and_kmean() 

1714# test_run_kmean_hierarchical_dummy() 

1715# test_logged_behavior_unlogged() 

1716# test_speech_to_text_m4a() 

1717# test_z_image_to_text() 

1718# test_speech_to_text() 

1719# asyncio.run(test_insert_json_datastore()) 

1720# 

1721# test_tentative_pyfvs_launch_fvs_and_kmean() 

1722# test_mail_from_prompt_apia() 

1723# test_get_project_info() 

1724# 

1725# 

1726# test_image_to_text() 

1727 #test_process_json() 

1728# test_create_setup() 

1729# test_mail_from_prompt_apia() 

1730# test_multiple_safia_import_for_batch_integration_dummy() 

1731 

1732 

1733 

1734 fcst = FirstClassSafiaTest() 

1735 fcst._pre_setup() 

1736 fcst.test_zz_setup_action_ok() 

1737 fcst.test_crash_reconnect() 

1738 fcst.test_kmean_multi_dim_dummy() 

1739 fcst.test_file_upload_and_import_git() 

1740# fcst.test_invitation() 

1741# fcst.test_logged_behavior_unlogged() 

1742 fcst.test_aaa_first_action_truncate_table() 

1743# 

1744# fcst.test_b_update_configuration_and_datou_dummy() 

1745# fcst.test_update_configuration_and_datou_dummy() 

1746# fcst.test_example() 

1747 

1748# fcst.test_aaa_first_action_truncate_table() 

1749 

1750# fcst.test_connected_ok() 

1751 

1752 # fcst.test_documents_page() 

1753 #fcst.test_connected_ok() 

1754 

1755 

1756# fcst.test_zz_setup_action_ok() 

1757 

1758 

1759 

1760# from flask.testing import FlaskClient 

1761# fcst.client = FlaskClient(fcst.app) 

1762# fcst.test_file_upload_and_import_git() 

1763 # 

1764# fcst.setUp() 

1765# fcst.setUpClass() 

1766# fcst.test_update_configuration_and_datou_dummy() 

1767# fcst.test_file_upload_and_import_git() 

1768 

1769