Coverage for test/main_test.py: 73%
828 statements
« prev ^ index » next coverage.py v7.9.1, created at 2026-02-18 02:40 +0100
« prev ^ index » next coverage.py v7.9.1, created at 2026-02-18 02:40 +0100
1# -*- coding: utf-8 -*-
2__author__ = 'moilerat'
4import shutil
5import time
7#
8# # Safia copyright 2022-2023 VR - LICENSETOBEDEFINED : on premise treatment custom
9#
10#! Package test main_test.py
11#! to be cleaned, but it is working
12#
14from flask_testing import TestCase
15from server.safia import app # import your Flask application
16import json
18import logging
19import pytest
20import asyncio
21import os, sys
23#sys.path.append(os.path.join(os.getenv("GITVELOURS"), "c_visual_search/pyfvs/install"))
24if "GITPYFVS" in os.environ:
25 sys.path.append(os.getenv("GITPYFVS"))
27project_nrt = 1
28#project_nrt = 3
29nrt_user_id = 1
31from lib.lib_safia_system import LibSafiaSystem
32from auth.lib_conf_system import LibConfSystem
33lcs = LibConfSystem()
34(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT) = lcs.get_pg_conf()
35OPENAI_API_KEY = lcs.get_openai_api_key()
36email_user = "anonymous@opio.fr"
38from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys
39lpgss_singleton = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT)
41from auth.lib_auth import LibUserDataExternalServices
42lib_auth_singleton = LibUserDataExternalServices(lib_at = None)
44from auth.lib_right import LibRight
45lib_right = LibRight(lib_internal_data=lpgss_singleton)
46lss_singleton = LibSafiaSystem(lib_user_data_internal=lpgss_singleton,
47 lib_user_data_external=lib_auth_singleton,
48 lib_right=lib_right)
52class FirstClassSafiaTest(TestCase):
53# def __init__(self):
54# super(self)
55# import logging
56# self.logger = logging.getLogger()
57# pass
60# def __init__(self):
61# super(self).__init__()
62# import logging
63# self.logger = logging.getLogger()
64# self.logger.info("FirstClassSafiaTest init")
65# # self.client = None
67 def setUp(self):
68 print("VR 9-6-25 : a mon avis il faut créer un user ! :-) et ca ne marchera plus quand on aura besoin de récupérer le code 2FA")
70 username = "vic"
71 # from server.bp.auth import create_login_bp.User
72 # user = User.query.filter_by(username=username).first()
73 from flask_login import login_user, UserMixin
74 user = UserMixin()
75 user.mtr_user_id = nrt_user_id
76 user.id = nrt_user_id
77 user.username = "victor@reutenauer.eu"
78 login_user(user)
80 response = self.client.post("/login",
81 # data={"username":"vic","password":"vic"},
82 data={"username": "moilerat", "password": "alaoui"},
83 content_type='multipart/form-data')
85 print(" Setup FirstClassSafiaTest " + str(type(self.client)))
88 def tearDown(self):
89 # _db.session.remove()
90 # _db.drop_all()
91 print(" tearDown FirstClassSafiaTest " + str(type(self.client)))
93 def create_app(self):
94 app.config['TESTING'] = True
95 return app
99 def test_aaa_first_action_truncate_table(self):
100 import logging
101 logger = logging.getLogger()
102 logger.info(str(type(self.client)))
104 logger.info(" PG_DB : " + str(PG_DB))
106 from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys
107 lpgss = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT)
109 curr_project_id = project_nrt
111 logger.info(" Before get_project_info ")
112 info_project = lss_singleton.get_project_info(curr_project_id)
113 logger.info(" Before truncate_project_documents (useless s'il y a des document_id unique ! ) ")
114 import time
115 time.sleep(3)
116 kill_all = True
117 ret = lss_singleton.truncate_project_documents(curr_project_id, kill_all = kill_all)
118 time.sleep(3)
119 logger.info(" After truncate_project_documents ")
121 lpgss.add_cost_to_project(curr_project_id, -0.1274092)
123 assert ret == "TRUNCATE"
125 def test_example(self):
126 response = self.client.get('/')
127 self.assertEqual(response.status_code, 200)
129 print(" response : " + str(response))
131 def test_documents_page(self, endpoint = "/docsmart",
132 search_into = "<h1>Documents Smart</h1>",
133 length_file = 1952,
134 filename = None,
135 save_comp = True):
136 self.client.set_cookie( 'otp', "c6ae5f7095abfe9ac59e75fc6624cea56038d9da")
137 self.client.set_cookie( 'project_id', str(project_nrt))
139 if filename == None:
140 filename = endpoint.lstrip("/")
142 response = self.client.get(endpoint)
143 self.assertEqual(response.status_code, 200)
145 if save_comp :
146 # Write the reference HTML content from the file
147 filepath = os.path.join("test/data/comp", filename)
148 if not os.path.exists("test/data/comp"):
149 os.makedirs("test/data/comp")
150 with open(filepath, 'w') as file:
151 file.write(response.data.decode('utf-8'))
153 # Read the reference HTML content from the file
154 filepath = os.path.join("test/data/ref", filename)
155 with open(filepath, 'r') as file:
156 reference_html = file.read()
158 import logging
159 self.logger = logging.getLogger()
160 self.logger.info(response.data)
161 self.logger.info(len(response.data))
163 # Compare the response data with the reference HTML
164 response_decoded = response.data.decode('utf-8')
165 if not save_comp:
166 self.assertEqual(response_decoded, reference_html, "Ca va pas du tout : " + str(response_decoded) + " VERSUCE" + response.data.decode("utf-8"))
168 assert len(response.data) == length_file
169 assert search_into in response.data.decode('utf-8')
170 else :
171 print(" len(response.data) : " + str(len(response.data)))
173 def test_connected_ok(self):
174 self.client.set_cookie( 'otp', "c6ae5f7095abfe9ac59e75fc6624cea56038d9da")
175 self.client.set_cookie( 'project_id', str(project_nrt))
177 response = self.client.get('/api/v1/project/queries/' + str(project_nrt))
178 self.assertEqual(response.status_code, 200)
179 response = self.client.post('/api/v1/project/select/' + str(project_nrt))
180 self.assertEqual(response.status_code, 200)
181 response = self.client.get('/project/admin/' + str(project_nrt))
182 self.assertEqual(response.status_code, 200)
184 response = self.client.get('/')
185 self.assertEqual(response.status_code, 200)
186 response = self.client.get('/account')
187 self.assertEqual(response.status_code, 200)
188 response = self.client.get('/datou')
189 self.assertEqual(response.status_code, 200)
190 response = self.client.get('/datou?json')
191 self.assertEqual(response.status_code, 200)
192 response = self.client.get('/documents.html')
193 self.assertEqual(response.status_code, 200)
194 response = self.client.get('/upload')
195 self.assertEqual(response.status_code, 200)
196 response = self.client.get('/doc')
197 self.assertEqual(response.status_code, 200)
199 response = self.client.get('/projects')
200 self.assertEqual(response.status_code, 200)
202 # TODO test
203 # - [ ] TODO VR on pourrait faire une boucle
204 def test_unlogged_ok(self):
205 response = self.client.get('/jscover')
206 self.assertEqual(response.status_code, 200)
207 if os.path.exists("temp_test_nrt_jscover"):
208 import shutil
209 shutil.rmtree("temp_test_nrt_jscover")
210 response = self.client.post('/jscoverage-store/temp_test_nrt_jscover', content_type='application/json', data=json.dumps({"just":"test"}))
211 self.assertEqual(response.status_code, 200)
212 response = self.client.get('/suiviprod/pg/admin')
213 self.assertEqual(response.status_code, 200)
215 response = self.client.post('/api/v1/form/getintouch', data=json.dumps({"just":"test"}), content_type='application/json')
216 self.assertEqual(response.status_code, 200)
217 self.assertEqual(response.json, {"Thank" : "you !"})
219 response = self.client.get('/landing')
220 self.assertEqual(response.status_code, 200)
222 response = self.client.get('/about')
223 self.assertEqual(response.status_code, 200)
224 response = self.client.get('/legal')
225 self.assertEqual(response.status_code, 200)
226 response = self.client.get('/privacy')
227 self.assertEqual(response.status_code, 200)
228 response = self.client.get('/terms')
229 self.assertEqual(response.status_code, 200)
230 response = self.client.get('/favicon.ico')
231 self.assertEqual(response.status_code, 200)
232# response = self.client.get('/zeitgeist.png')
233# self.assertEqual(response.status_code, 200)
235 def test_logged_behavior_unlogged(self):
236 response = self.client.get('/invitation/dummy_invitation_anyway')
237 self.assertEqual(response.status_code, 200)
238 self.you_have_no_valid = """
239 You have no valid OTP password or your are browsing anonymously.
240 Please create an account on <a href="https://www.openpromptia.com">opia</a> or https://opia.softr.app https://opia.softr.app/apia/assistant#otp2pe
241 Please create an account on <a href="https://reanna392.preview.softr.app/apia/assistant#otp2pe">opia</a> or https://reanna392.preview.softr.app/apia/assistant#otp2pe
242 and get an OTP password or EXTEND ITS EXPIRY DATE ON <a href="https://reanna392.preview.softr.app/apia/assistant#otp2pe">here</a> (valid 24 hours).
243 If you feel you got this message by error you can send and email to info@opio.fr .
244 Maybe openpromptia.com will come back soon
245 """
246 self.you_have_already = 'Invitation already used, but no date ?'
248 self.assertEqual(response.data.decode("utf-8"), self.you_have_already)
250 # - [ ] TODO VR c'est deux requetes doivent changer de comportement quand elles plantent
251 response = self.client.get('/api/v1/project/role/remove?project_id=1666&user_id_=1777')
252 self.assertEqual(response.status_code, 200)
254 response = self.client.get('/projects')
255 self.assertEqual(response.status_code, 200)
257 response = self.client.get('/project/admin/1666')
258 expected_content = """<!DOCTYPE html>
259 <html>
260 <head>
261 <title>Safia Project Admin</title>
262 </head>
263 <body>
264 <h1>Safia Project Admin</h1>
265 <table>
266 <tr>
267 <th>Project Name</th>
268 <th>Number of Documents</th>
269 <th>Size</th>
270 <th>Cost</th>
271 <th>Command</th>
272 </tr>
274 <tr>
275 <td></td>
276 <td>0</td>
277 <td>TO ESTIMATE</td>
278 <td></td>
279 <td></td>
280 </tr>
282 </table>
283 <a href="/documents.html">Listes complètes documents !</a>
284 </body>
285 </html>"""
286# self.assertEqual(expected_content, response.data.decode("utf-8"))
287 self.assertEqual(response.status_code, 200)
289 response = self.client.get('/editor')
290 self.assertEqual(response.status_code, 200)
291# self.assertEqual(response.data.decode("utf-8"), self.you_have_no_valid)
293 response = self.client.post('/api/v1/document/save')
294 self.assertEqual(response.status_code, 415)
296 response = self.client.post('/api/v1/document/save', data=json.dumps({}), content_type="application/json")
297 self.assertEqual(response.status_code, 200)
298 self.to_test_mets_a_jour = "TO TEST Mets à jour"
299# self.assertEqual(response.data.decode("utf-8"), self.to_test_mets_a_jour)
301 response = self.client.post('/api/v1/datou/delete')
302 self.assertEqual(response.status_code, 415)
303# self.assertEqual(response.data.decode("utf-8"), self.to_test_mets_a_jour)
305 response = self.client.post('/api/v1/datou/save')
306 self.assertEqual(response.status_code, 415)
307 # self.assertEqual(response.data.decode("utf-8"), self.to_test_mets_a_jour)
309 response = self.client.post('/api/v1/project/select/1666')
310 self.assertEqual(response.status_code, 200)
311 self.assertEqual(response.data.decode("utf-8"), '{"project_id": "1666"}')
313 response = self.client.get('/api/v1/project/detail/1666')
314 self.assertEqual(response.status_code, 200)
315 self.assertEqual(response.data.decode("utf-8"), '{"number_documents": 0, "latest_update": null, "size": "TO ESTIMATE", "created_at": "", "modified_at": null, "most_recent_date_h": "il y a 11 ans", "most_recent_date": "2014-09-06 00:00:00", "nb_doc_with_no_title": 0, "nb_doc_with_title": 0, "nb_doc_wo_info_on_title": 0}')
317 response = self.client.post('/api/v1/project/save', data=json.dumps({"just":"notlogged"}), content_type="application/json")
318 self.assertEqual(response.status_code, 200)
319 self.assertEqual(response.data.decode("utf-8"), '{\n "success": "Shortname updated"\n}\n')
321 try :
322 response = self.client.post('/api/v1/configuration/update', data=json.dumps({"just":"notlogged"}), content_type="application/json")
323 self.assertEqual(response.status_code, 200)
324 self.assertEqual(response.json, {})
325 except Exception as e:
326 print(str(e))
327 print("VR essaie de faire passer les tests avant de pouvoir les améliorer 6/2/26")
328 # assert False, "Should not raise exception"
330 response = self.client.get('/account')
331 self.assertEqual(response.status_code, 200)
332 self.assertTrue(len(response.data.decode("utf-8")) > 20000)
334 response = self.client.get('/doc')
335 self.assertEqual(response.status_code, 200)
336 self.assertTrue(len(response.data.decode("utf-8")) > 10000)
338 response = self.client.get('/datou')
339 self.assertEqual(response.status_code, 200)
340 self.assertTrue(len(response.data.decode("utf-8")) > 80000)
343 def te_st_crash_reconnect(self):
344 response = self.client.get('/')
345 self.assertEqual(response.status_code, 200)
347 response = self.client.get('/bugr')
348 self.assertEqual(response.status_code, 200)
350 result_ref = """syntax error at or near "duplicate"
351LINE 2: VALUES (49, 9, 'admin') on duplicate(group_i...
352 ^
354INTERNAL ERROR CAUSED BY MANUAL TEST (AND TO CHECK suiviprod) could be INTERNALAL ERROR otherwise"""
356# None reconnected None
358 result_comp = response.data.decode("utf-8")
359 import logging
360 self.logger = logging.getLogger()
361 self.logger.info(result_comp)
362 self.logger.info(result_ref)
363 self.assertEqual(result_comp, result_ref)
365 except_on_index = False
366 try :
367 response = self.client.get('/')
368 self.assertEqual(response.status_code, 500)
369 except_on_index = False
370 except Exception as e:
371 print(str(e))
372 except_on_index = True
374 self.assertEqual(except_on_index, True)
376 response = self.client.get('/reconnect')
377 self.assertEqual(response.status_code, 200)
378 message_ref = """ERROR: current transaction is aborted, commands ignored until end of transaction block
379 reconnected None"""
380 message_ref = """None reconnected None"""
381 message_ref2 = """syntax error at or near "duplicate"
382LINE 2: VALUES (49, 9, 'admin') on duplicate(group_i...
383 ^
385INTERNALAL ERROR CAUSED BY MANUAL TEST"""
386 message_comp = response.data.decode("utf-8")
387 self.assertEqual(message_comp, message_ref)
389 response = self.client.get('/')
390 self.assertEqual(response.status_code, 200)
392 def test_zz_setup_action_ok(self):
393 otp = "c6ae5f7095abfe9ac59e75fc6624cea56038d9da"
394 response = self.client.get("/otp/" + otp)
395 print(str(response))
396 self.assertEqual(response.status_code, 200) # ca ca marche plus de toute facon VR 6-6-25 # redirect
398 response = self.client.get("/otp/incorrectotp")
399 print(str(response))
400 self.assertEqual(response.status_code, 200) # redirect
402 def test_b_update_configuration_and_datou_dummy(self):
403 proj_nrt = project_nrt
405 logger = logging.getLogger()
406 # TODO VR le cookie va expirer !
407 self.client.set_cookie( 'otp', "c6ae5f7095abfe9ac59e75fc6624cea56038d9da")
408 self.client.set_cookie( 'project_id', str(proj_nrt))
410 data_update_configuration = {"nlp_chat" : "openai_gpt_api"}
412# response = self.client.post('/api/v1/configuration/update',
413# content_type='multipart/form-data',
414# data={'file': (f, 'lettre1erMoisCompletude-08-07-2022.pdf'), 'data': json.dumps(data)},
415# follow_redirects=True)
417 import json
418 response = self.client.post('/api/v1/configuration/update',
419 content_type='application/json',
420# content_type='multipart/form-data',
421 data=json.dumps(data_update_configuration))
423 logger.info("response : " + str(response))
424 self.assertEqual(response.status_code, 200)
426# data_update_datou = [{"id" : 1,
427# "param_json" : {"retest" : "etontest"}}]
429 data_update_datou = {"id" : 73, "name":"test_Datou", "list_steps" : [{"datou_step_type_id":1, "param_json":{"mont":"juiq"}, "order":1}]}
431 response = self.client.post('/api/v1/datou/save',
432 content_type='application/json',
433 data=json.dumps(data_update_datou))
435 logger.info("response : " + str(response))
436 self.assertEqual(response.status_code, 200)
438 logger.info(" PG_DB : " + str(PG_DB))
440 from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys
441 lpgss = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT)
443 datous = lpgss.get_datou_user(user_id=nrt_user_id)
444 logger.info(str(datous))
445# del datou["id"]
446 my_datou = list(filter(lambda x : x["filename"] == "test_Datou", datous))[0]
447 del my_datou["id"]
448 del my_datou["created_at"]
449 del my_datou["modified_at"]
450# for s in my_datou["list_steps"]:
451# del my_datou["list_steps"]["id"]
453 # RealDictRow([('user_id', 1), ('filename', 'test_Datou'), ('dot_file', None), ('public', False)])
455 datou_ref = {'owner_id' : 3, 'filename' :'test_Datou', 'dot_file': None, 'public' : True }
457 assert my_datou == datou_ref
459 data_update_datou = {"name":"new_datou_to_insert_and_delete", "list_steps" : [{"datou_step_type_id":1, "param_json":{"mont":"juiq"}, "order":1}]}
460 response = self.client.post('/api/v1/datou/save',
461 content_type='application/json',
462 data=json.dumps(data_update_datou))
463 self.assertEqual(response.status_code, 200)
465 # Comment fais t'on : On va choper le dernier ? on recherche par le nom ?
466 # Ou plutot on charge et on fait la recherche ici, mouais d'accord
468 # delete_datou or better to test
470 datous = lpgss.get_datou_user(user_id=nrt_user_id)
471 logger.info(str(datous))
472 my_datou = list(filter(lambda x : x["filename"] == "new_datou_to_insert_and_delete", datous))[0]
474 response = self.client.post('/api/v1/datou/delete',
475 content_type='application/json',
476 data=json.dumps({"id" : my_datou["id"]}))
477 self.assertEqual(response.status_code, 200)
479 datous = lpgss.get_datou_user(user_id=nrt_user_id)
481 list_datou_name = list(map(lambda x : x["filename"], datous))
483 assert "new_datou_to_insert_and_delete" not in list_datou_name
485# test/main_test.py::FirstClassSafiaTest::test_file_upload_and_import_git
486# @pytest.mark.asyncio
487# async def suspendest_file_upload(self):
488 #async
489# - [ ] TODO VR renommer ce test qui fait pas mal de choses
490 def test_file_upload_and_import_git(self):
491 proj_nrt = project_nrt
492 verbose = False
494 logger = logging.getLogger()
495# response = self.client.get('/')
496 # TODO VR le cookie va expirer !
497 self.client.set_cookie( 'otp', "c6ae5f7095abfe9ac59e75fc6624cea56038d9da")
498 self.client.set_cookie( 'project_id', str(proj_nrt))
499# import io
500# data = {
501# 'file': (io.BytesIO(b"my file contents"), 'test_file.txt'),
502# }
504 logger.info(" PG_DB : " + str(PG_DB))
507 from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys
508 lpgss = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT)
510 curr_project_id = project_nrt
512 logger.info(" Before get_project_info ")
513 info_project = lss_singleton.get_project_info(curr_project_id)
514 logger.info(" Before truncate_project_documents (useless s'il y a des document_id unique ! ) ")
515 import time
516 time.sleep(1)
517 lss_singleton.truncate_project_documents(curr_project_id, verbose=verbose)
518 time.sleep(1)
519 logger.info(" After truncate_project_documents ")
521 response = self.client.post("/login",
522 # data={"username":"vic","password":"vic"},
523 data={"username": "loc", "password": "loc"},
524 content_type='multipart/form-data')
525 # multipart/form-data. Otherwise, use application/x-www-form-urlencoded.
526 self.assertEqual(response.status_code, 200)
527 print(str(response))
529 username = "vic"
530# from server.bp.auth import create_login_bp.User
531# user = User.query.filter_by(username=username).first()
532 from flask_login import login_user, UserMixin
533 user = UserMixin()
534 user.mtr_user_id = 1
535 user.id = 1
536 user.username = "loc"
537 login_user(user)
539 try:
540 list = lpgss.get_usage_connection_ip_info()
541 except Exception as e:
542 print(str(e))
544 data_update_info = {'project_id' : curr_project_id,'shortname': "default"}
545 response = self.client.post("/api/v1/project/save",
546 data=json.dumps(data_update_info),
547 content_type='application/json')
548 self.assertEqual(response.status_code, 200)
550 response = self.client.get("/api/v1/project/detail/" + str(curr_project_id))
551 self.assertEqual(response.status_code, 200)
552 print(str(response))
553# print(str(response.json))
554 # VR TODO a mon avis le data store_in_project est inutile voir nefaste
555 data_json_ref ={'id': 1, 'owner_id': 1, 'description': 'non_reg_test_0623', 'table_documents': 'victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc', 'function_search_documents': 'match_c6ae5f7095abfe9ac59e75fc6624cea56038d9da', 'format': None, 'taille': None, 'created_at': '06/11/2023 04:30:41', 'modified_at': '06/11/2023 04:30:41', 'configuration': {'data': '{"store_in_project": "true", "project_id": "1"}', 'custom_display': {'data': '{"store_in_project": "true", "project_id": "1"}', 'home': {'data': '{"store_in_project": "true", "project_id": "1"}'}}}, 'shortlink': 'default', 'number_documents': 0, 'size': 'TO ESTIMATE', 'command': '\n cd $GITSAFIA/prompt/python/lib/import_util/lib_import_retrieval;\n export PYTHONPATH=`pwd`/../../..:`pwd`;\n python3.10 scripts/process_json/process_json.py --project_id=1 --pg_table=victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc --filepath=../../../io/Fotonower_Issues.json\n ', 'nb_doc_with_no_title': 0, 'nb_doc_with_title': 0, 'nb_doc_wo_info_on_title': 0,
556 'most_recent_date_h': 'il y a 8 secondes',
557 'most_recent_date': '2024-01-26 01:39:42.839445+01:00',
558 'latest_update': None}
559 # 30/10/24
560 data_json_ref = {'command': '\n'
561 ' cd '
562 '$GITSAFIA/prompt/python/lib/import_util/lib_import_retrieval;\n'
563 ' export PYTHONPATH=`pwd`/../../..:`pwd`;\n'
564 ' python3.10 scripts/process_json/process_json.py '
565 '--project_id=1 '
566 '--pg_table=victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc '
567 '--filepath=../../../io/Fotonower_Issues.json\n'
568 ' ',
569 'configuration': {'command': '\n'
570 ' cd '
571 '$GITSAFIA/prompt/python/lib/import_util/lib_import_retrieval;\n'
572 ' export '
573 'PYTHONPATH=`pwd`/../../..:`pwd`;\n'
574 ' python3.10 '
575 'scripts/process_json/process_json.py '
576 '--project_id=1 '
577 '--pg_table=victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc '
578 '--filepath=../../../io/Fotonower_Issues.json\n'
579 ' ',
580 'configuration': {'custom_display': {'data': '{"store_in_project": '
581 '"true", '
582 '"project_id": '
583 '"1"}',
584 'home': {'data': '{"store_in_project": '
585 '"true", '
586 '"project_id": '
587 '"1"}'}},
588 'data': '{"store_in_project": "true", '
589 '"project_id": "1"}'},
590 'created_at': '06/11/2023 04:30:41',
591 'custom_display': {'home': {'data': '{"store_in_project": '
592 '"true", "project_id": '
593 '"1"}'}},
594 'description': 'non_reg_test_0623',
595 'format': None,
596 'function_search_documents': 'match_c6ae5f7095abfe9ac59e75fc6624cea56038d9da',
597 'id': 1,
598 'latest_update': None,
599 'modified_at': '06/11/2023 04:30:41',
600 'most_recent_date': '2024-01-26 01:39:42.839445+01:00',
601 'most_recent_date_h': 'il y a 8 secondes',
602 'nb_doc_with_no_title': 0,
603 'nb_doc_with_title': 0,
604 'nb_doc_wo_info_on_title': 0,
605 'number_documents': 0,
606 'owner_id': 1,
607 'shortlink': 'default',
608 'size': 'TO ESTIMATE',
609 'table_documents': 'victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc',
610 'taille': None},
611 'created_at': '06/11/2023 04:30:41',
612 'description': 'non_reg_test_0623',
613 'format': None,
614 'function_search_documents': 'match_c6ae5f7095abfe9ac59e75fc6624cea56038d9da',
615 'id': 1,
616 'modified_at': '06/11/2023 04:30:41',
617 'nb_doc_with_no_title': 0,
618 'nb_doc_with_title': 0,
619 'nb_doc_wo_info_on_title': 0,
620 'number_documents': 0,
621 'owner_id': 1,
622 'shortlink': 'default',
623 'size': 'TO ESTIMATE',
624 'table_documents': 'victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc',
625 'taille': None}
626 # DB main no project 1
627 data_json_ref = {'number_documents': 0, 'latest_update': None, 'size': 'TO ESTIMATE', 'created_at': '', 'modified_at': None, 'most_recent_date_h': 'il y a 11 ans', 'most_recent_date': '2014-09-06 00:00:00', 'nb_doc_with_no_title': 0, 'nb_doc_with_title': 0, 'nb_doc_wo_info_on_title': 0}
628 data_json_ref = {'id': 1, 'owner_id': 1, 'description': 'non_reg_test_0623', 'table_documents': 'victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc', 'function_search_documents': 'match_c6ae5f7095abfe9ac59e75fc6624cea56038d9da', 'format': None, 'taille': None, 'created_at': '06/11/2023 04:30:41', 'modified_at': '06/11/2023 04:30:41', 'cost': -21.699331852300062, 'configuration': {'id': 1, 'owner_id': 1, 'description': 'non_reg_test_0623', 'table_documents': 'victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc', 'function_search_documents': 'match_c6ae5f7095abfe9ac59e75fc6624cea56038d9da', 'format': None, 'taille': None, 'created_at': '06/11/2023 04:30:41', 'modified_at': '06/11/2023 04:30:41', 'configuration': {'data': '{"store_in_project": "true", "project_id": "1"}', 'custom_display': {'data': '{"store_in_project": "true", "project_id": "1"}', 'home': {'data': '{"store_in_project": "true", "project_id": "1"}'}}}, 'shortlink': 'default', 'number_documents': 0, 'size': 'TO ESTIMATE', 'command': '\n cd $GITSAFIA/prompt/python/lib/import_util/lib_import_retrieval;\n export PYTHONPATH=`pwd`/../../..:`pwd`;\n python3.10 scripts/process_json/process_json.py --project_id=1 --pg_table=victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc --filepath=../../../io/Fotonower_Issues.json\n ', 'nb_doc_with_no_title': 0, 'nb_doc_with_title': 0, 'nb_doc_wo_info_on_title': 0, 'most_recent_date_h': 'il y a 8 secondes', 'most_recent_date': '2024-01-26 01:39:42.839445+01:00', 'latest_update': None, 'custom_display': {'home': {'data': '{"store_in_project": "true", "project_id": "1"}'}}}, 'shortlink': 'default', 'number_documents': 0, 'latest_update': None, 'size': 'TO ESTIMATE', 'command': '\n cd $GITSAFIA/prompt/python/lib/import_util/lib_import_retrieval;\n export PYTHONPATH=`pwd`/../../..:`pwd`;\n python3.10 scripts/process_json/process_json.py --project_id=1 --pg_table=victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc --filepath=../../../io/Fotonower_Issues.json\n ', 'most_recent_date_h': 'il y a 11 ans', 'most_recent_date': '2014-09-06 00:00:00', 'nb_doc_with_no_title': 0, 'nb_doc_with_title': 0, 'nb_doc_wo_info_on_title': 0}
629 data_json_ref = {'id': 1, 'owner_id': 1, 'description': 'non_reg_test_0623', 'table_documents': 'victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc', 'function_search_documents': 'match_c6ae5f7095abfe9ac59e75fc6624cea56038d9da', 'format': None, 'taille': None, 'created_at': '06/11/2023 04:30:41', 'modified_at': '06/11/2023 04:30:41', 'shortlink': 'shortlink_nrt', 'number_documents': 0, 'size': 'TO ESTIMATE', 'command': '\n cd $GITSAFIA/prompt/python/lib/import_util/lib_import_retrieval;\n export PYTHONPATH=`pwd`/../../..:`pwd`;\n python3.10 scripts/process_json/process_json.py --project_id=1 --pg_table=victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc --filepath=../../../io/Fotonower_Issues.json\n ', 'nb_doc_with_no_title': 0, 'nb_doc_with_title': 0, 'nb_doc_wo_info_on_title': 0}
630 data_json_ref = {'id': 1, 'owner_id': 1, 'description': 'plop', 'function_search_documents': 'match_8e3ce876cf8540318825808c81405950', 'format': None, 'taille': None, 'configuration': {'custom_display': {'home': {'data': '{"store_in_project": '
631 '"true", "project_id": '
632 '"1"}'}}}, 'shortlink': 'default', 'number_documents': 0, 'latest_update': None, 'size': 'TO ESTIMATE', 'command': '\n cd $GITSAFIA/prompt/python/lib/import_util/lib_import_retrieval;\n export PYTHONPATH=`pwd`/../../..:`pwd`;\n python3.10 scripts/process_json/process_json.py --project_id=1 --pg_table=melvic_documents.dataset_8e3ce876cf8540318825808c81405950 --filepath=../../../io/Fotonower_Issues.json\n ', 'most_recent_date_h': 'il y a 11 ans', 'most_recent_date': '2014-09-06 00:00:00', 'nb_doc_with_no_title': 0, 'nb_doc_with_title': 0, 'nb_doc_wo_info_on_title': 0}
634# del data_json_ref["modified_at"]
635# del data_json_ref["created_at"]
636# del data_json_ref["table_documents"]
638 if "cost" in data_json_ref:
639 del data_json_ref["cost"]
640 response_json = response.json
642 del response_json["modified_at"]
643 del response_json["created_at"]
644 del response_json["table_documents"]
646 # del response_json["cost"]
647 # VR TODO, le delete des nouveaux datous ne semble pas marcher (ou c'est plutot une interraction de merde avec les test jscover aussi)
648# del response_json["most_recent_date_h"]
649# if "most_recent_date_h" in data_json_ref:
650# del data_json_ref["most_recent_date_h"]
651# del response_json["most_recent_date"]
652# if "most_recent_date" in data_json_ref:
653# del data_json_ref["most_recent_date"]
654# del response_json["latest_update"]
655# if "latest_update" in data_json_ref:
656# del data_json_ref["latest_update"]
657 if "cost" in response_json:
658 del response_json["cost"]
659 print(self.maxDiff)
660 self.maxDiff = None
661 self.assertEqual(data_json_ref, response_json)
663 data_update_info = {'project_id' : curr_project_id,'shortname': "shortlink_nrt"}
664 response = self.client.post("/api/v1/project/save",
665 data=json.dumps(data_update_info),
666 content_type='application/json')
668 self.maxDiff = 480 # plutot que 640 par défaut
669 data_json_ref = {'id': 1, 'owner_id': 1, 'description': 'plop', 'table_documents': 'melvic_documents.dataset_8e3ce876cf8540318825808c81405950', 'function_search_documents': 'match_8e3ce876cf8540318825808c81405950', 'format': None, 'taille': None, 'created_at': '11/11/2024 02:26:30', 'modified_at': '11/11/2024 02:26:30', 'shortlink': 'shortlink_nrt', 'number_documents': 0, 'latest_update': None, 'size': 'TO ESTIMATE', 'command': '\n cd $GITSAFIA/prompt/python/lib/import_util/lib_import_retrieval;\n export PYTHONPATH=`pwd`/../../..:`pwd`;\n python3.10 scripts/process_json/process_json.py --project_id=1 --pg_table=melvic_documents.dataset_8e3ce876cf8540318825808c81405950 --filepath=../../../io/Fotonower_Issues.json\n ', 'most_recent_date_h': 'il y a 11 ans', 'most_recent_date': '2014-09-06 00:00:00', 'nb_doc_with_no_title': 0, 'nb_doc_with_title': 0, 'nb_doc_wo_info_on_title': 0}
670 data_json_ref["shortlink"] = "shortlink_nrt"
672 response = self.client.get("/api/v1/project/detail/" + str(curr_project_id))
673 response_json = response.json
674 del response_json["cost"]
675 del response_json["configuration"]
676# del response_json["most_recent_date_h"]
677# del response_json["most_recent_date"]
678# del response_json["latest_update"]
679 self.assertEqual(response.status_code, 200)
680 self.assertEqual(data_json_ref, response_json)
682 response = self.client.get("/shortlink_nrt")
683 self.assertEqual(response.status_code, 200)
684 response = self.client.get("/shortlink_nrtt")
685 self.assertEqual(response.status_code, 200)
686 self.assertEqual(response.data.decode("utf-8"), "GENERIC PATH") # - [ ] TODO manage these case !
688 data_update_info = {'project_id' : curr_project_id,'shortname': "default"}
689 response = self.client.post("/api/v1/project/save",
690 data=json.dumps(data_update_info),
691 content_type='application/json')
693 data = {
694 'store_in_project': 'true',
695 'project_id': str(curr_project_id)
696 }
698 path_to_your_file = "test/data/lettre1erMoisCompletude-08-07-2022.pdf"
700# from werkzeug.datastructures import FileStorage
701# # Open the file in binary mode and create a FileStorage object
702# with open(path_to_your_file, 'rb') as f:
703# file = FileStorage(f)
705# lettre1erMoisCompletude-08-07-2022.pdf
706# short_test_7JEI_RESCRIT.json
708 hostname = os.environ["HOSTANME"] if "HOSTANME" in os.environ else os.uname().nodename
710 os.system("pwd")
712 with open(path_to_your_file, 'rb') as f:
713 logger.info(" Before self.client.post : file : " + str(f))
714 # Send the file and the JSON data to the endpoint
716# application/pdf
717 # Test d'upload de l'instuction du dossier de permis de construire de l'ECA rue charlot
718 response = self.client.post('/api/v1/uploader',
719 content_type='multipart/form-data',
720 data={'file': (f, 'lettre1erMoisCompletude-08-07-2022.pdf'), 'data': json.dumps(data)},
721 follow_redirects=True)
723 print("response : " + str(response))
724 logger.info("response : " + str(response))
725 logger.info(" response keys " + str(response.json.keys()))
726 print(" response keys " + str(response.json.keys()))
727 logger.info("response manage_prompt : " + str(response.json["manage_prompt"]))
729 json_data = json.loads(response.data)
730 logger.info(" json_data keys " + str(json_data.keys()))
731 print(" json_data keys " + str(json_data.keys()))
732 file_to_manage = json_data["manage_prompt"]["file"]
733 print(" file_to_manage : " + str(file_to_manage))
735# # Check the response
736 self.assertEqual(response.status_code, 200)
738 # delete_document
740 # TODO VR : validité du cookie ! c6ae5f7095abfe9ac59e75fc6624cea56038d9da => update dans AirTable ?
741 data_query = {"input" : "Quel est le nom de famille de Perrine ?"} # => VR 17-6-23 : je en sais vraiment pas pourquoi je suis tombé sur Perrine Mouterde
742 data_query = {"input" : "Qui s'est occupé de la demande de permis de construire ?"}
744 response_query = self.client.get('/api/v1/safia/query',
745 # content_type='multipart/form-data',
746 query_string=data_query,
747 follow_redirects=True)
748 logger.info("response_query : " + str(response_query))
750 self.assertEqual(response_query.status_code, 200)
751 logger.info("response")
752 logger.info(str(response_query.data))
753 logger.info(" response_query keys " + str(response_query.json.keys()))
754 print(" response_query keys " + str(response_query.json.keys()))
755# logger.info("response_query manage_prompt : " + str(response_query.json["manage_prompt"]))
756# print("response_query manage_prompt : " + str(response_query.json["manage_prompt"]))
757 # references
759# logger.info(str(response_query.get_json())) => ce serait mieux !
760# self.assertIn("mouterde", str(response_query.data).lower())
762 self.assertIn("didier", str(response_query.data).lower())
764 lss = LibSafiaSystem(lib_user_data_internal=lpgss)
766 info_project = lss.get_project_info(project_nrt)
767 logger.info(" after get_project_info ")
768 logger.info(" info_project : " + str(info_project))
769 print(" info_project : " + str(info_project))
771 assert info_project["number_documents"] == 3 #4 #80 # 3 when run alone I think
773 document_id = "///FILE///moilerat@" + hostname + "//temp/lettre1erMoisCompletude-08-07-2022.pdf_1"
774 document_id = "///FILE///moilerat@" + hostname + "//" + file_to_manage + "_1"
776 response_delete = self.client.get('/api/v1/document/delete?id=' + document_id)
778 complete_document_id = "///FILE///moilerat@" + hostname + "//temp/lettre1erMoisCompletude-08-07-2022.pdf"
779 complete_document_id = "///FILE///moilerat@" + hostname + "//" + file_to_manage
780 self.test_documents_page(endpoint = "/editor?document_id=" + complete_document_id,
781 search_into = '<span class="ml-2">Voice Editor</span>',
782 length_file = 4351, #5795, #4327,
783 filename = "editor_document_lettre_completude.html",
784 save_comp = True)
786# time.sleep(5)
788 document_id = "///FILE///moilerat@" + hostname + "//temp/lettre1erMoisCompletude-08-07-2022.pdf_1"
789 document_id = "///FILE///moilerat@" + hostname + "//" + file_to_manage + "_1"
790 response_delete = self.client.get('/api/v1/document/delete?id=' + document_id)
792 self.assertEqual(response_delete.status_code, 302)
793 info_project = lss.get_project_info(project_nrt)
794 # VR 18-3-23 on ne test # plus la suppression
795 assert info_project["number_documents"] == 3 # 3 # 79 # 2
797 audit, list_inputs, list_datou_step = lss.resume_all_project(project_nrt) #, nb_cluster_input = 3)
798 logger.info("audit : " + str(audit))
799 print(str(audit))
800 del audit["created_at"]
802 assert audit == {'size_document': 1365, 'nb_chunks': 3, 'nb_tokens': 462, 'nb_input': 1,
803 'begin_content': " \nDirection de l'Urbanisme\nService du Permis de Construire et du Paysage de la Rue\nParis, le 08/07/2022\nAffaire suivie par :\nDidier MEURGUES\uf028 01 42 76 22 25\nmél : didier.meurgues@paris.fr6 Promenade Claude Lévi-StraussCS 51388 - 75639 PARIS CEDEX 13Monsieur Victor REUTENAUER\n \n30 rue Charlot\n75003 Paris\nRéférence dossier :DP 075 103 22 V0198\n30 RUE CHARLOT\n75003 PARIS\n*DP07510322V0198106* \nLettre Recommandée Electronique.\nMadame, Monsieur,\n"}
804# assert audit == {'size_document': 918, 'nb_chunks': 2, 'nb_tokens': 291, 'nb_input': 1}
805 assert len(list_inputs) == 1
806 assert "paris" in list_inputs[0]["text"].lower()
808 response_query = self.client.get('/resume_all_one_project')
809 logger.info("response_query : " + str(response_query.data))
810 print("response_query : " + str(response_query.data))
812 self.assertEqual(response_query.status_code, 200)
813# json_paris = json.loads("WIP please wait ![{\'text\': "Direction de l\'Urbanisme Service du Permis de Construire et du Paysage de la Rue Paris, le 08/07/2022 Affaire suivie par : Didier MEURGUES\\uf028 01 42 76 22 25 m\xc3\xa9l : didier.meurgues@paris.fr6 Promenade Claude L\xc3\xa9vi-StraussCS 51388 - 75639 PARIS CEDEX 13Monsieur Victor REUTENAUER 30 rue Charlot 75003 Paris R\xc3\xa9f\xc3\xa9rence dossier :DP 075 103 22 V0198 30 RUE CHARLOT 75003 PARIS *DP07510322V0198106* Lettre Recommand\xc3\xa9e Electronique. Madame, Monsieur,Je vous signale qu\'il s\'agit d\'un d\xc3\xa9lai maximum et que je m\xe2\x80\x99efforcerai de vous notifier ma d\xc3\xa9cision d\xc3\xa8s que possible avant son expiration. Je vous prie d\xe2\x80\x99agr\xc3\xa9er, Madame, Monsieur, l\xe2\x80\x99expression de mes salutations distingu\xc3\xa9es. Pour la Maire de Paris et par d\xc3\xa9l\xc3\xa9gation, #signature# Copie au Pr\xc3\xa9fet de Paris DP 075 103 22 V0198 Compl\xc3\xa9tude DP", \'preprompt\': "Merci de faire un r\xc3\xa9sum\xc3\xa9 en listant les points importants et les taches \xc3\xa0 accomplir de la transcription de l\'AG de Fotonower :", \'result\': "R\xc3\xa9sum\xc3\xa9 de la transcription de l\'AG de Fotonower:\\n\\n1. Direction de l\'Urbanisme Service du Permis de Construire et du Paysage de la Rue.\\n2. Date: 08/07/2022\\n3. Affaire suivie par Didier MEURGUES (01 42 76 22 25, didier.meurgues@paris.fr).\\n4. Adresse: 6 Promenade Claude L\xc3\xa9vi-Strauss, CS 51388 - 75639 PARIS CEDEX 13.\\n5. Destinataire: Victor REUTENAUER, 30 rue Charlot 75003 Paris.\\n6. R\xc3\xa9f\xc3\xa9rence dossier: DP 075 103 22 V0198 30 RUE CHARLOT 75003 PARIS.\\n\\nT\xc3\xa2ches \xc3\xa0 accomplir:\\n\\n1. Prendre note du d\xc3\xa9lai maximum pour notifier la d\xc3\xa9cision et s\'efforcer de la donner avant son expiration.\\n2. Transmettre les salutations distingu\xc3\xa9es de la Maire de Paris.\\n3. Envoyer une copie de la lettre recommand\xc3\xa9e \xc3\xa9lectronique au Pr\xc3\xa9fet de Paris (DP 075 103 22 V0198).\\n4. Assurer la compl\xc3\xa9tude du dossier DP.", \'request\': "Merci de faire un r\xc3\xa9sum\xc3\xa9 en listant les points importants et les taches \xc3\xa0 accomplir de la transcription de l\'AG de Fotonower :Direction de l\'Urbanisme Service du Permis de Construire et du Paysage de la Rue Paris, le 08/07/2022 Affaire suivie par : Didier MEURGUES\\uf028 01 42 76 22 25 m\xc3\xa9l : didier.meurgues@paris.fr6 Promenade Claude L\xc3\xa9vi-StraussCS 51388 - 75639 PARIS CEDEX 13Monsieur Victor REUTENAUER 30 rue Charlot 75003 Paris R\xc3\xa9f\xc3\xa9rence dossier :DP 075 103 22 V0198 30 RUE CHARLOT 75003 PARIS *DP07510322V0198106* Lettre Recommand\xc3\xa9e Electronique. Madame, Monsieur,Je vous signale qu\'il s\'agit d\'un d\xc3\xa9lai maximum et que je m\xe2\x80\x99efforcerai de vous notifier ma d\xc3\xa9cision d\xc3\xa8s que possible avant son expiration. Je vous prie d\xe2\x80\x99agr\xc3\xa9er, Madame, Monsieur, l\xe2\x80\x99expression de mes salutations distingu\xc3\xa9es. Pour la Maire de Paris et par d\xc3\xa9l\xc3\xa9gation, #signature# Copie au Pr\xc3\xa9fet de Paris DP 075 103 22 V0198 Compl\xc3\xa9tude DP", \'cost\': 0.03473999999999999}]")
814 assert "Service du Permis de Construire" in str(response_query.data)
816 response_query = self.client.get('/stat_all_one_project')
817# response_query = self.client.get(url_for('project.stat_all_one_project'))
818 logger.info("response_query : " + str(response_query.data))
819 print("response_query : " + str(response_query.data))
821 self.assertEqual(response_query.status_code, 200)
822 json_stat_ref = {'///FILE///moilerat@' + hostname + '//' + file_to_manage: {'nb_chunks': 2, 'nb_input': 1, 'nb_tokens': 273, 'size_document': 783}}
823 json_stat_ref = [{'begin_content': " \nDirection de l'Urbanisme\nService du Permis de Construire et du Paysage de la Rue\nParis, le 08/07/2022\nAffaire suivie par :\nDidier MEURGUES\uf028 01 42 76 22 25\nmél : didier.meurgues@paris.fr6 Promenade Claude Lévi-StraussCS 51388 - 75639 PARIS CEDEX 13Monsieur Victor REUTENAUER\n \n30 rue Charlot\n75003 Paris\nRéférence dossier :DP 075 103 22 V0198\n30 RUE CHARLOT\n75003 PARIS\n*DP07510322V0198106* \nLettre Recommandée Electronique.\nMadame, Monsieur,\n", 'created_at': '2023-10-10 23:51:14.149641+02:00', 'created_at_h': 'il y a 23 secondes', 'document_id': '///FILE///moilerat@MacBook-Air-Victor.local//temp/lettre1erMoisCompletude-08-07-2022adcfcdbd-19dd-46e3-b8db-b0faefc781bd.pdf', 'kunslash': 'FILEmoileratMacBook_Air_Victorlocaltemplettre_erMoisCompletude___________adcfcdbd___dd___e__b_db_b_faefc___bdpdf', 'nb_chunks': 3, 'nb_input': 1, 'nb_tokens': 462, 'size_document': 1365, 'title': 'No Title Found'}]
824 json_stat_resp = json.loads(response_query.data)
825 del json_stat_resp[0]["created_at"]
826 del json_stat_resp[0]["created_at_h"]
827 del json_stat_resp[0]["document_id"]
828 del json_stat_resp[0]["kunslash"]
829 del json_stat_ref[0]["created_at"]
830 del json_stat_ref[0]["created_at_h"]
831 del json_stat_ref[0]["document_id"]
832 del json_stat_ref[0]["kunslash"]
833 # TODO VR ce type d'assert est plus efficace
834 assert json_stat_ref == json_stat_resp
836 try :
837 response_import_git = self.client.get('/git?repo=fotonower/Safia&limit=1&last_nb_days=1234567890&verbose=True&with_comment=True')
838 expected_result_code = 200
839 self.assertEqual(response_import_git.status_code, expected_result_code)
840 info_project = lss.get_project_info(project_nrt)
842 assert info_project["number_documents"] >= 14 and info_project[
843 "number_documents"] <= 28 # 24 # 14 #21 #11 # 10 #8 #6 #21 # 19 # 10 au 11-7-23 je ne sais pas pourquoi # les tickets se rallonge 8 #80 # 3
844 except Exception as e:
845 print("Bad crednetials")
848 response_query = self.client.get('/resume_all_one_project?nb_cluster=2')
849 logger.info("response_query : " + str(response_query.data))
850 print("response_query : " + str(response_query.data))
852 self.assertEqual(response_query.status_code, 200)
856 data_update = {
857 # "id" : "///FILE///moilerat@Air-Victor//temp/lettre1erMoisCompletude-08-07-2022.pdf_0",
858 "id" : "///FILE///moilerat@" + hostname + "//" + file_to_manage + "_0",
859 "content" : "Le petrole c'est mieux que le solaire"
860 }
861# data_to_send = {"document_content" : document_content, "document_id" : document_id.id}
862 # TODO CA ca update juste un chunk et ce n'est pas interfacé
863 responseUpdate = self.client.post('/api/v1/document/update',
864 content_type='application/json',
865 data=json.dumps(data_update),
866 follow_redirects=True)
867 self.assertEqual(responseUpdate.status_code, 200)
869 # - [ ] TODO c'est assez dummy le test de document/save et document/update
871 data_update = {
872 "document_id" : "///FILE///moilerat@" + hostname + "//" + file_to_manage,
873 "document_content" : "Le petrole c'est mieux que le solaire"
874 }
875 # TODO CA ca ne supprime pas les chunks inutiles !
876 responseUpdate = self.client.post('/api/v1/document/save',
877 content_type='application/json',
878 data=json.dumps(data_update),
879 follow_redirects=True)
880 self.assertEqual(responseUpdate.status_code, 200)
882 data_query = {"input" : "Qu'est-ce qui est mieux que le solaire ? Répond par un mot seulement !"}
884 response_query = self.client.get('/api/v1/safia/query',
885 content_type='multipart/form-data',
886 query_string=data_query,
887 follow_redirects=True)
888 logger.info("response_query : " + str(response_query))
890 self.assertEqual(response_query.status_code, 200)
891 logger.info("response")
892 logger.info(str(response_query.data))
894 #self.assertIn("pétrole", str(response_query.data).lower())
895 #self.assertIn("preprompt", str(response_query.data).lower())
896 self.assertIn("merci de repondre", str(response_query.data).lower())
897# assert "pétrole" in str(response_query.data).lower()
898 # le pétrole est identifié comme étant préférable au solaire
899# assert "trole est mieux que le solaire" in str(response_query.data).lower() or "trole est meilleur que le solaire" in str(response_query.data).lower()
900 # self.assertIn("il est mentionn\xc3\xa9 que le p\xc3\xa9trole est meilleur que le solaire", str(response_query.data).lower())
901 # Bien que ce soit mal indexé, on dirait qu'il trouve, mais par contre je en sais pas comment lire ces caractères
902# self.assertIn("p\xc3\xa9trole", str(response_query.data).lower())
903# self.assertIn("pétrole", str(response_query.data).lower())
906 data_query = {"input" : "Quelle fleuve passe au Puy-En-Velay ?",
907 "store_in_project" : "true",
908 "object" : "simple_text_query"}
909 response_query = self.client.get('/api/v1/safia/query',
910 content_type='multipart/form-data',
911 query_string=data_query,
912 follow_redirects=True)
913 logger.info("response_query : " + str(response_query))
915 self.assertEqual(response_query.status_code, 200)
916 logger.info("response")
917 resp = response_query.data.decode("utf-8").lower()
918 found_river = "loire" in resp or "borne" in resp or "dolaizon" in resp
919 self.assertTrue(found_river)
922 # TODO VR 17-6 : set specific password for debug prompt from mail
924 def te_st_kmean_multi_dim_dummy(self):
925 self.client.set_cookie( 'otp', "c6ae5f7095abfe9ac59e75fc6624cea56038d9da")
927 response_query = self.client.get('/kmean?limit=10&pids=8,2&nb_cluster=12&dim=234')
929 self.assertEqual(response_query.status_code, 200)
931 # deprecated while NRT can't login !?!
932 def te_st_invitation(self):
933 # useless since we take an otp valid until 11/11/23
934# id_info = lpgss_singleton.setup_user()
936 # - [ ] TODO VR il faudrait qu'il chope un otp pour cookie de manière à pouvoir tester l'interface et pas que la création, grrrr !
937 # - [ ] on le fera plus tard de dupliquer la récupération des otp depuis lib_internal plutot que depuis AT
938 # - [ ] TODO VR ARCHI comment gérer la migration : service dupliquer ou migration hard
939 # - [ ] TODO VR test project inaccessible first
941 from lib.lib_util import get_unique_id
942 otp_for_info = get_unique_id()
944# print("id_info : " + str(id_info))
946 otp_info_opio_fr = "c6ae5f7095abfe9ac59e75fc6624cea56038d9da" # nathalie_share "3b221881f70bde52677697d1f37ea437baf5df2f"
947 response = self.client.get("/otp/" + otp_info_opio_fr)
948 print(str(response))
949 self.assertEqual(response.status_code, 200) # redirect plutot que 302
951 info = lpgss_singleton.get_user_from_mail("info@opio.fr")
952 print(str(info))
953 user_id_info = info["id"]
955 lpgss_singleton.get_admin_situation()
956 time.sleep(3)
958 self.client.set_cookie('otp', "c6ae5f7095abfe9ac59e75fc6624cea56038d9da")
959 response = self.client.get("/api/v1/project/role/remove?project_id=" + str(project_nrt) + "&user_id=" + str(user_id_info))
960 assert response.status_code == 200
962# list_projects = lss_singleton.get_project("info@opio.fr")
963 list_projects = lss_singleton.get_projects(user_id_info)
965 assert len(list_projects) == 1 # 2
967 data_to_send = {"project_id" : project_nrt,
968 "role" : "admin",
969 "mail" : "info@opio.fr"}
971 import json
972 self.client.set_cookie('otp', "c6ae5f7095abfe9ac59e75fc6624cea56038d9da")
973 response = self.client.post('/api/v1/invitation/create',
974 content_type='application/json',
975 data=json.dumps(data_to_send),
976 follow_redirects=True)
978 print(" response : " + str(response))
980 link = response.json["link"]["link"]
981 token_invitation = response.json["link"]["token"]
983 self.client.set_cookie( 'otp', otp_info_opio_fr)
984 response = self.client.get(link)
985# response = self.client.get('/invitation/' + token_invitation,
986# follow_redirects=True)
988 print(" response : " + str(response))
990# list_projects = lss_singleton.get_project("info@opio.fr")
991 list_projects = lss_singleton.get_projects(user_id_info)
993 assert len(list_projects) == 1 # 2
995 self.client.set_cookie( 'otp', "c6ae5f7095abfe9ac59e75fc6624cea56038d9da")
996 response = self.client.get('/')
998 from bs4 import BeautifulSoup
999 # Parsez le contenu HTML avec BeautifulSoup
1000 soup = BeautifulSoup(response.data, 'html.parser')
1002 # Recherchez l'élément span avec la classe 'pc_cov' et extrayez son texte
1003 span_value = soup.find_all('span', {'class': 'proj-desc'})
1005 print(str(span_value))
1007 assert len(span_value) == 0 # 37 # je ne vois pas bien pourquoi la précedente requete en avait 2, on a l'air d'utiliser la base local de dev
1009 print("Test done !")
1011 # To move from file when possible !
1012def test_speech_to_text(snapshot):
1013 """
1014 Test de transcription speech-to-text pour fichier .amr
1016 Utilise les snapshots pour valider la structure complète du résultat.
1017 Conserve des checks structurels pour vérifier que le résultat contient
1018 les mots clés attendus du contenu audio.
1019 """
1020 logger = logging.getLogger()
1021 email = "anonymous@opio.fr"
1022 object = "cr"
1023# file = "/Users/moilerat/Documents/Fotonower/Safia/prompt/python/test/data/recording-2023-05-07-13-15-54.amr"
1024 file = "test/data/recording-2023-05-07-13-15-54.amr"
1026 list_files = []
1028 list_files.append({"from_mail":email, "object":object ,"file":file})
1030 lib_auth = LibUserDataExternalServices(lib_at = None)
1032 from auth.lib_auth import get_datou_exec_context_as_complete_param_json
1033 complete_param_json = get_datou_exec_context_as_complete_param_json(email, verbose=True, privacy=False,
1034 openai_token=OPENAI_API_KEY,
1035 lss=lss_singleton, project_id=project_nrt)
1037 from lib.lib_util import select_datou_step_from_extension_type_upload
1038 is_managed, list_actions_datou, extension = select_datou_step_from_extension_type_upload(file, ".amr", "on s'en fout dans ce cas")
1040 input = list_files[0]
1041 ref_text = "Please make a report with a few main themes, todos and follow-up of the following by keeping the same language :"
1042 # 10-24
1043 ref_text = "Please make a report with a few main themes, todos and follow-up of the following by keeping the same language :J'aimerais bien que FTA m'aide à gérer mes peurs et mes attentes et mes désirs. Des choses très personnelles. Du coup, je veux bien que tu me renvoies une petite liste de... Non, renvoie-moi rien, je t'enverrai des releases plus tard. Mais pour l'instant, on va juste voir si ça marche de respecter la RGPD là-dessus."
1044 ref_text = "Please make a report with a few main themes, todos and follow-up of the following by keeping the same language :J'aimerais bien que FTA m'aide à gérer mes peurs et mes attentes et mes désirs. Des choses très personnelles. Du coup, je veux bien que tu me renvoies une petite liste de... Non, renvoie-moi rien, je t'enverrai des releases plus tard. Mais pour l'instant, on va juste voir si ça marche de respecter la RGPD là-dessus.J'aimerais bien que FTA m'aide à gérer mes peurs, mes attentes et mes désirs. Des choses très personnelles. Du coup, je veux bien que tu me renvoies une petite liste de... Non, renvoie-moi rien, je t'enverrai des releases plus tard. Mais pour l'instant, on va juste voir si ça marche de respecter la RGPD là-dessus."
1045 input["preprompt"] = ref_text
1047 from lib.datou.datou_exec import datou_exec
1048 result, audit_json = datou_exec(list_actions_datou, input, complete_param_json)
1050# from lib.lib_prompt import manage_prompt
1051# result = manage_prompt(list_files=list_files, openai_token=OPENAI_API_KEY,
1052# verbose=True, input=list_files[0]) # , debug=True
1053 # , openai_model="gpt-3-turbo")
1054 logger.info("Finished manage_prompt ! ")
1055 print("Finished manage_prompt !")
1057 logger.info(" result : " + str(result))
1059 # Vérifications structurelles importantes
1060 assert len(result) == 10 #8 # 10 # 16 #15 # 14 #7 #9 #7 # 9
1061 assert "request" in result
1062 assert len(result["result"]) > 10
1063 assert "peur" in result["result"].lower() or "attentes" in result["result"].lower() or "fear" in result["result"].lower()
1065# "peurs, mes attentes"
1066# "peurs et mes attentes"
1068 # Validation complète via snapshot
1069 assert result["request"][:450] == snapshot
1073 # To move from file when possible !
1074def test_speech_to_text_m4a(snapshot):
1075 """
1076 Test de transcription speech-to-text pour fichier .m4a
1078 Utilise les snapshots pour valider la structure complète du résultat.
1079 Conserve des checks structurels pour vérifier que le résultat contient
1080 les mots clés attendus du contenu audio.
1081 """
1082 logger = logging.getLogger()
1083 email = "anonymous@opio.fr"
1084 object = "cr"
1085# file = "/Users/moilerat/Documents/Fotonower/Safia/prompt/python/test/data/recording-2023-05-07-13-15-54.amr"
1086 file = "test/data/topgun_audio1681204600.m4a"
1088 list_files = []
1090 list_files.append({"from_mail":email, "object":object ,"file":file})
1092 lib_auth = LibUserDataExternalServices(lib_at = None)
1094 from auth.lib_auth import get_datou_exec_context_as_complete_param_json
1095 complete_param_json = get_datou_exec_context_as_complete_param_json(email, verbose=True, privacy=False,
1096 openai_token=OPENAI_API_KEY,
1097 lss=lss_singleton, project_id=project_nrt)
1099 from lib.lib_util import select_datou_step_from_extension_type_upload
1100 is_managed, list_actions_datou, extension = select_datou_step_from_extension_type_upload(file, ".amr", "on s'en fout dans ce cas")
1102 input = list_files[0]
1103 input["preprompt"]="Please make a report with a few main themes, todos and follow-up of the following by keeping the same language :"
1105 to_choose, list_action, extension = select_datou_step_from_extension_type_upload(file, ".m4a", "plop")
1107 if to_choose:
1108 from lib.datou.datou_exec import datou_exec
1109 result, audit_json = datou_exec(list_actions_datou, input, complete_param_json)
1111 # from lib.lib_prompt import manage_prompt
1112 # result = manage_prompt(list_files=list_files, openai_token=OPENAI_API_KEY,
1113 # verbose=True, input=list_files[0]) # , debug=True
1114 # , openai_model="gpt-3-turbo")
1115 logger.info("Finished manage_prompt ! ")
1116 print("Finished manage_prompt !")
1118 logger.info(" result : " + str(result))
1119 else :
1120 result = {}
1122 # Vérifications structurelles importantes
1123 assert len(result) == 10 #8 # 10 # 16 #15 # 14 # 7 #9
1124 assert "text" in result
1125 assert len(result["result"]) > 10
1126 assert "top gun" in result["result"].lower()
1127 assert "maverick" in result["result"].lower()
1129 request = result["request"] if "request" in result else None
1131 assert request[230] == snapshot
1133 # VR 6/2/26 : impossible de Validation complète via snapshot car il y a trop de variabilité dans le résultat (ex: "top gun : maverick" est parfois reconnu comme "top gun : maverick" et parfois comme "top gun : maverick" selon les runs, sans que l'on sache vraiment pourquoi, peut-être un problème de tokenisation ou de reconnaissance vocale qui est à la limite de la compréhension de l'IA)
1134 ## assert result == snapshot
1138 # To move from file when possible !
1139def test_z_image_to_text(snapshot):
1140 """
1141 Test d'extraction de texte depuis une image (OCR)
1143 Utilise les snapshots pour valider:
1144 - La structure complète du résultat d'extraction
1145 - Le texte extrait de l'image
1146 - Les clés du dictionnaire retourné
1148 Note: Les champs 'hash_id_treatment' peuvent être volatiles mais sont conservés
1149 dans le snapshot pour validation complète.
1150 """
1151 verbose = False
1152 logger = logging.getLogger()
1153 email = "anonymous@opio.fr"
1154 object = "cr"
1155 file = "test/data/ticket_macdo.jpg"
1157 list_files = []
1159 list_files.append({"from_mail":email, "object":object ,"file":file})
1161 lss_singleton.truncate_project_documents(project_nrt, verbose=verbose)
1162 logger.info("After truncate project documents")
1164 lib_auth = LibUserDataExternalServices(lib_at = None)
1166 from auth.lib_auth import get_datou_exec_context_as_complete_param_json
1167 complete_param_json = get_datou_exec_context_as_complete_param_json(email, verbose=True, privacy=False,
1168 openai_token=OPENAI_API_KEY,
1169 lss=lss_singleton, project_id=project_nrt)
1171 from lib.lib_util import select_datou_step_from_extension_type_upload
1172 is_managed, list_actions_datou, extension = select_datou_step_from_extension_type_upload(file, ".jpg", "on s'en fout dans ce cas")
1174 input = list_files[0]
1175# input["preprompt"]="Please make a report with a few main themes, todos and follow-up of the following by keeping the same language :"
1177 list_actions_datou.append("result_to_json")
1178 list_actions_datou.append("import_json")
1179# list_actions_datou.append("json_")
1181 from lib.datou.datou_exec import datou_exec
1182 result, audit_json = datou_exec(list_actions_datou, input, complete_param_json)
1184 logger.info("Finished manage_prompt ! ")
1185 print("Finished manage_prompt !")
1187 logger.info(" result : " + str(result))
1188 logger.info(" result.text : " + str(result["text"]))
1190 # Vérifications structurelles importantes
1191 assert len(result) == 26 # 34 #33 #24 #13 #9 # 13 #9 # 13
1192 assert "text" in result
1194 # Validation du projet après import
1195 info_project = lss_singleton.get_project_info(project_nrt)
1196 logger.info(" after get_project_info ")
1197 logger.info(" info_project : " + str(info_project))
1198 print(" info_project : " + str(info_project))
1200 assert info_project["number_documents"] >= 1 and info_project["number_documents"] < 12
1201 # gpt a sans doute rallonger son blabla !
1203 assert "mcchicken" in result["text"].lower()
1204 assert "nuggets" in result["text"].lower()
1205 assert "petit frite" in result["text"].lower()
1207 # Validation complète via snapshot
1208 assert result["request"] == snapshot
1212@pytest.mark.asyncio
1213async def test_process_json():
1214 proj_nrt = project_nrt
1216 from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys
1217 lpgss = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT)
1219 lss = LibSafiaSystem(lib_user_data_internal=lpgss)
1221 curr_project_id = proj_nrt
1223 import logging
1224 logger = logging.getLogger()
1225 logger.info(" before get_project_info ")
1226 info_project = lss.get_project_info(curr_project_id)
1227 logger.info(" after get_project_info ")
1229 logger.info(" info_project : " + str(info_project))
1231 PG_TABLE = info_project["table_documents"]
1233# del info_project["number_documents"]
1235 json_to_import_path = "test/data/short_test_7JEI_RESCRIT.json"
1237 from lib.import_util.lib_import_retrieval.scripts.process_json.process_json import process_json_dump
1238 logger.info(" before process_json_dump : json_to_import_path : " + str(json_to_import_path))
1239# asyncio.run(process_json_dump(json_to_import_path, {}, False, False, None, lpgss, OPENAI_API_KEY, PG_TABLE, verbose = True))
1240 result = await process_json_dump(json_to_import_path, {}, False, False, None, lpgss, OPENAI_API_KEY, PG_TABLE, verbose = True)
1242 logger.info(" result process_json : " + str(result))
1244 assert 0 == 0
1247@pytest.mark.asyncio
1248async def test_insert_json_datastore():
1249 logger = logging.getLogger()
1250 # /Users/moilerat/Documents/Fotonower/Safia/prompt/python/
1251 filepath = "test/data/extract_for_safia_Doc_Issues.json"
1253 custom_metadata = {}
1254 screen_for_pii = False
1255 extract_metadata = False
1256 datastore = None # hack while integrating
1257 verbose = True
1258 project_id = project_nrt
1259 pg_table = "victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc"
1261# export PYTHONPATH=$PYTHONPATH:`pwd`/../../..:`pwd` cd $GITSAFIA/python/lib/import_util/lib_import_retrieval; cd python/lib/import_util/lib_import_retrieval; export PYTHONPATH=`pwd`/../../..:`pwd` python3.10 scripts/process_json/process_json.py --pg_table=victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc --filepath=../../../io/Fotonower_Issues.json
1263 from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys
1264 lpgss = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT)
1266 curr_project_id = project_nrt
1267 lss_singleton.truncate_project_documents(curr_project_id, verbose=verbose)
1268 logger.info("After truncate project documents")
1270 # pas sur du await
1271 #await
1272 from lib.import_util.lib_import_retrieval.scripts.process_json.process_json import process_json_dump
1273# result, nb_token, model
1274 result = await process_json_dump(
1275 filepath, custom_metadata, screen_for_pii, extract_metadata, datastore,
1276 lpgss = lpgss,
1277 OPENAI_API_KEY = OPENAI_API_KEY,
1278 PG_TABLE = pg_table,
1279 verbose = verbose
1280 )
1282 logger.info("Finished process_json_dump NOW need to test what append to avoid having a dummy test ! ")
1283 print("Finished manage_prompt !")
1285 logger.info(" result : " + str(result))
1287 from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys
1288 lpgss = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT)
1290 logger.info(" Before getting get_project_info ")
1291 info_project = lss_singleton.get_project_info(curr_project_id)
1292## truncate_project_documents(curr_project_id, lpgss, user)
1293 # lss.update_project_costs(project_id, nb_token, model, lpgss)
1295 logger.info(" info_project : " + str(info_project))
1297 assert info_project["number_documents"] > 20 # 30 # 50 # 30 # == 35 # old 60 < == 63 #62 #63 #64 #62 #61 #62 # 61 #62 #61 # 62 # 61 #60 # 61 # 62 # 60 # 35 # finalement ? 67 #35 # 28 # 84 # 28 when run alone # Why did it change => because no systematic truncate (easy to do (now) or not due to failure in truncate)
1298 assert info_project["number_documents"] < 30 # 45 #65 # 40 #65
1300# delete_document
1302# VR deprecate 29-1-26
1303def te_st_create_setup():
1305 import logging
1306 logger = logging.getLogger()
1307 logger.info(" PG_DB : " + str(PG_DB))
1309 PG_DB_NON_REG_TEST = PG_DB + "_nrt"
1310 from test.lib_test_util import read_and_modify_migration_for_db_test
1311 path_to_file_migration_db = "../devops/migration/v0/20230603_safia_user.psql"
1312 migration_db_test, create_db_test = read_and_modify_migration_for_db_test(path_to_file_migration_db, DB_NAME_PROD = PG_DB, DB_NAME_TEST = PG_DB_NON_REG_TEST)
1314 from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys
1315 lpgss = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, "postgres", PG_PORT)
1317 import os
1318 # """ PGPASSWORD=""" + str(PG_PASSWORD) +
1319# command_pysql_creation = """ psql -p """ + str(PG_PORT) + """ -h localhost -U safia -d postgres -c "DROP DATABASE IF EXISTS mtr_nrt;" -c "CREATE DATABASE mtr_nrt;" """
1320# os.system(command_pysql_creation)
1321 LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT).create_statement(f"DROP DATABASE IF EXISTS {PG_DB_NON_REG_TEST};")
1322 LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT).create_statement(f"CREATE DATABASE {PG_DB_NON_REG_TEST};")
1325#def create_database(dbname, user, password, host):
1326# conn = psycopg2.connect(dbname='postgres', user=user, password=password, host=host)
1327# conn.autocommit = True # Autocommit must be on to create databases
1328# cur = conn.cursor()
1330 # Use SQL string composition as psycopg2 does not allow parameterized table creation
1331# cur.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(dbname)))
1333# cur.close()
1334# conn.close()
1336# DEbugging on 11-11-24
1337 # """ PGPASSWORD=""" + str(PG_PASSWORD) +
1338# create_extension_with_admin_role = """ psql -p """ + str(PG_PORT) + """ -h localhost -d mtr_nrt -c "CREATE EXTENSION vector;" """
1339# os.system(create_extension_with_admin_role)
1340 try:
1341 LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB_NON_REG_TEST, PG_PORT).create_statement("CREATE EXTENSION vector;")
1342 except Exception as e:
1343 print(" e : " + str(e))
1344 print("En fait c'est dejà fais donc on s'en fout !")
1345 create_extension_with_admin_role = """ psql -p """ + str(
1346 PG_PORT) + """ -h localhost -d mtr_nrt -c "CREATE EXTENSION vector;" """
1347 os.system(create_extension_with_admin_role)
1349# lpgss.create_statement(create_db_test)
1350 lpgss_nrt = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB_NON_REG_TEST, PG_PORT)
1352 lpgss_nrt.create_statement(migration_db_test)
1354 from lib.lib_airtable import LibAirTable
1356 (api_key_at, base_id_apia) = lcs.get_at_apia_conf()
1357 (api_key_at, base_id_pi) = lcs.get_at_pe_conf()
1359 latPE = LibAirTable(api_key_at, base_id_pi)
1360 lib_auth = LibUserDataExternalServices(lib_at=latPE)
1362 otp = "c6ae5f7095abfe9ac59e75fc6624cea56038d9da"
1363 # non_reg_test
1365 info_auth, is_valid = lib_auth.connect_with_otp(otp)
1367 # TODO VR deprecate get_data_user
1368 if is_valid:
1370 lss_for_new_setup = LibSafiaSystem(lib_user_data_internal=lpgss_nrt)
1372 info_setup = lss_for_new_setup.setup_user(info_auth['Email_User'], info_auth["Description"], "otp_apia", otp)
1374 logger.info(str(info_setup))
1375 print(str(info_setup))
1377 data_user = lss_for_new_setup.get_data_user(info_auth['Email_User'])
1379 print(" data_user : " + str(data_user))
1381 assert data_user[0]["table_documents"] == "victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc"
1382 else :
1383 message = "Please extend otp nonregtest validity"
1384 print(message)
1385 logger.info(message)
1386 assert 0 == 1
1387 assert message == ""
1391 print(" TO FINISH ")
1394# il faut tester
1395# il faut un autre lss_singleton
1396# lss_singleton.use_invitation()
1399def test_mail_from_prompt_apia(snapshot):
1400 """
1401 Test de récupération et traitement d'emails depuis un serveur mail
1403 Utilise les snapshots pour valider:
1404 - La structure complète du résultat (array d'emails traités)
1405 - Le contenu de chaque email
1406 - Les métadonnées (log_git_action, etc.)
1408 Note: Ce test dépend d'un serveur mail externe et peut échouer si la boîte
1409 mail est vide ou si les credentials changent.
1410 """
1411 import logging
1412 logger = logging.getLogger()
1414# username = "promptd@fotonower.com"
1415 username = "promptd@safia.app"
1416 password = "alalaoui"
1418 from lib.lib_prompt import prompt_from_mail
1419 server = "mail.gandi.net"
1420 port = 995
1421 from lib.lib_airtable import LibAirTable as lat
1422 lib_at = lat()
1423 # safia_nrt@rubbia.fr
1424 result = prompt_from_mail(server, username, password, port, openai_token=OPENAI_API_KEY, debug = True, lib_connect=lib_at)
1426 print(" result : " + str(result))
1428 logger.info(" result : " + str(result))
1430 # Vérifications structurelles importantes
1431 assert len(result) == 0 # 2
1432 if len(result) > 0:
1433 assert "result" in result[0]
1434 assert result[0]['log_git_action'] == 'git_action_done'
1435 assert 'log_git_action' not in result[1]
1437 # Validation complète via snapshot
1438 assert result == snapshot
1440def test_import_all():
1441 from lib.lib_util import import_all_for_coverage_with_zero_percent
1442 import_all_for_coverage_with_zero_percent()
1444def test_multiple_safia_import_for_batch_integration_dummy():
1446 from lib.lib_safia import safia_import
1447 in_folder = "test/data/multiple_import"
1448 out_file_json, list_detailed_time, unmanaged = safia_import(in_folder)
1449 # main(['foo', '-f','v'])
1451 # TODO VR reload out_file_json
1452# assert out_file_json == []
1454 from lib.import_util.lib_import_retrieval.scripts.process_json.process_json import main as main_pj
1455 sys.argv = ["this", '--filepath', out_file_json, "--project_id", 0, "--pg_table", "victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc"]
1456# asyncio.run(main_pj(['--filepath', out_file_json, "project_id", 0, "--pg_table", "victor_reutenauer_eu_documents.dataset_c6ae5f7095abfe9ac59e75fc"]))
1457 asyncio.run(main_pj())
1459 print("TO TEST ! :-)")
1463#def suspendest_get_project_info():
1464def test_get_project_info(snapshot):
1465 """
1466 Test de récupération des informations d'un projet.
1468 Ce test utilise les snapshots pour comparer la structure complète
1469 du projet retourné par get_project_info.
1471 Champs exclus de la comparaison (valeurs volatiles):
1472 - number_documents: varie selon les imports
1473 - latest_update: timestamp changeant
1474 - cost: coût de traitement variable
1475 """
1476 from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys
1477 lpgss = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT)
1479 lss = LibSafiaSystem(lib_user_data_internal=lpgss)
1481 curr_project_id = project_nrt
1483 import logging
1484 logger = logging.getLogger()
1485 logger.info(" before get_project_info ")
1486 info_project = lss.get_project_info(project_nrt)
1487 logger.info(" after get_project_info ")
1489 logger.info(" info_project : " + str(info_project))
1491 # Suppression des champs volatiles avant comparaison snapshot
1492 del info_project["number_documents"]
1494 import datetime
1496 latest_update = info_project["latest_update"]
1497 now = datetime.datetime.now()
1498 logger.info(" latest_update : " + str(latest_update))
1499 latest_update = latest_update.replace(tzinfo=None)
1500 logger.info(" latest_update : " + str(latest_update))
1501 logger.info(" now : " + str(now))
1502 diff_latest = now - latest_update
1503 logger.info(" diff_latest : " + str(diff_latest))
1505 logger.info(" info_project[cost] : " + str(info_project["cost"]))
1507 del info_project["latest_update"]
1509 cost = info_project["cost"]
1510 del info_project["cost"]
1512 logger.info(" cost project : " + str(cost))
1513 # Validation séparée du coût (non inclus dans snapshot)
1514 # Ces assertions sont commentées car le coût est variable
1515 # assert cost < 0.13
1516 # assert cost > 0.125
1518 # Comparaison avec snapshot au lieu de dictionnaire hardcodé
1519 assert info_project == snapshot
1521# TODO VR 23-11-23 finish this test
1522def te_st_run_kmean_hierarchical_dummy():
1523 #sys.path.append(os.path.join(os.getenv("GITVELOURS"), "c_visual_search/pyfvs/install"))
1524 if "GITPYFVS" in os.environ:
1525 sys.path.append(os.getenv("GITPYFVS"))
1526 from pyfvs.lib.lib_pyfvs_from_safia_to_refacto import FVSInterface
1527 #from lib.lib_ml.lib_tent_py_fvs import FVSInterface
1529# from lib.lib_ml.lib_kmean_pyfvs import query_and_output_of_kmeans
1530 from pyfvs.lib.advanced.lib_split_elbow import query_and_output_of_kmeans
1531 fvs_data = "data/data_fvs"
1532 fvs_interface = FVSInterface(fvs_data_folder=fvs_data)
1534 # {"limit" : limit, "pdt" : pdt, "out_folder" : out_folder,
1535 # "builded_at" : datetime.datetime.now()} (et les params de list_pdt_params)
1537 pdt = 450
1538 limit = 1000
1539 out_folder = "data/fvs_data"
1541 dimension = 4 # TODO AVEC FVSS on ré-écrit le conf file par defaut (et puis je ne l'avais pas commité, merdum)
1542 fvs_interface.data_.list_pdt_params[pdt] = {"dimension": dimension, "int_or_float": True, "photo_desc_type": pdt}
1544 fvs_interface.data_.list_fvs_builded[pdt] = {"limit" : limit, "pdt" : pdt, "out_folder" : out_folder}
1545# fvs_interface.data_.list_fvs_running[str(port)] = {"port": port, "pdt": pdt}
1547 # VR TODO REFACTO on pourrait chercher cette fonction depuis pyfvs ou mettre le fonctionnement dans une fonction interne
1548 from lib.lib_util import get_random_port
1549 port = get_random_port()
1550 # VR 19-11-23 : not sure if out_folder is correct and the same
1552 from auth.lib_conf_system import lcs_global_singleton
1553 fvs_path = lcs_global_singleton.get_fvs_path()
1555 port, p = fvs_interface.launch_fvs(port, pdt, limit, fvs_exe=fvs_path)
1557 nb_step = 2
1558 nb_new_cluster = 1 # at each step
1560 xmin, xmax, ymin, ymax = None, None, None, None
1562 save_cluster = ""
1563 load_cluster = ""
1564 verbose = False
1565 methodo = "split_one"
1566 nb_cluster_total_elbow = 4
1567 gpu_enable = False
1568 type_divergence = "E"
1570 metric = "E"
1571 keep_fvs_running = True
1572 build_fvs_desc = True
1573 dim = 32
1574 nb_project = 1
1575 list_project_ids = [1]
1576 launch_fvs = True
1577 filename = ""
1578 # VR 23-11-23 : ca c'est deja testé
1579 filename, list_legend_centroids, list_url_centroids, list_example_media, misc_info = lss_singleton.kmean_2d(list_project_ids, limit=limit,
1580 nb_project=nb_project, filename = filename,
1581 metric = metric, dim = dim,
1582 nb_cluster_input = nb_new_cluster, # Ca c'est inutile !
1583 gpu_enable = gpu_enable,
1584 nb_step = nb_step,
1585 keep_fvs_running = keep_fvs_running,
1586 port = port,
1587 pdt = pdt,
1588 build_fvs_desc = build_fvs_desc,
1589 save_cluster = save_cluster,
1590 load_cluster = load_cluster,
1591 launch_fvs = launch_fvs,
1592 verbose = verbose,
1593 fvs_interface = fvs_interface)
1595 port = misc_info["port"]
1597 ret = query_and_output_of_kmeans(fvs_interface=fvs_interface,
1598 port=port,
1599 nb_step=nb_step,
1600 # (fvs_interface : FVSInterface, port, nb_step,
1601 save_cluster=save_cluster,
1602 load_cluster=load_cluster,
1603 nb_new_cluster=nb_new_cluster,
1604 verbose=verbose,
1605 xmin=xmin,
1606 xmax=xmax,
1607 ymin=ymin,
1608 ymax=ymax,
1609 with_indices=True,
1610 type_divergence=type_divergence,
1611 methodo=methodo,
1612 nb_cluster_total_elbow=nb_cluster_total_elbow,
1613 gpu_enable=gpu_enable)
1615 print(" ret : " + str(ret))
1617 fvs_interface.kill()
1619# import re
1620 # définir l'expression régulière
1621# pattern = r'static\/temp\_tsne\/one\_gif\_(0-9a-f)*8*)\n'
1622 # compiler l'expression régulière
1623# pattern = re.compile(pattern, re.MULTILINE)
1624 # chercher cette expression dans le texte
1625# match = pattern.search(ret)
1627## assert match
1628 assert ret.startswith("static/temp_tsne/one_gif_")
1631def te_st_tentative_pyfvs_launch_fvs_and_kmean():
1632 import os, sys
1633 fvs_data = os.path.join(os.path.dirname(__file__), "data/fvs_data")
1634 output_image_folder = fvs_data
1635 out_folder = fvs_data
1636 if "GITPYFVS" in os.environ:
1637 sys.path.append(os.getenv("GITPYFVS"))
1638 from pyfvs.lib.lib_pyfvs_from_safia_to_refacto import FVSInterface
1639 fvs_interface = FVSInterface(fvs_data_folder=fvs_data)
1641 try :
1642 pdt = 450
1643 limit = 100
1645 dimension = 4 # TODO AVEC FVSS on ré-écrit le conf file par defaut (et puis je ne l'avais pas commité, merdum)
1646 fvs_interface.data_.list_pdt_params[pdt] = {"dimension": dimension, "int_or_float": True,
1647 "photo_desc_type": pdt}
1649 import os, sys
1650 sys.path.append(os.path.join(os.getenv("GITSAFIA"), "prompt/python"))
1651 from auth.lib_conf_system import lcs_global_singleton
1652 fvs_path = lcs_global_singleton.get_fvs_path()
1653 if fvs_path == None:
1654 fvs_path = os.path.join(os.getenv("GITVELOURS"), "c_visual_search/Debug/fvs")
1656 port, p = fvs_interface.launch_fvs(pdt, limit = limit, fvs_exe=fvs_path)
1658 init_kmean = ""
1659 nb_cluster = 2
1661 import time
1662 time.sleep(3)
1664 sys.path.append(os.path.join(os.getenv("GITVELOURS"), "python"))
1665 ret = fvs_interface.step_kmean(port = port, init_kmean = init_kmean,
1666 nb_iteration = 1, nb_new_clusters = nb_cluster,
1667 gpu_enable = False, verbose = False,
1668 type_divergence ="E", nb_cluster_loaded = 0,
1669 pdt = pdt)
1671 print(" ret : " + str(ret))
1672 ret_non_reg = '|1:0;99:inf|864203888,864208055|-100,-101|864203888;864208055,864215831,864233932,864236843,864245809,864246304,864249886,864250040,864252041,864264302,864272105,864272285,864281550,864284308,864291942,864292694,864292696,864295205,864298494,864299664|peof'
1673 assert ret_non_reg == ret
1675 list_ids_proj_doc_id = []
1677 sys.path.append(os.path.join(os.getenv("GITVELOURS"), "python"))
1678 one_step_tsne = fvs_interface.display_tsne(pdt, out_folder, list_ids_proj_doc_id, output_image_folder,
1679 extension="", root_api="https://safia.app",
1680 xmin=None,
1681 xmax=None,
1682 ymin=None,
1683 ymax=None,
1684 with_indices=False,
1685 nb_cluster_expected=nb_cluster)
1687 fvs_interface.kill()
1689 one_step_tsne_non_reg = ('data/fvs_data/tsne_450_2_E_8cdb1fb1-311c-468c-89e8-b9a77707c1cf.png', ['Point 0, id : 0, nb : 1', 'Point 1, id : 1, nb : 99'], ['https://safia.appeditor?document_id=list_not_loaded&project_id=proj_not_loaded', 'https://safia.appeditor?document_id=list_not_loaded&project_id=proj_not_loaded'], -0.05500000000000001, 0.05500000000000001, -0.05500000000000001, 0.05500000000000001)
1690 one_step_tsne_comp = one_step_tsne[1:]
1691 one_step_tsne_non_reg_comp = one_step_tsne_non_reg[1:]
1693 print("to compare")
1694 assert one_step_tsne_non_reg_comp == one_step_tsne_comp
1695 print("compared")
1697 except Exception as e:
1698 if str(e).startswith("assert"):
1699 raise e
1700 import logging
1701 logger = logging.getLogger()
1702 print(" str(e) : " + str(e))
1703 logger.info(" str(e) : " + str(e))
1704 assert 0 == 1
1705 fvs_interface.kill()
1707if __name__ == "__main__":
1708 from test.use_case.test_extractia import test_extractia_anon, test_extractia
1710 test_extractia()
1711 test_extractia_anon()
1712# test_mail_from_prompt_apia()
1713# test_tentative_pyfvs_launch_fvs_and_kmean()
1714# test_run_kmean_hierarchical_dummy()
1715# test_logged_behavior_unlogged()
1716# test_speech_to_text_m4a()
1717# test_z_image_to_text()
1718# test_speech_to_text()
1719# asyncio.run(test_insert_json_datastore())
1720#
1721# test_tentative_pyfvs_launch_fvs_and_kmean()
1722# test_mail_from_prompt_apia()
1723# test_get_project_info()
1724#
1725#
1726# test_image_to_text()
1727 #test_process_json()
1728# test_create_setup()
1729# test_mail_from_prompt_apia()
1730# test_multiple_safia_import_for_batch_integration_dummy()
1734 fcst = FirstClassSafiaTest()
1735 fcst._pre_setup()
1736 fcst.test_zz_setup_action_ok()
1737 fcst.test_crash_reconnect()
1738 fcst.test_kmean_multi_dim_dummy()
1739 fcst.test_file_upload_and_import_git()
1740# fcst.test_invitation()
1741# fcst.test_logged_behavior_unlogged()
1742 fcst.test_aaa_first_action_truncate_table()
1743#
1744# fcst.test_b_update_configuration_and_datou_dummy()
1745# fcst.test_update_configuration_and_datou_dummy()
1746# fcst.test_example()
1748# fcst.test_aaa_first_action_truncate_table()
1750# fcst.test_connected_ok()
1752 # fcst.test_documents_page()
1753 #fcst.test_connected_ok()
1756# fcst.test_zz_setup_action_ok()
1760# from flask.testing import FlaskClient
1761# fcst.client = FlaskClient(fcst.app)
1762# fcst.test_file_upload_and_import_git()
1763 #
1764# fcst.setUp()
1765# fcst.setUpClass()
1766# fcst.test_update_configuration_and_datou_dummy()
1767# fcst.test_file_upload_and_import_git()