Coverage for server/safia.py: 43%
2370 statements
« prev ^ index » next coverage.py v7.9.1, created at 2026-02-18 02:40 +0100
« prev ^ index » next coverage.py v7.9.1, created at 2026-02-18 02:40 +0100
1# -*- coding: utf-8 -*-
2import datetime
3import platform
4import termios
6import logging
7loggger = logging.getLogger()
9# debugging error 502 bad gateway when token is not up to date anymore
10logging.basicConfig(level=logging.DEBUG)
12print(platform.python_version())
13print(" Now starting !")
15# The rest
16#import flask.json.provider
17from flask import jsonify, Flask, request, send_from_directory, Response, url_for, render_template, make_response, redirect, render_template_string
18#from flask import jsonify, send_from_directory, url_for, render_template
19#from aioflask import make_response, redirect, render_template_string
20#from aioflask import Flask, request, Response
22import os, sys
23import json
24import time
26#from lib.lib_ml.lib_tent_py_fvs import FVSInterface
27# en attendant d'avoir accès à orson
28sys.path.append(os.getcwd())
29# sys.path.append(os.getenv("GITSAFIA") + "/prompt/python")
30sys.path.append(os.getcwd() + "../")
31sys.path.append("/home/safia/workarea/git/Safia/prompt/python")
32sys.path.append("/home/safia/mtr/.credentials")
34try :
35# sys.path.append(os.path.join(os.getenv("GITVELOURS"), "c_visual_search/pyfvs/install"))
36# sys.path.append(os.path.join(os.getenv("GITVELOURS"), "../git/pyfvs"))
37 if "GITPYFVS" in os.environ:
38 sys.path.append(os.getenv("GITPYFVS"))
39 from pyfvs.lib.lib_pyfvs_from_safia_to_refacto import FVSInterface
40 fvs_interface = FVSInterface()
41except Exception as e:
42 fvs_interface = None
44from lib.lib_live.lib_wsns import WebSocketAllocator
45ws_all_singleton = WebSocketAllocator()
47from flask_cors import CORS
49# - [ ] TODO TO REFACTO
50to_test_mets_a_jour = "TO TEST Mets à jour" # 'Please use just our interface and make no direct query' OR Please log correctly on opia/assistant
51# Former static_folder was just static
54from flask_openapi3 import Info, Tag
55from flask_openapi3 import OpenAPI
56from pydantic import BaseModel, Field
57from typing import Optional
59from lib.lib_version import version_name, version_date
60info = Info(title="Saxia API", version=version_name + " " + version_date)
61app = OpenAPI(__name__, info=info, static_folder=os.path.join(os.getcwd(), 'static/public'))
63# app = Flask(__name__, static_folder=os.path.join(os.getcwd(), 'static/public'))
64# finally useless
65# CORS(app)
67# VR tested 6/12/25 : makes scheme to https in debug with apache, but how to configure gunicorn ?
68from werkzeug.middleware.proxy_fix import ProxyFix
70# Nombre de proxies que vous attendez avant d'atteindre votre application
71# Changez ce nombre selon votre architecture
72# VR 6/12/25 : avoid p request.scheme : https
73app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_host=1, x_port=1, x_proto=1)
74# this doesn't work with gunicorn
75#app.wsgi_app = ProxyFix(app.wsgi_app, x_for=2, x_host=2, x_port=2, x_proto=2)
77app.config['MAX_CONTENT_LENGTH'] = 1073741824
79from flask import session
80from flask_session import Session
81SESSION_TYPE = 'filesystem'
82app.config.from_object(__name__)
83Session(app)
85#@app.route('/')
86#def reset():
87# from flask import session
88# session["counter"]=0
90print (sys.version)
91print (sys.version_info)
94@app.route('/.well-known/pki-validation/0BE9E75E5A8D040E0192B493800D792C.txt', methods=['GET'])
95def well_known():
96 return """82B9EE62A2D71A89A97E8C083B2CEE4F3C13374EDC90FA7327FA7204BD4FED75
97sectigo.com
989965d9fbc6401522fb28"""
102#@app.route('/api/v1/document/show_all', methods=['GET'])
103@app.route('/jscover', methods=['GET'])
104def jscover():
105 # Doesn't work !
106# return send_from_directory(directory="static/js_inst", path='jscoverage.html')
108#if (window.jscoverage_report) {
109# jscoverage_report(dir, callback, timeout);
110#}
112 return make_response(render_template('jscover_safia.html'))
116@app.route('/jscoverage-store/<folder>', methods=['POST'])
117def jscoverstore(folder):
119# f = request.files['file']
121 if not os.path.exists(folder):
122 os.makedirs(folder)
124 with open(os.path.join(folder, "jscoverage.json"), "w+") as f:
125# import ast, json
126# json_data = ast.literal_eval(json.dumps(request.json))
127 json.dump(request.json, f)
129# os.system("mv " + f.filename + " " + folder)
131 return "Stored !"
135print(str(sys.path))
136from auth.lib_conf_system import LibConfSystem as LCS
137lcs = LCS()
139from lib.stockage.lib_pg_safia_sys import LibPGSafiaSys
140(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT) = lcs.get_pg_conf()
141lpgss_singleton = LibPGSafiaSys(PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT)
143# Configure SQLAlchemy before creating blueprints
144# This ensures all blueprints use the same database configuration
145from flask_sqlalchemy import SQLAlchemy
147app.config['SQLALCHEMY_DATABASE_URI'] = f'postgresql+psycopg2://{PG_USER}:{PG_PASSWORD}@{PG_HOST}:{PG_PORT}/{PG_DB}'
148app.config['SECRET_KEY'] = 'your_secret_key_here' # TODO: Move to config file
149app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False # Disable modification tracking for performance
150app.config['OAUTH2_REFRESH_TOKEN_GENERATOR'] = True # Disable modification tracking for performance
151app.config['OAUTH2_TOKEN_EXPIRES_IN'] = {'authorization_code': 31530000, 'implicit': 3600, 'password': 864000, 'client_credentials': 864000}
152#app.config['OAUTH2_TOKEN_EXPIRES_IN'] = 150000000 # => 3 years
154# Create a single SQLAlchemy instance for the entire application
155db_app = SQLAlchemy()
156db_app.init_app(app)
158try :
159 from server.bp.auth import create_login_bp
160 # Pass the db_app instance to the blueprint to ensure we use the same instance
161 bp_auth, User = create_login_bp(app, db=db_app)
162 app.register_blueprint(bp_auth)
163except Exception as e:
164 print("No login")
165 print(str(e))
166 # print stacktrace
167 import traceback
168 traceback.print_exc()
170# VR 12/11/25 TO gather with the others bps
171from server.bp.oauth import create_oauth_bp
172oauth_bp, require_oauth = create_oauth_bp(app, db_app)
173#from lib.auth_oauth.oauth2 import config_oauth
174#config_oauth(app)
175#app.register_blueprint(oauth_bp)
176app.register_api(oauth_bp)
177#app.register_api_view(oauth_bp)
181@app.route('/saxia', methods=['GET'])
182def saxia_index():
184 return render_template('saxia/index.html')
188from lib.lib_airtable import LibAirTable
189(api_key_at, base_id_pi) = lcs.get_at_pe_conf()
190lib_at_ppe_singleton = LibAirTable(api_key_at, base_id_pi)
191(api_key_at, base_id_apia) = lcs.get_at_apia_conf()
192lib_att_apia_singleton = LibAirTable(api_key_at, base_id_apia)
194# - [ ] TODO VR utiliser lcs ou lss dans le code des requetes ou plutot dans la config ?
195# - [ ] TODO on veut quand meme protéger ces paramètres, donc il faut les injecter uniquement au moment ou c'est utile !
196OPENAI_API_KEY = lcs.get_openai_api_key()
198from lib.lib_safia import insert_query_log
199from lib.lib_util import count_and_display_elapsed_time
200from lib.lib_safia_system import LibSafiaSystem
201# TODO VR LibUserDataExternalServices est (pour l'instant lib_auth en fait)
202from auth.lib_auth import LibUserDataExternalServices
203# TODO VR pb with multiple user while there are user info kept (but maybe not used in this class !)
204lib_auth_now_from_at = LibUserDataExternalServices(lib_at = lib_at_ppe_singleton)
205lib_external_info_from_apia_at = LibUserDataExternalServices(lib_at = lib_att_apia_singleton)
207from auth.lib_auth import get_datou_exec_context_as_complete_param_json
209from auth.lib_right import LibRight
210lib_right_singleton = LibRight(lib_internal_data=lpgss_singleton)
212# Ou a mettre dans libAuth ?
213# TODO VR 13-6-23 : refacto localisation
214# Rename une partie connect_with_mail_to_validate_by_otp
215# et une autre partie get_context_lss_as_json
216# surtout prendre lss en argument au pire !
217# ca peut-aussi créer lss
220def register_user_get_datae(requeste = None, list_data_needed = [],
221 verbose = False):
222 lss = LibSafiaSystem(lib_user_data_internal=lpgss_singleton,
223 lib_user_data_external=lib_external_info_from_apia_at,
224 lib_auth_user_otp=lib_auth_now_from_at,
225 lib_right=lib_right_singleton)
227 from flask_login import current_user
229 print(str(current_user))
231 if requeste == None:
232 requeste = request
233 otp = requeste.cookies.get("otp")
234 project_id_str = requeste.cookies.get("project_id")
236 if verbose:
237 print("project_id_str : " + str(project_id_str))
238 if project_id_str == None or project_id_str == "undefined":
239 # TODO VR define anonymous project id
240 curr_project_id = -1
241 else:
242 curr_project_id = int(project_id_str)
244 if current_user.is_authenticated:
245 if current_user.mtr_user_id == None:
246 print(" TODO need to associate or create mtr_user_id")
247 is_valid = False
248 else :
249 if verbose:
250 print("current_user.mtr_user_id : " + str(current_user.mtr_user_id))
251 info_user = lss.lib_user_data_internal.select_user(current_user.mtr_user_id)
252 lss.user = info_user["mail"]
253 lss.user_id = info_user["id"]
254 is_valid = True
255 else :
256 print(" TODO VR 1-5-24 deprecate this old authentification method !")
258 info, is_valid = lss.connect_with_otp(otp)
259 if "Email_User" in info:
260 info_user = lss.lib_user_data_internal.get_user_from_mail(info["Email_User"])
261 has_access = lss.lib_right.get_role_on_project(info_user["id"], curr_project_id)
262 print(" has_access : " + str(has_access))
263 else :
264 is_valid = False
266# return info, is_valid, curr_project_id
268#def get_context_lss_as_json(lss, list_data_needed = []):
269 data_needed = {}
271 if is_valid:
272 user = lss.get_user()
274 # TODO VR 19-6-23 This architecture is quicker, but we are duplicating it from lss
275 if "table_documents" in list_data_needed:
276 info_project = lss.get_project_info(curr_project_id)
277 table_documents = info_project["table_documents"] if "table_documents" in info_project else "table_documents"
278 data_needed["table_documents"] = table_documents
279 if "project_id" in list_data_needed:
280 data_needed["project_id"] = curr_project_id
281 if "otp" in list_data_needed:
282 data_needed["otp"] = otp
284 current_user.lss = lss
285 return user, data_needed
286 else :
287 data_needed["user_id"] = -1
288 data_needed["project_id"] = -1
289 data_needed["otp"] = otp
290 data_needed["table_documents"] = "table_documents"
291 current_user.lss = lss
292 return "anonymous@opio.fr", data_needed
296#any([request.path.startswith('static/'),
297#request.endpoint != None and request.endpoint.startswith('static/'),
298#current_user.is_authenticated, # From Flask-Login
299#getattr(app.view_functions[request.endpoint], 'is_public', False)]):
300#return # Access granted
301#else:
302#return redirect(url_for('login.login') + "?redirecturl=" + request.url)
304# @app.before_request
305def register_user_get_data(requeste=None, list_data_needed=[]):
306# print(str(app.view_functions[request.endpoint]))
307 user, data_needed = register_user_get_datae(requeste, list_data_needed)
308 from flask_login import current_user
309 lss = current_user.lss
310 return user, data_needed, lss
312def make_page_anonymous(msg = ""):
313 html = """
314 You have no valid OTP password or your are browsing anonymously.
315 Please create an account on <a href="https://www.openpromptia.com">opia</a> or https://opia.softr.app https://opia.softr.app/apia/assistant#otp2pe
316 Please create an account on <a href="https://reanna392.preview.softr.app/apia/assistant#otp2pe">opia</a> or https://reanna392.preview.softr.app/apia/assistant#otp2pe
317 and get an OTP password or EXTEND ITS EXPIRY DATE ON <a href="https://reanna392.preview.softr.app/apia/assistant#otp2pe">here</a> (valid 24 hours).
318 If you feel you got this message by error you can send and email to info@opio.fr .
319 Maybe openpromptia.com will come back soon
320 """ + msg
321 response = make_response(render_template_string(html))
322 response.set_cookie('project_id', '', expires=0)
323 response.set_cookie('otp', '', expires=0)
324 return response
328@app.route('/api/v1/project/queries/<project_id>', methods=['GET'])
329def project_queries(project_id):
330 user, data_needed, lss = register_user_get_data(request)
332 parse = request.args.get("parse", None)
334 # TODO vr check right
335 res = lss.get_queries(project_id)
337 if parse == "tsne_for_latex":
338 from lib.lib_safia import parse_tsne_for_latex
339 res = parse_tsne_for_latex(res)
341 response = Response(
342 response=json.dumps(res, indent=4, sort_keys=True, default=str),
343 #json.dumps(res),
344 status=200,
345 mimetype='application/json'
346 )
347 return response
351@app.route('/invitation/<token>')
352def invitation_token(token):
353 user, data_needed, lss = register_user_get_data(request, list_data_needed = [])
354 if "anonymous@opio.fr" == user:
355 # TODO VR : tu n'es authentifié que par cookie et ne sera que temporaire ! The link is valid until !
356 return make_page_anonymous()
357 else :
358 return lss.use_invitation(token)
360# - [ ] VR TODO faire un edit plutot ?
361@app.route('/api/v1/project/role/remove', methods=['POST'])
362def project_role_remove():
363 user, data_needed, lss = register_user_get_data(request, list_data_needed = [])
365 project_id = int(request.json.get("project_id"))
366 user_id_to_change = int(request.json.get("user_id"))
368 if "anonymous@opio.fr" != user:
369 log = lss.remove_right(project_id, user_id_to_change)
371 projects = {"log" : log}
372 else :
373 projects = {"data" : [], "iter_pages" :[]}
374 return render_template('projects.html', projects=projects)
378@app.route('/project', methods=['GET'])
379def project_detail():
380 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
381 curr_project_id = data_needed["project_id"]
382 project_id = request.args.get("project_id")
383 key = request.args.get("key", None)
384 if project_id == None:
385 project_id = curr_project_id
387 if "anonymous@opio.fr" != user:
389# project_data = lss.get_project(project_id)
390 configuration = lss.load_conf_project(project_id, key)
391 else :
392 configuration = {}
394 # key
395 if key != None and key != "" and project_id != "all":
396 from lib.manaudit.lib_datou_audit import load_sub_json
397 configuration = load_sub_json(configuration, key)
399 project_data = {"configuration" : json.dumps(configuration)}
400 title = " Project Configuration Page Editor for project : " + str(project_id) + (str(key) if key != None else "")
402 # TODO 22/11 a supprimer ou mettre une condition json=Trye
403 response = Response(
404 response=json.dumps(project_data, indent=4, sort_keys=True, default=str),
405 status=200,
406 mimetype='application/json'
407 )
408# return response
410 dataframe_html = "all_configuration"
411 if project_id == 'all':
412 import pandas as pd
413 df = pd.DataFrame(configuration)
414 dataframe_html = df.to_html()
416 results = {}
417 results["project_id"] = project_id
418 results["user_id"] = user
419 results["document_name"] = ""
420 results["key"] = key
421 results["dataframe_html"] = dataframe_html
423 return render_template('project_details.html',
424 project_data=project_data,
425 title=title, results=results)
428@app.route('/projects', methods=['GET'])
429def all_projects():
430 user, data_needed, lss = register_user_get_data(request, list_data_needed = [])
432 if "anonymous@opio.fr" != user:
434 projects_data = lss.get_projects(lss.get_user_id())
435 projects = {"data" : projects_data, "iter_pages" : []}
436 else :
437 projects = {"data" : [], "iter_pages" :[]}
438 # - [ ] TODO VR why the proejct detail routes launch again projects.html
439 return render_template('projects.html', projects=projects)
441@app.route('/group', methods=['GET'])
442def group():
443 user, data_needed, lss = register_user_get_data(request, list_data_needed = [])
445 if "anonymous@opio.fr" != user:
446 group = lss.get_group()
447 groups = {"data" : group}
448 else :
449 groups = {"data" : []}
450 return render_template('group.html', groups=groups)
452# /api/v1/groups/' + groupId + '/revoke/' + userId,
453# method: 'DELETE',
455@app.route('/api/v1/groups/<groupId>/revoke/<userId>', methods=['POST'])
456def api_v1_groups_revoke(groupId, userId):
457 user, data_needed, lss = register_user_get_data(request, list_data_needed = [])
459 if "anonymous@opio.fr" != user:
460 log = lss.remove_right_group(lss.get_user_id(), groupId, userId)
462 response = Response(
463 response=json.dumps(log, indent=4, sort_keys=True, default=str),
464 status=200,
465 mimetype='application/json'
466 )
467 return response
469@app.route('/api/v1/groups/admin', methods=['POST'])
470def get_group():
471 user, data_needed, lss = register_user_get_data(request, list_data_needed = [])
473 if "anonymous@opio.fr" != user:
474 res = lss.get_group()
476 response = Response(
477 response=json.dumps(res, indent=4, sort_keys=True, default=str),
478 status=200,
479 mimetype='application/json'
480 )
481 return response
483@app.route('/api/v1/group/create', methods=['POST'])
484def create_group():
485 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["user_id"])
487 name = request.json.get("name", "DEFAULT_NAME")
489 if "anonymous@opio.fr" != user:
490 res = lss.create_group(name = name)
492 response = Response(
493 response=json.dumps(res, indent=4, sort_keys=True, default=str),
494 status=200,
495 mimetype='application/json'
496 )
497 return response
499@app.route('/api/v1/invitation/create', methods=['POST'])
500def api_v1_invitation_create():
501 user, data_needed, lss = register_user_get_data(request, list_data_needed = [])
503 project_id = request.json.get("project_id")
504 group_id = request.json.get("group_id")
505 role = request.json.get("role")
506 mail = request.json.get("mail")
508 res = lss.create_invitation(project_id, group_id, role, mail, request.host_url)
510 response = Response(
511 response=json.dumps(res, indent=4, sort_keys=True, default=str),
512 #json.dumps(res),
513 status=200,
514 mimetype='application/json'
515 )
516 return response
519@app.route("/bugr")
520def bugr():
521 try :
522 lpgss_singleton.add_role_to_user_group_buggue()
523 except Exception as e:
524 print("It always go through this !")
525 print(str(e))
526 message = str(e) + "\n" + "INTERNAL ERROR CAUSED BY MANUAL TEST (AND TO CHECK suiviprod) could be INTERNALAL ERROR otherwise"
527 print(message)
528 return message
531@app.route("/reconnect")
532def reconnect():
533 from server.lib_www_admin import reconnect_lib
534 return reconnect_lib(lpgss_singleton)
537@app.route('/suiviprod/pg/admin', methods=['GET'])
538def suiviprod_pg_admin():
539 l, ll = "", ""
540 verbose = bool(request.args.get("verbose", False))
541 error_message = lpgss_singleton.client.info.error_message
542 try :
543 l, ll = lpgss_singleton.get_admin_situation(verbose=verbose)
544 except Exception as e:
545 print(str(e))
546 return str(l) + "\n<br>" + str(ll) + "\n<br>" + str(error_message)
548@app.route('/project/admin/<project_id>', methods=['GET'])
549def project_admin(project_id):
550 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["otp", "project_id"])
551 otp = data_needed["otp"]
552 curr_project_id = data_needed["project_id"]
554 # TODO VR 6-6-44+79 : simplify : project_id and curr_project_id should be the same (so insane)
555 if int(project_id) != curr_project_id:
556 print("Project Id has been forced in interface, need right on project and admin in order to avoid this ! ")
557 curr_project_id = int(project_id)
559 if "anonymous@opio.fr" != lss.get_user():
560 data_user = lss.get_data_user(user)
562 is_project_owner = False
563 for d in data_user:
564 if d["id"] == curr_project_id:
565 is_project_owner = True
567 if is_project_owner or lss.user_id == 1:
568 info_project = lss.get_project_info(curr_project_id)
570 return render_template('project_admin.html', data=[info_project])
571 else :
572 return "You are the queen !"
573 else :
574 response = make_page_anonymous()
576 return response
577# make_page_anonymous()
579@app.route('/about', methods=['GET'])
580def about():
581 return "Consult fotonower.com or openpromptia.com"
585@app.route('/api/v1/form/getintouch', methods=['POST'])
586def landing_submit():
587 print("New Request !")
588 print(str(request.json))
589 print(str(request.form))
590 print(str(request.args))
591 return {"Thank" : "you !"}
594@app.route('/landing', methods=['GET'])
595def landing():
596 from lib.lib_version import version_name, version_date
597 result = {"version" : {"name" : version_name, "date" : version_date}}
599 resp = make_response(render_template('landing.html', results=result))
600 return resp
602# TODO VR 3-6-24 Type EP (sans doute à mettre en BP) safia / saxia
603# TODO VR rename data_needed into session_context_json
604@app.route('/ssaxia', methods=['GET'])
605def saxia_simple(project_id = None):
606 config_app = {
607 }
609 user, data_needed, lss = register_user_get_data(request, list_data_needed=["project_id", "otp"])
610 # from flask_login import current_user
611 # lss = current_user.lss
612 curr_project_id = data_needed["project_id"]
613 if project_id != None:
614 curr_project_id = project_id
616 auth = lss.lib_right.get_role_on_project(lss.get_user_id(), curr_project_id)
617 id_admin = lss.lib_right.is_user_admin_config(lss.get_user_id())
618 if auth == None and id_admin == False:
619 print(" We will select the first project where the user has right")
620 list_pids = lss.lib_right.get_projects_with_access(lss.get_user_id())
621 if len(list_pids) > 0 and "id" in list_pids[0]:
622 curr_project_id = list_pids[0]["id"]
625 raw_configuration = lss.load_conf_project(curr_project_id)
628 from lib.util.lib_formal_conf import formal_conf_prepare
629 # TODO example migration remove load_and_instantiate plop
630 begin_time = time.time()
631 try :
632 configuration_surannee = lcs.load_and_instantiate_config(raw_configuration)
633 except Exception as e:
634 print(str(e))
635 configuration_surannee = {}
637 begin_time, message = count_and_display_elapsed_time(begin_time, "load_and_instantiate_config", verbose=True)
638 configuration = formal_conf_prepare(raw_configuration, lss)
639 if configuration != configuration_surannee:
640 print("MIGRATION VERIFLFC DONE OR PB => NRT TO DO BY THE WAY !")
641 begin_time, message = count_and_display_elapsed_time(begin_time, "formal_conf_prepare", verbose=True)
642 custom_display = configuration["custom_display"] if "custom_display" in configuration else {}
643 if "home" in custom_display:
644 custom_display = custom_display["home"]
646 config_app = lcs.get_config_app()
648 type_suivi = "quick"
650 request_endpoint_config = "saxia_suivi"
651 endpoint_df_conf = configuration["custom_display"][request_endpoint_config] if "custom_display" in configuration and request_endpoint_config in configuration[
652 "custom_display"] else {}
653 endpoint_df_conf_type_suivi = endpoint_df_conf[type_suivi] if type_suivi in endpoint_df_conf else endpoint_df_conf[
654 "default"] if "default" in endpoint_df_conf else {}
655 condition = endpoint_df_conf_type_suivi["condition"] if "condition" in endpoint_df_conf_type_suivi else []
657 #if type_suivi == "customer" or project_id_arg != None:
658 if curr_project_id != None:
659 condition.append({'data_type': 'int', 'value': curr_project_id, 'variable': 'project_id'})
661 col_csv = "id,id_file,mtr_datou_id,created_at,launched_at,modified_at,deleted_at,valid_until,user_id,project_id,safia_doc_id,hash_id_treatment,cdn,input_data,nb_page,nb_modif,input_file,audit_resume,info_anon,info_customer,info_lab,info_cdn,info_back,info_date,info_suivi,info_action,info_usage_integration,info_ml,info_init,info_extract,info_consolidate,info_result,info_exec,info_page,info_stat"
662 # col_csv = "id,id_file,mtr_datou_id,created_at,info_date"
664 col_csv = request.args.get('col_csv', col_csv)
666 saxia_steps = []
668 try_to_count = lss.lib_user_data_internal.load_data_audit(col_csv="count(*)",
669 condition=condition)
670 print(" try_to_count : " + str(try_to_count))
671 limit = 100
672 nb_pagination = int(try_to_count[0]["count"] / limit) + 1
673 print(" nb_pagination : " + str(nb_pagination))
675 all_result = lss.lib_user_data_internal.load_data_audit(output_type="pd",
676 col_csv=col_csv,
677 limit=30,
678 condition=condition)
681 list_col_display = []
682 try:
683 from lib.batch.lib_batch import create_pandas_table_from_text, append_action_steps_button, set_custom_display
684 safia_suivi_prod_doc_id = "suivi_virtual"
685 safia_suivi_prod_proj_id = project_id
686 append_action_steps_button(all_result, safia_suivi_prod_doc_id, safia_suivi_prod_proj_id, saxia_steps)
687 list_col_display = set_custom_display(all_result, endpoint_df_conf_type_suivi, col_csv=col_csv)
688 except Exception as e:
689 print("ERROR while configuring suivi")
690 print(str(e))
692 list_datou_available = lss.get_datou()
693 # TODO since jenji does not work with {% if proj_conf["datou-{{k.id}}"] == 'true' %} checked {% endif %}
694 for d in list_datou_available:
695 d["checked"] = "checked" if "datou-" + str(d["id"]) in custom_display and custom_display["datou-" + str(d["id"])] == "true" else ""
697 result = {
698 "project_id" : curr_project_id,
699 "user_id" : "Docteur Victor",
700 "custom_display" : custom_display,
701 "df": all_result,
702 "list_col_display" : {"list" : list_col_display},
703 "datou" : list_datou_available
704 }
705 resp = make_response(render_template('saxia/saxia_simple.html', results=result,
706 config_app=config_app,
707 title="Bonjour Docteur Victor, Bienvenue dans l'assistant Saxia à la redaction de rapport d'expertise ",
708 custom_display = custom_display))
709 return resp
712# TODO VR 3-6-24 Type EP (sans doute à mettre en BP) safia / saxia
713# TODO VR rename data_needed into session_context_json
714@app.route('/chat', methods=['GET'])
715# @app.route('/', methods=['GET'])
716def index(project_id = None):
718 loggger.warning(" request.scheme : " + str(request.scheme))
719 print(" p request.scheme : " + str(request.scheme))
721 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id", "otp"])
722# from flask_login import current_user
723# lss = current_user.lss
724 curr_project_id = data_needed["project_id"]
725 otp = data_needed["otp"]
727 curr_project_id = int(request.args.get("project_id", curr_project_id))
728 if project_id != None: # pour gérer le short_name/shorlink endpoint => pas du tout ca marche pas et puis voilà !
729 curr_project_id = project_id
731 if lss.get_user_id() != None:
732 auth = lss.lib_right.get_role_on_project(lss.get_user_id(), curr_project_id)
733 id_admin = lss.lib_right.is_user_admin_config(lss.get_user_id())
734 if auth == None and id_admin == False:
735 print(" We will select the first project where the user has right")
736 list_pids = lss.lib_right.get_projects_with_access(lss.get_user_id())
737 if len(list_pids) > 0 and "id" in list_pids[0]:
738 curr_project_id = list_pids[0]["id"]
740 config_app = lcs.get_config_app()
742 configuration_as_json = lss.load_conf_project(curr_project_id)
743 custom_display = configuration_as_json["custom_display"] if "custom_display" in configuration_as_json else {}
744 if "home" in custom_display:
745 custom_display = custom_display["home"]
746 # VR TODO 27-4-24 : refactor this to avoid special case for saxia
747 saxia_conf = configuration_as_json["saxia"] if "saxia" in configuration_as_json else {}
748 assoc = saxia_conf["assoc"] if "assoc" in saxia_conf else {}
749 safia_result = assoc["safia"] if "safia" in assoc else {}
751 if user != "anonymous@opio.fr":
752 data_user = lss.get_data_user(user, user_id_hack=lss.get_user_id())
754 for d in data_user:
755 if not str(curr_project_id).isnumeric():
756 curr_project_id = d["id"]
758 if d["id"] == curr_project_id:
759 d["SELECTED"] = 1
761# list_datou_available = lss.lib_user_data_internal.get_datou_user(user _id)
762 list_datou_available = lss.get_datou()
763 # TODO since jenji does not work with {% if proj_conf["datou-{{k.id}}"] == 'true' %} checked {% endif %}
764 for d in list_datou_available:
765 d["checked"] = "checked" if "datou-" + str(d["id"]) in custom_display and custom_display["datou-" + str(d["id"])] == "true" else ""
767 import json
768 from lib.lib_version import version_name, version_date
769 result = {"projects" : data_user, "datou" : list_datou_available, "version" : {"name" : version_name, "date" : version_date}}
770 result["project_id"] = curr_project_id
771 result["user_id"] = user
772 result["document_name"] = None
773 name_app = "Saxia" if "app_type" in config_app and config_app["app_type"] == "saxia" else "Safia"
774 resp = make_response(render_template('chat.html', results=result,
775 custom_display = custom_display,
776 safia_result = safia_result,
777 config_app = config_app,
778 title="Welcome to " + name_app + " : Your personnal assistant with project_id : " + str(curr_project_id)))
779 # copyright=None
780 # resp.set_cookie('project_id', curr_project_id)
781 if otp != None:
782 resp.set_cookie("otp", otp, max_age = 60*60*24*15) # Le cookie expire après 15 jours
783 resp.set_cookie("project_id", str(curr_project_id), max_age = 60*60*24*15)
784# resp.headers["Cross-Origin-Embedder-Policy"] = "require-corp"
785# resp.headers["Cross-Origin-Opener-Policy"] = "same-origin"
786 return resp
787 else :
789 from lib.lib_version import version_name, version_date
790 result = {"version" : {"name" : version_name, "date" : version_date}}
792 if "app_type" in config_app and config_app["app_type"] == "saxia":
793 resp = make_response(render_template('saxia/index.html', results=result,
794 config_app = config_app,
795 title="Welcome to Saxia projecT_id : " + str(curr_project_id)))
796 else:
797 resp = make_response(render_template('landing.html', results=result,
798 title="Welcome to Safia projecT_id : " + str(curr_project_id)))
799# resp.headers["Cross-Origin-Embedder-Policy"] = "require-corp"
800# resp.headers["Cross-Origin-Opener-Policy"] = "same-origin"
801 return resp
804# TODO VR rename data_needed into session_context_json
805@app.route('/editor', methods=['GET'])
806def editor():
807 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id", "otp"])
808 project_id = data_needed["project_id"]
809 otp = data_needed["otp"]
811 document_id = request.args.get("document_id", None)
812 project_id = request.args.get("project_id", project_id)
813 # - [ ] TODO VR ultra hack
814 if document_id != None and "_" in document_id and document_id.split("_")[-1].isdigit() and int(document_id.split("_")[-1]) < 1000000000:
815 chunk_id = document_id.split("_")[-1]
816 document_id = "_".join(document_id.split("_")[0:-1])
817 else :
818 chunk_id = None
820 print("TODO check right of project_id !")
822 if user != "anonymous@opio.fr":
824 import json
825 document_content = lss.load_document(document_id, project_id, chunk_id)
827 result = {"document_id" : document_id, "document_content" : document_content, "chunk_id" : chunk_id, "project_id" : project_id}
828 resp = make_response(render_template('editor.html', results=result, title="SAFIA"))
829# resp.set_cookie('project_id', curr_project_id)
830 resp.set_cookie("project_id", str(project_id), max_age=60*60*24*15)
831 return resp
832 else :
833 response = make_page_anonymous()
835 return response
839# TODO VR rename data_needed into session_context_json
840@app.route('/smarteditor', methods=['GET'])
841def smarteditor():
842 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id", "otp"])
843 project_id = data_needed["project_id"]
844 otp = data_needed["otp"]
846 document_id = request.args.get("document_id", None)
847 project_id = request.args.get("project_id", project_id)
848 # - [ ] TODO VR ultra hack
849 if document_id != None and "_" in document_id and document_id.split("_")[-1].isdigit() and int(document_id.split("_")[-1]) < 1000000000:
850 chunk_id = document_id.split("_")[-1]
851 document_id = "_".join(document_id.split("_")[0:-1])
852 else :
853 chunk_id = None
855 print("TODO check right of project_id !")
857 if user != "anonymous@opio.fr":
859 import json
860 document_content = lss.load_document(document_id, project_id, chunk_id)
861 safia_suivi_prod_doc_id = document_id
862 safia_suivi_prod_proj_id = project_id
864 from lib.batch.lib_batch import create_pandas_table_from_text, append_action_steps_button
865 df = create_pandas_table_from_text(document_content)
867 configuration = lss.load_conf_project(project_id)
868 saxia_conf = configuration["saxia"] if "saxia" in configuration else {}
870# saxia_conf["format"]["info_format_intro"]
872 # pour les steps qui ont une col on va crée des boutons
873 # Clairement il faut dans l'interface la possibilité d'uploader des fichiers (mais il faudra aussi simplement les indexer depuis des dossiers)
874 # la ca ressemble au besoin lib_drive qui a été spécifié dans un commentaire
876 saxia_steps = saxia_conf["steps"] if "steps" in saxia_conf else {}
878 try:
879 append_action_steps_button(df, safia_suivi_prod_doc_id, safia_suivi_prod_proj_id, saxia_steps)
880 except Exception as e:
881 print(str(e))
882 print("omitting append_buttoon audit")
883 if type(df) == type(None):
884 print("df is None")
885 import pandas as pd
886 df = pd.DataFrame(columns=["Init Col 1"])
887 result = {"df" : df, "document_id" : document_id,
888 "document_content" : document_content, "chunk_id" : chunk_id,
889 "project_id" : project_id,
890 "list_col_display" : {"list" : []},
891 "custom_display" : {}}
893 resp = make_response(render_template('smarteditor.html', results=result, title="SAFIA"))
894 resp.set_cookie("project_id", str(project_id), max_age=60*60*24*15)
895 return resp
896 else :
897 response = make_page_anonymous()
899 return response
902@app.route('/api/v1/document/save', methods=['POST'])
903def document_save():
904 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
905 project_id = data_needed["project_id"]
906 project_id = request.json.get("project_id", project_id)
908 print(str("Pas trop content"))
910 if user != "anonymous@opio.fr":
911 save_document_data = request.json
912 result_json = lss.save_document(save_document_data, project_id, openai_token=OPENAI_API_KEY)
913# lss.lib_data.update_datou(update_datou_data) =>
915 # update_datou
917 response = Response(
918 response=json.dumps(result_json),
919 status=200,
920 mimetype='application/json'
921 )
922 return response
924 return to_test_mets_a_jour
926@app.route('/api/v1/user/audit/<user_id_used>', methods=['POST'])
927def api_v1_user_audit(user_id_used):
928 print("We are about to audit user_id")
930 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["user_id"])
931 user_id_connect = lss.get_user_id()
933 check = lss.lib_right.is_user_admin_config(user_id_connect)
934 warning_admin = False
935 if check and user_id_used != None:
936 user_id = user_id_used
937 lss.user_id = user_id
938 else :
939 user_id = user_id_connect
941 info = lss.audit_user()
943 return {"user_id" : user_id, "actions" : "User Audite", "info" : info}
946@app.route('/api/v1/user/delete/<user_id_used>', methods=['POST'])
947def api_v1_user_delete(user_id_used):
948 print("We are about to delete userid")
950 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["user_id"])
951 user_id_connect = lss.get_user_id()
953 check = lss.lib_right.is_user_admin_config(user_id_connect)
954 warning_admin = False
955 if check and user_id_used != None:
956 user_id = user_id_used
957 lss.user_id = user_id
958 else :
959 user_id = user_id_connect
961 ret = lss.delete_user()
963 return {"user_id" : user_id, "actions" : "Request user deletion", "info" : ret}
969# TODO VR move to specific util interface library
970def make_df_with_list_facture(info_invoices, type_column = "user"):
971 import pandas as pd
972 df_invoice = pd.DataFrame(info_invoices)
974 if type_column == "user":
975 list_columns_remove = ['type_storage', 'project_id', 'validated', 'price_ht']
976 elif type_column == "admin":
977 list_columns_remove = []
979 try:
980 if len(list_columns_remove) > 0:
981 print(df_invoice.columns)
982 df_invoice = df_invoice.drop(columns=list_columns_remove) # , 'project_id', 'project_id'
983 except Exception as e:
984 print("Error dropping columns in dataframe")
985 print(str(e))
987 # Boucle sur les données en rangée
988 for index, row in df_invoice.iterrows():
989 # On récupère le nom du fichier
990 filename = row['filename']
991 # On le met dans la colonne 'filename'
992 df_invoice.at[index, 'filename'] = "<a href='"+filename+"'>Télécharger</a>"
994 if type_column == "admin":
995 for index, row in df_invoice.iterrows():
996 # On récupère le nom du fichier
997 validated = row['validated']
998 id = row['id']
999 if validated:
1000 # On le met dans la colonne 'filename'
1001 df_invoice.at[index, 'set_valid'] = "You can't cancel a validated Bill, please make an 'avoir' by setting a fake folder with negative nb page !"
1002 else :
1003 df_invoice.at[index, 'set_valid'] = "<a href='/api/v1/admin/saxia/bill/"+ str(id) +"/set_valid'>Click to set Valid (can't be undone)</a>"
1005 # id
1006# filename
1007# created_at
1008# month
1009# payed_at
1010# nb_pages
1011# price_ttc
1013 return df_invoice
1015@app.route('/api/v1/admin/saxia/bill/generate', methods=['GET'])
1016def api_v1_admin_saxia_bill_generate():
1017 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["user_id"])
1018 is_admin = lss.lib_right.is_user_admin_config(lss.get_user_id())
1019 # TODO VR : need to check the month is finished !
1020 if is_admin:
1021 from lib.batch.lib_batch import generate_saxia_fact
1022 month = request.args.get("month", None)
1023 year = request.args.get("year", None)
1024 id_fact = request.args.get("id_fact", None)
1025 type_fact = request.args.get("type_fact", None)
1026 project_id = request.args.get("project_id", None)
1027 root_folder_fact = "/static/factures" # TODO VR Should come from config app grrr
1028 log = generate_saxia_fact(id_fact, month, year, type_fact, project_id, root_folder_fact =root_folder_fact)
1029 return log + "\nMaybe Done, please check on admin page"
1030 else:
1031 print("Please log in correctly !")
1033@app.route('/api/v1/admin/saxia/bill/<bill_id>/set_valid', methods=['GET'])
1034def api_v1_admin_saxia_bill_set_valid(bill_id):
1035 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["user_id"])
1036 is_admin = lss.lib_right.is_user_admin_config(lss.get_user_id())
1037 # TODO VR : need to check the month is finished !
1038 if is_admin:
1039 lss.lib_user_data_internal.set_valid_bill(bill_id)
1040 return "Maybe Done, please check on admin page"
1041 else:
1042 print("Please log in correctly !")
1044@app.route('/account', methods=['GET'])
1045def account():
1046 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["user_id"])
1048 project_id_arg = request.args.get("project_id", None)
1050 user_id_connect = lss.get_user_id()
1051 check = lss.lib_right.is_user_admin_config(user_id_connect)
1052 user_id_used = request.args.get("user_id", None)
1053 warning_admin = False
1054 if check and user_id_used != None:
1055 user_id = user_id_used
1056 lss.user_id = user_id
1057 warning_admin = "As Admin, you are using user_id " + str(user_id) + " instead of " + str(user_id_connect)
1058 else:
1059 user_id = user_id_connect
1061 project_id = None
1062 if project_id_arg != None:
1063 role = lss.lib_right.get_role_on_project(lss.get_user_id(), project_id_arg)
1064 user_id = lss.get_user_id()
1065 is_admin = lss.lib_right.is_user_admin_config(user_id)
1066 if is_admin or role != None:
1067 project_id = project_id_arg
1070 if user != "anonymous@opio.fr":
1071 import json
1072 result = {}
1073 user_test_to_remove_301123 = lss.get_user()
1074 info_user_from_mail = lss.lib_user_data_internal.select_user(user_id)
1075 lss.user_id = user_id
1076# info_user_from_mail = lss.lib_user_data_internal.get_user_from_mail(user)
1078 res = lss.lib_user_data_internal.get_configuration_option()
1079 # A mettre dans lss ?
1080 res_json = {}
1081 for k in res:
1082 res_json[k["layer_type"]] = k["options"].split(",")
1084 map_conf_selected, _ = lss.load_configuration(with_class_name = False)
1086 amount = lss.lib_user_data_internal.get_current_amount(lss.get_user_id())
1088 param = lss.lib_user_data_internal.select_user(user_id)
1090 import pandas as pd
1091 df_invoice = pd.DataFrame()
1093 project_id_selected, nb_projects = lss.select_one_project_id()
1094 # project_id
1095 if project_id == None:
1096 project_id = project_id_selected
1097 info_project = lss.get_project_info(project_id, with_facture = True, is_validated = True)
1098 if "facture" in info_project:
1099 df_invoice = make_df_with_list_facture(info_project["facture"])
1101 # TODO VR 20-4-25 : [x] refacto fonction select project, check only_one, - [ ] behavior if project_id already selected - [ ] use it every where
1102 # TODO VR CDC Archi fonction helper like reformat list facture (a bit like connect lss ? )
1103 message_multiple_project_id = ""
1104 if nb_projects > 1:
1105 message_multiple_project_id = "Vous avez " + str(nb_projects) + " projets, vos factures sont peut-etre incomplètes ! "
1107 result["configuration_option"] = res_json
1108 result["configuration_selected"] = map_conf_selected
1109 result["user"] = user
1110 result["user_id"] = lss.get_user_id() # VR TODO refacto
1111 result["amount"] = amount
1112 lastname = info_user_from_mail["lastname"] if info_user_from_mail != None and "lastname" in info_user_from_mail else ""
1113 firstname = info_user_from_mail["firstname"] if info_user_from_mail != None and "firstname" in info_user_from_mail else ""
1114 mail = info_user_from_mail["mail"] if info_user_from_mail != None and "mail" in info_user_from_mail else ""
1115 phone = info_user_from_mail["phone"] if info_user_from_mail != None and "phone" in info_user_from_mail else ""
1116 adresse = info_user_from_mail["miscinfo"]["adresse"] if info_user_from_mail != None and "miscinfo" in info_user_from_mail and info_user_from_mail["miscinfo"] != None and "adresse" in info_user_from_mail["miscinfo"] else ""
1117 siret = info_user_from_mail["miscinfo"]["siret"] if info_user_from_mail != None and "miscinfo" in info_user_from_mail and info_user_from_mail["miscinfo"] != None and "siret" in info_user_from_mail["miscinfo"] else ""
1118 nb_day_ahead_default = info_user_from_mail["miscinfo"]["nb_day_ahead_default"] if info_user_from_mail != None and "miscinfo" in info_user_from_mail and info_user_from_mail["miscinfo"] != None and "nb_day_ahead_default" in info_user_from_mail["miscinfo"] else 5
1119 make_resume_auto_and_consolidate = info_user_from_mail["miscinfo"]["make_resume_auto_and_consolidate"] if info_user_from_mail != None and "miscinfo" in info_user_from_mail and info_user_from_mail["miscinfo"] != None and "make_resume_auto_and_consolidate" in info_user_from_mail["miscinfo"] else False
1120 result["user_config"] = {"user_last_name": lastname,
1121 "user_first_name": firstname,
1122 "email": mail,
1123 "user_phone": phone,
1124 "adresse": adresse,
1125 "siret": siret,
1126 "apia_status": "",
1127 "datou_apia": "",
1128 "to_add": "",
1129 "nb_day_ahead_default": nb_day_ahead_default,
1130 "make_resume_auto_and_consolidate" : make_resume_auto_and_consolidate
1131 }
1133 result["warning_admin"] = warning_admin
1134 result["df_invoice"] = df_invoice
1135 result["message_multiple_project_id"] = message_multiple_project_id
1137 res_conso = lss.get_conso(project_id=project_id)
1139 result["conso"] = res_conso
1140 # {"current_month" : 23, "last_month" : 234, "gliding_month" : 234, "gliding_year" : 1234}
1142 return render_template('account.html', results=result)
1143 else :
1144 response = make_page_anonymous()
1146 return response
1148# api/v1/user/update route to update user data
1149@app.route('/api/v1/user/update', methods=['POST'])
1150def api_v1_user_update():
1151 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["user_id"])
1153 user_id_connect = lss.get_user_id()
1154 check = lss.lib_right.is_user_admin_config(user_id_connect)
1155 user_id_used = request.json.get("user_id", None)
1156 warning_admin = False
1157 if check and user_id_used != None:
1158 user_id = user_id_used
1159 lss.user_id = user_id
1160 warning_admin = "As Admin, you are using user_id " + str(user_id) + " instead of " + str(user_id_connect)
1161 else:
1162 user_id = user_id_connect
1164 if user != "anonymous@opio.fr":
1166# info_user_from_mail = lss.lib_user_data_internal.get_user_from_mail(user)
1167 info_user_from_mail = lss.lib_user_data_internal.select_user(user_id)
1169 first_name = request.json.get("user_first_name", None)
1170 last_name = request.json.get("user_last_name", None)
1171 adresse = request.json.get("user_adresse", None)
1172 email = request.json.get("user_email", None)
1173 phone = request.json.get("user_phone", None)
1174 siret = request.json.get("user_siret", None)
1175 nb_day_ahead_default = request.json.get("nb_day_ahead_default", None)
1176 make_resume_auto_and_consolidate = request.json.get("make_resume_auto_and_consolidate", None)
1177 misc_info_already_recorded = info_user_from_mail["miscinfo"] if info_user_from_mail != None and "miscinfo" in info_user_from_mail else {}
1178 if misc_info_already_recorded == None:
1179 misc_info_already_recorded = {}
1180 if siret != None:
1181 misc_info_already_recorded["siret"] = siret
1182 if adresse != None:
1183 misc_info_already_recorded["adresse"] = adresse
1184 if nb_day_ahead_default != None:
1185 misc_info_already_recorded["nb_day_ahead_default"] = nb_day_ahead_default
1186 if "make_resume_auto_and_consolidate" in locals() and make_resume_auto_and_consolidate != None:
1187 misc_info_already_recorded["make_resume_auto_and_consolidate"] = make_resume_auto_and_consolidate
1189 lss.lib_user_data_internal.update_user(user_id, first_name, last_name, phone, email, misc_info_already_recorded)
1191 return {"message" : "OK"}
1193 return {"message" : "NOK"}
1195# API routes for OAuth client management
1196@app.route('/api/v1/oauth/my-clients', methods=['GET'])
1197def api_v1_oauth_my_clients():
1198 """Get OAuth2 clients for the authenticated user"""
1199 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["user_id"])
1201 if user == "anonymous@opio.fr":
1202 return jsonify({'error': 'Authentication required'}), 401
1204 user_id = lss.get_user_id()
1206 # Use the lib_pg_safia_sys method to retrieve OAuth clients
1207 clients_data = lpgss_singleton.get_oauth_clients_by_user(user_id)
1209 result = []
1210 for client in clients_data:
1211 client_metadata = client.get('client_metadata', {}) if client.get('client_metadata') else {}
1212 client_metadata = json.loads(client_metadata) if isinstance(client_metadata, str) else client_metadata
1213 redirect_uris = client_metadata.get('redirect_uris', '') if 'redirect_uris' in client_metadata else []
1214 grant_types = client_metadata.get('grant_types', '') if 'grant_types' in client_metadata else []
1215 scope = client_metadata.get('scope', '') if 'scope' in client_metadata else ''
1216 result.append({
1217 'id': client.get('id'),
1218 'client_id': client.get('client_id'),
1219 'client_db_id': client.get('client_db_id'),
1220 'client_secret': client.get('client_secret'),
1221 'client_name': client.get('client_name', 'N/A'),
1222 'client_uri': client.get('client_uri', ''),
1223 'redirect_uris': redirect_uris,
1224 'grant_types': grant_types,
1225 'scope': scope,
1226 'client_id_issued_at': str(client.get('client_id_issued_at')) if client.get('client_id_issued_at') else None,
1227 'client_secret_expires_at': str(client.get('client_secret_expires_at')) if client.get('client_secret_expires_at') else None,
1228 })
1230 return jsonify({'clients': result}), 200
1232@app.route('/api/v1/oauth/create-client', methods=['POST'])
1233def api_v1_oauth_create_client():
1234 """Create a new OAuth2 client for the authenticated user"""
1235 from werkzeug.security import gen_salt
1236 import json
1238 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["user_id"])
1240 if user == "anonymous@opio.fr":
1241 return jsonify({'error': 'Authentication required'}), 401
1243 user_id = lss.get_user_id()
1244 data = request.get_json()
1246 # Validate required fields
1247 client_name = data.get('client_metadata', {}).get('client_name') if 'client_metadata' in data else data.get('client_name', data.get('client_id'))
1248 redirect_uris = data.get('redirect_uris', [])
1250 if not client_name:
1251 return jsonify({'error': 'client_name is required'}), 400
1253 if not redirect_uris or not isinstance(redirect_uris, list) or len(redirect_uris) == 0:
1254 return jsonify({'error': 'At least one redirect_uri is required'}), 400
1256 # Generate client_id and client_secret
1257 client_id = gen_salt(24)
1258 client_secret = gen_salt(48)
1260 # Prepare client metadata
1261 client_uri = data.get('client_uri', '')
1262 grant_types = data.get('grant_types', ['authorization_code', 'refresh_token'])
1263 scope = data.get('scope', 'profile email')
1265 # Convert lists to space-separated strings for storage
1266 redirect_uris_str = ' '.join(redirect_uris)
1267 grant_types_str = ' '.join(grant_types)
1269 # Client metadata as JSON
1270 client_metadata = {
1271 'client_name': client_name,
1272 'client_uri': client_uri,
1273 'grant_types': grant_types,
1274 'redirect_uris': redirect_uris,
1275 'response_types': ['code'],
1276 'scope': scope,
1277 'token_endpoint_auth_method': 'client_secret_basic'
1278 }
1279# 'token_endpoint_auth_method': 'client_secret_basic'
1280# client_secret_post
1282 try:
1283 # Use the lib_pg_safia_sys method to create OAuth client
1284 client_db_id = lpgss_singleton.create_oauth_client(
1285 client_id, client_secret, user_id, client_name,
1286 redirect_uris_str, grant_types_str, ['code'],
1287 scope, client_uri, client_metadata
1288 )
1290 return jsonify({
1291 'status': 'created',
1292 'message': 'OAuth2 client created successfully',
1293 'client_id': client_id,
1294 "client_db_id": client_db_id,
1295 'client_secret': client_secret,
1296 'client_name': client_name,
1297 'redirect_uris': redirect_uris
1298 }), 201
1299 except Exception as e:
1301# traceback.print_exc()
1302 print("Error creating OAuth client: " + str(e))
1303 return jsonify({'error': str(e)}), 500
1305@app.route('/doc', methods=['GET'])
1306def doc():
1307 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
1308 curr_project_id = data_needed["project_id"]
1310 if user != 'anonymous@opio.fr':
1312 data_user = lss.get_data_user(user, lss.get_user_id())
1314 for d in data_user:
1315 if d["id"] == curr_project_id:
1316 d["SELECTED"] = 1
1318 import json
1319 result = {"projects" : data_user}
1320 return render_template('doc.html', results=result)
1321 else :
1322 response = make_page_anonymous()
1325 return response
1326# make_page_anonymous()
1329@app.route('/doc/free.safia', methods=['GET'])
1330def doc_free_safia():
1331 return render_template('doc/free.safia.html')
1333@app.route('/datou', methods=['GET'])
1334def datou():
1335 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
1336 histo = bool(request.args.get("histo", False))
1337 # TODO est-ce que ce serait mieux de le mettre dans la config ?
1338 histo_folder = request.args.get("histo_folder", "/home/safia/workarea/git/Safia/prompt/python/server/static/onedrive/workarea_anon/list_datou_backup/")
1339 if histo_folder == "mba_vr":
1340 histo_folder = "/Users/moilerat/Documents/Fotonower/Safia/prompt/devops/histo_config/list_datou_backup"
1342 if user != "anonymous@opio.fr":
1344 id = request.args.get('id', None)
1345 datou_step_template = lss.get_datou_step_template()
1346 list_datou_with_step = lss.get_datou(id, dont_instantiate_config_prepare = True)
1348 if histo:
1349 from lib.manaudit.lib_datou_audit import get_list_backup
1350 try :
1351 list_backup, map_proj_id_date_backup = get_list_backup(histo_folder, mtd_id=id)
1352 if id != None:
1353 list_backup = list_backup[int(id)]
1354 else :
1355 list_backup_flat_one_per_datou = {}
1356 for id in list_backup:
1357 if type(list_backup[id]) == dict and len(list_backup[id]) == 1:
1358 one_and_only_key = list(list_backup[id].keys())[0]
1359 list_backup_flat_one_per_datou[str(id) + " " + str(one_and_only_key)] = list_backup[id][one_and_only_key]
1360 list_backup = list_backup_flat_one_per_datou
1361 except Exception as e:
1362 list_backup = {"error" : str(e)}
1363 print("Error In load list backup datou " + str(e))
1364 else:
1365 list_backup = {}
1367 import json
1368 for dst in datou_step_template:
1369 datou_step_template_str = json.dumps(dst, default=str)
1370 dst["data_str"] = datou_step_template_str
1371 for dst in list_backup:
1372 datou_step_template_str = json.dumps(list_backup[dst], default=str)
1373 list_backup[dst]["data_str"] = datou_step_template_str
1375 result = {"datou_step_template" : datou_step_template,
1376 "datou" : list_datou_with_step,
1377 "histo" : 'true' if histo else 'false',
1378 "list_backup" : list_backup}
1379 if "json" in request.args:
1380 return Response(
1381 response=json.dumps(result, indent=4, sort_keys=True, default=str),
1382 status=200,
1383 mimetype='application/json'
1384 )
1385 else :
1386 return render_template('datou.html', results=result)
1387 else :
1388 response = make_page_anonymous()
1390 return response
1391# make_page_anonymous()
1393@app.route('/admin/all_users', methods=['GET'])
1394def admin_all_users():
1395 print("Inside saxia admin/all_users endpoint")
1396 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
1397 project_id = data_needed["project_id"]
1399 user_id = lss.get_user_id()
1400 check = lss.lib_right.is_user_admin_config(user_id)
1401 if check:
1402 all_users = lss.lib_user_data_internal.get_all_users()
1403 for user in all_users:
1404 list_proj_id = lss.lib_right.get_projects_with_access(user["id"])
1405 if len(list_proj_id) == 1:
1406 user["project_ids"] = "<a href='/saxia/suivi?type_suivi=lab&project_id=" + str(list_proj_id[0]["id"]) + "' >Link To All Dossier of " + str(list_proj_id[0]["id"]) + "</a>"
1407 else :
1408 user["project_ids"] = list_proj_id
1409 import pandas as pd
1410 df = pd.json_normalize(all_users) #pd.read_json(all_users, orient="records")
1411# return df.to_html(classes="table pdt-table table-striped sortable", escape=False)
1413 result = {"df" : df}
1414 resp = make_response(render_template('saxia/admin_all_users.html',
1415 results=result,
1416 title="Saxia All Users"))
1417 resp.set_cookie("project_id", str(project_id), max_age=60 * 60 * 24 * 15)
1418 return resp
1420@app.route('/admin/all_factures', methods=['GET'])
1421def admin_all_factures():
1422 print("Inside saxia admin/all_factures endpoint")
1423 user, data_needed, lss = register_user_get_data(request, list_data_needed=["project_id"])
1424 project_id = data_needed["project_id"]
1425 validated = request.args.get("validated", None)
1427 user_id = lss.get_user_id()
1428 check = lss.lib_right.is_user_admin_config(user_id)
1429 if check:
1431 all_factures_json = lss.lib_user_data_internal.get_facture_from_project(is_validated=validated)
1432 df_factures = make_df_with_list_facture(all_factures_json, type_column = "admin")
1434 resp = make_response(df_factures.to_html(classes="table pdt-table table-striped sortable", escape=False, table_id="se-table"))
1435 resp.set_cookie("project_id", str(project_id), max_age=60 * 60 * 24 * 15)
1436 return resp
1439 # , escape=False
1441# return all_users
1442# return render_template('admin_all_users.html', results=all_users)
1443 else:
1444 return "You are the king !"
1448# Edition manuelle des données de traitement
1449@app.route('/api/v1/audit/load', methods=['GET'])
1450def api_v1_audit_load():
1451 print("Old endpoint saxia_admin /saxia/admin DEPRECATE 11/11, double check removing and recycled !")
1452 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
1453 project_id = data_needed["project_id"]
1455 if user != "anonymous@opio.fr":
1456 hash_id_treatment = request.args.get('hash_id_treatment', None)
1457 output_type = request.args.get('output_type', "dict")
1458 col_csv = request.args.get('col_csv', "sum(nb_page) as nb_page, 0 as id")
1459 limit = request.args.get('limit', None)
1460 offset = int(request.args.get('offset', 0))
1461 condition_csv = request.args.get('condition', "one_month,previous_month,last_year")
1463 res = lss.get_conso(hash_id_treatment = hash_id_treatment, output_type = output_type,
1464 col_csv = col_csv, limit = limit, offset = offset,
1465 condition_csv = condition_csv,
1466 project_id = project_id)
1468 return res
1470# for pagination
1471@app.route('/api/v1/saxia/running', methods=['GET'])
1472def api_v1_saxia_running():
1473 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
1474 project_id = data_needed["project_id"]
1475 limit = request.args.get('limit', 100)
1476 offset = request.args.get('offset', 0)
1477 nb_day = request.args.get("nb", 0)
1478 type_suivi = request.args.get('type_suivi', "admin")
1480 project_id_arg = request.args.get('project_id', None)
1481 verbose = False
1482 if project_id_arg != None:
1483 role = lss.lib_right.get_role_on_project(lss.get_user_id(), project_id_arg)
1484 user_id = lss.get_user_id()
1485 is_admin = lss.lib_right.is_user_admin_config(user_id)
1486 if is_admin or role != None:
1487 project_id = project_id_arg
1489 if project_id == -1:
1490 project_id, nb_projects = lss.select_one_project_id()
1492 ret = lss.lib_user_data_internal.running_job(project_id = project_id, verbose = verbose, nb_day = nb_day)
1493 # mtr_datou_id_upload = 51,
1494 # mtr_datou_id_anon = 25,
1495 # mtr_datou_id_extract = 40):
1496 # mtr_datou_audit = 55
1498 return ret
1501@app.route('/api/v1/saxia/step/<saxia_step_name>', methods=['GET', 'POST'])
1502def api_v1_saxia_step(saxia_step_name):
1503 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
1504 project_id = data_needed["project_id"]
1505 if request.method == 'POST':
1506 data_json = request.json
1507 else :
1508 data_json = request.args.to_dict()
1510 hash_id_treatment = data_json.get("hash_id_treatment", None)
1511 verbose = False
1513 if hash_id_treatment != None:
1514 print(" Plop plop")
1516 raw_configuration = lss.load_conf_project(project_id)
1517 from lib.util.lib_formal_conf import formal_conf_aux, formal_conf_switch_context, formal_conf_prepare
1518 # configuration = formal_conf_prepare(raw_configuration, lss)
1519 configuration = formal_conf_aux(raw_configuration, lss=lss,
1520 list_key_word=["safia.audit.col_REFACTO_NOW_DONE_LATER",
1521 "saxia.config.project", "format.string"]) # , "saxia.config.user", safia.datou.param ?
1522 # configuration_that_should_be_the_same = lcs.load_and_instantiate_config(raw_configuration)
1523 configuration_should_be_the_same = formal_conf_prepare(raw_configuration, lss)
1525 saxia_config = configuration["saxia"] if "saxia" in configuration else {}
1526 saxia_steps_config = saxia_config["steps"] if "steps" in saxia_config else {}
1527 saxia_current_step_config = saxia_steps_config[saxia_step_name] if saxia_step_name in saxia_steps_config else {}
1529 saxia_current_step_config_instanced = formal_conf_aux(saxia_current_step_config, lss=lss,
1530 list_key_word=["safia.audit.col"],
1531 hit=hash_id_treatment)
1533 mtr_datou_id = saxia_current_step_config["datou-id"] if "datou-id" in saxia_current_step_config else None
1535 saxia_current_step_config_instanced_format_string = formal_conf_aux(saxia_current_step_config_instanced, lss=lss,
1536 list_key_word=["format.string"])
1538 # assoc input_datou_from_input_saxia_step
1539 input_datou = {}
1540 for one_input_saxia in saxia_current_step_config_instanced_format_string["assoc_saxia_input_datou_input"]:
1541 in_name_datou = saxia_current_step_config_instanced_format_string["assoc_saxia_input_datou_input"][one_input_saxia]
1542 if one_input_saxia in saxia_current_step_config_instanced_format_string["input_saxia_step"] :
1543 input_datou[in_name_datou] = saxia_current_step_config_instanced_format_string["input_saxia_step"][one_input_saxia]
1545 complete_param_json = get_datou_exec_context_as_complete_param_json(lss = lss, project_id = project_id)
1547 datou = lss.get_datou(mtr_datou_id)
1548 datou = datou[0]
1549 list_actions_datou_query_texte = list(map(lambda x : x["name"], datou["steps"]))
1550 list_param_json = list(map(lambda x : x["param_json"], datou["steps"]))
1552 from lib.datou.datou_exec import datou_exec
1553 #input_datou["project_id"] = curr_project_id
1554 output, audit_json = datou_exec(list_actions_datou_query_texte,
1555 input_datou, complete_param_json,
1556 verbose=verbose, list_param_json_steps=list_param_json,
1557 with_audit=False)
1558 else:
1559 output = {"error":"No hash_id_treatment provided !"}
1561 return output
1563@app.route('/api/v1/saxia/suivi/upsert', methods=['GET', 'POST'])
1564def api_v1_saxia_suivi_upsert():
1565 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
1566 project_id = data_needed["project_id"]
1567 if request.method == 'POST':
1568 data_json = request.json
1569 else :
1570 data_json = request.args.to_dict()
1572 hash_id_treatment = ""
1573 verbose = False
1575 if user != "":
1576 print(" Plop plop")
1578 for d in data_json:
1579 if "hash_id_treatment" not in d or "value" not in d or "key" not in d or "col" not in d:
1580 print(" Missing data, nothing saved !")
1581 continue
1582 hash_id_treatment = d["hash_id_treatment"]
1583 type_save = d["type_save"] if "type_save" in d else "merge"
1584 value = d["value"]
1585 key = d["key"]
1586 col = d["col"]
1587 type_data = d["type_data"] if "type_data" in d else "json"
1588 value_to_save = {}
1589 if key == "None":
1590 key = None
1591 if type_save == "merge":
1592 data = lss.lib_user_data_internal.load_data_audit(hash_id_treatment=hash_id_treatment, col_csv=col, limit=1)
1593 value_to_save = data[col]
1595 if key == None:
1596 value_to_save = value
1597 elif type(value_to_save) == dict:
1598 value_to_save[key] = value
1599 else :
1600 print(" Mege impossible if asked !")
1601 value_to_save = {key : value}
1603 lss.lib_user_data_internal.upsert_audit_info(hash_id_treatment,
1604 list_json_update=[
1605 {"variable": col, "type": type_data}
1606 ],
1607 input_values={col:value_to_save},
1608 verbose=verbose)
1610 return "Should be done !"
1612# for pagination
1613@app.route('/api/v1/saxia/suivi', methods=['GET'])
1614def api_v1_saxia_suivi():
1615 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
1616 project_id = data_needed["project_id"]
1617 limit = request.args.get('limit', 100)
1618 offset = request.args.get('offset', 0)
1619 type_suivi = request.args.get('type_suivi', "admin")
1621 if user != "":
1622 print(" Plop plop")
1623 return "TODO plop plop"
1625# TODO 3-6-24 EP SAXIA/SAFIA
1626# Edition manuelle des données de traitement
1627@app.route('/saxia/suivi', methods=['GET'])
1628def saxia_suivi():
1629 print("Inside saxia admin saxia/suivi admin endpoint")
1630 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
1631 project_id = data_needed["project_id"]
1632 limit = request.args.get('limit', 100)
1633 offset = request.args.get('offset', 0)
1634 type_suivi = request.args.get('type_suivi', "customer")
1635 page = request.args.get('page', 1)
1636 offset = (int(page) - 1) * int(limit)
1637 project_id_arg = request.args.get('project_id', None)
1638 if project_id_arg != None:
1639 role = lss.lib_right.get_role_on_project(lss.get_user_id(), project_id_arg)
1640 user_id = lss.get_user_id()
1641 is_admin = lss.lib_right.is_user_admin_config(user_id)
1642 if is_admin or role != None:
1643 project_id = project_id_arg
1645 if user != "anonymous@opio.fr":
1646 config_app = lcs.get_config_app()
1648 hash_id_treatment = request.args.get('hash_id_treatment', None)
1650 # audit_info
1651 col_csv = "id,mtr_datou_id,created_at,launched_at,modified_at,deleted_at,valid_until,user_id,project_id,safia_doc_id,hash_id_treatment,cdn"
1652 col_csv = "id,id_file,mtr_datou_id,created_at,launched_at,modified_at,deleted_at,valid_until,user_id,project_id,safia_doc_id,hash_id_treatment,cdn,input_data,nb_page,nb_modif,input_file,audit_resume,info_anon,info_customer,info_lab,info_cdn,info_back,info_date,info_suivi,info_action,info_usage_integration,info_ml,info_init,info_extract,info_consolidate,info_result,info_exec,info_page,info_stat"
1653# col_csv = "id,id_file,mtr_datou_id,created_at,info_date"
1655 col_csv = request.args.get('col_csv', col_csv)
1657 raw_configuration = lss.load_conf_project(project_id)
1659# configuration = lcs.load_and_instantiate_config(raw_configuration)
1660 from lib.util.lib_formal_conf import formal_conf_prepare
1661 configuration = formal_conf_prepare(raw_configuration, lss)
1663 saxia_conf = configuration["saxia"] if "saxia" in configuration else {}
1664 saxia_steps = saxia_conf["steps"] if "steps" in saxia_conf else {}
1665 print(request.endpoint)
1666 endpoint_df_conf = configuration["custom_display"][request.endpoint] if "custom_display" in configuration and request.endpoint in configuration["custom_display"] else {}
1667 endpoint_df_conf_type_suivi = endpoint_df_conf[type_suivi] if type_suivi in endpoint_df_conf else endpoint_df_conf["default"] if "default" in endpoint_df_conf else {}
1668 condition = endpoint_df_conf_type_suivi["condition"] if "condition" in endpoint_df_conf_type_suivi else []
1670 if type_suivi == "customer" or project_id_arg != None:
1671 condition.append({'data_type': 'int', 'value': project_id, 'variable': 'project_id'})
1673 print(" Useless with hash_id_treatment not none : " + str(hash_id_treatment))
1674 try_to_count = lss.lib_user_data_internal.load_data_audit(hash_id_treatment = hash_id_treatment,
1675 col_csv = "count(*)",
1676 condition = condition)
1677 print(" try_to_count : " + str(try_to_count))
1678 nb_page = try_to_count[0]["count"] // int(limit) + 1
1679 print(" nb_page : " + str(nb_page))
1681 all_result = lss.lib_user_data_internal.load_data_audit(hash_id_treatment = hash_id_treatment,
1682 output_type = "pd",
1683 col_csv = col_csv,
1684 limit = limit,
1685 offset = offset,
1686 condition = condition)
1689 list_col_display = []
1690 try:
1691 from lib.batch.lib_batch import create_pandas_table_from_text, append_action_steps_button, set_custom_display
1692 safia_suivi_prod_doc_id = "suivi_virtual"
1693 safia_suivi_prod_proj_id = project_id
1694 append_action_steps_button(all_result, safia_suivi_prod_doc_id, safia_suivi_prod_proj_id, saxia_steps)
1695 list_col_display = set_custom_display(all_result, endpoint_df_conf_type_suivi, col_csv=col_csv)
1696 except Exception as e:
1697 print(str(e))
1699 from lib.lib_util import prepare_pagination
1700 list_to_display = prepare_pagination(int(nb_page), int(page))
1702 result = {"df": all_result, "document_id": safia_suivi_prod_doc_id, "document_content": "suivi_virtual_content", "chunk_id": None,
1703 "project_id": project_id,
1704 "list_col_display" : {"list" : list_col_display},
1705 "custom_display" : endpoint_df_conf_type_suivi,
1706 "info_pagination" : {"nb_page" : nb_page, "page" : page,
1707 "display" : list_to_display,
1708 "type_suivi" : type_suivi,
1709 "endpoint" : request.path}
1710 }
1712 resp = make_response(render_template('smarteditor.html',
1713 results=result,
1714 title="SAFIA SE SUIVI project_id " + str(project_id) + " type_suivi " + str(type_suivi),
1715 config_app=config_app))
1716 resp.set_cookie("project_id", str(project_id), max_age=60 * 60 * 24 * 15)
1717 return resp
1718 else :
1719 return("Please connect")
1721# return all_result.to_html(classes="table pdt-table table-striped sortable searchable")
1725list_acronym_from_flask_file = "list_acronym_from_flask_file.json"
1726@app.route('/api/v1/saxia/lexicon/acronym/add/<acre>', methods=['GET'])
1727def api_v1_saxia_lexicon_acronym_add(acre):
1728 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
1730 try :
1731 with open(list_acronym_from_flask_file, "r") as f:
1732 list_acronym_from_flask = json.load(f)
1733 except Exception as e:
1734 list_acronym_from_flask = {"list" : ["irm"]}
1736 if user != "info@opio.fr":
1737 if acre not in list_acronym_from_flask["list"]:
1738 list_acronym_from_flask["list"].append(acre)
1739 with open(list_acronym_from_flask_file, "w") as f:
1740 json.dump(list_acronym_from_flask, f)
1741 return json.dumps(list_acronym_from_flask)
1745# Edition manuelle des données de traitement
1746@app.route('/split', methods=['GET'])
1747def split():
1748 print("Inside split endpoint")
1749 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
1751 list_images_input = request.args.get('list_images', None)
1752# list_images_input = "id_0_nb_18_d_20250616:0,id_0_nb_18_d_20250616:10,id_0_nb_18_d_20250616:11,id_0_nb_18_d_20250616:13"
1753 list_id_file_page = list_images_input.split(",") if list_images_input != None else []
1754 map_id_file_page_nb = {}
1755 for id_file_page in list_id_file_page:
1756 if ":" in id_file_page:
1757 id_file, page_nb = id_file_page.split(":")
1758 if id_file not in map_id_file_page_nb:
1759 map_id_file_page_nb[id_file] = []
1760 map_id_file_page_nb[id_file].append(int(page_nb) + 1)
1761 else:
1762 print("Error in split input list_images : " + str(id_file_page))
1764 list_images = []
1765 for id_file in map_id_file_page_nb:
1766 list_hits = lss.lib_user_data_internal.select_datou_exec_from_id_file_and_mtr_datou_id(id_file, mtr_datou_id = 40)
1767 if len(list_hits) > 0:
1768 hit = list_hits[0]["hit"]
1769 data = lss.lib_user_data_internal.load_data_audit(hash_id_treatment=hit, col_csv="nb_page,info_back",
1770 limit=1)
1772 out_folder = data['info_back']['out_folder'] if 'info_back' in data and 'out_folder' in data[
1773 'info_back'] else ""
1774 nb_page = data.get('nb_page', None)
1775 for id_page in map_id_file_page_nb[id_file]:
1776 if id_page <= nb_page:
1777 list_images.append(out_folder + "/" + hit + "/page_" + str(id_page) + ".png")
1778 else:
1779 print("Warning : id_page " + str(id_page) + " is greater than nb_page " + str(nb_page))
1781 if len(list_images) > 0 and not os.path.exists(list_images[0]):
1782 from lib.lib_ml.lib_util_prepare_dataset import create_folder_and_copy, prepare_command_copy_files
1784 list_create, list_scp = prepare_command_copy_files(list_images)
1785 print("list_create : " + str(list_create))
1786 print("list_scp : " + str(list_scp))
1787 create_folder_and_copy(list_create, list_scp)
1789 hit = request.args.get('hash_id_treatment', None)
1791 if list_images == []:
1792 try :
1793 print("On y va")
1794 data = lss.lib_user_data_internal.load_data_audit(hash_id_treatment=hit, col_csv="nb_page,info_back", limit=1)
1796 out_folder = data['info_back']['out_folder'] if 'info_back' in data and 'out_folder' in data['info_back'] else ""
1797 nb_page = data.get('nb_page', None)
1798 list_images = [ out_folder + "/" + hit + "/page_" + str(id_page) + ".png" for id_page in range(1, nb_page + 1)]
1800 except Exception as e:
1801 print(str(e))
1802 list_images = []
1804 results = {"list_images" : list_images}
1805# var list_images = {{ result.list_images | tojson | safe}}
1807 return make_response(render_template('saxia/split.html',
1808 results=results))
1812# Edition manuelle des données de traitement
1813@app.route('/api/v1/create/pdf', methods=['GET'])
1814def api_v1_create_pdf():
1815 print("Inside create endpoint")
1816 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
1818 list_ccsv = request.args.get('list_ccsv', "")
1819 # list_pages = request.args.get('list_pages', [])
1820 hit = request.args.get('hash_id_treatment', None)
1822 new_pdf = ""
1823 try :
1824 data = lss.lib_user_data_internal.load_data_audit(hash_id_treatment=hit, col_csv="nb_page,info_back", limit=1)
1826 out_folder = data['info_back']['out_folder'] if 'info_back' in data and 'out_folder' in data['info_back'] else ""
1827 nb_page = data.get('nb_page', None)
1829 list_pages = [ out_folder + "/" + hit + "/page_" + str(id_page) + ".png" for id_page in range(1, nb_page + 1)]
1831 from lib.util.lib_simple_image import create_pdf
1832 new_pdf = create_pdf(list_pages=list_pages, list_ccsv=list_ccsv)
1834 except Exception as e:
1835 print(str(e))
1837 if new_pdf == "":
1838 return "Error creating PDF", 500
1839 return send_from_directory(
1840 "", new_pdf, as_attachment=True
1841 )
1844# Edition manuelle des données de traitement
1845@app.route('/manax', methods=['GET'])
1846def manax():
1847 print("Inside manax endpoint")
1848 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
1850 try :
1851 with open(list_acronym_from_flask_file, "r") as f:
1852 list_acronym_from_flask = json.load(f)
1853 except Exception as e:
1854 list_acronym_from_flask = {"list" : ["irm"]}
1856 if user != "anonymous@opio.fr":
1857 hash_id_treatment = request.args.get('hash_id_treatment', None)
1858 debug = bool(request.args.get('debug', False))
1859 ids_manual = request.args.get('ids_manual', None)
1860 suffix = request.args.get("suffix", "/") # static/saxia/output
1861 audit_file = request.args.get("audit_file", None)
1862 safia_suivi_prod_doc_id = request.args.get("safia_suivi_prod_doc_id", None)
1863 safia_suivi_prod_proj_id = request.args.get("safia_suivi_prod_proj_id", None)
1864 file = request.args.get("file", None)
1865 filename = os.path.basename(file) if file != None else None
1866 out_file = request.args.get("out_file", None)
1867 mtd_id = request.args.get("id", None)
1868 limit = request.args.get("limit", None)
1869 director = request.args.get("director", None)
1870 stat = request.args.get("stat", None)
1872 # type(data_needed["project_id"]) == type int
1873 project_id = data_needed["project_id"] if "project_id" in data_needed else 70
1874 # oups, is this really what we want, yes I think so, but needs to be documented somewhere else VR 7-4-24
1877 raw_configuration = lss.load_conf_project(project_id)
1879# configuration = lcs.load_and_instantiate_config(raw_configuration)
1880 from lib.util.lib_formal_conf import formal_conf_prepare
1881 configuration = formal_conf_prepare(raw_configuration, lss)
1883 saxia_conf = configuration["saxia"] if "saxia" in configuration else {}
1885 list_doc_type = []
1886 if ('format' in saxia_conf and
1887 'info_format_intro' in saxia_conf['format'] and
1888 'data' in saxia_conf['format']['info_format_intro'] and
1889 'validation' in saxia_conf['format']['info_format_intro']['data'] and
1890 'document_type' in saxia_conf['format']['info_format_intro']['data']['validation']):
1891 list_doc_type = saxia_conf['format']['info_format_intro']['data']['validation']['document_type']
1893 assoc_conf = saxia_conf["assoc"] if "assoc" in saxia_conf else {}
1894 if mtd_id != None and mtd_id in assoc_conf:
1895 assoc_conf = assoc_conf[mtd_id]
1896 # default hard coded value for behavior on datou 40 manax saxia correction
1897 if "paragraphs" not in assoc_conf:
1898 assoc_conf["paragraphs"] = "io_exec/3/input/paragraphs"
1899 manual_action_to_audit_data = assoc_conf["manual_action_to_audit_data"] if "manual_action_to_audit_data" in assoc_conf else {}
1900 type_entities = []
1901 # VR TODO enhance by checking if the action_interface contains manax_entity
1902 if mtd_id == "45":
1903 action_interface = saxia_conf["action_interface"][mtd_id] if "action_interface" in saxia_conf and mtd_id in saxia_conf["action_interface"] else []
1904 if "manax_entity" in action_interface:
1905 type_entities = saxia_conf["type_entities"] if "type_entities" in saxia_conf else []
1906 else:
1907 print("Unexpected behavior, configuration unexpected or to be correctedy")
1909 from lib.manaudit.lib_datou_audit import load_audit_info_and_apply_manual_correction
1911 df_auto, df_cons, hash_id_treatment_rerun, results, audit_info_count, id_file, audit_json_file_content_as_json, all_results, info_date = load_audit_info_and_apply_manual_correction(hash_id_treatment_auto = hash_id_treatment,
1912 hash_id_treatment_manual = hash_id_treatment,
1913 lpgss = lss.lib_user_data_internal,
1914 limit = limit,
1915 project_id = project_id,
1916 ids_manual = ids_manual,
1917 manual_action_to_audit_data = manual_action_to_audit_data)
1919 from lib.util.lib_audit_prediag import parse_error_case, compute_pre_diagnostic
1920 map_flag_error = parse_error_case(audit_json_file_content_as_json)
1921 trigger_high_cost = compute_pre_diagnostic(info_date)
1923 type_index_images = "page_number"
1924 type_index_images = "deprecated_page_number"
1926 images_path_key = "io_exec/0/input/images"
1927 paragraphs_key = "io_exec/6/input/paragraphs"
1928 # load_sub_json
1929 from lib.manaudit.lib_datou_audit import load_sub_json
1930 # list_left_right_decrire
1931 list_images_path = load_sub_json(audit_json_file_content_as_json, images_path_key)
1932 list_paragraphs = load_sub_json(audit_json_file_content_as_json, paragraphs_key)
1934 if stat != None:
1935 with_out_folder = bool(request.args.get("with_out_folder", False))
1936 from lib.manaudit.lib_datou_audit import study_qualite
1937 df_to_study = study_qualite(lpgss = lss.lib_user_data_internal,
1938 type_doc = "document_type_UNUSED",
1939 verbose = False,
1940 condition_query = "TODO",
1941 limit = limit,
1942 project_id = project_id,
1943 condition_intro_doc = True,
1944 only_correct_prediag = True,
1945 condition_only_one_page = True,
1946 with_out_folder = with_out_folder)
1947 list_images_path = []
1948 list_paragraphs = []
1949 results["df"] = df_to_study
1950 results["dataframe_html"] = df_to_study.to_html(classes="table pdt-table table-striped", escape=False)
1951 type_index_images = "line_idx"
1952 type_index_images = "from_df"
1956 info_format_intro_bo = saxia_conf["format"]["info_format_intro"] if "format" in saxia_conf and "info_format_intro" in saxia_conf["format"] else {}
1957 map_modif_df_manual = audit_info_count["map_modif_df_manual"] if "map_modif_df_manual" in audit_info_count else {}
1959 out_folder = "static/saxia/output"
1960 out_folder = "static/onedrive/output"
1962 # VR todo adapt per user, grrr 15-6-24
1963 out_folder = saxia_conf["out_folder"] if "out_folder" in saxia_conf else out_folder
1965 input_context_manax = {"out_folder" : out_folder, "out_file" : out_file}
1967 if director != None:
1968 from lib.lib_util import director_cut
1969 director = director_cut(id_file, results["df"], results["paragraphs"])
1971 results["trigger_high_cost"] = trigger_high_cost
1972 results["map_flag_error"] = map_flag_error
1973 results["suffix"] = suffix
1974# results["audit_json_file_content"] = audit_json_file_content
1975 results["audit_json_file_content_as_json"] = audit_json_file_content_as_json
1976 results["list_acronym_from_flask"] = list_acronym_from_flask
1977 results["hash_id_treatment"] = hash_id_treatment
1978 results["safia_suivi_prod_doc_id"] = safia_suivi_prod_doc_id
1979 results["safia_suivi_prod_proj_id"] = safia_suivi_prod_proj_id
1980 results["file"] = file
1981 results["filename"] = filename
1982 results["out_file"] = out_file
1983 # En fait on pourrait mettre toutes les infos de configuration à ce niveau là
1984 results["output_datou_to_col_consolidate"] = request.args.get("output_datou_to_col", "")
1985 results["info_format_intro_bo"] = info_format_intro_bo
1986 results["saxia_conf"] = saxia_conf
1987 results["map_modif_df_manual"] = map_modif_df_manual # maybe refacto set directly in resutls
1988 results["assoc_conf"] = assoc_conf
1989 results["type_entities"] = type_entities
1990 results["list_doc_type"] = list_doc_type
1992 results["project_id"] = project_id
1993 results["user_id"] = user
1994 results["id_file"] = id_file
1995 results["input_context_manax"] = input_context_manax
1996 results["debug"] = debug
1997 results["hash_id_treatment_rerun"] = hash_id_treatment_rerun
1998 results["list_images_path"] = list_images_path
1999 results["list_paragraphs"] = list_paragraphs
2000 results["type_index_images"] = type_index_images
2002 if df_cons != None:
2003 print("df_cons : " + str(df_cons))
2005# result = {"audit_json_file_content" : audit_json_file_content}
2006 resp = make_response(render_template('manax.html',
2007 results=results,
2008 title=" Safia Manax project_id " + str(project_id) + " id_file " + str(id_file)))
2009 resp.headers["Cross-Origin-Embedder-Policy"] = "require-corp"
2010 resp.headers["Cross-Origin-Opener-Policy"] = "same-origin"
2011 return resp
2012 else :
2013 response = make_page_anonymous()
2015 return response
2019# Edition manuelle des données de traitement
2020@app.route('/config_user', methods=['GET'])
2021def config_user():
2022 print("Inside config_user endpoint")
2023 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
2026# https://www.tiny.cloud/my-account/integrate/#html
2028 if user != "anonymous@opio.fr":
2030 config_app = lcs.get_config_app()
2032 project_id = data_needed["project_id"] if "project_id" in data_needed else 70
2033# oups, is this really what we want, yes I think so, but needs to be documented somewhere else VR 7-4-24
2034 configuration = lss.load_conf_project(project_id)
2036# result = {"audit_json_file_content" : audit_json_file_content}
2037 return render_template('saxia/config_user.html',
2038 results={"configuration" : configuration},
2039 config_app=config_app)
2040 else :
2041 response = make_page_anonymous()
2043 return response
2045# Suppression aux données manuelles
2046@app.route('/api/v1/manax/correction/delete', methods=['GET', 'POST'])
2047def api_v1_mana_correction_delete():
2048 print("Inside api_v1_mana_correction_delete endpoint")
2049 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
2051 if user != "anonymous@opio.fr":
2052 id = request.args.get('id', None)
2053 hash_id_treatment = request.args.get('hash_id_treatment', None)
2055 res = lss.lib_user_data_internal.delete_data_manual(hash_id_treatment = hash_id_treatment,
2056 id = id)
2057# return res
2058 return "Maybe done"
2060# Modification des données manuelles
2061@app.route('/api/v1/manax/correction/updated', methods=['GET', 'POST'])
2062def api_v1_mana_correction_update_duplicate():
2063 print("Inside api_v1_mana_correction_update_duplicate endpoint")
2064 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
2066 if user != "anonymous@opio.fr":
2067 id = request.args.get('id', None)
2068 new_hit = request.args.get('new_hit', None)
2069 hash_id_treatment = request.args.get('hash_id_treatment', None)
2071 res = lss.lib_user_data_internal.update_data_manual(hash_id_treatment = hash_id_treatment,
2072 id = id, new_hit = new_hit)
2073# return res
2074 return "Maybe done"
2076# Chargement des commentaires suite au traitement
2077@app.route('/api/v1/manax/comms/load', methods=['GET'])
2078def api_v1_mana_comms_load():
2079 print("Inside api_v1_mana_comms_load endpoint")
2080 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
2082 # VR TODO load from app_config hard-coded
2083 mtr_datou_id_extract = 40
2085 if user != "anonymous@opio.fr":
2086 hash_id_treatment = request.args.get('hash_id_treatment', None)
2087 limit = request.args.get('limit', None)
2088 offset = request.args.get('offset', None)
2089 # I am uysing space before col to replace the column name by the table.column_name value
2090 list_col_csv = request.args.get('list_col_csv', "mm.manual_input_info->'list_actions' as manual_corr,mm.created_at,mm.user_id,mm.id,mm.mtr_exec_id,json_array_length(mm.manual_input_info->'list_actions')")
2091 if list_col_csv == "all":
2092 list_col_csv = "*"
2093 if list_col_csv == "data":
2094 list_col_csv = "mm.manual_input_info->'list_actions' as manual_corr, mm.id, mm.mtr_exec_id,json_array_length( mm.manual_input_info->'list_actions')"
2095 if list_col_csv == "data_user":
2096 list_col_csv = "mm.manual_input_info->'list_actions' as manual_corr,mm.created_at,mm.user_id,mm.id,mm.mtr_exec_id,json_array_length( mm.manual_input_info->'list_actions')"
2098 res = lss.lib_user_data_internal.load_data_comms(hash_id_treatment = hash_id_treatment,
2099 mtr_datou_id_extract = mtr_datou_id_extract)
2101 return res
2104# Chargement de données de traitement
2105@app.route('/api/v1/manax/correction/load', methods=['GET'])
2106def api_v1_mana_correction_load():
2107 print("Inside api_v1_mana_correction_load endpoint")
2108 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
2110 if user != "anonymous@opio.fr":
2111 hash_id_treatment = request.args.get('hash_id_treatment', None)
2112 limit = request.args.get('limit', None)
2113 offset = request.args.get('offset', None)
2114 # I am uysing space before col to replace the column name by the table.column_name value
2115 list_col_csv = request.args.get('list_col_csv', "mm.manual_input_info->'list_actions' as manual_corr,mm.created_at,mm.user_id,mm.id,mm.mtr_exec_id,json_array_length(mm.manual_input_info->'list_actions')")
2116 if list_col_csv == "all":
2117 list_col_csv = "*"
2118 if list_col_csv == "data":
2119 list_col_csv = "mm.manual_input_info->'list_actions' as manual_corr, mm.id, mm.mtr_exec_id,json_array_length( mm.manual_input_info->'list_actions')"
2120 if list_col_csv == "data_user":
2121 list_col_csv = "mm.manual_input_info->'list_actions' as manual_corr,mm.created_at,mm.user_id,mm.id,mm.mtr_exec_id,json_array_length( mm.manual_input_info->'list_actions')"
2123 res = lss.lib_user_data_internal.load_data_manual(hash_id_treatment = hash_id_treatment,
2124 list_cols_as_csv = list_col_csv,
2125 limit = limit,
2126 offset = offset)
2127 i = int(limit) if limit != None else 0
2128 for r in res:
2129 r["nb"] = i
2130 i = i + 1
2132 return {"data" : res, "info" : {"nb" : len(res)}}
2134# Edition manuelle des données de traitement
2135@app.route('/api/v1/manax/correction/update', methods=['GET'])
2136def api_v1_mana_correction_update():
2137 print("Inside api_v1_mana_correction_update endpoint")
2138 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
2140 if user != "anonymous@opio.fr":
2141 hash_id_treatment = request.args.get('hash_id_treatment', None)
2142 new_hash_id_treatment = request.args.get('new_hash_id_treatment', None)
2143 id = request.args.get('id', None)
2145 res = lss.lib_user_data_internal.update_data_manual(hash_id_treatment = hash_id_treatment, new_hash_id_treatment = new_hash_id_treatment,
2146 id = id)
2147# return res
2148 return "Maybe done"
2151# Edition manuelle des données de traitement dupliquer avec update
2152@app.route('/api/v1/manax/correction/save', methods=['POST'])
2153def api_v1_mana_correction_save():
2154 print("Inside api_v1_mana_correction_save endpoint")
2155 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
2157 try :
2158 with open(list_acronym_from_flask_file, "r") as f:
2159 list_acronym_from_flask = json.load(f)
2160 except Exception as e:
2161 list_acronym_from_flask = {"list" : ["irm"]}
2163 if user != "anonymous@opio.fr":
2164 hash_id_treatment = request.json.get('hash_id_treatment', None)
2165 list_actions = request.json.get('list_actions', [])
2167 manual_result = {"list_actions" : list_actions}
2169 res = lss.lib_user_data_internal.save_data_manual(hash_id_treatment = hash_id_treatment, user_id = lss.get_user_id(), manual_result = manual_result)
2170# return res
2171 return "Maybe done"
2174@app.route('/api/v1/datou/delete', methods=['POST'])
2175def datou_delete():
2176 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
2178 print(str("Pas content de pas gérer les droits !"))
2180 if user != "anonymous@opio.fr":
2181 delete_datou = request.json
2182 mtr_datou_id = delete_datou["id"]
2184 result_json = lss.delete_datou(mtr_datou_id)
2185# lss.lib_data.update_datou(update_datou_data) =>
2187 # update_datou
2189 response = Response(
2190 response=json.dumps(result_json),
2191 status=200,
2192 mimetype='application/json'
2193 )
2194 return response
2196 return to_test_mets_a_jour
2198@app.route('/api/v1/datou/save', methods=['POST'])
2199def datou_step_edit():
2200 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
2202 print(str("Pas content"))
2204 if user != "anonymous@opio.fr":
2205 update_datou_data = request.json
2207 result_json = lss.update_datou(update_datou_data)
2208# lss.lib_data.update_datou(update_datou_data) =>
2210 # update_datou
2212 response = Response(
2213 response=json.dumps(result_json),
2214 status=200,
2215 mimetype='application/json'
2216 )
2217 return response
2219 return to_test_mets_a_jour
2223@app.route('/legal', methods=['GET'])
2224def legal():
2225 result = {"projects" : {}}
2226 return render_template('legal.html', results=result)
2228@app.route('/privacy', methods=['GET'])
2229def privacy():
2230 result = {"projects" : {}}
2231 return render_template('privacy.html', results=result)
2233@app.route('/terms', methods=['GET'])
2234def terms():
2235 result = {"projects" : {}}
2236 return render_template('terms.html', results=result)
2238def check_bearer_stuff_and_voila():
2239 print("TODO")
2241@app.route('/api/v1/safia/query', methods=['GET', 'POST']) # , 'POST'
2242# @oauth.require_oauth('email')
2243def query():
2244 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
2245 curr_project_id = data_needed["project_id"]
2247 if user == "anonymous@opio.fr":
2248 try :
2249 token = request.authorization.token
2250 info_if_is_valid = lss.lib_user_data_internal.check_bearer_token(token)
2251 if info_if_is_valid != None:
2252 lss.user_id = info_if_is_valid["mtr_user_id"]
2253 info_user = lss.lib_user_data_internal.select_user(info_if_is_valid["mtr_user_id"])
2254 user = info_user["mail"]
2255 except Exception as e:
2256 print(str(e))
2257 print("No auth !")
2260 start_time = time.time()
2262 preprompt = None
2263 import logging
2264 logger = logging.getLogger()
2265 logger.info(" In query() (Safia) ")
2267 # - [ ] TODO VR a normaliser en faisant une requete POST ?
2268 data_json = request.args if dict(request.args) != {} else request.json if isinstance(request.json, dict) else {}
2270 if curr_project_id == None or curr_project_id == "" or curr_project_id == -1:
2271 if "project_id" in data_json:
2272 curr_project_id = data_json["project_id"]
2274 saxia_step_name = data_json["saxia_step_name"] if "saxia_step_name" in data_json else None
2276 store_in_project = "store_in_project" in data_json and data_json["store_in_project"].lower() == 'true'
2277 replace_last_result = "replace_last_result" in data_json and data_json["replace_last_result"].lower() == 'true'
2278 if replace_last_result:
2279 print(" TO BE USED WISELEY TODO VR 13-9")
2280 include_last_request = "include_last_request" in data_json and data_json["include_last_request"].lower() == 'true'
2281 # - [ ] TODO VR work on this feature to understand the best way to do it
2282 replace_last_result = "replace_last_result" in data_json and data_json["replace_last_result"].lower() == 'true'
2283 last_text = data_json["last_text"].lower() if "last_text" in data_json else ""
2284 output_datou_to_col_csv = data_json["output_datou_to_col"] if "output_datou_to_col" in data_json else ""
2285 from lib.batch.lib_batch import prepare_ioput_col_datou
2286 output_column_name, output_datou_name = prepare_ioput_col_datou(output_datou_to_col_csv)
2287 value_index = data_json["value_index"] if "value_index" in data_json else None
2288 col_index = data_json["col_index"] if "col_index" in data_json else None
2289 safia_suivi_prod_proj_id = data_json["safia_suivi_prod_proj_id"] if "safia_suivi_prod_proj_id" in data_json else None
2290 safia_suivi_prod_doc_id = data_json["safia_suivi_prod_doc_id"] if "safia_suivi_prod_doc_id" in data_json else None
2292 elapsed_time = time.time() - start_time
2293 logger.info(" Before insert_query_log ! ")
2294 query_id = insert_query_log(lss, request.path, request, lss.get_user_id(), curr_project_id, elapsed_time) # "/api/v1/safia/query"
2296 logger.info(" Query inserted ! ")
2298 try:
2299 with_audit_arg = data_json.get("with_audit")
2300 with_audit = with_audit_arg != None and str(with_audit_arg).lower() != "false" # data_json est request.args ici
2301 input = request.args.get("input")
2302 print(" input : " + str(input))
2303 if include_last_request:
2304 input += last_text
2305 print(" input : " + str(input))
2307 input_csv = data_json.get("input_csv", "")
2308 from lib.lib_util import load_csv_as_dict
2309 input_from_csv = load_csv_as_dict(input_csv)
2311 doc_to_include = data_json["doc_to_include"] if "doc_to_include" in data_json else None
2312 if doc_to_include != None:
2313 input += lss.load_document(document_id_input = doc_to_include, project_id = curr_project_id, chunk_id = None)
2315 from lib.datou.datou_exec import list_datous
2316 list_actions_datou_query_texte = list_datous["query"]
2318 datou_id = -1
2319 list_param_json = []
2320 object_arg = None
2321 if request.method == "GET":
2322 object_arg = request.args.get("object", None)
2323 if request.method == "POST":
2324 object_arg = request.json.get("object", None)
2325 if object_arg == "simple_text_query":
2326# preprompt = ""
2327 print(" We want jsut to do a simple query")
2328 list_actions_datou_query_texte=["request_gpt"]
2330 to_search = data_json if dict(data_json) != {} else request.json if isinstance(data_json, dict) else {}
2331 for d in to_search:
2332 if d.startswith("datou") and str(to_search[d]).lower() == "true":
2333# data_json[d] = "true"
2334 print("One datou is checked : it overrides the rest : will be used : " + str(d))
2335 datou_id_str = d[6:]
2336 datou_id = int(datou_id_str)
2337 # VR TODO le chargement des datous dans uploader est vraiment tout pourri, il prend toutes les steps et reflechit apres
2338 # list_step_datou = lss.get_datou_step_user()
2339 # append_step_to_datou()
2340 # list_steps_datou = list(filter(lambda x : x["mtr_datou_id"] == datou_id, list_step_datou))
2342 list_datous = lss.get_datou(id = datou_id,
2343 project_id = curr_project_id)
2344 map_datou_datou_id = {int(l["id"]) : l for l in list_datous}
2345 list_steps_datou = map_datou_datou_id[datou_id]["steps"]
2346 list_param_json = list(map(lambda x : x["param_json"], list_steps_datou))
2348 list_actions_datou_query_texte = []
2349 for l in list_steps_datou:
2350 list_actions_datou_query_texte.append(l["name"])
2351 # TODO VR : je voudrais faire quelquechose de mieux que ce truc juste pour le document edit ! lié à doc edit
2352 if "preprompt" in l["param_json"] and l["param_json"]["preprompt"] != "":
2353 # preprompt = l["param_json"]["preprompt"] + "\n-----\nDEBUT DU DOCUMENT`A SUIVRE\n```markdown\n" + preprompt + "\n```\nLE DOCUMENT VIENT DE FINIR\n-----\nINSTRUCTION:\n-----\n"
2354 preprompt = l["param_json"]["preprompt"] + "{((O8<markdown\n" + preprompt + ">8O))}" if preprompt != None else ""
2356 print(" list_actions_datou : " + str(list_actions_datou_query_texte))
2358 print(" selected_datou in upload " + str(datou_id))
2360 if request.args.get("store_in_project") == "true":
2361 # list_actions_datou_query_texte.append("")
2362 list_actions_datou_query_texte.append("result_to_json")
2363 list_actions_datou_query_texte.append("import_json")
2366 verbose = request.args.get("verbose", False)
2368 complete_param_json = get_datou_exec_context_as_complete_param_json(user, verbose, privacy = False,
2369 openai_token=OPENAI_API_KEY, lss=lss,
2370 project_id=curr_project_id)
2372 from lib.datou.datou_exec import datou_exec
2373 input_datou = {}
2374 if request.method == "GET":
2375 input_datou = data_json.get("input_datou", {}) # GET ??
2376 if request.method == "POST":
2377 input_datou = request.json.get("input_datou", {}) # POST
2378 if input_datou == {}:
2379 input_datou = {"text" : input}
2380 input_datou.update(input_from_csv)
2381 if with_audit:
2382 # TODO c'est en fait optionnel, car la lliste des sorties on s'en fout un peu je crois
2383 input_datou["datou_exec_info"] = {"project_id": curr_project_id,
2384 "safia_doc_id": "list_treatment_from_api_front_safia", # ca c'est deprectaed 6/4/24
2385 "mtr_datou_id": datou_id,
2386 "launched_at" : datetime.datetime.now(),
2387 "user_id" : lss.get_user_id()}
2388 # VR 25-4-24 : to load saxia_conf, but why not doing it voila here and pass saxia_conf ? => is this too specific, grrr !
2389 input_datou["project_id"] = curr_project_id
2390 id_file = data_json["id_file"] if "id_file" in data_json else None
2391 if "datou_exec_info" in input_datou and (id_file != None and id_file != ""):
2392 input_datou["datou_exec_info"]["id_file"] = id_file
2394 hash_id_treatment_rerun = data_json[
2395 "hash_id_treatment_rerun"] if "hash_id_treatment_rerun" in data_json else None
2397 id_step_incomplete_args = None
2398 if request.method == "GET":
2399 id_step_incomplete_args = data_json.get("id_step_incomplete_args", None)
2400 if request.method == "POST":
2401 id_step_incomplete_args = request.json.get("id_step_incomplete_args", None)
2403 output, audit_json = datou_exec(list_actions_datou_query_texte,
2404 input_datou, complete_param_json,
2405 verbose=verbose, list_param_json_steps=list_param_json,
2406 with_audit = with_audit,
2407 hash_id_treatment_rerun = hash_id_treatment_rerun,
2408 id_step_incomplete_args = id_step_incomplete_args)
2410 info_store = {}
2411 if len(output_column_name) == len(output_datou_name) and len(output_column_name) > 0:
2412 if safia_suivi_prod_doc_id == None or str(safia_suivi_prod_doc_id) == "" or str(safia_suivi_prod_doc_id) == "None":
2413 print("DEPRECATED certainly VR 10-5-25")
2414 else:
2416 from lib.batch.lib_batch import create_pandas_table_from_text, fill_df_from_datou_result, \
2417 load_safia_doc_input_list, create_markdown_table_from_df
2419 total_content = load_safia_doc_input_list(safia_suivi_prod_doc_id, safia_suivi_prod_proj_id, lpgss_singleton, limit=100000)
2421 try:
2422 df = create_pandas_table_from_text(total_content)
2423 except Exception as e:
2424 print(str(e) + "in consolidate sans doute")
2425 import types
2426 if type(df) == types.NoneType:
2427 import pandas as pd
2428 df = pd.DataFrame(columns=output_column_name + [col_index])
2429 df.loc[len(df)] = pd.Series(dtype='string')
2430 df.loc[0, col_index] = value_index
2432 info_store = fill_df_from_datou_result(df, output_datou_name,
2433 output_column_name, output, value_index, col_index)
2435 total_content_new_text = create_markdown_table_from_df(df)
2437 lss.save_document({"document_id" : safia_suivi_prod_doc_id,"document_content" : total_content_new_text}, safia_suivi_prod_proj_id, openai_token=lcs.get_openai_api_key())
2440 print(" batch data updated in list treatment ")
2442 cost = 0
2443 if "cost" in output:
2444 cost = output["cost"]
2445 lss.lib_user_data_internal.add_cost_to_project(curr_project_id, cost)
2447 if info_store != {} and saxia_step_name != None and "hash_id_treatment" in output:
2448 hash_id_treatment = output["hash_id_treatment"]
2449 if "hash_id_treatment_input" in output:
2450 hash_id_treatment = output["hash_id_treatment_input"]
2451 name_info_step = "info_" + saxia_step_name
2452 saxia_step_name_at = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
2453 info_store[saxia_step_name + "_at"] = saxia_step_name_at
2455 lss.lib_user_data_internal.upsert_audit_info(hash_id_treatment,
2456 list_json_update = [
2457{"variable": name_info_step, "type": "json"}
2458],
2459 input_values = {name_info_step : info_store},
2460 verbose = verbose)
2463 detailed_time_safia_query = [] # to reset !
2465 print("About to finish !")
2467 except Exception as e: # pragma no cover except
2468 import traceback
2469 stack_trace = traceback.format_exc()
2470 print(stack_trace)
2472 elapsed_time = time.time() - start_time
2473 lss.update_query_end(query_id, response_status = 500, response_body = "", elapsed_time = elapsed_time,
2474 cost = 0, detailed_time = {})
2476 return Response('Error: {}'.format(str(e)), status=500)
2478 print("render")
2480 elapsed_time = time.time() - start_time
2481 lss.update_query_end(query_id, response_status = 200, response_body = output,
2482 elapsed_time = elapsed_time, cost = cost, detailed_time = {"list" : detailed_time_safia_query})
2484 response = Response(
2485 response=json.dumps(output, default=str),
2486 status=200,
2487 mimetype='application/json'
2488 )
2489 return response
2490# return str(output)
2494@app.route('/api/v1/project/select/<project_id>', methods=['POST'])
2495def api_v1_project_select_project_id(project_id):
2496 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["otp"])
2497 otp = data_needed["otp"]
2499 if request.method == 'POST':
2500 input = request.args.get("input")
2501 print(" input : " + str(input))
2502 result_json = {"project_id":project_id}
2503 if "anonymous@opio.fr" != lss.get_user():
2504 # TODO VR faire cet enregistrement de cookie dans toutes les query
2505# response = make_response("Data connexion " + str(info_auth))
2506 response = Response(
2507 response=json.dumps(result_json),
2508 status=200,
2509 mimetype='application/json'
2510 )
2511 print(" otp : " + str(otp))
2512 if otp != None:
2513 response.set_cookie("otp", otp, max_age=60*60*24*15) # Le cookie expire après 15 jours
2514 response.set_cookie("project_id", str(project_id), max_age=60*60*24*15)
2516 return response
2517 else :
2518 return to_test_mets_a_jour# 'Please log correctly on opia/assistant'
2523@app.route('/api/v1/project/create/<survey_id>', methods=['POST'])
2524def api_v1_project_create_from_survey(survey_id):
2525 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["otp"])
2526 otp = data_needed["otp"]
2528 if request.method == 'POST':
2529 input = request.json
2530 print(" input : " + str(input))
2532 if lss.get_user() != "anonymous@opio.fr":
2534 result_json = lss.create_project(input)
2536# result_json = {"sruvey_if_useless":survey_id}
2538 # TODO VR faire cet enregistrement de cookie dans toutes les query
2539# response = make_response("Data connexion " + str(info_auth))
2540 response = Response(
2541 response=json.dumps(result_json),
2542 status=200,
2543 mimetype='application/json'
2544 )
2545 return response
2546 else :
2547 return to_test_mets_a_jour# 'Please log correctly on opia/assistant'
2551@app.route('/api/v1/project/detail/<project_id>', methods=['GET'])
2552def api_v1_project_detail_project_id(project_id):
2553 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["otp"])
2554 otp = data_needed["otp"]
2556 if request.method == 'GET' and user != "anonymous@opio.fr":
2558 # - [ ] TODO VR check right
2559 limit = int(request.args.get("limit", 100))
2560 map_project_documents, result_json = lss.stat_all_one_project(project_id, limit = limit)
2562 nb_doc_with_no_title = 0
2563 nb_doc_with_title = 0
2564 nb_doc_wo_info_on_title = 0
2565 nb_chunks = 0
2566 for doc_info in map_project_documents:
2567 print(str(doc_info))
2568 if "title" not in doc_info or doc_info["title"] == None or doc_info["title"] == 'No Title Found':
2569 nb_doc_with_no_title += 1
2570 elif "title" in doc_info and doc_info["title"] != None:
2571 nb_doc_with_title += 1
2572 elif "title" not in doc_info:
2573 nb_doc_wo_info_on_title += 1
2575 result_json["nb_doc_with_no_title"] = nb_doc_with_no_title
2576 result_json["nb_doc_with_title"] = nb_doc_with_title
2577 result_json["nb_doc_wo_info_on_title"] = nb_doc_wo_info_on_title
2579# TODO to add in projects.html
2580 # (without title : <text id="nb-elem-notitle-proj-{{k.id}}" >Loading</text>/<text id="nb-elem-tottitle-proj-{{k.id}}" >Loading</text> AND noinfo <text id="nb-elem-noinfo-title-proj-{{k.id}}" >Loading</text>)
2582 response = Response(
2583 json.dumps(result_json, default=str),
2584 status=200,
2585 mimetype='application/json'
2586 )
2587 return response
2588 else :
2589 return to_test_mets_a_jour
2593@app.route('/api/v1/project/save', methods=['POST'])
2594def save_project():
2595 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
2596 project_id = data_needed["project_id"] if "project_id" in data_needed else 0
2598 if user == "anonymous@opio.fr":
2599 return to_test_mets_a_jour
2601 info_json = request.get_json()
2602 project_id = info_json["project_id"] if "project_id" in info_json else project_id
2603 key = info_json["key"] if "key" in info_json else None
2604# shortname = data.get('shortname')
2605# project_id = data.get('project_id')
2606# if not shortname or not project_id:
2607# return jsonify({'error': 'Missing shortname or project_id'}), 400
2609 if request.method == 'POST' and project_id != None and user != "anonymous@opio.fr":
2610 lss.update_project_info(project_id, info_json)
2611# configuration_option = request.json.get("configuration", {})
2612# if configuration_option != {}:
2613# lss.update_conf_project(project_id, configuration_option, replace = True)
2614 return jsonify({'success': 'Shortname updated'}), 200
2615 else:
2616 return jsonify({'error': 'Project not found'}), 200 # 404 not taken into account
2619@app.route('/upload')
2620def upload():
2621 return render_template('upload.html')
2624@app.route('/display')
2625def display():
2626 return render_template('display.html')
2628@app.route('/version', methods = ['GET', 'POST'])
2629# async
2630def version():
2631 email_user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
2632 curr_project_id = data_needed["project_id"] # A mon avis cela vient déjà des cookies
2634 # - [ ] TODO deduplicate this code
2635 curr_project_id = request.cookies.get("project_id")
2637 if request.method == 'POST':
2638 data_json = request.form
2639 if request.method == 'GET':
2640 data_json = request.args
2642 if "project_id" in data_json:
2643 curr_project_id = int(data_json["project_id"])
2645 all_configurations = []
2646 list_useful_project_id = [70, 91, 121, 94, 96]
2648 raw_configuration = lss.load_conf_project(curr_project_id)
2649 all_configurations.append(raw_configuration)
2651 from lib.util.lib_formal_conf import formal_conf_prepare
2652 configuration = formal_conf_prepare(raw_configuration, lss)
2654 list_datou_ids = configuration["saxia"]["list_datou_ids"] if "saxia" in configuration and "list_datou_ids" in configuration["saxia"] and "saxia" in configuration else []
2655 list_datous = lss.get_datou(list_datou_ids = list_datou_ids)
2657 for pid in list_useful_project_id:
2658 if pid != curr_project_id:
2659 raw_configuration = lss.load_conf_project(pid)
2660 all_configurations.append(raw_configuration)
2662 from auth.lib_conf_system import collect_version_from_datou_and_proj_and_app_recursively
2663 host_url = request.host_url
2664 hostname = ""
2665 version_string = collect_version_from_datou_and_proj_and_app_recursively(list_datous, all_configurations,
2666 host_url = host_url)
2668 return version_string
2672# user_ide
2673def upload_internal(requeste, email_usere, lsse, data_jsone, curr_project_ide,
2674 hc_datou_use = None,
2675 auth_type = "user_interface",
2676 auth_info = {}): # auth_type will be oauth_token, user_interface, # auth_info will be user logging in or time refresh_token
2678 request = requeste
2679 email_user = email_usere
2680 lss = lsse
2681 data_json = data_jsone
2682# query_id = query_ide
2683# list_files = list_filese
2684 curr_project_id = curr_project_ide
2688 start_time = time.time()
2689 count_detailed_time = time.time()
2690 detailed_time = [] # Where to append
2691 import logging
2692 logger = logging.getLogger()
2694 elapsed_time = time.time() - start_time
2695 query_id = insert_query_log(lss, request.path, request, lss.get_user_id(), curr_project_id, elapsed_time)
2696 count_detailed_time, message = count_and_display_elapsed_time(count_detailed_time, "get_project_info", verbose = False)
2697 detailed_time.append(message)
2700 from auth.lib_stat_usage import record_usage
2701 ipv4 = request.remote_addr
2702 record_usage(lss.get_user_id(), ipv4, type="endpoint_usage", info_message='/api/v1/uploader',
2703 lpgss=lpgss_singleton)
2705 first_input_file = ""
2706 if request.method == 'POST':
2707 nb_files = len(request.files)
2708 print(" nb_files : " + str(nb_files))
2709 list_files = []
2710 for f in request.files:
2711 list_files.append(request.files[f])
2712#
2714# previous_behavior_one_file = len(list_files) == 1
2715 host_url = request.host_url
2717 data_json = request.form
2719 input_csv = data_json.get("input_csv", "")
2720 from lib.lib_util import load_csv_as_dict
2721 input_from_csv = load_csv_as_dict(input_csv)
2724 object_safia_query = "from_www"
2725 if "object" in data_json:
2726 object_safia_query = data_json["object"]
2727 # TODO VR could be useful !
2728 if "project_id" in data_json:
2729 curr_project_id = int(data_json["project_id"])
2731 if curr_project_id == -1:
2732 list_pids = lss.lib_right.get_projects_with_access(lss.get_user_id())
2733 if len(list_pids) > 0 and "id" in list_pids[0]:
2734 curr_project_id = list_pids[0]["id"]
2737 # VR TODO 3-5-24 : remove these duplicate from save_input_data
2738 configuration = lss.load_conf_project(curr_project_id)
2739 cdn_conf_all = configuration["cdn"] if "cdn" in configuration else {}
2740 import socket
2741 hostname = socket.gethostname()
2742 cdn_conf = cdn_conf_all[hostname] if hostname in cdn_conf_all else cdn_conf_all["default"] if "default" in cdn_conf_all else {}
2744 root_directory_cdn = cdn_conf["root_local_dir"] if "root_local_dir" in cdn_conf else "static/temp" #/home/safia/workarea/git/Safia/prompt/python/server/static/onedrive/workarea_anon"
2746 # VR 3-5-24 : to be ported in lib_batch and used there also
2747 # What about action copy !
2748 input_dir = "from_api"
2749 today_output = "output_w" + input_dir + "_" + datetime.datetime.now().strftime("d_%d%m_%h_%H")
2750 working_current_dir = os.path.join(root_directory_cdn, "workarea_anon", today_output)
2752 preprompt = ""
2753 if "preprompt" in data_json:
2754 preprompt = data_json["preprompt"]
2756 doc_to_include = data_json["doc_to_include"] if "doc_to_include" in data_json else None
2757 if doc_to_include != None and doc_to_include != "":
2758 preprompt += lss.load_document(document_id_input = doc_to_include, project_id = curr_project_id, chunk_id = None)
2760 store_in_project = "store_in_project" in data_json and data_json["store_in_project"].lower() == 'true'
2761 # - [ ] TODO VR 13-9 a priori inutile ici
2762 include_last_request = "include_last_request" in data_json and data_json["include_last_request"].lower() == 'true'
2763 # - [ ] TODO VR 13-9 work on this feature to understand the best way to do it
2764 replace_last_result = "replace_last_result" in data_json and data_json["replace_last_result"].lower() == 'true'
2765 with_audit = "with_audit" in data_json and data_json["with_audit"].lower() == 'true'
2767 saxia_split_end_csv = data_json["saxia_split_end_csv"] if "saxia_split_end_csv" in data_json else ""
2768 saxia_all_doc_separated = "saxia_all_doc_separated" in data_json and data_json["saxia_all_doc_separated"].lower() == 'true'
2769 saxia_multiple_folder = "saxia_multiple_folder" in data_json and data_json["saxia_multiple_folder"].lower() == 'true'
2770 saxia_single_folder = "saxia_single_folder" in data_json and data_json["saxia_single_folder"].lower() == 'true'
2773 print("First try, we should then use something like temp and treat this file accordingly !")
2774 # f.save(secure_filename(f.filename))
2777 print(" list_files : " + str(list_files)) # f.content_type #
2778 logger.info(" list_files : " + str(list_files))
2780 file_name = [fi.filename for fi in list_files]
2781 if first_input_file == "":
2782 first_input_file = file_name[0] if len(file_name) > 0 else ""
2783 file_extension = [os.path.splitext(fi.filename)[1] for fi in list_files]
2784 file_content_type = [fj.content_type for fj in list_files]
2786# input_file_zip = zip([os.path.splitext(fi.filename) for fi in list_files])
2787# input_file_zip_unzip = list(map(list, input_file_zip))
2788# file_name, file_extension = input_file_zip_unzip
2790 if len(list_files) > 0:
2792 from lib.lib_util import select_datou_step_from_extension_type_upload
2793 from lib.datou.datou_exec import list_datous
2794 print(" list_datous that should stay always the same but seems to be edited by reference ! : " + str(list_datous))
2795 is_managed, list_actions_datou, extension = select_datou_step_from_extension_type_upload(file_name[0], file_extension[0], file_content_type[0])
2796 else:
2797 is_managed = False
2798 list_actions_datou = []
2799 extension = ".empty"
2800 print(" list_actions_datou : " + str(list_actions_datou))
2802 list_param_json = []
2803 datou_id = -1
2804 if hc_datou_use != None and type(hc_datou_use) == int:
2805 datou_id = hc_datou_use
2807 for d in data_json:
2808 if d.startswith("datou") and str(data_json[d]).lower() == "true":
2809# data_json[d] = "true"
2810 print("One datou is checked : it overrides the rest : will be used : " + str(d))
2811 datou_id_str = d[6:]
2812 datou_id = int(datou_id_str)
2813 # VR TODO le chargement des datous dans uploader est vraiment tout pourri, il prend toutes les steps et reflechit apres
2814# list_step_datou = lss.get_datou_step_user()
2815# append_step_to_datou()
2816# list_steps_datou = list(filter(lambda x : x["mtr_datou_id"] == datou_id, list_step_datou))
2817 break
2819 if datou_id > 0:
2821 list_datous = lss.get_datou(id = datou_id,
2822 project_id=curr_project_id)
2823 map_datou_datou_id = {int(l["id"]) : l for l in list_datous}
2824 list_steps_datou = map_datou_datou_id[datou_id]["steps"]
2825 list_param_json = list(map(lambda x : x["param_json"], list_steps_datou))
2827 list_actions_datou = []
2828 for l in list_steps_datou:
2829 list_actions_datou.append(l["name"])
2830 # TODO VR : je voudrais faire quelquechose de mieux que ce truc juste pour le document edit ! lié à doc edit
2831 if "preprompt" in l["param_json"] and l["param_json"]["preprompt"] != "":
2832# preprompt = l["param_json"]["preprompt"] + "\n-----\nDEBUT DU DOCUMENT`A SUIVRE\n```markdown\n" + preprompt + "\n```\nLE DOCUMENT VIENT DE FINIR\n-----\nINSTRUCTION:\n-----\n"
2833 preprompt = l["param_json"]["preprompt"] + "{((O8<markdown\n" + preprompt + ">8O))}"
2835 print(" list_actions_datou : " + str(list_actions_datou))
2838 list_datou_ids = configuration["saxia"]["list_datou_ids"] if "saxia" in configuration and "list_datou_ids" in configuration["saxia"] and "saxia" in configuration else []
2839 list_datous = lss.get_datou(list_datou_ids = list_datou_ids)
2841 from auth.lib_conf_system import collect_version_from_datou_and_proj_and_app_recursively
2842 hostname = ""
2843 version_string = collect_version_from_datou_and_proj_and_app_recursively(list_datous, [configuration])
2845 if auth_type == "user_interface":
2846 conf_to_update = {"custom_display" : {"home" : data_json}}
2847 lss.update_conf_project(curr_project_id, conf_to_update)
2849 if store_in_project and "prepare_json_for_safia" not in list_actions_datou:
2850 list_actions_datou.append("result_to_json")
2851 list_actions_datou.append("import_json")
2853 if include_last_request:
2854 print("TODO list_actions_datou !")
2856 if "query_voice" in data_json and data_json["query_voice"].lower() == "true":
2857 print("We expect only voice")
2858 list_actions_datou = ["speech_to_text", "get_embedding", "search_doc_NN", "request_gpt"]
2859 elif object_safia_query == "create_graph":
2860 list_actions_datou.append("create_graph") # SAuf qu'il faut la rajouter au debut
2861 elif object_safia_query == "edit_graph": # TODO VR 15-6-23 to merge with create_graph
2862 list_actions_datou.append("edit_graph")
2863 elif object_safia_query == "git":
2864 list_actions_datou.append("git_action")
2865 else :
2866 print("object " + str(object_safia_query) + " doesn't registered has a specific action (but maybe a preprompt) ")
2868 import uuid
2869 if file_extension[0] == "" and extension != "":
2870 local_filename = [os.path.join(working_current_dir, one_file_name + str(uuid.uuid4()) + extension) for one_file_name in file_name]
2871 elif file_extension[0] != "":
2872 local_filename = [os.path.join(working_current_dir, one_file_name + str(uuid.uuid4()) + one_file_extension) for one_file_name, one_file_extension in zip(file_name, file_extension)]
2873 else :
2874 print("We couldn't manage this file, nothing will happen now !")
2876 # VR TODO : sans le input data on ne pourra pas accéder au dossier temp à coté du dossier static, il faudrait qu'il soit dedans ou rajouter des endpoints
2877 # VR 3-5-24 : quand ai-je voulu faire cela ?
2878 # c'était bien avant saxia, et ca rendre dans le moule des cdns !
2879 save_input_data = "save_input_data" in data_json and data_json["save_input_data"].lower() == "true"
2880 if save_input_data:
2881 today_folder = datetime.datetime.now().strftime("%Y/%m/%d")
2882 local_filename = list(map(lambda l : os.path.join("static/cdn", today_folder, os.path.basename(l)), local_filename))
2884 # os.path.join("temp", f.filename)
2885# local_filename = "temp/from_upload.ogg"
2886 print(" We expect all local_filename to be in the same folder")
2887 if not os.path.exists(os.path.dirname(local_filename[0])):
2888 os.makedirs(os.path.dirname(local_filename[0]))
2889 for one_file, one_local_filename in zip(list_files, local_filename):
2890 one_file.save(one_local_filename)
2891 logger.info(" one_local_filename : " + str(one_local_filename))
2892 verbose = data_json.get("verbose", False)
2894 references = []
2895 if "link_input_as_reference" in data_json and data_json["link_input_as_reference"].lower() == "true":
2896 print(" We need to move this !")
2897 references = local_filename
2899 # TODO VR Test non reg impact fais chiez a supprimer result
2900 result_json = {"log" : "About to record "} #, "result" : {}}
2901 # if "import_json" in list_actions_datou or "search_doc_NN" in list_actions_datou:
2902 complete_param_json = get_datou_exec_context_as_complete_param_json(email_user, verbose, privacy = False,
2903 openai_token=OPENAI_API_KEY, lss=lss,
2904 project_id=curr_project_id)
2906 from lib.datou.datou_exec import datou_exec
2907 if type(local_filename) == list and len(local_filename) == 1:
2908 local_filename = local_filename[0]
2909 input = {"file" : local_filename, "references" : references}
2911 input["version"] = version_string
2912 complete_param_json["version"] = version_string
2913# complete_param_json["version"] = version_string
2915 if type(local_filename) == list and len(local_filename) > 1:
2916 input["multi_input"] = True
2917 if saxia_split_end_csv != "":
2918 print("TODO VR 11/9/24 better documentation : End of documents from saxia_split_end_csv will be ignored since we have many file !")
2919 saxia_split_end_csv = ""
2921 input["saxia_split_end_csv"] = saxia_split_end_csv.replace(" ", "")
2922 input["saxia_all_doc_separated"] = saxia_all_doc_separated
2923 input["saxia_single_folder"] = saxia_single_folder
2924 input["saxia_multiple_folder"] = saxia_multiple_folder
2925 input["input_file"] = first_input_file
2927 input.update(input_from_csv)
2929 if preprompt != "":
2930 input["preprompt"] = preprompt
2932 if "create_graph" in list_actions_datou:
2933 input["preprompt"] = "Renvoyer un graph au format dot de graphviz."
2935 out_folder = working_current_dir #os.path.join(root_directory_cdn, working_current_dir)
2936 input["out_folder"] = out_folder # "static/temp"
2937 # mayne useless
2938 if not os.path.exists("temp"):
2939 os.makedirs("temp")
2940 if not os.path.exists(input["out_folder"]):
2941 os.makedirs(input["out_folder"])
2943 if with_audit:
2944 # TODO c'est en fait optionnel, car la lliste des sorties on s'en fout un peu je crois
2945 input["datou_exec_info"] = {"project_id": curr_project_id,
2946 "safia_doc_id": "list_treatment_from_api_front_safia", # ca c'est deprectaed 6/4/24
2947 "mtr_datou_id": datou_id,
2948 "launched_at" : datetime.datetime.now()}
2949 output, audit_json = datou_exec(list_actions_datou, input,
2950 complete_param_json, verbose = verbose,
2951 list_param_json_steps = list_param_json,
2952 with_audit = with_audit)
2954 cost = 0
2955 if "cost" in output:
2956 cost = output["cost"]
2957 lss.lib_user_data_internal.add_cost_to_project(curr_project_id, cost)
2959 meta_instruction_display = {}
2961 if "create_graph" in list_actions_datou or "edit_graph" in list_actions_datou:
2962 result = output["result"]
2963 from lib.lib_graph import parse_graph_and_create
2964 my_graph = parse_graph_and_create(result, "static/temp/graph")
2965 meta_instruction_display = {"graph" : {"url_rel" : my_graph}}
2966 output["meta_instruction_display"] = meta_instruction_display
2967 else :
2968 print("No object specific query")
2969 result_json["manage_prompt"] = output
2970 # Sans doute à mettre dans datou
2971 tempdir = "temp"
2972 # TODO to refacto VR 26-4-24
2973 privacy_param = False
2974 if privacy_param :
2975 # TODO VR only delete the current file
2976 print("Deleting tempdir : " + tempdir)
2977 import shutil
2978 if os.path.exists(tempdir):
2979 shutil.rmtree(tempdir)
2980# shutil.rmtree(tempdir)
2983 list_detailed_time_safia_import_to_json = []
2985 count_detailed_time, message = count_and_display_elapsed_time(count_detailed_time, "prepare_data", verbose = False)
2986 detailed_time.append(message)
2990 if auth_type == "oauth_token":
2991 result_json["auth_info"] = auth_info
2992 if "manage_prompt" not in result_json:
2993 return jsonify({"Error":"Internal"}), 404
2994 folder_result = os.path.dirname(result_json["manage_prompt"]["file"])
2995 result_json["url_result"] = request.host_url + os.path.join(folder_result, result_json["manage_prompt"]["output_hit"]) + ".docx"
2996 result_json["manage_prompt"]["local_basename"] = os.path.basename(result_json["manage_prompt"]["file"])
2997 list_mp_to_del = []
2998 for d in result_json["manage_prompt"]:
2999 if d not in ["local_basename", "nb_page", "filename_at", "input_file_available_at", "size_file", "output_hit", "version", "hash_id_treatment", "cost", "output_hit"]:
3000 list_mp_to_del.append(d)
3001 for d in list_mp_to_del:
3002 del result_json["manage_prompt"][d]
3004# "file": "static/temp/workarea_anon/output_wfrom_api_d_0912_Dec_17/at_test_nb_1.pdf936f1b86-06a7-4b2e-83dd-edd33229bbab.pdf",
3005 import json
3006 result_json["query_id"] = query_id
3007 from lib.lib_util import remove_circular_refs
3008 result_json = remove_circular_refs(result_json)
3009 if "manage_prompt" in result_json:
3010 print("manage_prompt is temporary but has broken graph (among others things) ")
3011 print(" result_json : " + str(result_json)[:100])
3012 response = Response(
3013 response=json.dumps(result_json, default=str),
3014 status=200,
3015 mimetype='application/json'
3016 )
3018 list_prefix = list(map(lambda x : "safia_import_" + str(x), list_detailed_time_safia_import_to_json))
3019 detailed_time.extend(list_prefix)
3020 elapsed_time = time.time() - start_time
3021 lss.update_query_end(query_id, response_status = 200, response_body = json.dumps(result_json, default=str),
3022 elapsed_time = elapsed_time, cost = cost, detailed_time = str(detailed_time))
3024 return response
3025# return 'file uploaded successfully'
3029upload_tag = Tag(name="saxia_upload", description="Saxia Upload")
3031# Modèles Pydantic pour la documentation de l'endpoint uploader
3032class UploaderFormData(BaseModel):
3033 """Paramètres de formulaire pour l'endpoint uploader"""
3034 project_id: Optional[int] = Field(None, description="ID du projet (optionnel, -1 pour projet par défaut)", example=-1)
3035 object: Optional[str] = Field(None, description="Objet de la requête Saxia", example="from_www")
3036 preprompt: Optional[str] = Field(None, description="Pré-prompt à ajouter au traitement")
3037 doc_to_include: Optional[str] = Field(None, description="ID du document à inclure dans le preprompt")
3038 input_csv: Optional[str] = Field(None, description="Données CSV en entrée")
3039 store_in_project: Optional[bool] = Field(False, description="Stocker le résultat dans le projet", example=False)
3040 include_last_request: Optional[bool] = Field(False, description="Inclure la dernière requête", example=False)
3041 replace_last_result: Optional[bool] = Field(False, description="Remplacer le dernier résultat", example=False)
3042 with_audit: Optional[bool] = Field(False, description="Activer l'audit", example=False)
3043 saxia_split_end_csv: Optional[str] = Field(None, description="CSV de fin de séparation Saxia")
3044 saxia_all_doc_separated: Optional[bool] = Field(False, description="Tous les documents séparés", example=False)
3045 saxia_multiple_folder: Optional[bool] = Field(False, description="Dossiers multiples", example=False)
3046 saxia_single_folder: Optional[bool] = Field(False, description="Dossier unique", example=False)
3047 sdr: Optional[str] = Field("sleep,sizef,delf", description="Étapes de débogage (debug only)", example="sleep,sizef,delf")
3049class UploaderResponse(BaseModel):
3050 """Réponse de l'endpoint uploader"""
3051 log: str = Field(..., description="Message de log", example="About to record ")
3052 manage_prompt: dict = Field(..., description="Informations sur le traitement du document", example={
3053 "version": "v0.6.16alpha from 04-12-2025",
3054 "hash_id_treatment": "bc674dd6dd8b7dce26667f2e56c1ebb736ac8b07dcca18799df4e1e1f8bb2784",
3055 "nb_page": 1,
3056 "filename_at": "20251209",
3057 "input_file_available_at": "2025-12-09 18:00:16.814483",
3058 "size_file": 1076547,
3059 "output_hit": "id_0_nb_1_d_20251209_bc674dd6dd8b7dce26667f2e56c1ebb736ac8b07dcca18799df4e1e1f8bb2784",
3060 "cost": 0,
3061 "local_basename": "at_test_nb_1.pdfe0ffed24-9f56-48dc-985b-f1a40a5bf7cd.pdf"
3062 })
3063 auth_info: dict = Field(..., description="Informations d'authentification", example={
3064 "auth_type": "oauth_token",
3065 "user_id": 1,
3066 "token_valid_until": 1765970929,
3067 "nb_seconds_remaining": 671312.23
3068 })
3069 url_result: str = Field(..., description="URL du fichier résultat (DOCX)", example="https://app2.ultradocia.fr/static/temp/workarea_anon/output_wfrom_api_d_0912_Dec_18/id_0_nb_1_d_20251209_bc674dd6dd8b7dce26667f2e56c1ebb736ac8b07dcca18799df4e1e1f8bb2784.docx")
3070 query_id: int = Field(..., description="ID de la requête", example=-1)
3072class UploaderErrorResponse(BaseModel):
3073 """Réponse d'erreur de l'endpoint uploader"""
3074 error: str = Field(..., description="Code d'erreur", example="invalid_token")
3075 error_description: str = Field(..., description="Description de l'erreur", example="The access token provided is expired, revoked, malformed, or invalid for other reasons.")
3077#@app.errorhandler(401)
3078#@app.route('/api/v1/uploader', methods = ['GET', 'POST'])
3079@app.post('/api/v1/uploader',
3080 tags=[upload_tag],
3081 summary="Upload PDF document for processing",
3082 description="""
3083Upload a PDF document to be processed by Saxia AI.
3085**Authentication:**
3086Use Bearer token in the Authorization header.
3088**Example curl command (basic):**
3089```bash
3090curl -H "Authorization: Bearer YMWvsUJE15LGdsDKzQxHaRqEJxiIwZhBhpSvZqsC3o" \\
3091 -X POST \\
3092 -F 'file=@/path/to/your/document.pdf' \\
3093 https://app2.ultradocia.fr/api/v1/uploader
3094```
3096**Example curl command (with parameters):**
3097```bash
3098curl -H "Authorization: Bearer YMWvsUJE15LGdsDKzQxHaRqEJxiIwZhBhpSvZqsC3o" \\
3099 -X POST \\
3100 -F 'file=@/path/to/your/document.pdf' \\
3101 -F 'project_id=-1' \\
3102 -F 'preprompt=Extract all medical information' \\
3103 -F 'store_in_project=true' \\
3104 https://app2.ultradocia.fr/api/v1/uploader
3105```
3107**File upload:**
3108- Field name: `file` (multipart/form-data)
3109- PDF files only
3110- Maximum size: 1GB
3111- The file will be converted to DOCX format
3113**Optional form parameters:**
3114- `project_id`: ID du projet (-1 pour projet par défaut)
3115- `preprompt`: Texte de pré-prompt à ajouter au traitement
3116- `doc_to_include`: ID d'un document à inclure dans le preprompt
3117- `store_in_project`: Stocker le résultat dans le projet (true/false)
3118- `with_audit`: Activer l'audit (true/false)
3119- `object`: Objet de la requête (défaut: "from_www")
3120- `input_csv`: Données CSV en entrée
3121- `saxia_all_doc_separated`: Tous les documents séparés (true/false)
3122- `saxia_multiple_folder`: Dossiers multiples (true/false)
3123- `saxia_single_folder`: Dossier unique (true/false)
3125**Response:**
3126Returns information about the uploaded file, processing status, and URL to download the result (DOCX).
3128The `url_result` field contains the link to download the processed DOCX document.
3129 """,
3130 responses={
3131 "200": UploaderResponse,
3132 "401": UploaderErrorResponse
3133 })
3134def upload_file_compatible(form: UploaderFormData):
3135 """
3136 Unified uploader endpoint that works with both flask_login and Bearer token authentication.
3137 This function creates a Bearer token for flask_login users and calls upload_file_with_bearer().
3138 """
3139 email_user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
3141 token_auto_generated_from_flask_login = None
3144 import sys
3146 def print_mem_loc_upload(request = None):
3147 import logging
3148 logger = logging.getLogger()
3149 local_vars = list(locals().items())
3150 for var, obj in local_vars:
3151 print(var, sys.getsizeof(obj))
3152 logger.warning(" var : " + str(var) + " " + str(sys.getsizeof(obj)))
3153 if request != None:
3154 print(var, sys.getsizeof(request))
3155 logger.warning(" var request : " + str(request) + " " + str(sys.getsizeof(request)))
3159 if email_user != "" and email_user != "anonymous@opio.fr":
3160 # User is authenticated with flask_login
3161 # Get or create a Bearer token for this session
3162 user_id = lss.get_user_id()
3164 data_json = request.form
3165 curr_project_id = data_needed["project_id"]
3168 return upload_internal(requeste = request,
3169 email_usere = email_user,
3170 lsse = lss,
3171 data_jsone = data_json,
3172 curr_project_ide = curr_project_id)
3174 # Retrieve existing token or generate one
3175 try:
3176 # Try to get an existing valid token for this user
3177 tokens = lpgss_singleton.get_token(user_id)
3178 if tokens and len(tokens) > 0:
3179 # Use the first valid token
3180 token_auto_generated_from_flask_login = tokens[-1]['access_token']
3181 except Exception as e:
3182 print(f"Error retrieving token: {e}")
3184 # If no valid token exists, create one
3185 if not token_auto_generated_from_flask_login:
3186 from werkzeug.security import gen_salt
3187 token_auto_generated_from_flask_login = gen_salt(48) # en fait c'était 42, non ?
3188 # Create token valid for 7 days
3189 lpgss_singleton.create_token(
3190 user_id,
3191 description='Auto-generated for flask_login session',
3192 token=token_auto_generated_from_flask_login,
3193 configuration={},
3194 project_id=data_needed.get("project_id")
3195 )
3197# from flask import g
3198# g.bearer_token = token_auto_generated_from_flask_login
3200# VR 20/11/25 Claude fais de la mmmmerde avec request.authorization
3201# from werkzeug.datastructures import Authorization
3202# request.authorization = Authorization(
3203# auth_type='bearer',
3204# data={'token': token_auto_generated_from_flask_login}
3205# )
3206# VR 20/11/25 : en debugguant j'ai trouvé que request.headers est un EnvironHeaders
3208 bearer_auth = f'Bearer {token_auto_generated_from_flask_login}'
3209 request.headers.environ['HTTP_AUTHORIZATION'] = bearer_auth
3210 # request.headers['Authorization'] = bearer_auth
3211 return upload_file()
3212 else:
3213# return upload_file()
3215 import logging
3216 logger = logging.getLogger()
3217 # Call the Bearer token version with the token (if any)
3218# from authlib.oauth2 import OAuth2Error
3219 logger.info("info About to call upload_file l 2673")
3220 logger.warning("warning About to call upload_file l 2673")
3221 from werkzeug.exceptions import HTTPException
3222 # from lib.lib_github import print_mem
3223 loggger.warning(" request.scheme : " + str(request.scheme))
3224 print(" p request.scheme : " + str(request.scheme))
3225 print_mem_loc_upload()
3226 step_debug_reproduce = request.form.get("sdr", "sleep,sizef,delf")
3227 print(" p step_debug_reproduce : " + str(step_debug_reproduce))
3228 logger.warning(" step_debug_reproduce : " + str(step_debug_reproduce))
3229 if "direct" in step_debug_reproduce:
3230 return upload_file()
3231 try :
3232 print("About to call upload_file l 2673")
3233 return upload_file()
3234 except HTTPException as e:
3235 print_mem_loc_upload()
3236 import time
3237 logger.warning("HTTPException : sleep a bit")
3238# time.sleep(0.01) # VR 5/12/25 works with 0.01 for 1MB file and failed at 0.001 with debugguer vscode (wo gunicorn)
3240 if "sleep" in step_debug_reproduce:
3241# time.sleep(0.0001) # with this there is a bad gateway !
3242 time.sleep(0.1)
3244 json_Data_body = json.loads(e.get_body())
3245 print(str(e.get_body()))
3246 logger.warning(str(e.get_body()))
3247 logger.warning(str(e.code))
3248 logger.warning(str(e))
3249 print_mem_loc_upload(request)
3251 if "sizef" in step_debug_reproduce:
3252 for f in request.files:
3253 blob = request.files[f].read()
3254 size = len(blob)
3255 print(" size : " + str(size))
3256 logger.warning(" size : " + str(size))
3258 if "delf" in step_debug_reproduce:
3259 request.files = {}
3260 return jsonify(json_Data_body), e.code
3261 except Exception as e:
3262 print(str(e))
3263 return jsonify({"Error" : "Undefined", "body" : str(e)}), 401
3265@app.route('/error401', methods = ['GET', 'POST'])
3266#@require_oauth('profile')
3267def error401():
3268 loggger.warning(" request.scheme : " + str(request.scheme))
3269 print(" p request.scheme : " + str(request.scheme))
3270 print("Wbout to test 401")
3271 import logging
3272 logger = logging.getLogger()
3273 logger.warning("About to test")
3275# return jsonify({"Error" : "Fort test"}), 401
3277 if True:
3278 from werkzeug.exceptions import HTTPException
3279 try :
3280 require_oauth('profile')
3281 print("DEbug error401 About to call upload_file l 2673")
3282 logger.warning("DEbug error401 About to call upload_file l 2673")
3283 # raise HTTPException("Ca va pas")
3285 return upload_file()
3286 except HTTPException as e:
3287 import time
3289 logger.warning("HTTPException : sleep a bit")
3290 time.sleep(3) # VR 5/12/25 works with main endpoint with 0.01 and failed at 0.001 with debugguer vscode (wo gunicorn)
3291 time.sleep(0.0001) # without this there is a bad gateway !
3292 print(str(e.get_body()))
3293 logger.warning(str(e.get_body()))
3294 logger.warning(str(e.code))
3295 logger.warning(str(e))
3297 for f in request.files:
3298 blob = request.files[f].read()
3299 size = len(blob)
3300 print(" size : " + str(size))
3301 logger.warning(" size : " + str(size))
3303 request.files = {}
3305 return jsonify(e.get_body()), e.code
3306 except Exception as e:
3307 print(str(e))
3308 return jsonify({"Error" : "Undefined", "body" : str(e)}), e.code
3311#import asyncio
3312@app.route('/api/v1/oauth/uploader', methods = ['GET', 'POST'])
3313# async
3314@require_oauth('profile')
3315def upload_file():
3316 email_user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
3317 curr_project_id = data_needed["project_id"]
3319 from authlib.integrations.flask_oauth2 import current_token
3321 user = current_token.user
3322 email_user = user.username
3323 mtr_user_id = user.mtr_user_id
3324 lss.user_id = mtr_user_id
3326 data_json = request.form
3328 termination_date = current_token.issued_at + current_token.expires_in
3329 import datetime
3330 now = datetime.datetime.now()
3331 nb_seconds_remaining = termination_date - now.timestamp()
3333 auth_info = {"auth_type" : "oauth_token",
3334 "user_id" : mtr_user_id,
3335 "token_valid_until" : termination_date,
3336 "nb_seconds_remaining" : nb_seconds_remaining}
3338 return upload_internal(requeste = request,
3339 email_usere = email_user,
3340 lsse = lss,
3341 data_jsone = data_json,
3342 curr_project_ide = curr_project_id,
3343 hc_datou_use = 51,
3344 auth_type = "oauth_token",
3345 auth_info = auth_info)
3350#import asyncio
3351@app.route('/api/v1/saxia/result_correct/uploader', methods = ['POST'])
3352# async
3353def api_v1_saxia_result_correct_uploader():
3354 email_user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
3355 curr_project_id = data_needed["project_id"]
3357 user_id = lss.get_user_id()
3358 is_admin = lss.lib_right.is_user_admin_config(user_id)
3360 if request.method == 'POST' and is_admin:
3361 nb_files = len(request.files)
3362 print(" nb_files : " + str(nb_files))
3363 list_files = []
3364 for f in request.files:
3365 list_files.append(request.files[f])
3367 data_json = request.form
3368 hash_id_treatment = data_json["hash_id_treatment"]
3370 file_name = [fi.filename for fi in list_files]
3371 file_extension = [os.path.splitext(fi.filename)[1] for fi in list_files]
3372 file_content_type = [fj.content_type for fj in list_files]
3374 if len(list_files) == 1:
3375 if file_extension[0].lower() != ".docx":
3376 res = "WRONG_EXTENSION"
3377 else :
3378 this_treatment = lss.lib_user_data_internal.load_data_audit(hash_id_treatment)
3380 info_consolidate = this_treatment["info_consolidate"]
3381 infos = os.path.join(info_consolidate["out_folder"], info_consolidate["outfile_name_docx"])
3383 name = list_files[0].name
3384 one_file = request.files[name]
3386 if not os.path.exists(info_consolidate["out_folder"]):
3387 os.makedirs(info_consolidate["out_folder"])
3388 one_file.save(infos)
3391 res = "TODO"
3392 else:
3393 res = "NOK"
3395# import shutil
3396# if os.path.exists(tempdir):
3397# shutil.rmtree(tempdir)
3399 else :
3400 res = "CANT"
3402 result_json = {"ret":res}
3403 response = Response(
3404 response=json.dumps(result_json, default=str),
3405 status=200,
3406 mimetype='application/json'
3407 )
3409 return response
3413@app.route('/otp/<otp>')
3414def otp_safia(otp):
3415 try :
3416 print("About to check " + str(otp))
3417 info_auth, is_valid = lib_auth_now_from_at.connect_with_otp(otp)
3419 if is_valid:
3420 lss = LibSafiaSystem(lib_user_data_internal=lpgss_singleton,
3421 lib_user_data_external=lib_external_info_from_apia_at,
3422 lib_auth_user_otp=lib_auth_now_from_at,
3423 lib_right=lib_right_singleton)
3425 info_setup = lss.setup_user(info_auth['Email_User'], info_auth["Description"], "otp_apia", otp)
3427 response = make_response(redirect('/'))
3428 response.set_cookie("otp", otp, max_age=60*60*24*15) # Le cookie expire après 15 jours
3429 if "id" in info_setup["project"]:
3430 response.set_cookie("project_id", str(info_setup["project"]["id"]), max_age=60*60*24*15)
3431 else :
3432 print(" No project_id => no cookie (when creating project_id maybe) : info_setup : " + str(info_setup))
3433 return response
3434 else :
3435 return make_page_anonymous()
3436 except Exception as e : # pragma no cover except
3437 message = "\n<br>Got exception in " + request.path + " : " + str(e)
3438 print(message)
3439 return make_page_anonymous(message)
3443# TODO VR 13-6-23 : make modal and send query to <a href="{{ url_for('edit_document', id=document.id) }}">Edit</a>
3444# TODO VR : documents iteration to do for pagination !
3445# deprecated replaced by docsmart, to delete 15-8-25
3446#@app.route('/api/v1/document/show_all', methods=['GET'])
3447@app.route('/documents.html', methods=['GET'])
3448def show_all_documents():
3449 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["table_documents", "project_id"])
3450 table_documents = data_needed["table_documents"] if "table_documents" in data_needed else "" # TODO VR 18-6-40 + 83 or waterloo + 208 : we should send page to reconnect or something
3451 project_id = data_needed["project_id"]
3452 project_id = int(request.args.get("project_id", project_id))
3454 # - [ ] TODO VR check right
3455 info_project = lss.get_project_info(project_id)
3456 table_documents = info_project["table_documents"] if "table_documents" in info_project else table_documents
3458 print("Check right")
3460 if table_documents != "":
3462 document_datas = lss.lib_user_data_internal.get_documents(table_documents)
3463 documents = {"data" : document_datas, "iter_pages" : []}
3464 else :
3465 documents = {"data" : [], "iter_pages" :[]}
3466 return make_response(render_template('documents.html', documents=documents, project_id=project_id, title="SAFIA"))
3468@app.route('/api/v1/document/update', methods=['POST'])
3469def update_document():
3470 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["table_documents"])
3471 table_documents = data_needed["table_documents"] if "table_documents" in data_needed else "" # TODO VR 18-6-40 + 83 or waterloo + 208 : we should send page to reconnect or something
3473 if table_documents != "":
3474 document_id = request.json.get('id')
3475 new_values = {
3476 'source': request.json.get('source'),
3477 'source_id': request.json.get('source_id'),
3478 'content': request.json.get('content'),
3479 'document_id': request.json.get('document_id'),
3480 'author': request.json.get('author'),
3481 'url': request.json.get('url'),
3482 }
3484 lss.lib_user_data_internal.update_document(table_documents, document_id, new_values)
3485 return redirect(url_for('show_all_documents'))
3486 else :
3487 return redirect(url_for('show_all_documents'))
3489@app.route('/api/v1/document/delete', methods=['GET'])
3490def delete_document():
3491 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["table_documents", "project_id"])
3492 table_documents = data_needed["table_documents"] if "table_documents" in data_needed else ""
3493 project_id = data_needed["project_id"]
3494 # TODO VR 18-6-40 + 83 or waterloo + 208 : we should send page to reconnect or something
3495 # TODO VR 18-9-23 : we need also the project_id
3497 referer = request.referrer
3499 if table_documents != "":
3500 document_id = request.form.get('id')
3501 document_id = request.args["id"]
3502 project_id_input = int(request.args.get("project_id", 0))
3503 if project_id_input != project_id:
3504 print(" We have a pb houston, we do not delete anything, and should refresh the page and set the cookie plop plop or manage the project_id wisely !")
3506 lss.lib_user_data_internal.delete_document(table_documents, document_id)
3507 if referer == None or "editor" in referer:
3508 return make_response(redirect(url_for('docsmart'))) # show_all_documents
3509 else:
3510 return make_response(redirect(referer))
3511 else:
3512 return make_response(redirect(referer))
3515@app.route('/api/v1/document/rename', methods=['POST'])
3516def rename_document():
3517 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["table_documents", "project_id"])
3518 table_documents = data_needed["table_documents"] if "table_documents" in data_needed else ""
3519 project_id = data_needed["project_id"]
3520 # TODO VR 18-6-40 + 83 or waterloo + 208 : we should send page to reconnect or something
3521 # TODO VR 18-9-23 : we need also the project_id
3523 referer = request.referrer
3525 if table_documents != "":
3526 document_id = request.json["document_id"]
3527 rename = request.json["rename"]
3528 project_id_input = request.json["project_id"]
3529 if project_id_input != project_id:
3530 print(" We have a pb houston, we do not rename anything, and should refresh the page and set the cookie plop plop or manage the project_id wisely !")
3532 lss.lib_user_data_internal.rename_document(table_documents, document_id, rename)
3533 return redirect(referer)
3534 else:
3535 return redirect(referer)
3539# TODO VR : en fait c'est appelé à la génération de l'html
3540# TODO renommer data_needed en contexte
3541@app.route('/api/v1/configuration/update', methods=['POST'])
3542def configuration_option():
3543 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
3544 project_id = data_needed["project_id"]
3546 print(str(request.data))
3547 print(str(request.args))
3548# print(str(request.json))
3550 if user != "anonymous@opio.fr":
3552 list_configuration = [("nlp_chat", "openai_gpt_api", lss.get_user_id(), project_id)]
3553 list_configuration = []
3555 data_input = request.json
3557 for d in data_input:
3558 tuple_config = (d, request.json[d], lss.get_user_id(), project_id)
3559 list_configuration.append(tuple_config)
3560 # TODO VR il faut encapsuler la logique FK dans une classe adaptée
3561 res = lss.lib_user_data_internal.update_configuration(list_configuration)
3563 response = Response(
3564 response=json.dumps(res, indent=4, sort_keys=True, default=str),
3565 #json.dumps(res),
3566 status=200,
3567 mimetype='application/json'
3568 )
3569 return response
3570 else:
3571 return {}
3574# mtruser.user_payment_status
3576import stripe
3577# The library needs to be configured with your account's secret key.
3578# Ensure the key is kept out of any version control system you might be using.
3579#stripe.api_key = "sk_test_NWZuwN1jnWlMbO6dLbKL5ECC"
3580#endpoint_secret = "whsec_TnnQsaKmIKg2evaHoCUyy6IaAcuUffCL"
3582# This is your Stripe CLI webhook secret for testing your endpoint locally.
3583#endpoint_secret = 'whsec_54e4dbe23fb3af46cd455e80f3a130d57d2480bd0826f8db9db8f9e7f14b004d'
3585# And for webhook not locally (may change if website change isn't it ?) => but why don't I find it anymore ?
3586#endpoint_secret = "whsec_TnnQsaKmIKg2evaHoCUyy6IaAcuUffCL"
3588#endpoint_secret = 'whsec_54e4dbe23fb3af46cd455e80f3a130d57d2480bd0826f8db9db8f9e7f14b004d'
3589#app = Flask(__name__)
3591stripe.api_key = lcs.get_stripe_public_key()
3592endpoint_secret = lcs.get_stripe_secret_key()
3594@app.route('/get_test_stripe_webhooks', methods=['GET'])
3595def webhookt(): # pragma no cover payment
3596 print("Received stripe webhook payment !")
3597 print(" stripe.api_key : " + str(stripe.api_key))
3598 print(" endpoint_secret : " + str(endpoint_secret))
3599 return "TEST"
3601@app.route('/stripe_webhooks', methods=['POST'])
3602def webhook(): # pragma no cover payment
3603 print("Received stripe webhook payment !")
3604 print(" stripe.api_key : " + str(stripe.api_key))
3605 print(" endpoint_secret : " + str(endpoint_secret))
3606 event = None
3607 payload = request.data
3608 sig_header = request.headers['STRIPE_SIGNATURE']
3610 try:
3611 event = stripe.Webhook.construct_event(
3612 payload, sig_header, endpoint_secret
3613 )
3614 except ValueError as e:
3615 # Invalid payload
3616 raise e
3617 except stripe.error.SignatureVerificationError as e:
3618 # Invalid signature
3619 raise e
3621 invoice = None
3623 # Handle the event
3626# if event['type'] == 'invoice.created':
3627# invoice = event['data']['object']
3628# elif event['type'] == 'invoice.deleted':
3629# invoice = event['data']['object']
3630# elif event['type'] == 'invoice.finalization_failed':
3631# invoice = event['data']['object']
3632# elif event['type'] == 'invoice.finalized':
3633# invoice = event['data']['object']
3634# elif event['type'] == 'invoice.marked_uncollectible':
3635# invoice = event['data']['object']
3636# elif event['type'] == 'invoice.paid':
3637# invoice = event['data']['object']
3638# elif event['type'] == 'invoice.payment_action_required':
3639# invoice = event['data']['object']
3640# elif event['type'] == 'invoice.payment_failed':
3641# invoice = event['data']['object']
3643 invoice = None
3644 payment_intent = None
3645 payment_link = None
3646 checkout = None
3647 if event['type'] == 'invoice.payment_succeeded':
3648 invoice = event['data']['object']
3649 elif event['type'] == 'invoice.sent':
3650 invoice = event['data']['object']
3651 elif event['type'] == 'invoice.upcoming':
3652 invoice = event['data']['object']
3653 elif event['type'] == 'invoice.updated':
3654 invoice = event['data']['object']
3655 elif event['type'] == 'invoice.voided':
3656 invoice = event['data']['object']
3657 elif event['type'] == 'payment_intent.amount_capturable_updated':
3658 payment_intent = event['data']['object']
3659 elif event['type'] == 'payment_intent.canceled':
3660 payment_intent = event['data']['object']
3661 elif event['type'] == 'payment_intent.created':
3662 payment_intent = event['data']['object']
3663 elif event['type'] == 'payment_intent.partially_funded':
3664 payment_intent = event['data']['object']
3665 elif event['type'] == 'payment_intent.payment_failed':
3666 payment_intent = event['data']['object']
3667 elif event['type'] == 'payment_intent.processing':
3668 payment_intent = event['data']['object']
3669 elif event['type'] == 'payment_intent.requires_action':
3670 payment_intent = event['data']['object']
3671 elif event['type'] == 'payment_intent.succeeded':
3672 payment_intent = event['data']['object']
3673 elif event['type'] == 'payment_link.created':
3674 payment_link = event['data']['object']
3675 print(" payment_link.created : " + str(event))
3676 elif event['type'] == 'payment_link.updated':
3677 payment_link = event['data']['object']
3678 # ... handle other event types
3679 print(" payment_link.updated : " + str(event))
3680 elif event['type'] == "checkout.session.completed":
3681 checkout = event['data']['object']
3682 else:
3683 print('Unhandled event type {}'.format(event['type']))
3685 if invoice != None:
3686 amount = invoice["amount_paid"] / 100
3687 user = invoice["customer_email"]
3688 print(" amount : " + str(amount))
3689 print(" user : " + str(user))
3691 info_user = lpgss_singleton.get_user_from_mail(user.lower())
3692 if info_user != None and "id" in info_user and info_user["id"].isnumerical():
3693 user_id = info_user["id"]
3694 if event['type'] == 'invoice.payment_succeeded':
3695 new_val = lpgss_singleton.topup_money(user_id, int(amount))
3696 print("new_val : " + str(new_val))
3697 else :
3698 print("PROBLEM ENREGISTREMENT MONEY, who has pay : " + str(user) + " INTERNAL error, please send an email !")
3700 return jsonify(success=True)
3703@app.route('/vic') # pragma no cover dev
3704def vic():
3705 return render_template_string("<a href=/otp/0a76f14b131682eaa36fbef63d725f9352cfb85d >Vic Connect</a>")
3709# TODO VR refacto as datou with cost management
3710@app.route('/git') # pragma no cover scale
3711def git_import():
3712 start_time = time.time()
3713 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id", "otp"])
3714 project_id = data_needed["project_id"]
3716 repo = request.args.get("repo")
3717 limit_str = request.args.get("limit")
3718 last_nb_days_str = request.args.get("last_nb_days")
3719 with_comment = bool(request.args.get("with_comment"))
3720 verbose = bool(request.args.get("verbose"))
3721 load_all = bool(request.args.get("load_all", False))
3722 only_recent_update = bool(request.args.get("only_recent_update", False))
3724 list_repos = [repo]
3725# load_all for closed
3726 list_excluded_labels = []
3728 query_id = insert_query_log(lss, request.path, request, lss.get_user_id(), project_id, time.time() - start_time) # "/api/v1/safia/query"
3730 complete_param_json = get_datou_exec_context_as_complete_param_json(user, verbose, privacy = False,
3731 openai_token=OPENAI_API_KEY, lss=lss,
3732 project_id=project_id)
3734 from lib.datou.datou_exec import datou_exec
3736 github_token = complete_param_json["github_token"]
3738 limit_str = request.args.get("limit")
3739 if limit_str:
3740 limit = int(limit_str)
3741 else :
3742 limit = 0 # pragma no cover scale
3744 if with_comment:
3745 with_comment = True
3746 if last_nb_days_str:
3747 last_nb_days = int(last_nb_days_str)
3748 else :
3749 last_nb_days = None # pragma no cover scale
3751 print("About to import repo : " + str(repo))
3752 from lib.lib_github import git_issue_to_retrieval_json
3753 json_path, recently_updated_name = git_issue_to_retrieval_json(list_repos, list_excluded_labels, github_token,
3754# list_required_labels = color_labels,
3755 verbose = verbose,
3756# TODO VR entre ces trois paramètres, il y a la meme chose !
3757 with_comment = with_comment,
3758# with_closed_issue = with_closed_issue,
3759 out_path = "temp/import_git.json",
3760 check_issue_re = True,
3761 load_all = load_all,
3762 last_number_days = last_nb_days,
3763 limit = limit,
3764 only_recent_update = only_recent_update)
3766 input = {"json_path" : json_path}
3767 list_datou_steps = ["import_json"]
3768 output, audit_json = datou_exec(list_datou_steps, input, complete_param_json, verbose=verbose)
3770 cost = 0
3771 if "cost" in output:
3772 cost = output["cost"]
3773 lss.lib_user_data_internal.add_cost_to_project(project_id, cost)
3775 elapsed_time = time.time() - start_time
3776 lss.update_query_end(query_id, response_status = 200, response_body = output,
3777 elapsed_time = elapsed_time, cost = cost, detailed_time = {"list" : []})
3779 output["recently_updated_name"] = recently_updated_name
3781 return output
3783@app.route('/resume_all_one_project')
3784def resume_all_one_project():
3785 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id", "otp"])
3786 project_id = data_needed["project_id"]
3788 nb_cluster = int(request.args.get("nb_cluster", 5))
3790#limit = 10000,
3791# nb_cluster_input = 10,
3792# metric = "euclidean",
3793# verbose = True,
3794# strat = "default"
3796 audit, list_inputs, list_datou_step = lss.resume_all_project(project_id, nb_cluster_input = nb_cluster)
3798 if True:
3799 verbose = True
3801 complete_param_json = get_datou_exec_context_as_complete_param_json(user, verbose, privacy = False,
3802 openai_token=OPENAI_API_KEY, lss=lss,
3803 project_id=project_id)
3805# complete_param_json["preprompt"] = preprompt
3807 from lib.datou.datou_exec import datou_exec
3809 # chat gpt accepte moins de token que api gpt-4, du coup je fais ce hack sans etre sur de la bonne manière de faire du map_reduce (sachant que le map je le fais à la main de toute facon car je veux faire des choses plus smart avec les kmeans a un moment donné)
3811 to_fusion_then = []
3812 for input in list_inputs:
3813 output, audit_json = datou_exec(list_datou_step, input, complete_param_json)
3815 if "cost" in output:
3816 lss.lib_user_data_internal.add_cost_to_project(project_id, output["cost"])
3818 to_fusion_then.append(output)
3821 complete_resume = "WIP please wait !"
3822# return complete_resume + str(to_fusion_then)
3824 response = Response(
3825 response=json.dumps(to_fusion_then, indent=4, sort_keys=True, default=str),
3826 #json.dumps(res),
3827 status=200,
3828 mimetype='application/json'
3829 )
3830 return response
3834@app.route('/docsmart')
3835def docsmart():
3836 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id", "otp"])
3837 project_id = data_needed["project_id"]
3838 project_id = request.args.get("project_id", project_id)
3839 limit = int(request.args.get("limit", 10000))
3840 offset = int(request.args.get("offset", 0))
3841 order_by_recent = bool(request.args.get("order_by_recent", True))
3843 # VR 22-11-23 : implémentation sauvage des droits
3844 has_access = lss.lib_right.get_role_on_project(lss.get_user_id(), project_id)
3846 if has_access:
3847 print("has_access")
3848 list_documents, info_project = lss.stat_all_one_project(project_id, limit=limit, offset=offset, order_by_recent = order_by_recent)
3849 else:
3850 list_documents = []
3851 info_project = {"access" : False} # TODO VR 22-11-23 : à gérer dans le flask
3853 info_for_pagination = {"info_project" : info_project, "limit" : limit}
3854 number_documents = info_for_pagination["info_project"]["number_documents"] if "info_project" in info_for_pagination and "number_documents" in info_for_pagination["info_project"] else 0
3855 if limit < number_documents:
3856 info_for_pagination["nb_doc_displayed"] = limit # if "nb_doc_displayed" in info_for_pagination else 0
3857 info_for_pagination["nb_doc_total"] = number_documents # if "nb_doc_total" in info_for_pagination else 0
3858 else:
3859 info_for_pagination["nb_doc_displayed"] = number_documents # if "nb_doc_displayed" in info_for_pagination else 0
3860 info_for_pagination["nb_doc_total"] = number_documents # if "nb_doc_total" in info_for_pagination else 0
3864 response = make_response(render_template('docsmart.html', documents=list_documents,
3865 project_id=project_id,
3866 title="SAFIA Smart Documents For Project " + str(project_id),
3867 pagination=info_for_pagination))
3869 return response
3872from server.bp.project import create_project_bp
3873project_bp = create_project_bp()
3874app.register_blueprint(project_bp)
3877@app.route('/admin/typemb')
3878def admin_typemb(): # pragma no cover debug
3879 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id", "otp"])
3880 project_id = data_needed["project_id"]
3881 project_id = int(request.args.get("project_id", project_id))
3883 info_project = lpgss_singleton.get_project_info(project_id)
3884 table_documents = info_project["table_documents"]
3886 one_desc = lpgss_singleton.get_descs(table_documents, 1)
3888 if len(one_desc) > 0 and "embedding" in one_desc[0]:
3889 typee = str(type(one_desc[0]["embedding"]))
3890# response = make_response(render_template_string(typee))
3892 response = Response(
3893 response=json.dumps({"type" : typee}, indent=4, sort_keys=True, default=str),
3894 #json.dumps(res),
3895 status=200,
3896 mimetype='application/json'
3897 )
3898# return response
3899 return response
3900 else :
3901 return "None"
3905@app.route('/admin/emb')
3906def admin_emb(): # pragma no cover debug
3907 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id", "otp"])
3908 project_id = data_needed["project_id"]
3909 project_id = int(request.args.get("project_id", project_id))
3910 document_id = request.args.get("document_id", None)
3912 info_project = lpgss_singleton.get_project_info(project_id)
3913 table_documents = info_project["table_documents"]
3915 one_desc = lpgss_singleton.get_descs(table_documents, document_id, 1)
3917 response = Response(
3918 response=json.dumps(one_desc, indent=4, sort_keys=True, default=str),
3919 #json.dumps(res),
3920 status=200,
3921 mimetype='application/json'
3922 )
3923 return response
3925print("We should build optionnaly (like for fotonower apy only what is needed)")
3927from server.bp.fvs import create_fvs_bp
3928fvs_bp = create_fvs_bp()
3929app.register_blueprint(fvs_bp)
3931from server.bp.saxia import create_saxia_bo_bp, create_saxia_interface_bp
3932sabo = create_saxia_bo_bp()
3933app.register_blueprint(sabo)
3935sai = create_saxia_interface_bp()
3936app.register_blueprint(sai)
3938from server.bp.uld import create_uld_bp
3939bp_uld = create_uld_bp()
3940app.register_blueprint(bp_uld)
3942from server.bp.dev_api import create_dev_api_bp
3943bp_dev_api = create_dev_api_bp()
3944app.register_blueprint(bp_dev_api)
3946from server.bp.admin import create_admin_bp
3947bp_admin_api = create_admin_bp()
3948app.register_blueprint(bp_admin_api)
3952@app.route('/admin/kmean')
3953def admin_kmean():
3954 print(" PROTECT ")
3955 ret = fvs_interface.get_admin_data()
3956 rets = json.dumps(ret, default=str)
3957 retj = json.loads(rets)
3958 return retj
3961@app.route('/api/v1/kmean/kill')
3962def kill_fvs_kmean():
3963 port = int(request.args.get("port", 6666))
3965 if port == 6666:
3966 port = None
3967 port = request.args.get("port")
3969 ret = fvs_interface.kill(port)
3970 return str(ret)
3972# VR 23-4-24 Ceci n'est pas testé, et sans doute deprecated, quels sont les impacts à le déplacer dans un bp ou le supprimer ?
3973@app.route('/api/v1/kmean/query/ok')
3974def query_kmean_ok():
3975 return "ok"
3978@app.route('/api/v1/kmean/query')
3979def query_kmean():
3980 print(" This function is used only when dataset already exists ")
3981 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id"])
3982 project_id = data_needed["project_id"]
3983 start_time = time.time()
3984 query_id = insert_query_log(lss, request.path, request, lss.get_user_id(), project_id, start_time)
3986 nb_new_cluster = int(request.args.get("nb_new_cluster", 0))
3987 port = int(request.args.get("port", 6666))
3988 nb_step = int(request.args.get("nb_step", 10))
3989 save_cluster = request.args.get("save_cluster", '')
3990 load_cluster = request.args.get("load_cluster", '')
3991 verbose = bool(int(request.args.get("verbose", '0')))
3993 xmin = request.args.get("xmin")
3994 xmax = request.args.get("xmax")
3995 ymin = request.args.get("ymin")
3996 ymax = request.args.get("ymax")
3997 with_indices = bool(int(request.args.get("verbose", '1')))
3999# from lib.lib_ml.lib_kmean_pyfvs import query_and_output_of_kmeans
4000 from pyfvs.lib.advanced.lib_split_elbow import query_and_output_of_kmeans
4001 filename_gif = query_and_output_of_kmeans(fvs_interface, port, nb_step,
4002 save_cluster, load_cluster, nb_new_cluster, verbose,
4003 xmin = xmin, xmax=xmax, ymin = ymin, ymax = ymax,
4004 with_indices=with_indices)
4006 elapsed_time = time.time() - start_time
4007 lss.update_query_end(query_id, response_status=400, response_body="", elapsed_time=elapsed_time,
4008 cost=0, detailed_time={"filename_gif": filename_gif, "args": request.args,
4009 # "inputs": inputs,
4010 "total_inertia": "total_inertia"})
4012 if os.path.exists(filename_gif):
4013 return send_from_directory(os.path.dirname(filename_gif),
4014 os.path.basename(filename_gif), mimetype='image/gif')
4015 else:
4016 return filename_gif + " doesn't exists !"
4017# return filename_gif
4021@app.route('/kmean')
4022def kmean():
4023 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id", "otp"])
4024 project_id = data_needed["project_id"]
4026 start_time = time.time()
4027 elapsed_time = time.time() - start_time
4028 query_id = insert_query_log(lss, request.path, request, lss.get_user_id(), project_id, elapsed_time)
4030 limit = int(request.args.get("limit", 0))
4031 nb_project = int(request.args.get("nb_project", 0))
4032 list_project_ids_csv = request.args.get("pids", "")
4033 format = request.args.get("format", "png")
4034 metric = request.args.get("metric", "")
4035 dim = int(request.args.get("dim", 3))
4036 nb_cluster = int(request.args.get("nb_cluster", 10))
4037 gpu_enable = bool(int(request.args.get("gpu", '0')))
4038 nb_step = int(request.args.get("nb_step", 10))
4039 pdt = int(request.args.get("pdt", 1888))
4040 build_fvs_desc = bool(int(request.args.get("build_fvs_desc", '1')))
4041 get_list_doc = bool(int(request.args.get("get_list_doc", '0')))
4042 keep_fvs_running = bool(int(request.args.get("keep_fvs_running", '0')))
4043 keep_image = bool(int(request.args.get("keep_image", '0'))) # en fait c'est fait par défaut mais il faudrait les nettoyer
4044 port = request.args.get("port", None)
4045 save_cluster = request.args.get("save_cluster", '')
4046 load_cluster = request.args.get("load_cluster", '')
4047 launch_fvs = bool(int(request.args.get("launch_fvs", '1')))
4048 verbose = bool(int(request.args.get("verbose", '0')))
4049 # if pdt is set we search the folder, of course !
4051 inputs = {
4052 "limit": limit,
4053 "nb_project": nb_project,
4054 "pids": list_project_ids_csv,
4055 "format": format,
4056 "metric": metric,
4057 "dim": dim,
4058 "nb_cluster": nb_cluster,
4059 "gpu_enable": gpu_enable,
4060 "nb_step": nb_step,
4061 "pdt": pdt,
4062 "build_fvs_desc": build_fvs_desc,
4063 "get_list_doc": get_list_doc,
4064 "keep_fvs_running": keep_fvs_running,
4065 "port" : port,
4066 "save_cluster" : save_cluster,
4067 "launch_fvs" : launch_fvs,
4068 "verbose" : verbose
4069 }
4070 if list_project_ids_csv == "":
4071 list_projects = lss.get_project(user)
4073 list_project_ids = list(map(lambda x : x["id"], list_projects))
4074 else :
4075 list_project_ids = list(map(int, list_project_ids_csv.split(",")))
4077 filename = "static/temp/img." + format
4079 filename, list_legend_centroids, list_url_centroids, list_example_media, misc_info = lss.kmean_2d(list_project_ids, limit=limit,
4080 nb_project=nb_project, filename = filename,
4081 metric = metric, dim = dim,
4082 nb_cluster_input = nb_cluster,
4083 gpu_enable = gpu_enable,
4084 nb_step = nb_step,
4085 keep_fvs_running = keep_fvs_running,
4086 port = port,
4087 pdt = pdt,
4088 build_fvs_desc = build_fvs_desc,
4089 save_cluster = save_cluster,
4090 load_cluster = load_cluster,
4091 launch_fvs = launch_fvs,
4092 verbose = verbose,
4093 fvs_interface = fvs_interface)
4095 list_clusters_from_example_media = [x["pdt_dyn"] for x in list_example_media]
4097 mat_confusion = fvs_interface.make_confusion_matrix(list_clusters=list_clusters_from_example_media)
4099 try:
4100 from lib.lib_util import display_confusion_matrix
4101 ma_matrice = display_confusion_matrix(mat_confusion)
4102 except Exception as e:
4103 ma_matrice = mat_confusion.astype(str).values
4105 misc_info_str = misc_info["str"] if "str" in misc_info else None
4107 misc_info_str += str(mat_confusion)
4109 # list_nn_centroids
4111 if filename == None :
4112 return "No data available, please choose one metric among euclidean, KL, IS, EX, LX"
4113 else :
4114# return send_from_directory(os.path.dirname(filename), os.path.basename(filename))
4116 if len(list_url_centroids) != len(list_legend_centroids):
4117 return "Unexpected size different"
4118 list_legend_urls = []
4119 import numpy as np
4120 total_inertia = np.sum(list(map(lambda x : x["inertia"], list_example_media)))
4121 for i in range(len(list_url_centroids)):
4122 inertia = list_example_media[i]["inertia"] if i < len(list_example_media) else -1666
4123 pdt_dyn = list_example_media[i]["pdt_dyn"] if i < len(list_example_media) else -1777
4124 print(" - [ ] TODO VR : when reloading there is a mismatch and data are missing, but don't know why and how")
4125 list_legend_urls.append({"url" : list_url_centroids[i], "legend" : list_legend_centroids[i] + ", inertia : " + str(inertia) + ", pdt_dyn : " + str(pdt_dyn)})
4127 elapsed_time = time.time() - start_time
4128 lss.update_query_end(query_id, response_status = 400, response_body = list_legend_urls, elapsed_time = elapsed_time,
4129 cost = 0, detailed_time = {"filename" : filename, "args" : request.args, "inputs" : inputs, "total_inertia" : total_inertia})
4131 return render_template('kmean.html', image_path=filename,
4132 links=list_legend_urls, inputs=inputs,
4133 total_inertia=total_inertia,
4134 misc_info_str=misc_info_str,
4135 all_misc_info_str=str(misc_info),
4136 matrice=ma_matrice)
4138@app.route('/favicon.ico')
4139def favicon():
4140 return send_from_directory(os.path.join(app.root_path, 'static'),
4141 'favicon.ico', mimetype='image/vnd.microsoft.icon')
4143@app.route('/live')
4144def live():
4145 return render_template('live.html')
4147@app.route('/admin/live')
4148def admin_live():
4149 from lib.lib_live.lib_wsns import list_chunk_wav
4150 global list_chunk_wav
4151 ret = str(list_chunk_wav.keys())
4153 from lib.lib_live.lib_wsns import get_info_thread
4154 ret += str(get_info_thread())
4156 return ret
4158import asyncio
4160@app.route('/api/v1/live/ws/create')
4161def api_v1_live_ws_create():
4162 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["otp"])
4163 otp = data_needed["otp"]
4164 ret = ws_all_singleton.create_web_socket(lss = lss, request_host=request.host, otp = otp)
4165 response = Response(
4166 response=json.dumps(ret, indent=4, sort_keys=True, default=str),
4167 # json.dumps(res),
4168 status=200,
4169 mimetype='application/json'
4170 )
4171 return response
4173@app.route('/api/v1/live/ws/all')
4174def api_v1_live_ws_all():
4175 user, data_needed, lss = register_user_get_data(request, list_data_needed = [])
4177 # - [ ] TODO take care to the right
4179 ret = ws_all_singleton.list_all()
4180 response = Response(
4181 response=json.dumps(ret, indent=4, sort_keys=True, default=str),
4182 # json.dumps(res),
4183 status=200,
4184 mimetype='application/json'
4185 )
4186 return response
4188@app.route('/api/v1/live/ws/stop')
4189def api_v1_live_ws_stop():
4190 user, data_needed, lss = register_user_get_data(request, list_data_needed = [])
4191 user_id = lss.get_user_id()
4192 port = int(request.args["port"])
4193 ret = ws_all_singleton.stop_web_socket(port, user_id)
4194# ret = ws_all_singleton.list_all()
4195 response = Response(
4196 response=json.dumps(ret, indent=4, sort_keys=True, default=str),
4197 # json.dumps(res),
4198 status=200,
4199 mimetype='application/json'
4200 )
4201 return response
4203@app.route('/api/v1/live/ws/kill_all')
4204def api_v1_live_ws_kill_all():
4205 user, data_needed, lss = register_user_get_data(request, list_data_needed = [])
4206 ret = ws_all_singleton.kill_all()
4207# ret = ws_all_singleton.list_all()
4208 response = Response(
4209 response=json.dumps(ret, indent=4, sort_keys=True, default=str),
4210 # json.dumps(res),
4211 status=200,
4212 mimetype='application/json'
4213 )
4214 return response
4216@app.route('/ws/launch')
4217def ws_launch():
4218 user, data_needed, lss = register_user_get_data(request, list_data_needed = [])
4219 from lib.lib_live.lib_wsns import launch_ws
4220 import asyncio
4221 loop = asyncio.new_event_loop()
4222 asyncio.set_event_loop(loop)
4223 asyncio.run(launch_ws(lss=lss))
4225 # not accessed for now !
4227@app.route('/anon', methods=['GET', 'POST'])
4228def anon():
4229 if request.method == 'POST':
4230 # Ici, nous traiterions l'upload de fichier ou la sélection d'un fichier => non j'envoie sur api/v1/uploader
4231 pass
4233 # Simulons une liste de fichiers existants
4234 existing_files = ['document1.pdf', 'document2.pdf']
4235 from lib.lib_util import list_file_anon
4236 folder= request.args.get("folder", "/home/safia/workarea/git/Safia/prompt/python/doc_confi_medecin/test_ocr_extraction_expert/130124")
4237 existing_files = list_file_anon(folder)
4239 existing_files_for_html = json.dumps(existing_files, indent=4, sort_keys=True, default=str)
4240 list_files = existing_files.keys()
4242 return render_template('expert_anon.html', existing_files=list_files, data_anon_strat=existing_files_for_html, title="Anonymisation de Document")
4245@app.route('/uld', methods=['GET', 'POST'])
4246def uld():
4247 if request.method == 'POST':
4248 # Ici, nous traiterions l'upload de fichier ou la sélection d'un fichier => non j'envoie sur api/v1/uploader
4249 pass
4251 return render_template('uld/pres.html')
4253@app.route('/zeitgeist.png')
4254def zeitgeist():
4255 return send_from_directory(os.path.join(app.root_path, 'static'),
4256 'zeitgeist.png', mimetype='image/png')
4258def has_no_empty_params(rule):
4259 defaults = rule.defaults if rule.defaults is not None else ()
4260 arguments = rule.arguments if rule.arguments is not None else ()
4261 return len(defaults) >= len(arguments)
4263@app.route("/site-map")
4264def site_map():
4265 links = []
4266 for rule in app.url_map.iter_rules():
4267 # Filter out rules we can't navigate to in a browser
4268 # and rules that require parameters
4269 if "GET" in rule.methods and has_no_empty_params(rule):
4270 url = url_for(rule.endpoint, **(rule.defaults or {}))
4271 links.append((url, rule.endpoint))
4272 # links is now a list of url, endpoint tuples
4274 print (str(links))
4276 response = Response(
4277 response=json.dumps({"links" : links}, indent=4, sort_keys=True, default=str),
4278 #json.dumps(res),
4279 status=200,
4280 mimetype='application/json'
4281 )
4282 return response
4286@app.route('/static/factures/<path:path>')
4287def facture_path(path):
4288 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id", "otp"])
4290 print(" user : " + str(user))
4291 print(" path : " + str(path))
4293 print("TODO check right")
4295 has_access = lss.has_access_to_this_facture("/static/factures", path)
4296 if not has_access:
4297 is_admin = lss.lib_right.is_user_admin_config(lss.get_user_id())
4298 if is_admin:
4299 has_access = True
4301 if has_access:
4303# complete_dir = os.path.join(app.root_path, 'static/factures', os.path.dirname(path))
4304# basename = os.path.basename(path)
4305# return send_from_directory(complete_dir, basename, mimetype='file/pdf')
4306 return send_from_directory(os.path.join(app.root_path, 'static/factures'),
4307 path, mimetype='file/pdf')
4308 else:
4309 return ("Something went wrong, please login or contact doc@saxia.app if you believe you have access to this bill !")
4311# VR 5/11/25 : on dirait que ces trucs sont inutiles !
4312# /static/js
4313# /static/css
4314# /static/uld
4315# /static/image_static
4317# VR 5/11/25 : mais ceux-la le sont
4318@app.route('/public/<path:path>')
4319def public_custom_static_image_static(path):
4320 return send_from_directory('static', path)
4323@app.route('/public/<a>/<path:path>')
4324def static_1(path, a):
4325 return send_from_directory(os.path.join('static', a), path)
4327@app.route('/public/<a>/<b>/<path:path>')
4328def static_2(path, a, b):
4329 return send_from_directory(os.path.join('static', a, b), path)
4331@app.route('/public/<a>/<b>/<c>/<path:path>')
4332def static_3(path, a, b, c):
4333 return send_from_directory(os.path.join('static', a, b, c), path)
4335@app.route('/public/<a>/<b>/<c>/<d>/<path:path>')
4336def static_4(path, a, b, c, d):
4337 return send_from_directory(os.path.join('static', a, b, c, d), path)
4341result_tag = Tag(name="saxia_result", description="Saxia Result")
4343#@app.errorhandler(401)
4344#@app.route('/api/v1/uploader', methods = ['GET', 'POST'])
4345#@app.get(rule = '/static/<path:path>', tags = [result_tag])
4346@app.route('/static/<path:path>')
4347def generic_static_path(path):
4348 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id", "otp"])
4350 print(" user : " + str(user))
4351 print(" path : " + str(path))
4353 print("TODO check right like facture")
4355 # Authorize if path doesn't finished with docx
4356 authorize = False
4357 if not path.lower().endswith(".docx"):
4358 authorize = True
4359 else:
4360 has_access = lss.has_access_to_this_document(path)
4361 if has_access:
4362 authorize = True
4363 else:
4364 is_admin = lss.lib_right.is_user_admin_config(lss.get_user_id())
4365 if is_admin:
4366 authorize = True
4368 # if authorize :
4369 # serve file if exists
4370 if True:
4371 return send_from_directory(os.path.join(app.root_path, 'static'), path)
4372 else:
4373 print("Access denied : Something went wrong, please contact contact@safia.app if you believe you have access to this document !" + str(path) + " and user " + str(user))
4374 return ("Access denied : Something went wrong, please contact contact@safia.app if you believe you have access to this document !")
4376@app.route('/<path:path>')
4377def generic_path(path):
4378 user, data_needed, lss = register_user_get_data(request, list_data_needed = ["project_id", "otp"])
4380 print(" user : " + str(user))
4381 print(" path : " + str(path))
4383 print("TODO check right")
4385 project = lss.get_project_by_shortname(path)
4387 print("project : " + str(project))
4388 if len(project) == 1:
4389 print("On est bon")
4390 project_id = project[0]["id"]
4392# response =
4393 # (redirect('/?project_id=' + str(project_id)))
4394# response.set_cookie("project_id", str(project_id), max_age=60*60*24*15) # Le cookie expire après 15 jours
4395# return response
4396 return index(project_id)
4397 else :
4398 print("- [ ] TODO manage these case !")
4400 return "GENERIC PATH"
4402from flask import send_file
4404@app.route('/download',methods=["GET","POST"])
4405def downloadFile(): #In your case fname is your filename
4406 path = request.args.get("path")
4407 try:
4408 # ,mimetype='text/csv'
4409 # , attachment_filename=os.path.basename(path)
4410 return send_file(path, as_attachment=True)
4411 except Exception as e:
4412 return str(e)
4415base = "https://app2.ultradocia.fr"
4417@app.route('/.well-known/openid-configuration',methods=["GET","POST"])
4418def well_known_oid_config(): #In your case fname is your filename
4419 # inspired from https://accounts.google.com/.well-known/openid-configuration
4420 base = request.host_url
4421 data_dict = {
4422 "issuer": base,
4423 "authorization_endpoint": base + "/oauth/authorize",
4424 "token_endpoint": base + "/oauth/token",
4425 "revocation_endpoint": base + "/oauth/revoke",
4426 "response_types_supported": [
4427 "code",
4428 "token",
4429 "id_token",
4430 "code token",
4431 "code id_token",
4432 "token id_token",
4433 "code token id_token",
4434 "none"
4435 ],
4436 "response_modes_supported": [
4437 "query",
4438 "fragment",
4439 "form_post"
4440 ],
4441 "subject_types_supported": [
4442 "public"
4443 ],
4444 "id_token_signing_alg_values_supported": [
4445 "RS256"
4446 ],
4447 "scopes_supported": [
4448 "openid",
4449 "email",
4450 "profile"
4451 ],
4452 "token_endpoint_auth_methods_supported": [
4453 "client_secret_post",
4454 "client_secret_basic"
4455 ],
4456 "claims_supported": [
4457 "aud",
4458 "email",
4459 "email_verified",
4460 "exp",
4461 "family_name",
4462 "given_name",
4463 "iat",
4464 "iss",
4465 "name",
4466 "picture",
4467 "sub"
4468 ],
4469 "code_challenge_methods_supported": [
4470 "plain",
4471 "S256"
4472 ],
4473 "grant_types_supported": [
4474 "authorization_code",
4475 "refresh_token",
4476 "urn:ietf:params:oauth:grant-type:device_code",
4477 "urn:ietf:params:oauth:grant-type:jwt-bearer"
4478 ]
4479}
4480 return jsonify(data_dict)
4483def main(): # pragma no cover prod
4484 print(app.blueprints.keys())
4485 with app.app_context():
4486 if "login" in app.blueprints:
4487 app.blueprints["login"].db.create_all() # Crée des tables si elles n'existent pas déjà => pour test creation compte
4489 # app.run(debug=True, host="0.0.0.0")
4490 # app.run()
4492# - [ ] TODO VR 22/8/23 a mettre dans credentials
4493 if os.path.exists("/home/safia/.ssl/safia_rubbia_fr_by_gandi/safia.rubbia.fr.crt"):
4494# tuple_ssl_context=("/home/safia/.ssl/auto_signe/cert.pem", "/home/safia/.ssl/auto_signe/key.pem")
4495 tuple_ssl_context=("/home/safia/.ssl/safia_rubbia_fr_by_gandi/safia.rubbia.fr.crt", "/home/safia/.ssl/safia_rubbia_fr_by_gandi/safia.rubbia.fr.key")
4497 app.run(ssl_context=tuple_ssl_context, port=4999, host="0.0.0.0")
4498# app.run(port=4999)
4499 elif os.path.exists("/home/safia/.ssl/auto_signe/cert.pem"): # jl conf 19-6-23
4500 tuple_ssl_context=("/home/safia/.ssl/auto_signe/cert.pem", "/home/safia/.ssl/auto_signe/key.pem")
4501 # 'adhoc' instead of tuple_ssl_context for self-signed cert
4502 app.run(debug=False, ssl_context='adhoc', port=4998, host="0.0.0.0")
4503# app.run(port=4997, host="0.0.0.0")
4504# app.run(port=4997, host="0.0.0.0")
4505# until we have parameters and so on to choose this
4506# elif os.path.exists("/Users/moilerat/Documents/Fotonower/Safia/prompt/doc/conf_install/autosigne/cert.pem"): # jl conf 19-6-23
4507# tuple_ssl_context=("/Users/moilerat/Documents/Fotonower/Safia/prompt/doc/conf_install/autosigne/cert.pem", "/Users/moilerat/Documents/Fotonower/Safia/prompt/doc/conf_install/autosigne/key.pem")
4509# app.run(ssl_context=tuple_ssl_context, port=5000, host="0.0.0.0")
4510 else :
4511# UNCOMMENT LOCALHOST WSS
4512# cert_args = "/Users/moilerat/Documents/Fotonower/Safia/prompt/doc/conf_install/autosigne/cert.pem"
4513# key_args = "/Users/moilerat/Documents/Fotonower/Safia/prompt/doc/conf_install/autosigne/key.pem"
4514# tuple_ssl_context = (cert_args, key_args)
4515 app.run(debug=False, host="0.0.0.0", port=4999) #, ssl_context=tuple_ssl_context)
4517if __name__ == '__main__': # pragma no cover prod
4518 main()
4519# import_all_for_coverage_with_zero_percent()