Coverage for lib/import_util/lib_path_to_vec.py: 83%
203 statements
« prev ^ index » next coverage.py v7.9.1, created at 2026-02-10 01:10 +0100
« prev ^ index » next coverage.py v7.9.1, created at 2026-02-10 01:10 +0100
1# Sert à créer des embeddings à partir de dossier
5# Peux-tu me faire un script en python qui à partir d'un dossier va lister récursivement
6# les fichiers présent en gardant leur taille et date de création tout en créant des indexs
7# des tailles de chaque dossier et sous-dossier ?
11import os
12import time
14import os
15import time
17# BUG TODO, ca ne gère pas les noms de fichiers avec des . !
18def list_files(startpath, verbose = False,
19 managed_zip_extension_to_avoid = [".zip", ".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tbz"]):
20 index = {}
21 file_list = []
22 if os.path.isfile(startpath):
23 file_list.append({"path" : startpath, "size" : os.path.getsize(startpath), "extension": os.path.splitext(startpath)[1]})
24 index[startpath] = {"folder_size" : os.path.getsize(startpath), "nb_element" : 1}
25 for root, dirs, files in os.walk(startpath):
26 level = root.replace(startpath, '').count(os.sep)
27 indent = ' ' * 4 * (level)
28 if verbose:
29 print('{}{}/'.format(indent, os.path.basename(root)))
30 subindent = ' ' * 4 * (level + 1)
31 folder_size = 0
32 nb_element = 0
33 for f in files:
34 file_path = os.path.join(root, f)
35 file_size = os.path.getsize(file_path)
36 folder_size += file_size
37 nb_element += 1
38 creation_time = time.ctime(os.path.getctime(file_path))
39 if verbose:
40 print('{}{} - Size: {} bytes - Created at: {}'.format(subindent, f, file_size, creation_time))
41 tentative_extension = os.path.splitext(f)[1]
42 if ("." + tentative_extension) not in managed_zip_extension_to_avoid:
43 file_list.append({
44 'path': file_path,
45 'size': file_size,
46 'extension': os.path.splitext(f)[1]
47 })
48 if verbose:
49 print('{}Total folder size: {} bytes, : {} files'.format(subindent, folder_size, nb_element))
50 index[root] = {"folder_size" : folder_size, "nb_element" : nb_element}
51 return index, file_list
53# Utilisation de la fonction
54# index, file_list = list_files('/path/to/your/directory')
55# print(index)
56# print(file_list)
58import sys
59from lib.lib_util import count_and_display_elapsed_time
61def unzip_files(list_zip_files, managed_zip_extension = [".zip", ".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tbz"]):
62 list_folder_temp = []
64 import os
65 import shutil
67 for l in list_zip_files:
68 filepath = l["path"]
69 extension = l["extension"].lower()
70 try :
71 if extension == ".zip":
72 shutil.unpack_archive(filepath, os.path.dirname(filepath), 'zip')
73 elif extension == ".tar":
74 shutil.unpack_archive(filepath, os.path.dirname(filepath), 'tar')
75 elif extension in [".tar.gz", ".tgz"]:
76 shutil.unpack_archive(filepath, os.path.dirname(filepath), 'gztar')
77 elif extension in [".tar.bz2", ".tbz"]:
78 shutil.unpack_archive(filepath, os.path.dirname(filepath), 'bztar')
79 else:
80 print(f"File extension not supported: {filepath}")
81 except Exception as e:
82 print(str(e))
83 print("- [ ] TODO VR 14-9-23 We don't want to override the file for now, maybe we will miss some")
87def load_files(list_files, managed_extension, verbose = False):
88 import os
89 count_time = end_time = time.time()
91 list_detailed_time = []
92 hostname = os.environ["HOSTNAME"] if "HOSTNAME" in os.environ else os.uname().nodename
93#import uuid
95#def get_unique_id():
96# return ':'.join(['{:02x}'.format((uuid.getnode() >> i) & 0xff) for i in range(0,8*6,8)][::-1])
98# pour create_id uuid.uuid4() unique à chauq eexecution !
100 print(" hostname : " + str(hostname))
101 prefix_url = "///FILE///" + os.environ['USER'] + "@" + hostname + "//"
103 data_for_retrieval = []
104 unmanaged_extension = {}
105 nb_character_for_cost = 0
106 sum_size_parsed = 0
107 for f in list_files:
108 ext = f["extension"].lower()
109 sys.stdout.write("&")
110 if ext in managed_extension:
111 content = ""
112 try:
113 fpath = f["path"]
114 if verbose:
115 print(fpath)
116 if ext == ".txt" or ext == ".py" or ext == ".md":
117 sys.stdout.write("t")
118 content = load_file_txt(fpath)
119 elif ext == ".pdf":
120 sys.stdout.write("p")
121# content = load_file_pdf(fpath)
122 content = load_file_pdf2(fpath)
123# content = pdf_to_clipboard(fpath)
124 elif ext == ".docx":
125 sys.stdout.write("d")
126 content = docx_to_text(fpath)
127 elif ext == ".pages":
128 sys.stdout.write("d")
129 content = load_file_pages(fpath)
130 elif ext == ".pptx":
131 sys.stdout.write("d")
132 content = load_file_pptx(fpath)
133 content += pptx_to_markdown(fpath)
134 else :
135 print("ext : " + ext + " unmanaged !")
137 except Exception as e:
138 print(str(e))
139 print(" Couldn't read " + str(f))
141 nb_character_for_cost += len(content)
142 sum_size_parsed += f["size"]
144 f_to_json = {"id" : prefix_url + f["path"],
145 "text" : content,
146 "size" : f["size"], "url" : prefix_url + f["path"]}
148 data_for_retrieval.append(f_to_json)
149 else :
150 sys.stdout.write("U")
151 if ext not in unmanaged_extension:
152 unmanaged_extension[ext] = {"nb":0, "size":0}
153 unmanaged_extension[ext]["nb"] += 1
154 unmanaged_extension[ext]["size"] += f["size"]
156 count_time, detailed_time = count_and_display_elapsed_time(count_time, "one_file " + f["path"], min_time = 1)
157 list_detailed_time.append(detailed_time)
159 print(" nb_character_for_cost : " + str(nb_character_for_cost))
160 print(" for a total size of : " + str(sum_size_parsed))
162 # Trier le dictionnaire par taille
163 sorted_dict_unmanaged_extension = dict(sorted(unmanaged_extension.items(), key=lambda item: item[1]['size'],reverse=True))
165 print("unmanaged_extension : " + str(sorted_dict_unmanaged_extension))
167 return data_for_retrieval, list_detailed_time, sorted_dict_unmanaged_extension
171import json
172def manual_dump_id_path_url(out_path, json_data, verbose = False):
173 print("Dumping json in " + out_path)
174 with open(out_path, "w") as f:
175 f.write("[\n")
176 for d in json_data:
177 f.write(" {\n")
178 if "id" in d and "text" in d and "url" in d:
179 f.write(" \"id\" : \"" + str(d["id"]) + "\",\n")
180 f.write(" \"text\" : " + json.dumps(d["text"]) + ",\n")
181 f.write(" \"url\" : \"" + str(d["url"]) + "\"\n")
182 f.write(" }")
183 if json_data.index(d) < len(json_data) - 1 :
184 f.write(",")
185 f.write("\n")
186 f.write("]\n")
190def write_json_index_for_import(data_for_retrieval,
191 out_file_json,
192 verbose = False):
193 import ast
194 json_data = ast.literal_eval(json.dumps(data_for_retrieval))
195 if verbose :
196 print(json_data)
197 from lib.import_util.lib_path_to_vec import manual_dump_id_path_url
198 manual_dump_id_path_url(out_file_json, json_data, verbose = verbose)
202def load_file_txt(file):
203 try:
204 with open(file, "r") as ff:
205 text = ff.readlines()
206 return "\n".join(text)
207 except Exception as e:
208 print(str(e))
209 print(file + " not read !")
210 return ""
212# Peut etre utile
213# Chaîne de caractères à échapper
214# Échapper tous les caractères spéciaux
216#from urllib.parse import quote
217#s = "Hello World! @#"
218#escaped_s = quote(s, safe='')
219#print(escaped_s) # Output: Hello%20World%21%20%40%23
222# pdf to txt est peut-etre un meilleur nom !
223def load_file_pdf2(file_path):
224 import PyPDF2
225 # Open the PDF file in binary read mode
226 with open(file_path, 'rb') as file:
227 # Create a PdfReader object
228 pdfReader = PyPDF2.PdfReader(file)
230 text = ''
231 # for i in range(len(pdfReader.pages)):
232# page = pdfReader.getPage[i]
233 for page in pdfReader.pages:
234 # getting a specific page from the pdf file
236 etxt = page.extract_text()
237 if etxt != None:
238 text += etxt
239 else :
240 print("None texte in page")
242 return text
245# pdf to txt est peut-etre un meilleur nom !
246def load_file_pdf(file_path): # pragma no cover
247 import pdfreader
248 # Open the PDF file in binary read mode
249 with open(file_path, 'rb') as file:
250 # Create a PdfReader object
251 reader = pdfreader.SimplePDFViewer(file)
253 # printing number of pages in pdf file
254 print(len(reader.pages))
256 # Read the text from each page and add it to the text variable
257 text = ''
258 for canvas in reader:
259 text += canvas.text_content
261 # Help https://pdfreader.readthedocs.io/_/downloads/en/latest/pdf/ page 15
262 return text
263# Utilisation de la fonction
264#text = pdf_to_text('/path/to/your/document.pdf')
265#print(text)
268def pdf_to_clipboard(file_path): # pragma no cover
269 import pyperclip
270 import subprocess
271 import time
272 import webbrowser
274 # Ouvrir le fichier PDF en mode lecture binaire
275 with open(file_path, 'rb') as file:
277 webbrowser.open_new(file_path)
278 # Ouvrir le fichier PDF avec le logiciel par défaut
279 #subprocess.run(['open', file_path])
280# subprocess.Popen(file_path, shell=True)
282 # Attendre quelques secondes pour que le fichier s'ouvre
283 time.sleep(5)
285 # Récupérer le texte du presse-papiers
286 clipboard_text = pyperclip.paste()
288 return clipboard_text
290# Utilisation de la fonction
292# Utilisation de la fonction
293#clipboard_text = pdf_to_clipboard('/path/to/your/document.pdf')
294#print(clipboard_text)
297def load_file_pages(file_path):
298 import zipfile
299 import os
300 from bs4 import BeautifulSoup
302 # Extraire l'archive ZIP
303 with zipfile.ZipFile(file_path, 'r') as zip_ref:
304 zip_ref.extractall('temp')
306 # Lire le fichier XML
307 with open(os.path.join('temp', 'index.xml'), 'r') as file:
308 content = file.read()
310 # Utiliser BeautifulSoup pour analyser le contenu XML
311 soup = BeautifulSoup(content, 'xml')
312 text = soup.get_text()
314 # Supprimer le dossier temporaire
315 os.remove('temp')
317 return text
319# Utilisation de la fonction
320#text = load_file_pages('/path/to/your/document.pages')
321#print(text)
325def load_file_pptx(file_path):
326 from pptx import Presentation
327 # Charger la présentation
328 prs = Presentation(file_path)
330 # Parcourir toutes les diapositives et extraire le texte
331 text = ''
332 for slide in prs.slides:
333 for shape in slide.shapes:
334 if shape.has_text_frame:
335 for paragraph in shape.text_frame.paragraphs:
336 for run in paragraph.runs:
337 text += run.text
338 return text
340# Utilisation de la fonction
341# text = load_file_pptx('/path/to/your/presentation.pptx')
342# print(text)
346# A TESTER :
350def pptx_to_markdown(file_path):
351 from pptx import Presentation
352 # Charger la présentation
353 prs = Presentation(file_path)
355 # Parcourir toutes les diapositives et extraire le texte
356 markdown_text = ''
357 for i, slide in enumerate(prs.slides):
358 markdown_text += f"## Slide {i+1}\n\n"
359 for shape in slide.shapes:
360 if shape.has_text_frame:
361 for paragraph in shape.text_frame.paragraphs:
362 for run in paragraph.runs:
363 markdown_text += run.text
364 markdown_text += "\n"
365 elif shape.shape_type == 13: # This is the code for Picture
366 image = shape.image
367 image_bytes = image.blob
368 image_filename = image.ext
369 markdown_text += f"\n\n"
370 markdown_text += "\n---\n"
371 return markdown_text
373# Utilisation de la fonction
374#markdown_text = pptx_to_markdown('/path/to/your/presentation.pptx')
375#print(markdown_text)
379def docx_to_text(file_path):
380 from docx import Document
381 # Charger le document
382 doc = Document(file_path)
384 # Lire le texte de chaque paragraphe et l'ajouter à la variable text
385 text = ''
386 for paragraph in doc.paragraphs:
387 text += paragraph.text + '\n'
388 return text
390# Utilisation de la fonction
391#text = docx_to_text('/path/to/your/document.docx')
392#print(text)