From 48912032c497a6d80e60f55dd8ce6767f87c14ed Mon Sep 17 00:00:00 2001 From: Emmanuel Viennet Date: Thu, 7 Jul 2022 11:56:18 +0200 Subject: [PATCH] =?UTF-8?q?Am=C3=A9lioration=20sco=5Fexcel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/entreprises/routes.py | 4 +- app/scodoc/sco_excel.py | 159 ++++++++++++++++---------------------- 2 files changed, 70 insertions(+), 93 deletions(-) diff --git a/app/entreprises/routes.py b/app/entreprises/routes.py index ef15f32b..6a831d86 100644 --- a/app/entreprises/routes.py +++ b/app/entreprises/routes.py @@ -1419,7 +1419,7 @@ def get_import_donnees_file_sample(): @permission_required(Permission.RelationsEntreprisesExport) def import_donnees(): """ - Permet d'importer des entreprises a l'aide d'un fichier excel (.xlsx) + Permet d'importer des entreprises à partir d'un fichier excel (.xlsx) """ form = ImportForm() if form.validate_on_submit(): @@ -1428,7 +1428,7 @@ def import_donnees(): Config.SCODOC_VAR_DIR, "tmp", secure_filename(file.filename) ) file.save(file_path) - diag, lm = sco_excel.excel_file_to_list_are(file_path) + diag, lm = sco_excel.excel_workbook_to_list(file_path) os.remove(file_path) if lm is None or len(lm) < 2: flash("Veuillez utilisez la feuille excel à remplir") diff --git a/app/scodoc/sco_excel.py b/app/scodoc/sco_excel.py index b2165209..8e1ceabc 100644 --- a/app/scodoc/sco_excel.py +++ b/app/scodoc/sco_excel.py @@ -40,10 +40,9 @@ from openpyxl.comments import Comment from openpyxl import Workbook, load_workbook from openpyxl.cell import WriteOnlyCell from openpyxl.styles import Font, Border, Side, Alignment, PatternFill +from openpyxl.worksheet.worksheet import Worksheet import app.scodoc.sco_utils as scu -from app.scodoc import notesdb -from app.scodoc import sco_preferences from app import log from app.scodoc.sco_exceptions import ScoValueError @@ -593,71 +592,87 @@ def excel_feuille_saisie(e, titreannee, description, lines): def excel_bytes_to_list(bytes_content): try: filelike = io.BytesIO(bytes_content) - return _excel_to_list(filelike) - except: + except Exception as exc: raise ScoValueError( """Le fichier xlsx attendu n'est pas lisible ! Peut-être avez-vous fourni un fichier au mauvais format (txt, xls, ..) """ - ) + ) from exc + return _excel_to_list(filelike) def excel_file_to_list(filename): try: return _excel_to_list(filename) - except: + except Exception as exc: raise ScoValueError( """Le fichier xlsx attendu n'est pas lisible ! Peut-être avez-vous fourni un fichier au mauvais format (txt, xls, ...) """ - ) + ) from exc def excel_file_to_list_are(filename): try: return _excel_to_list_are(filename) - except: + except Exception as exc: raise ScoValueError( """Le fichier xlsx attendu n'est pas lisible ! Peut-être avez-vous fourni un fichier au mauvais format (txt, xls, ...) """ - ) + ) from exc + + +def _open_workbook(filelike, dump_debug=False) -> Workbook: + """Open document. + On error, if dump-debug is True, dump data in /tmp for debugging purpose + """ + try: + workbook = load_workbook(filename=filelike, read_only=True, data_only=True) + except Exception as exc: + log("Excel_to_list: failure to import document") + if dump_debug: + dump_filename = "/tmp/last_scodoc_import_failure" + scu.XLSX_SUFFIX + log(f"Dumping problemetic file on {dump_filename}") + with open(dump_filename, "wb") as f: + f.write(filelike) + raise ScoValueError( + "Fichier illisible: assurez-vous qu'il s'agit bien d'un document Excel xlsx !" + ) from exc + return workbook def _excel_to_list(filelike): - """returns list of list - convert_to_string is a conversion function applied to all non-string values (ie numbers) - """ - try: - wb = load_workbook(filename=filelike, read_only=True, data_only=True) - except: - log("Excel_to_list: failure to import document") - with open("/tmp/last_scodoc_import_failure" + scu.XLSX_SUFFIX, "wb") as f: - f.write(filelike) - raise ScoValueError( - "Fichier illisible: assurez-vous qu'il s'agit bien d'un document Excel !" - ) + """returns list of list""" + workbook = _open_workbook(filelike) diag = [] # liste de chaines pour former message d'erreur - # n'utilise que la première feuille - if len(wb.get_sheet_names()) < 1: + if len(workbook.get_sheet_names()) < 1: diag.append("Aucune feuille trouvée dans le classeur !") return diag, None - if len(wb.get_sheet_names()) > 1: + # n'utilise que la première feuille: + if len(workbook.get_sheet_names()) > 1: diag.append("Attention: n'utilise que la première feuille du classeur !") + sheet_name = workbook.get_sheet_names()[0] + ws = workbook[sheet_name] + matrix, diag_sheet = _excel_sheet_to_list(ws, sheet_name) + diag += diag_sheet + return diag, matrix + + +def _excel_sheet_to_list(sheet: Worksheet, sheet_name: str) -> tuple[list, list]: + """read a spreadsheet sheet, and returns: + - diag : a list of strings (error messages aimed at helping the user) + - a list of lists: the spreadsheet cells + """ + diag = [] # fill matrix - sheet_name = wb.get_sheet_names()[0] - ws = wb.get_sheet_by_name(sheet_name) - sheet_name = sheet_name.encode(scu.SCO_ENCODING, "backslashreplace") values = {} - for row in ws.iter_rows(): + for row in sheet.iter_rows(): for cell in row: if cell.value is not None: values[(cell.row - 1, cell.column - 1)] = str(cell.value) if not values: - diag.append( - "Aucune valeur trouvée dans la feuille %s !" - % sheet_name.decode(scu.SCO_ENCODING) - ) + diag.append(f"Aucune valeur trouvée dans la feuille {sheet_name} !") return diag, None indexes = list(values.keys()) # search numbers of rows and cols @@ -665,76 +680,38 @@ def _excel_to_list(filelike): cols = [x[1] for x in indexes] nbcols = max(cols) + 1 nbrows = max(rows) + 1 - m = [] + matrix = [] for _ in range(nbrows): - m.append([""] * nbcols) + matrix.append([""] * nbcols) for row_idx, col_idx in indexes: v = values[(row_idx, col_idx)] - # if isinstance(v, six.text_type): - # v = v.encode(scu.SCO_ENCODING, "backslashreplace") - # elif convert_to_string: - # v = convert_to_string(v) - m[row_idx][col_idx] = v - diag.append( - 'Feuille "%s", %d lignes' % (sheet_name.decode(scu.SCO_ENCODING), len(m)) - ) - # diag.append(str(M)) - # - return diag, m + matrix[row_idx][col_idx] = v + diag.append(f'Feuille "{sheet_name}", {len(matrix)} lignes') + + return diag, matrix -def _excel_to_list_are(filelike): - """returns list of list - convert_to_string is a conversion function applied to all non-string values (ie numbers) +def _excel_workbook_to_list(filelike): + """Lit un classeur (workbook): chaque feuille est lue + et est convertie en une liste de listes. + Returns: + - diag : a list of strings (error messages aimed at helping the user) + - a list of lists: the spreadsheet cells """ - try: - wb = load_workbook(filename=filelike, read_only=True, data_only=True) - except: - log("Excel_to_list: failure to import document") - with open("/tmp/last_scodoc_import_failure" + scu.XLSX_SUFFIX, "wb") as f: - f.write(filelike) - raise ScoValueError( - "Fichier illisible: assurez-vous qu'il s'agit bien d'un document Excel !" - ) + workbook = _open_workbook(filelike) diag = [] # liste de chaines pour former message d'erreur - if len(wb.get_sheet_names()) < 1: + if len(workbook.get_sheet_names()) < 1: diag.append("Aucune feuille trouvée dans le classeur !") return diag, None - lm = [] - for sheet_name in wb.get_sheet_names(): + matrix_list = [] + for sheet_name in workbook.get_sheet_names(): # fill matrix - ws = wb.get_sheet_by_name(sheet_name) - sheet_name = sheet_name.encode(scu.SCO_ENCODING, "backslashreplace") - values = {} - for row in ws.iter_rows(): - for cell in row: - if cell.value is not None: - values[(cell.row - 1, cell.column - 1)] = str(cell.value) - if not values: - diag.append( - "Aucune valeur trouvée dans la feuille %s !" - % sheet_name.decode(scu.SCO_ENCODING) - ) - return diag, None - indexes = list(values.keys()) - # search numbers of rows and cols - rows = [x[0] for x in indexes] - cols = [x[1] for x in indexes] - nbcols = max(cols) + 1 - nbrows = max(rows) + 1 - m = [] - for _ in range(nbrows): - m.append([""] * nbcols) - - for row_idx, col_idx in indexes: - v = values[(row_idx, col_idx)] - m[row_idx][col_idx] = v - diag.append( - 'Feuille "%s", %d lignes' % (sheet_name.decode(scu.SCO_ENCODING), len(m)) - ) - lm.append(m) - return diag, lm + sheet = workbook.get_sheet_by_name(sheet_name) + matrix, diag_sheet = _excel_sheet_to_list(sheet, sheet_name) + diag += diag_sheet + matrix_list.append(matrix) + return diag, matrix_list def excel_feuille_listeappel(