From cb0b890f1f605cbfbd91bf9ea0b78839eb76a829 Mon Sep 17 00:00:00 2001 From: Emmanuel Viennet Date: Wed, 19 Jul 2023 10:37:56 +0200 Subject: [PATCH] Optimisation migration absences->assiduites --- tools/migrate_abs_to_assiduites.py | 110 +++++++++++++++++++---------- 1 file changed, 71 insertions(+), 39 deletions(-) diff --git a/tools/migrate_abs_to_assiduites.py b/tools/migrate_abs_to_assiduites.py index 278873e9..2a096b98 100644 --- a/tools/migrate_abs_to_assiduites.py +++ b/tools/migrate_abs_to_assiduites.py @@ -7,6 +7,8 @@ from datetime import date, datetime, time, timedelta from json import dump, dumps from sqlalchemy import not_ +from flask import g + from app import db from app.models import ( Absence, @@ -30,6 +32,7 @@ from app.scodoc.sco_utils import ( localize_datetime, print_progress_bar, ) +from app.scodoc import notesdb as ndb class _glob: @@ -38,7 +41,6 @@ class _glob: DEBUG: bool = False PROBLEMS: dict[int, list[str]] = {} DEPT_ETUDIDS: dict[int, Identite] = {} - MODULES: dict[tuple[int, int]] = {} COMPTE: list[int, int] = [] ERR_ETU: list[int] = [] MERGER_ASSI: "_Merger" = None @@ -97,42 +99,70 @@ class _Merger: date_deb = _Merger._tuple_to_date(self.deb) date_fin = _Merger._tuple_to_date(self.fin, end=True) - retour = Justificatif.fast_create_justificatif( - etudid=self.etudid, - date_debut=date_deb, - date_fin=date_fin, - etat=EtatJustificatif.VALIDE, - raison=self.raison, - entry_date=self.entry_date, + _glob.cursor.execute( + """INSERT INTO justificatifs + (etudid,date_debut,date_fin,etat,raison,entry_date) + VALUES (%(etudid)s,%(date_debut)s,%(date_fin)s,%(etat)s,%(raison)s,%(entry_date)s) + """, + { + "etudid": self.etudid, + "date_debut": date_deb, + "date_fin": date_fin, + "etat": EtatJustificatif.VALIDE, + "raison": self.raison, + "entry_date": self.entry_date, + }, ) - return retour + # retour = Justificatif.fast_create_justificatif( + # etudid=self.etudid, + # date_debut=date_deb, + # date_fin=date_fin, + # etat=EtatJustificatif.VALIDE, + # raison=self.raison, + # entry_date=self.entry_date, + # ) + # return retour def _to_assi(self): date_deb = _Merger._tuple_to_date(self.deb) date_fin = _Merger._tuple_to_date(self.fin, end=True) - retour = Assiduite.fast_create_assiduite( - etudid=self.etudid, - date_debut=date_deb, - date_fin=date_fin, - etat=EtatAssiduite.ABSENT, - moduleimpl_id=self.moduleimpl, - description=self.raison, - entry_date=self.entry_date, + _glob.cursor.execute( + """INSERT INTO assiduites + (etudid,date_debut,date_fin,etat,moduleimpl_id,"desc",entry_date) + VALUES (%(etudid)s,%(date_debut)s,%(date_fin)s,%(etat)s,%(moduleimpl_id)s,%(desc)s,%(entry_date)s) + """, + { + "etudid": self.etudid, + "date_debut": date_deb, + "date_fin": date_fin, + "etat": EtatAssiduite.ABSENT, + "moduleimpl_id": self.moduleimpl, + "desc": self.raison, + "entry_date": self.entry_date, + }, ) - return retour + + # retour = Assiduite.fast_create_assiduite( + # etudid=self.etudid, + # date_debut=date_deb, + # date_fin=date_fin, + # etat=EtatAssiduite.ABSENT, + # moduleimpl_id=self.moduleimpl, + # description=self.raison, + # entry_date=self.entry_date, + # ) + # return retour def export(self): """Génère un nouvel objet Assiduité ou Justificatif""" obj: Assiduite or Justificatif = None if self.est_abs: _glob.COMPTE[0] += 1 - obj = self._to_assi() + self._to_assi() else: _glob.COMPTE[1] += 1 - obj = self._to_justif() - - db.session.add(obj) + self._to_justif() class _Statistics: @@ -243,6 +273,10 @@ def migrate_abs_to_assiduites( evening: list[str] = str(evening).split(":") _glob.EVENING = time(int(evening[0]), int(evening[1])) + ndb.open_db_connection() + _glob.cnx = g.db_conn + _glob.cursor = _glob.cnx.cursor() + if dept is None: prof_total = Profiler("MigrationTotal") prof_total.start() @@ -287,7 +321,6 @@ def migrate_dept(dept_name: str, stats: _Statistics, time_elapsed: Profiler): return _glob.DEPT_ETUDIDS = {e.id for e in Identite.query.filter_by(dept_id=dept.id)} - _glob.MODULES = {} _glob.COMPTE = [0, 0] _glob.ERR_ETU = [] _glob.MERGER_ASSI = None @@ -299,9 +332,17 @@ def migrate_dept(dept_name: str, stats: _Statistics, time_elapsed: Profiler): print_progress_bar(0, absences_len, "Progression", "effectué", autosize=True) + etuds_modimpl_ids = {} for i, abs_ in enumerate(absences): + etud_modimpl_ids = etuds_modimpl_ids.get(abs_.etudid) + if etud_modimpl_ids is None: + etud_modimpl_ids = { + ins.moduleimpl_id + for ins in ModuleImplInscription.query.filter_by(etudid=abs_.etudid) + } + etuds_modimpl_ids[abs_.etudid] = etud_modimpl_ids try: - _from_abs_to_assiduite_justificatif(abs_) + _from_abs_to_assiduite_justificatif(abs_, etud_modimpl_ids) except ValueError as e: stats.add_problem(abs_, e.args[0]) @@ -322,14 +363,14 @@ def migrate_dept(dept_name: str, stats: _Statistics, time_elapsed: Profiler): "effectué", autosize=True, ) - db.session.commit() + _glob.cnx.commit() if _glob.MERGER_ASSI is not None: _glob.MERGER_ASSI.export() if _glob.MERGER_JUST is not None: _glob.MERGER_JUST.export() - db.session.commit() + _glob.cnx.commit() print_progress_bar( absences_len, @@ -379,24 +420,15 @@ def migrate_dept(dept_name: str, stats: _Statistics, time_elapsed: Profiler): print(dumps(statistiques, indent=2)) -def _from_abs_to_assiduite_justificatif(_abs: Absence): +def _from_abs_to_assiduite_justificatif(_abs: Absence, etud_modimpl_ids: set[int]): if _abs.etudid not in _glob.DEPT_ETUDIDS: raise ValueError("Etudiant inexistant") if _abs.estabs: - moduleimpl_id: int = _abs.moduleimpl_id - if ( - moduleimpl_id is not None - and (_abs.etudid, _abs.moduleimpl_id) not in _glob.MODULES + if (_abs.moduleimpl_id is not None) and ( + _abs.moduleimpl_id not in etud_modimpl_ids ): - moduleimpl_inscription: ModuleImplInscription = ( - ModuleImplInscription.query.filter_by( - moduleimpl_id=_abs.moduleimpl_id, etudid=_abs.etudid - ).first() - ) - if moduleimpl_inscription is None: - raise ValueError("Moduleimpl_id incorrect ou étudiant non inscrit") - _glob.MODULES[(_abs.etudid, _abs.moduleimpl_id)] = True + raise ValueError("Moduleimpl_id incorrect ou étudiant non inscrit") if _glob.MERGER_ASSI is None: _glob.MERGER_ASSI = _Merger(_abs, True)