module assiduités : fusion cache WIP

This commit is contained in:
iziram 2023-02-13 17:50:58 +01:00
parent aa956f4530
commit 0f3e1ea95e
4 changed files with 196 additions and 179 deletions

View File

@ -42,7 +42,7 @@ from app.scodoc import sco_cache
from app.scodoc import sco_etud
from app.scodoc import sco_formsemestre_inscriptions
from app.scodoc import sco_preferences
from app.models import Assiduite
from app.models import Assiduite, Justificatif
import app.scodoc.sco_assiduites as scass
import app.scodoc.sco_utils as scu
@ -1059,28 +1059,37 @@ def get_assiduites_count_in_interval(etudid, date_debut_iso, date_fin_iso):
tuple (nb abs, nb abs justifiées)
Utilise un cache.
"""
key = str(etudid) + "_" + date_debut_iso + "_" + date_fin_iso
key = str(etudid) + "_" + date_debut_iso + "_" + date_fin_iso + "_assiduites"
r = sco_cache.AbsSemEtudCache.get(key)
if not r:
date_debut: datetime.datetime = scu.is_iso_formated(date_debut_iso, True)
date_fin: datetime.datetime = scu.is_iso_formated(date_debut_iso, True)
date_fin: datetime.datetime = scu.is_iso_formated(date_fin_iso, True)
assiduites: Assiduite = Assiduite.query.filter_by(etudid=etudid)
justificatifs: Justificatif = Justificatif.query.filter_by(etudid=etudid)
assiduites = scass.filter_assiduites_by_date(assiduites, date_debut, sup=True)
assiduites = scass.filter_assiduites_by_date(assiduites, date_fin, sup=False)
nb_abs = scass.get_count(assiduites)["demi"]
nb_abs_just = count_abs_just(
etudid=etudid,
debut=date_debut_iso,
fin=date_fin_iso,
assiduites = scass.filter_by_date(assiduites, Assiduite, date_debut, date_fin)
justificatifs = scass.filter_by_date(
justificatifs, Justificatif, date_debut, date_fin
)
calculator: scass.CountCalculator = scass.CountCalculator()
calculator.compute_assiduites(assiduites)
nb_abs: dict = calculator.to_dict()["demi"]
abs_just: list[Assiduite] = scass.get_all_justified(
justificatifs, date_debut, date_fin
)
calculator.reset()
calculator.compute_assiduites(abs_just)
nb_abs_just: dict = calculator.to_dict()["demi"]
r = (nb_abs, nb_abs_just)
ans = sco_cache.AbsSemEtudCache.set(key, r)
if not ans:
log("warning: get_abs_count failed to cache")
log("warning: get_assiduites_count failed to cache")
return r

View File

@ -4,6 +4,7 @@ import app.scodoc.sco_utils as scu
from app.models.assiduites import Assiduite, Justificatif
from app.models.etudiants import Identite
from app.models.formsemestre import FormSemestre, FormSemestreInscription
from app.profiler import Profiler
class CountCalculator:
@ -37,6 +38,12 @@ class CountCalculator:
self.count: int = 0
def reset(self):
self.days = []
self.half_days = []
self.hours = 0.0
self.count = 0
def add_half_day(self, day: date, is_morning: bool = True):
key: tuple[date, bool] = (day, is_morning)
if key not in self.half_days:
@ -53,7 +60,9 @@ class CountCalculator:
scu.localize_datetime(datetime.combine(period[0].date(), self.noon)),
)
in_morning: bool = scu.is_period_overlapping(period, interval_morning)
in_morning: bool = scu.is_period_overlapping(
period, interval_morning, bornes=False
)
return in_morning
def check_in_evening(self, period: tuple[datetime, datetime]) -> bool:
@ -113,7 +122,10 @@ class CountCalculator:
def compute_assiduites(self, assiduites: Assiduite):
assi: Assiduite
for assi in assiduites.all():
assiduites: list[Assiduite] = (
assiduites.all() if isinstance(assiduites, Assiduite) else assiduites
)
for assi in assiduites:
self.count += 1
delta: timedelta = assi.date_fin - assi.date_debut
@ -177,106 +189,6 @@ def get_assiduites_stats(
return output if output else count
# def big_counter(
# interval: tuple[datetime],
# pref_time: time = time(12, 0),
# ):
# curr_date: datetime
# if interval[0].time() >= pref_time:
# curr_date = scu.localize_datetime(
# datetime.combine(interval[0].date(), pref_time)
# )
# else:
# curr_date = scu.localize_datetime(
# datetime.combine(interval[0].date(), time(0, 0))
# )
# def next_(curr: datetime, journee):
# if curr.time() != pref_time:
# next_time = scu.localize_datetime(datetime.combine(curr.date(), pref_time))
# else:
# next_time = scu.localize_datetime(
# datetime.combine(curr.date() + timedelta(days=1), time(0, 0))
# )
# journee += 1
# return next_time, journee
# demi: int = 0
# j: int = 0
# while curr_date <= interval[1]:
# next_time: datetime
# next_time, j = next_(curr_date, j)
# if scu.is_period_overlapping((curr_date, next_time), interval, True):
# demi += 1
# curr_date = next_time
# delta: timedelta = interval[1] - interval[0]
# heures: float = delta.total_seconds() / 3600
# if delta.days >= 1:
# heures -= delta.days * 16
# return (demi, j, heures)
# def get_count(
# assiduites: Assiduite, noon: time = time(hour=12)
# ) -> dict[str, int or float]:
# """Fonction permettant de compter les assiduites
# -> seul "compte" est correcte lorsque les assiduites viennent de plusieurs étudiants
# """
# # TODO: Comptage demi journée / journée d'assiduité longue
# output: dict[str, int or float] = {}
# compte: int = assiduites.count()
# heure: float = 0.0
# journee: int = 0
# demi: int = 0
# all_assiduites: list[Assiduite] = assiduites.order_by(Assiduite.date_debut).all()
# current_day: date = None
# current_time: str = None
# midnight: time = time(hour=0)
# def time_check(dtime):
# return midnight <= dtime.time() <= noon
# for ass in all_assiduites:
# delta: timedelta = ass.date_fin - ass.date_debut
# if delta.days > 0:
# computed_values: tuple[int, int, float] = big_counter(
# (ass.date_debut, ass.date_fin), noon
# )
# demi += computed_values[0] - 1
# journee += computed_values[1] - 1
# heure += computed_values[2]
# current_day = ass.date_fin.date()
# continue
# heure += delta.total_seconds() / 3600
# ass_time: str = time_check(ass.date_debut)
# if current_day != ass.date_debut.date():
# current_day = ass.date_debut.date()
# current_time = ass_time
# demi += 1
# journee += 1
# if current_time != ass_time:
# current_time = ass_time
# demi += 1
# heure = round(heure, 2)
# return {"compte": compte, "journee": journee, "heure": heure, "demi": demi}
def filter_assiduites_by_etat(assiduites: Assiduite, etat: str) -> Assiduite:
"""
Filtrage d'une collection d'assiduites en fonction de leur état
@ -323,27 +235,6 @@ def filter_justificatifs_by_etat(
return justificatifs.filter(Justificatif.etat.in_(etats))
def filter_justificatifs_by_date(
justificatifs: Justificatif, date_: datetime, sup: bool = True
) -> Assiduite:
"""
Filtrage d'une collection d'assiduites en fonction d'une date
Sup == True -> les assiduites doivent débuter après 'date'\n
Sup == False -> les assiduites doivent finir avant 'date'
"""
if date_.tzinfo is None:
first_justificatif: Justificatif = justificatifs.first()
if first_justificatif is not None:
date_: datetime = date_.replace(tzinfo=first_justificatif.date_debut.tzinfo)
if sup:
return justificatifs.filter(Justificatif.date_debut >= date_)
return justificatifs.filter(Justificatif.date_fin <= date_)
def filter_by_module_impl(
assiduites: Assiduite, module_impl_id: int or None
) -> Assiduite:
@ -376,26 +267,48 @@ def filter_by_formsemestre(assiduites_query: Assiduite, formsemestre: FormSemest
return assiduites_query.filter(Assiduite.date_fin <= formsemestre.date_fin)
def justifies(justi: Justificatif) -> list[int]:
def justifies(justi: Justificatif, obj: bool = False) -> list[int]:
"""
Retourne la liste des assiduite_id qui sont justifié par la justification
Une assiduité est justifiée si elle est STRICTEMENT comprise dans la plage du justificatif
et que l'état du justificatif est "validé"
renvoie des id si obj == False, sinon les Assiduités
"""
justified: list[int] = []
if justi.etat != scu.EtatJustificatif.VALIDE:
return justified
return []
assiduites_query: Assiduite = Assiduite.query.join(
Justificatif, Assiduite.etudid == Justificatif.etudid
).filter(Assiduite.etat != scu.EtatAssiduite.PRESENT)
assiduites_query = filter_by_date(
assiduites_query, Assiduite, justi.date_debut, justi.date_fin
assiduites_query: Assiduite = (
Assiduite.query.join(Justificatif, Assiduite.etudid == Justificatif.etudid)
.filter(Assiduite.etat != scu.EtatAssiduite.PRESENT)
.filter(
Assiduite.date_debut >= justi.date_debut,
Assiduite.date_debut <= justi.date_fin,
Assiduite.date_fin >= justi.date_debut,
Assiduite.date_fin <= justi.date_fin,
)
)
justified = [assi.id for assi in assiduites_query.all()]
if not obj:
return [assi.id for assi in assiduites_query.all()]
return justified
return assiduites_query
def get_all_justified(
justificatifs: Justificatif, date_deb: datetime = None, date_fin: datetime = None
) -> list[Assiduite]:
if date_deb is None:
date_deb = datetime.min
if date_fin is None:
date_fin = datetime.max
date_deb = scu.localize_datetime(date_deb)
date_fin = scu.localize_datetime(date_fin)
assiduites: list[Assiduite] = []
for justi in justificatifs:
assis: list[Assiduite] = justifies(justi, obj=True)
assiduites.extend(assis)
return list(assiduites)

View File

@ -17,6 +17,12 @@ import app.scodoc.sco_assiduites as scass
from app.models import Assiduite, Justificatif, Identite, FormSemestre, ModuleImpl
from app.scodoc.sco_exceptions import ScoValueError
import app.scodoc.sco_utils as scu
from app.scodoc.sco_abs import (
get_abs_count_in_interval,
get_assiduites_count_in_interval,
)
from app.scodoc import sco_abs_views
from tools import migrate_abs_to_assiduites, downgrade_module
class BiInt(int, scu.BiDirectionalEnum):
@ -130,17 +136,100 @@ def test_general(test_client):
etud_faux_dict = g_fake.create_etud(code_nip=None, prenom="etudfaux")
etud_faux = Identite.query.filter_by(id=etud_faux_dict["id"]).first()
verif_migration_abs_assiduites(g_fake)
ajouter_assiduites(etuds, moduleimpls, etud_faux)
justificatifs: list[Justificatif] = ajouter_justificatifs(etuds[0])
verifier_comptage_et_filtrage_assiduites(
etuds, moduleimpls, (formsemestre_1, formsemestre_2, formsemestre_3)
)
verifier_filtrage_justificatifs(etuds[0], justificatifs)
editer_supprimer_assiduites(etuds, moduleimpls)
editer_supprimer_justificatif(etuds[0])
def verif_migration_abs_assiduites(g_fake):
downgrade_module(assiduites=True, justificatifs=True)
etudid: int = 1
for debut, fin, demijournee in [
(
"02/01/2023",
"10/01/2023",
2,
), # 2 assiduités 02/01: 08h -> 06/01: 18h & assiduités 09/01: 08h -> 10/01: 18h | 14dj
("16/01/2023", "16/01/2023", 1), # 1 assiduité 16/01: 08h -> 16/01: 12h | 1dj
("19/01/2023", "19/01/2023", 0), # 1 assiduité 19/01: 12h -> 19/01: 18h | 1dj
("18/01/2023", "18/01/2023", 2), # 1 assiduité 18/01: 08h -> 18/01: 18h | 2dj
("23/01/2023", "23/01/2023", 0), # 1 assiduité 23/01: 12h -> 24/01: 18h | 3dj
("24/01/2023", "24/01/2023", 2),
]:
sco_abs_views.doSignaleAbsence(
datedebut=debut,
datefin=fin,
demijournee=demijournee,
etudid=etudid,
)
# --- Justification de certaines absences
for debut, fin, demijournee in [
(
"02/01/2023",
"10/01/2023",
2,
), # 2 justificatif 02/01: 08h -> 06/01: 18h & justificatif 09/01: 08h -> 10/01: 18h | 14dj
(
"19/01/2023",
"19/01/2023",
0,
), # 1 justificatif 19/01: 12h -> 19/01: 18h | 1dj
(
"18/01/2023",
"18/01/2023",
2,
), # 1 justificatif 18/01: 08h -> 18/01: 18h | 2dj
]:
sco_abs_views.doJustifAbsence(
datedebut=debut,
datefin=fin,
demijournee=demijournee,
etudid=etudid,
)
migrate_abs_to_assiduites()
assert Assiduite.query.count() == 6, "Erreur migration assiduites"
assert Justificatif.query.count() == 4, "Erreur migration justificatifs"
essais_cache(etudid, g_fake)
downgrade_module(assiduites=True, justificatifs=True)
def essais_cache(etudid, g_fake):
date_deb: str = "2023-01-01T07:00"
date_fin: str = "2023-03-31T19:00"
abs_count_no_cache: int = get_abs_count_in_interval(etudid, date_deb, date_fin)
abs_count_cache = get_abs_count_in_interval(etudid, date_deb, date_fin)
assiduites_count_no_cache = get_assiduites_count_in_interval(
etudid, date_deb, date_fin
)
assiduites_count_cache = get_assiduites_count_in_interval(
etudid, date_deb, date_fin
)
assert (
abs_count_cache
== abs_count_no_cache
== assiduites_count_cache
== assiduites_count_no_cache
== (21, 17)
), "Erreur cache"
def ajouter_justificatifs(etud):
obj_justificatifs = [
@ -329,10 +418,10 @@ def editer_supprimer_justificatif(etud: Identite):
# Modification de l'état
justi.etat = scu.EtatJustificatif.MODIFIE
db.session.add(justi)
# Modification du moduleimpl
justi.date_debut = scu.localize_datetime("2023-02-03T11:00:01+01:00")
justi.fin = scu.localize_datetime("2023-02-03T12:00:01+01:00")
justi.date_fin = scu.localize_datetime("2023-02-03T12:00:01+01:00")
db.session.add(justi)
db.session.commit()
@ -340,12 +429,15 @@ def editer_supprimer_justificatif(etud: Identite):
assert (
scass.filter_justificatifs_by_etat(etud.justificatifs, "modifie").count() == 2
), "Edition de justificatif mauvais"
assert (
scass.filter_justificatifs_by_date(
etud.justificatifs, scu.localize_datetime("2023-02-03T11:00:00+01:00")
scass.filter_by_date(
etud.justificatifs,
Justificatif,
date_deb=scu.localize_datetime("2023-02-01T11:00:00+01:00"),
).count()
== 1
), "Edition de justificatif mauvais"
), "Edition de justificatif mauvais 2"
# Supression d'une assiduité

View File

@ -27,14 +27,13 @@ class _Merger:
class _glob:
DUPLICATIONS_ASSIDUITES: dict[tuple[date, bool, int], Assiduite] = {}
DUPLICATIONS_JUSTIFICATIFS: dict[tuple[date, bool, int], Justificatif] = {}
PROBLEMS: dict[int, list[str]] = {}
CURRENT_ETU: list = []
MODULES: list[tuple[int, int]] = []
COMPTE: list[int, int] = []
ERR_ETU: list[int] = []
MERGER: _Merger = None
MERGER_ASSI: _Merger = None
MERGER_JUST: _Merger = None
MORNING: time = None
NOON: time = None
@ -42,13 +41,12 @@ class _glob:
class _Merger:
def __init__(self, abs: Absence) -> None:
def __init__(self, abs: Absence, est_abs: bool) -> None:
self.deb = (abs.jour, abs.matin)
self.fin = (abs.jour, abs.matin)
self.moduleimpl = abs.moduleimpl_id
self.etudid = abs.etudid
self.est_abs = abs.estabs
self.est_just = abs.estjust
self.est_abs = est_abs
self.raison = abs.description
def merge(self, abs: Absence) -> bool:
@ -58,18 +56,14 @@ class _Merger:
# Cas d'une même absence enregistrée plusieurs fois
if self.fin == (abs.jour, abs.matin):
self.est_abs |= abs.estabs
self.est_just |= abs.estjust
self.moduleimpl = None
else:
if self.est_abs != abs.estabs or self.est_just != abs.estjust:
return False
if self.fin[1]:
if abs.jour != self.fin[0]:
return False
else:
if abs.jour - timedelta(days=1) != self.fin[0]:
day_after: date = abs.jour - timedelta(days=1) == self.fin[0]
if not (day_after and abs.matin):
return False
self.fin = (abs.jour, abs.matin)
@ -118,7 +112,7 @@ class _Merger:
if self.est_abs:
_glob.COMPTE[0] += 1
objects.append(self._to_assi())
if self.est_just:
else:
_glob.COMPTE[1] += 1
objects.append(self._to_justif())
@ -238,13 +232,12 @@ def migrate_abs_to_assiduites(
Absence.etudid, Absence.jour, not_(Absence.matin)
)
_glob.DUPLICATED = []
_glob.DUPLICATIONS_ASSIDUITES = {}
_glob.DUPLICATIONS_JUSTIFICATIFS = {}
_glob.CURRENT_ETU = []
_glob.MODULES = []
_glob.COMPTE = [0, 0]
_glob.ERR_ETU = []
_glob.MERGER_ASSI = None
_glob.MERGER_JUST = None
absences_len: int = absences.count()
@ -255,6 +248,7 @@ def migrate_abs_to_assiduites(
printProgressBar(0, absences_len, "Progression", "effectué", autosize=True)
for i, abs in enumerate(absences):
try:
_from_abs_to_assiduite_justificatif(abs)
except Exception as e:
@ -279,11 +273,8 @@ def migrate_abs_to_assiduites(
)
db.session.commit()
dup_assi = _glob.DUPLICATED
assi: Assiduite
for assi in dup_assi:
assi.moduleimpl_id = None
db.session.add(assi)
_glob.MERGER_ASSI.export()
_glob.MERGER_JUST.export()
db.session.commit()
@ -342,10 +333,22 @@ def _from_abs_to_assiduite_justificatif(_abs: Absence):
if moduleimpl_inscription is None:
raise Exception("Moduleimpl_id incorrect ou étudiant non inscrit")
if _glob.MERGER is None:
_glob.MERGER = _Merger(_abs)
elif _glob.MERGER.merge(_abs):
if _glob.MERGER_ASSI is None:
_glob.MERGER_ASSI = _Merger(_abs, True)
return True
elif _glob.MERGER_ASSI.merge(_abs):
return True
else:
_glob.MERGER_ASSI.export()
_glob.MERGER_ASSI = _Merger(_abs, True)
return False
if _glob.MERGER_JUST is None:
_glob.MERGER_JUST = _Merger(_abs, False)
return True
elif _glob.MERGER_JUST.merge(_abs):
return True
else:
_glob.MERGER.export()
_glob.MERGER = _Merger(_abs)
_glob.MERGER_JUST.export()
_glob.MERGER_JUST = _Merger(_abs, False)
return False