ScoDoc/ScoDoc
ScoDoc
/
ScoDoc
6
2
Fork 15
Code Issues 143 Pull Requests Projects 3 Releases 128 Wiki Activity

WIP: migrating to SQlAlchemy 2.0.8

This commit is contained in:
Emmanuel Viennet 2023-04-03 17:40:45 +02:00
parent c6e1a16b99
commit 2248090248
19 changed files with 71 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
app/__init__.py
View File

@ -13,11 +13,11 @@ import logging
from logging.handlers import SMTPHandler, WatchedFileHandler
from threading import Thread
import flask
from flask import current_app, g, request
from flask import Flask
from flask import abort, flash, has_request_context, jsonify
from flask import render_template
from flask.json import JSONEncoder
from flask.logging import default_handler
from flask_bootstrap import Bootstrap
@ -29,7 +29,7 @@ from flask_moment import Moment
from flask_sqlalchemy import SQLAlchemy
from jinja2 import select_autoescape
import sqlalchemy
import sqlalchemy as sa
from flask_cas import CAS
@ -140,7 +140,7 @@ def handle_invalid_usage(error):
# JSON ENCODING
class ScoDocJSONEncoder(JSONEncoder):
class ScoDocJSONEncoder(flask.json.provider.DefaultJSONProvider):
def default(self, o):
if isinstance(o, (datetime.datetime, datetime.date)):
return o.isoformat()
@ -248,9 +248,13 @@ def create_app(config_class=DevConfig):
CAS(app, url_prefix="/cas", configuration_function=cas.set_cas_configuration)
app.wsgi_app = ReverseProxied(app.wsgi_app)
app.json_encoder = ScoDocJSONEncoder
app.json_provider_class = ScoDocJSONEncoder
app.config.from_object(config_class)
# Pour conserver l'ordre des objets dans les JSON:
# e.g. l'ordre des UE dans les bulletins
app.json.sort_keys = False
# Evite de logguer toutes les requetes dans notre log
logging.getLogger("werkzeug").disabled = True
app.logger.setLevel(app.config["LOG_LEVEL"])
@ -405,7 +409,7 @@ def create_app(config_class=DevConfig):
with app.app_context():
try:
set_cas_configuration(app)
except sqlalchemy.exc.ProgrammingError:
except sa.exc.ProgrammingError:
# Si la base n'a pas été upgradée (arrive durrant l'install)
# il se peut que la table scodoc_site_config n'existe pas encore.
pass
@ -417,7 +421,7 @@ def set_sco_dept(scodoc_dept: str, open_cnx=True):
# Check that dept exists
try:
dept = Departement.query.filter_by(acronym=scodoc_dept).first()
except sqlalchemy.exc.OperationalError:
except sa.exc.OperationalError:
abort(503)
if not dept:
raise ScoValueError(f"Invalid dept: {scodoc_dept}")
@ -495,14 +499,15 @@ def truncate_database():
"""
# use a stored SQL function, see createtables.sql
try:
db.session.execute("SELECT truncate_tables('scodoc');")
db.session.execute(sa.text("SELECT truncate_tables('scodoc');"))
db.session.commit()
except:
db.session.rollback()
raise
# Remet les compteurs (séquences sql) à zéro
db.session.execute(
"""
sa.text(
"""
CREATE OR REPLACE FUNCTION reset_sequences(username IN VARCHAR) RETURNS void AS $$
DECLARE
statements CURSOR FOR
@ -518,6 +523,7 @@ def truncate_database():
SELECT reset_sequences('scodoc');
"""
)
)
db.session.commit()

2
app/models/__init__.py
View File

@ -21,8 +21,6 @@ convention = {
metadata_obj = sqlalchemy.MetaData(naming_convention=convention)
from app.models.raw_sql_init import create_database_functions
from app.models.absences import Absence, AbsenceNotification, BilletAbsence
from app.models.departements import Departement
from app.models.etudiants import (

7
app/models/but_refcomp.py
View File

@ -7,7 +7,7 @@
"""
from datetime import datetime
import flask_sqlalchemy
from flask_sqlalchemy.query import Query
from sqlalchemy.orm import class_mapper
import sqlalchemy
@ -307,6 +307,7 @@ class ApcSituationPro(db.Model, XMLModel):
nullable=False,
)
libelle = db.Column(db.Text(), nullable=False)
# aucun attribut (le text devient le libellé)
def to_dict(self):
return {"libelle": self.libelle}
@ -451,7 +452,7 @@ class ApcAppCritique(db.Model, XMLModel):
ref_comp: ApcReferentielCompetences,
annee: str,
competence: ApcCompetence = None,
) -> flask_sqlalchemy.BaseQuery:
) -> Query:
"Liste les AC de tous les parcours de ref_comp pour l'année indiquée"
assert annee in {"BUT1", "BUT2", "BUT3"}
query = cls.query.filter(
@ -550,7 +551,7 @@ class ApcParcours(db.Model, XMLModel):
d["annees"] = {x.ordre: x.to_dict() for x in self.annees}
return d
def query_competences(self) -> flask_sqlalchemy.BaseQuery:
def query_competences(self) -> Query:
"Les compétences associées à ce parcours"
return (
ApcCompetence.query.join(ApcParcoursNiveauCompetence, ApcAnneeParcours)

4
app/models/but_validations.py
View File

@ -4,7 +4,7 @@
"""
from typing import Union
import flask_sqlalchemy
from flask_sqlalchemy.query import Query
from app import db
from app.models import CODE_STR_LEN
@ -177,7 +177,7 @@ class RegroupementCoherentUE:
def query_validations(
self,
) -> flask_sqlalchemy.BaseQuery: # list[ApcValidationRCUE]
) -> Query: # list[ApcValidationRCUE]
"""Les validations de jury enregistrées pour ce RCUE"""
niveau = self.ue_2.niveau_competence

8
app/models/formations.py
View File

@ -1,6 +1,6 @@
"""ScoDoc 9 models : Formations
"""
import flask_sqlalchemy
from flask_sqlalchemy.query import Query
import app
from app import db
@ -213,7 +213,7 @@ class Formation(db.Model):
if change:
app.clear_scodoc_cache()
def query_ues_parcour(self, parcour: ApcParcours) -> flask_sqlalchemy.BaseQuery:
def query_ues_parcour(self, parcour: ApcParcours) -> Query:
"""Les UEs d'un parcours de la formation.
Si parcour est None, les UE sans parcours.
Exemple: pour avoir les UE du semestre 3, faire
@ -231,9 +231,7 @@ class Formation(db.Model):
ApcAnneeParcours.parcours_id == parcour.id,
)
def query_competences_parcour(
self, parcour: ApcParcours
) -> flask_sqlalchemy.BaseQuery:
def query_competences_parcour(self, parcour: ApcParcours) -> Query:
"""Les ApcCompetences d'un parcours de la formation.
None si pas de référentiel de compétences.
"""

9
app/models/formsemestre.py
View File

@ -14,7 +14,8 @@ import datetime
from functools import cached_property
from flask_login import current_user
import flask_sqlalchemy
from flask_sqlalchemy.query import Query
from flask import flash, g
from sqlalchemy import and_, or_
from sqlalchemy.sql import text
@ -281,7 +282,7 @@ class FormSemestre(db.Model):
)
return r or []
def query_ues(self, with_sport=False) -> flask_sqlalchemy.BaseQuery:
def query_ues(self, with_sport=False) -> Query:
"""UE des modules de ce semestre, triées par numéro.
- Formations classiques: les UEs auxquelles appartiennent
les modules mis en place dans ce semestre.
@ -311,7 +312,7 @@ class FormSemestre(db.Model):
sem_ues = sem_ues.filter(UniteEns.type != codes_cursus.UE_SPORT)
return sem_ues.order_by(UniteEns.numero)
def query_ues_parcours_etud(self, etudid: int) -> flask_sqlalchemy.BaseQuery:
def query_ues_parcours_etud(self, etudid: int) -> Query:
"""XXX inutilisé à part pour un test unitaire => supprimer ?
UEs que suit l'étudiant dans ce semestre BUT
en fonction du parcours dans lequel il est inscrit.
@ -967,7 +968,7 @@ class FormationModalite(db.Model):
"""Create default modalities"""
numero = 0
try:
for (code, titre) in (
for code, titre in (
(FormationModalite.DEFAULT_MODALITE, "Formation Initiale"),
("FAP", "Apprentissage"),
("FC", "Formation Continue"),

4
app/models/moduleimpls.py
View File

@ -2,7 +2,7 @@
"""ScoDoc models: moduleimpls
"""
import pandas as pd
import flask_sqlalchemy
from flask_sqlalchemy.query import Query
from app import db
from app.auth.models import User
@ -163,7 +163,7 @@ class ModuleImplInscription(db.Model):
@classmethod
def etud_modimpls_in_ue(
cls, formsemestre_id: int, etudid: int, ue_id: int
) -> flask_sqlalchemy.BaseQuery:
) -> Query:
"""moduleimpls de l'UE auxquels l'étudiant est inscrit.
(Attention: inutile en APC, il faut considérer les coefficients)
"""

7
app/models/notes.py
View File

@ -3,6 +3,7 @@
"""Notes, décisions de jury, évènements scolaires
"""
import sqlalchemy as sa
from app import db
import app.scodoc.sco_utils as scu
@ -86,7 +87,8 @@ def etud_has_notes_attente(etudid, formsemestre_id):
(ne compte que les notes en attente dans des évaluations avec coef. non nul).
"""
cursor = db.session.execute(
"""SELECT COUNT(*)
sa.text(
"""SELECT COUNT(*)
FROM notes_notes n, notes_evaluation e, notes_moduleimpl m,
notes_moduleimpl_inscription i
WHERE n.etudid = :etudid
@ -97,7 +99,8 @@ def etud_has_notes_attente(etudid, formsemestre_id):
and e.coefficient != 0
and m.id = i.moduleimpl_id
and i.etudid = :etudid
""",
"""
),
{
"formsemestre_id": formsemestre_id,
"etudid": etudid,

57
app/models/raw_sql_init.py
View File

@ -1,57 +0,0 @@
# -*- coding: UTF-8 -*
"""
Create some Postgresql sequences and functions used by ScoDoc
using raw SQL
"""
from app import db
def create_database_functions(): # XXX obsolete
"""Create specific SQL functions and sequences
XXX Obsolete: cette fonction est dans la première migration 9.0.3
Flask-Migrate fait maintenant (dans les versions >= 9.0.4) ce travail.
"""
# Important: toujours utiliser IF NOT EXISTS
# car cette fonction peut être appelée plusieurs fois sur la même db
db.session.execute(
"""
CREATE SEQUENCE IF NOT EXISTS notes_idgen_fcod;
CREATE OR REPLACE FUNCTION notes_newid_fcod() RETURNS TEXT
AS $$ SELECT 'FCOD' || to_char(nextval('notes_idgen_fcod'), 'FM999999999'); $$
LANGUAGE SQL;
CREATE OR REPLACE FUNCTION notes_newid_ucod() RETURNS TEXT
AS $$ SELECT 'UCOD' || to_char(nextval('notes_idgen_fcod'), 'FM999999999'); $$
LANGUAGE SQL;
CREATE OR REPLACE FUNCTION truncate_tables(username IN VARCHAR) RETURNS void AS $$
DECLARE
statements CURSOR FOR
SELECT tablename FROM pg_tables
WHERE tableowner = username AND schemaname = 'public'
AND tablename <> 'notes_semestres'
AND tablename <> 'notes_form_modalites'
AND tablename <> 'alembic_version';
BEGIN
FOR stmt IN statements LOOP
EXECUTE 'TRUNCATE TABLE ' || quote_ident(stmt.tablename) || ' CASCADE;';
END LOOP;
END;
$$ LANGUAGE plpgsql;
-- Fonction pour anonymisation:
-- inspirée par https://www.simononsoftware.com/random-string-in-postgresql/
CREATE OR REPLACE FUNCTION random_text_md5( integer ) returns text
LANGUAGE SQL
AS $$
select upper( substring( (SELECT string_agg(md5(random()::TEXT), '')
FROM generate_series(
1,
CEIL($1 / 32.)::integer)
), 1, $1) );
$$;
"""
)
db.session.commit()

6
app/scodoc/sco_abs_billets.py
View File

@ -29,16 +29,14 @@
"""
from flask import g, url_for
import flask_sqlalchemy
from flask_sqlalchemy.query import Query
from app.models.absences import BilletAbsence
from app.models.etudiants import Identite
from app.scodoc.gen_tables import GenTable
from app.scodoc import sco_preferences
def query_billets_etud(
etudid: int = None, etat: bool = None
) -> flask_sqlalchemy.BaseQuery:
def query_billets_etud(etudid: int = None, etat: bool = None) -> Query:
"""Billets d'absences pour un étudiant, ou tous si etudid is None.
Si etat, filtre par état.
Si dans un département et que la gestion des billets n'a pas été activée

3
app/scodoc/sco_edit_ue.py
View File

@ -30,6 +30,7 @@
"""
import re
import sqlalchemy as sa
import flask
from flask import flash, render_template, url_for
from flask import g, request
@ -127,7 +128,7 @@ def do_ue_create(args):
):
# évite les conflits de code
while True:
cursor = db.session.execute("select notes_newid_ucod();")
cursor = db.session.execute(sa.text("select notes_newid_ucod();"))
code = cursor.fetchone()[0]
if UniteEns.query.filter_by(ue_code=code).count() == 0:
break

2
app/scodoc/sco_utils.py
View File

@ -690,7 +690,7 @@ def sendPDFFile(data, filename): # DEPRECATED utiliser send_file
return send_file(data, filename=filename, mime=PDF_MIMETYPE, attached=True)
class ScoDocJSONEncoder(json.JSONEncoder):
class ScoDocJSONEncoder(flask.json.provider.DefaultJSONProvider):
def default(self, o): # pylint: disable=E0202
if isinstance(o, (datetime.date, datetime.datetime)):
return o.isoformat()

3
config.py
View File

@ -38,9 +38,6 @@ class Config:
SCODOC_ERR_FILE = os.path.join(SCODOC_VAR_DIR, "log", "scodoc_exc.log")
#
MAX_CONTENT_LENGTH = 16 * 1024 * 1024 # Flask uploads (16Mo, en ligne avec nginx)
# Pour conserver l'ordre des objets dans les JSON:
# e.g. l'ordre des UE dans les bulletins
JSON_SORT_KEYS = False
class ProdConfig(Config):

10
flask_cas/__init__.py
View File

@ -5,14 +5,6 @@ flask_cas.__init__
import flask
from flask import current_app
# Find the stack on which we want to store the database connection.
# Starting with Flask 0.9, the _app_ctx_stack is the correct one,
# before that we need to use the _request_ctx_stack.
try:
from flask import _app_ctx_stack as stack
except ImportError:
from flask import _request_ctx_stack as stack
from . import routing
from functools import wraps
@ -67,7 +59,7 @@ class CAS(object):
app.teardown_request(self.teardown)
def teardown(self, exception):
ctx = stack.top
pass # ctx = stack.top
@property
def app(self):

16
migrations/versions/25e3ca6cc063_dispenseue_par_semestre.py
View File

@ -43,11 +43,14 @@ def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
dispenses = session.execute(
"""SELECT id, ue_id, etudid FROM "dispenseUE" WHERE formsemestre_id IS NULL;"""
sa.text(
"""SELECT id, ue_id, etudid FROM "dispenseUE" WHERE formsemestre_id IS NULL;"""
)
).all()
for dispense_id, ue_id, etudid in dispenses:
formsemestre_ids = session.execute(
"""
sa.text(
"""
SELECT notes_formsemestre.id
FROM notes_formsemestre, notes_formations, notes_ue, notes_formsemestre_inscription
WHERE notes_formsemestre.formation_id = notes_formations.id
@ -58,14 +61,17 @@ def upgrade():
and notes_formsemestre_inscription.etudid = :etudid
ORDER BY notes_formsemestre.date_debut DESC
LIMIT 1;
""",
"""
),
{"ue_id": ue_id, "etudid": etudid},
).all()
if formsemestre_ids:
formsemestre_id = formsemestre_ids[0][0]
session.execute(
"""
UPDATE "dispenseUE" SET formsemestre_id=:formsemestre_id WHERE id=:dispense_id""",
sa.text(
"""
UPDATE "dispenseUE" SET formsemestre_id=:formsemestre_id WHERE id=:dispense_id"""
),
{"formsemestre_id": formsemestre_id, "dispense_id": dispense_id},
)

4
migrations/versions/5542cac8c34a_semset_periode.py
View File

@ -23,7 +23,9 @@ def upgrade():
#
bind = op.get_bind()
session = Session(bind=bind)
session.execute("""UPDATE notes_semset SET sem_id=0 WHERE sem_id IS NULL;""")
session.execute(
sa.text("""UPDATE notes_semset SET sem_id=0 WHERE sem_id IS NULL;""")
)
op.alter_column(
"notes_semset", "sem_id", existing_type=sa.INTEGER(), nullable=False
)

18
migrations/versions/ae9bb0feea7a_contraintes_identite.py
View File

@ -29,21 +29,25 @@ def upgrade():
session = Session(bind=bind)
# Corrige NIP
dups = session.execute(
"""SELECT dept_id, code_nip
sa.text(
"""SELECT dept_id, code_nip
FROM identite
WHERE code_nip IS NOT NULL
GROUP BY dept_id, code_nip
HAVING COUNT(*) > 1;"""
)
).all()
for dept_id, code_nip in dups:
etuds_dups = session.execute(
"""SELECT id, nom, prenom FROM identite
WHERE dept_id=:dept_id AND code_nip=:code_nip""",
sa.text(
"""SELECT id, nom, prenom FROM identite
WHERE dept_id=:dept_id AND code_nip=:code_nip"""
),
{"dept_id": dept_id, "code_nip": code_nip},
).all()
for i, (etudid, nom, prenom) in enumerate(etuds_dups[1:], start=1):
session.execute(
"""UPDATE identite SET code_nip=:code_nip WHERE id=:etudid""",
sa.text("""UPDATE identite SET code_nip=:code_nip WHERE id=:etudid"""),
{
"code_nip": f"{code_nip}-{i}",
"etudid": etudid,
@ -55,11 +59,13 @@ def upgrade():
session.commit()
# Corrige INE
dups = session.execute(
"""SELECT dept_id, code_ine
sa.text(
"""SELECT dept_id, code_ine
FROM identite
WHERE code_ine IS NOT NULL
GROUP BY dept_id, code_ine
HAVING COUNT(*) > 1;"""
)
).all()
for dept_id, code_ine in dups:
etuds_dups = session.execute(
@ -69,7 +75,7 @@ def upgrade():
).all()
for i, (etudid, nom, prenom) in enumerate(etuds_dups[1:], start=1):
session.execute(
"""UPDATE identite SET code_ine=:code_ine WHERE id=:etudid""",
sa.text("""UPDATE identite SET code_ine=:code_ine WHERE id=:etudid"""),
{
"code_ine": f"{code_ine}-{i}",
"etudid": etudid,

2
migrations/versions/b9aadc10227f_module_type_non_null.py
View File

@ -23,7 +23,7 @@ def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute(
"""UPDATE notes_modules SET module_type=0 WHERE module_type IS NULL;"""
sa.text("""UPDATE notes_modules SET module_type=0 WHERE module_type IS NULL;""")
)
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column(

4
migrations/versions/d74b4e16fb3c_scodoc_9_0_51_add_unicity_constraint_on_.py
View File

@ -24,13 +24,15 @@ def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute(
"""
sa.text(
"""
DELETE FROM notes_moduleimpl_inscription i1
USING notes_moduleimpl_inscription i2
WHERE i1.id < i2.id
AND i1.moduleimpl_id = i2.moduleimpl_id
AND i1.etudid = i2.etudid;
"""
)
)
# ### commands auto generated by Alembic - please adjust! ###