Modified install/upgrade scripts to use Flask-Migrate (Alembic)

This commit is contained in:
Emmanuel Viennet 2021-08-27 17:03:47 +02:00
parent 137a486329
commit 6090672089
14 changed files with 1457 additions and 19 deletions

View File

@ -130,6 +130,25 @@ un utilisateur:
**Attention:** les tests unitaires **effacent** complètement le contenu de la **Attention:** les tests unitaires **effacent** complètement le contenu de la
base de données (tous les départements, et les utilisateurs) avant de commencer ! base de données (tous les départements, et les utilisateurs) avant de commencer !
#### Modification du schéma de la base
On utilise SQLAlchemy avec Alembic et Flask-Migrate.
flask db migrate -m "ScoDoc 9.0.4" # ajuster le message !
flask db upgrade
Ne pas oublier de commiter les migrations (`git add migrations` ...).
Mémo pour développeurs: séquence re-création d'une base:
dropdb SCODOC_DEV
tools/create_database.sh SCODOC_DEV # créé base SQL
flask db upgrade # créé les tables à partir des migrations
flask sco-db-init # ajoute au besoin les constantes (todo: mettre en migration 0)
# puis imports:
flask import-scodoc7-users
flask import-scodoc7-dept STID SCOSTID
# Paquet debian 11 # Paquet debian 11

View File

@ -25,7 +25,7 @@ from config import DevConfig
import sco_version import sco_version
db = SQLAlchemy() db = SQLAlchemy()
migrate = Migrate() migrate = Migrate(compare_type=True)
login = LoginManager() login = LoginManager()
login.login_view = "auth.login" login.login_view = "auth.login"
login.login_message = "Please log in to access this page." login.login_message = "Please log in to access this page."
@ -140,7 +140,9 @@ def set_sco_dept(scodoc_dept: str):
def user_db_init(): def user_db_init():
"""Initialize the users database.""" """Initialize the users database.
Check that basic roles and admin user exist.
"""
from app.auth.models import User, Role from app.auth.models import User, Role
current_app.logger.info("Init User's db") current_app.logger.info("Init User's db")
@ -167,8 +169,8 @@ def user_db_init():
) )
def sco_db_init(): def sco_db_insert_constants():
"""Initialize Sco database""" """Initialize Sco database: insert some constants (modalités, ...)."""
from app import models from app import models
current_app.logger.info("Init Sco db") current_app.logger.info("Init Sco db")
@ -176,25 +178,25 @@ def sco_db_init():
models.NotesFormModalite.insert_modalites() models.NotesFormModalite.insert_modalites()
def initialize_scodoc_database(erase=False): def initialize_scodoc_database(erase=False, create_all=False):
"""Initialize the database. """Initialize the database for unit tests
Starts from an existing database and create all necessary Starts from an existing database and create all necessary
SQL tables and functions. SQL tables and functions.
If erase is True, _erase_ all database content. If erase is True, _erase_ all database content.
""" """
from app import models from app import models
# - our specific functions and sequences, not generated by SQLAlchemy
models.create_database_functions()
# - ERASE (the truncation sql function has been defined above) # - ERASE (the truncation sql function has been defined above)
if erase: if erase:
truncate_database() truncate_database()
# - Create all tables # - Create all tables
if create_all:
# managed by migrations, except for TESTS
db.create_all() db.create_all()
# - Insert initial roles and create super-admin user # - Insert initial roles and create super-admin user
user_db_init() user_db_init()
# - Insert some constant values (modalites, ...) # - Insert some constant values (modalites, ...)
sco_db_init() sco_db_insert_constants()
# - Flush cache # - Flush cache
clear_scodoc_cache() clear_scodoc_cache()

View File

@ -87,7 +87,8 @@ class NotesModule(db.Model):
module_id = db.synonym("id") module_id = db.synonym("id")
titre = db.Column(db.Text()) titre = db.Column(db.Text())
abbrev = db.Column(db.Text()) # nom court abbrev = db.Column(db.Text()) # nom court
code = db.Column(db.String(SHORT_STR_LEN), nullable=False) # certains départements ont des codes infiniment longs: donc Text !
code = db.Column(db.Text(), nullable=False)
heures_cours = db.Column(db.Float) heures_cours = db.Column(db.Float)
heures_td = db.Column(db.Float) heures_td = db.Column(db.Float)
heures_tp = db.Column(db.Float) heures_tp = db.Column(db.Float)

View File

@ -8,8 +8,12 @@ using raw SQL
from app import db from app import db
def create_database_functions(): def create_database_functions(): # XXX obsolete
"""Create specific SQL functions and sequences""" """Create specific SQL functions and sequences
XXX Obsolete: cette fonction est dans la première migration 9.0.3
Flask-Migrate fait maintenant (dans les versions >= 9.0.4) ce travail.
"""
# Important: toujours utiliser IF NOT EXISTS # Important: toujours utiliser IF NOT EXISTS
# car cette fonction peut être appelée plusieurs fois sur la même db # car cette fonction peut être appelée plusieurs fois sur la même db
db.session.execute( db.session.execute(

1
migrations/README Executable file
View File

@ -0,0 +1 @@
Single-database configuration for Flask.

50
migrations/alembic.ini Normal file
View File

@ -0,0 +1,50 @@
# A generic, single database configuration.
[alembic]
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic,flask_migrate
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[logger_flask_migrate]
level = INFO
handlers =
qualname = flask_migrate
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

91
migrations/env.py Executable file
View File

@ -0,0 +1,91 @@
from __future__ import with_statement
import logging
from logging.config import fileConfig
from flask import current_app
from alembic import context
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
logger = logging.getLogger('alembic.env')
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
config.set_main_option(
'sqlalchemy.url',
str(current_app.extensions['migrate'].db.get_engine().url).replace(
'%', '%%'))
target_metadata = current_app.extensions['migrate'].db.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url, target_metadata=target_metadata, literal_binds=True
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# this callback is used to prevent an auto-migration from being generated
# when there are no changes to the schema
# reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, 'autogenerate', False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
logger.info('No changes in schema detected.')
connectable = current_app.extensions['migrate'].db.get_engine()
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
process_revision_directives=process_revision_directives,
**current_app.extensions['migrate'].configure_args
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

24
migrations/script.py.mako Executable file
View File

@ -0,0 +1,24 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

View File

@ -0,0 +1,34 @@
"""ScoDoc 9.0.4: code module en Text
Revision ID: 6b071b7947e5
Revises: 993ce4a01d57
Create Date: 2021-08-27 16:00:27.322153
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '6b071b7947e5'
down_revision = '993ce4a01d57'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('notes_modules', 'code',
existing_type=sa.VARCHAR(length=32),
type_=sa.Text(),
existing_nullable=False)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('notes_modules', 'code',
existing_type=sa.Text(),
type_=sa.VARCHAR(length=32),
existing_nullable=False)
# ### end Alembic commands ###

File diff suppressed because it is too large Load Diff

View File

@ -66,7 +66,7 @@ def make_shell_context():
@app.cli.command() @app.cli.command()
def db_init(): # db-init def sco_db_init(): # sco-db-init
"""Initialize the database. """Initialize the database.
Starts from an existing database and create all Starts from an existing database and create all
the necessary SQL tables and functions. the necessary SQL tables and functions.

View File

@ -23,7 +23,7 @@ def test_client():
with apptest.app_context(): with apptest.app_context():
with apptest.test_request_context(): with apptest.test_request_context():
# erase and reset database: # erase and reset database:
initialize_scodoc_database(erase=True) initialize_scodoc_database(erase=True, create_all=True)
# Loge l'utilisateur super-admin # Loge l'utilisateur super-admin
admin_user = get_super_admin() admin_user = get_super_admin()
login_user(admin_user) login_user(admin_user)

View File

@ -121,7 +121,7 @@ then
echo echo
echo "Création des tables et du compte admin" echo "Création des tables et du compte admin"
echo echo
su -c "(cd /opt/scodoc; source venv/bin/activate; flask db-init; flask user-password admin)" "$SCODOC_USER" || die "Erreur: db-init" su -c "(cd /opt/scodoc; source venv/bin/activate; flask db upgrade; flask sco-db-init; flask user-password admin)" "$SCODOC_USER" || die "Erreur: sco-db-init"
echo echo
echo "base initialisée et admin créé." echo "base initialisée et admin créé."
echo echo

View File

@ -15,7 +15,7 @@ check_create_scodoc_user
# -- Répertoires /opt/scodoc donné à scodoc # -- Répertoires /opt/scodoc donné à scodoc
change_scodoc_file_ownership change_scodoc_file_ownership
# --- Création au bseoin de /opt/scodoc-data # --- Création au besoin de /opt/scodoc-data
set_scodoc_var_dir set_scodoc_var_dir
# ------------ LOCALES (pour compat bases ScoDoc 7 et plus anciennes) # ------------ LOCALES (pour compat bases ScoDoc 7 et plus anciennes)
@ -71,11 +71,11 @@ fi
# ------------ CREATION DU VIRTUALENV # ------------ CREATION DU VIRTUALENV
# donc re-créé sur le client à chaque install ou upgrade # donc re-créé sur le client à chaque install ou upgrade
#echo "Creating python3 virtualenv..." #echo "Creating python3 virtualenv..."
(cd $SCODOC_DIR && python3 -m venv venv) || die "Error creating Python 3 virtualenv" su -c "(cd $SCODOC_DIR && python3 -m venv venv)" "$SCODOC_USER" || die "Error creating Python 3 virtualenv"
# ------------ INSTALL DES PAQUETS PYTHON (3.9) # ------------ INSTALL DES PAQUETS PYTHON (3.9)
# pip in our env, as user "scodoc" # pip in our env, as user "scodoc"
(cd $SCODOC_DIR && source venv/bin/activate && pip install wheel && pip install -r requirements-3.9.txt) || die "Error installing python packages" su -c "(cd $SCODOC_DIR && source venv/bin/activate && pip install wheel && pip install -r requirements-3.9.txt)" "$SCODOC_USER" || die "Error installing python packages"
# --- NGINX # --- NGINX
if [ ! -L /etc/nginx/sites-enabled/scodoc9.nginx ] if [ ! -L /etc/nginx/sites-enabled/scodoc9.nginx ]
@ -89,6 +89,19 @@ fi
# --- Ensure postgres user "scodoc" ($POSTGRES_USER) exists # --- Ensure postgres user "scodoc" ($POSTGRES_USER) exists
init_postgres_user init_postgres_user
# ------------ BASE DE DONNEES
# gérées avec Flask-Migrate (Alembic/SQLAlchemy)
# Si la base SCODOC existe, tente de la mettre à jour
# (Ne gère pas les bases DEV et TEST)
n=$(su -c "psql -l | grep -c -E '^[[:blank:]]*SCODOC[[:blank:]]*\|'" "$SCODOC_USER")
if [ "$n" == 1 ]
then
echo "Upgrading existing SCODOC database..."
# utilise les scripts dans migrations/version/
# pour mettre à jour notre base (en tant qu'utilisateur scodoc)
export SQLALCHEMY_DATABASE_URI="postgresql:///SCODOC"
su -c "(cd $SCODOC_DIR && source venv/bin/activate && flask db upgrade)" "$SCODOC_USER"
fi
# ------------ CONFIG SERVICE SCODOC # ------------ CONFIG SERVICE SCODOC
echo echo