From d99b9aeb59d6c9e60e2733ab3c520a2d165650d7 Mon Sep 17 00:00:00 2001 From: Emmanuel Viennet Date: Wed, 18 Aug 2021 19:36:30 +0200 Subject: [PATCH] Fix script import --- tools/import_scodoc7_dept.py | 16 ++-- tools/import_scodoc7_user_db.py | 1 + tools/migrate_from_scodoc7.sh | 2 - tools/restore_scodoc_data.sh | 141 -------------------------------- tools/save_scodoc_data.sh | 70 ---------------- 5 files changed, 10 insertions(+), 220 deletions(-) delete mode 100755 tools/restore_scodoc_data.sh delete mode 100755 tools/save_scodoc_data.sh diff --git a/tools/import_scodoc7_dept.py b/tools/import_scodoc7_dept.py index 353e302b..be183716 100644 --- a/tools/import_scodoc7_dept.py +++ b/tools/import_scodoc7_dept.py @@ -116,9 +116,9 @@ def import_scodoc7_dept(dept_id: str, dept_db_uri=None): klass = get_class_for_table(table) t1 = time.time() n = convert_table(dept, cursor, id_from_scodoc7, klass, id_name, default_user) - logging.info(f" inserted {n} objects in {time.time()-t1}s.") + logging.info(f" inserted {n} objects in {time.time()-t1:3.2f}s.") - logging.info(f"All table imported in {time.time()-t0}s") + logging.info(f"All table imported in {time.time()-t0:3.2f}s") logging.info(f"clearing app caches...") clear_scodoc_cache() logging.info(f"Done.") @@ -236,7 +236,7 @@ def convert_table( if cnx: cnx.close() - db.session.commit() # écrit la table en une fois + db.session.commit() # écrit la table if has_id: update_table_sequence(table_name) return len(objects) @@ -339,9 +339,9 @@ def convert_object( statement = sqlalchemy.insert(klass).values(**obj) _ = cnx.execute(statement) else: - # new_obj = klass(**obj) # ORM object - # db.session.add(new_obj) - insert_object(cnx, table_name, obj) + new_obj = klass(**obj) # ORM object + db.session.add(new_obj) + # insert_object(cnx, table_name, obj) # Stocke l'id pour les références (foreign keys): if id_name and has_id: @@ -352,7 +352,9 @@ def convert_object( def insert_object(cnx, table_name: str, vals: dict) -> str: - """insert tuple in db""" + """insert tuple in db + version manuelle => ne semble pas plus rapide + """ cols = list(vals.keys()) colnames = ",".join(cols) fmt = ",".join(["%%(%s)s" % col for col in cols]) diff --git a/tools/import_scodoc7_user_db.py b/tools/import_scodoc7_user_db.py index 39827e87..aa2d1a0c 100644 --- a/tools/import_scodoc7_user_db.py +++ b/tools/import_scodoc7_user_db.py @@ -17,6 +17,7 @@ def import_scodoc7_user_db(scodoc7_db="dbname=SCOUSERS"): The resulting users are in SCO8USERS, handled via Flask/SQLAlchemy ORM. """ + current_app.logger.info("Importation des utilisateurs...") messages = [] cnx = psycopg2.connect(scodoc7_db) cursor = cnx.cursor(cursor_factory=psycopg2.extras.DictCursor) diff --git a/tools/migrate_from_scodoc7.sh b/tools/migrate_from_scodoc7.sh index abc5eeff..09663c38 100755 --- a/tools/migrate_from_scodoc7.sh +++ b/tools/migrate_from_scodoc7.sh @@ -104,8 +104,6 @@ echo "Les fichiers locaux de ScoDoc: configuration, photos, procès-verbaux..." echo "sont maintenant stockées dans $SCODOC_VAR_DIR" echo -echo "XXX STOPPING XXX" -exit 0 # ----- Migration base utilisateurs echo diff --git a/tools/restore_scodoc_data.sh b/tools/restore_scodoc_data.sh deleted file mode 100755 index 90a88f4f..00000000 --- a/tools/restore_scodoc_data.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash - -# -# ScoDoc: restore data (saved by save_scodoc_data) into current install -# -# Utile pour migrer ScoDoc d'un serveur a un autre -# A executer en tant que root sur le nouveau serveur -# -# E. Viennet, Sept 2011, Nov 2013, Mar 2017, Aug 2020, Jul 2021 -# - -# Le répertoire de ce script: -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -source "$SCRIPT_DIR/config.sh" -source "$SCRIPT_DIR/utils.sh" - -check_uid_root "$0" - -# Safety check -echo "Ce script va remplacer les donnees de votre installation ScoDoc par celles" -echo "enregistrees dans le fichier fourni." -echo "Ce fichier doit avoir ete cree par le script save_scodoc_data.sh, sur une autre machine." -echo -echo "Attention: TOUTES LES DONNEES DE CE SERVEUR SERONT REMPLACEES !" -echo "Notamment, tous les utilisateurs et departements existants seront effaces !" -echo -echo "TOUTES LES BASES POSTGRESQL SERONT EFFACEES !!!" -echo -echo -n "Voulez vous poursuivre cette operation ? (y/n) [n]" -read -r ans -if [ ! "$(norm_ans "$ans")" = 'Y' ] -then - echo "Annulation" - exit 1 -fi - -# Usage -if [ ! $# -eq 1 ] -then - echo "Usage: $0 directory_or_archive" - exit 1 -fi - -SRC=$1 - -if [ "${SRC:0:1}" != "/" ] -then - echo "Usage: $0 directory_or_archive" - echo "Erreur: utiliser un chemin absolu (commencant par /)" - exit 1 -fi - -# Source directory -if [ "${SRC##*.}" = 'tgz' ] -then - echo "Opening tgz archive..." - tmp=$(mktemp -d) - chmod a+rx "$tmp" - cd "$tmp" || terminate "directory error" - tar xfz "$SRC" - SRC=$(ls -1d "$tmp"/*) - IS_TMP=1 - # If source is a tgz, can use mv - COPY="mv" -else - IS_TMP=0 - # If source is a directory, does not modify its content - COPY="cp -rp" -fi - -echo "Source is $SRC" -echo "Stopping ScoDoc..." -scodocctl stop - -# Erase all postgres databases and load data -chmod a+rx "$SRC" -chmod a+r "$SRC"/scodoc.dump.txt -PG_DUMPFILE="$SRC/scodoc.dump.txt" - -su -c "$SCODOC_DIR/tools/psql_restore_databases.sh $PG_DUMPFILE" postgres - -# -echo Copying data files... - -rm -rf "${SCODOC_DIR:?}/var" -$COPY "$SRC/var" "$SCODOC_DIR" - -if [ ! -e "${SCODOC_VAR_DIR:?}/config/" ] -then - mkdir "${SCODOC_VAR_DIR:?}/config/" - chown "$SCODOC_USER"."$SCODOC_GROUP" "${SCODOC_VAR_DIR:?}/config/" - chmod 755 "${SCODOC_VAR_DIR:?}/config/" -fi - -rm -rf "${SCODOC_DIR:?}/config/depts" -if [ -e "${SRC:?}/depts" ] -then - # legacy depts => move them to var - $COPY "$SRC/depts" "${SCODOC_VAR_DIR}/config/" -fi - -rm -rf "$SCODOC_DIR/static/photos" -if [ -e "$SRC/photos" ] -then - # legacy photos (in /static/photos) => move them to var - $COPY "$SRC/photos" "${SCODOC_VAR_DIR}/" -fi - -rm -rf "${SCODOC_DIR:?}/logos" -$COPY "$SRC/logos" "$SCODOC_DIR/" - -mv "$SCODOC_DIR/config/scodoc_config.py" "$SCODOC_DIR/config/scodoc_config.py.$(date +%Y%m%d-%H%M%S)" -$COPY "$SRC/scodoc_config.py" "$SCODOC_DIR/config/" -# Verifie le codage de ce fichier: -if [ -z "$(file $SCODOC_DIR/config/scodoc_config.py | grep -i UTF-8)" ] -then - mv "$SCODOC_DIR/config/scodoc_config.py" "$SCODOC_DIR/config/scodoc_config.py.orig" - iconv -f iso8859-15 -t utf-8 "$SCODOC_DIR/config/scodoc_config.py.orig" > "$SCODOC_DIR/config/scodoc_config.py" -fi - -rm -rf "${SCODOC_DIR:?}/log" -$COPY "$SRC/log" "$SCODOC_DIR/" - -# Fix file ownership and access rights -chown -R "$SCODOC_USER"."$SCODOC_GROUP" "${SCODOC_DIR}" -chmod -R 755 "$SCODOC_DIR" - -# Remove tmp directory -if [ "$IS_TMP" = "1" ] -then - rm -rf "${tmp}" -fi - -# Mise a jour BD ScoDoc -cd ${SCODOC_DIR:?}/config || terminate "no config directory" -./upgrade.sh - -# -echo -scodocctl start \ No newline at end of file diff --git a/tools/save_scodoc_data.sh b/tools/save_scodoc_data.sh deleted file mode 100755 index 9dc7e6c1..00000000 --- a/tools/save_scodoc_data.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash - -# -# ScoDoc: save all user data (database, configs, images, archives...) in separate directory -# -# Utile pour migrer ScoDoc d'un serveur a un autre -# Executer en tant que root sur le serveur d'origine -# -# E. Viennet, Sept 2011, Aug 2020, Jul 2021 -# -# Le répertoire de ce script: -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -source "$SCRIPT_DIR/config.sh" -source "$SCRIPT_DIR/utils.sh" - - -check_uid_root "$0" - -# Destination directory -if [ ! $# -eq 1 ] -then - echo "Usage: $0 destination_directory" - exit 1 -fi -DEST=$1 -# remove trailing slashs if needed: -shopt -s extglob -DEST="${DEST%%+(/)}" - -if [ ! -e "$DEST" ] -then - echo Creating directory "$DEST" - mkdir "$DEST" -else - echo "Error: Directory " "$DEST" " exists" - echo "remove it or specify another destination !" - exit 2 -fi - -echo "Stopping ScoDoc..." -scodocctl stop - -# Dump all postgres databases -echo "Dumping SQL database..." -chown postgres "$DEST" -su -c "pg_dumpall > \"$DEST\"/scodoc.dump.txt" postgres -if [ ! "$?" -eq 0 ] -then - printf "Error dumping postgresql database\nPlease check that SQL server is running\nAborting.\n" - exit 1 -fi -chown root "$DEST" - -# ScoDoc archives, configuration, photos, etc. -echo "Copying var/ ..." -cp -rp "$SCODOC_DIR/var" "$DEST" - - -echo "Copying server logs..." -cp -rp "$SCODOC_DIR/log" "$DEST" - - -# --- Archive all files in a tarball to ease transfer -echo -echo "Archiving backup files in a $DEST.tgz..." -base=$(basename "$DEST") -(cd "$DEST"/.. || terminate "directory error"; tar cfz "$DEST".tgz "$base") - -echo "Done (you can copy " "$DEST"".tgz to destination machine)."