iziram/ScoDoc
1
0
Fork 0
forked from ScoDoc/ScoDoc
Code Issues 12 Pull Requests Projects 1 Releases Wiki Activity

mail auteur dump

Browse Source
This commit is contained in:
Emmanuel Viennet 2022-10-03 11:59:38 +02:00 committed by iziram
parent 53ffff4983
commit 3a37ffa0b0
1 changed files with 227 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

454
app/scodoc/sco_dump_db.py
View File

@ -1,227 +1,227 @@
# -*- mode: python -*- # -*- mode: python -*-
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
############################################################################## ##############################################################################
# #
# Gestion scolarite IUT # Gestion scolarite IUT
# #
# Copyright (c) 1999 - 2022 Emmanuel Viennet. All rights reserved. # Copyright (c) 1999 - 2022 Emmanuel Viennet. All rights reserved.
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by # it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or # the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# This program is distributed in the hope that it will be useful, # This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software # along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
# #
# Emmanuel Viennet emmanuel.viennet@viennet.net # Emmanuel Viennet emmanuel.viennet@viennet.net
# #
############################################################################## ##############################################################################
"""Dump base de données pour debug et support technique """Dump base de données pour debug et support technique
Le principe est le suivant: Le principe est le suivant:
1- S'il existe une base en cours d'anonymisation, s'arrête et affiche un msg d'erreur l'utilisateur, 1- S'il existe une base en cours d'anonymisation, s'arrête et affiche un msg d'erreur l'utilisateur,
qui peut décider de la supprimer. qui peut décider de la supprimer.
2- ScoDoc lance un script qui duplique la base (la copie de SCORT devient ANORT) 2- ScoDoc lance un script qui duplique la base (la copie de SCORT devient ANORT)
- (si elle existe deja, s'arrête) - (si elle existe deja, s'arrête)
createdb -E UTF-8 ANORT createdb -E UTF-8 ANORT
pg_dump SCORT | psql ANORT pg_dump SCORT | psql ANORT
3- ScoDoc lance le script d'anonymisation config/anonymize_db.py qui: 3- ScoDoc lance le script d'anonymisation config/anonymize_db.py qui:
- vide ou anonymise certaines colonnes - vide ou anonymise certaines colonnes
- dump cette base modifiée - dump cette base modifiée
- supprime cette base. - supprime cette base.
4- La copie dump anonymisé est uploadée. 4- La copie dump anonymisé est uploadée.
""" """
import base64 import base64
import fcntl import fcntl
import os import os
import requests import requests
import subprocess import subprocess
import traceback import traceback
from flask import g, request from flask import g, request
from flask_login import current_user from flask_login import current_user
import app.scodoc.notesdb as ndb import app.scodoc.notesdb as ndb
import app.scodoc.sco_utils as scu import app.scodoc.sco_utils as scu
from app import log from app import log
from app.scodoc import sco_users from app.scodoc import sco_users
import sco_version import sco_version
from app.scodoc.sco_exceptions import ScoValueError from app.scodoc.sco_exceptions import ScoValueError
SCO_DUMP_LOCK = "/tmp/scodump.lock" SCO_DUMP_LOCK = "/tmp/scodump.lock"
def sco_dump_and_send_db( def sco_dump_and_send_db(
message: str = "", request_url: str = "", traceback_str_base64: str = "" message: str = "", request_url: str = "", traceback_str_base64: str = ""
): ):
"""Dump base de données et l'envoie anonymisée pour debug""" """Dump base de données et l'envoie anonymisée pour debug"""
traceback_str = base64.urlsafe_b64decode(traceback_str_base64).decode( traceback_str = base64.urlsafe_b64decode(traceback_str_base64).decode(
scu.SCO_ENCODING scu.SCO_ENCODING
) )
# get current (dept) DB name: # get current (dept) DB name:
cursor = ndb.SimpleQuery("SELECT current_database()", {}) cursor = ndb.SimpleQuery("SELECT current_database()", {})
db_name = cursor.fetchone()[0] db_name = cursor.fetchone()[0]
ano_db_name = "ANO" + db_name ano_db_name = "ANO" + db_name
# Lock # Lock
try: try:
x = open(SCO_DUMP_LOCK, "w+") x = open(SCO_DUMP_LOCK, "w+")
fcntl.flock(x, fcntl.LOCK_EX | fcntl.LOCK_NB) fcntl.flock(x, fcntl.LOCK_EX | fcntl.LOCK_NB)
except (IOError, OSError): except (IOError, OSError):
raise ScoValueError( raise ScoValueError(
"Un envoi de la base " "Un envoi de la base "
+ db_name + db_name
+ " est déjà en cours, re-essayer plus tard" + " est déjà en cours, re-essayer plus tard"
) )
try: try:
# Drop if exists # Drop if exists
_drop_ano_db(ano_db_name) _drop_ano_db(ano_db_name)
# Duplicate database # Duplicate database
_duplicate_db(db_name, ano_db_name) _duplicate_db(db_name, ano_db_name)
# Anonymisation # Anonymisation
_anonymize_db(ano_db_name) _anonymize_db(ano_db_name)
# Send # Send
r = _send_db(ano_db_name, message, request_url, traceback_str=traceback_str) r = _send_db(ano_db_name, message, request_url, traceback_str=traceback_str)
code = r.status_code code = r.status_code
finally: finally:
# Drop anonymized database # Drop anonymized database
# XXX _drop_ano_db(ano_db_name) # XXX _drop_ano_db(ano_db_name)
# Remove lock # Remove lock
fcntl.flock(x, fcntl.LOCK_UN) fcntl.flock(x, fcntl.LOCK_UN)
log("sco_dump_and_send_db: done.") log("sco_dump_and_send_db: done.")
return code return code
def _duplicate_db(db_name, ano_db_name): def _duplicate_db(db_name, ano_db_name):
"""Create new database, and copy old one into""" """Create new database, and copy old one into"""
cmd = ["createdb", "-E", "UTF-8", ano_db_name] cmd = ["createdb", "-E", "UTF-8", ano_db_name]
log("sco_dump_and_send_db/_duplicate_db: {}".format(cmd)) log("sco_dump_and_send_db/_duplicate_db: {}".format(cmd))
try: try:
_ = subprocess.check_output(cmd) _ = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
log("sco_dump_and_send_db: exception createdb {}".format(e)) log("sco_dump_and_send_db: exception createdb {}".format(e))
raise ScoValueError( raise ScoValueError(
"erreur lors de la creation de la base {}".format(ano_db_name) "erreur lors de la creation de la base {}".format(ano_db_name)
) )
cmd = "pg_dump {} | psql {}".format(db_name, ano_db_name) cmd = "pg_dump {} | psql {}".format(db_name, ano_db_name)
log("sco_dump_and_send_db/_duplicate_db: {}".format(cmd)) log("sco_dump_and_send_db/_duplicate_db: {}".format(cmd))
try: try:
_ = subprocess.check_output(cmd, shell=1) _ = subprocess.check_output(cmd, shell=1)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
log("sco_dump_and_send_db: exception {}".format(e)) log("sco_dump_and_send_db: exception {}".format(e))
raise ScoValueError( raise ScoValueError(
"erreur lors de la duplication de la base {} vers {}".format( "erreur lors de la duplication de la base {} vers {}".format(
db_name, ano_db_name db_name, ano_db_name
) )
) )
def _anonymize_db(ano_db_name): def _anonymize_db(ano_db_name):
"""Anonymize a departement database""" """Anonymize a departement database"""
cmd = os.path.join(scu.SCO_TOOLS_DIR, "anonymize_db.py") cmd = os.path.join(scu.SCO_TOOLS_DIR, "anonymize_db.py")
log("_anonymize_db: {}".format(cmd)) log("_anonymize_db: {}".format(cmd))
try: try:
_ = subprocess.check_output([cmd, ano_db_name]) _ = subprocess.check_output([cmd, ano_db_name])
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
log("sco_dump_and_send_db: exception in anonymisation: {}".format(e)) log("sco_dump_and_send_db: exception in anonymisation: {}".format(e))
raise ScoValueError( raise ScoValueError(
"erreur lors de l'anonymisation de la base {}".format(ano_db_name) "erreur lors de l'anonymisation de la base {}".format(ano_db_name)
) )
def _get_scodoc_serial(): def _get_scodoc_serial():
try: try:
with open(os.path.join(scu.SCODOC_VERSION_DIR, "scodoc.sn")) as f: with open(os.path.join(scu.SCODOC_VERSION_DIR, "scodoc.sn")) as f:
return int(f.read()) return int(f.read())
except: except:
return 0 return 0
def _send_db( def _send_db(
ano_db_name: str, message: str = "", request_url: str = "", traceback_str: str = "" ano_db_name: str, message: str = "", request_url: str = "", traceback_str: str = ""
): ):
"""Dump this (anonymized) database and send it to tech support""" """Dump this (anonymized) database and send it to tech support"""
log(f"dumping anonymized database {ano_db_name}") log(f"dumping anonymized database {ano_db_name}")
try: try:
dump = subprocess.check_output( dump = subprocess.check_output(
f"pg_dump --format=custom {ano_db_name}", shell=1 f"pg_dump --format=custom {ano_db_name}", shell=1
) )
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
log(f"sco_dump_and_send_db: exception in anonymisation: {e}") log(f"sco_dump_and_send_db: exception in anonymisation: {e}")
raise ScoValueError( raise ScoValueError(
f"erreur lors de l'anonymisation de la base {ano_db_name}" f"erreur lors de l'anonymisation de la base {ano_db_name}"
) from e ) from e
log(f"traceback_str={traceback_str}") log(f"traceback_str={traceback_str}")
log("uploading anonymized dump...") log("uploading anonymized dump...")
files = {"file": (ano_db_name + ".dump", dump)} files = {"file": (ano_db_name + ".dump", dump)}
try: try:
r = requests.post( r = requests.post(
scu.SCO_DUMP_UP_URL, scu.SCO_DUMP_UP_URL,
files=files, files=files,
data={ data={
"dept_name": getattr(g, "scodoc_dept", "-"), "dept_name": getattr(g, "scodoc_dept", "-"),
"message": message or "", "message": message or "",
"request_url": request_url or request.url, "request_url": request_url or request.url,
"serial": _get_scodoc_serial(), "serial": _get_scodoc_serial(),
"sco_user": str(current_user), "sco_user": str(current_user),
"sent_by": sco_users.user_info(str(current_user))["nomcomplet"], "sent_by": f'"{current_user.get_nomcomplet()}" <{current_user.email}>',
"sco_version": sco_version.SCOVERSION, "sco_version": sco_version.SCOVERSION,
"sco_fullversion": scu.get_scodoc_version(), "sco_fullversion": scu.get_scodoc_version(),
"traceback_str": traceback_str, "traceback_str": traceback_str,
}, },
) )
except requests.exceptions.ConnectionError as exc: except requests.exceptions.ConnectionError as exc:
log("ConnectionError: Impossible de joindre le serveur d'assistance") log("ConnectionError: Impossible de joindre le serveur d'assistance")
raise ScoValueError( raise ScoValueError(
""" """
Impossible de joindre le serveur d'assistance (scodoc.org). Impossible de joindre le serveur d'assistance (scodoc.org).
Veuillez contacter le service informatique de votre établissement pour Veuillez contacter le service informatique de votre établissement pour
corriger la configuration de ScoDoc. Dans la plupart des cas, il corriger la configuration de ScoDoc. Dans la plupart des cas, il
s'agit d'un proxy mal configuré. s'agit d'un proxy mal configuré.
""" """
) from exc ) from exc
return r return r
def _drop_ano_db(ano_db_name): def _drop_ano_db(ano_db_name):
"""drop temp database if it exists""" """drop temp database if it exists"""
existing_databases = [ existing_databases = [
s.split("|")[0].strip() s.split("|")[0].strip()
for s in subprocess.check_output(["psql", "-l"]) for s in subprocess.check_output(["psql", "-l"])
.decode(scu.SCO_ENCODING) .decode(scu.SCO_ENCODING)
.split("\n")[3:] .split("\n")[3:]
] ]
if ano_db_name not in existing_databases: if ano_db_name not in existing_databases:
log("_drop_ano_db: no temp db, nothing to drop") log("_drop_ano_db: no temp db, nothing to drop")
return return
cmd = ["dropdb", ano_db_name] cmd = ["dropdb", ano_db_name]
log("sco_dump_and_send_db: {}".format(cmd)) log("sco_dump_and_send_db: {}".format(cmd))
try: try:
_ = subprocess.check_output(cmd) _ = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
log("sco_dump_and_send_db: exception dropdb {}".format(e)) log("sco_dump_and_send_db: exception dropdb {}".format(e))
raise ScoValueError( raise ScoValueError(
"erreur lors de la suppression de la base {}".format(ano_db_name) "erreur lors de la suppression de la base {}".format(ano_db_name)
) )