Add cli: photos-import-files

This commit is contained in:
Emmanuel Viennet 2021-10-20 16:47:41 +02:00
parent e56a97eaf6
commit 66dbec86bf
6 changed files with 243 additions and 69 deletions

View File

@ -27,9 +27,12 @@
"""Various HTML generation functions """Various HTML generation functions
""" """
from html.parser import HTMLParser
from html.entities import name2codepoint
import re
from flask import g, url_for from flask import g, url_for
import app.scodoc.sco_utils as scu
from . import listhistogram from . import listhistogram
@ -130,3 +133,63 @@ def make_menu(title, items, css_class="", alone=False):
if alone: if alone:
H.append("</ul>") H.append("</ul>")
return "".join(H) return "".join(H)
"""
HTML <-> text conversions.
http://stackoverflow.com/questions/328356/extracting-text-from-html-file-using-python
"""
class _HTMLToText(HTMLParser):
def __init__(self):
HTMLParser.__init__(self)
self._buf = []
self.hide_output = False
def handle_starttag(self, tag, attrs):
if tag in ("p", "br") and not self.hide_output:
self._buf.append("\n")
elif tag in ("script", "style"):
self.hide_output = True
def handle_startendtag(self, tag, attrs):
if tag == "br":
self._buf.append("\n")
def handle_endtag(self, tag):
if tag == "p":
self._buf.append("\n")
elif tag in ("script", "style"):
self.hide_output = False
def handle_data(self, text):
if text and not self.hide_output:
self._buf.append(re.sub(r"\s+", " ", text))
def handle_entityref(self, name):
if name in name2codepoint and not self.hide_output:
c = chr(name2codepoint[name])
self._buf.append(c)
def handle_charref(self, name):
if not self.hide_output:
n = int(name[1:], 16) if name.startswith("x") else int(name)
self._buf.append(chr(n))
def get_text(self):
return re.sub(r" +", " ", "".join(self._buf))
def html_to_text(html):
"""
Given a piece of HTML, return the plain text it contains.
This handles entities and char refs, but not javascript and stylesheets.
"""
parser = _HTMLToText()
try:
parser.feed(html)
parser.close()
except: # HTMLParseError: No good replacement?
pass
return parser.get_text()

View File

@ -30,7 +30,8 @@
les dossiers d'admission et autres pièces utiles. les dossiers d'admission et autres pièces utiles.
""" """
import flask import flask
from flask import url_for, g, request from flask import url_for, render_template
from flask import g, request
from flask_login import current_user from flask_login import current_user
import app.scodoc.sco_utils as scu import app.scodoc.sco_utils as scu
@ -328,9 +329,9 @@ def etudarchive_import_files_form(group_id):
if tf[0] == 0: if tf[0] == 0:
return "\n".join(H) + tf[1] + "</li></ol>" + F return "\n".join(H) + tf[1] + "</li></ol>" + F
elif tf[0] == -1: # retrouve le semestre à partir du groupe:
# retrouve le semestre à partir du groupe: group = sco_groups.get_group(group_id)
group = sco_groups.get_group(group_id) if tf[0] == -1:
return flask.redirect( return flask.redirect(
url_for( url_for(
"notes.formsemestre_status", "notes.formsemestre_status",
@ -340,21 +341,41 @@ def etudarchive_import_files_form(group_id):
) )
else: else:
return etudarchive_import_files( return etudarchive_import_files(
group_id=tf[2]["group_id"], formsemestre_id=group["formsemestre_id"],
xlsfile=tf[2]["xlsfile"], xlsfile=tf[2]["xlsfile"],
zipfile=tf[2]["zipfile"], zipfile=tf[2]["zipfile"],
description=tf[2]["description"], description=tf[2]["description"],
) )
def etudarchive_import_files(group_id=None, xlsfile=None, zipfile=None, description=""): def etudarchive_import_files(
formsemestre_id=None, xlsfile=None, zipfile=None, description=""
):
"Importe des fichiers"
def callback(etud, data, filename): def callback(etud, data, filename):
_store_etud_file_to_new_archive(etud["etudid"], data, filename, description) _store_etud_file_to_new_archive(etud["etudid"], data, filename, description)
filename_title = "fichier_a_charger" # Utilise la fontion developpée au depart pour les photos
page_title = "Téléchargement de fichiers associés aux étudiants" (
# Utilise la fontion au depart developpee pour les photos ignored_zipfiles,
r = sco_trombino.zip_excel_import_files( unmatched_files,
xlsfile, zipfile, callback, filename_title, page_title stored_etud_filename,
) = sco_trombino.zip_excel_import_files(
xlsfile=xlsfile,
zipfile=zipfile,
callback=callback,
filename_title="fichier_a_charger",
)
return render_template(
"scolar/photos_import_files.html",
page_title="Téléchargement de fichiers associés aux étudiants",
ignored_zipfiles=ignored_zipfiles,
unmatched_files=unmatched_files,
stored_etud_filename=stored_etud_filename,
next_page=url_for(
"scolar.groups_view",
scodoc_dept=g.scodoc_dept,
formsemestre_id=formsemestre_id,
),
) )
return r + html_sco_header.sco_footer()

View File

@ -30,6 +30,7 @@
import io import io
from zipfile import ZipFile, BadZipfile from zipfile import ZipFile, BadZipfile
from flask.templating import render_template
import reportlab import reportlab
from reportlab.lib.units import cm, mm from reportlab.lib.units import cm, mm
from reportlab.lib.enums import TA_LEFT, TA_RIGHT, TA_CENTER, TA_JUSTIFY from reportlab.lib.enums import TA_LEFT, TA_RIGHT, TA_CENTER, TA_JUSTIFY
@ -531,25 +532,33 @@ def photos_import_files_form(group_ids=[]):
elif tf[0] == -1: elif tf[0] == -1:
return flask.redirect(back_url) return flask.redirect(back_url)
else: else:
return photos_import_files(
group_ids=tf[2]["group_ids"], def callback(etud, data, filename):
sco_photos.store_photo(etud, data)
(
ignored_zipfiles,
unmatched_files,
stored_etud_filename,
) = zip_excel_import_files(
xlsfile=tf[2]["xlsfile"], xlsfile=tf[2]["xlsfile"],
zipfile=tf[2]["zipfile"], zipfile=tf[2]["zipfile"],
callback=callback,
filename_title="fichier_photo",
)
return render_template(
"scolar/photos_import_files.html",
page_title="Téléchargement des photos des étudiants",
ignored_zipfiles=ignored_zipfiles,
unmatched_files=unmatched_files,
stored_etud_filename=stored_etud_filename,
next_page=url_for(
"scolar.groups_view",
scodoc_dept=g.scodoc_dept,
formsemestre_id=groups_infos.formsemestre_id,
curtab="tab-photos",
),
) )
def photos_import_files(group_ids=[], xlsfile=None, zipfile=None):
"""Importation des photos"""
groups_infos = sco_groups_view.DisplayedGroupsInfos(group_ids)
back_url = "groups_view?%s&curtab=tab-photos" % groups_infos.groups_query_args
filename_title = "fichier_photo"
page_title = "Téléchargement des photos des étudiants"
def callback(etud, data, filename):
sco_photos.store_photo(etud, data)
zip_excel_import_files(xlsfile, zipfile, callback, filename_title, page_title)
return flask.redirect(back_url + "&head_message=photos%20 importees")
def zip_excel_import_files( def zip_excel_import_files(
@ -557,19 +566,19 @@ def zip_excel_import_files(
zipfile=None, zipfile=None,
callback=None, callback=None,
filename_title="", # doit obligatoirement etre specifié filename_title="", # doit obligatoirement etre specifié
page_title="",
): ):
"""Importation de fichiers à partir d'un excel et d'un zip """Importation de fichiers à partir d'un excel et d'un zip
La fonction La fonction
callback() callback()
est appelé pour chaque fichier trouvé. est appelée pour chaque fichier trouvé.
Fonction utilisée pour les photos et les fichiers étudiants (archives).
""" """
# 1- build mapping etudid -> filename # 1- build mapping etudid -> filename
exceldata = xlsfile.read() exceldata = xlsfile.read()
if not exceldata: if not exceldata:
raise ScoValueError("Fichier excel vide ou invalide") raise ScoValueError("Fichier excel vide ou invalide")
_, data = sco_excel.excel_bytes_to_list(exceldata) _, data = sco_excel.excel_bytes_to_list(exceldata)
if not data: # probably a bug if not data:
raise ScoValueError("Fichier excel vide !") raise ScoValueError("Fichier excel vide !")
# on doit avoir une colonne etudid et une colonne filename_title ('fichier_photo') # on doit avoir une colonne etudid et une colonne filename_title ('fichier_photo')
titles = data[0] titles = data[0]
@ -591,30 +600,30 @@ def zip_excel_import_files(
fn = fn.split("/")[-1] # use only last component, not directories fn = fn.split("/")[-1] # use only last component, not directories
return fn return fn
Filename2Etud = {} # filename : etudid filename_to_etud = {} # filename : etudid
for l in data[1:]: for l in data[1:]:
filename = l[filename_idx].strip() filename = l[filename_idx].strip()
if filename: if filename:
Filename2Etud[normfilename(filename)] = l[etudid_idx] filename_to_etud[normfilename(filename)] = l[etudid_idx]
# 2- Ouvre le zip et # 2- Ouvre le zip et
try: try:
z = ZipFile(zipfile) z = ZipFile(zipfile)
except BadZipfile: except BadZipfile:
raise ScoValueError("Fichier ZIP incorrect !") raise ScoValueError("Fichier ZIP incorrect !") from BadZipfile
ignored_zipfiles = [] ignored_zipfiles = []
stored = [] # [ (etud, filename) ] stored_etud_filename = [] # [ (etud, filename) ]
for name in z.namelist(): for name in z.namelist():
if len(name) > 4 and name[-1] != "/" and "." in name: if len(name) > 4 and name[-1] != "/" and "." in name:
data = z.read(name) data = z.read(name)
# match zip filename with name given in excel # match zip filename with name given in excel
normname = normfilename(name) normname = normfilename(name)
if normname in Filename2Etud: if normname in filename_to_etud:
etudid = Filename2Etud[normname] etudid = filename_to_etud[normname]
# ok, store photo # ok, store photo
try: try:
etud = sco_etud.get_etud_info(etudid=etudid, filled=True)[0] etud = sco_etud.get_etud_info(etudid=etudid, filled=True)[0]
del Filename2Etud[normname] del filename_to_etud[normname]
except: except:
raise ScoValueError("ID étudiant invalide: %s" % etudid) raise ScoValueError("ID étudiant invalide: %s" % etudid)
@ -624,7 +633,7 @@ def zip_excel_import_files(
normfilename(name, lowercase=False), normfilename(name, lowercase=False),
) )
stored.append((etud, name)) stored_etud_filename.append((etud, name))
else: else:
log("zip: zip name %s not in excel !" % name) log("zip: zip name %s not in excel !" % name)
ignored_zipfiles.append(name) ignored_zipfiles.append(name)
@ -632,35 +641,9 @@ def zip_excel_import_files(
if name[-1] != "/": if name[-1] != "/":
ignored_zipfiles.append(name) ignored_zipfiles.append(name)
log("zip: ignoring %s" % name) log("zip: ignoring %s" % name)
if Filename2Etud: if filename_to_etud:
# lignes excel non traitées # lignes excel non traitées
unmatched_files = list(Filename2Etud.keys()) unmatched_files = list(filename_to_etud.keys())
else: else:
unmatched_files = [] unmatched_files = []
# 3- Result page return ignored_zipfiles, unmatched_files, stored_etud_filename
H = [
_trombino_html_header(),
"""<h2 class="formsemestre">%s</h2>
<h3>Opération effectuée</h3>
"""
% page_title,
]
if ignored_zipfiles:
H.append("<h4>Fichiers ignorés dans le zip:</h4><ul>")
for name in ignored_zipfiles:
H.append("<li>%s</li>" % name)
H.append("</ul>")
if unmatched_files:
H.append(
"<h4>Fichiers indiqués dans feuille mais non trouvés dans le zip:</h4><ul>"
)
for name in unmatched_files:
H.append("<li>%s</li>" % name)
H.append("</ul>")
if stored:
H.append("<h4>Fichiers chargés:</h4><ul>")
for (etud, name) in stored:
H.append("<li>%s: <tt>%s</tt></li>" % (etud["nomprenom"], name))
H.append("</ul>")
return "\n".join(H)

View File

@ -0,0 +1,39 @@
{% extends 'base.html' %}
{% block app_content %}
<h2 class="formsemestre">{{ page_title }}</h2>
<h3>Opération effectuée</h3>
{% if ignored_zipfiles %}
<h4>Fichiers ignorés dans le zip:</h4>
<ul>
{% for name in ignored_zipfiles %}
<li>{{name}}</li>
{% endfor %}
</ul>
{% endif %}
{% if unmatched_files %}
<h4>Fichiers indiqués dans la feuille mais non trouvés dans le zip:</h4>
<ul>
{% for name in unmatched_files %}
<li>{{name}}</li>
{% endfor %}
</ul>
{% endif %}
{% if stored_etud_filename %}
<h4>Fichiers chargés:</h4>
<ul>
{% for (etud, name) in stored_etud_filename %}
<li>{{etud["nomprenom"]}}: <tt>{{name}}</tt></li>
{% endfor %}
</ul>
{% endif %}
<div>
<p><a href="{{ next_page | safe }}">Continuer</a>
</div>
{% endblock %}

View File

@ -0,0 +1,23 @@
Importation des photo effectuée
{% if ignored_zipfiles %}
# Fichiers ignorés dans le zip:
{% for name in ignored_zipfiles %}
- {{name}}
{% endfor %}
{% endif %}
{% if unmatched_files %}
# Fichiers indiqués dans la feuille mais non trouvés dans le zip:
{% for name in unmatched_files %}
- {{name}}
{% endfor %}
{% endif %}
{% if stored_etud_filename %}
# Fichiers chargés:
{% for (etud, name) in stored_etud_filename %}
- {{etud["nomprenom"]}}: <tt>{{name}}</tt></li>
{% endfor %}
{% endif %}

View File

@ -13,6 +13,7 @@ import sys
import click import click
import flask import flask
from flask.cli import with_appcontext from flask.cli import with_appcontext
from flask.templating import render_template
from app import create_app, cli, db from app import create_app, cli, db
from app import initialize_scodoc_database from app import initialize_scodoc_database
@ -323,6 +324,50 @@ def migrate_scodoc7_dept_archive(dept: str): # migrate-scodoc7-dept-archive
tools.migrate_scodoc7_dept_archive(dept) tools.migrate_scodoc7_dept_archive(dept)
@app.cli.command()
@click.argument("formsemestre_id", type=click.INT)
@click.argument("xlsfile", type=click.File("rb"))
@click.argument("zipfile", type=click.File("rb"))
def photos_import_files(formsemestre_id: int, xlsfile: str, zipfile: str):
import app as mapp
from app.scodoc import sco_trombino, sco_photos
from app.scodoc import notesdb as ndb
from flask_login import login_user
from app.auth.models import get_super_admin
sem = mapp.models.formsemestre.FormSemestre.query.get(formsemestre_id)
if not sem:
sys.stderr.write("photos-import-files: numéro de semestre invalide\n")
return 2
with app.test_request_context():
mapp.set_sco_dept(sem.departement.acronym)
admin_user = get_super_admin()
login_user(admin_user)
def callback(etud, data, filename):
sco_photos.store_photo(etud, data)
(
ignored_zipfiles,
unmatched_files,
stored_etud_filename,
) = sco_trombino.zip_excel_import_files(
xlsfile=xlsfile,
zipfile=zipfile,
callback=callback,
filename_title="fichier_photo",
)
print(
render_template(
"scolar/photos_import_files.txt",
ignored_zipfiles=ignored_zipfiles,
unmatched_files=unmatched_files,
stored_etud_filename=stored_etud_filename,
)
)
@app.cli.command() @app.cli.command()
@with_appcontext @with_appcontext
def clear_cache(): # clear-cache def clear_cache(): # clear-cache