From 4869d111ef13ff651c694883b9805bb60a0b2e0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sun, 19 Jan 2025 16:00:32 +0100 Subject: [PATCH 01/33] add write permission to md and ipynb files (when files submission are manually downloaded (because of a gitlab breakdown) they come with read permission only) --- travo/jupyter_course.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 91f2e1fc..dcc36d6b 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -198,10 +198,15 @@ class JupyterCourse(Course): ) continue ipynbname = mdname[:-3] + ".ipynb" - self.log.info(f"Converting {mdname} to {ipynbname}") - notebook = jupytext.read(mdname) - jupytext.write(notebook, ipynbname) - # run(["jupytext", mdname, "--to ipynb"]) + if not os.path.exists(ipynbname): + self.log.info(f"Converting {mdname} to {ipynbname}") + notebook = jupytext.read(mdname) + jupytext.write(notebook, ipynbname) + else: + self.log.info(f"{ipynbname} already exists") + # ensure user writing permission + os.chmod(ipynbname, 0o644) + os.chmod(mdname, 0o644) self.log.info("Updating cross-links to other notebooks (.md->.ipynb)") with open(ipynbname, "r") as file: filedata = file.read() @@ -216,9 +221,15 @@ class JupyterCourse(Course): for ipynbname in glob.glob(os.path.join(path, "*.ipynb")): mdname = ipynbname[:-6] + ".md" - self.log.info(f"Converting {ipynbname} to {mdname}") - notebook = jupytext.read(ipynbname) - jupytext.write(notebook, mdname) + if not os.path.exists(mdname): + self.log.info(f"Converting {ipynbname} to {mdname}") + notebook = jupytext.read(ipynbname) + jupytext.write(notebook, mdname) + else: + self.log.info(f"{mdname} already exists") + # ensure user writing permission + os.chmod(ipynbname, 0o644) + os.chmod(mdname, 0o644) self.log.info("Updating cross-links to other notebooks (.ipynb->.md)") with open(mdname, "r") as file: filedata = file.read() -- GitLab From d7e804bdd1fa68bd0239618f23c0ce659048b94d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sun, 19 Jan 2025 16:01:44 +0100 Subject: [PATCH 02/33] generate_assignment_content: check if folder is a dit folder before converting it --- travo/jupyter_course.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index dcc36d6b..a43e0806 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -258,11 +258,13 @@ class JupyterCourse(Course): release_path = assignment.release_path() self.convert_from_md_to_ipynb(path=source_path) with tempfile.TemporaryDirectory() as tmpdirname: - gitdir = os.path.join(release_path, ".git") - tmpgitdir = os.path.join(tmpdirname, ".git") db = os.path.join(release_path, ".gradebook.db") - self.log.info("Sauvegarde de l'historique git") - shutil.move(gitdir, tmpgitdir) + gitdir = os.path.join(release_path, ".git") + is_git = os.path.exists(gitdir) + if is_git: + tmpgitdir = os.path.join(tmpdirname, ".git") + self.log.info("Sauvegarde de l'historique git") + shutil.move(gitdir, tmpgitdir) assignment_basename = os.path.basename(assignment_name) try: run( -- GitLab From 5221e57a9a432be6dee20a09778eed4fff187b1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sun, 19 Jan 2025 16:02:30 +0100 Subject: [PATCH 03/33] generate_assignment_content: check if folder is a dit folder before converting it --- travo/jupyter_course.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index a43e0806..62a1907e 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -282,10 +282,11 @@ class JupyterCourse(Course): except (): pass finally: - self.log.info("Restauration de l'historique git") - # In case the target_path has been destroyed and not recreated - os.makedirs(release_path, exist_ok=True) - shutil.move(tmpgitdir, gitdir) + if is_git: + self.log.info("Restauration de l'historique git") + # In case the target_path has been destroyed and not recreated + os.makedirs(release_path, exist_ok=True) + shutil.move(tmpgitdir, gitdir) if add_gitlab_ci and self.gitlab_ci_yml is not None: io.open(os.path.join(release_path, ".gitlab-ci.yml"), "w").write( self.gitlab_ci_yml.format(assignment=assignment_basename) -- GitLab From 72259ebfd2cb2bb0f2a6004f35777288679347eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sun, 19 Jan 2025 16:02:55 +0100 Subject: [PATCH 04/33] autograde: default argument '*' for tag --- travo/jupyter_course.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 62a1907e..69a2ea48 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -342,7 +342,7 @@ class JupyterCourse(Course): nbgrader_config.append("--CourseDirectory.submitted_directory=submitted") return nbgrader_config - def autograde(self, assignment_name: str, tag: str) -> None: + def autograde(self, assignment_name: str, tag: str = "*") -> None: """ Autograde the assignment for the given student. The student name can be given with wildcard. -- GitLab From 4b995b15e6cfce1622adbe5e3fc96401f5789e45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sun, 19 Jan 2025 16:02:30 +0100 Subject: [PATCH 05/33] move to no_gitlab branch --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index 04ee53dc..4979a47f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ travo.code-workspace +.idea code/artifacts.zip forkedRepositories/froidevaux.pierre_thomas/activity.json .vscode/settings.json @@ -14,7 +15,13 @@ report.xml coverage.xml .coverage +# local git tests +.gitlab-ci-local* + # local rendered documentation docs/build .spyproject docs/sources/examples/*.ipynb + +# gradebooks +*gradebook.db* -- GitLab From 6f760a703578eeee068fd1cd005015a649fd5b16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sun, 19 Jan 2025 16:29:30 +0100 Subject: [PATCH 06/33] utf8 decoding for markdown files to deal with some accents --- travo/jupyter_course.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index a20e39d2..65258393 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -190,7 +190,7 @@ class JupyterCourse(Course): import jupytext # type: ignore for mdname in glob.glob(os.path.join(path, "*.md")): - with io.open(mdname) as fd: + with io.open(mdname, encoding="utf-8") as fd: if "nbgrader" not in fd.read(): self.log.debug( "Skip markdown file/notebook with no nbgrader metadata:" -- GitLab From 676f5cd98d36640e2ba5693e5fcdbb78ca0e436f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sun, 19 Jan 2025 20:22:11 +0100 Subject: [PATCH 07/33] autograde function to perform locally what is done with student_autograde in CI (in case submissions are manually downloaded) --- travo/jupyter_course.py | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 65258393..9cebf095 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -349,7 +349,6 @@ class JupyterCourse(Course): """ run(["nbgrader", "--version"]) assignment = os.path.basename(assignment_name) - nbgrader_config = self.get_nbgrader_config() self.nbgrader_update_student_list(tag=tag) for student_id in os.listdir("submitted"): if tag.replace("*", "") not in student_id: @@ -358,14 +357,36 @@ class JupyterCourse(Course): os.path.join("submitted", student_id, assignment) ) self.convert_from_md_to_ipynb(assignment) - run( - [ - "nbgrader", - "autograde", - *nbgrader_config, - os.path.basename(assignment_name), - f"--student={tag}", - ] + cwd = os.getcwd() + os.makedirs("autograded", exist_ok=True) + for submitted_dir in sorted( + glob.glob(f"submitted/{tag}/{os.path.basename(assignment_name)}") + ): + student = submitted_dir.split("/")[1] + autograded_dir = os.path.join( + "autograded", student, os.path.basename(assignment_name) + ) + # make a full copy of submitted dir, including .git repo + shutil.rmtree(autograded_dir, ignore_errors=True) + os.makedirs(autograded_dir, exist_ok=True) + shutil.copytree(submitted_dir, autograded_dir, dirs_exist_ok=True) + # make student_autograde as in CI + os.chdir(submitted_dir) + self.student_autograde(assignment_name=assignment_name, student=student) + # get autograded files result + shutil.copytree( + autograded_dir, os.path.join(cwd, autograded_dir), dirs_exist_ok=True + ) + # clean the submitted repo from nbgrader artifacts and leave the submitted repo + for subdir in ["submitted", "autograded", "feedback", "feedback_generated"]: + shutil.rmtree(subdir) + os.chdir(cwd) + self.collect_autograded_post( + assignment_name=os.path.basename(assignment_name), + tag=tag, + autograded=True, + on_inconsistency="ERROR", + new_score_policy="only_empty", ) def generate_feedback( -- GitLab From 9b6d21e4b8695e04f3774a46062d5f4ee98f2bbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sun, 19 Jan 2025 21:31:21 +0100 Subject: [PATCH 08/33] create local_autograde and nbgrader_autograde methods --- travo/jupyter_course.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 9cebf095..802f07d0 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -342,7 +342,33 @@ class JupyterCourse(Course): nbgrader_config.append("--CourseDirectory.submitted_directory=submitted") return nbgrader_config - def autograde(self, assignment_name: str, tag: str = "*") -> None: + def nbgrader_autograde(self, assignment_name: str, tag: str = "*") -> None: + """ + Autograde the assignment for the given student. The student name can be given + with wildcard. + """ + run(["nbgrader", "--version"]) + assignment = os.path.basename(assignment_name) + nbgrader_config = self.get_nbgrader_config() + self.nbgrader_update_student_list(tag=tag) + for student_id in os.listdir("submitted"): + if tag.replace("*", "") not in student_id: + continue + self.convert_from_md_to_ipynb( + os.path.join("submitted", student_id, assignment) + ) + self.convert_from_md_to_ipynb(assignment) + run( + [ + "nbgrader", + "autograde", + *nbgrader_config, + os.path.basename(assignment_name), + f"--student={tag}", + ] + ) + + def local_autograde(self, assignment_name: str, tag: str = "*") -> None: """ Autograde the assignment for the given student. The student name can be given with wildcard. -- GitLab From 1af5a84d7f8749af9e66189556983467d45768f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sun, 19 Jan 2025 22:33:50 +0100 Subject: [PATCH 09/33] add forge_autograde method --- travo/jupyter_course.py | 92 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 802f07d0..17ccf8db 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -22,6 +22,7 @@ from .assignment import SubmissionStatus from .course import Course from travo.i18n import _ from .utils import run +from .nbgrader_utils import remove_submission_gradebook, Gradebook if TYPE_CHECKING: from . import dashboards @@ -415,6 +416,97 @@ class JupyterCourse(Course): new_score_policy="only_empty", ) + def forge_autograde( + self, + assignment_name: str, + tag: Optional[str] = None, + new_score_policy: str = "only_empty", + ) -> None: + """ + Autograde the student's assignments + + Submit the corrected assignment from submitted, wait for the student_autograde + and collect the student gradebook. + + Examples: + + Autograde all students submissions in the current directory:: + + course.forge_autograde("Assignment1") + + Autograde all students submissions for a given student group, + laying them out in nbgrader's format, with the student's group + appended to the username: + + course.forge_autograde("Assignment1", + student_group="MP2") + """ + failed = [] + self.forge.login() + for assignment_dir in glob.glob( + f"submitted/{tag}/{os.path.basename(assignment_name)}" + ): + student = assignment_dir.split("/")[1] + # get student group + project = self.assignment(assignment_name, username=student) + submission = project.submission() + assert ( + submission.repo.forked_from_project is not None + ), "a student assignment should be a fork" + student_group = submission.repo.forked_from_project.namespace.name + self.log.info(f"Student: {student} (group {student_group})") + remove_submission_gradebook( + Gradebook("sqlite:///.gradebook.db"), + os.path.basename(assignment_name), + student, + ) + # re-submit the submitted + self.log.info("- Enregistrement des changements:") + self.forge.ensure_local_git_configuration(dir=os.getcwd()) + if ( + self.forge.git( + [ + "commit", + "--all", + "-m", + f"Correction par {self.forge.get_current_user().username}", + ], + check=False, + cwd=assignment_dir, + ).returncode + != 0 + ): + self.log.info(" Pas de changement à enregistrer") + + self.log.info("- Envoi des changements:") + branch = submission.repo.default_branch + url = submission.repo.web_url + self.forge.git(["push", url, branch], cwd=assignment_dir) + # Force an update of origin/master (or whichever the origin default branch) + # self.forge.git(["update-ref", f"refs/remotes/origin/{branch}", branch]) + self.log.info( + f"- Nouvelle soumission effectuée. " + f"Vous pouvez consulter le dépôt: {url}" + ) + # autograde + try: + job = submission.ensure_autograded(force_autograde=True) + except RuntimeError as e: + self.log.warning(e) + failed.append(assignment_dir) + continue + # collect gradebooks + self.log.info(f"fetch autograded for {assignment_dir}") + submission.fetch_artifacts(job, path=".", prefix="") + self.merge_autograded_db( + assignment_name=os.path.basename(assignment_name), + tag=tag, + on_inconsistency="WARNING", + new_score_policy=new_score_policy, + ) + if failed: + self.log.warning(f"Failed autograde: {' '.join(failed)}") + def generate_feedback( self, assignment_name: str, tag: str = "*", new_score_policy: str = "only_empty" ) -> None: -- GitLab From 27637cf613cff11eb587d44dd070e6dc0560e5db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Mon, 20 Jan 2025 10:11:50 +0100 Subject: [PATCH 10/33] update documentation and repair mypy typings --- travo/jupyter_course.py | 50 +++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 17ccf8db..eb56510d 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -345,8 +345,22 @@ class JupyterCourse(Course): def nbgrader_autograde(self, assignment_name: str, tag: str = "*") -> None: """ - Autograde the assignment for the given student. The student name can be given - with wildcard. + Run nbgrader autograde for the assignment of the given student. + Student submissions must follow nbgrader convention and be in `submitted` directory. + Student notebooks can be either md or ipynb files. + + The student name can be given with wildcard. + + Examples: + + Autograde all students submissions in the submitted directory:: + + course.nbgrader_autograde("Assignment1") + + Autograde submissions for a given student: + + course.nbgrader_autograde("Assignment1", + tag="firstname.lastname") """ run(["nbgrader", "--version"]) assignment = os.path.basename(assignment_name) @@ -371,8 +385,22 @@ class JupyterCourse(Course): def local_autograde(self, assignment_name: str, tag: str = "*") -> None: """ - Autograde the assignment for the given student. The student name can be given - with wildcard. + Autograde the assignment for the given student locally in the autograded folder. + Student submissions must follow nbgrader convention and be in `submitted` directory. + Gradebooks .gradebook.db files are produced in the `autograded` folder. + + The student name can be given with wildcard. + + Examples: + + Autograde all students submissions in the submitted directory:: + + course.local_autograde("Assignment1") + + Autograde submissions for a given student: + + course.local_autograde("Assignment1", + tag="firstname.lastname") """ run(["nbgrader", "--version"]) assignment = os.path.basename(assignment_name) @@ -419,11 +447,11 @@ class JupyterCourse(Course): def forge_autograde( self, assignment_name: str, - tag: Optional[str] = None, + tag: str = "*", new_score_policy: str = "only_empty", ) -> None: """ - Autograde the student's assignments + Autograde the student's assignments directly on the forge Submit the corrected assignment from submitted, wait for the student_autograde and collect the student gradebook. @@ -434,12 +462,10 @@ class JupyterCourse(Course): course.forge_autograde("Assignment1") - Autograde all students submissions for a given student group, - laying them out in nbgrader's format, with the student's group - appended to the username: + Autograde all students submissions for a given student: course.forge_autograde("Assignment1", - student_group="MP2") + tag="firstname.lastname") """ failed = [] self.forge.login() @@ -480,7 +506,7 @@ class JupyterCourse(Course): self.log.info("- Envoi des changements:") branch = submission.repo.default_branch - url = submission.repo.web_url + url = str(submission.repo.web_url) self.forge.git(["push", url, branch], cwd=assignment_dir) # Force an update of origin/master (or whichever the origin default branch) # self.forge.git(["update-ref", f"refs/remotes/origin/{branch}", branch]) @@ -497,7 +523,7 @@ class JupyterCourse(Course): continue # collect gradebooks self.log.info(f"fetch autograded for {assignment_dir}") - submission.fetch_artifacts(job, path=".", prefix="") + submission.repo.fetch_artifacts(job, path=".", prefix="") self.merge_autograded_db( assignment_name=os.path.basename(assignment_name), tag=tag, -- GitLab From 3e8babb0adc0ff385eebabff9b0d8e3e3f8db926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Wed, 5 Feb 2025 17:08:20 +0100 Subject: [PATCH 11/33] generate an entry in the local db when generating the assignment --- travo/jupyter_course.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index eb56510d..551f8cd7 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -22,7 +22,11 @@ from .assignment import SubmissionStatus from .course import Course from travo.i18n import _ from .utils import run -from .nbgrader_utils import remove_submission_gradebook, Gradebook +from .nbgrader_utils import ( + remove_submission_gradebook, + Gradebook, + merge_assignment_gradebook, +) if TYPE_CHECKING: from . import dashboards @@ -279,6 +283,11 @@ class JupyterCourse(Course): f"--db='sqlite:///{db}'", ] ) + self.log.info(f"Importation de {db} dans .gradebook.db") + merge_assignment_gradebook( + source=Gradebook(f"sqlite:///{db}"), + target=Gradebook("sqlite:///.gradebook.db"), + ) self.convert_from_ipynb_to_md(path=release_path) except (): pass -- GitLab From 2d09754eaa2973f44d10dc411c2d21e730519c87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Fri, 21 Mar 2025 15:45:29 +0100 Subject: [PATCH 12/33] convert into ipynb after download in submitted --- travo/jupyter_course.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 551f8cd7..48872c4d 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -775,6 +775,24 @@ class JupyterCourse(Course): if failed: self.log.warning(f"Failed force autograde: {' '.join(failed)}") + def collect_in_submitted( + self, assignment_name: str, student_group: Optional[str] = None + ) -> None: + """ + Collect the student's submissions following nbgrader's standard organization + and convert markdown into ipynb notebooks. + + This wrapper for `collect`: + - forces a login; + - reports more information to the user (at a cost); + - stores the output in the subdirectory `submitted/`, + following nbgrader's standard organization. + + This is used by the course dashboard. + """ + super().collect_in_submitted(assignment_name, student_group) + self.convert_from_md_to_ipynb(path=f"submitted/*/{assignment_name}") + def collect_status( self, assignment_name: str, student_group: Optional[str] = None ) -> List[SubmissionStatus]: -- GitLab From d637b9e567fcf68195d66ca8dafa12d7e9cc86fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Fri, 21 Mar 2025 15:47:45 +0100 Subject: [PATCH 13/33] generate assignment first in local gradebbok for local correction and then in release assignment --- travo/jupyter_course.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 48872c4d..0c9329f8 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -272,6 +272,17 @@ class JupyterCourse(Course): shutil.move(gitdir, tmpgitdir) assignment_basename = os.path.basename(assignment_name) try: + run( + [ + "nbgrader", + "generate_assignment", + "--force", + f"--CourseDirectory.source_directory={self.source_directory}", + f"--CourseDirectory.release_directory={self.release_directory}", + assignment_basename, + "--db='sqlite:///.gradebook.db'", + ] + ) run( [ "nbgrader", -- GitLab From a837b8c4277e978029551b57083816e3d754cd5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Fri, 21 Mar 2025 17:04:39 +0100 Subject: [PATCH 14/33] generate assignment first in local gradebbok for local correction and then in release assignment --- travo/jupyter_course.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 0c9329f8..a478a3d9 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -295,10 +295,10 @@ class JupyterCourse(Course): ] ) self.log.info(f"Importation de {db} dans .gradebook.db") - merge_assignment_gradebook( - source=Gradebook(f"sqlite:///{db}"), - target=Gradebook("sqlite:///.gradebook.db"), - ) + # merge_assignment_gradebook( + # source=Gradebook(f"sqlite:///{db}"), + # target=Gradebook("sqlite:///.gradebook.db"), + # ) self.convert_from_ipynb_to_md(path=release_path) except (): pass @@ -959,7 +959,6 @@ class JupyterCourse(Course): from nbgrader.api import Gradebook, MissingEntry from .nbgrader_utils import ( merge_submission_gradebook, - merge_assignment_gradebook, ) target = Gradebook("sqlite:///.gradebook.db") -- GitLab From 43e8014f786e75aab1883f24996f34883027a8cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sat, 22 Mar 2025 16:14:05 +0100 Subject: [PATCH 15/33] to stick to nbgrader pipeline: convert md to ipynb after downloading submission to submitted --- travo/jupyter_course.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index a478a3d9..2a459b57 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -802,7 +802,9 @@ class JupyterCourse(Course): This is used by the course dashboard. """ super().collect_in_submitted(assignment_name, student_group) - self.convert_from_md_to_ipynb(path=f"submitted/*/{assignment_name}") + self.convert_from_md_to_ipynb( + path=f"submitted/*/{os.path.basename(assignment_name)}" + ) def collect_status( self, assignment_name: str, student_group: Optional[str] = None -- GitLab From 8c6ec416497db8d4a66ac6f75ca3fa3079d614d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sat, 22 Mar 2025 16:15:17 +0100 Subject: [PATCH 16/33] to benefit from original nbgrader autograde function, kernelspec must be specified --- travo/nbgrader_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/travo/nbgrader_utils.py b/travo/nbgrader_utils.py index a47e34ed..fccf5ff4 100644 --- a/travo/nbgrader_utils.py +++ b/travo/nbgrader_utils.py @@ -27,6 +27,9 @@ def merge_assignment_gradebook(source: Gradebook, target: Gradebook) -> None: for notebook in assignment.notebooks: args, kwargs = to_args(notebook, ["name"]) + # kernelspec must be specified for nbgrader autograde + if notebook.kernelspec is not None: + kwargs["kernelspec"] = notebook.kernelspec target.update_or_create_notebook(*args, assignment.name, **kwargs) for cell in notebook.grade_cells: args, kwargs = to_args(cell, ["name", "notebook", "assignment"]) @@ -65,6 +68,7 @@ def merge_submission_gradebook( for assignment in source.assignments: for submission in source.assignment_submissions(assignment.name): args, kwargs = to_args(submission, ["student"]) + print(args, kwargs) del kwargs["first_name"] del kwargs["last_name"] # Don't pass in kwargs, because only the student name is @@ -90,7 +94,6 @@ def merge_submission_gradebook( continue else: raise - if back: grade, target_grade = target_grade, grade args, kwargs = to_args( -- GitLab From ee7bd1fa845d83136e80a4126ca2e67129a712f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Mon, 14 Apr 2025 13:07:12 +0200 Subject: [PATCH 17/33] cleaning nbgrader_utils --- travo/nbgrader_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travo/nbgrader_utils.py b/travo/nbgrader_utils.py index fccf5ff4..24c52bd3 100644 --- a/travo/nbgrader_utils.py +++ b/travo/nbgrader_utils.py @@ -68,7 +68,6 @@ def merge_submission_gradebook( for assignment in source.assignments: for submission in source.assignment_submissions(assignment.name): args, kwargs = to_args(submission, ["student"]) - print(args, kwargs) del kwargs["first_name"] del kwargs["last_name"] # Don't pass in kwargs, because only the student name is @@ -94,6 +93,7 @@ def merge_submission_gradebook( continue else: raise + if back: grade, target_grade = target_grade, grade args, kwargs = to_args( -- GitLab From b0218932147a1db5571b2369118f9d3020849618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sun, 19 Jan 2025 16:00:32 +0100 Subject: [PATCH 18/33] add write permission to md and ipynb files (when files submission are manually downloaded (because of a gitlab breakdown) they come with read permission only) --- travo/jupyter_course.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 68ce96af..09867e30 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -198,10 +198,15 @@ class JupyterCourse(Course): ) continue ipynbname = mdname[:-3] + ".ipynb" - self.log.info(f"Converting {mdname} to {ipynbname}") - notebook = jupytext.read(mdname) - jupytext.write(notebook, ipynbname) - # run(["jupytext", mdname, "--to ipynb"]) + if not os.path.exists(ipynbname): + self.log.info(f"Converting {mdname} to {ipynbname}") + notebook = jupytext.read(mdname) + jupytext.write(notebook, ipynbname) + else: + self.log.info(f"{ipynbname} already exists") + # ensure user writing permission + os.chmod(ipynbname, 0o644) + os.chmod(mdname, 0o644) self.log.info("Updating cross-links to other notebooks (.md->.ipynb)") with open(ipynbname, "r") as file: filedata = file.read() @@ -216,9 +221,15 @@ class JupyterCourse(Course): for ipynbname in glob.glob(os.path.join(path, "*.ipynb")): mdname = ipynbname[:-6] + ".md" - self.log.info(f"Converting {ipynbname} to {mdname}") - notebook = jupytext.read(ipynbname) - jupytext.write(notebook, mdname) + if not os.path.exists(mdname): + self.log.info(f"Converting {ipynbname} to {mdname}") + notebook = jupytext.read(ipynbname) + jupytext.write(notebook, mdname) + else: + self.log.info(f"{mdname} already exists") + # ensure user writing permission + os.chmod(ipynbname, 0o644) + os.chmod(mdname, 0o644) self.log.info("Updating cross-links to other notebooks (.ipynb->.md)") with open(mdname, "r") as file: filedata = file.read() -- GitLab From ed1ce112cfd81c4f82d7c1b4b2058030261c86e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sun, 19 Jan 2025 16:01:44 +0100 Subject: [PATCH 19/33] generate_assignment_content: check if folder is a dit folder before converting it --- travo/jupyter_course.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 09867e30..1cb26f67 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -258,11 +258,13 @@ class JupyterCourse(Course): release_path = assignment.release_path() self.convert_from_md_to_ipynb(path=source_path) with tempfile.TemporaryDirectory() as tmpdirname: - gitdir = os.path.join(release_path, ".git") - tmpgitdir = os.path.join(tmpdirname, ".git") db = os.path.join(release_path, ".gradebook.db") - self.log.info("Sauvegarde de l'historique git") - shutil.move(gitdir, tmpgitdir) + gitdir = os.path.join(release_path, ".git") + is_git = os.path.exists(gitdir) + if is_git: + tmpgitdir = os.path.join(tmpdirname, ".git") + self.log.info("Sauvegarde de l'historique git") + shutil.move(gitdir, tmpgitdir) assignment_basename = os.path.basename(assignment_name) try: run( -- GitLab From a2a0cfb2d2e996ceb2e078cf1253212a5e8b4540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sun, 19 Jan 2025 16:02:30 +0100 Subject: [PATCH 20/33] generate_assignment_content: check if folder is a dit folder before converting it --- travo/jupyter_course.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 1cb26f67..bfa5e725 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -282,10 +282,11 @@ class JupyterCourse(Course): except (): pass finally: - self.log.info("Restauration de l'historique git") - # In case the target_path has been destroyed and not recreated - os.makedirs(release_path, exist_ok=True) - shutil.move(tmpgitdir, gitdir) + if is_git: + self.log.info("Restauration de l'historique git") + # In case the target_path has been destroyed and not recreated + os.makedirs(release_path, exist_ok=True) + shutil.move(tmpgitdir, gitdir) if add_gitlab_ci and self.gitlab_ci_yml is not None: io.open(os.path.join(release_path, ".gitlab-ci.yml"), "w").write( self.gitlab_ci_yml.format(assignment=assignment_basename) -- GitLab From f9e2515c66b485cb4bc1545f1c8b153f070db2ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sun, 19 Jan 2025 16:02:55 +0100 Subject: [PATCH 21/33] autograde: default argument '*' for tag --- travo/jupyter_course.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index bfa5e725..a20e39d2 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -342,7 +342,7 @@ class JupyterCourse(Course): nbgrader_config.append("--CourseDirectory.submitted_directory=submitted") return nbgrader_config - def autograde(self, assignment_name: str, tag: str) -> None: + def autograde(self, assignment_name: str, tag: str = "*") -> None: """ Autograde the assignment for the given student. The student name can be given with wildcard. -- GitLab From e419985259308365de6e08191854fb0bc3f2130b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sun, 19 Jan 2025 16:29:30 +0100 Subject: [PATCH 22/33] utf8 decoding for markdown files to deal with some accents --- travo/jupyter_course.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index a20e39d2..65258393 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -190,7 +190,7 @@ class JupyterCourse(Course): import jupytext # type: ignore for mdname in glob.glob(os.path.join(path, "*.md")): - with io.open(mdname) as fd: + with io.open(mdname, encoding="utf-8") as fd: if "nbgrader" not in fd.read(): self.log.debug( "Skip markdown file/notebook with no nbgrader metadata:" -- GitLab From 6fa82c6e89125a0abdc6c765b8f317aa77f399d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sun, 19 Jan 2025 20:22:11 +0100 Subject: [PATCH 23/33] autograde function to perform locally what is done with student_autograde in CI (in case submissions are manually downloaded) --- travo/jupyter_course.py | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 65258393..9cebf095 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -349,7 +349,6 @@ class JupyterCourse(Course): """ run(["nbgrader", "--version"]) assignment = os.path.basename(assignment_name) - nbgrader_config = self.get_nbgrader_config() self.nbgrader_update_student_list(tag=tag) for student_id in os.listdir("submitted"): if tag.replace("*", "") not in student_id: @@ -358,14 +357,36 @@ class JupyterCourse(Course): os.path.join("submitted", student_id, assignment) ) self.convert_from_md_to_ipynb(assignment) - run( - [ - "nbgrader", - "autograde", - *nbgrader_config, - os.path.basename(assignment_name), - f"--student={tag}", - ] + cwd = os.getcwd() + os.makedirs("autograded", exist_ok=True) + for submitted_dir in sorted( + glob.glob(f"submitted/{tag}/{os.path.basename(assignment_name)}") + ): + student = submitted_dir.split("/")[1] + autograded_dir = os.path.join( + "autograded", student, os.path.basename(assignment_name) + ) + # make a full copy of submitted dir, including .git repo + shutil.rmtree(autograded_dir, ignore_errors=True) + os.makedirs(autograded_dir, exist_ok=True) + shutil.copytree(submitted_dir, autograded_dir, dirs_exist_ok=True) + # make student_autograde as in CI + os.chdir(submitted_dir) + self.student_autograde(assignment_name=assignment_name, student=student) + # get autograded files result + shutil.copytree( + autograded_dir, os.path.join(cwd, autograded_dir), dirs_exist_ok=True + ) + # clean the submitted repo from nbgrader artifacts and leave the submitted repo + for subdir in ["submitted", "autograded", "feedback", "feedback_generated"]: + shutil.rmtree(subdir) + os.chdir(cwd) + self.collect_autograded_post( + assignment_name=os.path.basename(assignment_name), + tag=tag, + autograded=True, + on_inconsistency="ERROR", + new_score_policy="only_empty", ) def generate_feedback( -- GitLab From a16b753af508bc12896d9cda7b1e7677b4d926c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sun, 19 Jan 2025 21:31:21 +0100 Subject: [PATCH 24/33] create local_autograde and nbgrader_autograde methods --- travo/jupyter_course.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 9cebf095..802f07d0 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -342,7 +342,33 @@ class JupyterCourse(Course): nbgrader_config.append("--CourseDirectory.submitted_directory=submitted") return nbgrader_config - def autograde(self, assignment_name: str, tag: str = "*") -> None: + def nbgrader_autograde(self, assignment_name: str, tag: str = "*") -> None: + """ + Autograde the assignment for the given student. The student name can be given + with wildcard. + """ + run(["nbgrader", "--version"]) + assignment = os.path.basename(assignment_name) + nbgrader_config = self.get_nbgrader_config() + self.nbgrader_update_student_list(tag=tag) + for student_id in os.listdir("submitted"): + if tag.replace("*", "") not in student_id: + continue + self.convert_from_md_to_ipynb( + os.path.join("submitted", student_id, assignment) + ) + self.convert_from_md_to_ipynb(assignment) + run( + [ + "nbgrader", + "autograde", + *nbgrader_config, + os.path.basename(assignment_name), + f"--student={tag}", + ] + ) + + def local_autograde(self, assignment_name: str, tag: str = "*") -> None: """ Autograde the assignment for the given student. The student name can be given with wildcard. -- GitLab From 17ea28e8009087bf73c05dfd42ef0bfb7de8672d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sun, 19 Jan 2025 22:33:50 +0100 Subject: [PATCH 25/33] add forge_autograde method --- travo/jupyter_course.py | 92 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 802f07d0..17ccf8db 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -22,6 +22,7 @@ from .assignment import SubmissionStatus from .course import Course from travo.i18n import _ from .utils import run +from .nbgrader_utils import remove_submission_gradebook, Gradebook if TYPE_CHECKING: from . import dashboards @@ -415,6 +416,97 @@ class JupyterCourse(Course): new_score_policy="only_empty", ) + def forge_autograde( + self, + assignment_name: str, + tag: Optional[str] = None, + new_score_policy: str = "only_empty", + ) -> None: + """ + Autograde the student's assignments + + Submit the corrected assignment from submitted, wait for the student_autograde + and collect the student gradebook. + + Examples: + + Autograde all students submissions in the current directory:: + + course.forge_autograde("Assignment1") + + Autograde all students submissions for a given student group, + laying them out in nbgrader's format, with the student's group + appended to the username: + + course.forge_autograde("Assignment1", + student_group="MP2") + """ + failed = [] + self.forge.login() + for assignment_dir in glob.glob( + f"submitted/{tag}/{os.path.basename(assignment_name)}" + ): + student = assignment_dir.split("/")[1] + # get student group + project = self.assignment(assignment_name, username=student) + submission = project.submission() + assert ( + submission.repo.forked_from_project is not None + ), "a student assignment should be a fork" + student_group = submission.repo.forked_from_project.namespace.name + self.log.info(f"Student: {student} (group {student_group})") + remove_submission_gradebook( + Gradebook("sqlite:///.gradebook.db"), + os.path.basename(assignment_name), + student, + ) + # re-submit the submitted + self.log.info("- Enregistrement des changements:") + self.forge.ensure_local_git_configuration(dir=os.getcwd()) + if ( + self.forge.git( + [ + "commit", + "--all", + "-m", + f"Correction par {self.forge.get_current_user().username}", + ], + check=False, + cwd=assignment_dir, + ).returncode + != 0 + ): + self.log.info(" Pas de changement à enregistrer") + + self.log.info("- Envoi des changements:") + branch = submission.repo.default_branch + url = submission.repo.web_url + self.forge.git(["push", url, branch], cwd=assignment_dir) + # Force an update of origin/master (or whichever the origin default branch) + # self.forge.git(["update-ref", f"refs/remotes/origin/{branch}", branch]) + self.log.info( + f"- Nouvelle soumission effectuée. " + f"Vous pouvez consulter le dépôt: {url}" + ) + # autograde + try: + job = submission.ensure_autograded(force_autograde=True) + except RuntimeError as e: + self.log.warning(e) + failed.append(assignment_dir) + continue + # collect gradebooks + self.log.info(f"fetch autograded for {assignment_dir}") + submission.fetch_artifacts(job, path=".", prefix="") + self.merge_autograded_db( + assignment_name=os.path.basename(assignment_name), + tag=tag, + on_inconsistency="WARNING", + new_score_policy=new_score_policy, + ) + if failed: + self.log.warning(f"Failed autograde: {' '.join(failed)}") + def generate_feedback( self, assignment_name: str, tag: str = "*", new_score_policy: str = "only_empty" ) -> None: -- GitLab From 5051bdfcf982ad0703d2b567cca8025a27f384fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Mon, 20 Jan 2025 10:11:50 +0100 Subject: [PATCH 26/33] update documentation and repair mypy typings --- travo/jupyter_course.py | 50 +++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 17ccf8db..eb56510d 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -345,8 +345,22 @@ class JupyterCourse(Course): def nbgrader_autograde(self, assignment_name: str, tag: str = "*") -> None: """ - Autograde the assignment for the given student. The student name can be given - with wildcard. + Run nbgrader autograde for the assignment of the given student. + Student submissions must follow nbgrader convention and be in `submitted` directory. + Student notebooks can be either md or ipynb files. + + The student name can be given with wildcard. + + Examples: + + Autograde all students submissions in the submitted directory:: + + course.nbgrader_autograde("Assignment1") + + Autograde submissions for a given student: + + course.nbgrader_autograde("Assignment1", + tag="firstname.lastname") """ run(["nbgrader", "--version"]) assignment = os.path.basename(assignment_name) @@ -371,8 +385,22 @@ class JupyterCourse(Course): def local_autograde(self, assignment_name: str, tag: str = "*") -> None: """ - Autograde the assignment for the given student. The student name can be given - with wildcard. + Autograde the assignment for the given student locally in the autograded folder. + Student submissions must follow nbgrader convention and be in `submitted` directory. + Gradebooks .gradebook.db files are produced in the `autograded` folder. + + The student name can be given with wildcard. + + Examples: + + Autograde all students submissions in the submitted directory:: + + course.local_autograde("Assignment1") + + Autograde submissions for a given student: + + course.local_autograde("Assignment1", + tag="firstname.lastname") """ run(["nbgrader", "--version"]) assignment = os.path.basename(assignment_name) @@ -419,11 +447,11 @@ class JupyterCourse(Course): def forge_autograde( self, assignment_name: str, - tag: Optional[str] = None, + tag: str = "*", new_score_policy: str = "only_empty", ) -> None: """ - Autograde the student's assignments + Autograde the student's assignments directly on the forge Submit the corrected assignment from submitted, wait for the student_autograde and collect the student gradebook. @@ -434,12 +462,10 @@ class JupyterCourse(Course): course.forge_autograde("Assignment1") - Autograde all students submissions for a given student group, - laying them out in nbgrader's format, with the student's group - appended to the username: + Autograde all students submissions for a given student: course.forge_autograde("Assignment1", - student_group="MP2") + tag="firstname.lastname") """ failed = [] self.forge.login() @@ -480,7 +506,7 @@ class JupyterCourse(Course): self.log.info("- Envoi des changements:") branch = submission.repo.default_branch - url = submission.repo.web_url + url = str(submission.repo.web_url) self.forge.git(["push", url, branch], cwd=assignment_dir) # Force an update of origin/master (or whichever the origin default branch) # self.forge.git(["update-ref", f"refs/remotes/origin/{branch}", branch]) @@ -497,7 +523,7 @@ class JupyterCourse(Course): continue # collect gradebooks self.log.info(f"fetch autograded for {assignment_dir}") - submission.fetch_artifacts(job, path=".", prefix="") + submission.repo.fetch_artifacts(job, path=".", prefix="") self.merge_autograded_db( assignment_name=os.path.basename(assignment_name), tag=tag, -- GitLab From 06ccc43f86bfb6f7a704b73846f66cc7634c2aa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Wed, 5 Feb 2025 17:08:20 +0100 Subject: [PATCH 27/33] generate an entry in the local db when generating the assignment --- travo/jupyter_course.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index eb56510d..551f8cd7 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -22,7 +22,11 @@ from .assignment import SubmissionStatus from .course import Course from travo.i18n import _ from .utils import run -from .nbgrader_utils import remove_submission_gradebook, Gradebook +from .nbgrader_utils import ( + remove_submission_gradebook, + Gradebook, + merge_assignment_gradebook, +) if TYPE_CHECKING: from . import dashboards @@ -279,6 +283,11 @@ class JupyterCourse(Course): f"--db='sqlite:///{db}'", ] ) + self.log.info(f"Importation de {db} dans .gradebook.db") + merge_assignment_gradebook( + source=Gradebook(f"sqlite:///{db}"), + target=Gradebook("sqlite:///.gradebook.db"), + ) self.convert_from_ipynb_to_md(path=release_path) except (): pass -- GitLab From 028d607e13ffbda4e389b4cf4f5cf3273dfc1461 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Fri, 21 Mar 2025 15:45:29 +0100 Subject: [PATCH 28/33] convert into ipynb after download in submitted --- travo/jupyter_course.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 551f8cd7..48872c4d 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -775,6 +775,24 @@ class JupyterCourse(Course): if failed: self.log.warning(f"Failed force autograde: {' '.join(failed)}") + def collect_in_submitted( + self, assignment_name: str, student_group: Optional[str] = None + ) -> None: + """ + Collect the student's submissions following nbgrader's standard organization + and convert markdown into ipynb notebooks. + + This wrapper for `collect`: + - forces a login; + - reports more information to the user (at a cost); + - stores the output in the subdirectory `submitted/`, + following nbgrader's standard organization. + + This is used by the course dashboard. + """ + super().collect_in_submitted(assignment_name, student_group) + self.convert_from_md_to_ipynb(path=f"submitted/*/{assignment_name}") + def collect_status( self, assignment_name: str, student_group: Optional[str] = None ) -> List[SubmissionStatus]: -- GitLab From 3e0bed79e6cd6da27a6e95408796b84d70bbe1cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Fri, 21 Mar 2025 15:47:45 +0100 Subject: [PATCH 29/33] generate assignment first in local gradebbok for local correction and then in release assignment --- travo/jupyter_course.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 48872c4d..0c9329f8 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -272,6 +272,17 @@ class JupyterCourse(Course): shutil.move(gitdir, tmpgitdir) assignment_basename = os.path.basename(assignment_name) try: + run( + [ + "nbgrader", + "generate_assignment", + "--force", + f"--CourseDirectory.source_directory={self.source_directory}", + f"--CourseDirectory.release_directory={self.release_directory}", + assignment_basename, + "--db='sqlite:///.gradebook.db'", + ] + ) run( [ "nbgrader", -- GitLab From 92891fbed9add0b012caa272ea96871f2beb6a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Fri, 21 Mar 2025 17:04:39 +0100 Subject: [PATCH 30/33] generate assignment first in local gradebbok for local correction and then in release assignment --- travo/jupyter_course.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index 0c9329f8..a478a3d9 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -295,10 +295,10 @@ class JupyterCourse(Course): ] ) self.log.info(f"Importation de {db} dans .gradebook.db") - merge_assignment_gradebook( - source=Gradebook(f"sqlite:///{db}"), - target=Gradebook("sqlite:///.gradebook.db"), - ) + # merge_assignment_gradebook( + # source=Gradebook(f"sqlite:///{db}"), + # target=Gradebook("sqlite:///.gradebook.db"), + # ) self.convert_from_ipynb_to_md(path=release_path) except (): pass @@ -959,7 +959,6 @@ class JupyterCourse(Course): from nbgrader.api import Gradebook, MissingEntry from .nbgrader_utils import ( merge_submission_gradebook, - merge_assignment_gradebook, ) target = Gradebook("sqlite:///.gradebook.db") -- GitLab From f9492e8134118d682c522b9dfeee21de1be0db49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sat, 22 Mar 2025 16:14:05 +0100 Subject: [PATCH 31/33] to stick to nbgrader pipeline: convert md to ipynb after downloading submission to submitted --- travo/jupyter_course.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index a478a3d9..2a459b57 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -802,7 +802,9 @@ class JupyterCourse(Course): This is used by the course dashboard. """ super().collect_in_submitted(assignment_name, student_group) - self.convert_from_md_to_ipynb(path=f"submitted/*/{assignment_name}") + self.convert_from_md_to_ipynb( + path=f"submitted/*/{os.path.basename(assignment_name)}" + ) def collect_status( self, assignment_name: str, student_group: Optional[str] = None -- GitLab From 23884efdf068b595973f045d4398264ed71a7c6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Sat, 22 Mar 2025 16:15:17 +0100 Subject: [PATCH 32/33] to benefit from original nbgrader autograde function, kernelspec must be specified --- travo/nbgrader_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/travo/nbgrader_utils.py b/travo/nbgrader_utils.py index a47e34ed..fccf5ff4 100644 --- a/travo/nbgrader_utils.py +++ b/travo/nbgrader_utils.py @@ -27,6 +27,9 @@ def merge_assignment_gradebook(source: Gradebook, target: Gradebook) -> None: for notebook in assignment.notebooks: args, kwargs = to_args(notebook, ["name"]) + # kernelspec must be specified for nbgrader autograde + if notebook.kernelspec is not None: + kwargs["kernelspec"] = notebook.kernelspec target.update_or_create_notebook(*args, assignment.name, **kwargs) for cell in notebook.grade_cells: args, kwargs = to_args(cell, ["name", "notebook", "assignment"]) @@ -65,6 +68,7 @@ def merge_submission_gradebook( for assignment in source.assignments: for submission in source.assignment_submissions(assignment.name): args, kwargs = to_args(submission, ["student"]) + print(args, kwargs) del kwargs["first_name"] del kwargs["last_name"] # Don't pass in kwargs, because only the student name is @@ -90,7 +94,6 @@ def merge_submission_gradebook( continue else: raise - if back: grade, target_grade = target_grade, grade args, kwargs = to_args( -- GitLab From 515d2a0c5c329f94da82b2e2aac999976135c798 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Mon, 14 Apr 2025 13:07:12 +0200 Subject: [PATCH 33/33] cleaning nbgrader_utils --- travo/nbgrader_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travo/nbgrader_utils.py b/travo/nbgrader_utils.py index fccf5ff4..24c52bd3 100644 --- a/travo/nbgrader_utils.py +++ b/travo/nbgrader_utils.py @@ -68,7 +68,6 @@ def merge_submission_gradebook( for assignment in source.assignments: for submission in source.assignment_submissions(assignment.name): args, kwargs = to_args(submission, ["student"]) - print(args, kwargs) del kwargs["first_name"] del kwargs["last_name"] # Don't pass in kwargs, because only the student name is @@ -94,6 +93,7 @@ def merge_submission_gradebook( continue else: raise + if back: grade, target_grade = target_grade, grade args, kwargs = to_args( -- GitLab