diff --git a/.coverage b/.coverage new file mode 100644 index 0000000000000000000000000000000000000000..f13de030cb8efbd054d831b065acd0eb51ed3403 Binary files /dev/null and b/.coverage differ diff --git a/.gitignore b/.gitignore index cc31b2ab2fa0a4f7e82a005eead9e14955632bfd..5af49d685fba41934eeeb166f725571e89af44b7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,12 @@ __pycache__ .travo travo.egg-info report.xml +.vs/ +flake8-reports/ +SonarQube-reports/ +.scannerwork/ +.pytest_cache/ +flake8-reports/ +.understand/ +.travo.und/ +.coverage.xml \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 57a9f6ac8ba6f5de3352bb9ba0d2e8fae335b61f..28420d76ef668aae5fbea295065d62cbb24721e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,7 @@ stages: - test + - linting + - sonarqube before_script: # - apt-get update @@ -7,19 +9,26 @@ before_script: # - wget https://download.docker.com/linux/debian/dists/jessie/pool/stable/amd64/docker-ce_18.06.3~ce~3-0~debian_amd64.deb # - dpkg -i docker-ce_18.06.3~ce~3-0~debian_amd64.deb - - pip install tox - - pip install . + - pip3 install tox --ignore-installed + - pip3 install . tests_python39: image: python:3.9 stage: test script: - tox -e py39 + variables: + CI_DEBUG_TRACE: "false" artifacts: when: always + paths: + - "${CI_PROJECT_DIR}" reports: junit: - report.xml + coverage_report: + coverage_format: cobertura + path: coverage.xml # tests_python38: # image: python:3.8 @@ -38,6 +47,8 @@ tests_python36: stage: test script: - tox -e py36 + variables: + CI_DEBUG_TRACE: "false" artifacts: when: always reports: @@ -49,3 +60,35 @@ tests_python36: # Also parallel runs currently tend to fail due to reaching the # rate limit of the test gitlab instance. needs: [tests_python39] + + +pylint: + stage: linting + image: registry.gitlab.com/pipeline-components/pylint:latest + script: + - pylint --exit-zero --load-plugins=pylint_gitlab --output-format=gitlab-codeclimate:codeclimate.json **/*.py + artifacts: + paths: + - "${CI_PROJECT_DIR}" + reports: + codequality: codeclimate.json + when: always + +sonarqube_check: + stage: sonarqube + image: + name: sonarsource/sonar-scanner-cli:latest + entrypoint: [""] + variables: + SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar" # Defines the location of the analysis task cache + GIT_DEPTH: "0" # Tells git to fetch all the branches of the project, required by the analysis task + CI_DEBUG_TRACE: "false" + cache: + key: "${CI_JOB_NAME}" + paths: + - .sonar/cache + script: + - sonar-scanner + # - chmod 755 sonar.sh + # - ./sonar.sh + when: on_success diff --git a/conftest.py b/conftest.py index 3fbd365b8ed479ffe370c19e172874b0f42730e2..866f36e3dca10f16d4bc7d61541da92f07d70ed7 100644 --- a/conftest.py +++ b/conftest.py @@ -41,7 +41,6 @@ def project_name(test_run_id: str) -> str: @pytest.fixture def project(gitlab: GitLab, project_path: str, project_name: str) -> Iterator[Project]: project = gitlab.ensure_project(project_path, project_name) - # TODO: factor out duplication with assignment_repo project.ensure_file("README.md") yield project gitlab.remove_project(project_path, force=True) @@ -197,3 +196,4 @@ def assignment_personal_repo( @pytest.fixture def student_work_dir(course: Course) -> str: return course.ensure_work_dir() + diff --git a/setup.py b/setup.py index 8eae2d1b827560939230aa28549f96a1dcca9d24..92b0274685adda52b0d6902feec73f3c924808e9 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( license='CC', classifiers=[ 'Development Status :: 4 - Alpha', - 'Intended Audience :: Information Technology' + 'Intended Audience :: Information Technology' + 'Topic :: Scientific/Engineering', 'Programming Language :: Python :: 3', ], # classifiers list: https://pypi.python.org/pypi?%3Aaction=list_classifiers @@ -33,4 +33,4 @@ setup( "travo-echo-travo-token = travo.console_scripts:travo_echo_travo_token", ], }, -) +) \ No newline at end of file diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000000000000000000000000000000000000..88915d3d27f4e857ba48ce342fd6ee4173b757ff --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,42 @@ +sonar.projectKey=tchapsdev_travo-classroom_AYkXEFkvwSgLzWhExlmG + +sonar.qualitygate.wait=true +sonar.qualitygate.timeout=900 + +sonar.organization=UQAM + +# This is the name and version displayed in the SonarCloud UI. + sonar.projectName=Travo Classroom +# sonar.projectVersion=1.0 + +# Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows. + sonar.sources=. + +#sonar.tests=/conftest.py + +# Encoding of the source code. Default is default system encoding +# sonar.sourceEncoding=UTF-8 + +sonar.host.url=http://72.167.48.165:9000 + +# Account for login to the SonarQube Server +sonar.login=travo +sonar.password=travo123 + +sonar.token=sqp_e4cbd578d908625af088d9fe4318c477f3e01c5b + +sonar.gitlab.max_major_issues_gate=0 + +# Source File Exclusions +sonar.exclusions=/build/**/*,understand/*,coverage-reports/*,travo.und/*,flake8*.*,SonarQube-reports/* + +sonar.python.version=3.6,3.7, 3.8, 3.9 + +# Path to coverage report(s) +#List of paths pointing to coverage reports. Ant patterns are accepted for relative path. The reports have to conform to the Cobertura XML format. +#Key: sonar.python.coverage.reportPaths +sonar.python.coverage.reportPaths=coverage.xml + +# Paths (absolute or relative) to report files with Pylint issues. +sonar.python.pylint.reportPaths=codeclimate.json +# sonar.python.file.suffixes=py \ No newline at end of file diff --git a/tox.ini b/tox.ini index 7f825d98c346bafae3c24d33d84d85870355652e..dfebfee8501870e7b30d93b58bf65e0c0ce317f9 100644 --- a/tox.ini +++ b/tox.ini @@ -14,11 +14,19 @@ deps = types-toml tqdm ipydatagrid + coverage + pytest-cov + flake8 commands = mypy - pytest --junitxml=report.xml + pytest --junitxml=report.xml --cov --cov-report term --cov-report xml:coverage.xml + coverage xml -o coverage.xml [testenv:py36] commands = pytest --junitxml=report.xml + +[coverage:run] +relative_files = True +branch = True \ No newline at end of file diff --git a/travo/assignment.py b/travo/assignment.py index 68117db6f4f834e232fd4b8a6ab9fdc5f83a031c..ca4d655adcf669254214771a3b773e9ebf8e1901 100644 --- a/travo/assignment.py +++ b/travo/assignment.py @@ -13,18 +13,18 @@ from .i18n import _ @dataclass class Assignment: - forge: Forge - log: Logger - repo_path: str - instructors_path: str - name: str - username: Optional[str] = None - leader_name: Optional[str] = None - assignment_dir: Optional[str] = None - script: str = field(default='travo') + forge: Forge + log: Logger + repo_path: str + instructors_path: str + name: str + username: Optional[str] = None + leader_name: Optional[str] = None + assignment_dir: Optional[str] = None + script: str = field(default='travo') jobs_enabled_for_students: bool = field(default=True) - expires_at: Optional[str] = None - _repo_cache: Optional[Project] = None + expires_at: Optional[str] = None + _repo_cache: Optional[Project] = None @classmethod def from_url(cls, @@ -151,7 +151,6 @@ class Assignment: else: return project.namespace.path - ########################################################################## def personal_repo(self) -> Project: @@ -248,7 +247,8 @@ Il sera reconstruit lors du prochain dépôt""") expires_at=self.expires_at) self.log.debug("- Configuration badge:") - artifacts_url = self.forge.base_url+'/%{project_path}/-/jobs/artifacts/%{default_branch}/' + artifacts_url = self.forge.base_url + \ + '/%{project_path}/-/jobs/artifacts/%{default_branch}/' link_url = artifacts_url + 'file/feedback/scores.html?job=autograde' image_url = artifacts_url + 'raw/feedback/scores.svg?job=autograde' @@ -260,7 +260,7 @@ Il sera reconstruit lors du prochain dépôt""") self.log.info(f" {my_repo.web_url}") return my_repo - def remove_personal_repo(self, force: bool=False) -> None: + def remove_personal_repo(self, force: bool = False) -> None: """ Remove the users' personal repository for this assignment """ @@ -471,6 +471,7 @@ Déplacez ou supprimez le """ self.forge.login(anonymous_ok=True) user = self.forge.get_current_user() + def git(args: List[str], **kwargs: Any) -> subprocess.CompletedProcess: return self.forge.git(args, cwd=assignment_dir, **kwargs) @@ -501,15 +502,18 @@ Déplacez ou supprimez le # Something special needs to be done for "." assert assignment_dir != "." backup_dir = f"{assignment_dir}_{now}" - self.log.warning(f"""La mise à jour n'a pas pu se faire du fait d'un conflit + self.log.warning( + f"""La mise à jour n'a pas pu se faire du fait d'un conflit Votre devoir local va être renommé en {backup_dir} et une copie fraîche du sujet téléchargée à la place """) os.rename(assignment_dir, backup_dir) self.fetch(assignment_dir) else: - self.log.error("- La mise à jour n'a pas pu se faire du fait d'un conflit.") - self.log.info(f""" Pour renommer votre devoir local et forcer la mise à jour, + self.log.error( + "- La mise à jour n'a pas pu se faire du fait d'un conflit.") + self.log.info( + f""" Pour renommer votre devoir local et forcer la mise à jour, utilisez l'option force; en ligne de commande: {self.script} fetch ... --force @@ -520,7 +524,8 @@ Déplacez ou supprimez le # fetch_branch may require a commit which require git to be configured # we could try harder if there is a use case for it self.fetch_branch(assignment_dir, branch=None, content=_("updates")) - self.fetch_branch(assignment_dir, branch="errata", content=_("erratas"), on_failure='warning') + self.fetch_branch(assignment_dir, branch="errata", + content=_("erratas"), on_failure='warning') def submit(self, assignment_dir: Optional[str] = None, @@ -554,7 +559,10 @@ Déplacez ou supprimez le return self.forge.git(args, cwd=assignment_dir, **kwargs) self.log.info("- Enregistrement des changements:") - if git(["commit", "--all", "-m", f"Soumission depuis {hostname} par {user.name}"], + if git(["commit", + "--all", + "-m", + f"Soumission depuis {hostname} par {user.name}"], check=False).returncode != 0: self.log.info(" Pas de changement à enregistrer") @@ -761,7 +769,8 @@ class Submission: progress_bar.desc = pipeline['status'] progress_bar.update() time.sleep(1) - pipeline = forge.get_json(f"/projects/{repo.id}/pipelines/{pipeline['id']}") + pipeline = forge.get_json( + f"/projects/{repo.id}/pipelines/{pipeline['id']}") job, = forge.get_json( f"/projects/{repo.id}/pipelines/{pipeline['id']}/jobs") @@ -863,4 +872,4 @@ class Submission: autograde_job=autograde_job, status=status, submission=self, - ) + ) diff --git a/travo/course.py b/travo/course.py index 5f0badbf0fd736144c3830f92832d543decd3ae2..7cdd11260986d171b012e14a0c05021772095e8d 100644 --- a/travo/course.py +++ b/travo/course.py @@ -42,7 +42,7 @@ class CourseAssignment(Assignment): >>> course = getfixture("rich_course") >>> course.assignment("SubCourse/Assignment1").personal_repo_path() 'travo-test-etu/TestCourse-2020-2021-SubCourse-Assignment1' - >>> course.assignment("SubCourse/Assignment1", + >>> course.assignment("SubCourse/Assignment1", ... student_group="Group1").personal_repo_path() 'travo-test-etu/TestCourse-2020-2021-SubCourse-Assignment1' """ @@ -114,7 +114,6 @@ class CourseAssignment(Assignment): repo = self.forge.get_project(path) return (repo, 1) - def get_submission_username(self, project: Project) -> Optional[str]: """Return the username for the given submission @@ -175,25 +174,25 @@ class Course: If you wish to use another course layout, you can set the above variables directly. """ - forge: Forge - path: str - name: str - student_dir: str + forge: Forge + path: str + name: str + student_dir: str assignments_group_path: str = "" assignments_group_name: str = "" - session_path: Optional[str] = None - session_name: Optional[str] = None # Will be defined as soon as session_path is - assignments: Optional[List[str]] = None - student_groups: Optional[List[str]] = None - script: str = "travo" - url: Optional[str] = None + session_path: Optional[str] = None + session_name: Optional[str] = None # Will be defined as soon as session_path is + assignments: Optional[List[str]] = None + student_groups: Optional[List[str]] = None + script: str = "travo" + url: Optional[str] = None jobs_enabled_for_students: bool = False - log: logging.Logger = field(default_factory=getLogger) - mail_extension: str = "" - expires_at: Optional[str] = None + log: logging.Logger = field(default_factory=getLogger) + mail_extension: str = "" + expires_at: Optional[str] = None def __post_init__(self) -> None: - # TODO: "Check that: name contains only letters, digits, emojis, '_', '.', dash, space. It must start with letter, digit, emoji or '_'." + # "Check that: name contains only letters, digits, emojis, '_', '.', dash, space. It must start with letter, digit, emoji or '_'." if self.session_path is not None: self.assignments_group_path = os.path.join( self.path, @@ -260,14 +259,14 @@ class Course: should some course want to use another one. """ assert role == "student" - dir = self.student_dir - if dir == ".": + dir1 = self.student_dir + if dir1 == ".": return "." - if dir[:2] == "~/": - dir = os.path.join(self.forge.home_dir, dir[2:]) + if dir1[:2] == "~/": + dir1 = os.path.join(self.forge.home_dir, dir1[2:]) if assignment is not None: - dir = os.path.join(dir, assignment) - return dir + dir1 = os.path.join(dir1, assignment) + return dir1 def ensure_work_dir(self) -> str: """ @@ -320,7 +319,6 @@ class Course: message += _('unknown group', student_group=student_group) + "\n" message += _('specify group', student_groups=', '.join(self.student_groups)) + "\n" - # message += _('help', script=self.script) raise RuntimeError(message) @@ -362,7 +360,7 @@ class Course: 'TestCourse/2020-2021/Subcourse/MI1/Assignment1' """ result = [self.assignments_group_path] - dirname = os.path.dirname(assignment) + dirname = os.path.dirname(assignment) assignment = os.path.basename(assignment) if dirname: result.append(dirname) @@ -424,20 +422,20 @@ class Course: self.check_student_group(student_group, none_ok=True) return self.Assignment( - forge=self.forge, - course=self, - log=self.log, - name=assignment_name, - instructors_path=self.assignments_group_path, - script=self.script, - expires_at=self.expires_at, - repo_path=repo_path, - student_group=student_group, - leader_name=leader_name, - username=username, - assignment_dir=self.work_dir(assignment_name), - jobs_enabled_for_students=self.jobs_enabled_for_students, - ) + forge=self.forge, + course=self, + log=self.log, + name=assignment_name, + instructors_path=self.assignments_group_path, + script=self.script, + expires_at=self.expires_at, + repo_path=repo_path, + student_group=student_group, + leader_name=leader_name, + username=username, + assignment_dir=self.work_dir(assignment_name), + jobs_enabled_for_students=self.jobs_enabled_for_students, + ) @deprecated(deprecated_in="0.2", removed_in="1.0", current_version=__version__, @@ -468,8 +466,9 @@ class Course: """ self.ensure_work_dir() assignment_dir = self.work_dir(assignment=assignment) - return self.assignment(assignment, student_group=student_group).fetch(assignment_dir=assignment_dir, - force=force) + return self.assignment( + assignment, student_group=student_group).fetch( + assignment_dir=assignment_dir, force=force) def submit(self, assignment: str, @@ -489,12 +488,14 @@ class Course: def share_with(self, assignment_name: str, username: str, - access_level: Union[int, Resource.AccessLevels] = Resource.AccessLevels.DEVELOPER, + access_level: Union[int, + Resource.AccessLevels] = Resource.AccessLevels.DEVELOPER, ) -> None: try: repo = self.assignment(assignment_name).personal_repo() except ResourceNotFoundError: - raise RuntimeError(_("no submission; please submit", assignment_name=assignment_name)) + raise RuntimeError(_("no submission; please submit", + assignment_name=assignment_name)) user = self.forge.get_user(username) repo.share_with(user, access=access_level) @@ -521,17 +522,18 @@ class Course: """ self.check_assignment(assignment) - self.log.info(f"Publish the assignment {assignment} with visibility {visibility}.") + self.log.info( + f"Publish the assignment {assignment} with visibility {visibility}.") # travo_gitlab_remove_project "${repo}" attributes = dict( - visibility = visibility, - issues_enabled = False, - merge_requests_enabled = False, - container_registry_enabled = False, - wiki_enabled = False, - snippets_enabled = False, - lfs_enabled = False, + visibility=visibility, + issues_enabled=False, + merge_requests_enabled=False, + container_registry_enabled=False, + wiki_enabled=False, + snippets_enabled=False, + lfs_enabled=False, ) path = self.assignment_repo_path(assignment) @@ -550,7 +552,8 @@ class Course: for student_group in self.student_groups: path = self.assignment_repo_path(assignment, student_group=student_group) name = self.assignment_repo_name(assignment) - self.log.info(f"- Publishing to the student group {student_group}' fork {path}.") + self.log.info( + f"- Publishing to the student group {student_group}' fork {path}.") self.forge.ensure_group(os.path.dirname(path), name=student_group, visibility="public") @@ -591,11 +594,11 @@ class Course: self.forge.remove_project(repo.path_with_namespace, force=force) def collect(self, - assignment: str, + assignment: str, student_group: Optional[str] = None, - student: Optional[str] = None, - template: str = "{username}", - date: Optional[str] = None) -> None: + student: Optional[str] = None, + template: str = "{username}", + date: Optional[str] = None) -> None: """ Collect the student's assignments @@ -620,8 +623,8 @@ class Course: date=date) def ensure_instructor_access(self, - assignment: str, - student_group: Optional[str] = None) -> None: + assignment: str, + student_group: Optional[str] = None) -> None: """ Ensure instructor access to the student repositories. @@ -644,7 +647,10 @@ class Course: """Run an arbitrary shell command""" return run(args) - def get_released_assignments(self, level: str = "", order_by: str = "created_at") -> List[str]: + def get_released_assignments( + self, + level: str = "", + order_by: str = "created_at") -> List[str]: group_path = self.assignments_group_path if level: group_path += "/" + level diff --git a/travo/dashboards.py b/travo/dashboards.py index e8bc140bec54c624ae387e09cec455f9cd1049a0..e6041af7829bc983eb2f3785d1420c14bfa575f6 100644 --- a/travo/dashboards.py +++ b/travo/dashboards.py @@ -50,8 +50,13 @@ from .utils import run from travo.i18n import _ from . import jupyter_course # only used by type hints -# TODO: should use the current foreground color rather than black -border_layout = {'border': '1px solid black'} +BORDER_LAYOUT_STYLE = '1px solid black' + +# TO DO: should use the current foreground color rather than black +border_layout = {'border': BORDER_LAYOUT_STYLE } + +ASSIGNMENT_STATUS_NOT_RELEASE = "not released" +ATTENTE_NOTATION_MANUELLE = "attente notation manuelle" def HTML(*args: Any, **kwargs: Any) -> ipywidgets.HTML: @@ -80,8 +85,10 @@ class AuthenticationWidget(VBox): forge.on_missing_credentials = self.on_missing_credentials self.forge = forge title = Label(_("authentication widget title", url=forge.base_url)) - self.usernameUI = Text(description=_("username"), layout={'width': 'fit-content'}) - self.passwordUI = Password(description=_("password"), layout={'width': 'fit-content'}) + self.usernameUI = Text(description=_("username"), + layout={'width': 'fit-content'}) + self.passwordUI = Password(description=_("password"), + layout={'width': 'fit-content'}) self.button = Button(description=_("sign in"), button_style="primary" ) @@ -98,7 +105,7 @@ class AuthenticationWidget(VBox): self.passwordUI, self.button, self.messageUI), - layout={'border': '1px solid black', + layout={'border': BORDER_LAYOUT_STYLE, 'display': 'none'}) def show_widget(self, message: str = "") -> None: @@ -224,6 +231,7 @@ class StatusBar(VBox): class AssignmentStudentDashboard(HBox): + def __init__(self, assignment: Assignment, status_bar: Optional[StatusBar] = None, @@ -300,7 +308,7 @@ class AssignmentStudentDashboard(HBox): status = self.assignment.status() except AuthenticationError: return - if status.status == "not released": + if status.status == ASSIGNMENT_STATUS_NOT_RELEASE: self.nameUI.value = self.name self.fetchUI.disabled = True else: @@ -397,12 +405,12 @@ class AssignmentStudentDashboard(HBox): # TODO : if README.md does not exist neither, try to open gitlab file browser index_files = ["index.md", "index.ipynb", "README.md", "README.ipynb"] for x in index_files: - file = os.path.join(self.assignment.assignment_dir, x) - if os.path.isfile(file): + file_1 = os.path.join(self.assignment.assignment_dir, x) + if os.path.isfile(file_1): self.jupyter_front_end.commands.execute( "docmanager:open", { - "path": path + "/" + file, # Generalize if there is no index.md + "path": path + "/" + file_1, # Generalize if there is no index.md "factory": "Notebook", # 'options': { # 'mode': 'split-right' @@ -427,11 +435,13 @@ class CourseStudentDashboard(VBox): student_group_UI: Dropdown assignments: Tuple[str, ...] = () assignment_dashboards: Dict[str, AssignmentStudentDashboard] + student_group_UI_label: str = "" def __init__(self, course: Course, student_group: Optional[str] = None): self.course = course + self.student_group_UI_label = 'student group' self.header = HBox( [HTML(f'{course.name}', @@ -439,7 +449,7 @@ class CourseStudentDashboard(VBox): layout=border_layout) if self.course.student_groups is not None: self.student_group_UI = Dropdown( - description=_('student group'), + description=_(self.student_group_UI_label), value=student_group, options=course.student_groups, ) @@ -487,8 +497,6 @@ class CourseStudentDashboard(VBox): def update(self, update_assignment_list: bool = False) -> None: student_group = self.student_group_UI.value - # if student_group is None: - # self.center = None if update_assignment_list: if self.course.assignments is not None: assignments = self.course.assignments @@ -513,13 +521,13 @@ class CourseStudentDashboard(VBox): self.grid.children = [ Label(label) for j, label in enumerate( - [_('assignment'), - '', - _('work directory'), - '', - _('submission'), - _('score'), - ]) + [_('assignment'), + '', + _('work directory'), + '', + _('submission'), + _('score'), + ]) ] + [widget for assignment in self.assignments for widget in self.assignment_dashboards[assignment].children] @@ -591,7 +599,8 @@ class AssignmentInstructorDashboard(HBox): disabled=False, ) self.collectButton.on_click(lambda event: self.collect()) - self.collectStatus = Textarea(self.count_submissions_cmd(submitted_directory="submitted"), layout=layout) + self.collectStatus = Textarea(self.count_submissions_cmd( + submitted_directory="submitted"), layout=layout) self.collectUI = VBox([self.collectButton, self.collectStatus]) self.formgraderButton = Button( @@ -616,7 +625,8 @@ class AssignmentInstructorDashboard(HBox): disabled=False, ) self.feedbackButton.on_click(lambda event: self.feedback()) - self.feedbackStatus = Textarea(self.release_feedback_status_cmd(self.student_group), layout=layout) + self.feedbackStatus = Textarea( + self.release_feedback_status_cmd(self.student_group), layout=layout) self.feedbackUI = VBox([self.feedbackButton, self.feedbackStatus]) HBox.__init__( @@ -644,7 +654,7 @@ class AssignmentInstructorDashboard(HBox): status = self.assignment.status() except AuthenticationError: return - if status.status == "not released": + if status.status == self.assignment_status_release: self.assignmentUI.value = "" self.collectButton.disabled = True self.feedbackButton.disabled = True @@ -654,7 +664,7 @@ class AssignmentInstructorDashboard(HBox): self.assignmentUI.value = ( f'{_("browse")}' ) - # self.generateButton.disabled = False + # Comment to remove: self.generateButton.disabled = False self.releaseStatus.value = self.release_status_cmd() if self.student_group is None: self.collectButton.disabled = True @@ -664,7 +674,8 @@ class AssignmentInstructorDashboard(HBox): self.feedbackButton.disabled = False self.formgraderButton.disabled = False - self.collectStatus.value = self.count_submissions_cmd(submitted_directory="submitted") + self.collectStatus.value = self.count_submissions_cmd( + submitted_directory="submitted") self.formgraderStatus.value = self.count_submissions_need_manual_grade_cmd() self.feedbackStatus.value = self.release_feedback_status_cmd(self.student_group) @@ -723,12 +734,20 @@ class AssignmentInstructorDashboard(HBox): if self.student_group is None: self.course.log.error("Must choose a student_group value.") return - self.course.collect_for_nbgrader(assignment_name=self.name, student_group=self.student_group) - self.course.ensure_autograded(assignment_name=self.name, student_group=self.student_group, - force_autograde=self.force_autograding) - self.course.collect_autograded(assignment_name=self.name, student_group=self.student_group) - self.course.merge_autograded_db(assignment_name=os.path.basename(self.name), on_inconsistency="WARNING", back=False, - new_score_policy=self.new_score_policy) + self.course.collect_for_nbgrader( + assignment_name=self.name, student_group=self.student_group) + self.course.ensure_autograded( + assignment_name=self.name, + student_group=self.student_group, + force_autograde=self.force_autograding) + self.course.collect_autograded( + assignment_name=self.name, student_group=self.student_group) + self.course.merge_autograded_db( + assignment_name=os.path.basename( + self.name), + on_inconsistency="WARNING", + back=False, + new_score_policy=self.new_score_policy) self.course.clear_needs_manual_grade(os.path.basename(self.name)) def feedback_cmd(self, tag: str = '*') -> None: @@ -738,23 +757,26 @@ class AssignmentInstructorDashboard(HBox): self.course.generate_feedback(os.path.basename(self.name), tag=tag, new_score_policy=self.new_score_policy) # self.course.merge_autograded_db(os.path.basename(self.name), back=True, on_inconsistency="WARNING", - # tag=tag, new_score_policy=self.new_score_policy.value) + # tag=tag, new_score_policy=self.new_score_policy.value) self.course.release_feedback(self.name, student_group=self.student_group) def open_formgrader_cmd(self, event: Any) -> None: try: # jupyterlab from ipylab import JupyterFrontEnd # type: ignore - # if self.assignment.assignment_dir is None: - # raise ValueError("Can't open work dir if assignment_dir is not set") - path = os.path.dirname(self.jupyter_front_end.sessions.current_session['path']) - file = self.course.formgrader(os.path.basename(self.name), in_notebook=False) + ## Comment to remove: if self.assignment.assignment_dir is None: + ## Comment to remove: raise ValueError("Can't open work dir if assignment_dir is not set") + path = os.path.dirname( + self.jupyter_front_end.sessions.current_session['path']) + file_form = self.course.formgrader( + os.path.basename(self.name), in_notebook=False) import pdb pdb.pm() self.jupyter_front_end.commands.execute( "docmanager:open", { - "path": os.path.join(path, file), # Generalize if there is no index.md + # Generalize if there is no index.md + "path": os.path.join(path, file_form), "factory": "Notebook", # 'options': { # 'mode': 'split-right' @@ -766,22 +788,32 @@ class AssignmentInstructorDashboard(HBox): ) except KeyError: # jupyter notebook from IPython.display import display # type: ignore - display(self.course.formgrader(os.path.basename(self.name), in_notebook=True)) # type: ignore + display(self.course.formgrader(os.path.basename( + self.name), in_notebook=True)) # type: ignore def generate_status_cmd(self) -> str: try: - path = os.path.join(self.course.release_directory, os.path.basename(self.assignment.name)) - date = subprocess.run(['git', 'log', '-1', '--format=%ai'], check=True, text=True, capture_output=True, + path = os.path.join(self.course.release_directory, + os.path.basename(self.assignment.name)) + date = subprocess.run(['git', + 'log', + '-1', + '--format=%ai'], + check=True, + text=True, + capture_output=True, cwd=path).stdout status = f"last generation: {date}" - except: + except BaseException: status = "not generated" return status def count_submissions_cmd(self, submitted_directory: str = "submitted") -> str: status = "" - number_of_copies = len(glob.glob(f"{submitted_directory}/*/{os.path.basename(self.assignment.name)}", - recursive=True)) + number_of_copies = len( + glob.glob( + f"{submitted_directory}/*/{os.path.basename(self.assignment.name)}", + recursive=True)) if number_of_copies > 0: status += f"#submissions: {number_of_copies}\n" return status @@ -801,7 +833,8 @@ class AssignmentInstructorDashboard(HBox): # `MissingEntry` exception will be raised, which means the student # didn't submit anything, so we assign them a score of zero. try: - submission = gb.find_submission(os.path.basename(self.assignment.name), student.id) + submission = gb.find_submission( + os.path.basename(self.assignment.name), student.id) except MissingEntry: pass else: @@ -814,7 +847,7 @@ class AssignmentInstructorDashboard(HBox): status += f"Needs manual grading: {counts}\nGrades: {np.mean(grades):.2f}+/-{np.std(grades):.2f}\n" else: status = "No grades" - except: + except BaseException: pass return status @@ -823,16 +856,18 @@ class AssignmentInstructorDashboard(HBox): p = self.course.forge.get_project(self.assignment.repo_path) status = f"last release: {p.visibility}\n{p.last_activity_at}" except ResourceNotFoundError: - status = "not released" + status = self.assignment_status_release return status def release_feedback_status_cmd(self, student_group: Optional[Any]) -> str: try: - url = self.course.forge.get_project(self.course.assignment(self.assignment.name, - student_group=student_group).repo_path).http_url_to_repo + url = self.course.forge.get_project( + self.course.assignment( + self.assignment.name, + student_group=student_group).repo_path).http_url_to_repo url = "go to " + url.replace(".git", "/-/forks") except ResourceNotFoundError: - url = "not released" + url = self.assignment_status_release return url @@ -849,13 +884,18 @@ class CourseInstructorDashboard(VBox): self.course = course self.course.forge.login() - self.release_mode = RadioButtons(description="", value=None, options=["public", "private"], - layout={'width': 'fit-content'}) - self.force_autograding = RadioButtons(description="", value=False, options=[False, True], - layout={'width': 'fit-content'}) - self.new_score_policy = RadioButtons(description="", value="only_greater", - options=["only_empty", "only_greater", "force_new_score"], - layout={'width': 'fit-content'}) + self.release_mode = RadioButtons( + description="", value=None, options=[ + "public", "private"], layout={ + 'width': 'fit-content'}) + self.force_autograding = RadioButtons( + description="", value=False, options=[ + False, True], layout={ + 'width': 'fit-content'}) + self.new_score_policy = RadioButtons( + description="", value="only_greater", options=[ + "only_empty", "only_greater", "force_new_score"], layout={ + 'width': 'fit-content'}) self.header = HBox( [HTML(f'{course.name}', @@ -863,14 +903,15 @@ class CourseInstructorDashboard(VBox): VBox([Label("release mode"), self.release_mode], width="10px"), VBox([Label("collect:\nforce new autograde"), self.force_autograding]), VBox([Label("new score policy"), self.new_score_policy])], - layout=Layout(border='1px solid black', grid_gap='5px 40px')) + layout=Layout(border=BORDER_LAYOUT_STYLE, grid_gap='5px 40px')) if self.course.student_groups is not None: self.student_group_UI = Dropdown( description="", value=student_group, options=course.student_groups, ) - self.header.children += (VBox([Label(_('student group')), self.student_group_UI]),) + self.header.children += (VBox([Label(_(ASSIGNMENT_STATUS_NOT_RELEASE)), + self.student_group_UI]),) self.student_group_UI.observe(lambda change: self.update(), names='value') self.release_mode.observe(lambda change: self.update(), @@ -933,16 +974,16 @@ class CourseInstructorDashboard(VBox): for assignment_name in self.course.assignments ] self.grid.children = [ - Label(label) - for j, label in enumerate( - [_('assignment'), - _('assignment repository'), - 'Gestion du cours', '', - 'Evaluations', '', '', - ]) - ] + [widget - for assignment_dashboard in self.assignment_dashboards - for widget in assignment_dashboard.children] + Label(label) + for j, label in enumerate( + [_('assignment'), + _('assignment repository'), + 'Gestion du cours', '', + 'Evaluations', '', '', + ]) + ] + [widget + for assignment_dashboard in self.assignment_dashboards + for widget in assignment_dashboard.children] class CourseGradeDashboard(VBox): @@ -960,13 +1001,13 @@ class CourseGradeDashboard(VBox): self.dashboard_grade_filename = "dashboard-grades.csv" if self.course.assignments is None: self.course.assignments = self.course.get_released_assignments() - self.assignments_UI = Dropdown(description="assignments", value=None, - options=["all"] + [a for a in self.course.assignments], width="350px") + self.assignments_UI = Dropdown(description="assignments", value=None, options=[ + "all"] + [a for a in self.course.assignments], width="350px") self.get_scores_UI = Button(description="update scores", - button_style="primary", - icon="book", - layout=layout) + button_style="primary", + icon="book", + layout=layout) self.get_scores_UI.on_click(lambda event: self.get_scores()) self.clear_csv_UI = Button(description="clear scores", @@ -1021,13 +1062,20 @@ class CourseGradeDashboard(VBox): renderers = {} for col in self.df.columns: if "note" in col and "/0" not in col: - renderers[col] = TextRenderer(text_color="black", background_color=Expr(_format_note)) + renderers[col] = TextRenderer( + text_color="black", background_color=Expr(_format_note)) elif "status" in col: - renderers[col] = TextRenderer(text_color="black", background_color=Expr(_format_status)) + renderers[col] = TextRenderer( + text_color="black", background_color=Expr(_format_status)) grid_layout = {"height": "300px"} - self.grid = DataGrid(self.df, base_column_size=200, base_header_size=200, layout=grid_layout, editable=True, - renderers=renderers) + self.grid = DataGrid( + self.df, + base_column_size=200, + base_header_size=200, + layout=grid_layout, + editable=True, + renderers=renderers) self.grid.auto_fit_params = {"area": "all", "padding": 100} self.grid.auto_fit_columns = True @@ -1076,12 +1124,14 @@ class CourseGradeDashboard(VBox): """ # load the requested assignments if self.assignments_UI.value is None or self.assignments_UI.value == "all": - assignments = [assignment_name for assignment_name in self.assignments_UI.options[1:]] + assignments = [ + assignment_name for assignment_name in self.assignments_UI.options[1:]] else: assignments = [self.assignments_UI.value] # load the requested groupes if self.student_group_UI.value is None or self.student_group_UI.value == "all": - student_groups = [student_group for student_group in self.student_group_UI.options[1:]] + student_groups = [ + student_group for student_group in self.student_group_UI.options[1:]] else: student_groups = [self.student_group_UI.value] @@ -1096,13 +1146,13 @@ class CourseGradeDashboard(VBox): total = 0 total_bad = 0 for student_group in student_groups: - # path = course.assignment_repo_path(assignment, student_group) + # # Comment to remove: path = course.assignment_repo_path(assignment, student_group) try: - # project = course.forge.get_project(path) + ## Comment to remove: project = course.forge.get_project(path) # forks = project.get_forks(recursive=True) - submissions_status = self.course.assignment(assignment_name=assignment, - student_group=student_group).collect_status() - # course.log.info(f"Collecting autograded for {len(submissions_status)} students") + submissions_status = self.course.assignment( + assignment_name=assignment, student_group=student_group).collect_status() + ## Comment to remove: course.log.info(f"Collecting autograded for {len(submissions_status)} students") bad_projects = [] if len(submissions_status) == 0: continue @@ -1112,34 +1162,45 @@ class CourseGradeDashboard(VBox): if student not in d.keys(): d[student] = {"group": student_group} # for assignment in assignments: - d[student]["email"] = student + "@" + self.course.mail_extension - d[student][assignment + "-status"] = "as " + student_group + " with autograding=" + status.autograde_status + d[student]["email"] = student + \ + "@" + self.course.mail_extension + d[student][assignment + "-status"] = "as " + student_group + \ + " with autograding=" + status.autograde_status if status.autograde_status != "success": - self.course.log.info(f"missing successful autograde for {student}") + self.course.log.info( + f"missing successful autograde for {student}") bad_projects.append(student) continue self.course.log.info(f"fetching scores for {student}") if status.submission is not None: repo = status.submission.repo job = status.autograde_job - path = f"feedback/scores.csv" - scores_txt = repo.fetch_artifact(job, artifact_path=path).text + path = "feedback/scores.csv" + scores_txt = repo.fetch_artifact( + job, artifact_path=path).text scores = pd.read_csv(io.StringIO(scores_txt)) total_score = np.sum(scores["total_score"].values) manual_score = np.nansum(scores["manual_score"].values) - max_manual_score = np.sum(scores["max_manual_score"].values) - if np.all(np.isnan(scores["manual_score"].values)) and ~np.isnan(max_manual_score) and max_manual_score > 0: - d[student][assignment + f"-note (/{int(scores['max_total_score'].values[0])})"] = "attente notation manuelle" + max_manual_score = np.sum( + scores["max_manual_score"].values) + if np.all(np.isnan(scores["manual_score"].values)) and ~np.isnan( + max_manual_score) and max_manual_score > 0: + d[student][assignment + + f"-note (/{int(scores['max_total_score'].values[0])})"] = ATTENTE_NOTATION_MANUELLE else: - d[student][assignment + f"-note (/{int(scores['max_total_score'].values[0])})"] = total_score + d[student][assignment + + f"-note (/{int(scores['max_total_score'].values[0])})"] = total_score else: - self.course.log.info(f"missing submission for {student}") + self.course.log.info( + f"missing submission for {student}") bad_projects.append(student) continue if len(bad_projects) > 0: - self.course.log.warning(f"{len(bad_projects)} dépôt(s) avec autograde_status!=success:") + self.course.log.warning( + f"{len(bad_projects)} dépôt(s) avec autograde_status!=success:") for student in bad_projects: - self.course.log.warning(f"student={student}, status={d[student][assignment + '-status']}") + self.course.log.warning( + f"student={student}, status={d[student][assignment + '-status']}") total += len(submissions_status) total_bad += len(bad_projects) self.course.log.warning( @@ -1147,7 +1208,8 @@ class CourseGradeDashboard(VBox): f" {len(bad_projects)} dépôt(s) corrompu(s).") except ResourceNotFoundError: pass - self.course.log.info(f"{total} projets soumis pour {assignment}, {total_bad} dépôt(s) corrompu(s).") + self.course.log.info( + f"{total} projets soumis pour {assignment}, {total_bad} dépôt(s) corrompu(s).") self.df = pd.DataFrame.from_dict(d, orient="index") self.df.index.name = "student_id" @@ -1160,12 +1222,12 @@ class CourseGradeDashboard(VBox): self.update() def copy_cmd(self) -> None: - df2 = self.df.replace("attente notation manuelle", np.nan) + df2 = self.df.replace(ATTENTE_NOTATION_MANUELLE, np.nan) df2.to_clipboard(excel=True) def _format_note(cell: Any) -> str: - return ("yellow" if cell.value == "attente notation manuelle" else "white") + return ("yellow" if cell.value == ATTENTE_NOTATION_MANUELLE else "white") def _format_status(cell: Any) -> str: diff --git a/travo/gitlab.py b/travo/gitlab.py index f488fdc30efa4dfcef5f09090eba1ea9524651cd..48fa1a779332c0c8ee2b5e1d8ce99b5749c569b5 100644 --- a/travo/gitlab.py +++ b/travo/gitlab.py @@ -24,26 +24,28 @@ from .i18n import _ from .utils import urlencode, run, getLogger R = TypeVar('R', 'Group', 'Project', 'Namespace') -# JSON = TypeAlias(Any) # Python 3.10 -# Job = TypeAlias(JSON) # Python 3.10 +#Comment To Remove : JSON = TypeAlias(Any) # Python 3.10 +#Comment To Remove : Job = TypeAlias(JSON) # Python 3.10 JSON = Any # Could be made more specific Job = JSON # Could be made more specific + class ResourceNotFoundError(RuntimeError): pass + class AuthenticationError(RuntimeError): pass def request_credentials_basic(forge: 'GitLab', - username: Optional[str] = None, - password: Optional[str] = None - ) -> Tuple[str, str]: + username: Optional[str] = None, + password: Optional[str] = None + ) -> Tuple[str, str]: """ Basic interactive UI for requesting credentials to the user """ - print(_("please authenticate on", url=forge.base_url)) + print(_("please authenticate on", url=forge.base_url)) if username is None: username = input(_('username') + ': ') if username == 'anonymous': @@ -55,30 +57,30 @@ def request_credentials_basic(forge: 'GitLab', class GitLab: - debug: bool = False + debug: bool = False home_dir: str = str(pathlib.Path.home()) - token: Optional[str] = None - token_expires_at: Optional[float] = None + token: Optional[str] = None + token_expires_at: Optional[float] = None _current_user: Optional[Union['User', 'AnonymousUser']] = None base_url: str - api: str - session: requests.Session - log: logging.Logger + api: str + session: requests.Session + log: logging.Logger # on_missing_credentials should be a callable that either # returns the credentials as a tuple (username, password) # (typically after requesting them interactively to the user), # or raise (typically after setting up a UI to request the # credentials) - on_missing_credentials: Any # Should be Callable + on_missing_credentials: Any # Should be Callable def __init__(self, base_url: str, - token: Optional[str] = None, - log: logging.Logger = getLogger(), + token: Optional[str] = None, + log: logging.Logger = getLogger(), home_dir: Optional[str] = None): if base_url[-1] != "/": - base_url = base_url+"/" - if not base_url[:8] == "https://": + base_url = base_url + "/" + if base_url[:8] != "https://": raise ValueError("Only the https protocol is supported") self.base_url = base_url self.api = base_url + "api/v4/" @@ -112,11 +114,11 @@ class GitLab: @return whether the token is valid """ t = time.time() - response = self.session.get(self.base_url+"/oauth/token/info", + response = self.session.get(self.base_url + "/oauth/token/info", data=dict(access_token=token)) try: json = response.json() - except: + except BaseException: response.raise_for_status() if 'error' in json: assert json['error'] == 'invalid_token' @@ -264,13 +266,13 @@ class GitLab: self._current_user = anonymous_user return - result = self.session.post(self.base_url+"/oauth/token", + result = self.session.post(self.base_url + "/oauth/token", params=dict(grant_type="password", username=username, password=password, scope="api")) token = result.json().get('access_token') - # TODO: handle connection failures + # TODO: handle connection failures if token is None: # TODO: pourrait réessayer raise AuthenticationError(_("invalid credentials", url=self.base_url)) @@ -418,7 +420,6 @@ class GitLab: return typing.cast(str, json[0]['id']) raise ResourceNotFoundError(f"Namespace {path} not found") - def get_resource(self, cls: Type[R], path: Union[R, int, str], **args: Any) -> R: """ Get a resource from its path @@ -438,7 +439,8 @@ class GitLab: raise RuntimeError(f"{cls.__name__} {path} not found: {json['error']}") message = json.get("message", "") if message and message[0] != "2": - raise ResourceNotFoundError(f"{cls.__name__} {path} not found: {json['message']}") + raise ResourceNotFoundError( + f"{cls.__name__} {path} not found: {json['message']}") return cls(gitlab=self, **json) def ensure_resource(self, @@ -491,8 +493,7 @@ class GitLab: assert isinstance(json, dict) return cls(gitlab=self, **json) - - def confirm(self, message: str)-> bool: + def confirm(self, message: str) -> bool: """ Asks the user to confirm a dangerous or irreversible operation. @@ -683,7 +684,7 @@ class GitLab: return self.ensure_resource(Group, path=path, name=name, - get_resource_args = dict(with_projects=with_projects), + get_resource_args=dict(with_projects=with_projects), **attributes) def remove_group(self, id_or_path: str, @@ -695,7 +696,7 @@ class GitLab: """ self.remove_resource(Group, id_or_path, force=force) - def get_current_user(self)-> Union['User', 'AnonymousUser']: + def get_current_user(self) -> Union['User', 'AnonymousUser']: """ Get the currently logged in user (which may be 'anonymous') @@ -717,7 +718,7 @@ class GitLab: """ if self._current_user is None: self._current_user = User(gitlab=self, - **self.get_json("/user")) + **self.get_json("/user")) return self._current_user @@ -827,10 +828,10 @@ class GitLab: cwd=dir) def collect_forks(self, - path: str, + path: str, username: Optional[str] = None, template: str = "{username}", - date: Optional[str] = None) -> None: + date: Optional[str] = None) -> None: """ Collect all forks of this project @@ -855,14 +856,16 @@ class GitLab: if fork.owner is not None: if username is not None and fork.owner.username != username: continue - self.log.info(f"Download/update repository for {fork.owner.username} at date {date}:") + self.log.info( + f"Download/update repository for {fork.owner.username} at date {date}:") path = template.format(username=fork.owner.username, path=fork.path) try: fork.clone_or_pull(path, date=date) except subprocess.CalledProcessError: bad_projects.append(fork.http_url_to_repo) if len(bad_projects) > 0: - self.log.warning(f"{len(bad_projects)} corrupted or empty project, check the links :") + self.log.warning( + f"{len(bad_projects)} corrupted or empty project, check the links :") for url in bad_projects: self.log.warning(url) @@ -937,9 +940,9 @@ class ResourceRef: # - setting mandatory attributes in __post_init__ # - setting frozen attributes in __post_init__ # see e.g. https://groups.google.com/forum/#!topic/dev-python/7vBAZn_jEfQ - forge: 'Forge' = field(default=cast('Forge', None)) - path: str = field(default=cast('str', None)) - url: InitVar[Optional[str]] = None + forge: 'Forge' = field(default=cast('Forge', None)) + path: str = field(default=cast('str', None)) + url: InitVar[Optional[str]] = None def __post_init__(self, url: Optional[str] = None) -> None: if (self.forge is None) == (url is None): @@ -949,17 +952,18 @@ class ResourceRef: if url is not None: u = urllib.parse.urlparse(url) if u.scheme != '': - root_url = urllib.parse.urlunparse([u.scheme, u.hostname, '', '', '', '']) + root_url = urllib.parse.urlunparse( + [u.scheme, u.hostname, '', '', '', '']) path = u.path else: # Assume a ssh - r = re.search("^(.*@)?([^:]+):(.*?)$", url) + r = re.search("^(.*@)?([^:]+):(.*?)$", url) if r is not None: root_url = "https://" + r.groups()[1] path = r.groups()[2] object.__setattr__(self, "forge", GitLab(root_url)) - object.__setattr__(self, "path", path) - object.__setattr__(self, "path", self.path.lstrip("/")) + object.__setattr__(self, "path", path) + object.__setattr__(self, "path", self.path.lstrip("/")) if self.path.endswith(".git"): object.__setattr__(self, "path", self.path[:-4]) @@ -976,6 +980,8 @@ class ClassCallMetaclass(type): get_type_hints_cache: Dict[Type, Dict] = {} + + def get_type_hints(cls: Type) -> Dict: type_hints = get_type_hints_cache.get(cls, None) if type_hints is None: @@ -988,11 +994,11 @@ def get_type_hints(cls: Type) -> Dict: class Resource(metaclass=ClassCallMetaclass): class AccessLevels(enum.IntEnum): - GUEST = 10 - REPORTER = 20 - DEVELOPER = 30 + GUEST = 10 + REPORTER = 20 + DEVELOPER = 30 MAINTAINER = 40 - OWNER = 50 + OWNER = 50 __initialized: bool = field(default=False, repr=False, @@ -1040,9 +1046,9 @@ class Resource(metaclass=ClassCallMetaclass): >>> assert reproject.description == project.description """ # Select only the attributes that were changed - attributes = { key: value - for key, value in attributes.items() - if key not in self.__dict__ or value != self.__dict__[key] } + attributes = {key: value + for key, value in attributes.items() + if key not in self.__dict__ or value != self.__dict__[key]} if self.__initialized: # Check that the attributes are valid @@ -1072,7 +1078,7 @@ class Resource(metaclass=ClassCallMetaclass): value = resource_type(gitlab=self.gitlab, **value) elif typing_utils.issubtype(type_hint, List[Resource]): resource_type = typing_utils.get_args(type_hint)[0] - value = [ resource_type(gitlab=self.gitlab, **v) for v in value ] + value = [resource_type(gitlab=self.gitlab, **v) for v in value] self.__dict__[key] = value def __setattr__(self, key: str, value: Any) -> None: @@ -1115,6 +1121,7 @@ class Resource(metaclass=ClassCallMetaclass): res[key] = value return res + @dataclass class Project(Resource): _resource_type_api_url = "projects" @@ -1142,76 +1149,77 @@ class Project(Resource): last_activity_at: str - web_url: Optional[str] = None - shared_with_groups: List[Dict[str, Any]] = field(default_factory=list) - visibility: Optional[str ] = None - merge_method: Optional[str ] = None - _links: Optional[dict] = None - archived: Optional[bool] = None + web_url: Optional[str] = None + shared_with_groups: List[Dict[str, Any]] = field( + default_factory=list) + visibility: Optional[str] = None + merge_method: Optional[str] = None + _links: Optional[dict] = None + archived: Optional[bool] = None resolve_outdated_diff_discussions: Optional[bool] = None - container_registry_enabled: Optional[bool] = None - container_expiration_policy: Optional[dict] = None - issues_enabled: Optional[bool] = None - merge_requests_enabled: Optional[bool] = None - wiki_enabled: Optional[bool] = None - jobs_enabled: Optional[bool] = None - snippets_enabled: Optional[bool] = None - shared_runners_enabled: Optional[bool] = None - lfs_enabled: Optional[bool] = None - packages_enabled: Optional[bool] = None - service_desk_enabled: Optional[bool] = None - service_desk_address: Optional[str ] = None - empty_repo: Optional[bool] = None - public_jobs: Optional[bool] = None + container_registry_enabled: Optional[bool] = None + container_expiration_policy: Optional[dict] = None + issues_enabled: Optional[bool] = None + merge_requests_enabled: Optional[bool] = None + wiki_enabled: Optional[bool] = None + jobs_enabled: Optional[bool] = None + snippets_enabled: Optional[bool] = None + shared_runners_enabled: Optional[bool] = None + lfs_enabled: Optional[bool] = None + packages_enabled: Optional[bool] = None + service_desk_enabled: Optional[bool] = None + service_desk_address: Optional[str] = None + empty_repo: Optional[bool] = None + public_jobs: Optional[bool] = None only_allow_merge_if_pipeline_succeeds: Optional[bool] = None - request_access_enabled: Optional[bool] = None + request_access_enabled: Optional[bool] = None only_allow_merge_if_all_discussions_are_resolved: Optional[bool] = None printing_merge_request_link_enabled: Optional[bool] = None - can_create_merge_request_in: Optional[bool] = None - issues_access_level: Optional[str ] = None - repository_access_level: Optional[str ] = None - merge_requests_access_level: Optional[str ] = None - forking_access_level: Optional[str ] = None - wiki_access_level: Optional[str ] = None - builds_access_level: Optional[str ] = None - snippets_access_level: Optional[str ] = None - pages_access_level: Optional[str ] = None - operations_access_level: Optional[str ] = None - analytics_access_level: Optional[str ] = None - emails_disabled: Optional[bool] = None - ci_default_git_depth: Optional[int ] = None - ci_forward_deployment_enabled: Optional[bool] = None - build_timeout: Optional[int ] = None - auto_cancel_pending_pipelines: Optional[bool] = None - build_coverage_regex: Optional[str ] = None - allow_merge_on_skipped_pipeline: Optional[bool] = None - remove_source_branch_after_merge: Optional[bool] = None - suggestion_commit_message: Optional[str ] = None - auto_devops_enabled: Optional[bool] = None - auto_devops_deploy_strategy: Optional[str ] = None - autoclose_referenced_issues: Optional[bool] = None - - creator_id: Optional[int ] = None - import_status: Optional[str ] = None - open_issues_count: Optional[int ] = None - ci_config_path: Optional[str ] = None - - repository_storage: Optional[str ] = None - - permissions: Optional[dict] = None - forked_from_project: Optional['Project'] = None - import_error: Optional[str ] = None - runners_token: Optional[str ] = None - owner: Optional['User'] = None - - build_git_strategy: Optional[str ] = None - restrict_user_defined_variables: Optional[bool] = None - - container_registry_image_prefix: Optional[str ] = None - topics: Optional[list] = None - ci_job_token_scope_enabled: Optional[bool] = None - squash_option: Optional[str ] = None - keep_latest_artifact: Optional[bool] = None + can_create_merge_request_in: Optional[bool] = None + issues_access_level: Optional[str] = None + repository_access_level: Optional[str] = None + merge_requests_access_level: Optional[str] = None + forking_access_level: Optional[str] = None + wiki_access_level: Optional[str] = None + builds_access_level: Optional[str] = None + snippets_access_level: Optional[str] = None + pages_access_level: Optional[str] = None + operations_access_level: Optional[str] = None + analytics_access_level: Optional[str] = None + emails_disabled: Optional[bool] = None + ci_default_git_depth: Optional[int] = None + ci_forward_deployment_enabled: Optional[bool] = None + build_timeout: Optional[int] = None + auto_cancel_pending_pipelines: Optional[bool] = None + build_coverage_regex: Optional[str] = None + allow_merge_on_skipped_pipeline: Optional[bool] = None + remove_source_branch_after_merge: Optional[bool] = None + suggestion_commit_message: Optional[str] = None + auto_devops_enabled: Optional[bool] = None + auto_devops_deploy_strategy: Optional[str] = None + autoclose_referenced_issues: Optional[bool] = None + + creator_id: Optional[int] = None + import_status: Optional[str] = None + open_issues_count: Optional[int] = None + ci_config_path: Optional[str] = None + + repository_storage: Optional[str] = None + + permissions: Optional[dict] = None + forked_from_project: Optional['Project'] = None + import_error: Optional[str] = None + runners_token: Optional[str] = None + owner: Optional['User'] = None + + build_git_strategy: Optional[str] = None + restrict_user_defined_variables: Optional[bool] = None + + container_registry_image_prefix: Optional[str] = None + topics: Optional[list] = None + ci_job_token_scope_enabled: Optional[bool] = None + squash_option: Optional[str] = None + keep_latest_artifact: Optional[bool] = None def archive(self) -> 'Project': """Archive current project""" @@ -1247,10 +1255,12 @@ class Project(Resource): if full_path is None: full_path = self.path_with_namespace - self.gitlab.log.info(f'Exporting project {self.path_with_namespace} to {full_path} on {forge}') + self.gitlab.log.info( + f'Exporting project {self.path_with_namespace} to {full_path} on {forge}') self.gitlab.post_json(f'/projects/{self.id}/export') - while self.gitlab.get_json(f'/projects/{self.id}/export')['export_status'] != 'finished': + while self.gitlab.get_json( + f'/projects/{self.id}/export')['export_status'] != 'finished': time.sleep(1) self.gitlab.log.info('waiting for export to complete') res = self.gitlab.get(f'/projects/{self.id}/export/download') @@ -1262,10 +1272,10 @@ class Project(Resource): "namespace": os.path.dirname(full_path) } json = forge.post_json('projects/import', data=data, files=files) - id = json['id'] - while forge.get_json(f'/projects/{id}/import')['import_status'] != 'finished': + project_id = json['id'] + while forge.get_json(f'/projects/{project_id}/import')['import_status'] != 'finished': time.sleep(1) - return forge.get_project(id) + return forge.get_project(project_id) def ensure_fork(self, path: str, @@ -1450,12 +1460,12 @@ class Project(Resource): >>> assert other_user.id in [u['id'] for u in project.get_members()] """ if isinstance(group_or_user, User): - # user_id = self.gitlab.get_user(group_or_user).id + #Comment To Remove : user_id = self.gitlab.get_user(group_or_user).id user_id = group_or_user.id if any(user_id == user['id'] for user in self.get_members()): return - data : dict = dict(user_id=user_id, - access_level=int(access)) + data: dict = dict(user_id=user_id, + access_level=int(access)) if expires_at is not None: data['expires_at'] = expires_at json = self.gitlab.post(f"/projects/{self.id}/members", @@ -1463,7 +1473,7 @@ class Project(Resource): assert json['id'] == user_id assert json['access_level'] == int(access) elif isinstance(group_or_user, Group): - # group_id = self.gitlab.get_group(group_or_user).id + #Comment To Remove : group_id = self.gitlab.get_group(group_or_user).id group_id = group_or_user.id data = dict(group_id=group_id, group_access=int(access)) @@ -1479,7 +1489,7 @@ class Project(Resource): def get_forks_ssh_url_to_repo(self) -> List[str]: forks = self.gitlab.get(f"/projects/{self.id}/forks").json() - return [ fork['ssh_url_to_repo'] for fork in forks ] + return [fork['ssh_url_to_repo'] for fork in forks] def get_forks(self, recursive: Union[int, bool] = False, @@ -1554,7 +1564,6 @@ class Project(Resource): simple=simple)] return forks - def get_origin_commit(self) -> JSON: """ Return the first (oldest) commit of the repository for the default branch. @@ -1563,11 +1572,12 @@ class Project(Resource): See `get_possible_forks`. """ - # Need to iterate because there is no direct link to the last page nor a ascending order + # Need to iterate because there is no direct link to the last page nor a + # ascending order page: Optional[str] = "1" while page is not None and page != '': res = self.gitlab.get(f"/projects/{self.id}/repository/commits", - data={"per_page": 100, "page": page}) + data={"per_page": 100, "page": page}) page = res.headers.get("X-Next-Page") json = res.json() if len(json) == 0: @@ -1575,8 +1585,11 @@ class Project(Resource): else: return json[-1] - - def get_possible_forks(self, deep:bool=False, nonfork:bool=False, progress:bool=False) -> List['Project']: + def get_possible_forks( + self, + deep: bool = False, + nonfork: bool = False, + progress: bool = False) -> List['Project']: """ Iterate onto newer projects to detect is they are possible forks but not identified as such. @@ -1609,13 +1622,13 @@ class Project(Resource): q = {"id_after": self.id, "page": page} if not deep: q["min_access_level"] = 40 - res = self.gitlab.get(f"/projects", data=q) + res = self.gitlab.get("/projects", data=q) page = res.headers.get("X-Next-Page") for json in res.json(): other = Project(self.gitlab, **json) if (other.forked_from_project is not None and (not deep or other.forked_from_project.id == self.id)): - continue # Already registered as a fork of something + continue # Already registered as a fork of something if progress: print(".", end='', flush=True) if nonfork and other.path == self.path: @@ -1632,7 +1645,6 @@ class Project(Resource): print("", flush=True) return result - def add_origin(self, origin: 'Project') -> JSON: """ Register `self` as a fork of `origin`. @@ -1641,7 +1653,6 @@ class Project(Resource): json = self.gitlab.post(f"/projects/{self.id}/fork/{origin.id}").json() return json - def get_branches(self) -> List[JSON]: """ Return the branches of this project @@ -1677,7 +1688,8 @@ class Project(Resource): 'protected': True, 'web_url': 'https://gitlab-test.info.uqam.ca/groupe-public-test/projet-public/-/tree/master'}] """ - return cast(list, self.gitlab.get(f"/projects/{self.id}/repository/branches").json()) + return cast(list, self.gitlab.get( + f"/projects/{self.id}/repository/branches").json()) def get_branch(self, branch_name: Optional[str] = None) -> JSON: """ @@ -1714,14 +1726,14 @@ class Project(Resource): file: str, ref: str) -> JSON: """Get a file from the repository""" - file = urlencode(file) - json = self.gitlab.get(f"/projects/{self.id}/repository/files/{file}", + file_encoded = urlencode(file) + json = self.gitlab.get(f"/projects/{self.id}/repository/files/{file_encoded}", data=dict( ref=ref, )).json() error = json.get('error', json.get('message')) if error is not None: - raise RuntimeError(f"get file {file} of ref {ref} " + raise RuntimeError(f"get file {file_encoded} of ref {ref} " f"of project {self.path_with_namespace} failed: " f"{error}") return json @@ -1790,14 +1802,14 @@ class Project(Resource): of project travo-test-etu/temporary-test-projet-20... failed: 404 Commit Not Found """ - file = urlencode(file) + file_1 = urlencode(file) if branch is None: branch = self.default_branch try: - oldcontent = self.get_file(file, branch)['content'] + oldcontent = self.get_file(file_1, branch)['content'] except RuntimeError: oldcontent = None - if oldcontent is not None: # Compare old and desired content + if oldcontent is not None: # Compare old and desired content if content is None: return if encoding == "text": @@ -1810,20 +1822,20 @@ class Project(Resource): content=content, branch=branch, commit_message=commit_message, - ) + ) if encoding != "text": data['encoding'] = encoding if oldcontent is None: # file does not yet exist - json = self.gitlab.post(f"/projects/{self.id}/repository/files/{file}", + json = self.gitlab.post(f"/projects/{self.id}/repository/files/{file_1}", data=data ).json() else: - json = self.gitlab.put(f"/projects/{self.id}/repository/files/{file}", + json = self.gitlab.put(f"/projects/{self.id}/repository/files/{file_1}", data=data ).json() error = json.get('error', json.get('message')) if error is not None: - raise RuntimeError(f"ensuring file {file} " + raise RuntimeError(f"ensuring file {file_1} " f"of project {self.path_with_namespace} failed: " f"{error}") @@ -1876,7 +1888,8 @@ class Project(Resource): json = self.gitlab.post(f"/projects/{self.id}/protected_branches/", data=dict(name=name)).json() if 'message' in json: - raise RuntimeError(f"protecting branch {name} of project {self.path_with_namespace} failed: {json['message']}") + raise RuntimeError( + f"protecting branch {name} of project {self.path_with_namespace} failed: {json['message']}") def unprotect_branch(self, name: str) -> None: """ @@ -1888,7 +1901,8 @@ class Project(Resource): return json = r.json() if 'message' in json: - raise RuntimeError(f"unprotecting branch {name} of project {self.path_with_namespace} failed: {json['message']}") + raise RuntimeError( + f"unprotecting branch {name} of project {self.path_with_namespace} failed: {json['message']}") def get_pipelines(self) -> JSON: """ @@ -1923,25 +1937,28 @@ class Project(Resource): for pipeline in pipelines: pref = pipeline['ref'] if ref is not None and pref != ref: - continue # Skip unwanted refs + continue # Skip unwanted refs if pref in suites: - continue # skip older pipeline for this ref + continue # skip older pipeline for this ref suites[pref] = pipeline prefix = pref + '.' - report = self.gitlab.get(f"/projects/{self.id}/pipelines/{pipeline['id']}/test_report").json() + report = self.gitlab.get( + f"/projects/{self.id}/pipelines/{pipeline['id']}/test_report").json() for suite in report['test_suites']: suites[prefix + suite['name']] = suite - jobs = self.gitlab.get(f"/projects/{self.id}/pipelines/{pipeline['id']}/jobs").json() + jobs = self.gitlab.get( + f"/projects/{self.id}/pipelines/{pipeline['id']}/jobs").json() for job in jobs: name = prefix + job['name'] if name in suites: - job.update(suites[name]) + job.update(suites[name]) suites[name] = job - log = self.gitlab.get(f"/projects/{self.id}/jobs/{job['id']}/trace").text + log = self.gitlab.get( + f"/projects/{self.id}/jobs/{job['id']}/trace").text job['log'] = log if 'total_count' not in job: ok = len(re.findall('^ok ', log, re.MULTILINE)) @@ -1985,7 +2002,8 @@ class Project(Resource): return self.gitlab.get_json(f"/projects/{self.id}/members/all", depaginate=True) - _get_owner_cache: Optional[JSON] = None + _get_owner_cache: Optional[JSON] = None + def get_owners(self) -> JSON: """ Return the owners including all indirect members with an access_level >= 50 @@ -2015,8 +2033,11 @@ class Project(Resource): self._get_owner_cache = res return res - - def get_compare(self, from_project: 'Project', ref: str = "", from_ref: str = "") -> JSON: + def get_compare( + self, + from_project: 'Project', + ref: str = "", + from_ref: str = "") -> JSON: """ Compare `self` to another project. @@ -2031,8 +2052,13 @@ class Project(Resource): if from_ref == "": from_ref = from_project.default_branch - res = self.gitlab.get(f"/projects/{self.id}/repository/compare", - data={"from": from_ref, "to": ref, "from_project_id": {from_project.id}}).json() + res = self.gitlab.get( + f"/projects/{self.id}/repository/compare", + data={ + "from": from_ref, + "to": ref, + "from_project_id": { + from_project.id}}).json() return res def get_badges(self, name: Optional[str] = None) -> List[JSON]: @@ -2069,10 +2095,10 @@ class Project(Resource): ... image_url='https://foo_image', ... ) {'name': 'Foo', - 'link_url': 'https://foo_link', + 'link_url': 'https://foo_link', 'image_url': 'https://foo_image', 'rendered_link_url': 'https://foo_link', - 'rendered_image_url': 'https://foo_image', + 'rendered_image_url': 'https://foo_image', 'id': ..., 'kind': 'project'} >>> project.ensure_badge(name='Bar', @@ -2084,7 +2110,7 @@ class Project(Resource): 'image_url': 'https://bar_image', 'rendered_link_url': 'https://bar_link', 'rendered_image_url': 'https://bar_image', - 'id': ..., + 'id': ..., 'kind': 'project'} >>> project.get_badges(name="Bar") [{'name': 'Bar', @@ -2092,14 +2118,14 @@ class Project(Resource): 'image_url': 'https://bar_image', 'rendered_link_url': 'https://bar_link', 'rendered_image_url': 'https://bar_image', - 'id': ..., + 'id': ..., 'kind': 'project'}] >>> project.get_badges() [{'name': 'Foo', - 'link_url': 'https://foo_link', + 'link_url': 'https://foo_link', 'image_url': 'https://foo_image', 'rendered_link_url': 'https://foo_link', - 'rendered_image_url': 'https://foo_image', + 'rendered_image_url': 'https://foo_image', 'id': ..., 'kind': 'project'}, {'name': 'Bar', @@ -2107,7 +2133,7 @@ class Project(Resource): 'image_url': 'https://bar_image', 'rendered_link_url': 'https://bar_link', 'rendered_image_url': 'https://bar_image', - 'id': ..., + 'id': ..., 'kind': 'project'}] >>> project.ensure_badge(name='Bar', @@ -2119,7 +2145,7 @@ class Project(Resource): 'image_url': 'https://bar2_image', 'rendered_link_url': 'https://bar2_link', 'rendered_image_url': 'https://bar2_image', - 'id': ..., + 'id': ..., 'kind': 'project'} >>> project.get_badges(name="Bar") [{'name': 'Bar', @@ -2127,7 +2153,7 @@ class Project(Resource): 'image_url': 'https://bar2_image', 'rendered_link_url': 'https://bar2_link', 'rendered_image_url': 'https://bar2_image', - 'id': ..., + 'id': ..., 'kind': 'project'}] """ @@ -2137,7 +2163,7 @@ class Project(Resource): if len(badges) == 0: return self.gitlab.post_json(f'/projects/{self.id}/badges', - data=data) + data=data) badge = badges[0] if badge['link_url'] == link_url and badge['image_url'] == image_url: return badge @@ -2149,7 +2175,7 @@ class Project(Resource): date: Optional[str] = None, pull_can_fail: bool = False, force: bool = False, - anonymous: bool=False) -> None: + anonymous: bool = False) -> None: """ Clone or pull the project on the file system (with git). @@ -2236,10 +2262,14 @@ class Project(Resource): self.http_url_to_repo, path], anonymous=anonymous) if date is not None: - self.gitlab.git(["-c", "advice.detachedHead=false", "checkout", "origin/HEAD@{'"+date+"'}"], + self.gitlab.git(["-c", + "advice.detachedHead=false", + "checkout", + "origin/HEAD@{'" + date + "'}"], cwd=path, anonymous=anonymous) + @dataclass class Namespace(Resource): _resource_type_api_url = "namespaces" @@ -2251,9 +2281,9 @@ class Namespace(Resource): path: str full_path: str kind: str = field(repr=False) - web_url: Optional[str] = None - parent_id: Optional[int] = None - avatar_url: Optional[str] = None + web_url: Optional[str] = None + parent_id: Optional[int] = None + avatar_url: Optional[str] = None members_count_with_descendants: Optional[int] = None @@ -2263,7 +2293,8 @@ class Group(Resource): # In the GitLab api, the namespace of a group is specified # in the attribute `parent_id` rather than `namespace_id` _resource_type_namespace_id_attribute = "parent_id" - _read_only = Namespace._read_only + ('full_name', 'parent_id', 'projects', 'shared_projects') + _read_only = Namespace._read_only + \ + ('full_name', 'parent_id', 'projects', 'shared_projects') name: str path: str full_name: str @@ -2273,23 +2304,26 @@ class Group(Resource): avatar_url: str lfs_enabled: bool request_access_enabled: bool - web_url: Optional[str] = None - shared_with_groups: List[Dict[str, Any]] = field(default_factory=list) - projects: List[Project] = field(repr=False, default_factory=list) - shared_projects: List[Project] = field(repr=False, default_factory=list) - - parent_id: Optional[int ] = None - created_at: Optional[str ] = None - default_branch_protection: Optional[str ] = None - subgroup_creation_level: Optional[str ] = None - project_creation_level: Optional[str ] = None - auto_devops_enabled: Optional[str ] = None - mentions_disabled: Optional[bool] = None - emails_disabled: Optional[bool] = None - two_factor_grace_period: Optional[bool] = None + web_url: Optional[str] = None + shared_with_groups: List[Dict[str, Any]] = field( + default_factory=list) + projects: List[Project] = field( + repr=False, default_factory=list) + shared_projects: List[Project] = field( + repr=False, default_factory=list) + + parent_id: Optional[int] = None + created_at: Optional[str] = None + default_branch_protection: Optional[str] = None + subgroup_creation_level: Optional[str] = None + project_creation_level: Optional[str] = None + auto_devops_enabled: Optional[str] = None + mentions_disabled: Optional[bool] = None + emails_disabled: Optional[bool] = None + two_factor_grace_period: Optional[bool] = None require_two_factor_authentication: Optional[bool] = None - share_with_group_lock: Optional[bool] = None - runners_token: Optional[str ] = None + share_with_group_lock: Optional[bool] = None + runners_token: Optional[str] = None def get_projects(self, owned: bool = False, @@ -2425,20 +2459,24 @@ class Group(Resource): for glob in ignore if fnmatch.fnmatch(project.path_with_namespace, glob)] if matches: - self.gitlab.log.info(f"Ignoring project {project.path_with_namespace} which matches {matches}") + self.gitlab.log.info( + f"Ignoring project {project.path_with_namespace} which matches {matches}") continue if project.archived: - self.gitlab.log.info(f'Ignoring archived project {project.path_with_namespace}') + self.gitlab.log.info( + f'Ignoring archived project {project.path_with_namespace}') continue if project.forked_from_project is not None: - self.gitlab.log.info(f'Ignoring project {project.path_with_namespace} which is a fork') + self.gitlab.log.info( + f'Ignoring project {project.path_with_namespace} which is a fork') continue project_full_path = os.path.join(full_path, project.path) try: forge.get_project(project_full_path) - self.gitlab.log.info(f'Ignoring project {project.path_with_namespace} which already exists in the target forge') + self.gitlab.log.info( + f'Ignoring project {project.path_with_namespace} which already exists in the target forge') continue - + except ResourceNotFoundError: pass project.export(forge, project_full_path) @@ -2449,45 +2487,45 @@ class Group(Resource): class User(Resource): _resource_type_api_url = "users" _read_only = ('id', 'path', 'namespace') - web_url: Optional[str] = None - name: Optional[str] = None - username: Optional[str] = None - state: Optional[str] = None - avatar_url: Optional[str] = None - created_at: Optional[str] = None - bio: Optional[str] = None - bio_html: Optional[str] = None - job_title: Optional[str] = None - work_information: Optional[str] = None - message: Optional[str] = None - location: Optional[str] = None - public_email: Optional[str] = None - skype: Optional[str] = None - linkedin: Optional[str] = None - twitter: Optional[str] = None - website_url: Optional[str] = None - organization: Optional[str] = None - last_sign_in_at: Optional[str] = None - confirmed_at: Optional[str] = None - last_activity_on: Optional[str] = None - email: Optional[str] = None - theme_id: Optional[int] = None - color_scheme_id: Optional[int] = None - projects_limit: Optional[int] = None - current_sign_in_at: Optional[str] = None - identities: Optional[List[Dict]] = None - can_create_group: Optional[bool] = None - can_create_project: Optional[bool] = None - two_factor_enabled: Optional[bool] = None - external: Optional[bool] = None - private_profile: Optional[bool] = None - - bot: Optional[bool] = None - followers: Optional[int ] = None - following: Optional[int ] = None - commit_email: Optional[str ] = None - is_admin: Optional[bool] = None - note: Optional[str ] = None + web_url: Optional[str] = None + name: Optional[str] = None + username: Optional[str] = None + state: Optional[str] = None + avatar_url: Optional[str] = None + created_at: Optional[str] = None + bio: Optional[str] = None + bio_html: Optional[str] = None + job_title: Optional[str] = None + work_information: Optional[str] = None + message: Optional[str] = None + location: Optional[str] = None + public_email: Optional[str] = None + skype: Optional[str] = None + linkedin: Optional[str] = None + twitter: Optional[str] = None + website_url: Optional[str] = None + organization: Optional[str] = None + last_sign_in_at: Optional[str] = None + confirmed_at: Optional[str] = None + last_activity_on: Optional[str] = None + email: Optional[str] = None + theme_id: Optional[int] = None + color_scheme_id: Optional[int] = None + projects_limit: Optional[int] = None + current_sign_in_at: Optional[str] = None + identities: Optional[List[Dict]] = None + can_create_group: Optional[bool] = None + can_create_project: Optional[bool] = None + two_factor_enabled: Optional[bool] = None + external: Optional[bool] = None + private_profile: Optional[bool] = None + + bot: Optional[bool] = None + followers: Optional[int] = None + following: Optional[int] = None + commit_email: Optional[str] = None + is_admin: Optional[bool] = None + note: Optional[str] = None def get_projects(self, with_shared: bool = False, @@ -2504,20 +2542,25 @@ class User(Resource): export = Group.export + @dataclass class AnonymousUser: username: str = "anonymous" + anonymous_user = AnonymousUser() class Unknown(enum.Enum): unknown = enum.auto() + def __repr__(self) -> str: return f'{self.name}' + unknown = Unknown.unknown + class GitLabTest(GitLab): """ A gitlab instance for testing purposes @@ -2534,7 +2577,7 @@ class GitLabTest(GitLab): self.tempdir: tempfile.TemporaryDirectory = tempfile.TemporaryDirectory() super().__init__(base_url=base_url, home_dir=self.tempdir.name) - def confirm(self, message:str) -> bool: + def confirm(self, message: str) -> bool: return True def login(self, diff --git a/travo/homework.py b/travo/homework.py index 374c84f901fbde162f41964753bb8454a8c533b1..4ee4fc03dcf7b88a667d2f0f5972a718c4b98c7b 100755 --- a/travo/homework.py +++ b/travo/homework.py @@ -4,6 +4,7 @@ from typing import Optional, List from travo.utils import git_get_origin from travo.gitlab import GitLab, ResourceRef, Project, User, Group + @dataclass class Homework: """ @@ -16,16 +17,15 @@ class Homework: """ gitlab: GitLab - project: Project # the original project - assignment: Project # the assignment (can be `project` or not) - directory: str # The working directory. Usually `.` - - group: Optional[Group] = None # The possible correction group - group_level: int = 20 # The access level of the group + project: Project # the original project + assignment: Project # the assignment (can be `project` or not) + directory: str # The working directory. Usually `.` - instructor: Optional[User] = None # The possible instructor - instructor_level: int = 40 # The access level of the instructor + group: Optional[Group] = None # The possible correction group + group_level: int = 20 # The access level of the group + instructor: Optional[User] = None # The possible instructor + instructor_level: int = 40 # The access level of the instructor def __init__(self, url: str = '.') -> None: """ @@ -45,11 +45,13 @@ class Homework: self.directory = url try: url = git_get_origin(url) - except: + except BaseException: if url == ".": - raise RuntimeError(f"The current directory is not a valid travo/git directory. Specify a valid directory or an project URL.") + raise RuntimeError( + "The current directory is not a valid travo/git directory. Specify a valid directory or an project URL.") else: - raise RuntimeError(f"{url} is not a valid travo/git directory. Specify a valid directory or an project URL.") + raise RuntimeError( + f"{url} is not a valid travo/git directory. Specify a valid directory or an project URL.") else: self.directory = '.' @@ -88,7 +90,8 @@ class Homework: # This is likely an assignment assignment = project.forked_from_project self.assignment = assignment - project.gitlab.log.info(f"Fork of: '{assignment.name_with_namespace}' {assignment.web_url}") + project.gitlab.log.info( + f"Fork of: '{assignment.name_with_namespace}' {assignment.web_url}") forks = [project] self.forks = forks return forks @@ -98,8 +101,7 @@ class Homework: self.copies = forks return forks - - def check_student(self, project: 'Project', fixup:bool=False) -> bool: + def check_student(self, project: 'Project', fixup: bool = False) -> bool: """ Check various configuration on a student copy. @@ -119,13 +121,14 @@ class Homework: # Check fork if assignment is not None: if project.forked_from_project is None: - print(f" ❌ not a fork") + print(" ❌ not a fork") result = False if fixup: project.add_origin(assignment) recheck = True elif assignment.id != project.forked_from_project.id: - print(f" ❌ bad fork of {project.forked_from_project.name_with_namespace}") + print( + f" ❌ bad fork of {project.forked_from_project.name_with_namespace}") result = False # Check visibiliy @@ -146,7 +149,7 @@ class Homework: if g['group_id'] == self.group.id: if g['group_access_level'] >= self.group_level: sharedok = True - continue + if not sharedok: print(f" ❌ bad group sharing with group {self.group.full_path}") result = False @@ -158,7 +161,7 @@ class Homework: if instructor is None and assignment is not None: owners = assignment.get_owners() if len(owners) == 0: - print(f" ❌ no assignment owner") + print(" ❌ no assignment owner") result = False elif len(owners) > 1: print(f" ❌ multiple assignment owners {owners}") @@ -171,22 +174,21 @@ class Homework: for member in project.get_members(): if member['id'] == instructor.id and member['access_level'] >= self.instructor_level: instructok = True - continue + if not instructok: print(f" ❌ bad membership of instructor {instructor.username}") result = False if fixup: - #FIXME share_with do not works on users yet - #project.share_with(instructor, self.instructor_level) + ##FIXME share_with do not works on users yet + # project.share_with(instructor, self.instructor_level) recheck = True if recheck: - print(f" Recheck after fixup!") + print(" Recheck after fixup!") result = self.check_student(project) return result - - def print_info(self, project: 'Project', fixup:bool=False) -> None: + def print_info(self, project: 'Project', fixup: bool = False) -> None: """ Print most information about a student project """ @@ -201,7 +203,7 @@ class Homework: self.check_student(project, fixup=fixup) if project.default_branch is None: - print(f" ❌ empty repository") + print(" ❌ empty repository") return nocommits = False @@ -213,12 +215,13 @@ class Homework: compare = None if compare is not None: if len(compare['commits']) == 0: - print(f" ❌ no commits") + print(" ❌ no commits") nocommits = True compare_rev = assignment.get_compare(project) if len(compare_rev['commits']) != 0: - print(f" not up to date with upstream; lags {len(compare_rev['commits'])}") + print( + f" not up to date with upstream; lags {len(compare_rev['commits'])}") if nocommits: return @@ -234,7 +237,8 @@ class Homework: mark = "❌" else: mark = " " - print(f" {mark} {k}: {v['status']} {v['success_count']}/{v['total_count']}") + print( + f" {mark} {k}: {v['status']} {v['success_count']}/{v['total_count']}") else: if status == "success": mark = "✅" diff --git a/travo/jupyter_course.py b/travo/jupyter_course.py index a1ddbaad6a3d145d69f43e481d1b1c9d51bb140f..7c4a14e00ff88f8a993d1b895358d0c6e19958c7 100644 --- a/travo/jupyter_course.py +++ b/travo/jupyter_course.py @@ -23,6 +23,7 @@ from .gitlab import GitLab, AuthenticationError from .gitlab import ResourceNotFoundError from . import dashboards + @contextlib.contextmanager def TrivialContextManager() -> Iterator[Any]: yield @@ -30,7 +31,7 @@ def TrivialContextManager() -> Iterator[Any]: # Currently just a dummy grade report, just for making some tests grade_report = """ - + @@ -133,6 +134,7 @@ class JupyterCourse(Course): ignore_nbgrader = ignore + [".*"] gitlab_ci_yml = None release_directory: str = "release" + gradebook_db: str = ".gradebook.db"; @staticmethod def validate(*files: str) -> None: @@ -141,16 +143,16 @@ class JupyterCourse(Course): """ errors = 0 failures = 0 - for file in files: - if file.endswith(".md"): - testfile = f".test.{file}.ipynb" - run(["jupytext", file, "-o", testfile]) - file = testfile + for file_1 in files: + if file_1.endswith(".md"): + testfile = f".test.{file_1}.ipynb" + run(["jupytext", file_1, "-o", testfile]) + file_1 = testfile else: testfile = "" - assert file.endswith(".ipynb") + assert file_1.endswith(".ipynb") command = ["nbgrader", "validate"] - process = subprocess.Popen([*command, file], + process = subprocess.Popen([*command, file_1], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, @@ -166,7 +168,7 @@ class JupyterCourse(Course): os.remove(testfile) if failures + errors: print(_("validation failed", errors=str(errors), failures=str(failures))) - exit(failures+errors) + exit(failures + errors) def convert_from_md_to_ipynb(self, path: str) -> None: import jupytext # type: ignore @@ -203,8 +205,12 @@ class JupyterCourse(Course): file.write(filedata) run(["jupytext", "--sync", ipynbname]) - def generate_assignment(self, assignment: str, release_dir: str = "release", - add_gitignore: bool = True, add_gitlab_ci: bool = True) -> None: + def generate_assignment( + self, + assignment: str, + release_dir: str = "release", + add_gitignore: bool = True, + add_gitlab_ci: bool = True) -> None: """ Generate the student version of the given assignment """ @@ -214,15 +220,17 @@ class JupyterCourse(Course): with tempfile.TemporaryDirectory() as tmpdirname: gitdir = os.path.join(target_path, ".git") tmpgitdir = os.path.join(tmpdirname, ".git") - db = os.path.join(target_path, ".gradebook.db") - # tmpdb = os.path.join(tmpdirname, ".gradebook.db") + db = os.path.join(target_path, self.gradebook_db) + #ToRemove: tmpdb = os.path.join(tmpdirname, ".gradebook.db") preserve_gitdir = os.path.exists(gitdir) if preserve_gitdir: self.log.info("Sauvegarde de l'historique git") shutil.move(gitdir, tmpgitdir) try: - run(["nbgrader", "generate_assignment", "--force", assignment, f"--db='sqlite:///{db}'"]) - self.convert_from_ipynb_to_md(path=os.path.join(release_dir, assignment)) + run(["nbgrader", "generate_assignment", "--force", + assignment, f"--db='sqlite:///{db}'"]) + self.convert_from_ipynb_to_md( + path=os.path.join(release_dir, assignment)) except (): pass finally: @@ -239,16 +247,18 @@ class JupyterCourse(Course): self.gitlab_ci_yml.format(assignment=assignment)) if add_gitignore: io.open(os.path.join(target_path, ".gitignore"), "w").write( - '\n'.join(self.ignore)+'\n') + '\n'.join(self.ignore) + '\n') run(["git", "add", "."], cwd=target_path) - run(["git", "commit", "-n", "--allow-empty", f"-m {assignment} {datetime.now()}"], cwd=target_path) + run(["git", "commit", "-n", "--allow-empty", + f"-m {assignment} {datetime.now()}"], cwd=target_path) def nbgrader_update_student_list(self, tag: str = "", submitted_directory: str = "submitted") -> None: """Piece of code specific to methnum, to be more developed and generalized... """ - student_list = run(["nbgrader", "db", "student", "list"], capture_output=True).stdout.decode("utf-8") + student_list = run(["nbgrader", "db", "student", "list"], + capture_output=True).stdout.decode("utf-8") for student_id in os.listdir(submitted_directory): if tag.replace('*', '') not in student_id: continue @@ -259,13 +269,20 @@ class JupyterCourse(Course): elif student_id.count(".") == 2: group, firstname, lastname = student_id.split(".") else: - raise ValueError(f"Unknown student format {student_id}. " - f"Must be group.firstname.lastname or firstname.lastname.") + raise ValueError( + f"Unknown student format {student_id}. " + f"Must be group.firstname.lastname or firstname.lastname.") firstname = firstname.lower() lastname = lastname.lower() email = f"{firstname}.{lastname}@{self.mail_extension}" - run(["nbgrader", "db", "student", "add", f"{student_id}", - f"--first-name={firstname}", f"--last-name={lastname}", f"--email={email}"]) + run(["nbgrader", + "db", + "student", + "add", + f"{student_id}", + f"--first-name={firstname}", + f"--last-name={lastname}", + f"--email={email}"]) def get_nbgrader_config(self) -> List[str]: nbgrader_config = ['--CourseDirectory.ignore=' + str(self.ignore_nbgrader)] @@ -287,10 +304,12 @@ class JupyterCourse(Course): for student_id in os.listdir("submitted"): if tag.replace('*', '') not in student_id: continue - self.convert_from_md_to_ipynb(os.path.join("submitted", student_id, assignment)) + self.convert_from_md_to_ipynb(os.path.join( + "submitted", student_id, assignment)) self.convert_from_md_to_ipynb(assignment) # run(["nbgrader", "generate_assignment", "--force", assignment]) - run(["nbgrader", "autograde", *nbgrader_config, os.path.basename(assignment_name), f"--student={tag}"]) + run(["nbgrader", "autograde", *nbgrader_config, + os.path.basename(assignment_name), f"--student={tag}"]) def generate_feedback(self, assignment_name: str, @@ -301,13 +320,22 @@ class JupyterCourse(Course): The student name can be given with wildcard. """ nbgrader_config = self.get_nbgrader_config() - run(["nbgrader", "generate_feedback", "--force", "--CourseDirectory.feedback_directory=feedback_generated", - *nbgrader_config, os.path.basename(assignment_name), f"--student={tag}"]) + run(["nbgrader", + "generate_feedback", + "--force", + "--CourseDirectory.feedback_directory=feedback_generated", + *nbgrader_config, + os.path.basename(assignment_name), + f"--student={tag}"]) if os.path.exists("feedback"): shutil.rmtree("feedback") shutil.copytree("feedback_generated", "feedback") - self.merge_autograded_db(os.path.basename(assignment_name), back=True, on_inconsistency="WARNING", - tag=tag, new_score_policy=new_score_policy) + self.merge_autograded_db( + os.path.basename(assignment_name), + back=True, + on_inconsistency="WARNING", + tag=tag, + new_score_policy=new_score_policy) def student_autograde(self, assignment_name: str, @@ -328,20 +356,28 @@ class JupyterCourse(Course): elif student.count(".") == 2: group, firstname, lastname = student.split(".") else: - raise ValueError(f"Unknown student format {student}. " - f"Must be group.firstname.lastname or firstname.lastname.") + raise ValueError( + f"Unknown student format {student}. " + f"Must be group.firstname.lastname or firstname.lastname.") firstname = firstname.lower() lastname = lastname.lower() email = f"{firstname}.{lastname}@{self.mail_extension}" - run(["nbgrader", "db", "student", "add", f"{student}", f"--first-name={firstname}", - f"--last-name={lastname}", f"--email={email}", "--db=sqlite:///.gradebook.db"]) + run(["nbgrader", + "db", + "student", + "add", + f"{student}", + f"--first-name={firstname}", + f"--last-name={lastname}", + f"--email={email}", + "--db=sqlite:///.gradebook.db"]) nbgrader_config = ["--db=sqlite:///.gradebook.db", "--force", "--CourseDirectory.submitted_directory=submitted", "--CourseDirectory.autograded_directory=autograded", "--CourseDirectory.feedback_directory=feedback_generated", - "--CourseDirectory.ignore="+str(self.ignore_nbgrader), + "--CourseDirectory.ignore=" + str(self.ignore_nbgrader), "--ExecutePreprocessor.allow_errors=True", "--ExecutePreprocessor.interrupt_on_timeout=True", ] @@ -354,14 +390,16 @@ class JupyterCourse(Course): for nb_md in notebooks_md: with io.open(nb_md) as fd: if "nbgrader" not in fd.read(): - self.log.debug(f"Skip markdown file/notebook with no nbgrader metadata: {nb_md}") + self.log.debug( + f"Skip markdown file/notebook with no nbgrader metadata: {nb_md}") continue run(["jupytext", "--to", "ipynb", nb_md]) run(["nbgrader", "autograde", *nbgrader_config, assignment_name]) run(["nbgrader", "generate_feedback", *nbgrader_config, assignment_name]) autograded = os.path.join("autograded", student, assignment_name) - shutil.copy(".gradebook.db", autograded) - feedback_generated = os.path.join("feedback_generated", student, assignment_name) + shutil.copy(self.gradebook_db, autograded) + feedback_generated = os.path.join( + "feedback_generated", student, assignment_name) for format in ["csv", "md", "html", "svg"]: io.open(os.path.join(feedback_generated, f"scores.{format}"), "w").write( self.export_scores(format, student=student, assignment=assignment_name)) @@ -406,7 +444,8 @@ class JupyterCourse(Course): submission = assignment.submission() job = submission.ensure_autograded(force_autograde=force_autograde) - self.log.info(f"Téléchargement des retours dans {self.student_dir}/{assignment_name}/feedback/") + self.log.info( + f"Téléchargement des retours dans {self.student_dir}/{assignment_name}/feedback/") submission.repo.fetch_artifacts(job, path=assignment_dir, prefix="feedback") student = self.forge.get_current_user().username assert student is not None @@ -438,7 +477,8 @@ class JupyterCourse(Course): failed = [] for submission in assignment.submissions(): try: - self.log.info(f"Ensuring {submission.student}'s submission has been autograded") + self.log.info( + f"Ensuring {submission.student}'s submission has been autograded") submission.ensure_autograded(force_autograde=force_autograde) except RuntimeError as e: self.log.warning(e) @@ -455,11 +495,13 @@ class JupyterCourse(Course): Force the autograding of all submissions. """ self.forge.login() - assignment = self.assignment(assignment_name=assignment_name, student_group=student_group) + assignment = self.assignment( + assignment_name=assignment_name, student_group=student_group) failed = [] for submission in assignment.submissions(): try: - self.log.info(f"Ensuring {submission.student}'s submission has been autograded") + self.log.info( + f"Ensuring {submission.student}'s submission has been autograded") submission.force_autograde() except RuntimeError as e: self.log.warning(e) @@ -488,11 +530,14 @@ class JupyterCourse(Course): Only files starting with the given prefix are extracted. """ self.forge.login() - submissions_status = self.assignment(assignment_name=assignment_name, - student_group=student_group).collect_status() + submissions_status = self.assignment( + assignment_name=assignment_name, + student_group=student_group).collect_status() self.log.info(f"Downloading submissions for {len(submissions_status)} students") - template = os.path.join("submitted", "{username}/" + f"{os.path.basename(assignment_name)}") - self.collect(assignment=assignment_name, student_group=student_group, template=template) + template = os.path.join( + "submitted", "{username}/" + f"{os.path.basename(assignment_name)}") + self.collect(assignment=assignment_name, + student_group=student_group, template=template) def collect_autograded(self, assignment_name: str, @@ -537,13 +582,14 @@ class JupyterCourse(Course): repo = status.submission.repo job = status.autograde_job repo.fetch_artifacts(job, path=".", prefix=prefix) - # autograded_anonymous = os.path.join("autograded", "student") + #ToRemove autograded_anonymous = os.path.join("autograded", "student") # if os.path.isdir(autograded_anonymous): # shutil.copytree(autograded_anonymous, # os.path.join("autograded", student), # dirs_exist_ok=True) # shutil.rmtree(autograded_anonymous) - feedback_path = os.path.join("feedback_generated", student, os.path.basename(assignment_name)) + feedback_path = os.path.join( + "feedback_generated", student, os.path.basename(assignment_name)) if os.path.isdir("feedback"): shutil.copytree("feedback", feedback_path, @@ -606,10 +652,10 @@ class JupyterCourse(Course): assert status.submission is not None and status.autograde_job is not None job = status.autograde_job repo = status.submission.repo - file = f"autograded/{student}/{assignment_name}/.gradebook.db" - content = repo.fetch_artifact(job, artifact_path=file).content - os.makedirs(os.path.dirname(file), exist_ok=True) - with io.open(file, 'wb') as f: + file_1 = f"autograded/{student}/{assignment_name}/.gradebook.db" + content = repo.fetch_artifact(job, artifact_path=file_1).content + os.makedirs(os.path.dirname(file_1), exist_ok=True) + with io.open(file_1, 'wb') as f: f.write(content) def merge_autograded_db(self, @@ -641,7 +687,8 @@ class JupyterCourse(Course): else: self.log.info( "Syncing students' gradebook to the global gradebook `.gradebook.db`") - for file in sorted(glob.glob(f"autograded/{tag}/{assignment_name}/.gradebook.db")): + for file in sorted( + glob.glob(f"autograded/{tag}/{assignment_name}/.gradebook.db")): self.log.info(f"Student gradebook `{file}`") source = Gradebook(f'sqlite:///{file}') try: @@ -687,7 +734,8 @@ class JupyterCourse(Course): student_group: Optional[str] = None, tag: Optional[str] = "*") -> None: self.forge.login() - for file in sorted(glob.glob(f"autograded/{tag}/{os.path.basename(assignment_name)}/.gradebook.db")): + for file in sorted( + glob.glob(f"autograded/{tag}/{os.path.basename(assignment_name)}/.gradebook.db")): content = base64.b64encode(io.open(file, 'rb').read()).decode("ascii") username = file.split('/')[1] project = self.assignment(assignment_name, @@ -699,7 +747,7 @@ class JupyterCourse(Course): if group != student_group: continue self.log.info(f"Release feedback for student gradebook `{file}`") - project.ensure_file(".gradebook.db", + project.ensure_file(self.gradebook_db, content=content, encoding="base64", commit_message="Release feedback" @@ -729,14 +777,14 @@ class JupyterCourse(Course): if 'JUPYTERHUB_SERVICE_PREFIX' in os.environ: print("Launching formgrader in the background") jurl = jupyter_notebook_in_hub(path=url, - background=True) - assert jurl is not None # TODO check if jurl can be None and what it means + background=True) + assert jurl is not None # TODO check if jurl can be None and what it means url = jurl return HTML("Follow this link to " f"" f"start grading {assignment}" - "") # type: ignore + "") # type: ignore jupyter_notebook(url) def student_dashboard(self, diff --git a/travo/nbgrader_utils.py b/travo/nbgrader_utils.py index 78dc562ad4ffb985e1411d6bb752313dfd1006ca..3113d90b396824fed4ac6cdb2d95a0cb4465f103 100644 --- a/travo/nbgrader_utils.py +++ b/travo/nbgrader_utils.py @@ -44,7 +44,7 @@ def merge_assignment_gradebook(source: Gradebook, target: Gradebook) -> None: def merge_submission_gradebook(source: Gradebook, target: Gradebook, - back : bool = False, + back: bool = False, on_inconsistency: str = "ERROR", # Literal("ERROR", "WARNING") new_score_policy: str = "only_empty" @@ -69,29 +69,48 @@ def merge_submission_gradebook(source: Gradebook, for notebook in submission.notebooks: for grade in notebook.grades: - args, kwargs = to_args(grade, ['name', 'notebook', 'assignment', 'student']) + args, kwargs = to_args( + grade, ['name', 'notebook', 'assignment', 'student']) try: target_grade = target.find_grade(*args) except MissingEntry: if on_inconsistency == "WARNING": log = getLogger() - log.warning(f"Skipping grade which does not exist in the target: {grade}") + log.warning( + f"Skipping grade which does not exist in the target: {grade}") continue else: raise if back: grade, target_grade = target_grade, grade - args, kwargs = to_args(grade, ['name', 'notebook', 'assignment', 'student']) + args, kwargs = to_args( + grade, ['name', 'notebook', 'assignment', 'student']) for key, value in kwargs.items(): - if ((new_score_policy=="only_empty" or new_score_policy=="only_greater") and getattr(target_grade, key) is None) \ - or new_score_policy=="force_new_score" \ - or (new_score_policy=="only_greater" and type(getattr(target_grade, key)) is float and type(getattr(grade, key)) is float and getattr(grade, key) > getattr(target_grade, key)): + if ( + ( + new_score_policy == "only_empty" or new_score_policy == "only_greater") and getattr( + target_grade, + key) is None) or new_score_policy == "force_new_score" or ( + new_score_policy == "only_greater" and isinstance( + getattr( + target_grade, + key), + float) and isinstance( + getattr( + grade, + key), + float) and getattr( + grade, + key) > getattr( + target_grade, + key)): setattr(target_grade, key, value) for comment in notebook.comments: - args, kwargs = to_args(comment, ['name', 'notebook', 'assignment', 'student']) + args, kwargs = to_args( + comment, ['name', 'notebook', 'assignment', 'student']) try: target_comment = target.find_comment(*args) except MissingEntry: @@ -99,42 +118,51 @@ def merge_submission_gradebook(source: Gradebook, or (comment.auto_comment is None and comment.manual_comment is None): log = getLogger() - log.warning(f"Skipping comment which does not exist in the target: {comment}") + log.warning( + f"Skipping comment which does not exist in the target: {comment}") continue else: raise if back: comment, target_comment = target_comment, comment - args, kwargs = to_args(comment, ['name', 'notebook', 'assignment', 'student']) + args, kwargs = to_args( + comment, ['name', 'notebook', 'assignment', 'student']) for key, value in kwargs.items(): if getattr(target_comment, key) is None: setattr(target_comment, key, value) + def remove_submission_gradebook(gb: Gradebook, assignment_name: str, student: str) -> None: log = getLogger() try: gb.find_submission(assignment_name, student) - answer = input(f'Do you really want to suppress {assignment_name} submission of {student} ' - f'from your local .gradebook.db ? [y/N] ') + answer = input( + f'Do you really want to suppress {assignment_name} submission of {student} ' + f'from your local .gradebook.db ? [y/N] ') if answer != "y": log.info("Submission not deleted from your local .gradebook.db.") return gb.remove_submission(assignment_name, student) except (MissingEntry): - log.info(f"Missing entry for {assignment_name} submission from student {student} in your local .gradebook.db") + log.info( + f"Missing entry for {assignment_name} submission from student {student} in your local .gradebook.db") -def remove_assignment_gradebook(gb: Gradebook, assignment_name: str, force: bool = False) -> None: +def remove_assignment_gradebook( + gb: Gradebook, + assignment_name: str, + force: bool = False) -> None: log = getLogger() try: gb.find_assignment(assignment_name) if not force: - answer = input(f'Do you really want to suppress all submissions from {assignment_name} ' - f'from your local .gradebook.db ? [y/N] ') + answer = input( + f'Do you really want to suppress all submissions from {assignment_name} ' + f'from your local .gradebook.db ? [y/N] ') if answer != "y": log.info("Assignment not deleted from your local .gradebook.db.") return @@ -147,7 +175,7 @@ def remove_assignment_gradebook(gb: Gradebook, assignment_name: str, force: bool try: for notebook in assignment.notebooks: gb.remove_notebook(notebook.name, assignment_name) - except: + except BaseException: pass gb.db.delete(assignment) @@ -161,7 +189,8 @@ def remove_assignment_gradebook(gb: Gradebook, assignment_name: str, force: bool gb.db.close() except (MissingEntry): - log.info(f"Missing entry for assignment {assignment_name} in your local .gradebook.db") + log.info( + f"Missing entry for assignment {assignment_name} in your local .gradebook.db") class GradebookExporter: @@ -170,6 +199,7 @@ class GradebookExporter: auto_score: float, max_auto_score: float, manual_score: Optional[float], max_manual_score: float, extra_credit: float) -> None: + # to be implemented pass def record_assignment(self, @@ -177,14 +207,17 @@ class GradebookExporter: auto_score: float, max_auto_score: float, manual_score: Optional[float], max_manual_score: float, extra_credit: float) -> None: + # to be implemented pass def export(self) -> Any: + # to be implemented pass + class DataFrameGradebookExporter(GradebookExporter): def __init__(self) -> None: - self.data : List[List] = [] + self.data: List[List] = [] def record(self, student: str, assignment: str, notebook_name: str, @@ -226,7 +259,7 @@ class DataFrameGradebookExporter(GradebookExporter): 'total_score', 'max_total_score', ]) df.set_index(['student', 'assignment', 'notebook'], inplace=True) - # df = df.astype('Int64') + #ToRemove: df = df.astype('Int64') return df @@ -387,7 +420,8 @@ def export_scores(gradebook: Gradebook, for grade in notebook.grades: # Try to guess whether this is a test cell, whose score can be computed automatically # At some point, auto_score was None in this case, but this seems to be gone - # For now, assumes that a code cell with a non trivial max_score is a test cell + # For now, assumes that a code cell with a non trivial max_score is + # a test cell if grade.cell_type == 'code' and grade.auto_score is not None: # Autograded cell max_auto_score += grade.max_score @@ -420,9 +454,9 @@ def export_scores(gradebook: Gradebook, assignment_max_manual_score += max_manual_score assignment_extra_credit += extra_credit gradebook_exporter.record_assignment( - student, submission.assignment.name, - assignment_auto_score, assignment_max_auto_score, - assignment_manual_score, assignment_max_manual_score, - assignment_extra_credit) + student, submission.assignment.name, + assignment_auto_score, assignment_max_auto_score, + assignment_manual_score, assignment_max_manual_score, + assignment_extra_credit) return gradebook_exporter.export() diff --git a/travo/script.py b/travo/script.py index 3e4a4a2cadf1baae55bafe74454f6e0256853377..6c1657292fd582da0259440e805e9d5349baea1c 100644 --- a/travo/script.py +++ b/travo/script.py @@ -151,10 +151,10 @@ def add_subparsers_for_object_methods( method = getattr(object, name) if not (inspect.ismethod(method) or inspect.isfunction(method)): continue - help = inspect.getdoc(method) - if help is not None: - help = help.splitlines()[0] - parser = subparsers.add_parser(name, help=help, description=help) + help_doc = inspect.getdoc(method) + if help_doc is not None: + help_doc = help_doc.splitlines()[0] + parser = subparsers.add_parser(name, help=help_doc, description=help_doc) add_parser_arguments_for_function(parser, method) @@ -367,7 +367,7 @@ def CLI( assert "--" + key not in kwargs # TODO this seems to fix a conflicting bug but I'm not sure what I'm doing function_args.append(value) - # function_kwargs[key] = value + #ToRemove function_kwargs[key] = value if debug: result = function(*function_args, **function_kwargs) diff --git a/travo/utils.py b/travo/utils.py index e0cd0c85c06e89ef69d5fab4b2c34bfbf1239c67..72319e53353dd453110aa45044492e0aaf86e3b0 100644 --- a/travo/utils.py +++ b/travo/utils.py @@ -4,9 +4,11 @@ import urllib import urllib.parse from typing import cast, Any, Sequence, Optional import logging -import colorlog # type: ignore +import colorlog # type: ignore + +_logger: Optional[logging.Logger] = None + -_logger : Optional[logging.Logger] = None def getLogger() -> logging.Logger: global _logger if _logger is None: @@ -17,6 +19,7 @@ def getLogger() -> logging.Logger: _logger.addHandler(handler) return _logger + def urlencode(s: str) -> str: """ Encode `s` for inclusion in a URL @@ -26,7 +29,8 @@ def urlencode(s: str) -> str: >>> urlencode("foo/bar!") 'foo%2Fbar%21' """ - return urllib.parse.urlencode({'':s})[1:] + return urllib.parse.urlencode({'': s})[1:] + def run(args: Sequence[str], check: bool = True, **kwargs: Any) -> subprocess.CompletedProcess: @@ -44,6 +48,7 @@ def run(args: Sequence[str], getLogger().info("Running: " + ' '.join(args)) return subprocess.run(args, check=check, **kwargs) + def git_get_origin(cwd: str = ".") -> str: """ Return the origin of the current repository @@ -57,7 +62,8 @@ def git_get_origin(cwd: str = ".") -> str: >>> git_get_origin() 'https://xxx.yy/truc.git' """ - result = run(["git", "remote", "get-url", "origin"], check=False, capture_output=True, cwd=cwd) + result = run(["git", "remote", "get-url", "origin"], + check=False, capture_output=True, cwd=cwd) if result.returncode != 0: raise RuntimeError(result.stdout.decode().strip()) for line in result.stdout.decode().splitlines():