From 26a071f125df26545b47a3b58bf8ca22d338968d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20M=2E=20Thi=C3=A9ry?= Date: Thu, 13 Oct 2022 22:11:24 +0200 Subject: [PATCH 1/5] Optimize Resource's initialization magic - by caching the fetching - calling Project(...) is now about 7 times faster: 40ms -> 6.93 ms --- travo/gitlab.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/travo/gitlab.py b/travo/gitlab.py index d745477c..71793ef5 100644 --- a/travo/gitlab.py +++ b/travo/gitlab.py @@ -937,6 +937,15 @@ class ClassCallMetaclass(type): return cast(R, super().__call__(*args, **kwargs)) +get_type_hints_cache: Dict = {} +def get_type_hints(cls: Type) -> Dict: + type_hints = get_type_hints_cache.get(cls, None) + if type_hints is None: + type_hints = cast(Any, typing.get_type_hints(cls)) + get_type_hints_cache[cls] = type_hints + return type_hints + + @dataclass class Resource(metaclass=ClassCallMetaclass): @@ -1007,12 +1016,14 @@ class Resource(metaclass=ClassCallMetaclass): # Update the value in GitLab self.gitlab.put(self.get_api_url(), attributes) + type_hints = get_type_hints(self.__class__) + for key, value in attributes.items(): # Set the value in `self`, using the type hints to # construct Resources from raw dictionaries when # relevant; for now are supported types of the form # Optional[Resource] or List[Resource] - type_hint = typing.get_type_hints(self.__class__)[key] + type_hint = type_hints[key] if typing_utils.issubtype(type_hint, Optional[Resource]): if typing_utils.issubtype(type_hint, Resource): resource_type = type_hint -- GitLab From daebd190f56fabd68f4305a8f9c6959464aff248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20M=2E=20Thi=C3=A9ry?= Date: Thu, 13 Oct 2022 22:14:05 +0200 Subject: [PATCH 2/5] Project.get_forks: add simple option to enable speed gains --- travo/gitlab.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/travo/gitlab.py b/travo/gitlab.py index 71793ef5..564b83b5 100644 --- a/travo/gitlab.py +++ b/travo/gitlab.py @@ -1411,15 +1411,21 @@ class Project(Resource): return [ fork['ssh_url_to_repo'] for fork in forks ] def get_forks(self, - recursive: Union[int, bool] = False) -> List['Project']: + recursive: Union[int, bool] = False, + simple: bool = False + ) -> List['Project']: """Return the forks of this project - @return a list + @return a list of projects If `recursive` is True, the forks of these forks are explored recursively. Alternatively, one may specify a recursion depth, a depth of `1` being equivalent to `recursive=False`. + If `simple` is True, then less information about each project + is returned. In particular, `owner` is not set. On the other + hand, it is about 5 times faster. + Example: >>> gitlab = getfixture('gitlab') @@ -1447,13 +1453,14 @@ class Project(Resource): # >>> project = gitlab.get_project("Info111/2020-2021/Semaine2") # >>> forks = project.get_forks(recursive=True) # >>> assert len(forks) >= 180 + """ json = self.gitlab.get_json(f"/projects/{self.id}/forks", # simple = True would reduce # output volume; however at this # stage e.g. owner is still needed # in some of our use_cases - data=dict(simple=False), + data=dict(simple=simple), depaginate=True, ) forks = [Project(gitlab=self.gitlab, **fork) @@ -1465,7 +1472,8 @@ class Project(Resource): if recursive: forks = forks + [subfork for fork in forks - for subfork in fork.get_forks(recursive=recursive)] + for subfork in fork.get_forks(recursive=recursive, + simple=simple)] return forks -- GitLab From 57e33b68b01c9241da700bc639f5452c6ccbe6a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20M=2E=20Thi=C3=A9ry?= Date: Thu, 13 Oct 2022 22:40:15 +0200 Subject: [PATCH 3/5] Project.get_forks: fixed recursion depth limitation get_forks used to do the full recursion in all cases, including on the leaves! --- travo/gitlab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travo/gitlab.py b/travo/gitlab.py index 564b83b5..d6bc75a7 100644 --- a/travo/gitlab.py +++ b/travo/gitlab.py @@ -1467,7 +1467,7 @@ class Project(Resource): for fork in json if 'default_branch' in fork # ignore broken forks with no repositories ] - if not(recursive, bool): + if not isinstance(recursive, bool): recursive -= 1 if recursive: forks = forks + [subfork -- GitLab From 08b0ec5fc94bcb4a0a965ab92ca4511c16483507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20M=2E=20Thi=C3=A9ry?= Date: Thu, 13 Oct 2022 22:42:03 +0200 Subject: [PATCH 4/5] Assignment submissions: don't rely on project.owner for speed! When searching for the forks of a project, requesting the simple information on the forks is much faster, but does not include 'owner'. --- travo/assignment.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/travo/assignment.py b/travo/assignment.py index 7cda0afe..0eb414f9 100644 --- a/travo/assignment.py +++ b/travo/assignment.py @@ -512,13 +512,20 @@ Déplacez ou supprimez le def submissions(self) -> List["Submission"]: """ Return all the submissions for this assignment + + The submissions are found among the forks or forks of forks. + To distinguish between actual submissions and, + e.g. intermediate forks used to model student groups, the + current criteria is whether the fork resides in a user + namespace, rather than a group namespace. The criteria used to + be whether the fork had an owner. """ depth = 2 if self.fork_of_fork else 1 project = self.repo() - forks = project.get_forks(recursive=depth) + forks = project.get_forks(recursive=depth, simple=True) return [Submission(fork) for fork in forks - if fork.owner is not None] + if fork.namespace.kind == 'user'] def status(self) -> "SubmissionStatus": """ @@ -572,9 +579,8 @@ class Submission: student: str = field(default=cast(str, None)) def __post_init__(self) -> None: - assert self.repo.owner is not None - assert self.repo.owner.username is not None - self.student = self.repo.owner.username + assert self.repo.namespace.kind == 'user' + self.student = self.repo.namespace.path def get_autograde_job(self) -> Optional[Job]: """ -- GitLab From 59d39c899e00e3bac638680b269de798c95e2e01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20M=2E=20Thi=C3=A9ry?= Date: Thu, 13 Oct 2022 23:02:02 +0200 Subject: [PATCH 5/5] Fixed type hints --- travo/gitlab.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/travo/gitlab.py b/travo/gitlab.py index d6bc75a7..91b854c0 100644 --- a/travo/gitlab.py +++ b/travo/gitlab.py @@ -937,11 +937,11 @@ class ClassCallMetaclass(type): return cast(R, super().__call__(*args, **kwargs)) -get_type_hints_cache: Dict = {} +get_type_hints_cache: Dict[Type, Dict] = {} def get_type_hints(cls: Type) -> Dict: type_hints = get_type_hints_cache.get(cls, None) if type_hints is None: - type_hints = cast(Any, typing.get_type_hints(cls)) + type_hints = typing.get_type_hints(cls) get_type_hints_cache[cls] = type_hints return type_hints -- GitLab