diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0cd8fc222091b1fe9e9f6faed2a39443ff7e449e..5ba31e3c81f779c4ad75f995925f1f7d3515baaa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,7 @@ stages: - /bin/false emulator-linux-ubuntu1804-install: + stage: build image: ubuntu:18.04 before_script: - apt-get update @@ -28,6 +29,15 @@ emulator-linux-ubuntu1804-install: - cd .. - gle -l +emulator-linux-py3.5-install: + stage: build + image: python:3.5-slim + script: + - cd emulator + - pip3 install . + - cd .. + - gle -l + emulator-linux-test: image: name: docker:stable-dind diff --git a/emulator/gitlabemu/gitlab/client.py b/emulator/gitlabemu/gitlab/client.py new file mode 100644 index 0000000000000000000000000000000000000000..a6e808f2e537e229c8420517ba95db01d279598c --- /dev/null +++ b/emulator/gitlabemu/gitlab/client.py @@ -0,0 +1,64 @@ +import zipfile +from typing import List + +from .. import stream_response +from ..userconfig import get_user_config_value + +try: + from gitlab import Gitlab + from gitlab.v4.objects import ProjectPipelineJob +except ImportError: + gitlab = None + + +class GitlabSupportError(Exception): + """Error with gitlab client support""" + def __init__(self, msg): + self.msg = msg + + +def gitlab_api(cfg: dict, alias: str, secure=True) -> "Gitlab": + """Create a Gitlab API client""" + if not gitlab: + raise GitlabSupportError("Gitlab support not available on this python") + servers = get_user_config_value(cfg, "gitlab", name="servers", default=[]) + for item in servers: + if item.get("name") == alias: + server = item.get("server") + token = item.get("token") + if not server: + raise GitlabSupportError(f"no server address for alias {alias}") + if not token: + raise GitlabSupportError(f"no api-token for alias {alias} ({server})") + client = Gitlab(url=server, private_token=token, ssl_verify=secure) + + return client + + raise GitlabSupportError(f"Cannot find local configuration for server {alias}") + + +def gitlab_download_artifacts(cfg, pipeline_str: str, jobnames: List[str], insecure=False): + """Download and extract the artifacts from the given jobs in a pipeline""" + server, extra = pipeline_str + project_path, pipeline_id = extra.rsplit("/", 1) + gl = gitlab_api(cfg, server, secure=not insecure) + # get project + project = gl.projects.get(project_path) + # get pipeline + pipeline = project.pipelines.get(int(pipeline_id)) + pipeline_jobs = pipeline.jobs.list(all=True) + # download what we need + upsteam_jobs: List[ProjectPipelineJob] = [x for x in pipeline_jobs if x.name in jobnames] + + for upstream in upsteam_jobs: + print(f"Fetching {upstream.name} artifacts from {pipeline_str}..") + artifact_url = f"{gl.api_url}/projects/{project.id}/jobs/{upstream.id}/artifacts" + + # stream it into zipfile + resp = gl.session.get(artifact_url, headers={"PRIVATE-TOKEN": gl.private_token}, stream=True) + resp.raise_for_status() + seekable = stream_response.ResponseStream(resp.iter_content(4096)) + with zipfile.ZipFile(seekable) as zf: + for item in zf.infolist(): + print(f"Unpacking {item.filename} ..") + zf.extract(item) diff --git a/emulator/gitlabemu/runner.py b/emulator/gitlabemu/runner.py index 2c03a87b56871a58acc38688bed87070935d080c..700d0e255417fe4e74480d3e9944a9499fb26dc6 100644 --- a/emulator/gitlabemu/runner.py +++ b/emulator/gitlabemu/runner.py @@ -2,20 +2,14 @@ import re import sys import os import argparse -import zipfile -from typing import Optional, List - -from gitlab.v4.objects import ProjectPipeline, ProjectPipelineJob from . import configloader -from . import stream_response from .docker import has_docker +from .gitlab.client import gitlab_download_artifacts, GitlabSupportError from .localfiles import restore_path_ownership from .helpers import is_apple, is_linux, is_windows, git_worktree from .userconfig import load_user_config, get_user_config_value, override_user_config_value, USER_CFG_ENV -from gitlab import Gitlab - CONFIG_DEFAULT = ".gitlab-ci.yml" parser = argparse.ArgumentParser(prog="{} -m gitlabemu".format(os.path.basename(sys.executable))) @@ -117,24 +111,6 @@ def execute_job(config, jobname, seen=None, recurse=False): seen.add(jobname) -def gitlab_api(cfg: dict, alias: str, secure=True) -> Gitlab: - """Create a Gitlab API client""" - servers = get_user_config_value(cfg, "gitlab", name="servers", default=[]) - for item in servers: - if item.get("name") == alias: - server = item.get("server") - token = item.get("token") - if not server: - die(f"no server address for alias {alias}") - if not token: - die(f"no api-token for alias {alias} ({server})") - client = Gitlab(url=server, private_token=token, ssl_verify=secure) - - return client - - die(f"Cannot find local configuration for server {alias}") - - def run(args=None): options = parser.parse_args(args) loader = configloader.Loader() @@ -242,31 +218,11 @@ def run(args=None): loader.config["variables"][name] = value if options.FROM: - server, extra = options.FROM.split("/", 1) - project_path, pipeline_id = extra.rsplit("/", 1) - gitlab = gitlab_api(cfg, server, secure=not options.insecure) - # get project - project = gitlab.projects.get(project_path) - # get pipeline - pipeline = project.pipelines.get(int(pipeline_id)) - pipeline_jobs = pipeline.jobs.list(all=True) - - jobobj = configloader.load_job(loader.config, jobname) - - # download what we need - upsteam_jobs: List[ProjectPipelineJob] = [x for x in pipeline_jobs if x.name in jobobj.dependencies] - for upstream in upsteam_jobs: - print(f"Fetching {upstream.name} artifacts from {options.FROM}..") - artifact_url = f"{gitlab.api_url}/projects/{project.id}/jobs/{upstream.id}/artifacts" - - # stream it into zipfile - resp = gitlab.session.get(artifact_url, headers={"PRIVATE-TOKEN": gitlab.private_token}, stream=True) - resp.raise_for_status() - seekable = stream_response.ResponseStream(resp.iter_content(4096)) - with zipfile.ZipFile(seekable) as zf: - for item in zf.infolist(): - print(f"Unpacking {item.filename} ..") - zf.extract(item) + try: + jobobj = configloader.load_job(loader.config, jobname) + gitlab_download_artifacts(cfg, options.FROM, jobobj.dependencies, insecure=options.insecure) + except GitlabSupportError as err: + die(err.msg) if options.enter_shell: if options.FULL: @@ -293,3 +249,5 @@ def run(args=None): restore_path_ownership(os.getcwd()) print("finished") print("Build complete!") + + diff --git a/emulator/setup.py b/emulator/setup.py index 7fb8d4a1ff7c9e0c5a68c5a70db9a486ede7b624..c9cf52cfd22054f7bc18193c544025424b452e4d 100644 --- a/emulator/setup.py +++ b/emulator/setup.py @@ -1,6 +1,8 @@ from distutils.core import setup -VERSION = "0.9.2" +VERSION = "0.9.3" + +primary_platforms = "platform_system=='Linux' or platform_system=='Darwin' or platform_system=='Windows'" setup( name="gitlab-emulator", @@ -16,11 +18,10 @@ setup( scripts=["locallab.py"], install_requires=[ "pyyaml>=3.13", - "docker>=5.0.2; platform_system=='Linux'", - "docker>=5.0.2; platform_system=='Darwin'", - "docker>=5.0.2; platform_system=='Windows'", - "python-gitlab>=3.2.0; python_version>='3.7'", - "python-gitlab==2.10.1; python_version<='3.6'", + "docker>=5.0.2; python_version>='3.6' and ({})".format(primary_platforms), + "docker>=4.4.4; python_version=='3.5' and ({})".format(primary_platforms), + "python-gitlab>=3.2.0; python_version>='3.7' and ({})".format(primary_platforms), + "python-gitlab==2.10.1; python_version=='3.6' and ({})".format(primary_platforms), ], platforms=["any"], license="License :: OSI Approved :: MIT License",