From 127ee32991240bdfb2df1ae2a5cc10394f65b1f2 Mon Sep 17 00:00:00 2001 From: Ian Norton Date: Fri, 24 Jun 2022 18:52:47 +0100 Subject: [PATCH 1/9] Pass tls_verify down more for --from --pipeline jobs --- emulator/gitlabemu/generator.py | 9 +++++++-- emulator/gitlabemu/runner.py | 12 ++++++++++-- emulator/setup.py | 2 +- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/emulator/gitlabemu/generator.py b/emulator/gitlabemu/generator.py index a7f6df2..4757567 100644 --- a/emulator/gitlabemu/generator.py +++ b/emulator/gitlabemu/generator.py @@ -13,12 +13,17 @@ from .helpers import git_top_level, git_commit_sha, git_uncommitted_changes, git def generate_artifact_fetch_job( loader: Loader, stage: str, - needed: Dict[str, str]) -> dict: + needed: Dict[str, str], + tls_verify: Optional[bool] = True +) -> dict: """Generate a job to fetch artifacts of needed jobs from a completed pipeline""" # use CI_JOB_TOKEN to fetch the artifacts script = [] paths = [] generated = {} + verify = "--cacert $CI_SERVER_TLS_CA_FILE" + if not tls_verify: + verify = "--insecure" for name in needed: job = loader.get_job(name) # does it define any artifacts? @@ -29,7 +34,7 @@ def generate_artifact_fetch_job( script.extend( [ 'apk add curl', - f'curl --location --output {name}-artifacts.zip --header "JOB-TOKEN: $CI_JOB_TOKEN" {url}', + f'curl {verify} --location --output {name}-artifacts.zip --header "JOB-TOKEN: $CI_JOB_TOKEN" {url}', f'unzip -o {name}-artifacts.zip', f'rm -f {name}-artifacts.zip', ] diff --git a/emulator/gitlabemu/runner.py b/emulator/gitlabemu/runner.py index 7ba34ed..3bdacf4 100644 --- a/emulator/gitlabemu/runner.py +++ b/emulator/gitlabemu/runner.py @@ -187,7 +187,12 @@ def do_pipeline(options: argparse.Namespace, loader): elif ident.gitref: note(f"Searching for latest pipeline on {ident.gitref} ..") # find the newest pipeline for this git reference - found = project.pipelines.list(sort="desc", ref=ident.gitref, order_by="updated_at", page=1, per_page=5, status='success') + found = project.pipelines.list( + sort="desc", + ref=ident.gitref, + order_by="updated_at", + page=1, per_page=5, + status='success') if not found: die(f"Could not find a completed pipeline for git reference {ident.gitref}") pipeline = found[0] @@ -218,7 +223,10 @@ def do_pipeline(options: argparse.Namespace, loader): if download_jobs: stages = generated.get("stages", ["test"]) - fetch_job = generate_artifact_fetch_job(loader, stages[0], download_jobs) + fetch_job = generate_artifact_fetch_job(loader, + stages[0], + download_jobs, + tls_verify=client.ssl_verify) generated["from_pipeline"] = fetch_job for job in jobs: generated[job]["needs"] = ["from_pipeline"] diff --git a/emulator/setup.py b/emulator/setup.py index 77f0527..b9b7d80 100644 --- a/emulator/setup.py +++ b/emulator/setup.py @@ -1,6 +1,6 @@ from distutils.core import setup -VERSION = "0.11.2" +VERSION = "0.11.3" requirements = [ "pyyaml>=5.1", -- GitLab From 0a229c4d221b960f63da0d782143e052674a4f5d Mon Sep 17 00:00:00 2001 From: Ian Norton Date: Fri, 24 Jun 2022 19:07:28 +0100 Subject: [PATCH 2/9] keep top level variables when doing --pipeline --- emulator/gitlabemu/generator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/emulator/gitlabemu/generator.py b/emulator/gitlabemu/generator.py index 4757567..6f7f7e8 100644 --- a/emulator/gitlabemu/generator.py +++ b/emulator/gitlabemu/generator.py @@ -63,6 +63,12 @@ def generate_pipeline_yaml(loader: Loader, stages = loader.config.get("stages", []) needed = set(goals) + # get the variables and defaults sections + generated["variables"] = dict(loader.config.get("variables", {})) + for item in ["image", "default"]: + if item in loader.config: + generated[item] = loader.config.get(item) + while len(needed): for name in list(needed): needed.remove(name) -- GitLab From 2760274ed83540835ba27d13918a49f266f59611 Mon Sep 17 00:00:00 2001 From: Ian Norton Date: Fri, 24 Jun 2022 19:13:49 +0100 Subject: [PATCH 3/9] Keep more top level things in subset pipelines --- emulator/gitlabemu/generator.py | 15 ++++++--------- emulator/gitlabemu/runner.py | 3 ++- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/emulator/gitlabemu/generator.py b/emulator/gitlabemu/generator.py index 6f7f7e8..259105a 100644 --- a/emulator/gitlabemu/generator.py +++ b/emulator/gitlabemu/generator.py @@ -63,12 +63,6 @@ def generate_pipeline_yaml(loader: Loader, stages = loader.config.get("stages", []) needed = set(goals) - # get the variables and defaults sections - generated["variables"] = dict(loader.config.get("variables", {})) - for item in ["image", "default"]: - if item in loader.config: - generated[item] = loader.config.get(item) - while len(needed): for name in list(needed): needed.remove(name) @@ -88,13 +82,16 @@ def generate_pipeline_yaml(loader: Loader, for item in loaded.dependencies: if isinstance(item, str): needed.add(item) - else: - pass - # else, download from an earlier pipeline if stages: generated["stages"] = list(stages) + # get the variables and defaults sections etc + generated["variables"] = dict(loader.config.get("variables", {})) + for item in ["image", "default", "before_script", "after_script", "services"]: + if item in loader.config: + generated[item] = loader.config.get(item) + return generated diff --git a/emulator/gitlabemu/runner.py b/emulator/gitlabemu/runner.py index 3bdacf4..984e682 100644 --- a/emulator/gitlabemu/runner.py +++ b/emulator/gitlabemu/runner.py @@ -8,6 +8,7 @@ from gitlab import GitlabGetError from . import configloader from .docker import has_docker +from .gitlab.types import RESERVED_TOP_KEYS from .gitlab_client_api import PipelineError, PipelineInvalid, get_gitlab_project_client, parse_gitlab_from_arg from .generator import generate_pipeline_yaml, create_pipeline_branch, wait_for_project_commit_pipeline, \ generate_artifact_fetch_job @@ -218,7 +219,7 @@ def do_pipeline(options: argparse.Namespace, loader): download_jobs[dep] = artifact_url generated = generate_pipeline_yaml(loader, *goals, recurse=recurse) - jobs = [name for name in generated.keys() if name != "stages"] + jobs = [name for name in generated.keys() if name in RESERVED_TOP_KEYS] note(f"Will build jobs: {jobs} ..") if download_jobs: -- GitLab From cf3616750e6eb16826ad94a66e7e71eb7e395f65 Mon Sep 17 00:00:00 2001 From: Ian Norton Date: Thu, 30 Jun 2022 14:45:32 +0100 Subject: [PATCH 4/9] Prevent CI_ emulator generated vars making it into --pipeline builds --- emulator/gitlabemu/configloader.py | 19 ++++++------- emulator/gitlabemu/generator.py | 3 ++- emulator/gitlabemu/jobs.py | 4 +++ emulator/gitlabemu/runner.py | 43 +++++++++++++++++------------- 4 files changed, 40 insertions(+), 29 deletions(-) diff --git a/emulator/gitlabemu/configloader.py b/emulator/gitlabemu/configloader.py index 9573363..8bf911c 100644 --- a/emulator/gitlabemu/configloader.py +++ b/emulator/gitlabemu/configloader.py @@ -347,7 +347,7 @@ def job_docker_image(config, name): return config[name].get("image") -def load_job(config, name): +def load_job(config, name, allow_add_variables=True): """ Load a job from the configuration :param config: @@ -362,16 +362,13 @@ def load_job(config, name): job = DockerJob() else: job = Job() - + job.allow_add_variables = allow_add_variables job.load(name, config) return job def do_variables(baseobj, yamlfile): - baseobj[".gitlab-emulator-workspace"] = os.path.abspath(os.path.dirname(yamlfile)) - if "variables" not in baseobj: - baseobj["variables"] = {} # set CI_ values baseobj["variables"]["CI_PIPELINE_ID"] = os.getenv( "CI_PIPELINE_ID", "0") @@ -447,10 +444,10 @@ class Loader(object): A configuration loader for gitlab pipelines """ - def __init__(self): + def __init__(self, emulator_variables=True): self.filename = None self.rootdir = None - + self.create_emulator_variables = emulator_variables self.config = {} self.included_files = [] @@ -506,7 +503,11 @@ class Loader(object): :param yamlfile: :return: """ - return do_variables(baseobj, yamlfile) + if "variables" not in baseobj: + baseobj["variables"] = {} + baseobj[".gitlab-emulator-workspace"] = os.path.abspath(os.path.dirname(yamlfile)) + if self.create_emulator_variables: + return do_variables(baseobj, yamlfile) def get_jobs(self): """ @@ -525,7 +526,7 @@ class Loader(object): def load_job(self, name) -> "Job": """Return a loaded job object""" - return load_job(self.config, name) + return load_job(self.config, name, allow_add_variables=self.create_emulator_variables) def get_stages(self): """ diff --git a/emulator/gitlabemu/generator.py b/emulator/gitlabemu/generator.py index 259105a..1ee9f38 100644 --- a/emulator/gitlabemu/generator.py +++ b/emulator/gitlabemu/generator.py @@ -87,7 +87,8 @@ def generate_pipeline_yaml(loader: Loader, generated["stages"] = list(stages) # get the variables and defaults sections etc - generated["variables"] = dict(loader.config.get("variables", {})) + vars = dict(loader.config.get("variables", {})) + generated["variables"] = vars for item in ["image", "default", "before_script", "after_script", "services"]: if item in loader.config: generated[item] = loader.config.get(item) diff --git a/emulator/gitlabemu/jobs.py b/emulator/gitlabemu/jobs.py index ba31eae..42cc8dd 100644 --- a/emulator/gitlabemu/jobs.py +++ b/emulator/gitlabemu/jobs.py @@ -44,6 +44,7 @@ class Job(object): self.tags = [] self.stage = "test" self.variables = {} + self.allow_add_variables = True self.dependencies = [] self.needed_artifacts = [] self._shell = None @@ -190,6 +191,9 @@ class Job(object): Set job variable defaults. If the variable is not present in self.variables, set it to the given value. If the variable is present in os.environ, use that value instead :return: """ + if not self.allow_add_variables: + return + if value is None: value = "" value = str(value) diff --git a/emulator/gitlabemu/runner.py b/emulator/gitlabemu/runner.py index 984e682..6d7eb28 100644 --- a/emulator/gitlabemu/runner.py +++ b/emulator/gitlabemu/runner.py @@ -219,18 +219,19 @@ def do_pipeline(options: argparse.Namespace, loader): download_jobs[dep] = artifact_url generated = generate_pipeline_yaml(loader, *goals, recurse=recurse) - jobs = [name for name in generated.keys() if name in RESERVED_TOP_KEYS] + jobs = [name for name in generated.keys() if name not in RESERVED_TOP_KEYS] note(f"Will build jobs: {jobs} ..") if download_jobs: + from_name = f"from_pipeline_{pipeline.id}" stages = generated.get("stages", ["test"]) fetch_job = generate_artifact_fetch_job(loader, stages[0], download_jobs, tls_verify=client.ssl_verify) - generated["from_pipeline"] = fetch_job + generated[from_name] = fetch_job for job in jobs: - generated[job]["needs"] = ["from_pipeline"] + generated[job]["needs"] = [from_name] branch_name = f"temp/{client.user.username}/{git_current_branch(cwd)}" note(f"Creating temporary pipeline branch '{branch_name}'..") @@ -350,12 +351,29 @@ def run(args=None): os.environ[USER_CFG_ENV] = options.USER_SETTINGS ctx = get_user_config_context() + fullpath = os.path.abspath(yamlfile) + rootdir = os.path.dirname(fullpath) + os.chdir(rootdir) + hide_dot_jobs = not options.hidden try: - fullpath = os.path.abspath(yamlfile) - rootdir = os.path.dirname(fullpath) - os.chdir(rootdir) + if options.pipeline: + loader = configloader.Loader(emulator_variables=False) + loader.load(fullpath) + do_pipeline(options, loader) + return + + if options.FULL and options.parallel: + die("--full and --parallel cannot be used together") + + if options.FROM: + loader = configloader.Loader(emulator_variables=False) + loader.load(fullpath) + do_gitlab_from(options, loader) + return + loader.load(fullpath) + except configloader.ConfigLoaderError as err: die("Config error: " + str(err)) @@ -368,19 +386,6 @@ def run(args=None): windows_shell = options.windows_shell loader.config[".gitlabemu-windows-shell"] = windows_shell - hide_dot_jobs = not options.hidden - - if options.pipeline: - do_pipeline(options, loader) - return - - if options.FULL and options.parallel: - die("--full and --parallel cannot be used together") - - if options.FROM: - do_gitlab_from(options, loader) - return - if options.LIST: for jobname in sorted(loader.get_jobs()): if jobname.startswith(".") and hide_dot_jobs: -- GitLab From e00b7f5f0d4699a6c2884272eb26d04d30ccf0ad Mon Sep 17 00:00:00 2001 From: Ian Norton Date: Thu, 30 Jun 2022 16:05:52 +0100 Subject: [PATCH 5/9] Generate a download job for each needed job instead of one big downloader Only ask for a small amount of k8s resource --- emulator/gitlabemu/generator.py | 8 ++++++-- emulator/gitlabemu/runner.py | 19 ++++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/emulator/gitlabemu/generator.py b/emulator/gitlabemu/generator.py index 1ee9f38..f0dd07b 100644 --- a/emulator/gitlabemu/generator.py +++ b/emulator/gitlabemu/generator.py @@ -49,6 +49,10 @@ def generate_artifact_fetch_job( "artifacts": { "paths": list(set(paths)), "expire_in": '1 day' + }, + "variables": { + "KUBERNETES_CPU_REQUEST": "1", + "KUBERNETES_MEMORY_REQUEST": "2G", } } @@ -87,8 +91,8 @@ def generate_pipeline_yaml(loader: Loader, generated["stages"] = list(stages) # get the variables and defaults sections etc - vars = dict(loader.config.get("variables", {})) - generated["variables"] = vars + variables = dict(loader.config.get("variables", {})) + generated["variables"] = variables for item in ["image", "default", "before_script", "after_script", "services"]: if item in loader.config: generated[item] = loader.config.get(item) diff --git a/emulator/gitlabemu/runner.py b/emulator/gitlabemu/runner.py index 6d7eb28..ae2b6fb 100644 --- a/emulator/gitlabemu/runner.py +++ b/emulator/gitlabemu/runner.py @@ -172,6 +172,7 @@ def do_pipeline(options: argparse.Namespace, loader): pipeline = None goals = [options.JOB] download_jobs = {} + deps = {} if options.EXTRA_JOBS: goals.extend(options.EXTRA_JOBS) note(f"Generate subset pipeline to build '{goals}'..") @@ -202,7 +203,7 @@ def do_pipeline(options: argparse.Namespace, loader): # now make sure the pipeline contains the jobs we need pipeline_jobs = {} - download_jobs = {} + for item in pipeline.jobs.list(all=True): if item.status == "success": pipeline_jobs[item.name] = item @@ -217,21 +218,25 @@ def do_pipeline(options: argparse.Namespace, loader): if hasattr(from_job, "artifacts_file"): # missing if it created no artifacts artifact_url = f"{client.api_url}/projects/{project.id}/jobs/{from_job.id}/artifacts" download_jobs[dep] = artifact_url + if goal not in deps: + deps[goal] = [] + deps[goal].append(dep) generated = generate_pipeline_yaml(loader, *goals, recurse=recurse) jobs = [name for name in generated.keys() if name not in RESERVED_TOP_KEYS] note(f"Will build jobs: {jobs} ..") + stages = generated.get("stages", ["test"]) - if download_jobs: - from_name = f"from_pipeline_{pipeline.id}" - stages = generated.get("stages", ["test"]) + for from_name in download_jobs: fetch_job = generate_artifact_fetch_job(loader, stages[0], - download_jobs, + {from_name: download_jobs[from_name]}, tls_verify=client.ssl_verify) generated[from_name] = fetch_job - for job in jobs: - generated[job]["needs"] = [from_name] + + if deps: + for job in goals: + generated[job]["needs"] = deps.get(job, []) branch_name = f"temp/{client.user.username}/{git_current_branch(cwd)}" note(f"Creating temporary pipeline branch '{branch_name}'..") -- GitLab From f7d997341a09cdfdcc8e0a580aadd9bf213e642f Mon Sep 17 00:00:00 2001 From: Ian Norton Date: Thu, 30 Jun 2022 23:59:37 +0100 Subject: [PATCH 6/9] Remove obsolete assert --- emulator/gitlabemu/tests/test_configloader_loader.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/emulator/gitlabemu/tests/test_configloader_loader.py b/emulator/gitlabemu/tests/test_configloader_loader.py index 17c8c20..ab12586 100644 --- a/emulator/gitlabemu/tests/test_configloader_loader.py +++ b/emulator/gitlabemu/tests/test_configloader_loader.py @@ -11,12 +11,9 @@ HERE = os.path.dirname(__file__) def test_load_simple(top_dir): loader = configloader.Loader() yamlfile = os.path.join(top_dir, ".gitlab-ci.yml") - expected = configloader.read(yamlfile) - + configloader.read(yamlfile) loader.load(yamlfile) - assert loader.config == expected - def test_load_callbacks(top_dir): loader = configloader.Loader() -- GitLab From 840d7399dcf39d3e9eb63937d77510920c9bec51 Mon Sep 17 00:00:00 2001 From: Ian Norton Date: Fri, 1 Jul 2022 06:56:24 +0100 Subject: [PATCH 7/9] Fix runner tests --- emulator/gitlabemu/configloader.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/emulator/gitlabemu/configloader.py b/emulator/gitlabemu/configloader.py index 8bf911c..8bae0dd 100644 --- a/emulator/gitlabemu/configloader.py +++ b/emulator/gitlabemu/configloader.py @@ -370,6 +370,9 @@ def load_job(config, name, allow_add_variables=True): def do_variables(baseobj, yamlfile): # set CI_ values + if "variables" not in baseobj: + baseobj["variables"] = {} + baseobj["variables"]["CI_PIPELINE_ID"] = os.getenv( "CI_PIPELINE_ID", "0") baseobj["variables"]["CI_COMMIT_REF_SLUG"] = os.getenv( -- GitLab From 2b645731e2f971f64c5ed593d3cc1bfec40d2459 Mon Sep 17 00:00:00 2001 From: Ian Norton Date: Fri, 1 Jul 2022 06:59:49 +0100 Subject: [PATCH 8/9] Bump to 1.0.1 --- emulator/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emulator/setup.py b/emulator/setup.py index b9b7d80..bcaf0c8 100644 --- a/emulator/setup.py +++ b/emulator/setup.py @@ -1,6 +1,6 @@ from distutils.core import setup -VERSION = "0.11.3" +VERSION = "1.0.1" requirements = [ "pyyaml>=5.1", -- GitLab From 3a223459676171e270798ba045c566f306a16fd5 Mon Sep 17 00:00:00 2001 From: Ian Norton Date: Fri, 1 Jul 2022 07:00:31 +0100 Subject: [PATCH 9/9] Bump runner version --- runner/GitlabPyRunner/consts.py | 2 +- runner/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/GitlabPyRunner/consts.py b/runner/GitlabPyRunner/consts.py index b8f300e..6f8aba5 100644 --- a/runner/GitlabPyRunner/consts.py +++ b/runner/GitlabPyRunner/consts.py @@ -2,5 +2,5 @@ Constants for gitlab-python-runner """ NAME = "gitlab-python-runner" -VERSION = "14.3.20" +VERSION = "14.3.21" USER_AGENT = "{} {}".format(NAME, VERSION) diff --git a/runner/setup.py b/runner/setup.py index d880c68..23113b2 100644 --- a/runner/setup.py +++ b/runner/setup.py @@ -13,7 +13,7 @@ setup( scripts=["gitlab-runner.py"], install_requires=[ "pyyaml>=5.1", - "gitlab-emulator>=0.11.1", + "gitlab-emulator>=1.0.1", "unidecode>=1.1.1"], platforms=["any"], license="License :: OSI Approved :: MIT License", -- GitLab