From d84e602c049de8878bfcb641f5292a6217d5a964 Mon Sep 17 00:00:00 2001 From: Ian Norton Date: Wed, 1 Nov 2023 13:02:46 +0000 Subject: [PATCH] Improve support for remote includes: resolve #102 support for `include:project` resolve #104 support for `include:template` Also resolve #111 since the above two were blocking this. --- .idea/gitlab-emulator.iml | 2 +- emulator/gitlabemu/configloader.py | 119 +++++++++++------- emulator/gitlabemu/configtool_runner.py | 3 +- emulator/gitlabemu/docker.py | 36 +++--- emulator/gitlabemu/gitlab/types.py | 1 + emulator/gitlabemu/gitlab/urls.py | 2 + emulator/gitlabemu/gitlab_client_api.py | 28 +++-- emulator/gitlabemu/helpers.py | 12 +- emulator/gitlabemu/localfiles.py | 5 +- emulator/gitlabemu/logmsg.py | 12 +- emulator/gitlabemu/pipelines.py | 51 ++++---- emulator/gitlabemu/runner.py | 49 ++++---- .../gitlabemu/tests/include-templates.yml | 10 ++ emulator/gitlabemu/tests/test_helpers.py | 4 +- emulator/gitlabemu/tests/test_include.py | 4 + emulator/gitlabemu/tests/test_local_docker.py | 4 +- emulator/gitlabemu/variables.py | 8 ++ emulator/setup.py | 2 +- 18 files changed, 210 insertions(+), 142 deletions(-) create mode 100644 emulator/gitlabemu/tests/include-templates.yml diff --git a/.idea/gitlab-emulator.iml b/.idea/gitlab-emulator.iml index 451be4b..1b8357a 100644 --- a/.idea/gitlab-emulator.iml +++ b/.idea/gitlab-emulator.iml @@ -16,7 +16,7 @@ - + diff --git a/emulator/gitlabemu/configloader.py b/emulator/gitlabemu/configloader.py index 1dc6171..a518b58 100644 --- a/emulator/gitlabemu/configloader.py +++ b/emulator/gitlabemu/configloader.py @@ -5,13 +5,17 @@ import os import copy import sys import tempfile +import urllib.parse from abc import ABC, abstractmethod from typing import Dict, Any, Union, Optional, List + import requests +from gitlab import Gitlab from .errors import ConfigLoaderError, BadSyntaxError, FeatureNotSupportedError from .gitlab.types import RESERVED_TOP_KEYS, DEFAULT_JOB_KEYS -from .gitlab.urls import GITLAB_ORG_TEMPLATE_BASEURL +from .gitlab.urls import GITLAB_SERVER_TEMPLATE_PATH +from .gitlab_client_api import get_current_project_client from .helpers import stringlist_if_string from .jobtypes import JobFactory, Job, DockerJob from .jobs import NoSuchJob @@ -36,10 +40,10 @@ def do_single_include(baseobj: Dict[str, Any], handle_read=None, variables: Optional[Dict[str, str]] = None, filename: Optional[str] = None, - handle_fetch_project=None) -> Dict[str, Any]: + handle_fetch=None) -> Dict[str, Any]: """ Load a single included file and return it's object graph - :param handle_fetch_project: called to fetch an included file from another project + :param handle_fetch: called to fetch an included file :param filename: the name of the parent file wanting to include this one :param handle_read: :param baseobj: previously loaded and included objects @@ -56,7 +60,6 @@ def do_single_include(baseobj: Dict[str, Any], handle_read = read location = None inc_type = None - temp_content = None if isinstance(inc, str): location = inc.lstrip("/\\") @@ -70,21 +73,6 @@ def do_single_include(baseobj: Dict[str, Any], if not supported: raise FeatureNotSupportedError(f"Do not understand how to include {inc}") - if "local" in inc: - location = inc["local"] - inc_type = "local" - elif "remote" in inc: - location = inc["remote"] - inc_type = "remote" - elif "template" in inc: # pragma: no cover - location = inc["template"] - inc_type = "template" - elif "project" in inc: # pragma: no cover - ref = inc.get("ref", "HEAD") - location = f"{inc['project']}/{inc['file']}#{ref}".replace("//", "/") - inc_type = "project" - fatal(f"Not Yet Implemented, {inc}") - rules = inc.get("rules", []) if not rules: debugrule(f"{filename} include: {inc}") @@ -103,35 +91,22 @@ def do_single_include(baseobj: Dict[str, Any], debugrule(f"{filename} not including: {inc}") return {} - assert location, f"cannot work out include source location: {inc}" - included = baseobj.get("include", []) - if location in included: - raise BadSyntaxError(f"{filename}: {location} has already been included") - baseobj["include"].append(location) - if inc_type == "local": + if location is None: + location = inc[inc_type] if os.sep != "/": # pragma: cover if windows location = location.replace("/", os.sep) - return handle_read(location, variables=False, validate_jobs=False, topdir=yamldir, baseobj=baseobj) - elif inc_type == "template": # pragma: no cover - # get the template from gitlab.com - warning(f"Including gitlab.com template: {location}") - inc_type = "remote" - location = f"{GITLAB_ORG_TEMPLATE_BASEURL}/{location}" - elif inc_type == "project": # pragma: no cover - warning(f"Including CI yaml from another project: {location}") - if not handle_fetch_project: - warning("no project handler added") - else: - temp_content = handle_fetch_project(inc) + included = baseobj.get("include", []) + if location: + if location in included: + raise BadSyntaxError(f"{filename}: {location} has already been included") + baseobj["include"].append(location) - if inc_type == "remote": - # fetch the file, no authentication needed - warning(f"Including remote CI yaml file: {location}") - resp = requests.get(location, allow_redirects=True) - if not resp.status_code == 200: # pragma: no cover - raise ConfigLoaderError(f"HTTP error getting {location}: status {resp.status_code}") - temp_content = resp.text + if inc_type == "local": + return handle_read(location, variables=False, validate_jobs=False, topdir=yamldir, baseobj=baseobj) + else: + warning(f"Including remote CI yaml file: {inc}") + temp_content = handle_fetch(inc) if temp_content is not None: with tempfile.TemporaryDirectory() as temp_folder: @@ -139,6 +114,7 @@ def do_single_include(baseobj: Dict[str, Any], with open(path, "w") as fd: fd.write(temp_content) return handle_read(path, variables=False, validate_jobs=False, topdir=str(temp_folder), baseobj=baseobj) + return {} def do_includes(baseobj: Dict[str, Any], @@ -691,6 +667,14 @@ class Loader(BaseLoader, JobLoaderMixin, ValidatorMixin, ExtendsMixin): def __init__(self, emulator_variables: Optional[bool] = True): super().__init__() self.create_emulator_variables = emulator_variables + self.gitlab_api: Optional[Gitlab] = None + self.tls_verify = True + + def get_gitlab_client(self) -> Gitlab: + if self.gitlab_api is None: + gitlab, _, _ = get_current_project_client(tls_verify=self.tls_verify, need_remote=False, need_project=False) + self.gitlab_api = gitlab + return self.gitlab_api def load_job(self, name: str, @@ -714,7 +698,46 @@ class Loader(BaseLoader, JobLoaderMixin, ValidatorMixin, ExtendsMixin): :param incs: :return: """ - return do_includes(baseobj, yamldir, incs, handle_include=self.do_single_include, filename=self.filename) + return do_includes(baseobj, yamldir, incs, + handle_include=self.do_single_include, + filename=self.filename) + + def fetch_include(self, inc) -> str: + """Download a ci yml file from a remote server/project""" + get_template = inc.get("template", None) + get_remote = inc.get("remote", None) + get_project = inc.get("project", None) + resp = None + if get_template or get_project: + gitlab = self.get_gitlab_client() + if get_template: + url = gitlab.api_url + GITLAB_SERVER_TEMPLATE_PATH + get_template + resp = gitlab.session.get(url) + resp.raise_for_status() + data = resp.json() + return data.get("content", "") + elif get_project: + get_file = inc.get("file", None) + get_ref = inc.get("ref", "HEAD") + get_params = None + if not get_file: + raise BadSyntaxError(f"project include has no file: {inc}") + get_file = get_file.lstrip("/") + encoded_file = urllib.parse.quote_plus(get_file) + encoded_project = urllib.parse.quote_plus(get_project) + url = gitlab.api_url + f"/projects/{encoded_project}/repository/files/{encoded_file}/raw" + if get_ref: + get_params = {"ref": get_ref} + resp = gitlab.session.get(url, params=get_params) + # have to decode the content + resp.raise_for_status() + return resp.text + + elif get_remote: + resp = requests.get(get_remote) + resp.raise_for_status() + return resp.text + return "" def do_single_include(self, baseobj: Dict[str, Any], @@ -725,7 +748,11 @@ class Loader(BaseLoader, JobLoaderMixin, ValidatorMixin, ExtendsMixin): """ Include a single file and process it """ - return do_single_include(baseobj, yamldir, inc, handle_read=self._read, variables=self.variables, filename=filename) + return do_single_include(baseobj, yamldir, inc, + handle_read=self._read, + handle_fetch=self.fetch_include, + variables=self.variables, + filename=filename) def do_validate(self, baseobj: Dict[str, Any]) -> None: """ diff --git a/emulator/gitlabemu/configtool_runner.py b/emulator/gitlabemu/configtool_runner.py index 4b607e8..bfc6c54 100644 --- a/emulator/gitlabemu/configtool_runner.py +++ b/emulator/gitlabemu/configtool_runner.py @@ -3,7 +3,8 @@ from argparse import Namespace import yaml -from .helpers import note, die, truth_string, setenv_string +from .helpers import note, die, setenv_string +from .variables import truth_string from .userconfig import get_user_config from .userconfigdata import GleRunnerConfig, DockerExecutorConfig, RUNNER_SHELL_SHELLS, RUNNER_EXECUTOR_TYPES, \ EXECUTOR_DOCKER, DEFAULT_DOCKER_CLI diff --git a/emulator/gitlabemu/docker.py b/emulator/gitlabemu/docker.py index f8a9bb9..ec937ef 100644 --- a/emulator/gitlabemu/docker.py +++ b/emulator/gitlabemu/docker.py @@ -14,7 +14,7 @@ from .helpers import communicate as comm, is_windows from .userconfig import get_user_config_context from .errors import DockerExecError, GitlabEmulatorError from .userconfigdata import GleRunnerConfig -from .variables import expand_variable +from .variables import expand_variable, truth_string PULL_POLICY_ALWAYS = "always" PULL_POLICY_IF_NOT_PRESENT = "if-not-present" @@ -219,8 +219,9 @@ class DockerTool(object): "-d", "--name", self.name, ] - if self.entrypoint is not None: - cmdline.extend(["--entrypoint", str(self.entrypoint)]) + if not is_windows(): + if self.entrypoint is not None: + cmdline.extend(["--entrypoint", str(self.entrypoint)]) if self.network is not None: cmdline.extend(["--network", self.network]) if priv: @@ -233,8 +234,13 @@ class DockerTool(object): cmdline.extend(["-e", f"{name}={value}"]) cmdline.extend(["-i", self.image]) + if not is_windows(): + if self.entrypoint == "": + cmdline.append("/bin/sh") + proc = self.docker_call(*cmdline) self.container = proc.stdout.strip() + info(f"started container {self.container}") except DockerToolFailed: # pragma: no cover if not self.image_present: fatal(f"Docker image {self.image} does not exist, (pull_policy={self.pull_policy})") @@ -312,10 +318,13 @@ class DockerJob(Job): @property def docker_entrypoint(self) -> Optional[List[str]]: - custom_entryppoint = None + entrypoint = [""] + envs = self.get_envs() + if truth_string(envs.get("DOCKER_DISABLE_ENTRYPOINT_OVERRIDE", "y")): + entrypoint = None if isinstance(self._image, dict): - custom_entryppoint = self._image.get("entrypoint", None) - return custom_entryppoint + entrypoint = self._image.get("entrypoint", None) + return entrypoint @property def docker_pull_policy(self) -> Optional[str]: @@ -551,15 +560,12 @@ class DockerJob(Job): self.docker.add_env(envname, environ[envname]) if self.docker_entrypoint is not None: - if is_windows(): # pragma: cover if windows - # windows can't have multiple args - args = self.docker_entrypoint - if len(args) > 0: - if len(args) > 1: - warning("windows docker entrypoint override may fail with several args") - self.docker.entrypoint = args[0] - else: - self.docker.entrypoint = self.docker_entrypoint + # can't have multiple args + args = self.docker_entrypoint + if len(args) > 0: + if len(args) > 1: + warning("windows docker entrypoint override may fail with several args") + self.docker.entrypoint = args[0] volumes = self.runner.docker.runtime_volumes() if volumes: info("Extra docker volumes registered:") diff --git a/emulator/gitlabemu/gitlab/types.py b/emulator/gitlabemu/gitlab/types.py index d3a5af0..64e2808 100644 --- a/emulator/gitlabemu/gitlab/types.py +++ b/emulator/gitlabemu/gitlab/types.py @@ -14,6 +14,7 @@ RESERVED_TOP_KEYS = ["stages", "variables", "include", "workflow", + "secret_detection", "default", ".gitlab-emulator-workspace" ] diff --git a/emulator/gitlabemu/gitlab/urls.py b/emulator/gitlabemu/gitlab/urls.py index 29ee0fb..633b305 100644 --- a/emulator/gitlabemu/gitlab/urls.py +++ b/emulator/gitlabemu/gitlab/urls.py @@ -1 +1,3 @@ GITLAB_ORG_TEMPLATE_BASEURL = "https://gitlab.com/gitlab-org/gitlab-foss/-/raw/HEAD/lib/gitlab/ci/templates" + +GITLAB_SERVER_TEMPLATE_PATH = "/templates/gitlab_ci_ymls/" \ No newline at end of file diff --git a/emulator/gitlabemu/gitlab_client_api.py b/emulator/gitlabemu/gitlab_client_api.py index 4abb6e7..7a821a5 100644 --- a/emulator/gitlabemu/gitlab_client_api.py +++ b/emulator/gitlabemu/gitlab_client_api.py @@ -471,7 +471,10 @@ def find_gitlab_project_config(servers: Dict[str, str]) -> Optional[GitlabIdent] return ident -def get_gitlab_project_client(repo: str, secure=True) -> Tuple[Optional[Gitlab], Optional[Project], Optional[str]]: +def get_gitlab_project_client(repo: str, + secure=True, + need_project=True, + ) -> Tuple[Optional[Gitlab], Optional[Project], Optional[str]]: """Get the gitlab client, project and git remote name for the given git repo""" override_server = os.getenv(GITLAB_SERVER_ENV) override_project = os.getenv(GITLAB_PROJECT_ENV) @@ -505,12 +508,15 @@ def get_gitlab_project_client(repo: str, secure=True) -> Tuple[Optional[Gitlab], # we have an ident, try to find the gitlab config api = gitlab_api(ident.server, secure=secure) api.auth() - for proj in api.projects.list(membership=True, all=True): - if ident.project: - if proj.path_with_namespace == ident.project: - project = proj - client = api - break + if need_project: + for proj in api.projects.list(membership=True, all=True): + if ident.project: + if proj.path_with_namespace == ident.project: + project = proj + client = api + break + else: + client = api if remotes and project: project_remotes = [project.ssh_url_to_repo, project.http_url_to_repo] @@ -523,12 +529,14 @@ def get_gitlab_project_client(repo: str, secure=True) -> Tuple[Optional[Gitlab], def get_current_project_client(tls_verify: Optional[bool] = True, - need_remote: Optional[bool] = True) -> Tuple[Gitlab, Project, str]: + need_remote: Optional[bool] = True, + need_project: Optional[bool] = True, + ) -> Tuple[Gitlab, Optional[Project], str]: """Get the requested/current gitlab client, gitlab project and optional git remote""" cwd = os.getcwd() - client, project, remotename = get_gitlab_project_client(cwd, tls_verify) + client, project, remotename = get_gitlab_project_client(cwd, tls_verify, need_project=need_project) - if not (client and project): + if not client: die("Could not find a gitlab server configuration, please add one with 'gle-config gitlab'") if need_remote: diff --git a/emulator/gitlabemu/helpers.py b/emulator/gitlabemu/helpers.py index cdd4d13..ec7966c 100644 --- a/emulator/gitlabemu/helpers.py +++ b/emulator/gitlabemu/helpers.py @@ -1,6 +1,8 @@ """ Various useful common funcs """ +from __future__ import print_function + import os.path from select import select from threading import Thread @@ -452,16 +454,10 @@ def notice(text: str) -> None: print(f"notice: {text}", file=sys.stderr, flush=True) -def truth_string(text: str) -> bool: - if text: - text = text.lower() - if text in ["y", "yes", "true", "on", "1"]: - return True - return False - - def setenv_string(text: str) -> Tuple[str, str]: parts = text.split("=", 1) if len(parts) == 2: return parts[0], parts[1] raise ValueError(f"{text} is not in the form NAME=VALUE") + + diff --git a/emulator/gitlabemu/localfiles.py b/emulator/gitlabemu/localfiles.py index abbec9c..0f1afdb 100644 --- a/emulator/gitlabemu/localfiles.py +++ b/emulator/gitlabemu/localfiles.py @@ -2,20 +2,21 @@ Utils for local file housekeeping """ import os -from .helpers import is_windows, has_docker +from .helpers import has_docker, is_linux from .docker import DockerTool def restore_path_ownership(path): path = os.path.abspath(path) chowner = os.path.abspath(os.path.join(os.path.dirname(__file__), "chown.py")) - if not is_windows(): # pragma: cover if not windows + if is_linux(): # pragma: cover if not windows if os.getuid() != 0: if has_docker(): from .resnamer import generate_resource_name dt = DockerTool() dt.name = generate_resource_name("chowner") dt.image = "python:3.9-alpine3.14" + dt.privileged = False dt.add_volume(path, path) dt.entrypoint = "/bin/sh" if not dt.image_present: diff --git a/emulator/gitlabemu/logmsg.py b/emulator/gitlabemu/logmsg.py index 6582498..8241f47 100644 --- a/emulator/gitlabemu/logmsg.py +++ b/emulator/gitlabemu/logmsg.py @@ -6,6 +6,7 @@ import sys import os import logging from .errors import GitlabEmulatorError +from .variables import truth_string FORMAT = '%(asctime)-15s %(name)s %(message)s' logging.basicConfig(format=FORMAT) @@ -15,15 +16,11 @@ LOGGER.setLevel(logging.INFO) FATAL_EXIT = True -AFIRMATIVE = ["y", "yes", "1", "enabled", "on"] - -def is_afirmative(txt: str) -> bool: - return str(txt).lower() in AFIRMATIVE - def enable_rule_debug(): os.environ["GLE_DEBUG_RULES"] = "y" + def enable_debug(): os.environ["GLE_DEBUG"] = "y" @@ -31,12 +28,13 @@ def enable_debug(): def info(msg): LOGGER.info(msg) + def debug_enabled() -> bool: - return is_afirmative(os.environ.get("GLE_DEBUG", "n")) + return truth_string(os.environ.get("GLE_DEBUG", "n")) def debugrule_enabled() -> bool: - return is_afirmative(os.environ.get("GLE_DEBUG_RULES", "n")) + return truth_string(os.environ.get("GLE_DEBUG_RULES", "n")) def debugrule(msg): diff --git a/emulator/gitlabemu/pipelines.py b/emulator/gitlabemu/pipelines.py index 0357dba..8f4e5c1 100644 --- a/emulator/gitlabemu/pipelines.py +++ b/emulator/gitlabemu/pipelines.py @@ -97,7 +97,6 @@ def generate_partial_pipeline(loader, *goals, variables: Optional[Dict[str, str] die("Cannot generate a pipeline with zero jobs") - def generate_pipeline(loader, *goals, variables: Optional[Dict[str, str]] = None, use_from: Optional[str] = None, @@ -107,13 +106,21 @@ def generate_pipeline(loader, *goals, pipeline = None download_jobs = {} deps = {} - client, project, remotename = get_current_project_client(tls_verify=tls_verify) + client = None + project = None + remotename = None + + if dump_only: + client = loader.get_gitlab_client() + cwd = os.getcwd() if not goals: die("Cannot generate a pipeline with zero jobs") note(f"Generate subset pipeline to build '{goals}'..") recurse = True + if use_from or not dump_only: + client, project, remotename = get_current_project_client(tls_verify=tls_verify) if use_from: recurse = False ident = parse_gitlab_from_arg(use_from, prefer_gitref=True) @@ -185,28 +192,28 @@ def generate_pipeline(loader, *goals, if dump_only: return generated + else: + branch_name = generate_subset_branch_name(client, cwd) + note(f"Creating temporary pipeline branch '{branch_name}'..") + commit = create_pipeline_branch(cwd, + remotename, + branch_name, + f"subset pipeline for {goals}", + { + ".gitlab-ci.yml": ordered_dump(generated) + }) + if commit: + note(f"Custom build commit is {commit}") + note(f"Waiting for new pipeline to start..") + pipeline = wait_for_project_commit_pipeline(project, commit) + if not pipeline: + die("Could not find the pipeline for our change") + else: + note(f"Building: {pipeline.web_url}") - branch_name = generate_subset_branch_name(client, cwd) - note(f"Creating temporary pipeline branch '{branch_name}'..") - commit = create_pipeline_branch(cwd, - remotename, - branch_name, - f"subset pipeline for {goals}", - { - ".gitlab-ci.yml": ordered_dump(generated) - }) - if commit: - note(f"Custom build commit is {commit}") - note(f"Waiting for new pipeline to start..") - pipeline = wait_for_project_commit_pipeline(project, commit) - if not pipeline: - die("Could not find the pipeline for our change") else: - note(f"Building: {pipeline.web_url}") - - else: - die("Could not make a custom pipeline branch, " - "please make sure your local changes are committed first") + die("Could not make a custom pipeline branch, " + "please make sure your local changes are committed first") def get_subset_prefix() -> str: diff --git a/emulator/gitlabemu/runner.py b/emulator/gitlabemu/runner.py index 02f6c4e..855526d 100644 --- a/emulator/gitlabemu/runner.py +++ b/emulator/gitlabemu/runner.py @@ -219,6 +219,7 @@ def execute_job(config: Dict[str, Any], options: Optional[Dict[str, Any]] = None, overrides: Optional[Dict[str, Any]] = None, jobfactory: Optional[JobFactory] = None, + chown=True, ): """ Run a job, optionally run required dependencies @@ -231,6 +232,7 @@ def execute_job(config: Dict[str, Any], :param noop: if True, print instead of execute commands :param options: If given, set attributes on the job before use. :param overrides: If given, replace properties in the top level of a job dictionary. + :param chown: If True and if using docker, attempt to restore the folder ownership. :return: """ if seen is None: @@ -243,7 +245,7 @@ def execute_job(config: Dict[str, Any], if recurse: for need in jobobj.dependencies: - execute_job(config, need, seen=seen, recurse=True, noop=noop, jobfactory=jobfactory) + execute_job(config, need, seen=seen, recurse=True, noop=noop, jobfactory=jobfactory, chown=chown) print(f">>> execute {jobobj.name}:", file=sys.stderr) if noop: if isinstance(jobobj, DockerJob): @@ -255,7 +257,10 @@ def execute_job(config: Dict[str, Any], print(f"script {line}") else: if use_runner: - gitlab_runner_exec(jobobj) + try: + gitlab_runner_exec(jobobj) + finally: + restore_path_ownership(os.getcwd()) else: jobobj.run() seen.add(jobname) @@ -543,28 +548,22 @@ def run(args=None): if options.error_shell: # pragma: no cover job_options["error_shell"] = [options.error_shell] overrides["timeout"] = None - try: - jobfactory = JobFactory() - if options.gen_script: - jobfactory = ScriptJobFactory() - - executed_jobs = set() - execute_job(loader.config, jobname, - seen=executed_jobs, - use_runner=options.exec, - recurse=options.FULL, - noop=options.noop, - options=job_options, - overrides=overrides, - jobfactory=jobfactory - ) - finally: - if not options.noop: - if has_docker() and fix_ownership: - if is_linux() or is_apple(): # pragma: cover if posix - if os.getuid() > 0: - note("Fixing up local file ownerships..") - restore_path_ownership(os.getcwd()) - note("finished") + + jobfactory = JobFactory() + if options.gen_script: + jobfactory = ScriptJobFactory() + + executed_jobs = set() + execute_job(loader.config, jobname, + seen=executed_jobs, + use_runner=options.exec, + recurse=options.FULL, + noop=options.noop, + options=job_options, + overrides=overrides, + jobfactory=jobfactory, + chown=fix_ownership + ) + if not options.gen_script: print("Build complete!") diff --git a/emulator/gitlabemu/tests/include-templates.yml b/emulator/gitlabemu/tests/include-templates.yml new file mode 100644 index 0000000..78f29fd --- /dev/null +++ b/emulator/gitlabemu/tests/include-templates.yml @@ -0,0 +1,10 @@ +include: + - template: Bash + - project: 'gitlab-org/quality/pipeline-common' + file: '/ci/danger-review.yml' + + +job1: + image: alpine:latest + script: + - env \ No newline at end of file diff --git a/emulator/gitlabemu/tests/test_helpers.py b/emulator/gitlabemu/tests/test_helpers.py index 53542b1..d0470c5 100644 --- a/emulator/gitlabemu/tests/test_helpers.py +++ b/emulator/gitlabemu/tests/test_helpers.py @@ -22,8 +22,8 @@ from ..helpers import ( remote_servers, sensitive_varname, setenv_string, - trim_quotes, truth_string, -) + trim_quotes, ) +from ..variables import truth_string from ..logmsg import debug_enabled diff --git a/emulator/gitlabemu/tests/test_include.py b/emulator/gitlabemu/tests/test_include.py index 58a9a31..195d9bf 100644 --- a/emulator/gitlabemu/tests/test_include.py +++ b/emulator/gitlabemu/tests/test_include.py @@ -18,6 +18,10 @@ def test_include_processing(top_dir: str, capfd: CaptureFixture): assert "BOOK=exforce" in stdout +def test_include_template(in_tests: str, capfd: CaptureFixture): + run(["-c", "include-templates.yml", "-l"]) + + def test_include_unknown(in_tests: str, capfd: CaptureFixture): with pytest.raises(SystemExit): run(["-c", str(Path("invalid") / "include-unknown-type.yaml"), "-l" ]) diff --git a/emulator/gitlabemu/tests/test_local_docker.py b/emulator/gitlabemu/tests/test_local_docker.py index 14e7a77..6e61447 100644 --- a/emulator/gitlabemu/tests/test_local_docker.py +++ b/emulator/gitlabemu/tests/test_local_docker.py @@ -313,7 +313,6 @@ def test_docker_fail_shell(top_dir, capfd): assert f"{magic}" in stdout -@pytest.mark.usefixtures("has_docker") @pytest.mark.usefixtures("posix_only") def test_docker_runner_exec(top_dir, capfd, temp_folder: Path): """Test running gitlab-runner exec""" @@ -321,12 +320,13 @@ def test_docker_runner_exec(top_dir, capfd, temp_folder: Path): fake_gitlab_runner = temp_folder / "gitlab-runner" with open(fake_gitlab_runner, "w") as fd: print("#!/bin/sh", file=fd) - print("echo running: \"$@\"", file=fd) + print("echo fake runner running: \"$@\"", file=fd) os.chmod(str(fake_gitlab_runner), 0o755) os.environ["PATH"] = str(temp_folder.absolute()) + os.pathsep + os.environ["PATH"] run(["-c", "test-ci.yml", "--exec", "alpine-test"]) stdout, stderr = capfd.readouterr() + assert "fake runner running" in stdout for required in ["--cicd-config-file", "--docker-volumes", "--custom_build_dir-enabled", diff --git a/emulator/gitlabemu/variables.py b/emulator/gitlabemu/variables.py index b56bbf7..05af393 100644 --- a/emulator/gitlabemu/variables.py +++ b/emulator/gitlabemu/variables.py @@ -26,3 +26,11 @@ def expand_variable(variables: Dict[str, str], haystack: Union[str, Dict[str, An else: break return haystack + + +def truth_string(text: Union[str, int]) -> bool: + if text: + text = str(text).lower() + if text in ["y", "yes", "true", "on", "1"]: + return True + return False diff --git a/emulator/setup.py b/emulator/setup.py index 435aab1..9bee051 100644 --- a/emulator/setup.py +++ b/emulator/setup.py @@ -1,6 +1,6 @@ from distutils.core import setup -VERSION = "16.0.2" +VERSION = "16.1.1" requirements = [ "pyyaml>=5.1", -- GitLab