diff --git a/dump_cfg.py b/dump_cfg.py new file mode 100644 index 0000000000000000000000000000000000000000..e750c2b1fbe95f61f930230aec2441f5f903df4e --- /dev/null +++ b/dump_cfg.py @@ -0,0 +1,11 @@ +import os +import yaml +from gitlabemu import configloader +cfg = configloader.find_ci_config(os.getcwd()) +loader = configloader.Loader() +loader.load(cfg) +for jobname in loader.get_jobs(): + print(jobname) + job = loader.get_job(jobname) + assert job + # job is a dictionary of the job image, stage, steps, artifacts etc diff --git a/emulator/gitlabemu/docker.py b/emulator/gitlabemu/docker.py index 9a9b9312aa146436783421ff5308e402db6abedc..ff0f43f72e10805851e53e680c444876c7d7226c 100644 --- a/emulator/gitlabemu/docker.py +++ b/emulator/gitlabemu/docker.py @@ -222,21 +222,26 @@ class DockerJob(Job): filename = "generated-gitlab-script" + self.get_script_fileext() temp = os.path.join(tempfile.gettempdir(), filename) try: - with open(temp, "w") as fd: - print(lines, file=fd) - # copy it to the container - dest = "/tmp" - if is_windows(): - dest = "c:\\windows\\temp" - target_script = os.path.join(dest, filename) - info("Copying {} to container as {} ..".format(temp, target_script)) - self.docker.add_file(temp, dest) + stdin = None + if self.is_powershell(): + stdin = lines + target_script = None + if stdin is None: + # copy script into the container + with open(temp, "w") as fd: + print(lines, file=fd) + dest = "/tmp" + if is_windows(): + dest = "c:\\windows\\temp" + target_script = os.path.join(dest, filename) + info("Copying {} to container as {} ..".format(temp, target_script)) + self.docker.add_file(temp, dest) while attempts > 0: try: cmdline = self.shell_command(target_script) - task = self.docker.exec(self.workspace, cmdline, user=user) - self.communicate(task, script=None) + task = self.docker.exec(self.workspace, cmdline, user=user, pipe=True) + self.communicate(task, script=stdin) break except DockerExecError: self.stdout.write( diff --git a/emulator/gitlabemu/helpers.py b/emulator/gitlabemu/helpers.py index aa7ec9098358f4135bc1701fc8bbde10e10fd092..7a391a181f1f7d04686347762fc3a4bcd3d52e16 100644 --- a/emulator/gitlabemu/helpers.py +++ b/emulator/gitlabemu/helpers.py @@ -44,6 +44,7 @@ class ProcessLineProxyThread(Thread): self.errors.append(err) def run(self): + data = None while True: try: data = self.process.stdout.readline() diff --git a/emulator/gitlabemu/jobs.py b/emulator/gitlabemu/jobs.py index 4594dffdee011f5a9ec171d56494283889c710b3..3446ee2a45c3f1284cb9719b8e9acc17e8bfdfb7 100644 --- a/emulator/gitlabemu/jobs.py +++ b/emulator/gitlabemu/jobs.py @@ -11,10 +11,12 @@ import subprocess import tempfile import threading import time + from .logmsg import info, fatal from .errors import GitlabEmulatorError from .helpers import communicate as comm, is_windows, is_apple, is_linux, debug_print from .helpers import parse_timeout +from .ansi import ANSI_GREEN, ANSI_RESET class NoSuchJob(GitlabEmulatorError): @@ -111,12 +113,12 @@ class Job(object): def shell_command(self, scriptfile): if is_windows(): - if self.shell == "powershell": + if self.is_powershell(): return ["powershell.exe", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", - "-Command", scriptfile] + "-Command", "-"] return ["powershell", "-Command", "& cmd /Q /C " + scriptfile] # else unix/linux interp = "/bin/sh" @@ -267,21 +269,27 @@ class Job(object): script = make_script(lines, powershell=self.is_powershell()) temp = tempfile.mkdtemp() try: - ext = self.get_script_fileext() - generated = os.path.join(temp, "generated-gitlab-script" + ext) - with open(generated, "w") as fd: - print(script, file=fd) + generated = None + stdin = None + proc_stdin = subprocess.DEVNULL + if self.is_powershell(): + stdin = script.encode() + proc_stdin = subprocess.PIPE + else: + generated = os.path.join(temp, "generated-gitlab-script" + self.get_script_fileext()) + with open(generated, "w") as fd: + print(script, file=fd) cmdline = self.shell_command(generated) debug_print("cmdline: {}".format(cmdline)) opened = subprocess.Popen(cmdline, env=envs, shell=False, cwd=self.workspace, - stdin=subprocess.DEVNULL, + stdin=proc_stdin, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) self.build_process = opened - self.communicate(opened, script=None) + self.communicate(opened, script=stdin) finally: shutil.rmtree(temp) @@ -381,6 +389,27 @@ class Job(object): fatal("Shell job {} failed".format(self.name)) +def powershell_escape(text: str, variable=False) -> str: + """Escape a powershell string""" + text = ANSI_GREEN + text + ANSI_RESET + # taken from: http://www.robvanderwoude.com/escapechars.php + text = text.replace("`", "``") + text = text.replace("\a", "`a") + text = text.replace("\b", "`b") + text = text.replace("\f", "^f") + text = text.replace("\r", "`r") + text = text.replace("\n", "`n") + text = text.replace("\t", "^t") + text = text.replace("\v", "^v") + text = text.replace("#", "`#") + text = text.replace("'", "`'") + text = text.replace("\"", "`\"") + if variable: + text = text.replace("$", "`$") + text = text.replace("``e", "`e") + return text + + def make_script(lines, powershell=False): """ Join lines together to make a script @@ -404,7 +433,7 @@ def make_script(lines, powershell=False): 'echo "Running on $([Environment]::MachineName)..."', ] line_wrap_before = [ - '& {', + '& { ', ] line_wrap_tail = [ '}', @@ -440,7 +469,8 @@ def make_script(lines, powershell=False): else: content += os.linesep.join(line_wrap_before) if powershell: - content += "& " + line + os.linesep + content += "echo" + powershell_escape(line, variable=True) + os.linesep + content += "& " + powershell_escape(line) + os.linesep content += "if(!$?) { Exit $LASTEXITCODE }" + os.linesep else: content += line + os.linesep diff --git a/emulator/gitlabemu/tests/conftest.py b/emulator/gitlabemu/tests/conftest.py index 02f5843ba193f749f66676b34d97f2f316845578..00cf21a02ed2bd5709bca6c0b24484604bb3ec59 100644 --- a/emulator/gitlabemu/tests/conftest.py +++ b/emulator/gitlabemu/tests/conftest.py @@ -34,6 +34,11 @@ def posix_only(): pytest.skip("System does not have the posix module") +@pytest.fixture(scope="session") +def windows_only(): + if platform.system() != "Windows": + pytest.skip("not a Windows platform") + @pytest.fixture(scope="session") def has_docker(): try: diff --git a/emulator/gitlabemu/tests/test-powershell-fail.yml b/emulator/gitlabemu/tests/test-powershell-fail.yml index 09e26e2277268f3a3ae3049f730e40d878cfff54..8d73c95c708be2b9aefd43a2f9cdbeddcd1fadcf 100644 --- a/emulator/gitlabemu/tests/test-powershell-fail.yml +++ b/emulator/gitlabemu/tests/test-powershell-fail.yml @@ -15,6 +15,7 @@ windows-powershell-ok: - .windows script: - echo hello + - cmd /C "echo this goes to stderr 1>&2" - echo 'this is powershell' - dir test-powershell-fail.yml diff --git a/emulator/gitlabemu/tests/test_local_shell.py b/emulator/gitlabemu/tests/test_local_shell.py new file mode 100644 index 0000000000000000000000000000000000000000..11a640d42af7af35fd24de467b568a539d9e355a --- /dev/null +++ b/emulator/gitlabemu/tests/test_local_shell.py @@ -0,0 +1,19 @@ +"""Tests running local shells""" +import pytest +import os +from ..runner import run + + +@pytest.mark.usefixtures("windows_only") +def test_local_shell_windows(top_dir, capfd): + """Test a basic shell script job works and fails correctly""" + folder = os.path.join(top_dir, "emulator", "gitlabemu", "tests") + os.chdir(folder) + # should print to stderr without failing the build + run(["-c", "test-powershell-fail.yml", "windows-powershell-ok", "--powershell", "--ignore-docker"]) + stdout, _ = capfd.readouterr() + assert "this goes to stderr" in stdout + assert "this is powershell" in stdout + + with pytest.raises(SystemExit): + run(["-c", "test-powershell-fail.yml", "windows-powershell-fail", "--powershell", "--ignore-docker"]) diff --git a/emulator/setup.py b/emulator/setup.py index b1c2c92c1bfc8d1a04049318ac2aa4547933282a..3e96e6fa9e1b6be87d6c188b2b11b11d70d3b13c 100644 --- a/emulator/setup.py +++ b/emulator/setup.py @@ -1,6 +1,6 @@ from distutils.core import setup -VERSION = "0.9.18" +VERSION = "0.9.19" setup( name="gitlab-emulator", diff --git a/runner/GitlabPyRunner/consts.py b/runner/GitlabPyRunner/consts.py index b30f8768e46171b22ec7bb2501fe8420f33290f9..5844aa2c6f724ca4729d64d2d32716ab6bdf0ff8 100644 --- a/runner/GitlabPyRunner/consts.py +++ b/runner/GitlabPyRunner/consts.py @@ -2,5 +2,5 @@ Constants for gitlab-python-runner """ NAME = "gitlab-python-runner" -VERSION = "14.3.13" +VERSION = "14.3.14" USER_AGENT = "{} {}".format(NAME, VERSION) diff --git a/runner/setup.py b/runner/setup.py index 28daeeffa58e97f76b459da6546c691a5fe00e0c..84096efeb14e50885e4e350b27c74abf33d4586d 100644 --- a/runner/setup.py +++ b/runner/setup.py @@ -13,7 +13,7 @@ setup( scripts=["gitlab-runner.py"], install_requires=[ "pyyaml>=5.1", - "gitlab-emulator>=0.9.18", + "gitlab-emulator>=0.9.19", "unidecode>=1.1.1"], platforms=["any"], license="License :: OSI Approved :: MIT License",