From 24a6303450ad0ed38933e09993cd8179eb224762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=BCnther?= Date: Fri, 6 Nov 2020 14:27:27 +0100 Subject: [PATCH 01/26] change default lanch parameter --- .vscode/launch.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.vscode/launch.json b/.vscode/launch.json index 17e15f2..5a9b2e5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,6 +9,10 @@ "type": "python", "request": "launch", "program": "${file}", + "args": [ + "backup", + "--verbose" + ], "console": "integratedTerminal" } ] -- GitLab From 68e3ac9d23ca27c697a4e837296b88ed1f5cf93e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=BCnther?= Date: Fri, 6 Nov 2020 14:31:39 +0100 Subject: [PATCH 02/26] refactor dobas python module - entrypoint outside module sources - test case base class - module logging --- dobas/__init__.py | 15 ++++ dobas/dobas.py | 125 ---------------------------- dobas/orchestrator.py | 56 +++++++++++++ dobas/test/__init__.py | 0 dobas/test/docker-compose.yml | 4 +- dobas/test/test_case_base.py | 24 ++++++ dobas/test/test_content_provider.py | 23 +---- dobas/test/test_dummy.py | 20 ----- dobas/worker.py | 55 ++++++++++++ entry.py | 31 +++++++ 10 files changed, 186 insertions(+), 167 deletions(-) delete mode 100644 dobas/dobas.py create mode 100644 dobas/test/__init__.py create mode 100644 dobas/test/test_case_base.py delete mode 100644 dobas/test/test_dummy.py create mode 100755 entry.py diff --git a/dobas/__init__.py b/dobas/__init__.py index e69de29..7a1380c 100644 --- a/dobas/__init__.py +++ b/dobas/__init__.py @@ -0,0 +1,15 @@ +import logging +import sys + +__version__ = "0.1.0" + +logger = logging.getLogger(__name__) + + +def configure_module_logger(): + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(logging.DEBUG) + logger.addHandler(handler) + + +configure_module_logger() diff --git a/dobas/dobas.py b/dobas/dobas.py deleted file mode 100644 index f1e72e6..0000000 --- a/dobas/dobas.py +++ /dev/null @@ -1,125 +0,0 @@ - -"""Borg backup -Usage: - dobas.py backup [options] - dobas.py restore [options] - dobas.py (-h | --help) - dobas.py --version - -Options: - -c , --config Borg configuration file [default: x] - -d, --dry Make dry run. Don't change anything. - --host Docker host [default: unix://var/run/docker.sock] - -v, --verbose Increase verbosity - -h, --help Show this screen. - --version Show version. - -- find subjects to backup -- for each subject: - do_subject specific backup - - -- subject types: - - pure docker volume -""" -from dataclasses import dataclass -import logging -import sys -import docker -import subprocess -from pathlib import PurePath - - -@dataclass -class DobasCfg: - dry_run: bool = False - verbosity: int = logging.WARNING - borg_prune_rules: str = "--keep-daily 14 --keep-monthly 10" - - -class DobasStrategy(list): - @staticmethod - def find_enabled_volumes() -> [docker.models.volumes.Volume]: - """ get volumes to be handled - - list docker volumes labled with "orchestrator.enable: true" - - return list of volumes - """ - client = docker.from_env() - - print(f"docker client: {client}") - - volumes = [volume for volume in client.volumes.list() if bool( - volume.attrs.get('Labels').get('orchestrator.enable'))] - - return volumes - - @staticmethod - def find_enabled_container_volumes() -> [docker.models.volumes.Volume]: - return [] - - @classmethod - def discover(cls): - s = DobasStrategy() - for rule in [DobasStrategy.find_enabled_volumes, DobasStrategy.find_enabled_container_volumes]: - s += rule() - return s - - -class DobasRunner(object): - def __init__(self, client: docker.DockerClient, cfg: DobasCfg = DobasCfg()): - self.cfg = cfg - self.client = client - - def run(self, strategy: DobasStrategy = None): - for element in strategy: - if type(element) is docker.models.volumes.Volume: - self.volume_backup_handler(volume=element) - - def prune(self): - ans = subprocess.run(["borg", "prune", "-v", "--list"] + - self.cfg.borg_prune_rules.split(" ") + ["::"]) - - if ans.returncode == 0: - pass - else: - print("oje") - - def volume_backup_handler(self, volume: docker.models.volumes.Volume = None): - if volume: - # risk of name duplicates here - mounted_volume = {f"{volume.id}": { - 'bind': f"/mnt/{volume.id}", 'mode': 'ro'}} - - self.client.containers.run("alpine", command="ls -la /mnt", - volumes=mounted_volume) - - ans = subprocess.run(["borg", "create", "--stats", - "::{now}"] + volume.get('bind')) - if ans.returncode > 0: - print(f"NO BACKUP CREATED. BORG Collective failed to assimilate biological " - "entity. (Exitcode: {creation.returncode})", file=sys.stderr) - else: - print("borg exec done") - - def simple_folder_handler(self, paths: [PurePath] = None): - if len(paths): - pass - - -module_cfg = DobasCfg() - - -def do_backup(): - strategy = DobasStrategy.discover() - runner = DobasRunner(client=docker.from_env(), cfg=module_cfg) - runner.run(strategy) - - -def do_restore(): - print(__name__) - - -if __name__ == "__main__": - do_backup() diff --git a/dobas/orchestrator.py b/dobas/orchestrator.py index e69de29..c046452 100644 --- a/dobas/orchestrator.py +++ b/dobas/orchestrator.py @@ -0,0 +1,56 @@ +import docker +from dobas import logger + + +class Strategy(list): + """ entities to process + """ + + @staticmethod + def find_enabled_volumes(client: docker.client.DockerClient = docker.from_env()) -> [docker.models.volumes.Volume]: + """ get volumes to be handled + + list docker volumes labled with "dobas.enable: true" + + return list of volumes + """ + logger.debug(f"fetch volumes fom docker client: {client}") + + volumes = [volume for volume in client.volumes.list() if bool( + volume.attrs.get('Labels').get('dobas.enable'))] + + logger.debug("found volumes: {volumes}") + + return volumes + + @staticmethod + def find_enabled_container_volumes(client: docker.client.DockerClient = docker.from_env()) -> [docker.models.volumes.Volume]: + """ + + return list of volumes + """ + return [] + + @classmethod + def discover(cls, client: docker.client.DockerClient = docker.from_env()): + logger.debug("discover subjects") + + s = Strategy() + + for rule in [Strategy.find_enabled_volumes, Strategy.find_enabled_container_volumes]: + s += rule(client=client) + + return s + + +class Orchestrator: + """ + """ + + def do_backup(self, strategy: Strategy, *args, **kwargs): + logger.info(__name__) + # worker = Worker(client=docker.from_env(), cfg=dobas.cfg) + # worker.process(strategy) + + def do_restore(self): + print(__name__) diff --git a/dobas/test/__init__.py b/dobas/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dobas/test/docker-compose.yml b/dobas/test/docker-compose.yml index 6eb6026..38c530f 100644 --- a/dobas/test/docker-compose.yml +++ b/dobas/test/docker-compose.yml @@ -1,8 +1,10 @@ version: '3' services: - nginx: + service: image: alpine + labels: + dobas.enabled: true volumes: - ./:/tmp/src - named-volume:/tmp/named-volume diff --git a/dobas/test/test_case_base.py b/dobas/test/test_case_base.py new file mode 100644 index 0000000..78285d8 --- /dev/null +++ b/dobas/test/test_case_base.py @@ -0,0 +1,24 @@ +import docker +import subprocess + +from unittest import TestCase + + +class ContentProviderBase(TestCase): + def setUp(self): + self.docker_client = docker.from_env() + # create example set of service volume combinations + process = subprocess.Popen(args=['docker-compose', '-f', 'dobas/test/docker-compose.yml', 'up', '-d'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + self.assertEqual(process.returncode, 0, + msg="process failed with stdout {} and stderr {}".format(stdout, stderr)) + + def tearDown(self): + process = subprocess.Popen(args=['docker-compose', '-f', 'dobas/test/docker-compose.yml', 'down', '--volumes'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + self.assertEqual(process.returncode, 0, + msg="process failed with stdout {} and stderr {}".format(stdout, stderr)) diff --git a/dobas/test/test_content_provider.py b/dobas/test/test_content_provider.py index d1e008b..f5e2844 100644 --- a/dobas/test/test_content_provider.py +++ b/dobas/test/test_content_provider.py @@ -1,28 +1,9 @@ -import docker import os -from unittest import TestCase -import subprocess +from test_case_base import ContentProviderBase -class TestContentProvider(TestCase): - def setUp(self): - self.docker_client = docker.from_env() - # create example set of service volume combinations - process = subprocess.Popen(args=['docker-compose', '-f', 'dobas/test/docker-compose.yml', 'up', '-d'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - self.assertEqual(process.returncode, 0, - msg="process failed with stdout {} and stderr {}".format(stdout, stderr)) - - def tearDown(self): - process = subprocess.Popen(args=['docker-compose', '-f', 'dobas/test/docker-compose.yml', 'down', '--volumes'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - self.assertEqual(process.returncode, 0, - msg="process failed with stdout {} and stderr {}".format(stdout, stderr)) +class TestContentProvider(ContentProviderBase): def test_startContainerWithExternalVolumesToBackup(self): # find volumes to backup volumes_to_backup = [] diff --git a/dobas/test/test_dummy.py b/dobas/test/test_dummy.py deleted file mode 100644 index 63cca4e..0000000 --- a/dobas/test/test_dummy.py +++ /dev/null @@ -1,20 +0,0 @@ -import unittest - - -class TestStringMethods(unittest.TestCase): - @classmethod - def setUpClass(cls): - pass - - @classmethod - def tearDownClass(cls): - pass - - def setUp(self): - self.setUp_passed = True - - def tearDown(self): - self.setUp_passed = True - - def test_dummy(self): - self.assertTrue(True) diff --git a/dobas/worker.py b/dobas/worker.py index e69de29..1ef58c3 100644 --- a/dobas/worker.py +++ b/dobas/worker.py @@ -0,0 +1,55 @@ +import docker +import sys +import subprocess +from pathlib import PurePath + +import dobas + + +class Worker(object): + def __init__(self, client: docker.DockerClient): + self.client = client + dobas.logger.debug(f"new worker {client}".format( + client=self.client)) + + def process(self, strategy: dobas.Strategy = None): + dobas.logger.debug("processing strategy {strategy}") + for element in strategy: + dobas.logger.info("processing {element}") + if type(element) is docker.models.volumes.Volume: + self.volume_backup_handler(volume=element) + else: + dobas.logger.warning( + "no handler for element {element}, skipping ...") + + def prune(self): + dobas.logger.info("prune") + dobas.logger.error("not implemented") + # ans = subprocess.run(["borg", "prune", "-v", "--list"] + + # self.cfg.borg_prune_rules.split(" ") + ["::"]) + + # if ans.returncode == 0: + # pass + # else: + # print("oje") + + def volume_backup_handler(self, volume: docker.models.volumes.Volume = None): + if volume: + # risk of name duplicates here + mounted_volume = {f"{volume.id}": { + 'bind': f"/mnt/{volume.id}", 'mode': 'ro'}} + + self.client.containers.run("alpine", command="ls -la /mnt", + volumes=mounted_volume) + + ans = subprocess.run(["borg", "create", "--stats", + "::{now}"] + volume.get('bind')) + if ans.returncode > 0: + print(f"NO BACKUP CREATED. BORG Collective failed to assimilate biological " + "entity. (Exitcode: {creation.returncode})", file=sys.stderr) + else: + print("borg exec done") + + def simple_folder_handler(self, paths: [PurePath] = None): + if len(paths): + pass diff --git a/entry.py b/entry.py new file mode 100755 index 0000000..685e6e5 --- /dev/null +++ b/entry.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Docker Backup Service +Usage: + dobas.py backup [options] + dobas.py restore [options] + dobas.py (-h | --help) + dobas.py --version + +Options: + -c , --config Borg configuration file [default: x] + -d, --dry Make dry run. Don't change anything. + --host Docker host [default: unix://var/run/docker.sock] + -v, --verbose Increase verbosity + -h, --help Show this screen. + --version Show version. +""" +from docopt import docopt + +import dobas +from logging import DEBUG + +if __name__ == '__main__': + arguments = docopt(__doc__) + + if(arguments['--verbose']): + dobas.logger.setLevel(DEBUG) + + dobas.logger.debug(arguments) + + if arguments.get('backup'): + print("do_backup()") -- GitLab From 425f63c3cf99d3ca504deb61b0729f51d3185f65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=BCnther?= Date: Fri, 6 Nov 2020 15:25:34 +0100 Subject: [PATCH 03/26] WIP: conception --- dobas/orchestrator.py | 89 +++++++++++++++++++++++------------- dobas/orchestrator_config.py | 10 ++++ dobas/subject.py | 10 ++++ dobas/worker.py | 28 ++++++------ dobas/worker_config.py | 6 +++ entry.py | 31 ------------- main.py | 47 +++++++++++++++++++ 7 files changed, 144 insertions(+), 77 deletions(-) create mode 100644 dobas/orchestrator_config.py create mode 100644 dobas/subject.py create mode 100644 dobas/worker_config.py delete mode 100755 entry.py create mode 100755 main.py diff --git a/dobas/orchestrator.py b/dobas/orchestrator.py index c046452..ad5390b 100644 --- a/dobas/orchestrator.py +++ b/dobas/orchestrator.py @@ -1,56 +1,79 @@ import docker -from dobas import logger +import os +from . import logger +from .subject import Subject +from .orchestrator_config import OrchestratorCfg -class Strategy(list): - """ entities to process + +def find_enabled_volumes(client: docker.client.DockerClient = docker.from_env()) -> [docker.models.volumes.Volume]: + """ get volumes to be handled + + list docker volumes labled with "dobas.enable: true" + + return list of volumes """ + logger.debug(f"fetch volumes fom docker client: {client}") - @staticmethod - def find_enabled_volumes(client: docker.client.DockerClient = docker.from_env()) -> [docker.models.volumes.Volume]: - """ get volumes to be handled + volumes = [volume for volume in client.volumes.list() if bool( + volume.attrs.get('Labels').get('dobas.enable'))] - list docker volumes labled with "dobas.enable: true" + logger.debug(f"found volumes: {volumes}") - return list of volumes - """ - logger.debug(f"fetch volumes fom docker client: {client}") + return volumes - volumes = [volume for volume in client.volumes.list() if bool( - volume.attrs.get('Labels').get('dobas.enable'))] - logger.debug("found volumes: {volumes}") +def find_enabled_container_volumes(client: docker.client.DockerClient = docker.from_env()) -> [docker.models.volumes.Volume]: + """ - return volumes + return list of volumes + """ + logger.error("find_enabled_container_volumes not implemented") + return [] - @staticmethod - def find_enabled_container_volumes(client: docker.client.DockerClient = docker.from_env()) -> [docker.models.volumes.Volume]: - """ - return list of volumes - """ - return [] +def orchestrator_config_from_env() -> OrchestratorCfg: + return OrchestratorCfg( + DOBAS_ORCHESTRATOR_DOCKER_HOST=os.getenv('DOCKER_HOST'), + DOBAS_ENABLE_DEFAULT=os.getenv('DOBAS_ENABLE_DEFAULT'), + DOBAS_WORKER_SCHEDULING=os.getenv('DOBAS_WORKER_SCHEDULING'), + DOBAS_WORKER_STORAGE_HOST=os.getenv('DOBAS_WORKER_STORAGE_HOST') + ) - @classmethod - def discover(cls, client: docker.client.DockerClient = docker.from_env()): - logger.debug("discover subjects") - s = Strategy() +def discover(client: docker.client.DockerClient = docker.from_env()) -> [Subject]: + logger.debug("discover subjects") - for rule in [Strategy.find_enabled_volumes, Strategy.find_enabled_container_volumes]: - s += rule(client=client) + volumes = find_enabled_volumes(client=client) + container_volumes = find_enabled_container_volumes(client=client) - return s + # TODO: get a better idea of grouping volumes and services + # Subject(volumes=[volume], worker_config=None, services=None) + return [Subject(worker_config=None, volumes=[volume], services=None) for volume in volumes].extend( + [Subject(worker_config=None, volumes=[volume], services=None) for volume in container_volumes]) class Orchestrator: """ """ - def do_backup(self, strategy: Strategy, *args, **kwargs): - logger.info(__name__) - # worker = Worker(client=docker.from_env(), cfg=dobas.cfg) - # worker.process(strategy) + def __init__(self, cfg: OrchestratorCfg): + self.cfg = cfg + logger.debug("implement typechecked cfg") + logger.info(f"create orchestrator {cfg}") + + self.docker_client = docker.DockerClient( + base_url=cfg.DOBAS_ORCHESTRATOR_DOCKER_HOST) + + discover(client=self.docker_client) + + def start_daemon(self): + logger.error("start_daemon not implemented") + + # def do_backup(self, strategy: Strategy, *args, **kwargs): + # logger.info(__name__) + # # worker = Worker(client=docker.from_env(), cfg=dobas.cfg) + # # worker.process(strategy) - def do_restore(self): - print(__name__) + # def do_restore(self): + # print(__name__) diff --git a/dobas/orchestrator_config.py b/dobas/orchestrator_config.py new file mode 100644 index 0000000..4dcc214 --- /dev/null +++ b/dobas/orchestrator_config.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass + + +@dataclass +class OrchestratorCfg: + # example: 'unix://var/run/docker.sock' + DOBAS_ORCHESTRATOR_DOCKER_HOST: str = None + DOBAS_ENABLE_DEFAULT: bool = None + DOBAS_WORKER_SCHEDULING: str = None + DOBAS_WORKER_STORAGE_HOST: str = None diff --git a/dobas/subject.py b/dobas/subject.py new file mode 100644 index 0000000..1e8499c --- /dev/null +++ b/dobas/subject.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass +from .worker_config import WorkerConfig +import docker + + +class Subject: + worker_config: WorkerConfig = None + volumes: [docker.models.volumes.Volume] = None + # TODO: implement services / docker-compose folder support in Subject + services: None = None diff --git a/dobas/worker.py b/dobas/worker.py index 1ef58c3..f8ec94e 100644 --- a/dobas/worker.py +++ b/dobas/worker.py @@ -3,28 +3,30 @@ import sys import subprocess from pathlib import PurePath -import dobas +from . import logger +from .subject import Subject class Worker(object): def __init__(self, client: docker.DockerClient): self.client = client - dobas.logger.debug(f"new worker {client}".format( + logger.debug(f"new worker {client}".format( client=self.client)) - def process(self, strategy: dobas.Strategy = None): - dobas.logger.debug("processing strategy {strategy}") - for element in strategy: - dobas.logger.info("processing {element}") - if type(element) is docker.models.volumes.Volume: - self.volume_backup_handler(volume=element) - else: - dobas.logger.warning( - "no handler for element {element}, skipping ...") + def process(self, subjects: [Subject] = None): + for subject in subjects: + logger.info(f"processing {subject}") + for volume in subject.volumes: + if type(subject) is docker.models.volumes.Volume: + self.volume_backup_handler(volume=volume) + else: + logger.warning( + f"no handler for volume {volume}, skipping ...") + # TODO: add functionality to capture subject completely def prune(self): - dobas.logger.info("prune") - dobas.logger.error("not implemented") + logger.info("prune") + logger.error("not implemented") # ans = subprocess.run(["borg", "prune", "-v", "--list"] + # self.cfg.borg_prune_rules.split(" ") + ["::"]) diff --git a/dobas/worker_config.py b/dobas/worker_config.py new file mode 100644 index 0000000..385eebc --- /dev/null +++ b/dobas/worker_config.py @@ -0,0 +1,6 @@ +from dataclasses import dataclass + + +@dataclass +class WorkerConfig: + pass diff --git a/entry.py b/entry.py deleted file mode 100755 index 685e6e5..0000000 --- a/entry.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python3 -"""Docker Backup Service -Usage: - dobas.py backup [options] - dobas.py restore [options] - dobas.py (-h | --help) - dobas.py --version - -Options: - -c , --config Borg configuration file [default: x] - -d, --dry Make dry run. Don't change anything. - --host Docker host [default: unix://var/run/docker.sock] - -v, --verbose Increase verbosity - -h, --help Show this screen. - --version Show version. -""" -from docopt import docopt - -import dobas -from logging import DEBUG - -if __name__ == '__main__': - arguments = docopt(__doc__) - - if(arguments['--verbose']): - dobas.logger.setLevel(DEBUG) - - dobas.logger.debug(arguments) - - if arguments.get('backup'): - print("do_backup()") diff --git a/main.py b/main.py new file mode 100755 index 0000000..20e1e0a --- /dev/null +++ b/main.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Docker Backup Service +Usage: + main.py backup [options] + main.py restore [options] + main.py (-h | --help) + main.py --version + +Options: + -c , --config Borg configuration file [default: x] + -d, --dry Make dry run. Don't change anything. + -v, --verbose Increase verbosity + -h, --help Show this screen. + --version Show version. + +We expect global docker client configuration via DOCKER_HOST environment variable, containing the docker host socket address to connect the client with. +""" +from docopt import docopt + +from dobas.orchestrator import Orchestrator, orchestrator_config_from_env +from dobas import logger +from logging import DEBUG + + +if __name__ == '__main__': + arguments = docopt(__doc__) + + if(arguments['--verbose']): + logger.setLevel(DEBUG) + + logger.debug(arguments) + + orchestrator_cfg = orchestrator_config_from_env() + # --> we need to discover the orchestrator config without usage of docker here, + # since we do not garantie to run the orchestrator daemon on the same dockerhost, where the global docker client ist connected to + # (where the volumes and services to backup are living). This is already the case in the tests, since the orchestrator container runs outside of the dind, + # where the services and volumes to backup are living. + # orchestrator behaviour, NOT container specific i.e.: + # - dobas.enable.default = (true | false) + # - dobas.worker.scheduling = "cron string" + # ... + # - dobas.worker.storage.host = xxx + # - default.values (pre.backup.action, post.backup.action) + + orchestrator = Orchestrator(cfg=orchestrator_cfg) + + orchestrator.start_daemon() # TODO: check python http server (must be stoppable) -- GitLab From 59247ce12d831d51bba9db1a473c8eefbbbfa095 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Sat, 7 Nov 2020 23:42:26 +0100 Subject: [PATCH 04/26] first implementation for orchestrator config and config cli interface --- .vscode/launch.json | 3 ++- dobas/orchestrator.py | 22 ++++++++++------------ dobas/orchestrator_config.py | 25 +++++++++++++++++++------ main.py | 30 +++++++++++++++++++++++------- 4 files changed, 54 insertions(+), 26 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 5a9b2e5..896bb6e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,7 +10,8 @@ "request": "launch", "program": "${file}", "args": [ - "backup", + "--orchestrator.mode.restore", + "--orchestrator.docker.host=${env:DOCKER_HOST}", "--verbose" ], "console": "integratedTerminal" diff --git a/dobas/orchestrator.py b/dobas/orchestrator.py index ad5390b..bc05ea4 100644 --- a/dobas/orchestrator.py +++ b/dobas/orchestrator.py @@ -32,15 +32,6 @@ def find_enabled_container_volumes(client: docker.client.DockerClient = docker.f return [] -def orchestrator_config_from_env() -> OrchestratorCfg: - return OrchestratorCfg( - DOBAS_ORCHESTRATOR_DOCKER_HOST=os.getenv('DOCKER_HOST'), - DOBAS_ENABLE_DEFAULT=os.getenv('DOBAS_ENABLE_DEFAULT'), - DOBAS_WORKER_SCHEDULING=os.getenv('DOBAS_WORKER_SCHEDULING'), - DOBAS_WORKER_STORAGE_HOST=os.getenv('DOBAS_WORKER_STORAGE_HOST') - ) - - def discover(client: docker.client.DockerClient = docker.from_env()) -> [Subject]: logger.debug("discover subjects") @@ -63,13 +54,20 @@ class Orchestrator: logger.info(f"create orchestrator {cfg}") self.docker_client = docker.DockerClient( - base_url=cfg.DOBAS_ORCHESTRATOR_DOCKER_HOST) + base_url=self.cfg.orchestrator_docker_host) discover(client=self.docker_client) - def start_daemon(self): - logger.error("start_daemon not implemented") + def run(self): + if(self.cfg.orchestrator_daemon): + self.__start_daemon() + def __start_daemon(self): + if(not self.cfg.orchestrator_mode_backup): + logger.error( + "Daemon requested without backup mode. Exiting daemon now.") + return + logger.error("backup daemon is not yet implemented.") # def do_backup(self, strategy: Strategy, *args, **kwargs): # logger.info(__name__) # # worker = Worker(client=docker.from_env(), cfg=dobas.cfg) diff --git a/dobas/orchestrator_config.py b/dobas/orchestrator_config.py index 4dcc214..5bb1540 100644 --- a/dobas/orchestrator_config.py +++ b/dobas/orchestrator_config.py @@ -1,10 +1,23 @@ from dataclasses import dataclass -@dataclass +@dataclass(frozen=True) class OrchestratorCfg: - # example: 'unix://var/run/docker.sock' - DOBAS_ORCHESTRATOR_DOCKER_HOST: str = None - DOBAS_ENABLE_DEFAULT: bool = None - DOBAS_WORKER_SCHEDULING: str = None - DOBAS_WORKER_STORAGE_HOST: str = None + """ static configuration generated at startup + + There are some szenarios, where the orchestrator is not able to query its own config from the references docker-host. This is the case for the test setup, where the orchestrator lives besides the dind, where the orchestrators docker-client is connected to. So the docker-client is not able to see it's own orchestrator container. + """ + orchestrator_docker: bool = True + orchestrator_docker_host: str = 'unix://var/run/docker.sock' + orchestrator_docker_volumes_enabledByDefault: bool = False + orchestrator_docker_services_enabledByDefault: bool = False + + orchestrator_daemon: bool = True + orchestrator_daemon_scheduling: str = "0 3 * * *" + orchestrator_mode_backup: bool = False + orchestrator_mode_restore: bool = False + + worker_storage_host: str = "" + worker_storage_ssh_knownHost: str = "" + worker_storage_ssh_username: str = "" + worker_storage_ssh_sshKeypairPath: str = "" diff --git a/main.py b/main.py index 20e1e0a..7a28183 100755 --- a/main.py +++ b/main.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 """Docker Backup Service Usage: - main.py backup [options] - main.py restore [options] + main.py [options] main.py (-h | --help) main.py --version @@ -13,24 +12,41 @@ Options: -h, --help Show this screen. --version Show version. + --orchestrator.docker + --orchestrator.docker.host= URL to docker daemon hosting the services and volumes to backup. [default: ] + --orchestrator.docker.volumes.enabledByDefault + --orchestrator.docker.services.enabledByDefault + --orchestrator.daemon + --orchestrator.daemon.scheduling= + --orchestrator.mode.restore + --orchestrator.mode.backup + + --worker.storage.host= URL to ssh host, which stores the backups. [default: ] + --worker.storage.ssh.knownHost= KnownHost entry referencing the ssh host. [default: ] + --worker.storage.ssh.username= Username used to authenticate against ssh host. [default: ] + --worker.storage.ssh.sshKeypairPath= Path to ssh key pair. [default: ] + We expect global docker client configuration via DOCKER_HOST environment variable, containing the docker host socket address to connect the client with. """ from docopt import docopt -from dobas.orchestrator import Orchestrator, orchestrator_config_from_env +from dobas.orchestrator import Orchestrator from dobas import logger from logging import DEBUG - +from dobas.orchestrator_config import OrchestratorCfg if __name__ == '__main__': arguments = docopt(__doc__) + orchestrator_config = OrchestratorCfg(**{ + key.strip("--").replace(".", "_"): value for key, value in arguments.items() if + key.startswith("--worker.") or key.startswith("--orchestrator.") + }) if(arguments['--verbose']): logger.setLevel(DEBUG) logger.debug(arguments) - orchestrator_cfg = orchestrator_config_from_env() # --> we need to discover the orchestrator config without usage of docker here, # since we do not garantie to run the orchestrator daemon on the same dockerhost, where the global docker client ist connected to # (where the volumes and services to backup are living). This is already the case in the tests, since the orchestrator container runs outside of the dind, @@ -42,6 +58,6 @@ if __name__ == '__main__': # - dobas.worker.storage.host = xxx # - default.values (pre.backup.action, post.backup.action) - orchestrator = Orchestrator(cfg=orchestrator_cfg) + orchestrator = Orchestrator(cfg=orchestrator_config) - orchestrator.start_daemon() # TODO: check python http server (must be stoppable) + orchestrator.run() # TODO: check python http server (must be stoppable) -- GitLab From 62f046e84a610e7ef1dd7afa6b2e0a707ded3194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Tue, 10 Nov 2020 18:08:54 +0100 Subject: [PATCH 05/26] WIP: implement orchestrator find volumes --- dobas/orchestrator.py | 25 +++++++++---------------- dobas/test/docker-compose.yml | 4 ++++ 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/dobas/orchestrator.py b/dobas/orchestrator.py index bc05ea4..5ef55cf 100644 --- a/dobas/orchestrator.py +++ b/dobas/orchestrator.py @@ -15,33 +15,26 @@ def find_enabled_volumes(client: docker.client.DockerClient = docker.from_env()) """ logger.debug(f"fetch volumes fom docker client: {client}") - volumes = [volume for volume in client.volumes.list() if bool( - volume.attrs.get('Labels').get('dobas.enable'))] + # find all volumes + volumes = [docker.models.volumes.Volume] + # volumes = client.volumes.list( + # filters={"label", "dobas.enabled=true"}) + + for container in client.containers.list(all=True): + volumes.extend( + [mount for mount in container.attrs.get('Mounts')]) logger.debug(f"found volumes: {volumes}") return volumes -def find_enabled_container_volumes(client: docker.client.DockerClient = docker.from_env()) -> [docker.models.volumes.Volume]: - """ - - return list of volumes - """ - logger.error("find_enabled_container_volumes not implemented") - return [] - - def discover(client: docker.client.DockerClient = docker.from_env()) -> [Subject]: logger.debug("discover subjects") - volumes = find_enabled_volumes(client=client) - container_volumes = find_enabled_container_volumes(client=client) - # TODO: get a better idea of grouping volumes and services # Subject(volumes=[volume], worker_config=None, services=None) - return [Subject(worker_config=None, volumes=[volume], services=None) for volume in volumes].extend( - [Subject(worker_config=None, volumes=[volume], services=None) for volume in container_volumes]) + return [Subject(worker_config=None, volumes=[volume], services=None) for volume in find_enabled_volumes(client=client)] class Orchestrator: diff --git a/dobas/test/docker-compose.yml b/dobas/test/docker-compose.yml index 38c530f..9fe4a22 100644 --- a/dobas/test/docker-compose.yml +++ b/dobas/test/docker-compose.yml @@ -12,7 +12,11 @@ services: volumes: named-volume: + labels: + dobas.enabled: true named-volume-bind: + labels: + dobas.enabled: true driver_opts: type: none device: / -- GitLab From db36230d4993857400d5d2dded7cb6d58cc742c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Tue, 10 Nov 2020 19:17:21 +0100 Subject: [PATCH 06/26] implement orchestrator named volume discovery --- dobas/orchestrator.py | 54 ++++++++++++++--------------------- dobas/subject.py | 1 + dobas/test/docker-compose.yml | 3 ++ 3 files changed, 26 insertions(+), 32 deletions(-) diff --git a/dobas/orchestrator.py b/dobas/orchestrator.py index 5ef55cf..b22b3b1 100644 --- a/dobas/orchestrator.py +++ b/dobas/orchestrator.py @@ -6,37 +6,6 @@ from .subject import Subject from .orchestrator_config import OrchestratorCfg -def find_enabled_volumes(client: docker.client.DockerClient = docker.from_env()) -> [docker.models.volumes.Volume]: - """ get volumes to be handled - - list docker volumes labled with "dobas.enable: true" - - return list of volumes - """ - logger.debug(f"fetch volumes fom docker client: {client}") - - # find all volumes - volumes = [docker.models.volumes.Volume] - # volumes = client.volumes.list( - # filters={"label", "dobas.enabled=true"}) - - for container in client.containers.list(all=True): - volumes.extend( - [mount for mount in container.attrs.get('Mounts')]) - - logger.debug(f"found volumes: {volumes}") - - return volumes - - -def discover(client: docker.client.DockerClient = docker.from_env()) -> [Subject]: - logger.debug("discover subjects") - - # TODO: get a better idea of grouping volumes and services - # Subject(volumes=[volume], worker_config=None, services=None) - return [Subject(worker_config=None, volumes=[volume], services=None) for volume in find_enabled_volumes(client=client)] - - class Orchestrator: """ """ @@ -49,7 +18,28 @@ class Orchestrator: self.docker_client = docker.DockerClient( base_url=self.cfg.orchestrator_docker_host) - discover(client=self.docker_client) + self._subjects = self._discover() + logger.info(f"created subjects {self._subjects}") + + def _discover(self) -> [Subject]: + logger.debug("discover subjects") + + # TODO: discover and handle also mounts without volumes + volume_label_filter = [] + if not self.cfg.orchestrator_docker_volumes_enabledByDefault: + volume_label_filter.append("dobas.enabled=True") + volume_filter = {"label": volume_label_filter} + + volumes = self.docker_client.volumes.list(filters=volume_filter) + + # remove every explicit disabled volume + for volume in volumes: + if ("dobas.disabled" in volume.attrs["Labels"]) and (volume.attrs["Labels"]["dobas.disabled"] == True): + volumes.remove(volume) + + # TODO: bundle services and volumes together + return [Subject(worker_config=None, volumes=[volume], services=None) + for volume in volumes] def run(self): if(self.cfg.orchestrator_daemon): diff --git a/dobas/subject.py b/dobas/subject.py index 1e8499c..faaea3a 100644 --- a/dobas/subject.py +++ b/dobas/subject.py @@ -3,6 +3,7 @@ from .worker_config import WorkerConfig import docker +@dataclass class Subject: worker_config: WorkerConfig = None volumes: [docker.models.volumes.Volume] = None diff --git a/dobas/test/docker-compose.yml b/dobas/test/docker-compose.yml index 9fe4a22..c29a806 100644 --- a/dobas/test/docker-compose.yml +++ b/dobas/test/docker-compose.yml @@ -11,6 +11,9 @@ services: - named-volume-bind:/tmp/named-volume-bind volumes: + named-volume-disabled: + labels: + dobas.disabled: true named-volume: labels: dobas.enabled: true -- GitLab From 39c7bf20b1f7122ba99bfbee72bce4ba9913267a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Wed, 11 Nov 2020 00:10:05 +0100 Subject: [PATCH 07/26] add first set of volume config discovery tests --- .devcontainer/devcontainer.json | 2 +- .gitlab-ci.yml | 4 +- dobas/orchestrator.py | 10 ++-- dobas/orchestrator_config.py | 3 +- dobas/test/__init__.py | 0 ...se-disable-named-volume-disabled-label.yml | 6 ++ ...compose-enable-named-volume-bind-label.yml | 6 ++ ...cker-compose-enable-named-volume-label.yml | 6 ++ .../docker-compose-enable-service-label.yml | 6 ++ ...pose.yml => docker-compose-monolythic.yml} | 8 +-- dobas/test/docker-compose-no-labels.yml | 18 ++++++ dobas/test/helper.py | 43 ++++++++++++++ dobas/test/test_case_base.py | 24 -------- dobas/test/test_content_provider.py | 18 ++++-- dobas/test/test_orchestrator.py | 56 +++++++++++++++++++ 15 files changed, 170 insertions(+), 40 deletions(-) delete mode 100644 dobas/test/__init__.py create mode 100644 dobas/test/docker-compose-disable-named-volume-disabled-label.yml create mode 100644 dobas/test/docker-compose-enable-named-volume-bind-label.yml create mode 100644 dobas/test/docker-compose-enable-named-volume-label.yml create mode 100644 dobas/test/docker-compose-enable-service-label.yml rename dobas/test/{docker-compose.yml => docker-compose-monolythic.yml} (77%) create mode 100644 dobas/test/docker-compose-no-labels.yml create mode 100644 dobas/test/helper.py delete mode 100644 dobas/test/test_case_base.py create mode 100644 dobas/test/test_orchestrator.py diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 590e892..7016c5e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -34,6 +34,6 @@ "ms-azuretools.vscode-docker", "mhutchie.git-graph", "timonwong.shellcheck", - "ms-python.python" + "ms-python.vscode-pylance" ] } \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 24cd1f4..f75aedf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,7 +58,7 @@ test-dev-environment: docker-compose up -d --build --remove-orphans dind storage sleep 5 docker-compose run -T orchestrator /bin/bash -c "./init-backup.sh" - docker-compose run -T orchestrator /bin/bash -c "python3 -m unittest discover -s ./dobas/test" + docker-compose run -T orchestrator /bin/bash -c "python3 -m unittest discover -s ./dobas/test -p test*.py" echo "check that source-mount into environment is working in expected path" cd "$CI_PROJECT_DIR" extends: @@ -86,6 +86,6 @@ test-orchestrator-build: docker-compose build orchestrator sleep 5 docker-compose run -T orchestrator /bin/bash -c "./init-backup.sh" - docker-compose run -T orchestrator /bin/bash -c "python3 -m unittest discover -s ./dobas/test" + docker-compose run -T orchestrator /bin/bash -c "python3 -m unittest discover -s ./dobas/test -p test*.py" extends: - .test-rules diff --git a/dobas/orchestrator.py b/dobas/orchestrator.py index b22b3b1..bf6ff56 100644 --- a/dobas/orchestrator.py +++ b/dobas/orchestrator.py @@ -1,3 +1,4 @@ +from typing import List import docker import os @@ -21,21 +22,22 @@ class Orchestrator: self._subjects = self._discover() logger.info(f"created subjects {self._subjects}") - def _discover(self) -> [Subject]: + def _discover(self) -> List[Subject]: logger.debug("discover subjects") # TODO: discover and handle also mounts without volumes volume_label_filter = [] if not self.cfg.orchestrator_docker_volumes_enabledByDefault: - volume_label_filter.append("dobas.enabled=True") + volume_label_filter.append("dobas.enable=True") volume_filter = {"label": volume_label_filter} volumes = self.docker_client.volumes.list(filters=volume_filter) # remove every explicit disabled volume for volume in volumes: - if ("dobas.disabled" in volume.attrs["Labels"]) and (volume.attrs["Labels"]["dobas.disabled"] == True): - volumes.remove(volume) + if ("dobas.disable" in volume.attrs.get("Labels")): + if (volume.attrs.get("Labels")["dobas.disable"] == 'True'): + volumes.remove(volume) # TODO: bundle services and volumes together return [Subject(worker_config=None, volumes=[volume], services=None) diff --git a/dobas/orchestrator_config.py b/dobas/orchestrator_config.py index 5bb1540..7a9321f 100644 --- a/dobas/orchestrator_config.py +++ b/dobas/orchestrator_config.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +import os @dataclass(frozen=True) @@ -8,7 +9,7 @@ class OrchestratorCfg: There are some szenarios, where the orchestrator is not able to query its own config from the references docker-host. This is the case for the test setup, where the orchestrator lives besides the dind, where the orchestrators docker-client is connected to. So the docker-client is not able to see it's own orchestrator container. """ orchestrator_docker: bool = True - orchestrator_docker_host: str = 'unix://var/run/docker.sock' + orchestrator_docker_host: str = os.environ['DOCKER_HOST'] orchestrator_docker_volumes_enabledByDefault: bool = False orchestrator_docker_services_enabledByDefault: bool = False diff --git a/dobas/test/__init__.py b/dobas/test/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dobas/test/docker-compose-disable-named-volume-disabled-label.yml b/dobas/test/docker-compose-disable-named-volume-disabled-label.yml new file mode 100644 index 0000000..499a890 --- /dev/null +++ b/dobas/test/docker-compose-disable-named-volume-disabled-label.yml @@ -0,0 +1,6 @@ +version: '3' + +volumes: + named-volume-disabled: + labels: + dobas.disable: true diff --git a/dobas/test/docker-compose-enable-named-volume-bind-label.yml b/dobas/test/docker-compose-enable-named-volume-bind-label.yml new file mode 100644 index 0000000..600a689 --- /dev/null +++ b/dobas/test/docker-compose-enable-named-volume-bind-label.yml @@ -0,0 +1,6 @@ +version: '3' + +volumes: + named-volume-bind: + labels: + dobas.enable: true diff --git a/dobas/test/docker-compose-enable-named-volume-label.yml b/dobas/test/docker-compose-enable-named-volume-label.yml new file mode 100644 index 0000000..970e0c7 --- /dev/null +++ b/dobas/test/docker-compose-enable-named-volume-label.yml @@ -0,0 +1,6 @@ +version: '3' + +volumes: + named-volume: + labels: + dobas.enable: true diff --git a/dobas/test/docker-compose-enable-service-label.yml b/dobas/test/docker-compose-enable-service-label.yml new file mode 100644 index 0000000..143f778 --- /dev/null +++ b/dobas/test/docker-compose-enable-service-label.yml @@ -0,0 +1,6 @@ +version: '3' + +services: + service: + labels: + dobas.enable: true diff --git a/dobas/test/docker-compose.yml b/dobas/test/docker-compose-monolythic.yml similarity index 77% rename from dobas/test/docker-compose.yml rename to dobas/test/docker-compose-monolythic.yml index c29a806..b2140b4 100644 --- a/dobas/test/docker-compose.yml +++ b/dobas/test/docker-compose-monolythic.yml @@ -4,7 +4,7 @@ services: service: image: alpine labels: - dobas.enabled: true + dobas.enable: true volumes: - ./:/tmp/src - named-volume:/tmp/named-volume @@ -13,13 +13,13 @@ services: volumes: named-volume-disabled: labels: - dobas.disabled: true + dobas.disable: true named-volume: labels: - dobas.enabled: true + dobas.enable: true named-volume-bind: labels: - dobas.enabled: true + dobas.enable: true driver_opts: type: none device: / diff --git a/dobas/test/docker-compose-no-labels.yml b/dobas/test/docker-compose-no-labels.yml new file mode 100644 index 0000000..666dba3 --- /dev/null +++ b/dobas/test/docker-compose-no-labels.yml @@ -0,0 +1,18 @@ +version: '3' + +services: + service: + image: alpine + volumes: + - ./:/tmp/src + - named-volume:/tmp/named-volume + - named-volume-bind:/tmp/named-volume-bind + +volumes: + named-volume-disabled: + named-volume: + named-volume-bind: + driver_opts: + type: none + device: / + o: bind diff --git a/dobas/test/helper.py b/dobas/test/helper.py new file mode 100644 index 0000000..526130c --- /dev/null +++ b/dobas/test/helper.py @@ -0,0 +1,43 @@ +import subprocess +import os +from typing import List + + +def _run_cli_command(args: List[str]): + print(args) + process = subprocess.Popen(args=args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + process.communicate() + # assert(process.returncode == 0) + + +def compose_up(docker_compose_file_names: List[str]): + file_list_packed = [['-f', os.path.join('dobas/test/', name)] + for name in docker_compose_file_names] + file_args = [] + for file in file_list_packed: + for parameter in file: + file_args.append(parameter) + + _run_cli_command(['docker-compose', + *file_args, + 'up', '-d']) + + +def compose_down(docker_compose_file_names: List[str]): + _run_cli_command(['docker-compose', '-f', + *[os.path.join('dobas/test/', name) + for name in docker_compose_file_names], + 'down', '--volumes']) + + +def docker_container_and_volume_prune(): + _run_cli_command(['/bin/bash', '-c', 'docker rm -f $(docker ps -a -q)']) + _run_cli_command( + ['/bin/bash', '-c', 'docker volume rm $(docker volume ls -q)']) + + +def docker_system_prune(): + _run_cli_command(['docker', 'system', 'prune', + '--all', '--volumes', '--force']) diff --git a/dobas/test/test_case_base.py b/dobas/test/test_case_base.py deleted file mode 100644 index 78285d8..0000000 --- a/dobas/test/test_case_base.py +++ /dev/null @@ -1,24 +0,0 @@ -import docker -import subprocess - -from unittest import TestCase - - -class ContentProviderBase(TestCase): - def setUp(self): - self.docker_client = docker.from_env() - # create example set of service volume combinations - process = subprocess.Popen(args=['docker-compose', '-f', 'dobas/test/docker-compose.yml', 'up', '-d'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - self.assertEqual(process.returncode, 0, - msg="process failed with stdout {} and stderr {}".format(stdout, stderr)) - - def tearDown(self): - process = subprocess.Popen(args=['docker-compose', '-f', 'dobas/test/docker-compose.yml', 'down', '--volumes'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - self.assertEqual(process.returncode, 0, - msg="process failed with stdout {} and stderr {}".format(stdout, stderr)) diff --git a/dobas/test/test_content_provider.py b/dobas/test/test_content_provider.py index f5e2844..848faab 100644 --- a/dobas/test/test_content_provider.py +++ b/dobas/test/test_content_provider.py @@ -1,13 +1,23 @@ import os +from unittest import TestCase +import docker -from test_case_base import ContentProviderBase +from helper import compose_up, compose_down -class TestContentProvider(ContentProviderBase): +class TestContentProvider(TestCase): + + def setUp(self): + compose_up(["docker-compose-monolythic.yml"]) + + def tearDown(self): + compose_down(["docker-compose-monolythic.yml"]) + def test_startContainerWithExternalVolumesToBackup(self): + docker_client = docker.from_env() # find volumes to backup volumes_to_backup = [] - for container in self.docker_client.containers.list(all=True): + for container in docker_client.containers.list(all=True): volumes_to_backup.extend( [mount for mount in container.attrs.get('Mounts')]) @@ -32,7 +42,7 @@ class TestContentProvider(ContentProviderBase): # The key is either the host path or a volume name, and the value is a dictionary with the keys: # - bind The path to mount the volume inside the container # - mode Either rw to mount the volume read/write, or ro to mount it read-only. - backup_folder_listing = self.docker_client.containers.run( + backup_folder_listing = docker_client.containers.run( image='alpine', command='bin/sh -c "ls -1 /backup/"', remove=True, diff --git a/dobas/test/test_orchestrator.py b/dobas/test/test_orchestrator.py new file mode 100644 index 0000000..966b21f --- /dev/null +++ b/dobas/test/test_orchestrator.py @@ -0,0 +1,56 @@ +import os +from unittest import TestCase + +from helper import compose_up, compose_down, docker_container_and_volume_prune +from dobas.orchestrator import Orchestrator +from dobas.orchestrator_config import OrchestratorCfg + + +class TestOrchestrator(TestCase): + + def tearDown(self): + docker_container_and_volume_prune() + + def test_discovery_default_behavior_without_labels(self): + compose_up(["docker-compose-no-labels.yml"]) + self.assertEqual(len(Orchestrator(OrchestratorCfg( + orchestrator_docker_volumes_enabledByDefault=False, + ))._discover()), 0) + + self.assertEqual(len(Orchestrator(OrchestratorCfg( + orchestrator_docker_volumes_enabledByDefault=True, + ))._discover()), 3) + + def test_discovery_with_labels(self): + compose_up(["docker-compose-no-labels.yml", + "docker-compose-disable-named-volume-disabled-label.yml"]) + self.assertEqual(len(Orchestrator(OrchestratorCfg( + orchestrator_docker_volumes_enabledByDefault=False, + ))._discover()), 0) + self.assertEqual(len(Orchestrator(OrchestratorCfg( + orchestrator_docker_volumes_enabledByDefault=True, + ))._discover()), 2) + docker_container_and_volume_prune() + + compose_up(["docker-compose-no-labels.yml", + "docker-compose-enable-named-volume-label.yml", + "docker-compose-disable-named-volume-disabled-label.yml"]) + self.assertEqual(len(Orchestrator(OrchestratorCfg( + orchestrator_docker_volumes_enabledByDefault=False, + ))._discover()), 1) + self.assertEqual(len(Orchestrator(OrchestratorCfg( + orchestrator_docker_volumes_enabledByDefault=True, + ))._discover()), 2) + docker_container_and_volume_prune() + + compose_up(["docker-compose-no-labels.yml", + "docker-compose-enable-named-volume-label.yml", + "docker-compose-enable-named-volume-bind-label.yml", + "docker-compose-disable-named-volume-disabled-label.yml"]) + self.assertEqual(len(Orchestrator(OrchestratorCfg( + orchestrator_docker_volumes_enabledByDefault=False, + ))._discover()), 2) + self.assertEqual(len(Orchestrator(OrchestratorCfg( + orchestrator_docker_volumes_enabledByDefault=True, + ))._discover()), 2) + docker_container_and_volume_prune() -- GitLab From 304f9a6eb6f610d2e8c6ce6a7561aab389f09a98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Fri, 13 Nov 2020 10:36:46 +0100 Subject: [PATCH 08/26] introduce verbosity levels on main interface --- main.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 7a28183..50e1c24 100755 --- a/main.py +++ b/main.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Docker Backup Service Usage: - main.py [options] + main.py [-v ...] [options] main.py (-h | --help) main.py --version @@ -32,7 +32,7 @@ from docopt import docopt from dobas.orchestrator import Orchestrator from dobas import logger -from logging import DEBUG +from logging import DEBUG, INFO, WARN from dobas.orchestrator_config import OrchestratorCfg if __name__ == '__main__': @@ -42,8 +42,12 @@ if __name__ == '__main__': key.startswith("--worker.") or key.startswith("--orchestrator.") }) - if(arguments['--verbose']): + if arguments['--verbose'] == 0: + logger.setLevel(WARN) + elif arguments['--verbose'] == 1: logger.setLevel(DEBUG) + elif arguments['--verbose'] == 2: + logger.setLevel(INFO) logger.debug(arguments) -- GitLab From b7d6873f62e5a370850bf8015b8a37c85ef09dc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Fri, 13 Nov 2020 13:11:53 +0100 Subject: [PATCH 09/26] reorganize orchestrator and worker config --- dobas/orchestrator.py | 21 ++++++---- dobas/orchestrator_config.py | 22 +++++----- dobas/subject.py | 3 +- dobas/test/test_orchestrator.py | 16 +++---- dobas/worker.py | 32 ++++---------- dobas/worker_config.py | 12 +++++- main.py | 74 ++++++++++++++++++++++----------- 7 files changed, 100 insertions(+), 80 deletions(-) diff --git a/dobas/orchestrator.py b/dobas/orchestrator.py index bf6ff56..8a2c724 100644 --- a/dobas/orchestrator.py +++ b/dobas/orchestrator.py @@ -11,13 +11,16 @@ class Orchestrator: """ """ - def __init__(self, cfg: OrchestratorCfg): - self.cfg = cfg + def __init__(self, config: OrchestratorCfg): + self.__config: OrchestratorCfg = config logger.debug("implement typechecked cfg") - logger.info(f"create orchestrator {cfg}") + logger.info(f"create orchestrator {config}") - self.docker_client = docker.DockerClient( - base_url=self.cfg.orchestrator_docker_host) + if(self.__config.docker_host): + self.docker_client = docker.DockerClient( + base_url=self.__config.docker_host) + else: + self.docker_client = docker.DockerClient.from_env() self._subjects = self._discover() logger.info(f"created subjects {self._subjects}") @@ -27,7 +30,7 @@ class Orchestrator: # TODO: discover and handle also mounts without volumes volume_label_filter = [] - if not self.cfg.orchestrator_docker_volumes_enabledByDefault: + if not self.__config.docker_volumes_enabledByDefault: volume_label_filter.append("dobas.enable=True") volume_filter = {"label": volume_label_filter} @@ -40,15 +43,15 @@ class Orchestrator: volumes.remove(volume) # TODO: bundle services and volumes together - return [Subject(worker_config=None, volumes=[volume], services=None) + return [Subject(worker_config=self.__config.default_worker_config, volumes=[volume], services=None) for volume in volumes] def run(self): - if(self.cfg.orchestrator_daemon): + if(self.__config.orchestrator_daemon): self.__start_daemon() def __start_daemon(self): - if(not self.cfg.orchestrator_mode_backup): + if(not self.__config.orchestrator_mode_backup): logger.error( "Daemon requested without backup mode. Exiting daemon now.") return diff --git a/dobas/orchestrator_config.py b/dobas/orchestrator_config.py index 7a9321f..82488b5 100644 --- a/dobas/orchestrator_config.py +++ b/dobas/orchestrator_config.py @@ -1,5 +1,6 @@ from dataclasses import dataclass import os +from .worker_config import WorkerConfig @dataclass(frozen=True) @@ -8,17 +9,14 @@ class OrchestratorCfg: There are some szenarios, where the orchestrator is not able to query its own config from the references docker-host. This is the case for the test setup, where the orchestrator lives besides the dind, where the orchestrators docker-client is connected to. So the docker-client is not able to see it's own orchestrator container. """ - orchestrator_docker: bool = True - orchestrator_docker_host: str = os.environ['DOCKER_HOST'] - orchestrator_docker_volumes_enabledByDefault: bool = False - orchestrator_docker_services_enabledByDefault: bool = False + docker: bool = True + docker_host: str = os.environ['DOCKER_HOST'] + docker_volumes_enabledByDefault: bool = False + docker_services_enabledByDefault: bool = False - orchestrator_daemon: bool = True - orchestrator_daemon_scheduling: str = "0 3 * * *" - orchestrator_mode_backup: bool = False - orchestrator_mode_restore: bool = False + daemon: bool = True + daemon_scheduling: str = "0 3 * * *" + mode_backup: bool = False + mode_restore: bool = False - worker_storage_host: str = "" - worker_storage_ssh_knownHost: str = "" - worker_storage_ssh_username: str = "" - worker_storage_ssh_sshKeypairPath: str = "" + default_worker_config: WorkerConfig = WorkerConfig() diff --git a/dobas/subject.py b/dobas/subject.py index faaea3a..ad719ca 100644 --- a/dobas/subject.py +++ b/dobas/subject.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from typing import List from .worker_config import WorkerConfig import docker @@ -6,6 +7,6 @@ import docker @dataclass class Subject: worker_config: WorkerConfig = None - volumes: [docker.models.volumes.Volume] = None + volumes: List[docker.models.volumes.Volume] = None # TODO: implement services / docker-compose folder support in Subject services: None = None diff --git a/dobas/test/test_orchestrator.py b/dobas/test/test_orchestrator.py index 966b21f..0eefa1f 100644 --- a/dobas/test/test_orchestrator.py +++ b/dobas/test/test_orchestrator.py @@ -14,21 +14,21 @@ class TestOrchestrator(TestCase): def test_discovery_default_behavior_without_labels(self): compose_up(["docker-compose-no-labels.yml"]) self.assertEqual(len(Orchestrator(OrchestratorCfg( - orchestrator_docker_volumes_enabledByDefault=False, + docker_volumes_enabledByDefault=False, ))._discover()), 0) self.assertEqual(len(Orchestrator(OrchestratorCfg( - orchestrator_docker_volumes_enabledByDefault=True, + docker_volumes_enabledByDefault=True, ))._discover()), 3) def test_discovery_with_labels(self): compose_up(["docker-compose-no-labels.yml", "docker-compose-disable-named-volume-disabled-label.yml"]) self.assertEqual(len(Orchestrator(OrchestratorCfg( - orchestrator_docker_volumes_enabledByDefault=False, + docker_volumes_enabledByDefault=False, ))._discover()), 0) self.assertEqual(len(Orchestrator(OrchestratorCfg( - orchestrator_docker_volumes_enabledByDefault=True, + docker_volumes_enabledByDefault=True, ))._discover()), 2) docker_container_and_volume_prune() @@ -36,10 +36,10 @@ class TestOrchestrator(TestCase): "docker-compose-enable-named-volume-label.yml", "docker-compose-disable-named-volume-disabled-label.yml"]) self.assertEqual(len(Orchestrator(OrchestratorCfg( - orchestrator_docker_volumes_enabledByDefault=False, + docker_volumes_enabledByDefault=False, ))._discover()), 1) self.assertEqual(len(Orchestrator(OrchestratorCfg( - orchestrator_docker_volumes_enabledByDefault=True, + docker_volumes_enabledByDefault=True, ))._discover()), 2) docker_container_and_volume_prune() @@ -48,9 +48,9 @@ class TestOrchestrator(TestCase): "docker-compose-enable-named-volume-bind-label.yml", "docker-compose-disable-named-volume-disabled-label.yml"]) self.assertEqual(len(Orchestrator(OrchestratorCfg( - orchestrator_docker_volumes_enabledByDefault=False, + docker_volumes_enabledByDefault=False, ))._discover()), 2) self.assertEqual(len(Orchestrator(OrchestratorCfg( - orchestrator_docker_volumes_enabledByDefault=True, + docker_volumes_enabledByDefault=True, ))._discover()), 2) docker_container_and_volume_prune() diff --git a/dobas/worker.py b/dobas/worker.py index f8ec94e..21cd689 100644 --- a/dobas/worker.py +++ b/dobas/worker.py @@ -13,28 +13,16 @@ class Worker(object): logger.debug(f"new worker {client}".format( client=self.client)) - def process(self, subjects: [Subject] = None): - for subject in subjects: - logger.info(f"processing {subject}") - for volume in subject.volumes: - if type(subject) is docker.models.volumes.Volume: - self.volume_backup_handler(volume=volume) - else: - logger.warning( - f"no handler for volume {volume}, skipping ...") + def process(self, subject: Subject = None): + logger.info(f"processing {subject}") + for volume in subject.volumes: + if type(subject) is docker.models.volumes.Volume: + self.volume_backup_handler(volume=volume) + else: + logger.warning( + f"no handler for volume {volume}, skipping ...") # TODO: add functionality to capture subject completely - def prune(self): - logger.info("prune") - logger.error("not implemented") - # ans = subprocess.run(["borg", "prune", "-v", "--list"] + - # self.cfg.borg_prune_rules.split(" ") + ["::"]) - - # if ans.returncode == 0: - # pass - # else: - # print("oje") - def volume_backup_handler(self, volume: docker.models.volumes.Volume = None): if volume: # risk of name duplicates here @@ -51,7 +39,3 @@ class Worker(object): "entity. (Exitcode: {creation.returncode})", file=sys.stderr) else: print("borg exec done") - - def simple_folder_handler(self, paths: [PurePath] = None): - if len(paths): - pass diff --git a/dobas/worker_config.py b/dobas/worker_config.py index 385eebc..acfb24c 100644 --- a/dobas/worker_config.py +++ b/dobas/worker_config.py @@ -1,6 +1,14 @@ from dataclasses import dataclass +import os -@dataclass +@dataclass(frozen=True) class WorkerConfig: - pass + # configured by orchestrator + source: os.path = "" + + # configured by labels + storage_destination: str = "" + storage_ssh_knownHost: str = "" + storage_ssh_username: str = "" + storage_ssh_sshKeypairPath: str = "" diff --git a/main.py b/main.py index 50e1c24..57b1d31 100755 --- a/main.py +++ b/main.py @@ -12,21 +12,20 @@ Options: -h, --help Show this screen. --version Show version. - --orchestrator.docker - --orchestrator.docker.host= URL to docker daemon hosting the services and volumes to backup. [default: ] - --orchestrator.docker.volumes.enabledByDefault - --orchestrator.docker.services.enabledByDefault - --orchestrator.daemon - --orchestrator.daemon.scheduling= - --orchestrator.mode.restore - --orchestrator.mode.backup - - --worker.storage.host= URL to ssh host, which stores the backups. [default: ] - --worker.storage.ssh.knownHost= KnownHost entry referencing the ssh host. [default: ] - --worker.storage.ssh.username= Username used to authenticate against ssh host. [default: ] - --worker.storage.ssh.sshKeypairPath= Path to ssh key pair. [default: ] - -We expect global docker client configuration via DOCKER_HOST environment variable, containing the docker host socket address to connect the client with. + --orchestrator.docker= [default: True] + --orchestrator.docker.host= URL to docker daemon hosting the services and volumes to backup. + If orchestrator.docker.host is empty, we use the content of DOCKER_HOST env variable. [default: ] + --orchestrator.docker.volumes.enabledByDefault= [default: False] + --orchestrator.docker.services.enabledByDefault= [default: False] + --orchestrator.daemon= [default: True] + --orchestrator.daemon.scheduling= [default: \"0 3 * * *\"] + --orchestrator.mode.restore= [default: False] + --orchestrator.mode.backup= [default: True] + + --worker.storage.destination= URL to ssh host, which stores the backups. [default: ] + --worker.storage.ssh.knownHost= KnownHost entry referencing the ssh host. [default: ] + --worker.storage.ssh.username= Username used to authenticate against ssh host. [default: ] + --worker.storage.ssh.sshKeypairPath= Path to ssh key pair. [default: ] """ from docopt import docopt @@ -34,22 +33,50 @@ from dobas.orchestrator import Orchestrator from dobas import logger from logging import DEBUG, INFO, WARN from dobas.orchestrator_config import OrchestratorCfg +from dobas.worker_config import WorkerConfig +import dataclasses + + +def __removeprefix(string: str, prefix: str) -> str: + if string.startswith(prefix): + return string[len(prefix):] + else: + return string[:] + + +def __dict_string_value_to_bool(dict_to_update): + for key, value in dict_to_update.items(): + if value == "True" or value == "true": + dict_to_update[key] = True + elif value == "False" or value == "false": + dict_to_update[key] = False + return dict_to_update + if __name__ == '__main__': arguments = docopt(__doc__) - orchestrator_config = OrchestratorCfg(**{ - key.strip("--").replace(".", "_"): value for key, value in arguments.items() if - key.startswith("--worker.") or key.startswith("--orchestrator.") - }) + + default_worker_config = WorkerConfig(**__dict_string_value_to_bool({ + __removeprefix(string=key, prefix="--worker.").replace(".", "_"): value for key, value in arguments.items() if + key.startswith("--worker.") + })) + + orchestrator_config = OrchestratorCfg( + **__dict_string_value_to_bool({ + __removeprefix(string=key, prefix="--orchestrator.").replace(".", "_"): value for key, value in arguments.items() if + key.startswith("--orchestrator.") + }), + default_worker_config=default_worker_config) if arguments['--verbose'] == 0: logger.setLevel(WARN) elif arguments['--verbose'] == 1: - logger.setLevel(DEBUG) - elif arguments['--verbose'] == 2: logger.setLevel(INFO) + elif arguments['--verbose'] >= 2: + logger.setLevel(DEBUG) logger.debug(arguments) + logger.info(orchestrator_config) # --> we need to discover the orchestrator config without usage of docker here, # since we do not garantie to run the orchestrator daemon on the same dockerhost, where the global docker client ist connected to @@ -62,6 +89,5 @@ if __name__ == '__main__': # - dobas.worker.storage.host = xxx # - default.values (pre.backup.action, post.backup.action) - orchestrator = Orchestrator(cfg=orchestrator_config) - - orchestrator.run() # TODO: check python http server (must be stoppable) + orchestrator = Orchestrator(config=orchestrator_config) + # orchestrator.run() # TODO: check python http server (must be stoppable) -- GitLab From 0b0e5829799396fa14a0c5996d7aff11863e8971 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Fri, 13 Nov 2020 15:00:00 +0100 Subject: [PATCH 10/26] initial implementation of configuratable worker backup runs --- dobas/__init__.py | 21 ++++++++++++++++++ dobas/orchestrator.py | 34 ++++++++++++++++++---------- dobas/worker.py | 50 +++++++++++++++++++++++++----------------- dobas/worker_config.py | 6 +++++ main.py | 8 +++++-- 5 files changed, 85 insertions(+), 34 deletions(-) diff --git a/dobas/__init__.py b/dobas/__init__.py index 7a1380c..266b1f1 100644 --- a/dobas/__init__.py +++ b/dobas/__init__.py @@ -1,10 +1,31 @@ import logging +from logging import log import sys __version__ = "0.1.0" logger = logging.getLogger(__name__) +# create console handler and set level to debug +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) + +# create formatter +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s", + "%Y-%m-%d %H:%M:%S") + +# add formatter to ch +ch.setFormatter(formatter) + +# add ch to logger +logger.addHandler(ch) + + +# add filehandler +# fh = logging.FileHandler("./dobas.log") +# fh.setLevel(logging.DEBUG) +# fh.setFormatter(formatter) +# logger.addHandler(fh) def configure_module_logger(): handler = logging.StreamHandler(sys.stdout) diff --git a/dobas/orchestrator.py b/dobas/orchestrator.py index 8a2c724..c373fa0 100644 --- a/dobas/orchestrator.py +++ b/dobas/orchestrator.py @@ -1,3 +1,4 @@ +from .worker import Worker from typing import List import docker import os @@ -17,10 +18,10 @@ class Orchestrator: logger.info(f"create orchestrator {config}") if(self.__config.docker_host): - self.docker_client = docker.DockerClient( + self.__docker_client = docker.DockerClient( base_url=self.__config.docker_host) else: - self.docker_client = docker.DockerClient.from_env() + self.__docker_client = docker.DockerClient.from_env() self._subjects = self._discover() logger.info(f"created subjects {self._subjects}") @@ -34,7 +35,7 @@ class Orchestrator: volume_label_filter.append("dobas.enable=True") volume_filter = {"label": volume_label_filter} - volumes = self.docker_client.volumes.list(filters=volume_filter) + volumes = self.__docker_client.volumes.list(filters=volume_filter) # remove every explicit disabled volume for volume in volumes: @@ -43,19 +44,28 @@ class Orchestrator: volumes.remove(volume) # TODO: bundle services and volumes together + # TODO: create individual worker config per subject return [Subject(worker_config=self.__config.default_worker_config, volumes=[volume], services=None) for volume in volumes] + def __processSubjects(self): + logger.debug("run process subjects") + for subject in self._subjects: + logger.debug("process subject {the_subject}".format( + the_subject=subject)) + + # TODO: target docket client? + worker = Worker(client=self.__docker_client) + worker.process(subject=subject) + def run(self): - if(self.__config.orchestrator_daemon): - self.__start_daemon() - - def __start_daemon(self): - if(not self.__config.orchestrator_mode_backup): - logger.error( - "Daemon requested without backup mode. Exiting daemon now.") - return - logger.error("backup daemon is not yet implemented.") + logger.debug("run orchestrator") + if(self.__config.daemon): + logger.error("daemon not implement yet.") + # self.__start_daemon() + else: + self.__processSubjects() + # def do_backup(self, strategy: Strategy, *args, **kwargs): # logger.info(__name__) # # worker = Worker(client=docker.from_env(), cfg=dobas.cfg) diff --git a/dobas/worker.py b/dobas/worker.py index 21cd689..e961ec2 100644 --- a/dobas/worker.py +++ b/dobas/worker.py @@ -1,3 +1,5 @@ +import os +from typing import List import docker import sys import subprocess @@ -5,37 +7,45 @@ from pathlib import PurePath from . import logger from .subject import Subject +from .worker_config import WorkerConfig class Worker(object): def __init__(self, client: docker.DockerClient): - self.client = client + self.__docker_client = client logger.debug(f"new worker {client}".format( - client=self.client)) + client=self.__docker_client)) - def process(self, subject: Subject = None): + def process(self, subject: Subject): logger.info(f"processing {subject}") + remapped_volumes = {} for volume in subject.volumes: - if type(subject) is docker.models.volumes.Volume: - self.volume_backup_handler(volume=volume) + if type(volume) is docker.models.volumes.Volume: + remapped_volumes[volume.name] = { + "bind": os.path.join(subject.worker_config.docker_sourceMountBasePath.strip('"\''), + str(volume.id)), + "mode": "ro" + } + # self.volume_backup_handler(volume=volume) else: logger.warning( f"no handler for volume {volume}, skipping ...") # TODO: add functionality to capture subject completely - def volume_backup_handler(self, volume: docker.models.volumes.Volume = None): - if volume: - # risk of name duplicates here - mounted_volume = {f"{volume.id}": { - 'bind': f"/mnt/{volume.id}", 'mode': 'ro'}} + # volumes: + # A dictionary to configure volumes mounted inside the container. + # The key is either the host path or a volume name, and the value is a dictionary with the keys: + # - bind The path to mount the volume inside the container + # - mode Either rw to mount the volume read/write, or ro to mount it read-only. + backup_folder_listing = self.__docker_client.containers.run( + image=subject.worker_config.docker_image, + entrypoint=subject.worker_config.docker_entrypoint, + command=subject.worker_config.docker_command, + working_dir=subject.worker_config.docker_workdir.strip("'\""), + volumes=remapped_volumes, + detach=False, + remove=True, + ) - self.client.containers.run("alpine", command="ls -la /mnt", - volumes=mounted_volume) - - ans = subprocess.run(["borg", "create", "--stats", - "::{now}"] + volume.get('bind')) - if ans.returncode > 0: - print(f"NO BACKUP CREATED. BORG Collective failed to assimilate biological " - "entity. (Exitcode: {creation.returncode})", file=sys.stderr) - else: - print("borg exec done") + logger.debug( + f"worker container run {backup_folder_listing.decode('ascii')}") diff --git a/dobas/worker_config.py b/dobas/worker_config.py index acfb24c..f2b8ca0 100644 --- a/dobas/worker_config.py +++ b/dobas/worker_config.py @@ -12,3 +12,9 @@ class WorkerConfig: storage_ssh_knownHost: str = "" storage_ssh_username: str = "" storage_ssh_sshKeypairPath: str = "" + + docker_image: str = "" + docker_entrypoint: str = "" + docker_command: str = "" + docker_workdir: str = "" + docker_sourceMountBasePath: str = "" diff --git a/main.py b/main.py index 57b1d31..765b7cb 100755 --- a/main.py +++ b/main.py @@ -26,6 +26,11 @@ Options: --worker.storage.ssh.knownHost= KnownHost entry referencing the ssh host. [default: ] --worker.storage.ssh.username= Username used to authenticate against ssh host. [default: ] --worker.storage.ssh.sshKeypairPath= Path to ssh key pair. [default: ] + --worker.docker.image= [default: alpine:latest] + --worker.docker.entrypoint= [default: ] + --worker.docker.command= [default: \"/bin/sh -c \'ls - la /backup/\'\"] + --worker.docker.workdir= [default: \"/backup\"] + --worker.docker.sourceMountBasePath= [default: \"/backup\"] """ from docopt import docopt @@ -34,7 +39,6 @@ from dobas import logger from logging import DEBUG, INFO, WARN from dobas.orchestrator_config import OrchestratorCfg from dobas.worker_config import WorkerConfig -import dataclasses def __removeprefix(string: str, prefix: str) -> str: @@ -90,4 +94,4 @@ if __name__ == '__main__': # - default.values (pre.backup.action, post.backup.action) orchestrator = Orchestrator(config=orchestrator_config) - # orchestrator.run() # TODO: check python http server (must be stoppable) + orchestrator.run() # TODO: check python http server (must be stoppable) -- GitLab From 4f2c0a789c1443b809f78c19856465e30e6117c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Sat, 14 Nov 2020 13:37:12 +0100 Subject: [PATCH 11/26] add dobas.docker.env labels for volumes; add additional tests to ensure discovery of volume ids --- dobas/helper.py | 5 ++ dobas/orchestrator.py | 21 ++++- dobas/subject.py | 2 +- ...e-enable-named-volume-label-docker-env.yml | 8 ++ dobas/test/test_orchestrator.py | 78 ++++++++++++++++--- dobas/worker.py | 25 +++--- dobas/worker_config.py | 3 +- main.py | 12 +-- 8 files changed, 115 insertions(+), 39 deletions(-) create mode 100644 dobas/helper.py create mode 100644 dobas/test/docker-compose-enable-named-volume-label-docker-env.yml diff --git a/dobas/helper.py b/dobas/helper.py new file mode 100644 index 0000000..0a2b7bb --- /dev/null +++ b/dobas/helper.py @@ -0,0 +1,5 @@ +def removeprefix(string: str, prefix: str) -> str: + if string.startswith(prefix): + return string[len(prefix):] + else: + return string[:] diff --git a/dobas/orchestrator.py b/dobas/orchestrator.py index c373fa0..201dee5 100644 --- a/dobas/orchestrator.py +++ b/dobas/orchestrator.py @@ -2,10 +2,13 @@ from .worker import Worker from typing import List import docker import os +import dataclasses from . import logger from .subject import Subject from .orchestrator_config import OrchestratorCfg +from .worker_config import WorkerConfig +from .helper import removeprefix class Orchestrator: @@ -37,7 +40,7 @@ class Orchestrator: volumes = self.__docker_client.volumes.list(filters=volume_filter) - # remove every explicit disabled volume + # remove every explicit disabled volume with "dobas.disable" for volume in volumes: if ("dobas.disable" in volume.attrs.get("Labels")): if (volume.attrs.get("Labels")["dobas.disable"] == 'True'): @@ -45,8 +48,20 @@ class Orchestrator: # TODO: bundle services and volumes together # TODO: create individual worker config per subject - return [Subject(worker_config=self.__config.default_worker_config, volumes=[volume], services=None) - for volume in volumes] + subjects = [] + for volume in volumes: + docker_envs = {} + # gather docker.env from volume labels + for docker_env_label in [label for label in volume.attrs.get("Labels") if label.startswith("dobas.docker.env.")]: + docker_envs[removeprefix(string=docker_env_label, prefix="dobas.docker.env.")] = volume.attrs.get( + "Labels")[docker_env_label] + + worker_config = WorkerConfig(self.__config.default_worker_config) + worker_config.docker_env = docker_envs + subjects.append(Subject(worker_config=worker_config, + volume=volume, services=None)) + + return subjects def __processSubjects(self): logger.debug("run process subjects") diff --git a/dobas/subject.py b/dobas/subject.py index ad719ca..1770d50 100644 --- a/dobas/subject.py +++ b/dobas/subject.py @@ -7,6 +7,6 @@ import docker @dataclass class Subject: worker_config: WorkerConfig = None - volumes: List[docker.models.volumes.Volume] = None + volume: docker.models.volumes.Volume = None # TODO: implement services / docker-compose folder support in Subject services: None = None diff --git a/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml b/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml new file mode 100644 index 0000000..2be2d44 --- /dev/null +++ b/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml @@ -0,0 +1,8 @@ +version: '3' + +volumes: + named-volume: + labels: + dobas.enable: true + dobas.docker.env.EnvLabelKey: envLabelValue + diff --git a/dobas/test/test_orchestrator.py b/dobas/test/test_orchestrator.py index 0eefa1f..75720b3 100644 --- a/dobas/test/test_orchestrator.py +++ b/dobas/test/test_orchestrator.py @@ -8,6 +8,9 @@ from dobas.orchestrator_config import OrchestratorCfg class TestOrchestrator(TestCase): + def setUp(self): + docker_container_and_volume_prune() + def tearDown(self): docker_container_and_volume_prune() @@ -17,9 +20,15 @@ class TestOrchestrator(TestCase): docker_volumes_enabledByDefault=False, ))._discover()), 0) - self.assertEqual(len(Orchestrator(OrchestratorCfg( + subjects = Orchestrator(OrchestratorCfg( docker_volumes_enabledByDefault=True, - ))._discover()), 3) + ))._discover() + self.assertEqual(len(subjects), 3) + + volume_ids = [subject.volume.id for subject in subjects] + self.assertIn("test_named-volume-disabled", volume_ids) + self.assertIn("test_named-volume", volume_ids) + self.assertIn("test_named-volume-bind", volume_ids) def test_discovery_with_labels(self): compose_up(["docker-compose-no-labels.yml", @@ -27,30 +36,75 @@ class TestOrchestrator(TestCase): self.assertEqual(len(Orchestrator(OrchestratorCfg( docker_volumes_enabledByDefault=False, ))._discover()), 0) - self.assertEqual(len(Orchestrator(OrchestratorCfg( + + subjects = Orchestrator(OrchestratorCfg( docker_volumes_enabledByDefault=True, - ))._discover()), 2) + ))._discover() + self.assertEqual(len(subjects), 2) + volume_ids = [subject.volume.id for subject in subjects] + self.assertIn("test_named-volume", volume_ids) + self.assertIn("test_named-volume-bind", volume_ids) docker_container_and_volume_prune() compose_up(["docker-compose-no-labels.yml", "docker-compose-enable-named-volume-label.yml", "docker-compose-disable-named-volume-disabled-label.yml"]) - self.assertEqual(len(Orchestrator(OrchestratorCfg( + subjects = Orchestrator(OrchestratorCfg( docker_volumes_enabledByDefault=False, - ))._discover()), 1) - self.assertEqual(len(Orchestrator(OrchestratorCfg( + ))._discover() + self.assertEqual(len(subjects), 1) + volume_ids = [subject.volume.id for subject in subjects] + self.assertIn("test_named-volume", volume_ids) + + subjects = Orchestrator(OrchestratorCfg( docker_volumes_enabledByDefault=True, - ))._discover()), 2) + ))._discover() + self.assertEqual(len(subjects), 2) + volume_ids = [subject.volume.id for subject in subjects] + self.assertIn("test_named-volume", volume_ids) + self.assertIn("test_named-volume-bind", volume_ids) docker_container_and_volume_prune() compose_up(["docker-compose-no-labels.yml", "docker-compose-enable-named-volume-label.yml", "docker-compose-enable-named-volume-bind-label.yml", "docker-compose-disable-named-volume-disabled-label.yml"]) - self.assertEqual(len(Orchestrator(OrchestratorCfg( + subjects = Orchestrator(OrchestratorCfg( docker_volumes_enabledByDefault=False, - ))._discover()), 2) - self.assertEqual(len(Orchestrator(OrchestratorCfg( + ))._discover() + self.assertEqual(len(subjects), 2) + volume_ids = [subject.volume.id for subject in subjects] + self.assertIn("test_named-volume", volume_ids) + self.assertIn("test_named-volume-bind", volume_ids) + + subjects = Orchestrator(OrchestratorCfg( docker_volumes_enabledByDefault=True, - ))._discover()), 2) + ))._discover() + self.assertEqual(len(subjects), 2) + volume_ids = [subject.volume.id for subject in subjects] + self.assertIn("test_named-volume", volume_ids) + self.assertIn("test_named-volume-bind", volume_ids) docker_container_and_volume_prune() + + def test_discovery_label_docker_envs(self): + compose_up(["docker-compose-no-labels.yml", + "docker-compose-enable-named-volume-label-docker-env.yml"]) + + def __testSubjects(orchestrator_config): + subjects = Orchestrator(orchestrator_config)._subjects + + envs_linearized_dict = {} + for subject in subjects: + envs_linearized_dict.update(subject.worker_config.docker_env) + + # self.assertIsInstance(envs, dict) + self.assertEqual(len(envs_linearized_dict.items()), 1) + self.assertEqual( + envs_linearized_dict["EnvLabelKey"], "envLabelValue") + + __testSubjects(OrchestratorCfg( + docker_volumes_enabledByDefault=False, + )) + __testSubjects(OrchestratorCfg( + docker_volumes_enabledByDefault=True, + )) diff --git a/dobas/worker.py b/dobas/worker.py index e961ec2..2f1ed69 100644 --- a/dobas/worker.py +++ b/dobas/worker.py @@ -18,18 +18,17 @@ class Worker(object): def process(self, subject: Subject): logger.info(f"processing {subject}") - remapped_volumes = {} - for volume in subject.volumes: - if type(volume) is docker.models.volumes.Volume: - remapped_volumes[volume.name] = { - "bind": os.path.join(subject.worker_config.docker_sourceMountBasePath.strip('"\''), - str(volume.id)), - "mode": "ro" - } - # self.volume_backup_handler(volume=volume) - else: - logger.warning( - f"no handler for volume {volume}, skipping ...") + remapped_volume = {} + if type(subject.volume) is docker.models.volumes.Volume: + remapped_volume[subject.volume.name] = { + "bind": os.path.join(subject.worker_config.docker_sourceMountBasePath.strip('"\''), + str(subject.volume.id)), + "mode": "ro" + } + # self.volume_backup_handler(volume=volume) + else: + logger.warning( + f"no handler for volume {subject.volume}, skipping ...") # TODO: add functionality to capture subject completely # volumes: @@ -42,7 +41,7 @@ class Worker(object): entrypoint=subject.worker_config.docker_entrypoint, command=subject.worker_config.docker_command, working_dir=subject.worker_config.docker_workdir.strip("'\""), - volumes=remapped_volumes, + volumes=remapped_volume, detach=False, remove=True, ) diff --git a/dobas/worker_config.py b/dobas/worker_config.py index f2b8ca0..b6079df 100644 --- a/dobas/worker_config.py +++ b/dobas/worker_config.py @@ -2,7 +2,7 @@ from dataclasses import dataclass import os -@dataclass(frozen=True) +@dataclass class WorkerConfig: # configured by orchestrator source: os.path = "" @@ -18,3 +18,4 @@ class WorkerConfig: docker_command: str = "" docker_workdir: str = "" docker_sourceMountBasePath: str = "" + docker_env: {} = None diff --git a/main.py b/main.py index 765b7cb..e039c6c 100755 --- a/main.py +++ b/main.py @@ -39,13 +39,7 @@ from dobas import logger from logging import DEBUG, INFO, WARN from dobas.orchestrator_config import OrchestratorCfg from dobas.worker_config import WorkerConfig - - -def __removeprefix(string: str, prefix: str) -> str: - if string.startswith(prefix): - return string[len(prefix):] - else: - return string[:] +from dobas.helper import removeprefix def __dict_string_value_to_bool(dict_to_update): @@ -61,13 +55,13 @@ if __name__ == '__main__': arguments = docopt(__doc__) default_worker_config = WorkerConfig(**__dict_string_value_to_bool({ - __removeprefix(string=key, prefix="--worker.").replace(".", "_"): value for key, value in arguments.items() if + removeprefix(string=key, prefix="--worker.").replace(".", "_"): value for key, value in arguments.items() if key.startswith("--worker.") })) orchestrator_config = OrchestratorCfg( **__dict_string_value_to_bool({ - __removeprefix(string=key, prefix="--orchestrator.").replace(".", "_"): value for key, value in arguments.items() if + removeprefix(string=key, prefix="--orchestrator.").replace(".", "_"): value for key, value in arguments.items() if key.startswith("--orchestrator.") }), default_worker_config=default_worker_config) -- GitLab From e71035ae83497f25ea04844e542a082f699ddf49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Sat, 14 Nov 2020 14:52:51 +0100 Subject: [PATCH 12/26] add first part of worker test --- ...e-enable-named-volume-label-docker-env.yml | 3 +-- dobas/test/test_orchestrator.py | 2 +- dobas/test/test_worker.py | 25 +++++++++++++++++++ dobas/worker.py | 7 ++++-- 4 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 dobas/test/test_worker.py diff --git a/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml b/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml index 2be2d44..e45e021 100644 --- a/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml +++ b/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml @@ -4,5 +4,4 @@ volumes: named-volume: labels: dobas.enable: true - dobas.docker.env.EnvLabelKey: envLabelValue - + dobas.docker.env.EnvLabelKey: EnvLabelValue diff --git a/dobas/test/test_orchestrator.py b/dobas/test/test_orchestrator.py index 75720b3..bfd5f28 100644 --- a/dobas/test/test_orchestrator.py +++ b/dobas/test/test_orchestrator.py @@ -100,7 +100,7 @@ class TestOrchestrator(TestCase): # self.assertIsInstance(envs, dict) self.assertEqual(len(envs_linearized_dict.items()), 1) self.assertEqual( - envs_linearized_dict["EnvLabelKey"], "envLabelValue") + envs_linearized_dict["EnvLabelKey"], "EnvLabelValue") __testSubjects(OrchestratorCfg( docker_volumes_enabledByDefault=False, diff --git a/dobas/test/test_worker.py b/dobas/test/test_worker.py new file mode 100644 index 0000000..331a151 --- /dev/null +++ b/dobas/test/test_worker.py @@ -0,0 +1,25 @@ +from unittest import TestCase + +from helper import compose_up, compose_down, docker_container_and_volume_prune +from dobas.orchestrator import Orchestrator +from dobas.orchestrator_config import OrchestratorCfg +from dobas.worker import Worker +from dobas.worker_config import WorkerConfig +from dobas.subject import Subject +import docker + + +class TestWorker(TestCase): + def test_docker_env(self): + worker = Worker(docker.from_env()) + + subject = Subject( + worker_config=WorkerConfig( + docker_image="alpine", + docker_command="/bin/sh -c \"env | grep EnvLabelKey\"", + docker_env={"EnvLabelKey": "EnvLabelValue"} + ) + ) + + output = worker.process(subject) + self.assertIn("EnvLabelKey=EnvLabelValue", output.decode("ascii")) diff --git a/dobas/worker.py b/dobas/worker.py index 2f1ed69..dcaaa4c 100644 --- a/dobas/worker.py +++ b/dobas/worker.py @@ -36,7 +36,7 @@ class Worker(object): # The key is either the host path or a volume name, and the value is a dictionary with the keys: # - bind The path to mount the volume inside the container # - mode Either rw to mount the volume read/write, or ro to mount it read-only. - backup_folder_listing = self.__docker_client.containers.run( + docker_run_output = self.__docker_client.containers.run( image=subject.worker_config.docker_image, entrypoint=subject.worker_config.docker_entrypoint, command=subject.worker_config.docker_command, @@ -44,7 +44,10 @@ class Worker(object): volumes=remapped_volume, detach=False, remove=True, + environment=subject.worker_config.docker_env, ) logger.debug( - f"worker container run {backup_folder_listing.decode('ascii')}") + f"worker container run {docker_run_output.decode('ascii')}") + + return docker_run_output -- GitLab From 8c1b46e641b688e85af0229aaaa9a291f896894e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Sat, 14 Nov 2020 17:20:16 +0100 Subject: [PATCH 13/26] prepare integration test of dobas worker config issued by orchestrator --- dobas/orchestrator.py | 1 - ...e-enable-named-volume-label-docker-env.yml | 2 ++ dobas/test/{helper.py => test_helper.py} | 0 dobas/test/test_integration.py | 32 +++++++++++++++++++ dobas/test/test_orchestrator.py | 5 +-- dobas/test/test_worker.py | 8 +++-- 6 files changed, 40 insertions(+), 8 deletions(-) rename dobas/test/{helper.py => test_helper.py} (100%) create mode 100644 dobas/test/test_integration.py diff --git a/dobas/orchestrator.py b/dobas/orchestrator.py index 201dee5..dacc155 100644 --- a/dobas/orchestrator.py +++ b/dobas/orchestrator.py @@ -69,7 +69,6 @@ class Orchestrator: logger.debug("process subject {the_subject}".format( the_subject=subject)) - # TODO: target docket client? worker = Worker(client=self.__docker_client) worker.process(subject=subject) diff --git a/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml b/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml index e45e021..e959c0f 100644 --- a/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml +++ b/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml @@ -5,3 +5,5 @@ volumes: labels: dobas.enable: true dobas.docker.env.EnvLabelKey: EnvLabelValue + dobas.docker.image: alpine:latest + dobas.docker.command: bin/sh -c "env | grep EnvLabelKey" diff --git a/dobas/test/helper.py b/dobas/test/test_helper.py similarity index 100% rename from dobas/test/helper.py rename to dobas/test/test_helper.py diff --git a/dobas/test/test_integration.py b/dobas/test/test_integration.py new file mode 100644 index 0000000..a5989c3 --- /dev/null +++ b/dobas/test/test_integration.py @@ -0,0 +1,32 @@ + +from unittest import TestCase + +from dobas.orchestrator import Orchestrator +from dobas.orchestrator_config import OrchestratorCfg +from dobas.worker import Worker +from dobas.worker_config import WorkerConfig +from dobas.subject import Subject +from test_helper import compose_up, docker_container_and_volume_prune + + +class TestOrchestratorWorkerIntegration(TestCase): + + def tearDown(self): + docker_container_and_volume_prune() + + def test_docker_env(self): + compose_up(["docker-compose-no-labels.yml", + "docker-compose-enable-named-volume-label-docker-env.yml"]) + + def __testSubjects(orchestrator_config): + orchestrator = Orchestrator(orchestrator_config) + orchestrator.run() + + __testSubjects(OrchestratorCfg( + docker_volumes_enabledByDefault=False, + daemon=False, + )) + __testSubjects(OrchestratorCfg( + docker_volumes_enabledByDefault=True, + daemon=False, + )) diff --git a/dobas/test/test_orchestrator.py b/dobas/test/test_orchestrator.py index bfd5f28..01e7ecc 100644 --- a/dobas/test/test_orchestrator.py +++ b/dobas/test/test_orchestrator.py @@ -1,16 +1,13 @@ import os from unittest import TestCase -from helper import compose_up, compose_down, docker_container_and_volume_prune +from test_helper import compose_up, docker_container_and_volume_prune from dobas.orchestrator import Orchestrator from dobas.orchestrator_config import OrchestratorCfg class TestOrchestrator(TestCase): - def setUp(self): - docker_container_and_volume_prune() - def tearDown(self): docker_container_and_volume_prune() diff --git a/dobas/test/test_worker.py b/dobas/test/test_worker.py index 331a151..f58ad6c 100644 --- a/dobas/test/test_worker.py +++ b/dobas/test/test_worker.py @@ -1,15 +1,17 @@ from unittest import TestCase -from helper import compose_up, compose_down, docker_container_and_volume_prune -from dobas.orchestrator import Orchestrator -from dobas.orchestrator_config import OrchestratorCfg from dobas.worker import Worker from dobas.worker_config import WorkerConfig from dobas.subject import Subject +from test_helper import docker_container_and_volume_prune import docker class TestWorker(TestCase): + + def tearDown(self): + docker_container_and_volume_prune() + def test_docker_env(self): worker = Worker(docker.from_env()) -- GitLab From 6c5e776066d0e7dbaa90617c1b56f9cfab851a36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Sun, 15 Nov 2020 13:36:56 +0100 Subject: [PATCH 14/26] move scheduling into worker config --- dobas/worker_config.py | 1 + main.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dobas/worker_config.py b/dobas/worker_config.py index b6079df..66278a7 100644 --- a/dobas/worker_config.py +++ b/dobas/worker_config.py @@ -4,6 +4,7 @@ import os @dataclass class WorkerConfig: + scheduling: str = "" # configured by orchestrator source: os.path = "" diff --git a/main.py b/main.py index e039c6c..6d054ef 100755 --- a/main.py +++ b/main.py @@ -18,10 +18,10 @@ Options: --orchestrator.docker.volumes.enabledByDefault= [default: False] --orchestrator.docker.services.enabledByDefault= [default: False] --orchestrator.daemon= [default: True] - --orchestrator.daemon.scheduling= [default: \"0 3 * * *\"] --orchestrator.mode.restore= [default: False] --orchestrator.mode.backup= [default: True] + --worker.scheduling= [default: \"0 3 * * *\"] --worker.storage.destination= URL to ssh host, which stores the backups. [default: ] --worker.storage.ssh.knownHost= KnownHost entry referencing the ssh host. [default: ] --worker.storage.ssh.username= Username used to authenticate against ssh host. [default: ] -- GitLab From 13aaf112f56f76f5f7d366ae8409ad1f8d989e6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Sun, 15 Nov 2020 14:49:19 +0100 Subject: [PATCH 15/26] named worker configuration --- dobas/orchestrator.py | 95 ++++++++++++++++--- dobas/orchestrator_config.py | 3 - dobas/subject.py | 2 +- ...e-enable-named-volume-label-docker-env.yml | 6 +- dobas/test/test_content_provider.py | 2 +- dobas/test/test_integration.py | 35 ++++--- dobas/test/test_orchestrator.py | 7 +- dobas/test/test_worker.py | 19 ++-- dobas/worker.py | 38 ++++---- dobas/worker_config.py | 5 +- main.py | 34 +++---- 11 files changed, 157 insertions(+), 89 deletions(-) diff --git a/dobas/orchestrator.py b/dobas/orchestrator.py index dacc155..ffde3bd 100644 --- a/dobas/orchestrator.py +++ b/dobas/orchestrator.py @@ -1,5 +1,5 @@ +from typing import Dict, List from .worker import Worker -from typing import List import docker import os import dataclasses @@ -26,9 +26,42 @@ class Orchestrator: else: self.__docker_client = docker.DockerClient.from_env() + self._default_worker_configs = self._discoverWorkerConfigsFromEnv() self._subjects = self._discover() logger.info(f"created subjects {self._subjects}") + def _discoverWorkerConfigsFromEnv(self) -> Dict[str, WorkerConfig]: + return self._dockerWorkerConfigsFromDict({removeprefix(string=key, prefix="dobas.worker"): value for key, + value in os.environ.items() if "dobas.worker." in key.lower()}) + + # expects already dobas.worker prefix stripped keys with dots as split symbols and worker_name as prefix + def _dockerWorkerConfigsFromDict(self, inputDict: Dict[str, str]) -> Dict[str, WorkerConfig]: + # collect all different worker names as keys + worker_configs = {key.split(".")[0]: None + for key, value in inputDict.items()} + + # collect worker config based on worker names and create WorkerConfig dataclasses + for worker_id, _ in worker_configs.items(): + # find alls envs for current worker_id and replace all . with _ + configs_for_worker = { + removeprefix(string=str(env)+".", prefix=worker_id).replace(".", "_"): value + for env, value in inputDict.items() if (env.startswith(worker_id) and "docker.env." not in env) + } + + docker_envs = {} + # mapping docker.env.=value is managed by WorkerConfig docker_env dict and should not be part of configs_for_worker + for docker_env, value in configs_for_worker: + if "docker.env." in docker_env: + docker_env_name = removeprefix( + string=docker_env, prefix="docker.env.").split(".")[0] + docker_envs[docker_env_name] = value + del configs_for_worker[docker_env] + + worker_configs[worker_id] = WorkerConfig( + identifier=worker_id, docker_env=docker_envs, **configs_for_worker) + + return worker_configs + def _discover(self) -> List[Subject]: logger.debug("discover subjects") @@ -50,15 +83,47 @@ class Orchestrator: # TODO: create individual worker config per subject subjects = [] for volume in volumes: - docker_envs = {} - # gather docker.env from volume labels - for docker_env_label in [label for label in volume.attrs.get("Labels") if label.startswith("dobas.docker.env.")]: - docker_envs[removeprefix(string=docker_env_label, prefix="dobas.docker.env.")] = volume.attrs.get( - "Labels")[docker_env_label] - - worker_config = WorkerConfig(self.__config.default_worker_config) - worker_config.docker_env = docker_envs - subjects.append(Subject(worker_config=worker_config, + # gather all dobas labels from the volume + volume_dobas_labels = {removeprefix(string=label, prefix="dobas.worker."): + volume.attrs.get("Labels")[label] for label in volume.attrs.get("Labels") + if label.startswith("dobas.worker.")} + + # determine which workers are active on this volume + # collect all different worker names as keys + volume_worker_configs = {key.split(".")[0]: WorkerConfig() + for key, _ in volume_dobas_labels.items()} + + # init with default WorkerConfig if present for the worker name + for volume_worker_name, _ in volume_worker_configs.items(): + if volume_worker_name in self._default_worker_configs.keys(): + volume_worker_configs[volume_worker_name] = WorkerConfig( + self._default_worker_configs[volume_worker_name]) + + # worker config with worker name specific volume labels + for volume_worker_name, volume_worker_config in volume_worker_configs.items(): + # gather docker.env from volume labels + volume_worker_docker_envs = {} + volume_worker_docker_env_label_prefix = "dobas.worker." + \ + str(volume_worker_name)+".docker.env." + for docker_env_label in [label for label in volume.attrs.get("Labels") if label.startswith(volume_worker_docker_env_label_prefix)]: + volume_worker_docker_envs[removeprefix(string=docker_env_label, prefix=volume_worker_docker_env_label_prefix)] = volume.attrs.get( + "Labels")[docker_env_label] + + worker_config_updates = dataclasses.asdict( + volume_worker_config) + worker_config_updates.update( + {removeprefix(string=key, prefix=str(volume_worker_name)+".").replace(".", "_"): value for key, + value in volume_dobas_labels.items() if key.split(".")[0] == volume_worker_name and "docker.env." not in key} + ) + if("docker_env" in worker_config_updates): + del worker_config_updates["docker_env"] + volume_worker_configs[volume_worker_name] = WorkerConfig( + **worker_config_updates, + docker_env=volume_worker_docker_envs, + ) + + # create one subjects per volume referencing all workers from labels + subjects.append(Subject(worker_configs=[config for _, config in volume_worker_configs.items()], volume=volume, services=None)) return subjects @@ -66,11 +131,11 @@ class Orchestrator: def __processSubjects(self): logger.debug("run process subjects") for subject in self._subjects: - logger.debug("process subject {the_subject}".format( - the_subject=subject)) - - worker = Worker(client=self.__docker_client) - worker.process(subject=subject) + for worker_config in subject.worker_configs: + worker = Worker(worker_config=worker_config, + client=self.__docker_client, + volume=subject.volume) + worker.process() def run(self): logger.debug("run orchestrator") diff --git a/dobas/orchestrator_config.py b/dobas/orchestrator_config.py index 82488b5..0cb72b7 100644 --- a/dobas/orchestrator_config.py +++ b/dobas/orchestrator_config.py @@ -15,8 +15,5 @@ class OrchestratorCfg: docker_services_enabledByDefault: bool = False daemon: bool = True - daemon_scheduling: str = "0 3 * * *" mode_backup: bool = False mode_restore: bool = False - - default_worker_config: WorkerConfig = WorkerConfig() diff --git a/dobas/subject.py b/dobas/subject.py index 1770d50..9054e4a 100644 --- a/dobas/subject.py +++ b/dobas/subject.py @@ -6,7 +6,7 @@ import docker @dataclass class Subject: - worker_config: WorkerConfig = None + worker_configs: List[WorkerConfig] = None volume: docker.models.volumes.Volume = None # TODO: implement services / docker-compose folder support in Subject services: None = None diff --git a/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml b/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml index e959c0f..8bb17ce 100644 --- a/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml +++ b/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml @@ -4,6 +4,6 @@ volumes: named-volume: labels: dobas.enable: true - dobas.docker.env.EnvLabelKey: EnvLabelValue - dobas.docker.image: alpine:latest - dobas.docker.command: bin/sh -c "env | grep EnvLabelKey" + dobas.worker.testworker1.docker.env.EnvLabelKey: EnvLabelValue + dobas.worker.testworker1.docker.image: alpine:latest + dobas.worker.testworker1.docker.command: bin/sh -c "env | grep EnvLabelKey" diff --git a/dobas/test/test_content_provider.py b/dobas/test/test_content_provider.py index 848faab..e214e6c 100644 --- a/dobas/test/test_content_provider.py +++ b/dobas/test/test_content_provider.py @@ -2,7 +2,7 @@ import os from unittest import TestCase import docker -from helper import compose_up, compose_down +from test_helper import compose_up, compose_down class TestContentProvider(TestCase): diff --git a/dobas/test/test_integration.py b/dobas/test/test_integration.py index a5989c3..320620f 100644 --- a/dobas/test/test_integration.py +++ b/dobas/test/test_integration.py @@ -5,28 +5,27 @@ from dobas.orchestrator import Orchestrator from dobas.orchestrator_config import OrchestratorCfg from dobas.worker import Worker from dobas.worker_config import WorkerConfig -from dobas.subject import Subject from test_helper import compose_up, docker_container_and_volume_prune -class TestOrchestratorWorkerIntegration(TestCase): +# class TestOrchestratorWorkerIntegration(TestCase): - def tearDown(self): - docker_container_and_volume_prune() +# def tearDown(self): +# docker_container_and_volume_prune() - def test_docker_env(self): - compose_up(["docker-compose-no-labels.yml", - "docker-compose-enable-named-volume-label-docker-env.yml"]) +# def test_docker_env(self): +# compose_up(["docker-compose-no-labels.yml", +# "docker-compose-enable-named-volume-label-docker-env.yml"]) - def __testSubjects(orchestrator_config): - orchestrator = Orchestrator(orchestrator_config) - orchestrator.run() +# def __testSubjects(orchestrator_config): +# orchestrator = Orchestrator(orchestrator_config) +# orchestrator.run() - __testSubjects(OrchestratorCfg( - docker_volumes_enabledByDefault=False, - daemon=False, - )) - __testSubjects(OrchestratorCfg( - docker_volumes_enabledByDefault=True, - daemon=False, - )) +# __testSubjects(OrchestratorCfg( +# docker_volumes_enabledByDefault=False, +# daemon=False, +# )) +# __testSubjects(OrchestratorCfg( +# docker_volumes_enabledByDefault=True, +# daemon=False, +# )) diff --git a/dobas/test/test_orchestrator.py b/dobas/test/test_orchestrator.py index 01e7ecc..ea96219 100644 --- a/dobas/test/test_orchestrator.py +++ b/dobas/test/test_orchestrator.py @@ -83,6 +83,8 @@ class TestOrchestrator(TestCase): self.assertIn("test_named-volume-bind", volume_ids) docker_container_and_volume_prune() + # TODO: write tests for orchestrator env worker default set configuration + def test_discovery_label_docker_envs(self): compose_up(["docker-compose-no-labels.yml", "docker-compose-enable-named-volume-label-docker-env.yml"]) @@ -92,9 +94,10 @@ class TestOrchestrator(TestCase): envs_linearized_dict = {} for subject in subjects: - envs_linearized_dict.update(subject.worker_config.docker_env) + for worker_config in subject.worker_configs: + if worker_config.docker_env: + envs_linearized_dict.update(worker_config.docker_env) - # self.assertIsInstance(envs, dict) self.assertEqual(len(envs_linearized_dict.items()), 1) self.assertEqual( envs_linearized_dict["EnvLabelKey"], "EnvLabelValue") diff --git a/dobas/test/test_worker.py b/dobas/test/test_worker.py index f58ad6c..a57978b 100644 --- a/dobas/test/test_worker.py +++ b/dobas/test/test_worker.py @@ -2,7 +2,6 @@ from unittest import TestCase from dobas.worker import Worker from dobas.worker_config import WorkerConfig -from dobas.subject import Subject from test_helper import docker_container_and_volume_prune import docker @@ -13,15 +12,13 @@ class TestWorker(TestCase): docker_container_and_volume_prune() def test_docker_env(self): - worker = Worker(docker.from_env()) + worker = Worker(worker_config=WorkerConfig( + docker_image="alpine", + docker_command="/bin/sh -c \"env | grep EnvLabelKey\"", + docker_env={"EnvLabelKey": "EnvLabelValue"} + ), + client=docker.from_env(), + volume=None) - subject = Subject( - worker_config=WorkerConfig( - docker_image="alpine", - docker_command="/bin/sh -c \"env | grep EnvLabelKey\"", - docker_env={"EnvLabelKey": "EnvLabelValue"} - ) - ) - - output = worker.process(subject) + output = worker.process() self.assertIn("EnvLabelKey=EnvLabelValue", output.decode("ascii")) diff --git a/dobas/worker.py b/dobas/worker.py index dcaaa4c..6a95e8d 100644 --- a/dobas/worker.py +++ b/dobas/worker.py @@ -3,32 +3,34 @@ from typing import List import docker import sys import subprocess -from pathlib import PurePath from . import logger -from .subject import Subject from .worker_config import WorkerConfig class Worker(object): - def __init__(self, client: docker.DockerClient): + def __init__(self, + worker_config: WorkerConfig, + client: docker.DockerClient, + volume: docker.models.volumes.Volume): self.__docker_client = client - logger.debug(f"new worker {client}".format( - client=self.__docker_client)) + self.__worker_config = worker_config + self.__volume = volume + logger.debug( + f"new worker {self.__worker_config.identifier} using client {self.__docker_client} with config {self.__worker_config} for volume {self.__volume}") - def process(self, subject: Subject): - logger.info(f"processing {subject}") + def process(self): remapped_volume = {} - if type(subject.volume) is docker.models.volumes.Volume: - remapped_volume[subject.volume.name] = { - "bind": os.path.join(subject.worker_config.docker_sourceMountBasePath.strip('"\''), - str(subject.volume.id)), + if type(self.__volume) is docker.models.volumes.Volume: + remapped_volume[self.__volume.name] = { + "bind": os.path.join(self.__worker_config.docker_sourceMountBasePath.strip('"\''), + str(self.__volume.id)), "mode": "ro" } # self.volume_backup_handler(volume=volume) else: logger.warning( - f"no handler for volume {subject.volume}, skipping ...") + f"no handler for volume {self.__volume}, skipping ...") # TODO: add functionality to capture subject completely # volumes: @@ -36,15 +38,17 @@ class Worker(object): # The key is either the host path or a volume name, and the value is a dictionary with the keys: # - bind The path to mount the volume inside the container # - mode Either rw to mount the volume read/write, or ro to mount it read-only. + logger.debug( + f"start {self.__worker_config.docker_image} with command {self.__worker_config.docker_command}") docker_run_output = self.__docker_client.containers.run( - image=subject.worker_config.docker_image, - entrypoint=subject.worker_config.docker_entrypoint, - command=subject.worker_config.docker_command, - working_dir=subject.worker_config.docker_workdir.strip("'\""), + image=self.__worker_config.docker_image, + entrypoint=self.__worker_config.docker_entrypoint, + command=self.__worker_config.docker_command, + working_dir=self.__worker_config.docker_workdir.strip("'\""), volumes=remapped_volume, detach=False, remove=True, - environment=subject.worker_config.docker_env, + environment=self.__worker_config.docker_env, ) logger.debug( diff --git a/dobas/worker_config.py b/dobas/worker_config.py index 66278a7..3a88579 100644 --- a/dobas/worker_config.py +++ b/dobas/worker_config.py @@ -1,9 +1,12 @@ from dataclasses import dataclass import os +import docker +from typing import Dict @dataclass class WorkerConfig: + identifier: str = "" scheduling: str = "" # configured by orchestrator source: os.path = "" @@ -19,4 +22,4 @@ class WorkerConfig: docker_command: str = "" docker_workdir: str = "" docker_sourceMountBasePath: str = "" - docker_env: {} = None + docker_env: Dict[str, str] = None diff --git a/main.py b/main.py index 6d054ef..7790a47 100755 --- a/main.py +++ b/main.py @@ -20,18 +20,19 @@ Options: --orchestrator.daemon= [default: True] --orchestrator.mode.restore= [default: False] --orchestrator.mode.backup= [default: True] - - --worker.scheduling= [default: \"0 3 * * *\"] - --worker.storage.destination= URL to ssh host, which stores the backups. [default: ] - --worker.storage.ssh.knownHost= KnownHost entry referencing the ssh host. [default: ] - --worker.storage.ssh.username= Username used to authenticate against ssh host. [default: ] - --worker.storage.ssh.sshKeypairPath= Path to ssh key pair. [default: ] - --worker.docker.image= [default: alpine:latest] - --worker.docker.entrypoint= [default: ] - --worker.docker.command= [default: \"/bin/sh -c \'ls - la /backup/\'\"] - --worker.docker.workdir= [default: \"/backup\"] - --worker.docker.sourceMountBasePath= [default: \"/backup\"] """ + +# --worker.scheduling= [default: \"0 3 * * *\"] +# --worker.storage.destination= URL to ssh host, which stores the backups. [default: ] +# --worker.storage.ssh.knownHost= KnownHost entry referencing the ssh host. [default: ] +# --worker.storage.ssh.username= Username used to authenticate against ssh host. [default: ] +# --worker.storage.ssh.sshKeypairPath= Path to ssh key pair. [default: ] +# --worker.docker.image= [default: alpine:latest] +# --worker.docker.entrypoint= [default: ] +# --worker.docker.command= [default: \"/bin/sh -c \'ls - la /backup/\'\"] +# --worker.docker.workdir= [default: \"/backup\"] +# --worker.docker.sourceMountBasePath= [default: \"/backup\"] + from docopt import docopt from dobas.orchestrator import Orchestrator @@ -54,17 +55,16 @@ def __dict_string_value_to_bool(dict_to_update): if __name__ == '__main__': arguments = docopt(__doc__) - default_worker_config = WorkerConfig(**__dict_string_value_to_bool({ - removeprefix(string=key, prefix="--worker.").replace(".", "_"): value for key, value in arguments.items() if - key.startswith("--worker.") - })) + # default_worker_config = WorkerConfig(**__dict_string_value_to_bool({ + # removeprefix(string=key, prefix="--worker.").replace(".", "_"): value for key, value in arguments.items() if + # key.startswith("--worker.") + # })) orchestrator_config = OrchestratorCfg( **__dict_string_value_to_bool({ removeprefix(string=key, prefix="--orchestrator.").replace(".", "_"): value for key, value in arguments.items() if key.startswith("--orchestrator.") - }), - default_worker_config=default_worker_config) + })) if arguments['--verbose'] == 0: logger.setLevel(WARN) -- GitLab From 2663e6bb1e5cfa335f28332be44aafaab01e25df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Mon, 16 Nov 2020 00:05:58 +0100 Subject: [PATCH 16/26] add orchestrator test_env_named_worker_defaults and bugfix to desired behavior --- dobas/orchestrator.py | 15 +++++----- dobas/orchestrator_config.py | 2 +- dobas/test/test_helper.py | 16 ++++++++++ dobas/test/test_integration.py | 6 ++-- dobas/test/test_orchestrator.py | 52 ++++++++++++++++++++++++--------- dobas/worker_config.py | 2 -- main.py | 15 ++-------- 7 files changed, 67 insertions(+), 41 deletions(-) diff --git a/dobas/orchestrator.py b/dobas/orchestrator.py index ffde3bd..6c1a7a5 100644 --- a/dobas/orchestrator.py +++ b/dobas/orchestrator.py @@ -6,7 +6,7 @@ import dataclasses from . import logger from .subject import Subject -from .orchestrator_config import OrchestratorCfg +from .orchestrator_config import OrchestratorConfig from .worker_config import WorkerConfig from .helper import removeprefix @@ -15,8 +15,8 @@ class Orchestrator: """ """ - def __init__(self, config: OrchestratorCfg): - self.__config: OrchestratorCfg = config + def __init__(self, config: OrchestratorConfig): + self.__config: OrchestratorConfig = config logger.debug("implement typechecked cfg") logger.info(f"create orchestrator {config}") @@ -31,7 +31,7 @@ class Orchestrator: logger.info(f"created subjects {self._subjects}") def _discoverWorkerConfigsFromEnv(self) -> Dict[str, WorkerConfig]: - return self._dockerWorkerConfigsFromDict({removeprefix(string=key, prefix="dobas.worker"): value for key, + return self._dockerWorkerConfigsFromDict({removeprefix(string=key, prefix="dobas.worker."): value for key, value in os.environ.items() if "dobas.worker." in key.lower()}) # expects already dobas.worker prefix stripped keys with dots as split symbols and worker_name as prefix @@ -44,18 +44,17 @@ class Orchestrator: for worker_id, _ in worker_configs.items(): # find alls envs for current worker_id and replace all . with _ configs_for_worker = { - removeprefix(string=str(env)+".", prefix=worker_id).replace(".", "_"): value + removeprefix(string=str(env), prefix=worker_id+".").replace(".", "_"): value for env, value in inputDict.items() if (env.startswith(worker_id) and "docker.env." not in env) } docker_envs = {} # mapping docker.env.=value is managed by WorkerConfig docker_env dict and should not be part of configs_for_worker - for docker_env, value in configs_for_worker: + for docker_env, value in inputDict.items(): if "docker.env." in docker_env: docker_env_name = removeprefix( - string=docker_env, prefix="docker.env.").split(".")[0] + string=docker_env, prefix=str(worker_id)+".docker.env.").split(".")[0] docker_envs[docker_env_name] = value - del configs_for_worker[docker_env] worker_configs[worker_id] = WorkerConfig( identifier=worker_id, docker_env=docker_envs, **configs_for_worker) diff --git a/dobas/orchestrator_config.py b/dobas/orchestrator_config.py index 0cb72b7..7e1e428 100644 --- a/dobas/orchestrator_config.py +++ b/dobas/orchestrator_config.py @@ -4,7 +4,7 @@ from .worker_config import WorkerConfig @dataclass(frozen=True) -class OrchestratorCfg: +class OrchestratorConfig: """ static configuration generated at startup There are some szenarios, where the orchestrator is not able to query its own config from the references docker-host. This is the case for the test setup, where the orchestrator lives besides the dind, where the orchestrators docker-client is connected to. So the docker-client is not able to see it's own orchestrator container. diff --git a/dobas/test/test_helper.py b/dobas/test/test_helper.py index 526130c..651fc8f 100644 --- a/dobas/test/test_helper.py +++ b/dobas/test/test_helper.py @@ -2,6 +2,22 @@ import subprocess import os from typing import List +dobas_env_worker_config_dict = { + "dobas.worker..scheduling": "0 3 * * *", + "dobas.worker..storage.destination": "/destination", + "dobas.worker..storage.ssh.knownHost": "knownHost", + "dobas.worker..storage.ssh.username": "username", + "dobas.worker..storage.ssh.sshKeypairPath": "sshKeyPairPath", + "dobas.worker..docker.image": "alpine:latest", + "dobas.worker..docker.entrypoint": "entrypoint", + "dobas.worker..docker.command": "/bin/sh -c \"echo hello world\"", + "dobas.worker..docker.workdir": "/root", + "dobas.worker..docker.sourceMountBasePath": "/backup", + "dobas.worker..docker.env.EnvLabelKey1": "EnvLabelValue1", + "dobas.worker..docker.env.EnvLabelKey2": "EnvLabelValue2", + "dobas.worker..docker.env.EnvLabelKey3": "EnvLabelValue3", +} + def _run_cli_command(args: List[str]): print(args) diff --git a/dobas/test/test_integration.py b/dobas/test/test_integration.py index 320620f..23654ae 100644 --- a/dobas/test/test_integration.py +++ b/dobas/test/test_integration.py @@ -2,7 +2,7 @@ from unittest import TestCase from dobas.orchestrator import Orchestrator -from dobas.orchestrator_config import OrchestratorCfg +from dobas.orchestrator_config import OrchestratorConfig from dobas.worker import Worker from dobas.worker_config import WorkerConfig from test_helper import compose_up, docker_container_and_volume_prune @@ -21,11 +21,11 @@ from test_helper import compose_up, docker_container_and_volume_prune # orchestrator = Orchestrator(orchestrator_config) # orchestrator.run() -# __testSubjects(OrchestratorCfg( +# __testSubjects(OrchestratorConfig( # docker_volumes_enabledByDefault=False, # daemon=False, # )) -# __testSubjects(OrchestratorCfg( +# __testSubjects(OrchestratorConfig( # docker_volumes_enabledByDefault=True, # daemon=False, # )) diff --git a/dobas/test/test_orchestrator.py b/dobas/test/test_orchestrator.py index ea96219..31f412f 100644 --- a/dobas/test/test_orchestrator.py +++ b/dobas/test/test_orchestrator.py @@ -1,9 +1,10 @@ import os -from unittest import TestCase +from unittest import TestCase, mock -from test_helper import compose_up, docker_container_and_volume_prune +from test_helper import compose_up, docker_container_and_volume_prune, dobas_env_worker_config_dict from dobas.orchestrator import Orchestrator -from dobas.orchestrator_config import OrchestratorCfg +from dobas.orchestrator_config import OrchestratorConfig +from dobas.worker_config import WorkerConfig class TestOrchestrator(TestCase): @@ -13,11 +14,11 @@ class TestOrchestrator(TestCase): def test_discovery_default_behavior_without_labels(self): compose_up(["docker-compose-no-labels.yml"]) - self.assertEqual(len(Orchestrator(OrchestratorCfg( + self.assertEqual(len(Orchestrator(OrchestratorConfig( docker_volumes_enabledByDefault=False, ))._discover()), 0) - subjects = Orchestrator(OrchestratorCfg( + subjects = Orchestrator(OrchestratorConfig( docker_volumes_enabledByDefault=True, ))._discover() self.assertEqual(len(subjects), 3) @@ -30,11 +31,11 @@ class TestOrchestrator(TestCase): def test_discovery_with_labels(self): compose_up(["docker-compose-no-labels.yml", "docker-compose-disable-named-volume-disabled-label.yml"]) - self.assertEqual(len(Orchestrator(OrchestratorCfg( + self.assertEqual(len(Orchestrator(OrchestratorConfig( docker_volumes_enabledByDefault=False, ))._discover()), 0) - subjects = Orchestrator(OrchestratorCfg( + subjects = Orchestrator(OrchestratorConfig( docker_volumes_enabledByDefault=True, ))._discover() self.assertEqual(len(subjects), 2) @@ -46,14 +47,14 @@ class TestOrchestrator(TestCase): compose_up(["docker-compose-no-labels.yml", "docker-compose-enable-named-volume-label.yml", "docker-compose-disable-named-volume-disabled-label.yml"]) - subjects = Orchestrator(OrchestratorCfg( + subjects = Orchestrator(OrchestratorConfig( docker_volumes_enabledByDefault=False, ))._discover() self.assertEqual(len(subjects), 1) volume_ids = [subject.volume.id for subject in subjects] self.assertIn("test_named-volume", volume_ids) - subjects = Orchestrator(OrchestratorCfg( + subjects = Orchestrator(OrchestratorConfig( docker_volumes_enabledByDefault=True, ))._discover() self.assertEqual(len(subjects), 2) @@ -66,7 +67,7 @@ class TestOrchestrator(TestCase): "docker-compose-enable-named-volume-label.yml", "docker-compose-enable-named-volume-bind-label.yml", "docker-compose-disable-named-volume-disabled-label.yml"]) - subjects = Orchestrator(OrchestratorCfg( + subjects = Orchestrator(OrchestratorConfig( docker_volumes_enabledByDefault=False, ))._discover() self.assertEqual(len(subjects), 2) @@ -74,7 +75,7 @@ class TestOrchestrator(TestCase): self.assertIn("test_named-volume", volume_ids) self.assertIn("test_named-volume-bind", volume_ids) - subjects = Orchestrator(OrchestratorCfg( + subjects = Orchestrator(OrchestratorConfig( docker_volumes_enabledByDefault=True, ))._discover() self.assertEqual(len(subjects), 2) @@ -83,7 +84,30 @@ class TestOrchestrator(TestCase): self.assertIn("test_named-volume-bind", volume_ids) docker_container_and_volume_prune() - # TODO: write tests for orchestrator env worker default set configuration + # TODO: test multiple worker names in + @mock.patch.dict(os.environ, {key.replace("", "TestWorker1"): value for key, value in dobas_env_worker_config_dict.items()}) + def test_env_named_worker_defaults(self): + orchestrator = Orchestrator(OrchestratorConfig()) + self.assertDictEqual({ + "TestWorker1": WorkerConfig( + identifier="TestWorker1", + scheduling="0 3 * * *", + storage_destination="/destination", + storage_ssh_knownHost="knownHost", + storage_ssh_username="username", + storage_ssh_sshKeypairPath="sshKeyPairPath", + docker_image="alpine:latest", + docker_entrypoint="entrypoint", + docker_command="/bin/sh -c \"echo hello world\"", + docker_workdir="/root", + docker_sourceMountBasePath="/backup", + docker_env={ + "EnvLabelKey1": "EnvLabelValue1", + "EnvLabelKey2": "EnvLabelValue2", + "EnvLabelKey3": "EnvLabelValue3", + } + ) + }, orchestrator._default_worker_configs) def test_discovery_label_docker_envs(self): compose_up(["docker-compose-no-labels.yml", @@ -102,9 +126,9 @@ class TestOrchestrator(TestCase): self.assertEqual( envs_linearized_dict["EnvLabelKey"], "EnvLabelValue") - __testSubjects(OrchestratorCfg( + __testSubjects(OrchestratorConfig( docker_volumes_enabledByDefault=False, )) - __testSubjects(OrchestratorCfg( + __testSubjects(OrchestratorConfig( docker_volumes_enabledByDefault=True, )) diff --git a/dobas/worker_config.py b/dobas/worker_config.py index 3a88579..3b130ac 100644 --- a/dobas/worker_config.py +++ b/dobas/worker_config.py @@ -8,8 +8,6 @@ from typing import Dict class WorkerConfig: identifier: str = "" scheduling: str = "" - # configured by orchestrator - source: os.path = "" # configured by labels storage_destination: str = "" diff --git a/main.py b/main.py index 7790a47..ea48eba 100755 --- a/main.py +++ b/main.py @@ -22,23 +22,12 @@ Options: --orchestrator.mode.backup= [default: True] """ -# --worker.scheduling= [default: \"0 3 * * *\"] -# --worker.storage.destination= URL to ssh host, which stores the backups. [default: ] -# --worker.storage.ssh.knownHost= KnownHost entry referencing the ssh host. [default: ] -# --worker.storage.ssh.username= Username used to authenticate against ssh host. [default: ] -# --worker.storage.ssh.sshKeypairPath= Path to ssh key pair. [default: ] -# --worker.docker.image= [default: alpine:latest] -# --worker.docker.entrypoint= [default: ] -# --worker.docker.command= [default: \"/bin/sh -c \'ls - la /backup/\'\"] -# --worker.docker.workdir= [default: \"/backup\"] -# --worker.docker.sourceMountBasePath= [default: \"/backup\"] - from docopt import docopt from dobas.orchestrator import Orchestrator from dobas import logger from logging import DEBUG, INFO, WARN -from dobas.orchestrator_config import OrchestratorCfg +from dobas.orchestrator_config import OrchestratorConfig from dobas.worker_config import WorkerConfig from dobas.helper import removeprefix @@ -60,7 +49,7 @@ if __name__ == '__main__': # key.startswith("--worker.") # })) - orchestrator_config = OrchestratorCfg( + orchestrator_config = OrchestratorConfig( **__dict_string_value_to_bool({ removeprefix(string=key, prefix="--orchestrator.").replace(".", "_"): value for key, value in arguments.items() if key.startswith("--orchestrator.") -- GitLab From 8b67cae32a26fe8b8266cb4c6dd948dff8c19784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Mon, 16 Nov 2020 22:21:59 +0100 Subject: [PATCH 17/26] test test_env_named_worker_defaults with 2 worker names and different content; bugfix exposed bug --- dobas/orchestrator.py | 2 +- dobas/test/test_helper.py | 44 ++++++++++++++++++++--------- dobas/test/test_orchestrator.py | 50 +++++++++++++++++++++++---------- 3 files changed, 66 insertions(+), 30 deletions(-) diff --git a/dobas/orchestrator.py b/dobas/orchestrator.py index 6c1a7a5..1cfffb3 100644 --- a/dobas/orchestrator.py +++ b/dobas/orchestrator.py @@ -51,7 +51,7 @@ class Orchestrator: docker_envs = {} # mapping docker.env.=value is managed by WorkerConfig docker_env dict and should not be part of configs_for_worker for docker_env, value in inputDict.items(): - if "docker.env." in docker_env: + if str(worker_id)+".docker.env." in docker_env: docker_env_name = removeprefix( string=docker_env, prefix=str(worker_id)+".docker.env.").split(".")[0] docker_envs[docker_env_name] = value diff --git a/dobas/test/test_helper.py b/dobas/test/test_helper.py index 651fc8f..8445031 100644 --- a/dobas/test/test_helper.py +++ b/dobas/test/test_helper.py @@ -2,20 +2,36 @@ import subprocess import os from typing import List -dobas_env_worker_config_dict = { - "dobas.worker..scheduling": "0 3 * * *", - "dobas.worker..storage.destination": "/destination", - "dobas.worker..storage.ssh.knownHost": "knownHost", - "dobas.worker..storage.ssh.username": "username", - "dobas.worker..storage.ssh.sshKeypairPath": "sshKeyPairPath", - "dobas.worker..docker.image": "alpine:latest", - "dobas.worker..docker.entrypoint": "entrypoint", - "dobas.worker..docker.command": "/bin/sh -c \"echo hello world\"", - "dobas.worker..docker.workdir": "/root", - "dobas.worker..docker.sourceMountBasePath": "/backup", - "dobas.worker..docker.env.EnvLabelKey1": "EnvLabelValue1", - "dobas.worker..docker.env.EnvLabelKey2": "EnvLabelValue2", - "dobas.worker..docker.env.EnvLabelKey3": "EnvLabelValue3", +dobas_env_worker_config_dict_testworker1 = { + "dobas.worker.TestWorker1.scheduling": "0 3 * * *1", + "dobas.worker.TestWorker1.storage.destination": "/destination1", + "dobas.worker.TestWorker1.storage.ssh.knownHost": "knownHost1", + "dobas.worker.TestWorker1.storage.ssh.username": "username1", + "dobas.worker.TestWorker1.storage.ssh.sshKeypairPath": "sshKeyPairPath1", + "dobas.worker.TestWorker1.docker.image": "alpine:latest1", + "dobas.worker.TestWorker1.docker.entrypoint": "entrypoint1", + "dobas.worker.TestWorker1.docker.command": "/bin/sh -c \"echo hello world\"1", + "dobas.worker.TestWorker1.docker.workdir": "/root1", + "dobas.worker.TestWorker1.docker.sourceMountBasePath": "/backup1", + "dobas.worker.TestWorker1.docker.env.EnvLabelKey1": "EnvLabelValue11", + "dobas.worker.TestWorker1.docker.env.EnvLabelKey2": "EnvLabelValue21", + "dobas.worker.TestWorker1.docker.env.EnvLabelKey3": "EnvLabelValue31", +} + +dobas_env_worker_config_dict_testworker2 = { + "dobas.worker.TestWorker2.scheduling": "0 3 * * *2", + "dobas.worker.TestWorker2.storage.destination": "/destination2", + "dobas.worker.TestWorker2.storage.ssh.knownHost": "knownHost2", + "dobas.worker.TestWorker2.storage.ssh.username": "username2", + "dobas.worker.TestWorker2.storage.ssh.sshKeypairPath": "sshKeyPairPath2", + "dobas.worker.TestWorker2.docker.image": "alpine:latest2", + "dobas.worker.TestWorker2.docker.entrypoint": "entrypoint2", + "dobas.worker.TestWorker2.docker.command": "/bin/sh -c \"echo hello world\"2", + "dobas.worker.TestWorker2.docker.workdir": "/root2", + "dobas.worker.TestWorker2.docker.sourceMountBasePath": "/backup2", + "dobas.worker.TestWorker2.docker.env.EnvLabelKey1": "EnvLabelValue12", + "dobas.worker.TestWorker2.docker.env.EnvLabelKey2": "EnvLabelValue22", + "dobas.worker.TestWorker2.docker.env.EnvLabelKey3": "EnvLabelValue32", } diff --git a/dobas/test/test_orchestrator.py b/dobas/test/test_orchestrator.py index 31f412f..fb2d9d3 100644 --- a/dobas/test/test_orchestrator.py +++ b/dobas/test/test_orchestrator.py @@ -1,7 +1,7 @@ import os from unittest import TestCase, mock -from test_helper import compose_up, docker_container_and_volume_prune, dobas_env_worker_config_dict +from test_helper import compose_up, docker_container_and_volume_prune, dobas_env_worker_config_dict_testworker1, dobas_env_worker_config_dict_testworker2 from dobas.orchestrator import Orchestrator from dobas.orchestrator_config import OrchestratorConfig from dobas.worker_config import WorkerConfig @@ -85,26 +85,46 @@ class TestOrchestrator(TestCase): docker_container_and_volume_prune() # TODO: test multiple worker names in - @mock.patch.dict(os.environ, {key.replace("", "TestWorker1"): value for key, value in dobas_env_worker_config_dict.items()}) + @mock.patch.dict(os.environ, {**dobas_env_worker_config_dict_testworker1, **dobas_env_worker_config_dict_testworker2}) def test_env_named_worker_defaults(self): + + print(os.environ) orchestrator = Orchestrator(OrchestratorConfig()) self.assertDictEqual({ "TestWorker1": WorkerConfig( identifier="TestWorker1", - scheduling="0 3 * * *", - storage_destination="/destination", - storage_ssh_knownHost="knownHost", - storage_ssh_username="username", - storage_ssh_sshKeypairPath="sshKeyPairPath", - docker_image="alpine:latest", - docker_entrypoint="entrypoint", - docker_command="/bin/sh -c \"echo hello world\"", - docker_workdir="/root", - docker_sourceMountBasePath="/backup", + scheduling="0 3 * * *1", + storage_destination="/destination1", + storage_ssh_knownHost="knownHost1", + storage_ssh_username="username1", + storage_ssh_sshKeypairPath="sshKeyPairPath1", + docker_image="alpine:latest1", + docker_entrypoint="entrypoint1", + docker_command="/bin/sh -c \"echo hello world\"1", + docker_workdir="/root1", + docker_sourceMountBasePath="/backup1", + docker_env={ + "EnvLabelKey1": "EnvLabelValue11", + "EnvLabelKey2": "EnvLabelValue21", + "EnvLabelKey3": "EnvLabelValue31", + } + ), + "TestWorker2": WorkerConfig( + identifier="TestWorker2", + scheduling="0 3 * * *2", + storage_destination="/destination2", + storage_ssh_knownHost="knownHost2", + storage_ssh_username="username2", + storage_ssh_sshKeypairPath="sshKeyPairPath2", + docker_image="alpine:latest2", + docker_entrypoint="entrypoint2", + docker_command="/bin/sh -c \"echo hello world\"2", + docker_workdir="/root2", + docker_sourceMountBasePath="/backup2", docker_env={ - "EnvLabelKey1": "EnvLabelValue1", - "EnvLabelKey2": "EnvLabelValue2", - "EnvLabelKey3": "EnvLabelValue3", + "EnvLabelKey1": "EnvLabelValue12", + "EnvLabelKey2": "EnvLabelValue22", + "EnvLabelKey3": "EnvLabelValue32", } ) }, orchestrator._default_worker_configs) -- GitLab From bd751fcc756422426eda946ebf6fcc3db4a01ece Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Thu, 19 Nov 2020 00:29:12 +0100 Subject: [PATCH 18/26] TDD: add expected default worker override by labels test, where the override mechaincs are not implemented yet --- ...e-enable-named-volume-label-docker-env.yml | 3 + dobas/test/test_orchestrator.py | 62 ++++++++++++++++++- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml b/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml index 8bb17ce..358a271 100644 --- a/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml +++ b/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml @@ -7,3 +7,6 @@ volumes: dobas.worker.testworker1.docker.env.EnvLabelKey: EnvLabelValue dobas.worker.testworker1.docker.image: alpine:latest dobas.worker.testworker1.docker.command: bin/sh -c "env | grep EnvLabelKey" + dobas.worker.testworker2.docker.env.EnvLabelKeyOverridden: EnvLabelValueOverridden + dobas.worker.testworker2.docker.image: alpine:edge-overridden + dobas.worker.testworker2.docker.command: bin/sh -c "env | grep EnvLabelKeyOverridden" \ No newline at end of file diff --git a/dobas/test/test_orchestrator.py b/dobas/test/test_orchestrator.py index fb2d9d3..bc291c1 100644 --- a/dobas/test/test_orchestrator.py +++ b/dobas/test/test_orchestrator.py @@ -84,7 +84,6 @@ class TestOrchestrator(TestCase): self.assertIn("test_named-volume-bind", volume_ids) docker_container_and_volume_prune() - # TODO: test multiple worker names in @mock.patch.dict(os.environ, {**dobas_env_worker_config_dict_testworker1, **dobas_env_worker_config_dict_testworker2}) def test_env_named_worker_defaults(self): @@ -129,6 +128,63 @@ class TestOrchestrator(TestCase): ) }, orchestrator._default_worker_configs) + @mock.patch.dict(os.environ, {**dobas_env_worker_config_dict_testworker1, **dobas_env_worker_config_dict_testworker2}) + def test_orchestrator_defaults_partial_overrides_by_volume_labels(self): + compose_up(["docker-compose-no-labels.yml", + "docker-compose-enable-named-volume-label-docker-env.yml"]) + orchestrator = Orchestrator(OrchestratorConfig( + docker_volumes_enabledByDefault=False, + )) + subjects = orchestrator._subjects + self.assertEqual(len(subjects), 1) + volume_ids = [subject.volume.id for subject in subjects] + self.assertIn("test_named-volume", volume_ids) + workerConfigs = {} + for subject in subjects: + for workerconfig in subject.worker_configs: + workerConfigs[str(workerconfig.identifier)] = workerconfig + + self.assertDictEqual({ + "TestWorker1": WorkerConfig( + identifier="TestWorker1", + scheduling="0 3 * * *1", + storage_destination="/destination1", + storage_ssh_knownHost="knownHost1", + storage_ssh_username="username1", + storage_ssh_sshKeypairPath="sshKeyPairPath1", + docker_image="alpine:latest", + docker_entrypoint="entrypoint1", + docker_command="/bin/sh -c \"env | grep EnvLabelKey\"", + docker_workdir="/root1", + docker_sourceMountBasePath="/backup1", + docker_env={ + "EnvLabelKey1": "EnvLabelValue11", + "EnvLabelKey2": "EnvLabelValue21", + "EnvLabelKey3": "EnvLabelValue31", + "EnvLabelKey": "EnvLabelValue" + } + ), + "TestWorker2": WorkerConfig( + identifier="TestWorker2", + scheduling="0 3 * * *2", + storage_destination="/destination2", + storage_ssh_knownHost="knownHost2", + storage_ssh_username="username2", + storage_ssh_sshKeypairPath="sshKeyPairPath2", + docker_image="alpine:edge-overridden", + docker_entrypoint="entrypoint2", + docker_command="/bin/sh -c \"env | grep EnvLabelKeyOverridden\"", + docker_workdir="/root2", + docker_sourceMountBasePath="/backup2", + docker_env={ + "EnvLabelKey1": "EnvLabelValue12", + "EnvLabelKey2": "EnvLabelValue22", + "EnvLabelKey3": "EnvLabelValue32", + "EnvLabelKeyOverridden": "EnvLabelValueOverridden" + } + ) + }, workerConfigs) + def test_discovery_label_docker_envs(self): compose_up(["docker-compose-no-labels.yml", "docker-compose-enable-named-volume-label-docker-env.yml"]) @@ -142,9 +198,11 @@ class TestOrchestrator(TestCase): if worker_config.docker_env: envs_linearized_dict.update(worker_config.docker_env) - self.assertEqual(len(envs_linearized_dict.items()), 1) + self.assertEqual(len(envs_linearized_dict.items()), 2) self.assertEqual( envs_linearized_dict["EnvLabelKey"], "EnvLabelValue") + self.assertEqual( + envs_linearized_dict["EnvLabelKeyOverridden"], "EnvLabelValueOverridden") __testSubjects(OrchestratorConfig( docker_volumes_enabledByDefault=False, -- GitLab From c38ba54f8d58883762aee06cba49ccdb68b33cb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Sat, 28 Nov 2020 17:58:10 +0100 Subject: [PATCH 19/26] use correct case sensitive worker names in labels for env config override test --- ...-compose-enable-named-volume-label-docker-env.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml b/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml index 358a271..f1667de 100644 --- a/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml +++ b/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml @@ -4,9 +4,9 @@ volumes: named-volume: labels: dobas.enable: true - dobas.worker.testworker1.docker.env.EnvLabelKey: EnvLabelValue - dobas.worker.testworker1.docker.image: alpine:latest - dobas.worker.testworker1.docker.command: bin/sh -c "env | grep EnvLabelKey" - dobas.worker.testworker2.docker.env.EnvLabelKeyOverridden: EnvLabelValueOverridden - dobas.worker.testworker2.docker.image: alpine:edge-overridden - dobas.worker.testworker2.docker.command: bin/sh -c "env | grep EnvLabelKeyOverridden" \ No newline at end of file + dobas.worker.TestWorker1.docker.env.EnvLabelKey: EnvLabelValue + dobas.worker.TestWorker1.docker.image: alpine:latest + dobas.worker.TestWorker1.docker.command: bin/sh -c "env | grep EnvLabelKey" + dobas.worker.TestWorker2.docker.env.EnvLabelKeyOverridden: EnvLabelValueOverridden + dobas.worker.TestWorker2.docker.image: alpine:edge-overridden + dobas.worker.TestWorker2.docker.command: bin/sh -c "env | grep EnvLabelKeyOverridden" \ No newline at end of file -- GitLab From bd4521aca7cb606c8f23c98f3087c4096d981c4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Sat, 28 Nov 2020 17:59:04 +0100 Subject: [PATCH 20/26] correctly reinstantiate WorkerConfig from default one, instead of not intended nesting them --- dobas/orchestrator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dobas/orchestrator.py b/dobas/orchestrator.py index 1cfffb3..5bb7363 100644 --- a/dobas/orchestrator.py +++ b/dobas/orchestrator.py @@ -96,7 +96,7 @@ class Orchestrator: for volume_worker_name, _ in volume_worker_configs.items(): if volume_worker_name in self._default_worker_configs.keys(): volume_worker_configs[volume_worker_name] = WorkerConfig( - self._default_worker_configs[volume_worker_name]) + **dataclasses.asdict(self._default_worker_configs[volume_worker_name])) # worker config with worker name specific volume labels for volume_worker_name, volume_worker_config in volume_worker_configs.items(): -- GitLab From 9666b14b893bd4a6a0330c984eca052b4d6a2259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Sat, 28 Nov 2020 18:10:57 +0100 Subject: [PATCH 21/26] maxDiff for assertdictEqual to None to get complete print outs --- dobas/test/test_orchestrator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dobas/test/test_orchestrator.py b/dobas/test/test_orchestrator.py index bc291c1..be6faae 100644 --- a/dobas/test/test_orchestrator.py +++ b/dobas/test/test_orchestrator.py @@ -89,6 +89,7 @@ class TestOrchestrator(TestCase): print(os.environ) orchestrator = Orchestrator(OrchestratorConfig()) + self.assertDictEqual.__self__.maxDiff = None self.assertDictEqual({ "TestWorker1": WorkerConfig( identifier="TestWorker1", @@ -144,6 +145,7 @@ class TestOrchestrator(TestCase): for workerconfig in subject.worker_configs: workerConfigs[str(workerconfig.identifier)] = workerconfig + self.assertDictEqual.__self__.maxDiff = None self.assertDictEqual({ "TestWorker1": WorkerConfig( identifier="TestWorker1", -- GitLab From e32eec37faf995fcad5583329d4d463bc5cee6a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Sat, 28 Nov 2020 18:11:40 +0100 Subject: [PATCH 22/26] ensure docker env override is initialized with prepopulated default config --- dobas/orchestrator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dobas/orchestrator.py b/dobas/orchestrator.py index 5bb7363..aab1924 100644 --- a/dobas/orchestrator.py +++ b/dobas/orchestrator.py @@ -100,8 +100,9 @@ class Orchestrator: # worker config with worker name specific volume labels for volume_worker_name, volume_worker_config in volume_worker_configs.items(): + # init docker.env from default config + volume_worker_docker_envs = volume_worker_configs[volume_worker_name].docker_env # gather docker.env from volume labels - volume_worker_docker_envs = {} volume_worker_docker_env_label_prefix = "dobas.worker." + \ str(volume_worker_name)+".docker.env." for docker_env_label in [label for label in volume.attrs.get("Labels") if label.startswith(volume_worker_docker_env_label_prefix)]: -- GitLab From a999706fdeffe22aca4c3ce0494a931c0f749320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Sat, 28 Nov 2020 18:21:27 +0100 Subject: [PATCH 23/26] use more readable AsserEqual for dataclasses instead of AssertDictEqual with nested data classes --- dobas/test/test_orchestrator.py | 83 ++++++++++++++++----------------- 1 file changed, 40 insertions(+), 43 deletions(-) diff --git a/dobas/test/test_orchestrator.py b/dobas/test/test_orchestrator.py index be6faae..aa14e52 100644 --- a/dobas/test/test_orchestrator.py +++ b/dobas/test/test_orchestrator.py @@ -89,7 +89,6 @@ class TestOrchestrator(TestCase): print(os.environ) orchestrator = Orchestrator(OrchestratorConfig()) - self.assertDictEqual.__self__.maxDiff = None self.assertDictEqual({ "TestWorker1": WorkerConfig( identifier="TestWorker1", @@ -129,7 +128,7 @@ class TestOrchestrator(TestCase): ) }, orchestrator._default_worker_configs) - @mock.patch.dict(os.environ, {**dobas_env_worker_config_dict_testworker1, **dobas_env_worker_config_dict_testworker2}) + @ mock.patch.dict(os.environ, {**dobas_env_worker_config_dict_testworker1, **dobas_env_worker_config_dict_testworker2}) def test_orchestrator_defaults_partial_overrides_by_volume_labels(self): compose_up(["docker-compose-no-labels.yml", "docker-compose-enable-named-volume-label-docker-env.yml"]) @@ -145,47 +144,45 @@ class TestOrchestrator(TestCase): for workerconfig in subject.worker_configs: workerConfigs[str(workerconfig.identifier)] = workerconfig - self.assertDictEqual.__self__.maxDiff = None - self.assertDictEqual({ - "TestWorker1": WorkerConfig( - identifier="TestWorker1", - scheduling="0 3 * * *1", - storage_destination="/destination1", - storage_ssh_knownHost="knownHost1", - storage_ssh_username="username1", - storage_ssh_sshKeypairPath="sshKeyPairPath1", - docker_image="alpine:latest", - docker_entrypoint="entrypoint1", - docker_command="/bin/sh -c \"env | grep EnvLabelKey\"", - docker_workdir="/root1", - docker_sourceMountBasePath="/backup1", - docker_env={ - "EnvLabelKey1": "EnvLabelValue11", - "EnvLabelKey2": "EnvLabelValue21", - "EnvLabelKey3": "EnvLabelValue31", - "EnvLabelKey": "EnvLabelValue" - } - ), - "TestWorker2": WorkerConfig( - identifier="TestWorker2", - scheduling="0 3 * * *2", - storage_destination="/destination2", - storage_ssh_knownHost="knownHost2", - storage_ssh_username="username2", - storage_ssh_sshKeypairPath="sshKeyPairPath2", - docker_image="alpine:edge-overridden", - docker_entrypoint="entrypoint2", - docker_command="/bin/sh -c \"env | grep EnvLabelKeyOverridden\"", - docker_workdir="/root2", - docker_sourceMountBasePath="/backup2", - docker_env={ - "EnvLabelKey1": "EnvLabelValue12", - "EnvLabelKey2": "EnvLabelValue22", - "EnvLabelKey3": "EnvLabelValue32", - "EnvLabelKeyOverridden": "EnvLabelValueOverridden" - } - ) - }, workerConfigs) + self.assertEqual(WorkerConfig( + identifier="TestWorker1", + scheduling="0 3 * * *1", + storage_destination="/destination1", + storage_ssh_knownHost="knownHost1", + storage_ssh_username="username1", + storage_ssh_sshKeypairPath="sshKeyPairPath1", + docker_image="alpine:latest", + docker_entrypoint="entrypoint1", + docker_command="/bin/sh -c \"env | grep EnvLabelKey\"", + docker_workdir="/root1", + docker_sourceMountBasePath="/backup1", + docker_env={ + "EnvLabelKey1": "EnvLabelValue11", + "EnvLabelKey2": "EnvLabelValue21", + "EnvLabelKey3": "EnvLabelValue31", + "EnvLabelKey": "EnvLabelValue" + } + ), workerConfigs["TestWorker1"]) + + self.assertEqual(WorkerConfig( + identifier="TestWorker2", + scheduling="0 3 * * *2", + storage_destination="/destination2", + storage_ssh_knownHost="knownHost2", + storage_ssh_username="username2", + storage_ssh_sshKeypairPath="sshKeyPairPath2", + docker_image="alpine:edge-overridden", + docker_entrypoint="entrypoint2", + docker_command="/bin/sh -c \"env | grep EnvLabelKeyOverridden\"", + docker_workdir="/root2", + docker_sourceMountBasePath="/backup2", + docker_env={ + "EnvLabelKey1": "EnvLabelValue12", + "EnvLabelKey2": "EnvLabelValue22", + "EnvLabelKey3": "EnvLabelValue32", + "EnvLabelKeyOverridden": "EnvLabelValueOverridden" + } + ), workerConfigs["TestWorker2"]) def test_discovery_label_docker_envs(self): compose_up(["docker-compose-no-labels.yml", -- GitLab From da73800632fb82c0d7204219cbea8ac006d4622c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Sat, 28 Nov 2020 18:28:53 +0100 Subject: [PATCH 24/26] fix up command for docker override test --- .../docker-compose-enable-named-volume-label-docker-env.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml b/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml index f1667de..14c5ab1 100644 --- a/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml +++ b/dobas/test/docker-compose-enable-named-volume-label-docker-env.yml @@ -6,7 +6,7 @@ volumes: dobas.enable: true dobas.worker.TestWorker1.docker.env.EnvLabelKey: EnvLabelValue dobas.worker.TestWorker1.docker.image: alpine:latest - dobas.worker.TestWorker1.docker.command: bin/sh -c "env | grep EnvLabelKey" + dobas.worker.TestWorker1.docker.command: /bin/sh -c "env | grep EnvLabelKey" dobas.worker.TestWorker2.docker.env.EnvLabelKeyOverridden: EnvLabelValueOverridden dobas.worker.TestWorker2.docker.image: alpine:edge-overridden - dobas.worker.TestWorker2.docker.command: bin/sh -c "env | grep EnvLabelKeyOverridden" \ No newline at end of file + dobas.worker.TestWorker2.docker.command: /bin/sh -c "env | grep EnvLabelKeyOverridden" \ No newline at end of file -- GitLab From 2d48bda47fa0bdf2da6e490cee9853cf731b317d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Sat, 28 Nov 2020 18:29:03 +0100 Subject: [PATCH 25/26] use raw strings instead of escaping --- dobas/test/test_orchestrator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dobas/test/test_orchestrator.py b/dobas/test/test_orchestrator.py index aa14e52..4e0e739 100644 --- a/dobas/test/test_orchestrator.py +++ b/dobas/test/test_orchestrator.py @@ -99,7 +99,7 @@ class TestOrchestrator(TestCase): storage_ssh_sshKeypairPath="sshKeyPairPath1", docker_image="alpine:latest1", docker_entrypoint="entrypoint1", - docker_command="/bin/sh -c \"echo hello world\"1", + docker_command=r'/bin/sh -c "echo hello world"1', docker_workdir="/root1", docker_sourceMountBasePath="/backup1", docker_env={ @@ -117,7 +117,7 @@ class TestOrchestrator(TestCase): storage_ssh_sshKeypairPath="sshKeyPairPath2", docker_image="alpine:latest2", docker_entrypoint="entrypoint2", - docker_command="/bin/sh -c \"echo hello world\"2", + docker_command=r'/bin/sh -c "echo hello world"2', docker_workdir="/root2", docker_sourceMountBasePath="/backup2", docker_env={ @@ -153,7 +153,7 @@ class TestOrchestrator(TestCase): storage_ssh_sshKeypairPath="sshKeyPairPath1", docker_image="alpine:latest", docker_entrypoint="entrypoint1", - docker_command="/bin/sh -c \"env | grep EnvLabelKey\"", + docker_command=r'/bin/sh -c "env | grep EnvLabelKey"', docker_workdir="/root1", docker_sourceMountBasePath="/backup1", docker_env={ @@ -173,7 +173,7 @@ class TestOrchestrator(TestCase): storage_ssh_sshKeypairPath="sshKeyPairPath2", docker_image="alpine:edge-overridden", docker_entrypoint="entrypoint2", - docker_command="/bin/sh -c \"env | grep EnvLabelKeyOverridden\"", + docker_command=r'/bin/sh -c "env | grep EnvLabelKeyOverridden"', docker_workdir="/root2", docker_sourceMountBasePath="/backup2", docker_env={ -- GitLab From 18464a29076ea63b8db9008b998d3ea824641628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=BCnther?= Date: Sat, 28 Nov 2020 18:35:34 +0100 Subject: [PATCH 26/26] fix None dereference, if docker envs are not provided in default config, but are provided in label worker config --- dobas/orchestrator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dobas/orchestrator.py b/dobas/orchestrator.py index aab1924..4f01097 100644 --- a/dobas/orchestrator.py +++ b/dobas/orchestrator.py @@ -101,7 +101,8 @@ class Orchestrator: # worker config with worker name specific volume labels for volume_worker_name, volume_worker_config in volume_worker_configs.items(): # init docker.env from default config - volume_worker_docker_envs = volume_worker_configs[volume_worker_name].docker_env + default_worker_docker_envs = volume_worker_configs[volume_worker_name].docker_env + volume_worker_docker_envs = default_worker_docker_envs if default_worker_docker_envs else {} # gather docker.env from volume labels volume_worker_docker_env_label_prefix = "dobas.worker." + \ str(volume_worker_name)+".docker.env." -- GitLab