From 8a44efc46420157b1eb7d3d962a9c0ca4057f546 Mon Sep 17 00:00:00 2001 From: Felipe Martin Date: Wed, 29 Apr 2020 20:08:06 +0200 Subject: [PATCH] Arguments --- jeeves/cli/cli.py | 18 +++++-- jeeves/core/actions/base.py | 12 ++++- jeeves/core/actions/docker.py | 78 ++++++++++++++++++++++-------- jeeves/core/executor.py | 10 +++- jeeves/core/objects.py | 4 ++ jeeves/core/tests/test_registry.py | 2 +- 6 files changed, 97 insertions(+), 27 deletions(-) diff --git a/jeeves/cli/cli.py b/jeeves/cli/cli.py index de40578..3f356aa 100644 --- a/jeeves/cli/cli.py +++ b/jeeves/cli/cli.py @@ -24,7 +24,8 @@ def main(): default=False, help="Display output for flow", ) -def execute(defintinion_file, print_output): +@click.option("--argument", "-a", "defined_arguments", multiple=True, default=[]) +def execute(defintinion_file, print_output, defined_arguments=list): extension = os.path.splitext(defintinion_file.name)[1][1:] if not hasattr(FlowParser, f"from_{extension}"): @@ -35,8 +36,13 @@ def execute(defintinion_file, print_output): flow = getattr(FlowParser, f"from_{extension}")(defintinion_file.read()) + arguments = {} + for argument in defined_arguments: + key, value = argument.split("=") + arguments[key] = value + title(f"Running flow: {flow.name}") - executor = Executor(flow) + executor = Executor(flow=flow, defined_arguments=arguments) for n, step in enumerate(executor.steps, start=1): click.echo( @@ -44,12 +50,14 @@ def execute(defintinion_file, print_output): nl=False, ) result = executor.execute_step(step) + if not result.success: error(f"Executing step [{n}/{executor.step_count}]: {step.task.name}") + if print_output: + click.echo(result.output) break else: success(f"Running step [{n}/{executor.step_count}]: {step.task.name}") - if print_output: - title("Full output:") - click.echo("\n".join(executor._execution.output)) + if print_output: + click.echo(result.output) diff --git a/jeeves/core/actions/base.py b/jeeves/core/actions/base.py index 430e032..8a47363 100644 --- a/jeeves/core/actions/base.py +++ b/jeeves/core/actions/base.py @@ -1,6 +1,7 @@ import logging from abc import abstractmethod +from jinja2 import Template import pydantic @@ -8,11 +9,20 @@ class Action: id = "" class Parameters(pydantic.BaseModel): - pass + PARSE_WITH_ARGUMENTS = set() def __init__(self, parameters=None): self.logger = logging.getLogger(self.__class__.__name__) self.parameters = self.Parameters(**(parameters or {})) + self.parsed_parameters = {} + + def parse_parameters_with_arguments(self, **arguments): + """ + Returns a dict with the parameters parsed in base of the provided arguments. + Parsing using jinja2 template themes only on the fields defined on the `Parameters.PARSE_WITH_ARGUMENTS`. + """ + for parameter_name in self.parameters.PARSE_WITH_ARGUMENTS: + self.parsed_parameters[parameter_name] = Template(self.parameters.dict()[parameter_name]).render(**arguments) @abstractmethod def execute(self, workspace, **kwargs): diff --git a/jeeves/core/actions/docker.py b/jeeves/core/actions/docker.py index 77e175e..d1a1bc3 100644 --- a/jeeves/core/actions/docker.py +++ b/jeeves/core/actions/docker.py @@ -1,4 +1,5 @@ -from typing import Text +import os.path +from typing import Text, Optional import docker import pydantic @@ -8,57 +9,96 @@ from jeeves.core.objects import Result from .base import Action -class DockerAction(Action): +class DockerBuildAction(Action): """ - .. automethod:: _run_container """ - id = "contrib/docker" + id = "contrib/docker/build" + verbose_name = "Build docker image" + + class Parameters(pydantic.BaseModel): + """ + """ + + dockerfile_path: Text + image_name: Text + tag: Text = "latest" + + def execute(self, **kwargs): + workspace = kwargs.get("workspace") + + client = docker.from_env() + + client.images.build( + path=workspace, + fileobj=os.path.join(workspace, self.parameters.dockerfile_path), + ) + + +class DockerRunAction(Action): + """ + """ + + id = "contrib/docker/run" verbose_name = "Execute docker container" class Parameters(pydantic.BaseModel): """ - +----------------+------+-----------+----------------------------------------------+ - | Parameter name | Type | Mandatory | Description | - +================+======+===========+==============================================+ - | ``image`` | text | no | Image to run (defaults to ``alpine:latest``) | - | ``command`` | text | yes | The command to be executed | - +----------------+------+-----------+----------------------------------------------+ + +----------------------+------+-----------+--------------------------------------------------------------+ + | Parameter name | Type | Mandatory | Description | + +======================+======+===========+==============================================================+ + | ``image`` | text | no | Image to run (defaults to ``alpine:latest``) | + | ``command`` | text | yes | The command to be executed | + | ``entrypoint`` | text | no | The entrypoint to use (defaults in image) | + | ``tty`` | bool | no | Allocate pseudo-tty (defaults to ``false``) | + | ``remove_container`` | text | no | Select to remove container after run (defaults to ``true`` ) | + +----------------------+------+-----------+--------------------------------------------------------------+ """ image: Text = "alpine:latest" command: Text + entrypoint: Optional[Text] = None + tty: bool = False remove_container: bool = True - def _run_container(self): - """ - """ - pass + PARSE_WITH_ARGUMENTS = {"command", "image"} def execute(self, **kwargs): workspace = kwargs.get("workspace") + arguments = kwargs.get("arguments") image = self.parameters.image - command = self.parameters.command environment = {"WORKSPACE_PATH": "/workspace"} + # Add arguments as uppercase environment variables prefixed with JEEVES_ + if arguments: + for key, value in arguments.items(): + environment[f"JEEVES_{key.upper}"] = value + client = docker.from_env() self.logger.info("Pulling image...") try: client.images.get(image) except docker.errors.ImageNotFound: - self.logger.error("Image does not exist") + self.logger.error(f"Image '{image}' does not exist") return Result(success=False) - self.logger.info("Execute command in container...") - container = client.containers.run( + run_kwargs = dict( image=image, - command=command, + command=self.parsed_parameters["command"], detach=True, environment=environment, + tty=self.parameters.tty, volumes={"/workspace": {"bind": str(workspace.path), "mode": "rw"}}, + working_dir="/workspace", ) + if self.parameters.entrypoint: + run_kwargs["entrypoint"] = self.parameters.entrypoint + + self.logger.info("Execute command in container...") + container = client.containers.run(**run_kwargs) + try: result = container.wait(timeout=30, condition="not-running") logs = container.logs() diff --git a/jeeves/core/executor.py b/jeeves/core/executor.py index 5144292..9953bc8 100644 --- a/jeeves/core/executor.py +++ b/jeeves/core/executor.py @@ -1,14 +1,21 @@ import traceback +from typing import Dict from jeeves.core.objects import Flow, Result, Execution, ExecutionStep from jeeves.core.registry import ActionRegistry class Executor: - def __init__(self, flow: Flow): + def __init__(self, flow: Flow, defined_arguments: Dict = None): + defined_arguments = defined_arguments or {} self.step_count = len(flow.tasks) self._flow: Flow = flow self._execution = Execution(flow=flow, steps=self._get_steps(flow)) + self._arguments = {} + if self._flow.arguments: + for argument in self._flow.arguments: + # TODO: What happens if not default? + self._arguments[argument.name] = defined_arguments.get(argument.name, argument.default) @property def steps(self): @@ -22,6 +29,7 @@ class Executor: def execute_step(self, step: ExecutionStep): try: action = ActionRegistry.get_action_cls(step.task.type)(parameters=step.task.parameters) + action.parse_parameters_with_arguments(**self._arguments) step.result = action.execute(workspace=self._execution.workspace) except Exception as error: # Catch unhandled exceptions, mark the result as unsuccessful diff --git a/jeeves/core/objects.py b/jeeves/core/objects.py index 00ffcbc..ae12ac3 100644 --- a/jeeves/core/objects.py +++ b/jeeves/core/objects.py @@ -32,6 +32,7 @@ class Argument(BaseObject): class Flow(BaseObject): name: Text tasks: List[Task] = field(default_factory=list) + arguments: Optional[List[Argument]] = None class ExecutionStep(BaseObject): @@ -60,12 +61,15 @@ class Execution(BaseObject): flow: Flow steps: List[ExecutionStep] workspace: Workspace = None # type: ignore + arguments: Optional[List[Argument]] = None success: bool = False @pydantic.validator("workspace", pre=True, always=True) def workspace_default(cls, v): # pylint: disable=no-self-argument return v or Workspace() + # TODO: @pydantic.validator("arguments") + @property def output(self): for step in self.steps: diff --git a/jeeves/core/tests/test_registry.py b/jeeves/core/tests/test_registry.py index 844696e..e0fec86 100644 --- a/jeeves/core/tests/test_registry.py +++ b/jeeves/core/tests/test_registry.py @@ -23,5 +23,5 @@ def test_registry_get_action_ok(): def test_registry_namespace_conflict_ok(): ActionRegistry.register_action(StubSuccessAction) - with pytest.raises(ActionRegistry.ActionNamespaceConflict): + with pytest.raises(ActionRegistry.ActionIDConflict): ActionRegistry.register_action(StubSuccessAction)