grim/local-pipelines

Add docker-py as a dependency and rework pipeline.py to use it which is a prereq for proper logging

2016-07-25, Gary Kramlich
7d2dac01ceb0
Parents 77dfff4f23f8
Children 707db01dc23a
Add docker-py as a dependency and rework pipeline.py to use it which is a prereq for proper logging
3 files changed, 70 insertions(+), 38 deletions(-)
  • +13 -1
    pipelines/core.py
  • +55 -37
    pipelines/pipeline.py
  • +2 -0
    requirements.txt
    • --- a/pipelines/core.py Mon Jul 25 20:41:45 2016 -0500
    +++ b/pipelines/core.py Mon Jul 25 23:55:16 2016 -0500
    @@ -8,6 +8,8 @@
    import yaml
    +from docker import Client
    +
    from pipelines import vcs
    from pipelines.pipeline import Pipeline
    @@ -23,6 +25,14 @@
    dest="pipeline_filename",
    )
    + parser.add_argument(
    + "-H", "--host",
    + help="Daemon socket to connect to",
    + metavar="SOCKET",
    + default="unix://var/run/docker.sock",
    + dest="host"
    + )
    +
    return parser.parse_args()
    @@ -39,10 +49,12 @@
    def main():
    args = parse_args()
    + client = Client(base_url=args.host)
    +
    path, config = _load_config(args.pipeline_filename)
    branch = vcs.get_branch(path)
    - pipeline = Pipeline(config, path, branch)
    + pipeline = Pipeline(client, config, path, branch)
    sys.exit(pipeline.run())
    --- a/pipelines/pipeline.py Mon Jul 25 20:41:45 2016 -0500
    +++ b/pipelines/pipeline.py Mon Jul 25 23:55:16 2016 -0500
    @@ -2,12 +2,12 @@
    import fnmatch
    import os
    -import subprocess
    import tempfile
    class Pipeline(object):
    - def __init__(self, config, path, branch):
    + def __init__(self, client, config, path, branch):
    + self.client = client
    self.config = config
    self.build_path = os.path.abspath(path)
    self.branch = branch
    @@ -32,52 +32,70 @@
    filename = os.path.realpath(filename)
    with os.fdopen(fd, "w") as ofp:
    + ofp.write("#!/bin/bash\n")
    +
    content = " && \\\n".join(step["script"])
    ofp.write(content)
    + ofp.write("\n")
    +
    return filename
    - def get_command(self, step):
    - cmd = [
    - "docker",
    - "run",
    - "--rm=true",
    - # TODONT cidfile This is only used by the host.
    - "--entrypoint=/bin/bash",
    - "--memory=2048m",
    - # TODO envfile
    - "-v {script_filename}:/tmp/{script_filename}:ro",
    - "-v {build_path}:/opt/atlassian/bitbucketci/agent/build:rw",
    - "-w /opt/atlassian/bitbucketci/agent/build",
    - "--label com.atlassian.pipelines.agent=\"local\"",
    - "{image}",
    - # TODO didn't work for me... "-i",
    - "/tmp/{script_filename}",
    - ]
    + def _container(self, step):
    + script_filename = self._build_script(step)
    +
    + labels = {
    + "com.atlassian.pipelines.agent": "local",
    + }
    - return " ".join(cmd).format(
    - script_filename=self._build_script(step),
    - build_path=self.build_path,
    + binds = {
    + script_filename: {
    + "bind": script_filename,
    + "mode": "ro",
    + },
    + self.build_path: {
    + "bind": "/opt/atlassian/bitbucketci/agent/build",
    + "mode": "rw"
    + },
    + }
    +
    + return self.client.create_container(
    image=self._determine_image(step),
    + command=script_filename,
    + working_dir="/opt/atlassian/bitbucketci/agent/build",
    + entrypoint="/bin/bash",
    + labels=labels,
    + host_config=self.client.create_host_config(
    + binds=binds,
    + mem_limit="2048m",
    + ),
    )
    - def _commands(self):
    - commands = []
    - print(self.steps)
    - print('-' * 40)
    - for container in self.steps:
    - step = container["step"]
    - commands.append(self.get_command(step))
    -
    - return commands
    + def _steps(self):
    + return [s["step"] for s in self.steps]
    def run(self):
    - for command in self._commands():
    - print('+ {}'.format(command))
    - proc = subprocess.Popen(command.split())
    + for n, step in enumerate(self._steps()):
    + container = self._container(step)
    +
    + try:
    + self.client.start(container=container.get("Id"))
    - return_code = proc.wait()
    - if return_code != 0:
    - return return_code
    + logs = self.client.logs(
    + container=container.get("Id"),
    + stdout=True,
    + stderr=True,
    + stream=True,
    + follow=True,
    + )
    +
    + for line in logs:
    + print('step-{} {}'.format(n, line.strip()))
    +
    + return_code = self.client.wait(container.get("Id"))
    + if return_code != 0:
    + return return_code
    + finally:
    + self.client.remove_container(container["Id"])
    return 0
    --- a/requirements.txt Mon Jul 25 20:41:45 2016 -0500
    +++ b/requirements.txt Mon Jul 25 23:55:16 2016 -0500
    @@ -1,1 +1,3 @@
    PyYAML==3.11
    +docker-py==1.8.1
    +