grim/convey

Use a custom network when running plans. Fixes #113

2017-07-05, Gary Kramlich
6df88f1baa6b
Parents 485ad02d17a9
Children fcc5de335fa9
Use a custom network when running plans. Fixes #113
17 files changed, 150 insertions(+), 58 deletions(-)
  • +1 -0
    ChangeLog
  • +3 -4
    docker/build.go
  • +4 -5
    docker/export.go
  • +2 -3
    docker/import.go
  • +1 -2
    docker/login.go
  • +1 -2
    docker/logout.go
  • +1 -2
    docker/pull.go
  • +1 -2
    docker/push.go
  • +1 -2
    docker/remove.go
  • +4 -3
    docker/run.go
  • +1 -2
    docker/tag.go
  • +1 -2
    intrinsic/clean.go
  • +67 -0
    network/network.go
  • +49 -20
    plans/plans.go
  • +5 -6
    stages/stages.go
  • +7 -1
    state/state.go
  • +1 -2
    tasks/tasks.go
    • --- a/ChangeLog Tue Jul 04 17:15:12 2017 -0500
    +++ b/ChangeLog Wed Jul 05 21:41:36 2017 -0500
    @@ -1,6 +1,7 @@
    0.8.0:
    * Allow labels for the Docker run and build tasks to use environment
    variables. (fixed #112)
    + * Use a custom network for each plan invocation. (fixed #113)
    0.7.0: 20170619
    * Added support for passing arguments to the Docker build task (--build-arg).
    --- a/docker/build.go Tue Jul 04 17:15:12 2017 -0500
    +++ b/docker/build.go Wed Jul 05 21:41:36 2017 -0500
    @@ -29,7 +29,6 @@
    "bitbucket.org/rw_grim/convey/environment"
    "bitbucket.org/rw_grim/convey/state"
    "bitbucket.org/rw_grim/convey/tasks"
    - "bitbucket.org/rw_grim/convey/workspace"
    )
    type Build struct {
    @@ -47,7 +46,7 @@
    {{range .Arguments}} --build-arg {{.}}{{end}}
    {{.buildContext}}`
    -func (b *Build) Execute(name string, ws *workspace.Workspace, logger *gomol.LogAdapter, env []string, st *state.State) error {
    +func (b *Build) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    // create out build directory
    @@ -65,7 +64,7 @@
    src, dest := tasks.ParseFilePath(base, environment.Mapper(src, fullEnv))
    cleanDest := filepath.Clean(filepath.Join(tmpDir, dest))
    - if err = exportFile(name, ws.Name, src, cleanDest, st.TaskTimeout); err != nil {
    + if err = exportFile(name, st.Workspace.Name, src, cleanDest, st.TaskTimeout); err != nil {
    return err
    }
    }
    @@ -74,7 +73,7 @@
    params := map[string]interface{}{
    "source": b.Dockerfile,
    "destination": tmpDir,
    - "workspace": ws.Name,
    + "workspace": st.Workspace.Name,
    }
    err = command.Run(name, exportTemplate, params, st.TaskTimeout)
    --- a/docker/export.go Tue Jul 04 17:15:12 2017 -0500
    +++ b/docker/export.go Wed Jul 05 21:41:36 2017 -0500
    @@ -31,7 +31,6 @@
    "bitbucket.org/rw_grim/convey/environment"
    "bitbucket.org/rw_grim/convey/state"
    "bitbucket.org/rw_grim/convey/tasks"
    - "bitbucket.org/rw_grim/convey/workspace"
    )
    type Export struct {
    @@ -86,7 +85,7 @@
    return nil
    }
    -func (e *Export) Execute(name string, ws *workspace.Workspace, logger *gomol.LogAdapter, env []string, st *state.State) error {
    +func (e *Export) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    for _, file := range e.Files {
    @@ -97,15 +96,15 @@
    }
    if strings.ContainsRune(file, '*') {
    - mountPoint := environment.Mapper(ws.MountPoint, fullEnv)
    + mountPoint := environment.Mapper(st.Workspace.MountPoint, fullEnv)
    - if err := exportGlob(name, ws.Name, mountPoint, file, st.TaskTimeout); err != nil {
    + if err := exportGlob(name, st.Workspace.Name, mountPoint, file, st.TaskTimeout); err != nil {
    return err
    }
    } else {
    src, dest := tasks.ParseFilePath("", file)
    - if err := exportFile(name, ws.Name, src, dest, st.TaskTimeout); err != nil {
    + if err := exportFile(name, st.Workspace.Name, src, dest, st.TaskTimeout); err != nil {
    return err
    }
    }
    --- a/docker/import.go Tue Jul 04 17:15:12 2017 -0500
    +++ b/docker/import.go Wed Jul 05 21:41:36 2017 -0500
    @@ -26,7 +26,6 @@
    "bitbucket.org/rw_grim/convey/environment"
    "bitbucket.org/rw_grim/convey/state"
    "bitbucket.org/rw_grim/convey/tasks"
    - "bitbucket.org/rw_grim/convey/workspace"
    )
    type Import struct {
    @@ -35,7 +34,7 @@
    const importTemplate = `docker cp {{.source}} {{.workspaceID}}:/workspace/{{.destination}}`
    -func (i *Import) Execute(name string, ws *workspace.Workspace, logger *gomol.LogAdapter, env []string, st *state.State) error {
    +func (i *Import) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    for _, file := range i.Files {
    @@ -44,7 +43,7 @@
    params := map[string]interface{}{
    "source": src,
    "destination": dest,
    - "workspaceID": ws.Name,
    + "workspaceID": st.Workspace.Name,
    }
    err := command.Run(name, importTemplate, params, st.TaskTimeout)
    --- a/docker/login.go Tue Jul 04 17:15:12 2017 -0500
    +++ b/docker/login.go Wed Jul 05 21:41:36 2017 -0500
    @@ -25,7 +25,6 @@
    "bitbucket.org/rw_grim/convey/command"
    "bitbucket.org/rw_grim/convey/state"
    "bitbucket.org/rw_grim/convey/tasks"
    - "bitbucket.org/rw_grim/convey/workspace"
    )
    type Login struct {
    @@ -36,7 +35,7 @@
    const loginTemplate = `docker login -u {{.username}} -p {{.password}}{{if .server}} {{.server}}{{end}}`
    -func (l *Login) Execute(name string, ws *workspace.Workspace, logger *gomol.LogAdapter, env []string, st *state.State) error {
    +func (l *Login) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    params := map[string]interface{}{
    "username": l.Username,
    "password": l.Password,
    --- a/docker/logout.go Tue Jul 04 17:15:12 2017 -0500
    +++ b/docker/logout.go Wed Jul 05 21:41:36 2017 -0500
    @@ -25,7 +25,6 @@
    "bitbucket.org/rw_grim/convey/command"
    "bitbucket.org/rw_grim/convey/state"
    "bitbucket.org/rw_grim/convey/tasks"
    - "bitbucket.org/rw_grim/convey/workspace"
    )
    type Logout struct {
    @@ -34,7 +33,7 @@
    const logoutTemplate = `docker logout{{if .server}} {{.server}}{{end}}`
    -func (l *Logout) Execute(name string, ws *workspace.Workspace, logger *gomol.LogAdapter, env []string, st *state.State) error {
    +func (l *Logout) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    params := map[string]interface{}{
    "server": l.Server,
    }
    --- a/docker/pull.go Tue Jul 04 17:15:12 2017 -0500
    +++ b/docker/pull.go Wed Jul 05 21:41:36 2017 -0500
    @@ -26,7 +26,6 @@
    "bitbucket.org/rw_grim/convey/environment"
    "bitbucket.org/rw_grim/convey/state"
    "bitbucket.org/rw_grim/convey/tasks"
    - "bitbucket.org/rw_grim/convey/workspace"
    )
    type Pull struct {
    @@ -36,7 +35,7 @@
    const pullTemplate = `docker pull {{.image}}`
    -func (p *Pull) Execute(name string, ws *workspace.Workspace, logger *gomol.LogAdapter, env []string, st *state.State) error {
    +func (p *Pull) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    for _, image := range p.Images {
    --- a/docker/push.go Tue Jul 04 17:15:12 2017 -0500
    +++ b/docker/push.go Wed Jul 05 21:41:36 2017 -0500
    @@ -26,7 +26,6 @@
    "bitbucket.org/rw_grim/convey/environment"
    "bitbucket.org/rw_grim/convey/state"
    "bitbucket.org/rw_grim/convey/tasks"
    - "bitbucket.org/rw_grim/convey/workspace"
    )
    type Push struct {
    @@ -36,7 +35,7 @@
    const pushTemplate = `docker push {{.image}}`
    -func (p *Push) Execute(name string, ws *workspace.Workspace, logger *gomol.LogAdapter, env []string, st *state.State) error {
    +func (p *Push) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    for _, image := range p.Images {
    --- a/docker/remove.go Tue Jul 04 17:15:12 2017 -0500
    +++ b/docker/remove.go Wed Jul 05 21:41:36 2017 -0500
    @@ -26,7 +26,6 @@
    "bitbucket.org/rw_grim/convey/environment"
    "bitbucket.org/rw_grim/convey/state"
    "bitbucket.org/rw_grim/convey/tasks"
    - "bitbucket.org/rw_grim/convey/workspace"
    )
    type Remove struct {
    @@ -36,7 +35,7 @@
    const removeTemplate = `docker rmi {{.image}}`
    -func (r *Remove) Execute(name string, ws *workspace.Workspace, logger *gomol.LogAdapter, env []string, st *state.State) error {
    +func (r *Remove) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    for _, image := range r.Images {
    --- a/docker/run.go Tue Jul 04 17:15:12 2017 -0500
    +++ b/docker/run.go Wed Jul 05 21:41:36 2017 -0500
    @@ -31,7 +31,6 @@
    "bitbucket.org/rw_grim/convey/environment"
    "bitbucket.org/rw_grim/convey/state"
    "bitbucket.org/rw_grim/convey/tasks"
    - "bitbucket.org/rw_grim/convey/workspace"
    )
    type Run struct {
    @@ -70,11 +69,12 @@
    {{if .SSHAgent }} -e SSH_AUTH_SOCK -v {{.SSHAuthSock}}:{{.SSHAuthSock}}{{end}}
    {{if .EntryPoint}} --entrypoint {{.EntryPoint}}{{end}}
    {{if .WorkDir}} -w {{.WorkDir}}{{end}}
    +{{if .Network}} --network {{.Network}}{{end}}
    {{range .Labels}} -l '{{.}}'{{end}}
    {{range .Environment}} -e {{.}}{{end}}
    {{.Image}}{{if .Command}} {{.Command}}{{end}}`
    -func (r *Run) Execute(name string, ws *workspace.Workspace, logger *gomol.LogAdapter, env []string, st *state.State) error {
    +func (r *Run) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, r.Environment)
    fullEnv = environment.Merge(fullEnv, st.Environment)
    @@ -157,12 +157,13 @@
    "Image": environment.Mapper(r.Image, fullEnv),
    "Labels": environment.SliceMapper(r.Labels, fullEnv),
    "Memory": st.Memory,
    + "Network": st.Network.Name,
    "ScriptFile": scriptFile,
    "SSHAgent": st.EnableSSHAgent,
    "SSHAuthSock": os.Getenv("SSH_AUTH_SOCK"),
    "UID": user.Uid,
    "WorkDir": environment.Mapper(r.WorkDir, fullEnv),
    - "WorkspacePath": environment.Mapper(ws.MountPoint, fullEnv),
    + "WorkspacePath": environment.Mapper(st.Workspace.MountPoint, fullEnv),
    "WorkspaceMount": environment.Mapper(workSpace, fullEnv),
    }
    --- a/docker/tag.go Tue Jul 04 17:15:12 2017 -0500
    +++ b/docker/tag.go Wed Jul 05 21:41:36 2017 -0500
    @@ -26,7 +26,6 @@
    "bitbucket.org/rw_grim/convey/environment"
    "bitbucket.org/rw_grim/convey/state"
    "bitbucket.org/rw_grim/convey/tasks"
    - "bitbucket.org/rw_grim/convey/workspace"
    )
    type Tag struct {
    @@ -37,7 +36,7 @@
    const tagTemplate = `docker tag {{.source}} {{.destination}}`
    -func (t *Tag) Execute(name string, ws *workspace.Workspace, logger *gomol.LogAdapter, env []string, st *state.State) error {
    +func (t *Tag) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    for _, destination := range t.Destinations {
    --- a/intrinsic/clean.go Tue Jul 04 17:15:12 2017 -0500
    +++ b/intrinsic/clean.go Wed Jul 05 21:41:36 2017 -0500
    @@ -29,7 +29,6 @@
    "bitbucket.org/rw_grim/convey/environment"
    "bitbucket.org/rw_grim/convey/state"
    "bitbucket.org/rw_grim/convey/tasks"
    - "bitbucket.org/rw_grim/convey/workspace"
    )
    type Clean struct {
    @@ -65,7 +64,7 @@
    return list, nil
    }
    -func (c *Clean) Execute(name string, ws *workspace.Workspace, logger *gomol.LogAdapter, env []string, st *state.State) error {
    +func (c *Clean) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    wd, err := os.Getwd()
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/network/network.go Wed Jul 05 21:41:36 2017 -0500
    @@ -0,0 +1,67 @@
    +/*
    + * Convey
    + * Copyright 2016-2017 Gary Kramlich <grim@reaperworld.com>
    + *
    + * This program is free software: you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License as published by
    + * the Free Software Foundation, either version 3 of the License, or
    + * (at your option) any later version.
    + *
    + * This program is distributed in the hope that it will be useful,
    + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    + * GNU General Public License for more details.
    + *
    + * You should have received a copy of the GNU General Public License
    + * along with this program. If not, see <http://www.gnu.org/licenses/>.
    + */
    +package network
    +
    +import (
    + "time"
    +
    + "github.com/aphistic/gomol"
    + "github.com/satori/go.uuid"
    +
    + "bitbucket.org/rw_grim/convey/command"
    + "bitbucket.org/rw_grim/convey/logging"
    +)
    +
    +type Network struct {
    + Name string
    +
    + logger *gomol.LogAdapter
    +}
    +
    +const (
    + createTemplate = `docker network create {{.Name}}`
    + destroyTemplate = `docker network rm {{.Name}}`
    +)
    +
    +func Create() (*Network, error) {
    + network := &Network{
    + Name: uuid.NewV4().String(),
    + logger: logging.NewAdapter("network"),
    + }
    +
    + params := map[string]interface{}{
    + "Name": network.Name,
    + }
    +
    + _, _, err := command.RunOutput("create network", createTemplate, params, time.Minute)
    + if err != nil {
    + return nil, err
    + }
    +
    + network.logger.Debugf("created network: %#v", network.Name)
    +
    + return network, nil
    +}
    +
    +func (network *Network) Destroy() error {
    + params := map[string]interface{}{
    + "Name": network.Name,
    + }
    +
    + return command.Run("remove network", destroyTemplate, params, time.Minute)
    +}
    --- a/plans/plans.go Tue Jul 04 17:15:12 2017 -0500
    +++ b/plans/plans.go Wed Jul 05 21:41:36 2017 -0500
    @@ -24,6 +24,7 @@
    "bitbucket.org/rw_grim/convey/environment"
    "bitbucket.org/rw_grim/convey/logging"
    + "bitbucket.org/rw_grim/convey/network"
    "bitbucket.org/rw_grim/convey/stages"
    "bitbucket.org/rw_grim/convey/state"
    "bitbucket.org/rw_grim/convey/tasks"
    @@ -35,9 +36,19 @@
    Stages []stages.Stage `yaml:"stages"`
    }
    -func (p *Plan) Execute(path string, tasks map[string]tasks.Task, env []string, st *state.State) error {
    - logger := logging.NewAdapter(path)
    +func (p *Plan) setup(logger *gomol.LogAdapter, st *state.State) error {
    + // create the network
    + logger.Info("creating network...")
    + network, err := network.Create()
    + if err != nil {
    + logger.Fatal("failed to create network")
    + return err
    + }
    + st.Network = network
    + logger.Infof("created network %s", st.Network.Name)
    +
    + // create the workspace
    logger.Info("creating workspace...")
    ws, err := workspace.Create()
    if err != nil {
    @@ -45,10 +56,44 @@
    return err
    }
    - logger.Infof("created workspace %s", ws.Name)
    + st.Workspace = ws
    + logger.Infof("created workspace %s", st.Workspace.Name)
    +
    + return nil
    +}
    +
    +func (p *Plan) teardown(logger *gomol.LogAdapter, st *state.State) {
    + // remove the workspace (if requested)
    + if st.KeepWorkspace {
    + logger.Infof("not removing workspace %s as requested", st.Workspace.Name)
    + } else {
    + logger.Infof("removing workspace %s", st.Workspace.Name)
    + if err := st.Workspace.Destroy(); err != nil {
    + logger.Fatalf("failed to remove workspace %s", st.Workspace.Name)
    + } else {
    + logger.Infof("removed workspace %s", st.Workspace.Name)
    + }
    + }
    +
    + // remove the network
    + logger.Infof("removing network %s")
    + if err := st.Network.Destroy(); err != nil {
    + logger.Fatalf("failed to remove network %s", st.Network.Name)
    + } else {
    + logger.Infof("removed network %s", st.Network.Name)
    + }
    +}
    +
    +func (p *Plan) Execute(path string, tasks map[string]tasks.Task, env []string, st *state.State) error {
    + logger := logging.NewAdapter(path)
    planEnv := environment.Merge(env, p.Environment)
    + defer p.teardown(logger, st)
    + if err := p.setup(logger, st); err != nil {
    + return err
    + }
    +
    // set a flag on whether or not we've failed
    var planErr error
    @@ -60,7 +105,7 @@
    stageLogger.Info("stage starting")
    - err := stage.Execute(absStageName, stageLogger, ws, tasks, planEnv, st)
    + err := stage.Execute(absStageName, stageLogger, tasks, planEnv, st)
    if err != nil {
    stageLogger.Fatal("stage failed")
    @@ -75,21 +120,5 @@
    }
    }
    - cleanupWorkspace(logger, ws, st.KeepWorkspace)
    -
    return planErr
    }
    -
    -func cleanupWorkspace(logger *gomol.LogAdapter, ws *workspace.Workspace, keep bool) {
    - if keep {
    - logger.Infof("not removing workspace %s as requested", ws.Name)
    - } else {
    - logger.Infof("removing workspace %s", ws.Name)
    - if err := ws.Destroy(); err != nil {
    - logger.Fatalf("failed to remove workspace %s", ws.Name)
    -
    - return
    - }
    - logger.Infof("removed workspace %s", ws.Name)
    - }
    -}
    --- a/stages/stages.go Tue Jul 04 17:15:12 2017 -0500
    +++ b/stages/stages.go Wed Jul 05 21:41:36 2017 -0500
    @@ -27,7 +27,6 @@
    "bitbucket.org/rw_grim/convey/logging"
    "bitbucket.org/rw_grim/convey/state"
    "bitbucket.org/rw_grim/convey/tasks"
    - "bitbucket.org/rw_grim/convey/workspace"
    )
    type Stage struct {
    @@ -52,7 +51,7 @@
    return nil
    }
    -func (s *Stage) Execute(path string, logger *gomol.LogAdapter, ws *workspace.Workspace, taskMap map[string]tasks.Task, env []string, st *state.State) error {
    +func (s *Stage) Execute(path string, logger *gomol.LogAdapter, taskMap map[string]tasks.Task, env []string, st *state.State) error {
    stageEnv := environment.Merge(env, s.Environment)
    if s.Concurrent && !st.ForceSequential {
    @@ -82,7 +81,7 @@
    wg.Add(1)
    go func(path, name string, res chan error) {
    defer wg.Done()
    - res <- s.runTask(path, name, stageEnv, ws, taskMap, st)
    + res <- s.runTask(path, name, stageEnv, taskMap, st)
    }(path, taskName, taskRes)
    }
    @@ -95,7 +94,7 @@
    } else {
    // serial execution
    for _, taskName := range s.Tasks {
    - err := s.runTask(path, taskName, stageEnv, ws, taskMap, st)
    + err := s.runTask(path, taskName, stageEnv, taskMap, st)
    if err != nil {
    return err
    }
    @@ -105,7 +104,7 @@
    return nil
    }
    -func (s *Stage) runTask(path, name string, stageEnv []string, ws *workspace.Workspace, taskMap map[string]tasks.Task, st *state.State) error {
    +func (s *Stage) runTask(path, name string, stageEnv []string, taskMap map[string]tasks.Task, st *state.State) error {
    absTaskName := fmt.Sprintf("%s/%s", path, name)
    taskLogger := logging.NewAdapter(fmt.Sprintf("%s/%s", path, name))
    taskLogger.Info("starting")
    @@ -117,7 +116,7 @@
    return fmt.Errorf("task %s not found", absTaskName)
    }
    - err := task.Execute(absTaskName, ws, taskLogger, stageEnv, st)
    + err := task.Execute(absTaskName, taskLogger, stageEnv, st)
    if err != nil {
    taskLogger.Fatalf("failed, %s", err)
    --- a/state/state.go Tue Jul 04 17:15:12 2017 -0500
    +++ b/state/state.go Wed Jul 05 21:41:36 2017 -0500
    @@ -21,10 +21,16 @@
    "fmt"
    "os"
    "time"
    +
    + "bitbucket.org/rw_grim/convey/network"
    + "bitbucket.org/rw_grim/convey/workspace"
    )
    type State struct {
    - KeepWorkspace bool
    + Network *network.Network
    + Workspace *workspace.Workspace
    + KeepWorkspace bool
    +
    ForceSequential bool
    EnableSSHAgent bool
    TaskTimeout time.Duration
    --- a/tasks/tasks.go Tue Jul 04 17:15:12 2017 -0500
    +++ b/tasks/tasks.go Wed Jul 05 21:41:36 2017 -0500
    @@ -21,11 +21,10 @@
    "github.com/aphistic/gomol"
    "bitbucket.org/rw_grim/convey/state"
    - "bitbucket.org/rw_grim/convey/workspace"
    )
    type Task interface {
    - Execute(name string, ws *workspace.Workspace, logger *gomol.LogAdapter, env []string, st *state.State) error
    + Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error
    New() Task
    Valid() error
    }