grim/convey

Add a signal handler.
cleanup-on-signal
2017-10-11, Eric Fritz
9f1221fc112f
Parents 4477a08d8a0b
Children b5b85405beca
Add a signal handler.
8 files changed, 250 insertions(+), 39 deletions(-)
  • +103 -0
    cleanup/cleanup.go
  • +26 -0
    cleanup/errors.go
  • +3 -1
    docker/build.go
  • +20 -2
    docker/network.go
  • +43 -29
    docker/run.go
  • +5 -1
    main.go
  • +3 -2
    plans/plans.go
  • +47 -4
    state/state.go
    • --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/cleanup/cleanup.go Wed Oct 11 22:00:46 2017 -0500
    @@ -0,0 +1,103 @@
    +/*
    + * Convey
    + * Copyright 2016-2017 Gary Kramlich <grim@reaperworld.com>
    + *
    + * This program is free software: you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License as published by
    + * the Free Software Foundation, either version 3 of the License, or
    + * (at your option) any later version.
    + *
    + * This program is distributed in the hope that it will be useful,
    + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    + * GNU General Public License for more details.
    + *
    + * You should have received a copy of the GNU General Public License
    + * along with this program. If not, see <http://www.gnu.org/licenses/>.
    + */
    +package cleanup
    +
    +import (
    + "os"
    + "os/signal"
    + "sync"
    + "syscall"
    +
    + "bitbucket.org/rw_grim/convey/logging"
    +
    + "github.com/satori/go.uuid"
    +)
    +
    +type (
    + // List is a list of things that should happen in case the
    + // program receives a signal. This should be things like making
    + // sure temp files are deleted, volumes are destroyed, etc.
    + List struct {
    + funcs map[string]Func
    + mutex sync.RWMutex
    + }
    +
    + // Func is a function to run on exit (it should be quick).
    + Func func()
    +)
    +
    +// NewList will create a new List that will run all registered
    +// functions on SIGINT or SIGTERM.
    +func NewList() *List {
    + var (
    + ch = make(chan os.Signal, 1)
    + list = &List{funcs: map[string]Func{}}
    + adapter = logging.NewAdapter("main")
    + )
    +
    + go func() {
    + <-ch
    + adapter.Fatal("Received signal, cleaning up")
    + list.run()
    + os.Exit(0)
    + }()
    +
    + signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
    + return list
    +}
    +
    +// Add will register a function to be invoked on system exit
    +// unless the returned function is called. The given function
    +// is wrapped in the return function, so it can be used in its
    +// place. This function MUST be called in order to deregister
    +// it on exit if it cannot be called idempotently.
    +func (c *List) Add(fn Func) func() {
    + c.mutex.Lock()
    + defer c.mutex.Unlock()
    +
    + name := ShortID()
    + c.funcs[name] = fn
    + return func() { c.fire(name) }
    +}
    +
    +func (c *List) fire(name string) error {
    + c.mutex.Lock()
    + defer c.mutex.Unlock()
    +
    + fn, ok := c.funcs[name]
    + if !ok {
    + return errNoFunc
    + }
    +
    + fn()
    + delete(c.funcs, name)
    + return nil
    +}
    +
    +func (c *List) run() {
    + c.mutex.Lock()
    + defer c.mutex.Unlock()
    +
    + for _, fn := range c.funcs {
    + fn()
    + }
    +}
    +
    +func ShortID() string {
    + return uuid.NewV4().String()[0:6]
    +}
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/cleanup/errors.go Wed Oct 11 22:00:46 2017 -0500
    @@ -0,0 +1,26 @@
    +/*
    + * Convey
    + * Copyright 2016-2017 Gary Kramlich <grim@reaperworld.com>
    + *
    + * This program is free software: you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License as published by
    + * the Free Software Foundation, either version 3 of the License, or
    + * (at your option) any later version.
    + *
    + * This program is distributed in the hope that it will be useful,
    + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    + * GNU General Public License for more details.
    + *
    + * You should have received a copy of the GNU General Public License
    + * along with this program. If not, see <http://www.gnu.org/licenses/>.
    + */
    +package cleanup
    +
    +import (
    + "errors"
    +)
    +
    +var (
    + errNoFunc = errors.New("no such cleanup function")
    +)
    --- a/docker/build.go Wed Oct 11 20:57:52 2017 -0500
    +++ b/docker/build.go Wed Oct 11 22:00:46 2017 -0500
    @@ -59,7 +59,9 @@
    if err != nil {
    return err
    }
    - defer os.RemoveAll(tmpDir)
    +
    + // Remove the dir on defer OR on signal exit (first one to happen)
    + defer st.CleanupList.Add(func() { os.RemoveAll(tmpDir) })()
    dockerfile, err := environment.Mapper(b.Dockerfile, fullEnv)
    if err != nil {
    --- a/docker/network.go Wed Oct 11 20:57:52 2017 -0500
    +++ b/docker/network.go Wed Oct 11 22:00:46 2017 -0500
    @@ -33,8 +33,9 @@
    }
    const (
    - networkCreateTemplate = `network create {{.Name}}`
    - networkDestroyTemplate = `network rm {{.Name}}`
    + networkCreateTemplate = `network create {{.Name}}`
    + networkDisconnectTemplate = `network disconnect --force {{.Name}} {{.ContainerName}}`
    + networkDestroyTemplate = `network rm {{.Name}}`
    )
    func NewNetwork(st *state.State) (*Network, error) {
    @@ -63,6 +64,23 @@
    }
    func (network *Network) Destroy() error {
    + for _, name := range network.state.GetRunning() {
    + params := map[string]interface{}{
    + "Name": network.name,
    + "ContainerName": name,
    + }
    +
    + network.logger.Infof("disconnecting container %s from network %s", name, network.Name())
    + err := Docker("disconnect container", networkDisconnectTemplate, params, network.state)
    +
    + if err != nil {
    + network.logger.Warningf("failed to disconnect container %s from network %s", name, network.Name())
    + continue
    + }
    +
    + network.logger.Infof("disconnected container %s from network %s", name, network.Name())
    + }
    +
    params := map[string]interface{}{
    "Name": network.name,
    }
    --- a/docker/run.go Wed Oct 11 20:57:52 2017 -0500
    +++ b/docker/run.go Wed Oct 11 22:00:46 2017 -0500
    @@ -28,6 +28,7 @@
    "github.com/aphistic/gomol"
    + "bitbucket.org/rw_grim/convey/cleanup"
    "bitbucket.org/rw_grim/convey/environment"
    "bitbucket.org/rw_grim/convey/normalize"
    "bitbucket.org/rw_grim/convey/state"
    @@ -64,6 +65,7 @@
    }
    const runTemplate = `run --rm
    +{{if .Name}} --name {{.Name}}{{end}}
    {{if .UID}} -e UID={{.UID}}{{end}}
    {{if .GID}} -e GID={{.GID}}{{end}}
    {{if .Detach}} -d{{end}}
    @@ -89,37 +91,13 @@
    {{end}}
    {{.Image}}{{if .Command}} {{.Command}}{{end}}`
    -// buildScript will create a shell script for the given commands
    -func (r *Run) buildScript(st *state.State, fullEnv []string) (string, string, string, error) {
    +// writeScript will write a shell script to the given tempfile for the given commands
    +func (r *Run) writeScript(st *state.State, script *os.File, fullEnv []string) (string, string, string, error) {
    entryPoint := r.Shell
    if entryPoint == "" {
    entryPoint = "/bin/sh"
    }
    - // figure out the current working directory
    - pwd, err := os.Getwd()
    - if err != nil {
    - return "", "", "", err
    - }
    -
    - // now get the absolute path
    - absPwd, err := filepath.Abs(pwd)
    - if err != nil {
    - return "", "", "", err
    - }
    -
    - // create the temp file to write the script to
    - script, err := ioutil.TempFile(absPwd, "convey-script-")
    - if err != nil {
    - return "", "", "", err
    - }
    -
    - // set scriptFile to the name of the temp file
    - scriptFile := script.Name()
    -
    - // set the run command argument to the script file
    - commandArg := scriptFile
    -
    // Scripts must retain order, so don't use st.MapSlice to
    // expand things (which results in a non-deterministically
    // ordered slice of the expanded input). It also doesn't
    @@ -130,6 +108,12 @@
    return "", "", "", err
    }
    + // set scriptFile to the name of the temp file
    + scriptFile := script.Name()
    +
    + // set the run command argument to the script file
    + commandArg := scriptFile
    +
    // write the script to the file
    script.WriteString(strings.Join(scripts, "\n"))
    script.Close()
    @@ -177,13 +161,31 @@
    // if we're using a script defined in the yaml, create it and override
    // some variables
    if len(r.Script) > 0 {
    - scriptFile, entryPoint, commandArg, err = r.buildScript(st, fullEnv)
    + // figure out the current working directory
    + pwd, err := os.Getwd()
    + if err != nil {
    + return err
    + }
    +
    + // now get the absolute path
    + absPwd, err := filepath.Abs(pwd)
    if err != nil {
    return err
    }
    - // remove the file when the function exits
    - defer os.Remove(scriptFile)
    + // create the temp file to write the script to
    + script, err := ioutil.TempFile(absPwd, "convey-script-")
    + if err != nil {
    + return err
    + }
    +
    + // Remove the script on defer OR on signal exit (first one to happen)
    + defer st.CleanupList.Add(func() { os.Remove(script.Name()) })()
    +
    + scriptFile, entryPoint, commandArg, err = r.writeScript(st, script, fullEnv)
    + if err != nil {
    + return err
    + }
    }
    taskLabel := normalize.Normalize(
    @@ -225,6 +227,9 @@
    return err
    }
    + runID := cleanup.ShortID()
    + logger.Infof("running container with id %s", runID)
    +
    // build the dict for the template
    params := map[string]interface{}{
    "Command": commandArg,
    @@ -247,6 +252,7 @@
    "WorkspacePath": workspacePath,
    "WorkspaceMount": workspaceMount,
    "TaskLabel": taskLabel,
    + "Name": runID,
    }
    if r.Detach {
    @@ -308,6 +314,14 @@
    return nil
    }
    + // Mark running so we can detach this container form the network
    + // when we've got a signal to shutdown. An active network cannot
    + // be removed, so we need to track things to kill along with it.
    + // This is no longer a problem once the subprocess returns.
    +
    + st.MarkRunning(runID)
    + defer st.UnmarkRunning(runID)
    +
    // run the command
    return Docker(name, runTemplate, params, st)
    }
    --- a/main.go Wed Oct 11 20:57:52 2017 -0500
    +++ b/main.go Wed Oct 11 22:00:46 2017 -0500
    @@ -25,6 +25,7 @@
    "github.com/alecthomas/kingpin"
    "github.com/aphistic/gomol"
    + "bitbucket.org/rw_grim/convey/cleanup"
    "bitbucket.org/rw_grim/convey/config"
    "bitbucket.org/rw_grim/convey/environment"
    "bitbucket.org/rw_grim/convey/loaders/bitbucket"
    @@ -54,7 +55,7 @@
    listTasks = app.Flag("list-tasks", "List the supported tasks").Short('L').Default("false").Bool()
    listMetaPlans = app.Flag("list-meta-plans", "List the meta plans that are available").Short('M').Default("false").Bool()
    listPlans = app.Flag("list-plans", "List the plans that are available").Short('P').Default("false").Bool()
    - memory = app.Flag("memory", "The ammount of memor to give the run task").Short('m').String()
    + memory = app.Flag("memory", "The amount of memory to give the run task").Short('m').String()
    showConfig = app.Flag("show-config", "Show a dump of the config file").Short('C').Hidden().Default("false").Bool()
    sshAgent = app.Flag("ssh-agent", "A shortcut for --ssh-identity=*").Default("false").Bool()
    sshIdentities = app.Flag("ssh-identity", "Enable ssh-agent for the given identities").Strings()
    @@ -131,7 +132,10 @@
    os.Exit(1)
    }
    + cleanupList := cleanup.NewList()
    +
    st := state.New()
    + st.CleanupList = cleanupList
    st.KeepWorkspace = *keep
    st.ForceSequential = *forceSequential
    st.EnableSSHAgent = enableSSHAgent
    --- a/plans/plans.go Wed Oct 11 20:57:52 2017 -0500
    +++ b/plans/plans.go Wed Oct 11 22:00:46 2017 -0500
    @@ -94,10 +94,11 @@
    func (p *Plan) Execute(path string, tasks map[string]tasks.Task, env []string, st *state.State) error {
    logger := logging.NewAdapter(path)
    -
    planEnv := environment.Merge(env, p.Environment)
    - defer p.teardown(logger, st)
    + // Teardown plan on defer OR on signal exit (first one to happen)
    + defer st.CleanupList.Add(func() { p.teardown(logger, st) })()
    +
    if err := p.setup(logger, st); err != nil {
    return err
    }
    --- a/state/state.go Wed Oct 11 20:57:52 2017 -0500
    +++ b/state/state.go Wed Oct 11 22:00:46 2017 -0500
    @@ -24,6 +24,7 @@
    "sync"
    "time"
    + "bitbucket.org/rw_grim/convey/cleanup"
    "bitbucket.org/rw_grim/convey/environment"
    "bitbucket.org/rw_grim/convey/network"
    "bitbucket.org/rw_grim/convey/workspace"
    @@ -32,6 +33,7 @@
    const ExpansionLimit = 100
    type State struct {
    + CleanupList *cleanup.List
    Network network.Network
    Workspace workspace.Workspace
    KeepWorkspace bool
    @@ -56,16 +58,18 @@
    expandables []string
    expandableDelimiter string
    - // This list is a stash of container names which are run
    - // in detached mode. Appending to this may happen from
    - // multiple goroutines, so this needs to be guarded via
    - // mutex.
    + // This list is a stash of container names which are either
    + // currently running or running in detached mode. Appending
    + // to this may happen from multiple goroutines, so this needs
    + // to be guarded via mutex.
    + runningContainers map[string]struct{}
    detachedContainers map[string]struct{}
    mutex *sync.RWMutex
    }
    func New() *State {
    return &State{
    + runningContainers: map[string]struct{}{},
    detachedContainers: map[string]struct{}{},
    mutex: &sync.RWMutex{},
    }
    @@ -204,6 +208,7 @@
    env = environment.Merge(env, st.Environment)
    return &State{
    + CleanupList: st.CleanupList,
    Network: st.Network,
    Workspace: st.Workspace,
    KeepWorkspace: st.KeepWorkspace,
    @@ -220,6 +225,44 @@
    }
    }
    +func (st *State) MarkRunning(name string) {
    + if st.parent != nil {
    + st.parent.MarkRunning(name)
    + return
    + }
    +
    + st.mutex.Lock()
    + defer st.mutex.Unlock()
    + st.runningContainers[name] = struct{}{}
    +}
    +
    +func (st *State) UnmarkRunning(name string) {
    + if st.parent != nil {
    + st.parent.UnmarkRunning(name)
    + return
    + }
    +
    + st.mutex.Lock()
    + defer st.mutex.Unlock()
    + delete(st.runningContainers, name)
    +}
    +
    +func (st *State) GetRunning() []string {
    + if st.parent != nil {
    + return st.parent.GetDetached()
    + }
    +
    + st.mutex.RLock()
    + defer st.mutex.RUnlock()
    +
    + names := []string{}
    + for key := range st.runningContainers {
    + names = append(names, key)
    + }
    +
    + return names
    +}
    +
    // MarkDetached will add the given container name into the list
    // of containers running in detached mode which must be shut down
    // at the end of the plan. This falls through directly to the root