grim/convey

Bump the version for release
v0.14.0-alpha3
2018-02-20, Gary Kramlich
166a6d1979fa
Bump the version for release
// Convey
// Copyright 2016-2018 Gary Kramlich <grim@reaperworld.com>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Package state contains the type that maintain the state during a run.
package state
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/aphistic/gomol"
"bitbucket.org/rw_grim/convey/environment"
"bitbucket.org/rw_grim/convey/logging"
"bitbucket.org/rw_grim/convey/network"
"bitbucket.org/rw_grim/convey/workspace"
)
const expansionLimit = 100
// State holds all of the runtime data during a run.
type State struct {
Directory string
logger *gomol.LogAdapter
cleanupList *cleanupList
CfgPath string
Network network.Network
Workspace workspace.Workspace
KeepWorkspace bool
DisableDeprecated bool
ForceSequential bool
EnableSSHAgent bool
PlanTimeout time.Duration
DockerConfig string
CPUShares string
Memory string
// Guard the environment around accessors so that the base
// environment can be modified while allowing all "wrapped"
// environments to get the same updates.
innerEnv []string
// Additional environment variables that are applied from
// the wrapper around the base env. This is applied on demand
// to the base environment so that we don't keep around a
// stale cache.
outerEnv []string
// States have the ability to "wrap" another one without
// changing the underlying state. This is used by the
// extends intrinsic in order to modify the stat without
// requiring unique access to the state object during the
// execution of the extended task. A past implementation
// had modified the stack directly, but that causes an
// extended task to clobber other tasks in concurrent mode.
parent *State
expandables []string
expandableDelimiter string
// This list is a stash of container names which are either
// currently running or running in detached mode. Appending
// to this may happen from multiple goroutines, so this needs
// to be guarded via mutex.
runningContainers map[string]struct{}
detachedContainers map[string]struct{}
mutex sync.RWMutex
}
// New creates a new state.
func New() *State {
dir, err := createDirectory()
if err != nil {
panic(err)
}
st := &State{
Directory: dir,
logger: logging.NewAdapter("state"),
cleanupList: newCleanupList(),
runningContainers: map[string]struct{}{},
detachedContainers: map[string]struct{}{},
}
st.cleanupOnSignal()
return st
}
// createDirectory creates the state directory for this run.
func createDirectory() (string, error) {
pwd, err := os.Getwd()
if err != nil {
return "", err
}
parent := filepath.Join(pwd, ".convey")
err = os.MkdirAll(parent, 0700)
if err != nil {
return "", err
}
dir, err := ioutil.TempDir(parent, "")
if err != nil {
return "", err
}
return dir, nil
}
// TaskDirectory will create a directory in the state directory for the named task.
func (st *State) TaskDirectory(name string) (string, error) {
dir := filepath.Join(st.Directory, name)
err := os.MkdirAll(dir, 0700)
return dir, err
}
// Valid validates whether the state is correct or not.
func (st *State) Valid() error {
if st.parent == nil && st.detachedContainers == nil {
return fmt.Errorf("state must be constructed via New")
}
if st.EnableSSHAgent {
if val := os.Getenv("SSH_AUTH_SOCK"); val == "" {
return fmt.Errorf("ssh-agent forwarding requested, but agent not running")
}
}
return nil
}
// GetEnv gets the fullEnv with the wrapped environments applied.
func (st *State) GetEnv() []string {
return st.mergeRecursive(st.getInnerEnv())
}
// getInnerEnv retrieves the original state environment (without wrapping).
func (st *State) getInnerEnv() []string {
if st.parent != nil {
return st.parent.getInnerEnv()
}
return st.innerEnv
}
// mergeRecursive applies the wrapped environments recursively.
func (st *State) mergeRecursive(env []string) []string {
if st.parent != nil {
// Merge recursively and then apply the wrapped environment so that
// the wrapping takes the highest precedence.
return environment.Merge(st.outerEnv, st.parent.mergeRecursive(env))
}
return env
}
// MergeEnv adds additional environment arguments with lower precedence to
// the original state environment. If they duplicate an existing environment
// variable, that value will be unchanged.
func (st *State) MergeEnv(env []string) {
if st.parent != nil {
st.parent.MergeEnv(env)
return
}
st.innerEnv = environment.Merge(env, st.innerEnv)
}
// MapSlice calls SliceMapper on the given environment, but also checks to
// see if the variable in the env parameter can be expanded into a list.
func (st *State) MapSlice(slice, fullEnv []string) ([]string, error) {
prev := slice
// Protect ourselves against a weird infinite expansion. This can
// happen if something occurs like X => A,$Y; Y => B,$X. This is
// never a useful case - a high expansion limit should catch this
// without ever hitting a practical edge.
for i := 0; i < expansionLimit; i++ {
next, err := st.expandSlice(prev, fullEnv)
if err != nil {
return nil, err
}
// If we haven't made a change, return the final result.
if isSame(next, prev) {
return environment.SliceMapper(next, fullEnv)
}
prev = next
}
return nil, fmt.Errorf("hit limit while expanding '%s'", slice)
}
func (st *State) expandSlice(env, fullEnv []string) ([]string, error) {
all := []string{}
for _, data := range env {
expanded, err := st.expand(data, fullEnv)
if err != nil {
return nil, err
}
all = append(all, expanded...)
}
return removeDuplicates(all), nil
}
// expand will attempt to expand any variables stored on the state expand
// stack. If no expandable variables are found, then the standard mapper
// is used. If multiple expandable variables are around, the mapper will
// be applied to the cartesian product of the variables.
func (st *State) expand(data string, fullEnv []string) ([]string, error) {
expansions := map[string][]string{}
// First, extract all of the expandable names and get their values
// in the full environment and split them by the proper delimiter.
for _, name := range getNames(data) {
if delimiter, ok := st.getDelimiter(name); ok {
expansions[name] = strings.SplitN(environment.Map(name, fullEnv), delimiter, -1)
}
}
// If we don't have any expandable variables, just use the standard
// mapper. If we don't do this here we won't return anything useful
// as product will return a nil map.
if len(expansions) == 0 {
mapped, err := environment.Mapper(data, fullEnv)
if err != nil {
return nil, err
}
return []string{mapped}, nil
}
// Construct the actual values. Product gives us a map of all possible
// combinations of expanded values. For each one, map using ONLY the
// expanded vars. Additional mapping will apply on the next iteration.
values := []string{}
for _, m := range product(expansions) {
env := []string{}
for k, v := range m {
env = append(env, fmt.Sprintf("%s=%s", k, v))
}
val, err := environment.Mapper(data, env)
if err != nil {
return nil, err
}
values = append(values, val)
}
return values, nil
}
// GetDelimiter returns the highest (outermost extend task) delimiter registered
// with a given expandable environment variable. Returns true if found by name.
func (st *State) getDelimiter(name string) (string, bool) {
if st.parent == nil {
return "", false
}
for _, expandable := range st.expandables {
if expandable == name {
return st.expandableDelimiter, true
}
}
return st.parent.getDelimiter(name)
}
// WrapWithExpandableEnv will create a shallow clone of the state with a reference
// to the current state as "parent" with a modified environment. This creates a local
// stack of states which do not interfere with other goroutines. A pop operation is
// the same as ignoring the wrapped values and using the underlying state. This stack
// is used to map a slice within an extended task.
func (st *State) WrapWithExpandableEnv(env, expandable []string, delimiter string) *State {
return &State{
Network: st.Network,
Workspace: st.Workspace,
KeepWorkspace: st.KeepWorkspace,
ForceSequential: st.ForceSequential,
EnableSSHAgent: st.EnableSSHAgent,
PlanTimeout: st.PlanTimeout,
DockerConfig: st.DockerConfig,
CPUShares: st.CPUShares,
Memory: st.Memory,
outerEnv: env,
parent: st,
expandables: expandable,
expandableDelimiter: delimiter,
}
}
// MarkRunning will add the given container name into the list
// of containers currently running. This falls through directly to
// the root state so that states wrapping the global one do not have
// to sync additional running container names.
func (st *State) MarkRunning(name string) {
if st.parent != nil {
st.parent.MarkRunning(name)
return
}
st.mutex.Lock()
defer st.mutex.Unlock()
st.runningContainers[name] = struct{}{}
}
// UnmarkRunning removes a container from the list of running containers.
func (st *State) UnmarkRunning(name string) {
if st.parent != nil {
st.parent.UnmarkRunning(name)
return
}
st.mutex.Lock()
defer st.mutex.Unlock()
delete(st.runningContainers, name)
}
// GetRunning returns a list of all running containers.
func (st *State) GetRunning() []string {
if st.parent != nil {
return st.parent.GetRunning()
}
st.mutex.RLock()
defer st.mutex.RUnlock()
names := []string{}
for key := range st.runningContainers {
names = append(names, key)
}
return names
}
// MarkDetached will add the given container name into the list
// of containers running in detached mode which must be shut down
// at the end of the plan. This falls through directly to the root
// state so that states wrapping the global one do not have to sync
// additional detached container names.
func (st *State) MarkDetached(name string) {
if st.parent != nil {
st.parent.MarkDetached(name)
return
}
st.mutex.Lock()
defer st.mutex.Unlock()
st.detachedContainers[name] = struct{}{}
}
// GetDetached returns a list of all detached containers.
func (st *State) GetDetached() []string {
if st.parent != nil {
return st.parent.GetDetached()
}
st.mutex.RLock()
defer st.mutex.RUnlock()
names := []string{}
for key := range st.detachedContainers {
names = append(names, key)
}
return names
}