grim/convey
Clone
Summary
Browse
Changes
Graph
Support directories for templating in the kubectl commands as well
2018-02-20, Gary Kramlich
fbe1cafcbb58
Support directories for templating in the kubectl commands as well
// Convey
// Copyright 2016-2018 Gary Kramlich <grim@reaperworld.com>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Package state contains the type that maintain the state during a run.
package
state
import
(
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/aphistic/gomol"
"bitbucket.org/rw_grim/convey/environment"
"bitbucket.org/rw_grim/convey/logging"
"bitbucket.org/rw_grim/convey/network"
"bitbucket.org/rw_grim/convey/workspace"
)
const
expansionLimit
=
100
// State holds all of the runtime data during a run.
type
State
struct
{
Directory
string
logger
*
gomol
.
LogAdapter
cleanupList
*
cleanupList
CfgPath
string
Network
network
.
Network
Workspace
workspace
.
Workspace
KeepWorkspace
bool
DisableDeprecated
bool
ForceSequential
bool
EnableSSHAgent
bool
PlanTimeout
time
.
Duration
DockerConfig
string
CPUShares
string
Memory
string
// Guard the environment around accessors so that the base
// environment can be modified while allowing all "wrapped"
// environments to get the same updates.
innerEnv
[]
string
// Additional environment variables that are applied from
// the wrapper around the base env. This is applied on demand
// to the base environment so that we don't keep around a
// stale cache.
outerEnv
[]
string
// States have the ability to "wrap" another one without
// changing the underlying state. This is used by the
// extends intrinsic in order to modify the stat without
// requiring unique access to the state object during the
// execution of the extended task. A past implementation
// had modified the stack directly, but that causes an
// extended task to clobber other tasks in concurrent mode.
parent
*
State
expandables
[]
string
expandableDelimiter
string
// This list is a stash of container names which are either
// currently running or running in detached mode. Appending
// to this may happen from multiple goroutines, so this needs
// to be guarded via mutex.
runningContainers
map
[
string
]
struct
{}
detachedContainers
map
[
string
]
struct
{}
mutex
sync
.
RWMutex
}
// New creates a new state.
func
New
()
*
State
{
dir
,
err
:=
createDirectory
()
if
err
!=
nil
{
panic
(
err
)
}
st
:=
&
State
{
Directory
:
dir
,
logger
:
logging
.
NewAdapter
(
"state"
),
cleanupList
:
newCleanupList
(),
runningContainers
:
map
[
string
]
struct
{}{},
detachedContainers
:
map
[
string
]
struct
{}{},
}
st
.
cleanupOnSignal
()
return
st
}
// createDirectory creates the state directory for this run.
func
createDirectory
()
(
string
,
error
)
{
pwd
,
err
:=
os
.
Getwd
()
if
err
!=
nil
{
return
""
,
err
}
parent
:=
filepath
.
Join
(
pwd
,
".convey"
)
err
=
os
.
MkdirAll
(
parent
,
0700
)
if
err
!=
nil
{
return
""
,
err
}
dir
,
err
:=
ioutil
.
TempDir
(
parent
,
""
)
if
err
!=
nil
{
return
""
,
err
}
return
dir
,
nil
}
// TaskDirectory will create a directory in the state directory for the named task.
func
(
st
*
State
)
TaskDirectory
(
name
string
)
(
string
,
error
)
{
dir
:=
filepath
.
Join
(
st
.
Directory
,
name
)
err
:=
os
.
MkdirAll
(
dir
,
0700
)
return
dir
,
err
}
// Valid validates whether the state is correct or not.
func
(
st
*
State
)
Valid
()
error
{
if
st
.
parent
==
nil
&&
st
.
detachedContainers
==
nil
{
return
fmt
.
Errorf
(
"state must be constructed via New"
)
}
if
st
.
EnableSSHAgent
{
if
val
:=
os
.
Getenv
(
"SSH_AUTH_SOCK"
);
val
==
""
{
return
fmt
.
Errorf
(
"ssh-agent forwarding requested, but agent not running"
)
}
}
return
nil
}
// GetEnv gets the fullEnv with the wrapped environments applied.
func
(
st
*
State
)
GetEnv
()
[]
string
{
return
st
.
mergeRecursive
(
st
.
getInnerEnv
())
}
// getInnerEnv retrieves the original state environment (without wrapping).
func
(
st
*
State
)
getInnerEnv
()
[]
string
{
if
st
.
parent
!=
nil
{
return
st
.
parent
.
getInnerEnv
()
}
return
st
.
innerEnv
}
// mergeRecursive applies the wrapped environments recursively.
func
(
st
*
State
)
mergeRecursive
(
env
[]
string
)
[]
string
{
if
st
.
parent
!=
nil
{
// Merge recursively and then apply the wrapped environment so that
// the wrapping takes the highest precedence.
return
environment
.
Merge
(
st
.
outerEnv
,
st
.
parent
.
mergeRecursive
(
env
))
}
return
env
}
// MergeEnv adds additional environment arguments with lower precedence to
// the original state environment. If they duplicate an existing environment
// variable, that value will be unchanged.
func
(
st
*
State
)
MergeEnv
(
env
[]
string
)
{
if
st
.
parent
!=
nil
{
st
.
parent
.
MergeEnv
(
env
)
return
}
st
.
innerEnv
=
environment
.
Merge
(
env
,
st
.
innerEnv
)
}
// MapSlice calls SliceMapper on the given environment, but also checks to
// see if the variable in the env parameter can be expanded into a list.
func
(
st
*
State
)
MapSlice
(
slice
,
fullEnv
[]
string
)
([]
string
,
error
)
{
prev
:=
slice
// Protect ourselves against a weird infinite expansion. This can
// happen if something occurs like X => A,$Y; Y => B,$X. This is
// never a useful case - a high expansion limit should catch this
// without ever hitting a practical edge.
for
i
:=
0
;
i
<
expansionLimit
;
i
++
{
next
,
err
:=
st
.
expandSlice
(
prev
,
fullEnv
)
if
err
!=
nil
{
return
nil
,
err
}
// If we haven't made a change, return the final result.
if
isSame
(
next
,
prev
)
{
return
environment
.
SliceMapper
(
next
,
fullEnv
)
}
prev
=
next
}
return
nil
,
fmt
.
Errorf
(
"hit limit while expanding '%s'"
,
slice
)
}
func
(
st
*
State
)
expandSlice
(
env
,
fullEnv
[]
string
)
([]
string
,
error
)
{
all
:=
[]
string
{}
for
_
,
data
:=
range
env
{
expanded
,
err
:=
st
.
expand
(
data
,
fullEnv
)
if
err
!=
nil
{
return
nil
,
err
}
all
=
append
(
all
,
expanded
...
)
}
return
removeDuplicates
(
all
),
nil
}
// expand will attempt to expand any variables stored on the state expand
// stack. If no expandable variables are found, then the standard mapper
// is used. If multiple expandable variables are around, the mapper will
// be applied to the cartesian product of the variables.
func
(
st
*
State
)
expand
(
data
string
,
fullEnv
[]
string
)
([]
string
,
error
)
{
expansions
:=
map
[
string
][]
string
{}
// First, extract all of the expandable names and get their values
// in the full environment and split them by the proper delimiter.
for
_
,
name
:=
range
getNames
(
data
)
{
if
delimiter
,
ok
:=
st
.
getDelimiter
(
name
);
ok
{
expansions
[
name
]
=
strings
.
SplitN
(
environment
.
Map
(
name
,
fullEnv
),
delimiter
,
-
1
)
}
}
// If we don't have any expandable variables, just use the standard
// mapper. If we don't do this here we won't return anything useful
// as product will return a nil map.
if
len
(
expansions
)
==
0
{
mapped
,
err
:=
environment
.
Mapper
(
data
,
fullEnv
)
if
err
!=
nil
{
return
nil
,
err
}
return
[]
string
{
mapped
},
nil
}
// Construct the actual values. Product gives us a map of all possible
// combinations of expanded values. For each one, map using ONLY the
// expanded vars. Additional mapping will apply on the next iteration.
values
:=
[]
string
{}
for
_
,
m
:=
range
product
(
expansions
)
{
env
:=
[]
string
{}
for
k
,
v
:=
range
m
{
env
=
append
(
env
,
fmt
.
Sprintf
(
"%s=%s"
,
k
,
v
))
}
val
,
err
:=
environment
.
Mapper
(
data
,
env
)
if
err
!=
nil
{
return
nil
,
err
}
values
=
append
(
values
,
val
)
}
return
values
,
nil
}
// GetDelimiter returns the highest (outermost extend task) delimiter registered
// with a given expandable environment variable. Returns true if found by name.
func
(
st
*
State
)
getDelimiter
(
name
string
)
(
string
,
bool
)
{
if
st
.
parent
==
nil
{
return
""
,
false
}
for
_
,
expandable
:=
range
st
.
expandables
{
if
expandable
==
name
{
return
st
.
expandableDelimiter
,
true
}
}
return
st
.
parent
.
getDelimiter
(
name
)
}
// WrapWithExpandableEnv will create a shallow clone of the state with a reference
// to the current state as "parent" with a modified environment. This creates a local
// stack of states which do not interfere with other goroutines. A pop operation is
// the same as ignoring the wrapped values and using the underlying state. This stack
// is used to map a slice within an extended task.
func
(
st
*
State
)
WrapWithExpandableEnv
(
env
,
expandable
[]
string
,
delimiter
string
)
*
State
{
return
&
State
{
Network
:
st
.
Network
,
Workspace
:
st
.
Workspace
,
KeepWorkspace
:
st
.
KeepWorkspace
,
ForceSequential
:
st
.
ForceSequential
,
EnableSSHAgent
:
st
.
EnableSSHAgent
,
PlanTimeout
:
st
.
PlanTimeout
,
DockerConfig
:
st
.
DockerConfig
,
CPUShares
:
st
.
CPUShares
,
Memory
:
st
.
Memory
,
outerEnv
:
env
,
parent
:
st
,
expandables
:
expandable
,
expandableDelimiter
:
delimiter
,
}
}
// MarkRunning will add the given container name into the list
// of containers currently running. This falls through directly to
// the root state so that states wrapping the global one do not have
// to sync additional running container names.
func
(
st
*
State
)
MarkRunning
(
name
string
)
{
if
st
.
parent
!=
nil
{
st
.
parent
.
MarkRunning
(
name
)
return
}
st
.
mutex
.
Lock
()
defer
st
.
mutex
.
Unlock
()
st
.
runningContainers
[
name
]
=
struct
{}{}
}
// UnmarkRunning removes a container from the list of running containers.
func
(
st
*
State
)
UnmarkRunning
(
name
string
)
{
if
st
.
parent
!=
nil
{
st
.
parent
.
UnmarkRunning
(
name
)
return
}
st
.
mutex
.
Lock
()
defer
st
.
mutex
.
Unlock
()
delete
(
st
.
runningContainers
,
name
)
}
// GetRunning returns a list of all running containers.
func
(
st
*
State
)
GetRunning
()
[]
string
{
if
st
.
parent
!=
nil
{
return
st
.
parent
.
GetRunning
()
}
st
.
mutex
.
RLock
()
defer
st
.
mutex
.
RUnlock
()
names
:=
[]
string
{}
for
key
:=
range
st
.
runningContainers
{
names
=
append
(
names
,
key
)
}
return
names
}
// MarkDetached will add the given container name into the list
// of containers running in detached mode which must be shut down
// at the end of the plan. This falls through directly to the root
// state so that states wrapping the global one do not have to sync
// additional detached container names.
func
(
st
*
State
)
MarkDetached
(
name
string
)
{
if
st
.
parent
!=
nil
{
st
.
parent
.
MarkDetached
(
name
)
return
}
st
.
mutex
.
Lock
()
defer
st
.
mutex
.
Unlock
()
st
.
detachedContainers
[
name
]
=
struct
{}{}
}
// GetDetached returns a list of all detached containers.
func
(
st
*
State
)
GetDetached
()
[]
string
{
if
st
.
parent
!=
nil
{
return
st
.
parent
.
GetDetached
()
}
st
.
mutex
.
RLock
()
defer
st
.
mutex
.
RUnlock
()
names
:=
[]
string
{}
for
key
:=
range
st
.
detachedContainers
{
names
=
append
(
names
,
key
)
}
return
names
}