grim/convey
Clone
Summary
Browse
Changes
Graph
Merged in efritz/convey/expand-list (pull request #26)
2017-10-04, Gary Kramlich
3263ee6583bd
Merged in efritz/convey/expand-list (pull request #26)
Expand list
Approved-by: Gary Kramlich
/*
* Convey
* Copyright 2016-2017 Gary Kramlich <grim@reaperworld.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package
state
import
(
"fmt"
"os"
"strings"
"sync"
"time"
"bitbucket.org/rw_grim/convey/environment"
"bitbucket.org/rw_grim/convey/network"
"bitbucket.org/rw_grim/convey/workspace"
)
const
ExpansionLimit
=
100
type
State
struct
{
Network
network
.
Network
Workspace
workspace
.
Workspace
KeepWorkspace
bool
ForceSequential
bool
EnableSSHAgent
bool
TaskTimeout
time
.
Duration
Environment
[]
string
DockerConfig
string
CPUShares
string
Memory
string
// States have the ability to "wrap" another one without
// changing the underlying state. This is used by the
// extends intrinsic in order to modify the stat without
// requiring unique access to the state object during the
// execution of the extended task. A past implementation
// had modified the stack directly, but that causes an
// extended task to clobber other tasks in concurrent mode.
parent
*
State
expandables
[]
string
expandableDelimiter
string
// This list is a stash of container names which are run
// in detached mode. Appending to this may happen from
// multiple goroutines, so this needs to be guarded via
// mutex.
detachedContainers
map
[
string
]
struct
{}
mutex
*
sync
.
RWMutex
}
func
New
()
*
State
{
return
&
State
{
detachedContainers
:
map
[
string
]
struct
{}{},
mutex
:
&
sync
.
RWMutex
{},
}
}
func
(
st
*
State
)
Valid
()
error
{
if
st
.
parent
==
nil
&&
(
st
.
detachedContainers
==
nil
||
st
.
mutex
==
nil
)
{
return
fmt
.
Errorf
(
"state must be constructed via New"
)
}
if
st
.
EnableSSHAgent
{
if
val
:=
os
.
Getenv
(
"SSH_AUTH_SOCK"
);
val
==
""
{
return
fmt
.
Errorf
(
"ssh-agent forwarding requested, but agent not running"
)
}
}
return
nil
}
// MapSlice calls SliceMapper on the given environment, but also checks to
// see if the variable in the env parameter can be expanded into a list.
func
(
st
*
State
)
MapSlice
(
env
,
fullEnv
[]
string
)
([]
string
,
error
)
{
prev
:=
env
// Protect ourselves against a weird infinite expansion. This can
// happen if something occurs like X => A,$Y; Y => B,$X. This is
// never a useful case - a high expansion limit should catch this
// without ever hitting a practical edge.
for
i
:=
0
;
i
<
ExpansionLimit
;
i
++
{
next
,
err
:=
st
.
expandSlice
(
prev
,
fullEnv
)
if
err
!=
nil
{
return
nil
,
err
}
// If we haven't made a change, return the final result.
if
isSame
(
next
,
prev
)
{
return
next
,
nil
}
prev
=
next
}
return
nil
,
fmt
.
Errorf
(
"hit limit while expanding '%s'"
,
env
)
}
func
(
st
*
State
)
expandSlice
(
env
,
fullEnv
[]
string
)
([]
string
,
error
)
{
all
:=
[]
string
{}
for
_
,
data
:=
range
env
{
expanded
,
err
:=
st
.
expand
(
data
,
fullEnv
)
if
err
!=
nil
{
return
nil
,
err
}
all
=
append
(
all
,
expanded
...
)
}
return
removeDuplicates
(
all
),
nil
}
// expand will attempt to expand any variables stored on the state expand
// stack. If no expandable variables are found, then the standard mapper
// is used. If multiple expandable variables are around, the mapper will
// be applied to the cartesian product of the variables.
func
(
st
*
State
)
expand
(
data
string
,
fullEnv
[]
string
)
([]
string
,
error
)
{
expansions
:=
map
[
string
][]
string
{}
// First, extract all of the expandable names and get their values
// in the full environment and split them by the proper delimiter.
for
_
,
name
:=
range
getNames
(
data
)
{
if
delimiter
,
ok
:=
st
.
getDelimiter
(
name
);
ok
{
expansions
[
name
]
=
strings
.
SplitN
(
environment
.
Map
(
name
,
fullEnv
),
delimiter
,
-
1
)
}
}
// If we don't have any expandable variables, just use the standard
// mapper. If we don't do this here we won't return anything useful
// as product will return a nil map.
if
len
(
expansions
)
==
0
{
mapped
,
err
:=
environment
.
Mapper
(
data
,
fullEnv
)
if
err
!=
nil
{
return
nil
,
err
}
return
[]
string
{
mapped
},
nil
}
// Construct the actual values. Product gives us a map of all possible
// combinations of expanded values. For each one, map using ONLY the
// expanded vars. Additional mapping will apply on the next iteration.
values
:=
[]
string
{}
for
_
,
m
:=
range
product
(
expansions
)
{
env
:=
[]
string
{}
for
k
,
v
:=
range
m
{
env
=
append
(
env
,
fmt
.
Sprintf
(
"%s=%s"
,
k
,
v
))
}
val
,
err
:=
environment
.
Mapper
(
data
,
env
)
if
err
!=
nil
{
return
nil
,
err
}
values
=
append
(
values
,
val
)
}
return
values
,
nil
}
// GetDelimiter returns the highest (outermost extend task) delimiter registered
// with a given expandable environment variable. Returns true if found by name.
func
(
st
*
State
)
getDelimiter
(
name
string
)
(
string
,
bool
)
{
if
st
.
parent
==
nil
{
return
""
,
false
}
for
_
,
expandable
:=
range
st
.
expandables
{
if
expandable
==
name
{
return
st
.
expandableDelimiter
,
true
}
}
return
st
.
parent
.
getDelimiter
(
name
)
}
// WrapWithExpandableEnv will create a shallow clone of the state with a reference
// to the current state as "parent" with a modified environment. This creates a local
// stack of states which do not interfere with other goroutines. A pop operation is
// the same as ignoring the wrapped values and using the underlying state. This stack
// is used to map a slice within an extended task.
func
(
st
*
State
)
WrapWithExpandableEnv
(
env
,
expandable
[]
string
,
delimiter
string
)
*
State
{
// Merge the environment into this map, but do NOT override anything that
// is currently in the state's environment - this has higher precedence.
env
=
environment
.
Merge
(
env
,
st
.
Environment
)
return
&
State
{
Network
:
st
.
Network
,
Workspace
:
st
.
Workspace
,
KeepWorkspace
:
st
.
KeepWorkspace
,
ForceSequential
:
st
.
ForceSequential
,
EnableSSHAgent
:
st
.
EnableSSHAgent
,
TaskTimeout
:
st
.
TaskTimeout
,
Environment
:
env
,
DockerConfig
:
st
.
DockerConfig
,
CPUShares
:
st
.
CPUShares
,
Memory
:
st
.
Memory
,
parent
:
st
,
expandables
:
expandable
,
expandableDelimiter
:
delimiter
,
}
}
// MarkDetached will add the given container name into the list
// of containers running in detached mode which must be shut down
// at the end of the plan. This falls through directly to the root
// state so that states wrapping the global one do not have to sync
// additional detached container names.
func
(
st
*
State
)
MarkDetached
(
name
string
)
{
if
st
.
parent
!=
nil
{
st
.
parent
.
MarkDetached
(
name
)
return
}
st
.
mutex
.
Lock
()
defer
st
.
mutex
.
Unlock
()
st
.
detachedContainers
[
name
]
=
struct
{}{}
}
// GetDetached returns a list of all detached containers.
func
(
st
*
State
)
GetDetached
()
[]
string
{
if
st
.
parent
!=
nil
{
return
st
.
parent
.
GetDetached
()
}
st
.
mutex
.
RLock
()
defer
st
.
mutex
.
RUnlock
()
names
:=
[]
string
{}
for
key
:=
range
st
.
detachedContainers
{
names
=
append
(
names
,
key
)
}
return
names
}