grim/convey

Refactor state environment.
inject
2017-10-11, Eric Fritz
93a8d6e1029d
Parents eba0fb6f4af8
Children e2ed1742f942
Refactor state environment.
18 files changed, 110 insertions(+), 52 deletions(-)
  • +1 -1
    docker/build.go
  • +1 -1
    docker/export.go
  • +1 -1
    docker/import.go
  • +1 -1
    docker/login.go
  • +1 -1
    docker/logout.go
  • +1 -1
    docker/pull.go
  • +1 -1
    docker/push.go
  • +1 -1
    docker/remove.go
  • +1 -1
    docker/run.go
  • +1 -1
    docker/tag.go
  • +1 -1
    environment/mapper.go
  • +2 -2
    environment/mapper_test.go
  • +1 -1
    intrinsic/clean.go
  • +6 -4
    intrinsic/extend_test.go
  • +1 -0
    intrinsic/intrinsic.go
  • +1 -1
    main.go
  • +56 -13
    state/state.go
  • +32 -20
    state/state_test.go
    • --- a/docker/build.go Wed Oct 11 09:22:28 2017 -0500
    +++ b/docker/build.go Wed Oct 11 19:39:23 2017 -0500
    @@ -47,7 +47,7 @@
    {{.buildContext}}`
    func (b *Build) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    files, err := st.MapSlice(b.Files, fullEnv)
    if err != nil {
    --- a/docker/export.go Wed Oct 11 09:22:28 2017 -0500
    +++ b/docker/export.go Wed Oct 11 19:39:23 2017 -0500
    @@ -84,7 +84,7 @@
    }
    func (e *Export) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    files, err := st.MapSlice(e.Files, fullEnv)
    if err != nil {
    --- a/docker/import.go Wed Oct 11 09:22:28 2017 -0500
    +++ b/docker/import.go Wed Oct 11 19:39:23 2017 -0500
    @@ -33,7 +33,7 @@
    const importTemplate = `cp {{.source}} {{.workspaceID}}:/workspace/{{.destination}}`
    func (i *Import) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    files, err := st.MapSlice(i.Files, fullEnv)
    if err != nil {
    --- a/docker/login.go Wed Oct 11 09:22:28 2017 -0500
    +++ b/docker/login.go Wed Oct 11 19:39:23 2017 -0500
    @@ -34,7 +34,7 @@
    const loginTemplate = `login -u {{.username}} -p {{.password}}{{if .server}} {{.server}}{{end}}`
    func (l *Login) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    username, err := environment.Mapper(l.Username, fullEnv)
    if err != nil {
    --- a/docker/logout.go Wed Oct 11 09:22:28 2017 -0500
    +++ b/docker/logout.go Wed Oct 11 19:39:23 2017 -0500
    @@ -32,7 +32,7 @@
    const logoutTemplate = `logout{{if .server}} {{.server}}{{end}}`
    func (l *Logout) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    server, err := environment.Mapper(l.Server, fullEnv)
    if err != nil {
    --- a/docker/pull.go Wed Oct 11 09:22:28 2017 -0500
    +++ b/docker/pull.go Wed Oct 11 19:39:23 2017 -0500
    @@ -34,7 +34,7 @@
    const pullTemplate = `pull {{.image}}`
    func (p *Pull) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    images, err := st.MapSlice(p.Images, fullEnv)
    if err != nil {
    --- a/docker/push.go Wed Oct 11 09:22:28 2017 -0500
    +++ b/docker/push.go Wed Oct 11 19:39:23 2017 -0500
    @@ -34,7 +34,7 @@
    const pushTemplate = `push {{.image}}`
    func (p *Push) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    images, err := st.MapSlice(p.Images, fullEnv)
    if err != nil {
    --- a/docker/remove.go Wed Oct 11 09:22:28 2017 -0500
    +++ b/docker/remove.go Wed Oct 11 19:39:23 2017 -0500
    @@ -37,7 +37,7 @@
    const removeTemplate = `rmi {{.image}}`
    func (r *Remove) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    images, err := st.MapSlice(r.Images, fullEnv)
    if err != nil {
    --- a/docker/run.go Wed Oct 11 09:22:28 2017 -0500
    +++ b/docker/run.go Wed Oct 11 19:39:23 2017 -0500
    @@ -143,7 +143,7 @@
    func (r *Run) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, r.Environment)
    - fullEnv = environment.Merge(fullEnv, st.Environment)
    + fullEnv = environment.Merge(fullEnv, st.GetEnv())
    user, err := user.Current()
    if err != nil {
    --- a/docker/tag.go Wed Oct 11 09:22:28 2017 -0500
    +++ b/docker/tag.go Wed Oct 11 19:39:23 2017 -0500
    @@ -35,7 +35,7 @@
    const tagTemplate = `tag {{.source}} {{.destination}}`
    func (t *Tag) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    source, err := environment.Mapper(t.Source, fullEnv)
    if err != nil {
    --- a/environment/mapper.go Wed Oct 11 09:22:28 2017 -0500
    +++ b/environment/mapper.go Wed Oct 11 19:39:23 2017 -0500
    @@ -38,7 +38,7 @@
    }
    }
    - return "$" + name
    + return fmt.Sprintf("${%s}", name)
    }
    // Map will return the value matching a KEY=VAL pair in the given environment.
    --- a/environment/mapper_test.go Wed Oct 11 09:22:28 2017 -0500
    +++ b/environment/mapper_test.go Wed Oct 11 19:39:23 2017 -0500
    @@ -62,11 +62,11 @@
    result, err := Mapper("${FOO}", []string{"FOOBAR"})
    Expect(err).To(BeNil())
    - Expect(result).To(Equal("$FOO"))
    + Expect(result).To(Equal("${FOO}"))
    result, err = Mapper("${BARBAZ}", []string{"BAR"})
    Expect(err).To(BeNil())
    - Expect(result).To(Equal("$BARBAZ"))
    + Expect(result).To(Equal("${BARBAZ}"))
    }
    func (e *environmentSuite) TestRecursiveMatch(t sweet.T) {
    --- a/intrinsic/clean.go Wed Oct 11 09:22:28 2017 -0500
    +++ b/intrinsic/clean.go Wed Oct 11 19:39:23 2017 -0500
    @@ -68,7 +68,7 @@
    }
    func (c *Clean) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    wd, err := os.Getwd()
    if err != nil {
    --- a/intrinsic/extend_test.go Wed Oct 11 09:22:28 2017 -0500
    +++ b/intrinsic/extend_test.go Wed Oct 11 19:39:23 2017 -0500
    @@ -30,11 +30,13 @@
    var (
    task = newMockTask()
    fullEnv = []string{}
    - st = &state.State{Environment: []string{"bar=bonk", "quux=honk"}}
    )
    + st := state.New()
    + st.MergeEnv([]string{"bar=bonk", "quux=honk"})
    +
    task.executeFn = func(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv = environment.Merge(environment.Merge(env, []string{"x=1", "foo=quux"}), st.Environment)
    + fullEnv = environment.Merge(environment.Merge(env, []string{"x=1", "foo=quux"}), st.GetEnv())
    return nil
    }
    @@ -48,8 +50,8 @@
    Expect(fullEnv).To(ConsistOf("x=1", "foo=bar", "bar=bonk", "quux=honk"))
    // Should revert state
    - Expect(st.Environment).To(HaveLen(2))
    - Expect(st.Environment).To(ConsistOf("bar=bonk", "quux=honk"))
    + Expect(st.GetEnv()).To(HaveLen(2))
    + Expect(st.GetEnv()).To(ConsistOf("bar=bonk", "quux=honk"))
    }
    func (s *intrinsicSuite) TestDependencies(t sweet.T) {
    --- a/intrinsic/intrinsic.go Wed Oct 11 09:22:28 2017 -0500
    +++ b/intrinsic/intrinsic.go Wed Oct 11 19:39:23 2017 -0500
    @@ -25,5 +25,6 @@
    Tasks = map[string]tasks.Task{
    "clean": &Clean{},
    "extend": &Extend{},
    + "inject": &Inject{},
    }
    )
    --- a/main.go Wed Oct 11 09:22:28 2017 -0500
    +++ b/main.go Wed Oct 11 19:39:23 2017 -0500
    @@ -136,10 +136,10 @@
    st.ForceSequential = *forceSequential
    st.EnableSSHAgent = enableSSHAgent
    st.TaskTimeout = *taskTimeout
    - st.Environment = environment.Merge(defEnv, *env)
    st.DockerConfig = *dockerConfig
    st.CPUShares = *cpuShares
    st.Memory = *memory
    + st.MergeEnv(environment.Merge(defEnv, *env))
    if err := st.Valid(); err != nil {
    fmt.Printf("%s\n", err)
    --- a/state/state.go Wed Oct 11 09:22:28 2017 -0500
    +++ b/state/state.go Wed Oct 11 19:39:23 2017 -0500
    @@ -39,12 +39,22 @@
    ForceSequential bool
    EnableSSHAgent bool
    TaskTimeout time.Duration
    - Environment []string
    DockerConfig string
    CPUShares string
    Memory string
    + // Guard the environment around accessors so that the base
    + // environment can be modified while allowing all "wrapped"
    + // environments to get the same updates.
    + innerEnv []string
    +
    + // Additional environment variables that are applied from
    + // the wrapper around the base env. This is applied on demand
    + // to the base environment so that we don't keep around a
    + // stale cache.
    + outerEnv []string
    +
    // States have the ability to "wrap" another one without
    // changing the underlying state. This is used by the
    // extends intrinsic in order to modify the stat without
    @@ -61,18 +71,17 @@
    // multiple goroutines, so this needs to be guarded via
    // mutex.
    detachedContainers map[string]struct{}
    - mutex *sync.RWMutex
    + mutex sync.RWMutex
    }
    func New() *State {
    return &State{
    detachedContainers: map[string]struct{}{},
    - mutex: &sync.RWMutex{},
    }
    }
    func (st *State) Valid() error {
    - if st.parent == nil && (st.detachedContainers == nil || st.mutex == nil) {
    + if st.parent == nil && st.detachedContainers == nil {
    return fmt.Errorf("state must be constructed via New")
    }
    @@ -85,10 +94,48 @@
    return nil
    }
    +// GetEnv gets the fullEnv with the wrapped environments applied.
    +func (st *State) GetEnv() []string {
    + // fmt.Printf("I/O > %#v %#v\n", st.innerEnv, st.outerEnv)
    + // fmt.Printf("I/O > %#v %#v\n", st.parent.innerEnv, st.parent.outerEnv)
    + // fmt.Printf("I/O > %#v %#v\n", st.parent.parent.innerEnv, st.parent.parent.outerEnv)
    + return st.mergeRecursive(st.getInnerEnv())
    +}
    +
    +// getInnerEnv retrieves the original state environment (without wrapping).
    +func (st *State) getInnerEnv() []string {
    + if st.parent != nil {
    + return st.parent.getInnerEnv()
    + }
    +
    + return st.innerEnv
    +}
    +
    +// mergeRecursive applies the wrapped environments recursively.
    +func (st *State) mergeRecursive(env []string) []string {
    + if st.parent != nil {
    + // Merge recursively and then apply the wrapped environment so that
    + // the wrapping takes the highest precedence.
    + return environment.Merge(st.outerEnv, st.parent.mergeRecursive(env))
    + }
    +
    + return env
    +}
    +
    +// MergeEnv adds additional environment arguments to the original state environment.
    +func (st *State) MergeEnv(env []string) {
    + if st.parent != nil {
    + st.parent.MergeEnv(env)
    + return
    + }
    +
    + st.innerEnv = environment.Merge(st.innerEnv, env)
    +}
    +
    // MapSlice calls SliceMapper on the given environment, but also checks to
    // see if the variable in the env parameter can be expanded into a list.
    -func (st *State) MapSlice(env, fullEnv []string) ([]string, error) {
    - prev := env
    +func (st *State) MapSlice(slice, fullEnv []string) ([]string, error) {
    + prev := slice
    // Protect ourselves against a weird infinite expansion. This can
    // happen if something occurs like X => A,$Y; Y => B,$X. This is
    @@ -103,13 +150,13 @@
    // If we haven't made a change, return the final result.
    if isSame(next, prev) {
    - return next, nil
    + return environment.SliceMapper(next, fullEnv)
    }
    prev = next
    }
    - return nil, fmt.Errorf("hit limit while expanding '%s'", env)
    + return nil, fmt.Errorf("hit limit while expanding '%s'", slice)
    }
    func (st *State) expandSlice(env, fullEnv []string) ([]string, error) {
    @@ -199,10 +246,6 @@
    // the same as ignoring the wrapped values and using the underlying state. This stack
    // is used to map a slice within an extended task.
    func (st *State) WrapWithExpandableEnv(env, expandable []string, delimiter string) *State {
    - // Merge the environment into this map, but do NOT override anything that
    - // is currently in the state's environment - this has higher precedence.
    - env = environment.Merge(env, st.Environment)
    -
    return &State{
    Network: st.Network,
    Workspace: st.Workspace,
    @@ -210,10 +253,10 @@
    ForceSequential: st.ForceSequential,
    EnableSSHAgent: st.EnableSSHAgent,
    TaskTimeout: st.TaskTimeout,
    - Environment: env,
    DockerConfig: st.DockerConfig,
    CPUShares: st.CPUShares,
    Memory: st.Memory,
    + outerEnv: env,
    parent: st,
    expandables: expandable,
    expandableDelimiter: delimiter,
    --- a/state/state_test.go Wed Oct 11 09:22:28 2017 -0500
    +++ b/state/state_test.go Wed Oct 11 19:39:23 2017 -0500
    @@ -37,11 +37,23 @@
    })
    }
    +func (s *stateSuite) TestWrappedEnvironment(t sweet.T) {
    + st1 := New()
    + st1.MergeEnv([]string{"Y=0"})
    + st2 := st1.WrapWithExpandableEnv([]string{"X=1", "Y=1"}, []string{}, ";")
    + st3 := st2.WrapWithExpandableEnv([]string{"X=2", "Y=2"}, []string{}, ";")
    + st4 := st3.WrapWithExpandableEnv([]string{"X=3", "Y=3", "Z=3"}, []string{}, ";")
    +
    + // Things in the "base" env are not override, but things
    + // that are wrapped take the highest precedence
    + Expect(st4.GetEnv()).To(ConsistOf([]string{"X=1", "Y=0", "Z=3"}))
    +}
    +
    func (s *stateSuite) TestMapSlice(t sweet.T) {
    - st1 := &State{}
    - st1.Environment = []string{"FOO=BAR"}
    + st1 := New()
    + st1.MergeEnv([]string{"FOO=BAR"})
    - Expect(mapEnv(st1, "$X")).To(ConsistOf([]string{"$X"}))
    + Expect(mapEnv(st1, "$X")).To(ConsistOf([]string{"${X}"}))
    Expect(mapEnv(st1, "$FOO")).To(ConsistOf([]string{"BAR"}))
    st2 := st1.WrapWithExpandableEnv([]string{"X=A;B;C"}, []string{"X"}, ";")
    @@ -55,8 +67,8 @@
    }
    func (s *stateSuite) TestMapSliceComplex(t sweet.T) {
    - st := &State{}
    - st.Environment = []string{"FOO=BAR"}
    + st := New()
    + st.MergeEnv([]string{"FOO=BAR"})
    st = st.WrapWithExpandableEnv([]string{"X=A;B;C", "Y=D;E;F"}, []string{"X", "Y"}, ";")
    // No expansion
    @@ -80,27 +92,27 @@
    }
    func (s *stateSuite) TestMapSliceIterative(t sweet.T) {
    - st := &State{}
    - st.Environment = []string{"FOO=BAR"}
    + st := New()
    + st.MergeEnv([]string{"FOO=BAR"})
    st = st.WrapWithExpandableEnv([]string{"X=A;x$Y", "Y=B;y$Z", "Z=C;D;E"}, []string{"X", "Y", "Z"}, ";")
    Expect(mapEnv(st, "$X")).To(ConsistOf([]string{"A", "xB", "xyC", "xyD", "xyE"}))
    }
    func (s *stateSuite) TestMapSliceIndirect(t sweet.T) {
    - st := &State{}
    - st.Environment = []string{"FOO=BAR"}
    + st := New()
    + st.MergeEnv([]string{"FOO=BAR"})
    st = st.WrapWithExpandableEnv([]string{"X=$Y", "Y=A;$Z;C", "Z=B;$W", "W=D"}, []string{"X", "Y", "Z"}, ";")
    Expect(mapEnv(st, "$X")).To(ConsistOf([]string{"A", "B", "C", "D"}))
    }
    func (s *stateSuite) TestMapSliceRecursive(t sweet.T) {
    - st := &State{}
    - st.Environment = []string{"FOO=BAR"}
    + st := New()
    + st.MergeEnv([]string{"FOO=BAR"})
    st = st.WrapWithExpandableEnv([]string{"X=A;x$Y", "Y=B;y$Z", "Z=C;D;$X"}, []string{"X", "Y", "Z"}, ";")
    - _, err := st.MapSlice([]string{"$X"}, st.Environment)
    + _, err := st.MapSlice([]string{"$X"}, st.GetEnv())
    Expect(err).NotTo(BeNil())
    Expect(err.Error()).To(ContainSubstring("hit limit"))
    }
    @@ -135,7 +147,7 @@
    }
    func (s *stateSuite) TestWrapParent(t sweet.T) {
    - st1 := &State{}
    + st1 := New()
    st1.WrapWithExpandableEnv(nil, nil, "")
    Expect(st1.parent).To(BeNil())
    @@ -144,14 +156,14 @@
    }
    func (s *stateSuite) TestWrapWithExpandableEnvMap(t sweet.T) {
    - st1 := &State{}
    - st1.Environment = []string{"FOO=BAR", "BAR=BAZ"}
    - Expect(st1.Environment).To(HaveLen(2))
    - Expect(st1.Environment).To(ConsistOf([]string{"FOO=BAR", "BAR=BAZ"}))
    + st1 := New()
    + st1.MergeEnv([]string{"FOO=BAR", "BAR=BAZ"})
    + Expect(st1.GetEnv()).To(HaveLen(2))
    + Expect(st1.GetEnv()).To(ConsistOf([]string{"FOO=BAR", "BAR=BAZ"}))
    st2 := st1.WrapWithExpandableEnv([]string{"FOO=BONK", "BAZ=BONK"}, nil, "")
    - Expect(st2.Environment).To(HaveLen(3))
    - Expect(st2.Environment).To(ConsistOf([]string{"FOO=BAR", "BAR=BAZ", "BAZ=BONK"}))
    + Expect(st2.GetEnv()).To(HaveLen(3))
    + Expect(st2.GetEnv()).To(ConsistOf([]string{"FOO=BAR", "BAR=BAZ", "BAZ=BONK"}))
    }
    func (s *stateSuite) TestDetached(t sweet.T) {
    @@ -173,7 +185,7 @@
    }
    func mapEnv(st *State, val string) []string {
    - mapped, err := st.MapSlice([]string{val}, st.Environment)
    + mapped, err := st.MapSlice([]string{val}, st.GetEnv())
    Expect(err).To(BeNil())
    return mapped
    }