grim/convey

Merged in efritz/convey/inject (pull request #32)

2017-10-15, Gary Kramlich
ff9eec994ade
Parents 04096b1153ea06923b191b21
Children 80c077406e10
Merged in efritz/convey/inject (pull request #32)

Inject

Approved-by: Gary Kramlich
23 files changed, 416 insertions(+), 62 deletions(-)
  • +35 -0
    REFERENCE.md
  • +1 -1
    docker/build.go
  • +11 -10
    docker/docker.go
  • +147 -0
    docker/environment.go
  • +58 -0
    docker/environment_test.go
  • +1 -0
    docker/errors.go
  • +1 -1
    docker/export.go
  • +1 -1
    docker/import.go
  • +1 -1
    docker/login.go
  • +1 -1
    docker/logout.go
  • +1 -1
    docker/pull.go
  • +1 -1
    docker/push.go
  • +1 -1
    docker/remove.go
  • +1 -1
    docker/run.go
  • +1 -1
    docker/tag.go
  • +1 -1
    environment/mapper.go
  • +2 -2
    environment/mapper_test.go
  • +56 -0
    examples/environment-from-file.yml
  • +1 -1
    intrinsic/clean.go
  • +6 -4
    intrinsic/extend_test.go
  • +1 -1
    main.go
  • +55 -13
    state/state.go
  • +32 -20
    state/state_test.go
    • --- a/REFERENCE.md Sun Oct 15 15:17:30 2017 -0500
    +++ b/REFERENCE.md Sun Oct 15 20:19:46 2017 +0000
    @@ -50,6 +50,10 @@
    This example shows how to expand an environment variable into a list in the context of a extended task.
    +## environment-from-file.yml
    +
    +This example shows how to inject variables into the environment from a file in the workspace.
    +
    ## login-logout.yml
    This example shows how you can login and logout of a Docker registry by using environment variables.
    @@ -172,6 +176,37 @@
    ----
    +### Environment task
    +
    +An environment task will read a file with environment variables and make them accessible to your plans.
    +
    +A file is expected to have lines of the form `variable=value`. Empty lines are ignored. Files are read
    +from the workspace, so files generated by the plan do not need to be exported in order to be a target
    +of an environment task. From the other direction, it is necessary to import files on the host if they are
    +to be read by an environment task.
    +
    +#### Attributes
    +
    +| Name | Required | Default | Description |
    +| ---------- | -------- | ------- | ----------- |
    +| from-file | | | A file that should be read. |
    +| from-files | | | A list of files that should be read. |
    +| prefix | | | A prefix to add to each variable read from a file |
    +
    +At least one file must be supplied by either the `from-file` or `from-files` attributes. If both are supplied,
    +`from-file` is inserted to the front of `from-files`. If the files being read are a result of environment variable
    +expansion, the order that the files are read are not guaranteed to be stable (or in the order supplied). Be cautious
    +of this if the environment files define overlapping variables.
    +
    +#### Example
    +
    + inject-version:
    + type: environment
    + from-file: version.txt
    + prefix: "APP_"
    +
    +----
    +
    ### Build task
    A build task will build a docker image.
    --- a/docker/build.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/docker/build.go Sun Oct 15 20:19:46 2017 +0000
    @@ -47,7 +47,7 @@
    {{.buildContext}}`
    func (b *Build) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    files, err := st.MapSlice(b.Files, fullEnv)
    if err != nil {
    --- a/docker/docker.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/docker/docker.go Sun Oct 15 20:19:46 2017 +0000
    @@ -25,16 +25,17 @@
    var (
    Tasks = map[string]tasks.Task{
    - "build": &Build{},
    - "export": &Export{},
    - "import": &Import{},
    - "login": &Login{},
    - "logout": &Logout{},
    - "pull": &Pull{},
    - "push": &Push{},
    - "remove": &Remove{},
    - "run": &Run{},
    - "tag": &Tag{},
    + "build": &Build{},
    + "export": &Export{},
    + "import": &Import{},
    + "login": &Login{},
    + "logout": &Logout{},
    + "pull": &Pull{},
    + "push": &Push{},
    + "remove": &Remove{},
    + "run": &Run{},
    + "tag": &Tag{},
    + "environment": &Environment{},
    }
    dockerTemplate = `docker {{if .DockerConfig}}--config {{.DockerConfig}} {{end}}`
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/docker/environment.go Sun Oct 15 20:19:46 2017 +0000
    @@ -0,0 +1,147 @@
    +/*
    + * Convey
    + * Copyright 2016-2017 Gary Kramlich <grim@reaperworld.com>
    + *
    + * This program is free software: you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License as published by
    + * the Free Software Foundation, either version 3 of the License, or
    + * (at your option) any later version.
    + *
    + * This program is distributed in the hope that it will be useful,
    + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    + * GNU General Public License for more details.
    + *
    + * You should have received a copy of the GNU General Public License
    + * along with this program. If not, see <http://www.gnu.org/licenses/>.
    + */
    +package docker
    +
    +import (
    + "fmt"
    + "io/ioutil"
    + "os"
    + "path/filepath"
    + "strings"
    +
    + "github.com/aphistic/gomol"
    +
    + "bitbucket.org/rw_grim/convey/environment"
    + "bitbucket.org/rw_grim/convey/state"
    + "bitbucket.org/rw_grim/convey/tasks"
    + "bitbucket.org/rw_grim/convey/yaml"
    +)
    +
    +type Environment struct {
    + File string `yaml:"from-file"`
    + Files yaml.StringOrSlice `yaml:"from-files"`
    + Prefix string `yaml:"prefix"`
    +}
    +
    +func (e *Environment) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    + fullEnv := environment.Merge(env, st.GetEnv())
    +
    + prefix, err := environment.Mapper(e.Prefix, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    + files, err := st.MapSlice(e.Files, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    + // Create a temp directory we can export files from the current
    + // container into. We can't read directly from the volume, and
    + // we don't want to require reading only environment files on
    + // the host.
    +
    + tmpDir, err := ioutil.TempDir("", "convey-environment-")
    + if err != nil {
    + return err
    + }
    +
    + // Remove temp dir on app exit
    + cleanupFn := st.CleanupList.Add(func() { os.RemoveAll(tmpDir) })
    +
    + // Call cleanup function on defer, which may instead be called
    + // if the cleanup thread traps a signal.
    + defer cleanupFn()
    +
    + for _, file := range files {
    + // Export the file into the temp directory and maintain the
    + // structure of the file (for ease of error messages, so we
    + // get the file a/b/c/env instead of env).
    + dest := filepath.Clean(filepath.Join(tmpDir, file))
    +
    + if err = exportFile(name, st.Workspace.Name(), file, dest, st); err != nil {
    + return err
    + }
    +
    + // Process the entries of the file and apply them to the
    + // state's base environment immediately.
    +
    + entries, err := processFile(dest, file, prefix)
    + if err != nil {
    + return err
    + }
    +
    + st.MergeEnv(entries)
    + }
    +
    + return nil
    +}
    +
    +func processFile(path, name, prefix string) ([]string, error) {
    + data, err := ioutil.ReadFile(path)
    + if err != nil {
    + return nil, fmt.Errorf("failed to read environment file '%s'", name)
    + }
    +
    + entries := []string{}
    + for _, line := range strings.Split(string(data), "\n") {
    + // Allow blank lines
    + if len(strings.TrimSpace(line)) == 0 {
    + continue
    + }
    +
    + // Each non-empty line requires the form key=val. Split the
    + // key and the val, then uppercase the key and add the prefix.
    + // We don't care what form val takes, it will be treated as a
    + // string (does not need to be quoted).
    +
    + parts := strings.SplitN(line, "=", 2)
    + if len(parts) != 2 {
    + return nil, fmt.Errorf("malformed entry in environments file '%s'", line)
    + }
    +
    + var (
    + key = strings.TrimSpace(parts[0])
    + val = strings.TrimSpace(parts[1])
    + )
    +
    + if len(key) == 0 {
    + return nil, fmt.Errorf("malformed entry in environments file '%s'", line)
    + }
    +
    + entries = append(entries, fmt.Sprintf("%s%s=%s", prefix, strings.ToUpper(key), val))
    + }
    +
    + return entries, nil
    +}
    +
    +func (e *Environment) New() tasks.Task {
    + return &Environment{}
    +}
    +
    +func (e *Environment) Valid() error {
    + if e.File != "" {
    + e.Files = append([]string{e.File}, e.Files...)
    + }
    +
    + if len(e.Files) == 0 {
    + return errNoFilesEnvironment
    + }
    +
    + return nil
    +}
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/docker/environment_test.go Sun Oct 15 20:19:46 2017 +0000
    @@ -0,0 +1,58 @@
    +/*
    + * Convey
    + * Copyright 2016-2017 Gary Kramlich <grim@reaperworld.com>
    + *
    + * This program is free software: you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License as published by
    + * the Free Software Foundation, either version 3 of the License, or
    + * (at your option) any later version.
    + *
    + * This program is distributed in the hope that it will be useful,
    + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    + * GNU General Public License for more details.
    + *
    + * You should have received a copy of the GNU General Public License
    + * along with this program. If not, see <http://www.gnu.org/licenses/>.
    + */
    +package docker
    +
    +import (
    + "github.com/aphistic/sweet"
    + "github.com/go-yaml/yaml"
    + . "github.com/onsi/gomega"
    +
    + cYaml "bitbucket.org/rw_grim/convey/yaml"
    +)
    +
    +func (s *dockerSuite) TestEnvironment(t sweet.T) {
    + i := &Environment{Files: cYaml.StringOrSlice{"foo"}}
    + Expect(i.Valid()).To(BeNil())
    +}
    +
    +func (s *dockerSuite) TestEnvironmentFilesRequired(t sweet.T) {
    + i := &Environment{}
    + Expect(i.Valid()).To(MatchError(errNoFilesEnvironment))
    +}
    +
    +func (s *dockerSuite) TestEnvironmentUnmarshalString(t sweet.T) {
    + data := `from-files: filename`
    +
    + imp := Environment{}
    + err := yaml.Unmarshal([]byte(data), &imp)
    +
    + Expect(err).To(BeNil())
    + Expect(imp.Files).To(Equal(cYaml.StringOrSlice{"filename"}))
    +}
    +
    +func (s *dockerSuite) TestEnvironmentUnmarshalStringSlice(t sweet.T) {
    + data := `from-files:
    + - filename1
    + - filename2`
    +
    + imp := Environment{}
    + err := yaml.Unmarshal([]byte(data), &imp)
    +
    + Expect(err).To(BeNil())
    + Expect(imp.Files).To(Equal(cYaml.StringOrSlice{"filename1", "filename2"}))
    +}
    --- a/docker/errors.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/docker/errors.go Sun Oct 15 20:19:46 2017 +0000
    @@ -32,4 +32,5 @@
    errNoImages = errors.New("no images specified")
    errNoSourceTag = errors.New("no source tag specified")
    errNoDestinationTags = errors.New("no destination tags specified")
    + errNoFilesEnvironment = errors.New("no environment files specified")
    )
    --- a/docker/export.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/docker/export.go Sun Oct 15 20:19:46 2017 +0000
    @@ -84,7 +84,7 @@
    }
    func (e *Export) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    files, err := st.MapSlice(e.Files, fullEnv)
    if err != nil {
    --- a/docker/import.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/docker/import.go Sun Oct 15 20:19:46 2017 +0000
    @@ -33,7 +33,7 @@
    const importTemplate = `cp {{.source}} {{.workspaceID}}:/workspace/{{.destination}}`
    func (i *Import) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    files, err := st.MapSlice(i.Files, fullEnv)
    if err != nil {
    --- a/docker/login.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/docker/login.go Sun Oct 15 20:19:46 2017 +0000
    @@ -34,7 +34,7 @@
    const loginTemplate = `login -u {{.username}} -p {{.password}}{{if .server}} {{.server}}{{end}}`
    func (l *Login) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    username, err := environment.Mapper(l.Username, fullEnv)
    if err != nil {
    --- a/docker/logout.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/docker/logout.go Sun Oct 15 20:19:46 2017 +0000
    @@ -32,7 +32,7 @@
    const logoutTemplate = `logout{{if .server}} {{.server}}{{end}}`
    func (l *Logout) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    server, err := environment.Mapper(l.Server, fullEnv)
    if err != nil {
    --- a/docker/pull.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/docker/pull.go Sun Oct 15 20:19:46 2017 +0000
    @@ -34,7 +34,7 @@
    const pullTemplate = `pull {{.image}}`
    func (p *Pull) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    images, err := st.MapSlice(p.Images, fullEnv)
    if err != nil {
    --- a/docker/push.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/docker/push.go Sun Oct 15 20:19:46 2017 +0000
    @@ -34,7 +34,7 @@
    const pushTemplate = `push {{.image}}`
    func (p *Push) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    images, err := st.MapSlice(p.Images, fullEnv)
    if err != nil {
    --- a/docker/remove.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/docker/remove.go Sun Oct 15 20:19:46 2017 +0000
    @@ -37,7 +37,7 @@
    const removeTemplate = `rmi {{.image}}`
    func (r *Remove) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    images, err := st.MapSlice(r.Images, fullEnv)
    if err != nil {
    --- a/docker/run.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/docker/run.go Sun Oct 15 20:19:46 2017 +0000
    @@ -127,7 +127,7 @@
    func (r *Run) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, r.Environment)
    - fullEnv = environment.Merge(fullEnv, st.Environment)
    + fullEnv = environment.Merge(fullEnv, st.GetEnv())
    user, err := user.Current()
    if err != nil {
    --- a/docker/tag.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/docker/tag.go Sun Oct 15 20:19:46 2017 +0000
    @@ -35,7 +35,7 @@
    const tagTemplate = `tag {{.source}} {{.destination}}`
    func (t *Tag) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    source, err := environment.Mapper(t.Source, fullEnv)
    if err != nil {
    --- a/environment/mapper.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/environment/mapper.go Sun Oct 15 20:19:46 2017 +0000
    @@ -42,7 +42,7 @@
    }
    }
    - return "$" + name
    + return fmt.Sprintf("${%s}", name)
    }
    // Map will return the value matching a KEY=VAL pair in the given environment.
    --- a/environment/mapper_test.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/environment/mapper_test.go Sun Oct 15 20:19:46 2017 +0000
    @@ -62,11 +62,11 @@
    result, err := Mapper("${FOO}", []string{"FOOBAR"})
    Expect(err).To(BeNil())
    - Expect(result).To(Equal("$FOO"))
    + Expect(result).To(Equal("${FOO}"))
    result, err = Mapper("${BARBAZ}", []string{"BAR"})
    Expect(err).To(BeNil())
    - Expect(result).To(Equal("$BARBAZ"))
    + Expect(result).To(Equal("${BARBAZ}"))
    }
    func (e *environmentSuite) TestRecursiveMatch(t sweet.T) {
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/examples/environment-from-file.yml Sun Oct 15 20:19:46 2017 +0000
    @@ -0,0 +1,56 @@
    +# This plan shows how the inject plan can add additional
    +# environment variables accessible to your tasks during
    +# runtime. Running this with `-e X_FOO=4`, the output
    +# will be (in two parts):
    +#
    +# - X_FOO=4
    +# - X_BAR=5
    +# - X_BAZ=
    +#
    +# - X_FOO=4
    +# - X_BAR=2
    +# - X_BAZ=3
    +#
    +# Notice that before the inject task, only X_FOO and X_BAR
    +# were set by the command line argument and the global env
    +# block, respectively. Afterwards, X_FOO's value did not
    +# change (as command line parameters take have precedence),
    +# X_BAR is overwritten, and X_BAZ is newly added.
    +
    +environment:
    + - X_BAR=5
    +
    +tasks:
    + import:
    + type: import
    + files: .
    +
    + generate:
    + image: gliderlabs/alpine:edge
    + script:
    + - echo "foo=1" >> env
    + - echo "bar=2" >> env
    + - echo "baz=3" >> env
    + workdir: /workspace
    +
    + inject:
    + type: environment
    + from-file: env
    + prefix: "X_"
    +
    + print:
    + image: gliderlabs/alpine:edge
    + script:
    + - echo "X_FOO=${X_FOO}"
    + - echo "X_BAR=${X_BAR}"
    + - echo "X_BAZ=${X_BAZ}"
    +
    +plans:
    + default:
    + stages:
    + - tasks:
    + - import
    + - generate
    + - print
    + - inject
    + - print
    --- a/intrinsic/clean.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/intrinsic/clean.go Sun Oct 15 20:19:46 2017 +0000
    @@ -68,7 +68,7 @@
    }
    func (c *Clean) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv := environment.Merge(env, st.Environment)
    + fullEnv := environment.Merge(env, st.GetEnv())
    wd, err := os.Getwd()
    if err != nil {
    --- a/intrinsic/extend_test.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/intrinsic/extend_test.go Sun Oct 15 20:19:46 2017 +0000
    @@ -30,11 +30,13 @@
    var (
    task = newMockTask()
    fullEnv = []string{}
    - st = &state.State{Environment: []string{"bar=bonk", "quux=honk"}}
    )
    + st := state.New()
    + st.MergeEnv([]string{"bar=bonk", "quux=honk"})
    +
    task.executeFn = func(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - fullEnv = environment.Merge(environment.Merge(env, []string{"x=1", "foo=quux"}), st.Environment)
    + fullEnv = environment.Merge(environment.Merge(env, []string{"x=1", "foo=quux"}), st.GetEnv())
    return nil
    }
    @@ -48,8 +50,8 @@
    Expect(fullEnv).To(ConsistOf("x=1", "foo=bar", "bar=bonk", "quux=honk"))
    // Should revert state
    - Expect(st.Environment).To(HaveLen(2))
    - Expect(st.Environment).To(ConsistOf("bar=bonk", "quux=honk"))
    + Expect(st.GetEnv()).To(HaveLen(2))
    + Expect(st.GetEnv()).To(ConsistOf("bar=bonk", "quux=honk"))
    }
    func (s *intrinsicSuite) TestDependencies(t sweet.T) {
    --- a/main.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/main.go Sun Oct 15 20:19:46 2017 +0000
    @@ -140,10 +140,10 @@
    st.ForceSequential = *forceSequential
    st.EnableSSHAgent = enableSSHAgent
    st.TaskTimeout = *taskTimeout
    - st.Environment = environment.Merge(defEnv, *env)
    st.DockerConfig = *dockerConfig
    st.CPUShares = *cpuShares
    st.Memory = *memory
    + st.MergeEnv(environment.Merge(defEnv, *env))
    if err := st.Valid(); err != nil {
    fmt.Printf("%s\n", err)
    --- a/state/state.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/state/state.go Sun Oct 15 20:19:46 2017 +0000
    @@ -41,12 +41,22 @@
    ForceSequential bool
    EnableSSHAgent bool
    TaskTimeout time.Duration
    - Environment []string
    DockerConfig string
    CPUShares string
    Memory string
    + // Guard the environment around accessors so that the base
    + // environment can be modified while allowing all "wrapped"
    + // environments to get the same updates.
    + innerEnv []string
    +
    + // Additional environment variables that are applied from
    + // the wrapper around the base env. This is applied on demand
    + // to the base environment so that we don't keep around a
    + // stale cache.
    + outerEnv []string
    +
    // States have the ability to "wrap" another one without
    // changing the underlying state. This is used by the
    // extends intrinsic in order to modify the stat without
    @@ -64,19 +74,18 @@
    // to be guarded via mutex.
    runningContainers map[string]struct{}
    detachedContainers map[string]struct{}
    - mutex *sync.RWMutex
    + mutex sync.RWMutex
    }
    func New() *State {
    return &State{
    runningContainers: map[string]struct{}{},
    detachedContainers: map[string]struct{}{},
    - mutex: &sync.RWMutex{},
    }
    }
    func (st *State) Valid() error {
    - if st.parent == nil && (st.detachedContainers == nil || st.mutex == nil) {
    + if st.parent == nil && st.detachedContainers == nil {
    return fmt.Errorf("state must be constructed via New")
    }
    @@ -89,10 +98,47 @@
    return nil
    }
    +// GetEnv gets the fullEnv with the wrapped environments applied.
    +func (st *State) GetEnv() []string {
    + return st.mergeRecursive(st.getInnerEnv())
    +}
    +
    +// getInnerEnv retrieves the original state environment (without wrapping).
    +func (st *State) getInnerEnv() []string {
    + if st.parent != nil {
    + return st.parent.getInnerEnv()
    + }
    +
    + return st.innerEnv
    +}
    +
    +// mergeRecursive applies the wrapped environments recursively.
    +func (st *State) mergeRecursive(env []string) []string {
    + if st.parent != nil {
    + // Merge recursively and then apply the wrapped environment so that
    + // the wrapping takes the highest precedence.
    + return environment.Merge(st.outerEnv, st.parent.mergeRecursive(env))
    + }
    +
    + return env
    +}
    +
    +// MergeEnv adds additional environment arguments with lower precedence to
    +// the original state environment. If they duplicate an existing environment
    +// variable, that value will be unchanged.
    +func (st *State) MergeEnv(env []string) {
    + if st.parent != nil {
    + st.parent.MergeEnv(env)
    + return
    + }
    +
    + st.innerEnv = environment.Merge(env, st.innerEnv)
    +}
    +
    // MapSlice calls SliceMapper on the given environment, but also checks to
    // see if the variable in the env parameter can be expanded into a list.
    -func (st *State) MapSlice(env, fullEnv []string) ([]string, error) {
    - prev := env
    +func (st *State) MapSlice(slice, fullEnv []string) ([]string, error) {
    + prev := slice
    // Protect ourselves against a weird infinite expansion. This can
    // happen if something occurs like X => A,$Y; Y => B,$X. This is
    @@ -107,13 +153,13 @@
    // If we haven't made a change, return the final result.
    if isSame(next, prev) {
    - return next, nil
    + return environment.SliceMapper(next, fullEnv)
    }
    prev = next
    }
    - return nil, fmt.Errorf("hit limit while expanding '%s'", env)
    + return nil, fmt.Errorf("hit limit while expanding '%s'", slice)
    }
    func (st *State) expandSlice(env, fullEnv []string) ([]string, error) {
    @@ -203,10 +249,6 @@
    // the same as ignoring the wrapped values and using the underlying state. This stack
    // is used to map a slice within an extended task.
    func (st *State) WrapWithExpandableEnv(env, expandable []string, delimiter string) *State {
    - // Merge the environment into this map, but do NOT override anything that
    - // is currently in the state's environment - this has higher precedence.
    - env = environment.Merge(env, st.Environment)
    -
    return &State{
    CleanupList: st.CleanupList,
    Network: st.Network,
    @@ -215,10 +257,10 @@
    ForceSequential: st.ForceSequential,
    EnableSSHAgent: st.EnableSSHAgent,
    TaskTimeout: st.TaskTimeout,
    - Environment: env,
    DockerConfig: st.DockerConfig,
    CPUShares: st.CPUShares,
    Memory: st.Memory,
    + outerEnv: env,
    parent: st,
    expandables: expandable,
    expandableDelimiter: delimiter,
    --- a/state/state_test.go Sun Oct 15 15:17:30 2017 -0500
    +++ b/state/state_test.go Sun Oct 15 20:19:46 2017 +0000
    @@ -37,11 +37,23 @@
    })
    }
    +func (s *stateSuite) TestWrappedEnvironment(t sweet.T) {
    + st1 := New()
    + st1.MergeEnv([]string{"Y=0"})
    + st2 := st1.WrapWithExpandableEnv([]string{"X=1", "Y=1"}, []string{}, ";")
    + st3 := st2.WrapWithExpandableEnv([]string{"X=2", "Y=2"}, []string{}, ";")
    + st4 := st3.WrapWithExpandableEnv([]string{"X=3", "Y=3", "Z=3"}, []string{}, ";")
    +
    + // Things in the "base" env are not overriden, but things
    + // that are wrapped take the highest precedence
    + Expect(st4.GetEnv()).To(ConsistOf([]string{"X=1", "Y=0", "Z=3"}))
    +}
    +
    func (s *stateSuite) TestMapSlice(t sweet.T) {
    - st1 := &State{}
    - st1.Environment = []string{"FOO=BAR"}
    + st1 := New()
    + st1.MergeEnv([]string{"FOO=BAR"})
    - Expect(mapEnv(st1, "$X")).To(ConsistOf([]string{"$X"}))
    + Expect(mapEnv(st1, "$X")).To(ConsistOf([]string{"${X}"}))
    Expect(mapEnv(st1, "$FOO")).To(ConsistOf([]string{"BAR"}))
    st2 := st1.WrapWithExpandableEnv([]string{"X=A;B;C"}, []string{"X"}, ";")
    @@ -55,8 +67,8 @@
    }
    func (s *stateSuite) TestMapSliceComplex(t sweet.T) {
    - st := &State{}
    - st.Environment = []string{"FOO=BAR"}
    + st := New()
    + st.MergeEnv([]string{"FOO=BAR"})
    st = st.WrapWithExpandableEnv([]string{"X=A;B;C", "Y=D;E;F"}, []string{"X", "Y"}, ";")
    // No expansion
    @@ -80,27 +92,27 @@
    }
    func (s *stateSuite) TestMapSliceIterative(t sweet.T) {
    - st := &State{}
    - st.Environment = []string{"FOO=BAR"}
    + st := New()
    + st.MergeEnv([]string{"FOO=BAR"})
    st = st.WrapWithExpandableEnv([]string{"X=A;x$Y", "Y=B;y$Z", "Z=C;D;E"}, []string{"X", "Y", "Z"}, ";")
    Expect(mapEnv(st, "$X")).To(ConsistOf([]string{"A", "xB", "xyC", "xyD", "xyE"}))
    }
    func (s *stateSuite) TestMapSliceIndirect(t sweet.T) {
    - st := &State{}
    - st.Environment = []string{"FOO=BAR"}
    + st := New()
    + st.MergeEnv([]string{"FOO=BAR"})
    st = st.WrapWithExpandableEnv([]string{"X=$Y", "Y=A;$Z;C", "Z=B;$W", "W=D"}, []string{"X", "Y", "Z"}, ";")
    Expect(mapEnv(st, "$X")).To(ConsistOf([]string{"A", "B", "C", "D"}))
    }
    func (s *stateSuite) TestMapSliceRecursive(t sweet.T) {
    - st := &State{}
    - st.Environment = []string{"FOO=BAR"}
    + st := New()
    + st.MergeEnv([]string{"FOO=BAR"})
    st = st.WrapWithExpandableEnv([]string{"X=A;x$Y", "Y=B;y$Z", "Z=C;D;$X"}, []string{"X", "Y", "Z"}, ";")
    - _, err := st.MapSlice([]string{"$X"}, st.Environment)
    + _, err := st.MapSlice([]string{"$X"}, st.GetEnv())
    Expect(err).NotTo(BeNil())
    Expect(err.Error()).To(ContainSubstring("hit limit"))
    }
    @@ -135,7 +147,7 @@
    }
    func (s *stateSuite) TestWrapParent(t sweet.T) {
    - st1 := &State{}
    + st1 := New()
    st1.WrapWithExpandableEnv(nil, nil, "")
    Expect(st1.parent).To(BeNil())
    @@ -144,14 +156,14 @@
    }
    func (s *stateSuite) TestWrapWithExpandableEnvMap(t sweet.T) {
    - st1 := &State{}
    - st1.Environment = []string{"FOO=BAR", "BAR=BAZ"}
    - Expect(st1.Environment).To(HaveLen(2))
    - Expect(st1.Environment).To(ConsistOf([]string{"FOO=BAR", "BAR=BAZ"}))
    + st1 := New()
    + st1.MergeEnv([]string{"FOO=BAR", "BAR=BAZ"})
    + Expect(st1.GetEnv()).To(HaveLen(2))
    + Expect(st1.GetEnv()).To(ConsistOf([]string{"FOO=BAR", "BAR=BAZ"}))
    st2 := st1.WrapWithExpandableEnv([]string{"FOO=BONK", "BAZ=BONK"}, nil, "")
    - Expect(st2.Environment).To(HaveLen(3))
    - Expect(st2.Environment).To(ConsistOf([]string{"FOO=BAR", "BAR=BAZ", "BAZ=BONK"}))
    + Expect(st2.GetEnv()).To(HaveLen(3))
    + Expect(st2.GetEnv()).To(ConsistOf([]string{"FOO=BAR", "BAR=BAZ", "BAZ=BONK"}))
    }
    func (s *stateSuite) TestRunning(t sweet.T) {
    @@ -195,7 +207,7 @@
    }
    func mapEnv(st *State, val string) []string {
    - mapped, err := st.MapSlice([]string{val}, st.Environment)
    + mapped, err := st.MapSlice([]string{val}, st.GetEnv())
    Expect(err).To(BeNil())
    return mapped
    }