grim/convey

Update state to hold a stack of expand frames.
auto-split
2017-09-21, Eric Fritz
98eb1323e154
Parents caa707f4e2d1
Children bd7a742de64a
Update state to hold a stack of expand frames.
15 files changed, 224 insertions(+), 73 deletions(-)
  • +8 -3
    docker/build.go
  • +1 -1
    docker/export.go
  • +1 -1
    docker/import.go
  • +1 -1
    docker/pull.go
  • +1 -1
    docker/push.go
  • +1 -1
    docker/remove.go
  • +7 -12
    docker/run.go
  • +1 -1
    docker/tag.go
  • +0 -16
    environment/mapper.go
  • +0 -14
    environment/mapper_test.go
  • +4 -1
    examples/extend-slices.yml
  • +1 -1
    examples/extend.yml
  • +14 -20
    intrinsic/extend.go
  • +84 -0
    state/state.go
  • +100 -0
    state/state_test.go
    • --- a/docker/build.go Thu Sep 21 15:49:58 2017 -0500
    +++ b/docker/build.go Thu Sep 21 19:25:24 2017 -0500
    @@ -48,7 +48,7 @@
    func (b *Build) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    - files, err := environment.SliceMapper(b.Files, fullEnv)
    + files, err := st.MapSlice(b.Files, fullEnv)
    if err != nil {
    return err
    }
    @@ -90,7 +90,12 @@
    return err
    }
    - labels, err := environment.SliceMapper(b.Labels, fullEnv)
    + labels, err := st.MapSlice(b.Labels, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    + arguments, err := st.MapSlice(b.Arguments, fullEnv)
    if err != nil {
    return err
    }
    @@ -101,7 +106,7 @@
    "tag": tag,
    "buildContext": tmpDir,
    "Labels": labels,
    - "Arguments": b.Arguments,
    + "Arguments": arguments,
    }
    return Docker(name, buildTemplate, params, st)
    --- a/docker/export.go Thu Sep 21 15:49:58 2017 -0500
    +++ b/docker/export.go Thu Sep 21 19:25:24 2017 -0500
    @@ -86,7 +86,7 @@
    func (e *Export) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    - files, err := environment.SliceMapper(e.Files, fullEnv)
    + files, err := st.MapSlice(e.Files, fullEnv)
    if err != nil {
    return err
    }
    --- a/docker/import.go Thu Sep 21 15:49:58 2017 -0500
    +++ b/docker/import.go Thu Sep 21 19:25:24 2017 -0500
    @@ -35,7 +35,7 @@
    func (i *Import) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    - files, err := environment.SliceMapper(i.Files, fullEnv)
    + files, err := st.MapSlice(i.Files, fullEnv)
    if err != nil {
    return err
    }
    --- a/docker/pull.go Thu Sep 21 15:49:58 2017 -0500
    +++ b/docker/pull.go Thu Sep 21 19:25:24 2017 -0500
    @@ -36,7 +36,7 @@
    func (p *Pull) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    - images, err := environment.SliceMapper(p.Images, fullEnv)
    + images, err := st.MapSlice(p.Images, fullEnv)
    if err != nil {
    return err
    }
    --- a/docker/push.go Thu Sep 21 15:49:58 2017 -0500
    +++ b/docker/push.go Thu Sep 21 19:25:24 2017 -0500
    @@ -36,7 +36,7 @@
    func (p *Push) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    - images, err := environment.SliceMapper(p.Images, fullEnv)
    + images, err := st.MapSlice(p.Images, fullEnv)
    if err != nil {
    return err
    }
    --- a/docker/remove.go Thu Sep 21 15:49:58 2017 -0500
    +++ b/docker/remove.go Thu Sep 21 19:25:24 2017 -0500
    @@ -36,7 +36,7 @@
    func (r *Remove) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    - images, err := environment.SliceMapper(r.Images, fullEnv)
    + images, err := st.MapSlice(r.Images, fullEnv)
    if err != nil {
    return err
    }
    --- a/docker/run.go Thu Sep 21 15:49:58 2017 -0500
    +++ b/docker/run.go Thu Sep 21 19:25:24 2017 -0500
    @@ -88,7 +88,7 @@
    {{.Image}}{{if .Command}} {{.Command}}{{end}}`
    // buildScript will create a shell script for the given commands
    -func (r *Run) buildScript(fullEnv []string) (string, string, string, error) {
    +func (r *Run) buildScript(st *state.State, fullEnv []string) (string, string, string, error) {
    entryPoint := r.Shell
    if entryPoint == "" {
    entryPoint = "/bin/sh"
    @@ -118,18 +118,13 @@
    // set the run command argument to the script file
    commandArg := scriptFile
    - // iterate the script and run the environment variable expansion on each line
    - for idx, item := range r.Script {
    - item, err := environment.Mapper(item, fullEnv)
    - if err != nil {
    - return "", "", "", err
    - }
    -
    - r.Script[idx] = item
    + scripts, err := st.MapSlice(r.Script, fullEnv)
    + if err != nil {
    + return "", "", "", err
    }
    // write the script to the file
    - script.WriteString(strings.Join(r.Script, "\n"))
    + script.WriteString(strings.Join(scripts, "\n"))
    script.Close()
    // make the script executable to the user
    @@ -175,7 +170,7 @@
    // if we're using a script defined in the yaml, create it and override
    // some variables
    if len(r.Script) > 0 {
    - scriptFile, entryPoint, commandArg, err = r.buildScript(fullEnv)
    + scriptFile, entryPoint, commandArg, err = r.buildScript(st, fullEnv)
    if err != nil {
    return err
    }
    @@ -198,7 +193,7 @@
    return err
    }
    - labels, err := environment.SliceMapper(r.Labels, fullEnv)
    + labels, err := st.MapSlice(r.Labels, fullEnv)
    if err != nil {
    return err
    }
    --- a/docker/tag.go Thu Sep 21 15:49:58 2017 -0500
    +++ b/docker/tag.go Thu Sep 21 19:25:24 2017 -0500
    @@ -42,7 +42,7 @@
    return err
    }
    - destinations, err := environment.SliceMapper(t.Destinations, fullEnv)
    + destinations, err := st.MapSlice(t.Destinations, fullEnv)
    if err != nil {
    return err
    }
    --- a/environment/mapper.go Thu Sep 21 15:49:58 2017 -0500
    +++ b/environment/mapper.go Thu Sep 21 19:25:24 2017 -0500
    @@ -79,19 +79,3 @@
    return mapped, nil
    }
    -
    -// SliceMapExpander expand yaml.StringOrSlice elements as follows: if value
    -// is a single item of the form `$var`, then split on ';'. Otherwise, expand
    -// every element separately in-place, just as in SliceMapper.
    -func SliceMapExpander(slice []string, env []string) ([]string, error) {
    - if len(slice) != 1 || strings.TrimSpace(os.Expand(slice[0], func(string) string { return "" })) != "" {
    - return SliceMapper(slice, env)
    - }
    -
    - expanded, err := Mapper(slice[0], env)
    - if err != nil {
    - return nil, err
    - }
    -
    - return strings.SplitN(expanded, ";", -1), nil
    -}
    --- a/environment/mapper_test.go Thu Sep 21 15:49:58 2017 -0500
    +++ b/environment/mapper_test.go Thu Sep 21 19:25:24 2017 -0500
    @@ -79,17 +79,3 @@
    _, err := Mapper("${FOO}", []string{"FOO=$BAR", "BAR=$FOO"})
    Expect(err).To(MatchError("infinite environment mapping loop while expanding '$BAR'"))
    }
    -
    -func (e *environmentSuite) TestSliceMapExpander(t sweet.T) {
    - // Single item
    - Expect(SliceMapExpander([]string{"${FOO}"}, []string{"FOO=bar"})).To(Equal([]string{"bar"}))
    -
    - // Expanding this as it's the only item containing a `;`
    - Expect(SliceMapExpander([]string{"${FOO}"}, []string{"FOO=bar;baz"})).To(Equal([]string{"bar", "baz"}))
    -
    - // Not ONLY expanding this, so we leave it
    - Expect(SliceMapExpander([]string{"1${FOO}2"}, []string{"FOO=bar;baz"})).To(Equal([]string{"1bar;baz2"}))
    -
    - // Multiple items in array
    - Expect(SliceMapExpander([]string{"${FOO}", "${FOO}"}, []string{"FOO=bar;baz"})).To(Equal([]string{"bar;baz", "bar;baz"}))
    -}
    --- a/examples/extend-slices.yml Thu Sep 21 15:49:58 2017 -0500
    +++ b/examples/extend-slices.yml Thu Sep 21 19:25:24 2017 -0500
    @@ -16,7 +16,10 @@
    type: extend
    task: tag-abstract
    environment:
    - - IMAGE_TAGS=foo;bar;baz
    + - IMAGE_TAGS=foo,bar,baz
    + expand:
    + - IMAGE_TAGS
    + expand_delimiter: ","
    plans:
    default:
    --- a/examples/extend.yml Thu Sep 21 15:49:58 2017 -0500
    +++ b/examples/extend.yml Thu Sep 21 19:25:24 2017 -0500
    @@ -17,7 +17,7 @@
    baz:
    type: extend
    - task: foo
    + task: bar
    environment:
    - MESSAGE=baz
    --- a/intrinsic/extend.go Thu Sep 21 15:49:58 2017 -0500
    +++ b/intrinsic/extend.go Thu Sep 21 19:25:24 2017 -0500
    @@ -23,17 +23,16 @@
    "github.com/aphistic/gomol"
    "github.com/mohae/deepcopy"
    - "bitbucket.org/rw_grim/convey/environment"
    "bitbucket.org/rw_grim/convey/state"
    "bitbucket.org/rw_grim/convey/tasks"
    "bitbucket.org/rw_grim/convey/yaml"
    )
    type Extend struct {
    - Task string `yaml:"task"`
    - Environment yaml.StringOrSlice `yaml:"environment"`
    - Split yaml.StringOrSlice `yaml:"split"`
    - SplitDelimiter string `yaml:"split_delimiter" default:";"`
    + Task string `yaml:"task"`
    + Environment yaml.StringOrSlice `yaml:"environment"`
    + Expand yaml.StringOrSlice `yaml:"expand"`
    + ExpandDelimiter string `yaml:"expand_delimiter"`
    // A copy of the extended task. Must be public so that deepcopy
    // can serialize it properly in case an extended task is extended
    @@ -43,22 +42,13 @@
    }
    func (e *Extend) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - // We want the extending environment to take precedence over the extended
    - // task environment (if it exists). In order to get the correct precedence,
    - // we stash it onto the state's environment. Once we're done executing the
    - // inner task we'll restore it to the previous values. Notice that we merge
    - // in what appears at first as backwards - we don't want to override command
    - // line arguments, so we let state keep a higher precedence.
    + // While extending, certain environment variables can be expanded into
    + // lists. We store this meta information as a stack on the state, which
    + // is passed into the inner task execute method (and so on).
    + st.PushExtendFrame(e.Environment, e.Expand, e.ExpandDelimiter)
    + defer st.PopExtendFrame()
    - // TODO - whitelist split things, need to add to state
    - // Split
    - // SplitDelimiter
    -
    - stashed := st.Environment
    - st.Environment = environment.Merge(e.Environment, st.Environment)
    - err := e.InnerTask.Execute(name, logger, env, st)
    - st.Environment = stashed
    - return err
    + return e.InnerTask.Execute(name, logger, env, st)
    }
    func (e *Extend) New() tasks.Task {
    @@ -66,6 +56,10 @@
    }
    func (e *Extend) Valid() error {
    + if e.ExpandDelimiter == "" {
    + e.ExpandDelimiter = ";"
    + }
    +
    return nil
    }
    --- a/state/state.go Thu Sep 21 15:49:58 2017 -0500
    +++ b/state/state.go Thu Sep 21 19:25:24 2017 -0500
    @@ -18,10 +18,13 @@
    package state
    import (
    + "errors"
    "fmt"
    "os"
    + "strings"
    "time"
    + "bitbucket.org/rw_grim/convey/environment"
    "bitbucket.org/rw_grim/convey/network"
    "bitbucket.org/rw_grim/convey/workspace"
    )
    @@ -35,6 +38,7 @@
    EnableSSHAgent bool
    TaskTimeout time.Duration
    Environment []string
    + ExtendFrames []extendFrame
    DockerConfig string
    CPUShares string
    @@ -43,6 +47,14 @@
    DetachedContainers []string
    }
    +type extendFrame struct {
    + expandable []string
    + delimiter string
    + stashedEnv []string
    +}
    +
    +var errEmptyExtendStack = errors.New("empty extend stack")
    +
    func (st *State) Valid() error {
    if st.EnableSSHAgent {
    if val := os.Getenv("SSH_AUTH_SOCK"); val == "" {
    @@ -52,3 +64,75 @@
    return nil
    }
    +
    +// MapSlice calls SliceMapper on the given environment, but also checks to
    +// see if the variable could be expanded into a list.
    +func (st *State) MapSlice(env, fullEnv []string) ([]string, error) {
    + if len(env) == 1 {
    + if delimiter, ok := st.getDelimiter(getName(env[0])); ok {
    + mapped, err := environment.Mapper(env[0], fullEnv)
    + if err != nil {
    + return nil, err
    + }
    +
    + return strings.SplitN(mapped, delimiter, -1), nil
    + }
    + }
    +
    + return environment.SliceMapper(env, fullEnv)
    +}
    +
    +func (st *State) getDelimiter(name string) (string, bool) {
    + for i := len(st.ExtendFrames) - 1; i >= 0; i-- {
    + frame := st.ExtendFrames[i]
    +
    + for _, expandable := range frame.expandable {
    + if expandable == name {
    + return frame.delimiter, true
    + }
    + }
    + }
    +
    + return "", false
    +}
    +
    +// PushExtendFrame will store the set of expandable environment variables
    +// and the current set of environments which are restored on pop. These frames
    +// are used to map a slice within an extneded task.
    +func (st *State) PushExtendFrame(env, expandable []string, delimiter string) {
    + st.ExtendFrames = append(st.ExtendFrames, extendFrame{
    + expandable: expandable,
    + delimiter: delimiter,
    + stashedEnv: st.Environment,
    + })
    +
    + // Merge the environment into this map, but do NOT override anything that
    + // is currently in the state's enviornment - this has higher precedence.
    + st.Environment = environment.Merge(env, st.Environment)
    +}
    +
    +// PopExtendFrame will remove one frame from the top of the stack.
    +func (st *State) PopExtendFrame() error {
    + idx := len(st.ExtendFrames) - 1
    + if idx < 0 {
    + return errEmptyExtendStack
    + }
    +
    + last := st.ExtendFrames[idx]
    + st.ExtendFrames = st.ExtendFrames[:idx]
    + st.Environment = last.stashedEnv
    +
    + return nil
    +}
    +
    +// getName returns the name of the variable `var` if the string is of the
    +// form $var or ${var}.
    +func getName(env string) string {
    + env = strings.TrimSpace(env)
    +
    + if strings.HasPrefix(env, "$") {
    + return strings.Trim(env[1:], "{}")
    + }
    +
    + return ""
    +}
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/state/state_test.go Thu Sep 21 19:25:24 2017 -0500
    @@ -0,0 +1,100 @@
    +/*
    + * Convey
    + * Copyright 2016-2017 Gary Kramlich <grim@reaperworld.com>
    + *
    + * This program is free software: you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License as published by
    + * the Free Software Foundation, either version 3 of the License, or
    + * (at your option) any later version.
    + *
    + * This program is distributed in the hope that it will be useful,
    + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    + * GNU General Public License for more details.
    + *
    + * You should have received a copy of the GNU General Public License
    + * along with this program. If not, see <http://www.gnu.org/licenses/>.
    + */
    +package state
    +
    +import (
    + "testing"
    +
    + "github.com/aphistic/sweet"
    + junit "github.com/aphistic/sweet-junit"
    + . "github.com/onsi/gomega"
    +)
    +
    +type stateSuite struct{}
    +
    +func TestMain(m *testing.M) {
    + RegisterFailHandler(sweet.GomegaFail)
    +
    + sweet.Run(m, func(s *sweet.S) {
    + s.RegisterPlugin(junit.NewPlugin())
    +
    + s.AddSuite(&stateSuite{})
    + })
    +}
    +
    +func (e *stateSuite) TestMap(t sweet.T) {
    + st := &State{}
    + st.Environment = []string{"FOO=BAR"}
    +
    + mapEnv := func(val string) []string {
    + mapped, err := st.MapSlice([]string{val}, st.Environment)
    + Expect(err).To(BeNil())
    + return mapped
    + }
    +
    + Expect(mapEnv("$X")).To(Equal([]string{"$X"}))
    + Expect(mapEnv("$FOO")).To(Equal([]string{"BAR"}))
    +
    + st.PushExtendFrame([]string{"X=A;B;C"}, []string{"X"}, ";")
    + Expect(mapEnv("$X")).To(Equal([]string{"A", "B", "C"}))
    + Expect(mapEnv("$FOO")).To(Equal([]string{"BAR"}))
    +
    + st.PushExtendFrame([]string{"BAR=B;A;R::B;A;Z", "FOO=SAME"}, []string{"BAR"}, "::")
    + Expect(mapEnv("$X")).To(Equal([]string{"A", "B", "C"}))
    + Expect(mapEnv("$FOO")).To(Equal([]string{"BAR"}))
    + Expect(mapEnv("$BAR")).To(Equal([]string{"B;A;R", "B;A;Z"}))
    +
    + Expect(st.PopExtendFrame()).To(BeNil())
    + Expect(st.PopExtendFrame()).To(BeNil())
    + Expect(mapEnv("$X")).To(Equal([]string{"$X"}))
    + Expect(mapEnv("$FOO")).To(Equal([]string{"BAR"}))
    +}
    +
    +func (e *stateSuite) TestPopExtendFrameSize(t sweet.T) {
    + st := &State{}
    + st.PushExtendFrame(nil, nil, "")
    + Expect(st.ExtendFrames).To(HaveLen(1))
    + st.PushExtendFrame(nil, nil, "")
    + Expect(st.ExtendFrames).To(HaveLen(2))
    + Expect(st.PopExtendFrame()).To(BeNil())
    + Expect(st.ExtendFrames).To(HaveLen(1))
    + Expect(st.PopExtendFrame()).To(BeNil())
    + Expect(st.ExtendFrames).To(HaveLen(0))
    +}
    +
    +func (e *stateSuite) TestPopExtendFrameRestoresEnv(t sweet.T) {
    + st := &State{}
    + st.Environment = []string{"FOO=BAR", "BAR=BAZ"}
    + st.PushExtendFrame([]string{"FOO=BONK", "BAZ=BONK"}, nil, "")
    + Expect(st.Environment).To(HaveLen(3))
    + Expect(st.Environment).To(ConsistOf([]string{"FOO=BAR", "BAR=BAZ", "BAZ=BONK"}))
    + st.PopExtendFrame()
    + Expect(st.Environment).To(HaveLen(2))
    + Expect(st.Environment).To(ConsistOf([]string{"FOO=BAR", "BAR=BAZ"}))
    +}
    +
    +func (e *stateSuite) TestPopExtendFrameEmpty(t sweet.T) {
    + st := &State{}
    + Expect(st.PopExtendFrame()).To(Equal(errEmptyExtendStack))
    +}
    +
    +func (e *stateSuite) TestGetName(t sweet.T) {
    + Expect(getName("foo")).To(Equal(""))
    + Expect(getName("$FOO")).To(Equal("FOO"))
    + Expect(getName("${FOO}")).To(Equal("FOO"))
    +}