grim/convey

Merged in efritz/convey/auto-split (pull request #22)

2017-09-26, Gary Kramlich
93c8bf5f9bfb
Parents 8861608b16c824a966ab69e9
Children 4fd7caa7c563
Merged in efritz/convey/auto-split (pull request #22)

[Possible Feature] Expand envvars into lists

Approved-by: Gary Kramlich
14 files changed, 320 insertions(+), 72 deletions(-)
  • +30 -7
    REFERENCE.md
  • +13 -8
    docker/build.go
  • +5 -5
    docker/export.go
  • +6 -1
    docker/import.go
  • +5 -5
    docker/pull.go
  • +5 -5
    docker/push.go
  • +5 -5
    docker/remove.go
  • +7 -12
    docker/run.go
  • +9 -9
    docker/tag.go
  • +31 -0
    examples/extend-slices.yml
  • +1 -1
    examples/extend.yml
  • +14 -14
    intrinsic/extend.go
  • +89 -0
    state/state.go
  • +100 -0
    state/state_test.go
    • --- a/REFERENCE.md Mon Sep 25 20:25:16 2017 -0500
    +++ b/REFERENCE.md Tue Sep 26 01:52:19 2017 +0000
    @@ -1,4 +1,4 @@
    -# Configuration
    +# Configuration
    Configuring convey is done via a file named `convey.yml`. This file defines the tasks as well as the plans. This document explains how that file is structured and what options are available.
    @@ -41,9 +41,13 @@
    This example shows how to have one file extend another, inheriting all of the plans and tasks that are
    not otherwise overwritten by the configuration file.
    -## login-logout.yml
    +## extends-slice.yml
    +
    +This example shows how to expand an environment variable into a list in the context of a extended task.
    -This examples show how you can login and logout of a Docker registry by using environment variables.
    +## login-logout.yml
    +
    +This example shows how you can login and logout of a Docker registry by using environment variables.
    ## script.yml
    @@ -122,10 +126,14 @@
    #### Attributes
    -| Name | Required | Default | Description |
    -| ----------- | -------- | ------- | ----------- |
    -| task | Yes | | The name of the task to extend |
    -| environment | | | The override environment |
    +| Name | Required | Default | Description |
    +| ---------------- | -------- | ------- | ----------- |
    +| task | Yes | | The name of the task to extend |
    +| environment | | | The override environment |
    +| expand | | | The envvars which can be expanded into lists on use |
    +| expand_delimiter | | ; | The string to split mapped values by |
    +
    +The `expand` and `expand_delimiter` attributes allow an environment variable, which is an unstructured string, to act as if it is an array. This is useful for supplying an "abstract" parent task with several arguments for the same attribute (e.g. tags in a build task). Variables that occur in the environment with a name in expand will be split by the expand delimiter to form a list, where applicable.
    #### Example
    @@ -142,6 +150,21 @@
    environment:
    - MESSAGE="Hi!"
    +#### Example with Variable Expansion
    +
    + tag-alpine:
    + type: tag
    + source: gliderlabs/alpine:edge
    + destinations: ${TAGS}
    +
    + tag-concrete:
    + type: extend
    + task: tag-alpine
    + environment:
    + - TAGS=foo;bar;baz
    + expand:
    + - TAGS
    +
    ----
    ### Build task
    --- a/docker/build.go Mon Sep 25 20:25:16 2017 -0500
    +++ b/docker/build.go Tue Sep 26 01:52:19 2017 +0000
    @@ -48,6 +48,11 @@
    func (b *Build) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    + files, err := st.MapSlice(b.Files, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    // create out build directory
    tmpDir, err := ioutil.TempDir("", "convey-build-")
    if err != nil {
    @@ -59,12 +64,7 @@
    base := filepath.Dir(b.Dockerfile)
    // export the files to it
    - for _, src := range b.Files {
    - src, err := environment.Mapper(src, fullEnv)
    - if err != nil {
    - return err
    - }
    -
    + for _, src := range files {
    src, dest := tasks.ParseFilePath(base, src)
    cleanDest := filepath.Clean(filepath.Join(tmpDir, dest))
    @@ -90,7 +90,12 @@
    return err
    }
    - labels, err := environment.SliceMapper(b.Labels, fullEnv)
    + labels, err := st.MapSlice(b.Labels, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    + arguments, err := st.MapSlice(b.Arguments, fullEnv)
    if err != nil {
    return err
    }
    @@ -101,7 +106,7 @@
    "tag": tag,
    "buildContext": tmpDir,
    "Labels": labels,
    - "Arguments": b.Arguments,
    + "Arguments": arguments,
    }
    return Docker(name, buildTemplate, params, st)
    --- a/docker/export.go Mon Sep 25 20:25:16 2017 -0500
    +++ b/docker/export.go Tue Sep 26 01:52:19 2017 +0000
    @@ -86,12 +86,12 @@
    func (e *Export) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    - for _, file := range e.Files {
    - file, err := environment.Mapper(file, fullEnv)
    - if err != nil {
    - return err
    - }
    + files, err := st.MapSlice(e.Files, fullEnv)
    + if err != nil {
    + return err
    + }
    + for _, file := range files {
    if err := checkFilePattern(file); err != nil {
    return err
    }
    --- a/docker/import.go Mon Sep 25 20:25:16 2017 -0500
    +++ b/docker/import.go Tue Sep 26 01:52:19 2017 +0000
    @@ -35,7 +35,12 @@
    func (i *Import) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    - for _, file := range i.Files {
    + files, err := st.MapSlice(i.Files, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    + for _, file := range files {
    file, err := environment.Mapper(file, fullEnv)
    if err != nil {
    return err
    --- a/docker/pull.go Mon Sep 25 20:25:16 2017 -0500
    +++ b/docker/pull.go Tue Sep 26 01:52:19 2017 +0000
    @@ -36,12 +36,12 @@
    func (p *Pull) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    - for _, image := range p.Images {
    - image, err := environment.Mapper(image, fullEnv)
    - if err != nil {
    - return err
    - }
    + images, err := st.MapSlice(p.Images, fullEnv)
    + if err != nil {
    + return err
    + }
    + for _, image := range images {
    params := map[string]interface{}{
    "image": image,
    }
    --- a/docker/push.go Mon Sep 25 20:25:16 2017 -0500
    +++ b/docker/push.go Tue Sep 26 01:52:19 2017 +0000
    @@ -36,12 +36,12 @@
    func (p *Push) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    - for _, image := range p.Images {
    - image, err := environment.Mapper(image, fullEnv)
    - if err != nil {
    - return err
    - }
    + images, err := st.MapSlice(p.Images, fullEnv)
    + if err != nil {
    + return err
    + }
    + for _, image := range images {
    params := map[string]interface{}{
    "image": image,
    }
    --- a/docker/remove.go Mon Sep 25 20:25:16 2017 -0500
    +++ b/docker/remove.go Tue Sep 26 01:52:19 2017 +0000
    @@ -36,12 +36,12 @@
    func (r *Remove) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    - for _, image := range r.Images {
    - image, err := environment.Mapper(image, fullEnv)
    - if err != nil {
    - return err
    - }
    + images, err := st.MapSlice(r.Images, fullEnv)
    + if err != nil {
    + return err
    + }
    + for _, image := range images {
    params := map[string]interface{}{
    "image": image,
    }
    --- a/docker/run.go Mon Sep 25 20:25:16 2017 -0500
    +++ b/docker/run.go Tue Sep 26 01:52:19 2017 +0000
    @@ -88,7 +88,7 @@
    {{.Image}}{{if .Command}} {{.Command}}{{end}}`
    // buildScript will create a shell script for the given commands
    -func (r *Run) buildScript(fullEnv []string) (string, string, string, error) {
    +func (r *Run) buildScript(st *state.State, fullEnv []string) (string, string, string, error) {
    entryPoint := r.Shell
    if entryPoint == "" {
    entryPoint = "/bin/sh"
    @@ -118,18 +118,13 @@
    // set the run command argument to the script file
    commandArg := scriptFile
    - // iterate the script and run the environment variable exapansion on each line
    - for idx, item := range r.Script {
    - item, err := environment.Mapper(item, fullEnv)
    - if err != nil {
    - return "", "", "", err
    - }
    -
    - r.Script[idx] = item
    + scripts, err := st.MapSlice(r.Script, fullEnv)
    + if err != nil {
    + return "", "", "", err
    }
    // write the script to the file
    - script.WriteString(strings.Join(r.Script, "\n"))
    + script.WriteString(strings.Join(scripts, "\n"))
    script.Close()
    // make the script executable to the user
    @@ -175,7 +170,7 @@
    // if we're using a script defined in the yaml, create it and override
    // some variables
    if len(r.Script) > 0 {
    - scriptFile, entryPoint, commandArg, err = r.buildScript(fullEnv)
    + scriptFile, entryPoint, commandArg, err = r.buildScript(st, fullEnv)
    if err != nil {
    return err
    }
    @@ -198,7 +193,7 @@
    return err
    }
    - labels, err := environment.SliceMapper(r.Labels, fullEnv)
    + labels, err := st.MapSlice(r.Labels, fullEnv)
    if err != nil {
    return err
    }
    --- a/docker/tag.go Mon Sep 25 20:25:16 2017 -0500
    +++ b/docker/tag.go Tue Sep 26 01:52:19 2017 +0000
    @@ -37,17 +37,17 @@
    func (t *Tag) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    fullEnv := environment.Merge(env, st.Environment)
    - for _, destination := range t.Destinations {
    - source, err := environment.Mapper(t.Source, fullEnv)
    - if err != nil {
    - return err
    - }
    + source, err := environment.Mapper(t.Source, fullEnv)
    + if err != nil {
    + return err
    + }
    - destination, err := environment.Mapper(destination, fullEnv)
    - if err != nil {
    - return err
    - }
    + destinations, err := st.MapSlice(t.Destinations, fullEnv)
    + if err != nil {
    + return err
    + }
    + for _, destination := range destinations {
    params := map[string]interface{}{
    "source": source,
    "destination": destination,
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/examples/extend-slices.yml Tue Sep 26 01:52:19 2017 +0000
    @@ -0,0 +1,31 @@
    +# This example shows how to use automatic splitting of expanded
    +# environment variables over directives that take arrays. Running
    +# this configuration file executes the following commands:
    +#
    +# - docker tag gliderlabs/alpine:edge baz
    +# - docker tag gliderlabs/alpine:edge bonk
    +
    +tasks:
    + tag-abstract:
    + type: tag
    + source: gliderlabs/alpine:edge
    + destinations: ${IMAGE_TAGS}
    +
    + tag-intermediate:
    + type: extend
    + task: tag-abstract
    + environment:
    + - IMAGE_TAGS=foo;bar;baz
    + expand:
    + - IMAGE_TAGS
    +
    + tag-concrete:
    + type: extend
    + task: tag-intermediate
    + environment:
    + - IMAGE_TAGS=baz;bonk
    +
    +plans:
    + default:
    + stages:
    + - tasks: tag-concrete
    --- a/examples/extend.yml Mon Sep 25 20:25:16 2017 -0500
    +++ b/examples/extend.yml Tue Sep 26 01:52:19 2017 +0000
    @@ -17,7 +17,7 @@
    baz:
    type: extend
    - task: foo
    + task: bar
    environment:
    - MESSAGE=baz
    --- a/intrinsic/extend.go Mon Sep 25 20:25:16 2017 -0500
    +++ b/intrinsic/extend.go Tue Sep 26 01:52:19 2017 +0000
    @@ -23,15 +23,16 @@
    "github.com/aphistic/gomol"
    "github.com/mohae/deepcopy"
    - "bitbucket.org/rw_grim/convey/environment"
    "bitbucket.org/rw_grim/convey/state"
    "bitbucket.org/rw_grim/convey/tasks"
    "bitbucket.org/rw_grim/convey/yaml"
    )
    type Extend struct {
    - Task string `yaml:"task"`
    - Environment yaml.StringOrSlice `yaml:"environment"`
    + Task string `yaml:"task"`
    + Environment yaml.StringOrSlice `yaml:"environment"`
    + Expand yaml.StringOrSlice `yaml:"expand"`
    + ExpandDelimiter string `yaml:"expand_delimiter"`
    // A copy of the extended task. Must be public so that deepcopy
    // can serialize it properly in case an extended task is extended
    @@ -41,18 +42,13 @@
    }
    func (e *Extend) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    - // We want the extending environment to take precedence over the extended
    - // task environment (if it exists). In order to get the correct precedence,
    - // we stash it onto the state's environment. Once we're done executing the
    - // inner task we'll restore it to the previous values. Notice that we merge
    - // in what appears at first as backwards - we don't want to override command
    - // line arguments, so we let state keep a higher precedence.
    + // While extending, certain environment variables can be expanded into
    + // lists. We store this meta information as a stack on the state, which
    + // is passed into the inner task execute method (and so on).
    + st.PushExtendFrame(e.Environment, e.Expand, e.ExpandDelimiter)
    + defer st.PopExtendFrame()
    - stashed := st.Environment
    - st.Environment = environment.Merge(e.Environment, st.Environment)
    - err := e.InnerTask.Execute(name, logger, env, st)
    - st.Environment = stashed
    - return err
    + return e.InnerTask.Execute(name, logger, env, st)
    }
    func (e *Extend) New() tasks.Task {
    @@ -60,6 +56,10 @@
    }
    func (e *Extend) Valid() error {
    + if e.ExpandDelimiter == "" {
    + e.ExpandDelimiter = ";"
    + }
    +
    return nil
    }
    --- a/state/state.go Mon Sep 25 20:25:16 2017 -0500
    +++ b/state/state.go Tue Sep 26 01:52:19 2017 +0000
    @@ -18,10 +18,13 @@
    package state
    import (
    + "errors"
    "fmt"
    "os"
    + "strings"
    "time"
    + "bitbucket.org/rw_grim/convey/environment"
    "bitbucket.org/rw_grim/convey/network"
    "bitbucket.org/rw_grim/convey/workspace"
    )
    @@ -35,6 +38,7 @@
    EnableSSHAgent bool
    TaskTimeout time.Duration
    Environment []string
    + ExtendFrames []extendFrame
    DockerConfig string
    CPUShares string
    @@ -43,6 +47,14 @@
    DetachedContainers []string
    }
    +type extendFrame struct {
    + expandable []string
    + delimiter string
    + stashedEnv []string
    +}
    +
    +var errEmptyExtendStack = errors.New("empty extend stack")
    +
    func (st *State) Valid() error {
    if st.EnableSSHAgent {
    if val := os.Getenv("SSH_AUTH_SOCK"); val == "" {
    @@ -52,3 +64,80 @@
    return nil
    }
    +
    +// MapSlice calls SliceMapper on the given environment, but also checks to
    +// see if the variable could be expanded into a list.
    +func (st *State) MapSlice(env, fullEnv []string) ([]string, error) {
    + if len(env) == 1 {
    + // If we only have one thing and it looks like $VAR or ${VAR}, then
    + // see if we have it in the list of things we can expand on the stack.
    + if delimiter, ok := st.getDelimiter(getName(env[0])); ok {
    + mapped, err := environment.Mapper(env[0], fullEnv)
    + if err != nil {
    + return nil, err
    + }
    +
    + // Split it!
    + return strings.SplitN(mapped, delimiter, -1), nil
    + }
    + }
    +
    + return environment.SliceMapper(env, fullEnv)
    +}
    +
    +// GetDelimiter returns the highest (outermost extend task) delimiter registered
    +// with a given expandable environment variable. Returns true if found by name.
    +func (st *State) getDelimiter(name string) (string, bool) {
    + for i := len(st.ExtendFrames) - 1; i >= 0; i-- {
    + frame := st.ExtendFrames[i]
    +
    + for _, expandable := range frame.expandable {
    + if expandable == name {
    + return frame.delimiter, true
    + }
    + }
    + }
    +
    + return "", false
    +}
    +
    +// PushExtendFrame will store the set of expandable environment variables
    +// and the current set of environments which are restored on pop. These frames
    +// are used to map a slice within an extended task.
    +func (st *State) PushExtendFrame(env, expandable []string, delimiter string) {
    + st.ExtendFrames = append(st.ExtendFrames, extendFrame{
    + expandable: expandable,
    + delimiter: delimiter,
    + stashedEnv: st.Environment,
    + })
    +
    + // Merge the environment into this map, but do NOT override anything that
    + // is currently in the state's enviornment - this has higher precedence.
    + st.Environment = environment.Merge(env, st.Environment)
    +}
    +
    +// PopExtendFrame will remove one frame from the top of the stack.
    +func (st *State) PopExtendFrame() error {
    + idx := len(st.ExtendFrames) - 1
    + if idx < 0 {
    + return errEmptyExtendStack
    + }
    +
    + last := st.ExtendFrames[idx]
    + st.ExtendFrames = st.ExtendFrames[:idx]
    + st.Environment = last.stashedEnv
    +
    + return nil
    +}
    +
    +// getName returns the name of the variable `var` if the string is of the
    +// form $var or ${var}.
    +func getName(env string) string {
    + env = strings.TrimSpace(env)
    +
    + if strings.HasPrefix(env, "$") {
    + return strings.Trim(env[1:], "{}")
    + }
    +
    + return ""
    +}
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/state/state_test.go Tue Sep 26 01:52:19 2017 +0000
    @@ -0,0 +1,100 @@
    +/*
    + * Convey
    + * Copyright 2016-2017 Gary Kramlich <grim@reaperworld.com>
    + *
    + * This program is free software: you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License as published by
    + * the Free Software Foundation, either version 3 of the License, or
    + * (at your option) any later version.
    + *
    + * This program is distributed in the hope that it will be useful,
    + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    + * GNU General Public License for more details.
    + *
    + * You should have received a copy of the GNU General Public License
    + * along with this program. If not, see <http://www.gnu.org/licenses/>.
    + */
    +package state
    +
    +import (
    + "testing"
    +
    + "github.com/aphistic/sweet"
    + junit "github.com/aphistic/sweet-junit"
    + . "github.com/onsi/gomega"
    +)
    +
    +type stateSuite struct{}
    +
    +func TestMain(m *testing.M) {
    + RegisterFailHandler(sweet.GomegaFail)
    +
    + sweet.Run(m, func(s *sweet.S) {
    + s.RegisterPlugin(junit.NewPlugin())
    +
    + s.AddSuite(&stateSuite{})
    + })
    +}
    +
    +func (e *stateSuite) TestMap(t sweet.T) {
    + st := &State{}
    + st.Environment = []string{"FOO=BAR"}
    +
    + mapEnv := func(val string) []string {
    + mapped, err := st.MapSlice([]string{val}, st.Environment)
    + Expect(err).To(BeNil())
    + return mapped
    + }
    +
    + Expect(mapEnv("$X")).To(Equal([]string{"$X"}))
    + Expect(mapEnv("$FOO")).To(Equal([]string{"BAR"}))
    +
    + st.PushExtendFrame([]string{"X=A;B;C"}, []string{"X"}, ";")
    + Expect(mapEnv("$X")).To(Equal([]string{"A", "B", "C"}))
    + Expect(mapEnv("$FOO")).To(Equal([]string{"BAR"}))
    +
    + st.PushExtendFrame([]string{"BAR=B;A;R::B;A;Z", "FOO=SAME"}, []string{"BAR"}, "::")
    + Expect(mapEnv("$X")).To(Equal([]string{"A", "B", "C"}))
    + Expect(mapEnv("$FOO")).To(Equal([]string{"BAR"}))
    + Expect(mapEnv("$BAR")).To(Equal([]string{"B;A;R", "B;A;Z"}))
    +
    + Expect(st.PopExtendFrame()).To(BeNil())
    + Expect(st.PopExtendFrame()).To(BeNil())
    + Expect(mapEnv("$X")).To(Equal([]string{"$X"}))
    + Expect(mapEnv("$FOO")).To(Equal([]string{"BAR"}))
    +}
    +
    +func (e *stateSuite) TestPopExtendFrameSize(t sweet.T) {
    + st := &State{}
    + st.PushExtendFrame(nil, nil, "")
    + Expect(st.ExtendFrames).To(HaveLen(1))
    + st.PushExtendFrame(nil, nil, "")
    + Expect(st.ExtendFrames).To(HaveLen(2))
    + Expect(st.PopExtendFrame()).To(BeNil())
    + Expect(st.ExtendFrames).To(HaveLen(1))
    + Expect(st.PopExtendFrame()).To(BeNil())
    + Expect(st.ExtendFrames).To(HaveLen(0))
    +}
    +
    +func (e *stateSuite) TestPopExtendFrameRestoresEnv(t sweet.T) {
    + st := &State{}
    + st.Environment = []string{"FOO=BAR", "BAR=BAZ"}
    + st.PushExtendFrame([]string{"FOO=BONK", "BAZ=BONK"}, nil, "")
    + Expect(st.Environment).To(HaveLen(3))
    + Expect(st.Environment).To(ConsistOf([]string{"FOO=BAR", "BAR=BAZ", "BAZ=BONK"}))
    + st.PopExtendFrame()
    + Expect(st.Environment).To(HaveLen(2))
    + Expect(st.Environment).To(ConsistOf([]string{"FOO=BAR", "BAR=BAZ"}))
    +}
    +
    +func (e *stateSuite) TestPopExtendFrameEmpty(t sweet.T) {
    + st := &State{}
    + Expect(st.PopExtendFrame()).To(Equal(errEmptyExtendStack))
    +}
    +
    +func (e *stateSuite) TestGetName(t sweet.T) {
    + Expect(getName("foo")).To(Equal(""))
    + Expect(getName("$FOO")).To(Equal("FOO"))
    + Expect(getName("${FOO}")).To(Equal("FOO"))
    +}