grim/convey

Merged in efritz/convey/env-loop (pull request #21)

2017-09-26, Gary Kramlich
47c2851788ff
Parents 4ec0cd5922a8bd5dcbe1bf33
Children 8861608b16c8
Merged in efritz/convey/env-loop (pull request #21)

Fix infinite env mapping loop

Fixes #124
23 files changed, 813 insertions(+), 121 deletions(-)
  • +28 -0
    REFERENCE.md
  • +18 -3
    docker/build.go
  • +8 -2
    docker/export.go
  • +7 -3
    docker/import.go
  • +6 -1
    docker/pull.go
  • +6 -1
    docker/push.go
  • +6 -1
    docker/remove.go
  • +50 -9
    docker/run.go
  • +12 -2
    docker/tag.go
  • +2 -16
    docker/workspace_test.go
  • +17 -6
    environment/mapper.go
  • +30 -9
    environment/mapper_test.go
  • +27 -0
    examples/extend.yml
  • +4 -1
    intrinsic/clean.go
  • +84 -0
    intrinsic/extend.go
  • +147 -0
    intrinsic/extend_test.go
  • +2 -1
    intrinsic/intrinsic.go
  • +82 -0
    loaders/convey/convey.go
  • +1 -0
    loaders/convey/convey_test.go
  • +182 -0
    loaders/convey/extend_test.go
  • +65 -66
    loaders/convey/tasks.go
  • +10 -0
    tasks/tasks.go
  • +19 -0
    tasks/util.go
    • --- a/REFERENCE.md Thu Sep 21 15:37:20 2017 -0500
    +++ b/REFERENCE.md Tue Sep 26 01:21:54 2017 +0000
    @@ -116,6 +116,34 @@
    ----
    +### Extend task
    +
    +An extend task overrides the environment of another task.
    +
    +#### Attributes
    +
    +| Name | Required | Default | Description |
    +| ----------- | -------- | ------- | ----------- |
    +| task | Yes | | The name of the task to extend |
    +| environment | | | The override environment |
    +
    +#### Example
    +
    + greeting:
    + image: gliderlabs/alpine:edge
    + script:
    + - echo "${MESSAGE}"
    + environment:
    + - MESSAGE="Hello, World"
    +
    + informal-greeting:
    + type: extend
    + task: greeting
    + environment:
    + - MESSAGE="Hi!"
    +
    +----
    +
    ### Build task
    A build task will build a docker image.
    --- a/docker/build.go Thu Sep 21 15:37:20 2017 -0500
    +++ b/docker/build.go Tue Sep 26 01:21:54 2017 +0000
    @@ -60,7 +60,12 @@
    // export the files to it
    for _, src := range b.Files {
    - src, dest := tasks.ParseFilePath(base, environment.Mapper(src, fullEnv))
    + src, err := environment.Mapper(src, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    + src, dest := tasks.ParseFilePath(base, src)
    cleanDest := filepath.Clean(filepath.Join(tmpDir, dest))
    if err = exportFile(name, st.Workspace.Name(), src, cleanDest, st); err != nil {
    @@ -80,12 +85,22 @@
    return err
    }
    + tag, err := environment.Mapper(b.Tag, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    + labels, err := environment.SliceMapper(b.Labels, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    // now run the build
    params = map[string]interface{}{
    "dockerfile": filepath.Join(tmpDir, filepath.Base(b.Dockerfile)),
    - "tag": environment.Mapper(b.Tag, fullEnv),
    + "tag": tag,
    "buildContext": tmpDir,
    - "Labels": environment.SliceMapper(b.Labels, fullEnv),
    + "Labels": labels,
    "Arguments": b.Arguments,
    }
    --- a/docker/export.go Thu Sep 21 15:37:20 2017 -0500
    +++ b/docker/export.go Tue Sep 26 01:21:54 2017 +0000
    @@ -87,7 +87,10 @@
    fullEnv := environment.Merge(env, st.Environment)
    for _, file := range e.Files {
    - file = environment.Mapper(file, fullEnv)
    + file, err := environment.Mapper(file, fullEnv)
    + if err != nil {
    + return err
    + }
    if err := checkFilePattern(file); err != nil {
    return err
    @@ -96,7 +99,10 @@
    dockerWorkspace := st.Workspace.(*Workspace)
    if strings.ContainsRune(file, '*') {
    - mountPoint := environment.Mapper(dockerWorkspace.mountPoint, fullEnv)
    + mountPoint, err := environment.Mapper(dockerWorkspace.mountPoint, fullEnv)
    + if err != nil {
    + return err
    + }
    if err := exportGlob(name, st.Workspace.Name(), mountPoint, file, st); err != nil {
    return err
    --- a/docker/import.go Thu Sep 21 15:37:20 2017 -0500
    +++ b/docker/import.go Tue Sep 26 01:21:54 2017 +0000
    @@ -36,7 +36,12 @@
    fullEnv := environment.Merge(env, st.Environment)
    for _, file := range i.Files {
    - src, dest := tasks.ParseFilePath("", environment.Mapper(file, fullEnv))
    + file, err := environment.Mapper(file, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    + src, dest := tasks.ParseFilePath("", file)
    params := map[string]interface{}{
    "source": src,
    @@ -44,8 +49,7 @@
    "workspaceID": st.Workspace.Name(),
    }
    - err := Docker(name, importTemplate, params, st)
    - if err != nil {
    + if err := Docker(name, importTemplate, params, st); err != nil {
    return err
    }
    }
    --- a/docker/pull.go Thu Sep 21 15:37:20 2017 -0500
    +++ b/docker/pull.go Tue Sep 26 01:21:54 2017 +0000
    @@ -37,8 +37,13 @@
    fullEnv := environment.Merge(env, st.Environment)
    for _, image := range p.Images {
    + image, err := environment.Mapper(image, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    params := map[string]interface{}{
    - "image": environment.Mapper(image, fullEnv),
    + "image": image,
    }
    if err := Docker(name, pullTemplate, params, st); err != nil {
    --- a/docker/push.go Thu Sep 21 15:37:20 2017 -0500
    +++ b/docker/push.go Tue Sep 26 01:21:54 2017 +0000
    @@ -37,8 +37,13 @@
    fullEnv := environment.Merge(env, st.Environment)
    for _, image := range p.Images {
    + image, err := environment.Mapper(image, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    params := map[string]interface{}{
    - "image": environment.Mapper(image, fullEnv),
    + "image": image,
    }
    if err := Docker(name, pushTemplate, params, st); err != nil {
    --- a/docker/remove.go Thu Sep 21 15:37:20 2017 -0500
    +++ b/docker/remove.go Tue Sep 26 01:21:54 2017 +0000
    @@ -37,8 +37,13 @@
    fullEnv := environment.Merge(env, st.Environment)
    for _, image := range r.Images {
    + image, err := environment.Mapper(image, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    params := map[string]interface{}{
    - "image": environment.Mapper(image, fullEnv),
    + "image": image,
    }
    if err := Docker(name, removeTemplate, params, st); err != nil {
    --- a/docker/run.go Thu Sep 21 15:37:20 2017 -0500
    +++ b/docker/run.go Tue Sep 26 01:21:54 2017 +0000
    @@ -120,7 +120,12 @@
    // iterate the script and run the environment variable exapansion on each line
    for idx, item := range r.Script {
    - r.Script[idx] = environment.Mapper(item, fullEnv)
    + item, err := environment.Mapper(item, fullEnv)
    + if err != nil {
    + return "", "", "", err
    + }
    +
    + r.Script[idx] = item
    }
    // write the script to the file
    @@ -145,7 +150,12 @@
    // now expand the environment
    for idx, v := range fullEnv {
    - fullEnv[idx] = environment.Mapper(v, fullEnv)
    + v, err := environment.Mapper(v, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    + fullEnv[idx] = v
    }
    // assign a default workspace location
    @@ -155,9 +165,12 @@
    }
    // initialize some variables
    + scriptFile := ""
    entryPoint := r.EntryPoint
    - commandArg := environment.Mapper(r.Command, fullEnv)
    - scriptFile := ""
    + commandArg, err := environment.Mapper(r.Command, fullEnv)
    + if err != nil {
    + return err
    + }
    // if we're using a script defined in the yaml, create it and override
    // some variables
    @@ -177,6 +190,34 @@
    dockerWorkspace := st.Workspace.(*Workspace)
    + //
    + // Map variables
    +
    + image, err := environment.Mapper(r.Image, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    + labels, err := environment.SliceMapper(r.Labels, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    + workdir, err := environment.Mapper(r.WorkDir, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    + workspacePath, err := environment.Mapper(dockerWorkspace.mountPoint, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    + workspaceMount, err := environment.Mapper(workSpace, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    // build the dict for the template
    params := map[string]interface{}{
    "Command": commandArg,
    @@ -186,17 +227,17 @@
    "EntryPoint": entryPoint,
    "GID": user.Gid,
    "HealthCheck": r.HealthCheck,
    - "Image": environment.Mapper(r.Image, fullEnv),
    - "Labels": environment.SliceMapper(r.Labels, fullEnv),
    + "Image": image,
    + "Labels": labels,
    "Memory": st.Memory,
    "Network": st.Network.Name(),
    "ScriptFile": scriptFile,
    "SSHAgent": st.EnableSSHAgent,
    "SSHAuthSock": os.Getenv("SSH_AUTH_SOCK"),
    "UID": user.Uid,
    - "WorkDir": environment.Mapper(r.WorkDir, fullEnv),
    - "WorkspacePath": environment.Mapper(dockerWorkspace.mountPoint, fullEnv),
    - "WorkspaceMount": environment.Mapper(workSpace, fullEnv),
    + "WorkDir": workdir,
    + "WorkspacePath": workspacePath,
    + "WorkspaceMount": workspaceMount,
    "TaskLabel": taskLabel,
    }
    --- a/docker/tag.go Thu Sep 21 15:37:20 2017 -0500
    +++ b/docker/tag.go Tue Sep 26 01:21:54 2017 +0000
    @@ -38,9 +38,19 @@
    fullEnv := environment.Merge(env, st.Environment)
    for _, destination := range t.Destinations {
    + source, err := environment.Mapper(t.Source, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    + destination, err := environment.Mapper(destination, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    params := map[string]interface{}{
    - "source": environment.Mapper(t.Source, fullEnv),
    - "destination": environment.Mapper(destination, fullEnv),
    + "source": source,
    + "destination": destination,
    }
    if err := Docker(name, tagTemplate, params, st); err != nil {
    --- a/docker/workspace_test.go Thu Sep 21 15:37:20 2017 -0500
    +++ b/docker/workspace_test.go Tue Sep 26 01:21:54 2017 +0000
    @@ -17,22 +17,8 @@
    */
    package docker
    -import (
    - "fmt"
    -
    - "github.com/aphistic/sweet"
    - . "github.com/onsi/gomega"
    -
    - "bitbucket.org/rw_grim/convey/workspace"
    -)
    +import "bitbucket.org/rw_grim/convey/workspace"
    type workspaceSuite struct{}
    -func (s *workspaceSuite) TestImplementsWorkspace(t sweet.T) {
    - var ws workspace.Workspace
    -
    - fmt.Printf("%#v %v", ws, ws)
    -
    - ws = &Workspace{}
    - Expect(ws).To(Not(BeNil()))
    -}
    +var _ workspace.Workspace = &Workspace{}
    --- a/environment/mapper.go Thu Sep 21 15:37:20 2017 -0500
    +++ b/environment/mapper.go Tue Sep 26 01:21:54 2017 +0000
    @@ -18,6 +18,7 @@
    package environment
    import (
    + "fmt"
    "os"
    "strings"
    )
    @@ -43,28 +44,38 @@
    // Mapper will replace ${TEMPLATE} patterns in the string with the KEY=VAL pairs
    // in the given environment. This function will replace patterns recursively, so
    // if VAL has the form ${TEMPLATE}, it will be replaced again.
    -func Mapper(str string, env []string) string {
    +func Mapper(str string, env []string) (string, error) {
    mapper := envMapper{env}
    last := str
    next := os.Expand(last, mapper.Map)
    + prev := map[string]struct{}{}
    - // TODO - add a max-depth or a list to detect replacement loops
    for last != next {
    + if _, ok := prev[next]; ok {
    + return "", fmt.Errorf("infinite environment mapping loop while expanding '%s'", next)
    + }
    +
    last = next
    next = os.Expand(last, mapper.Map)
    + prev[last] = struct{}{}
    }
    - return next
    + return next, nil
    }
    // SliceMapper calls Mapper for each item in a slice and returns a new slice.
    -func SliceMapper(slice []string, env []string) []string {
    +func SliceMapper(slice []string, env []string) ([]string, error) {
    mapped := []string{}
    for _, template := range slice {
    - mapped = append(mapped, Mapper(template, env))
    + expanded, err := Mapper(template, env)
    + if err != nil {
    + return nil, err
    + }
    +
    + mapped = append(mapped, expanded)
    }
    - return mapped
    + return mapped, nil
    }
    --- a/environment/mapper_test.go Thu Sep 21 15:37:20 2017 -0500
    +++ b/environment/mapper_test.go Tue Sep 26 01:21:54 2017 +0000
    @@ -25,36 +25,57 @@
    )
    func (e *environmentSuite) TestMapperNoOSWithValue(t sweet.T) {
    - Expect(Mapper("${FOO}", []string{"FOO=bar"})).To(Equal("bar"))
    + result, err := Mapper("${FOO}", []string{"FOO=bar"})
    + Expect(err).To(BeNil())
    + Expect(result).To(Equal("bar"))
    }
    func (e *environmentSuite) TestMapperNoOSWithoutValue(t sweet.T) {
    - Expect(Mapper("${FOO}", []string{"FOO="})).To(Equal(""))
    + result, err := Mapper("${FOO}", []string{"FOO="})
    + Expect(err).To(BeNil())
    + Expect(result).To(Equal(""))
    }
    func (e *environmentSuite) TestMultipleEquals(t sweet.T) {
    - Expect(Mapper("${FOO}", []string{"FOO=bar=baz"})).To(Equal("bar=baz"))
    + result, err := Mapper("${FOO}", []string{"FOO=bar=baz"})
    + Expect(err).To(BeNil())
    + Expect(result).To(Equal("bar=baz"))
    }
    func (e *environmentSuite) TestMapperOSWithValue(t sweet.T) {
    os.Setenv("FOO", "bar")
    - Expect(Mapper("${FOO}", []string{"FOO"})).To(Equal("bar"))
    + result, err := Mapper("${FOO}", []string{"FOO"})
    + Expect(err).To(BeNil())
    + Expect(result).To(Equal("bar"))
    os.Unsetenv("FOO")
    }
    func (e *environmentSuite) TestMapperOSWithoutValue(t sweet.T) {
    - Expect(Mapper("${FOO}", []string{"FOO"})).To(Equal(""))
    + result, err := Mapper("${FOO}", []string{"FOO"})
    + Expect(err).To(BeNil())
    + Expect(result).To(Equal(""))
    }
    func (e *environmentSuite) TestPartialMatches(t sweet.T) {
    os.Setenv("FOO", "bar")
    os.Setenv("BAR", "bonk")
    - Expect(Mapper("${FOO}", []string{"FOOBAR"})).To(Equal("$FOO"))
    - Expect(Mapper("${BARBAZ}", []string{"BAR"})).To(Equal("$BARBAZ"))
    +
    + result, err := Mapper("${FOO}", []string{"FOOBAR"})
    + Expect(err).To(BeNil())
    + Expect(result).To(Equal("$FOO"))
    +
    + result, err = Mapper("${BARBAZ}", []string{"BAR"})
    + Expect(err).To(BeNil())
    + Expect(result).To(Equal("$BARBAZ"))
    }
    func (e *environmentSuite) TestRecursiveMatch(t sweet.T) {
    - Expect(Mapper("1${FOO}a", []string{"FOO=2${BAR}b", "BAR=3${BAZ}c", "BAZ=-ohhai!-"})).To(Equal("123-ohhai!-cba"))
    + result, err := Mapper("1${FOO}a", []string{"FOO=2${BAR}b", "BAR=3${BAZ}c", "BAZ=-ohhai!-"})
    + Expect(err).To(BeNil())
    + Expect(result).To(Equal("123-ohhai!-cba"))
    }
    -// TODO - infinite expansion
    +func (e *environmentSuite) TestInfiniteExpansion(t sweet.T) {
    + _, err := Mapper("${FOO}", []string{"FOO=$BAR", "BAR=$FOO"})
    + Expect(err).To(MatchError("infinite environment mapping loop while expanding '$BAR'"))
    +}
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/examples/extend.yml Tue Sep 26 01:21:54 2017 +0000
    @@ -0,0 +1,27 @@
    +# This plan includes a task that extends another task with
    +# a different, higher precedence environment. Running this
    +# plan should print foo, bar, then baz.
    +
    +tasks:
    + foo:
    + image: gliderlabs/alpine:edge
    + command: echo "${MESSAGE}"
    + environment:
    + - MESSAGE=foo
    +
    + bar:
    + type: extend
    + task: foo
    + environment:
    + - MESSAGE=bar
    +
    + baz:
    + type: extend
    + task: foo
    + environment:
    + - MESSAGE=baz
    +
    +plans:
    + default:
    + stages:
    + - tasks: [foo, bar, baz]
    --- a/intrinsic/clean.go Thu Sep 21 15:37:20 2017 -0500
    +++ b/intrinsic/clean.go Tue Sep 26 01:21:54 2017 +0000
    @@ -36,7 +36,10 @@
    }
    func sanitizeFile(base, file string, env []string) (string, error) {
    - filename := environment.Mapper(file, env)
    + filename, err := environment.Mapper(file, env)
    + if err != nil {
    + return "", err
    + }
    pathname, err := filepath.Abs(filename)
    if err != nil {
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/intrinsic/extend.go Tue Sep 26 01:21:54 2017 +0000
    @@ -0,0 +1,84 @@
    +/*
    + * Convey
    + * Copyright 2016-2017 Gary Kramlich <grim@reaperworld.com>
    + *
    + * This program is free software: you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License as published by
    + * the Free Software Foundation, either version 3 of the License, or
    + * (at your option) any later version.
    + *
    + * This program is distributed in the hope that it will be useful,
    + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    + * GNU General Public License for more details.
    + *
    + * You should have received a copy of the GNU General Public License
    + * along with this program. If not, see <http://www.gnu.org/licenses/>.
    + */
    +package intrinsic
    +
    +import (
    + "fmt"
    +
    + "github.com/aphistic/gomol"
    + "github.com/mohae/deepcopy"
    +
    + "bitbucket.org/rw_grim/convey/environment"
    + "bitbucket.org/rw_grim/convey/state"
    + "bitbucket.org/rw_grim/convey/tasks"
    + "bitbucket.org/rw_grim/convey/yaml"
    +)
    +
    +type Extend struct {
    + Task string `yaml:"task"`
    + Environment yaml.StringOrSlice `yaml:"environment"`
    +
    + // A copy of the extended task. Must be public so that deepcopy
    + // can serialize it properly in case an extended task is extended
    + // itself (when private the innerTask is nil and causes a bad
    + // data access exception).
    + InnerTask tasks.Task
    +}
    +
    +func (e *Extend) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    + // We want the extending environment to take precedence over the extended
    + // task environment (if it exists). In order to get the correct precedence,
    + // we stash it onto the state's environment. Once we're done executing the
    + // inner task we'll restore it to the previous values. Notice that we merge
    + // in what appears at first as backwards - we don't want to override command
    + // line arguments, so we let state keep a higher precedence.
    +
    + stashed := st.Environment
    + st.Environment = environment.Merge(e.Environment, st.Environment)
    + err := e.InnerTask.Execute(name, logger, env, st)
    + st.Environment = stashed
    + return err
    +}
    +
    +func (e *Extend) New() tasks.Task {
    + return &Extend{}
    +}
    +
    +func (e *Extend) Valid() error {
    + return nil
    +}
    +
    +func (e *Extend) Dependencies() []string {
    + return []string{e.Task}
    +}
    +
    +func (e *Extend) Resolve(taskMap map[string]tasks.Task) error {
    + task, ok := taskMap[e.Task]
    + if !ok {
    + // Should never happen due to dependency order
    + return fmt.Errorf("Extending undeclared task '%s'", e.Task)
    + }
    +
    + // Some tasks may set their own fields, e.g. when mapping things
    + // to the correct runtime environment. In order to make sure that
    + // the inner task doesn't cache something from another run, we
    + // do a clone of the task here so we get our own copy to muck with.
    +
    + e.InnerTask = deepcopy.Copy(task).(tasks.Task)
    + return nil
    +}
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/intrinsic/extend_test.go Tue Sep 26 01:21:54 2017 +0000
    @@ -0,0 +1,147 @@
    +/*
    + * Convey
    + * Copyright 2016-2017 Gary Kramlich <grim@reaperworld.com>
    + *
    + * This program is free software: you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License as published by
    + * the Free Software Foundation, either version 3 of the License, or
    + * (at your option) any later version.
    + *
    + * This program is distributed in the hope that it will be useful,
    + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    + * GNU General Public License for more details.
    + *
    + * You should have received a copy of the GNU General Public License
    + * along with this program. If not, see <http://www.gnu.org/licenses/>.
    + */
    +package intrinsic
    +
    +import (
    + "bitbucket.org/rw_grim/convey/environment"
    + "bitbucket.org/rw_grim/convey/state"
    + "bitbucket.org/rw_grim/convey/tasks"
    + "github.com/aphistic/gomol"
    + "github.com/aphistic/sweet"
    + . "github.com/onsi/gomega"
    +)
    +
    +func (s *intrinsicSuite) TestExtendExecute(t sweet.T) {
    + var (
    + task = newMockTask()
    + fullEnv = []string{}
    + st = &state.State{Environment: []string{"bar=bonk", "quux=honk"}}
    + )
    +
    + task.executeFn = func(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    + fullEnv = environment.Merge(environment.Merge(env, []string{"x=1", "foo=quux"}), st.Environment)
    + return nil
    + }
    +
    + c := &Extend{
    + InnerTask: task,
    + Environment: []string{"foo=bar", "bar=baz"},
    + }
    +
    + Expect(c.Execute("task", nil, nil, st)).To(BeNil())
    + Expect(fullEnv).To(HaveLen(4))
    + Expect(fullEnv).To(ConsistOf("x=1", "foo=bar", "bar=bonk", "quux=honk"))
    +
    + // Should revert state
    + Expect(st.Environment).To(HaveLen(2))
    + Expect(st.Environment).To(ConsistOf("bar=bonk", "quux=honk"))
    +}
    +
    +func (s *intrinsicSuite) TestDependencies(t sweet.T) {
    + c := &Extend{Task: "test-task"}
    + Expect(c.Dependencies()).To(Equal([]string{"test-task"}))
    +}
    +
    +func (s *intrinsicSuite) TestResolve(t sweet.T) {
    + task := newMockTask()
    + task.X = 3
    + task.Y = []string{"foo", "bar"}
    + task.Z = struct {
    + A string
    + B string
    + C string
    + }{
    + A: "wynken",
    + B: "blynken",
    + C: "nod",
    + }
    +
    + c := &Extend{Task: "a"}
    + err := c.Resolve(map[string]tasks.Task{
    + "a": task,
    + })
    +
    + Expect(err).To(BeNil())
    +
    + // Must clone inner task
    + Expect(c.InnerTask).NotTo(BeNil())
    + Expect(c.InnerTask).NotTo(BeIdenticalTo(task))
    + Expect(c.InnerTask.(*mockTask).X).To(Equal(3))
    + Expect(c.InnerTask.(*mockTask).Y).To(Equal([]string{"foo", "bar"}))
    + Expect(c.InnerTask.(*mockTask).Z.A).To(Equal("wynken"))
    + Expect(c.InnerTask.(*mockTask).Z.B).To(Equal("blynken"))
    + Expect(c.InnerTask.(*mockTask).Z.C).To(Equal("nod"))
    +}
    +
    +func (s *intrinsicSuite) TestResolveChain(t sweet.T) {
    + task := newMockTask()
    + task.X = 3
    + task.Y = []string{"foo", "bar"}
    +
    + c1 := &Extend{Task: "a"}
    + c2 := &Extend{Task: "b"}
    + taskMapping := map[string]tasks.Task{
    + "a": task,
    + "b": c1,
    + }
    +
    + Expect(c1.Resolve(taskMapping)).To(BeNil())
    + Expect(c2.Resolve(taskMapping)).To(BeNil())
    +
    + // Must clone inner task
    + Expect(c2.InnerTask).NotTo(BeNil())
    + Expect(c2.InnerTask).NotTo(BeIdenticalTo(c1))
    + Expect(c2.InnerTask.(*Extend).InnerTask).NotTo(BeNil())
    + Expect(c2.InnerTask.(*Extend).InnerTask.(*mockTask).X).To(Equal(3))
    + Expect(c2.InnerTask.(*Extend).InnerTask.(*mockTask).Y).To(Equal([]string{"foo", "bar"}))
    +}
    +
    +//
    +// Mocks
    +
    +type mockTask struct {
    + X int
    + Y []string
    + Z struct {
    + A string
    + B string
    + C string
    + }
    +
    + executeFn func(string, *gomol.LogAdapter, []string, *state.State) error
    +}
    +
    +func newMockTask() *mockTask {
    + return &mockTask{
    + executeFn: func(string, *gomol.LogAdapter, []string, *state.State) error {
    + return nil
    + },
    + }
    +}
    +
    +func (t *mockTask) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    + return t.executeFn(name, logger, env, st)
    +}
    +
    +func (t *mockTask) New() tasks.Task {
    + return &mockTask{}
    +}
    +
    +func (t *mockTask) Valid() error {
    + return nil
    +}
    --- a/intrinsic/intrinsic.go Thu Sep 21 15:37:20 2017 -0500
    +++ b/intrinsic/intrinsic.go Tue Sep 26 01:21:54 2017 +0000
    @@ -23,6 +23,7 @@
    var (
    Tasks = map[string]tasks.Task{
    - "clean": &Clean{},
    + "clean": &Clean{},
    + "extend": &Extend{},
    }
    )
    --- a/loaders/convey/convey.go Thu Sep 21 15:37:20 2017 -0500
    +++ b/loaders/convey/convey.go Tue Sep 26 01:21:54 2017 +0000
    @@ -19,8 +19,11 @@
    import (
    "fmt"
    + "sort"
    + "strings"
    "github.com/go-yaml/yaml"
    + toposort "github.com/philopon/go-toposort"
    cConfig "bitbucket.org/rw_grim/convey/config"
    "bitbucket.org/rw_grim/convey/environment"
    @@ -142,6 +145,16 @@
    baseConfig.SSHIdentities = cfg.Options.SSHIdentities
    }
    + // Now that we have all of the tasks defined up to this point, any task
    + // that needs a reference to another task via the ResolvableTask interface
    + // is passed the entire map. We need to make sure that tasks are resolved
    + // in the correct order. This is a problem, for example, if the extend
    + // intrinsic clones a task that is not fully resolved.
    +
    + if err := c.resolve(baseConfig); err != nil {
    + return nil, err
    + }
    +
    // Return the base config at this point, but maybe possibly return an
    // error that we're missing something in order to proceed. This gets
    // kind of weird: if we're calling Load recursively through LoadFile,
    @@ -160,6 +173,75 @@
    return baseConfig, nil
    }
    +func (c *Loader) resolve(baseConfig *cConfig.Config) error {
    + // Extract resolvable tasks from config tasks
    +
    + resolvables := map[string]tasks.ResolvableTask{}
    + for name, task := range baseConfig.Tasks {
    + if rc, ok := task.(tasks.ResolvableTask); ok {
    + resolvables[name] = rc
    + }
    + }
    +
    + graph := toposort.NewGraph(len(resolvables))
    +
    + // Add all nodes to graph before edges, as it won't count
    + // the in/out degree correctly if we add an edge before one
    + // of the terminating nodes.
    +
    + for name, rc := range resolvables {
    + graph.AddNode(name)
    +
    + for _, dependency := range rc.Dependencies() {
    + if _, ok := baseConfig.Tasks[name]; !ok {
    + return fmt.Errorf("Unknown task dependency '%s'", dependency)
    + }
    +
    + graph.AddNode(dependency)
    + }
    +
    + }
    +
    + // Add all edges
    +
    + for name, rc := range resolvables {
    + for _, dependency := range rc.Dependencies() {
    + graph.AddEdge(dependency, name)
    + }
    + }
    +
    + // Sort and resolve all tasks
    +
    + order, ok := graph.Toposort()
    + if !ok {
    + for _, name := range order {
    + delete(resolvables, name)
    + }
    +
    + remaining := []string{}
    + for k := range resolvables {
    + remaining = append(remaining, k)
    + }
    +
    + sort.Strings(remaining)
    +
    + return fmt.Errorf("The following tasks are part of a dependency cycle: %s", strings.Join(remaining, ", "))
    + }
    +
    + for _, name := range order {
    + rc, ok := resolvables[name]
    + if !ok {
    + continue
    + }
    +
    + if err := rc.Resolve(baseConfig.Tasks); err != nil {
    + return fmt.Errorf("%s: %s", name, err.Error())
    + }
    + }
    +
    + return nil
    +}
    +
    func (c *Loader) loadFile(path string) (*cConfig.Config, error) {
    if c.fileLoader == nil {
    return cConfig.LoadFile(path, c)
    --- a/loaders/convey/convey_test.go Thu Sep 21 15:37:20 2017 -0500
    +++ b/loaders/convey/convey_test.go Tue Sep 26 01:21:54 2017 +0000
    @@ -37,6 +37,7 @@
    s.RegisterPlugin(junit.NewPlugin())
    s.AddSuite(&conveySuite{})
    + s.AddSuite(&extendSuite{})
    s.AddSuite(&extendsSuite{})
    s.AddSuite(&defaultPlanSuite{})
    s.AddSuite(&environmentSuite{})
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/loaders/convey/extend_test.go Tue Sep 26 01:21:54 2017 +0000
    @@ -0,0 +1,182 @@
    +/*
    + * Convey
    + * Copyright 2016-2017 Gary Kramlich <grim@reaperworld.com>
    + *
    + * This program is free software: you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License as published by
    + * the Free Software Foundation, either version 3 of the License, or
    + * (at your option) any later version.
    + *
    + * This program is distributed in the hope that it will be useful,
    + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    + * GNU General Public License for more details.
    + *
    + * You should have received a copy of the GNU General Public License
    + * along with this program. If not, see <http://www.gnu.org/licenses/>.
    + */
    +package convey
    +
    +import (
    + "github.com/aphistic/sweet"
    + . "github.com/onsi/gomega"
    +
    + cConfig "bitbucket.org/rw_grim/convey/config"
    + "bitbucket.org/rw_grim/convey/intrinsic"
    +)
    +
    +// This test suite tests
    +
    +type extendSuite struct{}
    +
    +func (d *extendSuite) TestSimple(t sweet.T) {
    + data := `
    +tasks:
    + a:
    + image: alpine:3.6
    + environment:
    + - X=1
    + - Y=2
    + b:
    + type: extend
    + task: a
    + environment:
    + - X=3
    +plans:
    + default:
    + stages:
    + - tasks: [a, b]
    +`
    +
    + loader := &Loader{}
    + cfg, err := loader.Load(".", ".", []byte(data))
    +
    + Expect(err).To(BeNil())
    + Expect(cfg.Tasks).To(HaveLen(2))
    + Expect(cfg.Tasks).To(HaveKey("a"))
    + Expect(cfg.Tasks).To(HaveKey("b"))
    + Expect(cfg.Tasks["b"].(*intrinsic.Extend).InnerTask).NotTo(BeIdenticalTo(cfg.Tasks["a"]))
    + Expect(cfg.Tasks["b"].(*intrinsic.Extend).InnerTask).To(BeEquivalentTo(cfg.Tasks["a"]))
    +}
    +
    +func (d *extendSuite) TestExtendChain(t sweet.T) {
    + data := `
    +tasks:
    + a:
    + image: alpine:3.6
    + environment:
    + - X=1
    + - Y=2
    + b:
    + type: extend
    + task: a
    + environment:
    + - X=3
    + c:
    + type: extend
    + task: b
    + environment:
    + - X=4
    +plans:
    + default:
    + stages:
    + - tasks: [a, b, c]
    +`
    +
    + loader := &Loader{}
    + cfg, err := loader.Load(".", ".", []byte(data))
    +
    + Expect(err).To(BeNil())
    + Expect(cfg.Tasks).To(HaveLen(3))
    + Expect(cfg.Tasks).To(HaveKey("a"))
    + Expect(cfg.Tasks).To(HaveKey("b"))
    + Expect(cfg.Tasks).To(HaveKey("c"))
    + Expect(cfg.Tasks["c"].(*intrinsic.Extend).InnerTask).NotTo(BeIdenticalTo(cfg.Tasks["b"]))
    + Expect(cfg.Tasks["c"].(*intrinsic.Extend).InnerTask).To(BeEquivalentTo(cfg.Tasks["b"]))
    + Expect(cfg.Tasks["b"].(*intrinsic.Extend).InnerTask).NotTo(BeIdenticalTo(cfg.Tasks["a"]))
    + Expect(cfg.Tasks["b"].(*intrinsic.Extend).InnerTask).To(BeEquivalentTo(cfg.Tasks["a"]))
    + Expect(cfg.Tasks["c"].(*intrinsic.Extend).InnerTask.(*intrinsic.Extend).InnerTask).NotTo(BeIdenticalTo(cfg.Tasks["a"]))
    + Expect(cfg.Tasks["c"].(*intrinsic.Extend).InnerTask.(*intrinsic.Extend).InnerTask).To(BeEquivalentTo(cfg.Tasks["a"]))
    +}
    +
    +func (d *extendSuite) TestCycle(t sweet.T) {
    + data := `
    +tasks:
    + a:
    + type: extend
    + task: b
    + b:
    + type: extend
    + task: a
    +plans:
    + default:
    + stages:
    + - tasks: [a, b]
    +`
    +
    + loader := &Loader{}
    + _, err := loader.Load(".", ".", []byte(data))
    + Expect(err).To(MatchError("The following tasks are part of a dependency cycle: a, b"))
    +}
    +
    +func (d *extendSuite) TestMissingTask(t sweet.T) {
    + data := `
    +tasks:
    + b:
    + type: extend
    + task: a
    +plans:
    + default:
    + stages:
    + - tasks: [a, b]
    +`
    +
    + loader := &Loader{}
    + _, err := loader.Load(".", ".", []byte(data))
    + Expect(err).To(MatchError("b: Extending undeclared task 'a'"))
    +}
    +
    +func (d *extendSuite) TestExtendExtends(t sweet.T) {
    + baseData := `
    +environment:
    + - x=1
    +tasks:
    + foo:
    + image: imaginary
    + environment:
    + - x=2
    +`
    +
    + extendedData := `
    +extends: base.yaml
    +environment:
    + - x=3
    +tasks:
    + bar:
    + type: extend
    + task: foo
    + environment:
    + - x=3
    +plans:
    + plan:
    + stages:
    + - tasks: [foo, bar]
    +`
    +
    + loader := &Loader{
    + fileLoader: func(name string, c *Loader) (*cConfig.Config, error) {
    + return c.Load(".", name, []byte(baseData))
    + },
    + }
    +
    + cfg, err := loader.Load(".", ".", []byte(extendedData))
    +
    + Expect(err).To(BeNil())
    + Expect(cfg.Tasks).To(HaveLen(2))
    + Expect(cfg.Tasks).To(HaveKey("foo"))
    + Expect(cfg.Tasks).To(HaveKey("bar"))
    + Expect(cfg.Tasks).To(HaveKey("foo"))
    + Expect(cfg.Tasks).To(HaveKey("bar"))
    + Expect(cfg.Tasks["bar"].(*intrinsic.Extend).InnerTask).NotTo(BeIdenticalTo(cfg.Tasks["foo"]))
    + Expect(cfg.Tasks["bar"].(*intrinsic.Extend).InnerTask).To(BeEquivalentTo(cfg.Tasks["foo"]))
    +}
    --- a/loaders/convey/tasks.go Thu Sep 21 15:37:20 2017 -0500
    +++ b/loaders/convey/tasks.go Tue Sep 26 01:21:54 2017 +0000
    @@ -36,79 +36,78 @@
    func loadTasks(path string, raw map[string]yaml.MapSlice) (map[string]tasks.Task, error) {
    realTasks := map[string]tasks.Task{}
    -
    for name, task := range raw {
    - // figure out the engine and type for the task
    - var (
    - rawEngine string
    - rawType string
    - )
    -
    - for _, item := range task {
    - switch item.Key.(string) {
    - case "engine":
    - rawEngine = item.Value.(string)
    - case "type":
    - rawType = item.Value.(string)
    - }
    - }
    -
    - var engineMap cConfig.TaskMap
    -
    - // if the engine wasn't specified search based on the task type
    - if rawEngine == "" {
    - // if the task type wasn't specified use the default engine
    - if rawType == "" {
    - engineMap = cConfig.TasksMap[cConfig.DefaultEngine]
    - } else {
    - eMap, err := findTask(rawType)
    - if err != nil {
    - return nil, err
    - }
    -
    - engineMap = eMap
    - }
    - } else {
    - // now see if we have the task for the given engine
    - eMap, found := cConfig.TasksMap[rawEngine]
    - if !found {
    - return nil, fmt.Errorf("unknown engine '%s'", rawEngine)
    - }
    -
    - engineMap = eMap
    - }
    -
    - // if we don't have a rawType use the default for the engine
    - if rawType == "" {
    - rawType = engineMap.Default
    - }
    -
    - taskType, found := engineMap.Tasks[rawType]
    - if !found {
    - return nil, fmt.Errorf("unknown task type '%s'", rawType)
    - }
    -
    - // now marshal the data again
    - rawTask, err := yaml.Marshal(task)
    + realTask, err := loadTask(name, task)
    if err != nil {
    return nil, err
    }
    - // unmarshal it into the correct struct
    - realTask := taskType.New()
    - err = yaml.Unmarshal(rawTask, realTask)
    - if err != nil {
    - return nil, err
    - }
    -
    - err = realTask.Valid()
    - if err != nil {
    - return nil, fmt.Errorf("%s: %s", name, err.Error())
    - }
    -
    - // store it
    realTasks[name] = realTask
    }
    return realTasks, nil
    }
    +
    +func loadTask(name string, task yaml.MapSlice) (tasks.Task, error) {
    + // figure out the engine and type for the task
    + var (
    + rawEngine string
    + rawType string
    + )
    +
    + for _, item := range task {
    + switch item.Key.(string) {
    + case "engine":
    + rawEngine = item.Value.(string)
    + case "type":
    + rawType = item.Value.(string)
    + }
    + }
    +
    + var engineMap cConfig.TaskMap
    +
    + // if the engine wasn't specified search based on the task type
    + if rawEngine == "" {
    + // if the task type wasn't specified use the default engine
    + if rawType == "" {
    + engineMap = cConfig.TasksMap[cConfig.DefaultEngine]
    + } else {
    + eMap, err := findTask(rawType)
    + if err != nil {
    + return nil, err
    + }
    +
    + engineMap = eMap
    + }
    + } else {
    + // now see if we have the task for the given engine
    + eMap, found := cConfig.TasksMap[rawEngine]
    + if !found {
    + return nil, fmt.Errorf("unknown engine '%s'", rawEngine)
    + }
    +
    + engineMap = eMap
    + }
    +
    + // if we don't have a rawType use the default for the engine
    + if rawType == "" {
    + rawType = engineMap.Default
    + }
    +
    + taskType, found := engineMap.Tasks[rawType]
    + if !found {
    + return nil, fmt.Errorf("unknown task type '%s'", rawType)
    + }
    +
    + realTask, err := tasks.CloneTask(task, taskType)
    + if err != nil {
    + return nil, err
    + }
    +
    + err = realTask.Valid()
    + if err != nil {
    + return nil, fmt.Errorf("%s: %s", name, err.Error())
    + }
    +
    + return realTask, nil
    +}
    --- a/tasks/tasks.go Thu Sep 21 15:37:20 2017 -0500
    +++ b/tasks/tasks.go Tue Sep 26 01:21:54 2017 +0000
    @@ -28,3 +28,13 @@
    New() Task
    Valid() error
    }
    +
    +type ResolvableTask interface {
    + // Resolve is called by the loader once the current set of tasks
    + // by name have been loaded from the config file.
    + Resolve(tasks map[string]Task) error
    +
    + // Dependencies will return a list of tasks which must be resolved
    + // before resolving this task.
    + Dependencies() []string
    +}
    --- a/tasks/util.go Thu Sep 21 15:37:20 2017 -0500
    +++ b/tasks/util.go Tue Sep 26 01:21:54 2017 +0000
    @@ -20,8 +20,27 @@
    import (
    "os"
    "strings"
    +
    + "github.com/go-yaml/yaml"
    )
    +// CloneTask creates a task of the given type from the given payload. It
    +// does this by creating a fresh instance of a task of the target type,
    +// then marshalling/unmarshalling the payload to that type.
    +func CloneTask(task interface{}, taskType Task) (Task, error) {
    + rawTask, err := yaml.Marshal(task)
    + if err != nil {
    + return nil, err
    + }
    +
    + realTask := taskType.New()
    + if err = yaml.Unmarshal(rawTask, realTask); err != nil {
    + return nil, err
    + }
    +
    + return realTask, nil
    +}
    +
    // ParseFilePath will split a string on the first : and return two pieces.
    // If there is no colon, then either "." (the current working directory)
    // will be returned as the destination if the source was an item in the