grim/convey

Add initial implementation for extend task.

2017-09-19, Eric Fritz
dc438a6fd296
Parents 5362b299f79c
Children 92fc215b4b32
Add initial implementation for extend task.
7 files changed, 290 insertions(+), 67 deletions(-)
  • +84 -0
    intrinsic/extend.go
  • +30 -0
    intrinsic/extend_test.go
  • +2 -1
    intrinsic/intrinsic.go
  • +80 -0
    loaders/convey/convey.go
  • +65 -66
    loaders/convey/tasks.go
  • +10 -0
    tasks/tasks.go
  • +19 -0
    tasks/util.go
    • --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/intrinsic/extend.go Tue Sep 19 20:14:49 2017 -0500
    @@ -0,0 +1,84 @@
    +/*
    + * Convey
    + * Copyright 2016-2017 Gary Kramlich <grim@reaperworld.com>
    + *
    + * This program is free software: you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License as published by
    + * the Free Software Foundation, either version 3 of the License, or
    + * (at your option) any later version.
    + *
    + * This program is distributed in the hope that it will be useful,
    + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    + * GNU General Public License for more details.
    + *
    + * You should have received a copy of the GNU General Public License
    + * along with this program. If not, see <http://www.gnu.org/licenses/>.
    + */
    +package intrinsic
    +
    +import (
    + "fmt"
    +
    + "github.com/aphistic/gomol"
    + "github.com/mohae/deepcopy"
    +
    + "bitbucket.org/rw_grim/convey/environment"
    + "bitbucket.org/rw_grim/convey/state"
    + "bitbucket.org/rw_grim/convey/tasks"
    + "bitbucket.org/rw_grim/convey/yaml"
    +)
    +
    +type Extend struct {
    + Task string `yaml:"task"`
    + Environment yaml.StringOrSlice `yaml:"environment"`
    +
    + // A copy of the extended task. Must be public so that deepcopy
    + // can serialize it properly in case an extended task is extended
    + // itself (when private the innerTask is nil and causes a bad
    + // data access exception).
    + InnerTask tasks.Task
    +}
    +
    +func (e *Extend) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    + // We want the extending environment to take precedence over the extended
    + // task environment (if it exists). In order to get the correct precedence,
    + // we stash it onto the state's environment. Once we're done executing the
    + // inner task we'll restore it to the previous values. Notice that we merge
    + // in what appears at first as backwards - we don't want to override command
    + // line arguments, so we let state keep a higher precedence.
    +
    + stashed := st.Environment
    + st.Environment = environment.Merge(e.Environment, st.Environment)
    + err := e.InnerTask.Execute(name, logger, env, st)
    + st.Environment = stashed
    + return err
    +}
    +
    +func (e *Extend) New() tasks.Task {
    + return &Extend{}
    +}
    +
    +func (e *Extend) Valid() error {
    + return nil
    +}
    +
    +func (e *Extend) Dependencies() []string {
    + return []string{e.Task}
    +}
    +
    +func (e *Extend) Resolve(taskMap map[string]tasks.Task) error {
    + task, ok := taskMap[e.Task]
    + if !ok {
    + // Should never happen due to dependency order
    + return fmt.Errorf("Extending undeclared task '%s'", e.Task)
    + }
    +
    + // Some tasks may set their own fields, e.g. when mapping things
    + // to the correct runtime environment. In order to make sure that
    + // the inner task doesn't cache something from another run, we
    + // do a clone of the task here so we get our own copy to muck with.
    +
    + e.InnerTask = deepcopy.Copy(task).(tasks.Task)
    + return nil
    +}
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/intrinsic/extend_test.go Tue Sep 19 20:14:49 2017 -0500
    @@ -0,0 +1,30 @@
    +/*
    + * Convey
    + * Copyright 2016-2017 Gary Kramlich <grim@reaperworld.com>
    + *
    + * This program is free software: you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License as published by
    + * the Free Software Foundation, either version 3 of the License, or
    + * (at your option) any later version.
    + *
    + * This program is distributed in the hope that it will be useful,
    + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    + * GNU General Public License for more details.
    + *
    + * You should have received a copy of the GNU General Public License
    + * along with this program. If not, see <http://www.gnu.org/licenses/>.
    + */
    +package intrinsic
    +
    +import (
    + "github.com/aphistic/sweet"
    + . "github.com/onsi/gomega"
    +)
    +
    +func (s *intrinsicSuite) TestExtend(t sweet.T) {
    + c := &Clean{
    + Files: []string{"foo", "bar", "**.pyc"},
    + }
    + Expect(c.Valid()).To(BeNil())
    +}
    --- a/intrinsic/intrinsic.go Sun Sep 17 18:49:52 2017 -0500
    +++ b/intrinsic/intrinsic.go Tue Sep 19 20:14:49 2017 -0500
    @@ -23,6 +23,7 @@
    var (
    Tasks = map[string]tasks.Task{
    - "clean": &Clean{},
    + "clean": &Clean{},
    + "extend": &Extend{},
    }
    )
    --- a/loaders/convey/convey.go Sun Sep 17 18:49:52 2017 -0500
    +++ b/loaders/convey/convey.go Tue Sep 19 20:14:49 2017 -0500
    @@ -19,8 +19,10 @@
    import (
    "fmt"
    + "strings"
    "github.com/go-yaml/yaml"
    + "github.com/philopon/go-toposort"
    cConfig "bitbucket.org/rw_grim/convey/config"
    "bitbucket.org/rw_grim/convey/environment"
    @@ -142,6 +144,16 @@
    baseConfig.SSHIdentities = cfg.Options.SSHIdentities
    }
    + // Now that we have all of the tasks defined up to this point, any task
    + // that needs a reference to another task via the ResolvableTask interface
    + // is passed the entire map. We need to make sure that tasks are resolved
    + // in the correct order. This is a problem, for example, if the extend
    + // intrinsic clones a task that is not fully resolved.
    +
    + if err := c.resolve(baseConfig); err != nil {
    + return nil, err
    + }
    +
    // Return the base config at this point, but maybe possibly return an
    // error that we're missing something in order to proceed. This gets
    // kind of weird: if we're calling Load recursively through LoadFile,
    @@ -160,6 +172,74 @@
    return baseConfig, nil
    }
    +func (c *Loader) resolve(baseConfig *cConfig.Config) error {
    + // Extract resolvable tasks from config tasks
    +
    + resolvables := map[string]tasks.ResolvableTask{}
    + for name, task := range baseConfig.Tasks {
    + if rc, ok := task.(tasks.ResolvableTask); ok {
    + resolvables[name] = rc
    + }
    + }
    +
    + graph := toposort.NewGraph(len(resolvables))
    +
    + // Add all nodes to graph before edges, as it won't count
    + // the in/out degree correctly if we add an edge before one
    + // of the terminating nodes.
    +
    + for name, rc := range resolvables {
    + graph.AddNode(name)
    +
    + for _, dependency := range rc.Dependencies() {
    + if _, ok := baseConfig.Tasks[name]; !ok {
    + return fmt.Errorf("Unknown task dependency '%s'", dependency)
    + }
    +
    + graph.AddNode(dependency)
    + }
    +
    + }
    +
    + // Add all edges
    +
    + for name, rc := range resolvables {
    + for _, dependency := range rc.Dependencies() {
    + graph.AddEdge(dependency, name)
    + }
    + }
    +
    + // Sort and resolve all tasks
    +
    + order, ok := graph.Toposort()
    + if !ok {
    + for _, name := range order {
    + delete(resolvables, name)
    + }
    +
    + remaining := []string{}
    + for k := range resolvables {
    + remaining = append(remaining, k)
    + }
    +
    + // TODO - get actual tasks
    + return fmt.Errorf("The following tasks are part of a dependency cycle: %s", strings.Join(remaining, ", "))
    + }
    +
    + for _, name := range order {
    + rc, ok := resolvables[name]
    + if !ok {
    + continue
    + }
    +
    + if err := rc.Resolve(baseConfig.Tasks); err != nil {
    + return fmt.Errorf("%s: %s", name, err.Error())
    + }
    + }
    +
    + return nil
    +}
    +
    func (c *Loader) loadFile(path string) (*cConfig.Config, error) {
    if c.fileLoader == nil {
    return cConfig.LoadFile(path, c)
    --- a/loaders/convey/tasks.go Sun Sep 17 18:49:52 2017 -0500
    +++ b/loaders/convey/tasks.go Tue Sep 19 20:14:49 2017 -0500
    @@ -36,79 +36,78 @@
    func loadTasks(path string, raw map[string]yaml.MapSlice) (map[string]tasks.Task, error) {
    realTasks := map[string]tasks.Task{}
    -
    for name, task := range raw {
    - // figure out the engine and type for the task
    - var (
    - rawEngine string
    - rawType string
    - )
    -
    - for _, item := range task {
    - switch item.Key.(string) {
    - case "engine":
    - rawEngine = item.Value.(string)
    - case "type":
    - rawType = item.Value.(string)
    - }
    - }
    -
    - var engineMap cConfig.TaskMap
    -
    - // if the engine wasn't specified search based on the task type
    - if rawEngine == "" {
    - // if the task type wasn't specified use the default engine
    - if rawType == "" {
    - engineMap = cConfig.TasksMap[cConfig.DefaultEngine]
    - } else {
    - eMap, err := findTask(rawType)
    - if err != nil {
    - return nil, err
    - }
    -
    - engineMap = eMap
    - }
    - } else {
    - // now see if we have the task for the given engine
    - eMap, found := cConfig.TasksMap[rawEngine]
    - if !found {
    - return nil, fmt.Errorf("unknown engine '%s'", rawEngine)
    - }
    -
    - engineMap = eMap
    - }
    -
    - // if we don't have a rawType use the default for the engine
    - if rawType == "" {
    - rawType = engineMap.Default
    - }
    -
    - taskType, found := engineMap.Tasks[rawType]
    - if !found {
    - return nil, fmt.Errorf("unknown task type '%s'", rawType)
    - }
    -
    - // now marshal the data again
    - rawTask, err := yaml.Marshal(task)
    + realTask, err := loadTask(name, task)
    if err != nil {
    return nil, err
    }
    - // unmarshal it into the correct struct
    - realTask := taskType.New()
    - err = yaml.Unmarshal(rawTask, realTask)
    - if err != nil {
    - return nil, err
    - }
    -
    - err = realTask.Valid()
    - if err != nil {
    - return nil, fmt.Errorf("%s: %s", name, err.Error())
    - }
    -
    - // store it
    realTasks[name] = realTask
    }
    return realTasks, nil
    }
    +
    +func loadTask(name string, task yaml.MapSlice) (tasks.Task, error) {
    + // figure out the engine and type for the task
    + var (
    + rawEngine string
    + rawType string
    + )
    +
    + for _, item := range task {
    + switch item.Key.(string) {
    + case "engine":
    + rawEngine = item.Value.(string)
    + case "type":
    + rawType = item.Value.(string)
    + }
    + }
    +
    + var engineMap cConfig.TaskMap
    +
    + // if the engine wasn't specified search based on the task type
    + if rawEngine == "" {
    + // if the task type wasn't specified use the default engine
    + if rawType == "" {
    + engineMap = cConfig.TasksMap[cConfig.DefaultEngine]
    + } else {
    + eMap, err := findTask(rawType)
    + if err != nil {
    + return nil, err
    + }
    +
    + engineMap = eMap
    + }
    + } else {
    + // now see if we have the task for the given engine
    + eMap, found := cConfig.TasksMap[rawEngine]
    + if !found {
    + return nil, fmt.Errorf("unknown engine '%s'", rawEngine)
    + }
    +
    + engineMap = eMap
    + }
    +
    + // if we don't have a rawType use the default for the engine
    + if rawType == "" {
    + rawType = engineMap.Default
    + }
    +
    + taskType, found := engineMap.Tasks[rawType]
    + if !found {
    + return nil, fmt.Errorf("unknown task type '%s'", rawType)
    + }
    +
    + realTask, err := tasks.CloneTask(task, taskType)
    + if err != nil {
    + return nil, err
    + }
    +
    + err = realTask.Valid()
    + if err != nil {
    + return nil, fmt.Errorf("%s: %s", name, err.Error())
    + }
    +
    + return realTask, nil
    +}
    --- a/tasks/tasks.go Sun Sep 17 18:49:52 2017 -0500
    +++ b/tasks/tasks.go Tue Sep 19 20:14:49 2017 -0500
    @@ -28,3 +28,13 @@
    New() Task
    Valid() error
    }
    +
    +type ResolvableTask interface {
    + // Resolve is called by the loader once the current set of tasks
    + // by name have been loaded from the config file.
    + Resolve(tasks map[string]Task) error
    +
    + // Dependencies will return a list of tasks which must be resolved
    + // before resolving this task.
    + Dependencies() []string
    +}
    --- a/tasks/util.go Sun Sep 17 18:49:52 2017 -0500
    +++ b/tasks/util.go Tue Sep 19 20:14:49 2017 -0500
    @@ -20,8 +20,27 @@
    import (
    "os"
    "strings"
    +
    + "github.com/go-yaml/yaml"
    )
    +// CloneTask creates a task of the given type from the given payload. It
    +// does this by creating a fresh instance of a task of the target type,
    +// then marshalling/unmarshalling the payload to that type.
    +func CloneTask(task interface{}, taskType Task) (Task, error) {
    + rawTask, err := yaml.Marshal(task)
    + if err != nil {
    + return nil, err
    + }
    +
    + realTask := taskType.New()
    + if err = yaml.Unmarshal(rawTask, realTask); err != nil {
    + return nil, err
    + }
    +
    + return realTask, nil
    +}
    +
    // ParseFilePath will split a string on the first : and return two pieces.
    // If there is no colon, then either "." (the current working directory)
    // will be returned as the destination if the source was an item in the