grim/convey

abstract out the script to convey tasks parser to the new script package

2018-01-21, Gary Kramlich
49fa7c253ba3
Parents 2909e17deafe
Children 14e4e891eda9
abstract out the script to convey tasks parser to the new script package
6 files changed, 262 insertions(+), 154 deletions(-)
  • +2 -1
    ChangeLog
  • +10 -59
    loaders/bitbucket/loader.go
  • +35 -35
    loaders/bitbucket/loader_test.go
  • +18 -59
    loaders/codebuild/loader.go
  • +88 -0
    script/script.go
  • +109 -0
    script/script_test.go
    • --- a/ChangeLog Sun Jan 21 00:24:21 2018 -0600
    +++ b/ChangeLog Sun Jan 21 00:24:55 2018 -0600
    @@ -1,5 +1,6 @@
    0.13.2dev:
    - * Nothing yet! Be the first!!
    + * Changed tasks names for bitbucket pipeline tasks.
    + * Abstracted the bitbucket pipeline script parsing to a library function.
    0.13.1: 20180114
    * Write warning, error, and fatal log messages to stderr. Fixed #156
    --- a/loaders/bitbucket/loader.go Sun Jan 21 00:24:21 2018 -0600
    +++ b/loaders/bitbucket/loader.go Sun Jan 21 00:24:55 2018 -0600
    @@ -20,13 +20,11 @@
    "fmt"
    "os"
    "path/filepath"
    - "strings"
    -
    - "github.com/kballard/go-shellquote"
    "bitbucket.org/rw_grim/convey/config"
    "bitbucket.org/rw_grim/convey/docker"
    "bitbucket.org/rw_grim/convey/plans"
    + "bitbucket.org/rw_grim/convey/script"
    "bitbucket.org/rw_grim/convey/stages"
    "bitbucket.org/rw_grim/convey/state"
    "bitbucket.org/rw_grim/convey/tasks"
    @@ -37,23 +35,6 @@
    // Loader is a loader.Loader that loads bitbucket-pipelines.yml files
    type Loader struct{}
    -func addTask(name string, task tasks.Task, plan plans.Plan, cfg *config.Config) {
    - cfg.Tasks[name] = task
    -
    - plan.Stages[0].Tasks = append(plan.Stages[0].Tasks, name)
    -}
    -
    -func addScript(name string, image image, script []string, plan plans.Plan, cfg *config.Config) {
    - task := &docker.Run{
    - Shell: "/bin/sh",
    - Script: script,
    - Image: image.Name,
    - WorkDir: "/workspace",
    - }
    -
    - addTask(name, task, plan, cfg)
    -}
    -
    func addPipeline(name string, defImage image, pipelines []pipeline, cfg *config.Config) error {
    plan := plans.Plan{
    Stages: []stages.Stage{
    @@ -103,47 +84,17 @@
    plan.Stages[idx].Tasks = append(plan.Stages[idx].Tasks, loginTask)
    }
    - // now figure out how to parse the script in the config
    - currentScript := []string{}
    - last := -1
    -
    - for scriptIdx, command := range pipeline.Steps.Script {
    - argv, err := shellquote.Split(command)
    - if err != nil {
    - return err
    - }
    -
    - if len(argv) > 0 {
    - if strings.TrimSpace(strings.ToLower(argv[0])) == "docker" {
    - taskName := fmt.Sprintf("%s-step-%d-%d", name, idx, scriptIdx)
    -
    - // if we have existing script commands add them now
    - if len(currentScript) > 0 {
    - taskName := fmt.Sprintf("%s-step-%d-%d", name, idx, scriptIdx-1)
    - addScript(taskName, image, currentScript, plan, cfg)
    - currentScript = []string{}
    - last = scriptIdx
    - }
    -
    - // now figure out what docker command we're running
    - task, err := docker.ParseCommand(argv)
    - if err != nil {
    - return err
    - }
    -
    - addTask(taskName, task, plan, cfg)
    - last = scriptIdx
    - } else {
    - currentScript = append(currentScript, command)
    - }
    -
    - }
    + parsedTasks, err := script.Parse(image.Name, "/bin/sh", pipeline.Steps.Script)
    + if err != nil {
    + return err
    }
    - // if we have existing script commands add them now
    - if len(currentScript) > 0 {
    - name := fmt.Sprintf("%s-step-%d-%d", name, idx, last+1)
    - addScript(name, image, currentScript, plan, cfg)
    + for idx, task := range parsedTasks {
    + name := fmt.Sprintf("%s-%d", name, idx)
    + cfg.Tasks[name] = task
    +
    + plan.Stages[0].Tasks = append(plan.Stages[0].Tasks, name)
    +
    }
    // if we have a logout task, add it to the tasks for this stage
    --- a/loaders/bitbucket/loader_test.go Sun Jan 21 00:24:21 2018 -0600
    +++ b/loaders/bitbucket/loader_test.go Sun Jan 21 00:24:55 2018 -0600
    @@ -42,7 +42,7 @@
    - flake8
    - PYTHONPATH=$(pwd) py.test --color=auto --cov=pipelines --cov-report=term-missing tests`)
    - cfg, err := l.Load("", "", data, true)
    + cfg, err := l.Load("", "", data, []string{}, true)
    Expect(err).To(BeNil())
    @@ -51,7 +51,7 @@
    "import": &docker.Import{
    Files: []string{"."},
    },
    - "default-step-0-0": &docker.Run{
    + "default-0": &docker.Run{
    Image: "python:3",
    WorkDir: "/workspace",
    Script: []string{
    @@ -74,7 +74,7 @@
    Concurrent: false,
    Run: "on-success",
    Environment: nil,
    - Tasks: []string{"import", "default-step-0-0"},
    + Tasks: []string{"import", "default-0"},
    },
    },
    },
    @@ -100,7 +100,7 @@
    - flake8
    - PYTHONPATH=$(pwd) py.test --color=auto --cov=pipelines --cov-report=term-missing tests`)
    - cfg, err := l.Load("", "", data, true)
    + cfg, err := l.Load("", "", data, []string{}, true)
    Expect(err).To(BeNil())
    @@ -109,7 +109,7 @@
    "import": &docker.Import{
    Files: []string{"."},
    },
    - "default-step-0-0": &docker.Run{
    + "default-0": &docker.Run{
    Image: "python:3",
    WorkDir: "/workspace",
    Script: []string{
    @@ -132,7 +132,7 @@
    Concurrent: false,
    Run: "on-success",
    Environment: nil,
    - Tasks: []string{"import", "default-step-0-0"},
    + Tasks: []string{"import", "default-0"},
    },
    },
    },
    @@ -161,7 +161,7 @@
    - flake8
    - PYTHONPATH=$(pwd) py.test --color=auto --cov=pipelines --cov-report=term-missing tests`)
    - cfg, err := l.Load("", "", data, true)
    + cfg, err := l.Load("", "", data, []string{}, true)
    Expect(err).To(BeNil())
    @@ -175,7 +175,7 @@
    Password: "bar",
    Server: "registry.docker.io",
    },
    - "default-step-0-0": &docker.Run{
    + "default-0": &docker.Run{
    Image: "registry.docker.io/python:3",
    WorkDir: "/workspace",
    Script: []string{
    @@ -201,7 +201,7 @@
    Run: "on-success",
    Concurrent: false,
    Environment: nil,
    - Tasks: []string{"import", "login", "default-step-0-0", "logout"},
    + Tasks: []string{"import", "login", "default-0", "logout"},
    },
    },
    },
    @@ -227,7 +227,7 @@
    - flake8
    - PYTHONPATH=$(pwd) py.test --color=auto --cov=pipelines --cov-report=term-missing tests`)
    - cfg, err := l.Load("", "", data, true)
    + cfg, err := l.Load("", "", data, []string{}, true)
    Expect(err).To(BeNil())
    @@ -236,7 +236,7 @@
    "import": &docker.Import{
    Files: []string{"."},
    },
    - "default-step-0-0": &docker.Run{
    + "default-0": &docker.Run{
    Image: "python:3",
    WorkDir: "/workspace",
    Script: []string{
    @@ -259,7 +259,7 @@
    Concurrent: false,
    Run: "on-success",
    Environment: nil,
    - Tasks: []string{"import", "default-step-0-0"},
    + Tasks: []string{"import", "default-0"},
    },
    },
    },
    @@ -287,7 +287,7 @@
    - flake8
    - PYTHONPATH=$(pwd) py.test --color=auto --cov=pipelines --cov-report=term-missing tests`)
    - cfg, err := l.Load("", "", data, true)
    + cfg, err := l.Load("", "", data, []string{}, true)
    Expect(err).To(BeNil())
    @@ -301,7 +301,7 @@
    Password: "pass",
    Server: "registry.docker.io",
    },
    - "default-step-0-0": &docker.Run{
    + "default-0": &docker.Run{
    Image: "registry.docker.io/python:3",
    WorkDir: "/workspace",
    Script: []string{
    @@ -327,7 +327,7 @@
    Concurrent: false,
    Run: "on-success",
    Environment: nil,
    - Tasks: []string{"import", "default-0-login", "default-step-0-0", "default-0-logout"},
    + Tasks: []string{"import", "default-0-login", "default-0", "default-0-logout"},
    },
    },
    },
    @@ -355,7 +355,7 @@
    docker: true
    `)
    - cfg, err := l.Load("", "", data, true)
    + cfg, err := l.Load("", "", data, []string{}, true)
    Expect(err).To(BeNil())
    @@ -364,30 +364,30 @@
    "import": &docker.Import{
    Files: []string{"."},
    },
    - "default-step-0-0": &docker.Run{
    + "default-0": &docker.Run{
    Image: "gliderlabs/alpine:edge",
    Script: []string{"hostname"},
    WorkDir: "/workspace",
    Shell: "/bin/sh",
    },
    - "default-step-0-1": &docker.Login{
    + "default-1": &docker.Login{
    Username: "$DOCKER_USERNAME",
    Password: "$DOCKER_PASSWORD",
    },
    - "default-step-0-2": &docker.Run{
    + "default-2": &docker.Run{
    Image: "gliderlabs/alpine:edge",
    Script: []string{"date +%s"},
    WorkDir: "/workspace",
    Shell: "/bin/sh",
    },
    - "default-step-0-3": &docker.Build{
    + "default-3": &docker.Build{
    Files: []string{"."},
    Tag: "atlassian/my-app:$BITBUCKET_COMMIT",
    },
    - "default-step-0-4": &docker.Push{
    + "default-4": &docker.Push{
    Image: "atlassian/my-app:$BITBUCKET_COMMIT",
    },
    - "default-step-0-5": &docker.Run{
    + "default-5": &docker.Run{
    Image: "gliderlabs/alpine:edge",
    Script: []string{"date +%s"},
    WorkDir: "/workspace",
    @@ -405,12 +405,12 @@
    Environment: nil,
    Tasks: []string{
    "import",
    - "default-step-0-0",
    - "default-step-0-1",
    - "default-step-0-2",
    - "default-step-0-3",
    - "default-step-0-4",
    - "default-step-0-5",
    + "default-0",
    + "default-1",
    + "default-2",
    + "default-3",
    + "default-4",
    + "default-5",
    },
    },
    },
    @@ -435,7 +435,7 @@
    docker: true
    `)
    - cfg, err := l.Load("", "", data, true)
    + cfg, err := l.Load("", "", data, []string{}, true)
    Expect(err).To(BeNil())
    @@ -444,15 +444,15 @@
    "import": &docker.Import{
    Files: []string{"."},
    },
    - "default-step-0-0": &docker.Login{
    + "default-0": &docker.Login{
    Username: "$DOCKER_USERNAME",
    Password: "$DOCKER_PASSWORD",
    },
    - "default-step-0-1": &docker.Build{
    + "default-1": &docker.Build{
    Files: []string{"."},
    Tag: "atlassian/my-app:$BITBUCKET_COMMIT",
    },
    - "default-step-0-2": &docker.Push{
    + "default-2": &docker.Push{
    Image: "atlassian/my-app:$BITBUCKET_COMMIT",
    },
    },
    @@ -467,9 +467,9 @@
    Environment: nil,
    Tasks: []string{
    "import",
    - "default-step-0-0",
    - "default-step-0-1",
    - "default-step-0-2",
    + "default-0",
    + "default-1",
    + "default-2",
    },
    },
    },
    --- a/loaders/codebuild/loader.go Sun Jan 21 00:24:21 2018 -0600
    +++ b/loaders/codebuild/loader.go Sun Jan 21 00:24:55 2018 -0600
    @@ -19,14 +19,13 @@
    import (
    "fmt"
    "path/filepath"
    - "strings"
    "github.com/go-yaml/yaml"
    - "github.com/kballard/go-shellquote"
    "bitbucket.org/rw_grim/convey/config"
    "bitbucket.org/rw_grim/convey/docker"
    "bitbucket.org/rw_grim/convey/plans"
    + "bitbucket.org/rw_grim/convey/script"
    "bitbucket.org/rw_grim/convey/stages"
    "bitbucket.org/rw_grim/convey/state"
    "bitbucket.org/rw_grim/convey/tasks"
    @@ -76,73 +75,33 @@
    }
    func (l *Loader) addPhase(cb CodeBuild, cfg *config.Config, phaseName, imageName string, plan *plans.Plan) error {
    - stage := stages.Stage{
    - Name: phaseName,
    - Enabled: true,
    - Run: "on-success",
    - }
    -
    phase, ok := cb.Phases[phaseName]
    // ignore phases that are not in the yaml
    if !ok {
    return nil
    }
    - currentScript := []string{}
    - last := -1
    -
    - for idx, cmd := range phase.Commands {
    - argv, err := shellquote.Split(cmd)
    - if err != nil {
    - return err
    - }
    -
    - if len(argv) == 0 {
    - continue
    - }
    -
    - if strings.TrimSpace(argv[0]) == "docker" {
    - taskName := fmt.Sprintf("%s-%d", phaseName, idx)
    -
    - if len(currentScript) > 0 {
    - // create and add the task
    - taskName := fmt.Sprintf("%s-%d", phaseName, idx-1)
    - cfg.Tasks[taskName] = &docker.Run{
    - Shell: "/bin/sh",
    - Script: currentScript,
    - Image: imageName,
    - }
    - stage.Tasks = append(stage.Tasks, taskName)
    -
    - // reset our state
    - currentScript = []string{}
    - last = idx
    - }
    -
    - task, err := docker.ParseCommand(argv)
    - if err != nil {
    - return err
    - }
    -
    - cfg.Tasks[taskName] = task
    - stage.Tasks = append(stage.Tasks, taskName)
    -
    - last = idx
    - } else {
    - currentScript = append(currentScript, cmd)
    - }
    + parsedTasks, err := script.Parse(imageName, "/bin/sh", phase.Commands)
    + if err != nil {
    + return err
    }
    - if len(currentScript) > 0 {
    - taskName := fmt.Sprintf("%s-%d", phaseName, last+1)
    - cfg.Tasks[taskName] = &docker.Run{
    - Shell: "/bin/sh",
    - Script: currentScript,
    - Image: imageName,
    - }
    - stage.Tasks = append(stage.Tasks, taskName)
    + // create our stage
    + stage := stages.Stage{
    + Name: phaseName,
    + Enabled: true,
    + Run: "on-success",
    }
    + // now add the tasks to the config and the stage
    + for idx, task := range parsedTasks {
    + name := fmt.Sprintf("%s-%d", phaseName, idx)
    + cfg.Tasks[name] = task
    +
    + stage.Tasks = append(stage.Tasks, name)
    + }
    +
    + // add the stage to the plan
    plan.Stages = append(plan.Stages, stage)
    return nil
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/script/script.go Sun Jan 21 00:24:55 2018 -0600
    @@ -0,0 +1,88 @@
    +// Convey
    +// Copyright 2016-2018 Gary Kramlich <grim@reaperworld.com>
    +//
    +// This program is free software: you can redistribute it and/or modify
    +// it under the terms of the GNU General Public License as published by
    +// the Free Software Foundation, either version 3 of the License, or
    +// (at your option) any later version.
    +//
    +// This program is distributed in the hope that it will be useful,
    +// but WITHOUT ANY WARRANTY; without even the implied warranty of
    +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    +// GNU General Public License for more details.
    +//
    +// You should have received a copy of the GNU General Public License
    +// along with this program. If not, see <http://www.gnu.org/licenses/>.
    +
    +package script
    +
    +import (
    + "strings"
    +
    + "github.com/kballard/go-shellquote"
    +
    + "bitbucket.org/rw_grim/convey/docker"
    + "bitbucket.org/rw_grim/convey/tasks"
    +)
    +
    +type ParseFunc func(argv []string) (tasks.Task, error)
    +
    +var (
    + commands = map[string]ParseFunc{
    + "docker": docker.ParseCommand,
    + }
    +)
    +
    +// Parse will process a slice of commands and turn it into the proper tasks for
    +// convey to run.
    +func Parse(image, shell string, script []string) ([]tasks.Task, error) {
    + newTasks := []tasks.Task{}
    + currentScript := []string{}
    +
    + for _, cmd := range script {
    + argv, err := shellquote.Split(cmd)
    + if err != nil {
    + return nil, err
    + }
    +
    + if len(argv) == 0 {
    + continue
    + }
    +
    + currentCmd := strings.TrimSpace(argv[0])
    + if parser, ok := commands[currentCmd]; ok {
    + if len(currentScript) > 0 {
    + newTasks = append(newTasks, &docker.Run{
    + Shell: shell,
    + Script: currentScript,
    + Image: image,
    + WorkDir: "/workspace",
    + })
    +
    + // reset our script
    + currentScript = []string{}
    + }
    +
    + task, err := parser(argv)
    + if err != nil {
    + return nil, err
    + }
    +
    + newTasks = append(newTasks, task)
    + } else {
    + currentScript = append(currentScript, cmd)
    + }
    + }
    +
    + // if we have any unfinished script commands create a task for them
    + if len(currentScript) > 0 {
    + newTasks = append(newTasks, &docker.Run{
    + Shell: shell,
    + Script: currentScript,
    + Image: image,
    + WorkDir: "/workspace",
    + })
    + }
    +
    + return newTasks, nil
    +}
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/script/script_test.go Sun Jan 21 00:24:55 2018 -0600
    @@ -0,0 +1,109 @@
    +// Convey
    +// Copyright 2016-2018 Gary Kramlich <grim@reaperworld.com>
    +//
    +// This program is free software: you can redistribute it and/or modify
    +// it under the terms of the GNU General Public License as published by
    +// the Free Software Foundation, either version 3 of the License, or
    +// (at your option) any later version.
    +//
    +// This program is distributed in the hope that it will be useful,
    +// but WITHOUT ANY WARRANTY; without even the implied warranty of
    +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    +// GNU General Public License for more details.
    +//
    +// You should have received a copy of the GNU General Public License
    +// along with this program. If not, see <http://www.gnu.org/licenses/>.
    +
    +package script
    +
    +import (
    + "testing"
    +
    + "github.com/aphistic/sweet"
    + junit "github.com/aphistic/sweet-junit"
    + . "github.com/onsi/gomega"
    +
    + "bitbucket.org/rw_grim/convey/docker"
    + "bitbucket.org/rw_grim/convey/tasks"
    +)
    +
    +type scriptSuite struct{}
    +
    +func TestMain(m *testing.M) {
    + RegisterFailHandler(sweet.GomegaFail)
    +
    + sweet.Run(m, func(s *sweet.S) {
    + s.RegisterPlugin(junit.NewPlugin())
    +
    + s.AddSuite(&scriptSuite{})
    + })
    +}
    +
    +func (s *scriptSuite) TestEmpty(t sweet.T) {
    + parsedTasks, err := Parse("", "", []string{})
    +
    + Expect(err).To(BeNil())
    + Expect(parsedTasks).To(BeEquivalentTo([]tasks.Task{}))
    +}
    +
    +func (s *scriptSuite) TestSimple(t sweet.T) {
    + commands := []string{
    + "#!/bin/sh",
    + "false",
    + }
    +
    + parsedTasks, err := Parse("", "", commands)
    +
    + Expect(err).To(BeNil())
    + Expect(len(parsedTasks)).To(Equal(1))
    +
    + Expect(parsedTasks[0]).To(BeAssignableToTypeOf(&docker.Run{}))
    +
    +}
    +
    +func (s *scriptSuite) TestDockerThenScript(t sweet.T) {
    + commands := []string{
    + "docker push foo",
    + "pwd",
    + }
    +
    + parsedTasks, err := Parse("", "", commands)
    +
    + Expect(err).To(BeNil())
    + Expect(len(parsedTasks)).To(Equal(2))
    +
    + Expect(parsedTasks[0]).To(BeAssignableToTypeOf(&docker.Push{}))
    + Expect(parsedTasks[1]).To(BeAssignableToTypeOf(&docker.Run{}))
    +}
    +
    +func (s *scriptSuite) TestScriptThenDocker(t sweet.T) {
    + commands := []string{
    + "pwd",
    + "docker push foo",
    + }
    +
    + parsedTasks, err := Parse("", "", commands)
    +
    + Expect(err).To(BeNil())
    + Expect(len(parsedTasks)).To(Equal(2))
    +
    + Expect(parsedTasks[0]).To(BeAssignableToTypeOf(&docker.Run{}))
    + Expect(parsedTasks[1]).To(BeAssignableToTypeOf(&docker.Push{}))
    +}
    +
    +func (s *scriptSuite) TestMixed(t sweet.T) {
    + commands := []string{
    + "pwd",
    + "docker push foo",
    + "pwd",
    + }
    +
    + parsedTasks, err := Parse("", "", commands)
    +
    + Expect(err).To(BeNil())
    + Expect(len(parsedTasks)).To(Equal(3))
    +
    + Expect(parsedTasks[0]).To(BeAssignableToTypeOf(&docker.Run{}))
    + Expect(parsedTasks[1]).To(BeAssignableToTypeOf(&docker.Push{}))
    + Expect(parsedTasks[2]).To(BeAssignableToTypeOf(&docker.Run{}))
    +}