grim/convey

We compile again, but stuff is broken including tests

2019-03-28, Gary Kramlich
f32e6c3ef3f9
Parents c5cf97738140
Children c3b54288d7dc
We compile again, but stuff is broken including tests
17 files changed, 198 insertions(+), 278 deletions(-)
  • +1 -21
    docker/build.go
  • +0 -1
    docker/docker.go
  • +1 -1
    docker/environment.go
  • +0 -164
    docker/export.go
  • +1 -1
    docker/import.go
  • +1 -5
    docker/run.go
  • +3 -3
    kubectl/command.go
  • +1 -1
    loaders/codebuild/loader.go
  • +0 -18
    plans/plans.go
  • +9 -50
    state/state.go
  • +72 -0
    state/workspace.go
  • +25 -0
    tasks/errors.go
  • +70 -0
    tasks/export.go
  • +1 -0
    tasks/tasks.go
  • +5 -5
    tests/breakout.yml
  • +2 -2
    tests/convey-clean-task.yml
  • +6 -6
    tests/tasks-types.yml
    • --- a/docker/build.go Mon Dec 31 23:09:04 2018 -0600
    +++ b/docker/build.go Thu Mar 28 04:40:01 2019 -0500
    @@ -44,35 +44,15 @@
    func (b *Build) Execute(name string, logger *gomol.LogAdapter, env *environment.Environment, rt *runtime.Runtime) error {
    fullEnv := env.Copy().Merge(rt.Environment)
    - files, err := fullEnv.MapSlice(b.Files)
    - if err != nil {
    - return err
    - }
    -
    // create out build directory
    buildDir := filepath.Join(rt.State.Directory, name)
    - err = os.MkdirAll(buildDir, 0700)
    + err := os.MkdirAll(buildDir, 0700)
    if err != nil {
    return err
    }
    dockerfile := fullEnv.Map(b.Dockerfile)
    - // Export the files from the workspace
    - fileExport := &Export{
    - Files: append(files, dockerfile),
    - Path: buildDir,
    - }
    - err = fileExport.Valid()
    - if err != nil {
    - return err
    - }
    -
    - err = fileExport.Execute(name, logger, fullEnv, rt)
    - if err != nil {
    - return err
    - }
    -
    tags, err := fullEnv.MapSlice(b.Tags)
    if err != nil {
    return err
    --- a/docker/docker.go Mon Dec 31 23:09:04 2018 -0600
    +++ b/docker/docker.go Thu Mar 28 04:40:01 2019 -0500
    @@ -27,7 +27,6 @@
    // Tasks is a map of all docker tasks.
    Tasks = map[string]tasks.Task{
    "build": &Build{},
    - "export": &Export{},
    "import": &Import{},
    "login": &Login{},
    "logout": &Logout{},
    --- a/docker/environment.go Mon Dec 31 23:09:04 2018 -0600
    +++ b/docker/environment.go Thu Mar 28 04:40:01 2019 -0500
    @@ -63,7 +63,7 @@
    // structure of the file (for ease of error messages, so we
    // get the file a/b/c/env instead of env).
    - e := Export{
    + e := tasks.Export{
    Files: []string{file},
    Path: envDir,
    }
    --- a/docker/export.go Mon Dec 31 23:09:04 2018 -0600
    +++ /dev/null Thu Jan 01 00:00:00 1970 +0000
    @@ -1,164 +0,0 @@
    -// Convey
    -// Copyright 2016-2018 Gary Kramlich <grim@reaperworld.com>
    -//
    -// This program is free software: you can redistribute it and/or modify
    -// it under the terms of the GNU General Public License as published by
    -// the Free Software Foundation, either version 3 of the License, or
    -// (at your option) any later version.
    -//
    -// This program is distributed in the hope that it will be useful,
    -// but WITHOUT ANY WARRANTY; without even the implied warranty of
    -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    -// GNU General Public License for more details.
    -//
    -// You should have received a copy of the GNU General Public License
    -// along with this program. If not, see <http://www.gnu.org/licenses/>.
    -
    -package docker
    -
    -import (
    - "fmt"
    - "os"
    - "path/filepath"
    - "strings"
    -
    - "github.com/aphistic/gomol"
    -
    - "bitbucket.org/rw_grim/convey/environment"
    - "bitbucket.org/rw_grim/convey/path"
    - "bitbucket.org/rw_grim/convey/runtime"
    - "bitbucket.org/rw_grim/convey/tasks"
    - "bitbucket.org/rw_grim/convey/yaml"
    -)
    -
    -// Export represents an export task which exports files from the workspace to
    -// the host.
    -type Export struct {
    - Files yaml.StringOrSlice `yaml:"files"`
    - Path string
    -}
    -
    -func checkFilePattern(file string) error {
    - if strings.ContainsRune(file, '*') && strings.ContainsRune(file, ':') {
    - return errWildcardWithDestination
    - }
    -
    - return nil
    -}
    -
    -func (e *Export) file(name, workSpace, src, dest string, rt *runtime.Runtime) error {
    - dir := filepath.Dir(dest)
    - if err := os.MkdirAll(dir, 0700); err != nil {
    - return err
    - }
    -
    - // build out the source path
    - source := workSpace + ":" + filepath.Join("/workspace", src)
    -
    - cmdv := []string{
    - "cp",
    - source,
    - dest,
    - }
    -
    - return Docker(name, cmdv, rt)
    -}
    -
    -func (e *Export) glob(name, workSpace, mountPoint, pattern string, rt *runtime.Runtime) error {
    - cmdv := []string{
    - "run",
    - "--rm",
    - "-v",
    - mountPoint + ":/workspace",
    - "convey/workspace-tools:latest",
    - "zglob",
    - pattern,
    - }
    -
    - out, _, err := DockerOutput(fmt.Sprintf("%s-zglob", name), cmdv, rt)
    - if err != nil {
    - return err
    - }
    -
    - for _, match := range strings.Split(out, "\n") {
    - if match == "" {
    - continue
    - }
    -
    - dest := e.Path
    - if dest == "" {
    - dest = path.DestFromSrc(match)
    - }
    -
    - err = e.file(name, workSpace, match, dest, rt)
    - if err != nil {
    - return err
    - }
    - }
    -
    - return nil
    -}
    -
    -// Execute runs the export task.
    -func (e *Export) Execute(name string, logger *gomol.LogAdapter, env *environment.Environment, rt *runtime.Runtime) error {
    - fullEnv := env.Copy().Merge(rt.Environment)
    -
    - files, err := fullEnv.MapSlice(e.Files)
    - if err != nil {
    - return err
    - }
    -
    - for _, file := range files {
    - if err := checkFilePattern(file); err != nil {
    - return err
    - }
    -
    - dockerWorkspace := rt.State.Workspace.(*Workspace)
    -
    - if strings.ContainsRune(file, '*') {
    - mountPoint := fullEnv.Map(dockerWorkspace.mountPoint)
    -
    - if err := e.glob(name, rt.State.Workspace.Name(), mountPoint, file, rt); err != nil {
    - return err
    - }
    - } else {
    - src, dest := path.ParseFilePath("", file)
    -
    - if e.Path != "" {
    - dest = e.Path
    - }
    -
    - // make sure the destination is in our build context
    - realDest, err := path.TraversesNonExistent(rt.State.CfgPath, dest)
    - if err != nil {
    - return err
    - }
    -
    - if err := e.file(name, rt.State.Workspace.Name(), src, realDest, rt); err != nil {
    - return err
    - }
    - }
    - }
    -
    - return nil
    -}
    -
    -// New creates a new Export task.
    -func (e *Export) New() tasks.Task {
    - return &Export{}
    -}
    -
    -// Valid validates the export task.
    -func (e *Export) Valid() error {
    - if len(e.Files) == 0 {
    - return errNoFiles
    - }
    -
    - for _, file := range e.Files {
    - if err := checkFilePattern(file); err != nil {
    - return err
    - }
    - }
    -
    - return nil
    -}
    --- a/docker/import.go Mon Dec 31 23:09:04 2018 -0600
    +++ b/docker/import.go Thu Mar 28 04:40:01 2019 -0500
    @@ -62,7 +62,7 @@
    cmdv := []string{
    "cp",
    realSrc,
    - rt.State.Workspace.Name() + ":/workspace/" + dest,
    + rt.State.Workspace.Path() + ":/workspace/" + dest,
    }
    if err := Docker(name, cmdv, rt); err != nil {
    --- a/docker/run.go Mon Dec 31 23:09:04 2018 -0600
    +++ b/docker/run.go Thu Mar 28 04:40:01 2019 -0500
    @@ -264,10 +264,6 @@
    cmd.Append("--network-alias", hostname)
    }
    - // figure out the workspace variables
    - dockerWorkspace := rt.State.Workspace.(*Workspace)
    - workspacePath := fullEnv.Map(dockerWorkspace.mountPoint)
    -
    // assign a default workspace location
    workSpace := r.WorkSpace
    if workSpace == "" {
    @@ -277,7 +273,7 @@
    workspaceMount := fullEnv.Map(workSpace)
    cmd.Append(
    - "-v", workspacePath+":"+workspaceMount,
    + "-v", rt.State.Workspace.Path()+":"+workspaceMount,
    "-e", "CONVEY_WORKSPACE="+workspaceMount,
    )
    --- a/kubectl/command.go Mon Dec 31 23:09:04 2018 -0600
    +++ b/kubectl/command.go Thu Mar 28 04:40:01 2019 -0500
    @@ -24,9 +24,9 @@
    "github.com/aphistic/gomol"
    "bitbucket.org/rw_grim/convey/command"
    - "bitbucket.org/rw_grim/convey/docker"
    "bitbucket.org/rw_grim/convey/environment"
    "bitbucket.org/rw_grim/convey/runtime"
    + "bitbucket.org/rw_grim/convey/tasks"
    "bitbucket.org/rw_grim/convey/yaml"
    )
    @@ -49,7 +49,7 @@
    }
    // now create an export task to get our files out of the workspace
    - export := &docker.Export{
    + export := &tasks.Export{
    Files: c.Files,
    Path: scratchDir,
    }
    @@ -159,7 +159,7 @@
    }
    // create our scratch directory
    - path, err := rt.State.TaskDirectory(name)
    + path, err := rt.State.Workspace.TaskDirectory(name)
    if err != nil {
    return err
    }
    --- a/loaders/codebuild/loader.go Mon Dec 31 23:09:04 2018 -0600
    +++ b/loaders/codebuild/loader.go Thu Mar 28 04:40:01 2019 -0500
    @@ -163,7 +163,7 @@
    return
    }
    - export := &docker.Export{
    + export := &tasks.Export{
    Files: cb.Artifacts.Files,
    }
    --- a/plans/plans.go Mon Dec 31 23:09:04 2018 -0600
    +++ b/plans/plans.go Thu Mar 28 04:40:01 2019 -0500
    @@ -81,24 +81,6 @@
    }
    })
    - // create the workspace
    - if lErr := logger.Info("creating workspace..."); lErr != nil {
    - fmt.Printf("error reporting info: %s\n", lErr)
    - }
    -
    - ws, err := docker.NewWorkspace(rt)
    - if err != nil {
    - if lErr := logger.Fatal("failed to create workspace"); lErr != nil {
    - fmt.Printf("error reporting fatal: %s\n", lErr)
    - }
    -
    - return err
    - }
    - rt.State.Workspace = ws
    - if lErr := logger.Infof("created workspace %s", rt.State.Workspace.Name()); lErr != nil {
    - fmt.Printf("error reporting info: %s\n", lErr)
    - }
    -
    return nil
    }
    --- a/state/state.go Mon Dec 31 23:09:04 2018 -0600
    +++ b/state/state.go Thu Mar 28 04:40:01 2019 -0500
    @@ -19,16 +19,13 @@
    import (
    "fmt"
    - "io/ioutil"
    "os"
    - "path/filepath"
    "time"
    "github.com/aphistic/gomol"
    "bitbucket.org/rw_grim/convey/logging"
    "bitbucket.org/rw_grim/convey/network"
    - "bitbucket.org/rw_grim/convey/workspace"
    )
    // State holds all of the runtime data during a run.
    @@ -38,7 +35,7 @@
    CfgPath string
    Network network.Network
    - Workspace workspace.Workspace
    + Workspace *Workspace
    KeepWorkspace bool
    DisableDeprecated bool
    @@ -53,13 +50,18 @@
    // New creates a new state.
    func New() *State {
    - dir, err := createDirectory()
    + pwd, err := os.Getwd()
    + if err != nil {
    + panic(err)
    + }
    +
    + ws, err := newWorkspace(pwd)
    if err != nil {
    panic(err)
    }
    return &State{
    - Directory: dir,
    + Workspace: ws,
    logger: logging.NewAdapter("state"),
    }
    }
    @@ -69,51 +71,8 @@
    func (st *State) Destroy() {
    // finally remove the workspace if requested
    if !st.KeepWorkspace {
    - if err := os.RemoveAll(st.Directory); err != nil {
    - if lErr := st.logger.Warningf("error removing %s: %s\n", st.Directory, err.Error()); lErr != nil {
    - fmt.Printf("error reporting warning: %s\n", lErr)
    - }
    - }
    -
    - // try to remove the .convey directory. This will only succeed when
    - // our state is the only one and that's intentional.
    - dir := filepath.Dir(st.Directory)
    - if err := os.Remove(dir); err != nil {
    - if lErr := st.logger.Warningf("error removing directory: %s : %s", dir, err); lErr != nil {
    - fmt.Printf("error reporting warning: %s\n", lErr)
    - }
    - }
    - }
    -}
    -
    -// createDirectory creates the state directory for this run.
    -func createDirectory() (string, error) {
    - pwd, err := os.Getwd()
    - if err != nil {
    - return "", err
    + st.Workspace.Destroy()
    }
    -
    - parent := filepath.Join(pwd, ".convey")
    - err = os.MkdirAll(parent, 0700)
    - if err != nil {
    - return "", err
    - }
    -
    - dir, err := ioutil.TempDir(parent, "")
    - if err != nil {
    - return "", err
    - }
    -
    - return dir, nil
    -}
    -
    -// TaskDirectory will create a directory in the state directory for the named task.
    -func (st *State) TaskDirectory(name string) (string, error) {
    - dir := filepath.Join(st.Directory, name)
    -
    - err := os.MkdirAll(dir, 0700)
    -
    - return dir, err
    }
    // Valid validates whether the state is correct or not.
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/state/workspace.go Thu Mar 28 04:40:01 2019 -0500
    @@ -0,0 +1,72 @@
    +// Convey
    +// Copyright 2016-2018 Gary Kramlich <grim@reaperworld.com>
    +//
    +// This program is free software: you can redistribute it and/or modify
    +// it under the terms of the GNU General Public License as published by
    +// the Free Software Foundation, either version 3 of the License, or
    +// (at your option) any later version.
    +//
    +// This program is distributed in the hope that it will be useful,
    +// but WITHOUT ANY WARRANTY; without even the implied warranty of
    +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    +// GNU General Public License for more details.
    +//
    +// You should have received a copy of the GNU General Public License
    +// along with this program. If not, see <http://www.gnu.org/licenses/>.
    +
    +package state
    +
    +import (
    + "fmt"
    + "io/ioutil"
    + "os"
    + "path/filepath"
    +)
    +
    +type Workspace struct {
    + path string
    +}
    +
    +func newWorkspace(root string) (*Workspace, error) {
    + parent := filepath.Join(root, ".convey")
    + err := os.MkdirAll(parent, 0700)
    + if err != nil {
    + return nil, err
    + }
    +
    + path, err := ioutil.TempDir(parent, "")
    + if err != nil {
    + return nil, err
    + }
    +
    + workspace := &Workspace{
    + path: path,
    + }
    +
    + return workspace, nil
    +}
    +
    +func (ws *Workspace) Destroy() {
    + if err := os.RemoveAll(ws.path); err != nil {
    + fmt.Printf("error removing worksace: %s\n", err)
    + }
    +
    + // try to remove the .convey directory. This will only succeed when
    + // our state is the only one and that's intentional.
    + dir := filepath.Dir(ws.path)
    + if err := os.Remove(dir); err != nil {
    + fmt.Printf("error removing .convey: %s\n", err)
    + }
    +}
    +
    +func (ws *Workspace) TaskDirectory(name string) (string, error) {
    + dir := filepath.Join(ws.path, name)
    +
    + err := os.MkdirAll(dir, 0700)
    +
    + return dir, err
    +}
    +
    +func (ws *Workspace) Path() string {
    + return ws.path
    +}
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/tasks/errors.go Thu Mar 28 04:40:01 2019 -0500
    @@ -0,0 +1,25 @@
    +// Convey
    +// Copyright 2016-2019 Gary Kramlich <grim@reaperworld.com>
    +//
    +// This program is free software: you can redistribute it and/or modify
    +// it under the terms of the GNU General Public License as published by
    +// the Free Software Foundation, either version 3 of the License, or
    +// (at your option) any later version.
    +//
    +// This program is distributed in the hope that it will be useful,
    +// but WITHOUT ANY WARRANTY; without even the implied warranty of
    +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    +// GNU General Public License for more details.
    +//
    +// You should have received a copy of the GNU General Public License
    +// along with this program. If not, see <http://www.gnu.org/licenses/>.
    +
    +package tasks
    +
    +import (
    + "errors"
    +)
    +
    +var (
    + errNoFiles = errors.New("no files specified")
    +)
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/tasks/export.go Thu Mar 28 04:40:01 2019 -0500
    @@ -0,0 +1,70 @@
    +// Convey
    +// Copyright 2016-2019 Gary Kramlich <grim@reaperworld.com>
    +//
    +// This program is free software: you can redistribute it and/or modify
    +// it under the terms of the GNU General Public License as published by
    +// the Free Software Foundation, either version 3 of the License, or
    +// (at your option) any later version.
    +//
    +// This program is distributed in the hope that it will be useful,
    +// but WITHOUT ANY WARRANTY; without even the implied warranty of
    +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    +// GNU General Public License for more details.
    +//
    +// You should have received a copy of the GNU General Public License
    +// along with this program. If not, see <http://www.gnu.org/licenses/>.
    +
    +package tasks
    +
    +import (
    + "errors"
    + "path/filepath"
    +
    + "github.com/aphistic/gomol"
    +
    + "bitbucket.org/rw_grim/convey/environment"
    + "bitbucket.org/rw_grim/convey/runtime"
    +)
    +
    +type Export struct {
    + Files []string `yaml:"files"`
    + Path string `yaml:"path"`
    +}
    +
    +// New creates a new Export task.
    +func (e *Export) New() Task {
    + return &Export{}
    +}
    +
    +// Valid validates the export task.
    +func (e *Export) Valid() error {
    + if len(e.Files) == 0 {
    + return errNoFiles
    + }
    +
    + return nil
    +}
    +
    +// Copies a file
    +func (e *Export) copyFile(filename string) error {
    + return errors.New("Not implemented")
    +}
    +
    +// Executes the task
    +func (e *Export) Execute(name string, logger *gomol.LogAdapter, env *environment.Environment, rt *runtime.Runtime) error {
    + for _, patterns := range e.Files {
    + matches, err := filepath.Glob(filepath.Join(rt.State.CfgPath, patterns))
    + if err != nil {
    + return err
    + }
    +
    + for _, match := range matches {
    + err = e.copyFile(match)
    + if err != nil {
    + return nil
    + }
    + }
    + }
    +
    + return nil
    +}
    --- a/tasks/tasks.go Mon Dec 31 23:09:04 2018 -0600
    +++ b/tasks/tasks.go Thu Mar 28 04:40:01 2019 -0500
    @@ -28,6 +28,7 @@
    // Tasks is the list of tasks defined in this package.
    Tasks = map[string]Task{
    "clean": &Clean{},
    + "export": &Export{},
    "extend": &Extend{},
    "noop": &Noop{},
    }
    --- a/tests/breakout.yml Mon Dec 31 23:09:04 2018 -0600
    +++ b/tests/breakout.yml Thu Mar 28 04:40:01 2019 -0500
    @@ -15,14 +15,14 @@
    default:
    stages:
    - tasks:
    - - import
    - - export
    + - import
    + - export
    import-bad:
    stages:
    - tasks:
    - - import-bad
    + - import-bad
    export-bad:
    stages:
    - tasks:
    - - import
    - - export-bad
    + - import
    + - export-bad
    --- a/tests/convey-clean-task.yml Mon Dec 31 23:09:04 2018 -0600
    +++ b/tests/convey-clean-task.yml Thu Mar 28 04:40:01 2019 -0500
    @@ -13,5 +13,5 @@
    default:
    stages:
    - tasks:
    - - clean-file
    - - clean-files
    + - clean-file
    + - clean-files
    --- a/tests/tasks-types.yml Mon Dec 31 23:09:04 2018 -0600
    +++ b/tests/tasks-types.yml Thu Mar 28 04:40:01 2019 -0500
    @@ -51,11 +51,11 @@
    default:
    stages:
    - tasks:
    - - implicit
    - - implicit-engine
    - - implicit-type
    - - explicit-old
    - - explicit-new
    + - implicit
    + - implicit-engine
    + - implicit-type
    + - explicit-old
    + - explicit-new
    - tasks:
    - - explicit-new-unknown
    + - explicit-new-unknown