grim/convey

A bunch of work for dealing with fileio but it's still seriously busted, needs rethinking

2020-05-27, Gary Kramlich
92f0295be27a
Parents 41133c801f52
Children ebde6315c3d3
A bunch of work for dealing with fileio but it's still seriously busted, needs rethinking
11 files changed, 215 insertions(+), 76 deletions(-)
  • +14 -5
    docker/build.go
  • +3 -3
    docker/environment.go
  • +3 -3
    docker/run.go
  • +3 -3
    kubectl/command.go
  • +2 -2
    tasks/export.go
  • +1 -1
    tasks/import.go
  • +2 -2
    tasks/import_test.go
  • +16 -47
    workspace/fileio.go
  • +86 -0
    workspace/taskdir.go
  • +71 -0
    workspace/volume.go
  • +14 -10
    workspace/workspace.go
    • --- a/docker/build.go Wed May 27 04:03:24 2020 -0500
    +++ b/docker/build.go Wed May 27 05:08:18 2020 -0500
    @@ -23,6 +23,7 @@
    "keep.imfreedom.org/grim/convey/environment"
    "keep.imfreedom.org/grim/convey/exec"
    + "keep.imfreedom.org/grim/convey/path"
    "keep.imfreedom.org/grim/convey/runtime"
    "keep.imfreedom.org/grim/convey/tasks"
    "keep.imfreedom.org/grim/convey/yaml"
    @@ -43,12 +44,20 @@
    func (b *Build) Execute(name string, logger *log.Entry, env *environment.Environment, rt *runtime.Runtime) error {
    fullEnv := env.Copy().Merge(rt.Environment)
    - // create out task directory
    - buildDir, err := rt.State.Workspace.TaskDirectory(name)
    + // create our task directory
    + td, err := rt.State.Workspace.CreateTaskDirectory(name)
    if err != nil {
    return err
    }
    + // import all of the declared files to our task directory
    + for _, filespec := range b.Files {
    + log.Debugf("filespec: %q", filespec)
    + src, dst := path.ParseFilePath("", filespec)
    + log.Debugf("src: %q, dst: %q", src, dst)
    + rt.State.Workspace.Volume().Export(src, dst)
    + }
    +
    dockerfile := fullEnv.Map(b.Dockerfile)
    tags, err := fullEnv.MapSlice(b.Tags)
    @@ -66,12 +75,12 @@
    return err
    }
    - rt.State.Workspace.Import(dockerfile, buildDir)
    + td.Import(dockerfile, filepath.Base(dockerfile))
    // create the basic command
    cmd := exec.NewGenerator(
    "build",
    - "-f", filepath.Join(buildDir, filepath.Base(dockerfile)),
    + "-f", filepath.Join(td.Path(), filepath.Base(dockerfile)),
    )
    // add any and all tags
    @@ -95,7 +104,7 @@
    }
    // finally add the build context
    - cmd.Append(buildDir)
    + cmd.Append(td.Path())
    return Docker(name, cmd.Command(), rt)
    }
    --- a/docker/environment.go Wed May 27 04:03:24 2020 -0500
    +++ b/docker/environment.go Wed May 27 05:08:18 2020 -0500
    @@ -51,7 +51,7 @@
    }
    // create a directory in our state for the task
    - envDir, err := rt.State.Workspace.TaskDirectory(name)
    + td, err := rt.State.Workspace.CreateTaskDirectory(name)
    if err != nil {
    return err
    }
    @@ -63,7 +63,7 @@
    e := tasks.Export{
    Files: []string{file},
    - Path: envDir,
    + Path: td.Path(),
    }
    err = e.Execute(name, logger, env, rt)
    @@ -73,7 +73,7 @@
    // Process the entries of the file and apply them to the
    // state's base environment immediately.
    - dest := filepath.Clean(filepath.Join(envDir, file))
    + dest := filepath.Clean(filepath.Join(td.Path(), file))
    entries, err := processFile(dest, file, prefix)
    if err != nil {
    return err
    --- a/docker/run.go Wed May 27 04:03:24 2020 -0500
    +++ b/docker/run.go Wed May 27 05:08:18 2020 -0500
    @@ -75,13 +75,13 @@
    // writeScript will write a shell script to the given tempfile for the given commands
    func (r *Run) writeScript(name string, rt *runtime.Runtime, fullEnv *environment.Environment) (string, string, string, error) {
    // create our task directory if it doesn't exist already
    - scriptsDir, err := rt.State.Workspace.TaskDirectory(name)
    + td, err := rt.State.Workspace.CreateTaskDirectory(name)
    if err != nil {
    return "", "", "", err
    }
    // create the temp file to write the script to
    - scriptFile := filepath.Join(scriptsDir, "script")
    + scriptFile := filepath.Join(td.Path(), "script")
    entryPoint := r.Shell
    if entryPoint == "" {
    @@ -245,7 +245,7 @@
    workspaceMount := fullEnv.Map(workSpace)
    cmd.Append(
    - "-v", rt.State.Workspace.Volume()+":"+workspaceMount,
    + "-v", rt.State.Workspace.Volume().Path()+":"+workspaceMount,
    "-e", "CONVEY_WORKSPACE="+workspaceMount,
    )
    --- a/kubectl/command.go Wed May 27 04:03:24 2020 -0500
    +++ b/kubectl/command.go Wed May 27 05:08:18 2020 -0500
    @@ -164,7 +164,7 @@
    }
    // create our scratch directory
    - path, err := rt.State.Workspace.TaskDirectory(name)
    + td, err := rt.State.Workspace.CreateTaskDirectory(name)
    if err != nil {
    return err
    }
    @@ -174,11 +174,11 @@
    case "":
    fallthrough
    case "none":
    - err = c.templateNone(name, path, cmd, logger, fullEnv, rt)
    + err = c.templateNone(name, td.Path(), cmd, logger, fullEnv, rt)
    case "env":
    fallthrough
    case "environment":
    - err = c.templateEnvironment(name, path, cmd, logger, fullEnv, rt)
    + err = c.templateEnvironment(name, td.Path(), cmd, logger, fullEnv, rt)
    }
    // if there was an error with templating bail
    --- a/tasks/export.go Wed May 27 04:03:24 2020 -0500
    +++ b/tasks/export.go Wed May 27 05:08:18 2020 -0500
    @@ -54,13 +54,13 @@
    pattern = fullEnv.Map(pattern)
    src, dst := path.ParseFilePath("", pattern)
    - matches, err := filepath.Glob(filepath.Join(rt.State.Workspace.Volume(), src))
    + matches, err := filepath.Glob(filepath.Join(rt.State.Workspace.Volume().Path(), src))
    if err != nil {
    return err
    }
    for _, match := range matches {
    - err = rt.State.Workspace.Export(match, dst)
    + err = rt.State.Workspace.Volume().Export(match, dst)
    if err != nil {
    return err
    }
    --- a/tasks/import.go Wed May 27 04:03:24 2020 -0500
    +++ b/tasks/import.go Wed May 27 05:08:18 2020 -0500
    @@ -61,7 +61,7 @@
    }
    for _, match := range matches {
    - err = rt.State.Workspace.Import(match, dst)
    + err = rt.State.Workspace.Volume().Import(match, dst)
    if err != nil {
    return err
    }
    --- a/tasks/import_test.go Wed May 27 04:03:24 2020 -0500
    +++ b/tasks/import_test.go Wed May 27 05:08:18 2020 -0500
    @@ -44,7 +44,7 @@
    rt := runtime.NewWithEnvironment(st, env)
    defer rt.Shutdown()
    - // create our task and runn it.
    + // create our task and run it.
    importTask := &Import{Files: files}
    err = importTask.Execute("", logging.NewAdapter("test"), env, rt)
    @@ -54,7 +54,7 @@
    for _, file := range files {
    _, dst := path.ParseFilePath(st.CfgPath, file)
    - _, err = os.Stat(filepath.Join(st.Workspace.Volume(), dst))
    + _, err = os.Stat(filepath.Join(st.Workspace.Volume().Path(), dst))
    assert.NotEqual(t, err, os.IsNotExist(err))
    }
    }
    --- a/workspace/fileio.go Wed May 27 04:03:24 2020 -0500
    +++ b/workspace/fileio.go Wed May 27 05:08:18 2020 -0500
    @@ -27,17 +27,6 @@
    log "github.com/sirupsen/logrus"
    )
    -func (ws *Workspace) TaskDirectory(name string) (string, error) {
    - dir := filepath.Join(ws.path, name)
    - if !strings.HasSuffix(dir, "/") {
    - dir += "/"
    - }
    -
    - err := os.MkdirAll(dir, 0700)
    -
    - return dir, err
    -}
    -
    // CopyFile copies the contents from src to dst atomically.
    // If dst does not exist, CopyFile creates it with permissions perm.
    // If the copy fails, CopyFile aborts and dst is preserved.
    @@ -69,7 +58,16 @@
    }
    func (ws *Workspace) copyDir(src, dst string, perm os.FileMode) error {
    - err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
    + absDst, err := filepath.Abs(dst)
    + if err != nil {
    + return err
    + }
    +
    + dst = absDst
    +
    + err = filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
    + log.Debugf("walk: path %q", path)
    +
    // make sure we're copying files that are at the same level as convey.yml or
    // deeper.
    if !strings.HasPrefix(path, ws.parent) {
    @@ -80,14 +78,17 @@
    )
    }
    - // make sure we're don't try to import anything from the .convey directory
    - if strings.HasPrefix(path, ws.root) {
    + // make sure we're don't descend into our destination directory.
    + if strings.HasPrefix(path, absDst) {
    + log.Debugf("%q is in our destination %q", path, absDst)
    return nil
    }
    // at this point we're going to do something so we need to the real
    // destination name
    - realDst := filepath.Join(ws.volumePath, strings.TrimPrefix(path, ws.parent))
    + realDst := filepath.Join(dst, strings.TrimPrefix(path, ws.parent))
    +
    + log.Debugf("src %q; dst %q; realDst %q", path, dst, realDst)
    // if path is a directory, create it
    if info.IsDir() {
    @@ -108,35 +109,3 @@
    return err
    }
    -
    -func (ws *Workspace) Import(src, dst string) error {
    - file, err := os.Stat(src)
    - if err != nil {
    - return err
    - }
    -
    - realDst := filepath.Join(ws.volumePath, dst)
    -
    - if file.IsDir() {
    - log.Debugf("importing directory %q to %q", src, realDst)
    -
    - return ws.copyDir(src, realDst, file.Mode())
    - }
    -
    - log.Debugf("importing file %q to %q", src, realDst)
    -
    - return ws.copyFile(src, realDst, file.Mode())
    -}
    -
    -func (ws *Workspace) Export(src, dst string) error {
    - file, err := os.Stat(src)
    - if err != nil {
    - return err
    - }
    -
    - if file.IsDir() {
    - return ws.copyDir(src, dst, file.Mode())
    - }
    -
    - return ws.copyFile(src, dst, file.Mode())
    -}
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/workspace/taskdir.go Wed May 27 05:08:18 2020 -0500
    @@ -0,0 +1,86 @@
    +// Convey
    +// Copyright 2016-2020 Gary Kramlich <grim@reaperworld.com>
    +//
    +// This program is free software: you can redistribute it and/or modify
    +// it under the terms of the GNU General Public License as published by
    +// the Free Software Foundation, either version 3 of the License, or
    +// (at your option) any later version.
    +//
    +// This program is distributed in the hope that it will be useful,
    +// but WITHOUT ANY WARRANTY; without even the implied warranty of
    +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    +// GNU General Public License for more details.
    +//
    +// You should have received a copy of the GNU General Public License
    +// along with this program. If not, see <http://www.gnu.org/licenses/>.
    +
    +package workspace
    +
    +import (
    + "os"
    + "path/filepath"
    + "strings"
    +
    + log "github.com/sirupsen/logrus"
    +)
    +
    +type TaskDirectory struct {
    + path string
    + ws *Workspace
    +}
    +
    +func (ws *Workspace) CreateTaskDirectory(name string) (*TaskDirectory, error) {
    + dir := filepath.Join(ws.path, name)
    + if !strings.HasSuffix(dir, "/") {
    + dir += "/"
    + }
    +
    + err := os.MkdirAll(dir, 0700)
    +
    + return &TaskDirectory{
    + path: dir,
    + ws: ws,
    + }, err
    +}
    +
    +func (td *TaskDirectory) Import(src, dst string) error {
    + file, err := os.Stat(src)
    + if err != nil {
    + return err
    + }
    +
    + realDst := filepath.Join(td.path, dst)
    +
    + if file.IsDir() {
    + log.Debugf("importing directory %q to %q", src, realDst)
    +
    + return td.ws.copyDir(src, realDst, file.Mode())
    + }
    +
    + log.Debugf("importing file %q to %q", src, realDst)
    +
    + return td.ws.copyFile(src, realDst, file.Mode())
    +}
    +
    +func (td *TaskDirectory) Export(src, dst string) error {
    + realSrc := filepath.Join(td.path, src)
    +
    + file, err := os.Stat(realSrc)
    + if err != nil {
    + return err
    + }
    +
    + if file.IsDir() {
    + log.Debugf("importing directory %q to %q", realSrc, dst)
    +
    + return td.ws.copyDir(realSrc, dst, file.Mode())
    + }
    +
    + log.Debugf("importing file %q to %q", realSrc, dst)
    +
    + return td.ws.copyFile(realSrc, dst, file.Mode())
    +}
    +
    +func (td *TaskDirectory) Path() string {
    + return td.path
    +}
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/workspace/volume.go Wed May 27 05:08:18 2020 -0500
    @@ -0,0 +1,71 @@
    +// Convey
    +// Copyright 2016-2019 Gary Kramlich <grim@reaperworld.com>
    +//
    +// This program is free software: you can redistribute it and/or modify
    +// it under the terms of the GNU General Public License as published by
    +// the Free Software Foundation, either version 3 of the License, or
    +// (at your option) any later version.
    +//
    +// This program is distributed in the hope that it will be useful,
    +// but WITHOUT ANY WARRANTY; without even the implied warranty of
    +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    +// GNU General Public License for more details.
    +//
    +// You should have received a copy of the GNU General Public License
    +// along with this program. If not, see <http://www.gnu.org/licenses/>.
    +
    +package workspace
    +
    +import (
    + "os"
    + "path/filepath"
    +
    + log "github.com/sirupsen/logrus"
    +)
    +
    +type Volume struct {
    + path string
    + ws *Workspace
    +}
    +
    +func (vol *Volume) Import(src, dst string) error {
    + file, err := os.Stat(src)
    + if err != nil {
    + return err
    + }
    +
    + realDst := filepath.Join(vol.path, dst)
    +
    + if file.IsDir() {
    + log.Debugf("importing directory %q to %q", src, realDst)
    +
    + return vol.ws.copyDir(src, realDst, file.Mode())
    + }
    +
    + log.Debugf("importing file %q to %q", src, realDst)
    +
    + return vol.ws.copyFile(src, realDst, file.Mode())
    +}
    +
    +func (vol *Volume) Export(src, dst string) error {
    + realSrc := filepath.Join(vol.path, src)
    +
    + file, err := os.Stat(realSrc)
    + if err != nil {
    + return err
    + }
    +
    + if file.IsDir() {
    + log.Debugf("importing directory %q to %q", realSrc, dst)
    +
    + return vol.ws.copyDir(realSrc, dst, file.Mode())
    + }
    +
    + log.Debugf("importing file %q to %q", realSrc, dst)
    +
    + return vol.ws.copyFile(realSrc, dst, file.Mode())
    +}
    +
    +func (vol *Volume) Path() string {
    + return vol.path
    +}
    --- a/workspace/workspace.go Wed May 27 04:03:24 2020 -0500
    +++ b/workspace/workspace.go Wed May 27 05:08:18 2020 -0500
    @@ -25,10 +25,10 @@
    )
    type Workspace struct {
    - parent string
    - root string
    - path string
    - volumePath string
    + parent string
    + root string
    + path string
    + volume *Volume
    }
    func New(parent string) (*Workspace, error) {
    @@ -50,10 +50,14 @@
    }
    workspace := &Workspace{
    - parent: parent,
    - root: root,
    - path: path,
    - volumePath: volumePath,
    + parent: parent,
    + root: root,
    + path: path,
    + }
    +
    + workspace.volume = &Volume{
    + path: volumePath,
    + ws: workspace,
    }
    return workspace, nil
    @@ -76,6 +80,6 @@
    return ws.path
    }
    -func (ws *Workspace) Volume() string {
    - return ws.volumePath
    +func (ws *Workspace) Volume() *Volume {
    + return ws.volume
    }