Finish up environments and get our convey.yml fully functional
--- a/convey.yml Tue Oct 05 06:34:28 2021 -0500
+++ b/convey.yml Wed Oct 06 23:04:17 2021 -0500
@@ -1,8 +1,7 @@
-required-version: 0.14.0-dev
- CONVEY_VERSION=0.14.0-dev
- - GO_IMAGE=golang:1.17-bullseye
+ - GO_IMAGE=docker.io/golang:1.17-bullseye # tasks for the default plan
@@ -25,43 +24,44 @@
workdir: ${CONVEY_WORKSPACE}
workdir: ${CONVEY_WORKSPACE}
- command: go build -o convey-${CONVEY_VERSION}-${GOOS}-${GOARCH}${SUFFIX}
+ command: go build -o convey-${CONVEY_VERSION}-${GOOS}-${GOARCH} + workdir: ${CONVEY_WORKSPACE}
+ command: go build -o convey-${CONVEY_VERSION}-${GOOS}-${GOARCH}.exe + workdir: ${CONVEY_WORKSPACE}
+ command: go build -o convey-${CONVEY_VERSION}-${GOOS}-${GOARCH} + workdir: ${CONVEY_WORKSPACE}
+ command: go build -o convey-${CONVEY_VERSION}-${GOOS}-${GOARCH}
@@ -71,10 +71,10 @@
--- a/environment/default.go Tue Oct 05 06:34:28 2021 -0500
+++ b/environment/default.go Wed Oct 06 23:04:17 2021 -0500
@@ -28,7 +28,19 @@
"keep.imfreedom.org/grim/convey/normalize"
-func (e *Environment) loadVCS(wd string) error {
+// setEnvOS will add the given envar by name only to the environment but also +// put value into the OS's environmanet variable of the same name. +func (e Environment) setEnvOS(name, value string) error { + if err := os.Setenv(name, value); err != nil { +func (e Environment) loadVCS(wd string) error { repo, err := govcs.Detect(wd)
@@ -36,38 +48,54 @@
name := strings.ToUpper(repo.Name())
- e.Set("COMMIT", repo.Commit())
- e.Set("COMMIT_SHORT", repo.ShortCommit())
- e.Set("COMMIT", repo.Commit())
- e.Set("BRANCH", repo.Branch())
- e.Set("BRANCH_NORMALIZED", normalize.Normalize(repo.Branch()))
- e.Set("REMOTE", repo.Remote(""))
+ if err := e.setEnvOS("VCS", name); err != nil { + if err := e.setEnvOS("COMMIT", repo.Commit()); err != nil { + if err := e.setEnvOS("COMMIT_SHORT", repo.ShortCommit()); err != nil { - e.Set(name+"_COMMIT", repo.Commit())
- e.Set(name+"_COMMIT_SHORT", repo.ShortCommit())
- e.Set(name+"_COMMIT", repo.Commit())
- e.Set(name+"_BRANCH", repo.Branch())
- e.Set(name+"_BRANCH_NORMALIZED", normalize.Normalize(repo.Branch()))
- e.Set(name+"_REMOTE", repo.Remote(""))
+ if err := e.setEnvOS("COMMIT", repo.Commit()); err != nil { + if err := e.setEnvOS("BRANCH", repo.Branch()); err != nil { + if err := e.setEnvOS("BRANCH_NORMALIZED", normalize.Normalize(repo.Branch())); err != nil { + if err := e.setEnvOS("REMOTE", repo.Remote("")); err != nil { // check for vcs specific values. If this grows past 1, make it a switch
if hg, ok := repo.(*hg.Mercurial); ok {
- e.Set("BOOKMARK", hg.Bookmark())
- e.Set(name+"_BOOKMARK", hg.Bookmark())
+ if err := e.setEnvOS("BOOKMARK", hg.Bookmark()); err != nil { // LoadDefaults will load the default environment variables for a convey run.
-func (e *Environment) LoadDefaults(wd string) error {
+func (e Environment) LoadDefaults(wd string) error { oldHome := os.Getenv("HOME")
- defer e.Set("HOME", oldHome)
- e.Set("RUN_TIME", time.Now().UTC().Format("2006-01-02T15:04:05-0700"))
+ e["RUN_TIME"] = time.Now().UTC().Format("2006-01-02T15:04:05-0700") --- a/environment/environment.go Tue Oct 05 06:34:28 2021 -0500
+++ b/environment/environment.go Wed Oct 06 23:04:17 2021 -0500
@@ -18,24 +18,10 @@
-type EnvironmentReader interface {
- ReadEnvironment() []string
-type Environment struct {
+type Environment map[string]string // splitEqual takes a string like "foo=bar" and returns it as ("foo", "bar")
func splitEqual(value string) (string, string) {
@@ -47,80 +33,56 @@
return parts[0], parts[1]
-func New(items ...string) *Environment {
+func New(items ...string) Environment { -func (e *Environment) Copy() *Environment {
- return New(e.Items()...)
-func NewExpandable(expandables []string, delimiter string, items ...string) *Environment {
- expandables: expandables,
-func (e *Environment) Items() []string {
+ return r.MergeSlice(items) -func (e *Environment) Set(name, value string) {
- if err := os.Setenv(name, value); err != nil {
- fmt.Printf("error setting environment variable '%s': %s\n", name, err)
- e.items = append(e.items, name)
+func (e Environment) Copy() Environment { -// Prune removes items from the environment if they don't have a value and are
-// not in the OS's environment.
-func (e *Environment) Prune() *Environment {
- for _, val := range e.items {
- k, v := splitEqual(val)
- if v == "" && os.Getenv(k) == "" {
- pruned = append(pruned, val)
+// Merge will merge another environment into the current environment +// overwriting any existing items +func (e Environment) Merge(env Environment) Environment { + for k, v := range env {
-// Merge will merge another environment into the current environment
-func (e *Environment) Merge(env *Environment) *Environment {
- return e.MergeSlice(env.items)
-func (e *Environment) MergeSlice(items []string) *Environment {
- e.items = Merge(e.items, items)
+// MergeSlice will merge a slice of strings in key or key=value formats. +func (e Environment) MergeSlice(items []string) Environment { + for _, item := range items { + k, v := splitEqual(item) -func (e *Environment) MapSlice(slice []string) ([]string, error) {
- return SliceMapper(slice, e.Items())
+func (e Environment) All() []string { + r := make([]string, len(e)) -func (e *Environment) Map(key string) string {
- return Map(key, e.Items())
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/environment/expand.go Wed Oct 06 23:04:17 2021 -0500
@@ -0,0 +1,37 @@
+func (e Environment) expandMapper(name string) string { + // We might have something like $(pwd), in which case we don't want + // to wrap the name in `{}`, which would cause it to be ${}(pwd) and + // cannot be evaluated by a run task. + if val, found := e[name]; found { +// Expand looks for the given key and returns the value if one is found +// recursing if the value contains another variable. +func (e Environment) Expand(key string) string { + return os.Expand(key, e.expandMapper) +func (e Environment) Expandv(items []string) []string { + expanded := make([]string, len(items)) + for idx, item := range items { + expanded[idx] = e.Expand(item) --- a/environment/merge.go Tue Oct 05 06:34:28 2021 -0500
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,47 +0,0 @@
-// Copyright 2016-2018 Gary Kramlich <grim@reaperworld.com>
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-// You should have received a copy of the GNU General Public License
-// along with this program. If not, see <http://www.gnu.org/licenses/>.
-// Package environment provides utilities for managing environment variables.
-// Merge combines two environments and returns the result.
-func Merge(orig, update []string) []string {
- env := map[string]string{}
- for _, val := range orig {
- k, v := splitEqual(val)
- for _, val := range update {
- k, v := splitEqual(val)
- for k, v := range env {
--- a/plans/plans.go Tue Oct 05 06:34:28 2021 -0500
+++ b/plans/plans.go Wed Oct 06 23:04:17 2021 -0500
@@ -24,6 +24,7 @@
log "github.com/sirupsen/logrus"
+ "keep.imfreedom.org/grim/convey/environment" "keep.imfreedom.org/grim/convey/logging"
"keep.imfreedom.org/grim/convey/runtime"
"keep.imfreedom.org/grim/convey/stages"
@@ -46,12 +47,12 @@
// Execute runs the plan.
-func (p *Plan) Execute(path string, tasks map[string]tasks.Task, rt *runtime.Runtime) error {
+func (p *Plan) Execute(path string, tasks map[string]tasks.Task, configEnv environment.Environment, rt *runtime.Runtime) error { if err := p.Valid(); err != nil {
- planEnv := rt.Environment.Copy().MergeSlice(p.Environment)
+ planEnv := configEnv.Copy().MergeSlice(p.Environment).Merge(rt.Environment) p.logger = logging.NewAdapter(path)
--- a/podman/run.go Tue Oct 05 06:34:28 2021 -0500
+++ b/podman/run.go Wed Oct 06 23:04:17 2021 -0500
@@ -21,14 +21,15 @@
-func (r *Run) Execute(name string, logger *log.Entry, stageEnv *environment.Environment, rt *runtime.Runtime) error {
+func (r *Run) Execute(name string, logger *log.Entry, stageEnv environment.Environment, rt *runtime.Runtime) error { // Create a new environment based on the stage's environment. Then merge
// the task's environment overriding anything from the stage. Finally merge
// the runtime enviroment which holds the environment from the command line.
- //env := environment.New().Merge(stageEnv).MergeSlice(r.Environment).Merge(rt.Environment)
+ env := stageEnv.Copy().MergeSlice(r.Environment).Merge(rt.Environment) - // Figure out where we're mounting the workspace.
- workspace := r.Workspace
+ // Figure out where we're mounting the workspace. This should be done + // before workdir is expanded as work is typically set to CONVEY_WORKSPACE. + workspace := env.Expand(r.Workspace) @@ -38,6 +39,8 @@
+ env["CONVEY_WORKSPACE"] = workspace generator := exec.NewGenerator(
@@ -47,14 +50,19 @@
// If the user specified a workdir, use it.
- generator.Append("--workdir", r.Workdir)
+ generator.Append("--workdir", env.Expand(r.Workdir)) + // Now add all the given environment variables + for _, name := range env.All() { + generator.Append("-e", name) // Append the image name.
- generator.Append(r.Image)
+ generator.Append(env.Expand(r.Image)) - generator.Appendv(r.commandv)
+ generator.Appendv(env.Expandv(r.commandv)) return exec.Run(name, generator.Command(), rt.Timeout)
--- a/podman/tasks.go Tue Oct 05 06:34:28 2021 -0500
+++ b/podman/tasks.go Wed Oct 06 23:04:17 2021 -0500
@@ -7,6 +7,8 @@
Tasks = map[string]tasks.Task{
--- a/runner/cmd.go Tue Oct 05 06:34:28 2021 -0500
+++ b/runner/cmd.go Wed Oct 06 23:04:17 2021 -0500
@@ -7,6 +7,7 @@
"keep.imfreedom.org/grim/convey/config"
+ "keep.imfreedom.org/grim/convey/environment" "keep.imfreedom.org/grim/convey/globals"
"keep.imfreedom.org/grim/convey/logging"
"keep.imfreedom.org/grim/convey/runtime"
@@ -52,10 +53,12 @@
rt := runtime.New(configPath, c.ForceSequential, c.KeepWorkspace, c.Timeout)
+ configEnv := environment.New(cfg.Environment...) for _, name := range c.Plans {
- err := plan.Execute(name, cfg.Tasks, rt)
+ err := plan.Execute(name, cfg.Tasks, configEnv, rt) --- a/runtime/runtime.go Tue Oct 05 06:34:28 2021 -0500
+++ b/runtime/runtime.go Wed Oct 06 23:04:17 2021 -0500
@@ -27,7 +27,7 @@
- Environment *environment.Environment
+ Environment environment.Environment @@ -47,7 +47,7 @@
-func NewWithEnvironment(configPath string, forceSequential bool, keepWorkspace bool, timeout time.Duration, env *environment.Environment) *Runtime {
+func NewWithEnvironment(configPath string, forceSequential bool, keepWorkspace bool, timeout time.Duration, env environment.Environment) *Runtime { @@ -56,6 +56,8 @@
+ rt.Environment.LoadDefaults(configPath) --- a/stages/stages.go Tue Oct 05 06:34:28 2021 -0500
+++ b/stages/stages.go Wed Oct 06 23:04:17 2021 -0500
@@ -72,8 +72,8 @@
// Execute runs the stage.
-func (s *Stage) Execute(path string, logger *log.Entry, taskMap map[string]tasks.Task, env *environment.Environment, rt *runtime.Runtime) error {
- stageEnv := env.Copy().MergeSlice(env.Items()).MergeSlice(s.Environment)
+func (s *Stage) Execute(path string, logger *log.Entry, taskMap map[string]tasks.Task, env environment.Environment, rt *runtime.Runtime) error { + stageEnv := env.Copy().Merge(env).MergeSlice(s.Environment) if s.Concurrent && !rt.ForceSequential {
taskRes := make(chan error)
@@ -125,7 +125,7 @@
-func (s *Stage) runTask(path, name string, stageEnv *environment.Environment, taskMap map[string]tasks.Task, rt *runtime.Runtime) error {
+func (s *Stage) runTask(path, name string, stageEnv environment.Environment, taskMap map[string]tasks.Task, rt *runtime.Runtime) error { absTaskName := fmt.Sprintf("%s/%s", path, name)
taskLogger := logging.NewAdapter(fmt.Sprintf("%s/%s", path, name))
--- a/tasks/README.md Tue Oct 05 06:34:28 2021 -0500
+++ b/tasks/README.md Wed Oct 06 23:04:17 2021 -0500
@@ -6,13 +6,13 @@
-A clean task will remove files from the host relative and limited to the working directory of convey.
+A clean task will remove files from the host relative and limited to the directory where the convey.yml for the run. | Name | Required | Default | Description |
| --------- | -------- | ------- | ----------- |
-| files | Yes | | A list of filenames relative to the convey working directory to remove. |
+| files | Yes | | A list of filenames relative to the convey.yml file to remove. | @@ -25,10 +25,33 @@
+An export task will move files from the workspace to the host underneath the directory where the convey.yml file was found. +| Name | Required | Default | Description | +| --------- | -------- | ------- | ----------- | +| files | Yes | | A single filename or a list of files to copy from the workspace to the host. Files can be specified in one of two forms which can be mixed. The first is `source:destination` and the other is `filename` where filename will be used for both the host and the volume. | + - binary:binary-linux-x86_64 -The fail task just automatically fails. It's similar to convey/noop in that it's
-mostly used for debugging.
+The fail task just automatically fails. It's similar to convey/noop in that it's mostly used for debugging. @@ -41,6 +64,30 @@
+An import task copies files from the host to the workspace. It has one required attribute named files. This is a list of files relative to the directory the convey.yml file that will be copied into the workspace. +| Name | Required | Default | Description | +| ----- | -------- | ------- | ----------- | +| files | Yes | | A single filename or a list of files to copy from the host to the workspace. Files can be specified in one of two forms which can be mixed. The first is `source:destination` and the other is `filename` where filename will be used for both the host and the volume. | A noop task does nothing. It is a "No Operation" task. It's used primarily in
@@ -54,38 +101,3 @@
-----
-A subtask lets reuse existing tasks by setting the environment.
-As of right now, a subtask can **NOT** override an environment variable in it's
-parent task, but you can mimic that but having all subtasks setting the
-| Name | Required | Default | Description |
-| -------- | -------- | ------- | ----------- |
-| base | Yes | | The name of the parent task |
- command: go build -o foo-${GOOS}-${GOARCH}
--- a/tasks/clean.go Tue Oct 05 06:34:28 2021 -0500
+++ b/tasks/clean.go Wed Oct 06 23:04:17 2021 -0500
@@ -35,10 +35,10 @@
Files yaml.StringOrSlice `yaml:"files"`
-func sanitizeFile(base, file string, env []string) (string, error) {
- filename, err := environment.Mapper(file, env)
+func sanitizeFile(base, file string, env environment.Environment) (string, error) { + filename := env.Expand(file) + return "", fmt.Errorf("no file name provided") pathname, err := filepath.Abs(filename)
@@ -53,7 +53,7 @@
-func sanitizeFiles(base string, files, env []string) ([]string, error) {
+func sanitizeFiles(base string, files, env environment.Environment) ([]string, error) { for _, file := range files {
sanitized, err := sanitizeFile(base, file, env)
@@ -68,7 +68,7 @@
// Execute runs the clean task.
-func (c *Clean) Execute(name string, logger *log.Entry, env *environment.Environment, rt *runtime.Runtime) error {
+func (c *Clean) Execute(name string, logger *log.Entry, env environment.Environment, rt *runtime.Runtime) error { fullEnv := environment.New()
fullEnv.Merge(rt.Environment)
@@ -79,9 +79,9 @@
for _, pattern := range c.Files {
- expandedPattern, err := environment.Mapper(pattern, fullEnv.Items())
+ expandedPattern := fullEnv.Expand(pattern) + if expandedPattern == "" { matches, err := zglob.Glob(expandedPattern)
@@ -96,7 +96,7 @@
for _, path := range matches {
- sanitized, err := sanitizeFile(wd, path, fullEnv.Items())
+ sanitized, err := sanitizeFile(wd, path, fullEnv) --- a/tasks/count.go Tue Oct 05 06:34:28 2021 -0500
+++ b/tasks/count.go Wed Oct 06 23:04:17 2021 -0500
@@ -29,7 +29,7 @@
// Execute runs the count task.
-func (c *Count) Execute(name string, logger *log.Entry, env *environment.Environment, rt *runtime.Runtime) error {
+func (c *Count) Execute(name string, logger *log.Entry, env environment.Environment, rt *runtime.Runtime) error { --- a/tasks/export.go Tue Oct 05 06:34:28 2021 -0500
+++ b/tasks/export.go Wed Oct 06 23:04:17 2021 -0500
@@ -17,8 +17,8 @@
log "github.com/sirupsen/logrus"
@@ -30,7 +30,6 @@
Files yaml.StringOrSlice `yaml:"files"`
- Path string `yaml:"path"`
// New creates a new Export task.
@@ -48,27 +47,34 @@
-func (e *Export) Execute(name string, logger *log.Entry, env *environment.Environment, rt *runtime.Runtime) error {
+func (e *Export) Execute(name string, logger *log.Entry, env environment.Environment, rt *runtime.Runtime) error { fullEnv := env.Copy().Merge(rt.Environment)
+ workspace, err := rt.Workspace() for _, pattern := range e.Files {
- pattern = fullEnv.Map(pattern)
+ pattern := fullEnv.Expand(pattern) spec := fs.ParsePathSpec(pattern)
- // matches, err := filepath.Glob(filepath.Join(rt.State.Workspace.Volume().Path(), spec.Source()))
+ matches, err := filepath.Glob(filepath.Join(workspace.Path(), spec.Source())) - // for _, match := range matches {
- // realSrc := strings.TrimPrefix(match, rt.State.Workspace.Volume().Path())
- // err = rt.State.Workspace.Volume().Export(realSrc, spec.Destination())
- log.Printf("spec: %q\n", spec)
+ for _, match := range matches { + realSrc := strings.TrimPrefix(match, workspace.Path()) + err = workspace.Export(realSrc, spec.Destination()) --- a/tasks/fail.go Tue Oct 05 06:34:28 2021 -0500
+++ b/tasks/fail.go Wed Oct 06 23:04:17 2021 -0500
@@ -29,7 +29,7 @@
// Execute runs the fail task.
-func (c *Fail) Execute(name string, logger *log.Entry, env *environment.Environment, rt *runtime.Runtime) error {
+func (c *Fail) Execute(name string, logger *log.Entry, env environment.Environment, rt *runtime.Runtime) error { return fmt.Errorf("convey/fail task always fails")
--- a/tasks/import.go Tue Oct 05 06:34:28 2021 -0500
+++ b/tasks/import.go Wed Oct 06 23:04:17 2021 -0500
@@ -48,8 +48,15 @@
-func (i *Import) Execute(name string, logger *log.Entry, env *environment.Environment, rt *runtime.Runtime) error {
+func (i *Import) Execute(name string, logger *log.Entry, env environment.Environment, rt *runtime.Runtime) error { + fullEnv := env.Copy().Merge(rt.Environment) for _, pattern := range i.Files {
+ pattern := fullEnv.Expand(pattern) spec := fs.ParsePathSpec(pattern)
matches, err := filepath.Glob(filepath.Join(rt.ConfigPath, spec.Source()))
--- a/tasks/noop.go Tue Oct 05 06:34:28 2021 -0500
+++ b/tasks/noop.go Wed Oct 06 23:04:17 2021 -0500
@@ -27,7 +27,7 @@
// Execute runs the noop task.
-func (c *Noop) Execute(name string, logger *log.Entry, env *environment.Environment, rt *runtime.Runtime) error {
+func (c *Noop) Execute(name string, logger *log.Entry, env environment.Environment, rt *runtime.Runtime) error { --- a/tasks/subtask.go Tue Oct 05 06:34:28 2021 -0500
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,58 +0,0 @@
-// Copyright 2016-2020 Gary Kramlich <grim@reaperworld.com>
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-// You should have received a copy of the GNU General Public License
-// along with this program. If not, see <http://www.gnu.org/licenses/>.
- log "github.com/sirupsen/logrus"
- "keep.imfreedom.org/grim/convey/environment"
- "keep.imfreedom.org/grim/convey/runtime"
- "keep.imfreedom.org/grim/convey/yaml"
- Base string `yaml:"base"`
- Environment yaml.StringOrSlice `yaml:"environment"`
-func (t *SubTask) New() Task {
-func (t *SubTask) Valid() error {
- return fmt.Errorf("no base task specified")
-func (t *SubTask) Execute(name string, logger *log.Entry, env *environment.Environment, rt *runtime.Runtime) error {
- _ = t.Parent.(environment.EnvironmentReader)
- if e, ok := t.Parent.(environment.EnvironmentReader); ok {
- fullEnv.MergeSlice(e.ReadEnvironment())
- fullEnv.MergeSlice(t.Environment).Merge(rt.Environment)
- return t.Parent.Execute(name, logger, fullEnv, rt)
--- a/tasks/tasks.go Tue Oct 05 06:34:28 2021 -0500
+++ b/tasks/tasks.go Wed Oct 06 23:04:17 2021 -0500
@@ -27,18 +27,21 @@
// Tasks is the list of tasks defined in this package.
- "convey/clean": &Clean{},
- "convey/export": &Export{},
- "convey/fail": &Fail{},
- "convey/import": &Import{},
- "convey/noop": &Noop{},
- "convey/subtask": &SubTask{},
+ "convey/clean": &Clean{}, + "convey/export": &Export{}, + "convey/fail": &Fail{}, + "convey/import": &Import{}, + "convey/noop": &Noop{}, + // compatibilty mode engage + "docker/import": &Import{}, + "docker/export": &Export{}, // Task is an interface for defining a task that can be run.
- Execute(name string, logger *log.Entry, env *environment.Environment, rt *runtime.Runtime) error
+ Execute(name string, logger *log.Entry, env environment.Environment, rt *runtime.Runtime) error