grim/convey

Draft an implementation of inject task.
inject
2017-10-11, Eric Fritz
ad2092e0f404
Parents e2ed1742f942
Children f37bad7a9b83
Draft an implementation of inject task.
8 files changed, 297 insertions(+), 3 deletions(-)
  • +36 -0
    REFERENCE.md
  • +1 -0
    docker/docker.go
  • +1 -0
    docker/errors.go
  • +141 -0
    docker/inject.go
  • +58 -0
    docker/inject_test.go
  • +56 -0
    examples/inject.yml
  • +0 -1
    intrinsic/intrinsic.go
  • +4 -2
    state/state.go
    • --- a/REFERENCE.md Wed Oct 11 19:40:16 2017 -0500
    +++ b/REFERENCE.md Wed Oct 11 20:48:50 2017 -0500
    @@ -50,6 +50,10 @@
    This example shows how to expand an environment variable into a list in the context of a extended task.
    +## inject.yml
    +
    +This example shows how to inject variables from a file in the build container into the plan.
    +
    ## login-logout.yml
    This example shows how you can login and logout of a Docker registry by using environment variables.
    @@ -172,6 +176,38 @@
    ----
    +### Inject task
    +
    +An inject task will read a variable with environment variables and make them accessible to your plans.
    +
    +A file is expected to have lines of the form `variable=value`. Empty lines are ignored. Files are read
    +from the docker volume, so files generated by the plan do not need to be exported in order to be a target
    +of an inject task. From the other direction, it is necessary to import files on the host if they are to be
    +read by an inject task.
    +
    +#### Attributes
    +
    +| Name | Required | Default | Description |
    +| ---------- | -------- | ------- | ----------- |
    +| file | | | A file that should be read. |
    +| files | | | A list of files that should be read. |
    +| prefix | | | A prefix to add to each variable read from a file |
    +
    +At least one file must be supplied by either the `file` or `files` attributes. If both are
    +supplied, `file` is inserted to the front of `files`. If the files being read are a result of
    +environment variable expansion, the order that the files are read are not guaranteed to be
    +stable (or in the order supplied). Be cautious of this if the environment files have define
    +overlapping variables.
    +
    +#### Example
    +
    + inject-version:
    + type: inject
    + file: version.txt
    + prefix: "APP_"
    +
    +----
    +
    ### Build task
    A build task will build a docker image.
    --- a/docker/docker.go Wed Oct 11 19:40:16 2017 -0500
    +++ b/docker/docker.go Wed Oct 11 20:48:50 2017 -0500
    @@ -35,6 +35,7 @@
    "remove": &Remove{},
    "run": &Run{},
    "tag": &Tag{},
    + "inject": &Inject{},
    }
    dockerTemplate = `docker {{if .DockerConfig}}--config {{.DockerConfig}} {{end}}`
    --- a/docker/errors.go Wed Oct 11 19:40:16 2017 -0500
    +++ b/docker/errors.go Wed Oct 11 20:48:50 2017 -0500
    @@ -32,4 +32,5 @@
    errNoImages = errors.New("no images specified")
    errNoSourceTag = errors.New("no source tag specified")
    errNoDestinationTags = errors.New("no destination tags specified")
    + errNoFilesInject = errors.New("no environment files specified")
    )
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/docker/inject.go Wed Oct 11 20:48:50 2017 -0500
    @@ -0,0 +1,141 @@
    +/*
    + * Convey
    + * Copyright 2016-2017 Gary Kramlich <grim@reaperworld.com>
    + *
    + * This program is free software: you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License as published by
    + * the Free Software Foundation, either version 3 of the License, or
    + * (at your option) any later version.
    + *
    + * This program is distributed in the hope that it will be useful,
    + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    + * GNU General Public License for more details.
    + *
    + * You should have received a copy of the GNU General Public License
    + * along with this program. If not, see <http://www.gnu.org/licenses/>.
    + */
    +package docker
    +
    +import (
    + "fmt"
    + "io/ioutil"
    + "os"
    + "path/filepath"
    + "strings"
    +
    + "github.com/aphistic/gomol"
    +
    + "bitbucket.org/rw_grim/convey/environment"
    + "bitbucket.org/rw_grim/convey/state"
    + "bitbucket.org/rw_grim/convey/tasks"
    + "bitbucket.org/rw_grim/convey/yaml"
    +)
    +
    +type Inject struct {
    + File string `yaml:"file"`
    + Files yaml.StringOrSlice `yaml:"files"`
    + Prefix string `yaml:"prefix"`
    +}
    +
    +func (i *Inject) Execute(name string, logger *gomol.LogAdapter, env []string, st *state.State) error {
    + fullEnv := environment.Merge(env, st.GetEnv())
    +
    + prefix, err := environment.Mapper(i.Prefix, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    + files, err := st.MapSlice(i.Files, fullEnv)
    + if err != nil {
    + return err
    + }
    +
    + // Create a temp directory we can export files from the current
    + // container into. We can't read directly from the volume, and
    + // we don't want to require reading only environment files on
    + // the host.
    +
    + tmpDir, err := ioutil.TempDir("", "convey-inject-")
    + if err != nil {
    + return err
    + }
    + defer os.RemoveAll(tmpDir)
    +
    + for _, file := range files {
    + // Export the file into the temp directory and maintain the
    + // structure of the file (for ease of error messages, so we
    + // get the file a/b/c/env instead of env).
    + dest := filepath.Clean(filepath.Join(tmpDir, file))
    +
    + if err = exportFile(name, st.Workspace.Name(), file, dest, st); err != nil {
    + return err
    + }
    +
    + // Process the entries of the file and apply them to the
    + // state's base environment immediately.
    +
    + entries, err := processFile(dest, file, prefix)
    + if err != nil {
    + return err
    + }
    +
    + st.MergeEnv(entries)
    + }
    +
    + return nil
    +}
    +
    +func processFile(path, name, prefix string) ([]string, error) {
    + data, err := ioutil.ReadFile(path)
    + if err != nil {
    + return nil, fmt.Errorf("failed to read environment file '%s'", name)
    + }
    +
    + entries := []string{}
    + for _, line := range strings.Split(string(data), "\n") {
    + // Allow blank lines
    + if len(strings.TrimSpace(line)) == 0 {
    + continue
    + }
    +
    + // Each non-empty line requires the form key=val. Split the
    + // key and the val, then uppercase the key and add the prefix.
    + // We don't care what form val takes, it will be treated as a
    + // string (does not need to be quoted).
    +
    + parts := strings.SplitN(line, "=", 2)
    + if len(parts) != 2 {
    + return nil, fmt.Errorf("malformed entry in environments file '%s'", line)
    + }
    +
    + var (
    + key = strings.TrimSpace(parts[0])
    + val = strings.TrimSpace(parts[1])
    + )
    +
    + if len(key) == 0 {
    + return nil, fmt.Errorf("malformed entry in environments file '%s'", line)
    + }
    +
    + entries = append(entries, fmt.Sprintf("%s%s=%s", prefix, strings.ToUpper(key), val))
    + }
    +
    + return entries, nil
    +}
    +
    +func (i *Inject) New() tasks.Task {
    + return &Inject{}
    +}
    +
    +func (i *Inject) Valid() error {
    + if i.File != "" {
    + i.Files = append([]string{i.File}, i.Files...)
    + }
    +
    + if len(i.Files) == 0 {
    + return errNoFilesInject
    + }
    +
    + return nil
    +}
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/docker/inject_test.go Wed Oct 11 20:48:50 2017 -0500
    @@ -0,0 +1,58 @@
    +/*
    + * Convey
    + * Copyright 2016-2017 Gary Kramlich <grim@reaperworld.com>
    + *
    + * This program is free software: you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License as published by
    + * the Free Software Foundation, either version 3 of the License, or
    + * (at your option) any later version.
    + *
    + * This program is distributed in the hope that it will be useful,
    + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    + * GNU General Public License for more details.
    + *
    + * You should have received a copy of the GNU General Public License
    + * along with this program. If not, see <http://www.gnu.org/licenses/>.
    + */
    +package docker
    +
    +import (
    + "github.com/aphistic/sweet"
    + "github.com/go-yaml/yaml"
    + . "github.com/onsi/gomega"
    +
    + cYaml "bitbucket.org/rw_grim/convey/yaml"
    +)
    +
    +func (s *dockerSuite) TestInject(t sweet.T) {
    + i := &Inject{Files: cYaml.StringOrSlice{"foo"}}
    + Expect(i.Valid()).To(BeNil())
    +}
    +
    +func (s *dockerSuite) TestInjectFilesRequired(t sweet.T) {
    + i := &Inject{}
    + Expect(i.Valid()).To(MatchError(errNoFilesInject))
    +}
    +
    +func (s *dockerSuite) TestInjectUnmarshalString(t sweet.T) {
    + data := `files: filename`
    +
    + imp := Inject{}
    + err := yaml.Unmarshal([]byte(data), &imp)
    +
    + Expect(err).To(BeNil())
    + Expect(imp.Files).To(Equal(cYaml.StringOrSlice{"filename"}))
    +}
    +
    +func (s *dockerSuite) TestInjectUnmarshalStringSlice(t sweet.T) {
    + data := `files:
    + - filename1
    + - filename2`
    +
    + imp := Inject{}
    + err := yaml.Unmarshal([]byte(data), &imp)
    +
    + Expect(err).To(BeNil())
    + Expect(imp.Files).To(Equal(cYaml.StringOrSlice{"filename1", "filename2"}))
    +}
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/examples/inject.yml Wed Oct 11 20:48:50 2017 -0500
    @@ -0,0 +1,56 @@
    +# This plan shows how the inject plan can add additional
    +# environment variables accessible to your tasks during
    +# runtime. Running this with `-e X_FOO=4`, the output
    +# will be (in two parts):
    +#
    +# - X_FOO=4
    +# - X_BAR=5
    +# - X_BAZ=
    +#
    +# - X_FOO=4
    +# - X_BAR=2
    +# - X_BAZ=3
    +#
    +# Notice that before the inject task, only X_FOO and X_BAR
    +# were set by the command line argument and the global env
    +# block, respectively. Afterwards, X_FOO's value did not
    +# change (as command line parameters take have precedence),
    +# X_BAR is overwritten, and X_BAZ is newly added.
    +
    +environment:
    + - X_BAR=5
    +
    +tasks:
    + import:
    + type: import
    + files: .
    +
    + generate:
    + image: gliderlabs/alpine:edge
    + script:
    + - echo "foo=1" >> env
    + - echo "bar=2" >> env
    + - echo "baz=3" >> env
    + workdir: /workspace
    +
    + inject:
    + type: inject
    + files: env
    + prefix: "X_"
    +
    + print:
    + image: gliderlabs/alpine:edge
    + script:
    + - echo "X_FOO=${X_FOO}"
    + - echo "X_BAR=${X_BAR}"
    + - echo "X_BAZ=${X_BAZ}"
    +
    +plans:
    + default:
    + stages:
    + - tasks:
    + - import
    + - generate
    + - print
    + - inject
    + - print
    --- a/intrinsic/intrinsic.go Wed Oct 11 19:40:16 2017 -0500
    +++ b/intrinsic/intrinsic.go Wed Oct 11 20:48:50 2017 -0500
    @@ -25,6 +25,5 @@
    Tasks = map[string]tasks.Task{
    "clean": &Clean{},
    "extend": &Extend{},
    - "inject": &Inject{},
    }
    )
    --- a/state/state.go Wed Oct 11 19:40:16 2017 -0500
    +++ b/state/state.go Wed Oct 11 20:48:50 2017 -0500
    @@ -119,14 +119,16 @@
    return env
    }
    -// MergeEnv adds additional environment arguments to the original state environment.
    +// MergeEnv adds additional environment arguments with lowere precedence to
    +// the original state environment. If they duplicate an existing environment
    +// variable, that value will be unchanged.
    func (st *State) MergeEnv(env []string) {
    if st.parent != nil {
    st.parent.MergeEnv(env)
    return
    }
    - st.innerEnv = environment.Merge(st.innerEnv, env)
    + st.innerEnv = environment.Merge(env, st.innerEnv)
    }
    // MapSlice calls SliceMapper on the given environment, but also checks to