grim/convey

Add indirect env map expansion.
expand-list
2017-10-02, Eric Fritz
cad327117fc4
Parents 2c14c5d09ab5
Children a5aa9f92ae63
Add indirect env map expansion.
4 files changed, 246 insertions(+), 125 deletions(-)
  • +6 -0
    environment/mapper.go
  • +33 -106
    state/state.go
  • +55 -19
    state/state_test.go
  • +152 -0
    state/util.go
    • --- a/environment/mapper.go Mon Oct 02 11:21:46 2017 -0500
    +++ b/environment/mapper.go Mon Oct 02 12:06:42 2017 -0500
    @@ -41,6 +41,12 @@
    return "$" + name
    }
    +// Map will return the value matching a KEY=VAL pair in the given environment.
    +func Map(key string, env []string) string {
    + mapper := envMapper{env}
    + return mapper.Map(key)
    +}
    +
    // Mapper will replace ${TEMPLATE} patterns in the string with the KEY=VAL pairs
    // in the given environment. This function will replace patterns recursively, so
    // if VAL has the form ${TEMPLATE}, it will be replaced again.
    --- a/state/state.go Mon Oct 02 11:21:46 2017 -0500
    +++ b/state/state.go Mon Oct 02 12:06:42 2017 -0500
    @@ -21,7 +21,6 @@
    "errors"
    "fmt"
    "os"
    - "regexp"
    "strings"
    "time"
    @@ -30,6 +29,8 @@
    "bitbucket.org/rw_grim/convey/workspace"
    )
    +const ExpansionLimit = 100
    +
    type State struct {
    Network network.Network
    Workspace workspace.Workspace
    @@ -69,19 +70,42 @@
    // MapSlice calls SliceMapper on the given environment, but also checks to
    // see if the variable in the env parameter can be expanded into a list.
    func (st *State) MapSlice(env, fullEnv []string) ([]string, error) {
    - clean := []string{}
    + prev := env
    +
    + // Protect ourselves against a weird infinite expansion. This can
    + // happen if something occurs like X => A,$Y; Y => B,$X. This is
    + // never a useful case - a high expansion limit should catch this
    + // without ever hitting a practical edge.
    +
    + for i := 0; i < ExpansionLimit; i++ {
    + next, err := st.expandSlice(prev, fullEnv)
    + if err != nil {
    + return nil, err
    + }
    - // TODO - make iterative
    + // If we haven't made a change, return the final result.
    + if isSame(next, prev) {
    + return next, nil
    + }
    +
    + prev = next
    + }
    +
    + return nil, fmt.Errorf("hit limit while expanding '%s'", env)
    +}
    +
    +func (st *State) expandSlice(env, fullEnv []string) ([]string, error) {
    + all := []string{}
    for _, data := range env {
    expanded, err := st.expand(data, fullEnv)
    if err != nil {
    return nil, err
    }
    - clean = append(clean, expanded...)
    + all = append(all, expanded...)
    }
    - return clean, nil
    + return removeDuplicates(all), nil
    }
    // expand will attempt to expand any variables stored on the state expand
    @@ -96,12 +120,7 @@
    for _, name := range getNames(data) {
    if delimiter, ok := st.getDelimiter(name); ok {
    - val, err := environment.Mapper("$"+name, fullEnv)
    - if err != nil {
    - return nil, err
    - }
    -
    - expansions[name] = strings.SplitN(val, delimiter, -1)
    + expansions[name] = strings.SplitN(environment.Map(name, fullEnv), delimiter, -1)
    }
    }
    @@ -119,8 +138,8 @@
    }
    // Construct the actual values. Product gives us a map of all possible
    - // combinations of expanded values. For each one, overwrite the vars
    - // in the full env and apply the map.
    + // combinations of expanded values. For each one, map using ONLY the
    + // expanded vars. Additional mapping will apply on the next iteration.
    values := []string{}
    for _, m := range product(expansions) {
    @@ -129,7 +148,7 @@
    env = append(env, fmt.Sprintf("%s=%s", k, v))
    }
    - val, err := environment.Mapper(data, environment.Merge(fullEnv, env))
    + val, err := environment.Mapper(data, env)
    if err != nil {
    return nil, err
    }
    @@ -184,95 +203,3 @@
    return nil
    }
    -
    -var re1 = regexp.MustCompile("\\$[A-Za-z0-9_]+")
    -var re2 = regexp.MustCompile("\\${[^}]+}")
    -
    -// getNames extracts all variables with the format $var or ${var} from
    -// the given string.
    -func getNames(env string) []string {
    - names := []string{}
    -
    - for _, match := range re1.FindAllString(env, -1) {
    - // Remove leading $
    - names = append(names, match[1:])
    - }
    -
    - for _, match := range re2.FindAllString(env, -1) {
    - // Remove leading ${ and trailing }
    - names = append(names, match[2:len(match)-1])
    - }
    -
    - return names
    -}
    -
    -// product will return the cartesian product of the input map. For
    -// example, if the input map is A => 1, 2, 3; B => 4, 5, 6, then
    -// the list of outputs will contain the following:
    -// - A => 1; B => 4
    -// - A => 1; B => 5
    -// - A => 1; B => 6
    -// - A => 2; B => 4
    -// - A => 2; B => 5
    -// - A => 2; B => 6
    -// - A => 3; B => 4
    -// - A => 3; B => 5
    -// - A => 3; B => 6
    -func product(expansions map[string][]string) []map[string]string {
    - var (
    - order = []string{}
    - indices = make([]int, len(expansions))
    - maps = []map[string]string{}
    - )
    -
    - if len(expansions) == 0 {
    - return nil
    - }
    -
    - // Create a stable order of keys - this will determine our
    - // iteration order (in the above [A B] is the list as the
    - // index for B will increase more frequently than A).
    - for key := range expansions {
    - order = append(order, key)
    - }
    -
    - // The index map starts out as [0 0 ... 0] and will increase
    - // in the following order:
    - // - [0 0 ... 0 1]
    - // - [0 0 ... 0 2]
    - // - [0 0 ... 0 (n-1)]
    - // - ...
    - // - [0 0 ... 1 0]
    - //
    - // Once the first index (which increases the most slowly)
    - // exceeds the length of that slice, we've seen everything.
    -
    - for indices[0] < len(expansions[order[0]]) {
    - // Create a single element from the current state
    - // of the index map.
    -
    - snapshot := map[string]string{}
    - for i, j := range indices {
    - snapshot[order[i]] = expansions[order[i]][j]
    - }
    -
    - maps = append(maps, snapshot)
    -
    - // Advance the index map. Start from the end of the
    - // index map and increment by one. If we end up rolling
    - // over from n - 1 to 0 (where n is the length of the
    - // associated slice), then we also flip the index
    - // immediately to the left.
    -
    - for j := len(indices) - 1; j >= 0; j-- {
    - indices[j]++
    - if j == 0 || indices[j] < len(expansions[order[j]]) {
    - break
    - }
    -
    - indices[j] = 0
    - }
    - }
    -
    - return maps
    -}
    --- a/state/state_test.go Mon Oct 02 11:21:46 2017 -0500
    +++ b/state/state_test.go Mon Oct 02 12:06:42 2017 -0500
    @@ -47,27 +47,28 @@
    return mapped
    }
    - Expect(mapEnv("$X")).To(Equal([]string{"$X"}))
    - Expect(mapEnv("$FOO")).To(Equal([]string{"BAR"}))
    + Expect(mapEnv("$X")).To(ConsistOf([]string{"$X"}))
    + Expect(mapEnv("$FOO")).To(ConsistOf([]string{"BAR"}))
    st.PushExtendFrame([]string{"X=A;B;C"}, []string{"X"}, ";")
    - Expect(mapEnv("$X")).To(Equal([]string{"A", "B", "C"}))
    - Expect(mapEnv("$FOO")).To(Equal([]string{"BAR"}))
    + Expect(mapEnv("$X")).To(ConsistOf([]string{"A", "B", "C"}))
    + Expect(mapEnv("$FOO")).To(ConsistOf([]string{"BAR"}))
    st.PushExtendFrame([]string{"BAR=B;A;R::B;A;Z", "FOO=SAME"}, []string{"BAR"}, "::")
    - Expect(mapEnv("$X")).To(Equal([]string{"A", "B", "C"}))
    - Expect(mapEnv("$FOO")).To(Equal([]string{"BAR"}))
    - Expect(mapEnv("$BAR")).To(Equal([]string{"B;A;R", "B;A;Z"}))
    + Expect(mapEnv("$X")).To(ConsistOf([]string{"A", "B", "C"}))
    + Expect(mapEnv("$FOO")).To(ConsistOf([]string{"BAR"}))
    + Expect(mapEnv("$BAR")).To(ConsistOf([]string{"B;A;R", "B;A;Z"}))
    Expect(st.PopExtendFrame()).To(BeNil())
    Expect(st.PopExtendFrame()).To(BeNil())
    - Expect(mapEnv("$X")).To(Equal([]string{"$X"}))
    - Expect(mapEnv("$FOO")).To(Equal([]string{"BAR"}))
    + Expect(mapEnv("$X")).To(ConsistOf([]string{"$X"}))
    + Expect(mapEnv("$FOO")).To(ConsistOf([]string{"BAR"}))
    }
    func (e *stateSuite) TestMapSliceComplex(t sweet.T) {
    st := &State{}
    st.Environment = []string{"FOO=BAR"}
    + st.PushExtendFrame([]string{"X=A;B;C", "Y=D;E;F"}, []string{"X", "Y"}, ";")
    mapEnv := func(val string) []string {
    mapped, err := st.MapSlice([]string{val}, st.Environment)
    @@ -75,29 +76,64 @@
    return mapped
    }
    - st.PushExtendFrame([]string{"X=A;B;C"}, []string{"X"}, ";")
    - st.PushExtendFrame([]string{"Y=D;E;F"}, []string{"Y"}, ";")
    -
    // No expansion
    - Expect(mapEnv("BONK")).To(Equal([]string{"BONK"}))
    - Expect(mapEnv("$FOO")).To(Equal([]string{"BAR"}))
    + Expect(mapEnv("BONK")).To(ConsistOf([]string{"BONK"}))
    + Expect(mapEnv("$FOO")).To(ConsistOf([]string{"BAR"}))
    // Simple
    - Expect(mapEnv("$X")).To(Equal([]string{"A", "B", "C"}))
    - Expect(mapEnv("$Y")).To(Equal([]string{"D", "E", "F"}))
    + Expect(mapEnv("$X")).To(ConsistOf([]string{"A", "B", "C"}))
    + Expect(mapEnv("$Y")).To(ConsistOf([]string{"D", "E", "F"}))
    // Templated
    - Expect(mapEnv("x${X}x")).To(Equal([]string{"xAx", "xBx", "xCx"}))
    - Expect(mapEnv("x${Y}x")).To(Equal([]string{"xDx", "xEx", "xFx"}))
    + Expect(mapEnv("x${X}x")).To(ConsistOf([]string{"xAx", "xBx", "xCx"}))
    + Expect(mapEnv("x${Y}x")).To(ConsistOf([]string{"xDx", "xEx", "xFx"}))
    // Cartesian product
    - Expect(mapEnv("$X/$Y")).To(Equal([]string{
    + Expect(mapEnv("$X/$Y")).To(ConsistOf([]string{
    "A/D", "A/E", "A/F",
    "B/D", "B/E", "B/F",
    "C/D", "C/E", "C/F",
    }))
    }
    +func (e *stateSuite) TestMapSliceIterative(t sweet.T) {
    + st := &State{}
    + st.Environment = []string{"FOO=BAR"}
    + st.PushExtendFrame([]string{"X=A;x$Y", "Y=B;y$Z", "Z=C;D;E"}, []string{"X", "Y", "Z"}, ";")
    +
    + mapEnv := func(val string) []string {
    + mapped, err := st.MapSlice([]string{val}, st.Environment)
    + Expect(err).To(BeNil())
    + return mapped
    + }
    +
    + Expect(mapEnv("$X")).To(ConsistOf([]string{"A", "xB", "xyC", "xyD", "xyE"}))
    +}
    +
    +func (e *stateSuite) TestMapSliceIndirect(t sweet.T) {
    + st := &State{}
    + st.Environment = []string{"FOO=BAR"}
    + st.PushExtendFrame([]string{"X=$Y", "Y=A;$Z;C", "Z=B;$W", "W=D"}, []string{"X", "Y", "Z"}, ";")
    +
    + mapEnv := func(val string) []string {
    + mapped, err := st.MapSlice([]string{val}, st.Environment)
    + Expect(err).To(BeNil())
    + return mapped
    + }
    +
    + Expect(mapEnv("$X")).To(ConsistOf([]string{"A", "B", "C", "D"}))
    +}
    +
    +func (e *stateSuite) TestMapSliceRecursive(t sweet.T) {
    + st := &State{}
    + st.Environment = []string{"FOO=BAR"}
    + st.PushExtendFrame([]string{"X=A;x$Y", "Y=B;y$Z", "Z=C;D;$X"}, []string{"X", "Y", "Z"}, ";")
    +
    + _, err := st.MapSlice([]string{"$X"}, st.Environment)
    + Expect(err).NotTo(BeNil())
    + Expect(err.Error()).To(ContainSubstring("hit limit"))
    +}
    +
    func (e *stateSuite) TestPopExtendFrameSize(t sweet.T) {
    st := &State{}
    st.PushExtendFrame(nil, nil, "")
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/state/util.go Mon Oct 02 12:06:42 2017 -0500
    @@ -0,0 +1,152 @@
    +/*
    + * Convey
    + * Copyright 2016-2017 Gary Kramlich <grim@reaperworld.com>
    + *
    + * This program is free software: you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License as published by
    + * the Free Software Foundation, either version 3 of the License, or
    + * (at your option) any later version.
    + *
    + * This program is distributed in the hope that it will be useful,
    + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    + * GNU General Public License for more details.
    + *
    + * You should have received a copy of the GNU General Public License
    + * along with this program. If not, see <http://www.gnu.org/licenses/>.
    + */
    +package state
    +
    +import (
    + "regexp"
    +)
    +
    +var re1 = regexp.MustCompile("\\$[A-Za-z0-9_]+")
    +var re2 = regexp.MustCompile("\\${[^}]+}")
    +
    +// getNames extracts all variables with the format $var or ${var} from
    +// the given string.
    +func getNames(env string) []string {
    + names := []string{}
    +
    + for _, match := range re1.FindAllString(env, -1) {
    + // Remove leading $
    + names = append(names, match[1:])
    + }
    +
    + for _, match := range re2.FindAllString(env, -1) {
    + // Remove leading ${ and trailing }
    + names = append(names, match[2:len(match)-1])
    + }
    +
    + return names
    +}
    +
    +// product will return the cartesian product of the input map. For
    +// example, if the input map is A => 1, 2, 3; B => 4, 5, 6, then
    +// the list of outputs will contain the following:
    +// - A => 1; B => 4
    +// - A => 1; B => 5
    +// - A => 1; B => 6
    +// - A => 2; B => 4
    +// - A => 2; B => 5
    +// - A => 2; B => 6
    +// - A => 3; B => 4
    +// - A => 3; B => 5
    +// - A => 3; B => 6
    +func product(expansions map[string][]string) []map[string]string {
    + var (
    + order = []string{}
    + indices = make([]int, len(expansions))
    + maps = []map[string]string{}
    + )
    +
    + if len(expansions) == 0 {
    + return nil
    + }
    +
    + // Create a stable order of keys - this will determine our
    + // iteration order (in the above [A B] is the list as the
    + // index for B will increase more frequently than A).
    + for key := range expansions {
    + order = append(order, key)
    + }
    +
    + // The index map starts out as [0 0 ... 0] and will increase
    + // in the following order:
    + // - [0 0 ... 0 1]
    + // - [0 0 ... 0 2]
    + // - [0 0 ... 0 (n-1)]
    + // - ...
    + // - [0 0 ... 1 0]
    + //
    + // Once the first index (which increases the most slowly)
    + // exceeds the length of that slice, we've seen everything.
    +
    + for indices[0] < len(expansions[order[0]]) {
    + // Create a single element from the current state
    + // of the index map.
    +
    + snapshot := map[string]string{}
    + for i, j := range indices {
    + snapshot[order[i]] = expansions[order[i]][j]
    + }
    +
    + maps = append(maps, snapshot)
    +
    + // Advance the index map. Start from the end of the
    + // index map and increment by one. If we end up rolling
    + // over from n - 1 to 0 (where n is the length of the
    + // associated slice), then we also flip the index
    + // immediately to the left.
    +
    + for j := len(indices) - 1; j >= 0; j-- {
    + indices[j]++
    + if j == 0 || indices[j] < len(expansions[order[j]]) {
    + break
    + }
    +
    + indices[j] = 0
    + }
    + }
    +
    + return maps
    +}
    +
    +// removeDupliates returns a list without duplicate elements.
    +func removeDuplicates(env []string) []string {
    + set := map[string]struct{}{}
    + for _, v := range env {
    + set[v] = struct{}{}
    + }
    +
    + pruned := []string{}
    + for k := range set {
    + pruned = append(pruned, k)
    + }
    +
    + return pruned
    +}
    +
    +// Determine if two slices have the same elements.
    +func isSame(a, b []string) bool {
    + if len(a) != len(b) {
    + return false
    + }
    +
    + for _, val := range a {
    + found := false
    + for _, match := range b {
    + if val == match {
    + found = true
    + break
    + }
    + }
    +
    + if !found {
    + return false
    + }
    + }
    +
    + return true
    +}