grim/convey

Merged in efritz/convey/expand-list (pull request #26)

2017-10-04, Gary Kramlich
3263ee6583bd
Parents 46472d8da780ee97ff3b6550
Children 7a5448099486
Merged in efritz/convey/expand-list (pull request #26)

Expand list

Approved-by: Gary Kramlich
4 files changed, 338 insertions(+), 44 deletions(-)
  • +6 -0
    environment/mapper.go
  • +86 -24
    state/state.go
  • +94 -20
    state/state_test.go
  • +152 -0
    state/util.go
    • --- a/environment/mapper.go Tue Oct 03 21:33:29 2017 -0500
    +++ b/environment/mapper.go Wed Oct 04 03:27:13 2017 +0000
    @@ -41,6 +41,12 @@
    return "$" + name
    }
    +// Map will return the value matching a KEY=VAL pair in the given environment.
    +func Map(key string, env []string) string {
    + mapper := envMapper{env}
    + return mapper.Map(key)
    +}
    +
    // Mapper will replace ${TEMPLATE} patterns in the string with the KEY=VAL pairs
    // in the given environment. This function will replace patterns recursively, so
    // if VAL has the form ${TEMPLATE}, it will be replaced again.
    --- a/state/state.go Tue Oct 03 21:33:29 2017 -0500
    +++ b/state/state.go Wed Oct 04 03:27:13 2017 +0000
    @@ -29,6 +29,8 @@
    "bitbucket.org/rw_grim/convey/workspace"
    )
    +const ExpansionLimit = 100
    +
    type State struct {
    Network network.Network
    Workspace workspace.Workspace
    @@ -84,23 +86,95 @@
    }
    // MapSlice calls SliceMapper on the given environment, but also checks to
    -// see if the variable could be expanded into a list.
    +// see if the variable in the env parameter can be expanded into a list.
    func (st *State) MapSlice(env, fullEnv []string) ([]string, error) {
    - if len(env) == 1 {
    - // If we only have one thing and it looks like $VAR or ${VAR}, then
    - // see if we have it in the list of things we can expand on the stack.
    - if delimiter, ok := st.getDelimiter(getName(env[0])); ok {
    - mapped, err := environment.Mapper(env[0], fullEnv)
    - if err != nil {
    - return nil, err
    - }
    + prev := env
    +
    + // Protect ourselves against a weird infinite expansion. This can
    + // happen if something occurs like X => A,$Y; Y => B,$X. This is
    + // never a useful case - a high expansion limit should catch this
    + // without ever hitting a practical edge.
    +
    + for i := 0; i < ExpansionLimit; i++ {
    + next, err := st.expandSlice(prev, fullEnv)
    + if err != nil {
    + return nil, err
    + }
    +
    + // If we haven't made a change, return the final result.
    + if isSame(next, prev) {
    + return next, nil
    + }
    +
    + prev = next
    + }
    +
    + return nil, fmt.Errorf("hit limit while expanding '%s'", env)
    +}
    - // Split it!
    - return strings.SplitN(mapped, delimiter, -1), nil
    +func (st *State) expandSlice(env, fullEnv []string) ([]string, error) {
    + all := []string{}
    + for _, data := range env {
    + expanded, err := st.expand(data, fullEnv)
    + if err != nil {
    + return nil, err
    + }
    +
    + all = append(all, expanded...)
    + }
    +
    + return removeDuplicates(all), nil
    +}
    +
    +// expand will attempt to expand any variables stored on the state expand
    +// stack. If no expandable variables are found, then the standard mapper
    +// is used. If multiple expandable variables are around, the mapper will
    +// be applied to the cartesian product of the variables.
    +func (st *State) expand(data string, fullEnv []string) ([]string, error) {
    + expansions := map[string][]string{}
    +
    + // First, extract all of the expandable names and get their values
    + // in the full environment and split them by the proper delimiter.
    +
    + for _, name := range getNames(data) {
    + if delimiter, ok := st.getDelimiter(name); ok {
    + expansions[name] = strings.SplitN(environment.Map(name, fullEnv), delimiter, -1)
    }
    }
    - return environment.SliceMapper(env, fullEnv)
    + // If we don't have any expandable variables, just use the standard
    + // mapper. If we don't do this here we won't return anything useful
    + // as product will return a nil map.
    +
    + if len(expansions) == 0 {
    + mapped, err := environment.Mapper(data, fullEnv)
    + if err != nil {
    + return nil, err
    + }
    +
    + return []string{mapped}, nil
    + }
    +
    + // Construct the actual values. Product gives us a map of all possible
    + // combinations of expanded values. For each one, map using ONLY the
    + // expanded vars. Additional mapping will apply on the next iteration.
    +
    + values := []string{}
    + for _, m := range product(expansions) {
    + env := []string{}
    + for k, v := range m {
    + env = append(env, fmt.Sprintf("%s=%s", k, v))
    + }
    +
    + val, err := environment.Mapper(data, env)
    + if err != nil {
    + return nil, err
    + }
    +
    + values = append(values, val)
    + }
    +
    + return values, nil
    }
    // GetDelimiter returns the highest (outermost extend task) delimiter registered
    @@ -178,15 +252,3 @@
    return names
    }
    -
    -// getName returns the name of the variable `var` if the string is of the
    -// form $var or ${var}.
    -func getName(env string) string {
    - env = strings.TrimSpace(env)
    -
    - if strings.HasPrefix(env, "$") {
    - return strings.Trim(env[1:], "{}")
    - }
    -
    - return ""
    -}
    --- a/state/state_test.go Tue Oct 03 21:33:29 2017 -0500
    +++ b/state/state_test.go Wed Oct 04 03:27:13 2017 +0000
    @@ -37,27 +37,101 @@
    })
    }
    -func (s *stateSuite) TestMap(t sweet.T) {
    - mapEnv := func(st *State, val string) []string {
    - mapped, err := st.MapSlice([]string{val}, st.Environment)
    - Expect(err).To(BeNil())
    - return mapped
    - }
    -
    +func (s *stateSuite) TestMapSlice(t sweet.T) {
    st1 := &State{}
    st1.Environment = []string{"FOO=BAR"}
    - Expect(mapEnv(st1, "$X")).To(Equal([]string{"$X"}))
    - Expect(mapEnv(st1, "$FOO")).To(Equal([]string{"BAR"}))
    + Expect(mapEnv(st1, "$X")).To(ConsistOf([]string{"$X"}))
    + Expect(mapEnv(st1, "$FOO")).To(ConsistOf([]string{"BAR"}))
    st2 := st1.WrapWithExpandableEnv([]string{"X=A;B;C"}, []string{"X"}, ";")
    - Expect(mapEnv(st2, "$X")).To(Equal([]string{"A", "B", "C"}))
    - Expect(mapEnv(st2, "$FOO")).To(Equal([]string{"BAR"}))
    + Expect(mapEnv(st2, "$X")).To(ConsistOf([]string{"A", "B", "C"}))
    + Expect(mapEnv(st2, "$FOO")).To(ConsistOf([]string{"BAR"}))
    st3 := st2.WrapWithExpandableEnv([]string{"BAR=B;A;R::B;A;Z", "FOO=SAME"}, []string{"BAR"}, "::")
    - Expect(mapEnv(st3, "$X")).To(Equal([]string{"A", "B", "C"}))
    - Expect(mapEnv(st3, "$FOO")).To(Equal([]string{"BAR"}))
    - Expect(mapEnv(st3, "$BAR")).To(Equal([]string{"B;A;R", "B;A;Z"}))
    + Expect(mapEnv(st3, "$X")).To(ConsistOf([]string{"A", "B", "C"}))
    + Expect(mapEnv(st3, "$FOO")).To(ConsistOf([]string{"BAR"}))
    + Expect(mapEnv(st3, "$BAR")).To(ConsistOf([]string{"B;A;R", "B;A;Z"}))
    +}
    +
    +func (s *stateSuite) TestMapSliceComplex(t sweet.T) {
    + st := &State{}
    + st.Environment = []string{"FOO=BAR"}
    + st = st.WrapWithExpandableEnv([]string{"X=A;B;C", "Y=D;E;F"}, []string{"X", "Y"}, ";")
    +
    + // No expansion
    + Expect(mapEnv(st, "BONK")).To(ConsistOf([]string{"BONK"}))
    + Expect(mapEnv(st, "$FOO")).To(ConsistOf([]string{"BAR"}))
    +
    + // Simple
    + Expect(mapEnv(st, "$X")).To(ConsistOf([]string{"A", "B", "C"}))
    + Expect(mapEnv(st, "$Y")).To(ConsistOf([]string{"D", "E", "F"}))
    +
    + // Templated
    + Expect(mapEnv(st, "x${X}x")).To(ConsistOf([]string{"xAx", "xBx", "xCx"}))
    + Expect(mapEnv(st, "x${Y}x")).To(ConsistOf([]string{"xDx", "xEx", "xFx"}))
    +
    + // Cartesian product
    + Expect(mapEnv(st, "$X/$Y")).To(ConsistOf([]string{
    + "A/D", "A/E", "A/F",
    + "B/D", "B/E", "B/F",
    + "C/D", "C/E", "C/F",
    + }))
    +}
    +
    +func (s *stateSuite) TestMapSliceIterative(t sweet.T) {
    + st := &State{}
    + st.Environment = []string{"FOO=BAR"}
    + st = st.WrapWithExpandableEnv([]string{"X=A;x$Y", "Y=B;y$Z", "Z=C;D;E"}, []string{"X", "Y", "Z"}, ";")
    +
    + Expect(mapEnv(st, "$X")).To(ConsistOf([]string{"A", "xB", "xyC", "xyD", "xyE"}))
    +}
    +
    +func (s *stateSuite) TestMapSliceIndirect(t sweet.T) {
    + st := &State{}
    + st.Environment = []string{"FOO=BAR"}
    + st = st.WrapWithExpandableEnv([]string{"X=$Y", "Y=A;$Z;C", "Z=B;$W", "W=D"}, []string{"X", "Y", "Z"}, ";")
    +
    + Expect(mapEnv(st, "$X")).To(ConsistOf([]string{"A", "B", "C", "D"}))
    +}
    +
    +func (s *stateSuite) TestMapSliceRecursive(t sweet.T) {
    + st := &State{}
    + st.Environment = []string{"FOO=BAR"}
    + st = st.WrapWithExpandableEnv([]string{"X=A;x$Y", "Y=B;y$Z", "Z=C;D;$X"}, []string{"X", "Y", "Z"}, ";")
    +
    + _, err := st.MapSlice([]string{"$X"}, st.Environment)
    + Expect(err).NotTo(BeNil())
    + Expect(err.Error()).To(ContainSubstring("hit limit"))
    +}
    +
    +func (e *stateSuite) TestGetNames(t sweet.T) {
    + names := getNames("foo $bar ${baz} ${bar} $bonk_quux honk")
    + Expect(names).To(HaveLen(4))
    + Expect(names).To(ConsistOf([]string{"bar", "baz", "bar", "bonk_quux"}))
    +}
    +
    +func (e *stateSuite) TestProduct(t sweet.T) {
    + p := product(map[string][]string{
    + "a": []string{"1", "2", "3"},
    + "b": []string{"4", "5"},
    + "c": []string{"6", "7"},
    + })
    +
    + Expect(p).To(ConsistOf([]map[string]string{
    + map[string]string{"a": "1", "b": "4", "c": "6"},
    + map[string]string{"a": "1", "b": "4", "c": "7"},
    + map[string]string{"a": "1", "b": "5", "c": "6"},
    + map[string]string{"a": "1", "b": "5", "c": "7"},
    + map[string]string{"a": "2", "b": "4", "c": "6"},
    + map[string]string{"a": "2", "b": "4", "c": "7"},
    + map[string]string{"a": "2", "b": "5", "c": "6"},
    + map[string]string{"a": "2", "b": "5", "c": "7"},
    + map[string]string{"a": "3", "b": "4", "c": "6"},
    + map[string]string{"a": "3", "b": "4", "c": "7"},
    + map[string]string{"a": "3", "b": "5", "c": "6"},
    + map[string]string{"a": "3", "b": "5", "c": "7"},
    + }))
    }
    func (s *stateSuite) TestWrapParent(t sweet.T) {
    @@ -80,12 +154,6 @@
    Expect(st2.Environment).To(ConsistOf([]string{"FOO=BAR", "BAR=BAZ", "BAZ=BONK"}))
    }
    -func (s *stateSuite) TestGetName(t sweet.T) {
    - Expect(getName("foo")).To(Equal(""))
    - Expect(getName("$FOO")).To(Equal("FOO"))
    - Expect(getName("${FOO}")).To(Equal("FOO"))
    -}
    -
    func (s *stateSuite) TestDetached(t sweet.T) {
    st1 := New()
    Expect(st1.GetDetached()).To(BeEmpty())
    @@ -103,3 +171,9 @@
    Expect(st1.GetDetached()).To(ConsistOf([]string{"foo", "bar"}))
    }
    +
    +func mapEnv(st *State, val string) []string {
    + mapped, err := st.MapSlice([]string{val}, st.Environment)
    + Expect(err).To(BeNil())
    + return mapped
    +}
    --- /dev/null Thu Jan 01 00:00:00 1970 +0000
    +++ b/state/util.go Wed Oct 04 03:27:13 2017 +0000
    @@ -0,0 +1,152 @@
    +/*
    + * Convey
    + * Copyright 2016-2017 Gary Kramlich <grim@reaperworld.com>
    + *
    + * This program is free software: you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License as published by
    + * the Free Software Foundation, either version 3 of the License, or
    + * (at your option) any later version.
    + *
    + * This program is distributed in the hope that it will be useful,
    + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    + * GNU General Public License for more details.
    + *
    + * You should have received a copy of the GNU General Public License
    + * along with this program. If not, see <http://www.gnu.org/licenses/>.
    + */
    +package state
    +
    +import (
    + "regexp"
    +)
    +
    +var re1 = regexp.MustCompile("\\$[A-Za-z_][A-Za-z0-9_]*")
    +var re2 = regexp.MustCompile("\\${[A-Za-z_][A-Za-z0-9_]*}")
    +
    +// getNames extracts all variables with the format $var or ${var} from
    +// the given string.
    +func getNames(env string) []string {
    + names := []string{}
    +
    + for _, match := range re1.FindAllString(env, -1) {
    + // Remove leading $
    + names = append(names, match[1:])
    + }
    +
    + for _, match := range re2.FindAllString(env, -1) {
    + // Remove leading ${ and trailing }
    + names = append(names, match[2:len(match)-1])
    + }
    +
    + return names
    +}
    +
    +// product will return the cartesian product of the input map. For
    +// example, if the input map is A => 1, 2, 3; B => 4, 5, 6, then
    +// the list of outputs will contain the following:
    +// - A => 1; B => 4
    +// - A => 1; B => 5
    +// - A => 1; B => 6
    +// - A => 2; B => 4
    +// - A => 2; B => 5
    +// - A => 2; B => 6
    +// - A => 3; B => 4
    +// - A => 3; B => 5
    +// - A => 3; B => 6
    +func product(expansions map[string][]string) []map[string]string {
    + var (
    + order = []string{}
    + indices = make([]int, len(expansions))
    + maps = []map[string]string{}
    + )
    +
    + if len(expansions) == 0 {
    + return nil
    + }
    +
    + // Create a stable order of keys - this will determine our
    + // iteration order (in the above [A B] is the list as the
    + // index for B will increase more frequently than A).
    + for key := range expansions {
    + order = append(order, key)
    + }
    +
    + // The index map starts out as [0 0 ... 0] and will increase
    + // in the following order:
    + // - [0 0 ... 0 1]
    + // - [0 0 ... 0 2]
    + // - [0 0 ... 0 (n-1)]
    + // - ...
    + // - [0 0 ... 1 0]
    + //
    + // Once the first index (which increases the most slowly)
    + // exceeds the length of that slice, we've seen everything.
    +
    + for indices[0] < len(expansions[order[0]]) {
    + // Create a single element from the current state
    + // of the index map.
    +
    + snapshot := map[string]string{}
    + for i, j := range indices {
    + snapshot[order[i]] = expansions[order[i]][j]
    + }
    +
    + maps = append(maps, snapshot)
    +
    + // Advance the index map. Start from the end of the
    + // index map and increment by one. If we end up rolling
    + // over from n - 1 to 0 (where n is the length of the
    + // associated slice), then we also flip the index
    + // immediately to the left.
    +
    + for j := len(indices) - 1; j >= 0; j-- {
    + indices[j]++
    + if j == 0 || indices[j] < len(expansions[order[j]]) {
    + break
    + }
    +
    + indices[j] = 0
    + }
    + }
    +
    + return maps
    +}
    +
    +// removeDupliates returns a list without duplicate elements.
    +func removeDuplicates(env []string) []string {
    + set := map[string]struct{}{}
    + for _, v := range env {
    + set[v] = struct{}{}
    + }
    +
    + pruned := []string{}
    + for k := range set {
    + pruned = append(pruned, k)
    + }
    +
    + return pruned
    +}
    +
    +// Determine if two slices have the same elements.
    +func isSame(a, b []string) bool {
    + if len(a) != len(b) {
    + return false
    + }
    +
    + for _, val := range a {
    + found := false
    + for _, match := range b {
    + if val == match {
    + found = true
    + break
    + }
    + }
    +
    + if !found {
    + return false
    + }
    + }
    +
    + return true
    +}