Add a crazy recursive parsing test...and fix some bugs

This change completes my testing of decorator parsing for now.  It tests the token
`*[]map[string]map[()*(bool,string,test/package:test/module/Crazy)number][][]test/package:test/module/Crazy`.

This turned up some bugs, most notably in the way we returned the "full" token for
the parsed types.  We need to extract the subset of the token consumed by the parsing
routine, rather than the entire thing.  To do this, we introduce a tokenBuffer type
that allows for convenient parsing of tokens (eating, advancing, extraction, etc).

pkg/tokens/decors.go

@@ -10,34 +10,84 @@ import (
 	"github.com/marapongo/mu/pkg/util/contract"
 )
 
+// tokenBuffer is a parseable token buffer that simply carries a position.
+type tokenBuffer struct {
+	Tok Type
+	Pos int
+}
+
+func newTokenBuffer(tok Type) *tokenBuffer {
+	return &tokenBuffer{
+		Tok: tok,
+		Pos: 0,
+	}
+}
+
+func (b *tokenBuffer) Curr() Type {
+	return b.Tok[b.Pos:]
+}
+
+func (b *tokenBuffer) From(from int) Type {
+	return b.Tok[from:b.Pos]
+}
+
+func (b *tokenBuffer) Eat(s string) {
+	ate := b.MayEat(s)
+	contract.Assertf(ate, "Expected to eat '%v'", s)
+}
+
+func (b *tokenBuffer) MayEat(s string) bool {
+	if strings.HasPrefix(string(b.Curr()), s) {
+		b.Advance(len(s))
+		return true
+	} else {
+		return false
+	}
+}
+
+func (b *tokenBuffer) Advance(by int) {
+	b.Pos += by
+}
+
+func (b *tokenBuffer) Done() bool {
+	return b.Pos == len(b.Tok)
+}
+
+func (b *tokenBuffer) Finish() {
+	b.Pos = len(b.Tok)
+}
+
 // typePartDelims are separator characters that are used to parse recursive types.
 var typePartDelims = MapTypeSeparator + FunctionTypeParamSeparator + FunctionTypeSeparator
 
-// parseNextType parses one type out of the given token, returning both the resulting type token plus the remainder of
+// parseNextType parses one type out of the given token, mutating the buffer in place and returning the resulting type
-// the string.  This allows recursive parsing of complex decorated types below (like `map[[]string]func(func())`).
+// token.  This allows recursive parsing of complex decorated types below (like `map[[]string]func(func())`).
-func parseNextType(tok Type) (Type, string) {
+func parseNextType(b *tokenBuffer) Type {
 	// First, check for decorated types.
+	tok := b.Curr()
 	if tok.Pointer() {
-		ptr, rest := parseNextPointerType(tok)
+		ptr := parseNextPointerType(b)
-		return ptr.Tok, rest
+		return ptr.Tok
 	} else if tok.Array() {
-		arr, rest := parseNextArrayType(tok)
+		arr := parseNextArrayType(b)
-		return arr.Tok, rest
+		return arr.Tok
 	} else if tok.Map() {
-		mam, rest := parseNextMapType(tok)
+		mam := parseNextMapType(b)
-		return mam.Tok, rest
+		return mam.Tok
 	} else if tok.Function() {
-		fnc, rest := parseNextFunctionType(tok)
+		fnc := parseNextFunctionType(b)
-		return fnc.Tok, rest
+		return fnc.Tok
 	} else {
 		// Otherwise, we have either a qualified or simple (primitive) name.  Since we might be deep in the middle
 		// of parsing another token, however, we only parse up to any other decorator termination/separator tokens.
 		s := string(tok)
 		sep := strings.IndexAny(s, typePartDelims)
 		if sep == -1 {
-			return tok, ""
+			b.Finish()
+			return tok
 		} else {
-			return tok[:sep], s[sep:]
+			b.Advance(sep)
+			return tok[:sep]
 		}
 	}
 }
@@ -70,17 +120,21 @@ func IsPointerType(tok Type) bool {
 
 // ParsePointerType removes the pointer decorations from a token and returns its underlying type.
 func ParsePointerType(tok Type) PointerType {
-	ptr, extra := parseNextPointerType(tok)
+	b := newTokenBuffer(tok)
-	contract.Assertf(extra == "", "Did not expect anything extra after the pointer type %v; got: '%v'", tok, extra)
+	ptr := parseNextPointerType(b)
+	if !b.Done() {
+		contract.Failf("Did not expect anything extra after the pointer type %v; got: '%v'", tok, b.Curr())
+	}
 	return ptr
 }
 
-// parseNextPointerType parses the next pointer type from the given token, returning any excess.
+// parseNextPointerType parses the next pointer type from the given buffer.
-func parseNextPointerType(tok Type) (PointerType, string) {
+func parseNextPointerType(b *tokenBuffer) PointerType {
-	contract.Requiref(IsPointerType(tok), "tok", "IsPointerType")
+	mark := b.Pos            // remember where this token begins.
-	rest := string(tok)[len(PointerTypePrefix):]
+	b.Eat(PointerTypePrefix) // eat the "*" part.
-	elem, rest := parseNextType(Type(rest))
+	elem := parseNextType(b) // parse the required element type token.
-	return PointerType{tok, elem}, rest
+	contract.Assert(elem != "")
+	return PointerType{Tok: b.From(mark), Elem: elem}
 }
 
 // ArrayType is a type token that decorates an element type token to turn it into an array: `"[]" <Elem>`.
@@ -111,17 +165,21 @@ func IsArrayType(tok Type) bool {
 
 // ParseArrayType removes the array decorations from a token and returns its underlying type.
 func ParseArrayType(tok Type) ArrayType {
-	ptr, extra := parseNextArrayType(tok)
+	b := newTokenBuffer(tok)
-	contract.Assertf(extra == "", "Did not expect anything extra after the array type %v; got: '%v'", tok, extra)
+	arr := parseNextArrayType(b)
-	return ptr
+	if !b.Done() {
+		contract.Failf("Did not expect anything extra after the array type %v; got: '%v'", tok, b.Curr())
+	}
+	return arr
 }
 
-// parseNextArrayType parses the next array type from the given token, returning any excess.
+// parseNextArrayType parses the next array type from the given buffer.
-func parseNextArrayType(tok Type) (ArrayType, string) {
+func parseNextArrayType(b *tokenBuffer) ArrayType {
-	contract.Requiref(IsArrayType(tok), "tok", "IsArrayType")
+	mark := b.Pos            // remember where this token begins.
-	rest := string(tok)[len(ArrayTypePrefix):]
+	b.Eat(ArrayTypePrefix)   // eat the "[]" part.
-	elem, rest := parseNextType(Type(rest))
+	elem := parseNextType(b) // parse the required element type token.
-	return ArrayType{tok, elem}, rest
+	contract.Assert(elem != "")
+	return ArrayType{Tok: b.From(mark), Elem: elem}
 }
 
 // MapType is a type token that decorates a key and element type token to turn them into a map: `"map[" <Key> "]" <Elem>`.
@@ -154,28 +212,30 @@ func IsMapType(tok Type) bool {
 
 // ParseMapType removes the map decorations from a token and returns its underlying type.
 func ParseMapType(tok Type) MapType {
-	ptr, extra := parseNextMapType(tok)
+	b := newTokenBuffer(tok)
-	contract.Assertf(extra == "", "Did not expect anything extra after the map type %v; got: '%v'", tok, extra)
+	mam := parseNextMapType(b)
-	return ptr
+	if !b.Done() {
+		contract.Failf("Did not expect anything extra after the map type %v; got: '%v'", tok, b.Curr())
+	}
+	return mam
 }
 
-// parseNextMapType parses the next map type from the given token, returning any excess.
+// parseNextMapType parses the next map type from the given buffer.
-func parseNextMapType(tok Type) (MapType, string) {
+func parseNextMapType(b *tokenBuffer) MapType {
-	contract.Requiref(IsMapType(tok), "tok", "IsMapType")
+	mark := b.Pos        // remember where this token begins.
-
+	b.Eat(MapTypePrefix) // eat the "map[" prefix.
-	// Strip off the "map[" part.
-	rest := string(tok)[len(MapTypePrefix):]
 
 	// Now parse the key part.
-	key, rest := parseNextType(Type(rest))
+	key := parseNextType(b)
+	contract.Assert(key != "")
 
 	// Next, we expect to find the "]" separator token; eat it.
-	contract.Assertf(len(rest) > 0 && strings.HasPrefix(rest, MapTypeSeparator), "Expected a map separator")
+	b.Eat(MapTypeSeparator)
-	rest = rest[1:]
 
 	// Next, parse the element type part.
-	elem, rest := parseNextType(Type(rest))
+	elem := parseNextType(b)
-	return MapType{tok, key, elem}, rest
+	contract.Assert(elem != "")
+	return MapType{Tok: b.From(mark), Key: key, Elem: elem}
 }
 
 // FunctionType is a type token that decorates a set of optional parameter and return tokens to turn them into a function
@@ -240,47 +300,45 @@ func IsFunctionType(tok Type) bool {
 
 // ParseFunctionType removes the function decorations from a token and returns its underlying type.
 func ParseFunctionType(tok Type) FunctionType {
-	ptr, extra := parseNextFunctionType(tok)
+	b := newTokenBuffer(tok)
-	contract.Assertf(extra == "", "Did not expect anything extra after the function type %v; got: '%v'", tok, extra)
+	fnc := parseNextFunctionType(b)
-	return ptr
+	if !b.Done() {
+		contract.Failf("Did not expect anything extra after the function type %v; got: '%v'", tok, b.Curr())
+	}
+	return fnc
 }
 
+// funcDelims are the set of characters that might delimit function type parameters.
+var funcDelims = FunctionTypeParamSeparator + FunctionTypeSeparator
+
 // parseNextFunctionType parses the next function type from the given token, returning any excess.
-func parseNextFunctionType(tok Type) (FunctionType, string) {
+func parseNextFunctionType(b *tokenBuffer) FunctionType {
-	contract.Requiref(IsFunctionType(tok), "tok", "IsFunctionType")
+	mark := b.Pos             // remember the start of this token.
-
+	b.Eat(FunctionTypePrefix) // eat the function prefix "(".
-	// Strip off the "(" part.
-	rest := string(tok)[len(FunctionTypePrefix):]
 
+	// Parse out parameters until we encounter and eat a ")".
 	var params []Type
-	for {
+	for !b.MayEat(FunctionTypeSeparator) {
-		if comma := strings.Index(rest, FunctionTypeParamSeparator); comma != -1 {
+		next := parseNextType(b)
-			// More parameters are coming... parse up to the comma.
+		if next == "" {
-			params = append(params, Type(rest[:comma]))
+			contract.Assert(strings.HasPrefix(string(b.Curr()), FunctionTypeSeparator))
-			rest = rest[comma+1:]
-			contract.Assert(len(rest) > 0 && !strings.HasPrefix(rest, FunctionTypeSeparator))
 		} else {
-			// The end is in sight.  Maybe there's more, maybe not.
+			params = append(params, next)
-			if term := strings.Index(rest, FunctionTypeSeparator); term != 0 {
+
-				params = append(params, Type(rest[:term]))
+			// Eat the separator, if any, and keep going.
-				rest = rest[term:]
+			if !b.MayEat(FunctionTypeParamSeparator) {
+				contract.Assert(strings.HasPrefix(string(b.Curr()), FunctionTypeSeparator))
 			}
-			break
 		}
 	}
 
-	// Next, we expect to find the ")" separator token; eat it.
-	contract.Assertf(
-		len(rest) > 0 && strings.HasPrefix(rest, FunctionTypeSeparator), "Expected a function separator")
-	rest = rest[1:]
-
 	// Next, if there is anything remaining, parse out the return type.
 	var ret *Type
-	if rest != "" {
+	if !b.Done() {
-		var rett Type
+		if rett := parseNextType(b); rett != "" {
-		rett, rest = parseNextType(Type(rest))
+			ret = &rett
-		ret = &rett
+		}
 	}
 
-	return FunctionType{tok, params, ret}, rest
+	return FunctionType{Tok: b.From(mark), Parameters: params, Return: ret}
 }

pkg/tokens/decors_test.go

@@ -131,3 +131,98 @@ func TestFunctionTypes(t *testing.T) {
 		}
 	}
 }
+
+func TestComplexTypes(t *testing.T) {
+	// Create a crazy nested type and make sure they parse correctly; essentially:
+	//		*[]map[string]map[()*(bool,string,Crazy)number][][]Crazy
+	// or, in the fully qualified form:
+	//		*[]map[string]map[()*(bool,string,test/package:test/module/Crazy)number][][]test/package:test/module/Crazy
+	// which should parse as
+	//		Pointer
+	//			Elem=Array
+	//				Elem=Map
+	//					Key=string
+	//					Elem=Map
+	//						Key=Func
+	//							Params=()
+	//							Return=Pointer
+	//								Func
+	//									Params=
+	//										bool
+	//										string
+	//										Crazy
+	//									Return=number
+	//						Elem=Array
+	//							Elem=Array
+	//								Elem=Crazy
+	crazy := newTestTypeToken("Crazy")
+
+	number := Type("number")
+	ptrret := NewPointerTypeToken(NewFunctionTypeToken([]Type{"bool", "string", crazy}, &number))
+
+	ptr := NewPointerTypeToken(
+		NewArrayTypeToken(
+			NewMapTypeToken(
+				Type("string"),
+				NewMapTypeToken(
+					NewFunctionTypeToken(
+						[]Type{},
+						&ptrret,
+					),
+					NewArrayTypeToken(
+						NewArrayTypeToken(
+							crazy,
+						),
+					),
+				),
+			),
+		),
+	)
+
+	assert.True(t, ptr.Pointer(), "Expected pointer type token to be an pointer")
+	p1 := ParsePointerType(ptr) // Pointer<Array>
+	{
+		assert.True(t, p1.Elem.Array())
+		p2 := ParseArrayType(p1.Elem) // Array<Map>
+		{
+			assert.True(t, p2.Elem.Map())
+			p3 := ParseMapType(p2.Elem) // Map<string, Map>
+			{
+				assert.Equal(t, "string", string(p3.Key))
+				assert.True(t, p3.Elem.Map())
+				p4 := ParseMapType(p3.Elem) // Map<Func, Array>
+				{
+					assert.True(t, p4.Key.Function())
+					p5 := ParseFunctionType(p4.Key) // Func<(), Pointer>
+					{
+						assert.Equal(t, 0, len(p5.Parameters))
+						assert.NotNil(t, p5.Return)
+						assert.True(t, (*p5.Return).Pointer())
+						p6 := ParsePointerType(*p5.Return) // Pointer<Func>
+						{
+							assert.True(t, p6.Elem.Function())
+							p7 := ParseFunctionType(p6.Elem) // Func<(bool,string,Crazy), number>
+							{
+								assert.Equal(t, 3, len(p7.Parameters))
+								assert.Equal(t, "bool", string(p7.Parameters[0]))
+								assert.Equal(t, "string", string(p7.Parameters[1]))
+								assert.Equal(t, string(crazy), string(p7.Parameters[2]))
+								assert.NotNil(t, p7.Return)
+								assert.Equal(t, "number", string(*p7.Return))
+							}
+						}
+					}
+					assert.True(t, p4.Elem.Array())
+					p8 := ParseArrayType(p4.Elem) // Array<Array>
+					{
+						assert.True(t, p8.Elem.Array())
+						p9 := ParseArrayType(p8.Elem) // Array<Crazy>
+						{
+							assert.Equal(t, string(crazy), string(p9.Elem))
+						}
+					}
+				}
+			}
+		}
+	}
+}