8fded5f29f
Token detection was broken for conditional and for expressions that
represent template control sequences. The code originally attempted to
determine whether or not a conditional or for expression was a control
sequence by inspecting the expression's parent. Unfortunately, that
approach is unable to distinguish between expressions that are control
sequences and those that are merely template parts. These changes
instead inspect the first token of the expression for a template control
token (i.e. `%{`): if such a token is found, the expression is detected
as a template control sequence.
830 lines
26 KiB
Go
830 lines
26 KiB
Go
// Copyright 2016-2020, Pulumi Corporation.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package syntax
|
|
|
|
import (
|
|
"bytes"
|
|
"regexp"
|
|
"strings"
|
|
|
|
"github.com/hashicorp/hcl/v2"
|
|
"github.com/hashicorp/hcl/v2/hclsyntax"
|
|
"github.com/pulumi/pulumi/pkg/v2/codegen"
|
|
"github.com/pulumi/pulumi/sdk/v2/go/common/util/contract"
|
|
)
|
|
|
|
// tokenList is a list of Tokens with methods to aid in mapping source positions to tokens.
|
|
type tokenList []Token
|
|
|
|
// offsetIndex returns the index of the token that contains the given byte offset or -1 if no such token exists.
|
|
func (l tokenList) offsetIndex(offset int) int {
|
|
base := 0
|
|
for len(l) > 0 {
|
|
i := len(l) / 2
|
|
r := l[i].Range()
|
|
switch {
|
|
case offset < r.Start.Byte:
|
|
l = l[:i]
|
|
case r.Start.Byte <= offset && offset < r.End.Byte:
|
|
return base + i
|
|
case r.End.Byte <= offset:
|
|
l, base = l[i+1:], base+i+1
|
|
default:
|
|
contract.Failf("unexpected index condition: %v, %v, %v", r.Start.Byte, r.End.Byte, offset)
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
// atOffset returns the token that contains the given byte offset or the zero value if no such token exists.
|
|
func (l tokenList) atOffset(offset int) Token {
|
|
if i := l.offsetIndex(offset); i >= 0 {
|
|
return l[i]
|
|
}
|
|
return Token{}
|
|
}
|
|
|
|
// atPos returns the token that contains the given hcl.Pos or the zero value if no such token exists.
|
|
func (l tokenList) atPos(p hcl.Pos) Token {
|
|
return l.atOffset(p.Byte)
|
|
}
|
|
|
|
// inRange returns a slice of the tokens that cover the given range or nil if either the start or end position is
|
|
// uncovered by a token.
|
|
func (l tokenList) inRange(r hcl.Range) []Token {
|
|
// If the range is empty, ignore it.
|
|
if r.Empty() {
|
|
return nil
|
|
}
|
|
|
|
// Find the index of the start and end tokens for this range.
|
|
start, end := l.offsetIndex(r.Start.Byte), l.offsetIndex(r.End.Byte-1)
|
|
if start == -1 || end == -1 {
|
|
return nil
|
|
}
|
|
return l[start : end+1]
|
|
}
|
|
|
|
// A TokenMap is used to map from syntax nodes to information about their tokens and leading whitespace/comments.
|
|
type TokenMap interface {
|
|
ForNode(n hclsyntax.Node) NodeTokens
|
|
|
|
isTokenMap()
|
|
}
|
|
|
|
type tokenMap map[hclsyntax.Node]NodeTokens
|
|
|
|
// ForNode returns the token information for the given node, if any.
|
|
func (m tokenMap) ForNode(n hclsyntax.Node) NodeTokens {
|
|
return m[n]
|
|
}
|
|
|
|
func (tokenMap) isTokenMap() {}
|
|
|
|
// NewTokenMapForFiles creates a new token map that can be used to look up tokens for nodes in any of the given files.
|
|
func NewTokenMapForFiles(files []*File) TokenMap {
|
|
tokens := tokenMap{}
|
|
for _, f := range files {
|
|
for node, ts := range f.Tokens.(tokenMap) {
|
|
tokens[node] = ts
|
|
}
|
|
}
|
|
return tokens
|
|
}
|
|
|
|
type tokenMapper struct {
|
|
tokenMap tokenMap
|
|
tokens tokenList
|
|
stack []hclsyntax.Node
|
|
templateControlExprs codegen.Set
|
|
}
|
|
|
|
func (m *tokenMapper) getParent() (hclsyntax.Node, bool) {
|
|
if len(m.stack) < 2 {
|
|
return nil, false
|
|
}
|
|
return m.stack[len(m.stack)-2], true
|
|
}
|
|
|
|
func (m *tokenMapper) isSingleFunctionCallArg() bool {
|
|
parent, ok := m.getParent()
|
|
if !ok {
|
|
return false
|
|
}
|
|
call, ok := parent.(*hclsyntax.FunctionCallExpr)
|
|
return ok && len(call.Args) == 1
|
|
}
|
|
|
|
func (m *tokenMapper) inTemplateControl() bool {
|
|
if len(m.stack) < 2 {
|
|
return false
|
|
}
|
|
parent := m.stack[len(m.stack)-2]
|
|
return m.templateControlExprs.Has(parent)
|
|
}
|
|
|
|
func (m *tokenMapper) collectLabelTokens(r hcl.Range) Token {
|
|
tokens := m.tokens.inRange(r)
|
|
if len(tokens) != 3 {
|
|
return m.tokens.atPos(r.Start)
|
|
}
|
|
open := tokens[0]
|
|
if open.Raw.Type != hclsyntax.TokenOQuote || len(open.TrailingTrivia) != 0 {
|
|
return m.tokens.atPos(r.Start)
|
|
}
|
|
str := tokens[1]
|
|
if str.Raw.Type != hclsyntax.TokenQuotedLit || len(str.LeadingTrivia) != 0 || len(str.TrailingTrivia) != 0 {
|
|
return m.tokens.atPos(r.Start)
|
|
}
|
|
close := tokens[2]
|
|
if close.Raw.Type != hclsyntax.TokenCQuote || len(close.LeadingTrivia) != 0 {
|
|
return m.tokens.atPos(r.Start)
|
|
}
|
|
return Token{
|
|
Raw: hclsyntax.Token{
|
|
Type: hclsyntax.TokenQuotedLit,
|
|
Bytes: append(append(open.Raw.Bytes, str.Raw.Bytes...), close.Raw.Bytes...),
|
|
Range: hcl.Range{
|
|
Filename: open.Raw.Range.Filename,
|
|
Start: open.Raw.Range.Start,
|
|
End: close.Raw.Range.End,
|
|
},
|
|
},
|
|
LeadingTrivia: open.LeadingTrivia,
|
|
TrailingTrivia: close.TrailingTrivia,
|
|
}
|
|
}
|
|
|
|
func (m *tokenMapper) mapRelativeTraversalTokens(traversal hcl.Traversal) []TraverserTokens {
|
|
if len(traversal) == 0 {
|
|
return nil
|
|
}
|
|
|
|
contract.Assert(traversal.IsRelative())
|
|
items := make([]TraverserTokens, len(traversal))
|
|
for i, t := range traversal {
|
|
rng := t.SourceRange()
|
|
leadingToken := m.tokens.atPos(rng.Start)
|
|
indexToken := m.tokens.atOffset(rng.Start.Byte + 1)
|
|
if leadingToken.Raw.Type == hclsyntax.TokenOBrack {
|
|
if indexToken.Raw.Type == hclsyntax.TokenOQuote {
|
|
indexToken = m.collectLabelTokens(hcl.Range{
|
|
Filename: rng.Filename,
|
|
Start: hcl.Pos{Byte: rng.Start.Byte + 1},
|
|
End: hcl.Pos{Byte: rng.End.Byte - 1},
|
|
})
|
|
}
|
|
items[i] = &BracketTraverserTokens{
|
|
OpenBracket: leadingToken,
|
|
Index: indexToken,
|
|
CloseBracket: m.tokens.atOffset(rng.End.Byte - 1),
|
|
}
|
|
} else {
|
|
items[i] = &DotTraverserTokens{
|
|
Dot: leadingToken,
|
|
Index: indexToken,
|
|
}
|
|
}
|
|
}
|
|
|
|
return items
|
|
}
|
|
|
|
func (m *tokenMapper) nodeRange(n hclsyntax.Node) hcl.Range {
|
|
tokens, ok := m.tokenMap[n]
|
|
if !ok {
|
|
return n.Range()
|
|
}
|
|
|
|
filename := n.Range().Filename
|
|
|
|
var parens Parentheses
|
|
var start, end hcl.Pos
|
|
switch n := n.(type) {
|
|
case *hclsyntax.Attribute:
|
|
tokens := tokens.(*AttributeTokens)
|
|
start, end = tokens.Name.Range().Start, m.nodeRange(n.Expr).End
|
|
case *hclsyntax.BinaryOpExpr:
|
|
tokens := tokens.(*BinaryOpTokens)
|
|
parens = tokens.Parentheses
|
|
start, end = m.nodeRange(n.LHS).Start, m.nodeRange(n.RHS).End
|
|
case *hclsyntax.Block:
|
|
tokens := tokens.(*BlockTokens)
|
|
start, end = tokens.Type.Range().Start, tokens.CloseBrace.Range().End
|
|
case *hclsyntax.ConditionalExpr:
|
|
switch tokens := tokens.(type) {
|
|
case *ConditionalTokens:
|
|
parens = tokens.Parentheses
|
|
start, end = m.nodeRange(n.Condition).Start, m.nodeRange(n.FalseResult).End
|
|
case *TemplateConditionalTokens:
|
|
start, end = tokens.OpenIf.Range().Start, tokens.CloseEndif.Range().End
|
|
}
|
|
case *hclsyntax.ForExpr:
|
|
switch tokens := tokens.(type) {
|
|
case *ForTokens:
|
|
parens = tokens.Parentheses
|
|
start, end = tokens.Open.Range().Start, tokens.Close.Range().End
|
|
case *TemplateForTokens:
|
|
start, end = tokens.OpenFor.Range().Start, tokens.CloseEndfor.Range().End
|
|
}
|
|
case *hclsyntax.FunctionCallExpr:
|
|
tokens := tokens.(*FunctionCallTokens)
|
|
parens = tokens.Parentheses
|
|
start, end = tokens.Name.Range().Start, tokens.CloseParen.Range().End
|
|
case *hclsyntax.IndexExpr:
|
|
tokens := tokens.(*IndexTokens)
|
|
parens = tokens.Parentheses
|
|
start, end = m.nodeRange(n.Collection).Start, tokens.CloseBracket.Range().End
|
|
case *hclsyntax.LiteralValueExpr, *hclsyntax.ObjectConsKeyExpr:
|
|
tokens := tokens.(*LiteralValueTokens)
|
|
parens = tokens.Parentheses
|
|
start, end = tokens.Value[0].Range().Start, tokens.Value[len(tokens.Value)-1].Range().End
|
|
case *hclsyntax.ObjectConsExpr:
|
|
tokens := tokens.(*ObjectConsTokens)
|
|
parens = tokens.Parentheses
|
|
start, end = tokens.OpenBrace.Range().Start, tokens.CloseBrace.Range().End
|
|
case *hclsyntax.RelativeTraversalExpr:
|
|
tokens := tokens.(*RelativeTraversalTokens)
|
|
parens = tokens.Parentheses
|
|
start, end = m.nodeRange(n.Source).Start, tokens.Traversal[len(tokens.Traversal)-1].Range().End
|
|
case *hclsyntax.ScopeTraversalExpr:
|
|
tokens := tokens.(*ScopeTraversalTokens)
|
|
parens = tokens.Parentheses
|
|
start, end = tokens.Root.Range().Start, tokens.Root.Range().End
|
|
if len(tokens.Traversal) > 0 {
|
|
end = tokens.Traversal[len(tokens.Traversal)-1].Range().End
|
|
}
|
|
case *hclsyntax.SplatExpr:
|
|
tokens := tokens.(*SplatTokens)
|
|
parens = tokens.Parentheses
|
|
start, end = m.nodeRange(n.Source).Start, m.nodeRange(n.Each).End
|
|
case *hclsyntax.TemplateExpr:
|
|
tokens := tokens.(*TemplateTokens)
|
|
parens = tokens.Parentheses
|
|
start, end = tokens.Open.Range().Start, tokens.Close.Range().End
|
|
case *hclsyntax.TemplateWrapExpr:
|
|
tokens := tokens.(*TemplateTokens)
|
|
parens = tokens.Parentheses
|
|
start, end = tokens.Open.Range().Start, tokens.Close.Range().End
|
|
case *hclsyntax.TupleConsExpr:
|
|
tokens := tokens.(*TupleConsTokens)
|
|
parens = tokens.Parentheses
|
|
start, end = tokens.OpenBracket.Range().Start, tokens.CloseBracket.Range().End
|
|
case *hclsyntax.UnaryOpExpr:
|
|
tokens := tokens.(*UnaryOpTokens)
|
|
parens = tokens.Parentheses
|
|
start, end = tokens.Operator.Range().Start, m.nodeRange(n.Val).End
|
|
}
|
|
|
|
return exprRange(filename, parens, start, end)
|
|
}
|
|
|
|
func (m *tokenMapper) Enter(n hclsyntax.Node) hcl.Diagnostics {
|
|
switch n := n.(type) {
|
|
case hclsyntax.Attributes, hclsyntax.Blocks, hclsyntax.ChildScope:
|
|
// Do nothing
|
|
default:
|
|
m.stack = append(m.stack, n)
|
|
}
|
|
|
|
switch n := n.(type) {
|
|
case *hclsyntax.ConditionalExpr:
|
|
open := m.tokens.atPos(n.SrcRange.Start)
|
|
if open.Raw.Type == hclsyntax.TokenTemplateControl {
|
|
m.templateControlExprs.Add(n)
|
|
}
|
|
case *hclsyntax.ForExpr:
|
|
open := m.tokens.atPos(n.OpenRange.Start)
|
|
if open.Raw.Type == hclsyntax.TokenTemplateControl {
|
|
m.templateControlExprs.Add(n)
|
|
}
|
|
}
|
|
|
|
// Work around a bug in the HCL syntax library. The walkChildNodes implementation for ObjectConsKeyExpr does not
|
|
// descend into its wrapped expression if the wrapped expression can be interpreted as a keyword even if the
|
|
// syntactical form that _forces_ the wrapped expression to be a non-literal is used.
|
|
if x, ok := n.(*hclsyntax.ObjectConsKeyExpr); ok && x.ForceNonLiteral && hcl.ExprAsKeyword(x.Wrapped) != "" {
|
|
return hclsyntax.Walk(x.Wrapped, m)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (m *tokenMapper) Exit(n hclsyntax.Node) hcl.Diagnostics {
|
|
// Gather parentheses.
|
|
var parens Parentheses
|
|
startParens, endParens := n.Range().Start.Byte-1, n.Range().End.Byte
|
|
for {
|
|
open, close := m.tokens.atOffset(startParens), m.tokens.atOffset(endParens)
|
|
if open.Raw.Type != hclsyntax.TokenOParen || close.Raw.Type != hclsyntax.TokenCParen {
|
|
break
|
|
}
|
|
parens.Open, parens.Close = append(parens.Open, open), append(parens.Close, close)
|
|
startParens, endParens = open.Range().Start.Byte-1, close.Range().End.Byte
|
|
}
|
|
if m.isSingleFunctionCallArg() && len(parens.Open) > 0 {
|
|
parens.Open, parens.Close = parens.Open[:len(parens.Open)-1], parens.Close[:len(parens.Close)-1]
|
|
}
|
|
|
|
var nodeTokens NodeTokens
|
|
switch n := n.(type) {
|
|
case *hclsyntax.Attribute:
|
|
nodeTokens = &AttributeTokens{
|
|
Name: m.tokens.atPos(n.NameRange.Start),
|
|
Equals: m.tokens.atPos(n.EqualsRange.Start),
|
|
}
|
|
case *hclsyntax.BinaryOpExpr:
|
|
nodeTokens = &BinaryOpTokens{
|
|
Parentheses: parens,
|
|
Operator: m.tokens.atPos(m.nodeRange(n.LHS).End),
|
|
}
|
|
case *hclsyntax.Block:
|
|
labels := make([]Token, len(n.Labels))
|
|
for i, r := range n.LabelRanges {
|
|
labels[i] = m.collectLabelTokens(r)
|
|
}
|
|
nodeTokens = &BlockTokens{
|
|
Type: m.tokens.atPos(n.TypeRange.Start),
|
|
Labels: labels,
|
|
OpenBrace: m.tokens.atPos(n.OpenBraceRange.Start),
|
|
CloseBrace: m.tokens.atPos(n.CloseBraceRange.Start),
|
|
}
|
|
case *hclsyntax.ConditionalExpr:
|
|
condRange := m.nodeRange(n.Condition)
|
|
trueRange := m.nodeRange(n.TrueResult)
|
|
falseRange := m.nodeRange(n.FalseResult)
|
|
|
|
if m.templateControlExprs.Has(n) {
|
|
condition := m.tokens.atPos(condRange.Start)
|
|
|
|
ift := m.tokens.atOffset(condition.Range().Start.Byte - 1)
|
|
openIf := m.tokens.atOffset(ift.Range().Start.Byte - 1)
|
|
closeIf := m.tokens.atPos(condRange.End)
|
|
|
|
openEndifOrElse := m.tokens.atPos(trueRange.End)
|
|
endifOrElse := m.tokens.atPos(openEndifOrElse.Range().End)
|
|
closeEndifOrElse := m.tokens.atPos(endifOrElse.Range().End)
|
|
|
|
var openElse, elset, closeElse *Token
|
|
if endifOrElse.Raw.Type == hclsyntax.TokenIdent && string(endifOrElse.Raw.Bytes) == "else" {
|
|
open, t, close := openEndifOrElse, endifOrElse, closeEndifOrElse
|
|
openElse, elset, closeElse = &open, &t, &close
|
|
|
|
openEndifOrElse = m.tokens.atPos(falseRange.End)
|
|
endifOrElse = m.tokens.atPos(openEndifOrElse.Range().End)
|
|
closeEndifOrElse = m.tokens.atPos(endifOrElse.Range().End)
|
|
}
|
|
|
|
nodeTokens = &TemplateConditionalTokens{
|
|
OpenIf: openIf,
|
|
If: ift,
|
|
CloseIf: closeIf,
|
|
OpenElse: openElse,
|
|
Else: elset,
|
|
CloseElse: closeElse,
|
|
OpenEndif: openEndifOrElse,
|
|
Endif: endifOrElse,
|
|
CloseEndif: closeEndifOrElse,
|
|
}
|
|
} else {
|
|
nodeTokens = &ConditionalTokens{
|
|
Parentheses: parens,
|
|
QuestionMark: m.tokens.atPos(condRange.End),
|
|
Colon: m.tokens.atPos(trueRange.End),
|
|
}
|
|
}
|
|
case *hclsyntax.ForExpr:
|
|
openToken := m.tokens.atPos(n.OpenRange.Start)
|
|
forToken := m.tokens.atPos(openToken.Range().End)
|
|
|
|
var keyToken, commaToken *Token
|
|
var valueToken Token
|
|
if n.KeyVar != "" {
|
|
key := m.tokens.atPos(forToken.Range().End)
|
|
comma := m.tokens.atPos(key.Range().End)
|
|
value := m.tokens.atPos(comma.Range().End)
|
|
|
|
keyToken, commaToken, valueToken = &key, &comma, value
|
|
} else {
|
|
valueToken = m.tokens.atPos(forToken.Range().End)
|
|
}
|
|
|
|
var arrowToken *Token
|
|
if n.KeyExpr != nil {
|
|
arrow := m.tokens.atPos(m.nodeRange(n.KeyExpr).End)
|
|
arrowToken = &arrow
|
|
}
|
|
|
|
valRange := m.nodeRange(n.ValExpr)
|
|
|
|
var groupToken *Token
|
|
if n.Group {
|
|
group := m.tokens.atPos(valRange.End)
|
|
groupToken = &group
|
|
}
|
|
|
|
var ifToken *Token
|
|
if n.CondExpr != nil {
|
|
pos := valRange.End
|
|
if groupToken != nil {
|
|
pos = groupToken.Range().End
|
|
}
|
|
ift := m.tokens.atPos(pos)
|
|
ifToken = &ift
|
|
}
|
|
|
|
if m.templateControlExprs.Has(n) {
|
|
closeFor := m.tokens.atPos(m.nodeRange(n.CollExpr).End)
|
|
|
|
openEndfor := m.tokens.atPos(valRange.End)
|
|
endfor := m.tokens.atPos(openEndfor.Range().End)
|
|
closeEndfor := m.tokens.atPos(endfor.Range().End)
|
|
|
|
nodeTokens = &TemplateForTokens{
|
|
OpenFor: openToken,
|
|
For: forToken,
|
|
Key: keyToken,
|
|
Comma: commaToken,
|
|
Value: valueToken,
|
|
In: m.tokens.atPos(valueToken.Range().End),
|
|
CloseFor: closeFor,
|
|
OpenEndfor: openEndfor,
|
|
Endfor: endfor,
|
|
CloseEndfor: closeEndfor,
|
|
}
|
|
} else {
|
|
nodeTokens = &ForTokens{
|
|
Parentheses: parens,
|
|
Open: openToken,
|
|
For: forToken,
|
|
Key: keyToken,
|
|
Comma: commaToken,
|
|
Value: valueToken,
|
|
In: m.tokens.atPos(valueToken.Range().End),
|
|
Colon: m.tokens.atPos(m.nodeRange(n.CollExpr).End),
|
|
Arrow: arrowToken,
|
|
Group: groupToken,
|
|
If: ifToken,
|
|
Close: m.tokens.atPos(n.CloseRange.Start),
|
|
}
|
|
}
|
|
case *hclsyntax.FunctionCallExpr:
|
|
args := n.Args
|
|
commas := make([]Token, 0, len(args))
|
|
if len(args) > 0 {
|
|
for _, ex := range args[:len(args)-1] {
|
|
commas = append(commas, m.tokens.atPos(m.nodeRange(ex).End))
|
|
}
|
|
if trailing := m.tokens.atPos(m.nodeRange(args[len(args)-1]).End); trailing.Raw.Type == hclsyntax.TokenComma {
|
|
commas = append(commas, trailing)
|
|
}
|
|
}
|
|
nodeTokens = &FunctionCallTokens{
|
|
Parentheses: parens,
|
|
Name: m.tokens.atPos(n.NameRange.Start),
|
|
OpenParen: m.tokens.atPos(n.OpenParenRange.Start),
|
|
Commas: commas,
|
|
CloseParen: m.tokens.atPos(n.CloseParenRange.Start),
|
|
}
|
|
case *hclsyntax.IndexExpr:
|
|
nodeTokens = &IndexTokens{
|
|
Parentheses: parens,
|
|
OpenBracket: m.tokens.atPos(n.OpenRange.Start),
|
|
CloseBracket: m.tokens.atOffset(n.BracketRange.End.Byte - 1),
|
|
}
|
|
case *hclsyntax.LiteralValueExpr:
|
|
nodeTokens = &LiteralValueTokens{
|
|
Parentheses: parens,
|
|
Value: m.tokens.inRange(n.Range()),
|
|
}
|
|
case *hclsyntax.ObjectConsKeyExpr:
|
|
nodeTokens = &LiteralValueTokens{
|
|
Parentheses: parens,
|
|
Value: m.tokens.inRange(n.Range()),
|
|
}
|
|
case *hclsyntax.ObjectConsExpr:
|
|
items := make([]ObjectConsItemTokens, len(n.Items))
|
|
for i, item := range n.Items {
|
|
var comma *Token
|
|
if t := m.tokens.atPos(m.nodeRange(item.ValueExpr).End); t.Raw.Type == hclsyntax.TokenComma {
|
|
comma = &t
|
|
}
|
|
items[i] = ObjectConsItemTokens{
|
|
Equals: m.tokens.atPos(m.nodeRange(item.KeyExpr).End),
|
|
Comma: comma,
|
|
}
|
|
}
|
|
nodeTokens = &ObjectConsTokens{
|
|
Parentheses: parens,
|
|
OpenBrace: m.tokens.atPos(n.OpenRange.Start),
|
|
Items: items,
|
|
CloseBrace: m.tokens.atOffset(n.SrcRange.End.Byte - 1),
|
|
}
|
|
case *hclsyntax.RelativeTraversalExpr:
|
|
nodeTokens = &RelativeTraversalTokens{
|
|
Parentheses: parens,
|
|
Traversal: m.mapRelativeTraversalTokens(n.Traversal),
|
|
}
|
|
case *hclsyntax.ScopeTraversalExpr:
|
|
nodeTokens = &ScopeTraversalTokens{
|
|
Parentheses: parens,
|
|
Root: m.tokens.atPos(n.Traversal[0].SourceRange().Start),
|
|
Traversal: m.mapRelativeTraversalTokens(n.Traversal[1:]),
|
|
}
|
|
case *hclsyntax.SplatExpr:
|
|
openToken := m.tokens.atPos(m.nodeRange(n.Source).End)
|
|
starToken := m.tokens.atPos(openToken.Range().End)
|
|
var closeToken *Token
|
|
if openToken.Raw.Type == hclsyntax.TokenOBrack {
|
|
cbrack := m.tokens.atPos(starToken.Range().End)
|
|
closeToken = &cbrack
|
|
}
|
|
nodeTokens = &SplatTokens{
|
|
Parentheses: parens,
|
|
Open: openToken,
|
|
Star: starToken,
|
|
Close: closeToken,
|
|
}
|
|
case *hclsyntax.TemplateExpr:
|
|
// NOTE: the HCL parser lifts control sequences into if or for expressions. This is handled in the correspoding
|
|
// mappers.
|
|
if m.inTemplateControl() {
|
|
nodeTokens = &TemplateTokens{
|
|
Open: Token{
|
|
Raw: hclsyntax.Token{
|
|
Range: hcl.Range{
|
|
Filename: n.SrcRange.Filename,
|
|
Start: n.SrcRange.Start,
|
|
End: n.SrcRange.Start,
|
|
},
|
|
},
|
|
},
|
|
Close: Token{
|
|
Raw: hclsyntax.Token{
|
|
Range: hcl.Range{
|
|
Filename: n.SrcRange.Filename,
|
|
Start: n.SrcRange.End,
|
|
End: n.SrcRange.End,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
} else {
|
|
// If we're inside the body of a template if or for, there cannot be any delimiting tokens.
|
|
nodeTokens = &TemplateTokens{
|
|
Parentheses: parens,
|
|
Open: m.tokens.atPos(n.SrcRange.Start),
|
|
Close: m.tokens.atOffset(n.SrcRange.End.Byte - 1),
|
|
}
|
|
}
|
|
case *hclsyntax.TemplateWrapExpr:
|
|
nodeTokens = &TemplateTokens{
|
|
Parentheses: parens,
|
|
Open: m.tokens.atPos(n.SrcRange.Start),
|
|
Close: m.tokens.atOffset(n.SrcRange.End.Byte - 1),
|
|
}
|
|
case *hclsyntax.TupleConsExpr:
|
|
exprs := n.Exprs
|
|
commas := make([]Token, 0, len(exprs))
|
|
if len(exprs) > 0 {
|
|
for _, ex := range exprs[:len(exprs)-1] {
|
|
commas = append(commas, m.tokens.atPos(m.nodeRange(ex).End))
|
|
}
|
|
if trailing := m.tokens.atPos(m.nodeRange(exprs[len(exprs)-1]).End); trailing.Raw.Type == hclsyntax.TokenComma {
|
|
commas = append(commas, trailing)
|
|
}
|
|
}
|
|
nodeTokens = &TupleConsTokens{
|
|
Parentheses: parens,
|
|
OpenBracket: m.tokens.atPos(n.OpenRange.Start),
|
|
Commas: commas,
|
|
CloseBracket: m.tokens.atOffset(n.SrcRange.End.Byte - 1),
|
|
}
|
|
case *hclsyntax.UnaryOpExpr:
|
|
nodeTokens = &UnaryOpTokens{
|
|
Parentheses: parens,
|
|
Operator: m.tokens.atPos(n.SymbolRange.Start),
|
|
}
|
|
}
|
|
if nodeTokens != nil {
|
|
m.tokenMap[n] = nodeTokens
|
|
}
|
|
|
|
if n == m.stack[len(m.stack)-1] {
|
|
m.stack = m.stack[:len(m.stack)-1]
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// mapTokens builds a mapping from the syntax nodes in the given source file to their tokens. The mapping is recorded
|
|
// in the map passed in to the function.
|
|
func mapTokens(rawTokens hclsyntax.Tokens, filename string, root hclsyntax.Node, contents []byte, tokenMap tokenMap,
|
|
initialPos hcl.Pos) {
|
|
|
|
// Turn the list of raw tokens into a list of trivia-carrying tokens.
|
|
lastEndPos := initialPos
|
|
var tokens tokenList
|
|
trivia := TriviaList{}
|
|
inControlSeq := false
|
|
for _, raw := range rawTokens {
|
|
// Snip whitespace out of the body and turn it in to trivia.
|
|
if startPos := raw.Range.Start; startPos.Byte != lastEndPos.Byte {
|
|
triviaBytes := contents[lastEndPos.Byte-initialPos.Byte : startPos.Byte-initialPos.Byte]
|
|
|
|
// If this trivia begins a new line, attach the current trivia to the last processed token, if any.
|
|
if len(tokens) > 0 {
|
|
if nl := bytes.IndexByte(triviaBytes, '\n'); nl != -1 {
|
|
trailingTriviaBytes := triviaBytes[:nl+1]
|
|
triviaBytes = triviaBytes[nl+1:]
|
|
|
|
endPos := hcl.Pos{Line: lastEndPos.Line + 1, Column: 0, Byte: lastEndPos.Byte + nl + 1}
|
|
rng := hcl.Range{Filename: filename, Start: lastEndPos, End: endPos}
|
|
trivia = append(trivia, Whitespace{rng: rng, bytes: trailingTriviaBytes})
|
|
tokens[len(tokens)-1].TrailingTrivia, trivia = trivia, TriviaList{}
|
|
|
|
lastEndPos = endPos
|
|
}
|
|
}
|
|
|
|
rng := hcl.Range{Filename: filename, Start: lastEndPos, End: startPos}
|
|
trivia = append(trivia, Whitespace{rng: rng, bytes: triviaBytes})
|
|
}
|
|
|
|
switch raw.Type {
|
|
case hclsyntax.TokenComment:
|
|
trivia = append(trivia, Comment{Lines: processComment(raw.Bytes), rng: raw.Range, bytes: raw.Bytes})
|
|
case hclsyntax.TokenTemplateInterp:
|
|
// Treat these as trailing trivia.
|
|
trivia = append(trivia, TemplateDelimiter{Type: raw.Type, rng: raw.Range, bytes: raw.Bytes})
|
|
if len(tokens) > 0 {
|
|
tokens[len(tokens)-1].TrailingTrivia, trivia = trivia, TriviaList{}
|
|
}
|
|
case hclsyntax.TokenTemplateControl:
|
|
tokens, trivia = append(tokens, Token{Raw: raw, LeadingTrivia: trivia}), TriviaList{}
|
|
inControlSeq = true
|
|
case hclsyntax.TokenTemplateSeqEnd:
|
|
// If this terminates a template control sequence, it is a proper token. Otherwise, it is treated as leading
|
|
// trivia.
|
|
if !inControlSeq {
|
|
tokens[len(tokens)-1].TrailingTrivia = trivia
|
|
trivia = TriviaList{TemplateDelimiter{Type: raw.Type, rng: raw.Range, bytes: raw.Bytes}}
|
|
} else {
|
|
tokens, trivia = append(tokens, Token{Raw: raw, LeadingTrivia: trivia}), TriviaList{}
|
|
}
|
|
inControlSeq = false
|
|
case hclsyntax.TokenNewline, hclsyntax.TokenBitwiseAnd, hclsyntax.TokenBitwiseOr,
|
|
hclsyntax.TokenBitwiseNot, hclsyntax.TokenBitwiseXor, hclsyntax.TokenStarStar, hclsyntax.TokenApostrophe,
|
|
hclsyntax.TokenBacktick, hclsyntax.TokenSemicolon, hclsyntax.TokenTabs, hclsyntax.TokenInvalid,
|
|
hclsyntax.TokenBadUTF8, hclsyntax.TokenQuotedNewline:
|
|
// Treat these as whitespace. We cannot omit their bytes from the list entirely, as the algorithm below
|
|
// that maps positions to tokens requires that every byte in the source file is covered by the token list.
|
|
continue
|
|
default:
|
|
tokens, trivia = append(tokens, Token{Raw: raw, LeadingTrivia: trivia}), TriviaList{}
|
|
}
|
|
lastEndPos = raw.Range.End
|
|
}
|
|
|
|
// If we had any tokens, we should have attached all trivia to something.
|
|
contract.Assert(len(trivia) == 0 || len(tokens) == 0)
|
|
|
|
// Now build the token map.
|
|
//
|
|
// If a node records the ranges of relevant tokens in its syntax node, the start position of those ranges is used
|
|
// to fetch the corresponding token.
|
|
//
|
|
// If a node does not record the range of a relevant token, the end position of a preceding element (e.g. an
|
|
// expression or a token) is used to find the token.
|
|
//
|
|
// TODO(pdg): handle parenthesized expressions
|
|
diags := hclsyntax.Walk(root, &tokenMapper{
|
|
tokenMap: tokenMap,
|
|
tokens: tokens,
|
|
templateControlExprs: codegen.Set{},
|
|
})
|
|
contract.Assert(diags == nil)
|
|
|
|
// If the root was a Body and there is a trailing end-of-file token, attach it to the body.
|
|
body, isBody := root.(*hclsyntax.Body)
|
|
if isBody && len(tokens) > 0 && tokens[len(tokens)-1].Raw.Type == hclsyntax.TokenEOF {
|
|
tokenMap[body] = &BodyTokens{EndOfFile: &tokens[len(tokens)-1]}
|
|
}
|
|
}
|
|
|
|
// processComment separates the given comment into lines and attempts to remove comment tokens.
|
|
func processComment(bytes []byte) []string {
|
|
comment := string(bytes)
|
|
|
|
// Each HCL comment may be either a line comment or a block comment. Line comments start with '#' or '//' and
|
|
// terminate with an EOL. Block comments begin with a '/*' and terminate with a '*/'. All comment delimiters are
|
|
// preserved in the HCL comment text.
|
|
//
|
|
// To make life easier for the code generators, HCL comments are pre-processed to remove comment delimiters. For
|
|
// line comments, this process is trivial. For block comments, things are a bit more involved.
|
|
switch {
|
|
case comment[0] == '#':
|
|
return []string{strings.TrimSuffix(comment[1:], "\n")}
|
|
case comment[0:2] == "//":
|
|
return []string{strings.TrimSuffix(comment[2:], "\n")}
|
|
default:
|
|
return processBlockComment(comment)
|
|
}
|
|
}
|
|
|
|
// These regexes are used by processBlockComment. The first matches a block comment start, the second a block comment
|
|
// end, and the third a block comment line prefix.
|
|
var blockStartPat = regexp.MustCompile(`^/\*+`)
|
|
var blockEndPat = regexp.MustCompile(`[[:space:]]*\*+/$`)
|
|
var blockPrefixPat = regexp.MustCompile(`^[[:space:]]*\*`)
|
|
|
|
// processBlockComment splits a block comment into mutiple lines, removes comment delimiters, and attempts to remove
|
|
// common comment prefixes from interior lines. For example, the following HCL block comment:
|
|
//
|
|
// /**
|
|
// * This is a block comment!
|
|
// *
|
|
// * It has multiple lines.
|
|
// */
|
|
//
|
|
// becomes this set of lines:
|
|
//
|
|
// []string{" This is a block comment!", "", " It has multiple lines."}
|
|
//
|
|
func processBlockComment(text string) []string {
|
|
lines := strings.Split(text, "\n")
|
|
|
|
// We will always trim the following:
|
|
// - '^/\*+' from the first line
|
|
// - a trailing '[[:space:]]\*+/$' from the last line
|
|
|
|
// In addition, we will trim '^[[:space:]]*\*' from the second through last lines iff all lines in that set share
|
|
// a prefix that matches that pattern.
|
|
|
|
prefix := ""
|
|
for i, l := range lines[1:] {
|
|
m := blockPrefixPat.FindString(l)
|
|
if i == 0 {
|
|
prefix = m
|
|
} else if m != prefix {
|
|
prefix = ""
|
|
break
|
|
}
|
|
}
|
|
|
|
for i, l := range lines {
|
|
switch i {
|
|
case 0:
|
|
start := blockStartPat.FindString(l)
|
|
contract.Assert(start != "")
|
|
l = l[len(start):]
|
|
|
|
// If this is a single-line block comment, trim the end pattern as well.
|
|
if len(lines) == 1 {
|
|
contract.Assert(prefix == "")
|
|
|
|
if end := blockEndPat.FindString(l); end != "" {
|
|
l = l[:len(l)-len(end)]
|
|
}
|
|
}
|
|
case len(lines) - 1:
|
|
// The prefix we're trimming may overlap with the end pattern we're trimming. In this case, trim the entire
|
|
// line.
|
|
if len(l)-len(prefix) == 1 {
|
|
l = ""
|
|
} else {
|
|
l = l[len(prefix):]
|
|
if end := blockEndPat.FindString(l); end != "" {
|
|
l = l[:len(l)-len(end)]
|
|
}
|
|
}
|
|
default:
|
|
// Trim the prefix.
|
|
l = l[len(prefix):]
|
|
}
|
|
|
|
lines[i] = l
|
|
}
|
|
|
|
// If the first or last line is empty, drop it.
|
|
if lines[0] == "" {
|
|
lines = lines[1:]
|
|
}
|
|
if len(lines) > 0 && lines[len(lines)-1] == "" {
|
|
lines = lines[:len(lines)-1]
|
|
}
|
|
|
|
return lines
|
|
}
|