This change includes a few steps towards AWS backend code-generation: * Add a BoundDependencies property to ast.Stack to remember the *ast.Stack objects bound during Stack binding. * Make a few CloudFormation properties optional (cfOutput Export/Condition). * Rename clouds.ArchMap, clouds.ArchNames, schedulers.ArchMap, and schedulers.ArchNames to clouds.Values, clouds.Names, schedulers.Values, and schedulers.Names, respectively. This reads much nicer to my eyes. * Create a new anonymous ast.Target for deployments if no specific target was specified; this is to support quick-and-easy "one off" deployments, as will be common when doing local development. * Sketch out more of the AWS Cloud implementation. We actually map the Mu Services into CloudFormation Resources; well, kinda sorta, since we don't actually have Service-specific logic in here yet, however all of the structure and scaffolding is now here.
349 lines
10 KiB
Go
349 lines
10 KiB
Go
// Copyright 2016 Marapongo, Inc. All rights reserved.
|
|
|
|
package compiler
|
|
|
|
import (
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"github.com/golang/glog"
|
|
"github.com/satori/go.uuid"
|
|
|
|
"github.com/marapongo/mu/pkg/ast"
|
|
"github.com/marapongo/mu/pkg/compiler/backends"
|
|
"github.com/marapongo/mu/pkg/compiler/backends/clouds"
|
|
"github.com/marapongo/mu/pkg/compiler/backends/schedulers"
|
|
"github.com/marapongo/mu/pkg/compiler/core"
|
|
"github.com/marapongo/mu/pkg/diag"
|
|
"github.com/marapongo/mu/pkg/errors"
|
|
"github.com/marapongo/mu/pkg/workspace"
|
|
)
|
|
|
|
// Compiler provides an interface into the many phases of the Mu compilation process.
|
|
type Compiler interface {
|
|
core.Phase
|
|
|
|
// Context returns the current compiler context.
|
|
Context() *core.Context
|
|
|
|
// Build detects and compiles inputs from the given location, storing build artifacts in the given destination.
|
|
Build(inp string, outp string)
|
|
// BuildFile uses the given Mufile directly, and stores build artifacts in the given destination.
|
|
BuildFile(mufile []byte, ext string, outp string)
|
|
}
|
|
|
|
// compiler is the canonical implementation of the Mu compiler.
|
|
type compiler struct {
|
|
ctx *core.Context
|
|
opts Options
|
|
}
|
|
|
|
// NewCompiler creates a new instance of the Mu compiler, with the given initialization settings.
|
|
func NewCompiler(opts Options) Compiler {
|
|
return &compiler{
|
|
ctx: &core.Context{},
|
|
opts: opts,
|
|
}
|
|
}
|
|
|
|
func (c *compiler) Context() *core.Context {
|
|
return c.ctx
|
|
}
|
|
|
|
func (c *compiler) Diag() diag.Sink {
|
|
return c.opts.Diag
|
|
}
|
|
|
|
func (c *compiler) Build(inp string, outp string) {
|
|
glog.Infof("Building target '%v' (out='%v')", inp, outp)
|
|
|
|
// First find the root of the current package based on the location of its Mufile.
|
|
mufile := c.detectMufile(inp)
|
|
if mufile == "" {
|
|
c.Diag().Errorf(errors.MissingMufile, inp)
|
|
return
|
|
}
|
|
|
|
// Read in the contents of the document and make it available to subsequent stages.
|
|
doc, err := diag.ReadDocument(mufile)
|
|
if err != nil {
|
|
c.Diag().Errorf(errors.CouldNotReadMufile.WithFile(mufile), err)
|
|
return
|
|
}
|
|
|
|
c.buildDocument(doc, outp)
|
|
}
|
|
|
|
func (c *compiler) BuildFile(mufile []byte, ext string, outp string) {
|
|
glog.Infof("Building in-memory %v file (bytes=%v out='%v')", ext, len(mufile), outp)
|
|
c.buildDocument(&diag.Document{File: workspace.MufileBase + ext, Body: mufile}, outp)
|
|
}
|
|
|
|
func (c *compiler) buildDocument(doc *diag.Document, outp string) {
|
|
glog.Infof("Building doc '%v' (bytes=%v out='%v')", doc.File, len(doc.Body), outp)
|
|
if glog.V(2) {
|
|
defer func() {
|
|
glog.V(2).Infof("Building doc '%v' completed w/ %v warnings and %v errors",
|
|
doc.File, c.Diag().Warnings(), c.Diag().Errors())
|
|
}()
|
|
}
|
|
|
|
// Perform the front-end passes to generate a stack AST.
|
|
stack, ok := c.parseStack(doc)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
// Perform the semantic analysis passes to validate, transform, and/or update the AST.
|
|
stack, ok = c.analyzeStack(doc, stack)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
if !c.opts.SkipCodegen {
|
|
// Figure out which cloud architecture we will be targeting during code-gen.
|
|
target, arch, ok := c.discoverTargetArch(doc, stack)
|
|
if !ok {
|
|
return
|
|
}
|
|
if glog.V(2) {
|
|
tname := "n/a"
|
|
if target != nil {
|
|
tname = target.Name
|
|
}
|
|
glog.V(2).Infof("Stack %v targets target=%v cloud=%v", stack.Name, tname, arch)
|
|
}
|
|
|
|
// Now get the backend cloud provider to process the stack from here on out.
|
|
be := backends.New(arch)
|
|
be.CodeGen(target, doc, stack)
|
|
}
|
|
}
|
|
|
|
// loadAndParseStack takes a Mufile document, parses and validates it, and returns a stack AST. If anything goes wrong
|
|
// during this process, the number of errors will be non-zero, and the bool will be false.
|
|
func (c *compiler) parseStack(doc *diag.Document) (*ast.Stack, bool) {
|
|
// To build the Mu package, first parse the input file.
|
|
p := NewParser(c)
|
|
stack := p.Parse(doc)
|
|
if p.Diag().Errors() > 0 {
|
|
// If any errors happened during parsing, exit.
|
|
return stack, false
|
|
}
|
|
|
|
// Do a pass over the parse tree to ensure that all is well.
|
|
ptAnalyzer := NewPTAnalyzer(c)
|
|
ptAnalyzer.Analyze(doc, stack)
|
|
if p.Diag().Errors() > 0 {
|
|
// If any errors happened during parse tree analysis, exit.
|
|
return stack, false
|
|
}
|
|
|
|
return stack, true
|
|
}
|
|
|
|
// discoverTargetArch uses a variety of mechanisms to discover the target architecture, returning it. If no
|
|
// architecture was discovered, an error is issued, and the bool return will be false.
|
|
func (c *compiler) discoverTargetArch(doc *diag.Document, stack *ast.Stack) (*ast.Target, backends.Arch, bool) {
|
|
// Target and architectures settings may come from one of three places, in order of search preference:
|
|
// 1) command line arguments.
|
|
// 2) settings specific to this stack.
|
|
// 3) cluster-wide settings in a Mucluster file.
|
|
// In other words, 1 overrides 2 which overrides 3.
|
|
arch := c.opts.Arch
|
|
|
|
// If a target was specified, look it up and load up its options.
|
|
var target *ast.Target
|
|
if c.opts.Target != "" {
|
|
// First, check the stack to see if it has a targets section.
|
|
if t, exists := stack.Targets[c.opts.Target]; exists {
|
|
target = &t
|
|
} else {
|
|
// If that didn't work, see if there's a clusters file we can consult.
|
|
// TODO: support Mucluster files.
|
|
c.Diag().Errorf(errors.CloudTargetNotFound.WithDocument(doc), c.opts.Target)
|
|
return target, arch, false
|
|
}
|
|
}
|
|
|
|
// If no target was specified or discovered yet, see if there is a default one to use.
|
|
if target == nil {
|
|
for _, t := range stack.Targets {
|
|
if t.Default {
|
|
target = &t
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if target == nil {
|
|
// If no target was found, and we don't have an architecture, error out.
|
|
if arch.Cloud == clouds.NoArch {
|
|
c.Diag().Errorf(errors.MissingTarget.WithDocument(doc))
|
|
return target, arch, false
|
|
}
|
|
|
|
// If we got here, generate an "anonymous" target, so that we at least have a name.
|
|
target = c.newAnonTarget(arch)
|
|
} else {
|
|
// If a target was found, go ahead and extract and validate the target architecture.
|
|
a, ok := c.getTargetArch(doc, target, arch)
|
|
if !ok {
|
|
return target, arch, false
|
|
}
|
|
arch = a
|
|
}
|
|
|
|
return target, arch, true
|
|
}
|
|
|
|
// newAnonTarget creates an anonymous target for stacks that didn't declare one.
|
|
func (c *compiler) newAnonTarget(arch backends.Arch) *ast.Target {
|
|
// TODO: ensure this is unique.
|
|
// TODO: we want to cache names somewhere (~/.mu/?) so that we can reuse temporary local stacks, etc.
|
|
return &ast.Target{
|
|
Name: uuid.NewV4().String(),
|
|
Cloud: clouds.Names[arch.Cloud],
|
|
Scheduler: schedulers.Names[arch.Scheduler],
|
|
}
|
|
}
|
|
|
|
// getTargetArch gets and validates the architecture from an existing target.
|
|
func (c *compiler) getTargetArch(doc *diag.Document, target *ast.Target, existing backends.Arch) (backends.Arch, bool) {
|
|
targetCloud := existing.Cloud
|
|
targetScheduler := existing.Scheduler
|
|
|
|
// If specified, look up the target's architecture settings.
|
|
if target.Cloud != "" {
|
|
tc, ok := clouds.Values[target.Cloud]
|
|
if !ok {
|
|
c.Diag().Errorf(errors.UnrecognizedCloudArch.WithDocument(doc), target.Cloud)
|
|
return existing, false
|
|
}
|
|
targetCloud = tc
|
|
}
|
|
if target.Scheduler != "" {
|
|
ts, ok := schedulers.Values[target.Scheduler]
|
|
if !ok {
|
|
c.Diag().Errorf(errors.UnrecognizedSchedulerArch.WithDocument(doc), target.Scheduler)
|
|
return existing, false
|
|
}
|
|
targetScheduler = ts
|
|
}
|
|
|
|
// Ensure there aren't any conflicts, comparing compiler options to target settings.
|
|
tarch := backends.Arch{targetCloud, targetScheduler}
|
|
if targetCloud != existing.Cloud && existing.Cloud != clouds.NoArch {
|
|
c.Diag().Errorf(errors.ConflictingTargetArchSelection.WithDocument(doc), existing, target.Name, tarch)
|
|
return tarch, false
|
|
}
|
|
if targetScheduler != existing.Scheduler && existing.Scheduler != schedulers.NoArch {
|
|
c.Diag().Errorf(errors.ConflictingTargetArchSelection.WithDocument(doc), existing, target.Name, tarch)
|
|
return tarch, false
|
|
}
|
|
|
|
return tarch, true
|
|
}
|
|
|
|
// analyzeStack performs semantic analysis on a stack -- validating, transforming, and/or updating it -- and then
|
|
// returns the result. If a problem occurs, errors will have been emitted, and the bool return will be false.
|
|
func (c *compiler) analyzeStack(doc *diag.Document, stack *ast.Stack) (*ast.Stack, bool) {
|
|
binder := NewBinder(c)
|
|
binder.Bind(doc, stack)
|
|
if c.Diag().Errors() > 0 {
|
|
// If any errors happened during binding, exit.
|
|
return stack, false
|
|
}
|
|
|
|
// TODO: perform semantic analysis on the bound tree.
|
|
|
|
return stack, true
|
|
}
|
|
|
|
// detectMufile locates the closest Mufile-looking file from the given path, searching "upwards" in the directory
|
|
// hierarchy. If no Mufile is found, an empty path is returned.
|
|
func (c *compiler) detectMufile(from string) string {
|
|
abs, err := filepath.Abs(from)
|
|
if err != nil {
|
|
glog.Fatalf("An IO error occurred while searching for a Mufile: %v", err)
|
|
return ""
|
|
}
|
|
|
|
// It's possible the target is already the file we seek; if so, return right away.
|
|
if c.isMufile(abs) {
|
|
return abs
|
|
}
|
|
|
|
curr := abs
|
|
for {
|
|
stop := false
|
|
|
|
// If the target is a directory, enumerate its files, checking each to see if it's a Mufile.
|
|
files, err := ioutil.ReadDir(curr)
|
|
if err != nil {
|
|
glog.Fatalf("An IO error occurred while searching for a Mufile: %v", err)
|
|
return ""
|
|
}
|
|
for _, file := range files {
|
|
name := file.Name()
|
|
path := filepath.Join(curr, name)
|
|
if c.isMufile(path) {
|
|
return path
|
|
} else if name == workspace.Muspace {
|
|
// If we hit a .muspace file, stop looking.
|
|
stop = true
|
|
}
|
|
}
|
|
|
|
// If we encountered a stop condition, break out of the loop.
|
|
if stop {
|
|
break
|
|
}
|
|
|
|
// If neither succeeded, keep looking in our parent directory.
|
|
curr = filepath.Dir(curr)
|
|
if os.IsPathSeparator(curr[len(curr)-1]) {
|
|
break
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
// isMufile returns true if the path references what appears to be a valid Mufile.
|
|
func (c *compiler) isMufile(path string) bool {
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
|
|
// Directories can't be Mufiles.
|
|
if info.IsDir() {
|
|
return false
|
|
}
|
|
|
|
// Ensure the base name is expected.
|
|
name := info.Name()
|
|
ext := filepath.Ext(name)
|
|
base := strings.TrimSuffix(name, ext)
|
|
if base != workspace.MufileBase {
|
|
if strings.EqualFold(base, workspace.MufileBase) {
|
|
// If the strings aren't equal, but case-insensitively match, issue a warning.
|
|
c.Diag().Warningf(errors.WarnIllegalMufileCasing.WithFile(name))
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Check all supported extensions.
|
|
for _, mufileExt := range workspace.MufileExts {
|
|
if name == workspace.MufileBase+mufileExt {
|
|
return true
|
|
}
|
|
}
|
|
|
|
// If we got here, it means the base name matched, but not the extension. Warn and return.
|
|
c.Diag().Warningf(errors.WarnIllegalMufileExt.WithFile(name), ext)
|
|
return false
|
|
}
|