Move all cloud switching to mu/x MuPackage

In the old system, the core runtime/toolset understood that we are targeting
specific cloud providers at a very deep level.  In fact, the whole code-generation
phase of the compiler was based on it.

In the new system, this difference is less of a "special" concern, and more of
a general one of mapping MuIL objects to resource providers, and letting *them*
gather up any configuration they need in a more general purpose way.

Therefore, most of this stuff can go.  I've merged in a small amount of it to
the mu/x MuPackage, since that has to switch on cloud IaaS and CaaS providers in
order to decide what kind of resources to provision.  For example, it has a
mu.x.Cluster stack type that itself provisions a lot of the barebone essential
resources, like a virtual private cloud and its associated networking components.

I suspect *some* knowledge of this will surface again as we implement more
runtime presence (discovery, etc).  But for the time being, it's a distraction
getting the core model running.  I've retained some of the old AWS code in the
new pkg/resource/providers/aws package, in case I want to reuse some of it when
implementing our first AWS resource providers.  (Although we won't be using
CloudFormation, some of the name generation code might be useful.)  So, the
ships aren't completely burned to the ground, but they are certainly on 🔥.
This commit is contained in:
joeduffy 2017-01-20 09:46:59 -08:00
parent 59aa09008c
commit 729af81e44
29 changed files with 98 additions and 587 deletions

1
.gitignore vendored
View file

@ -1,3 +1,4 @@
*.swp
mu
vendor/

View file

@ -1,109 +0,0 @@
// Copyright 2016 Marapongo, Inc. All rights reserved.
package cmd
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/golang/glog"
"github.com/spf13/cobra"
"github.com/marapongo/mu/pkg/cmdutil"
"github.com/marapongo/mu/pkg/compiler"
)
// defaultIn is where the Mu compiler looks for inputs by default.
const defaultInp = "."
// defaultOutput is where the Mu compiler places build artifacts by default.
const defaultOutp = ".mu"
func newBuildCmd() *cobra.Command {
var outp string
var cluster string
var targetArch string
var cmd = &cobra.Command{
Use: "build [source] -- [args]",
Short: "Compile a Mu Stack",
Run: func(cmd *cobra.Command, args []string) {
flags := cmd.Flags()
ddash := flags.ArgsLenAtDash()
// If there's a --, we need to separate out the command args from the stack args.
var sargs []string
if ddash != -1 {
sargs = args[ddash:]
args = args[0:ddash]
}
// Fetch the input source directory.
inp := defaultInp
if len(args) > 0 {
inp = args[0]
}
abs, err := filepath.Abs(inp)
if err != nil {
glog.Fatal(err)
}
opts := compiler.DefaultOptions()
// Set the cluster and architecture if specified.
opts.Cluster = cluster
cmdutil.SetCloudArchOptions(targetArch, opts)
// See if there are any arguments and, if so, accumulate them.
if len(sargs) > 0 {
opts.Args = make(map[string]interface{})
// TODO[marapongo/mu#7]: This is a very rudimentary parser. We can and should do better.
for i := 0; i < len(sargs); i++ {
sarg := sargs[i]
// Eat - or -- at the start.
if sarg[0] == '-' {
sarg = sarg[1:]
if sarg[0] == '-' {
sarg = sarg[1:]
}
}
// Now find an k=v, and split the k/v part.
if eq := strings.IndexByte(sarg, '='); eq != -1 {
opts.Args[sarg[:eq]] = sarg[eq+1:]
} else {
// No =; if the next arg doesn't start with '-', use it. Else it must be a boolean "true".
if i+1 < len(sargs) && sargs[i+1][0] != '-' {
opts.Args[sarg] = sargs[i+1]
i++
} else {
// TODO(joe): support --no-key style "false"s.
opts.Args[sarg] = "true"
}
}
}
}
// Now new up a compiler and actually perform the build.
mup, err := compiler.New(abs, opts)
if err != nil {
fmt.Fprintf(os.Stderr, "fatal: %v", err)
}
mup.Compile()
},
}
cmd.PersistentFlags().StringVar(
&outp, "out", defaultOutp,
"The directory in which to place build artifacts")
cmd.PersistentFlags().StringVarP(
&cluster, "cluster", "c", "",
"Generate output for an existing, named cluster")
cmd.PersistentFlags().StringVarP(
&targetArch, "target", "t", "",
"Generate output for the target cloud architecture (format: \"cloud[:scheduler]\")")
return cmd
}

View file

@ -1,17 +0,0 @@
// Copyright 2016 Marapongo, Inc. All rights reserved.
module mu
import mu/clouds/aws
// A base Mu cluster that can run on any cloud/scheduler target.
service Cluster {
new() {
switch context.arch.cloud {
case "aws":
cf := new aws.Cluster {}
default:
mu.Panic("Unrecognized cloud target: %v", context.arch.cloud)
}
}
}

View file

@ -1,47 +0,0 @@
// Copyright 2016 Marapongo, Inc. All rights reserved.
import * as mu from 'mu';
import * as aws from 'mu-aws';
// A base Mu cluster, ready to host stacks.
// @name: mu/cluster
export default class Cluster extends mu.Stack {
constructor(ctx: mu.Context) {
super(ctx);
switch (ctx.arch.cloud) {
case mu.clouds.AWS:
this.createAWSCloudResources(ctx);
break;
default:
throw new Error(`Unrecognized cloud target: ctx.arch.cloud`);
}
}
// This function creates all of the basic resources necessary for an AWS cluster ready to host Mu stacks.
private createAWSCloudResources(ctx: mu.Context): void {
// First set up a VPC with a single subnet.
let cidr = "10.0.0.0/16";
let vpc = new aws.ec2.VPC(ctx, { name: `${ctx.cluster.name}-VPC`, cidrBlock: cidr });
let subnet = new aws.ec2.Subnet(ctx, { name: `${ctx.cluster.name}-Subnet`, vpc: vpc, cidrBlock: cidr });
// Now create an Internet-facing gateway to expose this cluster's subnet to Internet traffic.
let internetGateway = new aws.ec2.InternetGateway(ctx, { name: `${ctx.cluster.name}-InternetGateway` });
let vpcGatewayAttachment = new aws.ec2.VPCGatewayAttachment(
ctx, { internetGateway: internetGateway, vpc: vpc });
let routeTable = new aws.ec2.RouteTable(ctx, { name: `${ctx.cluster.name}-RouteTable`, vpc: vpc });
let route = new aws.ec2.Route(ctx, {
destinationCidrBlock: "0.0.0.0/0",
internetGateway: internetGateway,
vpcGatewayAttachment: vpcGatewayAttachment,
routeTable: routeTable,
});
// Finally, create a sole security group to use for everything by default.
let securityGroup = new aws.ec2.SecurityGroup(ctx, {
name: `${ctx.cluster.name}-SecurityGroup`,
vpc: vpc,
groupDescription: "The primary cluster's security group.",
});
}
}

View file

@ -1,6 +1,4 @@
// Copyright 2016 Marapongo, Inc. All rights reserved.
export interface Cluster {
name: string;
}
export let todo = "todo";

82
lib/mu/x/cluster.ts Normal file
View file

@ -0,0 +1,82 @@
// Copyright 2016 Marapongo, Inc. All rights reserved.
import * as mu from 'mu';
import * as aws from 'mu-aws';
// A base Mu cluster, ready to host stacks.
export default class Cluster extends mu.Stack {
constructor(args: ClusterArgs) {
super();
// TODO: support anonymous clusters (e.g., for local testing).
// TODO: load cluster targeting information from other places:
// 1) workspace settings (e.g., map keyed by cluster name).
// 2) general configuration system (e.g., defaults).
switch(args.arch.cloud) {
case "aws":
this.createAWSCloudResources(args);
break;
default:
throw new Error(`Unrecognized/unimplemented cloud target: ${args.arch.cloud}`);
}
}
// This function creates all of the basic resources necessary for an AWS cluster ready to host Mu stacks.
private createAWSCloudResources(args: ClusterArgs): void {
// First set up a VPC with a single subnet.
let cidr = "10.0.0.0/16";
let vpc = new aws.ec2.VPC({ name: `${args.name}-VPC`, cidrBlock: cidr });
let subnet = new aws.ec2.Subnet({ name: `${args.name}-Subnet`, vpc: vpc, cidrBlock: cidr });
// Now create an Internet-facing gateway to expose this cluster's subnet to Internet traffic.
let internetGateway = new aws.ec2.InternetGateway({ name: `${args.name}-InternetGateway` });
let vpcGatewayAttachment = new aws.ec2.VPCGatewayAttachment(
{ internetGateway: internetGateway, vpc: vpc });
let routeTable = new aws.ec2.RouteTable({ name: `${args.name}-RouteTable`, vpc: vpc });
let route = new aws.ec2.Route({
destinationCidrBlock: "0.0.0.0/0",
internetGateway: internetGateway,
vpcGatewayAttachment: vpcGatewayAttachment,
routeTable: routeTable,
});
// Finally, create a sole security group to use for everything by default.
let securityGroup = new aws.ec2.SecurityGroup({
name: `${args.name}-SecurityGroup`,
vpc: vpc,
groupDescription: "The primary cluster's security group.",
});
}
}
export interface ClusterArgs {
name: string; // the cluster name.
arch: Arch; // the required architecture to target.
}
// Arch is the target cloud "architecture" to target. It combines a cloud IaaS provider with an optional
// cloud CaaS container management/scheduling layer. All mu/x abstractions switch on these to do the right thing.
export interface Arch {
cloud: Cloud; // the cloud IaaS provider.
scheduler?: Scheduler; // the optional cloud CaaS scheduler.
}
// Cloud specifies a cloud infrastructure (IaaS) to target.
export type Cloud =
"aws" | // Amazon Web Services
"gcp" | // Google Cloud Platform
"azure" | // Microsoft Azure
"vmware" // VMWare vSphere, etc.
;
// Scheduler specifies a cloud container management/scheduling layer (CaaS) to target.
export type Scheduler =
"swarm" | // Docker Swarm
"kubernetes" | // Kubernetes
"mesos" | // Apache Mesos
"awsecs" | // Amazon Elastic Container Service (only valid when targeting aws)
"gcpgke" | // Google Container Engine (only valid when targeting gcp)
"azurecs" // Microsoft Azure Container Service (only valid when targeting azure)
;

11
lib/mu/x/index.ts Normal file
View file

@ -0,0 +1,11 @@
// Copyright 2016 Marapongo, Inc. All rights reserved.
// Export some things directly.
export * from "./cluster";
// Export top-level submodules.
import * as bucket from "./bucket";
import * as clouds from "./clouds";
import * as schedulers from "./schedulers";
export {bucket, clouds, schedulers};

View file

@ -1,50 +0,0 @@
// Copyright 2016 Marapongo, Inc. All rights reserved.
package cmdutil
import (
"fmt"
"os"
"strings"
"github.com/marapongo/mu/pkg/compiler"
"github.com/marapongo/mu/pkg/compiler/backends"
"github.com/marapongo/mu/pkg/compiler/backends/clouds"
"github.com/marapongo/mu/pkg/compiler/backends/schedulers"
)
func SetCloudArchOptions(arch string, opts *compiler.Options) {
// If an architecture was specified, parse the pieces and set the options. This isn't required because stacks
// and workspaces can have defaults. This simply overrides or provides one where none exists.
if arch != "" {
// The format is "cloud[:scheduler]"; parse out the pieces.
var cloud string
var scheduler string
if delim := strings.IndexRune(arch, ':'); delim != -1 {
cloud = arch[:delim]
scheduler = arch[delim+1:]
} else {
cloud = arch
}
cloudArch, ok := clouds.Values[cloud]
if !ok {
fmt.Fprintf(os.Stderr, "Unrecognized cloud arch '%v'\n", cloud)
os.Exit(-1)
}
var schedulerArch schedulers.Arch
if scheduler != "" {
schedulerArch, ok = schedulers.Values[scheduler]
if !ok {
fmt.Fprintf(os.Stderr, "Unrecognized cloud scheduler arch '%v'\n", scheduler)
os.Exit(-1)
}
}
opts.Arch = backends.Arch{
Cloud: cloudArch,
Scheduler: schedulerArch,
}
}
}

View file

@ -1,22 +0,0 @@
// Copyright 2016 Marapongo, Inc. All rights reserved.
package backends
import (
"github.com/marapongo/mu/pkg/compiler/backends/clouds"
"github.com/marapongo/mu/pkg/compiler/backends/schedulers"
)
// Arch is the target cloud "architecture" we are compiling against.
type Arch struct {
Cloud clouds.Arch
Scheduler schedulers.Arch
}
func (a Arch) String() string {
s := clouds.Names[a.Cloud]
if a.Scheduler != schedulers.None {
s += ":" + schedulers.Names[a.Scheduler]
}
return s
}

View file

@ -1,13 +0,0 @@
// Copyright 2016 Marapongo, Inc. All rights reserved.
package clouds
import (
"github.com/marapongo/mu/pkg/compiler/core"
)
// Cloud is an interface for providers that can target a Mu stack to a specific cloud IaaS.
type Cloud interface {
core.Backend
Arch() Arch
}

View file

@ -1,40 +0,0 @@
// Copyright 2016 Marapongo, Inc. All rights reserved.
package clouds
// Arch selects a cloud infrastructure to target when compiling.
type Arch int
const (
None Arch = iota // no target specified.
AWS // Amazon Web Services.
GCP // Google Cloud Platform.
Azure // Microsoft Azure.
VMWare // VMWare vSphere, etc.
)
const (
none = ""
aws = "aws"
gcp = "gcp"
azure = "azure"
vmware = "vmware"
)
// Names maps Archs to human-friendly names.
var Names = map[Arch]string{
None: none,
AWS: aws,
GCP: gcp,
Azure: azure,
VMWare: vmware,
}
// Values maps human-friendly names to the Archs for those names.
var Values = map[string]Arch{
none: None,
aws: AWS,
gcp: GCP,
azure: Azure,
vmware: VMWare,
}

View file

@ -1,49 +0,0 @@
// Copyright 2016 Marapongo, Inc. All rights reserved.
package backends
import (
"github.com/marapongo/mu/pkg/compiler/backends/clouds"
"github.com/marapongo/mu/pkg/compiler/backends/clouds/aws"
"github.com/marapongo/mu/pkg/compiler/backends/schedulers"
"github.com/marapongo/mu/pkg/compiler/backends/schedulers/awsecs"
"github.com/marapongo/mu/pkg/compiler/core"
"github.com/marapongo/mu/pkg/diag"
"github.com/marapongo/mu/pkg/util/contract"
)
func New(arch Arch, d diag.Sink) core.Backend {
// Bind to the cloud provider.
var cloud clouds.Cloud
switch arch.Cloud {
case clouds.AWS:
// TODO(joe): come up with a way to get options from CLI/workspace/etc. to here.
cloud = aws.New(d, aws.Options{})
case clouds.None:
contract.Failf("Expected a non-None cloud architecture")
default:
contract.Failf("Cloud architecture '%v' not yet supported", clouds.Names[arch.Cloud])
}
contract.Assert(cloud != nil)
contract.Assert(cloud.Arch() == arch.Cloud)
// Set the backend to the cloud provider.
var be core.Backend = cloud
// Now bind to the scheduler, if any, wrapping the cloud and replacing the backend.
var scheduler schedulers.Scheduler
switch arch.Scheduler {
case schedulers.None:
// Nothing to do.
case schedulers.AWSECS:
scheduler = awsecs.New(d, cloud)
default:
contract.Failf("Scheduler architecture '%v' not yet supported", schedulers.Names[arch.Scheduler])
}
if scheduler != nil {
contract.Assert(scheduler.Arch() == arch.Scheduler)
be = scheduler
}
return be
}

View file

@ -1,42 +0,0 @@
// Copyright 2016 Marapongo, Inc. All rights reserved.
package awsecs
import (
"github.com/marapongo/mu/pkg/compiler/backends/clouds"
"github.com/marapongo/mu/pkg/compiler/backends/schedulers"
"github.com/marapongo/mu/pkg/compiler/core"
"github.com/marapongo/mu/pkg/compiler/errors"
"github.com/marapongo/mu/pkg/diag"
)
// New returns a fresh instance of an AWS ECS Scheduler implementation. This requires an AWS backend, since ECS only
// works in an AWS environment. The code-gen outputs are idiomatic ECS, using Tasks and so on.
//
// For more details, see https://github.com/marapongo/mu/blob/master/docs/targets.md#aws-ec2-container-service-ecs
func New(d diag.Sink, be clouds.Cloud) schedulers.Scheduler {
arch := be.Arch()
if arch != clouds.AWS {
d.Errorf(errors.ErrorIllegalCloudSchedulerCombination, clouds.Names[arch], "awsecs")
}
return &awsECSScheduler{d: d, be: be}
}
type awsECSScheduler struct {
schedulers.Scheduler
d diag.Sink
be clouds.Cloud // an AWS cloud provider.
}
func (s *awsECSScheduler) Arch() schedulers.Arch {
return schedulers.AWSECS
}
func (s *awsECSScheduler) Diag() diag.Sink {
return s.d
}
func (s *awsECSScheduler) CodeGen(comp core.Compiland) {
// For now, simply rely on the underlying AWS code-generator to do its thing.
s.be.CodeGen(comp)
}

View file

@ -1,13 +0,0 @@
// Copyright 2016 Marapongo, Inc. All rights reserved.
package schedulers
import (
"github.com/marapongo/mu/pkg/compiler/core"
)
// Scheduler is an interface for providers that can target a Mu stack to a specific cloud CaaS.
type Scheduler interface {
core.Backend
Arch() Arch
}

View file

@ -1,48 +0,0 @@
// Copyright 2016 Marapongo, Inc. All rights reserved.
package schedulers
// selects a cloud scheduler to target when compiling.
type Arch int
const (
None Arch = iota // no scheduler, just use native VMs.
DockerSwarm // Docker Swarm
Kubernetes // Kubernetes
Mesos // Apache Mesos
AWSECS // Amazon Elastic Container Service (only valid for AWS)
GCPGKE // Google Container Engine (only valid for GCP)
AzureACS // Microsoft Azure Container Service (only valid for Azure)
)
const (
none = ""
dockerSwarm = "swarm"
kubernetes = "kubernetes"
mesos = "mesos"
awsECS = "ecs"
gcpGKE = "gke"
azureACS = "acs"
)
// Names maps s to human-friendly names.
var Names = map[Arch]string{
None: none,
DockerSwarm: dockerSwarm,
Kubernetes: kubernetes,
Mesos: mesos,
AWSECS: awsECS,
GCPGKE: gcpGKE,
AzureACS: azureACS,
}
// Values maps human-friendly names to the s for those names.
var Values = map[string]Arch{
none: None,
dockerSwarm: DockerSwarm,
kubernetes: Kubernetes,
mesos: Mesos,
awsECS: AWSECS,
gcpGKE: GCPGKE,
azureACS: AzureACS,
}

View file

@ -1,113 +0,0 @@
// Copyright 2016 Marapongo, Inc. All rights reserved.
package compiler
import (
"github.com/satori/go.uuid"
"github.com/marapongo/mu/pkg/compiler/backends"
"github.com/marapongo/mu/pkg/compiler/backends/clouds"
"github.com/marapongo/mu/pkg/compiler/backends/schedulers"
"github.com/marapongo/mu/pkg/compiler/errors"
"github.com/marapongo/mu/pkg/config"
"github.com/marapongo/mu/pkg/workspace"
)
// detectClusterArch uses a variety of mechanisms to discover the target architecture, returning it. If no
// architecture was discovered, an error is issued, and the bool return will be false.
func (c *compiler) detectClusterArch(w workspace.W) (*config.Cluster, backends.Arch) {
// Cluster and architectures settings may come from one of two places, in order of search preference:
// 1) command line arguments.
// 2) cluster-wide settings in a workspace.
var arch backends.Arch
var cluster *config.Cluster
// If a cluster was specified, look it up and load up its options.
clname := c.Options().Cluster
if clname != "" {
if cl, exists := w.Settings().Clusters[clname]; exists {
cluster = cl
} else {
c.Diag().Errorf(errors.ErrorClusterNotFound, clname)
return nil, arch
}
}
// If no cluster was specified or discovered yet, see if there is a default one to use.
if cluster == nil {
for _, cl := range w.Settings().Clusters {
if cl.Default {
cluster = cl
break
}
}
}
if cluster == nil {
// If no target was found, and we don't have an architecture, error out.
if arch.Cloud == clouds.None {
c.Diag().Errorf(errors.ErrorMissingTarget)
return nil, arch
}
// If we got here, generate an "anonymous" cluster, so that we at least have a name.
cluster = c.newAnonCluster(arch)
} else {
// If a target was found, go ahead and extract and validate the target architecture.
a, ok := c.extractClusterArch(cluster, arch)
if !ok {
return nil, arch
}
arch = a
}
return cluster, arch
}
// newAnonCluster creates an anonymous cluster for stacks that didn't declare one.
func (c *compiler) newAnonCluster(arch backends.Arch) *config.Cluster {
// TODO: ensure this is unique.
// TODO: we want to cache names somewhere (~/.mu/?) so that we can reuse temporary local stacks, etc.
return &config.Cluster{
Name: uuid.NewV4().String(),
Cloud: clouds.Names[arch.Cloud],
Scheduler: schedulers.Names[arch.Scheduler],
}
}
// extractClusterArch gets and validates the architecture from an existing target.
func (c *compiler) extractClusterArch(cluster *config.Cluster, existing backends.Arch) (backends.Arch, bool) {
targetCloud := existing.Cloud
targetScheduler := existing.Scheduler
// If specified, look up the cluster's architecture settings.
if cluster.Cloud != "" {
tc, ok := clouds.Values[cluster.Cloud]
if !ok {
c.Diag().Errorf(errors.ErrorUnrecognizedCloudArch, cluster.Cloud)
return existing, false
}
targetCloud = tc
}
if cluster.Scheduler != "" {
ts, ok := schedulers.Values[cluster.Scheduler]
if !ok {
c.Diag().Errorf(errors.ErrorUnrecognizedSchedulerArch, cluster.Scheduler)
return existing, false
}
targetScheduler = ts
}
// Ensure there aren't any conflicts, comparing compiler options to cluster settings.
tarch := backends.Arch{targetCloud, targetScheduler}
if targetCloud != existing.Cloud && existing.Cloud != clouds.None {
c.Diag().Errorf(errors.ErrorConflictingClusterArchSelection, existing, cluster.Name, tarch)
return tarch, false
}
if targetScheduler != existing.Scheduler && existing.Scheduler != schedulers.None {
c.Diag().Errorf(errors.ErrorConflictingClusterArchSelection, existing, cluster.Name, tarch)
return tarch, false
}
return tarch, true
}

View file

@ -3,16 +3,13 @@
package compiler
import (
"github.com/marapongo/mu/pkg/compiler/backends"
"github.com/marapongo/mu/pkg/diag"
)
// Options contains all of the settings a user can use to control the compiler's behavior.
type Options struct {
Diag diag.Sink // a sink to use for all diagnostics.
Arch backends.Arch // a target cloud architecture.
Cluster string // a named cluster with predefined settings to target.
Args map[string]interface{} // optional blueprint arguments passed at the CLI.
Diag diag.Sink // a sink to use for all diagnostics.
Args map[string]interface{} // optional blueprint arguments passed at the CLI.
}
// DefaultOptions returns the default set of compiler options.

View file

@ -1,14 +1,10 @@
// Copyright 2016 Marapongo, Inc. All rights reserved.
import { Arch } from './arch';
import { Cluster } from './cluster';
import { Options } from './options';
// A collection of information pertaining to the current compilation activity, like target cloud architecture, the
// cluster name, any compile-time options, and so on.
export interface Context {
arch: Arch; // the cloud architecture to target.
cluster: Cluster; // the cluster we will be deploying to.
options: Options; // any compiler options supplied.
}

View file

@ -1,14 +1,8 @@
// Copyright 2016 Marapongo, Inc. All rights reserved.
export * from './arch';
export * from './cluster';
export * from './context';
export * from './extension';
export * from './json';
export * from './options';
export * from './stack';
import * as clouds from './clouds';
import * as schedulers from './schedulers';
export { clouds, schedulers };

View file

@ -18,16 +18,11 @@
"files": [
"index.ts",
"arch.ts",
"cluster.ts",
"context.ts",
"extension.ts",
"json.ts",
"options.ts",
"stack.ts",
"clouds/index.ts",
"schedulers/index.ts"
"stack.ts"
]
}