maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Do not share maps so tests can run in parallel (#8136)

Browse source 
* Do not share maps so tests can run in parallel

* Fix comment

* Try not to break dependencies

* Address PR feedback

* Fix downstream compilation failure

* Fix lint

* Address PR feedback
This commit is contained in:
Anton Tayanovskyy 2021-10-06 15:03:21 +00:00 committed by GitHub
parent b6c6108edd
commit 5f9f38c9c9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 253 additions and 153 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
pkg/codegen/docs/examples.go
View file

@ -43,12 +43,12 @@ type docInfo struct {
importDetails string
}
func decomposeDocstring(docstring string) docInfo {
func (dctx *docGenContext) decomposeDocstring(docstring string) docInfo {
if docstring == "" {
return docInfo{}
}
languages := codegen.NewStringSet(snippetLanguages...)
languages := codegen.NewStringSet(dctx.snippetLanguages...)
source := []byte(docstring)
parsed := schema.ParseDocs(source)
@ -70,7 +70,7 @@ func decomposeDocstring(docstring string) docInfo {
if exampleShortcode == nil {
exampleShortcode, title, snippets = shortcode, "", map[string]string{}
} else if !enter && shortcode == exampleShortcode {
for _, l := range snippetLanguages {
for _, l := range dctx.snippetLanguages {
if _, ok := snippets[l]; !ok {
snippets[l] = defaultMissingExampleSnippetPlaceholder
}

311
pkg/codegen/docs/gen.go
View file

@ -41,37 +41,23 @@ import (
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
)
var (
// modules is a map of a module name and information
// about it. This is crux of all API docs generation
// as the modContext carries information about the resources,
// functions, as well other modules within each module.
modules map[string]*modContext
// Populated in auto-generated `packaged.go`
var packagedTemplates map[string][]byte
supportedLanguages = []string{"csharp", "go", "nodejs", "python"}
snippetLanguages = []string{"csharp", "go", "python", "typescript"}
templates *template.Template
packagedTemplates map[string][]byte
docHelpers map[string]codegen.DocLanguageHelper
func init() {
packagedTemplates = map[string][]byte{}
}
// The language-specific info objects for a certain package (provider).
goPkgInfo go_gen.GoPackageInfo
csharpPkgInfo dotnet.CSharpPackageInfo
nodePkgInfo nodejs.NodePackageInfo
pythonPkgInfo python.PackageInfo
// langModuleNameLookup is a map of module name to its language-specific
// name.
langModuleNameLookup map[string]string
// TODO[pulumi/pulumi#7813]: Remove this lookup once display name is available in
// the Pulumi schema.
//
// NOTE: For the time being this lookup map and the one used by the resourcedocsgen
// tool in `pulumi/docs` must be kept up-to-date.
//
// titleLookup is a map of package name to the desired display name
// for display in the TOC menu under API Reference.
titleLookup = map[string]string{
// TODO[pulumi/pulumi#7813]: Remove this lookup once display name is available in
// the Pulumi schema.
//
// NOTE: For the time being this lookup map and the one used by the resourcedocsgen
// tool in `pulumi/docs` must be kept up-to-date.
//
// titleLookup is a map of package name to the desired display name
// for display in the TOC menu under API Reference.
func titleLookup(shortName string) (string, bool) {
v, ok := map[string]string{
"aiven": "Aiven",
"akamai": "Akamai",
"alicloud": "Alibaba Cloud",
@ -136,14 +122,52 @@ var (
"vsphere": "vSphere",
"wavefront": "Wavefront",
"yandex": "Yandex",
}
// Property anchor tag separator, used in a property anchor tag id to separate the
// property and language (e.g. property~lang).
propertyLangSeparator = "_"
)
}[shortName]
return v, ok
}
func init() {
docHelpers = make(map[string]codegen.DocLanguageHelper)
// Property anchor tag separator, used in a property anchor tag id to separate the
// property and language (e.g. property~lang).
const propertyLangSeparator = "_"
type docGenContext struct {
internalModMap map[string]*modContext
supportedLanguages []string
snippetLanguages []string
templates *template.Template
docHelpers map[string]codegen.DocLanguageHelper
// The language-specific info objects for a certain package (provider).
goPkgInfo go_gen.GoPackageInfo
csharpPkgInfo dotnet.CSharpPackageInfo
nodePkgInfo nodejs.NodePackageInfo
pythonPkgInfo python.PackageInfo
// langModuleNameLookup is a map of module name to its language-specific
// name.
langModuleNameLookup map[string]string
}
// modules is a map of a module name and information
// about it. This is crux of all API docs generation
// as the modContext carries information about the resources,
// functions, as well other modules within each module.
func (dctx *docGenContext) modules() map[string]*modContext {
return dctx.internalModMap
}
func (dctx *docGenContext) setModules(modules map[string]*modContext) {
m := map[string]*modContext{}
for k, v := range modules {
m[k] = v.withDocGenContext(dctx)
}
dctx.internalModMap = m
}
func newDocGenContext() *docGenContext {
supportedLanguages := []string{"csharp", "go", "nodejs", "python"}
docHelpers := make(map[string]codegen.DocLanguageHelper)
for _, lang := range supportedLanguages {
switch lang {
case "csharp":
@ -157,7 +181,12 @@ func init() {
}
}
langModuleNameLookup = map[string]string{}
return &docGenContext{
supportedLanguages: supportedLanguages,
snippetLanguages: []string{"csharp", "go", "python", "typescript"},
langModuleNameLookup: map[string]string{},
docHelpers: docHelpers,
}
}
type typeDetails struct {
@ -346,14 +375,29 @@ func (ss nestedTypeUsageInfo) contains(token string, input bool) bool {
}
type modContext struct {
pkg *schema.Package
mod string
inputTypes []*schema.ObjectType
resources []*schema.Resource
functions []*schema.Function
typeDetails map[*schema.ObjectType]*typeDetails
children []*modContext
tool string
pkg *schema.Package
mod string
inputTypes []*schema.ObjectType
resources []*schema.Resource
functions []*schema.Function
typeDetails map[*schema.ObjectType]*typeDetails
children []*modContext
tool string
docGenContext *docGenContext
}
func (mod *modContext) withDocGenContext(dctx *docGenContext) *modContext {
if mod == nil {
return nil
}
copy := *mod
copy.docGenContext = dctx
var children []*modContext
for _, c := range copy.children {
children = append(children, c.withDocGenContext(dctx))
}
copy.children = children
return &copy
}
func resourceName(r *schema.Resource) string {
@ -363,8 +407,8 @@ func resourceName(r *schema.Resource) string {
return strings.Title(tokenToName(r.Token))
}
func getLanguageDocHelper(lang string) codegen.DocLanguageHelper {
if h, ok := docHelpers[lang]; ok {
func (dctx *docGenContext) getLanguageDocHelper(lang string) codegen.DocLanguageHelper {
if h, ok := dctx.docHelpers[lang]; ok {
return h
}
panic(errors.Errorf("could not find a doc lang helper for %s", lang))
@ -391,9 +435,10 @@ func (mod *modContext) details(t *schema.ObjectType) *typeDetails {
// language-specific name using the language info, if any, for the
// current package.
func (mod *modContext) getLanguageModuleName(lang string) string {
dctx := mod.docGenContext
modName := mod.mod
lookupKey := lang + "_" + modName
if v, ok := langModuleNameLookup[lookupKey]; ok {
if v, ok := mod.docGenContext.langModuleNameLookup[lookupKey]; ok {
return v
}
@ -401,24 +446,24 @@ func (mod *modContext) getLanguageModuleName(lang string) string {
case "go":
// Go module names use lowercase.
modName = strings.ToLower(modName)
if override, ok := goPkgInfo.ModuleToPackage[modName]; ok {
if override, ok := dctx.goPkgInfo.ModuleToPackage[modName]; ok {
modName = override
}
case "csharp":
if override, ok := csharpPkgInfo.Namespaces[modName]; ok {
if override, ok := dctx.csharpPkgInfo.Namespaces[modName]; ok {
modName = override
}
case "nodejs":
if override, ok := nodePkgInfo.ModuleToPackage[modName]; ok {
if override, ok := dctx.nodePkgInfo.ModuleToPackage[modName]; ok {
modName = override
}
case "python":
if override, ok := pythonPkgInfo.ModuleNameOverrides[modName]; ok {
if override, ok := dctx.pythonPkgInfo.ModuleNameOverrides[modName]; ok {
modName = override
}
}
langModuleNameLookup[lookupKey] = modName
mod.docGenContext.langModuleNameLookup[lookupKey] = modName
return modName
}
@ -499,7 +544,7 @@ func (mod *modContext) cleanTypeString(t schema.Type, langTypeString, lang, modN
func (mod *modContext) typeString(t schema.Type, lang string, characteristics propertyCharacteristics, insertWordBreaks bool) propertyType {
t = codegen.PlainType(t)
docLanguageHelper := getLanguageDocHelper(lang)
docLanguageHelper := mod.docGenContext.getLanguageDocHelper(lang)
modName := mod.getLanguageModuleName(lang)
langTypeString := docLanguageHelper.GetLanguageTypeString(mod.pkg, modName, t, characteristics.input)
@ -587,7 +632,7 @@ const (
func (mod *modContext) genConstructorTS(r *schema.Resource, argsOptional bool) []formalParam {
name := resourceName(r)
docLangHelper := getLanguageDocHelper("nodejs")
docLangHelper := mod.docGenContext.getLanguageDocHelper("nodejs")
var argsType string
optsType := "CustomResourceOptions"
@ -654,7 +699,7 @@ func (mod *modContext) genConstructorGo(r *schema.Resource, argsOptional bool) [
argsFlag = "*"
}
docLangHelper := getLanguageDocHelper("go")
docLangHelper := mod.docGenContext.getLanguageDocHelper("go")
return []formalParam{
{
@ -710,7 +755,7 @@ func (mod *modContext) genConstructorCS(r *schema.Resource, argsOptional bool) [
argsFlag = "?"
}
docLangHelper := getLanguageDocHelper("csharp")
docLangHelper := mod.docGenContext.getLanguageDocHelper("csharp")
return []formalParam{
{
@ -744,7 +789,7 @@ func (mod *modContext) genConstructorCS(r *schema.Resource, argsOptional bool) [
}
func (mod *modContext) genConstructorPython(r *schema.Resource, argsOptional, argsOverload bool) []formalParam {
docLanguageHelper := getLanguageDocHelper("python")
docLanguageHelper := mod.docGenContext.getLanguageDocHelper("python")
isK8sOverlayMod := mod.isKubernetesOverlayModule()
isDockerImageResource := mod.pkg.Name == "docker" && resourceName(r) == "Image"
@ -831,6 +876,7 @@ func (mod *modContext) genConstructorPython(r *schema.Resource, argsOptional, ar
}
func (mod *modContext) genNestedTypes(member interface{}, resourceType bool) []docNestedType {
dctx := mod.docGenContext
tokens := nestedTypeUsageInfo{}
// Collect all of the types for this "member" as a map of resource names
// and if it appears in an input object and/or output object.
@ -847,7 +893,7 @@ func (mod *modContext) genNestedTypes(member interface{}, resourceType bool) []d
// Create a map to hold the per-language properties of this object.
props := make(map[string][]property)
for _, lang := range supportedLanguages {
for _, lang := range dctx.supportedLanguages {
props[lang] = mod.getProperties(typ.Properties, lang, true, true, false)
}
@ -864,8 +910,8 @@ func (mod *modContext) genNestedTypes(member interface{}, resourceType bool) []d
name := strings.Title(tokenToName(typ.Token))
enums := make(map[string][]enum)
for _, lang := range supportedLanguages {
docLangHelper := getLanguageDocHelper(lang)
for _, lang := range dctx.supportedLanguages {
docLangHelper := dctx.getLanguageDocHelper(lang)
var langEnumValues []enum
for _, e := range typ.Elements {
@ -911,6 +957,8 @@ func (mod *modContext) getProperties(properties []*schema.Property, lang string,
func (mod *modContext) getPropertiesWithIDPrefixAndExclude(properties []*schema.Property, lang string, input, nested,
isProvider bool, idPrefix string, exclude func(name string) bool) []property {
dctx := mod.docGenContext
if len(properties) == 0 {
return nil
}
@ -933,7 +981,7 @@ func (mod *modContext) getPropertiesWithIDPrefixAndExclude(properties []*schema.
characteristics := propertyCharacteristics{input: input}
langDocHelper := getLanguageDocHelper(lang)
langDocHelper := dctx.getLanguageDocHelper(lang)
name, err := langDocHelper.GetPropertyName(prop)
if err != nil {
panic(err)
@ -1023,11 +1071,12 @@ func getDockerImagePythonFormalParams() []formalParam {
// Returns the rendered HTML for the resource's constructor, as well as the specific arguments.
func (mod *modContext) genConstructors(r *schema.Resource, allOptionalInputs bool) (map[string]string, map[string][]formalParam) {
dctx := mod.docGenContext
renderedParams := make(map[string]string)
formalParams := make(map[string][]formalParam)
// Add an extra language for Python's ResourceArg __init__ overload.
langs := append(supportedLanguages, "pythonargs")
langs := append(dctx.supportedLanguages, "pythonargs")
for _, lang := range langs {
var (
@ -1061,11 +1110,11 @@ func (mod *modContext) genConstructors(r *schema.Resource, allOptionalInputs boo
for i, p := range params {
if i != 0 {
if err := templates.ExecuteTemplate(b, paramSeparatorTemplate, ps); err != nil {
if err := dctx.templates.ExecuteTemplate(b, paramSeparatorTemplate, ps); err != nil {
panic(err)
}
}
if err := templates.ExecuteTemplate(b, paramTemplate, p); err != nil {
if err := dctx.templates.ExecuteTemplate(b, paramTemplate, p); err != nil {
panic(err)
}
}
@ -1079,10 +1128,12 @@ func (mod *modContext) genConstructors(r *schema.Resource, allOptionalInputs boo
// getConstructorResourceInfo returns a map of per-language information about
// the resource being constructed.
func (mod *modContext) getConstructorResourceInfo(resourceTypeName string) map[string]propertyType {
dctx := mod.docGenContext
resourceMap := make(map[string]propertyType)
resourceDisplayName := resourceTypeName
for _, lang := range supportedLanguages {
for _, lang := range dctx.supportedLanguages {
// Use the module to package lookup to transform the module name to its normalized package name.
modName := mod.getLanguageModuleName(lang)
// Reset the type name back to the display name.
@ -1093,7 +1144,7 @@ func (mod *modContext) getConstructorResourceInfo(resourceTypeName string) map[s
// Intentionally left blank.
case "csharp":
namespace := title(mod.pkg.Name, lang)
if ns, ok := csharpPkgInfo.Namespaces[mod.pkg.Name]; ok {
if ns, ok := dctx.csharpPkgInfo.Namespaces[mod.pkg.Name]; ok {
namespace = ns
}
if mod.mod == "" {
@ -1119,7 +1170,8 @@ func (mod *modContext) getConstructorResourceInfo(resourceTypeName string) map[s
}
func (mod *modContext) getTSLookupParams(r *schema.Resource, stateParam string) []formalParam {
docLangHelper := getLanguageDocHelper("nodejs")
dctx := mod.docGenContext
docLangHelper := dctx.getLanguageDocHelper("nodejs")
return []formalParam{
{
@ -1155,7 +1207,8 @@ func (mod *modContext) getTSLookupParams(r *schema.Resource, stateParam string)
}
func (mod *modContext) getGoLookupParams(r *schema.Resource, stateParam string) []formalParam {
docLangHelper := getLanguageDocHelper("go")
dctx := mod.docGenContext
docLangHelper := dctx.getLanguageDocHelper("go")
return []formalParam{
{
@ -1198,7 +1251,8 @@ func (mod *modContext) getGoLookupParams(r *schema.Resource, stateParam string)
}
func (mod *modContext) getCSLookupParams(r *schema.Resource, stateParam string) []formalParam {
docLangHelper := getLanguageDocHelper("csharp")
dctx := mod.docGenContext
docLangHelper := dctx.getLanguageDocHelper("csharp")
return []formalParam{
{
@ -1234,9 +1288,10 @@ func (mod *modContext) getCSLookupParams(r *schema.Resource, stateParam string)
}
func (mod *modContext) getPythonLookupParams(r *schema.Resource, stateParam string) []formalParam {
dctx := mod.docGenContext
// The input properties for a resource needs to be exploded as
// individual constructor params.
docLanguageHelper := getLanguageDocHelper("python")
docLanguageHelper := dctx.getLanguageDocHelper("python")
params := make([]formalParam, 0, len(r.StateInputs.Properties))
for _, p := range r.StateInputs.Properties {
typ := docLanguageHelper.GetLanguageTypeString(mod.pkg, mod.mod, codegen.PlainType(codegen.OptionalType(p)), true /*input*/)
@ -1254,12 +1309,13 @@ func (mod *modContext) getPythonLookupParams(r *schema.Resource, stateParam stri
// genLookupParams generates a map of per-language way of rendering the formal parameters of the lookup function
// used to lookup an existing resource.
func (mod *modContext) genLookupParams(r *schema.Resource, stateParam string) map[string]string {
dctx := mod.docGenContext
lookupParams := make(map[string]string)
if r.StateInputs == nil {
return lookupParams
}
for _, lang := range supportedLanguages {
for _, lang := range dctx.supportedLanguages {
var (
paramTemplate string
params []formalParam
@ -1288,11 +1344,11 @@ func (mod *modContext) genLookupParams(r *schema.Resource, stateParam string) ma
n := len(params)
for i, p := range params {
if err := templates.ExecuteTemplate(b, paramTemplate, p); err != nil {
if err := dctx.templates.ExecuteTemplate(b, paramTemplate, p); err != nil {
panic(err)
}
if i != n-1 {
if err := templates.ExecuteTemplate(b, paramSeparatorTemplate, ps); err != nil {
if err := dctx.templates.ExecuteTemplate(b, paramSeparatorTemplate, ps); err != nil {
panic(err)
}
}
@ -1344,6 +1400,7 @@ func (mod *modContext) genResourceHeader(r *schema.Resource) header {
// genResource is the entrypoint for generating a doc for a resource
// from its Pulumi schema.
func (mod *modContext) genResource(r *schema.Resource) resourceDocArgs {
dctx := mod.docGenContext
// Create a resource module file into which all of this resource's types will go.
name := resourceName(r)
@ -1364,7 +1421,7 @@ func (mod *modContext) genResource(r *schema.Resource) resourceDocArgs {
Type: schema.StringType,
})
for _, lang := range supportedLanguages {
for _, lang := range dctx.supportedLanguages {
inputProps[lang] = mod.getProperties(r.InputProperties, lang, true, false, r.IsProvider)
outputProps[lang] = mod.getProperties(filteredOutputProps, lang, false, false, r.IsProvider)
if r.IsProvider {
@ -1400,7 +1457,7 @@ func (mod *modContext) genResource(r *schema.Resource) resourceDocArgs {
stateParam := name + "State"
docInfo := decomposeDocstring(r.Comment)
docInfo := dctx.decomposeDocstring(r.Comment)
data := resourceDocArgs{
Header: mod.genResourceHeader(r),
@ -1498,19 +1555,21 @@ func (fs fs) add(path string, contents []byte) {
// getModuleFileName returns the file name to use for a module.
func (mod *modContext) getModuleFileName() string {
dctx := mod.docGenContext
if !isKubernetesPackage(mod.pkg) {
return mod.mod
}
// For k8s packages, use the Go-language info to get the file name
// for the module.
if override, ok := goPkgInfo.ModuleToPackage[mod.mod]; ok {
if override, ok := dctx.goPkgInfo.ModuleToPackage[mod.mod]; ok {
return override
}
return mod.mod
}
func (mod *modContext) gen(fs fs) error {
dctx := mod.docGenContext
modName := mod.getModuleFileName()
addFile := func(name, contents string) {
@ -1525,7 +1584,7 @@ func (mod *modContext) gen(fs fs) error {
title := resourceName(r)
buffer := &bytes.Buffer{}
err := templates.ExecuteTemplate(buffer, "resource.tmpl", data)
err := dctx.templates.ExecuteTemplate(buffer, "resource.tmpl", data)
if err != nil {
return err
}
@ -1538,7 +1597,7 @@ func (mod *modContext) gen(fs fs) error {
data := mod.genFunction(f)
buffer := &bytes.Buffer{}
err := templates.ExecuteTemplate(buffer, "function.tmpl", data)
err := dctx.templates.ExecuteTemplate(buffer, "function.tmpl", data)
if err != nil {
return err
}
@ -1549,7 +1608,7 @@ func (mod *modContext) gen(fs fs) error {
// Generate the index files.
idxData := mod.genIndex()
buffer := &bytes.Buffer{}
err := templates.ExecuteTemplate(buffer, "index.tmpl", idxData)
err := dctx.templates.ExecuteTemplate(buffer, "index.tmpl", idxData)
if err != nil {
return err
}
@ -1708,20 +1767,28 @@ func (mod *modContext) genIndex() indexData {
func formatTitleText(title string) string {
// If title not found in titleLookup map, default back to title given.
if val, ok := titleLookup[title]; ok {
if val, ok := titleLookup(title); ok {
return val
}
return title
}
func getMod(pkg *schema.Package, token string, tokenPkg *schema.Package, modules map[string]*modContext, tool string, add bool) *modContext {
func (dctx *docGenContext) getMod(
pkg *schema.Package,
token string,
tokenPkg *schema.Package,
modules map[string]*modContext,
tool string,
add bool) *modContext {
modName := pkg.TokenToModule(token)
mod, ok := modules[modName]
if !ok {
mod = &modContext{
pkg: pkg,
mod: modName,
tool: tool,
pkg: pkg,
mod: modName,
tool: tool,
docGenContext: dctx,
}
if modName != "" && tokenPkg == pkg {
@ -1732,7 +1799,7 @@ func getMod(pkg *schema.Package, token string, tokenPkg *schema.Package, modules
} else {
parentName = ":" + parentName + ":"
}
parent := getMod(pkg, parentName, tokenPkg, modules, tool, add)
parent := dctx.getMod(pkg, parentName, tokenPkg, modules, tool, add)
if add {
parent.children = append(parent.children, mod)
}
@ -1747,7 +1814,7 @@ func getMod(pkg *schema.Package, token string, tokenPkg *schema.Package, modules
return mod
}
func generateModulesFromSchemaPackage(tool string, pkg *schema.Package) map[string]*modContext {
func (dctx *docGenContext) generateModulesFromSchemaPackage(tool string, pkg *schema.Package) map[string]*modContext {
// Group resources, types, and functions into modules.
modules := map[string]*modContext{}
@ -1760,38 +1827,38 @@ func generateModulesFromSchemaPackage(tool string, pkg *schema.Package) map[stri
}); err != nil {
panic(err)
}
goPkgInfo, _ = pkg.Language["go"].(go_gen.GoPackageInfo)
csharpPkgInfo, _ = pkg.Language["csharp"].(dotnet.CSharpPackageInfo)
nodePkgInfo, _ = pkg.Language["nodejs"].(nodejs.NodePackageInfo)
pythonPkgInfo, _ = pkg.Language["python"].(python.PackageInfo)
dctx.goPkgInfo, _ = pkg.Language["go"].(go_gen.GoPackageInfo)
dctx.csharpPkgInfo, _ = pkg.Language["csharp"].(dotnet.CSharpPackageInfo)
dctx.nodePkgInfo, _ = pkg.Language["nodejs"].(nodejs.NodePackageInfo)
dctx.pythonPkgInfo, _ = pkg.Language["python"].(python.PackageInfo)
goLangHelper := getLanguageDocHelper("go").(*go_gen.DocLanguageHelper)
goLangHelper := dctx.getLanguageDocHelper("go").(*go_gen.DocLanguageHelper)
// Generate the Go package map info now, so we can use that to get the type string
// names later.
goLangHelper.GeneratePackagesMap(pkg, tool, goPkgInfo)
goLangHelper.GeneratePackagesMap(pkg, tool, dctx.goPkgInfo)
csharpLangHelper := getLanguageDocHelper("csharp").(*dotnet.DocLanguageHelper)
csharpLangHelper.Namespaces = csharpPkgInfo.Namespaces
csharpLangHelper := dctx.getLanguageDocHelper("csharp").(*dotnet.DocLanguageHelper)
csharpLangHelper.Namespaces = dctx.csharpPkgInfo.Namespaces
visitObjects := func(r *schema.Resource) {
visitObjectTypes(r.InputProperties, func(t schema.Type) {
switch T := t.(type) {
case *schema.ObjectType:
getMod(pkg, T.Token, T.Package, modules, tool, true).details(T).inputType = true
dctx.getMod(pkg, T.Token, T.Package, modules, tool, true).details(T).inputType = true
}
})
if r.StateInputs != nil {
visitObjectTypes(r.StateInputs.Properties, func(t schema.Type) {
switch T := t.(type) {
case *schema.ObjectType:
getMod(pkg, T.Token, T.Package, modules, tool, true).details(T).inputType = true
dctx.getMod(pkg, T.Token, T.Package, modules, tool, true).details(T).inputType = true
}
})
}
}
scanResource := func(r *schema.Resource) {
mod := getMod(pkg, r.Token, r.Package, modules, tool, true)
mod := dctx.getMod(pkg, r.Token, r.Package, modules, tool, true)
mod.resources = append(mod.resources, r)
visitObjects(r)
}
@ -1818,7 +1885,7 @@ func generateModulesFromSchemaPackage(tool string, pkg *schema.Package) map[stri
for _, f := range pkg.Functions {
if !f.IsMethod {
mod := getMod(pkg, f.Token, f.Package, modules, tool, true)
mod := dctx.getMod(pkg, f.Token, f.Package, modules, tool, true)
mod.functions = append(mod.functions, f)
}
}
@ -1827,7 +1894,7 @@ func generateModulesFromSchemaPackage(tool string, pkg *schema.Package) map[stri
for _, t := range pkg.Types {
switch typ := t.(type) {
case *schema.ObjectType:
mod := getMod(pkg, typ.Token, typ.Package, modules, tool, false)
mod := dctx.getMod(pkg, typ.Token, typ.Package, modules, tool, false)
if mod.details(typ).inputType {
mod.inputTypes = append(mod.inputTypes, typ)
}
@ -1837,8 +1904,8 @@ func generateModulesFromSchemaPackage(tool string, pkg *schema.Package) map[stri
return modules
}
func Initialize(tool string, pkg *schema.Package) {
templates = template.New("").Funcs(template.FuncMap{
func (dctx *docGenContext) initialize(tool string, pkg *schema.Package) {
dctx.templates = template.New("").Funcs(template.FuncMap{
"htmlSafe": func(html string) template.HTML {
// Markdown fragments in the templates need to be rendered as-is,
// so that html/template package doesn't try to inject data into it,
@ -1855,19 +1922,16 @@ func Initialize(tool string, pkg *schema.Package) {
}
for name, b := range packagedTemplates {
template.Must(templates.New(name).Parse(string(b)))
template.Must(dctx.templates.New(name).Parse(string(b)))
}
// Generate the modules from the schema, and for every module
// run the generator functions to generate markdown files.
modules = generateModulesFromSchemaPackage(tool, pkg)
dctx.setModules(dctx.generateModulesFromSchemaPackage(tool, pkg))
}
// GeneratePackage generates docs for each resource given the Pulumi
// schema. The returned map contains the filename with path as the key
// and the contents as its value.
func GeneratePackage(tool string, pkg *schema.Package) (map[string][]byte, error) {
if modules == nil {
func (dctx *docGenContext) generatePackage(tool string, pkg *schema.Package) (map[string][]byte, error) {
if dctx.modules() == nil {
return nil, errors.New("must call Initialize before generating the docs package")
}
@ -1875,7 +1939,7 @@ func GeneratePackage(tool string, pkg *schema.Package) (map[string][]byte, error
glog.V(3).Infoln("generating package docs now...")
files := fs{}
for _, mod := range modules {
for _, mod := range dctx.modules() {
if err := mod.gen(files); err != nil {
return nil, err
}
@ -1885,8 +1949,8 @@ func GeneratePackage(tool string, pkg *schema.Package) (map[string][]byte, error
}
// GeneratePackageTree returns a navigable structure starting from the top-most module.
func GeneratePackageTree() ([]PackageTreeItem, error) {
if modules == nil {
func (dctx *docGenContext) generatePackageTree() ([]PackageTreeItem, error) {
if dctx.modules() == nil {
return nil, errors.New("must call Initialize before generating the docs package")
}
@ -1894,7 +1958,7 @@ func GeneratePackageTree() ([]PackageTreeItem, error) {
var packageTree []PackageTreeItem
// "" indicates the top-most module.
if rootMod, ok := modules[""]; ok {
if rootMod, ok := dctx.modules()[""]; ok {
tree, err := generatePackageTree(*rootMod)
if err != nil {
glog.Errorf("Error generating the package tree for package: %v", err)
@ -1916,3 +1980,24 @@ func visitObjectTypes(properties []*schema.Property, visitor func(t schema.Type)
}
})
}
// Export a default static context so as not to break external
// consumers of this API; prefer *WithContext API internally to ensure
// tests can run in parallel.
var defaultContext = newDocGenContext()
func Initialize(tool string, pkg *schema.Package) {
defaultContext.initialize(tool, pkg)
}
// GeneratePackage generates docs for each resource given the Pulumi
// schema. The returned map contains the filename with path as the key
// and the contents as its value.
func GeneratePackage(tool string, pkg *schema.Package) (map[string][]byte, error) {
return defaultContext.generatePackage(tool, pkg)
}
// GeneratePackageTree returns a navigable structure starting from the top-most module.
func GeneratePackageTree() ([]PackageTreeItem, error) {
return defaultContext.generatePackageTree()
}

34
pkg/codegen/docs/gen_function.go
View file

@ -65,11 +65,12 @@ type functionDocArgs struct {
// getFunctionResourceInfo returns a map of per-language information about
// the resource being looked-up using a static "getter" function.
func (mod *modContext) getFunctionResourceInfo(f *schema.Function) map[string]propertyType {
dctx := mod.docGenContext
resourceMap := make(map[string]propertyType)
var resultTypeName string
for _, lang := range supportedLanguages {
docLangHelper := getLanguageDocHelper(lang)
for _, lang := range dctx.supportedLanguages {
docLangHelper := dctx.getLanguageDocHelper(lang)
switch lang {
case "nodejs":
resultTypeName = docLangHelper.GetResourceFunctionResultName(mod.mod, f)
@ -77,7 +78,7 @@ func (mod *modContext) getFunctionResourceInfo(f *schema.Function) map[string]pr
resultTypeName = docLangHelper.GetResourceFunctionResultName(mod.mod, f)
case "csharp":
namespace := title(mod.pkg.Name, lang)
if ns, ok := csharpPkgInfo.Namespaces[mod.pkg.Name]; ok {
if ns, ok := dctx.csharpPkgInfo.Namespaces[mod.pkg.Name]; ok {
namespace = ns
}
resultTypeName = docLangHelper.GetResourceFunctionResultName(mod.mod, f)
@ -106,8 +107,9 @@ func (mod *modContext) getFunctionResourceInfo(f *schema.Function) map[string]pr
}
func (mod *modContext) genFunctionTS(f *schema.Function, funcName string) []formalParam {
dctx := mod.docGenContext
argsType := title(funcName+"Args", "nodejs")
docLangHelper := getLanguageDocHelper("nodejs")
docLangHelper := dctx.getLanguageDocHelper("nodejs")
var params []formalParam
if f.Inputs != nil {
params = append(params, formalParam{
@ -166,8 +168,9 @@ func (mod *modContext) genFunctionGo(f *schema.Function, funcName string) []form
}
func (mod *modContext) genFunctionCS(f *schema.Function, funcName string) []formalParam {
dctx := mod.docGenContext
argsType := funcName + "Args"
docLangHelper := getLanguageDocHelper("csharp")
docLangHelper := dctx.getLanguageDocHelper("csharp")
var params []formalParam
if f.Inputs != nil {
params = append(params, formalParam{
@ -193,7 +196,8 @@ func (mod *modContext) genFunctionCS(f *schema.Function, funcName string) []form
}
func (mod *modContext) genFunctionPython(f *schema.Function, resourceName string) []formalParam {
docLanguageHelper := getLanguageDocHelper("python")
dctx := mod.docGenContext
docLanguageHelper := dctx.getLanguageDocHelper("python")
var params []formalParam
// Some functions don't have any inputs other than the InvokeOptions.
@ -229,9 +233,10 @@ func (mod *modContext) genFunctionPython(f *schema.Function, resourceName string
// genFunctionArgs generates the arguments string for a given Function that can be
// rendered directly into a template.
func (mod *modContext) genFunctionArgs(f *schema.Function, funcNameMap map[string]string) map[string]string {
dctx := mod.docGenContext
functionParams := make(map[string]string)
for _, lang := range supportedLanguages {
for _, lang := range dctx.supportedLanguages {
var (
paramTemplate string
params []formalParam
@ -256,7 +261,7 @@ func (mod *modContext) genFunctionArgs(f *schema.Function, funcNameMap map[strin
paramTemplate = "py_formal_param"
paramSeparatorTemplate = "py_param_separator"
docHelper := getLanguageDocHelper(lang)
docHelper := dctx.getLanguageDocHelper(lang)
funcName := docHelper.GetFunctionName(mod.mod, f)
ps = paramSeparator{Indent: strings.Repeat(" ", len("def (")+len(funcName))}
}
@ -267,11 +272,11 @@ func (mod *modContext) genFunctionArgs(f *schema.Function, funcNameMap map[strin
}
for i, p := range params {
if err := templates.ExecuteTemplate(b, paramTemplate, p); err != nil {
if err := dctx.templates.ExecuteTemplate(b, paramTemplate, p); err != nil {
panic(err)
}
if i != n-1 {
if err := templates.ExecuteTemplate(b, paramSeparatorTemplate, ps); err != nil {
if err := dctx.templates.ExecuteTemplate(b, paramSeparatorTemplate, ps); err != nil {
panic(err)
}
}
@ -307,9 +312,10 @@ func (mod *modContext) genFunctionHeader(f *schema.Function) header {
// genFunction is the main entrypoint for generating docs for a Function.
// Returns args type that can be used to execute the `function.tmpl` doc template.
func (mod *modContext) genFunction(f *schema.Function) functionDocArgs {
dctx := mod.docGenContext
inputProps := make(map[string][]property)
outputProps := make(map[string][]property)
for _, lang := range supportedLanguages {
for _, lang := range dctx.supportedLanguages {
if f.Inputs != nil {
inputProps[lang] = mod.getProperties(f.Inputs.Properties, lang, true, false, false)
}
@ -322,8 +328,8 @@ func (mod *modContext) genFunction(f *schema.Function) functionDocArgs {
// Generate the per-language map for the function name.
funcNameMap := map[string]string{}
for _, lang := range supportedLanguages {
docHelper := getLanguageDocHelper(lang)
for _, lang := range dctx.supportedLanguages {
docHelper := dctx.getLanguageDocHelper(lang)
funcNameMap[lang] = docHelper.GetFunctionName(mod.mod, f)
}
@ -333,7 +339,7 @@ func (mod *modContext) genFunction(f *schema.Function) functionDocArgs {
Notes: mod.pkg.Attribution,
}
docInfo := decomposeDocstring(f.Comment)
docInfo := dctx.decomposeDocstring(f.Comment)
args := functionDocArgs{
Header: mod.genFunctionHeader(f),

27
pkg/codegen/docs/gen_method.go
View file

@ -63,9 +63,10 @@ func (mod *modContext) genMethods(r *schema.Resource) []methodDocArgs {
}
func (mod *modContext) genMethod(r *schema.Resource, m *schema.Method) methodDocArgs {
dctx := mod.docGenContext
f := m.Function
inputProps, outputProps := make(map[string][]property), make(map[string][]property)
for _, lang := range supportedLanguages {
for _, lang := range dctx.supportedLanguages {
if f.Inputs != nil {
exclude := func(name string) bool {
return name == "__self__"
@ -84,12 +85,12 @@ func (mod *modContext) genMethod(r *schema.Resource, m *schema.Method) methodDoc
// Generate the per-language map for the method name.
methodNameMap := map[string]string{}
for _, lang := range supportedLanguages {
docHelper := getLanguageDocHelper(lang)
for _, lang := range dctx.supportedLanguages {
docHelper := dctx.getLanguageDocHelper(lang)
methodNameMap[lang] = docHelper.GetMethodName(m)
}
docInfo := decomposeDocstring(f.Comment)
docInfo := dctx.decomposeDocstring(f.Comment)
args := methodDocArgs{
Title: title(m.Name, ""),
@ -187,7 +188,8 @@ func (mod *modContext) genMethodCS(f *schema.Function, resourceName, methodName
}
func (mod *modContext) genMethodPython(f *schema.Function) []formalParam {
docLanguageHelper := getLanguageDocHelper("python")
dctx := mod.docGenContext
docLanguageHelper := dctx.getLanguageDocHelper("python")
var params []formalParam
params = append(params, formalParam{
@ -236,10 +238,11 @@ func (mod *modContext) genMethodPython(f *schema.Function) []formalParam {
func (mod *modContext) genMethodArgs(r *schema.Resource, m *schema.Method,
methodNameMap map[string]string) map[string]string {
dctx := mod.docGenContext
f := m.Function
functionParams := make(map[string]string)
for _, lang := range supportedLanguages {
for _, lang := range dctx.supportedLanguages {
var (
paramTemplate string
params []formalParam
@ -283,7 +286,7 @@ func (mod *modContext) genMethodArgs(r *schema.Resource, m *schema.Method,
paramTemplate = "py_formal_param"
paramSeparatorTemplate = "py_param_separator"
docHelper := getLanguageDocHelper(lang)
docHelper := dctx.getLanguageDocHelper(lang)
methodName := docHelper.GetMethodName(m)
ps = paramSeparator{Indent: strings.Repeat(" ", len("def (")+len(methodName))}
}
@ -295,11 +298,11 @@ func (mod *modContext) genMethodArgs(r *schema.Resource, m *schema.Method,
}
for i, p := range params {
if err := templates.ExecuteTemplate(b, paramTemplate, p); err != nil {
if err := dctx.templates.ExecuteTemplate(b, paramTemplate, p); err != nil {
panic(err)
}
if i != n-1 {
if err := templates.ExecuteTemplate(b, paramSeparatorTemplate, ps); err != nil {
if err := dctx.templates.ExecuteTemplate(b, paramSeparatorTemplate, ps); err != nil {
panic(err)
}
}
@ -314,10 +317,12 @@ func (mod *modContext) genMethodArgs(r *schema.Resource, m *schema.Method,
func (mod *modContext) getMethodResult(r *schema.Resource, m *schema.Method) map[string]propertyType {
resourceMap := make(map[string]propertyType)
dctx := mod.docGenContext
var resultTypeName string
for _, lang := range supportedLanguages {
for _, lang := range dctx.supportedLanguages {
if m.Function.Outputs != nil && len(m.Function.Outputs.Properties) > 0 {
resultTypeName = getLanguageDocHelper(lang).GetMethodResultName(mod.pkg, mod.mod, r, m)
resultTypeName = dctx.getLanguageDocHelper(lang).GetMethodResultName(mod.pkg, mod.mod, r, m)
}
resourceMap[lang] = propertyType{
Name: resultTypeName,

14
pkg/codegen/docs/gen_test.go
View file

@ -283,6 +283,7 @@ func getFunctionFromModule(function string, mod *modContext) *schema.Function {
}
func TestFunctionHeaders(t *testing.T) {
dctx := newDocGenContext()
initTestPackageSpec(t)
schemaPkg, err := schema.ImportSpec(testPackageSpec, nil)
@ -309,7 +310,7 @@ func TestFunctionHeaders(t *testing.T) {
},
}
modules := generateModulesFromSchemaPackage(unitTestTool, schemaPkg)
modules := dctx.generateModulesFromSchemaPackage(unitTestTool, schemaPkg)
for _, test := range tests {
t.Run(test.FunctionName, func(t *testing.T) {
mod, ok := modules[test.ModuleName]
@ -329,6 +330,7 @@ func TestFunctionHeaders(t *testing.T) {
}
func TestResourceDocHeader(t *testing.T) {
dctx := newDocGenContext()
initTestPackageSpec(t)
schemaPkg, err := schema.ImportSpec(testPackageSpec, nil)
@ -358,7 +360,7 @@ func TestResourceDocHeader(t *testing.T) {
},
}
modules := generateModulesFromSchemaPackage(unitTestTool, schemaPkg)
modules := dctx.generateModulesFromSchemaPackage(unitTestTool, schemaPkg)
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
mod, ok := modules[test.ModuleName]
@ -379,9 +381,10 @@ func TestResourceDocHeader(t *testing.T) {
func TestExamplesProcessing(t *testing.T) {
initTestPackageSpec(t)
dctx := newDocGenContext()
description := testPackageSpec.Resources["prov:module/resource:Resource"].Description
docInfo := decomposeDocstring(description)
docInfo := dctx.decomposeDocstring(description)
examplesSection := docInfo.examples
importSection := docInfo.importDetails
@ -407,8 +410,9 @@ func TestExamplesProcessing(t *testing.T) {
}
func generatePackage(tool string, pkg *schema.Package, extraFiles map[string][]byte) (map[string][]byte, error) {
Initialize(tool, pkg)
return GeneratePackage(tool, pkg)
dctx := newDocGenContext()
dctx.initialize(tool, pkg)
return dctx.generatePackage(tool, pkg)
}
func TestGeneratePackage(t *testing.T) {

5
pkg/codegen/docs/package_tree_test.go
View file

@ -23,13 +23,14 @@ import (
)
func TestGeneratePackageTree(t *testing.T) {
dctx := newDocGenContext()
initTestPackageSpec(t)
schemaPkg, err := schema.ImportSpec(testPackageSpec, nil)
assert.NoError(t, err, "importing spec")
Initialize(unitTestTool, schemaPkg)
pkgTree, err := GeneratePackageTree()
dctx.initialize(unitTestTool, schemaPkg)
pkgTree, err := dctx.generatePackageTree()
if err != nil {
t.Errorf("Error generating the package tree for package %s: %v", schemaPkg.Name, err)
}

9
pkg/codegen/internal/test/sdk_driver.go
View file

@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/pulumi/pulumi/pkg/v3/codegen"
"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
)
// Defines an extra check logic that accepts the directory with the
@ -246,11 +247,9 @@ type SDKCodegenOptions struct {
func TestSDKCodegen(t *testing.T, opts *SDKCodegenOptions) { // revive:disable-line
testDir := filepath.Join("..", "internal", "test", "testdata")
// Motivation for flagging parallelism: not all tests are
// parallel-safe yet (for example, codegen/docs tests fail),
// and there are concerns about memory utilizaion in CI. It
// can be a nice feature for developing though.
parallel := os.Getenv("PULUMI_PARALLEL_SDK_CODEGEN_TESTS") != ""
// Motivation for flagging: concerns about memory utilizaion
// in CI. It can be a nice feature for developing though.
parallel := cmdutil.IsTruthy(os.Getenv("PULUMI_PARALLEL_SDK_CODEGEN_TESTS"))
for _, sdkTest := range sdkTests {
tt := sdkTest // avoid capturing loop variable `sdkTest` in the closure