From 0343a6de764f5a126ab424c05f0d6ccf0dfa96b7 Mon Sep 17 00:00:00 2001 From: Praneet Loke <1466314+praneetloke@users.noreply.github.com> Date: Wed, 10 Nov 2021 11:07:43 -0800 Subject: [PATCH] [codegen/docs] Use go:embed for templates and use DisplayName from schema for titles (#8389) * Use the display name from the schema, if available * Update comment on the titleLookup map * Add examples of reserved labels that can be used with the Keywords array in the schema --- .github/workflows/run-docs-generation.yml | 6 + Makefile | 9 +- pkg/codegen/docs/README.md | 16 --- pkg/codegen/docs/bundler.go | 34 ------ pkg/codegen/docs/bundler/bundler.go | 127 ---------------------- pkg/codegen/docs/gen.go | 39 ++++--- pkg/codegen/schema/schema.go | 12 ++ 7 files changed, 41 insertions(+), 202 deletions(-) delete mode 100644 pkg/codegen/docs/bundler.go delete mode 100644 pkg/codegen/docs/bundler/bundler.go diff --git a/.github/workflows/run-docs-generation.yml b/.github/workflows/run-docs-generation.yml index e73b366d4..39f0d194b 100644 --- a/.github/workflows/run-docs-generation.yml +++ b/.github/workflows/run-docs-generation.yml @@ -93,6 +93,12 @@ jobs: # If generating docs for more providers here, be sure to update # the description of the draft PR that is opened in the next step. pushd docs + + pushd tools/resourcedocsgen + go mod edit -replace github.com/pulumi/pulumi/pkg/v3=../../../pulumi/pkg + go mod edit -replace github.com/pulumi/pulumi/sdk/v3=../../../pulumi/sdk + popd + ./scripts/gen_resource_docs.sh aws true ./scripts/gen_resource_docs.sh kubernetes true popd diff --git a/Makefile b/Makefile index 59d13e674..19325a8db 100644 --- a/Makefile +++ b/Makefile @@ -29,19 +29,18 @@ build-proto:: .PHONY: generate generate:: $(call STEP_MESSAGE) - echo "Generate static assets bundle for docs generator" - cd pkg && go generate ./codegen/docs/gen.go + echo "This command does not do anything anymore. It will be removed in a future version." -build:: generate +build:: cd pkg && go install -ldflags "-X github.com/pulumi/pulumi/pkg/v3/version.Version=${VERSION}" ${PROJECT} -build_debug:: generate +build_debug:: cd pkg && go install -gcflags="all=-N -l" -ldflags "-X github.com/pulumi/pulumi/pkg/v3/version.Version=${VERSION}" ${PROJECT} developer_docs:: cd developer-docs && make html -install:: generate +install:: cd pkg && GOBIN=$(PULUMI_BIN) go install -ldflags "-X github.com/pulumi/pulumi/pkg/v3/version.Version=${VERSION}" ${PROJECT} install_all:: install diff --git a/pkg/codegen/docs/README.md b/pkg/codegen/docs/README.md index 369915de6..657edaa83 100644 --- a/pkg/codegen/docs/README.md +++ b/pkg/codegen/docs/README.md @@ -32,19 +32,3 @@ For minor diffs, you can just update the test files manually and include those u ``` PULUMI_ACCEPT=true pushd pkg/codegen/docs && go test . && popd ``` - -## `bundler.go` - -This file contains a `main` function and is part of the `main` package. We run it using the `go generate` command (see the `Makefile` and the starting comment in `pkg/codegen/gen.go`). - -> This file is ignored using a `+build ignore` comment at the top of the file, so it is not ignored during a `go build ...`. - -## `packaged.go` - -A file generated by `bundler.go` that contains formatted byte strings, that represent the string templates from the `./templates/` folder. This file is also git-ignored as it is intended to only be generated by the `docs` repo and is not used during runtime of the main Pulumi CLI. In fact, this whole package is not used during the runtime of the CLI itself. - -## `go:generate` - -> Read more [here](https://blog.golang.org/generate). - -`go:generate` is a special code comment that can be used to run custom commands by simply running `go generate `, which then scans for `go:generate` comments in all sources in the package ``. It also serves as a way to document, that a certain file relies on a command to have been executed before it can be used. diff --git a/pkg/codegen/docs/bundler.go b/pkg/codegen/docs/bundler.go deleted file mode 100644 index ec9f6eee9..000000000 --- a/pkg/codegen/docs/bundler.go +++ /dev/null @@ -1,34 +0,0 @@ -//go:build ignore -// +build ignore - -// Copyright 2016-2020, Pulumi Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Pulling out some of the repeated strings tokens into constants would harm readability, so we just ignore the -// goconst linter's warning. -// -// nolint: lll, goconst -package main - -import "github.com/pulumi/pulumi/pkg/v3/codegen/docs/bundler" - -// main reads files under the templates directory, and builds a map of filename to byte slice. -// Each file's contents are then written to a generated file. -// -// NOTE: Sub-directories are currently not supported. -func main() { - if err := bundler.GenerateTemplatesBundleFile(); err != nil { - panic(err) - } -} diff --git a/pkg/codegen/docs/bundler/bundler.go b/pkg/codegen/docs/bundler/bundler.go deleted file mode 100644 index 70d6648c7..000000000 --- a/pkg/codegen/docs/bundler/bundler.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2016-2021, Pulumi Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bundler - -import ( - "bytes" - "fmt" - "go/format" - "io/ioutil" - "os" - "strings" - "text/template" - - "github.com/pkg/errors" -) - -const ( - basePath = "." - docsTemplatesPath = basePath + "/templates" - generatedFileName = basePath + "/packaged.go" -) - -var conv = map[string]interface{}{"conv": fmtByteSlice} -var tmpl = template.Must(template.New("").Funcs(conv).Parse(` - // AUTO-GENERATED FILE! DO NOT EDIT THIS FILE MANUALLY. - - // Copyright 2016-2020, Pulumi Corporation. - // - // Licensed under the Apache License, Version 2.0 (the "License"); - // you may not use this file except in compliance with the License. - // You may obtain a copy of the License at - // - // http://www.apache.org/licenses/LICENSE-2.0 - // - // Unless required by applicable law or agreed to in writing, software - // distributed under the License is distributed on an "AS IS" BASIS, - // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - // See the License for the specific language governing permissions and - // limitations under the License. - - // Pulling out some of the repeated strings tokens into constants would harm readability, so we just ignore the - // goconst linter's warning. - // - // nolint: lll, goconst - package docs - - func init() { - packagedTemplates = make(map[string][]byte) - {{ range $key, $value := . }} - packagedTemplates["{{ $key }}"] = []byte{ {{ conv $value }} } - {{ println }} - {{- end }} - } -`)) - -// fmtByteSlice returns a formatted byte string for a given slice of bytes. -// We embed the raw bytes to avoid any formatting errors that can occur due to saving -// raw strings in a file. -func fmtByteSlice(s []byte) string { - builder := strings.Builder{} - - for _, v := range s { - builder.WriteString(fmt.Sprintf("%d,", int(v))) - } - - return builder.String() -} - -// GenerateTemplatesBundleFile reads the templates from `../templates/` and writes a git-ignored -// packaged.go file containing byte-slices of the templates. -func GenerateTemplatesBundleFile() error { - files, err := ioutil.ReadDir(docsTemplatesPath) - if err != nil { - return errors.Wrap(err, "reading the templates dir") - } - - contents := make(map[string][]byte) - for _, f := range files { - if f.IsDir() { - fmt.Printf("%q is a dir. Skipping...\n", f.Name()) - } - b, err := ioutil.ReadFile(docsTemplatesPath + "/" + f.Name()) - if err != nil { - return errors.Wrapf(err, "reading file %s", f.Name()) - } - if len(b) == 0 { - fmt.Printf("%q is empty. Skipping...\n", f.Name()) - continue - } - contents[f.Name()] = b - } - - // We overwrite the file every time the `go generate ...` command is run. - f, err := os.Create(generatedFileName) - if err != nil { - return errors.Wrap(err, "creating blob file") - } - defer f.Close() - - builder := &bytes.Buffer{} - if err = tmpl.Execute(builder, contents); err != nil { - return errors.Wrap(err, "executing template") - } - - data, err := format.Source(builder.Bytes()) - if err != nil { - return errors.Wrap(err, "formatting generated code") - } - - if err = ioutil.WriteFile(generatedFileName, data, os.ModePerm); err != nil { - return errors.Wrap(err, "writing file") - } - - return nil -} diff --git a/pkg/codegen/docs/gen.go b/pkg/codegen/docs/gen.go index f2792047c..5d53a9c65 100644 --- a/pkg/codegen/docs/gen.go +++ b/pkg/codegen/docs/gen.go @@ -22,6 +22,7 @@ package docs import ( "bytes" + "embed" "fmt" "html" "html/template" @@ -41,18 +42,13 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" ) -// Populated in auto-generated `packaged.go` -var packagedTemplates map[string][]byte +//go:embed templates/*.tmpl +var packagedTemplates embed.FS -func init() { - packagedTemplates = map[string][]byte{} -} - -// TODO[pulumi/pulumi#7813]: Remove this lookup once display name is available in -// the Pulumi schema. -// -// NOTE: For the time being this lookup map and the one used by the resourcedocsgen -// tool in `pulumi/docs` must be kept up-to-date. +// NOTE: This lookup map can be removed when all Pulumi-managed packages +// have a DisplayName in their schema. See pulumi/pulumi#7813. +// This lookup table no longer needs to be updated for new providers +// and is considered stale. // // titleLookup is a map of package name to the desired display name. func titleLookup(shortName string) (string, bool) { @@ -1706,8 +1702,13 @@ func (mod *modContext) genIndex() indexData { modName := mod.getModuleFileName() title := modName + // An empty string indicates that this is the root module. if title == "" { - title = formatTitleText(mod.pkg.Name) + if mod.pkg.DisplayName != "" { + title = mod.pkg.DisplayName + } else { + title = getPackageDisplayName(mod.pkg.Name) + } } // If there are submodules, list them. @@ -1759,7 +1760,7 @@ func (mod *modContext) genIndex() indexData { // assume top level package index page when formatting title tags otherwise, if contains modules, assume modules // top level page when generating title tags. if len(modules) > 0 { - titleTag = fmt.Sprintf("%s Package", formatTitleText(title)) + titleTag = fmt.Sprintf("%s Package", getPackageDisplayName(title)) } else { titleTag = fmt.Sprintf("%s.%s", mod.pkg.Name, title) packageDescription = fmt.Sprintf("Explore the resources and functions of the %s.%s module.", mod.pkg.Name, title) @@ -1784,7 +1785,9 @@ func (mod *modContext) genIndex() indexData { return data } -func formatTitleText(title string) string { +// getPackageDisplayName uses the title lookup map to look for a +// display name for the given title. +func getPackageDisplayName(title string) string { // If title not found in titleLookup map, default back to title given. if val, ok := titleLookup(title); ok { return val @@ -1936,12 +1939,8 @@ func (dctx *docGenContext) initialize(tool string, pkg *schema.Package) { defer glog.Flush() - if len(packagedTemplates) == 0 { - glog.Fatal(`packagedTemplates is empty. Did you run "make generate" first?`) - } - - for name, b := range packagedTemplates { - template.Must(dctx.templates.New(name).Parse(string(b))) + if _, err := dctx.templates.ParseFS(packagedTemplates, "templates/*.tmpl"); err != nil { + glog.Fatalf("initializing templates: %v", err) } // Generate the modules from the schema, and for every module diff --git a/pkg/codegen/schema/schema.go b/pkg/codegen/schema/schema.go index cf9e9f0ad..6c772599b 100644 --- a/pkg/codegen/schema/schema.go +++ b/pkg/codegen/schema/schema.go @@ -548,6 +548,12 @@ type Package struct { // Description is the description of the package. Description string // Keywords is the list of keywords that are associated with the package, if any. + // Some reserved keywords can be specified as well that help with categorizing the + // package in the Pulumi registry. `category/` and `kind/` are the only + // reserved keywords at this time, where `` can be one of: + // `cloud`, `database`, `infrastructure`, `monitoring`, `network`, `utility`, `vcs` + // and `` is either `native` or `component`. If the package is a bridged Terraform + // provider, then don't include the `kind/` label. Keywords []string // Homepage is the package's homepage. Homepage string @@ -1473,6 +1479,12 @@ type PackageSpec struct { // Description is the description of the package. Description string `json:"description,omitempty" yaml:"description,omitempty"` // Keywords is the list of keywords that are associated with the package, if any. + // Some reserved keywords can be specified as well that help with categorizing the + // package in the Pulumi registry. `category/` and `kind/` are the only + // reserved keywords at this time, where `` can be one of: + // `cloud`, `database`, `infrastructure`, `monitoring`, `network`, `utility`, `vcs` + // and `` is either `native` or `component`. If the package is a bridged Terraform + // provider, then don't include the `kind/` label. Keywords []string `json:"keywords,omitempty" yaml:"keywords,omitempty"` // Homepage is the package's homepage. Homepage string `json:"homepage,omitempty" yaml:"homepage,omitempty"`