maxmustermann/pulumi
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[codegen/docs] Use go:embed for templates and use DisplayName from schema for titles (#8389)

Browse source 
* Use the display name from the schema, if available

* Update comment on the titleLookup map

* Add examples of reserved labels that can be used with the Keywords array in the schema
This commit is contained in:
Praneet Loke 2021-11-10 11:07:43 -08:00 committed by GitHub
parent d7ec7ce06c
commit 0343a6de76
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 41 additions and 202 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.github/workflows/run-docs-generation.yml vendored
View file

@ -93,6 +93,12 @@ jobs:
# If generating docs for more providers here, be sure to update
# the description of the draft PR that is opened in the next step.
pushd docs
pushd tools/resourcedocsgen
go mod edit -replace github.com/pulumi/pulumi/pkg/v3=../../../pulumi/pkg
go mod edit -replace github.com/pulumi/pulumi/sdk/v3=../../../pulumi/sdk
popd
./scripts/gen_resource_docs.sh aws true
./scripts/gen_resource_docs.sh kubernetes true
popd

9
Makefile
View file

@ -29,19 +29,18 @@ build-proto::
.PHONY: generate
generate::
$(call STEP_MESSAGE)
echo "Generate static assets bundle for docs generator"
cd pkg && go generate ./codegen/docs/gen.go
echo "This command does not do anything anymore. It will be removed in a future version."
build:: generate
build::
cd pkg && go install -ldflags "-X github.com/pulumi/pulumi/pkg/v3/version.Version=${VERSION}" ${PROJECT}
build_debug:: generate
build_debug::
cd pkg && go install -gcflags="all=-N -l" -ldflags "-X github.com/pulumi/pulumi/pkg/v3/version.Version=${VERSION}" ${PROJECT}
developer_docs::
cd developer-docs && make html
install:: generate
install::
cd pkg && GOBIN=$(PULUMI_BIN) go install -ldflags "-X github.com/pulumi/pulumi/pkg/v3/version.Version=${VERSION}" ${PROJECT}
install_all:: install

16
pkg/codegen/docs/README.md
View file

@ -32,19 +32,3 @@ For minor diffs, you can just update the test files manually and include those u
```
PULUMI_ACCEPT=true pushd pkg/codegen/docs && go test . && popd
```
## `bundler.go`
This file contains a `main` function and is part of the `main` package. We run it using the `go generate` command (see the `Makefile` and the starting comment in `pkg/codegen/gen.go`).
> This file is ignored using a `+build ignore` comment at the top of the file, so it is not ignored during a `go build ...`.
## `packaged.go`
A file generated by `bundler.go` that contains formatted byte strings, that represent the string templates from the `./templates/` folder. This file is also git-ignored as it is intended to only be generated by the `docs` repo and is not used during runtime of the main Pulumi CLI. In fact, this whole package is not used during the runtime of the CLI itself.
## `go:generate`
> Read more [here](https://blog.golang.org/generate).
`go:generate` is a special code comment that can be used to run custom commands by simply running `go generate <package>`, which then scans for `go:generate` comments in all sources in the package `<package>`. It also serves as a way to document, that a certain file relies on a command to have been executed before it can be used.

34
pkg/codegen/docs/bundler.go
View file

@ -1,34 +0,0 @@
//go:build ignore
// +build ignore
// Copyright 2016-2020, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Pulling out some of the repeated strings tokens into constants would harm readability, so we just ignore the
// goconst linter's warning.
//
// nolint: lll, goconst
package main
import "github.com/pulumi/pulumi/pkg/v3/codegen/docs/bundler"
// main reads files under the templates directory, and builds a map of filename to byte slice.
// Each file's contents are then written to a generated file.
//
// NOTE: Sub-directories are currently not supported.
func main() {
if err := bundler.GenerateTemplatesBundleFile(); err != nil {
panic(err)
}
}

127
pkg/codegen/docs/bundler/bundler.go
View file

@ -1,127 +0,0 @@
// Copyright 2016-2021, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bundler
import (
"bytes"
"fmt"
"go/format"
"io/ioutil"
"os"
"strings"
"text/template"
"github.com/pkg/errors"
)
const (
basePath = "."
docsTemplatesPath = basePath + "/templates"
generatedFileName = basePath + "/packaged.go"
)
var conv = map[string]interface{}{"conv": fmtByteSlice}
var tmpl = template.Must(template.New("").Funcs(conv).Parse(`
// AUTO-GENERATED FILE! DO NOT EDIT THIS FILE MANUALLY.
// Copyright 2016-2020, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Pulling out some of the repeated strings tokens into constants would harm readability, so we just ignore the
// goconst linter's warning.
//
// nolint: lll, goconst
package docs
func init() {
packagedTemplates = make(map[string][]byte)
{{ range $key, $value := . }}
packagedTemplates["{{ $key }}"] = []byte{ {{ conv $value }} }
{{ println }}
{{- end }}
}
`))
// fmtByteSlice returns a formatted byte string for a given slice of bytes.
// We embed the raw bytes to avoid any formatting errors that can occur due to saving
// raw strings in a file.
func fmtByteSlice(s []byte) string {
builder := strings.Builder{}
for _, v := range s {
builder.WriteString(fmt.Sprintf("%d,", int(v)))
}
return builder.String()
}
// GenerateTemplatesBundleFile reads the templates from `../templates/` and writes a git-ignored
// packaged.go file containing byte-slices of the templates.
func GenerateTemplatesBundleFile() error {
files, err := ioutil.ReadDir(docsTemplatesPath)
if err != nil {
return errors.Wrap(err, "reading the templates dir")
}
contents := make(map[string][]byte)
for _, f := range files {
if f.IsDir() {
fmt.Printf("%q is a dir. Skipping...\n", f.Name())
}
b, err := ioutil.ReadFile(docsTemplatesPath + "/" + f.Name())
if err != nil {
return errors.Wrapf(err, "reading file %s", f.Name())
}
if len(b) == 0 {
fmt.Printf("%q is empty. Skipping...\n", f.Name())
continue
}
contents[f.Name()] = b
}
// We overwrite the file every time the `go generate ...` command is run.
f, err := os.Create(generatedFileName)
if err != nil {
return errors.Wrap(err, "creating blob file")
}
defer f.Close()
builder := &bytes.Buffer{}
if err = tmpl.Execute(builder, contents); err != nil {
return errors.Wrap(err, "executing template")
}
data, err := format.Source(builder.Bytes())
if err != nil {
return errors.Wrap(err, "formatting generated code")
}
if err = ioutil.WriteFile(generatedFileName, data, os.ModePerm); err != nil {
return errors.Wrap(err, "writing file")
}
return nil
}

39
pkg/codegen/docs/gen.go
View file

@ -22,6 +22,7 @@ package docs
import (
"bytes"
"embed"
"fmt"
"html"
"html/template"
@ -41,18 +42,13 @@ import (
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
)
// Populated in auto-generated `packaged.go`
var packagedTemplates map[string][]byte
//go:embed templates/*.tmpl
var packagedTemplates embed.FS
func init() {
packagedTemplates = map[string][]byte{}
}
// TODO[pulumi/pulumi#7813]: Remove this lookup once display name is available in
// the Pulumi schema.
//
// NOTE: For the time being this lookup map and the one used by the resourcedocsgen
// tool in `pulumi/docs` must be kept up-to-date.
// NOTE: This lookup map can be removed when all Pulumi-managed packages
// have a DisplayName in their schema. See pulumi/pulumi#7813.
// This lookup table no longer needs to be updated for new providers
// and is considered stale.
//
// titleLookup is a map of package name to the desired display name.
func titleLookup(shortName string) (string, bool) {
@ -1706,8 +1702,13 @@ func (mod *modContext) genIndex() indexData {
modName := mod.getModuleFileName()
title := modName
// An empty string indicates that this is the root module.
if title == "" {
title = formatTitleText(mod.pkg.Name)
if mod.pkg.DisplayName != "" {
title = mod.pkg.DisplayName
} else {
title = getPackageDisplayName(mod.pkg.Name)
}
}
// If there are submodules, list them.
@ -1759,7 +1760,7 @@ func (mod *modContext) genIndex() indexData {
// assume top level package index page when formatting title tags otherwise, if contains modules, assume modules
// top level page when generating title tags.
if len(modules) > 0 {
titleTag = fmt.Sprintf("%s Package", formatTitleText(title))
titleTag = fmt.Sprintf("%s Package", getPackageDisplayName(title))
} else {
titleTag = fmt.Sprintf("%s.%s", mod.pkg.Name, title)
packageDescription = fmt.Sprintf("Explore the resources and functions of the %s.%s module.", mod.pkg.Name, title)
@ -1784,7 +1785,9 @@ func (mod *modContext) genIndex() indexData {
return data
}
func formatTitleText(title string) string {
// getPackageDisplayName uses the title lookup map to look for a
// display name for the given title.
func getPackageDisplayName(title string) string {
// If title not found in titleLookup map, default back to title given.
if val, ok := titleLookup(title); ok {
return val
@ -1936,12 +1939,8 @@ func (dctx *docGenContext) initialize(tool string, pkg *schema.Package) {
defer glog.Flush()
if len(packagedTemplates) == 0 {
glog.Fatal(`packagedTemplates is empty. Did you run "make generate" first?`)
}
for name, b := range packagedTemplates {
template.Must(dctx.templates.New(name).Parse(string(b)))
if _, err := dctx.templates.ParseFS(packagedTemplates, "templates/*.tmpl"); err != nil {
glog.Fatalf("initializing templates: %v", err)
}
// Generate the modules from the schema, and for every module

12
pkg/codegen/schema/schema.go
View file

@ -548,6 +548,12 @@ type Package struct {
// Description is the description of the package.
Description string
// Keywords is the list of keywords that are associated with the package, if any.
// Some reserved keywords can be specified as well that help with categorizing the
// package in the Pulumi registry. `category/<name>` and `kind/<type>` are the only
// reserved keywords at this time, where `<name>` can be one of:
// `cloud`, `database`, `infrastructure`, `monitoring`, `network`, `utility`, `vcs`
// and `<type>` is either `native` or `component`. If the package is a bridged Terraform
// provider, then don't include the `kind/` label.
Keywords []string
// Homepage is the package's homepage.
Homepage string
@ -1473,6 +1479,12 @@ type PackageSpec struct {
// Description is the description of the package.
Description string `json:"description,omitempty" yaml:"description,omitempty"`
// Keywords is the list of keywords that are associated with the package, if any.
// Some reserved keywords can be specified as well that help with categorizing the
// package in the Pulumi registry. `category/<name>` and `kind/<type>` are the only
// reserved keywords at this time, where `<name>` can be one of:
// `cloud`, `database`, `infrastructure`, `monitoring`, `network`, `utility`, `vcs`
// and `<type>` is either `native` or `component`. If the package is a bridged Terraform
// provider, then don't include the `kind/` label.
Keywords []string `json:"keywords,omitempty" yaml:"keywords,omitempty"`
// Homepage is the package's homepage.
Homepage string `json:"homepage,omitempty" yaml:"homepage,omitempty"`