1 // Copyright 2020 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // Command generate creates API (settings, etc) documentation in JSON and
6 // Markdown for machine and human consumption.
29 "github.com/sanity-io/litter"
30 "golang.org/x/tools/go/ast/astutil"
31 "golang.org/x/tools/go/packages"
32 "golang.org/x/tools/internal/lsp/command"
33 "golang.org/x/tools/internal/lsp/command/commandmeta"
34 "golang.org/x/tools/internal/lsp/mod"
35 "golang.org/x/tools/internal/lsp/source"
39 if _, err := doMain("..", true); err != nil {
40 fmt.Fprintf(os.Stderr, "Generation failed: %v\n", err)
45 func doMain(baseDir string, write bool) (bool, error) {
51 if ok, err := rewriteFile(filepath.Join(baseDir, "internal/lsp/source/api_json.go"), api, write, rewriteAPI); !ok || err != nil {
54 if ok, err := rewriteFile(filepath.Join(baseDir, "gopls/doc/settings.md"), api, write, rewriteSettings); !ok || err != nil {
57 if ok, err := rewriteFile(filepath.Join(baseDir, "gopls/doc/commands.md"), api, write, rewriteCommands); !ok || err != nil {
60 if ok, err := rewriteFile(filepath.Join(baseDir, "gopls/doc/analyzers.md"), api, write, rewriteAnalyzers); !ok || err != nil {
67 func loadAPI() (*source.APIJSON, error) {
68 pkgs, err := packages.Load(
70 Mode: packages.NeedTypes | packages.NeedTypesInfo | packages.NeedSyntax | packages.NeedDeps,
72 "golang.org/x/tools/internal/lsp/source",
79 api := &source.APIJSON{
80 Options: map[string][]*source.OptionJSON{},
82 defaults := source.DefaultOptions()
84 api.Commands, err = loadCommands(pkg)
88 api.Lenses = loadLenses(api.Commands)
90 // Transform the internal command name to the external command name.
91 for _, c := range api.Commands {
92 c.Command = command.ID(c.Command)
94 for _, m := range []map[string]*source.Analyzer{
95 defaults.DefaultAnalyzers,
96 defaults.TypeErrorAnalyzers,
97 defaults.ConvenienceAnalyzers,
98 // Don't yet add staticcheck analyzers.
100 api.Analyzers = append(api.Analyzers, loadAnalyzers(m)...)
102 for _, category := range []reflect.Value{
103 reflect.ValueOf(defaults.UserOptions),
105 // Find the type information and ast.File corresponding to the category.
106 optsType := pkg.Types.Scope().Lookup(category.Type().Name())
108 return nil, fmt.Errorf("could not find %v in scope %v", category.Type().Name(), pkg.Types.Scope())
110 opts, err := loadOptions(category, optsType, pkg, "")
114 catName := strings.TrimSuffix(category.Type().Name(), "Options")
115 api.Options[catName] = opts
117 // Hardcode the expected values for the analyses and code lenses
118 // settings, since their keys are not enums.
119 for _, opt := range opts {
122 for _, a := range api.Analyzers {
123 opt.EnumKeys.Keys = append(opt.EnumKeys.Keys, source.EnumKey{
124 Name: fmt.Sprintf("%q", a.Name),
126 Default: strconv.FormatBool(a.Default),
130 // Hack: Lenses don't set default values, and we don't want to
131 // pass in the list of expected lenses to loadOptions. Instead,
132 // format the defaults using reflection here. The hackiest part
133 // is reversing lowercasing of the field name.
134 reflectField := category.FieldByName(upperFirst(opt.Name))
135 for _, l := range api.Lenses {
136 def, err := formatDefaultFromEnumBoolMap(reflectField, l.Lens)
140 opt.EnumKeys.Keys = append(opt.EnumKeys.Keys, source.EnumKey{
141 Name: fmt.Sprintf("%q", l.Lens),
152 func loadOptions(category reflect.Value, optsType types.Object, pkg *packages.Package, hierarchy string) ([]*source.OptionJSON, error) {
153 file, err := fileForPos(pkg, optsType.Pos())
158 enums, err := loadEnums(pkg)
163 var opts []*source.OptionJSON
164 optsStruct := optsType.Type().Underlying().(*types.Struct)
165 for i := 0; i < optsStruct.NumFields(); i++ {
166 // The types field gives us the type.
167 typesField := optsStruct.Field(i)
169 // If the field name ends with "Options", assume it is a struct with
170 // additional options and process it recursively.
171 if h := strings.TrimSuffix(typesField.Name(), "Options"); h != typesField.Name() {
172 // Keep track of the parent structs.
174 h = hierarchy + "." + h
176 options, err := loadOptions(category, typesField, pkg, strings.ToLower(h))
180 opts = append(opts, options...)
183 path, _ := astutil.PathEnclosingInterval(file, typesField.Pos(), typesField.Pos())
185 return nil, fmt.Errorf("could not find AST node for field %v", typesField)
187 // The AST field gives us the doc.
188 astField, ok := path[1].(*ast.Field)
190 return nil, fmt.Errorf("unexpected AST path %v", path)
193 // The reflect field gives us the default value.
194 reflectField := category.FieldByName(typesField.Name())
195 if !reflectField.IsValid() {
196 return nil, fmt.Errorf("could not find reflect field for %v", typesField.Name())
199 def, err := formatDefault(reflectField)
204 typ := typesField.Type().String()
205 if _, ok := enums[typesField.Type()]; ok {
208 name := lowerFirst(typesField.Name())
210 var enumKeys source.EnumKeys
211 if m, ok := typesField.Type().(*types.Map); ok {
212 e, ok := enums[m.Key()]
214 typ = strings.Replace(typ, m.Key().String(), m.Key().Underlying().String(), 1)
216 keys, err := collectEnumKeys(name, m, reflectField, e)
225 // Get the status of the field by checking its struct tags.
226 reflectStructField, ok := category.Type().FieldByName(typesField.Name())
228 return nil, fmt.Errorf("no struct field for %s", typesField.Name())
230 status := reflectStructField.Tag.Get("status")
232 opts = append(opts, &source.OptionJSON{
235 Doc: lowerFirst(astField.Doc.Text()),
238 EnumValues: enums[typesField.Type()],
240 Hierarchy: hierarchy,
246 func loadEnums(pkg *packages.Package) (map[types.Type][]source.EnumValue, error) {
247 enums := map[types.Type][]source.EnumValue{}
248 for _, name := range pkg.Types.Scope().Names() {
249 obj := pkg.Types.Scope().Lookup(name)
250 cnst, ok := obj.(*types.Const)
254 f, err := fileForPos(pkg, cnst.Pos())
256 return nil, fmt.Errorf("finding file for %q: %v", cnst.Name(), err)
258 path, _ := astutil.PathEnclosingInterval(f, cnst.Pos(), cnst.Pos())
259 spec := path[1].(*ast.ValueSpec)
260 value := cnst.Val().ExactString()
261 doc := valueDoc(cnst.Name(), value, spec.Doc.Text())
262 v := source.EnumValue{
266 enums[obj.Type()] = append(enums[obj.Type()], v)
271 func collectEnumKeys(name string, m *types.Map, reflectField reflect.Value, enumValues []source.EnumValue) (*source.EnumKeys, error) {
272 // Make sure the value type gets set for analyses and codelenses
274 if len(enumValues) == 0 && !hardcodedEnumKeys(name) {
277 keys := &source.EnumKeys{
278 ValueType: m.Elem().String(),
280 // We can get default values for enum -> bool maps.
281 var isEnumBoolMap bool
282 if basic, ok := m.Elem().(*types.Basic); ok && basic.Kind() == types.Bool {
285 for _, v := range enumValues {
289 def, err = formatDefaultFromEnumBoolMap(reflectField, v.Value)
294 keys.Keys = append(keys.Keys, source.EnumKey{
303 func formatDefaultFromEnumBoolMap(reflectMap reflect.Value, enumKey string) (string, error) {
304 if reflectMap.Kind() != reflect.Map {
308 if unquoted, err := strconv.Unquote(name); err == nil {
311 for _, e := range reflectMap.MapKeys() {
312 if e.String() == name {
313 value := reflectMap.MapIndex(e)
314 if value.Type().Kind() == reflect.Bool {
315 return formatDefault(value)
319 // Assume that if the value isn't mentioned in the map, it defaults to
320 // the default value, false.
321 return formatDefault(reflect.ValueOf(false))
324 // formatDefault formats the default value into a JSON-like string.
325 // VS Code exposes settings as JSON, so showing them as JSON is reasonable.
326 // TODO(rstambler): Reconsider this approach, as the VS Code Go generator now
328 func formatDefault(reflectField reflect.Value) (string, error) {
329 def := reflectField.Interface()
331 // Durations marshal as nanoseconds, but we want the stringy versions,
333 if t, ok := def.(time.Duration); ok {
336 defBytes, err := json.Marshal(def)
341 // Nil values format as "null" so print them as hardcoded empty values.
342 switch reflectField.Type().Kind() {
344 if reflectField.IsNil() {
345 defBytes = []byte("{}")
348 if reflectField.IsNil() {
349 defBytes = []byte("[]")
352 return string(defBytes), err
355 // valueDoc transforms a docstring documenting an constant identifier to a
356 // docstring documenting its value.
358 // If doc is of the form "Foo is a bar", it returns '`"fooValue"` is a bar'. If
359 // doc is non-standard ("this value is a bar"), it returns '`"fooValue"`: this
361 func valueDoc(name, value, doc string) string {
365 if strings.HasPrefix(doc, name) {
366 // docstring in standard form. Replace the subject with value.
367 return fmt.Sprintf("`%s`%s", value, doc[len(name):])
369 return fmt.Sprintf("`%s`: %s", value, doc)
372 func loadCommands(pkg *packages.Package) ([]*source.CommandJSON, error) {
374 var commands []*source.CommandJSON
376 _, cmds, err := commandmeta.Load()
380 // Parse the objects it contains.
381 for _, cmd := range cmds {
382 commands = append(commands, &source.CommandJSON{
386 ArgDoc: argsDoc(cmd.Args),
392 func argsDoc(args []*commandmeta.Field) string {
393 var b strings.Builder
394 for i, arg := range args {
395 b.WriteString(argDoc(arg, 0))
396 if i != len(args)-1 {
403 func argDoc(arg *commandmeta.Field, level int) string {
404 // Max level to expand struct fields.
406 if len(arg.Fields) > 0 {
407 if level < maxLevel {
408 return structDoc(arg.Fields, level)
412 under := arg.Type.Underlying()
413 switch u := under.(type) {
415 return fmt.Sprintf("[]%s", u.Elem().Underlying().String())
417 return types.TypeString(under, nil)
420 func structDoc(fields []*commandmeta.Field, level int) string {
421 var b strings.Builder
423 indent := strings.Repeat("\t", level)
424 for _, fld := range fields {
425 if fld.Doc != "" && level == 0 {
426 doclines := strings.Split(fld.Doc, "\n")
427 for _, line := range doclines {
428 fmt.Fprintf(&b, "%s\t// %s\n", indent, line)
435 fmt.Fprintf(&b, "%s\t%q: %s,\n", indent, tag, argDoc(fld, level+1))
437 fmt.Fprintf(&b, "%s}", indent)
441 func loadLenses(commands []*source.CommandJSON) []*source.LensJSON {
442 all := map[command.Command]struct{}{}
443 for k := range source.LensFuncs() {
446 for k := range mod.LensFuncs() {
447 if _, ok := all[k]; ok {
448 panic(fmt.Sprintf("duplicate lens %q", string(k)))
453 var lenses []*source.LensJSON
455 for _, cmd := range commands {
456 if _, ok := all[command.Command(cmd.Command)]; ok {
457 lenses = append(lenses, &source.LensJSON{
467 func loadAnalyzers(m map[string]*source.Analyzer) []*source.AnalyzerJSON {
469 for _, a := range m {
470 sorted = append(sorted, a.Analyzer.Name)
473 var json []*source.AnalyzerJSON
474 for _, name := range sorted {
476 json = append(json, &source.AnalyzerJSON{
477 Name: a.Analyzer.Name,
485 func lowerFirst(x string) string {
489 return strings.ToLower(x[:1]) + x[1:]
492 func upperFirst(x string) string {
496 return strings.ToUpper(x[:1]) + x[1:]
499 func fileForPos(pkg *packages.Package, pos token.Pos) (*ast.File, error) {
501 for _, f := range pkg.Syntax {
502 if fset.Position(f.Pos()).Filename == fset.Position(pos).Filename {
506 return nil, fmt.Errorf("no file for pos %v", pos)
509 func rewriteFile(file string, api *source.APIJSON, write bool, rewrite func([]byte, *source.APIJSON) ([]byte, error)) (bool, error) {
510 old, err := ioutil.ReadFile(file)
515 new, err := rewrite(old, api)
517 return false, fmt.Errorf("rewriting %q: %v", file, err)
521 return bytes.Equal(old, new), nil
524 if err := ioutil.WriteFile(file, new, 0); err != nil {
531 func rewriteAPI(_ []byte, api *source.APIJSON) ([]byte, error) {
532 buf := bytes.NewBuffer(nil)
533 apiStr := litter.Options{
534 HomePackage: "source",
536 // Massive hack: filter out redundant types from the composite literal.
537 apiStr = strings.ReplaceAll(apiStr, "&OptionJSON", "")
538 apiStr = strings.ReplaceAll(apiStr, ": []*OptionJSON", ":")
539 apiStr = strings.ReplaceAll(apiStr, "&CommandJSON", "")
540 apiStr = strings.ReplaceAll(apiStr, "&LensJSON", "")
541 apiStr = strings.ReplaceAll(apiStr, "&AnalyzerJSON", "")
542 apiStr = strings.ReplaceAll(apiStr, " EnumValue{", "{")
543 apiStr = strings.ReplaceAll(apiStr, " EnumKey{", "{")
544 apiBytes, err := format.Source([]byte(apiStr))
548 fmt.Fprintf(buf, "// Code generated by \"golang.org/x/tools/gopls/doc/generate\"; DO NOT EDIT.\n\npackage source\n\nvar GeneratedAPIJSON = %s\n", apiBytes)
549 return buf.Bytes(), nil
552 var parBreakRE = regexp.MustCompile("\n{2,}")
554 type optionsGroup struct {
558 options []*source.OptionJSON
561 func rewriteSettings(doc []byte, api *source.APIJSON) ([]byte, error) {
563 for category, opts := range api.Options {
564 groups := collectGroups(opts)
566 // First, print a table of contents.
567 section := bytes.NewBuffer(nil)
568 fmt.Fprintln(section, "")
569 for _, h := range groups {
570 writeBullet(section, h.final, h.level)
572 fmt.Fprintln(section, "")
574 // Currently, the settings document has a title and a subtitle, so
575 // start at level 3 for a header beginning with "###".
577 for _, h := range groups {
578 level := baseLevel + h.level
579 writeTitle(section, h.final, level)
580 for _, opt := range h.options {
581 header := strMultiply("#", level+1)
582 fmt.Fprintf(section, "%s **%v** *%v*\n\n", header, opt.Name, opt.Type)
583 writeStatus(section, opt.Status)
584 enumValues := collectEnums(opt)
585 fmt.Fprintf(section, "%v%v\nDefault: `%v`.\n\n", opt.Doc, enumValues, opt.Default)
589 result, err = replaceSection(result, category, section.Bytes())
595 section := bytes.NewBuffer(nil)
596 for _, lens := range api.Lenses {
597 fmt.Fprintf(section, "### **%v**\n\nIdentifier: `%v`\n\n%v\n", lens.Title, lens.Lens, lens.Doc)
599 return replaceSection(result, "Lenses", section.Bytes())
602 func collectGroups(opts []*source.OptionJSON) []optionsGroup {
603 optsByHierarchy := map[string][]*source.OptionJSON{}
604 for _, opt := range opts {
605 optsByHierarchy[opt.Hierarchy] = append(optsByHierarchy[opt.Hierarchy], opt)
608 // As a hack, assume that uncategorized items are less important to
609 // users and force the empty string to the end of the list.
610 var containsEmpty bool
612 for h := range optsByHierarchy {
617 sorted = append(sorted, h)
621 sorted = append(sorted, "")
623 var groups []optionsGroup
625 for _, h := range sorted {
626 split := strings.SplitAfter(h, ".")
627 last := split[len(split)-1]
628 // Hack to capitalize all of UI.
632 // A hierarchy may look like "ui.formatting". If "ui" has no
633 // options of its own, it may not be added to the map, but it
634 // still needs a heading.
635 components := strings.Split(h, ".")
636 for i := 1; i < len(components); i++ {
637 parent := strings.Join(components[0:i], ".")
638 if _, ok := optsByHierarchy[parent]; !ok {
639 groups = append(groups, optionsGroup{
642 level: baseLevel + i,
646 groups = append(groups, optionsGroup{
649 level: baseLevel + strings.Count(h, "."),
650 options: optsByHierarchy[h],
656 func collectEnums(opt *source.OptionJSON) string {
657 var b strings.Builder
658 write := func(name, doc string, index, len int) {
660 unbroken := parBreakRE.ReplaceAllString(doc, "\\\n")
661 fmt.Fprintf(&b, "* %s", unbroken)
663 fmt.Fprintf(&b, "* `%s`", name)
669 if len(opt.EnumValues) > 0 && opt.Type == "enum" {
670 b.WriteString("\nMust be one of:\n\n")
671 for i, val := range opt.EnumValues {
672 write(val.Value, val.Doc, i, len(opt.EnumValues))
674 } else if len(opt.EnumKeys.Keys) > 0 && shouldShowEnumKeysInSettings(opt.Name) {
675 b.WriteString("\nCan contain any of:\n\n")
676 for i, val := range opt.EnumKeys.Keys {
677 write(val.Name, val.Doc, i, len(opt.EnumKeys.Keys))
683 func shouldShowEnumKeysInSettings(name string) bool {
684 // Both of these fields have too many possible options to print.
685 return !hardcodedEnumKeys(name)
688 func hardcodedEnumKeys(name string) bool {
689 return name == "analyses" || name == "codelenses"
692 func writeBullet(w io.Writer, title string, level int) {
696 // Capitalize the first letter of each title.
697 prefix := strMultiply(" ", level)
698 fmt.Fprintf(w, "%s* [%s](#%s)\n", prefix, capitalize(title), strings.ToLower(title))
701 func writeTitle(w io.Writer, title string, level int) {
705 // Capitalize the first letter of each title.
706 fmt.Fprintf(w, "%s %s\n\n", strMultiply("#", level), capitalize(title))
709 func writeStatus(section io.Writer, status string) {
713 fmt.Fprint(section, "**This is an advanced setting and should not be configured by most `gopls` users.**\n\n")
715 fmt.Fprint(section, "**This setting is for debugging purposes only.**\n\n")
717 fmt.Fprint(section, "**This setting is experimental and may be deleted.**\n\n")
719 fmt.Fprintf(section, "**Status: %s.**\n\n", status)
723 func capitalize(s string) string {
724 return string(unicode.ToUpper(rune(s[0]))) + s[1:]
727 func strMultiply(str string, count int) string {
729 for i := 0; i < count; i++ {
730 result += string(str)
735 func rewriteCommands(doc []byte, api *source.APIJSON) ([]byte, error) {
736 section := bytes.NewBuffer(nil)
737 for _, command := range api.Commands {
738 fmt.Fprintf(section, "### **%v**\nIdentifier: `%v`\n\n%v\n\n", command.Title, command.Command, command.Doc)
739 if command.ArgDoc != "" {
740 fmt.Fprintf(section, "Args:\n\n```\n%s\n```\n\n", command.ArgDoc)
743 return replaceSection(doc, "Commands", section.Bytes())
746 func rewriteAnalyzers(doc []byte, api *source.APIJSON) ([]byte, error) {
747 section := bytes.NewBuffer(nil)
748 for _, analyzer := range api.Analyzers {
749 fmt.Fprintf(section, "## **%v**\n\n", analyzer.Name)
750 fmt.Fprintf(section, "%s\n\n", analyzer.Doc)
751 switch analyzer.Default {
753 fmt.Fprintf(section, "**Enabled by default.**\n\n")
755 fmt.Fprintf(section, "**Disabled by default. Enable it by setting `\"analyses\": {\"%s\": true}`.**\n\n", analyzer.Name)
758 return replaceSection(doc, "Analyzers", section.Bytes())
761 func replaceSection(doc []byte, sectionName string, replacement []byte) ([]byte, error) {
762 re := regexp.MustCompile(fmt.Sprintf(`(?s)<!-- BEGIN %v.* -->\n(.*?)<!-- END %v.* -->`, sectionName, sectionName))
763 idx := re.FindSubmatchIndex(doc)
765 return nil, fmt.Errorf("could not find section %q", sectionName)
767 result := append([]byte(nil), doc[:idx[2]]...)
768 result = append(result, replacement...)
769 result = append(result, doc[idx[3]:]...)