1 // Copyright 2018 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // Package checker defines the implementation of the checker commands.
6 // The same code drives the multi-analysis driver, the single-analysis
7 // driver that is conventionally provided for convenience along with
8 // each analysis package, and the test driver.
32 "golang.org/x/tools/go/analysis"
33 "golang.org/x/tools/go/analysis/internal/analysisflags"
34 "golang.org/x/tools/go/packages"
35 "golang.org/x/tools/internal/analysisinternal"
36 "golang.org/x/tools/internal/span"
40 // Debug is a set of single-letter flags:
42 // f show [f]acts as they are created
43 // p disable [p]arallel execution of analyzers
44 // s do additional [s]anity checks on fact types and serialization
45 // t show [t]iming info (NB: use 'p' flag to avoid GC/scheduler noise)
46 // v show [v]erbose logging
50 // Log files for optional performance tracing.
51 CPUProfile, MemProfile, Trace string
53 // Fix determines whether to apply all suggested fixes.
57 // RegisterFlags registers command-line flags used by the analysis driver.
58 func RegisterFlags() {
59 // When adding flags here, remember to update
60 // the list of suppressed flags in analysisflags.
62 flag.StringVar(&Debug, "debug", Debug, `debug flags, any subset of "fpstv"`)
64 flag.StringVar(&CPUProfile, "cpuprofile", "", "write CPU profile to this file")
65 flag.StringVar(&MemProfile, "memprofile", "", "write memory profile to this file")
66 flag.StringVar(&Trace, "trace", "", "write trace log to this file")
68 flag.BoolVar(&Fix, "fix", false, "apply all suggested fixes")
71 // Run loads the packages specified by args using go/packages,
72 // then applies the specified analyzers to them.
73 // Analysis flags must already have been set.
74 // It provides most of the logic for the main functions of both the
75 // singlechecker and the multi-analysis commands.
76 // It returns the appropriate exit code.
77 func Run(args []string, analyzers []*analysis.Analyzer) (exitcode int) {
79 f, err := os.Create(CPUProfile)
83 if err := pprof.StartCPUProfile(f); err != nil {
86 // NB: profile won't be written in case of error.
87 defer pprof.StopCPUProfile()
91 f, err := os.Create(Trace)
95 if err := trace.Start(f); err != nil {
98 // NB: trace log won't be written in case of error.
101 log.Printf("To view the trace, run:\n$ go tool trace view %s", Trace)
105 if MemProfile != "" {
106 f, err := os.Create(MemProfile)
110 // NB: memprofile won't be written in case of error.
112 runtime.GC() // get up-to-date statistics
113 if err := pprof.WriteHeapProfile(f); err != nil {
114 log.Fatalf("Writing memory profile: %v", err)
120 // Load the packages.
123 log.SetFlags(log.Lmicroseconds) // display timing
124 log.Printf("load %s", args)
127 // Optimization: if the selected analyzers don't produce/consume
128 // facts, we need source only for the initial packages.
129 allSyntax := needFacts(analyzers)
130 initial, err := load(args, allSyntax)
133 return 1 // load errors
136 // Print the results.
137 roots := analyze(initial, analyzers)
143 return printDiagnostics(roots)
146 // load loads the initial packages.
147 func load(patterns []string, allSyntax bool) ([]*packages.Package, error) {
148 mode := packages.LoadSyntax
150 mode = packages.LoadAllSyntax
152 conf := packages.Config{
156 initial, err := packages.Load(&conf, patterns...)
158 if n := packages.PrintErrors(initial); n > 1 {
159 err = fmt.Errorf("%d errors during loading", n)
161 err = fmt.Errorf("error during loading")
162 } else if len(initial) == 0 {
163 err = fmt.Errorf("%s matched no packages", strings.Join(patterns, " "))
170 // TestAnalyzer applies an analysis to a set of packages (and their
171 // dependencies if necessary) and returns the results.
173 // Facts about pkg are returned in a map keyed by object; package facts
176 // This entry point is used only by analysistest.
177 func TestAnalyzer(a *analysis.Analyzer, pkgs []*packages.Package) []*TestAnalyzerResult {
178 var results []*TestAnalyzerResult
179 for _, act := range analyze(pkgs, []*analysis.Analyzer{a}) {
180 facts := make(map[types.Object][]analysis.Fact)
181 for key, fact := range act.objectFacts {
182 if key.obj.Pkg() == act.pass.Pkg {
183 facts[key.obj] = append(facts[key.obj], fact)
186 for key, fact := range act.packageFacts {
187 if key.pkg == act.pass.Pkg {
188 facts[nil] = append(facts[nil], fact)
192 results = append(results, &TestAnalyzerResult{act.pass, act.diagnostics, facts, act.result, act.err})
197 type TestAnalyzerResult struct {
199 Diagnostics []analysis.Diagnostic
200 Facts map[types.Object][]analysis.Fact
205 func analyze(pkgs []*packages.Package, analyzers []*analysis.Analyzer) []*action {
206 // Construct the action graph.
208 log.Printf("building graph of analysis passes")
211 // Each graph node (action) is one unit of analysis.
212 // Edges express package-to-package (vertical) dependencies,
213 // and analysis-to-analysis (horizontal) dependencies.
218 actions := make(map[key]*action)
220 var mkAction func(a *analysis.Analyzer, pkg *packages.Package) *action
221 mkAction = func(a *analysis.Analyzer, pkg *packages.Package) *action {
223 act, ok := actions[k]
225 act = &action{a: a, pkg: pkg}
227 // Add a dependency on each required analyzers.
228 for _, req := range a.Requires {
229 act.deps = append(act.deps, mkAction(req, pkg))
232 // An analysis that consumes/produces facts
233 // must run on the package's dependencies too.
234 if len(a.FactTypes) > 0 {
235 paths := make([]string, 0, len(pkg.Imports))
236 for path := range pkg.Imports {
237 paths = append(paths, path)
239 sort.Strings(paths) // for determinism
240 for _, path := range paths {
241 dep := mkAction(a, pkg.Imports[path])
242 act.deps = append(act.deps, dep)
251 // Build nodes for initial packages.
253 for _, a := range analyzers {
254 for _, pkg := range pkgs {
255 root := mkAction(a, pkg)
257 roots = append(roots, root)
261 // Execute the graph in parallel.
267 func applyFixes(roots []*action) {
268 visited := make(map[*action]bool)
269 var apply func(*action) error
270 var visitAll func(actions []*action) error
271 visitAll = func(actions []*action) error {
272 for _, act := range actions {
276 if err := apply(act); err != nil {
284 // TODO(matloob): Is this tree business too complicated? (After all this is Go!)
285 // Just create a set (map) of edits, sort by pos and call it a day?
286 type offsetedit struct {
289 } // TextEdit using byteOffsets instead of pos
295 var insert func(tree **node, edit offsetedit) error
296 insert = func(treeptr **node, edit offsetedit) error {
298 *treeptr = &node{edit, nil, nil}
302 if edit.end <= tree.edit.start {
303 return insert(&tree.left, edit)
304 } else if edit.start >= tree.edit.end {
305 return insert(&tree.right, edit)
308 // Overlapping text edit.
309 return fmt.Errorf("analyses applying overlapping text edits affecting pos range (%v, %v) and (%v, %v)",
310 edit.start, edit.end, tree.edit.start, tree.edit.end)
314 editsForFile := make(map[*token.File]*node)
316 apply = func(act *action) error {
317 for _, diag := range act.diagnostics {
318 for _, sf := range diag.SuggestedFixes {
319 for _, edit := range sf.TextEdits {
320 // Validate the edit.
321 if edit.Pos > edit.End {
323 "diagnostic for analysis %v contains Suggested Fix with malformed edit: pos (%v) > end (%v)",
324 act.a.Name, edit.Pos, edit.End)
326 file, endfile := act.pkg.Fset.File(edit.Pos), act.pkg.Fset.File(edit.End)
327 if file == nil || endfile == nil || file != endfile {
329 "diagnostic for analysis %v contains Suggested Fix with malformed spanning files %v and %v",
330 act.a.Name, file.Name(), endfile.Name()))
332 start, end := file.Offset(edit.Pos), file.Offset(edit.End)
334 // TODO(matloob): Validate that edits do not affect other packages.
335 root := editsForFile[file]
336 if err := insert(&root, offsetedit{start, end, edit.NewText}); err != nil {
339 editsForFile[file] = root // In case the root changed
348 fset := token.NewFileSet() // Shared by parse calls below
349 // Now we've got a set of valid edits for each file. Get the new file contents.
350 for f, tree := range editsForFile {
351 contents, err := ioutil.ReadFile(f.Name())
356 cur := 0 // current position in the file
360 var recurse func(*node)
361 recurse = func(node *node) {
362 if node.left != nil {
367 if edit.start > cur {
368 out.Write(contents[cur:edit.start])
369 out.Write(edit.newText)
373 if node.right != nil {
378 // Write out the rest of the file.
379 if cur < len(contents) {
380 out.Write(contents[cur:])
383 // Try to format the file.
384 ff, err := parser.ParseFile(fset, f.Name(), out.Bytes(), parser.ParseComments)
387 if err = format.Node(&buf, fset, ff); err == nil {
392 ioutil.WriteFile(f.Name(), out.Bytes(), 0644)
396 // printDiagnostics prints the diagnostics for the root packages in either
397 // plain text or JSON format. JSON format also includes errors for any
400 // It returns the exitcode: in plain mode, 0 for success, 1 for analysis
401 // errors, and 3 for diagnostics. We avoid 2 since the flag package uses
402 // it. JSON mode always succeeds at printing errors and diagnostics in a
403 // structured form to stdout.
404 func printDiagnostics(roots []*action) (exitcode int) {
407 // Print diagnostics only for root packages,
408 // but errors for all packages.
409 printed := make(map[*action]bool)
410 var print func(*action)
411 var visitAll func(actions []*action)
412 visitAll = func(actions []*action) {
413 for _, act := range actions {
422 if analysisflags.JSON {
424 tree := make(analysisflags.JSONTree)
425 print = func(act *action) {
426 var diags []analysis.Diagnostic
428 diags = act.diagnostics
430 tree.Add(act.pkg.Fset, act.pkg.ID, act.a.Name, diags, act.err)
437 // De-duplicate diagnostics by position (not token.Pos) to
438 // avoid double-reporting in source files that belong to
439 // multiple packages, such as foo and foo.test.
446 seen := make(map[key]bool)
448 print = func(act *action) {
450 fmt.Fprintf(os.Stderr, "%s: %v\n", act.a.Name, act.err)
451 exitcode = 1 // analysis failed, at least partially
455 for _, diag := range act.diagnostics {
456 // We don't display a.Name/f.Category
457 // as most users don't care.
459 posn := act.pkg.Fset.Position(diag.Pos)
460 end := act.pkg.Fset.Position(diag.End)
461 k := key{posn, end, act.a, diag.Message}
463 continue // duplicate
467 analysisflags.PrintPlain(act.pkg.Fset, diag)
473 if exitcode == 0 && len(seen) > 0 {
474 exitcode = 3 // successfully produced diagnostics
478 // Print timing info.
481 log.Println("Warning: times are mostly GC/scheduler noise; use -debug=tp to disable parallelism")
484 var total time.Duration
485 for act := range printed {
486 all = append(all, act)
487 total += act.duration
489 sort.Slice(all, func(i, j int) bool {
490 return all[i].duration > all[j].duration
493 // Print actions accounting for 90% of the total.
494 var sum time.Duration
495 for _, act := range all {
496 fmt.Fprintf(os.Stderr, "%s\t%s\n", act.duration, act)
498 if sum >= total*9/10 {
507 // needFacts reports whether any analysis required by the specified set
508 // needs facts. If so, we must load the entire program from source.
509 func needFacts(analyzers []*analysis.Analyzer) bool {
510 seen := make(map[*analysis.Analyzer]bool)
511 var q []*analysis.Analyzer // for BFS
512 q = append(q, analyzers...)
518 if len(a.FactTypes) > 0 {
521 q = append(q, a.Requires...)
527 // An action represents one unit of analysis work: the application of
528 // one analysis to one package. Actions form a DAG, both within a
529 // package (as different analyzers are applied, either in sequence or
530 // parallel), and across packages (as dependencies are analyzed).
534 pkg *packages.Package
538 objectFacts map[objectFactKey]analysis.Fact
539 packageFacts map[packageFactKey]analysis.Fact
540 inputs map[*analysis.Analyzer]interface{}
542 diagnostics []analysis.Diagnostic
544 duration time.Duration
547 type objectFactKey struct {
552 type packageFactKey struct {
557 func (act *action) String() string {
558 return fmt.Sprintf("%s@%s", act.a, act.pkg)
561 func execAll(actions []*action) {
562 sequential := dbg('p')
563 var wg sync.WaitGroup
564 for _, act := range actions {
566 work := func(act *action) {
579 func (act *action) exec() { act.once.Do(act.execOnce) }
581 func (act *action) execOnce() {
582 // Analyze dependencies.
585 // TODO(adonovan): uncomment this during profiling.
586 // It won't build pre-go1.11 but conditional compilation
587 // using build tags isn't warranted.
589 // ctx, task := trace.NewTask(context.Background(), "exec")
590 // trace.Log(ctx, "pass", act.String())
593 // Record time spent in this node but not its dependencies.
594 // In parallel mode, due to GC/scheduler contention, the
595 // time is 5x higher than in sequential mode, even with a
596 // semaphore limiting the number of threads here.
600 defer func() { act.duration = time.Since(t0) }()
603 // Report an error if any dependency failed.
605 for _, dep := range act.deps {
607 failed = append(failed, dep.String())
612 act.err = fmt.Errorf("failed prerequisites: %s", strings.Join(failed, ", "))
616 // Plumb the output values of the dependencies
617 // into the inputs of this action. Also facts.
618 inputs := make(map[*analysis.Analyzer]interface{})
619 act.objectFacts = make(map[objectFactKey]analysis.Fact)
620 act.packageFacts = make(map[packageFactKey]analysis.Fact)
621 for _, dep := range act.deps {
622 if dep.pkg == act.pkg {
623 // Same package, different analysis (horizontal edge):
624 // in-memory outputs of prerequisite analyzers
625 // become inputs to this analysis pass.
626 inputs[dep.a] = dep.result
628 } else if dep.a == act.a { // (always true)
629 // Same analysis, different package (vertical edge):
630 // serialized facts produced by prerequisite analysis
631 // become available to this analysis pass.
632 inheritFacts(act, dep)
637 pass := &analysis.Pass{
640 Files: act.pkg.Syntax,
641 OtherFiles: act.pkg.OtherFiles,
642 IgnoredFiles: act.pkg.IgnoredFiles,
644 TypesInfo: act.pkg.TypesInfo,
645 TypesSizes: act.pkg.TypesSizes,
647 Report: func(d analysis.Diagnostic) { act.diagnostics = append(act.diagnostics, d) },
648 ImportObjectFact: act.importObjectFact,
649 ExportObjectFact: act.exportObjectFact,
650 ImportPackageFact: act.importPackageFact,
651 ExportPackageFact: act.exportPackageFact,
652 AllObjectFacts: act.allObjectFacts,
653 AllPackageFacts: act.allPackageFacts,
657 var errors []types.Error
658 // Get any type errors that are attributed to the pkg.
659 // This is necessary to test analyzers that provide
660 // suggested fixes for compiler/type errors.
661 for _, err := range act.pkg.Errors {
662 if err.Kind != packages.TypeError {
665 // err.Pos is a string of form: "file:line:col" or "file:line" or "" or "-"
666 spn := span.Parse(err.Pos)
667 // Extract the token positions from the error string.
668 line, col, offset := spn.Start().Line(), spn.Start().Column(), -1
669 act.pkg.Fset.Iterate(func(f *token.File) bool {
670 if f.Name() != spn.URI().Filename() {
673 offset = int(f.LineStart(line)) + col - 1
679 errors = append(errors, types.Error{
682 Pos: token.Pos(offset),
685 analysisinternal.SetTypeErrors(pass, errors)
688 if act.pkg.IllTyped && !pass.Analyzer.RunDespiteErrors {
689 err = fmt.Errorf("analysis skipped due to errors in package")
691 act.result, err = pass.Analyzer.Run(pass)
693 if got, want := reflect.TypeOf(act.result), pass.Analyzer.ResultType; got != want {
695 "internal error: on package %s, analyzer %s returned a result of type %v, but declared ResultType %v",
696 pass.Pkg.Path(), pass.Analyzer, got, want)
702 // disallow calls after Run
703 pass.ExportObjectFact = nil
704 pass.ExportPackageFact = nil
707 // inheritFacts populates act.facts with
708 // those it obtains from its dependency, dep.
709 func inheritFacts(act, dep *action) {
710 serialize := dbg('s')
712 for key, fact := range dep.objectFacts {
713 // Filter out facts related to objects
714 // that are irrelevant downstream
715 // (equivalently: not in the compiler export data).
716 if !exportedFrom(key.obj, dep.pkg.Types) {
718 log.Printf("%v: discarding %T fact from %s for %s: %s", act, fact, dep, key.obj, fact)
723 // Optionally serialize/deserialize fact
724 // to verify that it works across address spaces.
726 encodedFact, err := codeFact(fact)
728 log.Panicf("internal error: encoding of %T fact failed in %v", fact, act)
734 log.Printf("%v: inherited %T fact for %s: %s", act, fact, key.obj, fact)
736 act.objectFacts[key] = fact
739 for key, fact := range dep.packageFacts {
740 // TODO: filter out facts that belong to
741 // packages not mentioned in the export data
742 // to prevent side channels.
744 // Optionally serialize/deserialize fact
745 // to verify that it works across address spaces
746 // and is deterministic.
748 encodedFact, err := codeFact(fact)
750 log.Panicf("internal error: encoding of %T fact failed in %v", fact, act)
756 log.Printf("%v: inherited %T fact for %s: %s", act, fact, key.pkg.Path(), fact)
758 act.packageFacts[key] = fact
762 // codeFact encodes then decodes a fact,
763 // just to exercise that logic.
764 func codeFact(fact analysis.Fact) (analysis.Fact, error) {
765 // We encode facts one at a time.
766 // A real modular driver would emit all facts
767 // into one encoder to improve gob efficiency.
769 if err := gob.NewEncoder(&buf).Encode(fact); err != nil {
773 // Encode it twice and assert that we get the same bits.
774 // This helps detect nondeterministic Gob encoding (e.g. of maps).
775 var buf2 bytes.Buffer
776 if err := gob.NewEncoder(&buf2).Encode(fact); err != nil {
779 if !bytes.Equal(buf.Bytes(), buf2.Bytes()) {
780 return nil, fmt.Errorf("encoding of %T fact is nondeterministic", fact)
783 new := reflect.New(reflect.TypeOf(fact).Elem()).Interface().(analysis.Fact)
784 if err := gob.NewDecoder(&buf).Decode(new); err != nil {
790 // exportedFrom reports whether obj may be visible to a package that imports pkg.
791 // This includes not just the exported members of pkg, but also unexported
792 // constants, types, fields, and methods, perhaps belonging to oether packages,
793 // that find there way into the API.
794 // This is an overapproximation of the more accurate approach used by
795 // gc export data, which walks the type graph, but it's much simpler.
797 // TODO(adonovan): do more accurate filtering by walking the type graph.
798 func exportedFrom(obj types.Object, pkg *types.Package) bool {
799 switch obj := obj.(type) {
801 return obj.Exported() && obj.Pkg() == pkg ||
802 obj.Type().(*types.Signature).Recv() != nil
807 // we can't filter more aggressively than this because we need
808 // to consider function parameters exported, but have no way
809 // of telling apart function parameters from local variables.
810 return obj.Pkg() == pkg
811 case *types.TypeName, *types.Const:
814 return false // Nil, Builtin, Label, or PkgName
817 // importObjectFact implements Pass.ImportObjectFact.
818 // Given a non-nil pointer ptr of type *T, where *T satisfies Fact,
819 // importObjectFact copies the fact value to *ptr.
820 func (act *action) importObjectFact(obj types.Object, ptr analysis.Fact) bool {
824 key := objectFactKey{obj, factType(ptr)}
825 if v, ok := act.objectFacts[key]; ok {
826 reflect.ValueOf(ptr).Elem().Set(reflect.ValueOf(v).Elem())
832 // exportObjectFact implements Pass.ExportObjectFact.
833 func (act *action) exportObjectFact(obj types.Object, fact analysis.Fact) {
834 if act.pass.ExportObjectFact == nil {
835 log.Panicf("%s: Pass.ExportObjectFact(%s, %T) called after Run", act, obj, fact)
838 if obj.Pkg() != act.pkg.Types {
839 log.Panicf("internal error: in analysis %s of package %s: Fact.Set(%s, %T): can't set facts on objects belonging another package",
840 act.a, act.pkg, obj, fact)
843 key := objectFactKey{obj, factType(fact)}
844 act.objectFacts[key] = fact // clobber any existing entry
846 objstr := types.ObjectString(obj, (*types.Package).Name)
847 fmt.Fprintf(os.Stderr, "%s: object %s has fact %s\n",
848 act.pkg.Fset.Position(obj.Pos()), objstr, fact)
852 // allObjectFacts implements Pass.AllObjectFacts.
853 func (act *action) allObjectFacts() []analysis.ObjectFact {
854 facts := make([]analysis.ObjectFact, 0, len(act.objectFacts))
855 for k := range act.objectFacts {
856 facts = append(facts, analysis.ObjectFact{k.obj, act.objectFacts[k]})
861 // importPackageFact implements Pass.ImportPackageFact.
862 // Given a non-nil pointer ptr of type *T, where *T satisfies Fact,
863 // fact copies the fact value to *ptr.
864 func (act *action) importPackageFact(pkg *types.Package, ptr analysis.Fact) bool {
868 key := packageFactKey{pkg, factType(ptr)}
869 if v, ok := act.packageFacts[key]; ok {
870 reflect.ValueOf(ptr).Elem().Set(reflect.ValueOf(v).Elem())
876 // exportPackageFact implements Pass.ExportPackageFact.
877 func (act *action) exportPackageFact(fact analysis.Fact) {
878 if act.pass.ExportPackageFact == nil {
879 log.Panicf("%s: Pass.ExportPackageFact(%T) called after Run", act, fact)
882 key := packageFactKey{act.pass.Pkg, factType(fact)}
883 act.packageFacts[key] = fact // clobber any existing entry
885 fmt.Fprintf(os.Stderr, "%s: package %s has fact %s\n",
886 act.pkg.Fset.Position(act.pass.Files[0].Pos()), act.pass.Pkg.Path(), fact)
890 func factType(fact analysis.Fact) reflect.Type {
891 t := reflect.TypeOf(fact)
892 if t.Kind() != reflect.Ptr {
893 log.Fatalf("invalid Fact type: got %T, want pointer", t)
898 // allObjectFacts implements Pass.AllObjectFacts.
899 func (act *action) allPackageFacts() []analysis.PackageFact {
900 facts := make([]analysis.PackageFact, 0, len(act.packageFacts))
901 for k := range act.packageFacts {
902 facts = append(facts, analysis.PackageFact{k.pkg, act.packageFacts[k]})
907 func dbg(b byte) bool { return strings.IndexByte(Debug, b) >= 0 }