1 // Copyright 2018 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // Package analysistest provides utilities for testing analyzers.
24 "golang.org/x/tools/go/analysis"
25 "golang.org/x/tools/go/analysis/internal/checker"
26 "golang.org/x/tools/go/packages"
27 "golang.org/x/tools/internal/lsp/diff"
28 "golang.org/x/tools/internal/lsp/diff/myers"
29 "golang.org/x/tools/internal/span"
30 "golang.org/x/tools/internal/testenv"
31 "golang.org/x/tools/txtar"
34 // WriteFiles is a helper function that creates a temporary directory
35 // and populates it with a GOPATH-style project using filemap (which
36 // maps file names to contents). On success it returns the name of the
37 // directory and a cleanup function to delete it.
38 func WriteFiles(filemap map[string]string) (dir string, cleanup func(), err error) {
39 gopath, err := ioutil.TempDir("", "analysistest")
43 cleanup = func() { os.RemoveAll(gopath) }
45 for name, content := range filemap {
46 filename := filepath.Join(gopath, "src", name)
47 os.MkdirAll(filepath.Dir(filename), 0777) // ignore error
48 if err := ioutil.WriteFile(filename, []byte(content), 0666); err != nil {
53 return gopath, cleanup, nil
56 // TestData returns the effective filename of
57 // the program's "testdata" directory.
58 // This function may be overridden by projects using
59 // an alternative build system (such as Blaze) that
60 // does not run a test in its package directory.
61 var TestData = func() string {
62 testdata, err := filepath.Abs("testdata")
69 // Testing is an abstraction of a *testing.T.
70 type Testing interface {
71 Errorf(format string, args ...interface{})
74 // RunWithSuggestedFixes behaves like Run, but additionally verifies suggested fixes.
75 // It uses golden files placed alongside the source code under analysis:
76 // suggested fixes for code in example.go will be compared against example.go.golden.
78 // Golden files can be formatted in one of two ways: as plain Go source code, or as txtar archives.
79 // In the first case, all suggested fixes will be applied to the original source, which will then be compared against the golden file.
80 // In the second case, suggested fixes will be grouped by their messages, and each set of fixes will be applied and tested separately.
81 // Each section in the archive corresponds to a single message.
83 // A golden file using txtar may look like this:
84 // -- turn into single negation --
87 // func fn(b1, b2 bool) {
88 // if !b1 { // want `negating a boolean twice`
93 // -- remove double negation --
96 // func fn(b1, b2 bool) {
97 // if b1 { // want `negating a boolean twice`
101 func RunWithSuggestedFixes(t Testing, dir string, a *analysis.Analyzer, patterns ...string) []*Result {
102 r := Run(t, dir, a, patterns...)
104 // Process each result (package) separately, matching up the suggested
105 // fixes into a diff, which we will compare to the .golden file. We have
106 // to do this per-result in case a file appears in two packages, such as in
107 // packages with tests, where mypkg/a.go will appear in both mypkg and
108 // mypkg.test. In that case, the analyzer may suggest the same set of
109 // changes to a.go for each package. If we merge all the results, those
110 // changes get doubly applied, which will cause conflicts or mismatches.
111 // Validating the results separately means as long as the two analyses
112 // don't produce conflicting suggestions for a single file, everything
114 for _, act := range r {
115 // file -> message -> edits
116 fileEdits := make(map[*token.File]map[string][]diff.TextEdit)
117 fileContents := make(map[*token.File][]byte)
119 // Validate edits, prepare the fileEdits map and read the file contents.
120 for _, diag := range act.Diagnostics {
121 for _, sf := range diag.SuggestedFixes {
122 for _, edit := range sf.TextEdits {
123 // Validate the edit.
124 if edit.Pos > edit.End {
126 "diagnostic for analysis %v contains Suggested Fix with malformed edit: pos (%v) > end (%v)",
127 act.Pass.Analyzer.Name, edit.Pos, edit.End)
130 file, endfile := act.Pass.Fset.File(edit.Pos), act.Pass.Fset.File(edit.End)
131 if file == nil || endfile == nil || file != endfile {
133 "diagnostic for analysis %v contains Suggested Fix with malformed spanning files %v and %v",
134 act.Pass.Analyzer.Name, file.Name(), endfile.Name())
137 if _, ok := fileContents[file]; !ok {
138 contents, err := ioutil.ReadFile(file.Name())
140 t.Errorf("error reading %s: %v", file.Name(), err)
142 fileContents[file] = contents
144 spn, err := span.NewRange(act.Pass.Fset, edit.Pos, edit.End).Span()
146 t.Errorf("error converting edit to span %s: %v", file.Name(), err)
149 if _, ok := fileEdits[file]; !ok {
150 fileEdits[file] = make(map[string][]diff.TextEdit)
152 fileEdits[file][sf.Message] = append(fileEdits[file][sf.Message], diff.TextEdit{
154 NewText: string(edit.NewText),
160 for file, fixes := range fileEdits {
161 // Get the original file contents.
162 orig, ok := fileContents[file]
164 t.Errorf("could not find file contents for %s", file.Name())
168 // Get the golden file and read the contents.
169 ar, err := txtar.ParseFile(file.Name() + ".golden")
171 t.Errorf("error reading %s.golden: %v", file.Name(), err)
175 if len(ar.Files) > 0 {
176 // one virtual file per kind of suggested fix
178 if len(ar.Comment) != 0 {
179 // we allow either just the comment, or just virtual
180 // files, not both. it is not clear how "both" should
182 t.Errorf("%s.golden has leading comment; we don't know what to do with it", file.Name())
186 for sf, edits := range fixes {
188 for _, vf := range ar.Files {
191 out := diff.ApplyEdits(string(orig), edits)
192 // the file may contain multiple trailing
193 // newlines if the user places empty lines
194 // between files in the archive. normalize
195 // this to a single newline.
196 want := string(bytes.TrimRight(vf.Data, "\n")) + "\n"
197 formatted, err := format.Source([]byte(out))
201 if want != string(formatted) {
202 d, err := myers.ComputeEdits("", want, string(formatted))
204 t.Errorf("failed to compute suggested fixes: %v", err)
206 t.Errorf("suggested fixes failed for %s:\n%s", file.Name(), diff.ToUnified(fmt.Sprintf("%s.golden [%s]", file.Name(), sf), "actual", want, d))
212 t.Errorf("no section for suggested fix %q in %s.golden", sf, file.Name())
216 // all suggested fixes are represented by a single file
218 var catchallEdits []diff.TextEdit
219 for _, edits := range fixes {
220 catchallEdits = append(catchallEdits, edits...)
223 out := diff.ApplyEdits(string(orig), catchallEdits)
224 want := string(ar.Comment)
226 formatted, err := format.Source([]byte(out))
230 if want != string(formatted) {
231 d, err := myers.ComputeEdits("", want, string(formatted))
233 t.Errorf("failed to compute edits: %s", err)
235 t.Errorf("suggested fixes failed for %s:\n%s", file.Name(), diff.ToUnified(file.Name()+".golden", "actual", want, d))
243 // Run applies an analysis to the packages denoted by the "go list" patterns.
245 // It loads the packages from the specified GOPATH-style project
246 // directory using golang.org/x/tools/go/packages, runs the analysis on
247 // them, and checks that each analysis emits the expected diagnostics
248 // and facts specified by the contents of '// want ...' comments in the
249 // package's source files.
251 // An expectation of a Diagnostic is specified by a string literal
252 // containing a regular expression that must match the diagnostic
253 // message. For example:
255 // fmt.Printf("%s", 1) // want `cannot provide int 1 to %s`
257 // An expectation of a Fact associated with an object is specified by
258 // 'name:"pattern"', where name is the name of the object, which must be
259 // declared on the same line as the comment, and pattern is a regular
260 // expression that must match the string representation of the fact,
261 // fmt.Sprint(fact). For example:
263 // func panicf(format string, args interface{}) { // want panicf:"printfWrapper"
265 // Package facts are specified by the name "package" and appear on
266 // line 1 of the first source file of the package.
268 // A single 'want' comment may contain a mixture of diagnostic and fact
269 // expectations, including multiple facts about the same object:
271 // // want "diag" "diag2" x:"fact1" x:"fact2" y:"fact3"
273 // Unexpected diagnostics and facts, and unmatched expectations, are
274 // reported as errors to the Testing.
276 // Run reports an error to the Testing if loading or analysis failed.
277 // Run also returns a Result for each package for which analysis was
278 // attempted, even if unsuccessful. It is safe for a test to ignore all
279 // the results, but a test may use it to perform additional checks.
280 func Run(t Testing, dir string, a *analysis.Analyzer, patterns ...string) []*Result {
281 if t, ok := t.(testenv.Testing); ok {
282 testenv.NeedsGoPackages(t)
285 pkgs, err := loadPackages(dir, patterns...)
287 t.Errorf("loading %s: %v", patterns, err)
291 results := checker.TestAnalyzer(a, pkgs)
292 for _, result := range results {
293 if result.Err != nil {
294 t.Errorf("error analyzing %s: %v", result.Pass, result.Err)
296 check(t, dir, result.Pass, result.Diagnostics, result.Facts)
302 // A Result holds the result of applying an analyzer to a package.
303 type Result = checker.TestAnalyzerResult
305 // loadPackages uses go/packages to load a specified packages (from source, with
306 // dependencies) from dir, which is the root of a GOPATH-style project
307 // tree. It returns an error if any package had an error, or the pattern
308 // matched no packages.
309 func loadPackages(dir string, patterns ...string) ([]*packages.Package, error) {
310 // packages.Load loads the real standard library, not a minimal
311 // fake version, which would be more efficient, especially if we
312 // have many small tests that import, say, net/http.
313 // However there is no easy way to make go/packages to consume
314 // a list of packages we generate and then do the parsing and
315 // typechecking, though this feature seems to be a recurring need.
317 cfg := &packages.Config{
318 Mode: packages.LoadAllSyntax,
321 Env: append(os.Environ(), "GOPATH="+dir, "GO111MODULE=off", "GOPROXY=off"),
323 pkgs, err := packages.Load(cfg, patterns...)
328 // Print errors but do not stop:
329 // some Analyzers may be disposed to RunDespiteErrors.
330 packages.PrintErrors(pkgs)
333 return nil, fmt.Errorf("no packages matched %s", patterns)
338 // check inspects an analysis pass on which the analysis has already
339 // been run, and verifies that all reported diagnostics and facts match
340 // specified by the contents of "// want ..." comments in the package's
341 // source files, which must have been parsed with comments enabled.
342 func check(t Testing, gopath string, pass *analysis.Pass, diagnostics []analysis.Diagnostic, facts map[types.Object][]analysis.Fact) {
348 want := make(map[key][]expectation)
350 // processComment parses expectations out of comments.
351 processComment := func(filename string, linenum int, text string) {
352 text = strings.TrimSpace(text)
354 // Any comment starting with "want" is treated
355 // as an expectation, even without following whitespace.
356 if rest := strings.TrimPrefix(text, "want"); rest != text {
357 lineDelta, expects, err := parseExpectations(rest)
359 t.Errorf("%s:%d: in 'want' comment: %s", filename, linenum, err)
363 want[key{filename, linenum + lineDelta}] = expects
368 // Extract 'want' comments from parsed Go files.
369 for _, f := range pass.Files {
370 for _, cgroup := range f.Comments {
371 for _, c := range cgroup.List {
373 text := strings.TrimPrefix(c.Text, "//")
374 if text == c.Text { // not a //-comment.
375 text = strings.TrimPrefix(text, "/*")
376 text = strings.TrimSuffix(text, "*/")
379 // Hack: treat a comment of the form "//...// want..."
380 // or "/*...// want... */
381 // as if it starts at 'want'.
382 // This allows us to add comments on comments,
383 // as required when testing the buildtag analyzer.
384 if i := strings.Index(text, "// want"); i >= 0 {
385 text = text[i+len("// "):]
388 // It's tempting to compute the filename
389 // once outside the loop, but it's
390 // incorrect because it can change due
391 // to //line directives.
392 posn := pass.Fset.Position(c.Pos())
393 filename := sanitize(gopath, posn.Filename)
394 processComment(filename, posn.Line, text)
399 // Extract 'want' comments from non-Go files.
400 // TODO(adonovan): we may need to handle //line directives.
401 for _, filename := range pass.OtherFiles {
402 data, err := ioutil.ReadFile(filename)
404 t.Errorf("can't read '// want' comments from %s: %v", filename, err)
407 filename := sanitize(gopath, filename)
409 for _, line := range strings.Split(string(data), "\n") {
412 // Hack: treat a comment of the form "//...// want..."
413 // or "/*...// want... */
414 // as if it starts at 'want'.
415 // This allows us to add comments on comments,
416 // as required when testing the buildtag analyzer.
417 if i := strings.Index(line, "// want"); i >= 0 {
421 if i := strings.Index(line, "//"); i >= 0 {
422 line = line[i+len("//"):]
423 processComment(filename, linenum, line)
428 checkMessage := func(posn token.Position, kind, name, message string) {
429 posn.Filename = sanitize(gopath, posn.Filename)
430 k := key{posn.Filename, posn.Line}
432 var unmatched []string
433 for i, exp := range expects {
434 if exp.kind == kind && exp.name == name {
435 if exp.rx.MatchString(message) {
436 // matched: remove the expectation.
437 expects[i] = expects[len(expects)-1]
438 expects = expects[:len(expects)-1]
442 unmatched = append(unmatched, fmt.Sprintf("%q", exp.rx))
445 if unmatched == nil {
446 t.Errorf("%v: unexpected %s: %v", posn, kind, message)
448 t.Errorf("%v: %s %q does not match pattern %s",
449 posn, kind, message, strings.Join(unmatched, " or "))
453 // Check the diagnostics match expectations.
454 for _, f := range diagnostics {
455 // TODO(matloob): Support ranges in analysistest.
456 posn := pass.Fset.Position(f.Pos)
457 checkMessage(posn, "diagnostic", "", f.Message)
460 // Check the facts match expectations.
461 // Report errors in lexical order for determinism.
462 // (It's only deterministic within each file, not across files,
463 // because go/packages does not guarantee file.Pos is ascending
464 // across the files of a single compilation unit.)
465 var objects []types.Object
466 for obj := range facts {
467 objects = append(objects, obj)
469 sort.Slice(objects, func(i, j int) bool {
470 // Package facts compare less than object facts.
471 ip, jp := objects[i] == nil, objects[j] == nil // whether i, j is a package fact
475 return objects[i].Pos() < objects[j].Pos()
477 for _, obj := range objects {
478 var posn token.Position
481 // Object facts are reported on the declaring line.
483 posn = pass.Fset.Position(obj.Pos())
485 // Package facts are reported at the start of the file.
487 posn = pass.Fset.Position(pass.Files[0].Pos())
491 for _, fact := range facts[obj] {
492 checkMessage(posn, "fact", name, fmt.Sprint(fact))
496 // Reject surplus expectations.
498 // Sometimes an Analyzer reports two similar diagnostics on a
499 // line with only one expectation. The reader may be confused by
500 // the error message.
501 // TODO(adonovan): print a better error:
502 // "got 2 diagnostics here; each one needs its own expectation".
504 for key, expects := range want {
505 for _, exp := range expects {
506 err := fmt.Sprintf("%s:%d: no %s was reported matching %q", key.file, key.line, exp.kind, exp.rx)
507 surplus = append(surplus, err)
510 sort.Strings(surplus)
511 for _, err := range surplus {
516 type expectation struct {
517 kind string // either "fact" or "diagnostic"
518 name string // name of object to which fact belongs, or "package" ("fact" only)
522 func (ex expectation) String() string {
523 return fmt.Sprintf("%s %s:%q", ex.kind, ex.name, ex.rx) // for debugging
526 // parseExpectations parses the content of a "// want ..." comment
527 // and returns the expectations, a mixture of diagnostics ("rx") and
528 // facts (name:"rx").
529 func parseExpectations(text string) (lineDelta int, expects []expectation, err error) {
531 sc := new(scanner.Scanner).Init(strings.NewReader(text))
532 sc.Error = func(s *scanner.Scanner, msg string) {
533 scanErr = msg // e.g. bad string escape
535 sc.Mode = scanner.ScanIdents | scanner.ScanStrings | scanner.ScanRawStrings | scanner.ScanInts
537 scanRegexp := func(tok rune) (*regexp.Regexp, error) {
538 if tok != scanner.String && tok != scanner.RawString {
539 return nil, fmt.Errorf("got %s, want regular expression",
540 scanner.TokenString(tok))
542 pattern, _ := strconv.Unquote(sc.TokenText()) // can't fail
543 return regexp.Compile(pattern)
551 if tok != scanner.Int {
552 return 0, nil, fmt.Errorf("got +%s, want +Int", scanner.TokenString(tok))
554 lineDelta, _ = strconv.Atoi(sc.TokenText())
555 case scanner.String, scanner.RawString:
556 rx, err := scanRegexp(tok)
560 expects = append(expects, expectation{"diagnostic", "", rx})
563 name := sc.TokenText()
566 return 0, nil, fmt.Errorf("got %s after %s, want ':'",
567 scanner.TokenString(tok), name)
570 rx, err := scanRegexp(tok)
574 expects = append(expects, expectation{"fact", name, rx})
578 return 0, nil, fmt.Errorf("%s", scanErr)
580 return lineDelta, expects, nil
583 return 0, nil, fmt.Errorf("unexpected %s", scanner.TokenString(tok))
588 // sanitize removes the GOPATH portion of the filename,
589 // typically a gnarly /tmp directory, and returns the rest.
590 func sanitize(gopath, filename string) string {
591 prefix := gopath + string(os.PathSeparator) + "src" + string(os.PathSeparator)
592 return filepath.ToSlash(strings.TrimPrefix(filename, prefix))