1 // Package analysistest provides utilities for testing analyzers.
20 "golang.org/x/tools/go/analysis"
21 "golang.org/x/tools/go/analysis/internal/checker"
22 "golang.org/x/tools/go/packages"
23 "golang.org/x/tools/internal/lsp/diff"
24 "golang.org/x/tools/internal/lsp/diff/myers"
25 "golang.org/x/tools/internal/span"
26 "golang.org/x/tools/internal/testenv"
27 "golang.org/x/tools/txtar"
30 // WriteFiles is a helper function that creates a temporary directory
31 // and populates it with a GOPATH-style project using filemap (which
32 // maps file names to contents). On success it returns the name of the
33 // directory and a cleanup function to delete it.
34 func WriteFiles(filemap map[string]string) (dir string, cleanup func(), err error) {
35 gopath, err := ioutil.TempDir("", "analysistest")
39 cleanup = func() { os.RemoveAll(gopath) }
41 for name, content := range filemap {
42 filename := filepath.Join(gopath, "src", name)
43 os.MkdirAll(filepath.Dir(filename), 0777) // ignore error
44 if err := ioutil.WriteFile(filename, []byte(content), 0666); err != nil {
49 return gopath, cleanup, nil
52 // TestData returns the effective filename of
53 // the program's "testdata" directory.
54 // This function may be overridden by projects using
55 // an alternative build system (such as Blaze) that
56 // does not run a test in its package directory.
57 var TestData = func() string {
58 testdata, err := filepath.Abs("testdata")
65 // Testing is an abstraction of a *testing.T.
66 type Testing interface {
67 Errorf(format string, args ...interface{})
70 // RunWithSuggestedFixes behaves like Run, but additionally verifies suggested fixes.
71 // It uses golden files placed alongside the source code under analysis:
72 // suggested fixes for code in example.go will be compared against example.go.golden.
74 // Golden files can be formatted in one of two ways: as plain Go source code, or as txtar archives.
75 // In the first case, all suggested fixes will be applied to the original source, which will then be compared against the golden file.
76 // In the second case, suggested fixes will be grouped by their messages, and each set of fixes will be applied and tested separately.
77 // Each section in the archive corresponds to a single message.
79 // A golden file using txtar may look like this:
80 // -- turn into single negation --
83 // func fn(b1, b2 bool) {
84 // if !b1 { // want `negating a boolean twice`
89 // -- remove double negation --
92 // func fn(b1, b2 bool) {
93 // if b1 { // want `negating a boolean twice`
97 func RunWithSuggestedFixes(t Testing, dir string, a *analysis.Analyzer, patterns ...string) []*Result {
98 r := Run(t, dir, a, patterns...)
100 // Process each result (package) separately, matching up the suggested
101 // fixes into a diff, which we will compare to the .golden file. We have
102 // to do this per-result in case a file appears in two packages, such as in
103 // packages with tests, where mypkg/a.go will appear in both mypkg and
104 // mypkg.test. In that case, the analyzer may suggest the same set of
105 // changes to a.go for each package. If we merge all the results, those
106 // changes get doubly applied, which will cause conflicts or mismatches.
107 // Validating the results separately means as long as the two analyses
108 // don't produce conflicting suggestions for a single file, everything
110 for _, act := range r {
111 // file -> message -> edits
112 fileEdits := make(map[*token.File]map[string][]diff.TextEdit)
113 fileContents := make(map[*token.File][]byte)
115 // Validate edits, prepare the fileEdits map and read the file contents.
116 for _, diag := range act.Diagnostics {
117 for _, sf := range diag.SuggestedFixes {
118 for _, edit := range sf.TextEdits {
119 // Validate the edit.
120 if edit.Pos > edit.End {
122 "diagnostic for analysis %v contains Suggested Fix with malformed edit: pos (%v) > end (%v)",
123 act.Pass.Analyzer.Name, edit.Pos, edit.End)
126 file, endfile := act.Pass.Fset.File(edit.Pos), act.Pass.Fset.File(edit.End)
127 if file == nil || endfile == nil || file != endfile {
129 "diagnostic for analysis %v contains Suggested Fix with malformed spanning files %v and %v",
130 act.Pass.Analyzer.Name, file.Name(), endfile.Name())
133 if _, ok := fileContents[file]; !ok {
134 contents, err := ioutil.ReadFile(file.Name())
136 t.Errorf("error reading %s: %v", file.Name(), err)
138 fileContents[file] = contents
140 spn, err := span.NewRange(act.Pass.Fset, edit.Pos, edit.End).Span()
142 t.Errorf("error converting edit to span %s: %v", file.Name(), err)
145 if _, ok := fileEdits[file]; !ok {
146 fileEdits[file] = make(map[string][]diff.TextEdit)
148 fileEdits[file][sf.Message] = append(fileEdits[file][sf.Message], diff.TextEdit{
150 NewText: string(edit.NewText),
156 for file, fixes := range fileEdits {
157 // Get the original file contents.
158 orig, ok := fileContents[file]
160 t.Errorf("could not find file contents for %s", file.Name())
164 // Get the golden file and read the contents.
165 ar, err := txtar.ParseFile(file.Name() + ".golden")
167 t.Errorf("error reading %s.golden: %v", file.Name(), err)
171 if len(ar.Files) > 0 {
172 // one virtual file per kind of suggested fix
174 if len(ar.Comment) != 0 {
175 // we allow either just the comment, or just virtual
176 // files, not both. it is not clear how "both" should
178 t.Errorf("%s.golden has leading comment; we don't know what to do with it", file.Name())
182 for sf, edits := range fixes {
184 for _, vf := range ar.Files {
187 out := diff.ApplyEdits(string(orig), edits)
188 // the file may contain multiple trailing
189 // newlines if the user places empty lines
190 // between files in the archive. normalize
191 // this to a single newline.
192 want := string(bytes.TrimRight(vf.Data, "\n")) + "\n"
193 formatted, err := format.Source([]byte(out))
197 if want != string(formatted) {
198 d := myers.ComputeEdits("", want, string(formatted))
199 t.Errorf("suggested fixes failed for %s:\n%s", file.Name(), diff.ToUnified(fmt.Sprintf("%s.golden [%s]", file.Name(), sf), "actual", want, d))
205 t.Errorf("no section for suggested fix %q in %s.golden", sf, file.Name())
209 // all suggested fixes are represented by a single file
211 var catchallEdits []diff.TextEdit
212 for _, edits := range fixes {
213 catchallEdits = append(catchallEdits, edits...)
216 out := diff.ApplyEdits(string(orig), catchallEdits)
217 want := string(ar.Comment)
219 formatted, err := format.Source([]byte(out))
223 if want != string(formatted) {
224 d := myers.ComputeEdits("", want, string(formatted))
225 t.Errorf("suggested fixes failed for %s:\n%s", file.Name(), diff.ToUnified(file.Name()+".golden", "actual", want, d))
233 // Run applies an analysis to the packages denoted by the "go list" patterns.
235 // It loads the packages from the specified GOPATH-style project
236 // directory using golang.org/x/tools/go/packages, runs the analysis on
237 // them, and checks that each analysis emits the expected diagnostics
238 // and facts specified by the contents of '// want ...' comments in the
239 // package's source files.
241 // An expectation of a Diagnostic is specified by a string literal
242 // containing a regular expression that must match the diagnostic
243 // message. For example:
245 // fmt.Printf("%s", 1) // want `cannot provide int 1 to %s`
247 // An expectation of a Fact associated with an object is specified by
248 // 'name:"pattern"', where name is the name of the object, which must be
249 // declared on the same line as the comment, and pattern is a regular
250 // expression that must match the string representation of the fact,
251 // fmt.Sprint(fact). For example:
253 // func panicf(format string, args interface{}) { // want panicf:"printfWrapper"
255 // Package facts are specified by the name "package" and appear on
256 // line 1 of the first source file of the package.
258 // A single 'want' comment may contain a mixture of diagnostic and fact
259 // expectations, including multiple facts about the same object:
261 // // want "diag" "diag2" x:"fact1" x:"fact2" y:"fact3"
263 // Unexpected diagnostics and facts, and unmatched expectations, are
264 // reported as errors to the Testing.
266 // Run reports an error to the Testing if loading or analysis failed.
267 // Run also returns a Result for each package for which analysis was
268 // attempted, even if unsuccessful. It is safe for a test to ignore all
269 // the results, but a test may use it to perform additional checks.
270 func Run(t Testing, dir string, a *analysis.Analyzer, patterns ...string) []*Result {
271 if t, ok := t.(testenv.Testing); ok {
272 testenv.NeedsGoPackages(t)
275 pkgs, err := loadPackages(dir, patterns...)
277 t.Errorf("loading %s: %v", patterns, err)
281 results := checker.TestAnalyzer(a, pkgs)
282 for _, result := range results {
283 if result.Err != nil {
284 t.Errorf("error analyzing %s: %v", result.Pass, result.Err)
286 check(t, dir, result.Pass, result.Diagnostics, result.Facts)
292 // A Result holds the result of applying an analyzer to a package.
293 type Result = checker.TestAnalyzerResult
295 // loadPackages uses go/packages to load a specified packages (from source, with
296 // dependencies) from dir, which is the root of a GOPATH-style project
297 // tree. It returns an error if any package had an error, or the pattern
298 // matched no packages.
299 func loadPackages(dir string, patterns ...string) ([]*packages.Package, error) {
300 // packages.Load loads the real standard library, not a minimal
301 // fake version, which would be more efficient, especially if we
302 // have many small tests that import, say, net/http.
303 // However there is no easy way to make go/packages to consume
304 // a list of packages we generate and then do the parsing and
305 // typechecking, though this feature seems to be a recurring need.
307 cfg := &packages.Config{
308 Mode: packages.LoadAllSyntax,
311 Env: append(os.Environ(), "GOPATH="+dir, "GO111MODULE=off", "GOPROXY=off"),
313 pkgs, err := packages.Load(cfg, patterns...)
318 // Print errors but do not stop:
319 // some Analyzers may be disposed to RunDespiteErrors.
320 packages.PrintErrors(pkgs)
323 return nil, fmt.Errorf("no packages matched %s", patterns)
328 // check inspects an analysis pass on which the analysis has already
329 // been run, and verifies that all reported diagnostics and facts match
330 // specified by the contents of "// want ..." comments in the package's
331 // source files, which must have been parsed with comments enabled.
332 func check(t Testing, gopath string, pass *analysis.Pass, diagnostics []analysis.Diagnostic, facts map[types.Object][]analysis.Fact) {
338 want := make(map[key][]expectation)
340 // processComment parses expectations out of comments.
341 processComment := func(filename string, linenum int, text string) {
342 text = strings.TrimSpace(text)
344 // Any comment starting with "want" is treated
345 // as an expectation, even without following whitespace.
346 if rest := strings.TrimPrefix(text, "want"); rest != text {
347 expects, err := parseExpectations(rest)
349 t.Errorf("%s:%d: in 'want' comment: %s", filename, linenum, err)
353 want[key{filename, linenum}] = expects
358 // Extract 'want' comments from parsed Go files.
359 for _, f := range pass.Files {
360 for _, cgroup := range f.Comments {
361 for _, c := range cgroup.List {
363 text := strings.TrimPrefix(c.Text, "//")
364 if text == c.Text { // not a //-comment.
365 text = strings.TrimPrefix(text, "/*")
366 text = strings.TrimSuffix(text, "*/")
369 // Hack: treat a comment of the form "//...// want..."
370 // or "/*...// want... */
371 // as if it starts at 'want'.
372 // This allows us to add comments on comments,
373 // as required when testing the buildtag analyzer.
374 if i := strings.Index(text, "// want"); i >= 0 {
375 text = text[i+len("// "):]
378 // It's tempting to compute the filename
379 // once outside the loop, but it's
380 // incorrect because it can change due
381 // to //line directives.
382 posn := pass.Fset.Position(c.Pos())
383 filename := sanitize(gopath, posn.Filename)
384 processComment(filename, posn.Line, text)
389 // Extract 'want' comments from non-Go files.
390 // TODO(adonovan): we may need to handle //line directives.
391 for _, filename := range pass.OtherFiles {
392 data, err := ioutil.ReadFile(filename)
394 t.Errorf("can't read '// want' comments from %s: %v", filename, err)
397 filename := sanitize(gopath, filename)
399 for _, line := range strings.Split(string(data), "\n") {
402 // Hack: treat a comment of the form "//...// want..."
403 // or "/*...// want... */
404 // as if it starts at 'want'.
405 // This allows us to add comments on comments,
406 // as required when testing the buildtag analyzer.
407 if i := strings.Index(line, "// want"); i >= 0 {
411 if i := strings.Index(line, "//"); i >= 0 {
412 line = line[i+len("//"):]
413 processComment(filename, linenum, line)
418 checkMessage := func(posn token.Position, kind, name, message string) {
419 posn.Filename = sanitize(gopath, posn.Filename)
420 k := key{posn.Filename, posn.Line}
422 var unmatched []string
423 for i, exp := range expects {
424 if exp.kind == kind && exp.name == name {
425 if exp.rx.MatchString(message) {
426 // matched: remove the expectation.
427 expects[i] = expects[len(expects)-1]
428 expects = expects[:len(expects)-1]
432 unmatched = append(unmatched, fmt.Sprintf("%q", exp.rx))
435 if unmatched == nil {
436 t.Errorf("%v: unexpected %s: %v", posn, kind, message)
438 t.Errorf("%v: %s %q does not match pattern %s",
439 posn, kind, message, strings.Join(unmatched, " or "))
443 // Check the diagnostics match expectations.
444 for _, f := range diagnostics {
445 // TODO(matloob): Support ranges in analysistest.
446 posn := pass.Fset.Position(f.Pos)
447 checkMessage(posn, "diagnostic", "", f.Message)
450 // Check the facts match expectations.
451 // Report errors in lexical order for determinism.
452 // (It's only deterministic within each file, not across files,
453 // because go/packages does not guarantee file.Pos is ascending
454 // across the files of a single compilation unit.)
455 var objects []types.Object
456 for obj := range facts {
457 objects = append(objects, obj)
459 sort.Slice(objects, func(i, j int) bool {
460 // Package facts compare less than object facts.
461 ip, jp := objects[i] == nil, objects[j] == nil // whether i, j is a package fact
465 return objects[i].Pos() < objects[j].Pos()
467 for _, obj := range objects {
468 var posn token.Position
471 // Object facts are reported on the declaring line.
473 posn = pass.Fset.Position(obj.Pos())
475 // Package facts are reported at the start of the file.
477 posn = pass.Fset.Position(pass.Files[0].Pos())
481 for _, fact := range facts[obj] {
482 checkMessage(posn, "fact", name, fmt.Sprint(fact))
486 // Reject surplus expectations.
488 // Sometimes an Analyzer reports two similar diagnostics on a
489 // line with only one expectation. The reader may be confused by
490 // the error message.
491 // TODO(adonovan): print a better error:
492 // "got 2 diagnostics here; each one needs its own expectation".
494 for key, expects := range want {
495 for _, exp := range expects {
496 err := fmt.Sprintf("%s:%d: no %s was reported matching %q", key.file, key.line, exp.kind, exp.rx)
497 surplus = append(surplus, err)
500 sort.Strings(surplus)
501 for _, err := range surplus {
506 type expectation struct {
507 kind string // either "fact" or "diagnostic"
508 name string // name of object to which fact belongs, or "package" ("fact" only)
512 func (ex expectation) String() string {
513 return fmt.Sprintf("%s %s:%q", ex.kind, ex.name, ex.rx) // for debugging
516 // parseExpectations parses the content of a "// want ..." comment
517 // and returns the expectations, a mixture of diagnostics ("rx") and
518 // facts (name:"rx").
519 func parseExpectations(text string) ([]expectation, error) {
521 sc := new(scanner.Scanner).Init(strings.NewReader(text))
522 sc.Error = func(s *scanner.Scanner, msg string) {
523 scanErr = msg // e.g. bad string escape
525 sc.Mode = scanner.ScanIdents | scanner.ScanStrings | scanner.ScanRawStrings
527 scanRegexp := func(tok rune) (*regexp.Regexp, error) {
528 if tok != scanner.String && tok != scanner.RawString {
529 return nil, fmt.Errorf("got %s, want regular expression",
530 scanner.TokenString(tok))
532 pattern, _ := strconv.Unquote(sc.TokenText()) // can't fail
533 return regexp.Compile(pattern)
536 var expects []expectation
540 case scanner.String, scanner.RawString:
541 rx, err := scanRegexp(tok)
545 expects = append(expects, expectation{"diagnostic", "", rx})
548 name := sc.TokenText()
551 return nil, fmt.Errorf("got %s after %s, want ':'",
552 scanner.TokenString(tok), name)
555 rx, err := scanRegexp(tok)
559 expects = append(expects, expectation{"fact", name, rx})
563 return nil, fmt.Errorf("%s", scanErr)
568 return nil, fmt.Errorf("unexpected %s", scanner.TokenString(tok))
573 // sanitize removes the GOPATH portion of the filename,
574 // typically a gnarly /tmp directory, and returns the rest.
575 func sanitize(gopath, filename string) string {
576 prefix := gopath + string(os.PathSeparator) + "src" + string(os.PathSeparator)
577 return filepath.ToSlash(strings.TrimPrefix(filename, prefix))