1 // Copyright 2018 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
19 "golang.org/x/tools/internal/gocommand"
22 // processGolistOverlay provides rudimentary support for adding
23 // files that don't exist on disk to an overlay. The results can be
24 // sometimes incorrect.
25 // TODO(matloob): Handle unsupported cases, including the following:
26 // - determining the correct package to add given a new import path
27 func (state *golistState) processGolistOverlay(response *responseDeduper) (modifiedPkgs, needPkgs []string, err error) {
28 havePkgs := make(map[string]string) // importPath -> non-test package ID
29 needPkgsSet := make(map[string]bool)
30 modifiedPkgsSet := make(map[string]bool)
32 pkgOfDir := make(map[string][]*Package)
33 for _, pkg := range response.dr.Packages {
34 // This is an approximation of import path to id. This can be
35 // wrong for tests, vendored packages, and a number of other cases.
36 havePkgs[pkg.PkgPath] = pkg.ID
37 dir, err := commonDir(pkg.GoFiles)
42 pkgOfDir[dir] = append(pkgOfDir[dir], pkg)
46 // If no new imports are added, it is safe to avoid loading any needPkgs.
47 // Otherwise, it's hard to tell which package is actually being loaded
48 // (due to vendoring) and whether any modified package will show up
49 // in the transitive set of dependencies (because new imports are added,
50 // potentially modifying the transitive set of dependencies).
51 var overlayAddsImports bool
53 // If both a package and its test package are created by the overlay, we
54 // need the real package first. Process all non-test files before test
55 // files, and make the whole process deterministic while we're at it.
56 var overlayFiles []string
57 for opath := range state.cfg.Overlay {
58 overlayFiles = append(overlayFiles, opath)
60 sort.Slice(overlayFiles, func(i, j int) bool {
61 iTest := strings.HasSuffix(overlayFiles[i], "_test.go")
62 jTest := strings.HasSuffix(overlayFiles[j], "_test.go")
64 return !iTest // non-tests are before tests.
66 return overlayFiles[i] < overlayFiles[j]
68 for _, opath := range overlayFiles {
69 contents := state.cfg.Overlay[opath]
70 base := filepath.Base(opath)
71 dir := filepath.Dir(opath)
72 var pkg *Package // if opath belongs to both a package and its test variant, this will be the test variant
73 var testVariantOf *Package // if opath is a test file, this is the package it is testing
75 isTestFile := strings.HasSuffix(opath, "_test.go")
76 pkgName, ok := extractPackageName(opath, contents)
78 // Don't bother adding a file that doesn't even have a parsable package statement
82 // If all the overlay files belong to a different package, change the
83 // package name to that package.
84 maybeFixPackageName(pkgName, isTestFile, pkgOfDir[dir])
86 for _, p := range response.dr.Packages {
87 if pkgName != p.Name && p.ID != "command-line-arguments" {
90 for _, f := range p.GoFiles {
91 if !sameFile(filepath.Dir(f), dir) {
94 // Make sure to capture information on the package's test variant, if needed.
95 if isTestFile && !hasTestFiles(p) {
96 // TODO(matloob): Are there packages other than the 'production' variant
97 // of a package that this can match? This shouldn't match the test main package
98 // because the file is generated in another directory.
101 } else if !isTestFile && hasTestFiles(p) {
102 // We're examining a test variant, but the overlaid file is
103 // a non-test file. Because the overlay implementation
104 // (currently) only adds a file to one package, skip this
105 // package, so that we can add the file to the production
106 // variant of the package. (https://golang.org/issue/36857
107 // tracks handling overlays on both the production and test
108 // variant of a package).
111 if pkg != nil && p != pkg && pkg.PkgPath == p.PkgPath {
112 // We have already seen the production version of the
113 // for which p is a test variant.
119 if filepath.Base(f) == base {
124 // The overlay could have included an entirely new package or an
125 // ad-hoc package. An ad-hoc package is one that we have manually
126 // constructed from inadequate `go list` results for a file= query.
127 // It will have the ID command-line-arguments.
128 if pkg == nil || pkg.ID == "command-line-arguments" {
129 // Try to find the module or gopath dir the file is contained in.
130 // Then for modules, add the module opath to the beginning.
131 pkgPath, ok, err := state.getPkgPath(dir)
138 var forTest string // only set for x tests
139 isXTest := strings.HasSuffix(pkgName, "_test")
147 id = fmt.Sprintf("%s [%s.test]", pkgPath, forTest)
149 id = fmt.Sprintf("%s [%s.test]", pkgPath, pkgPath)
153 // TODO(rstambler): We should change the package's path and ID
154 // here. The only issue is that this messes with the roots.
156 // Try to reclaim a package with the same ID, if it exists in the response.
157 for _, p := range response.dr.Packages {
158 if reclaimPackage(p, id, opath, contents) {
163 // Otherwise, create a new package.
169 Imports: make(map[string]*Package),
171 response.addPackage(pkg)
172 havePkgs[pkg.PkgPath] = id
173 // Add the production package's sources for a test variant.
174 if isTestFile && !isXTest && testVariantOf != nil {
175 pkg.GoFiles = append(pkg.GoFiles, testVariantOf.GoFiles...)
176 pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, testVariantOf.CompiledGoFiles...)
177 // Add the package under test and its imports to the test variant.
178 pkg.forTest = testVariantOf.PkgPath
179 for k, v := range testVariantOf.Imports {
180 pkg.Imports[k] = &Package{ID: v.ID}
184 pkg.forTest = forTest
190 pkg.GoFiles = append(pkg.GoFiles, opath)
191 // TODO(matloob): Adding the file to CompiledGoFiles can exhibit the wrong behavior
192 // if the file will be ignored due to its build tags.
193 pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, opath)
194 modifiedPkgsSet[pkg.ID] = true
196 imports, err := extractImports(opath, contents)
198 // Let the parser or type checker report errors later.
201 for _, imp := range imports {
202 // TODO(rstambler): If the package is an x test and the import has
203 // a test variant, make sure to replace it.
204 if _, found := pkg.Imports[imp]; found {
207 overlayAddsImports = true
208 id, ok := havePkgs[imp]
211 id, err = state.resolveImport(dir, imp)
216 pkg.Imports[imp] = &Package{ID: id}
217 // Add dependencies to the non-test variant version of this package as well.
218 if testVariantOf != nil {
219 testVariantOf.Imports[imp] = &Package{ID: id}
224 // toPkgPath guesses the package path given the id.
225 toPkgPath := func(sourceDir, id string) (string, error) {
226 if i := strings.IndexByte(id, ' '); i >= 0 {
227 return state.resolveImport(sourceDir, id[:i])
229 return state.resolveImport(sourceDir, id)
232 // Now that new packages have been created, do another pass to determine
233 // the new set of missing packages.
234 for _, pkg := range response.dr.Packages {
235 for _, imp := range pkg.Imports {
236 if len(pkg.GoFiles) == 0 {
237 return nil, nil, fmt.Errorf("cannot resolve imports for package %q with no Go files", pkg.PkgPath)
239 pkgPath, err := toPkgPath(filepath.Dir(pkg.GoFiles[0]), imp.ID)
243 if _, ok := havePkgs[pkgPath]; !ok {
244 needPkgsSet[pkgPath] = true
249 if overlayAddsImports {
250 needPkgs = make([]string, 0, len(needPkgsSet))
251 for pkg := range needPkgsSet {
252 needPkgs = append(needPkgs, pkg)
255 modifiedPkgs = make([]string, 0, len(modifiedPkgsSet))
256 for pkg := range modifiedPkgsSet {
257 modifiedPkgs = append(modifiedPkgs, pkg)
259 return modifiedPkgs, needPkgs, err
262 // resolveImport finds the ID of a package given its import path.
263 // In particular, it will find the right vendored copy when in GOPATH mode.
264 func (state *golistState) resolveImport(sourceDir, importPath string) (string, error) {
265 env, err := state.getEnv()
269 if env["GOMOD"] != "" {
270 return importPath, nil
273 searchDir := sourceDir
275 vendorDir := filepath.Join(searchDir, "vendor")
276 exists, ok := state.vendorDirs[vendorDir]
278 info, err := os.Stat(vendorDir)
279 exists = err == nil && info.IsDir()
280 state.vendorDirs[vendorDir] = exists
284 vendoredPath := filepath.Join(vendorDir, importPath)
285 if info, err := os.Stat(vendoredPath); err == nil && info.IsDir() {
286 // We should probably check for .go files here, but shame on anyone who fools us.
287 path, ok, err := state.getPkgPath(vendoredPath)
297 // We know we've hit the top of the filesystem when we Dir / and get /,
298 // or C:\ and get C:\, etc.
299 next := filepath.Dir(searchDir)
300 if next == searchDir {
305 return importPath, nil
308 func hasTestFiles(p *Package) bool {
309 for _, f := range p.GoFiles {
310 if strings.HasSuffix(f, "_test.go") {
317 // determineRootDirs returns a mapping from absolute directories that could
318 // contain code to their corresponding import path prefixes.
319 func (state *golistState) determineRootDirs() (map[string]string, error) {
320 env, err := state.getEnv()
324 if env["GOMOD"] != "" {
325 state.rootsOnce.Do(func() {
326 state.rootDirs, state.rootDirsError = state.determineRootDirsModules()
329 state.rootsOnce.Do(func() {
330 state.rootDirs, state.rootDirsError = state.determineRootDirsGOPATH()
333 return state.rootDirs, state.rootDirsError
336 func (state *golistState) determineRootDirsModules() (map[string]string, error) {
337 // List all of the modules--the first will be the directory for the main
338 // module. Any replaced modules will also need to be treated as roots.
339 // Editing files in the module cache isn't a great idea, so we don't
340 // plan to ever support that.
341 out, err := state.invokeGo("list", "-m", "-json", "all")
343 // 'go list all' will fail if we're outside of a module and
344 // GO111MODULE=on. Try falling back without 'all'.
346 out, innerErr = state.invokeGo("list", "-m", "-json")
351 roots := map[string]string{}
352 modules := map[string]string{}
354 for dec := json.NewDecoder(out); dec.More(); {
355 mod := new(gocommand.ModuleJSON)
356 if err := dec.Decode(mod); err != nil {
359 if mod.Dir != "" && mod.Path != "" {
360 // This is a valid module; add it to the map.
361 absDir, err := filepath.Abs(mod.Dir)
365 modules[absDir] = mod.Path
366 // The first result is the main module.
367 if i == 0 || mod.Replace != nil && mod.Replace.Path != "" {
368 roots[absDir] = mod.Path
376 func (state *golistState) determineRootDirsGOPATH() (map[string]string, error) {
377 m := map[string]string{}
378 for _, dir := range filepath.SplitList(state.mustGetEnv()["GOPATH"]) {
379 absDir, err := filepath.Abs(dir)
383 m[filepath.Join(absDir, "src")] = ""
388 func extractImports(filename string, contents []byte) ([]string, error) {
389 f, err := parser.ParseFile(token.NewFileSet(), filename, contents, parser.ImportsOnly) // TODO(matloob): reuse fileset?
394 for _, imp := range f.Imports {
395 quotedPath := imp.Path.Value
396 path, err := strconv.Unquote(quotedPath)
400 res = append(res, path)
405 // reclaimPackage attempts to reuse a package that failed to load in an overlay.
407 // If the package has errors and has no Name, GoFiles, or Imports,
408 // then it's possible that it doesn't yet exist on disk.
409 func reclaimPackage(pkg *Package, id string, filename string, contents []byte) bool {
410 // TODO(rstambler): Check the message of the actual error?
411 // It differs between $GOPATH and module mode.
415 if len(pkg.Errors) != 1 {
418 if pkg.Name != "" || pkg.ExportFile != "" {
421 if len(pkg.GoFiles) > 0 || len(pkg.CompiledGoFiles) > 0 || len(pkg.OtherFiles) > 0 {
424 if len(pkg.Imports) > 0 {
427 pkgName, ok := extractPackageName(filename, contents)
436 func extractPackageName(filename string, contents []byte) (string, bool) {
437 // TODO(rstambler): Check the message of the actual error?
438 // It differs between $GOPATH and module mode.
439 f, err := parser.ParseFile(token.NewFileSet(), filename, contents, parser.PackageClauseOnly) // TODO(matloob): reuse fileset?
443 return f.Name.Name, true
446 // commonDir returns the directory that all files are in, "" if files is empty,
447 // or an error if they aren't in the same directory.
448 func commonDir(files []string) (string, error) {
449 seen := make(map[string]bool)
450 for _, f := range files {
451 seen[filepath.Dir(f)] = true
454 return "", fmt.Errorf("files (%v) are in more than one directory: %v", files, seen)
456 for k := range seen {
457 // seen has only one element; return it.
460 return "", nil // no files
463 // It is possible that the files in the disk directory dir have a different package
464 // name from newName, which is deduced from the overlays. If they all have a different
465 // package name, and they all have the same package name, then that name becomes
467 // It returns true if it changes the package name, false otherwise.
468 func maybeFixPackageName(newName string, isTestFile bool, pkgsOfDir []*Package) {
469 names := make(map[string]int)
470 for _, p := range pkgsOfDir {
474 // some files are in different packages
478 for k := range names {
481 if newName == oldName {
484 // We might have a case where all of the package names in the directory are
485 // the same, but the overlay file is for an x test, which belongs to its
486 // own package. If the x test does not yet exist on disk, we may not yet
487 // have its package name on disk, but we should not rename the packages.
489 // We use a heuristic to determine if this file belongs to an x test:
490 // The test file should have a package name whose package name has a _test
491 // suffix or looks like "newName_test".
492 maybeXTest := strings.HasPrefix(oldName+"_test", newName) || strings.HasSuffix(newName, "_test")
493 if isTestFile && maybeXTest {
496 for _, p := range pkgsOfDir {
501 // This function is copy-pasted from
502 // https://github.com/golang/go/blob/9706f510a5e2754595d716bd64be8375997311fb/src/cmd/go/internal/search/search.go#L360.
503 // It should be deleted when we remove support for overlays from go/packages.
505 // NOTE: This does not handle any ./... or ./ style queries, as this function
506 // doesn't know the working directory.
508 // matchPattern(pattern)(name) reports whether
509 // name matches pattern. Pattern is a limited glob
510 // pattern in which '...' means 'any string' and there
511 // is no other special syntax.
512 // Unfortunately, there are two special cases. Quoting "go help packages":
514 // First, /... at the end of the pattern can match an empty string,
515 // so that net/... matches both net and packages in its subdirectories, like net/http.
516 // Second, any slash-separated pattern element containing a wildcard never
517 // participates in a match of the "vendor" element in the path of a vendored
518 // package, so that ./... does not match packages in subdirectories of
519 // ./vendor or ./mycode/vendor, but ./vendor/... and ./mycode/vendor/... do.
520 // Note, however, that a directory named vendor that itself contains code
521 // is not a vendored package: cmd/vendor would be a command named vendor,
522 // and the pattern cmd/... matches it.
523 func matchPattern(pattern string) func(name string) bool {
524 // Convert pattern to regular expression.
525 // The strategy for the trailing /... is to nest it in an explicit ? expression.
526 // The strategy for the vendor exclusion is to change the unmatchable
527 // vendor strings to a disallowed code point (vendorChar) and to use
528 // "(anything but that codepoint)*" as the implementation of the ... wildcard.
529 // This is a bit complicated but the obvious alternative,
530 // namely a hand-written search like in most shell glob matchers,
531 // is too easy to make accidentally exponential.
532 // Using package regexp guarantees linear-time matching.
534 const vendorChar = "\x00"
536 if strings.Contains(pattern, vendorChar) {
537 return func(name string) bool { return false }
540 re := regexp.QuoteMeta(pattern)
541 re = replaceVendor(re, vendorChar)
543 case strings.HasSuffix(re, `/`+vendorChar+`/\.\.\.`):
544 re = strings.TrimSuffix(re, `/`+vendorChar+`/\.\.\.`) + `(/vendor|/` + vendorChar + `/\.\.\.)`
545 case re == vendorChar+`/\.\.\.`:
546 re = `(/vendor|/` + vendorChar + `/\.\.\.)`
547 case strings.HasSuffix(re, `/\.\.\.`):
548 re = strings.TrimSuffix(re, `/\.\.\.`) + `(/\.\.\.)?`
550 re = strings.ReplaceAll(re, `\.\.\.`, `[^`+vendorChar+`]*`)
552 reg := regexp.MustCompile(`^` + re + `$`)
554 return func(name string) bool {
555 if strings.Contains(name, vendorChar) {
558 return reg.MatchString(replaceVendor(name, vendorChar))
562 // replaceVendor returns the result of replacing
563 // non-trailing vendor path elements in x with repl.
564 func replaceVendor(x, repl string) string {
565 if !strings.Contains(x, "vendor") {
568 elem := strings.Split(x, "/")
569 for i := 0; i < len(elem)-1; i++ {
570 if elem[i] == "vendor" {
574 return strings.Join(elem, "/")