1 // Copyright 2019 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
21 "golang.org/x/mod/module"
22 "golang.org/x/tools/internal/gocommand"
23 "golang.org/x/tools/internal/gopathwalk"
26 // ModuleResolver implements resolver for modules using the go command as little
28 type ModuleResolver struct {
31 dummyVendorMod *gocommand.ModuleJSON // If vendoring is enabled, the pseudo-module that represents the /vendor directory.
32 roots []gopathwalk.Root
33 scanSema chan struct{} // scanSema prevents concurrent scans and guards scannedRoots.
34 scannedRoots map[gopathwalk.Root]bool
37 main *gocommand.ModuleJSON
38 modsByModPath []*gocommand.ModuleJSON // All modules, ordered by # of path components in module Path...
39 modsByDir []*gocommand.ModuleJSON // ...or Dir.
41 // moduleCacheCache stores information about the module cache.
42 moduleCacheCache *dirInfoCache
43 otherCache *dirInfoCache
46 func newModuleResolver(e *ProcessEnv) *ModuleResolver {
49 scanSema: make(chan struct{}, 1),
51 r.scanSema <- struct{}{}
55 func (r *ModuleResolver) init() error {
60 goenv, err := r.env.goEnv()
64 inv := gocommand.Invocation{
65 BuildFlags: r.env.BuildFlags,
66 ModFlag: r.env.ModFlag,
67 ModFile: r.env.ModFile,
70 WorkingDir: r.env.WorkingDir,
72 mainMod, vendorEnabled, err := gocommand.VendorEnabled(context.TODO(), inv, r.env.GocmdRunner)
77 if mainMod != nil && vendorEnabled {
78 // Vendor mode is on, so all the non-Main modules are irrelevant,
79 // and we need to search /vendor for everything.
81 r.dummyVendorMod = &gocommand.ModuleJSON{
83 Dir: filepath.Join(mainMod.Dir, "vendor"),
85 r.modsByModPath = []*gocommand.ModuleJSON{mainMod, r.dummyVendorMod}
86 r.modsByDir = []*gocommand.ModuleJSON{mainMod, r.dummyVendorMod}
88 // Vendor mode is off, so run go list -m ... to find everything.
89 err := r.initAllMods()
90 // We expect an error when running outside of a module with
91 // GO111MODULE=on. Other errors are fatal.
93 if errMsg := err.Error(); !strings.Contains(errMsg, "working directory is not part of a module") && !strings.Contains(errMsg, "go.mod file not found") {
99 if gmc := r.env.Env["GOMODCACHE"]; gmc != "" {
100 r.moduleCacheDir = gmc
102 gopaths := filepath.SplitList(goenv["GOPATH"])
103 if len(gopaths) == 0 {
104 return fmt.Errorf("empty GOPATH")
106 r.moduleCacheDir = filepath.Join(gopaths[0], "/pkg/mod")
109 sort.Slice(r.modsByModPath, func(i, j int) bool {
110 count := func(x int) int {
111 return strings.Count(r.modsByModPath[x].Path, "/")
113 return count(j) < count(i) // descending order
115 sort.Slice(r.modsByDir, func(i, j int) bool {
116 count := func(x int) int {
117 return strings.Count(r.modsByDir[x].Dir, "/")
119 return count(j) < count(i) // descending order
122 r.roots = []gopathwalk.Root{
123 {filepath.Join(goenv["GOROOT"], "/src"), gopathwalk.RootGOROOT},
126 r.roots = append(r.roots, gopathwalk.Root{r.main.Dir, gopathwalk.RootCurrentModule})
129 r.roots = append(r.roots, gopathwalk.Root{r.dummyVendorMod.Dir, gopathwalk.RootOther})
131 addDep := func(mod *gocommand.ModuleJSON) {
132 if mod.Replace == nil {
133 // This is redundant with the cache, but we'll skip it cheaply enough.
134 r.roots = append(r.roots, gopathwalk.Root{mod.Dir, gopathwalk.RootModuleCache})
136 r.roots = append(r.roots, gopathwalk.Root{mod.Dir, gopathwalk.RootOther})
139 // Walk dependent modules before scanning the full mod cache, direct deps first.
140 for _, mod := range r.modsByModPath {
141 if !mod.Indirect && !mod.Main {
145 for _, mod := range r.modsByModPath {
146 if mod.Indirect && !mod.Main {
150 r.roots = append(r.roots, gopathwalk.Root{r.moduleCacheDir, gopathwalk.RootModuleCache})
153 r.scannedRoots = map[gopathwalk.Root]bool{}
154 if r.moduleCacheCache == nil {
155 r.moduleCacheCache = &dirInfoCache{
156 dirs: map[string]*directoryPackageInfo{},
157 listeners: map[*int]cacheListener{},
160 if r.otherCache == nil {
161 r.otherCache = &dirInfoCache{
162 dirs: map[string]*directoryPackageInfo{},
163 listeners: map[*int]cacheListener{},
170 func (r *ModuleResolver) initAllMods() error {
171 stdout, err := r.env.invokeGo(context.TODO(), "list", "-m", "-e", "-json", "...")
175 for dec := json.NewDecoder(stdout); dec.More(); {
176 mod := &gocommand.ModuleJSON{}
177 if err := dec.Decode(mod); err != nil {
181 if r.env.Logf != nil {
182 r.env.Logf("module %v has not been downloaded and will be ignored", mod.Path)
184 // Can't do anything with a module that's not downloaded.
187 // golang/go#36193: the go command doesn't always clean paths.
188 mod.Dir = filepath.Clean(mod.Dir)
189 r.modsByModPath = append(r.modsByModPath, mod)
190 r.modsByDir = append(r.modsByDir, mod)
198 func (r *ModuleResolver) ClearForNewScan() {
200 r.scannedRoots = map[gopathwalk.Root]bool{}
201 r.otherCache = &dirInfoCache{
202 dirs: map[string]*directoryPackageInfo{},
203 listeners: map[*int]cacheListener{},
205 r.scanSema <- struct{}{}
208 func (r *ModuleResolver) ClearForNewMod() {
212 moduleCacheCache: r.moduleCacheCache,
213 otherCache: r.otherCache,
214 scanSema: r.scanSema,
217 r.scanSema <- struct{}{}
220 // findPackage returns the module and directory that contains the package at
221 // the given import path, or returns nil, "" if no module is in scope.
222 func (r *ModuleResolver) findPackage(importPath string) (*gocommand.ModuleJSON, string) {
223 // This can't find packages in the stdlib, but that's harmless for all
224 // the existing code paths.
225 for _, m := range r.modsByModPath {
226 if !strings.HasPrefix(importPath, m.Path) {
229 pathInModule := importPath[len(m.Path):]
230 pkgDir := filepath.Join(m.Dir, pathInModule)
231 if r.dirIsNestedModule(pkgDir, m) {
235 if info, ok := r.cacheLoad(pkgDir); ok {
236 if loaded, err := info.reachedStatus(nameLoaded); loaded {
238 continue // No package in this dir.
242 if scanned, err := info.reachedStatus(directoryScanned); scanned && err != nil {
243 continue // Dir is unreadable, etc.
245 // This is slightly wrong: a directory doesn't have to have an
246 // importable package to count as a package for package-to-module
247 // resolution. package main or _test files should count but
249 // TODO(heschi): fix this.
250 if _, err := r.cachePackageName(info); err == nil {
255 // Not cached. Read the filesystem.
256 pkgFiles, err := ioutil.ReadDir(pkgDir)
260 // A module only contains a package if it has buildable go
261 // files in that directory. If not, it could be provided by an
262 // outer module. See #29736.
263 for _, fi := range pkgFiles {
264 if ok, _ := r.env.matchFile(pkgDir, fi.Name()); ok {
272 func (r *ModuleResolver) cacheLoad(dir string) (directoryPackageInfo, bool) {
273 if info, ok := r.moduleCacheCache.Load(dir); ok {
276 return r.otherCache.Load(dir)
279 func (r *ModuleResolver) cacheStore(info directoryPackageInfo) {
280 if info.rootType == gopathwalk.RootModuleCache {
281 r.moduleCacheCache.Store(info.dir, info)
283 r.otherCache.Store(info.dir, info)
287 func (r *ModuleResolver) cacheKeys() []string {
288 return append(r.moduleCacheCache.Keys(), r.otherCache.Keys()...)
291 // cachePackageName caches the package name for a dir already in the cache.
292 func (r *ModuleResolver) cachePackageName(info directoryPackageInfo) (string, error) {
293 if info.rootType == gopathwalk.RootModuleCache {
294 return r.moduleCacheCache.CachePackageName(info)
296 return r.otherCache.CachePackageName(info)
299 func (r *ModuleResolver) cacheExports(ctx context.Context, env *ProcessEnv, info directoryPackageInfo) (string, []string, error) {
300 if info.rootType == gopathwalk.RootModuleCache {
301 return r.moduleCacheCache.CacheExports(ctx, env, info)
303 return r.otherCache.CacheExports(ctx, env, info)
306 // findModuleByDir returns the module that contains dir, or nil if no such
307 // module is in scope.
308 func (r *ModuleResolver) findModuleByDir(dir string) *gocommand.ModuleJSON {
309 // This is quite tricky and may not be correct. dir could be:
310 // - a package in the main module.
311 // - a replace target underneath the main module's directory.
312 // - a nested module in the above.
313 // - a replace target somewhere totally random.
314 // - a nested module in the above.
315 // - in the mod cache.
316 // - in /vendor/ in -mod=vendor mode.
317 // - nested module? Dunno.
318 // Rumor has it that replace targets cannot contain other replace targets.
319 for _, m := range r.modsByDir {
320 if !strings.HasPrefix(dir, m.Dir) {
324 if r.dirIsNestedModule(dir, m) {
333 // dirIsNestedModule reports if dir is contained in a nested module underneath
334 // mod, not actually in mod.
335 func (r *ModuleResolver) dirIsNestedModule(dir string, mod *gocommand.ModuleJSON) bool {
336 if !strings.HasPrefix(dir, mod.Dir) {
339 if r.dirInModuleCache(dir) {
340 // Nested modules in the module cache are pruned,
341 // so it cannot be a nested module.
344 if mod != nil && mod == r.dummyVendorMod {
345 // The /vendor pseudomodule is flattened and doesn't actually count.
348 modDir, _ := r.modInfo(dir)
352 return modDir != mod.Dir
355 func (r *ModuleResolver) modInfo(dir string) (modDir string, modName string) {
356 readModName := func(modFile string) string {
357 modBytes, err := ioutil.ReadFile(modFile)
361 return modulePath(modBytes)
364 if r.dirInModuleCache(dir) {
365 if matches := modCacheRegexp.FindStringSubmatch(dir); len(matches) == 3 {
366 index := strings.Index(dir, matches[1]+"@"+matches[2])
367 modDir := filepath.Join(dir[:index], matches[1]+"@"+matches[2])
368 return modDir, readModName(filepath.Join(modDir, "go.mod"))
372 if info, ok := r.cacheLoad(dir); ok {
373 return info.moduleDir, info.moduleName
375 f := filepath.Join(dir, "go.mod")
376 info, err := os.Stat(f)
377 if err == nil && !info.IsDir() {
378 return dir, readModName(f)
381 d := filepath.Dir(dir)
382 if len(d) >= len(dir) {
383 return "", "" // reached top of file system, no go.mod
389 func (r *ModuleResolver) dirInModuleCache(dir string) bool {
390 if r.moduleCacheDir == "" {
393 return strings.HasPrefix(dir, r.moduleCacheDir)
396 func (r *ModuleResolver) loadPackageNames(importPaths []string, srcDir string) (map[string]string, error) {
397 if err := r.init(); err != nil {
400 names := map[string]string{}
401 for _, path := range importPaths {
402 _, packageDir := r.findPackage(path)
403 if packageDir == "" {
406 name, err := packageDirToName(packageDir)
415 func (r *ModuleResolver) scan(ctx context.Context, callback *scanCallback) error {
416 if err := r.init(); err != nil {
420 processDir := func(info directoryPackageInfo) {
421 // Skip this directory if we were not able to get the package information successfully.
422 if scanned, err := info.reachedStatus(directoryScanned); !scanned || err != nil {
425 pkg, err := r.canonicalize(info)
430 if !callback.dirFound(pkg) {
433 pkg.packageName, err = r.cachePackageName(info)
438 if !callback.packageNameLoaded(pkg) {
441 _, exports, err := r.loadExports(ctx, pkg, false)
445 callback.exportsLoaded(pkg, exports)
448 // Start processing everything in the cache, and listen for the new stuff
449 // we discover in the walk below.
450 stop1 := r.moduleCacheCache.ScanAndListen(ctx, processDir)
452 stop2 := r.otherCache.ScanAndListen(ctx, processDir)
455 // We assume cached directories are fully cached, including all their
456 // children, and have not changed. We can skip them.
457 skip := func(root gopathwalk.Root, dir string) bool {
458 info, ok := r.cacheLoad(dir)
462 // This directory can be skipped as long as we have already scanned it.
463 // Packages with errors will continue to have errors, so there is no need
465 packageScanned, _ := info.reachedStatus(directoryScanned)
466 return packageScanned
469 // Add anything new to the cache, and process it if we're still listening.
470 add := func(root gopathwalk.Root, dir string) {
471 r.cacheStore(r.scanDirForPackage(root, dir))
474 // r.roots and the callback are not necessarily safe to use in the
475 // goroutine below. Process them eagerly.
476 roots := filterRoots(r.roots, callback.rootFound)
477 // We can't cancel walks, because we need them to finish to have a usable
478 // cache. Instead, run them in a separate goroutine and detach.
479 scanDone := make(chan struct{})
486 defer func() { r.scanSema <- struct{}{} }()
487 // We have the lock on r.scannedRoots, and no other scans can run.
488 for _, root := range roots {
489 if ctx.Err() != nil {
493 if r.scannedRoots[root] {
496 gopathwalk.WalkSkip([]gopathwalk.Root{root}, add, skip, gopathwalk.Options{Logf: r.env.Logf, ModulesEnabled: true})
497 r.scannedRoots[root] = true
508 func (r *ModuleResolver) scoreImportPath(ctx context.Context, path string) float64 {
509 if _, ok := stdlib[path]; ok {
512 mod, _ := r.findPackage(path)
513 return modRelevance(mod)
516 func modRelevance(mod *gocommand.ModuleJSON) float64 {
517 var relevance float64
519 case mod == nil: // out of scope
520 return MaxRelevance - 4
522 relevance = MaxRelevance - 3
524 relevance = MaxRelevance - 2
526 relevance = MaxRelevance - 1 // main module ties with stdlib
529 _, versionString, ok := module.SplitPathVersion(mod.Path)
531 index := strings.Index(versionString, "v")
535 if versionNumber, err := strconv.ParseFloat(versionString[index+1:], 64); err == nil {
536 relevance += versionNumber / 1000
543 // canonicalize gets the result of canonicalizing the packages using the results
544 // of initializing the resolver from 'go list -m'.
545 func (r *ModuleResolver) canonicalize(info directoryPackageInfo) (*pkg, error) {
546 // Packages in GOROOT are already canonical, regardless of the std/cmd modules.
547 if info.rootType == gopathwalk.RootGOROOT {
549 importPathShort: info.nonCanonicalImportPath,
551 packageName: path.Base(info.nonCanonicalImportPath),
552 relevance: MaxRelevance,
556 importPath := info.nonCanonicalImportPath
557 mod := r.findModuleByDir(info.dir)
558 // Check if the directory is underneath a module that's in scope.
560 // It is. If dir is the target of a replace directive,
561 // our guessed import path is wrong. Use the real one.
562 if mod.Dir == info.dir {
563 importPath = mod.Path
565 dirInMod := info.dir[len(mod.Dir)+len("/"):]
566 importPath = path.Join(mod.Path, filepath.ToSlash(dirInMod))
568 } else if !strings.HasPrefix(importPath, info.moduleName) {
569 // The module's name doesn't match the package's import path. It
570 // probably needs a replace directive we don't have.
571 return nil, fmt.Errorf("package in %q is not valid without a replace statement", info.dir)
575 importPathShort: importPath,
577 relevance: modRelevance(mod),
579 // We may have discovered a package that has a different version
580 // in scope already. Canonicalize to that one if possible.
581 if _, canonicalDir := r.findPackage(importPath); canonicalDir != "" {
582 res.dir = canonicalDir
587 func (r *ModuleResolver) loadExports(ctx context.Context, pkg *pkg, includeTest bool) (string, []string, error) {
588 if err := r.init(); err != nil {
591 if info, ok := r.cacheLoad(pkg.dir); ok && !includeTest {
592 return r.cacheExports(ctx, r.env, info)
594 return loadExportsFromFiles(ctx, r.env, pkg.dir, includeTest)
597 func (r *ModuleResolver) scanDirForPackage(root gopathwalk.Root, dir string) directoryPackageInfo {
599 if dir != root.Path {
600 subdir = dir[len(root.Path)+len("/"):]
602 importPath := filepath.ToSlash(subdir)
603 if strings.HasPrefix(importPath, "vendor/") {
604 // Only enter vendor directories if they're explicitly requested as a root.
605 return directoryPackageInfo{
606 status: directoryScanned,
607 err: fmt.Errorf("unwanted vendor directory"),
611 case gopathwalk.RootCurrentModule:
612 importPath = path.Join(r.main.Path, filepath.ToSlash(subdir))
613 case gopathwalk.RootModuleCache:
614 matches := modCacheRegexp.FindStringSubmatch(subdir)
615 if len(matches) == 0 {
616 return directoryPackageInfo{
617 status: directoryScanned,
618 err: fmt.Errorf("invalid module cache path: %v", subdir),
621 modPath, err := module.UnescapePath(filepath.ToSlash(matches[1]))
623 if r.env.Logf != nil {
624 r.env.Logf("decoding module cache path %q: %v", subdir, err)
626 return directoryPackageInfo{
627 status: directoryScanned,
628 err: fmt.Errorf("decoding module cache path %q: %v", subdir, err),
631 importPath = path.Join(modPath, filepath.ToSlash(matches[3]))
634 modDir, modName := r.modInfo(dir)
635 result := directoryPackageInfo{
636 status: directoryScanned,
639 nonCanonicalImportPath: importPath,
643 if root.Type == gopathwalk.RootGOROOT {
644 // stdlib packages are always in scope, despite the confusing go.mod
650 // modCacheRegexp splits a path in a module cache into module, module version, and package.
651 var modCacheRegexp = regexp.MustCompile(`(.*)@([^/\\]*)(.*)`)
654 slashSlash = []byte("//")
655 moduleStr = []byte("module")
658 // modulePath returns the module path from the gomod file text.
659 // If it cannot find a module path, it returns an empty string.
660 // It is tolerant of unrelated problems in the go.mod file.
662 // Copied from cmd/go/internal/modfile.
663 func modulePath(mod []byte) string {
667 if i := bytes.IndexByte(line, '\n'); i >= 0 {
668 line, mod = line[:i], line[i+1:]
670 if i := bytes.Index(line, slashSlash); i >= 0 {
673 line = bytes.TrimSpace(line)
674 if !bytes.HasPrefix(line, moduleStr) {
677 line = line[len(moduleStr):]
679 line = bytes.TrimSpace(line)
680 if len(line) == n || len(line) == 0 {
684 if line[0] == '"' || line[0] == '`' {
685 p, err := strconv.Unquote(string(line))
687 return "" // malformed quoted string or multiline module path
694 return "" // missing module path