1 // Copyright 2018 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // Package cache implements the caching layer for gopls.
24 "golang.org/x/mod/modfile"
25 "golang.org/x/mod/semver"
26 "golang.org/x/tools/internal/event"
27 "golang.org/x/tools/internal/gocommand"
28 "golang.org/x/tools/internal/imports"
29 "golang.org/x/tools/internal/lsp/source"
30 "golang.org/x/tools/internal/memoize"
31 "golang.org/x/tools/internal/span"
32 "golang.org/x/tools/internal/xcontext"
33 errors "golang.org/x/xerrors"
41 options *source.Options
43 // mu protects most mutable state of the view.
46 // baseCtx is the context handed to NewView. This is the parent of all
47 // background contexts created for this view.
48 baseCtx context.Context
50 // backgroundCtx is the current context used by background tasks initiated
52 backgroundCtx context.Context
54 // cancel is called when all action being performed by the current view
56 cancel context.CancelFunc
58 // name is the user visible name of this view.
61 // folder is the folder with which this view was constructed.
64 importsState *importsState
66 // keep track of files by uri and by basename, a single file may be mapped
67 // to multiple uris, and the same basename may map to multiple files
68 filesByURI map[span.URI]*fileBase
69 filesByBase map[string][]*fileBase
71 // initCancelFirstAttempt can be used to terminate the view's first
72 // attempt at initialization.
73 initCancelFirstAttempt context.CancelFunc
78 // initialWorkspaceLoad is closed when the first workspace initialization has
79 // completed. If we failed to load, we only retry if the go.mod file changes,
80 // to avoid too many go/packages calls.
81 initialWorkspaceLoad chan struct{}
83 // initializationSema is used limit concurrent initialization of snapshots in
84 // the view. We use a channel instead of a mutex to avoid blocking when a
85 // context is canceled.
86 initializationSema chan struct{}
88 // workspaceInformation tracks various details about this view's
89 // environment variables, go version, and use of modules.
92 // tempWorkspace is a temporary directory dedicated to holding the latest
93 // version of the workspace go.mod file. (TODO: also go.sum file)
94 tempWorkspace span.URI
97 type workspaceInformation struct {
98 // The Go version in use: X in Go 1.X.
101 // hasGopackagesDriver is true if the user has a value set for the
102 // GOPACKAGESDRIVER environment variable or a gopackagesdriver binary on
104 hasGopackagesDriver bool
106 // `go env` variables that need to be tracked by gopls.
109 // The value of GO111MODULE we want to run with.
112 // goEnv is the `go env` output collected when a view is created.
113 // It includes the values of the environment variables above.
114 goEnv map[string]string
116 // rootURI is the rootURI directory of this view. If we are in GOPATH mode, this
117 // is just the folder. If we are in module mode, this is the module rootURI.
121 type environmentVariables struct {
122 gocache, gopath, goprivate, gomodcache string
125 type workspaceMode int
128 moduleMode workspaceMode = 1 << iota
130 // tempModfile indicates whether or not the -modfile flag should be used.
133 // usesWorkspaceModule indicates support for the experimental workspace module
138 type builtinPackageHandle struct {
139 handle *memoize.Handle
142 type builtinPackageData struct {
143 parsed *source.BuiltinPackage
147 // fileBase holds the common functionality for all files.
148 // It is intended to be embedded in the file implementations
149 type fileBase struct {
156 func (f *fileBase) URI() span.URI {
160 func (f *fileBase) filename() string {
164 func (f *fileBase) addURI(uri span.URI) int {
165 f.uris = append(f.uris, uri)
169 func (v *View) ID() string { return v.id }
171 // tempModFile creates a temporary go.mod file based on the contents of the
172 // given go.mod file. It is the caller's responsibility to clean up the files
173 // when they are done using them.
174 func tempModFile(modFh source.FileHandle, gosum []byte) (tmpURI span.URI, cleanup func(), err error) {
175 filenameHash := hashContents([]byte(modFh.URI().Filename()))
176 tmpMod, err := ioutil.TempFile("", fmt.Sprintf("go.%s.*.mod", filenameHash))
182 tmpURI = span.URIFromPath(tmpMod.Name())
183 tmpSumName := sumFilename(tmpURI)
185 content, err := modFh.Read()
190 if _, err := tmpMod.Write(content); err != nil {
195 _ = os.Remove(tmpSumName)
196 _ = os.Remove(tmpURI.Filename())
199 // Be careful to clean up if we return an error from this function.
207 // Create an analogous go.sum, if one exists.
209 if err := ioutil.WriteFile(tmpSumName, gosum, 0655); err != nil {
210 return "", cleanup, err
214 return tmpURI, cleanup, nil
217 // Name returns the user visible name of this view.
218 func (v *View) Name() string {
222 // Folder returns the folder at the base of this view.
223 func (v *View) Folder() span.URI {
227 func (v *View) Options() *source.Options {
229 defer v.optionsMu.Unlock()
233 func minorOptionsChange(a, b *source.Options) bool {
234 // Check if any of the settings that modify our understanding of files have been changed
235 if !reflect.DeepEqual(a.Env, b.Env) {
238 aBuildFlags := make([]string, len(a.BuildFlags))
239 bBuildFlags := make([]string, len(b.BuildFlags))
240 copy(aBuildFlags, a.BuildFlags)
241 copy(bBuildFlags, b.BuildFlags)
242 sort.Strings(aBuildFlags)
243 sort.Strings(bBuildFlags)
244 // the rest of the options are benign
245 return reflect.DeepEqual(aBuildFlags, bBuildFlags)
248 func (v *View) SetOptions(ctx context.Context, options *source.Options) (source.View, error) {
249 // no need to rebuild the view if the options were not materially changed
251 if minorOptionsChange(v.options, options) {
257 newView, err := v.session.updateView(ctx, v, options)
261 func (v *View) Rebuild(ctx context.Context) (source.Snapshot, func(), error) {
262 newView, err := v.session.updateView(ctx, v, v.Options())
264 return nil, func() {}, err
266 snapshot, release := newView.Snapshot(ctx)
267 return snapshot, release, nil
270 func (s *snapshot) WriteEnv(ctx context.Context, w io.Writer) error {
271 s.view.optionsMu.Lock()
272 env := s.view.options.EnvSlice()
273 buildFlags := append([]string{}, s.view.options.BuildFlags...)
274 s.view.optionsMu.Unlock()
276 fullEnv := make(map[string]string)
277 for k, v := range s.view.goEnv {
280 for _, v := range env {
281 s := strings.SplitN(v, "=", 2)
285 if _, ok := fullEnv[s[0]]; ok {
289 goVersion, err := s.view.session.gocmdRunner.Run(ctx, gocommand.Invocation{
292 WorkingDir: s.view.rootURI.Filename(),
297 fmt.Fprintf(w, `go env for %v
300 (valid build configuration = %v)
303 s.view.folder.Filename(),
304 s.view.rootURI.Filename(),
305 strings.TrimRight(goVersion.String(), "\n"),
306 s.ValidBuildConfiguration(),
308 for k, v := range fullEnv {
309 fmt.Fprintf(w, "%s=%s\n", k, v)
314 func (s *snapshot) RunProcessEnvFunc(ctx context.Context, fn func(*imports.Options) error) error {
315 return s.view.importsState.runProcessEnvFunc(ctx, s, fn)
318 func (v *View) contains(uri span.URI) bool {
319 return strings.HasPrefix(string(uri), string(v.rootURI))
322 func (v *View) mapFile(uri span.URI, f *fileBase) {
323 v.filesByURI[uri] = f
324 if f.addURI(uri) == 1 {
325 basename := basename(f.filename())
326 v.filesByBase[basename] = append(v.filesByBase[basename], f)
330 func basename(filename string) string {
331 return strings.ToLower(filepath.Base(filename))
334 func (v *View) relevantChange(c source.FileModification) bool {
335 // If the file is known to the view, the change is relevant.
336 known := v.knownFile(c.URI)
338 // If the file is not known to the view, and the change is only on-disk,
339 // we should not invalidate the snapshot. This is necessary because Emacs
340 // sends didChangeWatchedFiles events for temp files.
341 if !known && c.OnDisk && (c.Action == source.Change || c.Action == source.Delete) {
344 return v.contains(c.URI) || known
347 func (v *View) knownFile(uri span.URI) bool {
351 f, err := v.findFile(uri)
352 return f != nil && err == nil
355 // getFile returns a file for the given URI. It will always succeed because it
356 // adds the file to the managed set if needed.
357 func (v *View) getFile(uri span.URI) (*fileBase, error) {
361 f, err := v.findFile(uri)
369 fname: uri.Filename(),
375 // findFile checks the cache for any file matching the given uri.
377 // An error is only returned for an irreparable failure, for example, if the
378 // filename in question does not exist.
379 func (v *View) findFile(uri span.URI) (*fileBase, error) {
380 if f := v.filesByURI[uri]; f != nil {
384 // no exact match stored, time to do some real work
385 // check for any files with the same basename
386 fname := uri.Filename()
387 basename := basename(fname)
388 if candidates := v.filesByBase[basename]; candidates != nil {
389 pathStat, err := os.Stat(fname)
390 if os.IsNotExist(err) {
394 return nil, nil // the file may exist, return without an error
396 for _, c := range candidates {
397 if cStat, err := os.Stat(c.filename()); err == nil {
398 if os.SameFile(pathStat, cStat) {
406 // no file with a matching name was found, it wasn't in our cache
410 func (v *View) Shutdown(ctx context.Context) {
411 v.session.removeView(ctx, v)
414 // TODO(rFindley): probably some of this should also be one in View.Shutdown
416 func (v *View) shutdown(ctx context.Context) {
417 // Cancel the initial workspace load if it is still running.
418 v.initCancelFirstAttempt()
427 go v.snapshot.generation.Destroy()
428 v.snapshotMu.Unlock()
429 if v.tempWorkspace != "" {
430 if err := os.RemoveAll(v.tempWorkspace.Filename()); err != nil {
431 event.Error(ctx, "removing temp workspace", err)
436 func (v *View) BackgroundContext() context.Context {
440 return v.backgroundCtx
443 func (s *snapshot) IgnoredFile(uri span.URI) bool {
444 filename := uri.Filename()
445 var prefixes []string
446 if len(s.workspace.activeModFiles()) == 0 {
447 for _, entry := range filepath.SplitList(s.view.gopath) {
448 prefixes = append(prefixes, filepath.Join(entry, "src"))
451 prefixes = append(prefixes, s.view.gomodcache)
452 for m := range s.workspace.activeModFiles() {
453 prefixes = append(prefixes, dirURI(m).Filename())
456 for _, prefix := range prefixes {
457 if strings.HasPrefix(filename, prefix) {
458 return checkIgnored(filename[len(prefix):])
464 // checkIgnored implements go list's exclusion rules. go help list:
465 // Directory and file names that begin with "." or "_" are ignored
466 // by the go tool, as are directories named "testdata".
467 func checkIgnored(suffix string) bool {
468 for _, component := range strings.Split(suffix, string(filepath.Separator)) {
469 if len(component) == 0 {
472 if component[0] == '.' || component[0] == '_' || component == "testdata" {
479 func (v *View) Snapshot(ctx context.Context) (source.Snapshot, func()) {
481 defer v.snapshotMu.Unlock()
482 return v.snapshot, v.snapshot.generation.Acquire(ctx)
485 func (s *snapshot) initialize(ctx context.Context, firstAttempt bool) {
489 case s.view.initializationSema <- struct{}{}:
493 <-s.view.initializationSema
496 if s.initializeOnce == nil {
499 s.initializeOnce.Do(func() {
501 s.initializeOnce = nil
503 close(s.view.initialWorkspaceLoad)
507 // If we have multiple modules, we need to load them by paths.
508 var scopes []interface{}
509 var modErrors *source.ErrorList
510 addError := func(uri span.URI, err error) {
511 if modErrors == nil {
512 modErrors = &source.ErrorList{}
514 *modErrors = append(*modErrors, &source.Error{
516 Category: "compiler",
517 Kind: source.ListError,
518 Message: err.Error(),
521 for modURI := range s.workspace.activeModFiles() {
522 fh, err := s.GetFile(ctx, modURI)
524 addError(modURI, err)
527 parsed, err := s.ParseMod(ctx, fh)
529 addError(modURI, err)
532 if parsed.File == nil || parsed.File.Module == nil {
533 addError(modURI, fmt.Errorf("no module path for %s", modURI))
536 path := parsed.File.Module.Mod.Path
537 scopes = append(scopes, moduleLoadScope(path))
539 if len(scopes) == 0 {
540 scopes = append(scopes, viewLoadScope("LOAD_VIEW"))
542 err := s.load(ctx, append(scopes, packagePath("builtin"))...)
543 if ctx.Err() != nil {
547 event.Error(ctx, "initial workspace load failed", err)
548 if modErrors != nil {
549 s.initializedErr = errors.Errorf("errors loading modules: %v: %w", err, modErrors)
551 s.initializedErr = err
557 // invalidateContent invalidates the content of a Go file,
558 // including any position and type information that depends on it.
559 // It returns true if we were already tracking the given file, false otherwise.
560 func (v *View) invalidateContent(ctx context.Context, changes map[span.URI]*fileChange, forceReloadMetadata bool) (source.Snapshot, func()) {
561 // Detach the context so that content invalidation cannot be canceled.
562 ctx = xcontext.Detach(ctx)
564 // Cancel all still-running previous requests, since they would be
565 // operating on stale data.
568 // Do not clone a snapshot until its view has finished initializing.
569 v.snapshot.AwaitInitialized(ctx)
571 // This should be the only time we hold the view's snapshot lock for any period of time.
573 defer v.snapshotMu.Unlock()
575 oldSnapshot := v.snapshot
577 var workspaceChanged bool
578 v.snapshot, workspaceChanged = oldSnapshot.clone(ctx, changes, forceReloadMetadata)
579 if workspaceChanged && v.tempWorkspace != "" {
582 wsdir, err := snap.getWorkspaceDir(ctx)
584 event.Error(ctx, "getting workspace dir", err)
586 if err := copyWorkspace(v.tempWorkspace, wsdir); err != nil {
587 event.Error(ctx, "copying workspace dir", err)
591 go oldSnapshot.generation.Destroy()
593 return v.snapshot, v.snapshot.generation.Acquire(ctx)
596 func copyWorkspace(dst span.URI, src span.URI) error {
597 srcMod := filepath.Join(src.Filename(), "go.mod")
598 srcf, err := os.Open(srcMod)
600 return errors.Errorf("opening snapshot mod file: %w", err)
603 dstMod := filepath.Join(dst.Filename(), "go.mod")
604 dstf, err := os.Create(dstMod)
606 return errors.Errorf("truncating view mod file: %w", err)
609 if _, err := io.Copy(dstf, srcf); err != nil {
610 return errors.Errorf("copying modfiles: %w", err)
615 func (v *View) cancelBackground() {
619 // this can happen during shutdown
623 v.backgroundCtx, v.cancel = context.WithCancel(v.baseCtx)
626 func (s *Session) getWorkspaceInformation(ctx context.Context, folder span.URI, options *source.Options) (*workspaceInformation, error) {
627 if err := checkPathCase(folder.Filename()); err != nil {
628 return nil, errors.Errorf("invalid workspace configuration: %w", err)
631 inv := gocommand.Invocation{
632 WorkingDir: folder.Filename(),
633 Env: options.EnvSlice(),
635 goversion, err := gocommand.GoVersion(ctx, inv, s.gocmdRunner)
640 go111module := os.Getenv("GO111MODULE")
641 if v, ok := options.Env["GO111MODULE"]; ok {
644 // If using 1.16, change the default back to auto. The primary effect of
645 // GO111MODULE=on is to break GOPATH, which we aren't too interested in.
646 if goversion >= 16 && go111module == "" {
650 // Make sure to get the `go env` before continuing with initialization.
651 envVars, env, err := s.getGoEnv(ctx, folder.Filename(), append(options.EnvSlice(), "GO111MODULE="+go111module))
655 // The value of GOPACKAGESDRIVER is not returned through the go command.
656 gopackagesdriver := os.Getenv("GOPACKAGESDRIVER")
657 for _, s := range env {
658 split := strings.SplitN(s, "=", 2)
659 if split[0] == "GOPACKAGESDRIVER" {
660 gopackagesdriver = split[1]
663 // A user may also have a gopackagesdriver binary on their machine, which
664 // works the same way as setting GOPACKAGESDRIVER.
665 tool, _ := exec.LookPath("gopackagesdriver")
666 hasGopackagesDriver := gopackagesdriver != "off" && (gopackagesdriver != "" || tool != "")
669 if options.ExpandWorkspaceToModule {
670 wsRoot, err := findWorkspaceRoot(ctx, root, s)
678 return &workspaceInformation{
679 hasGopackagesDriver: hasGopackagesDriver,
680 go111module: go111module,
681 goversion: goversion,
683 environmentVariables: envVars,
688 func findWorkspaceRoot(ctx context.Context, folder span.URI, fs source.FileSource) (span.URI, error) {
689 for _, basename := range []string{"gopls.mod", "go.mod"} {
690 dir, err := findRootPattern(ctx, folder, basename, fs)
692 return "", errors.Errorf("finding %s: %w", basename, err)
701 func findRootPattern(ctx context.Context, folder span.URI, basename string, fs source.FileSource) (span.URI, error) {
702 dir := folder.Filename()
704 target := filepath.Join(dir, basename)
705 exists, err := fileExists(ctx, span.URIFromPath(target), fs)
710 return span.URIFromPath(dir), nil
712 next, _ := filepath.Split(dir)
721 // OS-specific path case check, for case-insensitive filesystems.
722 var checkPathCase = defaultCheckPathCase
724 func defaultCheckPathCase(path string) error {
728 func validBuildConfiguration(folder span.URI, ws *workspaceInformation, modFiles map[span.URI]struct{}) bool {
729 // Since we only really understand the `go` command, if the user has a
730 // different GOPACKAGESDRIVER, assume that their configuration is valid.
731 if ws.hasGopackagesDriver {
734 // Check if the user is working within a module or if we have found
735 // multiple modules in the workspace.
736 if len(modFiles) > 0 {
739 // The user may have a multiple directories in their GOPATH.
740 // Check if the workspace is within any of them.
741 for _, gp := range filepath.SplitList(ws.gopath) {
742 if source.InDir(filepath.Join(gp, "src"), folder.Filename()) {
749 // getGoEnv gets the view's various GO* values.
750 func (s *Session) getGoEnv(ctx context.Context, folder string, configEnv []string) (environmentVariables, map[string]string, error) {
751 envVars := environmentVariables{}
752 vars := map[string]*string{
753 "GOCACHE": &envVars.gocache,
754 "GOPATH": &envVars.gopath,
755 "GOPRIVATE": &envVars.goprivate,
756 "GOMODCACHE": &envVars.gomodcache,
758 // We can save ~200 ms by requesting only the variables we care about.
759 args := append([]string{"-json"}, imports.RequiredGoEnvVars...)
760 for k := range vars {
761 args = append(args, k)
764 inv := gocommand.Invocation{
770 // Don't go through runGoCommand, as we don't need a temporary -modfile to
772 stdout, err := s.gocmdRunner.Run(ctx, inv)
774 return environmentVariables{}, nil, err
776 env := make(map[string]string)
777 if err := json.Unmarshal(stdout.Bytes(), &env); err != nil {
778 return environmentVariables{}, nil, err
781 for key, ptr := range vars {
785 // Old versions of Go don't have GOMODCACHE, so emulate it.
786 if envVars.gomodcache == "" && envVars.gopath != "" {
787 envVars.gomodcache = filepath.Join(filepath.SplitList(envVars.gopath)[0], "pkg/mod")
789 return envVars, env, err
792 func (v *View) IsGoPrivatePath(target string) bool {
793 return globsMatchPath(v.goprivate, target)
797 // https://cs.opensource.google/go/go/+/master:src/cmd/go/internal/str/path.go;l=58;drc=2910c5b4a01a573ebc97744890a07c1a3122c67a
798 func globsMatchPath(globs, target string) bool {
800 // Extract next non-empty glob in comma-separated list.
802 if i := strings.Index(globs, ","); i >= 0 {
803 glob, globs = globs[:i], globs[i+1:]
805 glob, globs = globs, ""
811 // A glob with N+1 path elements (N slashes) needs to be matched
812 // against the first N+1 path elements of target,
813 // which end just before the N+1'th slash.
814 n := strings.Count(glob, "/")
816 // Walk target, counting slashes, truncating at the N+1'th slash.
817 for i := 0; i < len(target); i++ {
818 if target[i] == '/' {
827 // Not enough prefix elements.
830 matched, _ := path.Match(glob, prefix)
838 var modFlagRegexp = regexp.MustCompile(`-mod[ =](\w+)`)
840 // TODO(rstambler): Consolidate modURI and modContent back into a FileHandle
841 // after we have a version of the workspace go.mod file on disk. Getting a
842 // FileHandle from the cache for temporary files is problematic, since we
844 func (s *snapshot) needsModEqualsMod(ctx context.Context, modURI span.URI, modContent []byte) (bool, error) {
845 if s.view.goversion < 16 || s.workspaceMode()&moduleMode == 0 {
849 matches := modFlagRegexp.FindStringSubmatch(s.view.goEnv["GOFLAGS"])
851 if len(matches) != 0 {
855 // Don't override an explicit '-mod=vendor' argument.
856 // We do want to override '-mod=readonly': it would break various module code lenses,
857 // and on 1.16 we know -modfile is available, so we won't mess with go.mod anyway.
858 return modFlag == "vendor", nil
861 modFile, err := modfile.Parse(modURI.Filename(), modContent, nil)
865 if fi, err := os.Stat(filepath.Join(s.view.rootURI.Filename(), "vendor")); err != nil || !fi.IsDir() {
868 vendorEnabled := modFile.Go != nil && modFile.Go.Version != "" && semver.Compare("v"+modFile.Go.Version, "v1.14") >= 0
869 return !vendorEnabled, nil