1 // Copyright 2018 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
7 // See doc.go for package documentation and implementation notes.
26 "golang.org/x/tools/go/gcexportdata"
27 "golang.org/x/tools/internal/gocommand"
28 "golang.org/x/tools/internal/packagesinternal"
29 "golang.org/x/tools/internal/typesinternal"
32 // A LoadMode controls the amount of detail to return when loading.
33 // The bits below can be combined to specify which fields should be
34 // filled in the result packages.
35 // The zero value is a special case, equivalent to combining
36 // the NeedName, NeedFiles, and NeedCompiledGoFiles bits.
37 // ID and Errors (if present) will always be filled.
38 // Load may return more information than requested.
41 // TODO(matloob): When a V2 of go/packages is released, rename NeedExportsFile to
42 // NeedExportFile to make it consistent with the Package field it's adding.
45 // NeedName adds Name and PkgPath.
46 NeedName LoadMode = 1 << iota
48 // NeedFiles adds GoFiles and OtherFiles.
51 // NeedCompiledGoFiles adds CompiledGoFiles.
54 // NeedImports adds Imports. If NeedDeps is not set, the Imports field will contain
55 // "placeholder" Packages with only the ID set.
58 // NeedDeps adds the fields requested by the LoadMode in the packages in Imports.
61 // NeedExportsFile adds ExportFile.
64 // NeedTypes adds Types, Fset, and IllTyped.
67 // NeedSyntax adds Syntax.
70 // NeedTypesInfo adds TypesInfo.
73 // NeedTypesSizes adds TypesSizes.
76 // typecheckCgo enables full support for type checking cgo. Requires Go 1.15+.
77 // Modifies CompiledGoFiles and Types, and has no effect on its own.
80 // NeedModule adds Module.
85 // Deprecated: LoadFiles exists for historical compatibility
86 // and should not be used. Please directly specify the needed fields using the Need values.
87 LoadFiles = NeedName | NeedFiles | NeedCompiledGoFiles
89 // Deprecated: LoadImports exists for historical compatibility
90 // and should not be used. Please directly specify the needed fields using the Need values.
91 LoadImports = LoadFiles | NeedImports
93 // Deprecated: LoadTypes exists for historical compatibility
94 // and should not be used. Please directly specify the needed fields using the Need values.
95 LoadTypes = LoadImports | NeedTypes | NeedTypesSizes
97 // Deprecated: LoadSyntax exists for historical compatibility
98 // and should not be used. Please directly specify the needed fields using the Need values.
99 LoadSyntax = LoadTypes | NeedSyntax | NeedTypesInfo
101 // Deprecated: LoadAllSyntax exists for historical compatibility
102 // and should not be used. Please directly specify the needed fields using the Need values.
103 LoadAllSyntax = LoadSyntax | NeedDeps
106 // A Config specifies details about how packages should be loaded.
107 // The zero value is a valid configuration.
108 // Calls to Load do not modify this struct.
110 // Mode controls the level of information returned for each package.
113 // Context specifies the context for the load operation.
114 // If the context is cancelled, the loader may stop early
115 // and return an ErrCancelled error.
116 // If Context is nil, the load cannot be cancelled.
117 Context context.Context
119 // Logf is the logger for the config.
120 // If the user provides a logger, debug logging is enabled.
121 // If the GOPACKAGESDEBUG environment variable is set to true,
122 // but the logger is nil, default to log.Printf.
123 Logf func(format string, args ...interface{})
125 // Dir is the directory in which to run the build system's query tool
126 // that provides information about the packages.
127 // If Dir is empty, the tool is run in the current directory.
130 // Env is the environment to use when invoking the build system's query tool.
131 // If Env is nil, the current environment is used.
132 // As in os/exec's Cmd, only the last value in the slice for
133 // each environment key is used. To specify the setting of only
134 // a few variables, append to the current environment, as in:
136 // opt.Env = append(os.Environ(), "GOOS=plan9", "GOARCH=386")
140 // gocmdRunner guards go command calls from concurrency errors.
141 gocmdRunner *gocommand.Runner
143 // BuildFlags is a list of command-line flags to be passed through to
144 // the build system's query tool.
147 // modFile will be used for -modfile in go command invocations.
150 // modFlag will be used for -modfile in go command invocations.
153 // Fset provides source position information for syntax trees and types.
154 // If Fset is nil, Load will use a new fileset, but preserve Fset's value.
157 // ParseFile is called to read and parse each file
158 // when preparing a package's type-checked syntax tree.
159 // It must be safe to call ParseFile simultaneously from multiple goroutines.
160 // If ParseFile is nil, the loader will uses parser.ParseFile.
162 // ParseFile should parse the source from src and use filename only for
163 // recording position information.
165 // An application may supply a custom implementation of ParseFile
166 // to change the effective file contents or the behavior of the parser,
167 // or to modify the syntax tree. For example, selectively eliminating
168 // unwanted function bodies can significantly accelerate type checking.
169 ParseFile func(fset *token.FileSet, filename string, src []byte) (*ast.File, error)
171 // If Tests is set, the loader includes not just the packages
172 // matching a particular pattern but also any related test packages,
173 // including test-only variants of the package and the test executable.
175 // For example, when using the go command, loading "fmt" with Tests=true
176 // returns four packages, with IDs "fmt" (the standard package),
177 // "fmt [fmt.test]" (the package as compiled for the test),
178 // "fmt_test" (the test functions from source files in package fmt_test),
179 // and "fmt.test" (the test binary).
181 // In build systems with explicit names for tests,
182 // setting Tests may have no effect.
185 // Overlay provides a mapping of absolute file paths to file contents.
186 // If the file with the given path already exists, the parser will use the
187 // alternative file contents provided by the map.
189 // Overlays provide incomplete support for when a given file doesn't
190 // already exist on disk. See the package doc above for more details.
191 Overlay map[string][]byte
194 // driver is the type for functions that query the build system for the
195 // packages named by the patterns.
196 type driver func(cfg *Config, patterns ...string) (*driverResponse, error)
198 // driverResponse contains the results for a driver query.
199 type driverResponse struct {
200 // NotHandled is returned if the request can't be handled by the current
201 // driver. If an external driver returns a response with NotHandled, the
202 // rest of the driverResponse is ignored, and go/packages will fallback
203 // to the next driver. If go/packages is extended in the future to support
204 // lists of multiple drivers, go/packages will fall back to the next driver.
207 // Sizes, if not nil, is the types.Sizes to use when type checking.
208 Sizes *types.StdSizes
210 // Roots is the set of package IDs that make up the root packages.
211 // We have to encode this separately because when we encode a single package
212 // we cannot know if it is one of the roots as that requires knowledge of the
213 // graph it is part of.
214 Roots []string `json:",omitempty"`
216 // Packages is the full set of packages in the graph.
217 // The packages are not connected into a graph.
218 // The Imports if populated will be stubs that only have their ID set.
219 // Imports will be connected and then type and syntax information added in a
220 // later pass (see refine).
224 // Load loads and returns the Go packages named by the given patterns.
226 // Config specifies loading options;
227 // nil behaves the same as an empty Config.
229 // Load returns an error if any of the patterns was invalid
230 // as defined by the underlying build system.
231 // It may return an empty list of packages without an error,
232 // for instance for an empty expansion of a valid wildcard.
233 // Errors associated with a particular package are recorded in the
234 // corresponding Package's Errors list, and do not cause Load to
235 // return an error. Clients may need to handle such errors before
236 // proceeding with further analysis. The PrintErrors function is
237 // provided for convenient display of all errors.
238 func Load(cfg *Config, patterns ...string) ([]*Package, error) {
240 response, err := defaultDriver(&l.Config, patterns...)
244 l.sizes = response.Sizes
245 return l.refine(response.Roots, response.Packages...)
248 // defaultDriver is a driver that implements go/packages' fallback behavior.
249 // It will try to request to an external driver, if one exists. If there's
250 // no external driver, or the driver returns a response with NotHandled set,
251 // defaultDriver will fall back to the go list driver.
252 func defaultDriver(cfg *Config, patterns ...string) (*driverResponse, error) {
253 driver := findExternalDriver(cfg)
255 driver = goListDriver
257 response, err := driver(cfg, patterns...)
260 } else if response.NotHandled {
261 return goListDriver(cfg, patterns...)
266 // A Package describes a loaded Go package.
267 type Package struct {
268 // ID is a unique identifier for a package,
269 // in a syntax provided by the underlying build system.
271 // Because the syntax varies based on the build system,
272 // clients should treat IDs as opaque and not attempt to
276 // Name is the package name as it appears in the package source code.
279 // PkgPath is the package path as used by the go/types package.
282 // Errors contains any errors encountered querying the metadata
283 // of the package, or while parsing or type-checking its files.
286 // GoFiles lists the absolute file paths of the package's Go source files.
289 // CompiledGoFiles lists the absolute file paths of the package's source
290 // files that are suitable for type checking.
291 // This may differ from GoFiles if files are processed before compilation.
292 CompiledGoFiles []string
294 // OtherFiles lists the absolute file paths of the package's non-Go source files,
295 // including assembly, C, C++, Fortran, Objective-C, SWIG, and so on.
298 // IgnoredFiles lists source files that are not part of the package
299 // using the current build configuration but that might be part of
300 // the package using other build configurations.
301 IgnoredFiles []string
303 // ExportFile is the absolute path to a file containing type
304 // information for the package as provided by the build system.
307 // Imports maps import paths appearing in the package's Go source files
308 // to corresponding loaded Packages.
309 Imports map[string]*Package
311 // Types provides type information for the package.
312 // The NeedTypes LoadMode bit sets this field for packages matching the
313 // patterns; type information for dependencies may be missing or incomplete,
314 // unless NeedDeps and NeedImports are also set.
317 // Fset provides position information for Types, TypesInfo, and Syntax.
318 // It is set only when Types is set.
321 // IllTyped indicates whether the package or any dependency contains errors.
322 // It is set only when Types is set.
325 // Syntax is the package's syntax trees, for the files listed in CompiledGoFiles.
327 // The NeedSyntax LoadMode bit populates this field for packages matching the patterns.
328 // If NeedDeps and NeedImports are also set, this field will also be populated
332 // TypesInfo provides type information about the package's syntax trees.
333 // It is set only when Syntax is set.
334 TypesInfo *types.Info
336 // TypesSizes provides the effective size function for types in TypesInfo.
337 TypesSizes types.Sizes
339 // forTest is the package under test, if any.
342 // depsErrors is the DepsErrors field from the go list response, if any.
343 depsErrors []*packagesinternal.PackageError
345 // module is the module information for the package if it exists.
349 // Module provides module information for a package.
351 Path string // module path
352 Version string // module version
353 Replace *Module // replaced by this module
354 Time *time.Time // time version was created
355 Main bool // is this the main module?
356 Indirect bool // is this module only an indirect dependency of main module?
357 Dir string // directory holding files for this module, if any
358 GoMod string // path to go.mod file used when loading this module, if any
359 GoVersion string // go version used in module
360 Error *ModuleError // error loading module
363 // ModuleError holds errors loading a module.
364 type ModuleError struct {
365 Err string // the error itself
369 packagesinternal.GetForTest = func(p interface{}) string {
370 return p.(*Package).forTest
372 packagesinternal.GetDepsErrors = func(p interface{}) []*packagesinternal.PackageError {
373 return p.(*Package).depsErrors
375 packagesinternal.GetGoCmdRunner = func(config interface{}) *gocommand.Runner {
376 return config.(*Config).gocmdRunner
378 packagesinternal.SetGoCmdRunner = func(config interface{}, runner *gocommand.Runner) {
379 config.(*Config).gocmdRunner = runner
381 packagesinternal.SetModFile = func(config interface{}, value string) {
382 config.(*Config).modFile = value
384 packagesinternal.SetModFlag = func(config interface{}, value string) {
385 config.(*Config).modFlag = value
387 packagesinternal.TypecheckCgo = int(typecheckCgo)
390 // An Error describes a problem with a package's metadata, syntax, or types.
392 Pos string // "file:line:col" or "file:line" or "" or "-"
397 // ErrorKind describes the source of the error, allowing the user to
398 // differentiate between errors generated by the driver, the parser, or the
403 UnknownError ErrorKind = iota
409 func (err Error) Error() string {
412 pos = "-" // like token.Position{}.String()
414 return pos + ": " + err.Msg
417 // flatPackage is the JSON form of Package
418 // It drops all the type and syntax fields, and transforms the Imports
420 // TODO(adonovan): identify this struct with Package, effectively
421 // publishing the JSON protocol.
422 type flatPackage struct {
424 Name string `json:",omitempty"`
425 PkgPath string `json:",omitempty"`
426 Errors []Error `json:",omitempty"`
427 GoFiles []string `json:",omitempty"`
428 CompiledGoFiles []string `json:",omitempty"`
429 OtherFiles []string `json:",omitempty"`
430 IgnoredFiles []string `json:",omitempty"`
431 ExportFile string `json:",omitempty"`
432 Imports map[string]string `json:",omitempty"`
435 // MarshalJSON returns the Package in its JSON form.
436 // For the most part, the structure fields are written out unmodified, and
437 // the type and syntax fields are skipped.
438 // The imports are written out as just a map of path to package id.
439 // The errors are written using a custom type that tries to preserve the
440 // structure of error types we know about.
442 // This method exists to enable support for additional build systems. It is
443 // not intended for use by clients of the API and we may change the format.
444 func (p *Package) MarshalJSON() ([]byte, error) {
445 flat := &flatPackage{
451 CompiledGoFiles: p.CompiledGoFiles,
452 OtherFiles: p.OtherFiles,
453 IgnoredFiles: p.IgnoredFiles,
454 ExportFile: p.ExportFile,
456 if len(p.Imports) > 0 {
457 flat.Imports = make(map[string]string, len(p.Imports))
458 for path, ipkg := range p.Imports {
459 flat.Imports[path] = ipkg.ID
462 return json.Marshal(flat)
465 // UnmarshalJSON reads in a Package from its JSON format.
466 // See MarshalJSON for details about the format accepted.
467 func (p *Package) UnmarshalJSON(b []byte) error {
468 flat := &flatPackage{}
469 if err := json.Unmarshal(b, &flat); err != nil {
475 PkgPath: flat.PkgPath,
477 GoFiles: flat.GoFiles,
478 CompiledGoFiles: flat.CompiledGoFiles,
479 OtherFiles: flat.OtherFiles,
480 ExportFile: flat.ExportFile,
482 if len(flat.Imports) > 0 {
483 p.Imports = make(map[string]*Package, len(flat.Imports))
484 for path, id := range flat.Imports {
485 p.Imports[path] = &Package{ID: id}
491 func (p *Package) String() string { return p.ID }
493 // loaderPackage augments Package with state used during the loading phase
494 type loaderPackage struct {
496 importErrors map[string]error // maps each bad import to its error
498 color uint8 // for cycle detection
499 needsrc bool // load from source (Mode >= LoadTypes)
500 needtypes bool // type information is either requested or depended on
501 initial bool // package was matched by a pattern
504 // loader holds the working state of a single call to load.
506 pkgs map[string]*loaderPackage
509 parseCache map[string]*parseValue
510 parseCacheMu sync.Mutex
511 exportMu sync.Mutex // enforces mutual exclusion of exportdata operations
513 // Config.Mode contains the implied mode (see impliedLoadMode).
514 // Implied mode contains all the fields we need the data for.
515 // In requestedMode there are the actually requested fields.
516 // We'll zero them out before returning packages to the user.
517 // This makes it easier for us to get the conditions where
518 // we need certain modes right.
519 requestedMode LoadMode
522 type parseValue struct {
528 func newLoader(cfg *Config) *loader {
530 parseCache: map[string]*parseValue{},
534 // If the user has provided a logger, use it.
535 ld.Config.Logf = cfg.Logf
537 if ld.Config.Logf == nil {
538 // If the GOPACKAGESDEBUG environment variable is set to true,
539 // but the user has not provided a logger, default to log.Printf.
541 ld.Config.Logf = log.Printf
543 ld.Config.Logf = func(format string, args ...interface{}) {}
546 if ld.Config.Mode == 0 {
547 ld.Config.Mode = NeedName | NeedFiles | NeedCompiledGoFiles // Preserve zero behavior of Mode for backwards compatibility.
549 if ld.Config.Env == nil {
550 ld.Config.Env = os.Environ()
552 if ld.Config.gocmdRunner == nil {
553 ld.Config.gocmdRunner = &gocommand.Runner{}
555 if ld.Context == nil {
556 ld.Context = context.Background()
559 if dir, err := os.Getwd(); err == nil {
564 // Save the actually requested fields. We'll zero them out before returning packages to the user.
565 ld.requestedMode = ld.Mode
566 ld.Mode = impliedLoadMode(ld.Mode)
568 if ld.Mode&NeedTypes != 0 || ld.Mode&NeedSyntax != 0 {
570 ld.Fset = token.NewFileSet()
573 // ParseFile is required even in LoadTypes mode
574 // because we load source if export data is missing.
575 if ld.ParseFile == nil {
576 ld.ParseFile = func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) {
577 const mode = parser.AllErrors | parser.ParseComments
578 return parser.ParseFile(fset, filename, src, mode)
586 // refine connects the supplied packages into a graph and then adds type and
587 // and syntax information as requested by the LoadMode.
588 func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
589 rootMap := make(map[string]int, len(roots))
590 for i, root := range roots {
593 ld.pkgs = make(map[string]*loaderPackage)
594 // first pass, fixup and build the map and roots
595 var initial = make([]*loaderPackage, len(roots))
596 for _, pkg := range list {
598 if i, found := rootMap[pkg.ID]; found {
602 // Overlays can invalidate export data.
603 // TODO(matloob): make this check fine-grained based on dependencies on overlaid files
604 exportDataInvalid := len(ld.Overlay) > 0 || pkg.ExportFile == "" && pkg.PkgPath != "unsafe"
605 // This package needs type information if the caller requested types and the package is
606 // either a root, or it's a non-root and the user requested dependencies ...
607 needtypes := (ld.Mode&NeedTypes|NeedTypesInfo != 0 && (rootIndex >= 0 || ld.Mode&NeedDeps != 0))
608 // This package needs source if the call requested source (or types info, which implies source)
609 // and the package is either a root, or itas a non- root and the user requested dependencies...
610 needsrc := ((ld.Mode&(NeedSyntax|NeedTypesInfo) != 0 && (rootIndex >= 0 || ld.Mode&NeedDeps != 0)) ||
611 // ... or if we need types and the exportData is invalid. We fall back to (incompletely)
612 // typechecking packages from source if they fail to compile.
613 (ld.Mode&NeedTypes|NeedTypesInfo != 0 && exportDataInvalid)) && pkg.PkgPath != "unsafe"
614 lpkg := &loaderPackage{
616 needtypes: needtypes,
619 ld.pkgs[lpkg.ID] = lpkg
621 initial[rootIndex] = lpkg
625 for i, root := range roots {
626 if initial[i] == nil {
627 return nil, fmt.Errorf("root package %v is missing", root)
631 // Materialize the import graph.
635 grey = 1 // in progress
636 black = 2 // complete
639 // visit traverses the import graph, depth-first,
640 // and materializes the graph as Packages.Imports.
642 // Valid imports are saved in the Packages.Import map.
643 // Invalid imports (cycles and missing nodes) are saved in the importErrors map.
644 // Thus, even in the presence of both kinds of errors, the Import graph remains a DAG.
646 // visit returns whether the package needs src or has a transitive
647 // dependency on a package that does. These are the only packages
648 // for which we load source code.
649 var stack []*loaderPackage
650 var visit func(lpkg *loaderPackage) bool
651 var srcPkgs []*loaderPackage
652 visit = func(lpkg *loaderPackage) bool {
657 panic("internal error: grey node")
660 stack = append(stack, lpkg) // push
661 stubs := lpkg.Imports // the structure form has only stubs with the ID in the Imports
662 // If NeedImports isn't set, the imports fields will all be zeroed out.
663 if ld.Mode&NeedImports != 0 {
664 lpkg.Imports = make(map[string]*Package, len(stubs))
665 for importPath, ipkg := range stubs {
667 imp := ld.pkgs[ipkg.ID]
669 // (includes package "C" when DisableCgo)
670 importErr = fmt.Errorf("missing package: %q", ipkg.ID)
671 } else if imp.color == grey {
672 importErr = fmt.Errorf("import cycle: %s", stack)
674 if importErr != nil {
675 if lpkg.importErrors == nil {
676 lpkg.importErrors = make(map[string]error)
678 lpkg.importErrors[importPath] = importErr
685 lpkg.Imports[importPath] = imp.Package
689 srcPkgs = append(srcPkgs, lpkg)
691 if ld.Mode&NeedTypesSizes != 0 {
692 lpkg.TypesSizes = ld.sizes
694 stack = stack[:len(stack)-1] // pop
700 if ld.Mode&NeedImports == 0 {
701 // We do this to drop the stub import packages that we are not even going to try to resolve.
702 for _, lpkg := range initial {
706 // For each initial package, create its import DAG.
707 for _, lpkg := range initial {
711 if ld.Mode&NeedImports != 0 && ld.Mode&NeedTypes != 0 {
712 for _, lpkg := range srcPkgs {
713 // Complete type information is required for the
714 // immediate dependencies of each source package.
715 for _, ipkg := range lpkg.Imports {
716 imp := ld.pkgs[ipkg.ID]
721 // Load type data and syntax if needed, starting at
722 // the initial packages (roots of the import DAG).
723 if ld.Mode&NeedTypes != 0 || ld.Mode&NeedSyntax != 0 {
724 var wg sync.WaitGroup
725 for _, lpkg := range initial {
727 go func(lpkg *loaderPackage) {
728 ld.loadRecursive(lpkg)
735 result := make([]*Package, len(initial))
736 for i, lpkg := range initial {
737 result[i] = lpkg.Package
739 for i := range ld.pkgs {
740 // Clear all unrequested fields,
741 // to catch programs that use more than they request.
742 if ld.requestedMode&NeedName == 0 {
744 ld.pkgs[i].PkgPath = ""
746 if ld.requestedMode&NeedFiles == 0 {
747 ld.pkgs[i].GoFiles = nil
748 ld.pkgs[i].OtherFiles = nil
749 ld.pkgs[i].IgnoredFiles = nil
751 if ld.requestedMode&NeedCompiledGoFiles == 0 {
752 ld.pkgs[i].CompiledGoFiles = nil
754 if ld.requestedMode&NeedImports == 0 {
755 ld.pkgs[i].Imports = nil
757 if ld.requestedMode&NeedExportsFile == 0 {
758 ld.pkgs[i].ExportFile = ""
760 if ld.requestedMode&NeedTypes == 0 {
761 ld.pkgs[i].Types = nil
762 ld.pkgs[i].Fset = nil
763 ld.pkgs[i].IllTyped = false
765 if ld.requestedMode&NeedSyntax == 0 {
766 ld.pkgs[i].Syntax = nil
768 if ld.requestedMode&NeedTypesInfo == 0 {
769 ld.pkgs[i].TypesInfo = nil
771 if ld.requestedMode&NeedTypesSizes == 0 {
772 ld.pkgs[i].TypesSizes = nil
774 if ld.requestedMode&NeedModule == 0 {
775 ld.pkgs[i].Module = nil
782 // loadRecursive loads the specified package and its dependencies,
783 // recursively, in parallel, in topological order.
784 // It is atomic and idempotent.
785 // Precondition: ld.Mode&NeedTypes.
786 func (ld *loader) loadRecursive(lpkg *loaderPackage) {
787 lpkg.loadOnce.Do(func() {
788 // Load the direct dependencies, in parallel.
789 var wg sync.WaitGroup
790 for _, ipkg := range lpkg.Imports {
791 imp := ld.pkgs[ipkg.ID]
793 go func(imp *loaderPackage) {
794 ld.loadRecursive(imp)
803 // loadPackage loads the specified package.
804 // It must be called only once per Package,
805 // after immediate dependencies are loaded.
806 // Precondition: ld.Mode & NeedTypes.
807 func (ld *loader) loadPackage(lpkg *loaderPackage) {
808 if lpkg.PkgPath == "unsafe" {
809 // Fill in the blanks to avoid surprises.
810 lpkg.Types = types.Unsafe
812 lpkg.Syntax = []*ast.File{}
813 lpkg.TypesInfo = new(types.Info)
814 lpkg.TypesSizes = ld.sizes
818 // Call NewPackage directly with explicit name.
819 // This avoids skew between golist and go/types when the files'
820 // package declarations are inconsistent.
821 lpkg.Types = types.NewPackage(lpkg.PkgPath, lpkg.Name)
824 // Subtle: we populate all Types fields with an empty Package
825 // before loading export data so that export data processing
826 // never has to create a types.Package for an indirect dependency,
827 // which would then require that such created packages be explicitly
828 // inserted back into the Import graph as a final step after export data loading.
829 // The Diamond test exercises this case.
830 if !lpkg.needtypes && !lpkg.needsrc {
834 ld.loadFromExportData(lpkg)
835 return // not a source package, don't get syntax trees
838 appendError := func(err error) {
839 // Convert various error types into the one true Error.
841 switch err := err.(type) {
844 errs = append(errs, err)
848 errs = append(errs, Error{
849 Pos: err.Path + ":1",
850 Msg: err.Err.Error(),
854 case scanner.ErrorList:
856 for _, err := range err {
857 errs = append(errs, Error{
858 Pos: err.Pos.String(),
866 errs = append(errs, Error{
867 Pos: err.Fset.Position(err.Pos).String(),
873 // unexpected impoverished error from parser?
874 errs = append(errs, Error{
880 // If you see this error message, please file a bug.
881 log.Printf("internal error: error %q (%T) without position", err, err)
884 lpkg.Errors = append(lpkg.Errors, errs...)
887 if ld.Config.Mode&NeedTypes != 0 && len(lpkg.CompiledGoFiles) == 0 && lpkg.ExportFile != "" {
888 // The config requested loading sources and types, but sources are missing.
889 // Add an error to the package and fall back to loading from export data.
890 appendError(Error{"-", fmt.Sprintf("sources missing for package %s", lpkg.ID), ParseError})
891 ld.loadFromExportData(lpkg)
892 return // can't get syntax trees for this package
895 files, errs := ld.parseFiles(lpkg.CompiledGoFiles)
896 for _, err := range errs {
901 if ld.Config.Mode&NeedTypes == 0 {
905 lpkg.TypesInfo = &types.Info{
906 Types: make(map[ast.Expr]types.TypeAndValue),
907 Defs: make(map[*ast.Ident]types.Object),
908 Uses: make(map[*ast.Ident]types.Object),
909 Implicits: make(map[ast.Node]types.Object),
910 Scopes: make(map[ast.Node]*types.Scope),
911 Selections: make(map[*ast.SelectorExpr]*types.Selection),
913 lpkg.TypesSizes = ld.sizes
915 importer := importerFunc(func(path string) (*types.Package, error) {
916 if path == "unsafe" {
917 return types.Unsafe, nil
920 // The imports map is keyed by import path.
921 ipkg := lpkg.Imports[path]
923 if err := lpkg.importErrors[path]; err != nil {
926 // There was skew between the metadata and the
927 // import declarations, likely due to an edit
928 // race, or because the ParseFile feature was
929 // used to supply alternative file contents.
930 return nil, fmt.Errorf("no metadata for %s", path)
933 if ipkg.Types != nil && ipkg.Types.Complete() {
934 return ipkg.Types, nil
936 log.Fatalf("internal error: package %q without types was imported from %q", path, lpkg)
944 // Type-check bodies of functions only in non-initial packages.
945 // Example: for import graph A->B->C and initial packages {A,C},
946 // we can ignore function bodies in B.
947 IgnoreFuncBodies: ld.Mode&NeedDeps == 0 && !lpkg.initial,
952 if (ld.Mode & typecheckCgo) != 0 {
953 if !typesinternal.SetUsesCgo(tc) {
955 Msg: "typecheckCgo requires Go 1.15+",
961 types.NewChecker(tc, ld.Fset, lpkg.Types, lpkg.TypesInfo).Files(lpkg.Syntax)
963 lpkg.importErrors = nil // no longer needed
965 // If !Cgo, the type-checker uses FakeImportC mode, so
966 // it doesn't invoke the importer for import "C",
967 // nor report an error for the import,
968 // or for any undefined C.f reference.
969 // We must detect this explicitly and correctly
970 // mark the package as IllTyped (by reporting an error).
971 // TODO(adonovan): if these errors are annoying,
972 // we could just set IllTyped quietly.
975 for _, f := range lpkg.Syntax {
976 for _, imp := range f.Imports {
977 if imp.Path.Value == `"C"` {
978 err := types.Error{Fset: ld.Fset, Pos: imp.Pos(), Msg: `import "C" ignored`}
986 // Record accumulated errors.
987 illTyped := len(lpkg.Errors) > 0
989 for _, imp := range lpkg.Imports {
996 lpkg.IllTyped = illTyped
999 // An importFunc is an implementation of the single-method
1000 // types.Importer interface based on a function value.
1001 type importerFunc func(path string) (*types.Package, error)
1003 func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) }
1005 // We use a counting semaphore to limit
1006 // the number of parallel I/O calls per process.
1007 var ioLimit = make(chan bool, 20)
1009 func (ld *loader) parseFile(filename string) (*ast.File, error) {
1010 ld.parseCacheMu.Lock()
1011 v, ok := ld.parseCache[filename]
1014 ld.parseCacheMu.Unlock()
1018 v = &parseValue{ready: make(chan struct{})}
1019 ld.parseCache[filename] = v
1020 ld.parseCacheMu.Unlock()
1023 for f, contents := range ld.Config.Overlay {
1024 if sameFile(f, filename) {
1030 ioLimit <- true // wait
1031 src, err = ioutil.ReadFile(filename)
1037 v.f, v.err = ld.ParseFile(ld.Fset, filename, src)
1045 // parseFiles reads and parses the Go source files and returns the ASTs
1046 // of the ones that could be at least partially parsed, along with a
1047 // list of I/O and parse errors encountered.
1049 // Because files are scanned in parallel, the token.Pos
1050 // positions of the resulting ast.Files are not ordered.
1052 func (ld *loader) parseFiles(filenames []string) ([]*ast.File, []error) {
1053 var wg sync.WaitGroup
1055 parsed := make([]*ast.File, n)
1056 errors := make([]error, n)
1057 for i, file := range filenames {
1058 if ld.Config.Context.Err() != nil {
1060 errors[i] = ld.Config.Context.Err()
1064 go func(i int, filename string) {
1065 parsed[i], errors[i] = ld.parseFile(filename)
1071 // Eliminate nils, preserving order.
1073 for _, f := range parsed {
1082 for _, err := range errors {
1090 return parsed, errors
1093 // sameFile returns true if x and y have the same basename and denote
1096 func sameFile(x, y string) bool {
1098 // It could be the case that y doesn't exist.
1099 // For instance, it may be an overlay file that
1100 // hasn't been written to disk. To handle that case
1101 // let x == y through. (We added the exact absolute path
1102 // string to the CompiledGoFiles list, so the unwritten
1103 // overlay case implies x==y.)
1106 if strings.EqualFold(filepath.Base(x), filepath.Base(y)) { // (optimisation)
1107 if xi, err := os.Stat(x); err == nil {
1108 if yi, err := os.Stat(y); err == nil {
1109 return os.SameFile(xi, yi)
1116 // loadFromExportData returns type information for the specified
1117 // package, loading it from an export data file on the first request.
1118 func (ld *loader) loadFromExportData(lpkg *loaderPackage) (*types.Package, error) {
1119 if lpkg.PkgPath == "" {
1120 log.Fatalf("internal error: Package %s has no PkgPath", lpkg)
1123 // Because gcexportdata.Read has the potential to create or
1124 // modify the types.Package for each node in the transitive
1125 // closure of dependencies of lpkg, all exportdata operations
1126 // must be sequential. (Finer-grained locking would require
1127 // changes to the gcexportdata API.)
1129 // The exportMu lock guards the Package.Pkg field and the
1130 // types.Package it points to, for each Package in the graph.
1132 // Not all accesses to Package.Pkg need to be protected by exportMu:
1133 // graph ordering ensures that direct dependencies of source
1134 // packages are fully loaded before the importer reads their Pkg field.
1136 defer ld.exportMu.Unlock()
1138 if tpkg := lpkg.Types; tpkg != nil && tpkg.Complete() {
1139 return tpkg, nil // cache hit
1142 lpkg.IllTyped = true // fail safe
1144 if lpkg.ExportFile == "" {
1145 // Errors while building export data will have been printed to stderr.
1146 return nil, fmt.Errorf("no export data file")
1148 f, err := os.Open(lpkg.ExportFile)
1154 // Read gc export data.
1156 // We don't currently support gccgo export data because all
1157 // underlying workspaces use the gc toolchain. (Even build
1158 // systems that support gccgo don't use it for workspace
1160 r, err := gcexportdata.NewReader(f)
1162 return nil, fmt.Errorf("reading %s: %v", lpkg.ExportFile, err)
1167 // The gcexportdata machinery has no concept of package ID.
1168 // It identifies packages by their PkgPath, which although not
1169 // globally unique is unique within the scope of one invocation
1170 // of the linker, type-checker, or gcexportdata.
1172 // So, we must build a PkgPath-keyed view of the global
1173 // (conceptually ID-keyed) cache of packages and pass it to
1174 // gcexportdata. The view must contain every existing
1175 // package that might possibly be mentioned by the
1176 // current package---its transitive closure.
1178 // In loadPackage, we unconditionally create a types.Package for
1179 // each dependency so that export data loading does not
1182 // TODO(adonovan): it would be simpler and more efficient
1183 // if the export data machinery invoked a callback to
1184 // get-or-create a package instead of a map.
1186 view := make(map[string]*types.Package) // view seen by gcexportdata
1187 seen := make(map[*loaderPackage]bool) // all visited packages
1188 var visit func(pkgs map[string]*Package)
1189 visit = func(pkgs map[string]*Package) {
1190 for _, p := range pkgs {
1191 lpkg := ld.pkgs[p.ID]
1194 view[lpkg.PkgPath] = lpkg.Types
1201 viewLen := len(view) + 1 // adding the self package
1202 // Parse the export data.
1203 // (May modify incomplete packages in view but not create new ones.)
1204 tpkg, err := gcexportdata.Read(r, ld.Fset, view, lpkg.PkgPath)
1206 return nil, fmt.Errorf("reading %s: %v", lpkg.ExportFile, err)
1208 if viewLen != len(view) {
1209 log.Fatalf("Unexpected package creation during export data loading")
1213 lpkg.IllTyped = false
1218 // impliedLoadMode returns loadMode with its dependencies.
1219 func impliedLoadMode(loadMode LoadMode) LoadMode {
1220 if loadMode&NeedTypesInfo != 0 && loadMode&NeedImports == 0 {
1221 // If NeedTypesInfo, go/packages needs to do typechecking itself so it can
1222 // associate type info with the AST. To do so, we need the export data
1223 // for dependencies, which means we need to ask for the direct dependencies.
1224 // NeedImports is used to ask for the direct dependencies.
1225 loadMode |= NeedImports
1228 if loadMode&NeedDeps != 0 && loadMode&NeedImports == 0 {
1229 // With NeedDeps we need to load at least direct dependencies.
1230 // NeedImports is used to ask for the direct dependencies.
1231 loadMode |= NeedImports
1237 func usesExportData(cfg *Config) bool {
1238 return cfg.Mode&NeedExportsFile != 0 || cfg.Mode&NeedTypes != 0 && cfg.Mode&NeedDeps == 0