+++ /dev/null
-package lint
-
-/*
-Package loading
-
-Conceptually, package loading in the runner can be imagined as a
-graph-shaped work list. We iteratively pop off leaf nodes (packages
-that have no unloaded dependencies) and load data from export data,
-our cache, or source.
-
-Specifically, non-initial packages are loaded from export data and the
-fact cache if possible, otherwise from source. Initial packages are
-loaded from export data, the fact cache and the (problems, ignores,
-config) cache if possible, otherwise from source.
-
-The appeal of this approach is that it is both simple to implement and
-easily parallelizable. Each leaf node can be processed independently,
-and new leaf nodes appear as their dependencies are being processed.
-
-The downside of this approach, however, is that we're doing more work
-than necessary. Imagine an initial package A, which has the following
-dependency chain: A->B->C->D – in the current implementation, we will
-load all 4 packages. However, if package A can be loaded fully from
-cached information, then none of its dependencies are necessary, and
-we could avoid loading them.
-
-
-Parallelism
-
-Runner implements parallel processing of packages by spawning one
-goroutine per package in the dependency graph, without any semaphores.
-Each goroutine initially waits on the completion of all of its
-dependencies, thus establishing correct order of processing. Once all
-dependencies finish processing, the goroutine will load the package
-from export data or source – this loading is guarded by a semaphore,
-sized according to the number of CPU cores. This way, we only have as
-many packages occupying memory and CPU resources as there are actual
-cores to process them.
-
-This combination of unbounded goroutines but bounded package loading
-means that if we have many parallel, independent subgraphs, they will
-all execute in parallel, while not wasting resources for long linear
-chains or trying to process more subgraphs in parallel than the system
-can handle.
-
-
-Caching
-
-We make use of several caches. These caches are Go's export data, our
-facts cache, and our (problems, ignores, config) cache.
-
-Initial packages will either be loaded from a combination of all three
-caches, or from source. Non-initial packages will either be loaded
-from a combination of export data and facts cache, or from source.
-
-The facts cache is separate from the (problems, ignores, config) cache
-because when we process non-initial packages, we generate facts, but
-we discard problems and ignores.
-
-The facts cache is keyed by (package, analyzer), whereas the
-(problems, ignores, config) cache is keyed by (package, list of
-analyzes). The difference between the two exists because there are
-only a handful of analyses that produce facts, but hundreds of
-analyses that don't. Creating one cache entry per fact-generating
-analysis is feasible, creating one cache entry per normal analysis has
-significant performance and storage overheads.
-
-The downside of keying by the list of analyzes is, naturally, that a
-change in list of analyzes changes the cache key. `staticcheck -checks
-A` and `staticcheck -checks A,B` will therefore need their own cache
-entries and not reuse each other's work. This problem does not affect
-the facts cache.
-
-*/
-
-import (
- "bytes"
- "encoding/gob"
- "encoding/hex"
- "fmt"
- "go/ast"
- "go/token"
- "go/types"
- "reflect"
- "regexp"
- "runtime"
- "sort"
- "strconv"
- "strings"
- "sync"
- "sync/atomic"
- "time"
-
- "golang.org/x/tools/go/analysis"
- "golang.org/x/tools/go/packages"
- "golang.org/x/tools/go/types/objectpath"
- "honnef.co/go/tools/config"
- "honnef.co/go/tools/facts"
- "honnef.co/go/tools/internal/cache"
- "honnef.co/go/tools/loader"
-)
-
-func init() {
- gob.Register(&FileIgnore{})
- gob.Register(&LineIgnore{})
-}
-
-// If enabled, abuse of the go/analysis API will lead to panics
-const sanityCheck = true
-
-// OPT(dh): for a dependency tree A->B->C->D, if we have cached data
-// for B, there should be no need to load C and D individually. Go's
-// export data for B contains all the data we need on types, and our
-// fact cache could store the union of B, C and D in B.
-//
-// This may change unused's behavior, however, as it may observe fewer
-// interfaces from transitive dependencies.
-
-// OPT(dh): every single package will have the same value for
-// canClearTypes. We could move the Package.decUse method to runner to
-// eliminate this field. This is probably not worth it, though. There
-// are only thousands of packages, so the field only takes up
-// kilobytes of memory.
-
-// OPT(dh): do we really need the Package.gen field? it's based
-// trivially on pkg.results and merely caches the result of a type
-// assertion. How often do we actually use the field?
-
-type Package struct {
- // dependents is initially set to 1 plus the number of packages
- // that directly import this package. It is atomically decreased
- // by 1 every time a dependent has been processed or when the
- // package itself has been processed. Once the value reaches zero,
- // the package is no longer needed.
- dependents uint64
-
- *packages.Package
- Imports []*Package
- initial bool
- // fromSource is set to true for packages that have been loaded
- // from source. This is the case for initial packages, packages
- // with missing export data, and packages with no cached facts.
- fromSource bool
- // hash stores the package hash, as computed by packageHash
- hash string
- actionID cache.ActionID
- done chan struct{}
-
- resultsMu sync.Mutex
- // results maps analyzer IDs to analyzer results. it is
- // implemented as a deduplicating concurrent cache.
- results []*result
-
- cfg *config.Config
- // gen maps file names to the code generator that created them
- gen map[string]facts.Generator
- problems []Problem
- ignores []Ignore
- errs []error
-
- // these slices are indexed by analysis
- facts []map[types.Object][]analysis.Fact
- pkgFacts [][]analysis.Fact
-
- // canClearTypes is set to true if we can discard type
- // information after the package and its dependents have been
- // processed. This is the case when no cumulative checkers are
- // being run.
- canClearTypes bool
-}
-
-type cachedPackage struct {
- Problems []Problem
- Ignores []Ignore
- Config *config.Config
-}
-
-func (pkg *Package) decUse() {
- ret := atomic.AddUint64(&pkg.dependents, ^uint64(0))
- if ret == 0 {
- // nobody depends on this package anymore
- if pkg.canClearTypes {
- pkg.Types = nil
- }
- pkg.facts = nil
- pkg.pkgFacts = nil
-
- for _, imp := range pkg.Imports {
- imp.decUse()
- }
- }
-}
-
-type result struct {
- v interface{}
- err error
- ready chan struct{}
-}
-
-type Runner struct {
- cache *cache.Cache
- goVersion int
- stats *Stats
- repeatAnalyzers uint
-
- analyzerIDs analyzerIDs
- problemsCacheKey string
-
- // limits parallelism of loading packages
- loadSem chan struct{}
-}
-
-type analyzerIDs struct {
- m map[*analysis.Analyzer]int
-}
-
-func (ids analyzerIDs) get(a *analysis.Analyzer) int {
- id, ok := ids.m[a]
- if !ok {
- panic(fmt.Sprintf("no analyzer ID for %s", a.Name))
- }
- return id
-}
-
-type Fact struct {
- Path string
- Fact analysis.Fact
-}
-
-type analysisAction struct {
- analyzer *analysis.Analyzer
- analyzerID int
- pkg *Package
- newPackageFacts []analysis.Fact
- problems []Problem
-
- pkgFacts map[*types.Package][]analysis.Fact
-}
-
-func (ac *analysisAction) String() string {
- return fmt.Sprintf("%s @ %s", ac.analyzer, ac.pkg)
-}
-
-func (ac *analysisAction) allObjectFacts() []analysis.ObjectFact {
- out := make([]analysis.ObjectFact, 0, len(ac.pkg.facts[ac.analyzerID]))
- for obj, facts := range ac.pkg.facts[ac.analyzerID] {
- for _, fact := range facts {
- out = append(out, analysis.ObjectFact{
- Object: obj,
- Fact: fact,
- })
- }
- }
- return out
-}
-
-func (ac *analysisAction) allPackageFacts() []analysis.PackageFact {
- out := make([]analysis.PackageFact, 0, len(ac.pkgFacts))
- for pkg, facts := range ac.pkgFacts {
- for _, fact := range facts {
- out = append(out, analysis.PackageFact{
- Package: pkg,
- Fact: fact,
- })
- }
- }
- return out
-}
-
-func (ac *analysisAction) importObjectFact(obj types.Object, fact analysis.Fact) bool {
- if sanityCheck && len(ac.analyzer.FactTypes) == 0 {
- panic("analysis doesn't export any facts")
- }
- for _, f := range ac.pkg.facts[ac.analyzerID][obj] {
- if reflect.TypeOf(f) == reflect.TypeOf(fact) {
- reflect.ValueOf(fact).Elem().Set(reflect.ValueOf(f).Elem())
- return true
- }
- }
- return false
-}
-
-func (ac *analysisAction) importPackageFact(pkg *types.Package, fact analysis.Fact) bool {
- if sanityCheck && len(ac.analyzer.FactTypes) == 0 {
- panic("analysis doesn't export any facts")
- }
- for _, f := range ac.pkgFacts[pkg] {
- if reflect.TypeOf(f) == reflect.TypeOf(fact) {
- reflect.ValueOf(fact).Elem().Set(reflect.ValueOf(f).Elem())
- return true
- }
- }
- return false
-}
-
-func (ac *analysisAction) exportObjectFact(obj types.Object, fact analysis.Fact) {
- if sanityCheck && len(ac.analyzer.FactTypes) == 0 {
- panic("analysis doesn't export any facts")
- }
- ac.pkg.facts[ac.analyzerID][obj] = append(ac.pkg.facts[ac.analyzerID][obj], fact)
-}
-
-func (ac *analysisAction) exportPackageFact(fact analysis.Fact) {
- if sanityCheck && len(ac.analyzer.FactTypes) == 0 {
- panic("analysis doesn't export any facts")
- }
- ac.pkgFacts[ac.pkg.Types] = append(ac.pkgFacts[ac.pkg.Types], fact)
- ac.newPackageFacts = append(ac.newPackageFacts, fact)
-}
-
-func (ac *analysisAction) report(pass *analysis.Pass, d analysis.Diagnostic) {
- p := Problem{
- Pos: DisplayPosition(pass.Fset, d.Pos),
- End: DisplayPosition(pass.Fset, d.End),
- Message: d.Message,
- Check: pass.Analyzer.Name,
- }
- for _, r := range d.Related {
- p.Related = append(p.Related, Related{
- Pos: DisplayPosition(pass.Fset, r.Pos),
- End: DisplayPosition(pass.Fset, r.End),
- Message: r.Message,
- })
- }
- ac.problems = append(ac.problems, p)
-}
-
-func (r *Runner) runAnalysis(ac *analysisAction) (ret interface{}, err error) {
- ac.pkg.resultsMu.Lock()
- res := ac.pkg.results[r.analyzerIDs.get(ac.analyzer)]
- if res != nil {
- ac.pkg.resultsMu.Unlock()
- <-res.ready
- return res.v, res.err
- } else {
- res = &result{
- ready: make(chan struct{}),
- }
- ac.pkg.results[r.analyzerIDs.get(ac.analyzer)] = res
- ac.pkg.resultsMu.Unlock()
-
- defer func() {
- res.v = ret
- res.err = err
- close(res.ready)
- }()
-
- pass := new(analysis.Pass)
- *pass = analysis.Pass{
- Analyzer: ac.analyzer,
- Fset: ac.pkg.Fset,
- Files: ac.pkg.Syntax,
- // type information may be nil or may be populated. if it is
- // nil, it will get populated later.
- Pkg: ac.pkg.Types,
- TypesInfo: ac.pkg.TypesInfo,
- TypesSizes: ac.pkg.TypesSizes,
- ResultOf: map[*analysis.Analyzer]interface{}{},
- ImportObjectFact: ac.importObjectFact,
- ImportPackageFact: ac.importPackageFact,
- ExportObjectFact: ac.exportObjectFact,
- ExportPackageFact: ac.exportPackageFact,
- Report: func(d analysis.Diagnostic) {
- ac.report(pass, d)
- },
- AllObjectFacts: ac.allObjectFacts,
- AllPackageFacts: ac.allPackageFacts,
- }
-
- if !ac.pkg.initial {
- // Don't report problems in dependencies
- pass.Report = func(analysis.Diagnostic) {}
- }
- return r.runAnalysisUser(pass, ac)
- }
-}
-
-func (r *Runner) loadCachedPackage(pkg *Package, analyzers []*analysis.Analyzer) (cachedPackage, bool) {
- // OPT(dh): we can cache this computation, it'll be the same for all packages
- id := cache.Subkey(pkg.actionID, "data "+r.problemsCacheKey)
-
- b, _, err := r.cache.GetBytes(id)
- if err != nil {
- return cachedPackage{}, false
- }
- var cpkg cachedPackage
- if err := gob.NewDecoder(bytes.NewReader(b)).Decode(&cpkg); err != nil {
- return cachedPackage{}, false
- }
- return cpkg, true
-}
-
-func (r *Runner) loadCachedFacts(a *analysis.Analyzer, pkg *Package) ([]Fact, bool) {
- if len(a.FactTypes) == 0 {
- return nil, true
- }
-
- var facts []Fact
- // Look in the cache for facts
- aID := passActionID(pkg, a)
- aID = cache.Subkey(aID, "facts")
- b, _, err := r.cache.GetBytes(aID)
- if err != nil {
- // No cached facts, analyse this package like a user-provided one, but ignore diagnostics
- return nil, false
- }
-
- if err := gob.NewDecoder(bytes.NewReader(b)).Decode(&facts); err != nil {
- // Cached facts are broken, analyse this package like a user-provided one, but ignore diagnostics
- return nil, false
- }
- return facts, true
-}
-
-type dependencyError struct {
- dep string
- err error
-}
-
-func (err dependencyError) nested() dependencyError {
- if o, ok := err.err.(dependencyError); ok {
- return o.nested()
- }
- return err
-}
-
-func (err dependencyError) Error() string {
- if o, ok := err.err.(dependencyError); ok {
- return o.Error()
- }
- return fmt.Sprintf("error running dependency %s: %s", err.dep, err.err)
-}
-
-func (r *Runner) makeAnalysisAction(a *analysis.Analyzer, pkg *Package) *analysisAction {
- aid := r.analyzerIDs.get(a)
- ac := &analysisAction{
- analyzer: a,
- analyzerID: aid,
- pkg: pkg,
- }
-
- if len(a.FactTypes) == 0 {
- return ac
- }
-
- // Merge all package facts of dependencies
- ac.pkgFacts = map[*types.Package][]analysis.Fact{}
- seen := map[*Package]struct{}{}
- var dfs func(*Package)
- dfs = func(pkg *Package) {
- if _, ok := seen[pkg]; ok {
- return
- }
- seen[pkg] = struct{}{}
- s := pkg.pkgFacts[aid]
- ac.pkgFacts[pkg.Types] = s[0:len(s):len(s)]
- for _, imp := range pkg.Imports {
- dfs(imp)
- }
- }
- dfs(pkg)
-
- return ac
-}
-
-// analyzes that we always want to run, even if they're not being run
-// explicitly or as dependencies. these are necessary for the inner
-// workings of the runner.
-var injectedAnalyses = []*analysis.Analyzer{facts.Generated, config.Analyzer}
-
-func (r *Runner) runAnalysisUser(pass *analysis.Pass, ac *analysisAction) (interface{}, error) {
- if !ac.pkg.fromSource {
- panic(fmt.Sprintf("internal error: %s was not loaded from source", ac.pkg))
- }
-
- // User-provided package, analyse it
- // First analyze it with dependencies
- for _, req := range ac.analyzer.Requires {
- acReq := r.makeAnalysisAction(req, ac.pkg)
- ret, err := r.runAnalysis(acReq)
- if err != nil {
- // We couldn't run a dependency, no point in going on
- return nil, dependencyError{req.Name, err}
- }
-
- pass.ResultOf[req] = ret
- }
-
- // Then with this analyzer
- var ret interface{}
- for i := uint(0); i < r.repeatAnalyzers+1; i++ {
- var err error
- t := time.Now()
- ret, err = ac.analyzer.Run(pass)
- r.stats.MeasureAnalyzer(ac.analyzer, ac.pkg, time.Since(t))
- if err != nil {
- return nil, err
- }
- }
-
- if len(ac.analyzer.FactTypes) > 0 {
- // Merge new facts into the package and persist them.
- var facts []Fact
- for _, fact := range ac.newPackageFacts {
- id := r.analyzerIDs.get(ac.analyzer)
- ac.pkg.pkgFacts[id] = append(ac.pkg.pkgFacts[id], fact)
- facts = append(facts, Fact{"", fact})
- }
- for obj, afacts := range ac.pkg.facts[ac.analyzerID] {
- if obj.Pkg() != ac.pkg.Package.Types {
- continue
- }
- path, err := objectpath.For(obj)
- if err != nil {
- continue
- }
- for _, fact := range afacts {
- facts = append(facts, Fact{string(path), fact})
- }
- }
-
- if err := r.cacheData(facts, ac.pkg, ac.analyzer, "facts"); err != nil {
- return nil, err
- }
- }
-
- return ret, nil
-}
-
-func (r *Runner) cacheData(v interface{}, pkg *Package, a *analysis.Analyzer, subkey string) error {
- buf := &bytes.Buffer{}
- if err := gob.NewEncoder(buf).Encode(v); err != nil {
- return err
- }
- aID := passActionID(pkg, a)
- aID = cache.Subkey(aID, subkey)
- if err := r.cache.PutBytes(aID, buf.Bytes()); err != nil {
- return err
- }
- return nil
-}
-
-func NewRunner(stats *Stats) (*Runner, error) {
- cache, err := cache.Default()
- if err != nil {
- return nil, err
- }
-
- return &Runner{
- cache: cache,
- stats: stats,
- }, nil
-}
-
-// Run loads packages corresponding to patterns and analyses them with
-// analyzers. It returns the loaded packages, which contain reported
-// diagnostics as well as extracted ignore directives.
-//
-// Note that diagnostics have not been filtered at this point yet, to
-// accommodate cumulative analyzes that require additional steps to
-// produce diagnostics.
-func (r *Runner) Run(cfg *packages.Config, patterns []string, analyzers []*analysis.Analyzer, hasCumulative bool) ([]*Package, error) {
- checkerNames := make([]string, len(analyzers))
- for i, a := range analyzers {
- checkerNames[i] = a.Name
- }
- sort.Strings(checkerNames)
- r.problemsCacheKey = strings.Join(checkerNames, " ")
-
- var allAnalyzers []*analysis.Analyzer
- r.analyzerIDs = analyzerIDs{m: map[*analysis.Analyzer]int{}}
- id := 0
- seen := map[*analysis.Analyzer]struct{}{}
- var dfs func(a *analysis.Analyzer)
- dfs = func(a *analysis.Analyzer) {
- if _, ok := seen[a]; ok {
- return
- }
- seen[a] = struct{}{}
- allAnalyzers = append(allAnalyzers, a)
- r.analyzerIDs.m[a] = id
- id++
- for _, f := range a.FactTypes {
- gob.Register(f)
- }
- for _, req := range a.Requires {
- dfs(req)
- }
- }
- for _, a := range analyzers {
- if v := a.Flags.Lookup("go"); v != nil {
- v.Value.Set(fmt.Sprintf("1.%d", r.goVersion))
- }
- dfs(a)
- }
- for _, a := range injectedAnalyses {
- dfs(a)
- }
- // Run all analyzers on all packages (subject to further
- // restrictions enforced later). This guarantees that if analyzer
- // A1 depends on A2, and A2 has facts, that A2 will run on the
- // dependencies of user-provided packages, even though A1 won't.
- analyzers = allAnalyzers
-
- var dcfg packages.Config
- if cfg != nil {
- dcfg = *cfg
- }
-
- atomic.StoreUint32(&r.stats.State, StateGraph)
- initialPkgs, err := loader.Graph(dcfg, patterns...)
- if err != nil {
- return nil, err
- }
- defer r.cache.Trim()
-
- var allPkgs []*Package
- m := map[*packages.Package]*Package{}
- packages.Visit(initialPkgs, nil, func(l *packages.Package) {
- m[l] = &Package{
- Package: l,
- results: make([]*result, len(r.analyzerIDs.m)),
- facts: make([]map[types.Object][]analysis.Fact, len(r.analyzerIDs.m)),
- pkgFacts: make([][]analysis.Fact, len(r.analyzerIDs.m)),
- done: make(chan struct{}),
- // every package needs itself
- dependents: 1,
- canClearTypes: !hasCumulative,
- }
- allPkgs = append(allPkgs, m[l])
- for i := range m[l].facts {
- m[l].facts[i] = map[types.Object][]analysis.Fact{}
- }
- for _, err := range l.Errors {
- m[l].errs = append(m[l].errs, err)
- }
- for _, v := range l.Imports {
- m[v].dependents++
- m[l].Imports = append(m[l].Imports, m[v])
- }
-
- m[l].hash, err = r.packageHash(m[l])
- m[l].actionID = packageActionID(m[l])
- if err != nil {
- m[l].errs = append(m[l].errs, err)
- }
- })
-
- pkgs := make([]*Package, len(initialPkgs))
- for i, l := range initialPkgs {
- pkgs[i] = m[l]
- pkgs[i].initial = true
- }
-
- atomic.StoreUint32(&r.stats.InitialPackages, uint32(len(initialPkgs)))
- atomic.StoreUint32(&r.stats.TotalPackages, uint32(len(allPkgs)))
- atomic.StoreUint32(&r.stats.State, StateProcessing)
-
- var wg sync.WaitGroup
- wg.Add(len(allPkgs))
- r.loadSem = make(chan struct{}, runtime.GOMAXPROCS(-1))
- atomic.StoreUint32(&r.stats.TotalWorkers, uint32(cap(r.loadSem)))
- for _, pkg := range allPkgs {
- pkg := pkg
- go func() {
- r.processPkg(pkg, analyzers)
-
- if pkg.initial {
- atomic.AddUint32(&r.stats.ProcessedInitialPackages, 1)
- }
- atomic.AddUint32(&r.stats.Problems, uint32(len(pkg.problems)))
- wg.Done()
- }()
- }
- wg.Wait()
-
- return pkgs, nil
-}
-
-var posRe = regexp.MustCompile(`^(.+?):(\d+)(?::(\d+)?)?`)
-
-func parsePos(pos string) (token.Position, int, error) {
- if pos == "-" || pos == "" {
- return token.Position{}, 0, nil
- }
- parts := posRe.FindStringSubmatch(pos)
- if parts == nil {
- return token.Position{}, 0, fmt.Errorf("malformed position %q", pos)
- }
- file := parts[1]
- line, _ := strconv.Atoi(parts[2])
- col, _ := strconv.Atoi(parts[3])
- return token.Position{
- Filename: file,
- Line: line,
- Column: col,
- }, len(parts[0]), nil
-}
-
-// loadPkg loads a Go package. It may be loaded from a combination of
-// caches, or from source.
-func (r *Runner) loadPkg(pkg *Package, analyzers []*analysis.Analyzer) error {
- if pkg.Types != nil {
- panic(fmt.Sprintf("internal error: %s has already been loaded", pkg.Package))
- }
-
- if pkg.initial {
- // Try to load cached package
- cpkg, ok := r.loadCachedPackage(pkg, analyzers)
- if ok {
- pkg.problems = cpkg.Problems
- pkg.ignores = cpkg.Ignores
- pkg.cfg = cpkg.Config
- } else {
- pkg.fromSource = true
- return loader.LoadFromSource(pkg.Package)
- }
- }
-
- // At this point we're either working with a non-initial package,
- // or we managed to load cached problems for the package. We still
- // need export data and facts.
-
- // OPT(dh): we don't need type information for this package if no
- // other package depends on it. this may be the case for initial
- // packages.
-
- // Load package from export data
- if err := loader.LoadFromExport(pkg.Package); err != nil {
- // We asked Go to give us up to date export data, yet
- // we can't load it. There must be something wrong.
- //
- // Attempt loading from source. This should fail (because
- // otherwise there would be export data); we just want to
- // get the compile errors. If loading from source succeeds
- // we discard the result, anyway. Otherwise we'll fail
- // when trying to reload from export data later.
- //
- // FIXME(dh): we no longer reload from export data, so
- // theoretically we should be able to continue
- pkg.fromSource = true
- if err := loader.LoadFromSource(pkg.Package); err != nil {
- return err
- }
- // Make sure this package can't be imported successfully
- pkg.Package.Errors = append(pkg.Package.Errors, packages.Error{
- Pos: "-",
- Msg: fmt.Sprintf("could not load export data: %s", err),
- Kind: packages.ParseError,
- })
- return fmt.Errorf("could not load export data: %s", err)
- }
-
- failed := false
- seen := make([]bool, len(r.analyzerIDs.m))
- var dfs func(*analysis.Analyzer)
- dfs = func(a *analysis.Analyzer) {
- if seen[r.analyzerIDs.get(a)] {
- return
- }
- seen[r.analyzerIDs.get(a)] = true
-
- if len(a.FactTypes) > 0 {
- facts, ok := r.loadCachedFacts(a, pkg)
- if !ok {
- failed = true
- return
- }
-
- for _, f := range facts {
- if f.Path == "" {
- // This is a package fact
- pkg.pkgFacts[r.analyzerIDs.get(a)] = append(pkg.pkgFacts[r.analyzerIDs.get(a)], f.Fact)
- continue
- }
- obj, err := objectpath.Object(pkg.Types, objectpath.Path(f.Path))
- if err != nil {
- // Be lenient about these errors. For example, when
- // analysing io/ioutil from source, we may get a fact
- // for methods on the devNull type, and objectpath
- // will happily create a path for them. However, when
- // we later load io/ioutil from export data, the path
- // no longer resolves.
- //
- // If an exported type embeds the unexported type,
- // then (part of) the unexported type will become part
- // of the type information and our path will resolve
- // again.
- continue
- }
- pkg.facts[r.analyzerIDs.get(a)][obj] = append(pkg.facts[r.analyzerIDs.get(a)][obj], f.Fact)
- }
- }
-
- for _, req := range a.Requires {
- dfs(req)
- }
- }
- for _, a := range analyzers {
- dfs(a)
- }
-
- if !failed {
- return nil
- }
-
- // We failed to load some cached facts
- pkg.fromSource = true
- // XXX we added facts to the maps, we need to get rid of those
- return loader.LoadFromSource(pkg.Package)
-}
-
-type analysisError struct {
- analyzer *analysis.Analyzer
- pkg *Package
- err error
-}
-
-func (err analysisError) Error() string {
- return fmt.Sprintf("error running analyzer %s on %s: %s", err.analyzer, err.pkg, err.err)
-}
-
-// processPkg processes a package. This involves loading the package,
-// either from export data or from source. For packages loaded from
-// source, the provides analyzers will be run on the package.
-func (r *Runner) processPkg(pkg *Package, analyzers []*analysis.Analyzer) {
- defer func() {
- // Clear information we no longer need. Make sure to do this
- // when returning from processPkg so that we clear
- // dependencies, not just initial packages.
- pkg.TypesInfo = nil
- pkg.Syntax = nil
- pkg.results = nil
-
- atomic.AddUint32(&r.stats.ProcessedPackages, 1)
- pkg.decUse()
- close(pkg.done)
- }()
-
- // Ensure all packages have the generated map and config. This is
- // required by internals of the runner. Analyses that themselves
- // make use of either have an explicit dependency so that other
- // runners work correctly, too.
- analyzers = append(analyzers[0:len(analyzers):len(analyzers)], injectedAnalyses...)
-
- if len(pkg.errs) != 0 {
- return
- }
-
- for _, imp := range pkg.Imports {
- <-imp.done
- if len(imp.errs) > 0 {
- if imp.initial {
- // Don't print the error of the dependency since it's
- // an initial package and we're already printing the
- // error.
- pkg.errs = append(pkg.errs, fmt.Errorf("could not analyze dependency %s of %s", imp, pkg))
- } else {
- var s string
- for _, err := range imp.errs {
- s += "\n\t" + err.Error()
- }
- pkg.errs = append(pkg.errs, fmt.Errorf("could not analyze dependency %s of %s: %s", imp, pkg, s))
- }
- return
- }
- }
- if pkg.PkgPath == "unsafe" {
- pkg.Types = types.Unsafe
- return
- }
-
- r.loadSem <- struct{}{}
- atomic.AddUint32(&r.stats.ActiveWorkers, 1)
- defer func() {
- <-r.loadSem
- atomic.AddUint32(&r.stats.ActiveWorkers, ^uint32(0))
- }()
- if err := r.loadPkg(pkg, analyzers); err != nil {
- pkg.errs = append(pkg.errs, err)
- return
- }
-
- // A package's object facts is the union of all of its dependencies.
- for _, imp := range pkg.Imports {
- for ai, m := range imp.facts {
- for obj, facts := range m {
- pkg.facts[ai][obj] = facts[0:len(facts):len(facts)]
- }
- }
- }
-
- if !pkg.fromSource {
- // Nothing left to do for the package.
- return
- }
-
- // Run analyses on initial packages and those missing facts
- var wg sync.WaitGroup
- wg.Add(len(analyzers))
- errs := make([]error, len(analyzers))
- var acs []*analysisAction
- for i, a := range analyzers {
- i := i
- a := a
- ac := r.makeAnalysisAction(a, pkg)
- acs = append(acs, ac)
- go func() {
- defer wg.Done()
- // Only initial packages and packages with missing
- // facts will have been loaded from source.
- if pkg.initial || len(a.FactTypes) > 0 {
- if _, err := r.runAnalysis(ac); err != nil {
- errs[i] = analysisError{a, pkg, err}
- return
- }
- }
- }()
- }
- wg.Wait()
-
- depErrors := map[dependencyError]int{}
- for _, err := range errs {
- if err == nil {
- continue
- }
- switch err := err.(type) {
- case analysisError:
- switch err := err.err.(type) {
- case dependencyError:
- depErrors[err.nested()]++
- default:
- pkg.errs = append(pkg.errs, err)
- }
- default:
- pkg.errs = append(pkg.errs, err)
- }
- }
- for err, count := range depErrors {
- pkg.errs = append(pkg.errs,
- fmt.Errorf("could not run %s@%s, preventing %d analyzers from running: %s", err.dep, pkg, count, err.err))
- }
-
- // We can't process ignores at this point because `unused` needs
- // to see more than one package to make its decision.
- //
- // OPT(dh): can't we guard this block of code by pkg.initial?
- ignores, problems := parseDirectives(pkg.Package)
- pkg.ignores = append(pkg.ignores, ignores...)
- pkg.problems = append(pkg.problems, problems...)
- for _, ac := range acs {
- pkg.problems = append(pkg.problems, ac.problems...)
- }
-
- if pkg.initial {
- // Only initial packages have these analyzers run, and only
- // initial packages need these.
- if pkg.results[r.analyzerIDs.get(config.Analyzer)].v != nil {
- pkg.cfg = pkg.results[r.analyzerIDs.get(config.Analyzer)].v.(*config.Config)
- }
- pkg.gen = pkg.results[r.analyzerIDs.get(facts.Generated)].v.(map[string]facts.Generator)
- }
-
- // In a previous version of the code, we would throw away all type
- // information and reload it from export data. That was
- // nonsensical. The *types.Package doesn't keep any information
- // live that export data wouldn't also. We only need to discard
- // the AST and the TypesInfo maps; that happens after we return
- // from processPkg.
-}
-
-func parseDirective(s string) (cmd string, args []string) {
- if !strings.HasPrefix(s, "//lint:") {
- return "", nil
- }
- s = strings.TrimPrefix(s, "//lint:")
- fields := strings.Split(s, " ")
- return fields[0], fields[1:]
-}
-
-// parseDirectives extracts all linter directives from the source
-// files of the package. Malformed directives are returned as problems.
-func parseDirectives(pkg *packages.Package) ([]Ignore, []Problem) {
- var ignores []Ignore
- var problems []Problem
-
- for _, f := range pkg.Syntax {
- found := false
- commentLoop:
- for _, cg := range f.Comments {
- for _, c := range cg.List {
- if strings.Contains(c.Text, "//lint:") {
- found = true
- break commentLoop
- }
- }
- }
- if !found {
- continue
- }
- cm := ast.NewCommentMap(pkg.Fset, f, f.Comments)
- for node, cgs := range cm {
- for _, cg := range cgs {
- for _, c := range cg.List {
- if !strings.HasPrefix(c.Text, "//lint:") {
- continue
- }
- cmd, args := parseDirective(c.Text)
- switch cmd {
- case "ignore", "file-ignore":
- if len(args) < 2 {
- p := Problem{
- Pos: DisplayPosition(pkg.Fset, c.Pos()),
- Message: "malformed linter directive; missing the required reason field?",
- Severity: Error,
- Check: "compile",
- }
- problems = append(problems, p)
- continue
- }
- default:
- // unknown directive, ignore
- continue
- }
- checks := strings.Split(args[0], ",")
- pos := DisplayPosition(pkg.Fset, node.Pos())
- var ig Ignore
- switch cmd {
- case "ignore":
- ig = &LineIgnore{
- File: pos.Filename,
- Line: pos.Line,
- Checks: checks,
- Pos: DisplayPosition(pkg.Fset, c.Pos()),
- }
- case "file-ignore":
- ig = &FileIgnore{
- File: pos.Filename,
- Checks: checks,
- }
- }
- ignores = append(ignores, ig)
- }
- }
- }
- }
-
- return ignores, problems
-}
-
-// packageHash computes a package's hash. The hash is based on all Go
-// files that make up the package, as well as the hashes of imported
-// packages.
-func (r *Runner) packageHash(pkg *Package) (string, error) {
- key := cache.NewHash("package hash")
- fmt.Fprintf(key, "pkgpath %s\n", pkg.PkgPath)
- fmt.Fprintf(key, "go %d\n", r.goVersion)
- for _, f := range pkg.CompiledGoFiles {
- h, err := cache.FileHash(f)
- if err != nil {
- return "", err
- }
- fmt.Fprintf(key, "file %s %x\n", f, h)
- }
-
- // Actually load the configuration to calculate its hash. This
- // will take into consideration inheritance of configuration
- // files, as well as the default configuration.
- //
- // OPT(dh): doing this means we'll load the config twice: once for
- // computing the hash, and once when analyzing the package from
- // source.
- cdir := config.Dir(pkg.GoFiles)
- if cdir == "" {
- fmt.Fprintf(key, "file %s %x\n", config.ConfigName, [cache.HashSize]byte{})
- } else {
- cfg, err := config.Load(cdir)
- if err != nil {
- return "", err
- }
- h := cache.NewHash(config.ConfigName)
- if _, err := h.Write([]byte(cfg.String())); err != nil {
- return "", err
- }
- fmt.Fprintf(key, "file %s %x\n", config.ConfigName, h.Sum())
- }
-
- imps := make([]*Package, len(pkg.Imports))
- copy(imps, pkg.Imports)
- sort.Slice(imps, func(i, j int) bool {
- return imps[i].PkgPath < imps[j].PkgPath
- })
- for _, dep := range imps {
- if dep.PkgPath == "unsafe" {
- continue
- }
-
- fmt.Fprintf(key, "import %s %s\n", dep.PkgPath, dep.hash)
- }
- h := key.Sum()
- return hex.EncodeToString(h[:]), nil
-}
-
-func packageActionID(pkg *Package) cache.ActionID {
- key := cache.NewHash("package ID")
- fmt.Fprintf(key, "pkgpath %s\n", pkg.PkgPath)
- fmt.Fprintf(key, "pkghash %s\n", pkg.hash)
- return key.Sum()
-}
-
-// passActionID computes an ActionID for an analysis pass.
-func passActionID(pkg *Package, analyzer *analysis.Analyzer) cache.ActionID {
- return cache.Subkey(pkg.actionID, fmt.Sprintf("analyzer %s", analyzer.Name))
-}
projects / dotfiles / .git / blobdiff