--- /dev/null
+package lint
+
+/*
+Package loading
+
+Conceptually, package loading in the runner can be imagined as a
+graph-shaped work list. We iteratively pop off leaf nodes (packages
+that have no unloaded dependencies) and load data from export data,
+our cache, or source.
+
+Specifically, non-initial packages are loaded from export data and the
+fact cache if possible, otherwise from source. Initial packages are
+loaded from export data, the fact cache and the (problems, ignores,
+config) cache if possible, otherwise from source.
+
+The appeal of this approach is that it is both simple to implement and
+easily parallelizable. Each leaf node can be processed independently,
+and new leaf nodes appear as their dependencies are being processed.
+
+The downside of this approach, however, is that we're doing more work
+than necessary. Imagine an initial package A, which has the following
+dependency chain: A->B->C->D – in the current implementation, we will
+load all 4 packages. However, if package A can be loaded fully from
+cached information, then none of its dependencies are necessary, and
+we could avoid loading them.
+
+
+Parallelism
+
+Runner implements parallel processing of packages by spawning one
+goroutine per package in the dependency graph, without any semaphores.
+Each goroutine initially waits on the completion of all of its
+dependencies, thus establishing correct order of processing. Once all
+dependencies finish processing, the goroutine will load the package
+from export data or source – this loading is guarded by a semaphore,
+sized according to the number of CPU cores. This way, we only have as
+many packages occupying memory and CPU resources as there are actual
+cores to process them.
+
+This combination of unbounded goroutines but bounded package loading
+means that if we have many parallel, independent subgraphs, they will
+all execute in parallel, while not wasting resources for long linear
+chains or trying to process more subgraphs in parallel than the system
+can handle.
+
+
+Caching
+
+We make use of several caches. These caches are Go's export data, our
+facts cache, and our (problems, ignores, config) cache.
+
+Initial packages will either be loaded from a combination of all three
+caches, or from source. Non-initial packages will either be loaded
+from a combination of export data and facts cache, or from source.
+
+The facts cache is separate from the (problems, ignores, config) cache
+because when we process non-initial packages, we generate facts, but
+we discard problems and ignores.
+
+The facts cache is keyed by (package, analyzer), whereas the
+(problems, ignores, config) cache is keyed by (package, list of
+analyzes). The difference between the two exists because there are
+only a handful of analyses that produce facts, but hundreds of
+analyses that don't. Creating one cache entry per fact-generating
+analysis is feasible, creating one cache entry per normal analysis has
+significant performance and storage overheads.
+
+The downside of keying by the list of analyzes is, naturally, that a
+change in list of analyzes changes the cache key. `staticcheck -checks
+A` and `staticcheck -checks A,B` will therefore need their own cache
+entries and not reuse each other's work. This problem does not affect
+the facts cache.
+
+*/
+
+import (
+ "bytes"
+ "encoding/gob"
+ "encoding/hex"
+ "fmt"
+ "go/ast"
+ "go/token"
+ "go/types"
+ "reflect"
+ "regexp"
+ "runtime"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "golang.org/x/tools/go/analysis"
+ "golang.org/x/tools/go/packages"
+ "golang.org/x/tools/go/types/objectpath"
+ "honnef.co/go/tools/config"
+ "honnef.co/go/tools/facts"
+ "honnef.co/go/tools/internal/cache"
+ "honnef.co/go/tools/loader"
+)
+
+func init() {
+ gob.Register(&FileIgnore{})
+ gob.Register(&LineIgnore{})
+}
+
+// If enabled, abuse of the go/analysis API will lead to panics
+const sanityCheck = true
+
+// OPT(dh): for a dependency tree A->B->C->D, if we have cached data
+// for B, there should be no need to load C and D individually. Go's
+// export data for B contains all the data we need on types, and our
+// fact cache could store the union of B, C and D in B.
+//
+// This may change unused's behavior, however, as it may observe fewer
+// interfaces from transitive dependencies.
+
+// OPT(dh): every single package will have the same value for
+// canClearTypes. We could move the Package.decUse method to runner to
+// eliminate this field. This is probably not worth it, though. There
+// are only thousands of packages, so the field only takes up
+// kilobytes of memory.
+
+// OPT(dh): do we really need the Package.gen field? it's based
+// trivially on pkg.results and merely caches the result of a type
+// assertion. How often do we actually use the field?
+
+type Package struct {
+ // dependents is initially set to 1 plus the number of packages
+ // that directly import this package. It is atomically decreased
+ // by 1 every time a dependent has been processed or when the
+ // package itself has been processed. Once the value reaches zero,
+ // the package is no longer needed.
+ dependents uint64
+
+ *packages.Package
+ Imports []*Package
+ initial bool
+ // fromSource is set to true for packages that have been loaded
+ // from source. This is the case for initial packages, packages
+ // with missing export data, and packages with no cached facts.
+ fromSource bool
+ // hash stores the package hash, as computed by packageHash
+ hash string
+ actionID cache.ActionID
+ done chan struct{}
+
+ resultsMu sync.Mutex
+ // results maps analyzer IDs to analyzer results. it is
+ // implemented as a deduplicating concurrent cache.
+ results []*result
+
+ cfg *config.Config
+ // gen maps file names to the code generator that created them
+ gen map[string]facts.Generator
+ problems []Problem
+ ignores []Ignore
+ errs []error
+
+ // these slices are indexed by analysis
+ facts []map[types.Object][]analysis.Fact
+ pkgFacts [][]analysis.Fact
+
+ // canClearTypes is set to true if we can discard type
+ // information after the package and its dependents have been
+ // processed. This is the case when no cumulative checkers are
+ // being run.
+ canClearTypes bool
+}
+
+type cachedPackage struct {
+ Problems []Problem
+ Ignores []Ignore
+ Config *config.Config
+}
+
+func (pkg *Package) decUse() {
+ ret := atomic.AddUint64(&pkg.dependents, ^uint64(0))
+ if ret == 0 {
+ // nobody depends on this package anymore
+ if pkg.canClearTypes {
+ pkg.Types = nil
+ }
+ pkg.facts = nil
+ pkg.pkgFacts = nil
+
+ for _, imp := range pkg.Imports {
+ imp.decUse()
+ }
+ }
+}
+
+type result struct {
+ v interface{}
+ err error
+ ready chan struct{}
+}
+
+type Runner struct {
+ cache *cache.Cache
+ goVersion int
+ stats *Stats
+ repeatAnalyzers uint
+
+ analyzerIDs analyzerIDs
+ problemsCacheKey string
+
+ // limits parallelism of loading packages
+ loadSem chan struct{}
+}
+
+type analyzerIDs struct {
+ m map[*analysis.Analyzer]int
+}
+
+func (ids analyzerIDs) get(a *analysis.Analyzer) int {
+ id, ok := ids.m[a]
+ if !ok {
+ panic(fmt.Sprintf("no analyzer ID for %s", a.Name))
+ }
+ return id
+}
+
+type Fact struct {
+ Path string
+ Fact analysis.Fact
+}
+
+type analysisAction struct {
+ analyzer *analysis.Analyzer
+ analyzerID int
+ pkg *Package
+ newPackageFacts []analysis.Fact
+ problems []Problem
+
+ pkgFacts map[*types.Package][]analysis.Fact
+}
+
+func (ac *analysisAction) String() string {
+ return fmt.Sprintf("%s @ %s", ac.analyzer, ac.pkg)
+}
+
+func (ac *analysisAction) allObjectFacts() []analysis.ObjectFact {
+ out := make([]analysis.ObjectFact, 0, len(ac.pkg.facts[ac.analyzerID]))
+ for obj, facts := range ac.pkg.facts[ac.analyzerID] {
+ for _, fact := range facts {
+ out = append(out, analysis.ObjectFact{
+ Object: obj,
+ Fact: fact,
+ })
+ }
+ }
+ return out
+}
+
+func (ac *analysisAction) allPackageFacts() []analysis.PackageFact {
+ out := make([]analysis.PackageFact, 0, len(ac.pkgFacts))
+ for pkg, facts := range ac.pkgFacts {
+ for _, fact := range facts {
+ out = append(out, analysis.PackageFact{
+ Package: pkg,
+ Fact: fact,
+ })
+ }
+ }
+ return out
+}
+
+func (ac *analysisAction) importObjectFact(obj types.Object, fact analysis.Fact) bool {
+ if sanityCheck && len(ac.analyzer.FactTypes) == 0 {
+ panic("analysis doesn't export any facts")
+ }
+ for _, f := range ac.pkg.facts[ac.analyzerID][obj] {
+ if reflect.TypeOf(f) == reflect.TypeOf(fact) {
+ reflect.ValueOf(fact).Elem().Set(reflect.ValueOf(f).Elem())
+ return true
+ }
+ }
+ return false
+}
+
+func (ac *analysisAction) importPackageFact(pkg *types.Package, fact analysis.Fact) bool {
+ if sanityCheck && len(ac.analyzer.FactTypes) == 0 {
+ panic("analysis doesn't export any facts")
+ }
+ for _, f := range ac.pkgFacts[pkg] {
+ if reflect.TypeOf(f) == reflect.TypeOf(fact) {
+ reflect.ValueOf(fact).Elem().Set(reflect.ValueOf(f).Elem())
+ return true
+ }
+ }
+ return false
+}
+
+func (ac *analysisAction) exportObjectFact(obj types.Object, fact analysis.Fact) {
+ if sanityCheck && len(ac.analyzer.FactTypes) == 0 {
+ panic("analysis doesn't export any facts")
+ }
+ ac.pkg.facts[ac.analyzerID][obj] = append(ac.pkg.facts[ac.analyzerID][obj], fact)
+}
+
+func (ac *analysisAction) exportPackageFact(fact analysis.Fact) {
+ if sanityCheck && len(ac.analyzer.FactTypes) == 0 {
+ panic("analysis doesn't export any facts")
+ }
+ ac.pkgFacts[ac.pkg.Types] = append(ac.pkgFacts[ac.pkg.Types], fact)
+ ac.newPackageFacts = append(ac.newPackageFacts, fact)
+}
+
+func (ac *analysisAction) report(pass *analysis.Pass, d analysis.Diagnostic) {
+ p := Problem{
+ Pos: DisplayPosition(pass.Fset, d.Pos),
+ End: DisplayPosition(pass.Fset, d.End),
+ Message: d.Message,
+ Check: pass.Analyzer.Name,
+ }
+ for _, r := range d.Related {
+ p.Related = append(p.Related, Related{
+ Pos: DisplayPosition(pass.Fset, r.Pos),
+ End: DisplayPosition(pass.Fset, r.End),
+ Message: r.Message,
+ })
+ }
+ ac.problems = append(ac.problems, p)
+}
+
+func (r *Runner) runAnalysis(ac *analysisAction) (ret interface{}, err error) {
+ ac.pkg.resultsMu.Lock()
+ res := ac.pkg.results[r.analyzerIDs.get(ac.analyzer)]
+ if res != nil {
+ ac.pkg.resultsMu.Unlock()
+ <-res.ready
+ return res.v, res.err
+ } else {
+ res = &result{
+ ready: make(chan struct{}),
+ }
+ ac.pkg.results[r.analyzerIDs.get(ac.analyzer)] = res
+ ac.pkg.resultsMu.Unlock()
+
+ defer func() {
+ res.v = ret
+ res.err = err
+ close(res.ready)
+ }()
+
+ pass := new(analysis.Pass)
+ *pass = analysis.Pass{
+ Analyzer: ac.analyzer,
+ Fset: ac.pkg.Fset,
+ Files: ac.pkg.Syntax,
+ // type information may be nil or may be populated. if it is
+ // nil, it will get populated later.
+ Pkg: ac.pkg.Types,
+ TypesInfo: ac.pkg.TypesInfo,
+ TypesSizes: ac.pkg.TypesSizes,
+ ResultOf: map[*analysis.Analyzer]interface{}{},
+ ImportObjectFact: ac.importObjectFact,
+ ImportPackageFact: ac.importPackageFact,
+ ExportObjectFact: ac.exportObjectFact,
+ ExportPackageFact: ac.exportPackageFact,
+ Report: func(d analysis.Diagnostic) {
+ ac.report(pass, d)
+ },
+ AllObjectFacts: ac.allObjectFacts,
+ AllPackageFacts: ac.allPackageFacts,
+ }
+
+ if !ac.pkg.initial {
+ // Don't report problems in dependencies
+ pass.Report = func(analysis.Diagnostic) {}
+ }
+ return r.runAnalysisUser(pass, ac)
+ }
+}
+
+func (r *Runner) loadCachedPackage(pkg *Package, analyzers []*analysis.Analyzer) (cachedPackage, bool) {
+ // OPT(dh): we can cache this computation, it'll be the same for all packages
+ id := cache.Subkey(pkg.actionID, "data "+r.problemsCacheKey)
+
+ b, _, err := r.cache.GetBytes(id)
+ if err != nil {
+ return cachedPackage{}, false
+ }
+ var cpkg cachedPackage
+ if err := gob.NewDecoder(bytes.NewReader(b)).Decode(&cpkg); err != nil {
+ return cachedPackage{}, false
+ }
+ return cpkg, true
+}
+
+func (r *Runner) loadCachedFacts(a *analysis.Analyzer, pkg *Package) ([]Fact, bool) {
+ if len(a.FactTypes) == 0 {
+ return nil, true
+ }
+
+ var facts []Fact
+ // Look in the cache for facts
+ aID := passActionID(pkg, a)
+ aID = cache.Subkey(aID, "facts")
+ b, _, err := r.cache.GetBytes(aID)
+ if err != nil {
+ // No cached facts, analyse this package like a user-provided one, but ignore diagnostics
+ return nil, false
+ }
+
+ if err := gob.NewDecoder(bytes.NewReader(b)).Decode(&facts); err != nil {
+ // Cached facts are broken, analyse this package like a user-provided one, but ignore diagnostics
+ return nil, false
+ }
+ return facts, true
+}
+
+type dependencyError struct {
+ dep string
+ err error
+}
+
+func (err dependencyError) nested() dependencyError {
+ if o, ok := err.err.(dependencyError); ok {
+ return o.nested()
+ }
+ return err
+}
+
+func (err dependencyError) Error() string {
+ if o, ok := err.err.(dependencyError); ok {
+ return o.Error()
+ }
+ return fmt.Sprintf("error running dependency %s: %s", err.dep, err.err)
+}
+
+func (r *Runner) makeAnalysisAction(a *analysis.Analyzer, pkg *Package) *analysisAction {
+ aid := r.analyzerIDs.get(a)
+ ac := &analysisAction{
+ analyzer: a,
+ analyzerID: aid,
+ pkg: pkg,
+ }
+
+ if len(a.FactTypes) == 0 {
+ return ac
+ }
+
+ // Merge all package facts of dependencies
+ ac.pkgFacts = map[*types.Package][]analysis.Fact{}
+ seen := map[*Package]struct{}{}
+ var dfs func(*Package)
+ dfs = func(pkg *Package) {
+ if _, ok := seen[pkg]; ok {
+ return
+ }
+ seen[pkg] = struct{}{}
+ s := pkg.pkgFacts[aid]
+ ac.pkgFacts[pkg.Types] = s[0:len(s):len(s)]
+ for _, imp := range pkg.Imports {
+ dfs(imp)
+ }
+ }
+ dfs(pkg)
+
+ return ac
+}
+
+// analyzes that we always want to run, even if they're not being run
+// explicitly or as dependencies. these are necessary for the inner
+// workings of the runner.
+var injectedAnalyses = []*analysis.Analyzer{facts.Generated, config.Analyzer}
+
+func (r *Runner) runAnalysisUser(pass *analysis.Pass, ac *analysisAction) (interface{}, error) {
+ if !ac.pkg.fromSource {
+ panic(fmt.Sprintf("internal error: %s was not loaded from source", ac.pkg))
+ }
+
+ // User-provided package, analyse it
+ // First analyze it with dependencies
+ for _, req := range ac.analyzer.Requires {
+ acReq := r.makeAnalysisAction(req, ac.pkg)
+ ret, err := r.runAnalysis(acReq)
+ if err != nil {
+ // We couldn't run a dependency, no point in going on
+ return nil, dependencyError{req.Name, err}
+ }
+
+ pass.ResultOf[req] = ret
+ }
+
+ // Then with this analyzer
+ var ret interface{}
+ for i := uint(0); i < r.repeatAnalyzers+1; i++ {
+ var err error
+ t := time.Now()
+ ret, err = ac.analyzer.Run(pass)
+ r.stats.MeasureAnalyzer(ac.analyzer, ac.pkg, time.Since(t))
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ if len(ac.analyzer.FactTypes) > 0 {
+ // Merge new facts into the package and persist them.
+ var facts []Fact
+ for _, fact := range ac.newPackageFacts {
+ id := r.analyzerIDs.get(ac.analyzer)
+ ac.pkg.pkgFacts[id] = append(ac.pkg.pkgFacts[id], fact)
+ facts = append(facts, Fact{"", fact})
+ }
+ for obj, afacts := range ac.pkg.facts[ac.analyzerID] {
+ if obj.Pkg() != ac.pkg.Package.Types {
+ continue
+ }
+ path, err := objectpath.For(obj)
+ if err != nil {
+ continue
+ }
+ for _, fact := range afacts {
+ facts = append(facts, Fact{string(path), fact})
+ }
+ }
+
+ if err := r.cacheData(facts, ac.pkg, ac.analyzer, "facts"); err != nil {
+ return nil, err
+ }
+ }
+
+ return ret, nil
+}
+
+func (r *Runner) cacheData(v interface{}, pkg *Package, a *analysis.Analyzer, subkey string) error {
+ buf := &bytes.Buffer{}
+ if err := gob.NewEncoder(buf).Encode(v); err != nil {
+ return err
+ }
+ aID := passActionID(pkg, a)
+ aID = cache.Subkey(aID, subkey)
+ if err := r.cache.PutBytes(aID, buf.Bytes()); err != nil {
+ return err
+ }
+ return nil
+}
+
+func NewRunner(stats *Stats) (*Runner, error) {
+ cache, err := cache.Default()
+ if err != nil {
+ return nil, err
+ }
+
+ return &Runner{
+ cache: cache,
+ stats: stats,
+ }, nil
+}
+
+// Run loads packages corresponding to patterns and analyses them with
+// analyzers. It returns the loaded packages, which contain reported
+// diagnostics as well as extracted ignore directives.
+//
+// Note that diagnostics have not been filtered at this point yet, to
+// accommodate cumulative analyzes that require additional steps to
+// produce diagnostics.
+func (r *Runner) Run(cfg *packages.Config, patterns []string, analyzers []*analysis.Analyzer, hasCumulative bool) ([]*Package, error) {
+ checkerNames := make([]string, len(analyzers))
+ for i, a := range analyzers {
+ checkerNames[i] = a.Name
+ }
+ sort.Strings(checkerNames)
+ r.problemsCacheKey = strings.Join(checkerNames, " ")
+
+ var allAnalyzers []*analysis.Analyzer
+ r.analyzerIDs = analyzerIDs{m: map[*analysis.Analyzer]int{}}
+ id := 0
+ seen := map[*analysis.Analyzer]struct{}{}
+ var dfs func(a *analysis.Analyzer)
+ dfs = func(a *analysis.Analyzer) {
+ if _, ok := seen[a]; ok {
+ return
+ }
+ seen[a] = struct{}{}
+ allAnalyzers = append(allAnalyzers, a)
+ r.analyzerIDs.m[a] = id
+ id++
+ for _, f := range a.FactTypes {
+ gob.Register(f)
+ }
+ for _, req := range a.Requires {
+ dfs(req)
+ }
+ }
+ for _, a := range analyzers {
+ if v := a.Flags.Lookup("go"); v != nil {
+ v.Value.Set(fmt.Sprintf("1.%d", r.goVersion))
+ }
+ dfs(a)
+ }
+ for _, a := range injectedAnalyses {
+ dfs(a)
+ }
+ // Run all analyzers on all packages (subject to further
+ // restrictions enforced later). This guarantees that if analyzer
+ // A1 depends on A2, and A2 has facts, that A2 will run on the
+ // dependencies of user-provided packages, even though A1 won't.
+ analyzers = allAnalyzers
+
+ var dcfg packages.Config
+ if cfg != nil {
+ dcfg = *cfg
+ }
+
+ atomic.StoreUint32(&r.stats.State, StateGraph)
+ initialPkgs, err := loader.Graph(dcfg, patterns...)
+ if err != nil {
+ return nil, err
+ }
+ defer r.cache.Trim()
+
+ var allPkgs []*Package
+ m := map[*packages.Package]*Package{}
+ packages.Visit(initialPkgs, nil, func(l *packages.Package) {
+ m[l] = &Package{
+ Package: l,
+ results: make([]*result, len(r.analyzerIDs.m)),
+ facts: make([]map[types.Object][]analysis.Fact, len(r.analyzerIDs.m)),
+ pkgFacts: make([][]analysis.Fact, len(r.analyzerIDs.m)),
+ done: make(chan struct{}),
+ // every package needs itself
+ dependents: 1,
+ canClearTypes: !hasCumulative,
+ }
+ allPkgs = append(allPkgs, m[l])
+ for i := range m[l].facts {
+ m[l].facts[i] = map[types.Object][]analysis.Fact{}
+ }
+ for _, err := range l.Errors {
+ m[l].errs = append(m[l].errs, err)
+ }
+ for _, v := range l.Imports {
+ m[v].dependents++
+ m[l].Imports = append(m[l].Imports, m[v])
+ }
+
+ m[l].hash, err = r.packageHash(m[l])
+ m[l].actionID = packageActionID(m[l])
+ if err != nil {
+ m[l].errs = append(m[l].errs, err)
+ }
+ })
+
+ pkgs := make([]*Package, len(initialPkgs))
+ for i, l := range initialPkgs {
+ pkgs[i] = m[l]
+ pkgs[i].initial = true
+ }
+
+ atomic.StoreUint32(&r.stats.InitialPackages, uint32(len(initialPkgs)))
+ atomic.StoreUint32(&r.stats.TotalPackages, uint32(len(allPkgs)))
+ atomic.StoreUint32(&r.stats.State, StateProcessing)
+
+ var wg sync.WaitGroup
+ wg.Add(len(allPkgs))
+ r.loadSem = make(chan struct{}, runtime.GOMAXPROCS(-1))
+ atomic.StoreUint32(&r.stats.TotalWorkers, uint32(cap(r.loadSem)))
+ for _, pkg := range allPkgs {
+ pkg := pkg
+ go func() {
+ r.processPkg(pkg, analyzers)
+
+ if pkg.initial {
+ atomic.AddUint32(&r.stats.ProcessedInitialPackages, 1)
+ }
+ atomic.AddUint32(&r.stats.Problems, uint32(len(pkg.problems)))
+ wg.Done()
+ }()
+ }
+ wg.Wait()
+
+ return pkgs, nil
+}
+
+var posRe = regexp.MustCompile(`^(.+?):(\d+)(?::(\d+)?)?`)
+
+func parsePos(pos string) (token.Position, int, error) {
+ if pos == "-" || pos == "" {
+ return token.Position{}, 0, nil
+ }
+ parts := posRe.FindStringSubmatch(pos)
+ if parts == nil {
+ return token.Position{}, 0, fmt.Errorf("malformed position %q", pos)
+ }
+ file := parts[1]
+ line, _ := strconv.Atoi(parts[2])
+ col, _ := strconv.Atoi(parts[3])
+ return token.Position{
+ Filename: file,
+ Line: line,
+ Column: col,
+ }, len(parts[0]), nil
+}
+
+// loadPkg loads a Go package. It may be loaded from a combination of
+// caches, or from source.
+func (r *Runner) loadPkg(pkg *Package, analyzers []*analysis.Analyzer) error {
+ if pkg.Types != nil {
+ panic(fmt.Sprintf("internal error: %s has already been loaded", pkg.Package))
+ }
+
+ if pkg.initial {
+ // Try to load cached package
+ cpkg, ok := r.loadCachedPackage(pkg, analyzers)
+ if ok {
+ pkg.problems = cpkg.Problems
+ pkg.ignores = cpkg.Ignores
+ pkg.cfg = cpkg.Config
+ } else {
+ pkg.fromSource = true
+ return loader.LoadFromSource(pkg.Package)
+ }
+ }
+
+ // At this point we're either working with a non-initial package,
+ // or we managed to load cached problems for the package. We still
+ // need export data and facts.
+
+ // OPT(dh): we don't need type information for this package if no
+ // other package depends on it. this may be the case for initial
+ // packages.
+
+ // Load package from export data
+ if err := loader.LoadFromExport(pkg.Package); err != nil {
+ // We asked Go to give us up to date export data, yet
+ // we can't load it. There must be something wrong.
+ //
+ // Attempt loading from source. This should fail (because
+ // otherwise there would be export data); we just want to
+ // get the compile errors. If loading from source succeeds
+ // we discard the result, anyway. Otherwise we'll fail
+ // when trying to reload from export data later.
+ //
+ // FIXME(dh): we no longer reload from export data, so
+ // theoretically we should be able to continue
+ pkg.fromSource = true
+ if err := loader.LoadFromSource(pkg.Package); err != nil {
+ return err
+ }
+ // Make sure this package can't be imported successfully
+ pkg.Package.Errors = append(pkg.Package.Errors, packages.Error{
+ Pos: "-",
+ Msg: fmt.Sprintf("could not load export data: %s", err),
+ Kind: packages.ParseError,
+ })
+ return fmt.Errorf("could not load export data: %s", err)
+ }
+
+ failed := false
+ seen := make([]bool, len(r.analyzerIDs.m))
+ var dfs func(*analysis.Analyzer)
+ dfs = func(a *analysis.Analyzer) {
+ if seen[r.analyzerIDs.get(a)] {
+ return
+ }
+ seen[r.analyzerIDs.get(a)] = true
+
+ if len(a.FactTypes) > 0 {
+ facts, ok := r.loadCachedFacts(a, pkg)
+ if !ok {
+ failed = true
+ return
+ }
+
+ for _, f := range facts {
+ if f.Path == "" {
+ // This is a package fact
+ pkg.pkgFacts[r.analyzerIDs.get(a)] = append(pkg.pkgFacts[r.analyzerIDs.get(a)], f.Fact)
+ continue
+ }
+ obj, err := objectpath.Object(pkg.Types, objectpath.Path(f.Path))
+ if err != nil {
+ // Be lenient about these errors. For example, when
+ // analysing io/ioutil from source, we may get a fact
+ // for methods on the devNull type, and objectpath
+ // will happily create a path for them. However, when
+ // we later load io/ioutil from export data, the path
+ // no longer resolves.
+ //
+ // If an exported type embeds the unexported type,
+ // then (part of) the unexported type will become part
+ // of the type information and our path will resolve
+ // again.
+ continue
+ }
+ pkg.facts[r.analyzerIDs.get(a)][obj] = append(pkg.facts[r.analyzerIDs.get(a)][obj], f.Fact)
+ }
+ }
+
+ for _, req := range a.Requires {
+ dfs(req)
+ }
+ }
+ for _, a := range analyzers {
+ dfs(a)
+ }
+
+ if !failed {
+ return nil
+ }
+
+ // We failed to load some cached facts
+ pkg.fromSource = true
+ // XXX we added facts to the maps, we need to get rid of those
+ return loader.LoadFromSource(pkg.Package)
+}
+
+type analysisError struct {
+ analyzer *analysis.Analyzer
+ pkg *Package
+ err error
+}
+
+func (err analysisError) Error() string {
+ return fmt.Sprintf("error running analyzer %s on %s: %s", err.analyzer, err.pkg, err.err)
+}
+
+// processPkg processes a package. This involves loading the package,
+// either from export data or from source. For packages loaded from
+// source, the provides analyzers will be run on the package.
+func (r *Runner) processPkg(pkg *Package, analyzers []*analysis.Analyzer) {
+ defer func() {
+ // Clear information we no longer need. Make sure to do this
+ // when returning from processPkg so that we clear
+ // dependencies, not just initial packages.
+ pkg.TypesInfo = nil
+ pkg.Syntax = nil
+ pkg.results = nil
+
+ atomic.AddUint32(&r.stats.ProcessedPackages, 1)
+ pkg.decUse()
+ close(pkg.done)
+ }()
+
+ // Ensure all packages have the generated map and config. This is
+ // required by internals of the runner. Analyses that themselves
+ // make use of either have an explicit dependency so that other
+ // runners work correctly, too.
+ analyzers = append(analyzers[0:len(analyzers):len(analyzers)], injectedAnalyses...)
+
+ if len(pkg.errs) != 0 {
+ return
+ }
+
+ for _, imp := range pkg.Imports {
+ <-imp.done
+ if len(imp.errs) > 0 {
+ if imp.initial {
+ // Don't print the error of the dependency since it's
+ // an initial package and we're already printing the
+ // error.
+ pkg.errs = append(pkg.errs, fmt.Errorf("could not analyze dependency %s of %s", imp, pkg))
+ } else {
+ var s string
+ for _, err := range imp.errs {
+ s += "\n\t" + err.Error()
+ }
+ pkg.errs = append(pkg.errs, fmt.Errorf("could not analyze dependency %s of %s: %s", imp, pkg, s))
+ }
+ return
+ }
+ }
+ if pkg.PkgPath == "unsafe" {
+ pkg.Types = types.Unsafe
+ return
+ }
+
+ r.loadSem <- struct{}{}
+ atomic.AddUint32(&r.stats.ActiveWorkers, 1)
+ defer func() {
+ <-r.loadSem
+ atomic.AddUint32(&r.stats.ActiveWorkers, ^uint32(0))
+ }()
+ if err := r.loadPkg(pkg, analyzers); err != nil {
+ pkg.errs = append(pkg.errs, err)
+ return
+ }
+
+ // A package's object facts is the union of all of its dependencies.
+ for _, imp := range pkg.Imports {
+ for ai, m := range imp.facts {
+ for obj, facts := range m {
+ pkg.facts[ai][obj] = facts[0:len(facts):len(facts)]
+ }
+ }
+ }
+
+ if !pkg.fromSource {
+ // Nothing left to do for the package.
+ return
+ }
+
+ // Run analyses on initial packages and those missing facts
+ var wg sync.WaitGroup
+ wg.Add(len(analyzers))
+ errs := make([]error, len(analyzers))
+ var acs []*analysisAction
+ for i, a := range analyzers {
+ i := i
+ a := a
+ ac := r.makeAnalysisAction(a, pkg)
+ acs = append(acs, ac)
+ go func() {
+ defer wg.Done()
+ // Only initial packages and packages with missing
+ // facts will have been loaded from source.
+ if pkg.initial || len(a.FactTypes) > 0 {
+ if _, err := r.runAnalysis(ac); err != nil {
+ errs[i] = analysisError{a, pkg, err}
+ return
+ }
+ }
+ }()
+ }
+ wg.Wait()
+
+ depErrors := map[dependencyError]int{}
+ for _, err := range errs {
+ if err == nil {
+ continue
+ }
+ switch err := err.(type) {
+ case analysisError:
+ switch err := err.err.(type) {
+ case dependencyError:
+ depErrors[err.nested()]++
+ default:
+ pkg.errs = append(pkg.errs, err)
+ }
+ default:
+ pkg.errs = append(pkg.errs, err)
+ }
+ }
+ for err, count := range depErrors {
+ pkg.errs = append(pkg.errs,
+ fmt.Errorf("could not run %s@%s, preventing %d analyzers from running: %s", err.dep, pkg, count, err.err))
+ }
+
+ // We can't process ignores at this point because `unused` needs
+ // to see more than one package to make its decision.
+ //
+ // OPT(dh): can't we guard this block of code by pkg.initial?
+ ignores, problems := parseDirectives(pkg.Package)
+ pkg.ignores = append(pkg.ignores, ignores...)
+ pkg.problems = append(pkg.problems, problems...)
+ for _, ac := range acs {
+ pkg.problems = append(pkg.problems, ac.problems...)
+ }
+
+ if pkg.initial {
+ // Only initial packages have these analyzers run, and only
+ // initial packages need these.
+ if pkg.results[r.analyzerIDs.get(config.Analyzer)].v != nil {
+ pkg.cfg = pkg.results[r.analyzerIDs.get(config.Analyzer)].v.(*config.Config)
+ }
+ pkg.gen = pkg.results[r.analyzerIDs.get(facts.Generated)].v.(map[string]facts.Generator)
+ }
+
+ // In a previous version of the code, we would throw away all type
+ // information and reload it from export data. That was
+ // nonsensical. The *types.Package doesn't keep any information
+ // live that export data wouldn't also. We only need to discard
+ // the AST and the TypesInfo maps; that happens after we return
+ // from processPkg.
+}
+
+func parseDirective(s string) (cmd string, args []string) {
+ if !strings.HasPrefix(s, "//lint:") {
+ return "", nil
+ }
+ s = strings.TrimPrefix(s, "//lint:")
+ fields := strings.Split(s, " ")
+ return fields[0], fields[1:]
+}
+
+// parseDirectives extracts all linter directives from the source
+// files of the package. Malformed directives are returned as problems.
+func parseDirectives(pkg *packages.Package) ([]Ignore, []Problem) {
+ var ignores []Ignore
+ var problems []Problem
+
+ for _, f := range pkg.Syntax {
+ found := false
+ commentLoop:
+ for _, cg := range f.Comments {
+ for _, c := range cg.List {
+ if strings.Contains(c.Text, "//lint:") {
+ found = true
+ break commentLoop
+ }
+ }
+ }
+ if !found {
+ continue
+ }
+ cm := ast.NewCommentMap(pkg.Fset, f, f.Comments)
+ for node, cgs := range cm {
+ for _, cg := range cgs {
+ for _, c := range cg.List {
+ if !strings.HasPrefix(c.Text, "//lint:") {
+ continue
+ }
+ cmd, args := parseDirective(c.Text)
+ switch cmd {
+ case "ignore", "file-ignore":
+ if len(args) < 2 {
+ p := Problem{
+ Pos: DisplayPosition(pkg.Fset, c.Pos()),
+ Message: "malformed linter directive; missing the required reason field?",
+ Severity: Error,
+ Check: "compile",
+ }
+ problems = append(problems, p)
+ continue
+ }
+ default:
+ // unknown directive, ignore
+ continue
+ }
+ checks := strings.Split(args[0], ",")
+ pos := DisplayPosition(pkg.Fset, node.Pos())
+ var ig Ignore
+ switch cmd {
+ case "ignore":
+ ig = &LineIgnore{
+ File: pos.Filename,
+ Line: pos.Line,
+ Checks: checks,
+ Pos: DisplayPosition(pkg.Fset, c.Pos()),
+ }
+ case "file-ignore":
+ ig = &FileIgnore{
+ File: pos.Filename,
+ Checks: checks,
+ }
+ }
+ ignores = append(ignores, ig)
+ }
+ }
+ }
+ }
+
+ return ignores, problems
+}
+
+// packageHash computes a package's hash. The hash is based on all Go
+// files that make up the package, as well as the hashes of imported
+// packages.
+func (r *Runner) packageHash(pkg *Package) (string, error) {
+ key := cache.NewHash("package hash")
+ fmt.Fprintf(key, "pkgpath %s\n", pkg.PkgPath)
+ fmt.Fprintf(key, "go %d\n", r.goVersion)
+ for _, f := range pkg.CompiledGoFiles {
+ h, err := cache.FileHash(f)
+ if err != nil {
+ return "", err
+ }
+ fmt.Fprintf(key, "file %s %x\n", f, h)
+ }
+
+ // Actually load the configuration to calculate its hash. This
+ // will take into consideration inheritance of configuration
+ // files, as well as the default configuration.
+ //
+ // OPT(dh): doing this means we'll load the config twice: once for
+ // computing the hash, and once when analyzing the package from
+ // source.
+ cdir := config.Dir(pkg.GoFiles)
+ if cdir == "" {
+ fmt.Fprintf(key, "file %s %x\n", config.ConfigName, [cache.HashSize]byte{})
+ } else {
+ cfg, err := config.Load(cdir)
+ if err != nil {
+ return "", err
+ }
+ h := cache.NewHash(config.ConfigName)
+ if _, err := h.Write([]byte(cfg.String())); err != nil {
+ return "", err
+ }
+ fmt.Fprintf(key, "file %s %x\n", config.ConfigName, h.Sum())
+ }
+
+ imps := make([]*Package, len(pkg.Imports))
+ copy(imps, pkg.Imports)
+ sort.Slice(imps, func(i, j int) bool {
+ return imps[i].PkgPath < imps[j].PkgPath
+ })
+ for _, dep := range imps {
+ if dep.PkgPath == "unsafe" {
+ continue
+ }
+
+ fmt.Fprintf(key, "import %s %s\n", dep.PkgPath, dep.hash)
+ }
+ h := key.Sum()
+ return hex.EncodeToString(h[:]), nil
+}
+
+func packageActionID(pkg *Package) cache.ActionID {
+ key := cache.NewHash("package ID")
+ fmt.Fprintf(key, "pkgpath %s\n", pkg.PkgPath)
+ fmt.Fprintf(key, "pkghash %s\n", pkg.hash)
+ return key.Sum()
+}
+
+// passActionID computes an ActionID for an analysis pass.
+func passActionID(pkg *Package, analyzer *analysis.Analyzer) cache.ActionID {
+ return cache.Subkey(pkg.actionID, fmt.Sprintf("analyzer %s", analyzer.Name))
+}
projects / dotfiles / .git / blobdiff