1 // Copyright 2015 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // Compilebench benchmarks the speed of the Go compiler.
9 // compilebench [options]
11 // It times the compilation of various packages and prints results in
12 // the format used by package testing (and expected by golang.org/x/perf/cmd/benchstat).
17 // Report allocations.
20 // Use exe as the path to the cmd/compile binary.
22 // -compileflags 'list'
23 // Pass the space-separated list of flags to the compilation.
26 // Use exe as the path to the cmd/link binary.
29 // Pass the space-separated list of flags to the linker.
32 // Run each benchmark n times (default 1).
35 // Write a CPU profile of the compiler to file.
38 // Path to "go" command (default "go").
41 // Write a memory profile of the compiler to file.
43 // -memprofilerate rate
44 // Set runtime.MemProfileRate during compilation.
47 // Report object file statistics.
50 // Benchmark compiling a single package.
53 // Only run benchmarks with names matching regexp.
56 // Skip long-running benchmarks.
58 // Although -cpuprofile and -memprofile are intended to write a
59 // combined profile for all the executed benchmarks to file,
60 // today they write only the profile for the last benchmark executed.
62 // The default memory profiling rate is one profile sample per 512 kB
63 // allocated (see ``go doc runtime.MemProfileRate'').
64 // Lowering the rate (for example, -memprofilerate 64000) produces
65 // a more fine-grained and therefore accurate profile, but it also incurs
66 // execution cost. For benchmark comparisons, never use timings
67 // obtained with a low -memprofilerate option.
71 // Assuming the base version of the compiler has been saved with
72 // ``toolstash save,'' this sequence compares the old and new compiler:
74 // compilebench -count 10 -compile $(toolstash -n compile) >old.txt
75 // compilebench -count 10 >new.txt
76 // benchstat old.txt new.txt
85 exec "golang.org/x/sys/execabs"
105 flagGoCmd = flag.String("go", "go", "path to \"go\" command")
106 flagAlloc = flag.Bool("alloc", false, "report allocations")
107 flagObj = flag.Bool("obj", false, "report object file stats")
108 flagCompiler = flag.String("compile", "", "use `exe` as the cmd/compile binary")
109 flagCompilerFlags = flag.String("compileflags", "", "additional `flags` to pass to compile")
110 flagLinker = flag.String("link", "", "use `exe` as the cmd/link binary")
111 flagLinkerFlags = flag.String("linkflags", "", "additional `flags` to pass to link")
112 flagRun = flag.String("run", "", "run benchmarks matching `regexp`")
113 flagCount = flag.Int("count", 1, "run benchmarks `n` times")
114 flagCpuprofile = flag.String("cpuprofile", "", "write CPU profile to `file`")
115 flagMemprofile = flag.String("memprofile", "", "write memory profile to `file`")
116 flagMemprofilerate = flag.Int64("memprofilerate", -1, "set memory profile `rate`")
117 flagPackage = flag.String("pkg", "", "if set, benchmark the package at path `pkg`")
118 flagShort = flag.Bool("short", false, "skip long-running benchmarks")
126 type runner interface {
128 run(name string, count int) error
132 {"BenchmarkTemplate", compile{"html/template"}},
133 {"BenchmarkUnicode", compile{"unicode"}},
134 {"BenchmarkGoTypes", compile{"go/types"}},
135 {"BenchmarkCompiler", compile{"cmd/compile/internal/gc"}},
136 {"BenchmarkSSA", compile{"cmd/compile/internal/ssa"}},
137 {"BenchmarkFlate", compile{"compress/flate"}},
138 {"BenchmarkGoParser", compile{"go/parser"}},
139 {"BenchmarkReflect", compile{"reflect"}},
140 {"BenchmarkTar", compile{"archive/tar"}},
141 {"BenchmarkXML", compile{"encoding/xml"}},
142 {"BenchmarkLinkCompiler", link{"cmd/compile", ""}},
143 {"BenchmarkExternalLinkCompiler", link{"cmd/compile", "-linkmode=external"}},
144 {"BenchmarkLinkWithoutDebugCompiler", link{"cmd/compile", "-w"}},
145 {"BenchmarkStdCmd", goBuild{[]string{"std", "cmd"}}},
146 {"BenchmarkHelloSize", size{"$GOROOT/test/helloworld.go", false}},
147 {"BenchmarkCmdGoSize", size{"cmd/go", true}},
151 fmt.Fprintf(os.Stderr, "usage: compilebench [options]\n")
152 fmt.Fprintf(os.Stderr, "options:\n")
159 log.SetPrefix("compilebench: ")
162 if flag.NArg() != 0 {
166 s, err := exec.Command(*flagGoCmd, "env", "GOROOT").CombinedOutput()
168 log.Fatalf("%s env GOROOT: %v", *flagGoCmd, err)
170 goroot = strings.TrimSpace(string(s))
171 os.Setenv("GOROOT", goroot) // for any subcommands
173 compiler = *flagCompiler
176 foundTool, compiler = toolPath("compile", "6g")
177 if foundTool == "6g" {
183 if linker == "" && !is6g { // TODO: Support 6l
184 _, linker = toolPath("link")
188 *flagMemprofilerate = -1
195 r, err := regexp.Compile(*flagRun)
197 log.Fatalf("invalid -run argument: %v", err)
202 if *flagPackage != "" {
204 {"BenchmarkPkg", compile{*flagPackage}},
205 {"BenchmarkPkgLink", link{*flagPackage, ""}},
210 for i := 0; i < *flagCount; i++ {
211 for _, tt := range tests {
212 if tt.r.long() && *flagShort {
215 if runRE == nil || runRE.MatchString(tt.name) {
216 if err := tt.r.run(tt.name, i); err != nil {
217 log.Printf("%s: %v", tt.name, err)
224 func toolPath(names ...string) (found, path string) {
227 for i, name := range names {
228 out, err := exec.Command(*flagGoCmd, "tool", "-n", name).CombinedOutput()
230 return name, strings.TrimSpace(string(out))
233 out1, err1 = out, err
236 log.Fatalf("go tool -n %s: %v\n%s", names[0], err1, out1)
245 func goList(dir string) (*Pkg, error) {
247 out, err := exec.Command(*flagGoCmd, "list", "-json", dir).Output()
249 return nil, fmt.Errorf("go list -json %s: %v", dir, err)
251 if err := json.Unmarshal(out, &pkg); err != nil {
252 return nil, fmt.Errorf("go list -json %s: unmarshal: %v", dir, err)
257 func runCmd(name string, cmd *exec.Cmd) error {
259 out, err := cmd.CombinedOutput()
261 return fmt.Errorf("%v\n%s", err, out)
263 fmt.Printf("%s 1 %d ns/op\n", name, time.Since(start).Nanoseconds())
267 type goBuild struct{ pkgs []string }
269 func (goBuild) long() bool { return true }
271 func (r goBuild) run(name string, count int) error {
272 args := []string{"build", "-a"}
273 if *flagCompilerFlags != "" {
274 args = append(args, "-gcflags", *flagCompilerFlags)
276 args = append(args, r.pkgs...)
277 cmd := exec.Command(*flagGoCmd, args...)
278 cmd.Dir = filepath.Join(goroot, "src")
279 return runCmd(name, cmd)
283 // path is either a path to a file ("$GOROOT/test/helloworld.go") or a package path ("cmd/go").
288 func (r size) long() bool { return r.isLong }
290 func (r size) run(name string, count int) error {
291 if strings.HasPrefix(r.path, "$GOROOT/") {
292 r.path = goroot + "/" + r.path[len("$GOROOT/"):]
295 cmd := exec.Command(*flagGoCmd, "build", "-o", "_compilebenchout_", r.path)
296 cmd.Stdout = os.Stderr
297 cmd.Stderr = os.Stderr
298 if err := cmd.Run(); err != nil {
301 defer os.Remove("_compilebenchout_")
302 info, err := os.Stat("_compilebenchout_")
306 out, err := exec.Command("size", "_compilebenchout_").CombinedOutput()
308 return fmt.Errorf("size: %v\n%s", err, out)
310 lines := strings.Split(string(out), "\n")
312 return fmt.Errorf("not enough output from size: %s", out)
314 f := strings.Fields(lines[1])
315 if strings.HasPrefix(lines[0], "__TEXT") && len(f) >= 2 { // OS X
316 fmt.Printf("%s 1 %s text-bytes %s data-bytes %v exe-bytes\n", name, f[0], f[1], info.Size())
317 } else if strings.Contains(lines[0], "bss") && len(f) >= 3 {
318 fmt.Printf("%s 1 %s text-bytes %s data-bytes %s bss-bytes %v exe-bytes\n", name, f[0], f[1], f[2], info.Size())
323 type compile struct{ dir string }
325 func (compile) long() bool { return false }
327 func (c compile) run(name string, count int) error {
328 // Make sure dependencies needed by go tool compile are installed to GOROOT/pkg.
329 out, err := exec.Command(*flagGoCmd, "build", "-i", c.dir).CombinedOutput()
331 return fmt.Errorf("go build -i %s: %v\n%s", c.dir, err, out)
334 // Find dir and source file list.
335 pkg, err := goList(c.dir)
340 args := []string{"-o", "_compilebench_.o"}
341 args = append(args, strings.Fields(*flagCompilerFlags)...)
342 args = append(args, pkg.GoFiles...)
343 if err := runBuildCmd(name, count, pkg.Dir, compiler, args); err != nil {
347 opath := pkg.Dir + "/_compilebench_.o"
349 // TODO(josharian): object files are big; just read enough to find what we seek.
350 data, err := ioutil.ReadFile(opath)
354 // Find start of export data.
355 i := bytes.Index(data, []byte("\n$$B\n")) + len("\n$$B\n")
356 // Count bytes to end of export data.
357 nexport := bytes.Index(data[i:], []byte("\n$$\n"))
358 fmt.Printf(" %d object-bytes %d export-bytes", len(data), nexport)
366 type link struct{ dir, flags string }
368 func (link) long() bool { return false }
370 func (r link) run(name string, count int) error {
372 // No linker. Skip the test.
376 // Build dependencies.
377 out, err := exec.Command(*flagGoCmd, "build", "-i", "-o", "/dev/null", r.dir).CombinedOutput()
379 return fmt.Errorf("go build -i %s: %v\n%s", r.dir, err, out)
382 // Build the main package.
383 pkg, err := goList(r.dir)
387 args := []string{"-o", "_compilebench_.o"}
388 args = append(args, pkg.GoFiles...)
389 cmd := exec.Command(compiler, args...)
391 cmd.Stdout = os.Stderr
392 cmd.Stderr = os.Stderr
395 return fmt.Errorf("compiling: %v", err)
397 defer os.Remove(pkg.Dir + "/_compilebench_.o")
399 // Link the main package.
400 args = []string{"-o", "_compilebench_.exe"}
401 args = append(args, strings.Fields(*flagLinkerFlags)...)
402 args = append(args, strings.Fields(r.flags)...)
403 args = append(args, "_compilebench_.o")
404 if err := runBuildCmd(name, count, pkg.Dir, linker, args); err != nil {
408 defer os.Remove(pkg.Dir + "/_compilebench_.exe")
413 // runBuildCmd runs "tool args..." in dir, measures standard build
414 // tool metrics, and prints a benchmark line. The caller may print
415 // additional metrics and then must print a newline.
417 // This assumes tool accepts standard build tool flags like
418 // -memprofilerate, -memprofile, and -cpuprofile.
419 func runBuildCmd(name string, count int, dir, tool string, args []string) error {
421 if *flagMemprofilerate >= 0 {
422 preArgs = append(preArgs, "-memprofilerate", fmt.Sprint(*flagMemprofilerate))
424 if *flagAlloc || *flagCpuprofile != "" || *flagMemprofile != "" {
425 if *flagAlloc || *flagMemprofile != "" {
426 preArgs = append(preArgs, "-memprofile", "_compilebench_.memprof")
428 if *flagCpuprofile != "" {
429 preArgs = append(preArgs, "-cpuprofile", "_compilebench_.cpuprof")
432 cmd := exec.Command(tool, append(preArgs, args...)...)
434 cmd.Stdout = os.Stderr
435 cmd.Stderr = os.Stderr
443 haveAllocs, haveRSS := false, false
444 var allocs, allocbytes, rssbytes int64
445 if *flagAlloc || *flagMemprofile != "" {
446 out, err := ioutil.ReadFile(dir + "/_compilebench_.memprof")
448 log.Print("cannot find memory profile after compilation")
450 for _, line := range strings.Split(string(out), "\n") {
451 f := strings.Fields(line)
452 if len(f) < 4 || f[0] != "#" || f[2] != "=" {
455 val, err := strconv.ParseInt(f[3], 0, 64)
471 log.Println("missing stats in memprof (golang.org/issue/18641)")
474 if *flagMemprofile != "" {
475 outpath := *flagMemprofile
477 outpath = fmt.Sprintf("%s_%d", outpath, count)
479 if err := ioutil.WriteFile(outpath, out, 0666); err != nil {
483 os.Remove(dir + "/_compilebench_.memprof")
486 if *flagCpuprofile != "" {
487 out, err := ioutil.ReadFile(dir + "/_compilebench_.cpuprof")
491 outpath := *flagCpuprofile
493 outpath = fmt.Sprintf("%s_%d", outpath, count)
495 if err := ioutil.WriteFile(outpath, out, 0666); err != nil {
498 os.Remove(dir + "/_compilebench_.cpuprof")
501 wallns := end.Sub(start).Nanoseconds()
502 userns := cmd.ProcessState.UserTime().Nanoseconds()
504 fmt.Printf("%s 1 %d ns/op %d user-ns/op", name, wallns, userns)
506 fmt.Printf(" %d B/op %d allocs/op", allocbytes, allocs)
509 fmt.Printf(" %d maxRSS/op", rssbytes)