1 // Copyright 2019 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // Indexed binary package export.
6 // This file was derived from $GOROOT/src/cmd/compile/internal/gc/iexport.go;
7 // see that file for specification of the format.
24 // Current indexed export format version. Increase with each format change.
26 const iexportVersion = 0
28 // Current bundled export format version. Increase with each format change.
29 // 0: initial implementation
30 const bundleVersion = 0
32 // IExportData writes indexed export data for pkg to out.
34 // If no file set is provided, position info will be missing.
35 // The package path of the top-level package will not be recorded,
36 // so that calls to IImportData can override with a provided package path.
37 func IExportData(out io.Writer, fset *token.FileSet, pkg *types.Package) error {
38 return iexportCommon(out, fset, false, []*types.Package{pkg})
41 // IExportBundle writes an indexed export bundle for pkgs to out.
42 func IExportBundle(out io.Writer, fset *token.FileSet, pkgs []*types.Package) error {
43 return iexportCommon(out, fset, true, pkgs)
46 func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, pkgs []*types.Package) (err error) {
48 if e := recover(); e != nil {
49 if ierr, ok := e.(internalError); ok {
53 // Not an internal error; panic again.
60 allPkgs: map[*types.Package]bool{},
61 stringIndex: map[string]uint64{},
62 declIndex: map[types.Object]uint64{},
63 typIndex: map[types.Type]uint64{},
69 for i, pt := range predeclared() {
70 p.typIndex[pt] = uint64(i)
72 if len(p.typIndex) > predeclReserved {
73 panic(internalErrorf("too many predeclared types: %d > %d", len(p.typIndex), predeclReserved))
76 // Initialize work queue with exported declarations.
77 for _, pkg := range pkgs {
79 for _, name := range scope.Names() {
80 if ast.IsExported(name) {
81 p.pushDecl(scope.Lookup(name))
86 // Ensure pkg and its imports are included in the index.
88 for _, imp := range pkg.Imports() {
94 // Loop until no more work.
95 for !p.declTodo.empty() {
96 p.doDecl(p.declTodo.popHead())
99 // Append indices to data0 section.
100 dataLen := uint64(p.data0.Len())
102 w.writeIndex(p.declIndex)
105 w.uint64(uint64(len(pkgs)))
106 for _, pkg := range pkgs {
108 imps := pkg.Imports()
109 w.uint64(uint64(len(imps)))
110 for _, imp := range imps {
120 hdr.uint64(bundleVersion)
122 hdr.uint64(iexportVersion)
123 hdr.uint64(uint64(p.strings.Len()))
128 io.Copy(out, &p.strings)
129 io.Copy(out, &p.data0)
134 // writeIndex writes out an object index. mainIndex indicates whether
135 // we're writing out the main index, which is also read by
136 // non-compiler tools and includes a complete package description
137 // (i.e., name and height).
138 func (w *exportWriter) writeIndex(index map[types.Object]uint64) {
139 // Build a map from packages to objects from that package.
140 pkgObjs := map[*types.Package][]types.Object{}
142 // For the main index, make sure to include every package that
143 // we reference, even if we're not exporting (or reexporting)
144 // any symbols from it.
145 if w.p.localpkg != nil {
146 pkgObjs[w.p.localpkg] = nil
148 for pkg := range w.p.allPkgs {
152 for obj := range index {
153 pkgObjs[obj.Pkg()] = append(pkgObjs[obj.Pkg()], obj)
156 var pkgs []*types.Package
157 for pkg, objs := range pkgObjs {
158 pkgs = append(pkgs, pkg)
160 sort.Slice(objs, func(i, j int) bool {
161 return objs[i].Name() < objs[j].Name()
165 sort.Slice(pkgs, func(i, j int) bool {
166 return w.exportPath(pkgs[i]) < w.exportPath(pkgs[j])
169 w.uint64(uint64(len(pkgs)))
170 for _, pkg := range pkgs {
171 w.string(w.exportPath(pkg))
173 w.uint64(uint64(0)) // package height is not needed for go/types
176 w.uint64(uint64(len(objs)))
177 for _, obj := range objs {
184 type iexporter struct {
188 localpkg *types.Package
190 // allPkgs tracks all packages that have been referenced by
191 // the export data, so we can ensure to include them in the
193 allPkgs map[*types.Package]bool
198 stringIndex map[string]uint64
201 declIndex map[types.Object]uint64
202 typIndex map[types.Type]uint64
205 // stringOff returns the offset of s within the string section.
206 // If not already present, it's added to the end.
207 func (p *iexporter) stringOff(s string) uint64 {
208 off, ok := p.stringIndex[s]
210 off = uint64(p.strings.Len())
211 p.stringIndex[s] = off
213 p.strings.uint64(uint64(len(s)))
214 p.strings.WriteString(s)
219 // pushDecl adds n to the declaration work queue, if not already present.
220 func (p *iexporter) pushDecl(obj types.Object) {
221 // Package unsafe is known to the compiler and predeclared.
222 assert(obj.Pkg() != types.Unsafe)
224 if _, ok := p.declIndex[obj]; ok {
228 p.declIndex[obj] = ^uint64(0) // mark n present in work queue
229 p.declTodo.pushTail(obj)
232 // exportWriter handles writing out individual data section chunks.
233 type exportWriter struct {
237 currPkg *types.Package
242 func (w *exportWriter) exportPath(pkg *types.Package) string {
243 if pkg == w.p.localpkg {
249 func (p *iexporter) doDecl(obj types.Object) {
251 w.setPkg(obj.Pkg(), false)
253 switch obj := obj.(type) {
257 w.typ(obj.Type(), obj.Pkg())
260 sig, _ := obj.Type().(*types.Signature)
261 if sig.Recv() != nil {
262 panic(internalErrorf("unexpected method: %v", sig))
271 w.value(obj.Type(), obj.Val())
273 case *types.TypeName:
277 w.typ(obj.Type(), obj.Pkg())
285 underlying := obj.Type().Underlying()
286 w.typ(underlying, obj.Pkg())
289 if types.IsInterface(t) {
293 named, ok := t.(*types.Named)
295 panic(internalErrorf("%s is not a defined type", t))
298 n := named.NumMethods()
300 for i := 0; i < n; i++ {
304 sig, _ := m.Type().(*types.Signature)
310 panic(internalErrorf("unexpected object: %v", obj))
313 p.declIndex[obj] = w.flush()
316 func (w *exportWriter) tag(tag byte) {
317 w.data.WriteByte(tag)
320 func (w *exportWriter) pos(pos token.Pos) {
326 p := w.p.fset.Position(pos)
328 line := int64(p.Line)
330 // When file is the same as the last position (common case),
331 // we can save a few bytes by delta encoding just the line
334 // Note: Because data objects may be read out of order (or not
335 // at all), we can only apply delta encoding within a single
336 // object. This is handled implicitly by tracking prevFile and
337 // prevLine as fields of exportWriter.
339 if file == w.prevFile {
340 delta := line - w.prevLine
342 if delta == deltaNewFile {
346 w.int64(deltaNewFile)
347 w.int64(line) // line >= 0
354 func (w *exportWriter) pkg(pkg *types.Package) {
355 // Ensure any referenced packages are declared in the main index.
356 w.p.allPkgs[pkg] = true
358 w.string(w.exportPath(pkg))
361 func (w *exportWriter) qualifiedIdent(obj types.Object) {
362 // Ensure any referenced declarations are written out too.
369 func (w *exportWriter) typ(t types.Type, pkg *types.Package) {
370 w.data.uint64(w.p.typOff(t, pkg))
373 func (p *iexporter) newWriter() *exportWriter {
374 return &exportWriter{p: p}
377 func (w *exportWriter) flush() uint64 {
378 off := uint64(w.p.data0.Len())
379 io.Copy(&w.p.data0, &w.data)
383 func (p *iexporter) typOff(t types.Type, pkg *types.Package) uint64 {
384 off, ok := p.typIndex[t]
388 off = predeclReserved + w.flush()
394 func (w *exportWriter) startType(k itag) {
395 w.data.uint64(uint64(k))
398 func (w *exportWriter) doTyp(t types.Type, pkg *types.Package) {
399 switch t := t.(type) {
401 w.startType(definedType)
402 w.qualifiedIdent(t.Obj())
405 w.startType(pointerType)
409 w.startType(sliceType)
413 w.startType(arrayType)
414 w.uint64(uint64(t.Len()))
418 w.startType(chanType)
419 // 1 RecvOnly; 2 SendOnly; 3 SendRecv
437 case *types.Signature:
438 w.startType(signatureType)
443 w.startType(structType)
448 for i := 0; i < n; i++ {
453 w.bool(f.Anonymous())
454 w.string(t.Tag(i)) // note (or tag)
457 case *types.Interface:
458 w.startType(interfaceType)
461 n := t.NumEmbeddeds()
463 for i := 0; i < n; i++ {
466 w.typ(f.Obj().Type(), f.Obj().Pkg())
469 n = t.NumExplicitMethods()
471 for i := 0; i < n; i++ {
472 m := t.ExplicitMethod(i)
475 sig, _ := m.Type().(*types.Signature)
480 panic(internalErrorf("unexpected type: %v, %v", t, reflect.TypeOf(t)))
484 func (w *exportWriter) setPkg(pkg *types.Package, write bool) {
492 func (w *exportWriter) signature(sig *types.Signature) {
493 w.paramList(sig.Params())
494 w.paramList(sig.Results())
495 if sig.Params().Len() > 0 {
496 w.bool(sig.Variadic())
500 func (w *exportWriter) paramList(tup *types.Tuple) {
503 for i := 0; i < n; i++ {
508 func (w *exportWriter) param(obj types.Object) {
511 w.typ(obj.Type(), obj.Pkg())
514 func (w *exportWriter) value(typ types.Type, v constant.Value) {
517 switch b := typ.Underlying().(*types.Basic); b.Info() & types.IsConstType {
518 case types.IsBoolean:
519 w.bool(constant.BoolVal(v))
520 case types.IsInteger:
522 if i64, exact := constant.Int64Val(v); exact {
524 } else if ui64, exact := constant.Uint64Val(v); exact {
527 i.SetString(v.ExactString(), 10)
531 f := constantToFloat(v)
533 case types.IsComplex:
534 w.mpfloat(constantToFloat(constant.Real(v)), typ)
535 w.mpfloat(constantToFloat(constant.Imag(v)), typ)
537 w.string(constant.StringVal(v))
539 if b.Kind() == types.Invalid {
540 // package contains type errors
543 panic(internalErrorf("unexpected type %v (%v)", typ, typ.Underlying()))
547 // constantToFloat converts a constant.Value with kind constant.Float to a
549 func constantToFloat(x constant.Value) *big.Float {
550 x = constant.ToFloat(x)
551 // Use the same floating-point precision (512) as cmd/compile
552 // (see Mpprec in cmd/compile/internal/gc/mpfloat.go).
556 if v, exact := constant.Float64Val(x); exact {
559 } else if num, denom := constant.Num(x), constant.Denom(x); num.Kind() == constant.Int {
560 // TODO(gri): add big.Rat accessor to constant.Value.
562 d := valueToRat(denom)
563 f.SetRat(n.Quo(n, d))
565 // Value too large to represent as a fraction => inaccessible.
566 // TODO(gri): add big.Float accessor to constant.Value.
567 _, ok := f.SetString(x.ExactString())
573 // mpint exports a multi-precision integer.
575 // For unsigned types, small values are written out as a single
576 // byte. Larger values are written out as a length-prefixed big-endian
577 // byte string, where the length prefix is encoded as its complement.
578 // For example, bytes 0, 1, and 2 directly represent the integer
579 // values 0, 1, and 2; while bytes 255, 254, and 253 indicate a 1-,
580 // 2-, and 3-byte big-endian string follow.
582 // Encoding for signed types use the same general approach as for
583 // unsigned types, except small values use zig-zag encoding and the
584 // bottom bit of length prefix byte for large values is reserved as a
587 // The exact boundary between small and large encodings varies
588 // according to the maximum number of bytes needed to encode a value
589 // of type typ. As a special case, 8-bit types are always encoded as a
592 // TODO(mdempsky): Is this level of complexity really worthwhile?
593 func (w *exportWriter) mpint(x *big.Int, typ types.Type) {
594 basic, ok := typ.Underlying().(*types.Basic)
596 panic(internalErrorf("unexpected type %v (%T)", typ.Underlying(), typ.Underlying()))
599 signed, maxBytes := intSize(basic)
601 negative := x.Sign() < 0
602 if !signed && negative {
603 panic(internalErrorf("negative unsigned integer; type %v, value %v", typ, x))
607 if len(b) > 0 && b[0] == 0 {
608 panic(internalErrorf("leading zeros"))
610 if uint(len(b)) > maxBytes {
611 panic(internalErrorf("bad mpint length: %d > %d (type %v, value %v)", len(b), maxBytes, typ, x))
614 maxSmall := 256 - maxBytes
616 maxSmall = 256 - 2*maxBytes
622 // Check if x can use small value encoding.
635 w.data.WriteByte(byte(ux))
640 n := 256 - uint(len(b))
642 n = 256 - 2*uint(len(b))
647 if n < maxSmall || n >= 256 {
648 panic(internalErrorf("encoding mistake: %d, %v, %v => %d", len(b), signed, negative, n))
651 w.data.WriteByte(byte(n))
655 // mpfloat exports a multi-precision floating point number.
657 // The number's value is decomposed into mantissa × 2**exponent, where
658 // mantissa is an integer. The value is written out as mantissa (as a
659 // multi-precision integer) and then the exponent, except exponent is
660 // omitted if mantissa is zero.
661 func (w *exportWriter) mpfloat(f *big.Float, typ types.Type) {
663 panic("infinite constant")
666 // Break into f = mant × 2**exp, with 0.5 <= mant < 1.
668 exp := int64(f.MantExp(&mant))
670 // Scale so that mant is an integer.
671 prec := mant.MinPrec()
672 mant.SetMantExp(&mant, int(prec))
675 manti, acc := mant.Int(nil)
676 if acc != big.Exact {
677 panic(internalErrorf("mantissa scaling failed for %f (%s)", f, acc))
680 if manti.Sign() != 0 {
685 func (w *exportWriter) bool(b bool) bool {
694 func (w *exportWriter) int64(x int64) { w.data.int64(x) }
695 func (w *exportWriter) uint64(x uint64) { w.data.uint64(x) }
696 func (w *exportWriter) string(s string) { w.uint64(w.p.stringOff(s)) }
698 func (w *exportWriter) localIdent(obj types.Object) {
699 // Anonymous parameters.
714 type intWriter struct {
718 func (w *intWriter) int64(x int64) {
719 var buf [binary.MaxVarintLen64]byte
720 n := binary.PutVarint(buf[:], x)
724 func (w *intWriter) uint64(x uint64) {
725 var buf [binary.MaxVarintLen64]byte
726 n := binary.PutUvarint(buf[:], x)
730 func assert(cond bool) {
732 panic("internal error: assertion failed")
736 // The below is copied from go/src/cmd/compile/internal/gc/syntax.go.
738 // objQueue is a FIFO queue of types.Object. The zero value of objQueue is
739 // a ready-to-use empty queue.
740 type objQueue struct {
745 // empty returns true if q contains no Nodes.
746 func (q *objQueue) empty() bool {
747 return q.head == q.tail
750 // pushTail appends n to the tail of the queue.
751 func (q *objQueue) pushTail(obj types.Object) {
752 if len(q.ring) == 0 {
753 q.ring = make([]types.Object, 16)
754 } else if q.head+len(q.ring) == q.tail {
756 nring := make([]types.Object, len(q.ring)*2)
757 // Copy the old elements.
758 part := q.ring[q.head%len(q.ring):]
759 if q.tail-q.head <= len(part) {
760 part = part[:q.tail-q.head]
763 pos := copy(nring, part)
764 copy(nring[pos:], q.ring[:q.tail%len(q.ring)])
766 q.ring, q.head, q.tail = nring, 0, q.tail-q.head
769 q.ring[q.tail%len(q.ring)] = obj
773 // popHead pops a node from the head of the queue. It panics if q is empty.
774 func (q *objQueue) popHead() types.Object {
776 panic("dequeue empty")
778 obj := q.ring[q.head%len(q.ring)]