1 // Copyright 2020 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
20 "golang.org/x/tools/internal/lsp"
21 "golang.org/x/tools/internal/lsp/protocol"
22 "golang.org/x/tools/internal/lsp/source"
23 "golang.org/x/tools/internal/span"
26 // generate semantic tokens and interpolate them in the file
28 // The output is the input file decorated with comments showing the
29 // syntactic tokens. The comments are stylized:
30 // /*<arrow><length>,<token type>,[<modifiers]*/
31 // For most occurrences, the comment comes just before the token it
32 // describes, and arrow is a right arrow. If the token is inside a string
33 // the comment comes just after the string, and the arrow is a left arrow.
34 // <length> is the length of the token in runes, <token type> is one
35 // of the supported semantic token types, and <modifiers. is a
36 // (possibly empty) list of token type modifiers.
38 // There are 3 coordinate systems for lines and character offsets in lines
39 // LSP (what's returned from semanticTokens()):
40 // 0-based: the first line is line 0, the first character of a line
41 // is character 0, and characters are counted as UTF-16 code points
42 // gopls (and Go error messages):
43 // 1-based: the first line is line1, the first chararcter of a line
44 // is character 0, and characters are counted as bytes
45 // internal (as used in marks, and lines:=bytes.Split(buf, '\n'))
46 // 0-based: lines and character positions are 1 less than in
47 // the gopls coordinate system
53 var colmap *protocol.ColumnMapper
55 func (c *semtok) Name() string { return "semtok" }
56 func (c *semtok) Usage() string { return "<filename>" }
57 func (c *semtok) ShortHelp() string { return "show semantic tokens for the specified file" }
58 func (c *semtok) DetailedHelp(f *flag.FlagSet) {
60 _, f, l, ok := runtime.Caller(i)
64 log.Printf("%d: %s:%d", i, f, l)
66 fmt.Fprint(f.Output(), `
67 Example: show the semantic tokens for this file:
69 $ gopls semtok internal/lsp/cmd/semtok.go
71 gopls semtok flags are:
76 // Run performs the semtok on the files specified by args and prints the
77 // results to stdout. PJW: fix this description
78 func (c *semtok) Run(ctx context.Context, args ...string) error {
79 log.SetFlags(log.Lshortfile)
81 return fmt.Errorf("expected one file name, got %d", len(args))
83 // perhaps simpler if app had just had a FlagSet member
84 origOptions := c.app.options
85 c.app.options = func(opts *source.Options) {
87 opts.SemanticTokens = true
89 conn, err := c.app.connect(ctx)
93 defer conn.terminate(ctx)
94 uri := span.URIFromPath(args[0])
95 file := conn.AddFile(ctx, uri)
100 resp, err := conn.semanticTokens(ctx, uri)
104 buf, err := ioutil.ReadFile(args[0])
108 fset := token.NewFileSet()
109 f, err := parser.ParseFile(fset, args[0], buf, 0)
111 log.Printf("parsing %s failed %v", args[0], err)
114 tok := fset.File(f.Pos())
116 // can't happen; just parsed this file
117 return fmt.Errorf("can't find %s in fset", args[0])
119 tc := span.NewContentConverter(args[0], buf)
120 colmap = &protocol.ColumnMapper{
121 URI: span.URI(args[0]),
125 memo = lsp.SemanticMemo
126 err = decorate(file.uri.Filename(), resp.Data)
133 var memo *lsp.SemMemo
136 line, offset int // 1-based, from RangeSpan
137 len int // bytes, not runes
142 // prefixes for semantic token comments
145 SemanticRight = "/*⇒"
148 func markLine(m mark, lines [][]byte) {
149 l := lines[m.line-1] // mx is 1-based
150 length := utf8.RuneCount(l[m.offset-1 : m.offset-1+m.len])
151 splitAt := m.offset - 1
153 if m.typ == "namespace" && m.offset-1+m.len < len(l) && l[m.offset-1+m.len] == '"' {
154 // it is the last component of an import spec
155 // cannot put a comment inside a string
156 insert = fmt.Sprintf("%s%d,namespace,[]*/", SemanticLeft, length)
157 splitAt = m.offset + m.len
159 insert = fmt.Sprintf("%s%d,%s,%v*/", SemanticRight, length, m.typ, m.mods)
161 x := append([]byte(insert), l[splitAt:]...)
162 l = append(l[:splitAt], x...)
166 func decorate(file string, result []float64) error {
167 buf, err := ioutil.ReadFile(file)
171 marks := newMarks(result)
175 lines := bytes.Split(buf, []byte{'\n'})
176 for i := len(marks) - 1; i >= 0; i-- {
180 os.Stdout.Write(bytes.Join(lines, []byte{'\n'}))
184 func newMarks(d []float64) []mark {
186 // the following two loops could be merged, at the cost
187 // of making the logic slightly more complicated to understand
188 // first, convert from deltas to absolute, in LSP coordinates
189 lspLine := make([]float64, len(d)/5)
190 lspChar := make([]float64, len(d)/5)
191 line, char := 0.0, 0.0
192 for i := 0; 5*i < len(d); i++ {
193 lspLine[i] = line + d[5*i+0]
197 lspChar[i] = char + d[5*i+1]
201 // second, convert to gopls coordinates
202 for i := 0; 5*i < len(d); i++ {
203 pr := protocol.Range{
204 Start: protocol.Position{
206 Character: lspChar[i],
208 End: protocol.Position{
210 Character: lspChar[i] + d[5*i+2],
213 spn, err := colmap.RangeSpan(pr)
218 line: spn.Start().Line(),
219 offset: spn.Start().Column(),
220 len: spn.End().Column() - spn.Start().Column(),
221 typ: memo.Type(int(d[5*i+3])),
222 mods: memo.Mods(int(d[5*i+4])),