1 // Copyright 2018 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
18 "golang.org/x/mod/modfile"
21 const commentStart = "@"
22 const commentStartLen = len(commentStart)
24 // Identifier is the type for an identifier in an Note argument list.
25 type Identifier string
27 // Parse collects all the notes present in a file.
28 // If content is nil, the filename specified is read and parsed, otherwise the
29 // content is used and the filename is used for positions and error messages.
30 // Each comment whose text starts with @ is parsed as a comma-separated
32 // See the package documentation for details about the syntax of those
34 func Parse(fset *token.FileSet, filename string, content []byte) ([]*Note, error) {
39 switch filepath.Ext(filename) {
41 // TODO: We should write this in terms of the scanner.
42 // there are ways you can break the parser such that it will not add all the
43 // comments to the ast, which may result in files where the tests are silently
45 file, err := parser.ParseFile(fset, filename, src, parser.ParseComments)
49 return ExtractGo(fset, file)
51 file, err := modfile.Parse(filename, content, nil)
55 f := fset.AddFile(filename, -1, len(content))
56 f.SetLinesForContent(content)
57 notes, err := extractMod(fset, file)
61 // Since modfile.Parse does not return an *ast, we need to add the offset
62 // within the file's contents to the file's base relative to the fileset.
63 for _, note := range notes {
64 note.Pos += token.Pos(f.Base())
71 // extractMod collects all the notes present in a go.mod file.
72 // Each comment whose text starts with @ is parsed as a comma-separated
74 // See the package documentation for details about the syntax of those
76 // Only allow notes to appear with the following format: "//@mark()" or // @mark()
77 func extractMod(fset *token.FileSet, file *modfile.File) ([]*Note, error) {
79 for _, stmt := range file.Syntax.Stmt {
80 comment := stmt.Comment()
84 // Handle the case for markers of `// indirect` to be on the line before
85 // the require statement.
86 // TODO(golang/go#36894): have a more intuitive approach for // indirect
87 for _, cmt := range comment.Before {
88 text, adjust := getAdjustedNote(cmt.Token)
92 parsed, err := parse(fset, token.Pos(int(cmt.Start.Byte)+adjust), text)
96 notes = append(notes, parsed...)
98 // Handle the normal case for markers on the same line.
99 for _, cmt := range comment.Suffix {
100 text, adjust := getAdjustedNote(cmt.Token)
104 parsed, err := parse(fset, token.Pos(int(cmt.Start.Byte)+adjust), text)
108 notes = append(notes, parsed...)
114 // ExtractGo collects all the notes present in an AST.
115 // Each comment whose text starts with @ is parsed as a comma-separated
116 // sequence of notes.
117 // See the package documentation for details about the syntax of those
119 func ExtractGo(fset *token.FileSet, file *ast.File) ([]*Note, error) {
121 for _, g := range file.Comments {
122 for _, c := range g.List {
123 text, adjust := getAdjustedNote(c.Text)
127 parsed, err := parse(fset, token.Pos(int(c.Pos())+adjust), text)
131 notes = append(notes, parsed...)
137 func getAdjustedNote(text string) (string, int) {
138 if strings.HasPrefix(text, "/*") {
139 text = strings.TrimSuffix(text, "*/")
141 text = text[2:] // remove "//" or "/*" prefix
143 // Allow notes to appear within comments.
145 // "// //@mark()" is valid.
146 // "// @mark()" is not valid.
147 // "// /*@mark()*/" is not valid.
149 if i := strings.Index(text, commentStart); i > 2 {
150 // Get the text before the commentStart.
158 if !strings.HasPrefix(text, commentStart) {
161 text = text[commentStartLen:]
162 return text, commentStartLen + adjust + 1
165 const invalidToken rune = 0
168 scanner scanner.Scanner
174 func (t *tokens) Init(base token.Pos, text string) *tokens {
176 t.scanner.Init(strings.NewReader(text))
177 t.scanner.Mode = scanner.GoTokens
178 t.scanner.Whitespace ^= 1 << '\n' // don't skip new lines
179 t.scanner.Error = func(s *scanner.Scanner, msg string) {
185 func (t *tokens) Consume() string {
186 t.current = invalidToken
187 return t.scanner.TokenText()
190 func (t *tokens) Token() rune {
194 if t.current == invalidToken {
195 t.current = t.scanner.Scan()
200 func (t *tokens) Skip(r rune) int {
202 for t.Token() == '\n' {
209 func (t *tokens) TokenString() string {
210 return scanner.TokenString(t.Token())
213 func (t *tokens) Pos() token.Pos {
214 return t.base + token.Pos(t.scanner.Position.Offset)
217 func (t *tokens) Errorf(msg string, args ...interface{}) {
221 t.err = fmt.Errorf(msg, args...)
224 func parse(fset *token.FileSet, base token.Pos, text string) ([]*Note, error) {
225 t := new(tokens).Init(base, text)
226 notes := parseComment(t)
228 return nil, fmt.Errorf("%v:%s", fset.Position(t.Pos()), t.err)
233 func parseComment(t *tokens) []*Note {
241 notes = append(notes, parseNote(t))
243 t.Errorf("unexpected %s parsing comment, expect identifier", t.TokenString())
252 t.Errorf("unexpected %s parsing comment, expect separator", t.TokenString())
258 func parseNote(t *tokens) *Note {
265 case ',', '\n', scanner.EOF:
266 // no argument list present
269 n.Args = parseArgumentList(t)
272 t.Errorf("unexpected %s parsing note", t.TokenString())
277 func parseArgumentList(t *tokens) []interface{} {
278 args := []interface{}{} // @name() is represented by a non-nil empty slice.
281 for t.Token() != ')' {
282 args = append(args, parseArgument(t))
283 if t.Token() != ',' {
289 if t.Token() != ')' {
290 t.Errorf("unexpected %s parsing argument list", t.TokenString())
297 func parseArgument(t *tokens) interface{} {
309 if t.Token() != scanner.String && t.Token() != scanner.RawString {
310 t.Errorf("re must be followed by string, got %s", t.TokenString())
313 pattern, _ := strconv.Unquote(t.Consume()) // can't fail
314 re, err := regexp.Compile(pattern)
316 t.Errorf("invalid regular expression %s: %v", pattern, err)
324 case scanner.String, scanner.RawString:
325 v, _ := strconv.Unquote(t.Consume()) // can't fail
330 v, err := strconv.ParseInt(s, 0, 0)
332 t.Errorf("cannot convert %v to int: %v", s, err)
338 v, err := strconv.ParseFloat(s, 64)
340 t.Errorf("cannot convert %v to float: %v", s, err)
345 t.Errorf("unexpected char literal %s", t.Consume())
349 t.Errorf("unexpected %s parsing argument", t.TokenString())