1 // Copyright 2018 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // Package modfile implements a parser and formatter for go.mod files.
7 // The go.mod syntax is described in
8 // https://golang.org/cmd/go/#hdr-The_go_mod_file.
10 // The Parse and ParseLax functions both parse a go.mod file and return an
11 // abstract syntax tree. ParseLax ignores unknown statements and may be used to
12 // parse go.mod files that may have been developed with newer versions of Go.
14 // The File struct returned by Parse and ParseLax represent an abstract
15 // go.mod file. File has several methods like AddNewRequire and DropReplace
16 // that can be used to programmatically edit a file.
18 // The Format function formats a File back to a byte slice which can be
31 "golang.org/x/mod/internal/lazyregexp"
32 "golang.org/x/mod/module"
35 // A File is the parsed, interpreted form of a go.mod file.
46 // A Module is the module statement.
52 // A Go is the go statement.
54 Version string // "1.23"
58 // A Require is a single require statement.
61 Indirect bool // has "// indirect" comment
65 // An Exclude is a single exclude statement.
71 // A Replace is a single replace statement.
78 func (f *File) AddModuleStmt(path string) error {
80 f.Syntax = new(FileSyntax)
84 Mod: module.Version{Path: path},
85 Syntax: f.Syntax.addLine(nil, "module", AutoQuote(path)),
88 f.Module.Mod.Path = path
89 f.Syntax.updateLine(f.Module.Syntax, "module", AutoQuote(path))
94 func (f *File) AddComment(text string) {
96 f.Syntax = new(FileSyntax)
98 f.Syntax.Stmt = append(f.Syntax.Stmt, &CommentBlock{
109 type VersionFixer func(path, version string) (string, error)
111 // Parse parses the data, reported in errors as being from file,
112 // into a File struct. It applies fix, if non-nil, to canonicalize all module versions found.
113 func Parse(file string, data []byte, fix VersionFixer) (*File, error) {
114 return parseToFile(file, data, fix, true)
117 // ParseLax is like Parse but ignores unknown statements.
118 // It is used when parsing go.mod files other than the main module,
119 // under the theory that most statement types we add in the future will
120 // only apply in the main module, like exclude and replace,
121 // and so we get better gradual deployments if old go commands
122 // simply ignore those statements when found in go.mod files
124 func ParseLax(file string, data []byte, fix VersionFixer) (*File, error) {
125 return parseToFile(file, data, fix, false)
128 func parseToFile(file string, data []byte, fix VersionFixer, strict bool) (*File, error) {
129 fs, err := parse(file, data)
138 for _, x := range fs.Stmt {
139 switch x := x.(type) {
141 f.add(&errs, x, x.Token[0], x.Token[1:], fix, strict)
144 if len(x.Token) > 1 {
146 errs = append(errs, Error{
149 Err: fmt.Errorf("unknown block type: %s", strings.Join(x.Token, " ")),
157 errs = append(errs, Error{
160 Err: fmt.Errorf("unknown block type: %s", strings.Join(x.Token, " ")),
164 case "module", "require", "exclude", "replace":
165 for _, l := range x.Line {
166 f.add(&errs, l, x.Token[0], l.Token, fix, strict)
178 var GoVersionRE = lazyregexp.New(`^([1-9][0-9]*)\.(0|[1-9][0-9]*)$`)
180 func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix VersionFixer, strict bool) {
181 // If strict is false, this module is a dependency.
182 // We ignore all unknown directives as well as main-module-only
183 // directives like replace and exclude. It will work better for
184 // forward compatibility if we can depend on modules that have unknown
185 // statements (presumed relevant only when acting as the main module)
186 // and simply ignore those statements.
189 case "module", "require", "go":
190 // want these even for dependency go.mods
196 wrapModPathError := func(modPath string, err error) {
197 *errs = append(*errs, Error{
198 Filename: f.Syntax.Name,
205 wrapError := func(err error) {
206 *errs = append(*errs, Error{
207 Filename: f.Syntax.Name,
212 errorf := func(format string, args ...interface{}) {
213 wrapError(fmt.Errorf(format, args...))
218 errorf("unknown directive: %s", verb)
222 errorf("repeated go statement")
226 errorf("go directive expects exactly one argument")
228 } else if !GoVersionRE.MatchString(args[0]) {
229 errorf("invalid go version '%s': must match format 1.23", args[0])
233 f.Go = &Go{Syntax: line}
234 f.Go.Version = args[0]
237 errorf("repeated module statement")
240 f.Module = &Module{Syntax: line}
242 errorf("usage: module module/path")
245 s, err := parseString(&args[0])
247 errorf("invalid quoted string: %v", err)
250 f.Module.Mod = module.Version{Path: s}
251 case "require", "exclude":
253 errorf("usage: %s module/path v1.2.3", verb)
256 s, err := parseString(&args[0])
258 errorf("invalid quoted string: %v", err)
261 v, err := parseVersion(verb, s, &args[1], fix)
266 pathMajor, err := modulePathMajor(s)
271 if err := module.CheckPathMajor(v, pathMajor); err != nil {
272 wrapModPathError(s, err)
275 if verb == "require" {
276 f.Require = append(f.Require, &Require{
277 Mod: module.Version{Path: s, Version: v},
279 Indirect: isIndirect(line),
282 f.Exclude = append(f.Exclude, &Exclude{
283 Mod: module.Version{Path: s, Version: v},
289 if len(args) >= 2 && args[1] == "=>" {
292 if len(args) < arrow+2 || len(args) > arrow+3 || args[arrow] != "=>" {
293 errorf("usage: %s module/path [v1.2.3] => other/module v1.4\n\t or %s module/path [v1.2.3] => ../local/directory", verb, verb)
296 s, err := parseString(&args[0])
298 errorf("invalid quoted string: %v", err)
301 pathMajor, err := modulePathMajor(s)
303 wrapModPathError(s, err)
308 v, err = parseVersion(verb, s, &args[1], fix)
313 if err := module.CheckPathMajor(v, pathMajor); err != nil {
314 wrapModPathError(s, err)
318 ns, err := parseString(&args[arrow+1])
320 errorf("invalid quoted string: %v", err)
324 if len(args) == arrow+2 {
325 if !IsDirectoryPath(ns) {
326 errorf("replacement module without version must be directory path (rooted or starting with ./ or ../)")
329 if filepath.Separator == '/' && strings.Contains(ns, `\`) {
330 errorf("replacement directory appears to be Windows path (on a non-windows system)")
334 if len(args) == arrow+3 {
335 nv, err = parseVersion(verb, ns, &args[arrow+2], fix)
340 if IsDirectoryPath(ns) {
341 errorf("replacement module directory path %q cannot have version", ns)
345 f.Replace = append(f.Replace, &Replace{
346 Old: module.Version{Path: s, Version: v},
347 New: module.Version{Path: ns, Version: nv},
353 // isIndirect reports whether line has a "// indirect" comment,
354 // meaning it is in go.mod only for its effect on indirect dependencies,
355 // so that it can be dropped entirely once the effective version of the
356 // indirect dependency reaches the given minimum version.
357 func isIndirect(line *Line) bool {
358 if len(line.Suffix) == 0 {
361 f := strings.Fields(strings.TrimPrefix(line.Suffix[0].Token, string(slashSlash)))
362 return (len(f) == 1 && f[0] == "indirect" || len(f) > 1 && f[0] == "indirect;")
365 // setIndirect sets line to have (or not have) a "// indirect" comment.
366 func setIndirect(line *Line, indirect bool) {
367 if isIndirect(line) == indirect {
372 if len(line.Suffix) == 0 {
374 line.Suffix = []Comment{{Token: "// indirect", Suffix: true}}
378 com := &line.Suffix[0]
379 text := strings.TrimSpace(strings.TrimPrefix(com.Token, string(slashSlash)))
382 com.Token = "// indirect"
386 // Insert at beginning of existing comment.
387 com.Token = "// indirect; " + text
392 f := strings.Fields(line.Suffix[0].Token)
394 // Remove whole comment.
399 // Remove comment prefix.
400 com := &line.Suffix[0]
401 i := strings.Index(com.Token, "indirect;")
402 com.Token = "//" + com.Token[i+len("indirect;"):]
405 // IsDirectoryPath reports whether the given path should be interpreted
406 // as a directory path. Just like on the go command line, relative paths
407 // and rooted paths are directory paths; the rest are module paths.
408 func IsDirectoryPath(ns string) bool {
409 // Because go.mod files can move from one system to another,
410 // we check all known path syntaxes, both Unix and Windows.
411 return strings.HasPrefix(ns, "./") || strings.HasPrefix(ns, "../") || strings.HasPrefix(ns, "/") ||
412 strings.HasPrefix(ns, `.\`) || strings.HasPrefix(ns, `..\`) || strings.HasPrefix(ns, `\`) ||
413 len(ns) >= 2 && ('A' <= ns[0] && ns[0] <= 'Z' || 'a' <= ns[0] && ns[0] <= 'z') && ns[1] == ':'
416 // MustQuote reports whether s must be quoted in order to appear as
417 // a single token in a go.mod line.
418 func MustQuote(s string) bool {
419 for _, r := range s {
421 case ' ', '"', '\'', '`':
424 case '(', ')', '[', ']', '{', '}', ',':
430 if !unicode.IsPrint(r) {
435 return s == "" || strings.Contains(s, "//") || strings.Contains(s, "/*")
438 // AutoQuote returns s or, if quoting is required for s to appear in a go.mod,
439 // the quotation of s.
440 func AutoQuote(s string) string {
442 return strconv.Quote(s)
447 func parseString(s *string) (string, error) {
449 if strings.HasPrefix(t, `"`) {
451 if t, err = strconv.Unquote(t); err != nil {
454 } else if strings.ContainsAny(t, "\"'`") {
455 // Other quotes are reserved both for possible future expansion
456 // and to avoid confusion. For example if someone types 'x'
457 // we want that to be a syntax error and not a literal x in literal quotation marks.
458 return "", fmt.Errorf("unquoted string cannot contain quote")
464 type ErrorList []Error
466 func (e ErrorList) Error() string {
467 errStrs := make([]string, len(e))
468 for i, err := range e {
469 errStrs[i] = err.Error()
471 return strings.Join(errStrs, "\n")
482 func (e *Error) Error() string {
484 if e.Pos.LineRune > 1 {
485 // Don't print LineRune if it's 1 (beginning of line).
486 // It's always 1 except in scanner errors, which are rare.
487 pos = fmt.Sprintf("%s:%d:%d: ", e.Filename, e.Pos.Line, e.Pos.LineRune)
488 } else if e.Pos.Line > 0 {
489 pos = fmt.Sprintf("%s:%d: ", e.Filename, e.Pos.Line)
490 } else if e.Filename != "" {
491 pos = fmt.Sprintf("%s: ", e.Filename)
496 directive = fmt.Sprintf("%s %s: ", e.Verb, e.ModPath)
499 return pos + directive + e.Err.Error()
502 func (e *Error) Unwrap() error { return e.Err }
504 func parseVersion(verb string, path string, s *string, fix VersionFixer) (string, error) {
505 t, err := parseString(s)
510 Err: &module.InvalidVersionError{
518 t, err = fix(path, t)
520 if err, ok := err.(*module.ModuleError); ok {
530 if v := module.CanonicalVersion(t); v != "" {
537 Err: &module.InvalidVersionError{
539 Err: errors.New("must be of the form v1.2.3"),
544 func modulePathMajor(path string) (string, error) {
545 _, major, ok := module.SplitPathVersion(path)
547 return "", fmt.Errorf("invalid module path")
552 func (f *File) Format() ([]byte, error) {
553 return Format(f.Syntax), nil
556 // Cleanup cleans up the file f after any edit operations.
557 // To avoid quadratic behavior, modifications like DropRequire
558 // clear the entry but do not remove it from the slice.
559 // Cleanup cleans out all the cleared entries.
560 func (f *File) Cleanup() {
562 for _, r := range f.Require {
563 if r.Mod.Path != "" {
568 f.Require = f.Require[:w]
571 for _, x := range f.Exclude {
572 if x.Mod.Path != "" {
577 f.Exclude = f.Exclude[:w]
580 for _, r := range f.Replace {
581 if r.Old.Path != "" {
586 f.Replace = f.Replace[:w]
591 func (f *File) AddGoStmt(version string) error {
592 if !GoVersionRE.MatchString(version) {
593 return fmt.Errorf("invalid language version string %q", version)
597 if f.Module != nil && f.Module.Syntax != nil {
598 hint = f.Module.Syntax
602 Syntax: f.Syntax.addLine(hint, "go", version),
605 f.Go.Version = version
606 f.Syntax.updateLine(f.Go.Syntax, "go", version)
611 func (f *File) AddRequire(path, vers string) error {
613 for _, r := range f.Require {
614 if r.Mod.Path == path {
617 f.Syntax.updateLine(r.Syntax, "require", AutoQuote(path), vers)
620 f.Syntax.removeLine(r.Syntax)
627 f.AddNewRequire(path, vers, false)
632 func (f *File) AddNewRequire(path, vers string, indirect bool) {
633 line := f.Syntax.addLine(nil, "require", AutoQuote(path), vers)
634 setIndirect(line, indirect)
635 f.Require = append(f.Require, &Require{module.Version{Path: path, Version: vers}, indirect, line})
638 func (f *File) SetRequire(req []*Require) {
639 need := make(map[string]string)
640 indirect := make(map[string]bool)
641 for _, r := range req {
642 need[r.Mod.Path] = r.Mod.Version
643 indirect[r.Mod.Path] = r.Indirect
646 for _, r := range f.Require {
647 if v, ok := need[r.Mod.Path]; ok {
649 r.Indirect = indirect[r.Mod.Path]
656 for _, stmt := range f.Syntax.Stmt {
657 switch stmt := stmt.(type) {
659 if len(stmt.Token) > 0 && stmt.Token[0] == "require" {
661 for _, line := range stmt.Line {
662 if p, err := parseString(&line.Token[0]); err == nil && need[p] != "" {
663 if len(line.Comments.Before) == 1 && len(line.Comments.Before[0].Token) == 0 {
664 line.Comments.Before = line.Comments.Before[:0]
666 line.Token[1] = need[p]
668 setIndirect(line, indirect[p])
669 newLines = append(newLines, line)
672 if len(newLines) == 0 {
673 continue // drop stmt
679 if len(stmt.Token) > 0 && stmt.Token[0] == "require" {
680 if p, err := parseString(&stmt.Token[1]); err == nil && need[p] != "" {
681 stmt.Token[2] = need[p]
683 setIndirect(stmt, indirect[p])
685 continue // drop stmt
689 newStmts = append(newStmts, stmt)
691 f.Syntax.Stmt = newStmts
693 for path, vers := range need {
694 f.AddNewRequire(path, vers, indirect[path])
699 func (f *File) DropRequire(path string) error {
700 for _, r := range f.Require {
701 if r.Mod.Path == path {
702 f.Syntax.removeLine(r.Syntax)
709 func (f *File) AddExclude(path, vers string) error {
711 for _, x := range f.Exclude {
712 if x.Mod.Path == path && x.Mod.Version == vers {
715 if x.Mod.Path == path {
720 f.Exclude = append(f.Exclude, &Exclude{Mod: module.Version{Path: path, Version: vers}, Syntax: f.Syntax.addLine(hint, "exclude", AutoQuote(path), vers)})
724 func (f *File) DropExclude(path, vers string) error {
725 for _, x := range f.Exclude {
726 if x.Mod.Path == path && x.Mod.Version == vers {
727 f.Syntax.removeLine(x.Syntax)
734 func (f *File) AddReplace(oldPath, oldVers, newPath, newVers string) error {
736 old := module.Version{Path: oldPath, Version: oldVers}
737 new := module.Version{Path: newPath, Version: newVers}
738 tokens := []string{"replace", AutoQuote(oldPath)}
740 tokens = append(tokens, oldVers)
742 tokens = append(tokens, "=>", AutoQuote(newPath))
744 tokens = append(tokens, newVers)
748 for _, r := range f.Replace {
749 if r.Old.Path == oldPath && (oldVers == "" || r.Old.Version == oldVers) {
751 // Found replacement for old; update to use new.
753 f.Syntax.updateLine(r.Syntax, tokens...)
757 // Already added; delete other replacements for same.
758 f.Syntax.removeLine(r.Syntax)
761 if r.Old.Path == oldPath {
766 f.Replace = append(f.Replace, &Replace{Old: old, New: new, Syntax: f.Syntax.addLine(hint, tokens...)})
771 func (f *File) DropReplace(oldPath, oldVers string) error {
772 for _, r := range f.Replace {
773 if r.Old.Path == oldPath && r.Old.Version == oldVers {
774 f.Syntax.removeLine(r.Syntax)
781 func (f *File) SortBlocks() {
782 f.removeDups() // otherwise sorting is unsafe
784 for _, stmt := range f.Syntax.Stmt {
785 block, ok := stmt.(*LineBlock)
789 sort.Slice(block.Line, func(i, j int) bool {
792 for k := 0; k < len(li.Token) && k < len(lj.Token); k++ {
793 if li.Token[k] != lj.Token[k] {
794 return li.Token[k] < lj.Token[k]
797 return len(li.Token) < len(lj.Token)
802 func (f *File) removeDups() {
803 have := make(map[module.Version]bool)
804 kill := make(map[*Line]bool)
805 for _, x := range f.Exclude {
807 kill[x.Syntax] = true
813 for _, x := range f.Exclude {
815 excl = append(excl, x)
820 have = make(map[module.Version]bool)
821 // Later replacements take priority over earlier ones.
822 for i := len(f.Replace) - 1; i >= 0; i-- {
825 kill[x.Syntax] = true
831 for _, x := range f.Replace {
833 repl = append(repl, x)
839 for _, stmt := range f.Syntax.Stmt {
840 switch stmt := stmt.(type) {
847 for _, line := range stmt.Line {
849 lines = append(lines, line)
857 stmts = append(stmts, stmt)
859 f.Syntax.Stmt = stmts