1 // Copyright 2018 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // Package module defines the module.Version type along with support code.
7 // The module.Version type is a simple Path, Version pair:
9 // type Version struct {
14 // There are no restrictions imposed directly by use of this structure,
15 // but additional checking functions, most notably Check, verify that
16 // a particular path, version pair is valid.
20 // Module paths appear as substrings of file system paths
21 // (in the download cache) and of web server URLs in the proxy protocol.
22 // In general we cannot rely on file systems to be case-sensitive,
23 // nor can we rely on web servers, since they read from file systems.
24 // That is, we cannot rely on the file system to keep rsc.io/QUOTE
25 // and rsc.io/quote separate. Windows and macOS don't.
26 // Instead, we must never require two different casings of a file path.
27 // Because we want the download cache to match the proxy protocol,
28 // and because we want the proxy protocol to be possible to serve
29 // from a tree of static files (which might be stored on a case-insensitive
30 // file system), the proxy protocol must never require two different casings
31 // of a URL path either.
33 // One possibility would be to make the escaped form be the lowercase
34 // hexadecimal encoding of the actual path bytes. This would avoid ever
35 // needing different casings of a file path, but it would be fairly illegible
36 // to most programmers when those paths appeared in the file system
37 // (including in file paths in compiler errors and stack traces)
38 // in web server logs, and so on. Instead, we want a safe escaped form that
39 // leaves most paths unaltered.
41 // The safe escaped form is to replace every uppercase letter
42 // with an exclamation mark followed by the letter's lowercase equivalent.
46 // github.com/Azure/azure-sdk-for-go -> github.com/!azure/azure-sdk-for-go.
47 // github.com/GoogleCloudPlatform/cloudsql-proxy -> github.com/!google!cloud!platform/cloudsql-proxy
48 // github.com/Sirupsen/logrus -> github.com/!sirupsen/logrus.
50 // Import paths that avoid upper-case letters are left unchanged.
51 // Note that because import paths are ASCII-only and avoid various
52 // problematic punctuation (like : < and >), the escaped form is also ASCII-only
53 // and avoids the same problematic punctuation.
55 // Import paths have never allowed exclamation marks, so there is no
56 // need to define how to escape a literal !.
58 // Unicode Restrictions
60 // Today, paths are disallowed from using Unicode.
62 // Although paths are currently disallowed from using Unicode,
63 // we would like at some point to allow Unicode letters as well, to assume that
64 // file systems and URLs are Unicode-safe (storing UTF-8), and apply
65 // the !-for-uppercase convention for escaping them in the file system.
66 // But there are at least two subtle considerations.
68 // First, note that not all case-fold equivalent distinct runes
69 // form an upper/lower pair.
70 // For example, U+004B ('K'), U+006B ('k'), and U+212A ('K' for Kelvin)
71 // are three distinct runes that case-fold to each other.
72 // When we do add Unicode letters, we must not assume that upper/lower
73 // are the only case-equivalent pairs.
74 // Perhaps the Kelvin symbol would be disallowed entirely, for example.
75 // Or perhaps it would escape as "!!k", or perhaps as "(212A)".
77 // Second, it would be nice to allow Unicode marks as well as letters,
78 // but marks include combining marks, and then we must deal not
79 // only with case folding but also normalization: both U+00E9 ('é')
80 // and U+0065 U+0301 ('e' followed by combining acute accent)
81 // look the same on the page and are treated by some file systems
82 // as the same path. If we do allow Unicode marks in paths, there
83 // must be some kind of normalization to allow only one canonical
84 // encoding of any character used in an import path.
89 // This file essentially defines the set of valid import paths for the go command.
90 // There are many subtle considerations, including Unicode ambiguity,
91 // security, network, and file system representations.
93 // This file also defines the set of valid module path and version combinations,
94 // another topic with many subtle considerations.
96 // Changes to the semantics in this file require approval from rsc.
106 "golang.org/x/mod/semver"
107 errors "golang.org/x/xerrors"
110 // A Version (for clients, a module.Version) is defined by a module path and version pair.
111 // These are stored in their plain (unescaped) form.
112 type Version struct {
113 // Path is a module path, like "golang.org/x/text" or "rsc.io/quote/v2".
116 // Version is usually a semantic version in canonical form.
117 // There are three exceptions to this general rule.
118 // First, the top-level target of a build has no specific version
119 // and uses Version = "".
120 // Second, during MVS calculations the version "none" is used
121 // to represent the decision to take no version of a given module.
122 // Third, filesystem paths found in "replace" directives are
123 // represented by a path with an empty version.
124 Version string `json:",omitempty"`
127 // String returns a representation of the Version suitable for logging
128 // (Path@Version, or just Path if Version is empty).
129 func (m Version) String() string {
133 return m.Path + "@" + m.Version
136 // A ModuleError indicates an error specific to a module.
137 type ModuleError struct {
143 // VersionError returns a ModuleError derived from a Version and error,
144 // or err itself if it is already such an error.
145 func VersionError(v Version, err error) error {
146 var mErr *ModuleError
147 if errors.As(err, &mErr) && mErr.Path == v.Path && mErr.Version == v.Version {
157 func (e *ModuleError) Error() string {
158 if v, ok := e.Err.(*InvalidVersionError); ok {
159 return fmt.Sprintf("%s@%s: invalid %s: %v", e.Path, v.Version, v.noun(), v.Err)
162 return fmt.Sprintf("%s@%s: %v", e.Path, e.Version, e.Err)
164 return fmt.Sprintf("module %s: %v", e.Path, e.Err)
167 func (e *ModuleError) Unwrap() error { return e.Err }
169 // An InvalidVersionError indicates an error specific to a version, with the
170 // module path unknown or specified externally.
172 // A ModuleError may wrap an InvalidVersionError, but an InvalidVersionError
173 // must not wrap a ModuleError.
174 type InvalidVersionError struct {
180 // noun returns either "version" or "pseudo-version", depending on whether
181 // e.Version is a pseudo-version.
182 func (e *InvalidVersionError) noun() string {
184 return "pseudo-version"
189 func (e *InvalidVersionError) Error() string {
190 return fmt.Sprintf("%s %q invalid: %s", e.noun(), e.Version, e.Err)
193 func (e *InvalidVersionError) Unwrap() error { return e.Err }
195 // Check checks that a given module path, version pair is valid.
196 // In addition to the path being a valid module path
197 // and the version being a valid semantic version,
198 // the two must correspond.
199 // For example, the path "yaml/v2" only corresponds to
200 // semantic versions beginning with "v2.".
201 func Check(path, version string) error {
202 if err := CheckPath(path); err != nil {
205 if !semver.IsValid(version) {
208 Err: &InvalidVersionError{Version: version, Err: errors.New("not a semantic version")},
211 _, pathMajor, _ := SplitPathVersion(path)
212 if err := CheckPathMajor(version, pathMajor); err != nil {
213 return &ModuleError{Path: path, Err: err}
218 // firstPathOK reports whether r can appear in the first element of a module path.
219 // The first element of the path must be an LDH domain name, at least for now.
220 // To avoid case ambiguity, the domain name must be entirely lower case.
221 func firstPathOK(r rune) bool {
222 return r == '-' || r == '.' ||
223 '0' <= r && r <= '9' ||
227 // pathOK reports whether r can appear in an import path element.
228 // Paths can be ASCII letters, ASCII digits, and limited ASCII punctuation: - . _ and ~.
229 // This matches what "go get" has historically recognized in import paths.
230 // TODO(rsc): We would like to allow Unicode letters, but that requires additional
231 // care in the safe encoding (see "escaped paths" above).
232 func pathOK(r rune) bool {
233 if r < utf8.RuneSelf {
234 return r == '-' || r == '.' || r == '_' || r == '~' ||
235 '0' <= r && r <= '9' ||
236 'A' <= r && r <= 'Z' ||
242 // fileNameOK reports whether r can appear in a file name.
243 // For now we allow all Unicode letters but otherwise limit to pathOK plus a few more punctuation characters.
244 // If we expand the set of allowed characters here, we have to
245 // work harder at detecting potential case-folding and normalization collisions.
246 // See note about "escaped paths" above.
247 func fileNameOK(r rune) bool {
248 if r < utf8.RuneSelf {
249 // Entire set of ASCII punctuation, from which we remove characters:
250 // ! " # $ % & ' ( ) * + , - . / : ; < = > ? @ [ \ ] ^ _ ` { | } ~
251 // We disallow some shell special characters: " ' * < > ? ` |
252 // (Note that some of those are disallowed by the Windows file system as well.)
253 // We also disallow path separators / : and \ (fileNameOK is only called on path element characters).
254 // We allow spaces (U+0020) in file names.
255 const allowed = "!#$%&()+,-.=@[]^_{}~ "
256 if '0' <= r && r <= '9' || 'A' <= r && r <= 'Z' || 'a' <= r && r <= 'z' {
259 for i := 0; i < len(allowed); i++ {
260 if rune(allowed[i]) == r {
266 // It may be OK to add more ASCII punctuation here, but only carefully.
267 // For example Windows disallows < > \, and macOS disallows :, so we must not allow those.
268 return unicode.IsLetter(r)
271 // CheckPath checks that a module path is valid.
272 // A valid module path is a valid import path, as checked by CheckImportPath,
273 // with two additional constraints.
274 // First, the leading path element (up to the first slash, if any),
275 // by convention a domain name, must contain only lower-case ASCII letters,
276 // ASCII digits, dots (U+002E), and dashes (U+002D);
277 // it must contain at least one dot and cannot start with a dash.
278 // Second, for a final path element of the form /vN, where N looks numeric
279 // (ASCII digits and dots) must not begin with a leading zero, must not be /v1,
280 // and must not contain any dots. For paths beginning with "gopkg.in/",
281 // this second requirement is replaced by a requirement that the path
282 // follow the gopkg.in server's conventions.
283 func CheckPath(path string) error {
284 if err := checkPath(path, false); err != nil {
285 return fmt.Errorf("malformed module path %q: %v", path, err)
287 i := strings.Index(path, "/")
292 return fmt.Errorf("malformed module path %q: leading slash", path)
294 if !strings.Contains(path[:i], ".") {
295 return fmt.Errorf("malformed module path %q: missing dot in first path element", path)
298 return fmt.Errorf("malformed module path %q: leading dash in first path element", path)
300 for _, r := range path[:i] {
302 return fmt.Errorf("malformed module path %q: invalid char %q in first path element", path, r)
305 if _, _, ok := SplitPathVersion(path); !ok {
306 return fmt.Errorf("malformed module path %q: invalid version", path)
311 // CheckImportPath checks that an import path is valid.
313 // A valid import path consists of one or more valid path elements
314 // separated by slashes (U+002F). (It must not begin with nor end in a slash.)
316 // A valid path element is a non-empty string made up of
317 // ASCII letters, ASCII digits, and limited ASCII punctuation: - . _ and ~.
318 // It must not begin or end with a dot (U+002E), nor contain two dots in a row.
320 // The element prefix up to the first dot must not be a reserved file name
321 // on Windows, regardless of case (CON, com1, NuL, and so on). The element
322 // must not have a suffix of a tilde followed by one or more ASCII digits
323 // (to exclude paths elements that look like Windows short-names).
325 // CheckImportPath may be less restrictive in the future, but see the
326 // top-level package documentation for additional information about
327 // subtleties of Unicode.
328 func CheckImportPath(path string) error {
329 if err := checkPath(path, false); err != nil {
330 return fmt.Errorf("malformed import path %q: %v", path, err)
335 // checkPath checks that a general path is valid.
336 // It returns an error describing why but not mentioning path.
337 // Because these checks apply to both module paths and import paths,
338 // the caller is expected to add the "malformed ___ path %q: " prefix.
339 // fileName indicates whether the final element of the path is a file name
340 // (as opposed to a directory name).
341 func checkPath(path string, fileName bool) error {
342 if !utf8.ValidString(path) {
343 return fmt.Errorf("invalid UTF-8")
346 return fmt.Errorf("empty string")
349 return fmt.Errorf("leading dash")
351 if strings.Contains(path, "//") {
352 return fmt.Errorf("double slash")
354 if path[len(path)-1] == '/' {
355 return fmt.Errorf("trailing slash")
358 for i, r := range path {
360 if err := checkElem(path[elemStart:i], fileName); err != nil {
366 if err := checkElem(path[elemStart:], fileName); err != nil {
372 // checkElem checks whether an individual path element is valid.
373 // fileName indicates whether the element is a file name (not a directory name).
374 func checkElem(elem string, fileName bool) error {
376 return fmt.Errorf("empty path element")
378 if strings.Count(elem, ".") == len(elem) {
379 return fmt.Errorf("invalid path element %q", elem)
381 if elem[0] == '.' && !fileName {
382 return fmt.Errorf("leading dot in path element")
384 if elem[len(elem)-1] == '.' {
385 return fmt.Errorf("trailing dot in path element")
391 for _, r := range elem {
393 return fmt.Errorf("invalid char %q", r)
397 // Windows disallows a bunch of path elements, sadly.
398 // See https://docs.microsoft.com/en-us/windows/desktop/fileio/naming-a-file
400 if i := strings.Index(short, "."); i >= 0 {
403 for _, bad := range badWindowsNames {
404 if strings.EqualFold(bad, short) {
405 return fmt.Errorf("%q disallowed as path element component on Windows", short)
410 // don't check for Windows short-names in file names. They're
411 // only an issue for import paths.
415 // Reject path components that look like Windows short-names.
416 // Those usually end in a tilde followed by one or more ASCII digits.
417 if tilde := strings.LastIndexByte(short, '~'); tilde >= 0 && tilde < len(short)-1 {
418 suffix := short[tilde+1:]
419 suffixIsDigits := true
420 for _, r := range suffix {
421 if r < '0' || r > '9' {
422 suffixIsDigits = false
427 return fmt.Errorf("trailing tilde and digits in path element")
434 // CheckFilePath checks that a slash-separated file path is valid.
435 // The definition of a valid file path is the same as the definition
436 // of a valid import path except that the set of allowed characters is larger:
437 // all Unicode letters, ASCII digits, the ASCII space character (U+0020),
438 // and the ASCII punctuation characters
439 // “!#$%&()+,-.=@[]^_{}~”.
440 // (The excluded punctuation characters, " * < > ? ` ' | / \ and :,
441 // have special meanings in certain shells or operating systems.)
443 // CheckFilePath may be less restrictive in the future, but see the
444 // top-level package documentation for additional information about
445 // subtleties of Unicode.
446 func CheckFilePath(path string) error {
447 if err := checkPath(path, true); err != nil {
448 return fmt.Errorf("malformed file path %q: %v", path, err)
453 // badWindowsNames are the reserved file path elements on Windows.
454 // See https://docs.microsoft.com/en-us/windows/desktop/fileio/naming-a-file
455 var badWindowsNames = []string{
480 // SplitPathVersion returns prefix and major version such that prefix+pathMajor == path
481 // and version is either empty or "/vN" for N >= 2.
482 // As a special case, gopkg.in paths are recognized directly;
483 // they require ".vN" instead of "/vN", and for all N, not just N >= 2.
484 // SplitPathVersion returns with ok = false when presented with
485 // a path whose last path element does not satisfy the constraints
486 // applied by CheckPath, such as "example.com/pkg/v1" or "example.com/pkg/v1.2".
487 func SplitPathVersion(path string) (prefix, pathMajor string, ok bool) {
488 if strings.HasPrefix(path, "gopkg.in/") {
489 return splitGopkgIn(path)
494 for i > 0 && ('0' <= path[i-1] && path[i-1] <= '9' || path[i-1] == '.') {
495 if path[i-1] == '.' {
500 if i <= 1 || i == len(path) || path[i-1] != 'v' || path[i-2] != '/' {
501 return path, "", true
503 prefix, pathMajor = path[:i-2], path[i-2:]
504 if dot || len(pathMajor) <= 2 || pathMajor[2] == '0' || pathMajor == "/v1" {
505 return path, "", false
507 return prefix, pathMajor, true
510 // splitGopkgIn is like SplitPathVersion but only for gopkg.in paths.
511 func splitGopkgIn(path string) (prefix, pathMajor string, ok bool) {
512 if !strings.HasPrefix(path, "gopkg.in/") {
513 return path, "", false
516 if strings.HasSuffix(path, "-unstable") {
517 i -= len("-unstable")
519 for i > 0 && ('0' <= path[i-1] && path[i-1] <= '9') {
522 if i <= 1 || path[i-1] != 'v' || path[i-2] != '.' {
523 // All gopkg.in paths must end in vN for some N.
524 return path, "", false
526 prefix, pathMajor = path[:i-2], path[i-2:]
527 if len(pathMajor) <= 2 || pathMajor[2] == '0' && pathMajor != ".v0" {
528 return path, "", false
530 return prefix, pathMajor, true
533 // MatchPathMajor reports whether the semantic version v
534 // matches the path major version pathMajor.
536 // MatchPathMajor returns true if and only if CheckPathMajor returns nil.
537 func MatchPathMajor(v, pathMajor string) bool {
538 return CheckPathMajor(v, pathMajor) == nil
541 // CheckPathMajor returns a non-nil error if the semantic version v
542 // does not match the path major version pathMajor.
543 func CheckPathMajor(v, pathMajor string) error {
544 // TODO(jayconrod): return errors or panic for invalid inputs. This function
545 // (and others) was covered by integration tests for cmd/go, and surrounding
546 // code protected against invalid inputs like non-canonical versions.
547 if strings.HasPrefix(pathMajor, ".v") && strings.HasSuffix(pathMajor, "-unstable") {
548 pathMajor = strings.TrimSuffix(pathMajor, "-unstable")
550 if strings.HasPrefix(v, "v0.0.0-") && pathMajor == ".v1" {
551 // Allow old bug in pseudo-versions that generated v0.0.0- pseudoversion for gopkg .v1.
552 // For example, gopkg.in/yaml.v2@v2.2.1's go.mod requires gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405.
557 if m == "v0" || m == "v1" || semver.Build(v) == "+incompatible" {
560 pathMajor = "v0 or v1"
561 } else if pathMajor[0] == '/' || pathMajor[0] == '.' {
562 if m == pathMajor[1:] {
565 pathMajor = pathMajor[1:]
567 return &InvalidVersionError{
569 Err: fmt.Errorf("should be %s, not %s", pathMajor, semver.Major(v)),
573 // PathMajorPrefix returns the major-version tag prefix implied by pathMajor.
574 // An empty PathMajorPrefix allows either v0 or v1.
576 // Note that MatchPathMajor may accept some versions that do not actually begin
577 // with this prefix: namely, it accepts a 'v0.0.0-' prefix for a '.v1'
578 // pathMajor, even though that pathMajor implies 'v1' tagging.
579 func PathMajorPrefix(pathMajor string) string {
583 if pathMajor[0] != '/' && pathMajor[0] != '.' {
584 panic("pathMajor suffix " + pathMajor + " passed to PathMajorPrefix lacks separator")
586 if strings.HasPrefix(pathMajor, ".v") && strings.HasSuffix(pathMajor, "-unstable") {
587 pathMajor = strings.TrimSuffix(pathMajor, "-unstable")
590 if m != semver.Major(m) {
591 panic("pathMajor suffix " + pathMajor + "passed to PathMajorPrefix is not a valid major version")
596 // CanonicalVersion returns the canonical form of the version string v.
597 // It is the same as semver.Canonical(v) except that it preserves the special build suffix "+incompatible".
598 func CanonicalVersion(v string) string {
599 cv := semver.Canonical(v)
600 if semver.Build(v) == "+incompatible" {
601 cv += "+incompatible"
606 // Sort sorts the list by Path, breaking ties by comparing Version fields.
607 // The Version fields are interpreted as semantic versions (using semver.Compare)
608 // optionally followed by a tie-breaking suffix introduced by a slash character,
609 // like in "v0.0.1/go.mod".
610 func Sort(list []Version) {
611 sort.Slice(list, func(i, j int) bool {
614 if mi.Path != mj.Path {
615 return mi.Path < mj.Path
617 // To help go.sum formatting, allow version/file.
618 // Compare semver prefix by semver rules,
619 // file by string order.
623 if k := strings.Index(vi, "/"); k >= 0 {
624 vi, fi = vi[:k], vi[k:]
626 if k := strings.Index(vj, "/"); k >= 0 {
627 vj, fj = vj[:k], vj[k:]
630 return semver.Compare(vi, vj) < 0
636 // EscapePath returns the escaped form of the given module path.
637 // It fails if the module path is invalid.
638 func EscapePath(path string) (escaped string, err error) {
639 if err := CheckPath(path); err != nil {
643 return escapeString(path)
646 // EscapeVersion returns the escaped form of the given module version.
647 // Versions are allowed to be in non-semver form but must be valid file names
648 // and not contain exclamation marks.
649 func EscapeVersion(v string) (escaped string, err error) {
650 if err := checkElem(v, true); err != nil || strings.Contains(v, "!") {
651 return "", &InvalidVersionError{
653 Err: fmt.Errorf("disallowed version string"),
656 return escapeString(v)
659 func escapeString(s string) (escaped string, err error) {
661 for _, r := range s {
662 if r == '!' || r >= utf8.RuneSelf {
663 // This should be disallowed by CheckPath, but diagnose anyway.
664 // The correctness of the escaping loop below depends on it.
665 return "", fmt.Errorf("internal error: inconsistency in EscapePath")
667 if 'A' <= r && r <= 'Z' {
677 for _, r := range s {
678 if 'A' <= r && r <= 'Z' {
679 buf = append(buf, '!', byte(r+'a'-'A'))
681 buf = append(buf, byte(r))
684 return string(buf), nil
687 // UnescapePath returns the module path for the given escaped path.
688 // It fails if the escaped path is invalid or describes an invalid path.
689 func UnescapePath(escaped string) (path string, err error) {
690 path, ok := unescapeString(escaped)
692 return "", fmt.Errorf("invalid escaped module path %q", escaped)
694 if err := CheckPath(path); err != nil {
695 return "", fmt.Errorf("invalid escaped module path %q: %v", escaped, err)
700 // UnescapeVersion returns the version string for the given escaped version.
701 // It fails if the escaped form is invalid or describes an invalid version.
702 // Versions are allowed to be in non-semver form but must be valid file names
703 // and not contain exclamation marks.
704 func UnescapeVersion(escaped string) (v string, err error) {
705 v, ok := unescapeString(escaped)
707 return "", fmt.Errorf("invalid escaped version %q", escaped)
709 if err := checkElem(v, true); err != nil {
710 return "", fmt.Errorf("invalid escaped version %q: %v", v, err)
715 func unescapeString(escaped string) (string, bool) {
719 for _, r := range escaped {
720 if r >= utf8.RuneSelf {
725 if r < 'a' || 'z' < r {
728 buf = append(buf, byte(r+'A'-'a'))
735 if 'A' <= r && r <= 'Z' {
738 buf = append(buf, byte(r))
743 return string(buf), true
746 // MatchPrefixPatterns reports whether any path prefix of target matches one of
747 // the glob patterns (as defined by path.Match) in the comma-separated globs
748 // list. This implements the algorithm used when matching a module path to the
749 // GOPRIVATE environment variable, as described by 'go help module-private'.
751 // It ignores any empty or malformed patterns in the list.
752 func MatchPrefixPatterns(globs, target string) bool {
754 // Extract next non-empty glob in comma-separated list.
756 if i := strings.Index(globs, ","); i >= 0 {
757 glob, globs = globs[:i], globs[i+1:]
759 glob, globs = globs, ""
765 // A glob with N+1 path elements (N slashes) needs to be matched
766 // against the first N+1 path elements of target,
767 // which end just before the N+1'th slash.
768 n := strings.Count(glob, "/")
770 // Walk target, counting slashes, truncating at the N+1'th slash.
771 for i := 0; i < len(target); i++ {
772 if target[i] == '/' {
781 // Not enough prefix elements.
784 matched, _ := path.Match(glob, prefix)