1 // Copyright (c) 2015, Daniel Martà <mvdan@mvdan.cc>
2 // See LICENSE for licensing information
25 matching = flag.String("m", "", "")
26 relaxed = flag.Bool("r", false, "")
27 fix = flag.Bool("fix", false, "")
32 p := func(format string, a ...interface{}) {
33 fmt.Fprintf(os.Stderr, format, a...)
35 p("Usage: xurls [-h] [files]\n\n")
36 p("If no files are given, it reads from standard input.\n\n")
37 p(" -m <regexp> only match urls whose scheme matches a regexp\n")
38 p(" example: 'https?://|mailto:'\n")
39 p(" -r also match urls without a scheme (relaxed)\n")
40 p(" -fix overwrite urls that redirect\n")
44 func scanPath(re *regexp.Regexp, path string) error {
48 f, err = os.Open(path)
54 bufr := bufio.NewReader(f)
55 var fixedBuf bytes.Buffer
59 line, err := bufr.ReadBytes('\n')
61 for _, pair := range re.FindAllIndex(line, -1) {
62 // The indexes are based on the original line.
65 match := line[pair[0]:pair[1]]
67 fmt.Printf("%s\n", match)
70 u, err := url.Parse(string(match))
77 // See if the URL redirects somewhere.
78 client := &http.Client{
79 Timeout: 10 * time.Second,
80 CheckRedirect: func(req *http.Request, via []*http.Request) error {
82 return errors.New("stopped after 10 redirects")
84 // Keep the fragment around.
85 req.URL.Fragment = u.Fragment
86 fixed = req.URL.String()
90 resp, err := client.Get(fixed)
94 if resp.StatusCode >= 400 {
95 broken = append(broken, string(match))
99 if fixed != string(match) {
100 // Replace the url, and update the offset.
101 newLine := line[:pair[0]]
102 newLine = append(newLine, fixed...)
103 newLine = append(newLine, line[pair[1]:]...)
104 offset += len(newLine) - len(line)
111 os.Stdout.Write(line)
118 } else if err != nil {
122 if anyFixed && path != "-" {
124 // Overwrite the file, if we weren't reading stdin. Report its
127 if err := ioutil.WriteFile(path, fixedBuf.Bytes(), 0666); err != nil {
132 return fmt.Errorf("found %d broken urls in %q:\n%s", len(broken),
133 path, strings.Join(broken, "\n"))
138 func main() { os.Exit(main1()) }
142 if *relaxed && *matching != "" {
143 fmt.Fprintln(os.Stderr, "-r and -m at the same time don't make much sense")
146 var re *regexp.Regexp
149 } else if *matching != "" {
151 if re, err = xurls.StrictMatchingScheme(*matching); err != nil {
152 fmt.Fprintln(os.Stderr, err)
162 for _, path := range args {
163 if err := scanPath(re, path); err != nil {
164 fmt.Fprintln(os.Stderr, err)