1 // Copyright (c) 2015, Daniel Martà <mvdan@mvdan.cc>
2 // See LICENSE for licensing information
19 const path = "tlds.go"
21 var tldsTmpl = template.Must(template.New("tlds").Parse(`// Generated by tldsgen
25 // TLDs is a sorted list of all public top-level domains.
27 // Sources:{{range $_, $url := .URLs}}
30 {{range $_, $tld := .TLDs}}` + "\t`" + `{{$tld}}` + "`" + `,
34 func cleanTld(tld string) string {
35 tld = strings.ToLower(tld)
36 if strings.HasPrefix(tld, "xn--") {
42 func fetchFromURL(url, pat string) {
44 log.Printf("Fetching %s", url)
45 resp, err := http.Get(url)
46 if err == nil && resp.StatusCode >= 400 {
47 err = errors.New(resp.Status)
53 defer resp.Body.Close()
54 scanner := bufio.NewScanner(resp.Body)
55 re := regexp.MustCompile(pat)
57 line := scanner.Text()
58 tld := re.FindString(line)
65 if err := scanner.Err(); err != nil {
72 tldChan = make(chan string)
73 errChan = make(chan error)
76 func tldList() ([]string, []string, error) {
78 fromURL := func(url, pat string) {
79 urls = append(urls, url)
81 go fetchFromURL(url, pat)
83 fromURL("https://data.iana.org/TLD/tlds-alpha-by-domain.txt",
85 fromURL("https://publicsuffix.org/list/effective_tld_names.dat",
88 tldSet := make(map[string]struct{})
93 case tld := <-tldChan:
94 tldSet[tld] = struct{}{}
95 case err := <-errChan:
104 return nil, nil, errors.New("there were some errors while fetching the TLDs")
107 tlds := make([]string, 0, len(tldSet))
108 for tld := range tldSet {
109 tlds = append(tlds, tld)
113 return tlds, urls, nil
116 func writeTlds(tlds, urls []string) error {
117 f, err := os.Create(path)
122 return tldsTmpl.Execute(f, struct {
132 tlds, urls, err := tldList()
134 log.Fatalf("Could not get TLD list: %v", err)
136 log.Printf("Generating %s...", path)
137 if err := writeTlds(tlds, urls); err != nil {
138 log.Fatalf("Could not write path: %v", err)