.config/coc/extensions/coc-go-data/tools/pkg/mod/golang.org/x/mod@v0.4.1/zip/zip_test.go

   1 // Copyright 2019 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 package zip_test
   6
   7 import (
   8         "archive/zip"
   9         "bytes"
  10         "crypto/sha256"
  11         "encoding/hex"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "os"
  16         "os/exec"
  17         "path"
  18         "path/filepath"
  19         "runtime"
  20         "strings"
  21         "sync/atomic"
  22         "testing"
  23         "time"
  24
  25         "golang.org/x/mod/module"
  26         "golang.org/x/mod/sumdb/dirhash"
  27         modzip "golang.org/x/mod/zip"
  28         "golang.org/x/tools/txtar"
  29 )
  30
  31 const emptyHash = "h1:47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="
  32
  33 type testParams struct {
  34         path, version, wantErr, hash string
  35         archive                      *txtar.Archive
  36 }
  37
  38 // readTest loads a test from a txtar file. The comment section of the file
  39 // should contain lines with key=value pairs. Valid keys are the field names
  40 // from testParams.
  41 func readTest(file string) (testParams, error) {
  42         var test testParams
  43         var err error
  44         test.archive, err = txtar.ParseFile(file)
  45         if err != nil {
  46                 return testParams{}, err
  47         }
  48
  49         lines := strings.Split(string(test.archive.Comment), "\n")
  50         for n, line := range lines {
  51                 n++ // report line numbers starting with 1
  52                 if i := strings.IndexByte(line, '#'); i >= 0 {
  53                         line = line[:i]
  54                 }
  55                 line = strings.TrimSpace(line)
  56                 if line == "" {
  57                         continue
  58                 }
  59                 eq := strings.IndexByte(line, '=')
  60                 if eq < 0 {
  61                         return testParams{}, fmt.Errorf("%s:%d: missing = separator", file, n)
  62                 }
  63                 key, value := strings.TrimSpace(line[:eq]), strings.TrimSpace(line[eq+1:])
  64                 switch key {
  65                 case "path":
  66                         test.path = value
  67                 case "version":
  68                         test.version = value
  69                 case "wantErr":
  70                         test.wantErr = value
  71                 case "hash":
  72                         test.hash = value
  73                 default:
  74                         return testParams{}, fmt.Errorf("%s:%d: unknown key %q", file, n, key)
  75                 }
  76         }
  77
  78         return test, nil
  79 }
  80
  81 func extractTxtarToTempDir(arc *txtar.Archive) (dir string, err error) {
  82         dir, err = ioutil.TempDir("", "zip_test-*")
  83         if err != nil {
  84                 return "", err
  85         }
  86         defer func() {
  87                 if err != nil {
  88                         os.RemoveAll(dir)
  89                 }
  90         }()
  91         for _, f := range arc.Files {
  92                 filePath := filepath.Join(dir, f.Name)
  93                 if err := os.MkdirAll(filepath.Dir(filePath), 0777); err != nil {
  94                         return "", err
  95                 }
  96                 if err := ioutil.WriteFile(filePath, f.Data, 0666); err != nil {
  97                         return "", err
  98                 }
  99         }
 100         return dir, nil
 101 }
 102
 103 func extractTxtarToTempZip(arc *txtar.Archive) (zipPath string, err error) {
 104         zipFile, err := ioutil.TempFile("", "zip_test-*.zip")
 105         if err != nil {
 106                 return "", err
 107         }
 108         defer func() {
 109                 if cerr := zipFile.Close(); err == nil && cerr != nil {
 110                         err = cerr
 111                 }
 112                 if err != nil {
 113                         os.Remove(zipFile.Name())
 114                 }
 115         }()
 116         zw := zip.NewWriter(zipFile)
 117         for _, f := range arc.Files {
 118                 zf, err := zw.Create(f.Name)
 119                 if err != nil {
 120                         return "", err
 121                 }
 122                 if _, err := zf.Write(f.Data); err != nil {
 123                         return "", err
 124                 }
 125         }
 126         if err := zw.Close(); err != nil {
 127                 return "", err
 128         }
 129         return zipFile.Name(), nil
 130 }
 131
 132 type fakeFile struct {
 133         name string
 134         size uint64
 135         data []byte // if nil, Open will access a sequence of 0-bytes
 136 }
 137
 138 func (f fakeFile) Path() string                { return f.name }
 139 func (f fakeFile) Lstat() (os.FileInfo, error) { return fakeFileInfo{f}, nil }
 140 func (f fakeFile) Open() (io.ReadCloser, error) {
 141         if f.data != nil {
 142                 return ioutil.NopCloser(bytes.NewReader(f.data)), nil
 143         }
 144         if f.size >= uint64(modzip.MaxZipFile<<1) {
 145                 return nil, fmt.Errorf("cannot open fakeFile of size %d", f.size)
 146         }
 147         return ioutil.NopCloser(io.LimitReader(zeroReader{}, int64(f.size))), nil
 148 }
 149
 150 type fakeFileInfo struct {
 151         f fakeFile
 152 }
 153
 154 func (fi fakeFileInfo) Name() string       { return path.Base(fi.f.name) }
 155 func (fi fakeFileInfo) Size() int64        { return int64(fi.f.size) }
 156 func (fi fakeFileInfo) Mode() os.FileMode  { return 0644 }
 157 func (fi fakeFileInfo) ModTime() time.Time { return time.Time{} }
 158 func (fi fakeFileInfo) IsDir() bool        { return false }
 159 func (fi fakeFileInfo) Sys() interface{}   { return nil }
 160
 161 type zeroReader struct{}
 162
 163 func (r zeroReader) Read(b []byte) (int, error) {
 164         for i := range b {
 165                 b[i] = 0
 166         }
 167         return len(b), nil
 168 }
 169
 170 func formatCheckedFiles(cf modzip.CheckedFiles) string {
 171         buf := &bytes.Buffer{}
 172         fmt.Fprintf(buf, "valid:\n")
 173         for _, f := range cf.Valid {
 174                 fmt.Fprintln(buf, f)
 175         }
 176         fmt.Fprintf(buf, "\nomitted:\n")
 177         for _, f := range cf.Omitted {
 178                 fmt.Fprintf(buf, "%s: %v\n", f.Path, f.Err)
 179         }
 180         fmt.Fprintf(buf, "\ninvalid:\n")
 181         for _, f := range cf.Invalid {
 182                 fmt.Fprintf(buf, "%s: %v\n", f.Path, f.Err)
 183         }
 184         return buf.String()
 185 }
 186
 187 // TestCheckFiles verifies behavior of CheckFiles. Note that CheckFiles is also
 188 // covered by TestCreate, TestCreateDir, and TestCreateSizeLimits, so this test
 189 // focuses on how multiple errors and omissions are reported, rather than trying
 190 // to cover every case.
 191 func TestCheckFiles(t *testing.T) {
 192         testPaths, err := filepath.Glob(filepath.FromSlash("testdata/check_files/*.txt"))
 193         if err != nil {
 194                 t.Fatal(err)
 195         }
 196         for _, testPath := range testPaths {
 197                 testPath := testPath
 198                 name := strings.TrimSuffix(filepath.Base(testPath), ".txt")
 199                 t.Run(name, func(t *testing.T) {
 200                         t.Parallel()
 201
 202                         // Load the test.
 203                         test, err := readTest(testPath)
 204                         if err != nil {
 205                                 t.Fatal(err)
 206                         }
 207                         files := make([]modzip.File, 0, len(test.archive.Files))
 208                         var want string
 209                         for _, tf := range test.archive.Files {
 210                                 if tf.Name == "want" {
 211                                         want = string(tf.Data)
 212                                         continue
 213                                 }
 214                                 files = append(files, fakeFile{
 215                                         name: tf.Name,
 216                                         size: uint64(len(tf.Data)),
 217                                         data: tf.Data,
 218                                 })
 219                         }
 220
 221                         // Check the files.
 222                         cf, _ := modzip.CheckFiles(files)
 223                         got := formatCheckedFiles(cf)
 224                         if got != want {
 225                                 t.Errorf("got:\n%s\n\nwant:\n%s", got, want)
 226                         }
 227
 228                         // Check that the error (if any) is just a list of invalid files.
 229                         // SizeError is not covered in this test.
 230                         var gotErr, wantErr string
 231                         if len(cf.Invalid) > 0 {
 232                                 wantErr = modzip.FileErrorList(cf.Invalid).Error()
 233                         }
 234                         if err := cf.Err(); err != nil {
 235                                 gotErr = err.Error()
 236                         }
 237                         if gotErr != wantErr {
 238                                 t.Errorf("got error:\n%s\n\nwant error:\n%s", gotErr, wantErr)
 239                         }
 240                 })
 241         }
 242 }
 243
 244 // TestCheckDir verifies behavior of the CheckDir function. Note that CheckDir
 245 // relies on CheckFiles and listFilesInDir (called by CreateFromDir), so this
 246 // test focuses on how multiple errors and omissions are reported, rather than
 247 // trying to cover every case.
 248 func TestCheckDir(t *testing.T) {
 249         testPaths, err := filepath.Glob(filepath.FromSlash("testdata/check_dir/*.txt"))
 250         if err != nil {
 251                 t.Fatal(err)
 252         }
 253         for _, testPath := range testPaths {
 254                 testPath := testPath
 255                 name := strings.TrimSuffix(filepath.Base(testPath), ".txt")
 256                 t.Run(name, func(t *testing.T) {
 257                         t.Parallel()
 258
 259                         // Load the test and extract the files to a temporary directory.
 260                         test, err := readTest(testPath)
 261                         if err != nil {
 262                                 t.Fatal(err)
 263                         }
 264                         var want string
 265                         for i, f := range test.archive.Files {
 266                                 if f.Name == "want" {
 267                                         want = string(f.Data)
 268                                         test.archive.Files = append(test.archive.Files[:i], test.archive.Files[i+1:]...)
 269                                         break
 270                                 }
 271                         }
 272                         tmpDir, err := extractTxtarToTempDir(test.archive)
 273                         if err != nil {
 274                                 t.Fatal(err)
 275                         }
 276                         defer func() {
 277                                 if err := os.RemoveAll(tmpDir); err != nil {
 278                                         t.Errorf("removing temp directory: %v", err)
 279                                 }
 280                         }()
 281
 282                         // Check the directory.
 283                         cf, err := modzip.CheckDir(tmpDir)
 284                         if err != nil && err.Error() != cf.Err().Error() {
 285                                 // I/O error
 286                                 t.Fatal(err)
 287                         }
 288                         rep := strings.NewReplacer(tmpDir, "$work", `'\''`, `'\''`, string(os.PathSeparator), "/")
 289                         got := rep.Replace(formatCheckedFiles(cf))
 290                         if got != want {
 291                                 t.Errorf("got:\n%s\n\nwant:\n%s", got, want)
 292                         }
 293
 294                         // Check that the error (if any) is just a list of invalid files.
 295                         // SizeError is not covered in this test.
 296                         var gotErr, wantErr string
 297                         if len(cf.Invalid) > 0 {
 298                                 wantErr = modzip.FileErrorList(cf.Invalid).Error()
 299                         }
 300                         if err := cf.Err(); err != nil {
 301                                 gotErr = err.Error()
 302                         }
 303                         if gotErr != wantErr {
 304                                 t.Errorf("got error:\n%s\n\nwant error:\n%s", gotErr, wantErr)
 305                         }
 306                 })
 307         }
 308 }
 309
 310 // TestCheckZip verifies behavior of CheckZip. Note that CheckZip is also
 311 // covered by TestUnzip, so this test focuses on how multiple errors are
 312 // reported, rather than trying to cover every case.
 313 func TestCheckZip(t *testing.T) {
 314         testPaths, err := filepath.Glob(filepath.FromSlash("testdata/check_zip/*.txt"))
 315         if err != nil {
 316                 t.Fatal(err)
 317         }
 318         for _, testPath := range testPaths {
 319                 testPath := testPath
 320                 name := strings.TrimSuffix(filepath.Base(testPath), ".txt")
 321                 t.Run(name, func(t *testing.T) {
 322                         t.Parallel()
 323
 324                         // Load the test and extract the files to a temporary zip file.
 325                         test, err := readTest(testPath)
 326                         if err != nil {
 327                                 t.Fatal(err)
 328                         }
 329                         var want string
 330                         for i, f := range test.archive.Files {
 331                                 if f.Name == "want" {
 332                                         want = string(f.Data)
 333                                         test.archive.Files = append(test.archive.Files[:i], test.archive.Files[i+1:]...)
 334                                         break
 335                                 }
 336                         }
 337                         tmpZipPath, err := extractTxtarToTempZip(test.archive)
 338                         if err != nil {
 339                                 t.Fatal(err)
 340                         }
 341                         defer func() {
 342                                 if err := os.Remove(tmpZipPath); err != nil {
 343                                         t.Errorf("removing temp zip file: %v", err)
 344                                 }
 345                         }()
 346
 347                         // Check the zip.
 348                         m := module.Version{Path: test.path, Version: test.version}
 349                         cf, err := modzip.CheckZip(m, tmpZipPath)
 350                         if err != nil && err.Error() != cf.Err().Error() {
 351                                 // I/O error
 352                                 t.Fatal(err)
 353                         }
 354                         got := formatCheckedFiles(cf)
 355                         if got != want {
 356                                 t.Errorf("got:\n%s\n\nwant:\n%s", got, want)
 357                         }
 358
 359                         // Check that the error (if any) is just a list of invalid files.
 360                         // SizeError is not covered in this test.
 361                         var gotErr, wantErr string
 362                         if len(cf.Invalid) > 0 {
 363                                 wantErr = modzip.FileErrorList(cf.Invalid).Error()
 364                         }
 365                         if err := cf.Err(); err != nil {
 366                                 gotErr = err.Error()
 367                         }
 368                         if gotErr != wantErr {
 369                                 t.Errorf("got error:\n%s\n\nwant error:\n%s", gotErr, wantErr)
 370                         }
 371                 })
 372         }
 373 }
 374
 375 func TestCreate(t *testing.T) {
 376         testDir := filepath.FromSlash("testdata/create")
 377         testInfos, err := ioutil.ReadDir(testDir)
 378         if err != nil {
 379                 t.Fatal(err)
 380         }
 381         for _, testInfo := range testInfos {
 382                 testInfo := testInfo
 383                 base := filepath.Base(testInfo.Name())
 384                 if filepath.Ext(base) != ".txt" {
 385                         continue
 386                 }
 387                 t.Run(base[:len(base)-len(".txt")], func(t *testing.T) {
 388                         t.Parallel()
 389
 390                         // Load the test.
 391                         testPath := filepath.Join(testDir, testInfo.Name())
 392                         test, err := readTest(testPath)
 393                         if err != nil {
 394                                 t.Fatal(err)
 395                         }
 396
 397                         // Write zip to temporary file.
 398                         tmpZip, err := ioutil.TempFile("", "TestCreate-*.zip")
 399                         if err != nil {
 400                                 t.Fatal(err)
 401                         }
 402                         tmpZipPath := tmpZip.Name()
 403                         defer func() {
 404                                 tmpZip.Close()
 405                                 os.Remove(tmpZipPath)
 406                         }()
 407                         m := module.Version{Path: test.path, Version: test.version}
 408                         files := make([]modzip.File, len(test.archive.Files))
 409                         for i, tf := range test.archive.Files {
 410                                 files[i] = fakeFile{
 411                                         name: tf.Name,
 412                                         size: uint64(len(tf.Data)),
 413                                         data: tf.Data,
 414                                 }
 415                         }
 416                         if err := modzip.Create(tmpZip, m, files); err != nil {
 417                                 if test.wantErr == "" {
 418                                         t.Fatalf("unexpected error: %v", err)
 419                                 } else if !strings.Contains(err.Error(), test.wantErr) {
 420                                         t.Fatalf("got error %q; want error containing %q", err.Error(), test.wantErr)
 421                                 } else {
 422                                         return
 423                                 }
 424                         } else if test.wantErr != "" {
 425                                 t.Fatalf("unexpected success; wanted error containing %q", test.wantErr)
 426                         }
 427                         if err := tmpZip.Close(); err != nil {
 428                                 t.Fatal(err)
 429                         }
 430
 431                         // Hash zip file, compare with known value.
 432                         if hash, err := dirhash.HashZip(tmpZipPath, dirhash.Hash1); err != nil {
 433                                 t.Fatal(err)
 434                         } else if hash != test.hash {
 435                                 t.Fatalf("got hash: %q\nwant: %q", hash, test.hash)
 436                         }
 437                 })
 438         }
 439 }
 440
 441 func TestCreateFromDir(t *testing.T) {
 442         testDir := filepath.FromSlash("testdata/create_from_dir")
 443         testInfos, err := ioutil.ReadDir(testDir)
 444         if err != nil {
 445                 t.Fatal(err)
 446         }
 447         for _, testInfo := range testInfos {
 448                 testInfo := testInfo
 449                 base := filepath.Base(testInfo.Name())
 450                 if filepath.Ext(base) != ".txt" {
 451                         continue
 452                 }
 453                 t.Run(base[:len(base)-len(".txt")], func(t *testing.T) {
 454                         t.Parallel()
 455
 456                         // Load the test.
 457                         testPath := filepath.Join(testDir, testInfo.Name())
 458                         test, err := readTest(testPath)
 459                         if err != nil {
 460                                 t.Fatal(err)
 461                         }
 462
 463                         // Write files to a temporary directory.
 464                         tmpDir, err := extractTxtarToTempDir(test.archive)
 465                         if err != nil {
 466                                 t.Fatal(err)
 467                         }
 468                         defer func() {
 469                                 if err := os.RemoveAll(tmpDir); err != nil {
 470                                         t.Errorf("removing temp directory: %v", err)
 471                                 }
 472                         }()
 473
 474                         // Create zip from the directory.
 475                         tmpZip, err := ioutil.TempFile("", "TestCreateFromDir-*.zip")
 476                         if err != nil {
 477                                 t.Fatal(err)
 478                         }
 479                         tmpZipPath := tmpZip.Name()
 480                         defer func() {
 481                                 tmpZip.Close()
 482                                 os.Remove(tmpZipPath)
 483                         }()
 484                         m := module.Version{Path: test.path, Version: test.version}
 485                         if err := modzip.CreateFromDir(tmpZip, m, tmpDir); err != nil {
 486                                 if test.wantErr == "" {
 487                                         t.Fatalf("unexpected error: %v", err)
 488                                 } else if !strings.Contains(err.Error(), test.wantErr) {
 489                                         t.Fatalf("got error %q; want error containing %q", err, test.wantErr)
 490                                 } else {
 491                                         return
 492                                 }
 493                         } else if test.wantErr != "" {
 494                                 t.Fatalf("unexpected success; want error containing %q", test.wantErr)
 495                         }
 496
 497                         // Hash zip file, compare with known value.
 498                         if hash, err := dirhash.HashZip(tmpZipPath, dirhash.Hash1); err != nil {
 499                                 t.Fatal(err)
 500                         } else if hash != test.hash {
 501                                 t.Fatalf("got hash: %q\nwant: %q", hash, test.hash)
 502                         }
 503                 })
 504         }
 505 }
 506
 507 func TestCreateFromDirSpecial(t *testing.T) {
 508         for _, test := range []struct {
 509                 desc     string
 510                 setup    func(t *testing.T, tmpDir string) string
 511                 wantHash string
 512         }{
 513                 {
 514                         desc: "ignore_empty_dir",
 515                         setup: func(t *testing.T, tmpDir string) string {
 516                                 if err := os.Mkdir(filepath.Join(tmpDir, "empty"), 0777); err != nil {
 517                                         t.Fatal(err)
 518                                 }
 519                                 return tmpDir
 520                         },
 521                         wantHash: emptyHash,
 522                 }, {
 523                         desc: "ignore_symlink",
 524                         setup: func(t *testing.T, tmpDir string) string {
 525                                 if err := os.Symlink(tmpDir, filepath.Join(tmpDir, "link")); err != nil {
 526                                         switch runtime.GOOS {
 527                                         case "plan9", "windows":
 528                                                 t.Skipf("could not create symlink: %v", err)
 529                                         default:
 530                                                 t.Fatal(err)
 531                                         }
 532                                 }
 533                                 return tmpDir
 534                         },
 535                         wantHash: emptyHash,
 536                 }, {
 537                         desc: "dir_is_vendor",
 538                         setup: func(t *testing.T, tmpDir string) string {
 539                                 vendorDir := filepath.Join(tmpDir, "vendor")
 540                                 if err := os.Mkdir(vendorDir, 0777); err != nil {
 541                                         t.Fatal(err)
 542                                 }
 543                                 goModData := []byte("module example.com/m\n\ngo 1.13\n")
 544                                 if err := ioutil.WriteFile(filepath.Join(vendorDir, "go.mod"), goModData, 0666); err != nil {
 545                                         t.Fatal(err)
 546                                 }
 547                                 return vendorDir
 548                         },
 549                         wantHash: "h1:XduFAgX/GaspZa8Jv4pfzoGEzNaU/r88PiCunijw5ok=",
 550                 },
 551         } {
 552                 t.Run(test.desc, func(t *testing.T) {
 553                         tmpDir, err := ioutil.TempDir("", "TestCreateFromDirSpecial-"+test.desc)
 554                         if err != nil {
 555                                 t.Fatal(err)
 556                         }
 557                         defer os.RemoveAll(tmpDir)
 558                         dir := test.setup(t, tmpDir)
 559
 560                         tmpZipFile, err := ioutil.TempFile("", "TestCreateFromDir-*.zip")
 561                         if err != nil {
 562                                 t.Fatal(err)
 563                         }
 564                         tmpZipPath := tmpZipFile.Name()
 565                         defer func() {
 566                                 tmpZipFile.Close()
 567                                 os.Remove(tmpZipPath)
 568                         }()
 569
 570                         m := module.Version{Path: "example.com/m", Version: "v1.0.0"}
 571                         if err := modzip.CreateFromDir(tmpZipFile, m, dir); err != nil {
 572                                 t.Fatal(err)
 573                         }
 574                         if err := tmpZipFile.Close(); err != nil {
 575                                 t.Fatal(err)
 576                         }
 577
 578                         if hash, err := dirhash.HashZip(tmpZipPath, dirhash.Hash1); err != nil {
 579                                 t.Fatal(err)
 580                         } else if hash != test.wantHash {
 581                                 t.Fatalf("got hash %q; want %q", hash, emptyHash)
 582                         }
 583                 })
 584         }
 585 }
 586
 587 func TestUnzip(t *testing.T) {
 588         testDir := filepath.FromSlash("testdata/unzip")
 589         testInfos, err := ioutil.ReadDir(testDir)
 590         if err != nil {
 591                 t.Fatal(err)
 592         }
 593         for _, testInfo := range testInfos {
 594                 base := filepath.Base(testInfo.Name())
 595                 if filepath.Ext(base) != ".txt" {
 596                         continue
 597                 }
 598                 t.Run(base[:len(base)-len(".txt")], func(t *testing.T) {
 599                         // Load the test.
 600                         testPath := filepath.Join(testDir, testInfo.Name())
 601                         test, err := readTest(testPath)
 602                         if err != nil {
 603                                 t.Fatal(err)
 604                         }
 605
 606                         // Convert txtar to temporary zip file.
 607                         tmpZipPath, err := extractTxtarToTempZip(test.archive)
 608                         if err != nil {
 609                                 t.Fatal(err)
 610                         }
 611                         defer func() {
 612                                 if err := os.Remove(tmpZipPath); err != nil {
 613                                         t.Errorf("removing temp zip file: %v", err)
 614                                 }
 615                         }()
 616
 617                         // Extract to a temporary directory.
 618                         tmpDir, err := ioutil.TempDir("", "TestUnzip")
 619                         if err != nil {
 620                                 t.Fatal(err)
 621                         }
 622                         defer os.RemoveAll(tmpDir)
 623                         m := module.Version{Path: test.path, Version: test.version}
 624                         if err := modzip.Unzip(tmpDir, m, tmpZipPath); err != nil {
 625                                 if test.wantErr == "" {
 626                                         t.Fatalf("unexpected error: %v", err)
 627                                 } else if !strings.Contains(err.Error(), test.wantErr) {
 628                                         t.Fatalf("got error %q; want error containing %q", err.Error(), test.wantErr)
 629                                 } else {
 630                                         return
 631                                 }
 632                         } else if test.wantErr != "" {
 633                                 t.Fatalf("unexpected success; wanted error containing %q", test.wantErr)
 634                         }
 635
 636                         // Hash the directory, compare to known value.
 637                         prefix := fmt.Sprintf("%s@%s/", test.path, test.version)
 638                         if hash, err := dirhash.HashDir(tmpDir, prefix, dirhash.Hash1); err != nil {
 639                                 t.Fatal(err)
 640                         } else if hash != test.hash {
 641                                 t.Fatalf("got hash %q\nwant: %q", hash, test.hash)
 642                         }
 643                 })
 644         }
 645 }
 646
 647 type sizeLimitTest struct {
 648         desc              string
 649         files             []modzip.File
 650         wantErr           string
 651         wantCheckFilesErr string
 652         wantCreateErr     string
 653         wantCheckZipErr   string
 654         wantUnzipErr      string
 655 }
 656
 657 // sizeLimitTests is shared by TestCreateSizeLimits and TestUnzipSizeLimits.
 658 var sizeLimitTests = [...]sizeLimitTest{
 659         {
 660                 desc: "one_large",
 661                 files: []modzip.File{fakeFile{
 662                         name: "large.go",
 663                         size: modzip.MaxZipFile,
 664                 }},
 665         }, {
 666                 desc: "one_too_large",
 667                 files: []modzip.File{fakeFile{
 668                         name: "large.go",
 669                         size: modzip.MaxZipFile + 1,
 670                 }},
 671                 wantCheckFilesErr: "module source tree too large",
 672                 wantCreateErr:     "module source tree too large",
 673                 wantCheckZipErr:   "total uncompressed size of module contents too large",
 674                 wantUnzipErr:      "total uncompressed size of module contents too large",
 675         }, {
 676                 desc: "total_large",
 677                 files: []modzip.File{
 678                         fakeFile{
 679                                 name: "small.go",
 680                                 size: 10,
 681                         },
 682                         fakeFile{
 683                                 name: "large.go",
 684                                 size: modzip.MaxZipFile - 10,
 685                         },
 686                 },
 687         }, {
 688                 desc: "total_too_large",
 689                 files: []modzip.File{
 690                         fakeFile{
 691                                 name: "small.go",
 692                                 size: 10,
 693                         },
 694                         fakeFile{
 695                                 name: "large.go",
 696                                 size: modzip.MaxZipFile - 9,
 697                         },
 698                 },
 699                 wantCheckFilesErr: "module source tree too large",
 700                 wantCreateErr:     "module source tree too large",
 701                 wantCheckZipErr:   "total uncompressed size of module contents too large",
 702                 wantUnzipErr:      "total uncompressed size of module contents too large",
 703         }, {
 704                 desc: "large_gomod",
 705                 files: []modzip.File{fakeFile{
 706                         name: "go.mod",
 707                         size: modzip.MaxGoMod,
 708                 }},
 709         }, {
 710                 desc: "too_large_gomod",
 711                 files: []modzip.File{fakeFile{
 712                         name: "go.mod",
 713                         size: modzip.MaxGoMod + 1,
 714                 }},
 715                 wantErr: "go.mod file too large",
 716         }, {
 717                 desc: "large_license",
 718                 files: []modzip.File{fakeFile{
 719                         name: "LICENSE",
 720                         size: modzip.MaxLICENSE,
 721                 }},
 722         }, {
 723                 desc: "too_large_license",
 724                 files: []modzip.File{fakeFile{
 725                         name: "LICENSE",
 726                         size: modzip.MaxLICENSE + 1,
 727                 }},
 728                 wantErr: "LICENSE file too large",
 729         },
 730 }
 731
 732 var sizeLimitVersion = module.Version{Path: "example.com/large", Version: "v1.0.0"}
 733
 734 func TestCreateSizeLimits(t *testing.T) {
 735         if testing.Short() {
 736                 t.Skip("creating large files takes time")
 737         }
 738         tests := append(sizeLimitTests[:], sizeLimitTest{
 739                 // negative file size may happen when size is represented as uint64
 740                 // but is cast to int64, as is the case in zip files.
 741                 desc: "negative",
 742                 files: []modzip.File{fakeFile{
 743                         name: "neg.go",
 744                         size: 0x8000000000000000,
 745                 }},
 746                 wantErr: "module source tree too large",
 747         }, sizeLimitTest{
 748                 desc: "size_is_a_lie",
 749                 files: []modzip.File{fakeFile{
 750                         name: "lie.go",
 751                         size: 1,
 752                         data: []byte(`package large`),
 753                 }},
 754                 wantCreateErr: "larger than declared size",
 755         })
 756
 757         for _, test := range tests {
 758                 test := test
 759                 t.Run(test.desc, func(t *testing.T) {
 760                         t.Parallel()
 761
 762                         wantCheckFilesErr := test.wantCheckFilesErr
 763                         if wantCheckFilesErr == "" {
 764                                 wantCheckFilesErr = test.wantErr
 765                         }
 766                         if _, err := modzip.CheckFiles(test.files); err == nil && wantCheckFilesErr != "" {
 767                                 t.Fatalf("CheckFiles: unexpected success; want error containing %q", wantCheckFilesErr)
 768                         } else if err != nil && wantCheckFilesErr == "" {
 769                                 t.Fatalf("CheckFiles: got error %q; want success", err)
 770                         } else if err != nil && !strings.Contains(err.Error(), wantCheckFilesErr) {
 771                                 t.Fatalf("CheckFiles: got error %q; want error containing %q", err, wantCheckFilesErr)
 772                         }
 773
 774                         wantCreateErr := test.wantCreateErr
 775                         if wantCreateErr == "" {
 776                                 wantCreateErr = test.wantErr
 777                         }
 778                         if err := modzip.Create(ioutil.Discard, sizeLimitVersion, test.files); err == nil && wantCreateErr != "" {
 779                                 t.Fatalf("Create: unexpected success; want error containing %q", wantCreateErr)
 780                         } else if err != nil && wantCreateErr == "" {
 781                                 t.Fatalf("Create: got error %q; want success", err)
 782                         } else if err != nil && !strings.Contains(err.Error(), wantCreateErr) {
 783                                 t.Fatalf("Create: got error %q; want error containing %q", err, wantCreateErr)
 784                         }
 785                 })
 786         }
 787 }
 788
 789 func TestUnzipSizeLimits(t *testing.T) {
 790         if testing.Short() {
 791                 t.Skip("creating large files takes time")
 792         }
 793         for _, test := range sizeLimitTests {
 794                 test := test
 795                 t.Run(test.desc, func(t *testing.T) {
 796                         t.Parallel()
 797                         tmpZipFile, err := ioutil.TempFile("", "TestUnzipSizeLimits-*.zip")
 798                         if err != nil {
 799                                 t.Fatal(err)
 800                         }
 801                         tmpZipPath := tmpZipFile.Name()
 802                         defer func() {
 803                                 tmpZipFile.Close()
 804                                 if err := os.Remove(tmpZipPath); err != nil {
 805                                         t.Errorf("removing temp zip file: %v", err)
 806                                 }
 807                         }()
 808
 809                         zw := zip.NewWriter(tmpZipFile)
 810                         prefix := fmt.Sprintf("%s@%s/", sizeLimitVersion.Path, sizeLimitVersion.Version)
 811                         for _, tf := range test.files {
 812                                 zf, err := zw.Create(prefix + tf.Path())
 813                                 if err != nil {
 814                                         t.Fatal(err)
 815                                 }
 816                                 rc, err := tf.Open()
 817                                 if err != nil {
 818                                         t.Fatal(err)
 819                                 }
 820                                 _, err = io.Copy(zf, rc)
 821                                 rc.Close()
 822                                 if err != nil {
 823                                         t.Fatal(err)
 824                                 }
 825                         }
 826                         if err := zw.Close(); err != nil {
 827                                 t.Fatal(err)
 828                         }
 829                         if err := tmpZipFile.Close(); err != nil {
 830                                 t.Fatal(err)
 831                         }
 832
 833                         tmpDir, err := ioutil.TempDir("", "TestUnzipSizeLimits")
 834                         if err != nil {
 835                                 t.Fatal(err)
 836                         }
 837                         defer func() {
 838                                 if err := os.RemoveAll(tmpDir); err != nil {
 839                                         t.Errorf("removing temp dir: %v", err)
 840                                 }
 841                         }()
 842
 843                         wantCheckZipErr := test.wantCheckZipErr
 844                         if wantCheckZipErr == "" {
 845                                 wantCheckZipErr = test.wantErr
 846                         }
 847                         cf, err := modzip.CheckZip(sizeLimitVersion, tmpZipPath)
 848                         if err == nil {
 849                                 err = cf.Err()
 850                         }
 851                         if err == nil && wantCheckZipErr != "" {
 852                                 t.Fatalf("CheckZip: unexpected success; want error containing %q", wantCheckZipErr)
 853                         } else if err != nil && wantCheckZipErr == "" {
 854                                 t.Fatalf("CheckZip: got error %q; want success", err)
 855                         } else if err != nil && !strings.Contains(err.Error(), wantCheckZipErr) {
 856                                 t.Fatalf("CheckZip: got error %q; want error containing %q", err, wantCheckZipErr)
 857                         }
 858
 859                         wantUnzipErr := test.wantUnzipErr
 860                         if wantUnzipErr == "" {
 861                                 wantUnzipErr = test.wantErr
 862                         }
 863                         if err := modzip.Unzip(tmpDir, sizeLimitVersion, tmpZipPath); err == nil && wantUnzipErr != "" {
 864                                 t.Fatalf("Unzip: unexpected success; want error containing %q", wantUnzipErr)
 865                         } else if err != nil && wantUnzipErr == "" {
 866                                 t.Fatalf("Unzip: got error %q; want success", err)
 867                         } else if err != nil && !strings.Contains(err.Error(), wantUnzipErr) {
 868                                 t.Fatalf("Unzip: got error %q; want error containing %q", err, wantUnzipErr)
 869                         }
 870                 })
 871         }
 872 }
 873
 874 func TestUnzipSizeLimitsSpecial(t *testing.T) {
 875         if testing.Short() {
 876                 t.Skip("skipping test; creating large files takes time")
 877         }
 878
 879         for _, test := range []struct {
 880                 desc, wantErr string
 881                 m             module.Version
 882                 writeZip      func(t *testing.T, zipFile *os.File)
 883         }{
 884                 {
 885                         desc: "large_zip",
 886                         m:    module.Version{Path: "example.com/m", Version: "v1.0.0"},
 887                         writeZip: func(t *testing.T, zipFile *os.File) {
 888                                 if err := zipFile.Truncate(modzip.MaxZipFile); err != nil {
 889                                         t.Fatal(err)
 890                                 }
 891                         },
 892                         // this is not an error we care about; we're just testing whether
 893                         // Unzip checks the size of the file before opening.
 894                         // It's harder to create a valid zip file of exactly the right size.
 895                         wantErr: "not a valid zip file",
 896                 }, {
 897                         desc: "too_large_zip",
 898                         m:    module.Version{Path: "example.com/m", Version: "v1.0.0"},
 899                         writeZip: func(t *testing.T, zipFile *os.File) {
 900                                 if err := zipFile.Truncate(modzip.MaxZipFile + 1); err != nil {
 901                                         t.Fatal(err)
 902                                 }
 903                         },
 904                         wantErr: "module zip file is too large",
 905                 }, {
 906                         desc: "size_is_a_lie",
 907                         m:    module.Version{Path: "example.com/m", Version: "v1.0.0"},
 908                         writeZip: func(t *testing.T, zipFile *os.File) {
 909                                 // Create a normal zip file in memory containing one file full of zero
 910                                 // bytes. Use a distinctive size so we can find it later.
 911                                 zipBuf := &bytes.Buffer{}
 912                                 zw := zip.NewWriter(zipBuf)
 913                                 f, err := zw.Create("example.com/m@v1.0.0/go.mod")
 914                                 if err != nil {
 915                                         t.Fatal(err)
 916                                 }
 917                                 realSize := 0x0BAD
 918                                 buf := make([]byte, realSize)
 919                                 if _, err := f.Write(buf); err != nil {
 920                                         t.Fatal(err)
 921                                 }
 922                                 if err := zw.Close(); err != nil {
 923                                         t.Fatal(err)
 924                                 }
 925
 926                                 // Replace the uncompressed size of the file. As a shortcut, we just
 927                                 // search-and-replace the byte sequence. It should occur twice because
 928                                 // the 32- and 64-byte sizes are stored separately. All multi-byte
 929                                 // values are little-endian.
 930                                 zipData := zipBuf.Bytes()
 931                                 realSizeData := []byte{0xAD, 0x0B}
 932                                 fakeSizeData := []byte{0xAC, 0x00}
 933                                 s := zipData
 934                                 n := 0
 935                                 for {
 936                                         if i := bytes.Index(s, realSizeData); i < 0 {
 937                                                 break
 938                                         } else {
 939                                                 s = s[i:]
 940                                         }
 941                                         copy(s[:len(fakeSizeData)], fakeSizeData)
 942                                         n++
 943                                 }
 944                                 if n != 2 {
 945                                         t.Fatalf("replaced size %d times; expected 2", n)
 946                                 }
 947
 948                                 // Write the modified zip to the actual file.
 949                                 if _, err := zipFile.Write(zipData); err != nil {
 950                                         t.Fatal(err)
 951                                 }
 952                         },
 953                         wantErr: "uncompressed size of file example.com/m@v1.0.0/go.mod is larger than declared size",
 954                 },
 955         } {
 956                 test := test
 957                 t.Run(test.desc, func(t *testing.T) {
 958                         t.Parallel()
 959                         tmpZipFile, err := ioutil.TempFile("", "TestUnzipSizeLimitsSpecial-*.zip")
 960                         if err != nil {
 961                                 t.Fatal(err)
 962                         }
 963                         tmpZipPath := tmpZipFile.Name()
 964                         defer func() {
 965                                 tmpZipFile.Close()
 966                                 os.Remove(tmpZipPath)
 967                         }()
 968
 969                         test.writeZip(t, tmpZipFile)
 970                         if err := tmpZipFile.Close(); err != nil {
 971                                 t.Fatal(err)
 972                         }
 973
 974                         tmpDir, err := ioutil.TempDir("", "TestUnzipSizeLimitsSpecial")
 975                         if err != nil {
 976                                 t.Fatal(err)
 977                         }
 978                         defer os.RemoveAll(tmpDir)
 979                         if err := modzip.Unzip(tmpDir, test.m, tmpZipPath); err == nil && test.wantErr != "" {
 980                                 t.Fatalf("unexpected success; want error containing %q", test.wantErr)
 981                         } else if err != nil && test.wantErr == "" {
 982                                 t.Fatalf("got error %q; want success", err)
 983                         } else if err != nil && !strings.Contains(err.Error(), test.wantErr) {
 984                                 t.Fatalf("got error %q; want error containing %q", err, test.wantErr)
 985                         }
 986                 })
 987         }
 988 }
 989
 990 // TestVCS clones a repository, creates a zip for a known version,
 991 // and verifies the zip file itself has the same SHA-256 hash as the one
 992 // 'go mod download' produces.
 993 //
 994 // This test is intended to build confidence that this implementation produces
 995 // the same output as the go command, given the same VCS zip input. This is
 996 // not intended to be a complete conformance test. The code that produces zip
 997 // archives from VCS repos is based on the go command, but it's for testing
 998 // only, and we don't export it.
 999 //
1000 // Note that we test the hash of the zip file itself. This is stricter than
1001 // testing the hash of the content, which is what we've promised users.
1002 // It's okay if the zip hash changes without changing the content hash, but
1003 // we should not let that happen accidentally.
1004 func TestVCS(t *testing.T) {
1005         if testing.Short() {
1006                 t.Skip()
1007         }
1008
1009         var downloadErrorCount int32
1010         const downloadErrorLimit = 3
1011
1012         haveVCS := make(map[string]bool)
1013         for _, vcs := range []string{"git", "hg"} {
1014                 _, err := exec.LookPath(vcs)
1015                 haveVCS[vcs] = err == nil
1016         }
1017
1018         for _, test := range []struct {
1019                 m                            module.Version
1020                 vcs, url, subdir, rev        string
1021                 wantContentHash, wantZipHash string
1022         }{
1023                 // Simple tests: all versions of rsc.io/quote + newer major versions
1024                 {
1025                         m:               module.Version{Path: "rsc.io/quote", Version: "v1.0.0"},
1026                         vcs:             "git",
1027                         url:             "https://github.com/rsc/quote",
1028                         rev:             "v1.0.0",
1029                         wantContentHash: "h1:haUSojyo3j2M9g7CEUFG8Na09dtn7QKxvPGaPVQdGwM=",
1030                         wantZipHash:     "5c08ba2c09a364f93704aaa780e7504346102c6ef4fe1333a11f09904a732078",
1031                 },
1032                 {
1033                         m:               module.Version{Path: "rsc.io/quote", Version: "v1.1.0"},
1034                         vcs:             "git",
1035                         url:             "https://github.com/rsc/quote",
1036                         rev:             "v1.1.0",
1037                         wantContentHash: "h1:n/ElL9GOlVEwL0mVjzaYj0UxTI/TX9aQ7lR5LHqP/Rw=",
1038                         wantZipHash:     "730a5ae6e5c4e216e4f84bb93aa9785a85630ad73f96954ebb5f9daa123dcaa9",
1039                 },
1040                 {
1041                         m:               module.Version{Path: "rsc.io/quote", Version: "v1.2.0"},
1042                         vcs:             "git",
1043                         url:             "https://github.com/rsc/quote",
1044                         rev:             "v1.2.0",
1045                         wantContentHash: "h1:fFMCNi0A97hfNrtUZVQKETbuc3h7bmfFQHnjutpPYCg=",
1046                         wantZipHash:     "fe1bd62652e9737a30d6b7fd396ea13e54ad13fb05f295669eb63d6d33290b06",
1047                 },
1048                 {
1049                         m:               module.Version{Path: "rsc.io/quote", Version: "v1.2.1"},
1050                         vcs:             "git",
1051                         url:             "https://github.com/rsc/quote",
1052                         rev:             "v1.2.1",
1053                         wantContentHash: "h1:l+HtgC05eds8qgXNApuv6g1oK1q3B144BM5li1akqXY=",
1054                         wantZipHash:     "9f0e74de55a6bd20c1567a81e707814dc221f07df176af2a0270392c6faf32fd",
1055                 },
1056                 {
1057                         m:               module.Version{Path: "rsc.io/quote", Version: "v1.3.0"},
1058                         vcs:             "git",
1059                         url:             "https://github.com/rsc/quote",
1060                         rev:             "v1.3.0",
1061                         wantContentHash: "h1:aPUoHx/0Cd7BTZs4SAaknT4TaKryH766GcFTvJjVbHU=",
1062                         wantZipHash:     "03872ee7d6747bc2ee0abadbd4eb09e60f6df17d0a6142264abe8a8a00af50e7",
1063                 },
1064                 {
1065                         m:               module.Version{Path: "rsc.io/quote", Version: "v1.4.0"},
1066                         vcs:             "git",
1067                         url:             "https://github.com/rsc/quote",
1068                         rev:             "v1.4.0",
1069                         wantContentHash: "h1:tYuJspOzwTRMUOX6qmSDRTEKFVV80GM0/l89OLZuVNg=",
1070                         wantZipHash:     "f60be8193c607bf197da01da4bedb3d683fe84c30de61040eb5d7afaf7869f2e",
1071                 },
1072                 {
1073                         m:               module.Version{Path: "rsc.io/quote", Version: "v1.5.0"},
1074                         vcs:             "git",
1075                         url:             "https://github.com/rsc/quote",
1076                         rev:             "v1.5.0",
1077                         wantContentHash: "h1:mVjf/WMWxfIw299sOl/O3EXn5qEaaJPMDHMsv7DBDlw=",
1078                         wantZipHash:     "a2d281834ce159703540da94425fa02c7aec73b88b560081ed0d3681bfe9cd1f",
1079                 },
1080                 {
1081                         m:               module.Version{Path: "rsc.io/quote", Version: "v1.5.1"},
1082                         vcs:             "git",
1083                         url:             "https://github.com/rsc/quote",
1084                         rev:             "v1.5.1",
1085                         wantContentHash: "h1:ptSemFtffEBvMed43o25vSUpcTVcqxfXU8Jv0sfFVJs=",
1086                         wantZipHash:     "4ecd78a6d9f571e84ed2baac1688fd150400db2c5b017b496c971af30aaece02",
1087                 },
1088                 {
1089                         m:               module.Version{Path: "rsc.io/quote", Version: "v1.5.2"},
1090                         vcs:             "git",
1091                         url:             "https://github.com/rsc/quote",
1092                         rev:             "v1.5.2",
1093                         wantContentHash: "h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y=",
1094                         wantZipHash:     "643fcf8ef4e4cbb8f910622c42df3f9a81f3efe8b158a05825a81622c121ca0a",
1095                 },
1096                 {
1097                         m:               module.Version{Path: "rsc.io/quote", Version: "v1.5.3-pre1"},
1098                         vcs:             "git",
1099                         url:             "https://github.com/rsc/quote",
1100                         rev:             "v1.5.3-pre1",
1101                         wantContentHash: "h1:c3EJ21kn75/hyrOL/Dvj45+ifxGFSY8Wf4WBcoWTxF0=",
1102                         wantZipHash:     "24106f0f15384949df51fae5d34191bf120c3b80c1c904721ca2872cf83126b2",
1103                 },
1104                 {
1105                         m:               module.Version{Path: "rsc.io/quote/v2", Version: "v2.0.1"},
1106                         vcs:             "git",
1107                         url:             "https://github.com/rsc/quote",
1108                         rev:             "v2.0.1",
1109                         wantContentHash: "h1:DF8hmGbDhgiIa2tpqLjHLIKkJx6WjCtLEqZBAU+hACI=",
1110                         wantZipHash:     "009ed42474a59526fe56a14a9dd02bd7f977d1bd3844398bd209d0da0484aade",
1111                 },
1112                 {
1113                         m:               module.Version{Path: "rsc.io/quote/v3", Version: "v3.0.0"},
1114                         vcs:             "git",
1115                         url:             "https://github.com/rsc/quote",
1116                         rev:             "v3.0.0",
1117                         subdir:          "v3",
1118                         wantContentHash: "h1:OEIXClZHFMyx5FdatYfxxpNEvxTqHlu5PNdla+vSYGg=",
1119                         wantZipHash:     "cf3ff89056b785d7b3ef3a10e984efd83b47d9e65eabe8098b927b3370d5c3eb",
1120                 },
1121
1122                 // Test cases from vcs-test.golang.org
1123                 {
1124                         m:               module.Version{Path: "vcs-test.golang.org/git/v3pkg.git/v3", Version: "v3.0.0"},
1125                         vcs:             "git",
1126                         url:             "https://vcs-test.golang.org/git/v3pkg",
1127                         rev:             "v3.0.0",
1128                         wantContentHash: "h1:mZhljS1BaiW8lODR6wqY5pDxbhXja04rWPFXPwRAtvA=",
1129                         wantZipHash:     "9c65f0d235e531008dc04e977f6fa5d678febc68679bb63d4148dadb91d3fe57",
1130                 },
1131                 {
1132                         m:               module.Version{Path: "vcs-test.golang.org/go/custom-hg-hello", Version: "v0.0.0-20171010233936-a8c8e7a40da9"},
1133                         vcs:             "hg",
1134                         url:             "https://vcs-test.golang.org/hg/custom-hg-hello",
1135                         rev:             "a8c8e7a40da9",
1136                         wantContentHash: "h1:LU6jFCbwn5VVgTcj+y4LspOpJHLZvl5TGPE+LwwpMw4=",
1137                         wantZipHash:     "a1b12047da979d618c639ee98f370767a13d0507bd77785dc2f8dad66b40e2e6",
1138                 },
1139
1140                 // Latest versions of selected golang.org/x repos
1141                 {
1142                         m:               module.Version{Path: "golang.org/x/arch", Version: "v0.0.0-20190927153633-4e8777c89be4"},
1143                         vcs:             "git",
1144                         url:             "https://go.googlesource.com/arch",
1145                         rev:             "4e8777c89be4d9e61691fbe5d4e6c8838a7806f3",
1146                         wantContentHash: "h1:QlVATYS7JBoZMVaf+cNjb90WD/beKVHnIxFKT4QaHVI=",
1147                         wantZipHash:     "d17551a0c4957180ec1507065d13dcdd0f5cd8bfd7dd735fb81f64f3e2b31b68",
1148                 },
1149                 {
1150                         m:               module.Version{Path: "golang.org/x/blog", Version: "v0.0.0-20191017104857-0cd0cdff05c2"},
1151                         vcs:             "git",
1152                         url:             "https://go.googlesource.com/blog",
1153                         rev:             "0cd0cdff05c251ad0c796cc94d7059e013311fc6",
1154                         wantContentHash: "h1:IKGICrORhR1aH2xG/WqrnpggSNolSj5urQxggCfmj28=",
1155                         wantZipHash:     "0fed6b400de54da34b52b464ef2cdff45167236aaaf9a99ba8eba8855036faff",
1156                 },
1157                 {
1158                         m:               module.Version{Path: "golang.org/x/crypto", Version: "v0.0.0-20191011191535-87dc89f01550"},
1159                         vcs:             "git",
1160                         url:             "https://go.googlesource.com/crypto",
1161                         rev:             "87dc89f01550277dc22b74ffcf4cd89fa2f40f4c",
1162                         wantContentHash: "h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=",
1163                         wantZipHash:     "88e47aa05eb25c6abdad7387ccccfc39e74541896d87b7b1269e9dd2fa00100d",
1164                 },
1165                 {
1166                         m:               module.Version{Path: "golang.org/x/net", Version: "v0.0.0-20191014212845-da9a3fd4c582"},
1167                         vcs:             "git",
1168                         url:             "https://go.googlesource.com/net",
1169                         rev:             "da9a3fd4c5820e74b24a6cb7fb438dc9b0dd377c",
1170                         wantContentHash: "h1:p9xBe/w/OzkeYVKm234g55gMdD1nSIooTir5kV11kfA=",
1171                         wantZipHash:     "34901a85e6c15475a40457c2393ce66fb0999accaf2d6aa5b64b4863751ddbde",
1172                 },
1173                 {
1174                         m:               module.Version{Path: "golang.org/x/sync", Version: "v0.0.0-20190911185100-cd5d95a43a6e"},
1175                         vcs:             "git",
1176                         url:             "https://go.googlesource.com/sync",
1177                         rev:             "cd5d95a43a6e21273425c7ae415d3df9ea832eeb",
1178                         wantContentHash: "h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=",
1179                         wantZipHash:     "9c63fe51b0c533b258d3acc30d9319fe78679ce1a051109c9dea3105b93e2eef",
1180                 },
1181                 {
1182                         m:               module.Version{Path: "golang.org/x/sys", Version: "v0.0.0-20191010194322-b09406accb47"},
1183                         vcs:             "git",
1184                         url:             "https://go.googlesource.com/sys",
1185                         rev:             "b09406accb4736d857a32bf9444cd7edae2ffa79",
1186                         wantContentHash: "h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=",
1187                         wantZipHash:     "f26f2993757670b4d1fee3156d331513259757f17133a36966c158642c3f61df",
1188                 },
1189                 {
1190                         m:               module.Version{Path: "golang.org/x/talks", Version: "v0.0.0-20191010201600-067e0d331fee"},
1191                         vcs:             "git",
1192                         url:             "https://go.googlesource.com/talks",
1193                         rev:             "067e0d331feee4f8d0fa17d47444db533bd904e7",
1194                         wantContentHash: "h1:8fnBMBUwliuiHuzfFw6kSSx79AzQpqkjZi3FSNIoqYs=",
1195                         wantZipHash:     "fab2129f3005f970dbf2247378edb3220f6bd36726acdc7300ae3bb0f129e2f2",
1196                 },
1197                 {
1198                         m:               module.Version{Path: "golang.org/x/tools", Version: "v0.0.0-20191017205301-920acffc3e65"},
1199                         vcs:             "git",
1200                         url:             "https://go.googlesource.com/tools",
1201                         rev:             "920acffc3e65862cb002dae6b227b8d9695e3d29",
1202                         wantContentHash: "h1:GwXwgmbrvlcHLDsENMqrQTTIC2C0kIPszsq929NruKI=",
1203                         wantZipHash:     "7f0ab7466448190f8ad1b8cfb05787c3fb08f4a8f9953cd4b40a51c76ddebb28",
1204                 },
1205                 {
1206                         m:               module.Version{Path: "golang.org/x/tour", Version: "v0.0.0-20191002171047-6bb846ce41cd"},
1207                         vcs:             "git",
1208                         url:             "https://go.googlesource.com/tour",
1209                         rev:             "6bb846ce41cdca087b14c8e3560a679691c424b6",
1210                         wantContentHash: "h1:EUlK3Rq8iTkQERnCnveD654NvRJ/ZCM9XCDne+S5cJ8=",
1211                         wantZipHash:     "d6a7e03e02e5f7714bd12653d319a3b0f6e1099c01b1f9a17bc3613fb31c9170",
1212                 },
1213         } {
1214                 test := test
1215                 testName := strings.ReplaceAll(test.m.String(), "/", "_")
1216                 t.Run(testName, func(t *testing.T) {
1217                         if have, ok := haveVCS[test.vcs]; !ok {
1218                                 t.Fatalf("unknown vcs: %s", test.vcs)
1219                         } else if !have {
1220                                 t.Skip()
1221                         }
1222                         t.Parallel()
1223
1224                         repo, dl, cleanup, err := downloadVCSZip(test.vcs, test.url, test.rev, test.subdir)
1225                         defer cleanup()
1226                         if err != nil {
1227                                 // This may fail if there's a problem with the network or upstream
1228                                 // repository. The package being tested doesn't directly interact with
1229                                 // VCS tools; the test just does this to simulate what the go command
1230                                 // does. So an error should cause a skip instead of a failure. But we
1231                                 // should fail after too many errors so we don't lose test coverage
1232                                 // when something changes permanently.
1233                                 n := atomic.AddInt32(&downloadErrorCount, 1)
1234                                 if n < downloadErrorLimit {
1235                                         t.Skipf("failed to download zip from repository: %v", err)
1236                                 } else {
1237                                         t.Fatalf("failed to download zip from repository (repeated failure): %v", err)
1238                                 }
1239                         }
1240
1241                         // Create a module zip from that archive.
1242                         // (adapted from cmd/go/internal/modfetch.codeRepo.Zip)
1243                         info, err := dl.Stat()
1244                         if err != nil {
1245                                 t.Fatal(err)
1246                         }
1247                         zr, err := zip.NewReader(dl, info.Size())
1248                         if err != nil {
1249                                 t.Fatal(err)
1250                         }
1251
1252                         var files []modzip.File
1253                         topPrefix := ""
1254                         subdir := test.subdir
1255                         if subdir != "" && !strings.HasSuffix(subdir, "/") {
1256                                 subdir += "/"
1257                         }
1258                         haveLICENSE := false
1259                         for _, f := range zr.File {
1260                                 if !f.FileInfo().Mode().IsRegular() {
1261                                         continue
1262                                 }
1263                                 if topPrefix == "" {
1264                                         i := strings.Index(f.Name, "/")
1265                                         if i < 0 {
1266                                                 t.Fatal("missing top-level directory prefix")
1267                                         }
1268                                         topPrefix = f.Name[:i+1]
1269                                 }
1270                                 if strings.HasSuffix(f.Name, "/") { // drop directory dummy entries
1271                                         continue
1272                                 }
1273                                 if !strings.HasPrefix(f.Name, topPrefix) {
1274                                         t.Fatal("zip file contains more than one top-level directory")
1275                                 }
1276                                 name := strings.TrimPrefix(f.Name, topPrefix)
1277                                 if !strings.HasPrefix(name, subdir) {
1278                                         continue
1279                                 }
1280                                 name = strings.TrimPrefix(name, subdir)
1281                                 if name == ".hg_archival.txt" {
1282                                         // Inserted by hg archive.
1283                                         // Not correct to drop from other version control systems, but too bad.
1284                                         continue
1285                                 }
1286                                 if name == "LICENSE" {
1287                                         haveLICENSE = true
1288                                 }
1289                                 files = append(files, zipFile{name: name, f: f})
1290                         }
1291                         if !haveLICENSE && subdir != "" {
1292                                 license, err := downloadVCSFile(test.vcs, repo, test.rev, "LICENSE")
1293                                 if err != nil {
1294                                         t.Fatal(err)
1295                                 }
1296                                 files = append(files, fakeFile{
1297                                         name: "LICENSE",
1298                                         size: uint64(len(license)),
1299                                         data: license,
1300                                 })
1301                         }
1302
1303                         tmpModZipFile, err := ioutil.TempFile("", "TestVCS-*.zip")
1304                         if err != nil {
1305                                 t.Fatal(err)
1306                         }
1307                         tmpModZipPath := tmpModZipFile.Name()
1308                         defer func() {
1309                                 tmpModZipFile.Close()
1310                                 os.Remove(tmpModZipPath)
1311                         }()
1312                         h := sha256.New()
1313                         w := io.MultiWriter(tmpModZipFile, h)
1314                         if err := modzip.Create(w, test.m, files); err != nil {
1315                                 t.Fatal(err)
1316                         }
1317                         if err := tmpModZipFile.Close(); err != nil {
1318                                 t.Fatal(err)
1319                         }
1320
1321                         gotZipHash := hex.EncodeToString(h.Sum(nil))
1322                         if test.wantZipHash != gotZipHash {
1323                                 // If the test fails because the hash of the zip file itself differs,
1324                                 // that may be okay as long as the hash of the data within the zip file
1325                                 // does not change. For example, we might change the compression,
1326                                 // order, or alignment of files without affecting the extracted output.
1327                                 // We shouldn't make such a change unintentionally though, so this
1328                                 // test will fail either way.
1329                                 if gotSum, err := dirhash.HashZip(tmpModZipPath, dirhash.Hash1); err == nil && test.wantContentHash != gotSum {
1330                                         t.Fatalf("zip content hash: got %s, want %s", gotSum, test.wantContentHash)
1331                                 } else {
1332                                         t.Fatalf("zip file hash: got %s, want %s", gotZipHash, test.wantZipHash)
1333                                 }
1334                         }
1335                 })
1336         }
1337 }
1338
1339 func downloadVCSZip(vcs, url, rev, subdir string) (repoDir string, dl *os.File, cleanup func(), err error) {
1340         var cleanups []func()
1341         cleanup = func() {
1342                 for i := len(cleanups) - 1; i >= 0; i-- {
1343                         cleanups[i]()
1344                 }
1345         }
1346         repoDir, err = ioutil.TempDir("", "downloadVCSZip")
1347         if err != nil {
1348                 return "", nil, cleanup, err
1349         }
1350         cleanups = append(cleanups, func() { os.RemoveAll(repoDir) })
1351
1352         switch vcs {
1353         case "git":
1354                 // Create a repository and download the revision we want.
1355                 if err := run(repoDir, "git", "init", "--bare"); err != nil {
1356                         return "", nil, cleanup, err
1357                 }
1358                 if err := os.MkdirAll(filepath.Join(repoDir, "info"), 0777); err != nil {
1359                         return "", nil, cleanup, err
1360                 }
1361                 attrFile, err := os.OpenFile(filepath.Join(repoDir, "info", "attributes"), os.O_CREATE|os.O_APPEND|os.O_RDWR, 0666)
1362                 if err != nil {
1363                         return "", nil, cleanup, err
1364                 }
1365                 if _, err := attrFile.Write([]byte("\n* -export-subst -export-ignore\n")); err != nil {
1366                         attrFile.Close()
1367                         return "", nil, cleanup, err
1368                 }
1369                 if err := attrFile.Close(); err != nil {
1370                         return "", nil, cleanup, err
1371                 }
1372                 if err := run(repoDir, "git", "remote", "add", "origin", "--", url); err != nil {
1373                         return "", nil, cleanup, err
1374                 }
1375                 var refSpec string
1376                 if strings.HasPrefix(rev, "v") {
1377                         refSpec = fmt.Sprintf("refs/tags/%[1]s:refs/tags/%[1]s", rev)
1378                 } else {
1379                         refSpec = fmt.Sprintf("%s:refs/dummy", rev)
1380                 }
1381                 if err := run(repoDir, "git", "fetch", "-f", "--depth=1", "origin", refSpec); err != nil {
1382                         return "", nil, cleanup, err
1383                 }
1384
1385                 // Create an archive.
1386                 tmpZipFile, err := ioutil.TempFile("", "downloadVCSZip-*.zip")
1387                 if err != nil {
1388                         return "", nil, cleanup, err
1389                 }
1390                 cleanups = append(cleanups, func() {
1391                         name := tmpZipFile.Name()
1392                         tmpZipFile.Close()
1393                         os.Remove(name)
1394                 })
1395                 subdirArg := subdir
1396                 if subdir == "" {
1397                         subdirArg = "."
1398                 }
1399                 cmd := exec.Command("git", "-c", "core.autocrlf=input", "-c", "core.eol=lf", "archive", "--format=zip", "--prefix=prefix/", rev, "--", subdirArg)
1400                 cmd.Dir = repoDir
1401                 cmd.Stdout = tmpZipFile
1402                 if err := cmd.Run(); err != nil {
1403                         return "", nil, cleanup, err
1404                 }
1405                 if _, err := tmpZipFile.Seek(0, 0); err != nil {
1406                         return "", nil, cleanup, err
1407                 }
1408                 return repoDir, tmpZipFile, cleanup, nil
1409
1410         case "hg":
1411                 // Clone the whole repository.
1412                 if err := run(repoDir, "hg", "clone", "-U", "--", url, "."); err != nil {
1413                         return "", nil, cleanup, err
1414                 }
1415
1416                 // Create an archive.
1417                 tmpZipFile, err := ioutil.TempFile("", "downloadVCSZip-*.zip")
1418                 if err != nil {
1419                         return "", nil, cleanup, err
1420                 }
1421                 tmpZipPath := tmpZipFile.Name()
1422                 tmpZipFile.Close()
1423                 cleanups = append(cleanups, func() { os.Remove(tmpZipPath) })
1424                 args := []string{"archive", "-t", "zip", "--no-decode", "-r", rev, "--prefix=prefix/"}
1425                 if subdir != "" {
1426                         args = append(args, "-I", subdir+"/**")
1427                 }
1428                 args = append(args, "--", tmpZipPath)
1429                 if err := run(repoDir, "hg", args...); err != nil {
1430                         return "", nil, cleanup, err
1431                 }
1432                 if tmpZipFile, err = os.Open(tmpZipPath); err != nil {
1433                         return "", nil, cleanup, err
1434                 }
1435                 cleanups = append(cleanups, func() { tmpZipFile.Close() })
1436                 return repoDir, tmpZipFile, cleanup, err
1437
1438         default:
1439                 return "", nil, cleanup, fmt.Errorf("vcs %q not supported", vcs)
1440         }
1441 }
1442
1443 func downloadVCSFile(vcs, repo, rev, file string) ([]byte, error) {
1444         switch vcs {
1445         case "git":
1446                 cmd := exec.Command("git", "cat-file", "blob", rev+":"+file)
1447                 cmd.Dir = repo
1448                 return cmd.Output()
1449         default:
1450                 return nil, fmt.Errorf("vcs %q not supported", vcs)
1451         }
1452 }
1453
1454 func run(dir string, name string, args ...string) error {
1455         cmd := exec.Command(name, args...)
1456         cmd.Dir = dir
1457         if err := cmd.Run(); err != nil {
1458                 return fmt.Errorf("%s: %v", strings.Join(args, " "), err)
1459         }
1460         return nil
1461 }
1462
1463 type zipFile struct {
1464         name string
1465         f    *zip.File
1466 }
1467
1468 func (f zipFile) Path() string                 { return f.name }
1469 func (f zipFile) Lstat() (os.FileInfo, error)  { return f.f.FileInfo(), nil }
1470 func (f zipFile) Open() (io.ReadCloser, error) { return f.f.Open() }