1// Copyright 2015 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package zip 16 17import ( 18 "bytes" 19 "compress/flate" 20 "errors" 21 "fmt" 22 "hash/crc32" 23 "io" 24 "io/ioutil" 25 "os" 26 "path/filepath" 27 "sort" 28 "strings" 29 "sync" 30 "syscall" 31 "time" 32 "unicode" 33 34 "github.com/google/blueprint/pathtools" 35 36 "android/soong/jar" 37 "android/soong/third_party/zip" 38) 39 40// Block size used during parallel compression of a single file. 41const parallelBlockSize = 1 * 1024 * 1024 // 1MB 42 43// Minimum file size to use parallel compression. It requires more 44// flate.Writer allocations, since we can't change the dictionary 45// during Reset 46const minParallelFileSize = parallelBlockSize * 6 47 48// Size of the ZIP compression window (32KB) 49const windowSize = 32 * 1024 50 51type nopCloser struct { 52 io.Writer 53} 54 55func (nopCloser) Close() error { 56 return nil 57} 58 59type byteReaderCloser struct { 60 *bytes.Reader 61 io.Closer 62} 63 64type pathMapping struct { 65 dest, src string 66 zipMethod uint16 67} 68 69type FileArg struct { 70 PathPrefixInZip, SourcePrefixToStrip string 71 SourceFiles []string 72 JunkPaths bool 73 GlobDir string 74} 75 76type FileArgsBuilder struct { 77 state FileArg 78 err error 79 fs pathtools.FileSystem 80 81 fileArgs []FileArg 82} 83 84func NewFileArgsBuilder() *FileArgsBuilder { 85 return &FileArgsBuilder{ 86 fs: pathtools.OsFs, 87 } 88} 89 90func (b *FileArgsBuilder) JunkPaths(v bool) *FileArgsBuilder { 91 b.state.JunkPaths = v 92 b.state.SourcePrefixToStrip = "" 93 return b 94} 95 96func (b *FileArgsBuilder) SourcePrefixToStrip(prefixToStrip string) *FileArgsBuilder { 97 b.state.JunkPaths = false 98 b.state.SourcePrefixToStrip = prefixToStrip 99 return b 100} 101 102func (b *FileArgsBuilder) PathPrefixInZip(rootPrefix string) *FileArgsBuilder { 103 b.state.PathPrefixInZip = rootPrefix 104 return b 105} 106 107func (b *FileArgsBuilder) File(name string) *FileArgsBuilder { 108 if b.err != nil { 109 return b 110 } 111 112 arg := b.state 113 arg.SourceFiles = []string{name} 114 b.fileArgs = append(b.fileArgs, arg) 115 return b 116} 117 118func (b *FileArgsBuilder) Dir(name string) *FileArgsBuilder { 119 if b.err != nil { 120 return b 121 } 122 123 arg := b.state 124 arg.GlobDir = name 125 b.fileArgs = append(b.fileArgs, arg) 126 return b 127} 128 129func (b *FileArgsBuilder) List(name string) *FileArgsBuilder { 130 if b.err != nil { 131 return b 132 } 133 134 f, err := b.fs.Open(name) 135 if err != nil { 136 b.err = err 137 return b 138 } 139 defer f.Close() 140 141 list, err := ioutil.ReadAll(f) 142 if err != nil { 143 b.err = err 144 return b 145 } 146 147 arg := b.state 148 arg.SourceFiles = strings.Fields(string(list)) 149 b.fileArgs = append(b.fileArgs, arg) 150 return b 151} 152 153func (b *FileArgsBuilder) Error() error { 154 if b == nil { 155 return nil 156 } 157 return b.err 158} 159 160func (b *FileArgsBuilder) FileArgs() []FileArg { 161 if b == nil { 162 return nil 163 } 164 return b.fileArgs 165} 166 167type IncorrectRelativeRootError struct { 168 RelativeRoot string 169 Path string 170} 171 172func (x IncorrectRelativeRootError) Error() string { 173 return fmt.Sprintf("path %q is outside relative root %q", x.Path, x.RelativeRoot) 174} 175 176type ZipWriter struct { 177 time time.Time 178 createdFiles map[string]string 179 createdDirs map[string]string 180 directories bool 181 182 errors chan error 183 writeOps chan chan *zipEntry 184 185 cpuRateLimiter *CPURateLimiter 186 memoryRateLimiter *MemoryRateLimiter 187 188 compressorPool sync.Pool 189 compLevel int 190 191 followSymlinks pathtools.ShouldFollowSymlinks 192 ignoreMissingFiles bool 193 194 stderr io.Writer 195 fs pathtools.FileSystem 196} 197 198type zipEntry struct { 199 fh *zip.FileHeader 200 201 // List of delayed io.Reader 202 futureReaders chan chan io.Reader 203 204 // Only used for passing into the MemoryRateLimiter to ensure we 205 // release as much memory as much as we request 206 allocatedSize int64 207} 208 209type ZipArgs struct { 210 FileArgs []FileArg 211 OutputFilePath string 212 EmulateJar bool 213 SrcJar bool 214 AddDirectoryEntriesToZip bool 215 CompressionLevel int 216 ManifestSourcePath string 217 NumParallelJobs int 218 NonDeflatedFiles map[string]bool 219 WriteIfChanged bool 220 StoreSymlinks bool 221 IgnoreMissingFiles bool 222 223 Stderr io.Writer 224 Filesystem pathtools.FileSystem 225} 226 227const NOQUOTE = '\x00' 228 229func ReadRespFile(bytes []byte) []string { 230 var args []string 231 var arg []rune 232 233 isEscaping := false 234 quotingStart := NOQUOTE 235 for _, c := range string(bytes) { 236 switch { 237 case isEscaping: 238 if quotingStart == '"' { 239 if !(c == '"' || c == '\\') { 240 // '\"' or '\\' will be escaped under double quoting. 241 arg = append(arg, '\\') 242 } 243 } 244 arg = append(arg, c) 245 isEscaping = false 246 case c == '\\' && quotingStart != '\'': 247 isEscaping = true 248 case quotingStart == NOQUOTE && (c == '\'' || c == '"'): 249 quotingStart = c 250 case quotingStart != NOQUOTE && c == quotingStart: 251 quotingStart = NOQUOTE 252 case quotingStart == NOQUOTE && unicode.IsSpace(c): 253 // Current character is a space outside quotes 254 if len(arg) != 0 { 255 args = append(args, string(arg)) 256 } 257 arg = arg[:0] 258 default: 259 arg = append(arg, c) 260 } 261 } 262 263 if len(arg) != 0 { 264 args = append(args, string(arg)) 265 } 266 267 return args 268} 269 270func ZipTo(args ZipArgs, w io.Writer) error { 271 if args.EmulateJar { 272 args.AddDirectoryEntriesToZip = true 273 } 274 275 // Have Glob follow symlinks if they are not being stored as symlinks in the zip file. 276 followSymlinks := pathtools.ShouldFollowSymlinks(!args.StoreSymlinks) 277 278 z := &ZipWriter{ 279 time: jar.DefaultTime, 280 createdDirs: make(map[string]string), 281 createdFiles: make(map[string]string), 282 directories: args.AddDirectoryEntriesToZip, 283 compLevel: args.CompressionLevel, 284 followSymlinks: followSymlinks, 285 ignoreMissingFiles: args.IgnoreMissingFiles, 286 stderr: args.Stderr, 287 fs: args.Filesystem, 288 } 289 290 if z.fs == nil { 291 z.fs = pathtools.OsFs 292 } 293 294 if z.stderr == nil { 295 z.stderr = os.Stderr 296 } 297 298 pathMappings := []pathMapping{} 299 300 noCompression := args.CompressionLevel == 0 301 302 for _, fa := range args.FileArgs { 303 var srcs []string 304 for _, s := range fa.SourceFiles { 305 s = strings.TrimSpace(s) 306 if s == "" { 307 continue 308 } 309 310 globbed, _, err := z.fs.Glob(s, nil, followSymlinks) 311 if err != nil { 312 return err 313 } 314 if len(globbed) == 0 { 315 err := &os.PathError{ 316 Op: "lstat", 317 Path: s, 318 Err: os.ErrNotExist, 319 } 320 if args.IgnoreMissingFiles { 321 fmt.Fprintln(z.stderr, "warning:", err) 322 } else { 323 return err 324 } 325 } 326 srcs = append(srcs, globbed...) 327 } 328 if fa.GlobDir != "" { 329 if exists, isDir, err := z.fs.Exists(fa.GlobDir); err != nil { 330 return err 331 } else if !exists && !args.IgnoreMissingFiles { 332 err := &os.PathError{ 333 Op: "lstat", 334 Path: fa.GlobDir, 335 Err: os.ErrNotExist, 336 } 337 if args.IgnoreMissingFiles { 338 fmt.Fprintln(z.stderr, "warning:", err) 339 } else { 340 return err 341 } 342 } else if !isDir && !args.IgnoreMissingFiles { 343 err := &os.PathError{ 344 Op: "lstat", 345 Path: fa.GlobDir, 346 Err: syscall.ENOTDIR, 347 } 348 if args.IgnoreMissingFiles { 349 fmt.Fprintln(z.stderr, "warning:", err) 350 } else { 351 return err 352 } 353 } 354 globbed, _, err := z.fs.Glob(filepath.Join(fa.GlobDir, "**/*"), nil, followSymlinks) 355 if err != nil { 356 return err 357 } 358 srcs = append(srcs, globbed...) 359 } 360 for _, src := range srcs { 361 err := fillPathPairs(fa, src, &pathMappings, args.NonDeflatedFiles, noCompression) 362 if err != nil { 363 return err 364 } 365 } 366 } 367 368 return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.SrcJar, args.NumParallelJobs) 369} 370 371func Zip(args ZipArgs) error { 372 if args.OutputFilePath == "" { 373 return fmt.Errorf("output file path must be nonempty") 374 } 375 376 buf := &bytes.Buffer{} 377 var out io.Writer = buf 378 379 if !args.WriteIfChanged { 380 f, err := os.Create(args.OutputFilePath) 381 if err != nil { 382 return err 383 } 384 385 defer f.Close() 386 defer func() { 387 if err != nil { 388 os.Remove(args.OutputFilePath) 389 } 390 }() 391 392 out = f 393 } 394 395 err := ZipTo(args, out) 396 if err != nil { 397 return err 398 } 399 400 if args.WriteIfChanged { 401 err := pathtools.WriteFileIfChanged(args.OutputFilePath, buf.Bytes(), 0666) 402 if err != nil { 403 return err 404 } 405 } 406 407 return nil 408} 409 410func fillPathPairs(fa FileArg, src string, pathMappings *[]pathMapping, 411 nonDeflatedFiles map[string]bool, noCompression bool) error { 412 413 var dest string 414 415 if fa.JunkPaths { 416 dest = filepath.Base(src) 417 } else { 418 var err error 419 dest, err = filepath.Rel(fa.SourcePrefixToStrip, src) 420 if err != nil { 421 return err 422 } 423 if strings.HasPrefix(dest, "../") { 424 return IncorrectRelativeRootError{ 425 Path: src, 426 RelativeRoot: fa.SourcePrefixToStrip, 427 } 428 } 429 430 } 431 dest = filepath.Join(fa.PathPrefixInZip, dest) 432 433 zipMethod := zip.Deflate 434 if _, found := nonDeflatedFiles[dest]; found || noCompression { 435 zipMethod = zip.Store 436 } 437 *pathMappings = append(*pathMappings, 438 pathMapping{dest: dest, src: src, zipMethod: zipMethod}) 439 440 return nil 441} 442 443func jarSort(mappings []pathMapping) { 444 less := func(i int, j int) (smaller bool) { 445 return jar.EntryNamesLess(mappings[i].dest, mappings[j].dest) 446 } 447 sort.SliceStable(mappings, less) 448} 449 450func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar, srcJar bool, 451 parallelJobs int) error { 452 453 z.errors = make(chan error) 454 defer close(z.errors) 455 456 // This channel size can be essentially unlimited -- it's used as a fifo 457 // queue decouple the CPU and IO loads. Directories don't require any 458 // compression time, but still cost some IO. Similar with small files that 459 // can be very fast to compress. Some files that are more difficult to 460 // compress won't take a corresponding longer time writing out. 461 // 462 // The optimum size here depends on your CPU and IO characteristics, and 463 // the the layout of your zip file. 1000 was chosen mostly at random as 464 // something that worked reasonably well for a test file. 465 // 466 // The RateLimit object will put the upper bounds on the number of 467 // parallel compressions and outstanding buffers. 468 z.writeOps = make(chan chan *zipEntry, 1000) 469 z.cpuRateLimiter = NewCPURateLimiter(int64(parallelJobs)) 470 z.memoryRateLimiter = NewMemoryRateLimiter(0) 471 defer func() { 472 z.cpuRateLimiter.Stop() 473 z.memoryRateLimiter.Stop() 474 }() 475 476 if manifest != "" && !emulateJar { 477 return errors.New("must specify --jar when specifying a manifest via -m") 478 } 479 480 if emulateJar { 481 // manifest may be empty, in which case addManifest will fill in a default 482 pathMappings = append(pathMappings, pathMapping{jar.ManifestFile, manifest, zip.Deflate}) 483 484 jarSort(pathMappings) 485 } 486 487 go func() { 488 var err error 489 defer close(z.writeOps) 490 491 for _, ele := range pathMappings { 492 if emulateJar && ele.dest == jar.ManifestFile { 493 err = z.addManifest(ele.dest, ele.src, ele.zipMethod) 494 } else { 495 err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar, srcJar) 496 } 497 if err != nil { 498 z.errors <- err 499 return 500 } 501 } 502 }() 503 504 zipw := zip.NewWriter(f) 505 506 var currentWriteOpChan chan *zipEntry 507 var currentWriter io.WriteCloser 508 var currentReaders chan chan io.Reader 509 var currentReader chan io.Reader 510 var done bool 511 512 for !done { 513 var writeOpsChan chan chan *zipEntry 514 var writeOpChan chan *zipEntry 515 var readersChan chan chan io.Reader 516 517 if currentReader != nil { 518 // Only read and process errors 519 } else if currentReaders != nil { 520 readersChan = currentReaders 521 } else if currentWriteOpChan != nil { 522 writeOpChan = currentWriteOpChan 523 } else { 524 writeOpsChan = z.writeOps 525 } 526 527 select { 528 case writeOp, ok := <-writeOpsChan: 529 if !ok { 530 done = true 531 } 532 533 currentWriteOpChan = writeOp 534 535 case op := <-writeOpChan: 536 currentWriteOpChan = nil 537 538 var err error 539 if op.fh.Method == zip.Deflate { 540 currentWriter, err = zipw.CreateCompressedHeader(op.fh) 541 } else { 542 var zw io.Writer 543 544 op.fh.CompressedSize64 = op.fh.UncompressedSize64 545 546 zw, err = zipw.CreateHeaderAndroid(op.fh) 547 currentWriter = nopCloser{zw} 548 } 549 if err != nil { 550 return err 551 } 552 553 currentReaders = op.futureReaders 554 if op.futureReaders == nil { 555 currentWriter.Close() 556 currentWriter = nil 557 } 558 z.memoryRateLimiter.Finish(op.allocatedSize) 559 560 case futureReader, ok := <-readersChan: 561 if !ok { 562 // Done with reading 563 currentWriter.Close() 564 currentWriter = nil 565 currentReaders = nil 566 } 567 568 currentReader = futureReader 569 570 case reader := <-currentReader: 571 _, err := io.Copy(currentWriter, reader) 572 if err != nil { 573 return err 574 } 575 576 currentReader = nil 577 578 case err := <-z.errors: 579 return err 580 } 581 } 582 583 // One last chance to catch an error 584 select { 585 case err := <-z.errors: 586 return err 587 default: 588 zipw.Close() 589 return nil 590 } 591} 592 593// imports (possibly with compression) <src> into the zip at sub-path <dest> 594func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar, srcJar bool) error { 595 var fileSize int64 596 var executable bool 597 598 var s os.FileInfo 599 var err error 600 if z.followSymlinks { 601 s, err = z.fs.Stat(src) 602 } else { 603 s, err = z.fs.Lstat(src) 604 } 605 606 if err != nil { 607 if os.IsNotExist(err) && z.ignoreMissingFiles { 608 fmt.Fprintln(z.stderr, "warning:", err) 609 return nil 610 } 611 return err 612 } 613 614 createParentDirs := func(dest, src string) error { 615 if err := z.writeDirectory(filepath.Dir(dest), src, emulateJar); err != nil { 616 return err 617 } 618 619 if prev, exists := z.createdDirs[dest]; exists { 620 return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src) 621 } 622 if prev, exists := z.createdFiles[dest]; exists { 623 return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src) 624 } 625 626 z.createdFiles[dest] = src 627 628 return nil 629 } 630 631 if s.IsDir() { 632 if z.directories { 633 return z.writeDirectory(dest, src, emulateJar) 634 } 635 return nil 636 } else if s.Mode()&os.ModeSymlink != 0 { 637 err = createParentDirs(dest, src) 638 if err != nil { 639 return err 640 } 641 642 return z.writeSymlink(dest, src) 643 } else if s.Mode().IsRegular() { 644 r, err := z.fs.Open(src) 645 if err != nil { 646 return err 647 } 648 649 if srcJar && filepath.Ext(src) == ".java" { 650 // rewrite the destination using the package path if it can be determined 651 pkg, err := jar.JavaPackage(r, src) 652 if err != nil { 653 // ignore errors for now, leaving the file at in its original location in the zip 654 } else { 655 dest = filepath.Join(filepath.Join(strings.Split(pkg, ".")...), filepath.Base(src)) 656 } 657 658 _, err = r.Seek(0, io.SeekStart) 659 if err != nil { 660 return err 661 } 662 } 663 664 fileSize = s.Size() 665 executable = s.Mode()&0100 != 0 666 667 header := &zip.FileHeader{ 668 Name: dest, 669 Method: method, 670 UncompressedSize64: uint64(fileSize), 671 } 672 673 if executable { 674 header.SetMode(0700) 675 } 676 677 err = createParentDirs(dest, src) 678 if err != nil { 679 return err 680 } 681 682 return z.writeFileContents(header, r) 683 } else { 684 return fmt.Errorf("%s is not a file, directory, or symlink", src) 685 } 686} 687 688func (z *ZipWriter) addManifest(dest string, src string, method uint16) error { 689 if prev, exists := z.createdDirs[dest]; exists { 690 return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src) 691 } 692 if prev, exists := z.createdFiles[dest]; exists { 693 return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src) 694 } 695 696 if err := z.writeDirectory(filepath.Dir(dest), src, true); err != nil { 697 return err 698 } 699 700 var contents []byte 701 if src != "" { 702 f, err := z.fs.Open(src) 703 if err != nil { 704 return err 705 } 706 707 contents, err = ioutil.ReadAll(f) 708 f.Close() 709 if err != nil { 710 return err 711 } 712 } 713 714 fh, buf, err := jar.ManifestFileContents(contents) 715 if err != nil { 716 return err 717 } 718 719 reader := &byteReaderCloser{bytes.NewReader(buf), ioutil.NopCloser(nil)} 720 721 return z.writeFileContents(fh, reader) 722} 723 724func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r pathtools.ReaderAtSeekerCloser) (err error) { 725 726 header.SetModTime(z.time) 727 728 compressChan := make(chan *zipEntry, 1) 729 z.writeOps <- compressChan 730 731 // Pre-fill a zipEntry, it will be sent in the compressChan once 732 // we're sure about the Method and CRC. 733 ze := &zipEntry{ 734 fh: header, 735 } 736 737 ze.allocatedSize = int64(header.UncompressedSize64) 738 z.cpuRateLimiter.Request() 739 z.memoryRateLimiter.Request(ze.allocatedSize) 740 741 fileSize := int64(header.UncompressedSize64) 742 if fileSize == 0 { 743 fileSize = int64(header.UncompressedSize) 744 } 745 746 if header.Method == zip.Deflate && fileSize >= minParallelFileSize { 747 wg := new(sync.WaitGroup) 748 749 // Allocate enough buffer to hold all readers. We'll limit 750 // this based on actual buffer sizes in RateLimit. 751 ze.futureReaders = make(chan chan io.Reader, (fileSize/parallelBlockSize)+1) 752 753 // Calculate the CRC in the background, since reading the entire 754 // file could take a while. 755 // 756 // We could split this up into chunks as well, but it's faster 757 // than the compression. Due to the Go Zip API, we also need to 758 // know the result before we can begin writing the compressed 759 // data out to the zipfile. 760 wg.Add(1) 761 go z.crcFile(r, ze, compressChan, wg) 762 763 for start := int64(0); start < fileSize; start += parallelBlockSize { 764 sr := io.NewSectionReader(r, start, parallelBlockSize) 765 resultChan := make(chan io.Reader, 1) 766 ze.futureReaders <- resultChan 767 768 z.cpuRateLimiter.Request() 769 770 last := !(start+parallelBlockSize < fileSize) 771 var dict []byte 772 if start >= windowSize { 773 dict, err = ioutil.ReadAll(io.NewSectionReader(r, start-windowSize, windowSize)) 774 if err != nil { 775 return err 776 } 777 } 778 779 wg.Add(1) 780 go z.compressPartialFile(sr, dict, last, resultChan, wg) 781 } 782 783 close(ze.futureReaders) 784 785 // Close the file handle after all readers are done 786 go func(wg *sync.WaitGroup, closer io.Closer) { 787 wg.Wait() 788 closer.Close() 789 }(wg, r) 790 } else { 791 go func() { 792 z.compressWholeFile(ze, r, compressChan) 793 r.Close() 794 }() 795 } 796 797 return nil 798} 799 800func (z *ZipWriter) crcFile(r io.Reader, ze *zipEntry, resultChan chan *zipEntry, wg *sync.WaitGroup) { 801 defer wg.Done() 802 defer z.cpuRateLimiter.Finish() 803 804 crc := crc32.NewIEEE() 805 _, err := io.Copy(crc, r) 806 if err != nil { 807 z.errors <- err 808 return 809 } 810 811 ze.fh.CRC32 = crc.Sum32() 812 resultChan <- ze 813 close(resultChan) 814} 815 816func (z *ZipWriter) compressPartialFile(r io.Reader, dict []byte, last bool, resultChan chan io.Reader, wg *sync.WaitGroup) { 817 defer wg.Done() 818 819 result, err := z.compressBlock(r, dict, last) 820 if err != nil { 821 z.errors <- err 822 return 823 } 824 825 z.cpuRateLimiter.Finish() 826 827 resultChan <- result 828} 829 830func (z *ZipWriter) compressBlock(r io.Reader, dict []byte, last bool) (*bytes.Buffer, error) { 831 buf := new(bytes.Buffer) 832 var fw *flate.Writer 833 var err error 834 if len(dict) > 0 { 835 // There's no way to Reset a Writer with a new dictionary, so 836 // don't use the Pool 837 fw, err = flate.NewWriterDict(buf, z.compLevel, dict) 838 } else { 839 var ok bool 840 if fw, ok = z.compressorPool.Get().(*flate.Writer); ok { 841 fw.Reset(buf) 842 } else { 843 fw, err = flate.NewWriter(buf, z.compLevel) 844 } 845 defer z.compressorPool.Put(fw) 846 } 847 if err != nil { 848 return nil, err 849 } 850 851 _, err = io.Copy(fw, r) 852 if err != nil { 853 return nil, err 854 } 855 if last { 856 fw.Close() 857 } else { 858 fw.Flush() 859 } 860 861 return buf, nil 862} 863 864func (z *ZipWriter) compressWholeFile(ze *zipEntry, r io.ReadSeeker, compressChan chan *zipEntry) { 865 866 crc := crc32.NewIEEE() 867 _, err := io.Copy(crc, r) 868 if err != nil { 869 z.errors <- err 870 return 871 } 872 873 ze.fh.CRC32 = crc.Sum32() 874 875 _, err = r.Seek(0, 0) 876 if err != nil { 877 z.errors <- err 878 return 879 } 880 881 readFile := func(reader io.ReadSeeker) ([]byte, error) { 882 _, err := reader.Seek(0, 0) 883 if err != nil { 884 return nil, err 885 } 886 887 buf, err := ioutil.ReadAll(reader) 888 if err != nil { 889 return nil, err 890 } 891 892 return buf, nil 893 } 894 895 ze.futureReaders = make(chan chan io.Reader, 1) 896 futureReader := make(chan io.Reader, 1) 897 ze.futureReaders <- futureReader 898 close(ze.futureReaders) 899 900 if ze.fh.Method == zip.Deflate { 901 compressed, err := z.compressBlock(r, nil, true) 902 if err != nil { 903 z.errors <- err 904 return 905 } 906 if uint64(compressed.Len()) < ze.fh.UncompressedSize64 { 907 futureReader <- compressed 908 } else { 909 buf, err := readFile(r) 910 if err != nil { 911 z.errors <- err 912 return 913 } 914 ze.fh.Method = zip.Store 915 futureReader <- bytes.NewReader(buf) 916 } 917 } else { 918 buf, err := readFile(r) 919 if err != nil { 920 z.errors <- err 921 return 922 } 923 ze.fh.Method = zip.Store 924 futureReader <- bytes.NewReader(buf) 925 } 926 927 z.cpuRateLimiter.Finish() 928 929 close(futureReader) 930 931 compressChan <- ze 932 close(compressChan) 933} 934 935// writeDirectory annotates that dir is a directory created for the src file or directory, and adds 936// the directory entry to the zip file if directories are enabled. 937func (z *ZipWriter) writeDirectory(dir string, src string, emulateJar bool) error { 938 // clean the input 939 dir = filepath.Clean(dir) 940 941 // discover any uncreated directories in the path 942 zipDirs := []string{} 943 for dir != "" && dir != "." { 944 if _, exists := z.createdDirs[dir]; exists { 945 break 946 } 947 948 if prev, exists := z.createdFiles[dir]; exists { 949 return fmt.Errorf("destination %q is both a directory %q and a file %q", dir, src, prev) 950 } 951 952 z.createdDirs[dir] = src 953 // parent directories precede their children 954 zipDirs = append([]string{dir}, zipDirs...) 955 956 dir = filepath.Dir(dir) 957 } 958 959 if z.directories { 960 // make a directory entry for each uncreated directory 961 for _, cleanDir := range zipDirs { 962 var dirHeader *zip.FileHeader 963 964 if emulateJar && cleanDir+"/" == jar.MetaDir { 965 dirHeader = jar.MetaDirFileHeader() 966 } else { 967 dirHeader = &zip.FileHeader{ 968 Name: cleanDir + "/", 969 } 970 dirHeader.SetMode(0700 | os.ModeDir) 971 } 972 973 dirHeader.SetModTime(z.time) 974 975 ze := make(chan *zipEntry, 1) 976 ze <- &zipEntry{ 977 fh: dirHeader, 978 } 979 close(ze) 980 z.writeOps <- ze 981 } 982 } 983 984 return nil 985} 986 987func (z *ZipWriter) writeSymlink(rel, file string) error { 988 fileHeader := &zip.FileHeader{ 989 Name: rel, 990 } 991 fileHeader.SetModTime(z.time) 992 fileHeader.SetMode(0777 | os.ModeSymlink) 993 994 dest, err := z.fs.Readlink(file) 995 if err != nil { 996 return err 997 } 998 999 fileHeader.UncompressedSize64 = uint64(len(dest)) 1000 fileHeader.CRC32 = crc32.ChecksumIEEE([]byte(dest)) 1001 1002 ze := make(chan *zipEntry, 1) 1003 futureReaders := make(chan chan io.Reader, 1) 1004 futureReader := make(chan io.Reader, 1) 1005 futureReaders <- futureReader 1006 close(futureReaders) 1007 futureReader <- bytes.NewBufferString(dest) 1008 close(futureReader) 1009 1010 ze <- &zipEntry{ 1011 fh: fileHeader, 1012 futureReaders: futureReaders, 1013 } 1014 close(ze) 1015 z.writeOps <- ze 1016 1017 return nil 1018} 1019