1// Copyright 2015 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package zip
16
17import (
18	"bytes"
19	"compress/flate"
20	"errors"
21	"fmt"
22	"hash/crc32"
23	"io"
24	"io/ioutil"
25	"os"
26	"path/filepath"
27	"sort"
28	"strings"
29	"sync"
30	"syscall"
31	"time"
32	"unicode"
33
34	"github.com/google/blueprint/pathtools"
35
36	"android/soong/jar"
37	"android/soong/third_party/zip"
38)
39
40// Block size used during parallel compression of a single file.
41const parallelBlockSize = 1 * 1024 * 1024 // 1MB
42
43// Minimum file size to use parallel compression. It requires more
44// flate.Writer allocations, since we can't change the dictionary
45// during Reset
46const minParallelFileSize = parallelBlockSize * 6
47
48// Size of the ZIP compression window (32KB)
49const windowSize = 32 * 1024
50
51type nopCloser struct {
52	io.Writer
53}
54
55func (nopCloser) Close() error {
56	return nil
57}
58
59type byteReaderCloser struct {
60	*bytes.Reader
61	io.Closer
62}
63
64type pathMapping struct {
65	dest, src string
66	zipMethod uint16
67}
68
69type FileArg struct {
70	PathPrefixInZip, SourcePrefixToStrip string
71	SourceFiles                          []string
72	JunkPaths                            bool
73	GlobDir                              string
74}
75
76type FileArgsBuilder struct {
77	state FileArg
78	err   error
79	fs    pathtools.FileSystem
80
81	fileArgs []FileArg
82}
83
84func NewFileArgsBuilder() *FileArgsBuilder {
85	return &FileArgsBuilder{
86		fs: pathtools.OsFs,
87	}
88}
89
90func (b *FileArgsBuilder) JunkPaths(v bool) *FileArgsBuilder {
91	b.state.JunkPaths = v
92	b.state.SourcePrefixToStrip = ""
93	return b
94}
95
96func (b *FileArgsBuilder) SourcePrefixToStrip(prefixToStrip string) *FileArgsBuilder {
97	b.state.JunkPaths = false
98	b.state.SourcePrefixToStrip = prefixToStrip
99	return b
100}
101
102func (b *FileArgsBuilder) PathPrefixInZip(rootPrefix string) *FileArgsBuilder {
103	b.state.PathPrefixInZip = rootPrefix
104	return b
105}
106
107func (b *FileArgsBuilder) File(name string) *FileArgsBuilder {
108	if b.err != nil {
109		return b
110	}
111
112	arg := b.state
113	arg.SourceFiles = []string{name}
114	b.fileArgs = append(b.fileArgs, arg)
115	return b
116}
117
118func (b *FileArgsBuilder) Dir(name string) *FileArgsBuilder {
119	if b.err != nil {
120		return b
121	}
122
123	arg := b.state
124	arg.GlobDir = name
125	b.fileArgs = append(b.fileArgs, arg)
126	return b
127}
128
129func (b *FileArgsBuilder) List(name string) *FileArgsBuilder {
130	if b.err != nil {
131		return b
132	}
133
134	f, err := b.fs.Open(name)
135	if err != nil {
136		b.err = err
137		return b
138	}
139	defer f.Close()
140
141	list, err := ioutil.ReadAll(f)
142	if err != nil {
143		b.err = err
144		return b
145	}
146
147	arg := b.state
148	arg.SourceFiles = strings.Fields(string(list))
149	b.fileArgs = append(b.fileArgs, arg)
150	return b
151}
152
153func (b *FileArgsBuilder) Error() error {
154	if b == nil {
155		return nil
156	}
157	return b.err
158}
159
160func (b *FileArgsBuilder) FileArgs() []FileArg {
161	if b == nil {
162		return nil
163	}
164	return b.fileArgs
165}
166
167type IncorrectRelativeRootError struct {
168	RelativeRoot string
169	Path         string
170}
171
172func (x IncorrectRelativeRootError) Error() string {
173	return fmt.Sprintf("path %q is outside relative root %q", x.Path, x.RelativeRoot)
174}
175
176type ZipWriter struct {
177	time         time.Time
178	createdFiles map[string]string
179	createdDirs  map[string]string
180	directories  bool
181
182	errors   chan error
183	writeOps chan chan *zipEntry
184
185	cpuRateLimiter    *CPURateLimiter
186	memoryRateLimiter *MemoryRateLimiter
187
188	compressorPool sync.Pool
189	compLevel      int
190
191	followSymlinks     pathtools.ShouldFollowSymlinks
192	ignoreMissingFiles bool
193
194	stderr io.Writer
195	fs     pathtools.FileSystem
196}
197
198type zipEntry struct {
199	fh *zip.FileHeader
200
201	// List of delayed io.Reader
202	futureReaders chan chan io.Reader
203
204	// Only used for passing into the MemoryRateLimiter to ensure we
205	// release as much memory as much as we request
206	allocatedSize int64
207}
208
209type ZipArgs struct {
210	FileArgs                 []FileArg
211	OutputFilePath           string
212	EmulateJar               bool
213	SrcJar                   bool
214	AddDirectoryEntriesToZip bool
215	CompressionLevel         int
216	ManifestSourcePath       string
217	NumParallelJobs          int
218	NonDeflatedFiles         map[string]bool
219	WriteIfChanged           bool
220	StoreSymlinks            bool
221	IgnoreMissingFiles       bool
222
223	Stderr     io.Writer
224	Filesystem pathtools.FileSystem
225}
226
227const NOQUOTE = '\x00'
228
229func ReadRespFile(bytes []byte) []string {
230	var args []string
231	var arg []rune
232
233	isEscaping := false
234	quotingStart := NOQUOTE
235	for _, c := range string(bytes) {
236		switch {
237		case isEscaping:
238			if quotingStart == '"' {
239				if !(c == '"' || c == '\\') {
240					// '\"' or '\\' will be escaped under double quoting.
241					arg = append(arg, '\\')
242				}
243			}
244			arg = append(arg, c)
245			isEscaping = false
246		case c == '\\' && quotingStart != '\'':
247			isEscaping = true
248		case quotingStart == NOQUOTE && (c == '\'' || c == '"'):
249			quotingStart = c
250		case quotingStart != NOQUOTE && c == quotingStart:
251			quotingStart = NOQUOTE
252		case quotingStart == NOQUOTE && unicode.IsSpace(c):
253			// Current character is a space outside quotes
254			if len(arg) != 0 {
255				args = append(args, string(arg))
256			}
257			arg = arg[:0]
258		default:
259			arg = append(arg, c)
260		}
261	}
262
263	if len(arg) != 0 {
264		args = append(args, string(arg))
265	}
266
267	return args
268}
269
270func ZipTo(args ZipArgs, w io.Writer) error {
271	if args.EmulateJar {
272		args.AddDirectoryEntriesToZip = true
273	}
274
275	// Have Glob follow symlinks if they are not being stored as symlinks in the zip file.
276	followSymlinks := pathtools.ShouldFollowSymlinks(!args.StoreSymlinks)
277
278	z := &ZipWriter{
279		time:               jar.DefaultTime,
280		createdDirs:        make(map[string]string),
281		createdFiles:       make(map[string]string),
282		directories:        args.AddDirectoryEntriesToZip,
283		compLevel:          args.CompressionLevel,
284		followSymlinks:     followSymlinks,
285		ignoreMissingFiles: args.IgnoreMissingFiles,
286		stderr:             args.Stderr,
287		fs:                 args.Filesystem,
288	}
289
290	if z.fs == nil {
291		z.fs = pathtools.OsFs
292	}
293
294	if z.stderr == nil {
295		z.stderr = os.Stderr
296	}
297
298	pathMappings := []pathMapping{}
299
300	noCompression := args.CompressionLevel == 0
301
302	for _, fa := range args.FileArgs {
303		var srcs []string
304		for _, s := range fa.SourceFiles {
305			s = strings.TrimSpace(s)
306			if s == "" {
307				continue
308			}
309
310			globbed, _, err := z.fs.Glob(s, nil, followSymlinks)
311			if err != nil {
312				return err
313			}
314			if len(globbed) == 0 {
315				err := &os.PathError{
316					Op:   "lstat",
317					Path: s,
318					Err:  os.ErrNotExist,
319				}
320				if args.IgnoreMissingFiles {
321					fmt.Fprintln(z.stderr, "warning:", err)
322				} else {
323					return err
324				}
325			}
326			srcs = append(srcs, globbed...)
327		}
328		if fa.GlobDir != "" {
329			if exists, isDir, err := z.fs.Exists(fa.GlobDir); err != nil {
330				return err
331			} else if !exists && !args.IgnoreMissingFiles {
332				err := &os.PathError{
333					Op:   "lstat",
334					Path: fa.GlobDir,
335					Err:  os.ErrNotExist,
336				}
337				if args.IgnoreMissingFiles {
338					fmt.Fprintln(z.stderr, "warning:", err)
339				} else {
340					return err
341				}
342			} else if !isDir && !args.IgnoreMissingFiles {
343				err := &os.PathError{
344					Op:   "lstat",
345					Path: fa.GlobDir,
346					Err:  syscall.ENOTDIR,
347				}
348				if args.IgnoreMissingFiles {
349					fmt.Fprintln(z.stderr, "warning:", err)
350				} else {
351					return err
352				}
353			}
354			globbed, _, err := z.fs.Glob(filepath.Join(fa.GlobDir, "**/*"), nil, followSymlinks)
355			if err != nil {
356				return err
357			}
358			srcs = append(srcs, globbed...)
359		}
360		for _, src := range srcs {
361			err := fillPathPairs(fa, src, &pathMappings, args.NonDeflatedFiles, noCompression)
362			if err != nil {
363				return err
364			}
365		}
366	}
367
368	return z.write(w, pathMappings, args.ManifestSourcePath, args.EmulateJar, args.SrcJar, args.NumParallelJobs)
369}
370
371func Zip(args ZipArgs) error {
372	if args.OutputFilePath == "" {
373		return fmt.Errorf("output file path must be nonempty")
374	}
375
376	buf := &bytes.Buffer{}
377	var out io.Writer = buf
378
379	if !args.WriteIfChanged {
380		f, err := os.Create(args.OutputFilePath)
381		if err != nil {
382			return err
383		}
384
385		defer f.Close()
386		defer func() {
387			if err != nil {
388				os.Remove(args.OutputFilePath)
389			}
390		}()
391
392		out = f
393	}
394
395	err := ZipTo(args, out)
396	if err != nil {
397		return err
398	}
399
400	if args.WriteIfChanged {
401		err := pathtools.WriteFileIfChanged(args.OutputFilePath, buf.Bytes(), 0666)
402		if err != nil {
403			return err
404		}
405	}
406
407	return nil
408}
409
410func fillPathPairs(fa FileArg, src string, pathMappings *[]pathMapping,
411	nonDeflatedFiles map[string]bool, noCompression bool) error {
412
413	var dest string
414
415	if fa.JunkPaths {
416		dest = filepath.Base(src)
417	} else {
418		var err error
419		dest, err = filepath.Rel(fa.SourcePrefixToStrip, src)
420		if err != nil {
421			return err
422		}
423		if strings.HasPrefix(dest, "../") {
424			return IncorrectRelativeRootError{
425				Path:         src,
426				RelativeRoot: fa.SourcePrefixToStrip,
427			}
428		}
429
430	}
431	dest = filepath.Join(fa.PathPrefixInZip, dest)
432
433	zipMethod := zip.Deflate
434	if _, found := nonDeflatedFiles[dest]; found || noCompression {
435		zipMethod = zip.Store
436	}
437	*pathMappings = append(*pathMappings,
438		pathMapping{dest: dest, src: src, zipMethod: zipMethod})
439
440	return nil
441}
442
443func jarSort(mappings []pathMapping) {
444	less := func(i int, j int) (smaller bool) {
445		return jar.EntryNamesLess(mappings[i].dest, mappings[j].dest)
446	}
447	sort.SliceStable(mappings, less)
448}
449
450func (z *ZipWriter) write(f io.Writer, pathMappings []pathMapping, manifest string, emulateJar, srcJar bool,
451	parallelJobs int) error {
452
453	z.errors = make(chan error)
454	defer close(z.errors)
455
456	// This channel size can be essentially unlimited -- it's used as a fifo
457	// queue decouple the CPU and IO loads. Directories don't require any
458	// compression time, but still cost some IO. Similar with small files that
459	// can be very fast to compress. Some files that are more difficult to
460	// compress won't take a corresponding longer time writing out.
461	//
462	// The optimum size here depends on your CPU and IO characteristics, and
463	// the the layout of your zip file. 1000 was chosen mostly at random as
464	// something that worked reasonably well for a test file.
465	//
466	// The RateLimit object will put the upper bounds on the number of
467	// parallel compressions and outstanding buffers.
468	z.writeOps = make(chan chan *zipEntry, 1000)
469	z.cpuRateLimiter = NewCPURateLimiter(int64(parallelJobs))
470	z.memoryRateLimiter = NewMemoryRateLimiter(0)
471	defer func() {
472		z.cpuRateLimiter.Stop()
473		z.memoryRateLimiter.Stop()
474	}()
475
476	if manifest != "" && !emulateJar {
477		return errors.New("must specify --jar when specifying a manifest via -m")
478	}
479
480	if emulateJar {
481		// manifest may be empty, in which case addManifest will fill in a default
482		pathMappings = append(pathMappings, pathMapping{jar.ManifestFile, manifest, zip.Deflate})
483
484		jarSort(pathMappings)
485	}
486
487	go func() {
488		var err error
489		defer close(z.writeOps)
490
491		for _, ele := range pathMappings {
492			if emulateJar && ele.dest == jar.ManifestFile {
493				err = z.addManifest(ele.dest, ele.src, ele.zipMethod)
494			} else {
495				err = z.addFile(ele.dest, ele.src, ele.zipMethod, emulateJar, srcJar)
496			}
497			if err != nil {
498				z.errors <- err
499				return
500			}
501		}
502	}()
503
504	zipw := zip.NewWriter(f)
505
506	var currentWriteOpChan chan *zipEntry
507	var currentWriter io.WriteCloser
508	var currentReaders chan chan io.Reader
509	var currentReader chan io.Reader
510	var done bool
511
512	for !done {
513		var writeOpsChan chan chan *zipEntry
514		var writeOpChan chan *zipEntry
515		var readersChan chan chan io.Reader
516
517		if currentReader != nil {
518			// Only read and process errors
519		} else if currentReaders != nil {
520			readersChan = currentReaders
521		} else if currentWriteOpChan != nil {
522			writeOpChan = currentWriteOpChan
523		} else {
524			writeOpsChan = z.writeOps
525		}
526
527		select {
528		case writeOp, ok := <-writeOpsChan:
529			if !ok {
530				done = true
531			}
532
533			currentWriteOpChan = writeOp
534
535		case op := <-writeOpChan:
536			currentWriteOpChan = nil
537
538			var err error
539			if op.fh.Method == zip.Deflate {
540				currentWriter, err = zipw.CreateCompressedHeader(op.fh)
541			} else {
542				var zw io.Writer
543
544				op.fh.CompressedSize64 = op.fh.UncompressedSize64
545
546				zw, err = zipw.CreateHeaderAndroid(op.fh)
547				currentWriter = nopCloser{zw}
548			}
549			if err != nil {
550				return err
551			}
552
553			currentReaders = op.futureReaders
554			if op.futureReaders == nil {
555				currentWriter.Close()
556				currentWriter = nil
557			}
558			z.memoryRateLimiter.Finish(op.allocatedSize)
559
560		case futureReader, ok := <-readersChan:
561			if !ok {
562				// Done with reading
563				currentWriter.Close()
564				currentWriter = nil
565				currentReaders = nil
566			}
567
568			currentReader = futureReader
569
570		case reader := <-currentReader:
571			_, err := io.Copy(currentWriter, reader)
572			if err != nil {
573				return err
574			}
575
576			currentReader = nil
577
578		case err := <-z.errors:
579			return err
580		}
581	}
582
583	// One last chance to catch an error
584	select {
585	case err := <-z.errors:
586		return err
587	default:
588		zipw.Close()
589		return nil
590	}
591}
592
593// imports (possibly with compression) <src> into the zip at sub-path <dest>
594func (z *ZipWriter) addFile(dest, src string, method uint16, emulateJar, srcJar bool) error {
595	var fileSize int64
596	var executable bool
597
598	var s os.FileInfo
599	var err error
600	if z.followSymlinks {
601		s, err = z.fs.Stat(src)
602	} else {
603		s, err = z.fs.Lstat(src)
604	}
605
606	if err != nil {
607		if os.IsNotExist(err) && z.ignoreMissingFiles {
608			fmt.Fprintln(z.stderr, "warning:", err)
609			return nil
610		}
611		return err
612	}
613
614	createParentDirs := func(dest, src string) error {
615		if err := z.writeDirectory(filepath.Dir(dest), src, emulateJar); err != nil {
616			return err
617		}
618
619		if prev, exists := z.createdDirs[dest]; exists {
620			return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src)
621		}
622		if prev, exists := z.createdFiles[dest]; exists {
623			return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src)
624		}
625
626		z.createdFiles[dest] = src
627
628		return nil
629	}
630
631	if s.IsDir() {
632		if z.directories {
633			return z.writeDirectory(dest, src, emulateJar)
634		}
635		return nil
636	} else if s.Mode()&os.ModeSymlink != 0 {
637		err = createParentDirs(dest, src)
638		if err != nil {
639			return err
640		}
641
642		return z.writeSymlink(dest, src)
643	} else if s.Mode().IsRegular() {
644		r, err := z.fs.Open(src)
645		if err != nil {
646			return err
647		}
648
649		if srcJar && filepath.Ext(src) == ".java" {
650			// rewrite the destination using the package path if it can be determined
651			pkg, err := jar.JavaPackage(r, src)
652			if err != nil {
653				// ignore errors for now, leaving the file at in its original location in the zip
654			} else {
655				dest = filepath.Join(filepath.Join(strings.Split(pkg, ".")...), filepath.Base(src))
656			}
657
658			_, err = r.Seek(0, io.SeekStart)
659			if err != nil {
660				return err
661			}
662		}
663
664		fileSize = s.Size()
665		executable = s.Mode()&0100 != 0
666
667		header := &zip.FileHeader{
668			Name:               dest,
669			Method:             method,
670			UncompressedSize64: uint64(fileSize),
671		}
672
673		if executable {
674			header.SetMode(0700)
675		}
676
677		err = createParentDirs(dest, src)
678		if err != nil {
679			return err
680		}
681
682		return z.writeFileContents(header, r)
683	} else {
684		return fmt.Errorf("%s is not a file, directory, or symlink", src)
685	}
686}
687
688func (z *ZipWriter) addManifest(dest string, src string, method uint16) error {
689	if prev, exists := z.createdDirs[dest]; exists {
690		return fmt.Errorf("destination %q is both a directory %q and a file %q", dest, prev, src)
691	}
692	if prev, exists := z.createdFiles[dest]; exists {
693		return fmt.Errorf("destination %q has two files %q and %q", dest, prev, src)
694	}
695
696	if err := z.writeDirectory(filepath.Dir(dest), src, true); err != nil {
697		return err
698	}
699
700	var contents []byte
701	if src != "" {
702		f, err := z.fs.Open(src)
703		if err != nil {
704			return err
705		}
706
707		contents, err = ioutil.ReadAll(f)
708		f.Close()
709		if err != nil {
710			return err
711		}
712	}
713
714	fh, buf, err := jar.ManifestFileContents(contents)
715	if err != nil {
716		return err
717	}
718
719	reader := &byteReaderCloser{bytes.NewReader(buf), ioutil.NopCloser(nil)}
720
721	return z.writeFileContents(fh, reader)
722}
723
724func (z *ZipWriter) writeFileContents(header *zip.FileHeader, r pathtools.ReaderAtSeekerCloser) (err error) {
725
726	header.SetModTime(z.time)
727
728	compressChan := make(chan *zipEntry, 1)
729	z.writeOps <- compressChan
730
731	// Pre-fill a zipEntry, it will be sent in the compressChan once
732	// we're sure about the Method and CRC.
733	ze := &zipEntry{
734		fh: header,
735	}
736
737	ze.allocatedSize = int64(header.UncompressedSize64)
738	z.cpuRateLimiter.Request()
739	z.memoryRateLimiter.Request(ze.allocatedSize)
740
741	fileSize := int64(header.UncompressedSize64)
742	if fileSize == 0 {
743		fileSize = int64(header.UncompressedSize)
744	}
745
746	if header.Method == zip.Deflate && fileSize >= minParallelFileSize {
747		wg := new(sync.WaitGroup)
748
749		// Allocate enough buffer to hold all readers. We'll limit
750		// this based on actual buffer sizes in RateLimit.
751		ze.futureReaders = make(chan chan io.Reader, (fileSize/parallelBlockSize)+1)
752
753		// Calculate the CRC in the background, since reading the entire
754		// file could take a while.
755		//
756		// We could split this up into chunks as well, but it's faster
757		// than the compression. Due to the Go Zip API, we also need to
758		// know the result before we can begin writing the compressed
759		// data out to the zipfile.
760		wg.Add(1)
761		go z.crcFile(r, ze, compressChan, wg)
762
763		for start := int64(0); start < fileSize; start += parallelBlockSize {
764			sr := io.NewSectionReader(r, start, parallelBlockSize)
765			resultChan := make(chan io.Reader, 1)
766			ze.futureReaders <- resultChan
767
768			z.cpuRateLimiter.Request()
769
770			last := !(start+parallelBlockSize < fileSize)
771			var dict []byte
772			if start >= windowSize {
773				dict, err = ioutil.ReadAll(io.NewSectionReader(r, start-windowSize, windowSize))
774				if err != nil {
775					return err
776				}
777			}
778
779			wg.Add(1)
780			go z.compressPartialFile(sr, dict, last, resultChan, wg)
781		}
782
783		close(ze.futureReaders)
784
785		// Close the file handle after all readers are done
786		go func(wg *sync.WaitGroup, closer io.Closer) {
787			wg.Wait()
788			closer.Close()
789		}(wg, r)
790	} else {
791		go func() {
792			z.compressWholeFile(ze, r, compressChan)
793			r.Close()
794		}()
795	}
796
797	return nil
798}
799
800func (z *ZipWriter) crcFile(r io.Reader, ze *zipEntry, resultChan chan *zipEntry, wg *sync.WaitGroup) {
801	defer wg.Done()
802	defer z.cpuRateLimiter.Finish()
803
804	crc := crc32.NewIEEE()
805	_, err := io.Copy(crc, r)
806	if err != nil {
807		z.errors <- err
808		return
809	}
810
811	ze.fh.CRC32 = crc.Sum32()
812	resultChan <- ze
813	close(resultChan)
814}
815
816func (z *ZipWriter) compressPartialFile(r io.Reader, dict []byte, last bool, resultChan chan io.Reader, wg *sync.WaitGroup) {
817	defer wg.Done()
818
819	result, err := z.compressBlock(r, dict, last)
820	if err != nil {
821		z.errors <- err
822		return
823	}
824
825	z.cpuRateLimiter.Finish()
826
827	resultChan <- result
828}
829
830func (z *ZipWriter) compressBlock(r io.Reader, dict []byte, last bool) (*bytes.Buffer, error) {
831	buf := new(bytes.Buffer)
832	var fw *flate.Writer
833	var err error
834	if len(dict) > 0 {
835		// There's no way to Reset a Writer with a new dictionary, so
836		// don't use the Pool
837		fw, err = flate.NewWriterDict(buf, z.compLevel, dict)
838	} else {
839		var ok bool
840		if fw, ok = z.compressorPool.Get().(*flate.Writer); ok {
841			fw.Reset(buf)
842		} else {
843			fw, err = flate.NewWriter(buf, z.compLevel)
844		}
845		defer z.compressorPool.Put(fw)
846	}
847	if err != nil {
848		return nil, err
849	}
850
851	_, err = io.Copy(fw, r)
852	if err != nil {
853		return nil, err
854	}
855	if last {
856		fw.Close()
857	} else {
858		fw.Flush()
859	}
860
861	return buf, nil
862}
863
864func (z *ZipWriter) compressWholeFile(ze *zipEntry, r io.ReadSeeker, compressChan chan *zipEntry) {
865
866	crc := crc32.NewIEEE()
867	_, err := io.Copy(crc, r)
868	if err != nil {
869		z.errors <- err
870		return
871	}
872
873	ze.fh.CRC32 = crc.Sum32()
874
875	_, err = r.Seek(0, 0)
876	if err != nil {
877		z.errors <- err
878		return
879	}
880
881	readFile := func(reader io.ReadSeeker) ([]byte, error) {
882		_, err := reader.Seek(0, 0)
883		if err != nil {
884			return nil, err
885		}
886
887		buf, err := ioutil.ReadAll(reader)
888		if err != nil {
889			return nil, err
890		}
891
892		return buf, nil
893	}
894
895	ze.futureReaders = make(chan chan io.Reader, 1)
896	futureReader := make(chan io.Reader, 1)
897	ze.futureReaders <- futureReader
898	close(ze.futureReaders)
899
900	if ze.fh.Method == zip.Deflate {
901		compressed, err := z.compressBlock(r, nil, true)
902		if err != nil {
903			z.errors <- err
904			return
905		}
906		if uint64(compressed.Len()) < ze.fh.UncompressedSize64 {
907			futureReader <- compressed
908		} else {
909			buf, err := readFile(r)
910			if err != nil {
911				z.errors <- err
912				return
913			}
914			ze.fh.Method = zip.Store
915			futureReader <- bytes.NewReader(buf)
916		}
917	} else {
918		buf, err := readFile(r)
919		if err != nil {
920			z.errors <- err
921			return
922		}
923		ze.fh.Method = zip.Store
924		futureReader <- bytes.NewReader(buf)
925	}
926
927	z.cpuRateLimiter.Finish()
928
929	close(futureReader)
930
931	compressChan <- ze
932	close(compressChan)
933}
934
935// writeDirectory annotates that dir is a directory created for the src file or directory, and adds
936// the directory entry to the zip file if directories are enabled.
937func (z *ZipWriter) writeDirectory(dir string, src string, emulateJar bool) error {
938	// clean the input
939	dir = filepath.Clean(dir)
940
941	// discover any uncreated directories in the path
942	zipDirs := []string{}
943	for dir != "" && dir != "." {
944		if _, exists := z.createdDirs[dir]; exists {
945			break
946		}
947
948		if prev, exists := z.createdFiles[dir]; exists {
949			return fmt.Errorf("destination %q is both a directory %q and a file %q", dir, src, prev)
950		}
951
952		z.createdDirs[dir] = src
953		// parent directories precede their children
954		zipDirs = append([]string{dir}, zipDirs...)
955
956		dir = filepath.Dir(dir)
957	}
958
959	if z.directories {
960		// make a directory entry for each uncreated directory
961		for _, cleanDir := range zipDirs {
962			var dirHeader *zip.FileHeader
963
964			if emulateJar && cleanDir+"/" == jar.MetaDir {
965				dirHeader = jar.MetaDirFileHeader()
966			} else {
967				dirHeader = &zip.FileHeader{
968					Name: cleanDir + "/",
969				}
970				dirHeader.SetMode(0700 | os.ModeDir)
971			}
972
973			dirHeader.SetModTime(z.time)
974
975			ze := make(chan *zipEntry, 1)
976			ze <- &zipEntry{
977				fh: dirHeader,
978			}
979			close(ze)
980			z.writeOps <- ze
981		}
982	}
983
984	return nil
985}
986
987func (z *ZipWriter) writeSymlink(rel, file string) error {
988	fileHeader := &zip.FileHeader{
989		Name: rel,
990	}
991	fileHeader.SetModTime(z.time)
992	fileHeader.SetMode(0777 | os.ModeSymlink)
993
994	dest, err := z.fs.Readlink(file)
995	if err != nil {
996		return err
997	}
998
999	fileHeader.UncompressedSize64 = uint64(len(dest))
1000	fileHeader.CRC32 = crc32.ChecksumIEEE([]byte(dest))
1001
1002	ze := make(chan *zipEntry, 1)
1003	futureReaders := make(chan chan io.Reader, 1)
1004	futureReader := make(chan io.Reader, 1)
1005	futureReaders <- futureReader
1006	close(futureReaders)
1007	futureReader <- bytes.NewBufferString(dest)
1008	close(futureReader)
1009
1010	ze <- &zipEntry{
1011		fh:            fileHeader,
1012		futureReaders: futureReaders,
1013	}
1014	close(ze)
1015	z.writeOps <- ze
1016
1017	return nil
1018}
1019