1// Copyright 2014 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package pathtools
16
17import (
18	"errors"
19	"fmt"
20	"io/ioutil"
21	"os"
22	"path/filepath"
23	"strings"
24
25	"github.com/google/blueprint/deptools"
26)
27
28var GlobMultipleRecursiveErr = errors.New("pattern contains multiple '**'")
29var GlobLastRecursiveErr = errors.New("pattern has '**' as last path element")
30var GlobInvalidRecursiveErr = errors.New("pattern contains other characters between '**' and path separator")
31
32// Glob returns the list of files and directories that match the given pattern
33// but do not match the given exclude patterns, along with the list of
34// directories and other dependencies that were searched to construct the file
35// list.  The supported glob and exclude patterns are equivalent to
36// filepath.Glob, with an extension that recursive glob (** matching zero or
37// more complete path entries) is supported. Any directories in the matches
38// list will have a '/' suffix.
39//
40// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps
41// should be used instead, as they will automatically set up dependencies
42// to rerun the primary builder when the list of matching files changes.
43func Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (matches, deps []string, err error) {
44	return startGlob(OsFs, pattern, excludes, follow)
45}
46
47func startGlob(fs FileSystem, pattern string, excludes []string,
48	follow ShouldFollowSymlinks) (matches, deps []string, err error) {
49
50	if filepath.Base(pattern) == "**" {
51		return nil, nil, GlobLastRecursiveErr
52	} else {
53		matches, deps, err = glob(fs, pattern, false, follow)
54	}
55
56	if err != nil {
57		return nil, nil, err
58	}
59
60	matches, err = filterExcludes(matches, excludes)
61	if err != nil {
62		return nil, nil, err
63	}
64
65	// If the pattern has wildcards, we added dependencies on the
66	// containing directories to know about changes.
67	//
68	// If the pattern didn't have wildcards, and didn't find matches, the
69	// most specific found directories were added.
70	//
71	// But if it didn't have wildcards, and did find a match, no
72	// dependencies were added, so add the match itself to detect when it
73	// is removed.
74	if !isWild(pattern) {
75		deps = append(deps, matches...)
76	}
77
78	for i, match := range matches {
79		isSymlink, err := fs.IsSymlink(match)
80		if err != nil {
81			return nil, nil, err
82		}
83		if !(isSymlink && follow == DontFollowSymlinks) {
84			isDir, err := fs.IsDir(match)
85			if os.IsNotExist(err) {
86				if isSymlink {
87					return nil, nil, fmt.Errorf("%s: dangling symlink", match)
88				}
89			}
90			if err != nil {
91				return nil, nil, fmt.Errorf("%s: %s", match, err.Error())
92			}
93
94			if isDir {
95				matches[i] = match + "/"
96			}
97		}
98	}
99
100	return matches, deps, nil
101}
102
103// glob is a recursive helper function to handle globbing each level of the pattern individually,
104// allowing searched directories to be tracked.  Also handles the recursive glob pattern, **.
105func glob(fs FileSystem, pattern string, hasRecursive bool,
106	follow ShouldFollowSymlinks) (matches, dirs []string, err error) {
107
108	if !isWild(pattern) {
109		// If there are no wilds in the pattern, check whether the file exists or not.
110		// Uses filepath.Glob instead of manually statting to get consistent results.
111		pattern = filepath.Clean(pattern)
112		matches, err = fs.glob(pattern)
113		if err != nil {
114			return matches, dirs, err
115		}
116
117		if len(matches) == 0 {
118			// Some part of the non-wild pattern didn't exist.  Add the last existing directory
119			// as a dependency.
120			var matchDirs []string
121			for len(matchDirs) == 0 {
122				pattern = filepath.Dir(pattern)
123				matchDirs, err = fs.glob(pattern)
124				if err != nil {
125					return matches, dirs, err
126				}
127			}
128			dirs = append(dirs, matchDirs...)
129		}
130		return matches, dirs, err
131	}
132
133	dir, file := saneSplit(pattern)
134
135	if file == "**" {
136		if hasRecursive {
137			return matches, dirs, GlobMultipleRecursiveErr
138		}
139		hasRecursive = true
140	} else if strings.Contains(file, "**") {
141		return matches, dirs, GlobInvalidRecursiveErr
142	}
143
144	dirMatches, dirs, err := glob(fs, dir, hasRecursive, follow)
145	if err != nil {
146		return nil, nil, err
147	}
148
149	for _, m := range dirMatches {
150		isDir, err := fs.IsDir(m)
151		if os.IsNotExist(err) {
152			if isSymlink, _ := fs.IsSymlink(m); isSymlink {
153				return nil, nil, fmt.Errorf("dangling symlink: %s", m)
154			}
155		}
156		if err != nil {
157			return nil, nil, fmt.Errorf("unexpected error after glob: %s", err)
158		}
159
160		if isDir {
161			if file == "**" {
162				recurseDirs, err := fs.ListDirsRecursive(m, follow)
163				if err != nil {
164					return nil, nil, err
165				}
166				matches = append(matches, recurseDirs...)
167			} else {
168				dirs = append(dirs, m)
169				newMatches, err := fs.glob(filepath.Join(MatchEscape(m), file))
170				if err != nil {
171					return nil, nil, err
172				}
173				if file[0] != '.' {
174					newMatches = filterDotFiles(newMatches)
175				}
176				matches = append(matches, newMatches...)
177			}
178		}
179	}
180
181	return matches, dirs, nil
182}
183
184// Faster version of dir, file := filepath.Dir(path), filepath.File(path) with no allocations
185// Similar to filepath.Split, but returns "." if dir is empty and trims trailing slash if dir is
186// not "/".  Returns ".", "" if path is "."
187func saneSplit(path string) (dir, file string) {
188	if path == "." {
189		return ".", ""
190	}
191	dir, file = filepath.Split(path)
192	switch dir {
193	case "":
194		dir = "."
195	case "/":
196		// Nothing
197	default:
198		dir = dir[:len(dir)-1]
199	}
200	return dir, file
201}
202
203func isWild(pattern string) bool {
204	return strings.ContainsAny(pattern, "*?[")
205}
206
207// Filters the strings in matches based on the glob patterns in excludes.  Hierarchical (a/*) and
208// recursive (**) glob patterns are supported.
209func filterExcludes(matches []string, excludes []string) ([]string, error) {
210	if len(excludes) == 0 {
211		return matches, nil
212	}
213
214	var ret []string
215matchLoop:
216	for _, m := range matches {
217		for _, e := range excludes {
218			exclude, err := Match(e, m)
219			if err != nil {
220				return nil, err
221			}
222			if exclude {
223				continue matchLoop
224			}
225		}
226		ret = append(ret, m)
227	}
228
229	return ret, nil
230}
231
232// filterDotFiles filters out files that start with '.'
233func filterDotFiles(matches []string) []string {
234	ret := make([]string, 0, len(matches))
235
236	for _, match := range matches {
237		_, name := filepath.Split(match)
238		if name[0] == '.' {
239			continue
240		}
241		ret = append(ret, match)
242	}
243
244	return ret
245}
246
247// Match returns true if name matches pattern using the same rules as filepath.Match, but supporting
248// recursive globs (**).
249func Match(pattern, name string) (bool, error) {
250	if filepath.Base(pattern) == "**" {
251		return false, GlobLastRecursiveErr
252	}
253
254	patternDir := pattern[len(pattern)-1] == '/'
255	nameDir := name[len(name)-1] == '/'
256
257	if patternDir != nameDir {
258		return false, nil
259	}
260
261	if nameDir {
262		name = name[:len(name)-1]
263		pattern = pattern[:len(pattern)-1]
264	}
265
266	for {
267		var patternFile, nameFile string
268		pattern, patternFile = filepath.Dir(pattern), filepath.Base(pattern)
269
270		if patternFile == "**" {
271			if strings.Contains(pattern, "**") {
272				return false, GlobMultipleRecursiveErr
273			}
274			// Test if the any prefix of name matches the part of the pattern before **
275			for {
276				if name == "." || name == "/" {
277					return name == pattern, nil
278				}
279				if match, err := filepath.Match(pattern, name); err != nil {
280					return false, err
281				} else if match {
282					return true, nil
283				}
284				name = filepath.Dir(name)
285			}
286		} else if strings.Contains(patternFile, "**") {
287			return false, GlobInvalidRecursiveErr
288		}
289
290		name, nameFile = filepath.Dir(name), filepath.Base(name)
291
292		if nameFile == "." && patternFile == "." {
293			return true, nil
294		} else if nameFile == "/" && patternFile == "/" {
295			return true, nil
296		} else if nameFile == "." || patternFile == "." || nameFile == "/" || patternFile == "/" {
297			return false, nil
298		}
299
300		match, err := filepath.Match(patternFile, nameFile)
301		if err != nil || !match {
302			return match, err
303		}
304	}
305}
306
307func GlobPatternList(patterns []string, prefix string) (globedList []string, depDirs []string, err error) {
308	var (
309		matches []string
310		deps    []string
311	)
312
313	globedList = make([]string, 0)
314	depDirs = make([]string, 0)
315
316	for _, pattern := range patterns {
317		if isWild(pattern) {
318			matches, deps, err = Glob(filepath.Join(prefix, pattern), nil, FollowSymlinks)
319			if err != nil {
320				return nil, nil, err
321			}
322			globedList = append(globedList, matches...)
323			depDirs = append(depDirs, deps...)
324		} else {
325			globedList = append(globedList, filepath.Join(prefix, pattern))
326		}
327	}
328	return globedList, depDirs, nil
329}
330
331// IsGlob returns true if the pattern contains any glob characters (*, ?, or [).
332func IsGlob(pattern string) bool {
333	return strings.IndexAny(pattern, "*?[") >= 0
334}
335
336// HasGlob returns true if any string in the list contains any glob characters (*, ?, or [).
337func HasGlob(in []string) bool {
338	for _, s := range in {
339		if IsGlob(s) {
340			return true
341		}
342	}
343
344	return false
345}
346
347// GlobWithDepFile finds all files and directories that match glob.  Directories
348// will have a trailing '/'.  It compares the list of matches against the
349// contents of fileListFile, and rewrites fileListFile if it has changed.  It
350// also writes all of the the directories it traversed as dependencies on
351// fileListFile to depFile.
352//
353// The format of glob is either path/*.ext for a single directory glob, or
354// path/**/*.ext for a recursive glob.
355//
356// Returns a list of file paths, and an error.
357//
358// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps
359// should be used instead, as they will automatically set up dependencies
360// to rerun the primary builder when the list of matching files changes.
361func GlobWithDepFile(glob, fileListFile, depFile string, excludes []string) (files []string, err error) {
362	files, deps, err := Glob(glob, excludes, FollowSymlinks)
363	if err != nil {
364		return nil, err
365	}
366
367	fileList := strings.Join(files, "\n") + "\n"
368
369	WriteFileIfChanged(fileListFile, []byte(fileList), 0666)
370	deptools.WriteDepFile(depFile, fileListFile, deps)
371
372	return
373}
374
375// WriteFileIfChanged wraps ioutil.WriteFile, but only writes the file if
376// the files does not already exist with identical contents.  This can be used
377// along with ninja restat rules to skip rebuilding downstream rules if no
378// changes were made by a rule.
379func WriteFileIfChanged(filename string, data []byte, perm os.FileMode) error {
380	var isChanged bool
381
382	dir := filepath.Dir(filename)
383	err := os.MkdirAll(dir, 0777)
384	if err != nil {
385		return err
386	}
387
388	info, err := os.Stat(filename)
389	if err != nil {
390		if os.IsNotExist(err) {
391			// The file does not exist yet.
392			isChanged = true
393		} else {
394			return err
395		}
396	} else {
397		if info.Size() != int64(len(data)) {
398			isChanged = true
399		} else {
400			oldData, err := ioutil.ReadFile(filename)
401			if err != nil {
402				return err
403			}
404
405			if len(oldData) != len(data) {
406				isChanged = true
407			} else {
408				for i := range data {
409					if oldData[i] != data[i] {
410						isChanged = true
411						break
412					}
413				}
414			}
415		}
416	}
417
418	if isChanged {
419		err = ioutil.WriteFile(filename, data, perm)
420		if err != nil {
421			return err
422		}
423	}
424
425	return nil
426}
427
428var matchEscaper = strings.NewReplacer(
429	`*`, `\*`,
430	`?`, `\?`,
431	`[`, `\[`,
432	`]`, `\]`,
433)
434
435// MatchEscape returns its inputs with characters that would be interpreted by
436func MatchEscape(s string) string {
437	return matchEscaper.Replace(s)
438}
439