1// Copyright 2014 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package pathtools 16 17import ( 18 "errors" 19 "fmt" 20 "io/ioutil" 21 "os" 22 "path/filepath" 23 "strings" 24 25 "github.com/google/blueprint/deptools" 26) 27 28var GlobMultipleRecursiveErr = errors.New("pattern contains multiple '**'") 29var GlobLastRecursiveErr = errors.New("pattern has '**' as last path element") 30var GlobInvalidRecursiveErr = errors.New("pattern contains other characters between '**' and path separator") 31 32// Glob returns the list of files and directories that match the given pattern 33// but do not match the given exclude patterns, along with the list of 34// directories and other dependencies that were searched to construct the file 35// list. The supported glob and exclude patterns are equivalent to 36// filepath.Glob, with an extension that recursive glob (** matching zero or 37// more complete path entries) is supported. Any directories in the matches 38// list will have a '/' suffix. 39// 40// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps 41// should be used instead, as they will automatically set up dependencies 42// to rerun the primary builder when the list of matching files changes. 43func Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (matches, deps []string, err error) { 44 return startGlob(OsFs, pattern, excludes, follow) 45} 46 47func startGlob(fs FileSystem, pattern string, excludes []string, 48 follow ShouldFollowSymlinks) (matches, deps []string, err error) { 49 50 if filepath.Base(pattern) == "**" { 51 return nil, nil, GlobLastRecursiveErr 52 } else { 53 matches, deps, err = glob(fs, pattern, false, follow) 54 } 55 56 if err != nil { 57 return nil, nil, err 58 } 59 60 matches, err = filterExcludes(matches, excludes) 61 if err != nil { 62 return nil, nil, err 63 } 64 65 // If the pattern has wildcards, we added dependencies on the 66 // containing directories to know about changes. 67 // 68 // If the pattern didn't have wildcards, and didn't find matches, the 69 // most specific found directories were added. 70 // 71 // But if it didn't have wildcards, and did find a match, no 72 // dependencies were added, so add the match itself to detect when it 73 // is removed. 74 if !isWild(pattern) { 75 deps = append(deps, matches...) 76 } 77 78 for i, match := range matches { 79 isSymlink, err := fs.IsSymlink(match) 80 if err != nil { 81 return nil, nil, err 82 } 83 if !(isSymlink && follow == DontFollowSymlinks) { 84 isDir, err := fs.IsDir(match) 85 if os.IsNotExist(err) { 86 if isSymlink { 87 return nil, nil, fmt.Errorf("%s: dangling symlink", match) 88 } 89 } 90 if err != nil { 91 return nil, nil, fmt.Errorf("%s: %s", match, err.Error()) 92 } 93 94 if isDir { 95 matches[i] = match + "/" 96 } 97 } 98 } 99 100 return matches, deps, nil 101} 102 103// glob is a recursive helper function to handle globbing each level of the pattern individually, 104// allowing searched directories to be tracked. Also handles the recursive glob pattern, **. 105func glob(fs FileSystem, pattern string, hasRecursive bool, 106 follow ShouldFollowSymlinks) (matches, dirs []string, err error) { 107 108 if !isWild(pattern) { 109 // If there are no wilds in the pattern, check whether the file exists or not. 110 // Uses filepath.Glob instead of manually statting to get consistent results. 111 pattern = filepath.Clean(pattern) 112 matches, err = fs.glob(pattern) 113 if err != nil { 114 return matches, dirs, err 115 } 116 117 if len(matches) == 0 { 118 // Some part of the non-wild pattern didn't exist. Add the last existing directory 119 // as a dependency. 120 var matchDirs []string 121 for len(matchDirs) == 0 { 122 pattern = filepath.Dir(pattern) 123 matchDirs, err = fs.glob(pattern) 124 if err != nil { 125 return matches, dirs, err 126 } 127 } 128 dirs = append(dirs, matchDirs...) 129 } 130 return matches, dirs, err 131 } 132 133 dir, file := saneSplit(pattern) 134 135 if file == "**" { 136 if hasRecursive { 137 return matches, dirs, GlobMultipleRecursiveErr 138 } 139 hasRecursive = true 140 } else if strings.Contains(file, "**") { 141 return matches, dirs, GlobInvalidRecursiveErr 142 } 143 144 dirMatches, dirs, err := glob(fs, dir, hasRecursive, follow) 145 if err != nil { 146 return nil, nil, err 147 } 148 149 for _, m := range dirMatches { 150 isDir, err := fs.IsDir(m) 151 if os.IsNotExist(err) { 152 if isSymlink, _ := fs.IsSymlink(m); isSymlink { 153 return nil, nil, fmt.Errorf("dangling symlink: %s", m) 154 } 155 } 156 if err != nil { 157 return nil, nil, fmt.Errorf("unexpected error after glob: %s", err) 158 } 159 160 if isDir { 161 if file == "**" { 162 recurseDirs, err := fs.ListDirsRecursive(m, follow) 163 if err != nil { 164 return nil, nil, err 165 } 166 matches = append(matches, recurseDirs...) 167 } else { 168 dirs = append(dirs, m) 169 newMatches, err := fs.glob(filepath.Join(MatchEscape(m), file)) 170 if err != nil { 171 return nil, nil, err 172 } 173 if file[0] != '.' { 174 newMatches = filterDotFiles(newMatches) 175 } 176 matches = append(matches, newMatches...) 177 } 178 } 179 } 180 181 return matches, dirs, nil 182} 183 184// Faster version of dir, file := filepath.Dir(path), filepath.File(path) with no allocations 185// Similar to filepath.Split, but returns "." if dir is empty and trims trailing slash if dir is 186// not "/". Returns ".", "" if path is "." 187func saneSplit(path string) (dir, file string) { 188 if path == "." { 189 return ".", "" 190 } 191 dir, file = filepath.Split(path) 192 switch dir { 193 case "": 194 dir = "." 195 case "/": 196 // Nothing 197 default: 198 dir = dir[:len(dir)-1] 199 } 200 return dir, file 201} 202 203func isWild(pattern string) bool { 204 return strings.ContainsAny(pattern, "*?[") 205} 206 207// Filters the strings in matches based on the glob patterns in excludes. Hierarchical (a/*) and 208// recursive (**) glob patterns are supported. 209func filterExcludes(matches []string, excludes []string) ([]string, error) { 210 if len(excludes) == 0 { 211 return matches, nil 212 } 213 214 var ret []string 215matchLoop: 216 for _, m := range matches { 217 for _, e := range excludes { 218 exclude, err := Match(e, m) 219 if err != nil { 220 return nil, err 221 } 222 if exclude { 223 continue matchLoop 224 } 225 } 226 ret = append(ret, m) 227 } 228 229 return ret, nil 230} 231 232// filterDotFiles filters out files that start with '.' 233func filterDotFiles(matches []string) []string { 234 ret := make([]string, 0, len(matches)) 235 236 for _, match := range matches { 237 _, name := filepath.Split(match) 238 if name[0] == '.' { 239 continue 240 } 241 ret = append(ret, match) 242 } 243 244 return ret 245} 246 247// Match returns true if name matches pattern using the same rules as filepath.Match, but supporting 248// recursive globs (**). 249func Match(pattern, name string) (bool, error) { 250 if filepath.Base(pattern) == "**" { 251 return false, GlobLastRecursiveErr 252 } 253 254 patternDir := pattern[len(pattern)-1] == '/' 255 nameDir := name[len(name)-1] == '/' 256 257 if patternDir != nameDir { 258 return false, nil 259 } 260 261 if nameDir { 262 name = name[:len(name)-1] 263 pattern = pattern[:len(pattern)-1] 264 } 265 266 for { 267 var patternFile, nameFile string 268 pattern, patternFile = filepath.Dir(pattern), filepath.Base(pattern) 269 270 if patternFile == "**" { 271 if strings.Contains(pattern, "**") { 272 return false, GlobMultipleRecursiveErr 273 } 274 // Test if the any prefix of name matches the part of the pattern before ** 275 for { 276 if name == "." || name == "/" { 277 return name == pattern, nil 278 } 279 if match, err := filepath.Match(pattern, name); err != nil { 280 return false, err 281 } else if match { 282 return true, nil 283 } 284 name = filepath.Dir(name) 285 } 286 } else if strings.Contains(patternFile, "**") { 287 return false, GlobInvalidRecursiveErr 288 } 289 290 name, nameFile = filepath.Dir(name), filepath.Base(name) 291 292 if nameFile == "." && patternFile == "." { 293 return true, nil 294 } else if nameFile == "/" && patternFile == "/" { 295 return true, nil 296 } else if nameFile == "." || patternFile == "." || nameFile == "/" || patternFile == "/" { 297 return false, nil 298 } 299 300 match, err := filepath.Match(patternFile, nameFile) 301 if err != nil || !match { 302 return match, err 303 } 304 } 305} 306 307func GlobPatternList(patterns []string, prefix string) (globedList []string, depDirs []string, err error) { 308 var ( 309 matches []string 310 deps []string 311 ) 312 313 globedList = make([]string, 0) 314 depDirs = make([]string, 0) 315 316 for _, pattern := range patterns { 317 if isWild(pattern) { 318 matches, deps, err = Glob(filepath.Join(prefix, pattern), nil, FollowSymlinks) 319 if err != nil { 320 return nil, nil, err 321 } 322 globedList = append(globedList, matches...) 323 depDirs = append(depDirs, deps...) 324 } else { 325 globedList = append(globedList, filepath.Join(prefix, pattern)) 326 } 327 } 328 return globedList, depDirs, nil 329} 330 331// IsGlob returns true if the pattern contains any glob characters (*, ?, or [). 332func IsGlob(pattern string) bool { 333 return strings.IndexAny(pattern, "*?[") >= 0 334} 335 336// HasGlob returns true if any string in the list contains any glob characters (*, ?, or [). 337func HasGlob(in []string) bool { 338 for _, s := range in { 339 if IsGlob(s) { 340 return true 341 } 342 } 343 344 return false 345} 346 347// GlobWithDepFile finds all files and directories that match glob. Directories 348// will have a trailing '/'. It compares the list of matches against the 349// contents of fileListFile, and rewrites fileListFile if it has changed. It 350// also writes all of the the directories it traversed as dependencies on 351// fileListFile to depFile. 352// 353// The format of glob is either path/*.ext for a single directory glob, or 354// path/**/*.ext for a recursive glob. 355// 356// Returns a list of file paths, and an error. 357// 358// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps 359// should be used instead, as they will automatically set up dependencies 360// to rerun the primary builder when the list of matching files changes. 361func GlobWithDepFile(glob, fileListFile, depFile string, excludes []string) (files []string, err error) { 362 files, deps, err := Glob(glob, excludes, FollowSymlinks) 363 if err != nil { 364 return nil, err 365 } 366 367 fileList := strings.Join(files, "\n") + "\n" 368 369 WriteFileIfChanged(fileListFile, []byte(fileList), 0666) 370 deptools.WriteDepFile(depFile, fileListFile, deps) 371 372 return 373} 374 375// WriteFileIfChanged wraps ioutil.WriteFile, but only writes the file if 376// the files does not already exist with identical contents. This can be used 377// along with ninja restat rules to skip rebuilding downstream rules if no 378// changes were made by a rule. 379func WriteFileIfChanged(filename string, data []byte, perm os.FileMode) error { 380 var isChanged bool 381 382 dir := filepath.Dir(filename) 383 err := os.MkdirAll(dir, 0777) 384 if err != nil { 385 return err 386 } 387 388 info, err := os.Stat(filename) 389 if err != nil { 390 if os.IsNotExist(err) { 391 // The file does not exist yet. 392 isChanged = true 393 } else { 394 return err 395 } 396 } else { 397 if info.Size() != int64(len(data)) { 398 isChanged = true 399 } else { 400 oldData, err := ioutil.ReadFile(filename) 401 if err != nil { 402 return err 403 } 404 405 if len(oldData) != len(data) { 406 isChanged = true 407 } else { 408 for i := range data { 409 if oldData[i] != data[i] { 410 isChanged = true 411 break 412 } 413 } 414 } 415 } 416 } 417 418 if isChanged { 419 err = ioutil.WriteFile(filename, data, perm) 420 if err != nil { 421 return err 422 } 423 } 424 425 return nil 426} 427 428var matchEscaper = strings.NewReplacer( 429 `*`, `\*`, 430 `?`, `\?`, 431 `[`, `\[`, 432 `]`, `\]`, 433) 434 435// MatchEscape returns its inputs with characters that would be interpreted by 436func MatchEscape(s string) string { 437 return matchEscaper.Replace(s) 438} 439