1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.providers.media.scan; 18 19 import static android.media.MediaMetadataRetriever.METADATA_KEY_ALBUM; 20 import static android.media.MediaMetadataRetriever.METADATA_KEY_ALBUMARTIST; 21 import static android.media.MediaMetadataRetriever.METADATA_KEY_ARTIST; 22 import static android.media.MediaMetadataRetriever.METADATA_KEY_CD_TRACK_NUMBER; 23 import static android.media.MediaMetadataRetriever.METADATA_KEY_COLOR_RANGE; 24 import static android.media.MediaMetadataRetriever.METADATA_KEY_COLOR_STANDARD; 25 import static android.media.MediaMetadataRetriever.METADATA_KEY_COLOR_TRANSFER; 26 import static android.media.MediaMetadataRetriever.METADATA_KEY_COMPILATION; 27 import static android.media.MediaMetadataRetriever.METADATA_KEY_COMPOSER; 28 import static android.media.MediaMetadataRetriever.METADATA_KEY_DATE; 29 import static android.media.MediaMetadataRetriever.METADATA_KEY_DURATION; 30 import static android.media.MediaMetadataRetriever.METADATA_KEY_GENRE; 31 import static android.media.MediaMetadataRetriever.METADATA_KEY_IS_DRM; 32 import static android.media.MediaMetadataRetriever.METADATA_KEY_TITLE; 33 import static android.media.MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT; 34 import static android.media.MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH; 35 import static android.media.MediaMetadataRetriever.METADATA_KEY_YEAR; 36 import static android.os.Trace.TRACE_TAG_DATABASE; 37 import static android.provider.MediaStore.AUTHORITY; 38 import static android.provider.MediaStore.UNKNOWN_STRING; 39 import static android.text.format.DateUtils.HOUR_IN_MILLIS; 40 import static android.text.format.DateUtils.MINUTE_IN_MILLIS; 41 42 import android.annotation.CurrentTimeMillisLong; 43 import android.annotation.CurrentTimeSecondsLong; 44 import android.annotation.NonNull; 45 import android.annotation.Nullable; 46 import android.content.ContentProviderClient; 47 import android.content.ContentProviderOperation; 48 import android.content.ContentProviderResult; 49 import android.content.ContentResolver; 50 import android.content.ContentUris; 51 import android.content.Context; 52 import android.content.OperationApplicationException; 53 import android.database.Cursor; 54 import android.database.sqlite.SQLiteDatabase; 55 import android.media.ExifInterface; 56 import android.media.MediaFile; 57 import android.media.MediaMetadataRetriever; 58 import android.mtp.MtpConstants; 59 import android.net.Uri; 60 import android.os.Build; 61 import android.os.CancellationSignal; 62 import android.os.Environment; 63 import android.os.FileUtils; 64 import android.os.OperationCanceledException; 65 import android.os.RemoteException; 66 import android.os.Trace; 67 import android.provider.MediaStore; 68 import android.provider.MediaStore.Audio.AudioColumns; 69 import android.provider.MediaStore.Audio.PlaylistsColumns; 70 import android.provider.MediaStore.Files.FileColumns; 71 import android.provider.MediaStore.Images.ImageColumns; 72 import android.provider.MediaStore.MediaColumns; 73 import android.provider.MediaStore.Video.VideoColumns; 74 import android.text.TextUtils; 75 import android.util.ArrayMap; 76 import android.util.Log; 77 import android.util.LongArray; 78 79 import com.android.internal.annotations.GuardedBy; 80 import com.android.internal.annotations.VisibleForTesting; 81 import com.android.providers.media.util.IsoInterface; 82 import com.android.providers.media.util.XmpInterface; 83 84 import java.io.File; 85 import java.io.FileInputStream; 86 import java.io.IOException; 87 import java.nio.file.FileVisitResult; 88 import java.nio.file.FileVisitor; 89 import java.nio.file.Files; 90 import java.nio.file.Path; 91 import java.nio.file.attribute.BasicFileAttributes; 92 import java.text.ParseException; 93 import java.text.SimpleDateFormat; 94 import java.util.ArrayList; 95 import java.util.Arrays; 96 import java.util.List; 97 import java.util.Locale; 98 import java.util.Optional; 99 import java.util.TimeZone; 100 import java.util.regex.Pattern; 101 102 /** 103 * Modern implementation of media scanner. 104 * <p> 105 * This is a bug-compatible reimplementation of the legacy media scanner, but 106 * written purely in managed code for better testability and long-term 107 * maintainability. 108 * <p> 109 * Initial tests shows it performing roughly on-par with the legacy scanner. 110 * <p> 111 * In general, we start by populating metadata based on file attributes, and 112 * then overwrite with any valid metadata found using 113 * {@link MediaMetadataRetriever}, {@link ExifInterface}, and 114 * {@link XmpInterface}, each with increasing levels of trust. 115 */ 116 public class ModernMediaScanner implements MediaScanner { 117 private static final String TAG = "ModernMediaScanner"; 118 private static final boolean LOGW = Log.isLoggable(TAG, Log.WARN); 119 private static final boolean LOGD = Log.isLoggable(TAG, Log.DEBUG); 120 private static final boolean LOGV = Log.isLoggable(TAG, Log.VERBOSE); 121 122 // TODO: add DRM support 123 124 // TODO: refactor to use UPSERT once we have SQLite 3.24.0 125 126 // TODO: deprecate playlist editing 127 // TODO: deprecate PARENT column, since callers can't see directories 128 129 private static final SimpleDateFormat sDateFormat; 130 131 static { 132 sDateFormat = new SimpleDateFormat("yyyyMMdd'T'HHmmss"); 133 sDateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); 134 } 135 136 private static final int BATCH_SIZE = 32; 137 138 private static final Pattern PATTERN_VISIBLE = Pattern.compile( 139 "(?i)^/storage/[^/]+(?:/[0-9]+)?(?:/Android/sandbox/([^/]+))?$"); 140 private static final Pattern PATTERN_INVISIBLE = Pattern.compile( 141 "(?i)^/storage/[^/]+(?:/[0-9]+)?(?:/Android/sandbox/([^/]+))?/Android/(?:data|obb)$"); 142 143 private final Context mContext; 144 145 /** 146 * Map from volume name to signals that can be used to cancel any active 147 * scan operations on those volumes. 148 */ 149 @GuardedBy("mSignals") 150 private final ArrayMap<String, CancellationSignal> mSignals = new ArrayMap<>(); 151 ModernMediaScanner(Context context)152 public ModernMediaScanner(Context context) { 153 mContext = context; 154 } 155 156 @Override getContext()157 public Context getContext() { 158 return mContext; 159 } 160 161 @Override scanDirectory(File file)162 public void scanDirectory(File file) { 163 try (Scan scan = new Scan(file)) { 164 scan.run(); 165 } catch (OperationCanceledException ignored) { 166 } 167 } 168 169 @Override scanFile(File file)170 public Uri scanFile(File file) { 171 try (Scan scan = new Scan(file)) { 172 scan.run(); 173 return scan.mFirstResult; 174 } catch (OperationCanceledException ignored) { 175 return null; 176 } 177 } 178 179 @Override onDetachVolume(String volumeName)180 public void onDetachVolume(String volumeName) { 181 synchronized (mSignals) { 182 final CancellationSignal signal = mSignals.remove(volumeName); 183 if (signal != null) { 184 signal.cancel(); 185 } 186 } 187 } 188 getOrCreateSignal(String volumeName)189 private CancellationSignal getOrCreateSignal(String volumeName) { 190 synchronized (mSignals) { 191 CancellationSignal signal = mSignals.get(volumeName); 192 if (signal == null) { 193 signal = new CancellationSignal(); 194 mSignals.put(volumeName, signal); 195 } 196 return signal; 197 } 198 } 199 200 /** 201 * Individual scan request for a specific file or directory. When run it 202 * will traverse all included media files under the requested location, 203 * reconciling them against {@link MediaStore}. 204 */ 205 private class Scan implements Runnable, FileVisitor<Path>, AutoCloseable { 206 private final ContentProviderClient mClient; 207 private final ContentResolver mResolver; 208 209 private final File mRoot; 210 private final String mVolumeName; 211 private final Uri mFilesUri; 212 private final CancellationSignal mSignal; 213 214 private final boolean mSingleFile; 215 private final ArrayList<ContentProviderOperation> mPending = new ArrayList<>(); 216 private LongArray mScannedIds = new LongArray(); 217 private LongArray mUnknownIds = new LongArray(); 218 private LongArray mPlaylistIds = new LongArray(); 219 220 private Uri mFirstResult; 221 Scan(File root)222 public Scan(File root) { 223 Trace.traceBegin(TRACE_TAG_DATABASE, "ctor"); 224 225 mClient = mContext.getContentResolver() 226 .acquireContentProviderClient(MediaStore.AUTHORITY); 227 mResolver = ContentResolver.wrap(mClient.getLocalContentProvider()); 228 229 mRoot = root; 230 mVolumeName = MediaStore.getVolumeName(root); 231 mFilesUri = MediaStore.setIncludePending(MediaStore.Files.getContentUri(mVolumeName)); 232 mSignal = getOrCreateSignal(mVolumeName); 233 234 mSingleFile = mRoot.isFile(); 235 236 Trace.traceEnd(TRACE_TAG_DATABASE); 237 } 238 239 @Override run()240 public void run() { 241 // First, scan everything that should be visible under requested 242 // location, tracking scanned IDs along the way 243 walkFileTree(); 244 245 // Second, reconcile all items known in the database against all the 246 // items we scanned above 247 if (mSingleFile && mScannedIds.size() == 1) { 248 // We can safely skip this step if the scan targeted a single 249 // file which we scanned above 250 } else { 251 reconcileAndClean(); 252 } 253 254 // Third, resolve any playlists that we scanned 255 if (mPlaylistIds.size() > 0) { 256 resolvePlaylists(); 257 } 258 } 259 walkFileTree()260 private void walkFileTree() { 261 mSignal.throwIfCanceled(); 262 if (!isDirectoryHiddenRecursive(mSingleFile ? mRoot.getParentFile() : mRoot)) { 263 Trace.traceBegin(Trace.TRACE_TAG_DATABASE, "walkFileTree"); 264 try { 265 Files.walkFileTree(mRoot.toPath(), this); 266 } catch (IOException e) { 267 // This should never happen, so yell loudly 268 throw new IllegalStateException(e); 269 } finally { 270 Trace.traceEnd(Trace.TRACE_TAG_DATABASE); 271 } 272 applyPending(); 273 } 274 } 275 reconcileAndClean()276 private void reconcileAndClean() { 277 final long[] scannedIds = mScannedIds.toArray(); 278 Arrays.sort(scannedIds); 279 280 // The query phase is split from the delete phase so that our query 281 // remains stable if we need to paginate across multiple windows. 282 mSignal.throwIfCanceled(); 283 Trace.traceBegin(Trace.TRACE_TAG_DATABASE, "reconcile"); 284 try (Cursor c = mResolver.query(mFilesUri, 285 new String[]{FileColumns._ID}, 286 FileColumns.FORMAT + "!=? AND " + FileColumns.DATA + " LIKE ? ESCAPE '\\'", 287 new String[]{ 288 // Ignore abstract playlists which don't have files on disk 289 String.valueOf(MtpConstants.FORMAT_ABSTRACT_AV_PLAYLIST), 290 escapeForLike(mRoot.getAbsolutePath()) + '%' 291 }, 292 FileColumns._ID + " DESC", mSignal)) { 293 while (c.moveToNext()) { 294 final long id = c.getLong(0); 295 if (Arrays.binarySearch(scannedIds, id) < 0) { 296 mUnknownIds.add(id); 297 } 298 } 299 } finally { 300 Trace.traceEnd(Trace.TRACE_TAG_DATABASE); 301 } 302 303 // Third, clean all the unknown database entries found above 304 mSignal.throwIfCanceled(); 305 Trace.traceBegin(Trace.TRACE_TAG_DATABASE, "clean"); 306 try { 307 for (int i = 0; i < mUnknownIds.size(); i++) { 308 final long id = mUnknownIds.get(i); 309 if (LOGV) Log.v(TAG, "Cleaning " + id); 310 final Uri uri = MediaStore.Files.getContentUri(mVolumeName, id).buildUpon() 311 .appendQueryParameter(MediaStore.PARAM_DELETE_DATA, "false") 312 .build(); 313 mPending.add(ContentProviderOperation.newDelete(uri).build()); 314 maybeApplyPending(); 315 } 316 applyPending(); 317 } finally { 318 Trace.traceEnd(Trace.TRACE_TAG_DATABASE); 319 } 320 } 321 resolvePlaylists()322 private void resolvePlaylists() { 323 mSignal.throwIfCanceled(); 324 for (int i = 0; i < mPlaylistIds.size(); i++) { 325 final Uri uri = MediaStore.Files.getContentUri(mVolumeName, mPlaylistIds.get(i)); 326 try { 327 mPending.addAll( 328 PlaylistResolver.resolvePlaylist(mResolver, uri)); 329 maybeApplyPending(); 330 } catch (IOException e) { 331 if (LOGW) Log.w(TAG, "Ignoring troubled playlist: " + uri, e); 332 } 333 applyPending(); 334 } 335 } 336 337 @Override close()338 public void close() { 339 // Sanity check that we drained any pending operations 340 if (!mPending.isEmpty()) { 341 throw new IllegalStateException(); 342 } 343 344 mClient.close(); 345 } 346 347 @Override preVisitDirectory(Path dir, BasicFileAttributes attrs)348 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) 349 throws IOException { 350 // Possibly bail before digging into each directory 351 mSignal.throwIfCanceled(); 352 353 if (isDirectoryHidden(dir.toFile())) { 354 return FileVisitResult.SKIP_SUBTREE; 355 } 356 357 // Scan this directory as a normal file so that "parent" database 358 // entries are created 359 return visitFile(dir, attrs); 360 } 361 362 @Override visitFile(Path file, BasicFileAttributes attrs)363 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) 364 throws IOException { 365 if (LOGV) Log.v(TAG, "Visiting " + file); 366 367 // Skip files that have already been scanned, and which haven't 368 // changed since they were last scanned 369 final File realFile = file.toFile(); 370 long existingId = -1; 371 Trace.traceBegin(Trace.TRACE_TAG_DATABASE, "checkChanged"); 372 try (Cursor c = mResolver.query(mFilesUri, 373 new String[] { FileColumns._ID, FileColumns.DATE_MODIFIED, FileColumns.SIZE }, 374 FileColumns.DATA + "=?", new String[] { realFile.getAbsolutePath() }, null)) { 375 if (c.moveToFirst()) { 376 existingId = c.getLong(0); 377 final long dateModified = c.getLong(1); 378 final long size = c.getLong(2); 379 380 // Remember visiting this existing item, even if we skipped 381 // due to it being unchanged; this is needed so we don't 382 // delete the item during a later cleaning phase 383 mScannedIds.add(existingId); 384 385 // We also technically found our first result 386 if (mFirstResult == null) { 387 mFirstResult = MediaStore.Files.getContentUri(mVolumeName, existingId); 388 } 389 390 final boolean sameTime = (lastModifiedTime(realFile, attrs) == dateModified); 391 final boolean sameSize = (attrs.size() == size); 392 if (attrs.isDirectory() || (sameTime && sameSize)) { 393 if (LOGV) Log.v(TAG, "Skipping unchanged " + file); 394 return FileVisitResult.CONTINUE; 395 } 396 } 397 } finally { 398 Trace.traceEnd(Trace.TRACE_TAG_DATABASE); 399 } 400 401 final ContentProviderOperation op; 402 Trace.traceBegin(Trace.TRACE_TAG_DATABASE, "scanItem"); 403 try { 404 op = scanItem(existingId, file.toFile(), attrs, mVolumeName); 405 } finally { 406 Trace.traceEnd(Trace.TRACE_TAG_DATABASE); 407 } 408 if (op != null) { 409 mPending.add(op); 410 maybeApplyPending(); 411 } 412 return FileVisitResult.CONTINUE; 413 } 414 415 @Override visitFileFailed(Path file, IOException exc)416 public FileVisitResult visitFileFailed(Path file, IOException exc) 417 throws IOException { 418 Log.w(TAG, "Failed to visit " + file + ": " + exc); 419 return FileVisitResult.CONTINUE; 420 } 421 422 @Override postVisitDirectory(Path dir, IOException exc)423 public FileVisitResult postVisitDirectory(Path dir, IOException exc) 424 throws IOException { 425 return FileVisitResult.CONTINUE; 426 } 427 maybeApplyPending()428 private void maybeApplyPending() { 429 if (mPending.size() > BATCH_SIZE) { 430 applyPending(); 431 } 432 } 433 applyPending()434 private void applyPending() { 435 Trace.traceBegin(Trace.TRACE_TAG_DATABASE, "applyPending"); 436 try { 437 ContentProviderResult[] results = mResolver.applyBatch(AUTHORITY, mPending); 438 for (int index = 0; index < results.length; index++) { 439 ContentProviderResult result = results[index]; 440 ContentProviderOperation operation = mPending.get(index); 441 442 Uri uri = result.uri; 443 if (uri != null) { 444 if (mFirstResult == null) { 445 mFirstResult = uri; 446 } 447 final long id = ContentUris.parseId(uri); 448 mScannedIds.add(id); 449 } 450 451 // Some operations don't return a URI, so check the original if necessary 452 Uri uriToCheck = uri == null ? operation.getUri() : uri; 453 if (uriToCheck != null) { 454 if (isPlaylist(uriToCheck)) { 455 // If this was a playlist, remember it so we can resolve 456 // its contents once all other media has been scanned 457 mPlaylistIds.add(ContentUris.parseId(uriToCheck)); 458 } 459 } 460 } 461 } catch (RemoteException | OperationApplicationException e) { 462 Log.w(TAG, "Failed to apply: " + e); 463 } finally { 464 mPending.clear(); 465 Trace.traceEnd(Trace.TRACE_TAG_DATABASE); 466 } 467 } 468 } 469 470 /** 471 * Scan the requested file, returning a {@link ContentProviderOperation} 472 * containing all indexed metadata, suitable for passing to a 473 * {@link SQLiteDatabase#replace} operation. 474 */ scanItem(long existingId, File file, BasicFileAttributes attrs, String volumeName)475 private static @Nullable ContentProviderOperation scanItem(long existingId, File file, 476 BasicFileAttributes attrs, String volumeName) { 477 final String name = file.getName(); 478 if (name.startsWith(".")) { 479 if (LOGD) Log.d(TAG, "Ignoring hidden file: " + file); 480 return null; 481 } 482 483 try { 484 final String mimeType; 485 if (attrs.isDirectory()) { 486 mimeType = null; 487 } else { 488 mimeType = MediaFile.getMimeTypeForFile(file.getPath()); 489 } 490 491 if (attrs.isDirectory()) { 492 return scanItemDirectory(existingId, file, attrs, mimeType, volumeName); 493 } else if (MediaFile.isPlayListMimeType(mimeType)) { 494 return scanItemPlaylist(existingId, file, attrs, mimeType, volumeName); 495 } else if (MediaFile.isAudioMimeType(mimeType)) { 496 return scanItemAudio(existingId, file, attrs, mimeType, volumeName); 497 } else if (MediaFile.isVideoMimeType(mimeType)) { 498 return scanItemVideo(existingId, file, attrs, mimeType, volumeName); 499 } else if (MediaFile.isImageMimeType(mimeType)) { 500 return scanItemImage(existingId, file, attrs, mimeType, volumeName); 501 } else { 502 return scanItemFile(existingId, file, attrs, mimeType, volumeName); 503 } 504 } catch (IOException e) { 505 if (LOGW) Log.w(TAG, "Ignoring troubled file: " + file, e); 506 return null; 507 } 508 } 509 510 /** 511 * Populate the given {@link ContentProviderOperation} with the generic 512 * {@link MediaColumns} values that can be determined directly from the file 513 * or its attributes. 514 */ withGenericValues(ContentProviderOperation.Builder op, File file, BasicFileAttributes attrs, String mimeType)515 private static void withGenericValues(ContentProviderOperation.Builder op, 516 File file, BasicFileAttributes attrs, String mimeType) { 517 op.withValue(MediaColumns.DATA, file.getAbsolutePath()); 518 op.withValue(MediaColumns.SIZE, attrs.size()); 519 op.withValue(MediaColumns.TITLE, extractName(file)); 520 op.withValue(MediaColumns.DATE_MODIFIED, lastModifiedTime(file, attrs)); 521 op.withValue(MediaColumns.DATE_TAKEN, null); 522 op.withValue(MediaColumns.MIME_TYPE, mimeType); 523 op.withValue(MediaColumns.IS_DRM, 0); 524 op.withValue(MediaColumns.WIDTH, null); 525 op.withValue(MediaColumns.HEIGHT, null); 526 op.withValue(MediaColumns.DOCUMENT_ID, null); 527 op.withValue(MediaColumns.INSTANCE_ID, null); 528 op.withValue(MediaColumns.ORIGINAL_DOCUMENT_ID, null); 529 op.withValue(MediaColumns.DURATION, null); 530 op.withValue(MediaColumns.ORIENTATION, null); 531 } 532 533 /** 534 * Populate the given {@link ContentProviderOperation} with the generic 535 * {@link MediaColumns} values using the given XMP metadata. 536 */ withXmpValues(ContentProviderOperation.Builder op, XmpInterface xmp, String mimeType)537 private static void withXmpValues(ContentProviderOperation.Builder op, 538 XmpInterface xmp, String mimeType) { 539 op.withValue(MediaColumns.MIME_TYPE, 540 maybeOverrideMimeType(mimeType, xmp.getFormat())); 541 op.withValue(MediaColumns.DOCUMENT_ID, xmp.getDocumentId()); 542 op.withValue(MediaColumns.INSTANCE_ID, xmp.getInstanceId()); 543 op.withValue(MediaColumns.ORIGINAL_DOCUMENT_ID, xmp.getOriginalDocumentId()); 544 } 545 546 /** 547 * Overwrite a value in the given {@link ContentProviderOperation}, but only 548 * when the given {@link Optional} value is present. 549 */ withOptionalValue(ContentProviderOperation.Builder op, String key, Optional<?> value)550 private static void withOptionalValue(ContentProviderOperation.Builder op, 551 String key, Optional<?> value) { 552 if (value.isPresent()) { 553 op.withValue(key, value.get()); 554 } 555 } 556 scanItemDirectory(long existingId, File file, BasicFileAttributes attrs, String mimeType, String volumeName)557 private static @NonNull ContentProviderOperation scanItemDirectory(long existingId, File file, 558 BasicFileAttributes attrs, String mimeType, String volumeName) throws IOException { 559 final ContentProviderOperation.Builder op = newUpsert( 560 MediaStore.Files.getContentUri(volumeName), existingId); 561 try { 562 withGenericValues(op, file, attrs, mimeType); 563 op.withValue(FileColumns.MEDIA_TYPE, 0); 564 op.withValue(FileColumns.FORMAT, MtpConstants.FORMAT_ASSOCIATION); 565 op.withValue(FileColumns.MIME_TYPE, null); 566 } catch (Exception e) { 567 throw new IOException(e); 568 } 569 return op.build(); 570 } 571 572 private static ArrayMap<String, String> sAudioTypes = new ArrayMap<>(); 573 574 static { sAudioTypes.put(Environment.DIRECTORY_RINGTONES, AudioColumns.IS_RINGTONE)575 sAudioTypes.put(Environment.DIRECTORY_RINGTONES, AudioColumns.IS_RINGTONE); sAudioTypes.put(Environment.DIRECTORY_NOTIFICATIONS, AudioColumns.IS_NOTIFICATION)576 sAudioTypes.put(Environment.DIRECTORY_NOTIFICATIONS, AudioColumns.IS_NOTIFICATION); sAudioTypes.put(Environment.DIRECTORY_ALARMS, AudioColumns.IS_ALARM)577 sAudioTypes.put(Environment.DIRECTORY_ALARMS, AudioColumns.IS_ALARM); sAudioTypes.put(Environment.DIRECTORY_PODCASTS, AudioColumns.IS_PODCAST)578 sAudioTypes.put(Environment.DIRECTORY_PODCASTS, AudioColumns.IS_PODCAST); sAudioTypes.put(Environment.DIRECTORY_AUDIOBOOKS, AudioColumns.IS_AUDIOBOOK)579 sAudioTypes.put(Environment.DIRECTORY_AUDIOBOOKS, AudioColumns.IS_AUDIOBOOK); sAudioTypes.put(Environment.DIRECTORY_MUSIC, AudioColumns.IS_MUSIC)580 sAudioTypes.put(Environment.DIRECTORY_MUSIC, AudioColumns.IS_MUSIC); 581 } 582 scanItemAudio(long existingId, File file, BasicFileAttributes attrs, String mimeType, String volumeName)583 private static @NonNull ContentProviderOperation scanItemAudio(long existingId, File file, 584 BasicFileAttributes attrs, String mimeType, String volumeName) throws IOException { 585 final ContentProviderOperation.Builder op = newUpsert( 586 MediaStore.Audio.Media.getContentUri(volumeName), existingId); 587 588 withGenericValues(op, file, attrs, mimeType); 589 op.withValue(AudioColumns.ARTIST, UNKNOWN_STRING); 590 op.withValue(AudioColumns.ALBUM_ARTIST, null); 591 op.withValue(AudioColumns.COMPILATION, null); 592 op.withValue(AudioColumns.COMPOSER, null); 593 op.withValue(AudioColumns.ALBUM, file.getParentFile().getName()); 594 op.withValue(AudioColumns.TRACK, null); 595 op.withValue(AudioColumns.YEAR, null); 596 op.withValue(AudioColumns.GENRE, null); 597 598 final String lowPath = file.getAbsolutePath().toLowerCase(Locale.ROOT); 599 boolean anyMatch = false; 600 for (int i = 0; i < sAudioTypes.size(); i++) { 601 final boolean match = lowPath 602 .contains('/' + sAudioTypes.keyAt(i).toLowerCase(Locale.ROOT) + '/'); 603 op.withValue(sAudioTypes.valueAt(i), match ? 1 : 0); 604 anyMatch |= match; 605 } 606 if (!anyMatch) { 607 op.withValue(AudioColumns.IS_MUSIC, 1); 608 } 609 610 try (FileInputStream is = new FileInputStream(file)) { 611 try (MediaMetadataRetriever mmr = new MediaMetadataRetriever()) { 612 mmr.setDataSource(is.getFD()); 613 614 withOptionalValue(op, MediaColumns.TITLE, 615 parseOptional(mmr.extractMetadata(METADATA_KEY_TITLE))); 616 withOptionalValue(op, MediaColumns.IS_DRM, 617 parseOptional(mmr.extractMetadata(METADATA_KEY_IS_DRM))); 618 withOptionalValue(op, MediaColumns.DURATION, 619 parseOptional(mmr.extractMetadata(METADATA_KEY_DURATION))); 620 621 withOptionalValue(op, AudioColumns.ARTIST, 622 parseOptional(mmr.extractMetadata(METADATA_KEY_ARTIST))); 623 withOptionalValue(op, AudioColumns.ALBUM_ARTIST, 624 parseOptional(mmr.extractMetadata(METADATA_KEY_ALBUMARTIST))); 625 withOptionalValue(op, AudioColumns.COMPILATION, 626 parseOptional(mmr.extractMetadata(METADATA_KEY_COMPILATION))); 627 withOptionalValue(op, AudioColumns.COMPOSER, 628 parseOptional(mmr.extractMetadata(METADATA_KEY_COMPOSER))); 629 withOptionalValue(op, AudioColumns.ALBUM, 630 parseOptional(mmr.extractMetadata(METADATA_KEY_ALBUM))); 631 withOptionalValue(op, AudioColumns.TRACK, 632 parseOptional(mmr.extractMetadata(METADATA_KEY_CD_TRACK_NUMBER))); 633 withOptionalValue(op, AudioColumns.YEAR, 634 parseOptionalOrZero(mmr.extractMetadata(METADATA_KEY_YEAR))); 635 withOptionalValue(op, AudioColumns.GENRE, 636 parseOptional(mmr.extractMetadata(METADATA_KEY_GENRE))); 637 } 638 639 // Also hunt around for XMP metadata 640 final IsoInterface iso = IsoInterface.fromFileDescriptor(is.getFD()); 641 final XmpInterface xmp = XmpInterface.fromContainer(iso); 642 withXmpValues(op, xmp, mimeType); 643 644 } catch (Exception e) { 645 throw new IOException(e); 646 } 647 return op.build(); 648 } 649 scanItemPlaylist(long existingId, File file, BasicFileAttributes attrs, String mimeType, String volumeName)650 private static @NonNull ContentProviderOperation scanItemPlaylist(long existingId, File file, 651 BasicFileAttributes attrs, String mimeType, String volumeName) throws IOException { 652 final ContentProviderOperation.Builder op = newUpsert( 653 MediaStore.Audio.Playlists.getContentUri(volumeName), existingId); 654 try { 655 withGenericValues(op, file, attrs, mimeType); 656 op.withValue(PlaylistsColumns.NAME, extractName(file)); 657 } catch (Exception e) { 658 throw new IOException(e); 659 } 660 return op.build(); 661 } 662 scanItemVideo(long existingId, File file, BasicFileAttributes attrs, String mimeType, String volumeName)663 private static @NonNull ContentProviderOperation scanItemVideo(long existingId, File file, 664 BasicFileAttributes attrs, String mimeType, String volumeName) throws IOException { 665 final ContentProviderOperation.Builder op = newUpsert( 666 MediaStore.Video.Media.getContentUri(volumeName), existingId); 667 668 withGenericValues(op, file, attrs, mimeType); 669 op.withValue(VideoColumns.ARTIST, UNKNOWN_STRING); 670 op.withValue(VideoColumns.ALBUM, file.getParentFile().getName()); 671 op.withValue(VideoColumns.RESOLUTION, null); 672 op.withValue(VideoColumns.COLOR_STANDARD, null); 673 op.withValue(VideoColumns.COLOR_TRANSFER, null); 674 op.withValue(VideoColumns.COLOR_RANGE, null); 675 676 try (FileInputStream is = new FileInputStream(file)) { 677 try (MediaMetadataRetriever mmr = new MediaMetadataRetriever()) { 678 mmr.setDataSource(is.getFD()); 679 680 withOptionalValue(op, MediaColumns.TITLE, 681 parseOptional(mmr.extractMetadata(METADATA_KEY_TITLE))); 682 withOptionalValue(op, MediaColumns.IS_DRM, 683 parseOptional(mmr.extractMetadata(METADATA_KEY_IS_DRM))); 684 withOptionalValue(op, MediaColumns.WIDTH, 685 parseOptional(mmr.extractMetadata(METADATA_KEY_VIDEO_WIDTH))); 686 withOptionalValue(op, MediaColumns.HEIGHT, 687 parseOptional(mmr.extractMetadata(METADATA_KEY_VIDEO_HEIGHT))); 688 withOptionalValue(op, MediaColumns.DURATION, 689 parseOptional(mmr.extractMetadata(METADATA_KEY_DURATION))); 690 withOptionalValue(op, MediaColumns.DATE_TAKEN, 691 parseOptionalDate(mmr.extractMetadata(METADATA_KEY_DATE))); 692 693 withOptionalValue(op, VideoColumns.ARTIST, 694 parseOptional(mmr.extractMetadata(METADATA_KEY_ARTIST))); 695 withOptionalValue(op, VideoColumns.ALBUM, 696 parseOptional(mmr.extractMetadata(METADATA_KEY_ALBUM))); 697 withOptionalValue(op, VideoColumns.RESOLUTION, 698 parseOptionalResolution(mmr)); 699 withOptionalValue(op, VideoColumns.COLOR_STANDARD, 700 parseOptional(mmr.extractMetadata(METADATA_KEY_COLOR_STANDARD))); 701 withOptionalValue(op, VideoColumns.COLOR_TRANSFER, 702 parseOptional(mmr.extractMetadata(METADATA_KEY_COLOR_TRANSFER))); 703 withOptionalValue(op, VideoColumns.COLOR_RANGE, 704 parseOptional(mmr.extractMetadata(METADATA_KEY_COLOR_RANGE))); 705 } 706 707 // Also hunt around for XMP metadata 708 final IsoInterface iso = IsoInterface.fromFileDescriptor(is.getFD()); 709 final XmpInterface xmp = XmpInterface.fromContainer(iso); 710 withXmpValues(op, xmp, mimeType); 711 712 } catch (Exception e) { 713 throw new IOException(e); 714 } 715 return op.build(); 716 } 717 scanItemImage(long existingId, File file, BasicFileAttributes attrs, String mimeType, String volumeName)718 private static @NonNull ContentProviderOperation scanItemImage(long existingId, File file, 719 BasicFileAttributes attrs, String mimeType, String volumeName) throws IOException { 720 final ContentProviderOperation.Builder op = newUpsert( 721 MediaStore.Images.Media.getContentUri(volumeName), existingId); 722 723 withGenericValues(op, file, attrs, mimeType); 724 op.withValue(ImageColumns.DESCRIPTION, null); 725 726 try (FileInputStream is = new FileInputStream(file)) { 727 final ExifInterface exif = new ExifInterface(is); 728 729 withOptionalValue(op, MediaColumns.WIDTH, 730 parseOptionalOrZero(exif.getAttribute(ExifInterface.TAG_IMAGE_WIDTH))); 731 withOptionalValue(op, MediaColumns.HEIGHT, 732 parseOptionalOrZero(exif.getAttribute(ExifInterface.TAG_IMAGE_LENGTH))); 733 withOptionalValue(op, MediaColumns.DATE_TAKEN, 734 parseOptionalDateTaken(exif, lastModifiedTime(file, attrs) * 1000)); 735 withOptionalValue(op, MediaColumns.ORIENTATION, 736 parseOptionalOrientation(exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, 737 ExifInterface.ORIENTATION_UNDEFINED))); 738 739 withOptionalValue(op, ImageColumns.DESCRIPTION, 740 parseOptional(exif.getAttribute(ExifInterface.TAG_IMAGE_DESCRIPTION))); 741 742 // Also hunt around for XMP metadata 743 final XmpInterface xmp = XmpInterface.fromContainer(exif); 744 withXmpValues(op, xmp, mimeType); 745 746 } catch (Exception e) { 747 throw new IOException(e); 748 } 749 return op.build(); 750 } 751 scanItemFile(long existingId, File file, BasicFileAttributes attrs, String mimeType, String volumeName)752 private static @NonNull ContentProviderOperation scanItemFile(long existingId, File file, 753 BasicFileAttributes attrs, String mimeType, String volumeName) throws IOException { 754 final ContentProviderOperation.Builder op = newUpsert( 755 MediaStore.Files.getContentUri(volumeName), existingId); 756 try { 757 withGenericValues(op, file, attrs, mimeType); 758 } catch (Exception e) { 759 throw new IOException(e); 760 } 761 return op.build(); 762 } 763 newUpsert(Uri uri, long existingId)764 private static @NonNull ContentProviderOperation.Builder newUpsert(Uri uri, long existingId) { 765 if (existingId == -1) { 766 return ContentProviderOperation.newInsert(uri) 767 .withFailureAllowed(true); 768 } else { 769 return ContentProviderOperation.newUpdate(ContentUris.withAppendedId(uri, existingId)) 770 .withExpectedCount(1) 771 .withFailureAllowed(true); 772 } 773 } 774 extractExtension(File file)775 public static @Nullable String extractExtension(File file) { 776 final String name = file.getName(); 777 final int lastDot = name.lastIndexOf('.'); 778 return (lastDot == -1) ? null : name.substring(lastDot + 1); 779 } 780 extractName(File file)781 public static @NonNull String extractName(File file) { 782 final String name = file.getName(); 783 final int lastDot = name.lastIndexOf('.'); 784 return (lastDot == -1) ? name : name.substring(0, lastDot); 785 } 786 parseOptional(@ullable T value)787 private static @NonNull <T> Optional<T> parseOptional(@Nullable T value) { 788 if (value == null) { 789 return Optional.empty(); 790 } else if (value instanceof String && ((String) value).length() == 0) { 791 return Optional.empty(); 792 } else if (value instanceof String && ((String) value).equals("-1")) { 793 return Optional.empty(); 794 } else if (value instanceof Number && ((Number) value).intValue() == -1) { 795 return Optional.empty(); 796 } else { 797 return Optional.of(value); 798 } 799 } 800 parseOptionalOrZero(@ullable T value)801 private static @NonNull <T> Optional<T> parseOptionalOrZero(@Nullable T value) { 802 if (value instanceof String && ((String) value).equals("0")) { 803 return Optional.empty(); 804 } else if (value instanceof Number && ((Number) value).intValue() == 0) { 805 return Optional.empty(); 806 } else { 807 return parseOptional(value); 808 } 809 } 810 811 /** 812 * Try our best to calculate {@link MediaColumns#DATE_TAKEN} in reference to 813 * the epoch, making our best guess from unrelated fields when offset 814 * information isn't directly available. 815 */ parseOptionalDateTaken(@onNull ExifInterface exif, @CurrentTimeMillisLong long lastModifiedTime)816 static @NonNull Optional<Long> parseOptionalDateTaken(@NonNull ExifInterface exif, 817 @CurrentTimeMillisLong long lastModifiedTime) { 818 final long originalTime = exif.getDateTimeOriginal(); 819 if (exif.hasAttribute(ExifInterface.TAG_OFFSET_TIME_ORIGINAL)) { 820 // We have known offset information, return it directly! 821 return Optional.of(originalTime); 822 } else { 823 // Otherwise we need to guess the offset from unrelated fields 824 final long smallestZone = 15 * MINUTE_IN_MILLIS; 825 final long gpsTime = exif.getGpsDateTime(); 826 if (gpsTime > 0) { 827 final long offset = gpsTime - originalTime; 828 if (Math.abs(offset) < 24 * HOUR_IN_MILLIS) { 829 final long rounded = Math.round((float) offset / smallestZone) * smallestZone; 830 return Optional.of(originalTime + rounded); 831 } 832 } 833 if (lastModifiedTime > 0) { 834 final long offset = lastModifiedTime - originalTime; 835 if (Math.abs(offset) < 24 * HOUR_IN_MILLIS) { 836 final long rounded = Math.round((float) offset / smallestZone) * smallestZone; 837 return Optional.of(originalTime + rounded); 838 } 839 } 840 return Optional.empty(); 841 } 842 } 843 parseOptionalOrientation(int orientation)844 private static @NonNull Optional<Integer> parseOptionalOrientation(int orientation) { 845 switch (orientation) { 846 case ExifInterface.ORIENTATION_NORMAL: return Optional.of(0); 847 case ExifInterface.ORIENTATION_ROTATE_90: return Optional.of(90); 848 case ExifInterface.ORIENTATION_ROTATE_180: return Optional.of(180); 849 case ExifInterface.ORIENTATION_ROTATE_270: return Optional.of(270); 850 default: return Optional.empty(); 851 } 852 } 853 parseOptionalResolution( @onNull MediaMetadataRetriever mmr)854 private static @NonNull Optional<String> parseOptionalResolution( 855 @NonNull MediaMetadataRetriever mmr) { 856 final Optional<?> width = parseOptional(mmr.extractMetadata(METADATA_KEY_VIDEO_WIDTH)); 857 final Optional<?> height = parseOptional(mmr.extractMetadata(METADATA_KEY_VIDEO_HEIGHT)); 858 if (width.isPresent() && height.isPresent()) { 859 return Optional.of(width.get() + "\u00d7" + height.get()); 860 } else { 861 return Optional.empty(); 862 } 863 } 864 parseOptionalDate(@ullable String date)865 private static @NonNull Optional<Long> parseOptionalDate(@Nullable String date) { 866 if (TextUtils.isEmpty(date)) return Optional.empty(); 867 try { 868 final long value = sDateFormat.parse(date).getTime(); 869 return (value > 0) ? Optional.of(value) : Optional.empty(); 870 } catch (ParseException e) { 871 return Optional.empty(); 872 } 873 } 874 875 /** 876 * Maybe replace the MIME type from extension with the MIME type from the 877 * XMP metadata, but only when the top-level MIME type agrees. 878 */ 879 @VisibleForTesting maybeOverrideMimeType(@onNull String extMimeType, @Nullable String xmpMimeType)880 public static @NonNull String maybeOverrideMimeType(@NonNull String extMimeType, 881 @Nullable String xmpMimeType) { 882 // Ignore XMP when missing 883 if (TextUtils.isEmpty(xmpMimeType)) return extMimeType; 884 885 // Ignore XMP when invalid 886 final int xmpSplit = xmpMimeType.indexOf('/'); 887 if (xmpSplit == -1) return extMimeType; 888 889 if (extMimeType.regionMatches(0, xmpMimeType, 0, xmpSplit + 1)) { 890 return xmpMimeType; 891 } else { 892 return extMimeType; 893 } 894 } 895 896 /** 897 * Return last modified time of given file. This value is typically read 898 * from the given {@link BasicFileAttributes}, except in the case of 899 * read-only partitions, where {@link Build#TIME} is used instead. 900 */ lastModifiedTime(@onNull File file, @NonNull BasicFileAttributes attrs)901 public static @CurrentTimeSecondsLong long lastModifiedTime(@NonNull File file, 902 @NonNull BasicFileAttributes attrs) { 903 if (FileUtils.contains(Environment.getStorageDirectory(), file)) { 904 return attrs.lastModifiedTime().toMillis() / 1000; 905 } else { 906 return Build.TIME / 1000; 907 } 908 } 909 910 /** 911 * Test if any parents of given directory should be considered hidden. 912 */ isDirectoryHiddenRecursive(File dir)913 static boolean isDirectoryHiddenRecursive(File dir) { 914 Trace.traceBegin(TRACE_TAG_DATABASE, "isDirectoryHiddenRecursive"); 915 try { 916 while (dir != null) { 917 if (isDirectoryHidden(dir)) { 918 return true; 919 } 920 dir = dir.getParentFile(); 921 } 922 return false; 923 } finally { 924 Trace.traceEnd(TRACE_TAG_DATABASE); 925 } 926 } 927 928 /** 929 * Test if this given directory should be considered hidden. 930 */ isDirectoryHidden(File dir)931 static boolean isDirectoryHidden(File dir) { 932 final File nomedia = new File(dir, ".nomedia"); 933 934 // Handle well-known paths that should always be visible or invisible, 935 // regardless of .nomedia presence 936 if (PATTERN_VISIBLE.matcher(dir.getAbsolutePath()).matches()) { 937 nomedia.delete(); 938 return false; 939 } 940 if (PATTERN_INVISIBLE.matcher(dir.getAbsolutePath()).matches()) { 941 try { 942 nomedia.createNewFile(); 943 } catch (IOException ignored) { 944 } 945 return true; 946 } 947 948 // Otherwise fall back to directory name or .nomedia presence 949 final String name = dir.getName(); 950 if (name.startsWith(".")) { 951 return true; 952 } 953 if (nomedia.exists()) { 954 return true; 955 } 956 return false; 957 } 958 959 /** 960 * Test if this given {@link Uri} is a 961 * {@link android.provider.MediaStore.Audio.Playlists} item. 962 */ isPlaylist(Uri uri)963 static boolean isPlaylist(Uri uri) { 964 final List<String> path = uri.getPathSegments(); 965 return (path.size() == 4) && path.get(1).equals("audio") && path.get(2).equals("playlists"); 966 } 967 968 /** 969 * Escape the given argument for use in a {@code LIKE} statement. 970 */ escapeForLike(String arg)971 static String escapeForLike(String arg) { 972 final StringBuilder sb = new StringBuilder(); 973 for (int i = 0; i < arg.length(); i++) { 974 final char c = arg.charAt(i); 975 switch (c) { 976 case '%': sb.append('\\'); 977 case '_': sb.append('\\'); 978 } 979 sb.append(c); 980 } 981 return sb.toString(); 982 } 983 } 984