1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.compat.annotation.UnsupportedAppUsage; 20 import android.graphics.Canvas; 21 import android.media.MediaPlayer.TrackInfo; 22 import android.os.Handler; 23 import android.util.Log; 24 import android.util.LongSparseArray; 25 import android.util.Pair; 26 27 import java.util.Iterator; 28 import java.util.NoSuchElementException; 29 import java.util.SortedMap; 30 import java.util.TreeMap; 31 import java.util.Vector; 32 33 /** 34 * A subtitle track abstract base class that is responsible for parsing and displaying 35 * an instance of a particular type of subtitle. 36 * 37 * @hide 38 */ 39 public abstract class SubtitleTrack implements MediaTimeProvider.OnMediaTimeListener { 40 private static final String TAG = "SubtitleTrack"; 41 private long mLastUpdateTimeMs; 42 private long mLastTimeMs; 43 44 private Runnable mRunnable; 45 46 /** @hide TODO private */ 47 final protected LongSparseArray<Run> mRunsByEndTime = new LongSparseArray<Run>(); 48 /** @hide TODO private */ 49 final protected LongSparseArray<Run> mRunsByID = new LongSparseArray<Run>(); 50 51 /** @hide TODO private */ 52 protected CueList mCues; 53 /** @hide TODO private */ 54 final protected Vector<Cue> mActiveCues = new Vector<Cue>(); 55 /** @hide */ 56 protected boolean mVisible; 57 58 /** @hide */ 59 public boolean DEBUG = false; 60 61 /** @hide */ 62 protected Handler mHandler = new Handler(); 63 64 private MediaFormat mFormat; 65 SubtitleTrack(MediaFormat format)66 public SubtitleTrack(MediaFormat format) { 67 mFormat = format; 68 mCues = new CueList(); 69 clearActiveCues(); 70 mLastTimeMs = -1; 71 } 72 73 /** @hide */ getFormat()74 public final MediaFormat getFormat() { 75 return mFormat; 76 } 77 78 private long mNextScheduledTimeMs = -1; 79 onData(SubtitleData data)80 protected void onData(SubtitleData data) { 81 long runID = data.getStartTimeUs() + 1; 82 onData(data.getData(), true /* eos */, runID); 83 setRunDiscardTimeMs( 84 runID, 85 (data.getStartTimeUs() + data.getDurationUs()) / 1000); 86 } 87 88 /** 89 * Called when there is input data for the subtitle track. The 90 * complete subtitle for a track can include multiple whole units 91 * (runs). Each of these units can have multiple sections. The 92 * contents of a run are submitted in sequential order, with eos 93 * indicating the last section of the run. Calls from different 94 * runs must not be intermixed. 95 * 96 * @param data subtitle data byte buffer 97 * @param eos true if this is the last section of the run. 98 * @param runID mostly-unique ID for this run of data. Subtitle cues 99 * with runID of 0 are discarded immediately after 100 * display. Cues with runID of ~0 are discarded 101 * only at the deletion of the track object. Cues 102 * with other runID-s are discarded at the end of the 103 * run, which defaults to the latest timestamp of 104 * any of its cues (with this runID). 105 */ onData(byte[] data, boolean eos, long runID)106 public abstract void onData(byte[] data, boolean eos, long runID); 107 108 /** 109 * Called when adding the subtitle rendering widget to the view hierarchy, 110 * as well as when showing or hiding the subtitle track, or when the video 111 * surface position has changed. 112 * 113 * @return the widget that renders this subtitle track. For most renderers 114 * there should be a single shared instance that is used for all 115 * tracks supported by that renderer, as at most one subtitle track 116 * is visible at one time. 117 */ getRenderingWidget()118 public abstract RenderingWidget getRenderingWidget(); 119 120 /** 121 * Called when the active cues have changed, and the contents of the subtitle 122 * view should be updated. 123 * 124 * @hide 125 */ updateView(Vector<Cue> activeCues)126 public abstract void updateView(Vector<Cue> activeCues); 127 128 /** @hide */ updateActiveCues(boolean rebuild, long timeMs)129 protected synchronized void updateActiveCues(boolean rebuild, long timeMs) { 130 // out-of-order times mean seeking or new active cues being added 131 // (during their own timespan) 132 if (rebuild || mLastUpdateTimeMs > timeMs) { 133 clearActiveCues(); 134 } 135 136 for(Iterator<Pair<Long, Cue> > it = 137 mCues.entriesBetween(mLastUpdateTimeMs, timeMs).iterator(); it.hasNext(); ) { 138 Pair<Long, Cue> event = it.next(); 139 Cue cue = event.second; 140 141 if (cue.mEndTimeMs == event.first) { 142 // remove past cues 143 if (DEBUG) Log.v(TAG, "Removing " + cue); 144 mActiveCues.remove(cue); 145 if (cue.mRunID == 0) { 146 it.remove(); 147 } 148 } else if (cue.mStartTimeMs == event.first) { 149 // add new cues 150 // TRICKY: this will happen in start order 151 if (DEBUG) Log.v(TAG, "Adding " + cue); 152 if (cue.mInnerTimesMs != null) { 153 cue.onTime(timeMs); 154 } 155 mActiveCues.add(cue); 156 } else if (cue.mInnerTimesMs != null) { 157 // cue is modified 158 cue.onTime(timeMs); 159 } 160 } 161 162 /* complete any runs */ 163 while (mRunsByEndTime.size() > 0 && 164 mRunsByEndTime.keyAt(0) <= timeMs) { 165 removeRunsByEndTimeIndex(0); // removes element 166 } 167 mLastUpdateTimeMs = timeMs; 168 } 169 removeRunsByEndTimeIndex(int ix)170 private void removeRunsByEndTimeIndex(int ix) { 171 Run run = mRunsByEndTime.valueAt(ix); 172 while (run != null) { 173 Cue cue = run.mFirstCue; 174 while (cue != null) { 175 mCues.remove(cue); 176 Cue nextCue = cue.mNextInRun; 177 cue.mNextInRun = null; 178 cue = nextCue; 179 } 180 mRunsByID.remove(run.mRunID); 181 Run nextRun = run.mNextRunAtEndTimeMs; 182 run.mPrevRunAtEndTimeMs = null; 183 run.mNextRunAtEndTimeMs = null; 184 run = nextRun; 185 } 186 mRunsByEndTime.removeAt(ix); 187 } 188 189 @Override finalize()190 protected void finalize() throws Throwable { 191 /* remove all cues (untangle all cross-links) */ 192 int size = mRunsByEndTime.size(); 193 for(int ix = size - 1; ix >= 0; ix--) { 194 removeRunsByEndTimeIndex(ix); 195 } 196 197 super.finalize(); 198 } 199 takeTime(long timeMs)200 private synchronized void takeTime(long timeMs) { 201 mLastTimeMs = timeMs; 202 } 203 204 /** @hide */ clearActiveCues()205 protected synchronized void clearActiveCues() { 206 if (DEBUG) Log.v(TAG, "Clearing " + mActiveCues.size() + " active cues"); 207 mActiveCues.clear(); 208 mLastUpdateTimeMs = -1; 209 } 210 211 /** @hide */ scheduleTimedEvents()212 protected void scheduleTimedEvents() { 213 /* get times for the next event */ 214 if (mTimeProvider != null) { 215 mNextScheduledTimeMs = mCues.nextTimeAfter(mLastTimeMs); 216 if (DEBUG) Log.d(TAG, "sched @" + mNextScheduledTimeMs + " after " + mLastTimeMs); 217 mTimeProvider.notifyAt( 218 mNextScheduledTimeMs >= 0 ? 219 (mNextScheduledTimeMs * 1000) : MediaTimeProvider.NO_TIME, 220 this); 221 } 222 } 223 224 /** 225 * @hide 226 */ 227 @Override onTimedEvent(long timeUs)228 public void onTimedEvent(long timeUs) { 229 if (DEBUG) Log.d(TAG, "onTimedEvent " + timeUs); 230 synchronized (this) { 231 long timeMs = timeUs / 1000; 232 updateActiveCues(false, timeMs); 233 takeTime(timeMs); 234 } 235 updateView(mActiveCues); 236 scheduleTimedEvents(); 237 } 238 239 /** 240 * @hide 241 */ 242 @Override onSeek(long timeUs)243 public void onSeek(long timeUs) { 244 if (DEBUG) Log.d(TAG, "onSeek " + timeUs); 245 synchronized (this) { 246 long timeMs = timeUs / 1000; 247 updateActiveCues(true, timeMs); 248 takeTime(timeMs); 249 } 250 updateView(mActiveCues); 251 scheduleTimedEvents(); 252 } 253 254 /** 255 * @hide 256 */ 257 @Override onStop()258 public void onStop() { 259 synchronized (this) { 260 if (DEBUG) Log.d(TAG, "onStop"); 261 clearActiveCues(); 262 mLastTimeMs = -1; 263 } 264 updateView(mActiveCues); 265 mNextScheduledTimeMs = -1; 266 if (mTimeProvider != null) { 267 mTimeProvider.notifyAt(MediaTimeProvider.NO_TIME, this); 268 } 269 } 270 271 /** @hide */ 272 protected MediaTimeProvider mTimeProvider; 273 274 /** @hide */ show()275 public void show() { 276 if (mVisible) { 277 return; 278 } 279 280 mVisible = true; 281 RenderingWidget renderingWidget = getRenderingWidget(); 282 if (renderingWidget != null) { 283 renderingWidget.setVisible(true); 284 } 285 if (mTimeProvider != null) { 286 mTimeProvider.scheduleUpdate(this); 287 } 288 } 289 290 /** @hide */ hide()291 public void hide() { 292 if (!mVisible) { 293 return; 294 } 295 296 if (mTimeProvider != null) { 297 mTimeProvider.cancelNotifications(this); 298 } 299 RenderingWidget renderingWidget = getRenderingWidget(); 300 if (renderingWidget != null) { 301 renderingWidget.setVisible(false); 302 } 303 mVisible = false; 304 } 305 306 /** @hide */ addCue(Cue cue)307 protected synchronized boolean addCue(Cue cue) { 308 mCues.add(cue); 309 310 if (cue.mRunID != 0) { 311 Run run = mRunsByID.get(cue.mRunID); 312 if (run == null) { 313 run = new Run(); 314 mRunsByID.put(cue.mRunID, run); 315 run.mEndTimeMs = cue.mEndTimeMs; 316 } else if (run.mEndTimeMs < cue.mEndTimeMs) { 317 run.mEndTimeMs = cue.mEndTimeMs; 318 } 319 320 // link-up cues in the same run 321 cue.mNextInRun = run.mFirstCue; 322 run.mFirstCue = cue; 323 } 324 325 // if a cue is added that should be visible, need to refresh view 326 long nowMs = -1; 327 if (mTimeProvider != null) { 328 try { 329 nowMs = mTimeProvider.getCurrentTimeUs( 330 false /* precise */, true /* monotonic */) / 1000; 331 } catch (IllegalStateException e) { 332 // handle as it we are not playing 333 } 334 } 335 336 if (DEBUG) Log.v(TAG, "mVisible=" + mVisible + ", " + 337 cue.mStartTimeMs + " <= " + nowMs + ", " + 338 cue.mEndTimeMs + " >= " + mLastTimeMs); 339 340 if (mVisible && 341 cue.mStartTimeMs <= nowMs && 342 // we don't trust nowMs, so check any cue since last callback 343 cue.mEndTimeMs >= mLastTimeMs) { 344 if (mRunnable != null) { 345 mHandler.removeCallbacks(mRunnable); 346 } 347 final SubtitleTrack track = this; 348 final long thenMs = nowMs; 349 mRunnable = new Runnable() { 350 @Override 351 public void run() { 352 // even with synchronized, it is possible that we are going 353 // to do multiple updates as the runnable could be already 354 // running. 355 synchronized (track) { 356 mRunnable = null; 357 updateActiveCues(true, thenMs); 358 updateView(mActiveCues); 359 } 360 } 361 }; 362 // delay update so we don't update view on every cue. TODO why 10? 363 if (mHandler.postDelayed(mRunnable, 10 /* delay */)) { 364 if (DEBUG) Log.v(TAG, "scheduling update"); 365 } else { 366 if (DEBUG) Log.w(TAG, "failed to schedule subtitle view update"); 367 } 368 return true; 369 } 370 371 if (mVisible && 372 cue.mEndTimeMs >= mLastTimeMs && 373 (cue.mStartTimeMs < mNextScheduledTimeMs || 374 mNextScheduledTimeMs < 0)) { 375 scheduleTimedEvents(); 376 } 377 378 return false; 379 } 380 381 /** @hide */ setTimeProvider(MediaTimeProvider timeProvider)382 public synchronized void setTimeProvider(MediaTimeProvider timeProvider) { 383 if (mTimeProvider == timeProvider) { 384 return; 385 } 386 if (mTimeProvider != null) { 387 mTimeProvider.cancelNotifications(this); 388 } 389 mTimeProvider = timeProvider; 390 if (mTimeProvider != null) { 391 mTimeProvider.scheduleUpdate(this); 392 } 393 } 394 395 396 /** @hide */ 397 static class CueList { 398 private static final String TAG = "CueList"; 399 // simplistic, inefficient implementation 400 private SortedMap<Long, Vector<Cue> > mCues; 401 public boolean DEBUG = false; 402 addEvent(Cue cue, long timeMs)403 private boolean addEvent(Cue cue, long timeMs) { 404 Vector<Cue> cues = mCues.get(timeMs); 405 if (cues == null) { 406 cues = new Vector<Cue>(2); 407 mCues.put(timeMs, cues); 408 } else if (cues.contains(cue)) { 409 // do not duplicate cues 410 return false; 411 } 412 413 cues.add(cue); 414 return true; 415 } 416 removeEvent(Cue cue, long timeMs)417 private void removeEvent(Cue cue, long timeMs) { 418 Vector<Cue> cues = mCues.get(timeMs); 419 if (cues != null) { 420 cues.remove(cue); 421 if (cues.size() == 0) { 422 mCues.remove(timeMs); 423 } 424 } 425 } 426 add(Cue cue)427 public void add(Cue cue) { 428 // ignore non-positive-duration cues 429 if (cue.mStartTimeMs >= cue.mEndTimeMs) 430 return; 431 432 if (!addEvent(cue, cue.mStartTimeMs)) { 433 return; 434 } 435 436 long lastTimeMs = cue.mStartTimeMs; 437 if (cue.mInnerTimesMs != null) { 438 for (long timeMs: cue.mInnerTimesMs) { 439 if (timeMs > lastTimeMs && timeMs < cue.mEndTimeMs) { 440 addEvent(cue, timeMs); 441 lastTimeMs = timeMs; 442 } 443 } 444 } 445 446 addEvent(cue, cue.mEndTimeMs); 447 } 448 remove(Cue cue)449 public void remove(Cue cue) { 450 removeEvent(cue, cue.mStartTimeMs); 451 if (cue.mInnerTimesMs != null) { 452 for (long timeMs: cue.mInnerTimesMs) { 453 removeEvent(cue, timeMs); 454 } 455 } 456 removeEvent(cue, cue.mEndTimeMs); 457 } 458 entriesBetween( final long lastTimeMs, final long timeMs)459 public Iterable<Pair<Long, Cue>> entriesBetween( 460 final long lastTimeMs, final long timeMs) { 461 return new Iterable<Pair<Long, Cue> >() { 462 @Override 463 public Iterator<Pair<Long, Cue> > iterator() { 464 if (DEBUG) Log.d(TAG, "slice (" + lastTimeMs + ", " + timeMs + "]="); 465 try { 466 return new EntryIterator( 467 mCues.subMap(lastTimeMs + 1, timeMs + 1)); 468 } catch(IllegalArgumentException e) { 469 return new EntryIterator(null); 470 } 471 } 472 }; 473 } 474 nextTimeAfter(long timeMs)475 public long nextTimeAfter(long timeMs) { 476 SortedMap<Long, Vector<Cue>> tail = null; 477 try { 478 tail = mCues.tailMap(timeMs + 1); 479 if (tail != null) { 480 return tail.firstKey(); 481 } else { 482 return -1; 483 } 484 } catch(IllegalArgumentException e) { 485 return -1; 486 } catch(NoSuchElementException e) { 487 return -1; 488 } 489 } 490 491 class EntryIterator implements Iterator<Pair<Long, Cue> > { 492 @Override hasNext()493 public boolean hasNext() { 494 return !mDone; 495 } 496 497 @Override next()498 public Pair<Long, Cue> next() { 499 if (mDone) { 500 throw new NoSuchElementException(""); 501 } 502 mLastEntry = new Pair<Long, Cue>( 503 mCurrentTimeMs, mListIterator.next()); 504 mLastListIterator = mListIterator; 505 if (!mListIterator.hasNext()) { 506 nextKey(); 507 } 508 return mLastEntry; 509 } 510 511 @Override remove()512 public void remove() { 513 // only allow removing end tags 514 if (mLastListIterator == null || 515 mLastEntry.second.mEndTimeMs != mLastEntry.first) { 516 throw new IllegalStateException(""); 517 } 518 519 // remove end-cue 520 mLastListIterator.remove(); 521 mLastListIterator = null; 522 if (mCues.get(mLastEntry.first).size() == 0) { 523 mCues.remove(mLastEntry.first); 524 } 525 526 // remove rest of the cues 527 Cue cue = mLastEntry.second; 528 removeEvent(cue, cue.mStartTimeMs); 529 if (cue.mInnerTimesMs != null) { 530 for (long timeMs: cue.mInnerTimesMs) { 531 removeEvent(cue, timeMs); 532 } 533 } 534 } 535 EntryIterator(SortedMap<Long, Vector<Cue> > cues)536 public EntryIterator(SortedMap<Long, Vector<Cue> > cues) { 537 if (DEBUG) Log.v(TAG, cues + ""); 538 mRemainingCues = cues; 539 mLastListIterator = null; 540 nextKey(); 541 } 542 nextKey()543 private void nextKey() { 544 do { 545 try { 546 if (mRemainingCues == null) { 547 throw new NoSuchElementException(""); 548 } 549 mCurrentTimeMs = mRemainingCues.firstKey(); 550 mListIterator = 551 mRemainingCues.get(mCurrentTimeMs).iterator(); 552 try { 553 mRemainingCues = 554 mRemainingCues.tailMap(mCurrentTimeMs + 1); 555 } catch (IllegalArgumentException e) { 556 mRemainingCues = null; 557 } 558 mDone = false; 559 } catch (NoSuchElementException e) { 560 mDone = true; 561 mRemainingCues = null; 562 mListIterator = null; 563 return; 564 } 565 } while (!mListIterator.hasNext()); 566 } 567 568 private long mCurrentTimeMs; 569 private Iterator<Cue> mListIterator; 570 private boolean mDone; 571 private SortedMap<Long, Vector<Cue> > mRemainingCues; 572 private Iterator<Cue> mLastListIterator; 573 private Pair<Long,Cue> mLastEntry; 574 } 575 CueList()576 CueList() { 577 mCues = new TreeMap<Long, Vector<Cue>>(); 578 } 579 } 580 581 /** @hide */ 582 public static class Cue { 583 public long mStartTimeMs; 584 public long mEndTimeMs; 585 public long[] mInnerTimesMs; 586 public long mRunID; 587 588 /** @hide */ 589 public Cue mNextInRun; 590 591 public void onTime(long timeMs) { } 592 } 593 594 /** @hide update mRunsByEndTime (with default end time) */ 595 protected void finishedRun(long runID) { 596 if (runID != 0 && runID != ~0) { 597 Run run = mRunsByID.get(runID); 598 if (run != null) { 599 run.storeByEndTimeMs(mRunsByEndTime); 600 } 601 } 602 } 603 604 /** @hide update mRunsByEndTime with given end time */ 605 public void setRunDiscardTimeMs(long runID, long timeMs) { 606 if (runID != 0 && runID != ~0) { 607 Run run = mRunsByID.get(runID); 608 if (run != null) { 609 run.mEndTimeMs = timeMs; 610 run.storeByEndTimeMs(mRunsByEndTime); 611 } 612 } 613 } 614 615 /** @hide whether this is a text track who fires events instead getting rendered */ 616 public int getTrackType() { 617 return getRenderingWidget() == null 618 ? TrackInfo.MEDIA_TRACK_TYPE_TIMEDTEXT 619 : TrackInfo.MEDIA_TRACK_TYPE_SUBTITLE; 620 } 621 622 623 /** @hide */ 624 private static class Run { 625 public Cue mFirstCue; 626 public Run mNextRunAtEndTimeMs; 627 public Run mPrevRunAtEndTimeMs; 628 public long mEndTimeMs = -1; 629 public long mRunID = 0; 630 private long mStoredEndTimeMs = -1; 631 632 public void storeByEndTimeMs(LongSparseArray<Run> runsByEndTime) { 633 // remove old value if any 634 int ix = runsByEndTime.indexOfKey(mStoredEndTimeMs); 635 if (ix >= 0) { 636 if (mPrevRunAtEndTimeMs == null) { 637 assert(this == runsByEndTime.valueAt(ix)); 638 if (mNextRunAtEndTimeMs == null) { 639 runsByEndTime.removeAt(ix); 640 } else { 641 runsByEndTime.setValueAt(ix, mNextRunAtEndTimeMs); 642 } 643 } 644 removeAtEndTimeMs(); 645 } 646 647 // add new value 648 if (mEndTimeMs >= 0) { 649 mPrevRunAtEndTimeMs = null; 650 mNextRunAtEndTimeMs = runsByEndTime.get(mEndTimeMs); 651 if (mNextRunAtEndTimeMs != null) { 652 mNextRunAtEndTimeMs.mPrevRunAtEndTimeMs = this; 653 } 654 runsByEndTime.put(mEndTimeMs, this); 655 mStoredEndTimeMs = mEndTimeMs; 656 } 657 } 658 659 public void removeAtEndTimeMs() { 660 Run prev = mPrevRunAtEndTimeMs; 661 662 if (mPrevRunAtEndTimeMs != null) { 663 mPrevRunAtEndTimeMs.mNextRunAtEndTimeMs = mNextRunAtEndTimeMs; 664 mPrevRunAtEndTimeMs = null; 665 } 666 if (mNextRunAtEndTimeMs != null) { 667 mNextRunAtEndTimeMs.mPrevRunAtEndTimeMs = prev; 668 mNextRunAtEndTimeMs = null; 669 } 670 } 671 } 672 673 /** 674 * Interface for rendering subtitles onto a Canvas. 675 */ 676 public interface RenderingWidget { 677 /** 678 * Sets the widget's callback, which is used to send updates when the 679 * rendered data has changed. 680 * 681 * @param callback update callback 682 */ 683 @UnsupportedAppUsage 684 public void setOnChangedListener(OnChangedListener callback); 685 686 /** 687 * Sets the widget's size. 688 * 689 * @param width width in pixels 690 * @param height height in pixels 691 */ 692 @UnsupportedAppUsage 693 public void setSize(int width, int height); 694 695 /** 696 * Sets whether the widget should draw subtitles. 697 * 698 * @param visible true if subtitles should be drawn, false otherwise 699 */ 700 public void setVisible(boolean visible); 701 702 /** 703 * Renders subtitles onto a {@link Canvas}. 704 * 705 * @param c canvas on which to render subtitles 706 */ 707 @UnsupportedAppUsage 708 public void draw(Canvas c); 709 710 /** 711 * Called when the widget is attached to a window. 712 */ 713 @UnsupportedAppUsage 714 public void onAttachedToWindow(); 715 716 /** 717 * Called when the widget is detached from a window. 718 */ 719 @UnsupportedAppUsage 720 public void onDetachedFromWindow(); 721 722 /** 723 * Callback used to send updates about changes to rendering data. 724 */ 725 public interface OnChangedListener { 726 /** 727 * Called when the rendering data has changed. 728 * 729 * @param renderingWidget the widget whose data has changed 730 */ 731 public void onChanged(RenderingWidget renderingWidget); 732 } 733 } 734 } 735