1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.mediav2.cts;
18 
19 import android.content.pm.PackageManager;
20 import android.graphics.ImageFormat;
21 import android.media.Image;
22 import android.media.MediaCodec;
23 import android.media.MediaCodecInfo;
24 import android.media.MediaCodecList;
25 import android.media.MediaExtractor;
26 import android.media.MediaFormat;
27 import android.os.Build;
28 import android.os.PersistableBundle;
29 import android.util.Log;
30 import android.util.Pair;
31 import android.view.Surface;
32 
33 import androidx.annotation.NonNull;
34 import androidx.test.platform.app.InstrumentationRegistry;
35 
36 import org.junit.Assert;
37 
38 import java.io.File;
39 import java.io.FileInputStream;
40 import java.io.IOException;
41 import java.nio.ByteBuffer;
42 import java.nio.ByteOrder;
43 import java.util.ArrayList;
44 import java.util.Arrays;
45 import java.util.Collections;
46 import java.util.HashMap;
47 import java.util.LinkedList;
48 import java.util.List;
49 import java.util.Map;
50 import java.util.concurrent.locks.Condition;
51 import java.util.concurrent.locks.Lock;
52 import java.util.concurrent.locks.ReentrantLock;
53 import java.util.zip.CRC32;
54 
55 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible;
56 import static org.junit.Assert.assertEquals;
57 import static org.junit.Assert.assertTrue;
58 import static org.junit.Assert.fail;
59 
60 class CodecAsyncHandler extends MediaCodec.Callback {
61     private static final String LOG_TAG = CodecAsyncHandler.class.getSimpleName();
62     private final Lock mLock = new ReentrantLock();
63     private final Condition mCondition = mLock.newCondition();
64     private final LinkedList<Pair<Integer, MediaCodec.BufferInfo>> mCbInputQueue;
65     private final LinkedList<Pair<Integer, MediaCodec.BufferInfo>> mCbOutputQueue;
66     private MediaFormat mOutFormat;
67     private boolean mSignalledOutFormatChanged;
68     private volatile boolean mSignalledError;
69 
CodecAsyncHandler()70     CodecAsyncHandler() {
71         mCbInputQueue = new LinkedList<>();
72         mCbOutputQueue = new LinkedList<>();
73         mSignalledError = false;
74         mSignalledOutFormatChanged = false;
75     }
76 
clearQueues()77     void clearQueues() {
78         mLock.lock();
79         mCbInputQueue.clear();
80         mCbOutputQueue.clear();
81         mLock.unlock();
82     }
83 
resetContext()84     void resetContext() {
85         clearQueues();
86         mOutFormat = null;
87         mSignalledOutFormatChanged = false;
88         mSignalledError = false;
89     }
90 
91     @Override
onInputBufferAvailable(@onNull MediaCodec codec, int bufferIndex)92     public void onInputBufferAvailable(@NonNull MediaCodec codec, int bufferIndex) {
93         assertTrue(bufferIndex >= 0);
94         mLock.lock();
95         mCbInputQueue.add(new Pair<>(bufferIndex, (MediaCodec.BufferInfo) null));
96         mCondition.signalAll();
97         mLock.unlock();
98     }
99 
100     @Override
onOutputBufferAvailable(@onNull MediaCodec codec, int bufferIndex, @NonNull MediaCodec.BufferInfo info)101     public void onOutputBufferAvailable(@NonNull MediaCodec codec, int bufferIndex,
102             @NonNull MediaCodec.BufferInfo info) {
103         assertTrue(bufferIndex >= 0);
104         mLock.lock();
105         mCbOutputQueue.add(new Pair<>(bufferIndex, info));
106         mCondition.signalAll();
107         mLock.unlock();
108     }
109 
110     @Override
onError(@onNull MediaCodec codec, MediaCodec.CodecException e)111     public void onError(@NonNull MediaCodec codec, MediaCodec.CodecException e) {
112         mLock.lock();
113         mSignalledError = true;
114         mCondition.signalAll();
115         mLock.unlock();
116         Log.e(LOG_TAG, "received media codec error : " + e.getMessage());
117     }
118 
119     @Override
onOutputFormatChanged(@onNull MediaCodec codec, @NonNull MediaFormat format)120     public void onOutputFormatChanged(@NonNull MediaCodec codec, @NonNull MediaFormat format) {
121         mOutFormat = format;
122         mSignalledOutFormatChanged = true;
123         Log.i(LOG_TAG, "Output format changed: " + format.toString());
124     }
125 
setCallBack(MediaCodec codec, boolean isCodecInAsyncMode)126     void setCallBack(MediaCodec codec, boolean isCodecInAsyncMode) {
127         if (isCodecInAsyncMode) {
128             codec.setCallback(this);
129         } else {
130             codec.setCallback(null);
131         }
132     }
133 
getInput()134     Pair<Integer, MediaCodec.BufferInfo> getInput() throws InterruptedException {
135         Pair<Integer, MediaCodec.BufferInfo> element = null;
136         mLock.lock();
137         while (!mSignalledError) {
138             if (mCbInputQueue.isEmpty()) {
139                 mCondition.await();
140             } else {
141                 element = mCbInputQueue.remove(0);
142                 break;
143             }
144         }
145         mLock.unlock();
146         return element;
147     }
148 
getOutput()149     Pair<Integer, MediaCodec.BufferInfo> getOutput() throws InterruptedException {
150         Pair<Integer, MediaCodec.BufferInfo> element = null;
151         mLock.lock();
152         while (!mSignalledError) {
153             if (mCbOutputQueue.isEmpty()) {
154                 mCondition.await();
155             } else {
156                 element = mCbOutputQueue.remove(0);
157                 break;
158             }
159         }
160         mLock.unlock();
161         return element;
162     }
163 
getWork()164     Pair<Integer, MediaCodec.BufferInfo> getWork() throws InterruptedException {
165         Pair<Integer, MediaCodec.BufferInfo> element = null;
166         mLock.lock();
167         while (!mSignalledError) {
168             if (mCbInputQueue.isEmpty() && mCbOutputQueue.isEmpty()) {
169                 mCondition.await();
170             } else {
171                 if (!mCbOutputQueue.isEmpty()) {
172                     element = mCbOutputQueue.remove(0);
173                     break;
174                 }
175                 if (!mCbInputQueue.isEmpty()) {
176                     element = mCbInputQueue.remove(0);
177                     break;
178                 }
179             }
180         }
181         mLock.unlock();
182         return element;
183     }
184 
isInputQueueEmpty()185     boolean isInputQueueEmpty() {
186         mLock.lock();
187         boolean isEmpty = mCbInputQueue.isEmpty();
188         mLock.unlock();
189         return isEmpty;
190     }
191 
hasSeenError()192     boolean hasSeenError() {
193         return mSignalledError;
194     }
195 
hasOutputFormatChanged()196     boolean hasOutputFormatChanged() {
197         return mSignalledOutFormatChanged;
198     }
199 
getOutputFormat()200     MediaFormat getOutputFormat() {
201         return mOutFormat;
202     }
203 }
204 
205 class OutputManager {
206     private static final String LOG_TAG = OutputManager.class.getSimpleName();
207     private byte[] memory;
208     private int memIndex;
209     private ArrayList<Long> crc32List;
210     private ArrayList<Long> inpPtsList;
211     private ArrayList<Long> outPtsList;
212 
OutputManager()213     OutputManager() {
214         memory = new byte[1024];
215         memIndex = 0;
216         crc32List = new ArrayList<>();
217         inpPtsList = new ArrayList<>();
218         outPtsList = new ArrayList<>();
219     }
220 
saveInPTS(long pts)221     void saveInPTS(long pts) {
222         // Add only Unique timeStamp, discarding any duplicate frame / non-display frame
223         if (!inpPtsList.contains(pts)) {
224             inpPtsList.add(pts);
225         }
226     }
227 
saveOutPTS(long pts)228     void saveOutPTS(long pts) {
229         outPtsList.add(pts);
230     }
231 
isPtsStrictlyIncreasing(long lastPts)232     boolean isPtsStrictlyIncreasing(long lastPts) {
233         boolean res = true;
234         for (int i = 0; i < outPtsList.size(); i++) {
235             if (lastPts < outPtsList.get(i)) {
236                 lastPts = outPtsList.get(i);
237             } else {
238                 Log.e(LOG_TAG, "Timestamp ordering check failed: last timestamp: " + lastPts +
239                         " current timestamp:" + outPtsList.get(i));
240                 res = false;
241                 break;
242             }
243         }
244         return res;
245     }
246 
isOutPtsListIdenticalToInpPtsList(boolean requireSorting)247     boolean isOutPtsListIdenticalToInpPtsList(boolean requireSorting) {
248         boolean res;
249         Collections.sort(inpPtsList);
250         if (requireSorting) {
251             Collections.sort(outPtsList);
252         }
253         if (outPtsList.size() != inpPtsList.size()) {
254             Log.e(LOG_TAG, "input and output presentation timestamp list sizes are not identical" +
255                     "exp/rec" + inpPtsList.size() + '/' + outPtsList.size());
256             return false;
257         } else {
258             int count = 0;
259             for (int i = 0; i < outPtsList.size(); i++) {
260                 if (!outPtsList.get(i).equals(inpPtsList.get(i))) {
261                     count ++;
262                     Log.e(LOG_TAG, "input output pts mismatch, exp/rec " + outPtsList.get(i) + '/' +
263                             inpPtsList.get(i));
264                     if (count == 20) {
265                         Log.e(LOG_TAG, "stopping after 20 mismatches, ...");
266                         break;
267                     }
268                 }
269             }
270             res = (count == 0);
271         }
272         return res;
273     }
274 
getOutStreamSize()275     int getOutStreamSize() {
276         return memIndex;
277     }
278 
checksum(ByteBuffer buf, int size)279     void checksum(ByteBuffer buf, int size) {
280         int cap = buf.capacity();
281         assertTrue("checksum() params are invalid: size = " + size + " cap = " + cap,
282                 size > 0 && size <= cap);
283         CRC32 crc = new CRC32();
284         if (buf.hasArray()) {
285             crc.update(buf.array(), buf.position() + buf.arrayOffset(), size);
286         } else {
287             int pos = buf.position();
288             final int rdsize = Math.min(4096, size);
289             byte[] bb = new byte[rdsize];
290             int chk;
291             for (int i = 0; i < size; i += chk) {
292                 chk = Math.min(rdsize, size - i);
293                 buf.get(bb, 0, chk);
294                 crc.update(bb, 0, chk);
295             }
296             buf.position(pos);
297         }
298         crc32List.add(crc.getValue());
299     }
300 
checksum(Image image)301     void checksum(Image image) {
302         int format = image.getFormat();
303         if (format != ImageFormat.YUV_420_888) {
304             crc32List.add(-1L);
305             return;
306         }
307         CRC32 crc = new CRC32();
308         int imageWidth = image.getWidth();
309         int imageHeight = image.getHeight();
310         Image.Plane[] planes = image.getPlanes();
311         for (int i = 0; i < planes.length; ++i) {
312             ByteBuffer buf = planes[i].getBuffer();
313             int width, height, rowStride, pixelStride, x, y;
314             rowStride = planes[i].getRowStride();
315             pixelStride = planes[i].getPixelStride();
316             if (i == 0) {
317                 width = imageWidth;
318                 height = imageHeight;
319             } else {
320                 width = imageWidth / 2;
321                 height = imageHeight / 2;
322             }
323             // local contiguous pixel buffer
324             byte[] bb = new byte[width * height];
325             if (buf.hasArray()) {
326                 byte[] b = buf.array();
327                 int offs = buf.arrayOffset();
328                 if (pixelStride == 1) {
329                     for (y = 0; y < height; ++y) {
330                         System.arraycopy(bb, y * width, b, y * rowStride + offs, width);
331                     }
332                 } else {
333                     // do it pixel-by-pixel
334                     for (y = 0; y < height; ++y) {
335                         int lineOffset = offs + y * rowStride;
336                         for (x = 0; x < width; ++x) {
337                             bb[y * width + x] = b[lineOffset + x * pixelStride];
338                         }
339                     }
340                 }
341             } else { // almost always ends up here due to direct buffers
342                 int pos = buf.position();
343                 if (pixelStride == 1) {
344                     for (y = 0; y < height; ++y) {
345                         buf.position(pos + y * rowStride);
346                         buf.get(bb, y * width, width);
347                     }
348                 } else {
349                     // local line buffer
350                     byte[] lb = new byte[rowStride];
351                     // do it pixel-by-pixel
352                     for (y = 0; y < height; ++y) {
353                         buf.position(pos + y * rowStride);
354                         // we're only guaranteed to have pixelStride * (width - 1) + 1 bytes
355                         buf.get(lb, 0, pixelStride * (width - 1) + 1);
356                         for (x = 0; x < width; ++x) {
357                             bb[y * width + x] = lb[x * pixelStride];
358                         }
359                     }
360                 }
361                 buf.position(pos);
362             }
363             crc.update(bb, 0, width * height);
364         }
365         crc32List.add(crc.getValue());
366     }
367 
saveToMemory(ByteBuffer buf, MediaCodec.BufferInfo info)368     void saveToMemory(ByteBuffer buf, MediaCodec.BufferInfo info) {
369         if (memIndex + info.size >= memory.length) {
370             memory = Arrays.copyOf(memory, memIndex + info.size);
371         }
372         buf.position(info.offset);
373         buf.get(memory, memIndex, info.size);
374         memIndex += info.size;
375     }
376 
position(int index)377     void position(int index) {
378         if (index < 0 || index >= memory.length) index = 0;
379         memIndex = index;
380     }
381 
getBuffer()382     ByteBuffer getBuffer() {
383         return ByteBuffer.wrap(memory);
384     }
385 
reset()386     void reset() {
387         position(0);
388         crc32List.clear();
389         inpPtsList.clear();
390         outPtsList.clear();
391     }
392 
getRmsError(short[] refData)393     float getRmsError(short[] refData) {
394         long totalErrorSquared = 0;
395         assertTrue(0 == (memory.length & 1));
396         short[] shortData = new short[memory.length / 2];
397         ByteBuffer.wrap(memory).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer().get(shortData);
398         if (refData.length != shortData.length) return Float.MAX_VALUE;
399         for (int i = 0; i < shortData.length; i++) {
400             int d = shortData[i] - refData[i];
401             totalErrorSquared += d * d;
402         }
403         long avgErrorSquared = (totalErrorSquared / shortData.length);
404         return (float) Math.sqrt(avgErrorSquared);
405     }
406 
407     @Override
equals(Object o)408     public boolean equals(Object o) {
409         if (this == o) return true;
410         if (o == null || getClass() != o.getClass()) return false;
411         OutputManager that = (OutputManager) o;
412         boolean isEqual = true;
413         if (!crc32List.equals(that.crc32List)) {
414             isEqual = false;
415             Log.e(LOG_TAG, "ref and test crc32 checksums mismatch");
416         }
417         if (!outPtsList.equals(that.outPtsList)) {
418             isEqual = false;
419             Log.e(LOG_TAG, "ref and test presentation timestamp mismatch");
420         }
421         if (memIndex == that.memIndex) {
422             int count = 0;
423             for (int i = 0; i < memIndex; i++) {
424                 if (memory[i] != that.memory[i]) {
425                     count++;
426                     if (count < 20) {
427                         Log.d(LOG_TAG, "sample at offset " + i + " exp/got:: " + memory[i] + '/' +
428                                 that.memory[i]);
429                     }
430                 }
431             }
432             if (count != 0) {
433                 isEqual = false;
434                 Log.e(LOG_TAG, "ref and test o/p samples mismatch " + count);
435             }
436         } else {
437             isEqual = false;
438             Log.e(LOG_TAG, "ref and test o/p sizes mismatch " + memIndex + '/' + that.memIndex);
439         }
440         return isEqual;
441     }
442 }
443 
444 abstract class CodecTestBase {
445     private static final String LOG_TAG = CodecTestBase.class.getSimpleName();
446     static final String CODEC_SEL_KEY = "codec-sel";
447     static final String CODEC_SEL_VALUE = "default";
448     static final Map<String, String> codecSelKeyMimeMap = new HashMap<>();
449     static final boolean ENABLE_LOGS = false;
450     static final int PER_TEST_TIMEOUT_LARGE_TEST_MS = 300000;
451     static final int PER_TEST_TIMEOUT_SMALL_TEST_MS = 60000;
452     static final long Q_DEQ_TIMEOUT_US = 5000;
453     static final String mInpPrefix = WorkDir.getMediaDirString();
454     static final PackageManager pm =
455             InstrumentationRegistry.getInstrumentation().getContext().getPackageManager();
456     static String codecSelKeys;
457 
458     CodecAsyncHandler mAsyncHandle;
459     boolean mIsCodecInAsyncMode;
460     boolean mSawInputEOS;
461     boolean mSawOutputEOS;
462     boolean mSignalEOSWithLastFrame;
463     int mInputCount;
464     int mOutputCount;
465     long mPrevOutputPts;
466     boolean mSignalledOutFormatChanged;
467     MediaFormat mOutFormat;
468     boolean mIsAudio;
469 
470     boolean mSaveToMem;
471     OutputManager mOutputBuff;
472 
473     MediaCodec mCodec;
474     Surface mSurface;
475 
476     static {
477         System.loadLibrary("ctsmediav2codec_jni");
478 
479         codecSelKeyMimeMap.put("vp8", MediaFormat.MIMETYPE_VIDEO_VP8);
480         codecSelKeyMimeMap.put("vp9", MediaFormat.MIMETYPE_VIDEO_VP9);
481         codecSelKeyMimeMap.put("av1", MediaFormat.MIMETYPE_VIDEO_AV1);
482         codecSelKeyMimeMap.put("avc", MediaFormat.MIMETYPE_VIDEO_AVC);
483         codecSelKeyMimeMap.put("hevc", MediaFormat.MIMETYPE_VIDEO_HEVC);
484         codecSelKeyMimeMap.put("mpeg4", MediaFormat.MIMETYPE_VIDEO_MPEG4);
485         codecSelKeyMimeMap.put("h263", MediaFormat.MIMETYPE_VIDEO_H263);
486         codecSelKeyMimeMap.put("mpeg2", MediaFormat.MIMETYPE_VIDEO_MPEG2);
487         codecSelKeyMimeMap.put("vraw", MediaFormat.MIMETYPE_VIDEO_RAW);
488         codecSelKeyMimeMap.put("amrnb", MediaFormat.MIMETYPE_AUDIO_AMR_NB);
489         codecSelKeyMimeMap.put("amrwb", MediaFormat.MIMETYPE_AUDIO_AMR_WB);
490         codecSelKeyMimeMap.put("mp3", MediaFormat.MIMETYPE_AUDIO_MPEG);
491         codecSelKeyMimeMap.put("aac", MediaFormat.MIMETYPE_AUDIO_AAC);
492         codecSelKeyMimeMap.put("vorbis", MediaFormat.MIMETYPE_AUDIO_VORBIS);
493         codecSelKeyMimeMap.put("opus", MediaFormat.MIMETYPE_AUDIO_OPUS);
494         codecSelKeyMimeMap.put("g711alaw", MediaFormat.MIMETYPE_AUDIO_G711_ALAW);
495         codecSelKeyMimeMap.put("g711mlaw", MediaFormat.MIMETYPE_AUDIO_G711_MLAW);
496         codecSelKeyMimeMap.put("araw", MediaFormat.MIMETYPE_AUDIO_RAW);
497         codecSelKeyMimeMap.put("flac", MediaFormat.MIMETYPE_AUDIO_FLAC);
498         codecSelKeyMimeMap.put("gsm", MediaFormat.MIMETYPE_AUDIO_MSGSM);
499 
500         android.os.Bundle args = InstrumentationRegistry.getArguments();
501         codecSelKeys = args.getString(CODEC_SEL_KEY);
502         if (codecSelKeys == null) codecSelKeys = CODEC_SEL_VALUE;
503     }
504 
isTv()505     static boolean isTv() {
506         return pm.hasSystemFeature(PackageManager.FEATURE_LEANBACK);
507     }
508 
hasMicrophone()509     static boolean hasMicrophone() {
510         return pm.hasSystemFeature(PackageManager.FEATURE_MICROPHONE);
511     }
512 
hasCamera()513     static boolean hasCamera() {
514         return pm.hasSystemFeature(PackageManager.FEATURE_CAMERA_ANY);
515     }
516 
isWatch()517     static boolean isWatch() {
518         return pm.hasSystemFeature(PackageManager.FEATURE_WATCH);
519     }
520 
isAutomotive()521     static boolean isAutomotive() {
522         return pm.hasSystemFeature(PackageManager.FEATURE_AUTOMOTIVE);
523     }
524 
hasAudioOutput()525     static boolean hasAudioOutput() {
526         return pm.hasSystemFeature(PackageManager.FEATURE_AUDIO_OUTPUT);
527     }
528 
isHandheld()529     static boolean isHandheld() {
530         // handheld nature is not exposed to package manager, for now
531         // we check for touchscreen and NOT watch and NOT tv
532         return pm.hasSystemFeature(PackageManager.FEATURE_TOUCHSCREEN) && !isWatch() && !isTv() &&
533                 !isAutomotive();
534     }
535 
prepareParamList(ArrayList<String> cddRequiredMimeList, List<Object[]> exhaustiveArgsList, boolean isEncoder)536     static List<Object[]> prepareParamList(ArrayList<String> cddRequiredMimeList,
537             List<Object[]> exhaustiveArgsList, boolean isEncoder) {
538         ArrayList<String> mimes = new ArrayList<>();
539         if (codecSelKeys.contains(CODEC_SEL_VALUE)) {
540             MediaCodecList codecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
541             MediaCodecInfo[] codecInfos = codecList.getCodecInfos();
542             for (MediaCodecInfo codecInfo : codecInfos) {
543                 if (codecInfo.isEncoder() != isEncoder) continue;
544                 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && codecInfo.isAlias()) continue;
545                 String[] types = codecInfo.getSupportedTypes();
546                 for (String type : types) {
547                     if (!mimes.contains(type)) {
548                         mimes.add(type);
549                     }
550                 }
551             }
552             // TODO(b/154423708): add checks for video o/p port and display length >= 2.5"
553             /* sec 5.2: device implementations include an embedded screen display with the
554             diagonal length of at least 2.5inches or include a video output port or declare the
555             support of a camera */
556             if (isEncoder && hasCamera() && !mimes.contains(MediaFormat.MIMETYPE_VIDEO_AVC) &&
557                     !mimes.contains(MediaFormat.MIMETYPE_VIDEO_VP8)) {
558                 fail("device must support at least one of VP8 or AVC video encoders");
559             }
560             for (String mime : cddRequiredMimeList) {
561                 if (!mimes.contains(mime)) {
562                     fail("no codec found for mime " + mime + " as required by cdd");
563                 }
564             }
565         } else {
566             for (Map.Entry<String, String> entry : codecSelKeyMimeMap.entrySet()) {
567                 String key = entry.getKey();
568                 String value = entry.getValue();
569                 if (codecSelKeys.contains(key) && !mimes.contains(value)) mimes.add(value);
570             }
571         }
572         final List<Object[]> argsList = new ArrayList<>();
573         for (String mime : mimes) {
574             boolean miss = true;
575             for (Object[] arg : exhaustiveArgsList) {
576                 if (mime.equals(arg[0])) {
577                     argsList.add(arg);
578                     miss = false;
579                 }
580             }
581             if (miss) {
582                 if (cddRequiredMimeList.contains(mime)) {
583                     fail("no test vectors for required mimetype " + mime);
584                 }
585                 Log.w(LOG_TAG, "no test vectors available for optional mime type " + mime);
586             }
587         }
588         return argsList;
589     }
590 
enqueueInput(int bufferIndex)591     abstract void enqueueInput(int bufferIndex) throws IOException;
592 
dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info)593     abstract void dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info);
594 
configureCodec(MediaFormat format, boolean isAsync, boolean signalEOSWithLastFrame, boolean isEncoder)595     void configureCodec(MediaFormat format, boolean isAsync, boolean signalEOSWithLastFrame,
596             boolean isEncoder) {
597         resetContext(isAsync, signalEOSWithLastFrame);
598         mAsyncHandle.setCallBack(mCodec, isAsync);
599         // signalEOS flag has nothing to do with configure. We are using this flag to try all
600         // available configure apis
601         if (signalEOSWithLastFrame) {
602             mCodec.configure(format, mSurface, null,
603                     isEncoder ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0);
604         } else {
605             mCodec.configure(format, mSurface, isEncoder ? MediaCodec.CONFIGURE_FLAG_ENCODE : 0,
606                     null);
607         }
608         if (ENABLE_LOGS) {
609             Log.v(LOG_TAG, "codec configured");
610         }
611     }
612 
flushCodec()613     void flushCodec() {
614         mCodec.flush();
615         // TODO(b/147576107): is it ok to clearQueues right away or wait for some signal
616         mAsyncHandle.clearQueues();
617         mSawInputEOS = false;
618         mSawOutputEOS = false;
619         mInputCount = 0;
620         mOutputCount = 0;
621         mPrevOutputPts = Long.MIN_VALUE;
622         if (ENABLE_LOGS) {
623             Log.v(LOG_TAG, "codec flushed");
624         }
625     }
626 
reConfigureCodec(MediaFormat format, boolean isAsync, boolean signalEOSWithLastFrame, boolean isEncoder)627     void reConfigureCodec(MediaFormat format, boolean isAsync, boolean signalEOSWithLastFrame,
628             boolean isEncoder) {
629         /* TODO(b/147348711) */
630         if (false) mCodec.stop();
631         else mCodec.reset();
632         configureCodec(format, isAsync, signalEOSWithLastFrame, isEncoder);
633     }
634 
resetContext(boolean isAsync, boolean signalEOSWithLastFrame)635     void resetContext(boolean isAsync, boolean signalEOSWithLastFrame) {
636         mAsyncHandle.resetContext();
637         mIsCodecInAsyncMode = isAsync;
638         mSawInputEOS = false;
639         mSawOutputEOS = false;
640         mSignalEOSWithLastFrame = signalEOSWithLastFrame;
641         mInputCount = 0;
642         mOutputCount = 0;
643         mPrevOutputPts = Long.MIN_VALUE;
644         mSignalledOutFormatChanged = false;
645     }
646 
enqueueEOS(int bufferIndex)647     void enqueueEOS(int bufferIndex) {
648         if (!mSawInputEOS) {
649             mCodec.queueInputBuffer(bufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
650             mSawInputEOS = true;
651             if (ENABLE_LOGS) {
652                 Log.v(LOG_TAG, "Queued End of Stream");
653             }
654         }
655     }
656 
doWork(int frameLimit)657     void doWork(int frameLimit) throws InterruptedException, IOException {
658         int frameCount = 0;
659         if (mIsCodecInAsyncMode) {
660             // dequeue output after inputEOS is expected to be done in waitForAllOutputs()
661             while (!mAsyncHandle.hasSeenError() && !mSawInputEOS && frameCount < frameLimit) {
662                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getWork();
663                 if (element != null) {
664                     int bufferID = element.first;
665                     MediaCodec.BufferInfo info = element.second;
666                     if (info != null) {
667                         // <id, info> corresponds to output callback. Handle it accordingly
668                         dequeueOutput(bufferID, info);
669                     } else {
670                         // <id, null> corresponds to input callback. Handle it accordingly
671                         enqueueInput(bufferID);
672                         frameCount++;
673                     }
674                 }
675             }
676         } else {
677             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
678             // dequeue output after inputEOS is expected to be done in waitForAllOutputs()
679             while (!mSawInputEOS && frameCount < frameLimit) {
680                 int outputBufferId = mCodec.dequeueOutputBuffer(outInfo, Q_DEQ_TIMEOUT_US);
681                 if (outputBufferId >= 0) {
682                     dequeueOutput(outputBufferId, outInfo);
683                 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
684                     mOutFormat = mCodec.getOutputFormat();
685                     mSignalledOutFormatChanged = true;
686                 }
687                 int inputBufferId = mCodec.dequeueInputBuffer(Q_DEQ_TIMEOUT_US);
688                 if (inputBufferId != -1) {
689                     enqueueInput(inputBufferId);
690                     frameCount++;
691                 }
692             }
693         }
694     }
695 
queueEOS()696     void queueEOS() throws InterruptedException {
697         if (mIsCodecInAsyncMode) {
698             while (!mAsyncHandle.hasSeenError() && !mSawInputEOS) {
699                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getWork();
700                 if (element != null) {
701                     int bufferID = element.first;
702                     MediaCodec.BufferInfo info = element.second;
703                     if (info != null) {
704                         dequeueOutput(bufferID, info);
705                     } else {
706                         enqueueEOS(element.first);
707                     }
708                 }
709             }
710         } else if (!mSawInputEOS) {
711             enqueueEOS(mCodec.dequeueInputBuffer(-1));
712         }
713     }
714 
waitForAllOutputs()715     void waitForAllOutputs() throws InterruptedException {
716         if (mIsCodecInAsyncMode) {
717             while (!mAsyncHandle.hasSeenError() && !mSawOutputEOS) {
718                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getOutput();
719                 if (element != null) {
720                     dequeueOutput(element.first, element.second);
721                 }
722             }
723         } else {
724             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
725             while (!mSawOutputEOS) {
726                 int outputBufferId = mCodec.dequeueOutputBuffer(outInfo, Q_DEQ_TIMEOUT_US);
727                 if (outputBufferId >= 0) {
728                     dequeueOutput(outputBufferId, outInfo);
729                 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
730                     mOutFormat = mCodec.getOutputFormat();
731                     mSignalledOutFormatChanged = true;
732                 }
733             }
734         }
735     }
736 
selectCodecs(String mime, ArrayList<MediaFormat> formats, String[] features, boolean isEncoder)737     static ArrayList<String> selectCodecs(String mime, ArrayList<MediaFormat> formats,
738             String[] features, boolean isEncoder) {
739         MediaCodecList codecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
740         MediaCodecInfo[] codecInfos = codecList.getCodecInfos();
741         ArrayList<String> listOfCodecs = new ArrayList<>();
742         for (MediaCodecInfo codecInfo : codecInfos) {
743             if (codecInfo.isEncoder() != isEncoder) continue;
744             if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && codecInfo.isAlias()) continue;
745             String[] types = codecInfo.getSupportedTypes();
746             for (String type : types) {
747                 if (type.equalsIgnoreCase(mime)) {
748                     boolean isOk = true;
749                     MediaCodecInfo.CodecCapabilities codecCapabilities =
750                             codecInfo.getCapabilitiesForType(type);
751                     if (formats != null) {
752                         for (MediaFormat format : formats) {
753                             if (!codecCapabilities.isFormatSupported(format)) {
754                                 isOk = false;
755                                 break;
756                             }
757                         }
758                     }
759                     if (features != null) {
760                         for (String feature : features) {
761                             if (!codecCapabilities.isFeatureSupported(feature)) {
762                                 isOk = false;
763                                 break;
764                             }
765                         }
766                     }
767                     if (isOk) listOfCodecs.add(codecInfo.getName());
768                 }
769             }
770         }
771         return listOfCodecs;
772     }
773 
getWidth(MediaFormat format)774     static int getWidth(MediaFormat format) {
775         int width = format.getInteger(MediaFormat.KEY_WIDTH, -1);
776         if (format.containsKey("crop-left") && format.containsKey("crop-right")) {
777             width = format.getInteger("crop-right") + 1 - format.getInteger("crop-left");
778         }
779         return width;
780     }
781 
getHeight(MediaFormat format)782     static int getHeight(MediaFormat format) {
783         int height = format.getInteger(MediaFormat.KEY_HEIGHT, -1);
784         if (format.containsKey("crop-top") && format.containsKey("crop-bottom")) {
785             height = format.getInteger("crop-bottom") + 1 - format.getInteger("crop-top");
786         }
787         return height;
788     }
789 
isFormatSimilar(MediaFormat inpFormat, MediaFormat outFormat)790     boolean isFormatSimilar(MediaFormat inpFormat, MediaFormat outFormat) {
791         if (inpFormat == null || outFormat == null) return false;
792         String inpMime = inpFormat.getString(MediaFormat.KEY_MIME);
793         String outMime = outFormat.getString(MediaFormat.KEY_MIME);
794         // not comparing input and output mimes because for a codec, mime is raw on one side and
795         // encoded type on the other
796         if (outMime.startsWith("audio/")) {
797             return inpFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT, -1) ==
798                     outFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT, -2) &&
799                     inpFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE, -1) ==
800                             outFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE, -2) &&
801                     inpMime.startsWith("audio/");
802         } else if (outMime.startsWith("video/")) {
803             return getWidth(inpFormat) == getWidth(outFormat) &&
804                     getHeight(inpFormat) == getHeight(outFormat) && inpMime.startsWith("video/");
805         }
806         return true;
807     }
808 
validateMetrics(String codec)809     PersistableBundle validateMetrics(String codec) {
810         PersistableBundle metrics = mCodec.getMetrics();
811         assertTrue("metrics is null", metrics != null);
812         assertTrue(metrics.getString(MediaCodec.MetricsConstants.CODEC).equals(codec));
813         if (mIsAudio) {
814             assertTrue(metrics.getString(MediaCodec.MetricsConstants.MODE)
815                     .equals(MediaCodec.MetricsConstants.MODE_AUDIO));
816         } else {
817             assertTrue(metrics.getString(MediaCodec.MetricsConstants.MODE)
818                     .equals(MediaCodec.MetricsConstants.MODE_VIDEO));
819         }
820         return metrics;
821     }
822 
validateMetrics(String codec, MediaFormat format)823     PersistableBundle validateMetrics(String codec, MediaFormat format) {
824         PersistableBundle metrics = validateMetrics(codec);
825         if (!mIsAudio) {
826             assertTrue(metrics.getInt(MediaCodec.MetricsConstants.WIDTH) == getWidth(format));
827             assertTrue(metrics.getInt(MediaCodec.MetricsConstants.HEIGHT) == getHeight(format));
828         }
829         assertTrue(metrics.getInt(MediaCodec.MetricsConstants.SECURE) == 0);
830         return metrics;
831     }
832 
validateColorAspects(MediaFormat fmt, int range, int standard, int transfer)833     void validateColorAspects(MediaFormat fmt, int range, int standard, int transfer) {
834         int colorRange = fmt.getInteger(MediaFormat.KEY_COLOR_RANGE, 0);
835         int colorStandard = fmt.getInteger(MediaFormat.KEY_COLOR_STANDARD, 0);
836         int colorTransfer = fmt.getInteger(MediaFormat.KEY_COLOR_TRANSFER, 0);
837         assertEquals("range mismatch ", range, colorRange);
838         assertEquals("color mismatch ", standard, colorStandard);
839         assertEquals("transfer mismatch ", transfer, colorTransfer);
840     }
841 }
842 
843 class CodecDecoderTestBase extends CodecTestBase {
844     private static final String LOG_TAG = CodecDecoderTestBase.class.getSimpleName();
845 
846     String mMime;
847     String mTestFile;
848 
849     ArrayList<ByteBuffer> mCsdBuffers;
850     private int mCurrCsdIdx;
851 
852     MediaExtractor mExtractor;
853 
CodecDecoderTestBase(String mime, String testFile)854     CodecDecoderTestBase(String mime, String testFile) {
855         mMime = mime;
856         mTestFile = testFile;
857         mAsyncHandle = new CodecAsyncHandler();
858         mCsdBuffers = new ArrayList<>();
859         mIsAudio = mMime.startsWith("audio/");
860     }
861 
setUpSource(String srcFile)862     MediaFormat setUpSource(String srcFile) throws IOException {
863         return setUpSource(mInpPrefix, srcFile);
864     }
865 
setUpSource(String prefix, String srcFile)866     MediaFormat setUpSource(String prefix, String srcFile) throws IOException {
867         mExtractor = new MediaExtractor();
868         mExtractor.setDataSource(prefix + srcFile);
869         for (int trackID = 0; trackID < mExtractor.getTrackCount(); trackID++) {
870             MediaFormat format = mExtractor.getTrackFormat(trackID);
871             if (mMime.equalsIgnoreCase(format.getString(MediaFormat.KEY_MIME))) {
872                 mExtractor.selectTrack(trackID);
873                 if (!mIsAudio) {
874                     // COLOR_FormatYUV420Flexible by default should be supported by all components
875                     // This call shouldn't effect configure() call for any codec
876                     format.setInteger(MediaFormat.KEY_COLOR_FORMAT, COLOR_FormatYUV420Flexible);
877                 }
878                 return format;
879             }
880         }
881         fail("No track with mime: " + mMime + " found in file: " + srcFile);
882         return null;
883     }
884 
hasCSD(MediaFormat format)885     boolean hasCSD(MediaFormat format) {
886         return format.containsKey("csd-0");
887     }
888 
enqueueCodecConfig(int bufferIndex)889     void enqueueCodecConfig(int bufferIndex) {
890         ByteBuffer inputBuffer = mCodec.getInputBuffer(bufferIndex);
891         ByteBuffer csdBuffer = mCsdBuffers.get(mCurrCsdIdx);
892         inputBuffer.put((ByteBuffer) csdBuffer.rewind());
893         mCodec.queueInputBuffer(bufferIndex, 0, csdBuffer.limit(), 0,
894                 MediaCodec.BUFFER_FLAG_CODEC_CONFIG);
895         if (ENABLE_LOGS) {
896             Log.v(LOG_TAG, "queued csd: id: " + bufferIndex + " size: " + csdBuffer.limit());
897         }
898     }
899 
enqueueInput(int bufferIndex)900     void enqueueInput(int bufferIndex) {
901         if (mExtractor.getSampleSize() < 0) {
902             enqueueEOS(bufferIndex);
903         } else {
904             ByteBuffer inputBuffer = mCodec.getInputBuffer(bufferIndex);
905             mExtractor.readSampleData(inputBuffer, 0);
906             int size = (int) mExtractor.getSampleSize();
907             long pts = mExtractor.getSampleTime();
908             int extractorFlags = mExtractor.getSampleFlags();
909             int codecFlags = 0;
910             if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) {
911                 codecFlags |= MediaCodec.BUFFER_FLAG_KEY_FRAME;
912             }
913             if ((extractorFlags & MediaExtractor.SAMPLE_FLAG_PARTIAL_FRAME) != 0) {
914                 codecFlags |= MediaCodec.BUFFER_FLAG_PARTIAL_FRAME;
915             }
916             if (!mExtractor.advance() && mSignalEOSWithLastFrame) {
917                 codecFlags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM;
918                 mSawInputEOS = true;
919             }
920             if (ENABLE_LOGS) {
921                 Log.v(LOG_TAG, "input: id: " + bufferIndex + " size: " + size + " pts: " + pts +
922                         " flags: " + codecFlags);
923             }
924             mCodec.queueInputBuffer(bufferIndex, 0, size, pts, codecFlags);
925             if (size > 0 && (codecFlags & (MediaCodec.BUFFER_FLAG_CODEC_CONFIG |
926                     MediaCodec.BUFFER_FLAG_PARTIAL_FRAME)) == 0) {
927                 mOutputBuff.saveInPTS(pts);
928                 mInputCount++;
929             }
930         }
931     }
932 
enqueueInput(int bufferIndex, ByteBuffer buffer, MediaCodec.BufferInfo info)933     void enqueueInput(int bufferIndex, ByteBuffer buffer, MediaCodec.BufferInfo info) {
934         ByteBuffer inputBuffer = mCodec.getInputBuffer(bufferIndex);
935         buffer.position(info.offset);
936         for (int i = 0; i < info.size; i++) {
937             inputBuffer.put(buffer.get());
938         }
939         if (ENABLE_LOGS) {
940             Log.v(LOG_TAG, "input: id: " + bufferIndex + " flags: " + info.flags + " size: " +
941                     info.size + " timestamp: " + info.presentationTimeUs);
942         }
943         mCodec.queueInputBuffer(bufferIndex, 0, info.size, info.presentationTimeUs,
944                 info.flags);
945         if (info.size > 0 && ((info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) &&
946                 ((info.flags & MediaCodec.BUFFER_FLAG_PARTIAL_FRAME) == 0)) {
947             mOutputBuff.saveInPTS(info.presentationTimeUs);
948             mInputCount++;
949         }
950     }
951 
dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info)952     void dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info) {
953         if (info.size > 0 && mSaveToMem) {
954             if (mIsAudio) {
955                 ByteBuffer buf = mCodec.getOutputBuffer(bufferIndex);
956                 mOutputBuff.saveToMemory(buf, info);
957             } else {
958                 // tests both getOutputImage and getOutputBuffer. Can do time division
959                 // multiplexing but lets allow it for now
960                 Image img = mCodec.getOutputImage(bufferIndex);
961                 assertTrue(img != null);
962                 mOutputBuff.checksum(img);
963 
964                 ByteBuffer buf = mCodec.getOutputBuffer(bufferIndex);
965                 mOutputBuff.checksum(buf, info.size);
966             }
967         }
968         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
969             mSawOutputEOS = true;
970         }
971         if (ENABLE_LOGS) {
972             Log.v(LOG_TAG, "output: id: " + bufferIndex + " flags: " + info.flags + " size: " +
973                     info.size + " timestamp: " + info.presentationTimeUs);
974         }
975         if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
976             mOutputBuff.saveOutPTS(info.presentationTimeUs);
977             mOutputCount++;
978         }
979         mCodec.releaseOutputBuffer(bufferIndex, false);
980     }
981 
doWork(ByteBuffer buffer, ArrayList<MediaCodec.BufferInfo> list)982     void doWork(ByteBuffer buffer, ArrayList<MediaCodec.BufferInfo> list)
983             throws InterruptedException {
984         int frameCount = 0;
985         if (mIsCodecInAsyncMode) {
986             // output processing after queuing EOS is done in waitForAllOutputs()
987             while (!mAsyncHandle.hasSeenError() && !mSawInputEOS && frameCount < list.size()) {
988                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getWork();
989                 if (element != null) {
990                     int bufferID = element.first;
991                     MediaCodec.BufferInfo info = element.second;
992                     if (info != null) {
993                         dequeueOutput(bufferID, info);
994                     } else {
995                         enqueueInput(bufferID, buffer, list.get(frameCount));
996                         frameCount++;
997                     }
998                 }
999             }
1000         } else {
1001             MediaCodec.BufferInfo outInfo = new MediaCodec.BufferInfo();
1002             // output processing after queuing EOS is done in waitForAllOutputs()
1003             while (!mSawInputEOS && frameCount < list.size()) {
1004                 int outputBufferId = mCodec.dequeueOutputBuffer(outInfo, Q_DEQ_TIMEOUT_US);
1005                 if (outputBufferId >= 0) {
1006                     dequeueOutput(outputBufferId, outInfo);
1007                 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
1008                     mOutFormat = mCodec.getOutputFormat();
1009                     mSignalledOutFormatChanged = true;
1010                 }
1011                 int inputBufferId = mCodec.dequeueInputBuffer(Q_DEQ_TIMEOUT_US);
1012                 if (inputBufferId != -1) {
1013                     enqueueInput(inputBufferId, buffer, list.get(frameCount));
1014                     frameCount++;
1015                 }
1016             }
1017         }
1018     }
1019 
queueCodecConfig()1020     void queueCodecConfig() throws InterruptedException {
1021         if (mIsCodecInAsyncMode) {
1022             for (mCurrCsdIdx = 0; !mAsyncHandle.hasSeenError() && mCurrCsdIdx < mCsdBuffers.size();
1023                  mCurrCsdIdx++) {
1024                 Pair<Integer, MediaCodec.BufferInfo> element = mAsyncHandle.getInput();
1025                 if (element != null) {
1026                     enqueueCodecConfig(element.first);
1027                 }
1028             }
1029         } else {
1030             for (mCurrCsdIdx = 0; mCurrCsdIdx < mCsdBuffers.size(); mCurrCsdIdx++) {
1031                 enqueueCodecConfig(mCodec.dequeueInputBuffer(-1));
1032             }
1033         }
1034     }
1035 
decodeToMemory(String file, String decoder, long pts, int mode, int frameLimit)1036     void decodeToMemory(String file, String decoder, long pts, int mode, int frameLimit)
1037             throws IOException, InterruptedException {
1038         mSaveToMem = true;
1039         mOutputBuff = new OutputManager();
1040         mCodec = MediaCodec.createByCodecName(decoder);
1041         MediaFormat format = setUpSource(file);
1042         configureCodec(format, false, true, false);
1043         mCodec.start();
1044         mExtractor.seekTo(pts, mode);
1045         doWork(frameLimit);
1046         queueEOS();
1047         waitForAllOutputs();
1048         mCodec.stop();
1049         mCodec.release();
1050         mExtractor.release();
1051         mSaveToMem = false;
1052     }
1053 
1054     @Override
validateMetrics(String decoder, MediaFormat format)1055     PersistableBundle validateMetrics(String decoder, MediaFormat format) {
1056         PersistableBundle metrics = super.validateMetrics(decoder, format);
1057         assertTrue(metrics.getString(MediaCodec.MetricsConstants.MIME_TYPE).equals(mMime));
1058         assertTrue(metrics.getInt(MediaCodec.MetricsConstants.ENCODER) == 0);
1059         return metrics;
1060     }
1061 
validateColorAspects(String decoder, String parent, String name, int range, int standard, int transfer)1062     void validateColorAspects(String decoder, String parent, String name, int range, int standard,
1063             int transfer) throws IOException, InterruptedException {
1064         mOutputBuff = new OutputManager();
1065         MediaFormat format = setUpSource(parent, name);
1066         if (decoder == null) {
1067             MediaCodecList codecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
1068             decoder = codecList.findDecoderForFormat(format);
1069         }
1070         mCodec = MediaCodec.createByCodecName(decoder);
1071         configureCodec(format, true, true, false);
1072         mCodec.start();
1073         doWork(1);
1074         queueEOS();
1075         waitForAllOutputs();
1076         validateColorAspects(mCodec.getOutputFormat(), range, standard, transfer);
1077         mCodec.stop();
1078         mCodec.release();
1079         mExtractor.release();
1080     }
1081 
validateColorAspects(String decoder, MediaFormat format, ByteBuffer buffer, ArrayList<MediaCodec.BufferInfo> infos, int range, int standard, int transfer)1082     void validateColorAspects(String decoder, MediaFormat format, ByteBuffer buffer,
1083             ArrayList<MediaCodec.BufferInfo> infos, int range, int standard, int transfer)
1084             throws IOException, InterruptedException {
1085         mOutputBuff = new OutputManager();
1086         if (decoder == null) {
1087             MediaCodecList codecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
1088             decoder = codecList.findDecoderForFormat(format);
1089         }
1090         mCodec = MediaCodec.createByCodecName(decoder);
1091         configureCodec(format, true, true, false);
1092         mCodec.start();
1093         doWork(buffer, infos);
1094         queueEOS();
1095         waitForAllOutputs();
1096         validateColorAspects(mCodec.getOutputFormat(), range, standard, transfer);
1097         mCodec.stop();
1098         mCodec.release();
1099     }
1100 }
1101 
1102 class CodecEncoderTestBase extends CodecTestBase {
1103     private static final String LOG_TAG = CodecEncoderTestBase.class.getSimpleName();
1104 
1105     // files are in WorkDir.getMediaDirString();
1106     private static final String mInputAudioFile = "bbb_2ch_44kHz_s16le.raw";
1107     private static final String mInputVideoFile = "bbb_cif_yuv420p_30fps.yuv";
1108     private final int INP_FRM_WIDTH = 352;
1109     private final int INP_FRM_HEIGHT = 288;
1110 
1111     final String mMime;
1112     final String mInputFile;
1113     byte[] mInputData;
1114     int mNumBytesSubmitted;
1115     long mInputOffsetPts;
1116 
1117     int mWidth, mHeight;
1118     int mFrameRate;
1119     int mMaxBFrames;
1120     int mChannels;
1121     int mSampleRate;
1122 
CodecEncoderTestBase(String mime)1123     CodecEncoderTestBase(String mime) {
1124         mMime = mime;
1125         mWidth = INP_FRM_WIDTH;
1126         mHeight = INP_FRM_HEIGHT;
1127         mChannels = 1;
1128         mSampleRate = 8000;
1129         mFrameRate = 30;
1130         mMaxBFrames = 0;
1131         if (mime.equals(MediaFormat.MIMETYPE_VIDEO_MPEG4)) mFrameRate = 12;
1132         else if (mime.equals(MediaFormat.MIMETYPE_VIDEO_H263)) mFrameRate = 12;
1133         mAsyncHandle = new CodecAsyncHandler();
1134         mIsAudio = mMime.startsWith("audio/");
1135         mInputFile = mIsAudio ? mInputAudioFile : mInputVideoFile;
1136     }
1137 
1138     @Override
resetContext(boolean isAsync, boolean signalEOSWithLastFrame)1139     void resetContext(boolean isAsync, boolean signalEOSWithLastFrame) {
1140         super.resetContext(isAsync, signalEOSWithLastFrame);
1141         mNumBytesSubmitted = 0;
1142         mInputOffsetPts = 0;
1143     }
1144 
1145     @Override
flushCodec()1146     void flushCodec() {
1147         super.flushCodec();
1148         if (mIsAudio) {
1149             mInputOffsetPts =
1150                     (mNumBytesSubmitted + 1024) * 1000000L / (2 * mChannels * mSampleRate);
1151         } else {
1152             mInputOffsetPts = (mInputCount + 5) * 1000000L / mFrameRate;
1153         }
1154         mPrevOutputPts = mInputOffsetPts - 1;
1155         mNumBytesSubmitted = 0;
1156     }
1157 
setUpSource(String srcFile)1158     void setUpSource(String srcFile) throws IOException {
1159         String inpPath = mInpPrefix + srcFile;
1160         try (FileInputStream fInp = new FileInputStream(inpPath)) {
1161             int size = (int) new File(inpPath).length();
1162             mInputData = new byte[size];
1163             fInp.read(mInputData, 0, size);
1164         }
1165     }
1166 
fillImage(Image image)1167     void fillImage(Image image) {
1168         Assert.assertTrue(image.getFormat() == ImageFormat.YUV_420_888);
1169         int imageWidth = image.getWidth();
1170         int imageHeight = image.getHeight();
1171         Image.Plane[] planes = image.getPlanes();
1172         int offset = mNumBytesSubmitted;
1173         for (int i = 0; i < planes.length; ++i) {
1174             ByteBuffer buf = planes[i].getBuffer();
1175             int width = imageWidth;
1176             int height = imageHeight;
1177             int tileWidth = INP_FRM_WIDTH;
1178             int tileHeight = INP_FRM_HEIGHT;
1179             int rowStride = planes[i].getRowStride();
1180             int pixelStride = planes[i].getPixelStride();
1181             if (i != 0) {
1182                 width = imageWidth / 2;
1183                 height = imageHeight / 2;
1184                 tileWidth = INP_FRM_WIDTH / 2;
1185                 tileHeight = INP_FRM_HEIGHT / 2;
1186             }
1187             if (pixelStride == 1) {
1188                 if (width == rowStride && width == tileWidth && height == tileHeight) {
1189                     buf.put(mInputData, offset, width * height);
1190                 } else {
1191                     for (int z = 0; z < height; z += tileHeight) {
1192                         int rowsToCopy = Math.min(height - z, tileHeight);
1193                         for (int y = 0; y < rowsToCopy; y++) {
1194                             for (int x = 0; x < width; x += tileWidth) {
1195                                 int colsToCopy = Math.min(width - x, tileWidth);
1196                                 buf.position((z + y) * rowStride + x);
1197                                 buf.put(mInputData, offset + y * tileWidth, colsToCopy);
1198                             }
1199                         }
1200                     }
1201                 }
1202             } else {
1203                 // do it pixel-by-pixel
1204                 for (int z = 0; z < height; z += tileHeight) {
1205                     int rowsToCopy = Math.min(height - z, tileHeight);
1206                     for (int y = 0; y < rowsToCopy; y++) {
1207                         int lineOffset = (z + y) * rowStride;
1208                         for (int x = 0; x < width; x += tileWidth) {
1209                             int colsToCopy = Math.min(width - x, tileWidth);
1210                             for (int w = 0; w < colsToCopy; w++) {
1211                                 buf.position(lineOffset + (x + w) * pixelStride);
1212                                 buf.put(mInputData[offset + y * tileWidth + w]);
1213                             }
1214                         }
1215                     }
1216                 }
1217             }
1218             offset += tileWidth * tileHeight;
1219         }
1220     }
1221 
fillByteBuffer(ByteBuffer inputBuffer)1222     void fillByteBuffer(ByteBuffer inputBuffer) {
1223         int offset = 0, frmOffset = mNumBytesSubmitted;
1224         for (int plane = 0; plane < 3; plane++) {
1225             int width = mWidth;
1226             int height = mHeight;
1227             int tileWidth = INP_FRM_WIDTH;
1228             int tileHeight = INP_FRM_HEIGHT;
1229             if (plane != 0) {
1230                 width = mWidth / 2;
1231                 height = mHeight / 2;
1232                 tileWidth = INP_FRM_WIDTH / 2;
1233                 tileHeight = INP_FRM_HEIGHT / 2;
1234             }
1235             for (int k = 0; k < height; k += tileHeight) {
1236                 int rowsToCopy = Math.min(height - k, tileHeight);
1237                 for (int j = 0; j < rowsToCopy; j++) {
1238                     for (int i = 0; i < width; i += tileWidth) {
1239                         int colsToCopy = Math.min(width - i, tileWidth);
1240                         inputBuffer.position(offset + (k + j) * width + i);
1241                         inputBuffer.put(mInputData, frmOffset + j * tileWidth, colsToCopy);
1242                     }
1243                 }
1244             }
1245             offset += width * height;
1246             frmOffset += tileWidth * tileHeight;
1247         }
1248     }
1249 
enqueueInput(int bufferIndex)1250     void enqueueInput(int bufferIndex) {
1251         ByteBuffer inputBuffer = mCodec.getInputBuffer(bufferIndex);
1252         if (mNumBytesSubmitted >= mInputData.length) {
1253             enqueueEOS(bufferIndex);
1254         } else {
1255             int size;
1256             int flags = 0;
1257             long pts = mInputOffsetPts;
1258             if (mIsAudio) {
1259                 pts += mNumBytesSubmitted * 1000000L / (2 * mChannels * mSampleRate);
1260                 size = Math.min(inputBuffer.capacity(), mInputData.length - mNumBytesSubmitted);
1261                 inputBuffer.put(mInputData, mNumBytesSubmitted, size);
1262                 if (mNumBytesSubmitted + size >= mInputData.length && mSignalEOSWithLastFrame) {
1263                     flags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM;
1264                     mSawInputEOS = true;
1265                 }
1266                 mNumBytesSubmitted += size;
1267             } else {
1268                 pts += mInputCount * 1000000L / mFrameRate;
1269                 size = mWidth * mHeight * 3 / 2;
1270                 int frmSize = INP_FRM_WIDTH * INP_FRM_HEIGHT * 3 / 2;
1271                 if (mNumBytesSubmitted + frmSize > mInputData.length) {
1272                     fail("received partial frame to encode");
1273                 } else {
1274                     Image img = mCodec.getInputImage(bufferIndex);
1275                     if (img != null) {
1276                         fillImage(img);
1277                     } else {
1278                         if (mWidth == INP_FRM_WIDTH && mHeight == INP_FRM_HEIGHT) {
1279                             inputBuffer.put(mInputData, mNumBytesSubmitted, size);
1280                         } else {
1281                             fillByteBuffer(inputBuffer);
1282                         }
1283                     }
1284                 }
1285                 if (mNumBytesSubmitted + frmSize >= mInputData.length && mSignalEOSWithLastFrame) {
1286                     flags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM;
1287                     mSawInputEOS = true;
1288                 }
1289                 mNumBytesSubmitted += frmSize;
1290             }
1291             if (ENABLE_LOGS) {
1292                 Log.v(LOG_TAG, "input: id: " + bufferIndex + " size: " + size + " pts: " + pts +
1293                         " flags: " + flags);
1294             }
1295             mCodec.queueInputBuffer(bufferIndex, 0, size, pts, flags);
1296             mOutputBuff.saveInPTS(pts);
1297             mInputCount++;
1298         }
1299     }
1300 
dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info)1301     void dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info) {
1302         if (ENABLE_LOGS) {
1303             Log.v(LOG_TAG, "output: id: " + bufferIndex + " flags: " + info.flags + " size: " +
1304                     info.size + " timestamp: " + info.presentationTimeUs);
1305         }
1306         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
1307             mSawOutputEOS = true;
1308         }
1309         if (info.size > 0) {
1310             if (mSaveToMem) {
1311                 ByteBuffer buf = mCodec.getOutputBuffer(bufferIndex);
1312                 mOutputBuff.saveToMemory(buf, info);
1313             }
1314             if ((info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
1315                 mOutputBuff.saveOutPTS(info.presentationTimeUs);
1316                 mOutputCount++;
1317             }
1318         }
1319         mCodec.releaseOutputBuffer(bufferIndex, false);
1320     }
1321 
1322     @Override
validateMetrics(String codec, MediaFormat format)1323     PersistableBundle validateMetrics(String codec, MediaFormat format) {
1324         PersistableBundle metrics = super.validateMetrics(codec, format);
1325         assertTrue(metrics.getString(MediaCodec.MetricsConstants.MIME_TYPE).equals(mMime));
1326         assertTrue(metrics.getInt(MediaCodec.MetricsConstants.ENCODER) == 1);
1327         return metrics;
1328     }
1329 }
1330