1 /*
2  * Copyright 2012, The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef MEDIA_CODEC_H_
18 
19 #define MEDIA_CODEC_H_
20 
21 #include <memory>
22 #include <vector>
23 
24 #include <gui/IGraphicBufferProducer.h>
25 #include <media/hardware/CryptoAPI.h>
26 #include <media/MediaCodecInfo.h>
27 #include <media/MediaResource.h>
28 #include <media/MediaMetrics.h>
29 #include <media/stagefright/foundation/AHandler.h>
30 #include <media/stagefright/FrameRenderTracker.h>
31 #include <utils/Vector.h>
32 
33 namespace android {
34 
35 struct ABuffer;
36 struct AMessage;
37 struct AReplyToken;
38 struct AString;
39 struct BatteryChecker;
40 class BufferChannelBase;
41 struct CodecBase;
42 class IBatteryStats;
43 struct ICrypto;
44 class MediaCodecBuffer;
45 class IMemory;
46 class IResourceManagerClient;
47 class IResourceManagerService;
48 struct PersistentSurface;
49 class SoftwareRenderer;
50 class Surface;
51 namespace hardware {
52 namespace cas {
53 namespace native {
54 namespace V1_0 {
55 struct IDescrambler;
56 }}}}
57 using hardware::cas::native::V1_0::IDescrambler;
58 
59 struct MediaCodec : public AHandler {
60     enum ConfigureFlags {
61         CONFIGURE_FLAG_ENCODE   = 1,
62     };
63 
64     enum BufferFlags {
65         BUFFER_FLAG_SYNCFRAME     = 1,
66         BUFFER_FLAG_CODECCONFIG   = 2,
67         BUFFER_FLAG_EOS           = 4,
68         BUFFER_FLAG_PARTIAL_FRAME = 8,
69         BUFFER_FLAG_MUXER_DATA    = 16,
70     };
71 
72     enum {
73         CB_INPUT_AVAILABLE = 1,
74         CB_OUTPUT_AVAILABLE = 2,
75         CB_ERROR = 3,
76         CB_OUTPUT_FORMAT_CHANGED = 4,
77         CB_RESOURCE_RECLAIMED = 5,
78     };
79 
80     static const pid_t kNoPid = -1;
81     static const uid_t kNoUid = -1;
82 
83     static sp<MediaCodec> CreateByType(
84             const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err = NULL,
85             pid_t pid = kNoPid, uid_t uid = kNoUid);
86 
87     static sp<MediaCodec> CreateByComponentName(
88             const sp<ALooper> &looper, const AString &name, status_t *err = NULL,
89             pid_t pid = kNoPid, uid_t uid = kNoUid);
90 
91     static sp<PersistentSurface> CreatePersistentInputSurface();
92 
93     status_t configure(
94             const sp<AMessage> &format,
95             const sp<Surface> &nativeWindow,
96             const sp<ICrypto> &crypto,
97             uint32_t flags);
98 
99     status_t configure(
100             const sp<AMessage> &format,
101             const sp<Surface> &nativeWindow,
102             const sp<ICrypto> &crypto,
103             const sp<IDescrambler> &descrambler,
104             uint32_t flags);
105 
106     status_t releaseCrypto();
107 
108     status_t setCallback(const sp<AMessage> &callback);
109 
110     status_t setOnFrameRenderedNotification(const sp<AMessage> &notify);
111 
112     status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer);
113 
114     status_t setInputSurface(const sp<PersistentSurface> &surface);
115 
116     status_t start();
117 
118     // Returns to a state in which the component remains allocated but
119     // unconfigured.
120     status_t stop();
121 
122     // Resets the codec to the INITIALIZED state.  Can be called after an error
123     // has occured to make the codec usable.
124     status_t reset();
125 
126     // Client MUST call release before releasing final reference to this
127     // object.
128     status_t release();
129 
130     status_t flush();
131 
132     status_t queueInputBuffer(
133             size_t index,
134             size_t offset,
135             size_t size,
136             int64_t presentationTimeUs,
137             uint32_t flags,
138             AString *errorDetailMsg = NULL);
139 
140     status_t queueSecureInputBuffer(
141             size_t index,
142             size_t offset,
143             const CryptoPlugin::SubSample *subSamples,
144             size_t numSubSamples,
145             const uint8_t key[16],
146             const uint8_t iv[16],
147             CryptoPlugin::Mode mode,
148             const CryptoPlugin::Pattern &pattern,
149             int64_t presentationTimeUs,
150             uint32_t flags,
151             AString *errorDetailMsg = NULL);
152 
153     status_t dequeueInputBuffer(size_t *index, int64_t timeoutUs = 0ll);
154 
155     status_t dequeueOutputBuffer(
156             size_t *index,
157             size_t *offset,
158             size_t *size,
159             int64_t *presentationTimeUs,
160             uint32_t *flags,
161             int64_t timeoutUs = 0ll);
162 
163     status_t renderOutputBufferAndRelease(size_t index, int64_t timestampNs);
164     status_t renderOutputBufferAndRelease(size_t index);
165     status_t releaseOutputBuffer(size_t index);
166 
167     status_t signalEndOfInputStream();
168 
169     status_t getOutputFormat(sp<AMessage> *format) const;
170     status_t getInputFormat(sp<AMessage> *format) const;
171 
172     status_t getInputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const;
173     status_t getOutputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const;
174 
175     status_t getOutputBuffer(size_t index, sp<MediaCodecBuffer> *buffer);
176     status_t getOutputFormat(size_t index, sp<AMessage> *format);
177     status_t getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer);
178 
179     status_t setSurface(const sp<Surface> &nativeWindow);
180 
181     status_t requestIDRFrame();
182 
183     // Notification will be posted once there "is something to do", i.e.
184     // an input/output buffer has become available, a format change is
185     // pending, an error is pending.
186     void requestActivityNotification(const sp<AMessage> &notify);
187 
188     status_t getName(AString *componentName) const;
189 
190     status_t getCodecInfo(sp<MediaCodecInfo> *codecInfo) const;
191 
192     status_t getMetrics(mediametrics_handle_t &reply);
193 
194     status_t setParameters(const sp<AMessage> &params);
195 
196     // Create a MediaCodec notification message from a list of rendered or dropped render infos
197     // by adding rendered frame information to a base notification message. Returns the number
198     // of frames that were rendered.
199     static size_t CreateFramesRenderedMessage(
200             const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg);
201 
202 protected:
203     virtual ~MediaCodec();
204     virtual void onMessageReceived(const sp<AMessage> &msg);
205 
206 private:
207     // used by ResourceManagerClient
208     status_t reclaim(bool force = false);
209     friend struct ResourceManagerClient;
210 
211 private:
212     enum State {
213         UNINITIALIZED,
214         INITIALIZING,
215         INITIALIZED,
216         CONFIGURING,
217         CONFIGURED,
218         STARTING,
219         STARTED,
220         FLUSHING,
221         FLUSHED,
222         STOPPING,
223         RELEASING,
224     };
225     std::string stateString(State state);
226 
227     enum {
228         kPortIndexInput         = 0,
229         kPortIndexOutput        = 1,
230     };
231 
232     enum {
233         kWhatInit                           = 'init',
234         kWhatConfigure                      = 'conf',
235         kWhatSetSurface                     = 'sSur',
236         kWhatCreateInputSurface             = 'cisf',
237         kWhatSetInputSurface                = 'sisf',
238         kWhatStart                          = 'strt',
239         kWhatStop                           = 'stop',
240         kWhatRelease                        = 'rele',
241         kWhatDequeueInputBuffer             = 'deqI',
242         kWhatQueueInputBuffer               = 'queI',
243         kWhatDequeueOutputBuffer            = 'deqO',
244         kWhatReleaseOutputBuffer            = 'relO',
245         kWhatSignalEndOfInputStream         = 'eois',
246         kWhatGetBuffers                     = 'getB',
247         kWhatFlush                          = 'flus',
248         kWhatGetOutputFormat                = 'getO',
249         kWhatGetInputFormat                 = 'getI',
250         kWhatDequeueInputTimedOut           = 'dITO',
251         kWhatDequeueOutputTimedOut          = 'dOTO',
252         kWhatCodecNotify                    = 'codc',
253         kWhatRequestIDRFrame                = 'ridr',
254         kWhatRequestActivityNotification    = 'racN',
255         kWhatGetName                        = 'getN',
256         kWhatGetCodecInfo                   = 'gCoI',
257         kWhatSetParameters                  = 'setP',
258         kWhatSetCallback                    = 'setC',
259         kWhatSetNotification                = 'setN',
260         kWhatDrmReleaseCrypto               = 'rDrm',
261         kWhatCheckBatteryStats              = 'chkB',
262     };
263 
264     enum {
265         kFlagUsesSoftwareRenderer       = 1,
266         kFlagOutputFormatChanged        = 2,
267         kFlagOutputBuffersChanged       = 4,
268         kFlagStickyError                = 8,
269         kFlagDequeueInputPending        = 16,
270         kFlagDequeueOutputPending       = 32,
271         kFlagIsSecure                   = 64,
272         kFlagSawMediaServerDie          = 128,
273         kFlagIsEncoder                  = 256,
274         // 512 skipped
275         kFlagIsAsync                    = 1024,
276         kFlagIsComponentAllocated       = 2048,
277         kFlagPushBlankBuffersOnShutdown = 4096,
278     };
279 
280     struct BufferInfo {
281         BufferInfo();
282 
283         sp<MediaCodecBuffer> mData;
284         bool mOwnedByClient;
285     };
286 
287     struct ResourceManagerServiceProxy : public IBinder::DeathRecipient {
288         ResourceManagerServiceProxy(pid_t pid, uid_t uid);
289         ~ResourceManagerServiceProxy();
290 
291         void init();
292 
293         // implements DeathRecipient
294         virtual void binderDied(const wp<IBinder>& /*who*/);
295 
296         void addResource(
297                 int64_t clientId,
298                 const sp<IResourceManagerClient> &client,
299                 const Vector<MediaResource> &resources);
300 
301         void removeResource(
302                 int64_t clientId,
303                 const Vector<MediaResource> &resources);
304 
305         void removeClient(int64_t clientId);
306 
307         bool reclaimResource(const Vector<MediaResource> &resources);
308 
309     private:
310         Mutex mLock;
311         sp<IResourceManagerService> mService;
312         pid_t mPid;
313         uid_t mUid;
314     };
315 
316     State mState;
317     uid_t mUid;
318     bool mReleasedByResourceManager;
319     sp<ALooper> mLooper;
320     sp<ALooper> mCodecLooper;
321     sp<CodecBase> mCodec;
322     AString mComponentName;
323     AString mOwnerName;
324     sp<MediaCodecInfo> mCodecInfo;
325     sp<AReplyToken> mReplyID;
326     uint32_t mFlags;
327     status_t mStickyError;
328     sp<Surface> mSurface;
329     SoftwareRenderer *mSoftRenderer;
330 
331     mediametrics_handle_t mMetricsHandle;
332     void initMediametrics();
333     void updateMediametrics();
334     void flushMediametrics();
335     void updateEphemeralMediametrics(mediametrics_handle_t item);
336 
337     sp<AMessage> mOutputFormat;
338     sp<AMessage> mInputFormat;
339     sp<AMessage> mCallback;
340     sp<AMessage> mOnFrameRenderedNotification;
341 
342     sp<IResourceManagerClient> mResourceManagerClient;
343     sp<ResourceManagerServiceProxy> mResourceManagerService;
344 
345     bool mIsVideo;
346     int32_t mVideoWidth;
347     int32_t mVideoHeight;
348     int32_t mRotationDegrees;
349     int32_t mAllowFrameDroppingBySurface;
350 
351     // initial create parameters
352     AString mInitName;
353 
354     // configure parameter
355     sp<AMessage> mConfigureMsg;
356 
357     // Used only to synchronize asynchronous getBufferAndFormat
358     // across all the other (synchronous) buffer state change
359     // operations, such as de/queueIn/OutputBuffer, start and
360     // stop/flush/reset/release.
361     Mutex mBufferLock;
362 
363     List<size_t> mAvailPortBuffers[2];
364     std::vector<BufferInfo> mPortBuffers[2];
365 
366     int32_t mDequeueInputTimeoutGeneration;
367     sp<AReplyToken> mDequeueInputReplyID;
368 
369     int32_t mDequeueOutputTimeoutGeneration;
370     sp<AReplyToken> mDequeueOutputReplyID;
371 
372     sp<ICrypto> mCrypto;
373 
374     sp<IDescrambler> mDescrambler;
375 
376     List<sp<ABuffer> > mCSD;
377 
378     sp<AMessage> mActivityNotify;
379 
380     bool mHaveInputSurface;
381     bool mHavePendingInputBuffers;
382     bool mCpuBoostRequested;
383 
384     std::shared_ptr<BufferChannelBase> mBufferChannel;
385 
386     MediaCodec(const sp<ALooper> &looper, pid_t pid, uid_t uid);
387 
388     static sp<CodecBase> GetCodecBase(const AString &name, const char *owner = nullptr);
389 
390     static status_t PostAndAwaitResponse(
391             const sp<AMessage> &msg, sp<AMessage> *response);
392 
393     void PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err);
394 
395     status_t init(const AString &name);
396 
397     void setState(State newState);
398     void returnBuffersToCodec(bool isReclaim = false);
399     void returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim = false);
400     size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg);
401     status_t onQueueInputBuffer(const sp<AMessage> &msg);
402     status_t onReleaseOutputBuffer(const sp<AMessage> &msg);
403     ssize_t dequeuePortBuffer(int32_t portIndex);
404 
405     status_t getBufferAndFormat(
406             size_t portIndex, size_t index,
407             sp<MediaCodecBuffer> *buffer, sp<AMessage> *format);
408 
409     bool handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
410     bool handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest = false);
411     void cancelPendingDequeueOperations();
412 
413     void extractCSD(const sp<AMessage> &format);
414     status_t queueCSDInputBuffer(size_t bufferIndex);
415 
416     status_t handleSetSurface(const sp<Surface> &surface);
417     status_t connectToSurface(const sp<Surface> &surface);
418     status_t disconnectFromSurface();
419 
hasCryptoOrDescramblerMediaCodec420     bool hasCryptoOrDescrambler() {
421         return mCrypto != NULL || mDescrambler != NULL;
422     }
423 
424     void postActivityNotificationIfPossible();
425 
426     void onInputBufferAvailable();
427     void onOutputBufferAvailable();
428     void onError(status_t err, int32_t actionCode, const char *detail = NULL);
429     void onOutputFormatChanged();
430 
431     status_t onSetParameters(const sp<AMessage> &params);
432 
433     status_t amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> &buffer);
434     bool isExecuting() const;
435 
436     uint64_t getGraphicBufferSize();
437     void addResource(MediaResource::Type type, MediaResource::SubType subtype, uint64_t value);
438     void removeResource(MediaResource::Type type, MediaResource::SubType subtype, uint64_t value);
439     void requestCpuBoostIfNeeded();
440 
441     bool hasPendingBuffer(int portIndex);
442     bool hasPendingBuffer();
443 
444     /* called to get the last codec error when the sticky flag is set.
445      * if no such codec error is found, returns UNKNOWN_ERROR.
446      */
getStickyErrorMediaCodec447     inline status_t getStickyError() const {
448         return mStickyError != 0 ? mStickyError : UNKNOWN_ERROR;
449     }
450 
setStickyErrorMediaCodec451     inline void setStickyError(status_t err) {
452         mFlags |= kFlagStickyError;
453         mStickyError = err;
454     }
455 
456     void onReleaseCrypto(const sp<AMessage>& msg);
457 
458     // managing time-of-flight aka latency
459     typedef struct {
460             int64_t presentationUs;
461             int64_t startedNs;
462     } BufferFlightTiming_t;
463     std::deque<BufferFlightTiming_t> mBuffersInFlight;
464     Mutex mLatencyLock;
465     int64_t mLatencyUnknown;    // buffers for which we couldn't calculate latency
466 
467     sp<BatteryChecker> mBatteryChecker;
468 
469     void statsBufferSent(int64_t presentationUs);
470     void statsBufferReceived(int64_t presentationUs);
471 
472     enum {
473         // the default shape of our latency histogram buckets
474         // XXX: should these be configurable in some way?
475         kLatencyHistBuckets = 20,
476         kLatencyHistWidth = 2000,
477         kLatencyHistFloor = 2000,
478 
479         // how many samples are in the 'recent latency' histogram
480         // 300 frames = 5 sec @ 60fps or ~12 sec @ 24fps
481         kRecentLatencyFrames = 300,
482 
483         // how we initialize mRecentSamples
484         kRecentSampleInvalid = -1,
485     };
486 
487     int64_t mRecentSamples[kRecentLatencyFrames];
488     int mRecentHead;
489     Mutex mRecentLock;
490 
491     class Histogram {
492       public:
HistogramMediaCodec493         Histogram() : mFloor(0), mWidth(0), mBelow(0), mAbove(0),
494                       mMin(INT64_MAX), mMax(INT64_MIN), mSum(0), mCount(0),
495                       mBucketCount(0), mBuckets(NULL) {};
~HistogramMediaCodec496         ~Histogram() { clear(); };
clearMediaCodec497         void clear() { if (mBuckets != NULL) free(mBuckets); mBuckets = NULL; };
498         bool setup(int nbuckets, int64_t width, int64_t floor = 0);
499         void insert(int64_t sample);
getMinMediaCodec500         int64_t getMin() const { return mMin; }
getMaxMediaCodec501         int64_t getMax() const { return mMax; }
getCountMediaCodec502         int64_t getCount() const { return mCount; }
getSumMediaCodec503         int64_t getSum() const { return mSum; }
getAvgMediaCodec504         int64_t getAvg() const { return mSum / (mCount == 0 ? 1 : mCount); }
505         std::string emit();
506       private:
507         int64_t mFloor, mCeiling, mWidth;
508         int64_t mBelow, mAbove;
509         int64_t mMin, mMax, mSum, mCount;
510 
511         int mBucketCount;
512         int64_t *mBuckets;
513     };
514 
515     Histogram mLatencyHist;
516 
517     DISALLOW_EVIL_CONSTRUCTORS(MediaCodec);
518 };
519 
520 }  // namespace android
521 
522 #endif  // MEDIA_CODEC_H_
523