1 /*
2  * Copyright 2018, The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "Codec2BufferUtils"
19 #include <utils/Log.h>
20 
21 #include <libyuv.h>
22 
23 #include <list>
24 #include <mutex>
25 
26 #include <media/hardware/HardwareAPI.h>
27 #include <media/stagefright/foundation/AUtils.h>
28 
29 #include <C2Debug.h>
30 
31 #include "Codec2BufferUtils.h"
32 
33 namespace android {
34 
35 namespace {
36 
37 /**
38  * A flippable, optimizable memcpy. Constructs such as (from ? src : dst) do not work as the results are
39  * always const.
40  */
41 template<bool ToA, size_t S>
42 struct MemCopier {
43     template<typename A, typename B>
copyandroid::__anonc0be1d460111::MemCopier44     inline static void copy(A *a, const B *b, size_t size) {
45         __builtin_memcpy(a, b, size);
46     }
47 };
48 
49 template<size_t S>
50 struct MemCopier<false, S> {
51     template<typename A, typename B>
copyandroid::__anonc0be1d460111::MemCopier52     inline static void copy(const A *a, B *b, size_t size) {
53         MemCopier<true, S>::copy(b, a, size);
54     }
55 };
56 
57 /**
58  * Copies between a MediaImage and a graphic view.
59  *
60  * \param ToMediaImage whether to copy to (or from) the MediaImage
61  * \param view graphic view (could be ConstGraphicView or GraphicView depending on direction)
62  * \param img MediaImage data
63  * \param imgBase base of MediaImage (could be const uint8_t* or uint8_t* depending on direction)
64  */
65 template<bool ToMediaImage, typename View, typename ImagePixel>
_ImageCopy(View & view,const MediaImage2 * img,ImagePixel * imgBase)66 static status_t _ImageCopy(View &view, const MediaImage2 *img, ImagePixel *imgBase) {
67     // TODO: more efficient copying --- e.g. copy interleaved planes together, etc.
68     const C2PlanarLayout &layout = view.layout();
69     const size_t bpp = divUp(img->mBitDepthAllocated, 8u);
70 
71     for (uint32_t i = 0; i < layout.numPlanes; ++i) {
72         typename std::conditional<ToMediaImage, uint8_t, const uint8_t>::type *imgRow =
73             imgBase + img->mPlane[i].mOffset;
74         typename std::conditional<ToMediaImage, const uint8_t, uint8_t>::type *viewRow =
75             viewRow = view.data()[i];
76         const C2PlaneInfo &plane = layout.planes[i];
77         if (plane.colSampling != img->mPlane[i].mHorizSubsampling
78                 || plane.rowSampling != img->mPlane[i].mVertSubsampling
79                 || plane.allocatedDepth != img->mBitDepthAllocated
80                 || plane.allocatedDepth < plane.bitDepth
81                 // MediaImage only supports MSB values
82                 || plane.rightShift != plane.allocatedDepth - plane.bitDepth
83                 || (bpp > 1 && plane.endianness != plane.NATIVE)) {
84             return BAD_VALUE;
85         }
86 
87         uint32_t planeW = img->mWidth / plane.colSampling;
88         uint32_t planeH = img->mHeight / plane.rowSampling;
89 
90         bool canCopyByRow = (plane.colInc == 1) && (img->mPlane[i].mColInc == 1);
91         bool canCopyByPlane = canCopyByRow && (plane.rowInc == img->mPlane[i].mRowInc);
92         if (canCopyByPlane) {
93             MemCopier<ToMediaImage, 0>::copy(imgRow, viewRow, plane.rowInc * planeH);
94         } else if (canCopyByRow) {
95             for (uint32_t row = 0; row < planeH; ++row) {
96                 MemCopier<ToMediaImage, 0>::copy(
97                         imgRow, viewRow, std::min(plane.rowInc, img->mPlane[i].mRowInc));
98                 imgRow += img->mPlane[i].mRowInc;
99                 viewRow += plane.rowInc;
100             }
101         } else {
102             for (uint32_t row = 0; row < planeH; ++row) {
103                 decltype(imgRow) imgPtr = imgRow;
104                 decltype(viewRow) viewPtr = viewRow;
105                 for (uint32_t col = 0; col < planeW; ++col) {
106                     MemCopier<ToMediaImage, 0>::copy(imgPtr, viewPtr, bpp);
107                     imgPtr += img->mPlane[i].mColInc;
108                     viewPtr += plane.colInc;
109                 }
110                 imgRow += img->mPlane[i].mRowInc;
111                 viewRow += plane.rowInc;
112             }
113         }
114     }
115     return OK;
116 }
117 
118 }  // namespace
119 
ImageCopy(uint8_t * imgBase,const MediaImage2 * img,const C2GraphicView & view)120 status_t ImageCopy(uint8_t *imgBase, const MediaImage2 *img, const C2GraphicView &view) {
121     if (view.crop().width != img->mWidth || view.crop().height != img->mHeight) {
122         return BAD_VALUE;
123     }
124     if ((IsNV12(view) && IsI420(img)) || (IsI420(view) && IsNV12(img))) {
125         // Take shortcuts to use libyuv functions between NV12 and I420 conversion.
126         const uint8_t* src_y = view.data()[0];
127         const uint8_t* src_u = view.data()[1];
128         const uint8_t* src_v = view.data()[2];
129         int32_t src_stride_y = view.layout().planes[0].rowInc;
130         int32_t src_stride_u = view.layout().planes[1].rowInc;
131         int32_t src_stride_v = view.layout().planes[2].rowInc;
132         uint8_t* dst_y = imgBase + img->mPlane[0].mOffset;
133         uint8_t* dst_u = imgBase + img->mPlane[1].mOffset;
134         uint8_t* dst_v = imgBase + img->mPlane[2].mOffset;
135         int32_t dst_stride_y = img->mPlane[0].mRowInc;
136         int32_t dst_stride_u = img->mPlane[1].mRowInc;
137         int32_t dst_stride_v = img->mPlane[2].mRowInc;
138         if (IsNV12(view) && IsI420(img)) {
139             if (!libyuv::NV12ToI420(src_y, src_stride_y, src_u, src_stride_u, dst_y, dst_stride_y,
140                                     dst_u, dst_stride_u, dst_v, dst_stride_v, view.crop().width,
141                                     view.crop().height)) {
142                 return OK;
143             }
144         } else {
145             if (!libyuv::I420ToNV12(src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v,
146                                     dst_y, dst_stride_y, dst_u, dst_stride_u, view.crop().width,
147                                     view.crop().height)) {
148                 return OK;
149             }
150         }
151     }
152     return _ImageCopy<true>(view, img, imgBase);
153 }
154 
ImageCopy(C2GraphicView & view,const uint8_t * imgBase,const MediaImage2 * img)155 status_t ImageCopy(C2GraphicView &view, const uint8_t *imgBase, const MediaImage2 *img) {
156     if (view.crop().width != img->mWidth || view.crop().height != img->mHeight) {
157         return BAD_VALUE;
158     }
159     if ((IsNV12(img) && IsI420(view)) || (IsI420(img) && IsNV12(view))) {
160         // Take shortcuts to use libyuv functions between NV12 and I420 conversion.
161         const uint8_t* src_y = imgBase + img->mPlane[0].mOffset;
162         const uint8_t* src_u = imgBase + img->mPlane[1].mOffset;
163         const uint8_t* src_v = imgBase + img->mPlane[2].mOffset;
164         int32_t src_stride_y = img->mPlane[0].mRowInc;
165         int32_t src_stride_u = img->mPlane[1].mRowInc;
166         int32_t src_stride_v = img->mPlane[2].mRowInc;
167         uint8_t* dst_y = view.data()[0];
168         uint8_t* dst_u = view.data()[1];
169         uint8_t* dst_v = view.data()[2];
170         int32_t dst_stride_y = view.layout().planes[0].rowInc;
171         int32_t dst_stride_u = view.layout().planes[1].rowInc;
172         int32_t dst_stride_v = view.layout().planes[2].rowInc;
173         if (IsNV12(img) && IsI420(view)) {
174             if (!libyuv::NV12ToI420(src_y, src_stride_y, src_u, src_stride_u, dst_y, dst_stride_y,
175                                     dst_u, dst_stride_u, dst_v, dst_stride_v, view.width(),
176                                     view.height())) {
177                 return OK;
178             }
179         } else {
180             if (!libyuv::I420ToNV12(src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v,
181                                     dst_y, dst_stride_y, dst_u, dst_stride_u, view.width(),
182                                     view.height())) {
183                 return OK;
184             }
185         }
186     }
187     return _ImageCopy<false>(view, img, imgBase);
188 }
189 
IsYUV420(const C2GraphicView & view)190 bool IsYUV420(const C2GraphicView &view) {
191     const C2PlanarLayout &layout = view.layout();
192     return (layout.numPlanes == 3
193             && layout.type == C2PlanarLayout::TYPE_YUV
194             && layout.planes[layout.PLANE_Y].channel == C2PlaneInfo::CHANNEL_Y
195             && layout.planes[layout.PLANE_Y].allocatedDepth == 8
196             && layout.planes[layout.PLANE_Y].bitDepth == 8
197             && layout.planes[layout.PLANE_Y].rightShift == 0
198             && layout.planes[layout.PLANE_Y].colSampling == 1
199             && layout.planes[layout.PLANE_Y].rowSampling == 1
200             && layout.planes[layout.PLANE_U].channel == C2PlaneInfo::CHANNEL_CB
201             && layout.planes[layout.PLANE_U].allocatedDepth == 8
202             && layout.planes[layout.PLANE_U].bitDepth == 8
203             && layout.planes[layout.PLANE_U].rightShift == 0
204             && layout.planes[layout.PLANE_U].colSampling == 2
205             && layout.planes[layout.PLANE_U].rowSampling == 2
206             && layout.planes[layout.PLANE_V].channel == C2PlaneInfo::CHANNEL_CR
207             && layout.planes[layout.PLANE_V].allocatedDepth == 8
208             && layout.planes[layout.PLANE_V].bitDepth == 8
209             && layout.planes[layout.PLANE_V].rightShift == 0
210             && layout.planes[layout.PLANE_V].colSampling == 2
211             && layout.planes[layout.PLANE_V].rowSampling == 2);
212 }
213 
IsNV12(const C2GraphicView & view)214 bool IsNV12(const C2GraphicView &view) {
215     if (!IsYUV420(view)) {
216         return false;
217     }
218     const C2PlanarLayout &layout = view.layout();
219     return (layout.rootPlanes == 2
220             && layout.planes[layout.PLANE_U].colInc == 2
221             && layout.planes[layout.PLANE_U].rootIx == layout.PLANE_U
222             && layout.planes[layout.PLANE_U].offset == 0
223             && layout.planes[layout.PLANE_V].colInc == 2
224             && layout.planes[layout.PLANE_V].rootIx == layout.PLANE_U
225             && layout.planes[layout.PLANE_V].offset == 1);
226 }
227 
IsI420(const C2GraphicView & view)228 bool IsI420(const C2GraphicView &view) {
229     if (!IsYUV420(view)) {
230         return false;
231     }
232     const C2PlanarLayout &layout = view.layout();
233     return (layout.rootPlanes == 3
234             && layout.planes[layout.PLANE_U].colInc == 1
235             && layout.planes[layout.PLANE_U].rootIx == layout.PLANE_U
236             && layout.planes[layout.PLANE_U].offset == 0
237             && layout.planes[layout.PLANE_V].colInc == 1
238             && layout.planes[layout.PLANE_V].rootIx == layout.PLANE_V
239             && layout.planes[layout.PLANE_V].offset == 0);
240 }
241 
IsYUV420(const MediaImage2 * img)242 bool IsYUV420(const MediaImage2 *img) {
243     return (img->mType == MediaImage2::MEDIA_IMAGE_TYPE_YUV
244             && img->mNumPlanes == 3
245             && img->mBitDepth == 8
246             && img->mBitDepthAllocated == 8
247             && img->mPlane[0].mHorizSubsampling == 1
248             && img->mPlane[0].mVertSubsampling == 1
249             && img->mPlane[1].mHorizSubsampling == 2
250             && img->mPlane[1].mVertSubsampling == 2
251             && img->mPlane[2].mHorizSubsampling == 2
252             && img->mPlane[2].mVertSubsampling == 2);
253 }
254 
IsNV12(const MediaImage2 * img)255 bool IsNV12(const MediaImage2 *img) {
256     if (!IsYUV420(img)) {
257         return false;
258     }
259     return (img->mPlane[1].mColInc == 2
260             && img->mPlane[2].mColInc == 2
261             && (img->mPlane[2].mOffset - img->mPlane[1].mOffset == 1));
262 }
263 
IsI420(const MediaImage2 * img)264 bool IsI420(const MediaImage2 *img) {
265     if (!IsYUV420(img)) {
266         return false;
267     }
268     return (img->mPlane[1].mColInc == 1
269             && img->mPlane[2].mColInc == 1
270             && img->mPlane[2].mOffset > img->mPlane[1].mOffset);
271 }
272 
CreateYUV420PlanarMediaImage2(uint32_t width,uint32_t height,uint32_t stride,uint32_t vstride)273 MediaImage2 CreateYUV420PlanarMediaImage2(
274         uint32_t width, uint32_t height, uint32_t stride, uint32_t vstride) {
275     return MediaImage2 {
276         .mType = MediaImage2::MEDIA_IMAGE_TYPE_YUV,
277         .mNumPlanes = 3,
278         .mWidth = width,
279         .mHeight = height,
280         .mBitDepth = 8,
281         .mBitDepthAllocated = 8,
282         .mPlane = {
283             {
284                 .mOffset = 0,
285                 .mColInc = 1,
286                 .mRowInc = (int32_t)stride,
287                 .mHorizSubsampling = 1,
288                 .mVertSubsampling = 1,
289             },
290             {
291                 .mOffset = stride * vstride,
292                 .mColInc = 1,
293                 .mRowInc = (int32_t)stride / 2,
294                 .mHorizSubsampling = 2,
295                 .mVertSubsampling = 2,
296             },
297             {
298                 .mOffset = stride * vstride * 5 / 4,
299                 .mColInc = 1,
300                 .mRowInc = (int32_t)stride / 2,
301                 .mHorizSubsampling = 2,
302                 .mVertSubsampling = 2,
303             }
304         },
305     };
306 }
307 
CreateYUV420SemiPlanarMediaImage2(uint32_t width,uint32_t height,uint32_t stride,uint32_t vstride)308 MediaImage2 CreateYUV420SemiPlanarMediaImage2(
309         uint32_t width, uint32_t height, uint32_t stride, uint32_t vstride) {
310     return MediaImage2 {
311         .mType = MediaImage2::MEDIA_IMAGE_TYPE_YUV,
312         .mNumPlanes = 3,
313         .mWidth = width,
314         .mHeight = height,
315         .mBitDepth = 8,
316         .mBitDepthAllocated = 8,
317         .mPlane = {
318             {
319                 .mOffset = 0,
320                 .mColInc = 1,
321                 .mRowInc = (int32_t)stride,
322                 .mHorizSubsampling = 1,
323                 .mVertSubsampling = 1,
324             },
325             {
326                 .mOffset = stride * vstride,
327                 .mColInc = 2,
328                 .mRowInc = (int32_t)stride,
329                 .mHorizSubsampling = 2,
330                 .mVertSubsampling = 2,
331             },
332             {
333                 .mOffset = stride * vstride + 1,
334                 .mColInc = 2,
335                 .mRowInc = (int32_t)stride,
336                 .mHorizSubsampling = 2,
337                 .mVertSubsampling = 2,
338             }
339         },
340     };
341 }
342 
ConvertRGBToPlanarYUV(uint8_t * dstY,size_t dstStride,size_t dstVStride,size_t bufferSize,const C2GraphicView & src)343 status_t ConvertRGBToPlanarYUV(
344         uint8_t *dstY, size_t dstStride, size_t dstVStride, size_t bufferSize,
345         const C2GraphicView &src) {
346     CHECK(dstY != nullptr);
347     CHECK((src.width() & 1) == 0);
348     CHECK((src.height() & 1) == 0);
349 
350     if (dstStride * dstVStride * 3 / 2 > bufferSize) {
351         ALOGD("conversion buffer is too small for converting from RGB to YUV");
352         return NO_MEMORY;
353     }
354 
355     uint8_t *dstU = dstY + dstStride * dstVStride;
356     uint8_t *dstV = dstU + (dstStride >> 1) * (dstVStride >> 1);
357 
358     const C2PlanarLayout &layout = src.layout();
359     const uint8_t *pRed   = src.data()[C2PlanarLayout::PLANE_R];
360     const uint8_t *pGreen = src.data()[C2PlanarLayout::PLANE_G];
361     const uint8_t *pBlue  = src.data()[C2PlanarLayout::PLANE_B];
362 
363 #define CLIP3(x,y,z) (((z) < (x)) ? (x) : (((z) > (y)) ? (y) : (z)))
364     for (size_t y = 0; y < src.height(); ++y) {
365         for (size_t x = 0; x < src.width(); ++x) {
366             uint8_t red = *pRed;
367             uint8_t green = *pGreen;
368             uint8_t blue = *pBlue;
369 
370             // using ITU-R BT.601 conversion matrix
371             unsigned luma =
372                 CLIP3(0, (((red * 66 + green * 129 + blue * 25) >> 8) + 16), 255);
373 
374             dstY[x] = luma;
375 
376             if ((x & 1) == 0 && (y & 1) == 0) {
377                 unsigned U =
378                     CLIP3(0, (((-red * 38 - green * 74 + blue * 112) >> 8) + 128), 255);
379 
380                 unsigned V =
381                     CLIP3(0, (((red * 112 - green * 94 - blue * 18) >> 8) + 128), 255);
382 
383                 dstU[x >> 1] = U;
384                 dstV[x >> 1] = V;
385             }
386             pRed   += layout.planes[C2PlanarLayout::PLANE_R].colInc;
387             pGreen += layout.planes[C2PlanarLayout::PLANE_G].colInc;
388             pBlue  += layout.planes[C2PlanarLayout::PLANE_B].colInc;
389         }
390 
391         if ((y & 1) == 0) {
392             dstU += dstStride >> 1;
393             dstV += dstStride >> 1;
394         }
395 
396         pRed   -= layout.planes[C2PlanarLayout::PLANE_R].colInc * src.width();
397         pGreen -= layout.planes[C2PlanarLayout::PLANE_G].colInc * src.width();
398         pBlue  -= layout.planes[C2PlanarLayout::PLANE_B].colInc * src.width();
399         pRed   += layout.planes[C2PlanarLayout::PLANE_R].rowInc;
400         pGreen += layout.planes[C2PlanarLayout::PLANE_G].rowInc;
401         pBlue  += layout.planes[C2PlanarLayout::PLANE_B].rowInc;
402 
403         dstY += dstStride;
404     }
405     return OK;
406 }
407 
408 namespace {
409 
410 /**
411  * A block of raw allocated memory.
412  */
413 struct MemoryBlockPoolBlock {
MemoryBlockPoolBlockandroid::__anonc0be1d460211::MemoryBlockPoolBlock414     MemoryBlockPoolBlock(size_t size)
415         : mData(new uint8_t[size]), mSize(mData ? size : 0) { }
416 
~MemoryBlockPoolBlockandroid::__anonc0be1d460211::MemoryBlockPoolBlock417     ~MemoryBlockPoolBlock() {
418         delete[] mData;
419     }
420 
dataandroid::__anonc0be1d460211::MemoryBlockPoolBlock421     const uint8_t *data() const {
422         return mData;
423     }
424 
sizeandroid::__anonc0be1d460211::MemoryBlockPoolBlock425     size_t size() const {
426         return mSize;
427     }
428 
429     C2_DO_NOT_COPY(MemoryBlockPoolBlock);
430 
431 private:
432     uint8_t *mData;
433     size_t mSize;
434 };
435 
436 /**
437  * A simple raw memory block pool implementation.
438  */
439 struct MemoryBlockPoolImpl {
releaseandroid::__anonc0be1d460211::MemoryBlockPoolImpl440     void release(std::list<MemoryBlockPoolBlock>::const_iterator block) {
441         std::lock_guard<std::mutex> lock(mMutex);
442         // return block to free blocks if it is the current size; otherwise, discard
443         if (block->size() == mCurrentSize) {
444             mFreeBlocks.splice(mFreeBlocks.begin(), mBlocksInUse, block);
445         } else {
446             mBlocksInUse.erase(block);
447         }
448     }
449 
fetchandroid::__anonc0be1d460211::MemoryBlockPoolImpl450     std::list<MemoryBlockPoolBlock>::const_iterator fetch(size_t size) {
451         std::lock_guard<std::mutex> lock(mMutex);
452         mFreeBlocks.remove_if([size](const MemoryBlockPoolBlock &block) -> bool {
453             return block.size() != size;
454         });
455         mCurrentSize = size;
456         if (mFreeBlocks.empty()) {
457             mBlocksInUse.emplace_front(size);
458         } else {
459             mBlocksInUse.splice(mBlocksInUse.begin(), mFreeBlocks, mFreeBlocks.begin());
460         }
461         return mBlocksInUse.begin();
462     }
463 
464     MemoryBlockPoolImpl() = default;
465 
466     C2_DO_NOT_COPY(MemoryBlockPoolImpl);
467 
468 private:
469     std::mutex mMutex;
470     std::list<MemoryBlockPoolBlock> mFreeBlocks;
471     std::list<MemoryBlockPoolBlock> mBlocksInUse;
472     size_t mCurrentSize;
473 };
474 
475 } // namespace
476 
477 struct MemoryBlockPool::Impl : MemoryBlockPoolImpl {
478 };
479 
480 struct MemoryBlock::Impl {
Implandroid::MemoryBlock::Impl481     Impl(std::list<MemoryBlockPoolBlock>::const_iterator block,
482          std::shared_ptr<MemoryBlockPoolImpl> pool)
483         : mBlock(block), mPool(pool) {
484     }
485 
~Implandroid::MemoryBlock::Impl486     ~Impl() {
487         mPool->release(mBlock);
488     }
489 
dataandroid::MemoryBlock::Impl490     const uint8_t *data() const {
491         return mBlock->data();
492     }
493 
sizeandroid::MemoryBlock::Impl494     size_t size() const {
495         return mBlock->size();
496     }
497 
498 private:
499     std::list<MemoryBlockPoolBlock>::const_iterator mBlock;
500     std::shared_ptr<MemoryBlockPoolImpl> mPool;
501 };
502 
fetch(size_t size)503 MemoryBlock MemoryBlockPool::fetch(size_t size) {
504     std::list<MemoryBlockPoolBlock>::const_iterator poolBlock = mImpl->fetch(size);
505     return MemoryBlock(std::make_shared<MemoryBlock::Impl>(
506             poolBlock, std::static_pointer_cast<MemoryBlockPoolImpl>(mImpl)));
507 }
508 
MemoryBlockPool()509 MemoryBlockPool::MemoryBlockPool()
510     : mImpl(std::make_shared<MemoryBlockPool::Impl>()) {
511 }
512 
MemoryBlock(std::shared_ptr<MemoryBlock::Impl> impl)513 MemoryBlock::MemoryBlock(std::shared_ptr<MemoryBlock::Impl> impl)
514     : mImpl(impl) {
515 }
516 
517 MemoryBlock::MemoryBlock() = default;
518 
519 MemoryBlock::~MemoryBlock() = default;
520 
data() const521 const uint8_t* MemoryBlock::data() const {
522     return mImpl ? mImpl->data() : nullptr;
523 }
524 
size() const525 size_t MemoryBlock::size() const {
526     return mImpl ? mImpl->size() : 0;
527 }
528 
Allocate(size_t size)529 MemoryBlock MemoryBlock::Allocate(size_t size) {
530     return MemoryBlockPool().fetch(size);
531 }
532 
533 }  // namespace android
534