1 /*
2 * Copyright (C) 2009 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "SoftwareRenderer"
18 #include <utils/Log.h>
19
20 #include "../include/SoftwareRenderer.h"
21 #include <cutils/properties.h> // for property_get
22 #include <media/stagefright/foundation/ADebug.h>
23 #include <media/stagefright/foundation/AMessage.h>
24 #include <media/stagefright/foundation/ColorUtils.h>
25 #include <media/stagefright/SurfaceUtils.h>
26 #include <system/window.h>
27 #include <ui/Fence.h>
28 #include <ui/GraphicBufferMapper.h>
29 #include <ui/GraphicBuffer.h>
30 #include <ui/Rect.h>
31
32 namespace android {
33
initDstYUV(const android_ycbcr & ycbcr,int32_t cropTop,int32_t cropLeft,uint8_t ** dst_y,uint8_t ** dst_u,uint8_t ** dst_v)34 inline void initDstYUV(
35 const android_ycbcr &ycbcr, int32_t cropTop, int32_t cropLeft,
36 uint8_t **dst_y, uint8_t **dst_u, uint8_t **dst_v) {
37 *dst_y = (uint8_t *)ycbcr.y + cropTop * ycbcr.ystride + cropLeft;
38
39 int32_t c_offset = (cropTop / 2) * ycbcr.cstride + cropLeft / 2;
40 *dst_v = (uint8_t *)ycbcr.cr + c_offset;
41 *dst_u = (uint8_t *)ycbcr.cb + c_offset;
42 }
43
SoftwareRenderer(const sp<ANativeWindow> & nativeWindow,int32_t rotation)44 SoftwareRenderer::SoftwareRenderer(
45 const sp<ANativeWindow> &nativeWindow, int32_t rotation)
46 : mColorFormat(OMX_COLOR_FormatUnused),
47 mConverter(NULL),
48 mYUVMode(None),
49 mNativeWindow(nativeWindow),
50 mWidth(0),
51 mHeight(0),
52 mStride(0),
53 mCropLeft(0),
54 mCropTop(0),
55 mCropRight(0),
56 mCropBottom(0),
57 mCropWidth(0),
58 mCropHeight(0),
59 mRotationDegrees(rotation),
60 mDataSpace(HAL_DATASPACE_UNKNOWN) {
61 memset(&mHDRStaticInfo, 0, sizeof(mHDRStaticInfo));
62 }
63
~SoftwareRenderer()64 SoftwareRenderer::~SoftwareRenderer() {
65 delete mConverter;
66 mConverter = NULL;
67 }
68
resetFormatIfChanged(const sp<AMessage> & format,size_t numOutputBuffers)69 void SoftwareRenderer::resetFormatIfChanged(
70 const sp<AMessage> &format, size_t numOutputBuffers) {
71 CHECK(format != NULL);
72
73 int32_t colorFormatNew;
74 CHECK(format->findInt32("color-format", &colorFormatNew));
75
76 int32_t widthNew, heightNew, strideNew;
77 CHECK(format->findInt32("width", &widthNew));
78 CHECK(format->findInt32("slice-height", &heightNew));
79 CHECK(format->findInt32("stride", &strideNew));
80
81 int32_t cropLeftNew, cropTopNew, cropRightNew, cropBottomNew;
82 if (!format->findRect(
83 "crop", &cropLeftNew, &cropTopNew, &cropRightNew, &cropBottomNew)) {
84 cropLeftNew = cropTopNew = 0;
85 cropRightNew = widthNew - 1;
86 cropBottomNew = heightNew - 1;
87 }
88
89 // The native window buffer format for high-bitdepth content could
90 // depend on the dataspace also.
91 android_dataspace dataSpace;
92 bool dataSpaceChangedForPlanar16 = false;
93 if (colorFormatNew == OMX_COLOR_FormatYUV420Planar16
94 && format->findInt32("android._dataspace", (int32_t *)&dataSpace)
95 && dataSpace != mDataSpace) {
96 // Do not modify mDataSpace here, it's only modified at last
97 // when we do native_window_set_buffers_data_space().
98 dataSpaceChangedForPlanar16 = true;
99 }
100
101 if (static_cast<int32_t>(mColorFormat) == colorFormatNew &&
102 mWidth == widthNew &&
103 mHeight == heightNew &&
104 mCropLeft == cropLeftNew &&
105 mCropTop == cropTopNew &&
106 mCropRight == cropRightNew &&
107 mCropBottom == cropBottomNew &&
108 !dataSpaceChangedForPlanar16) {
109 // Nothing changed, no need to reset renderer.
110 return;
111 }
112
113 mColorFormat = static_cast<OMX_COLOR_FORMATTYPE>(colorFormatNew);
114 mWidth = widthNew;
115 mHeight = heightNew;
116 mStride = strideNew;
117 mCropLeft = cropLeftNew;
118 mCropTop = cropTopNew;
119 mCropRight = cropRightNew;
120 mCropBottom = cropBottomNew;
121
122 mCropWidth = mCropRight - mCropLeft + 1;
123 mCropHeight = mCropBottom - mCropTop + 1;
124
125 // by default convert everything to RGB565
126 int halFormat = HAL_PIXEL_FORMAT_RGB_565;
127 size_t bufWidth = mCropWidth;
128 size_t bufHeight = mCropHeight;
129
130 // hardware has YUV12 and RGBA8888 support, so convert known formats
131 {
132 switch (mColorFormat) {
133 case OMX_COLOR_FormatYUV420Planar:
134 case OMX_COLOR_FormatYUV420SemiPlanar:
135 case OMX_TI_COLOR_FormatYUV420PackedSemiPlanar:
136 {
137 halFormat = HAL_PIXEL_FORMAT_YV12;
138 bufWidth = (mCropWidth + 1) & ~1;
139 bufHeight = (mCropHeight + 1) & ~1;
140 break;
141 }
142 case OMX_COLOR_Format24bitRGB888:
143 {
144 halFormat = HAL_PIXEL_FORMAT_RGB_888;
145 bufWidth = (mCropWidth + 1) & ~1;
146 bufHeight = (mCropHeight + 1) & ~1;
147 break;
148 }
149 case OMX_COLOR_Format32bitARGB8888:
150 case OMX_COLOR_Format32BitRGBA8888:
151 {
152 halFormat = HAL_PIXEL_FORMAT_RGBA_8888;
153 bufWidth = (mCropWidth + 1) & ~1;
154 bufHeight = (mCropHeight + 1) & ~1;
155 break;
156 }
157 case OMX_COLOR_FormatYUV420Planar16:
158 {
159 if (((dataSpace & HAL_DATASPACE_STANDARD_MASK) == HAL_DATASPACE_STANDARD_BT2020)
160 && ((dataSpace & HAL_DATASPACE_TRANSFER_MASK) == HAL_DATASPACE_TRANSFER_ST2084)) {
161 // Here we would convert OMX_COLOR_FormatYUV420Planar16 into
162 // OMX_COLOR_FormatYUV444Y410, and put it inside a buffer with
163 // format HAL_PIXEL_FORMAT_RGBA_1010102. Surfaceflinger will
164 // use render engine to convert it to RGB if needed.
165 halFormat = HAL_PIXEL_FORMAT_RGBA_1010102;
166 } else {
167 halFormat = HAL_PIXEL_FORMAT_YV12;
168 }
169 bufWidth = (mCropWidth + 1) & ~1;
170 bufHeight = (mCropHeight + 1) & ~1;
171 break;
172 }
173 default:
174 {
175 break;
176 }
177 }
178 }
179
180 if (halFormat == HAL_PIXEL_FORMAT_RGB_565) {
181 mConverter = new ColorConverter(
182 mColorFormat, OMX_COLOR_Format16bitRGB565);
183 CHECK(mConverter->isValid());
184 } else if (halFormat == HAL_PIXEL_FORMAT_RGBA_1010102) {
185 mConverter = new ColorConverter(
186 mColorFormat, OMX_COLOR_FormatYUV444Y410);
187 CHECK(mConverter->isValid());
188 }
189
190 CHECK(mNativeWindow != NULL);
191 CHECK(mCropWidth > 0);
192 CHECK(mCropHeight > 0);
193 CHECK(mConverter == NULL || mConverter->isValid());
194
195 CHECK_EQ(0,
196 native_window_set_usage(
197 mNativeWindow.get(),
198 GRALLOC_USAGE_SW_READ_NEVER | GRALLOC_USAGE_SW_WRITE_RARELY
199 | GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_EXTERNAL_DISP));
200
201 CHECK_EQ(0,
202 native_window_set_scaling_mode(
203 mNativeWindow.get(),
204 NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW));
205
206 // Width must be multiple of 32???
207 CHECK_EQ(0, native_window_set_buffers_dimensions(
208 mNativeWindow.get(),
209 bufWidth,
210 bufHeight));
211 CHECK_EQ(0, native_window_set_buffers_format(
212 mNativeWindow.get(),
213 halFormat));
214 if (OK != native_window_set_buffer_count(
215 mNativeWindow.get(), numOutputBuffers + 4)) {
216 ALOGE("Failed to set native window buffer count to (%zu + 4)",
217 numOutputBuffers);
218 }
219
220 // NOTE: native window uses extended right-bottom coordinate
221 android_native_rect_t crop;
222 crop.left = mCropLeft;
223 crop.top = mCropTop;
224 crop.right = mCropRight + 1;
225 crop.bottom = mCropBottom + 1;
226 ALOGV("setting crop: [%d, %d, %d, %d] for size [%zu, %zu]",
227 crop.left, crop.top, crop.right, crop.bottom, bufWidth, bufHeight);
228
229 CHECK_EQ(0, native_window_set_crop(mNativeWindow.get(), &crop));
230
231 int32_t rotationDegrees;
232 if (!format->findInt32("rotation-degrees", &rotationDegrees)) {
233 rotationDegrees = mRotationDegrees;
234 }
235 uint32_t transform;
236 switch (rotationDegrees) {
237 case 0: transform = 0; break;
238 case 90: transform = HAL_TRANSFORM_ROT_90; break;
239 case 180: transform = HAL_TRANSFORM_ROT_180; break;
240 case 270: transform = HAL_TRANSFORM_ROT_270; break;
241 default: transform = 0; break;
242 }
243
244 CHECK_EQ(0, native_window_set_buffers_transform(
245 mNativeWindow.get(), transform));
246 }
247
clearTracker()248 void SoftwareRenderer::clearTracker() {
249 mRenderTracker.clear(-1 /* lastRenderTimeNs */);
250 }
251
render(const void * data,size_t,int64_t mediaTimeUs,nsecs_t renderTimeNs,size_t numOutputBuffers,const sp<AMessage> & format)252 std::list<FrameRenderTracker::Info> SoftwareRenderer::render(
253 const void *data, size_t , int64_t mediaTimeUs, nsecs_t renderTimeNs,
254 size_t numOutputBuffers, const sp<AMessage>& format) {
255 resetFormatIfChanged(format, numOutputBuffers);
256 FrameRenderTracker::Info *info = NULL;
257
258 ANativeWindowBuffer *buf;
259 int fenceFd = -1;
260 int err = mNativeWindow->dequeueBuffer(mNativeWindow.get(), &buf, &fenceFd);
261 if (err == 0 && fenceFd >= 0) {
262 info = mRenderTracker.updateInfoForDequeuedBuffer(buf, fenceFd, 0);
263 sp<Fence> fence = new Fence(fenceFd);
264 err = fence->waitForever("SoftwareRenderer::render");
265 }
266 if (err != 0) {
267 ALOGW("Surface::dequeueBuffer returned error %d", err);
268 // complete (drop) dequeued frame if fence wait failed; otherwise,
269 // this returns an empty list as no frames should have rendered and not yet returned.
270 return mRenderTracker.checkFencesAndGetRenderedFrames(info, false /* dropIncomplete */);
271 }
272
273 GraphicBufferMapper &mapper = GraphicBufferMapper::get();
274
275 Rect bounds(mCropWidth, mCropHeight);
276
277 void *dst = NULL;
278 struct android_ycbcr ycbcr;
279 if ( !mConverter &&
280 (mColorFormat == OMX_COLOR_FormatYUV420Planar ||
281 mColorFormat == OMX_COLOR_FormatYUV420SemiPlanar ||
282 mColorFormat == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar ||
283 mColorFormat == OMX_COLOR_FormatYUV420Planar16)) {
284 CHECK_EQ(0, mapper.lockYCbCr(buf->handle,
285 GRALLOC_USAGE_SW_READ_NEVER | GRALLOC_USAGE_SW_WRITE_RARELY,
286 bounds, &ycbcr));
287 } else {
288 CHECK_EQ(0, mapper.lock(buf->handle,
289 GRALLOC_USAGE_SW_READ_NEVER | GRALLOC_USAGE_SW_WRITE_RARELY,
290 bounds, &dst));
291 }
292
293 // TODO move the other conversions also into ColorConverter, and
294 // fix cropping issues (when mCropLeft/Top != 0 or mWidth != mCropWidth)
295 if (mConverter) {
296 mConverter->convert(
297 data,
298 mWidth, mHeight, mStride,
299 mCropLeft, mCropTop, mCropRight, mCropBottom,
300 dst,
301 buf->stride, buf->height, 0,
302 0, 0, mCropWidth - 1, mCropHeight - 1);
303 } else if (mColorFormat == OMX_COLOR_FormatYUV420Planar) {
304 const uint8_t *src_y = (const uint8_t *)data + mCropTop * mStride + mCropLeft;
305 const uint8_t *src_u = (const uint8_t *)data + mStride * mHeight + mCropTop * mStride / 4;
306 const uint8_t *src_v = (const uint8_t *)src_u + mStride * mHeight / 4;
307
308 uint8_t *dst_y, *dst_u, *dst_v;
309 initDstYUV(ycbcr, mCropTop, mCropLeft, &dst_y, &dst_u, &dst_v);
310
311 for (int y = 0; y < mCropHeight; ++y) {
312 memcpy(dst_y, src_y, mCropWidth);
313
314 src_y += mStride;
315 dst_y += ycbcr.ystride;
316 }
317
318 for (int y = 0; y < (mCropHeight + 1) / 2; ++y) {
319 memcpy(dst_u, src_u, (mCropWidth + 1) / 2);
320 memcpy(dst_v, src_v, (mCropWidth + 1) / 2);
321
322 src_u += mStride / 2;
323 src_v += mStride / 2;
324 dst_u += ycbcr.cstride;
325 dst_v += ycbcr.cstride;
326 }
327 } else if (mColorFormat == OMX_COLOR_FormatYUV420Planar16) {
328 const uint8_t *src_y = (const uint8_t *)data + mCropTop * mStride + mCropLeft * 2;
329 const uint8_t *src_u = (const uint8_t *)data + mStride * mHeight + mCropTop * mStride / 4;
330 const uint8_t *src_v = (const uint8_t *)src_u + mStride * mHeight / 4;
331
332 uint8_t *dst_y, *dst_u, *dst_v;
333 initDstYUV(ycbcr, mCropTop, mCropLeft, &dst_y, &dst_u, &dst_v);
334
335 for (int y = 0; y < mCropHeight; ++y) {
336 for (int x = 0; x < mCropWidth; ++x) {
337 dst_y[x] = (uint8_t)(((uint16_t *)src_y)[x] >> 2);
338 }
339
340 src_y += mStride;
341 dst_y += ycbcr.ystride;
342 }
343
344 for (int y = 0; y < (mCropHeight + 1) / 2; ++y) {
345 for (int x = 0; x < (mCropWidth + 1) / 2; ++x) {
346 dst_u[x] = (uint8_t)(((uint16_t *)src_u)[x] >> 2);
347 dst_v[x] = (uint8_t)(((uint16_t *)src_v)[x] >> 2);
348 }
349
350 src_u += mStride / 2;
351 src_v += mStride / 2;
352 dst_u += ycbcr.cstride;
353 dst_v += ycbcr.cstride;
354 }
355 } else if (mColorFormat == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar
356 || mColorFormat == OMX_COLOR_FormatYUV420SemiPlanar) {
357 const uint8_t *src_y = (const uint8_t *)data;
358 const uint8_t *src_uv = (const uint8_t *)data
359 + mWidth * mHeight;
360
361 src_y += mCropLeft + mCropTop * mWidth;
362 src_uv += (mCropLeft + mCropTop * mWidth) / 2;
363
364 uint8_t *dst_y, *dst_u, *dst_v;
365 initDstYUV(ycbcr, mCropTop, mCropLeft, &dst_y, &dst_u, &dst_v);
366
367 for (int y = 0; y < mCropHeight; ++y) {
368 memcpy(dst_y, src_y, mCropWidth);
369
370 src_y += mWidth;
371 dst_y += ycbcr.ystride;
372 }
373
374 for (int y = 0; y < (mCropHeight + 1) / 2; ++y) {
375 size_t tmp = (mCropWidth + 1) / 2;
376 for (size_t x = 0; x < tmp; ++x) {
377 dst_u[x] = src_uv[2 * x];
378 dst_v[x] = src_uv[2 * x + 1];
379 }
380
381 src_uv += mWidth;
382 dst_u += ycbcr.cstride;
383 dst_v += ycbcr.cstride;
384 }
385 } else if (mColorFormat == OMX_COLOR_Format24bitRGB888) {
386 uint8_t* srcPtr = (uint8_t*)data + mWidth * mCropTop * 3 + mCropLeft * 3;
387 uint8_t* dstPtr = (uint8_t*)dst + buf->stride * mCropTop * 3 + mCropLeft * 3;
388
389 for (size_t y = 0; y < (size_t)mCropHeight; ++y) {
390 memcpy(dstPtr, srcPtr, mCropWidth * 3);
391 srcPtr += mWidth * 3;
392 dstPtr += buf->stride * 3;
393 }
394 } else if (mColorFormat == OMX_COLOR_Format32bitARGB8888) {
395 uint8_t *srcPtr, *dstPtr;
396
397 for (size_t y = 0; y < (size_t)mCropHeight; ++y) {
398 srcPtr = (uint8_t*)data + mWidth * 4 * (y + mCropTop) + mCropLeft * 4;
399 dstPtr = (uint8_t*)dst + buf->stride * 4 * (y + mCropTop) + mCropLeft * 4;
400 for (size_t x = 0; x < (size_t)mCropWidth; ++x) {
401 uint8_t a = *srcPtr++;
402 for (size_t i = 0; i < 3; ++i) { // copy RGB
403 *dstPtr++ = *srcPtr++;
404 }
405 *dstPtr++ = a; // alpha last (ARGB to RGBA)
406 }
407 }
408 } else if (mColorFormat == OMX_COLOR_Format32BitRGBA8888) {
409 uint8_t* srcPtr = (uint8_t*)data + mWidth * mCropTop * 4 + mCropLeft * 4;
410 uint8_t* dstPtr = (uint8_t*)dst + buf->stride * mCropTop * 4 + mCropLeft * 4;
411
412 for (size_t y = 0; y < (size_t)mCropHeight; ++y) {
413 memcpy(dstPtr, srcPtr, mCropWidth * 4);
414 srcPtr += mWidth * 4;
415 dstPtr += buf->stride * 4;
416 }
417 } else {
418 LOG_ALWAYS_FATAL("bad color format %#x", mColorFormat);
419 }
420
421 skip_copying:
422 CHECK_EQ(0, mapper.unlock(buf->handle));
423
424 if (renderTimeNs >= 0) {
425 if ((err = native_window_set_buffers_timestamp(mNativeWindow.get(),
426 renderTimeNs)) != 0) {
427 ALOGW("Surface::set_buffers_timestamp returned error %d", err);
428 }
429 }
430
431 // TODO: propagate color aspects to software renderer to allow better
432 // color conversion to RGB. For now, just mark dataspace for YUV rendering.
433 android_dataspace dataSpace;
434 if (format->findInt32("android._dataspace", (int32_t *)&dataSpace) && dataSpace != mDataSpace) {
435 mDataSpace = dataSpace;
436
437 if (mConverter != NULL && mConverter->isDstRGB()) {
438 // graphics only supports full range RGB. ColorConverter should have
439 // converted any YUV to full range.
440 dataSpace = (android_dataspace)
441 ((dataSpace & ~HAL_DATASPACE_RANGE_MASK) | HAL_DATASPACE_RANGE_FULL);
442 }
443
444 ALOGD("setting dataspace on output surface to #%x", dataSpace);
445 if ((err = native_window_set_buffers_data_space(mNativeWindow.get(), dataSpace))) {
446 ALOGW("failed to set dataspace on surface (%d)", err);
447 }
448 }
449 if (format->contains("hdr-static-info")) {
450 HDRStaticInfo info;
451 if (ColorUtils::getHDRStaticInfoFromFormat(format, &info)
452 && memcmp(&mHDRStaticInfo, &info, sizeof(info))) {
453 setNativeWindowHdrMetadata(mNativeWindow.get(), &info);
454 mHDRStaticInfo = info;
455 }
456 }
457
458 if ((err = mNativeWindow->queueBuffer(mNativeWindow.get(), buf, -1)) != 0) {
459 ALOGW("Surface::queueBuffer returned error %d", err);
460 } else {
461 mRenderTracker.onFrameQueued(mediaTimeUs, (GraphicBuffer *)buf, Fence::NO_FENCE);
462 }
463
464 buf = NULL;
465 return mRenderTracker.checkFencesAndGetRenderedFrames(info, info != NULL /* dropIncomplete */);
466 }
467
468 } // namespace android
469