/* Copyright 2017 The Chromium OS Authors. All rights reserved. * Use of this source code is governed by a BSD-style license that can be * found in the LICENSE file. */ #include "arc/image_processor.h" #include #include #include #include #include "arc/common.h" #include "arc/exif_utils.h" #include "arc/jpeg_compressor.h" namespace arc { using android::CameraMetadata; /* * Formats have different names in different header files. Here is the mapping * table: * * android_pixel_format_t videodev2.h FOURCC in libyuv * ----------------------------------------------------------------------------- * HAL_PIXEL_FORMAT_YV12 = V4L2_PIX_FMT_YVU420 = FOURCC_YV12 * HAL_PIXEL_FORMAT_YCrCb_420_SP = V4L2_PIX_FMT_NV21 = FOURCC_NV21 * HAL_PIXEL_FORMAT_RGBA_8888 = V4L2_PIX_FMT_RGB32 = FOURCC_BGR4 * HAL_PIXEL_FORMAT_YCbCr_422_I = V4L2_PIX_FMT_YUYV = FOURCC_YUYV * = FOURCC_YUY2 * V4L2_PIX_FMT_YUV420 = FOURCC_I420 * = FOURCC_YU12 * V4L2_PIX_FMT_MJPEG = FOURCC_MJPG * * Camera device generates FOURCC_YUYV and FOURCC_MJPG. * Preview needs FOURCC_ARGB format. * Software video encoder needs FOURCC_YU12. * CTS requires FOURCC_YV12 and FOURCC_NV21 for applications. * * Android stride requirement: * YV12 horizontal stride should be a multiple of 16 pixels. See * android.graphics.ImageFormat.YV12. * The stride of ARGB, YU12, and NV21 are always equal to the width. * * Conversion Path: * MJPG/YUYV (from camera) -> YU12 -> ARGB (preview) * -> NV21 (apps) * -> YV12 (apps) * -> YU12 (video encoder) */ // YV12 horizontal stride should be a multiple of 16 pixels for each plane. // |dst_stride_uv| is the pixel stride of u or v plane. static int YU12ToYV12(const void* yv12, void* yu12, int width, int height, int dst_stride_y, int dst_stride_uv); static int YU12ToNV21(const void* yv12, void* nv21, int width, int height); static bool ConvertToJpeg(const CameraMetadata& metadata, const FrameBuffer& in_frame, FrameBuffer* out_frame); static bool SetExifTags(const CameraMetadata& metadata, ExifUtils* utils); // How precise the float-to-rational conversion for EXIF tags would be. static const int kRationalPrecision = 10000; // Default JPEG quality settings. static const int DEFAULT_JPEG_QUALITY = 80; inline static size_t Align16(size_t value) { return (value + 15) & ~15; } size_t ImageProcessor::GetConvertedSize(int fourcc, uint32_t width, uint32_t height) { if ((width % 2) || (height % 2)) { LOGF(ERROR) << "Width or height is not even (" << width << " x " << height << ")"; return 0; } switch (fourcc) { case V4L2_PIX_FMT_YVU420: // YV12 return Align16(width) * height + Align16(width / 2) * height; case V4L2_PIX_FMT_YUV420: // YU12 // Fall-through. case V4L2_PIX_FMT_NV21: // NV21 return width * height * 3 / 2; case V4L2_PIX_FMT_BGR32: case V4L2_PIX_FMT_RGB32: return width * height * 4; case V4L2_PIX_FMT_JPEG: return 0; // For JPEG real size will be calculated after conversion. default: LOGF(ERROR) << "Pixel format " << FormatToString(fourcc) << " is unsupported."; return 0; } } bool ImageProcessor::SupportsConversion(uint32_t from_fourcc, uint32_t to_fourcc) { switch (from_fourcc) { case V4L2_PIX_FMT_YUYV: return (to_fourcc == V4L2_PIX_FMT_YUV420); case V4L2_PIX_FMT_YUV420: return ( to_fourcc == V4L2_PIX_FMT_YUV420 || to_fourcc == V4L2_PIX_FMT_YVU420 || to_fourcc == V4L2_PIX_FMT_NV21 || to_fourcc == V4L2_PIX_FMT_RGB32 || to_fourcc == V4L2_PIX_FMT_BGR32 || to_fourcc == V4L2_PIX_FMT_JPEG); case V4L2_PIX_FMT_MJPEG: return (to_fourcc == V4L2_PIX_FMT_YUV420); default: return false; } } int ImageProcessor::ConvertFormat(const CameraMetadata& metadata, const FrameBuffer& in_frame, FrameBuffer* out_frame) { if ((in_frame.GetWidth() % 2) || (in_frame.GetHeight() % 2)) { LOGF(ERROR) << "Width or height is not even (" << in_frame.GetWidth() << " x " << in_frame.GetHeight() << ")"; return -EINVAL; } size_t data_size = GetConvertedSize( out_frame->GetFourcc(), in_frame.GetWidth(), in_frame.GetHeight()); if (out_frame->SetDataSize(data_size)) { LOGF(ERROR) << "Set data size failed"; return -EINVAL; } if (in_frame.GetFourcc() == V4L2_PIX_FMT_YUYV) { switch (out_frame->GetFourcc()) { case V4L2_PIX_FMT_YUV420: // YU12 { int res = libyuv::YUY2ToI420( in_frame.GetData(), /* src_yuy2 */ in_frame.GetWidth() * 2, /* src_stride_yuy2 */ out_frame->GetData(), /* dst_y */ out_frame->GetWidth(), /* dst_stride_y */ out_frame->GetData() + out_frame->GetWidth() * out_frame->GetHeight(), /* dst_u */ out_frame->GetWidth() / 2, /* dst_stride_u */ out_frame->GetData() + out_frame->GetWidth() * out_frame->GetHeight() * 5 / 4, /* dst_v */ out_frame->GetWidth() / 2, /* dst_stride_v */ in_frame.GetWidth(), in_frame.GetHeight()); LOGF_IF(ERROR, res) << "YUY2ToI420() for YU12 returns " << res; return res ? -EINVAL : 0; } default: LOGF(ERROR) << "Destination pixel format " << FormatToString(out_frame->GetFourcc()) << " is unsupported for YUYV source format."; return -EINVAL; } } else if (in_frame.GetFourcc() == V4L2_PIX_FMT_YUV420) { // V4L2_PIX_FMT_YVU420 is YV12. I420 is usually referred to YU12 // (V4L2_PIX_FMT_YUV420), and YV12 is similar to YU12 except that U/V // planes are swapped. switch (out_frame->GetFourcc()) { case V4L2_PIX_FMT_YVU420: // YV12 { int ystride = Align16(in_frame.GetWidth()); int uvstride = Align16(in_frame.GetWidth() / 2); int res = YU12ToYV12(in_frame.GetData(), out_frame->GetData(), in_frame.GetWidth(), in_frame.GetHeight(), ystride, uvstride); LOGF_IF(ERROR, res) << "YU12ToYV12() returns " << res; return res ? -EINVAL : 0; } case V4L2_PIX_FMT_YUV420: // YU12 { memcpy(out_frame->GetData(), in_frame.GetData(), in_frame.GetDataSize()); return 0; } case V4L2_PIX_FMT_NV21: // NV21 { // TODO(henryhsu): Use libyuv::I420ToNV21. int res = YU12ToNV21(in_frame.GetData(), out_frame->GetData(), in_frame.GetWidth(), in_frame.GetHeight()); LOGF_IF(ERROR, res) << "YU12ToNV21() returns " << res; return res ? -EINVAL : 0; } case V4L2_PIX_FMT_BGR32: { int res = libyuv::I420ToABGR( in_frame.GetData(), /* src_y */ in_frame.GetWidth(), /* src_stride_y */ in_frame.GetData() + in_frame.GetWidth() * in_frame.GetHeight(), /* src_u */ in_frame.GetWidth() / 2, /* src_stride_u */ in_frame.GetData() + in_frame.GetWidth() * in_frame.GetHeight() * 5 / 4, /* src_v */ in_frame.GetWidth() / 2, /* src_stride_v */ out_frame->GetData(), /* dst_abgr */ out_frame->GetWidth() * 4, /* dst_stride_abgr */ in_frame.GetWidth(), in_frame.GetHeight()); LOGF_IF(ERROR, res) << "I420ToABGR() returns " << res; return res ? -EINVAL : 0; } case V4L2_PIX_FMT_RGB32: { int res = libyuv::I420ToARGB( in_frame.GetData(), /* src_y */ in_frame.GetWidth(), /* src_stride_y */ in_frame.GetData() + in_frame.GetWidth() * in_frame.GetHeight(), /* src_u */ in_frame.GetWidth() / 2, /* src_stride_u */ in_frame.GetData() + in_frame.GetWidth() * in_frame.GetHeight() * 5 / 4, /* src_v */ in_frame.GetWidth() / 2, /* src_stride_v */ out_frame->GetData(), /* dst_argb */ out_frame->GetWidth() * 4, /* dst_stride_argb */ in_frame.GetWidth(), in_frame.GetHeight()); LOGF_IF(ERROR, res) << "I420ToARGB() returns " << res; return res ? -EINVAL : 0; } case V4L2_PIX_FMT_JPEG: { bool res = ConvertToJpeg(metadata, in_frame, out_frame); LOGF_IF(ERROR, !res) << "ConvertToJpeg() returns " << res; return res ? -EINVAL : 0; } default: LOGF(ERROR) << "Destination pixel format " << FormatToString(out_frame->GetFourcc()) << " is unsupported for YU12 source format."; return -EINVAL; } } else if (in_frame.GetFourcc() == V4L2_PIX_FMT_MJPEG) { switch (out_frame->GetFourcc()) { case V4L2_PIX_FMT_YUV420: // YU12 { int res = libyuv::MJPGToI420( in_frame.GetData(), /* sample */ in_frame.GetDataSize(), /* sample_size */ out_frame->GetData(), /* dst_y */ out_frame->GetWidth(), /* dst_stride_y */ out_frame->GetData() + out_frame->GetWidth() * out_frame->GetHeight(), /* dst_u */ out_frame->GetWidth() / 2, /* dst_stride_u */ out_frame->GetData() + out_frame->GetWidth() * out_frame->GetHeight() * 5 / 4, /* dst_v */ out_frame->GetWidth() / 2, /* dst_stride_v */ in_frame.GetWidth(), in_frame.GetHeight(), out_frame->GetWidth(), out_frame->GetHeight()); LOGF_IF(ERROR, res) << "MJPEGToI420() returns " << res; return res ? -EINVAL : 0; } default: LOGF(ERROR) << "Destination pixel format " << FormatToString(out_frame->GetFourcc()) << " is unsupported for MJPEG source format."; return -EINVAL; } } else { LOGF(ERROR) << "Convert format doesn't support source format " << FormatToString(in_frame.GetFourcc()); return -EINVAL; } } int ImageProcessor::Scale(const FrameBuffer& in_frame, FrameBuffer* out_frame) { if (in_frame.GetFourcc() != V4L2_PIX_FMT_YUV420) { LOGF(ERROR) << "Pixel format " << FormatToString(in_frame.GetFourcc()) << " is unsupported."; return -EINVAL; } size_t data_size = GetConvertedSize( in_frame.GetFourcc(), out_frame->GetWidth(), out_frame->GetHeight()); if (out_frame->SetDataSize(data_size)) { LOGF(ERROR) << "Set data size failed"; return -EINVAL; } out_frame->SetFourcc(in_frame.GetFourcc()); VLOGF(1) << "Scale image from " << in_frame.GetWidth() << "x" << in_frame.GetHeight() << " to " << out_frame->GetWidth() << "x" << out_frame->GetHeight(); int ret = libyuv::I420Scale( in_frame.GetData(), in_frame.GetWidth(), in_frame.GetData() + in_frame.GetWidth() * in_frame.GetHeight(), in_frame.GetWidth() / 2, in_frame.GetData() + in_frame.GetWidth() * in_frame.GetHeight() * 5 / 4, in_frame.GetWidth() / 2, in_frame.GetWidth(), in_frame.GetHeight(), out_frame->GetData(), out_frame->GetWidth(), out_frame->GetData() + out_frame->GetWidth() * out_frame->GetHeight(), out_frame->GetWidth() / 2, out_frame->GetData() + out_frame->GetWidth() * out_frame->GetHeight() * 5 / 4, out_frame->GetWidth() / 2, out_frame->GetWidth(), out_frame->GetHeight(), libyuv::FilterMode::kFilterNone); LOGF_IF(ERROR, ret) << "I420Scale failed: " << ret; return ret; } static int YU12ToYV12(const void* yu12, void* yv12, int width, int height, int dst_stride_y, int dst_stride_uv) { if ((width % 2) || (height % 2)) { LOGF(ERROR) << "Width or height is not even (" << width << " x " << height << ")"; return -EINVAL; } if (dst_stride_y < width || dst_stride_uv < width / 2) { LOGF(ERROR) << "Y plane stride (" << dst_stride_y << ") or U/V plane stride (" << dst_stride_uv << ") is invalid for width " << width; return -EINVAL; } const uint8_t* src = reinterpret_cast(yu12); uint8_t* dst = reinterpret_cast(yv12); const uint8_t* u_src = src + width * height; uint8_t* u_dst = dst + dst_stride_y * height + dst_stride_uv * height / 2; const uint8_t* v_src = src + width * height * 5 / 4; uint8_t* v_dst = dst + dst_stride_y * height; return libyuv::I420Copy(src, width, u_src, width / 2, v_src, width / 2, dst, dst_stride_y, u_dst, dst_stride_uv, v_dst, dst_stride_uv, width, height); } static int YU12ToNV21(const void* yu12, void* nv21, int width, int height) { if ((width % 2) || (height % 2)) { LOGF(ERROR) << "Width or height is not even (" << width << " x " << height << ")"; return -EINVAL; } const uint8_t* src = reinterpret_cast(yu12); uint8_t* dst = reinterpret_cast(nv21); const uint8_t* u_src = src + width * height; const uint8_t* v_src = src + width * height * 5 / 4; uint8_t* vu_dst = dst + width * height; memcpy(dst, src, width * height); for (int i = 0; i < height / 2; i++) { for (int j = 0; j < width / 2; j++) { *vu_dst++ = *v_src++; *vu_dst++ = *u_src++; } } return 0; } static bool ConvertToJpeg(const CameraMetadata& metadata, const FrameBuffer& in_frame, FrameBuffer* out_frame) { ExifUtils utils; int jpeg_quality, thumbnail_jpeg_quality; camera_metadata_ro_entry entry; if (metadata.exists(ANDROID_JPEG_QUALITY)) { entry = metadata.find(ANDROID_JPEG_QUALITY); jpeg_quality = entry.data.u8[0]; } else { LOGF(ERROR) << "Could not find jpeg quality in metadata, defaulting to " << DEFAULT_JPEG_QUALITY; jpeg_quality = DEFAULT_JPEG_QUALITY; } if (metadata.exists(ANDROID_JPEG_THUMBNAIL_QUALITY)) { entry = metadata.find(ANDROID_JPEG_THUMBNAIL_QUALITY); thumbnail_jpeg_quality = entry.data.u8[0]; } else { thumbnail_jpeg_quality = jpeg_quality; } if (!utils.Initialize(in_frame.GetData(), in_frame.GetWidth(), in_frame.GetHeight(), thumbnail_jpeg_quality)) { LOGF(ERROR) << "ExifUtils initialization failed."; return false; } if (!SetExifTags(metadata, &utils)) { LOGF(ERROR) << "Setting Exif tags failed."; return false; } if (!utils.GenerateApp1()) { LOGF(ERROR) << "Generating APP1 segment failed."; return false; } JpegCompressor compressor; if (!compressor.CompressImage(in_frame.GetData(), in_frame.GetWidth(), in_frame.GetHeight(), jpeg_quality, utils.GetApp1Buffer(), utils.GetApp1Length())) { LOGF(ERROR) << "JPEG image compression failed"; return false; } size_t buffer_length = compressor.GetCompressedImageSize(); if (out_frame->SetDataSize(buffer_length)) { return false; } memcpy(out_frame->GetData(), compressor.GetCompressedImagePtr(), buffer_length); return true; } static bool SetExifTags(const CameraMetadata& metadata, ExifUtils* utils) { time_t raw_time = 0; struct tm time_info; bool time_available = time(&raw_time) != -1; localtime_r(&raw_time, &time_info); if (!utils->SetDateTime(time_info)) { LOGF(ERROR) << "Setting data time failed."; return false; } float focal_length; camera_metadata_ro_entry entry = metadata.find(ANDROID_LENS_FOCAL_LENGTH); if (entry.count) { focal_length = entry.data.f[0]; } else { LOGF(ERROR) << "Cannot find focal length in metadata."; return false; } if (!utils->SetFocalLength( static_cast(focal_length * kRationalPrecision), kRationalPrecision)) { LOGF(ERROR) << "Setting focal length failed."; return false; } if (metadata.exists(ANDROID_JPEG_GPS_COORDINATES)) { entry = metadata.find(ANDROID_JPEG_GPS_COORDINATES); if (entry.count < 3) { LOGF(ERROR) << "Gps coordinates in metadata is not complete."; return false; } if (!utils->SetGpsLatitude(entry.data.d[0])) { LOGF(ERROR) << "Setting gps latitude failed."; return false; } if (!utils->SetGpsLongitude(entry.data.d[1])) { LOGF(ERROR) << "Setting gps longitude failed."; return false; } if (!utils->SetGpsAltitude(entry.data.d[2])) { LOGF(ERROR) << "Setting gps altitude failed."; return false; } } if (metadata.exists(ANDROID_JPEG_GPS_PROCESSING_METHOD)) { entry = metadata.find(ANDROID_JPEG_GPS_PROCESSING_METHOD); std::string method_str(reinterpret_cast(entry.data.u8)); if (!utils->SetGpsProcessingMethod(method_str)) { LOGF(ERROR) << "Setting gps processing method failed."; return false; } } if (time_available && metadata.exists(ANDROID_JPEG_GPS_TIMESTAMP)) { entry = metadata.find(ANDROID_JPEG_GPS_TIMESTAMP); time_t timestamp = static_cast(entry.data.i64[0]); if (gmtime_r(×tamp, &time_info)) { if (!utils->SetGpsTimestamp(time_info)) { LOGF(ERROR) << "Setting gps timestamp failed."; return false; } } else { LOGF(ERROR) << "Time tranformation failed."; return false; } } if (metadata.exists(ANDROID_JPEG_ORIENTATION)) { entry = metadata.find(ANDROID_JPEG_ORIENTATION); if (!utils->SetOrientation(entry.data.i32[0])) { LOGF(ERROR) << "Setting orientation failed."; return false; } } if (metadata.exists(ANDROID_JPEG_THUMBNAIL_SIZE)) { entry = metadata.find(ANDROID_JPEG_THUMBNAIL_SIZE); if (entry.count < 2) { LOGF(ERROR) << "Thumbnail size in metadata is not complete."; return false; } int thumbnail_width = entry.data.i32[0]; int thumbnail_height = entry.data.i32[1]; if (thumbnail_width > 0 && thumbnail_height > 0) { if (!utils->SetThumbnailSize(static_cast(thumbnail_width), static_cast(thumbnail_height))) { LOGF(ERROR) << "Setting thumbnail size failed."; return false; } } } return true; } } // namespace arc