安卓Camera2用ImageReader获取NV21源码分析 原创

文章目录

• • • 以前如何得到Camera预览流回调
    • 如何使用Camera2得到预览流的回调
    • 是否可以通过ImageReader直接得到NV21回调呢
    • ImageReader数据流回调的流程源码分析

以前如何得到Camera预览流回调

• 可以通过如下方法，得到一路预览回调流 Camera#setPreviewCallbackWithBuffer(Camera.PreviewCallback)，
• 可以通过如下方法，设置回调数据的格式，比如 ImageFormat.NV21 Camera.Parameters#setPreviewForma
• 从API21开始，android.hardware.Camera 相关API 已经被谷歌废弃，
  当我们切换到Camera2相关后，如何得到一路NV21回调流呢？

如何使用Camera2得到预览流的回调

一般我们使用的方法如下，

private final ImageReader.OnImageAvailableListener mOnImageAvailableListener = reader -> {
    Image image = reader.acquireNextImage();
    // do sth with reader
    image.close();
};

mImageReader = ImageReader.newInstance(mPreviewSize.getWidth(), mPreviewSize.getHeight(),
        ImageFormat.YUV_420_888, 1);
mImageReader.setOnImageAvailableListener(mOnImageAvailableListener, mCameraHandler);
// 看起来是不是跟Camera2拍照很相似，关键在配流时需要添加如下代码
mPreviewRequestBuilder.addTarget(mImageReader.getSurface());

这样我们就可以通过 android.media.Image 读取回调的内容，内容是啥呢？

The order of planes in the array returned by Image#getPlanes() is guaranteed such that plane #0 is always Y, plane #1 is always U (Cb), and plane #2 is always V (Cr).

Image#getPlanes 返回三个数组，plane 0 是Y，plane 1 是U (Cb)，plane 2 是 V (Cr)
我们可以把他们转给NV21格式，如何转换呢？
可以参考如下答案
stackoverflow camera2-conversion-from-yuv-420-888-to-nv21

是否可以通过ImageReader直接得到NV21回调呢

上面我们设置的是YUV_420_888，是否可以直接改成NV21呢?
不行，在 ImageReader 我们可以看到如下，会直接闪退

if (format == ImageFormat.NV21) { 
    throw new IllegalArgumentException( "NV21 format is not supported")
}

本文地址 https://blog.csdn.net/CSqingchen/article/details/128941420

ImageReader数据流回调的流程源码分析

1. android.media.ImageReader 收流接口如下

/**
* Called from Native code when an Event happens. 
* This may be called from an arbitrary Binder thread, 
* so access to the *ImageReader must be synchronized appropriately.
*/
private static void postEventFromNative(Object selfRef) {
    ....
    if (executor != null && listener != null && isReaderValid) {
        executor.execute(new Runnable() {
            @Override
            public void run() {
                listener.onImageAvailable(ir);
            }
        });
    }
}

也就是这个流从 Native 回调过来的
2. android_media_ImageReader 收流代码如下

// 在 frameworks/base/media/jni/android_media_ImageReader.cpp 中部分源码如下
static struct {
    jfieldID mNativeContext;
    jmethodID postEventFromNative;
} gImageReaderClassInfo;

gImageReaderClassInfo.postEventFromNative = env->GetStaticMethodID(clazz, "postEventFromNative", "(Ljava/lang/Object;)V");

void JNIImageReaderContext::onFrameAvailable(const BufferItem& /*item*/)
{
    bool needsDetach = false;
    JNIEnv* env = getJNIEnv(&needsDetach);
    if (env != NULL) {
        env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz);
    } else {
        ALOGW("onFrameAvailable event will not posted");
    }
    if (needsDetach) {
        detachJNI();
    }
}

3. 作为一个消费者，通过 ConsumerBase 中 onFrameAvailable 回调到 android_media_ImageReader

// frameworks/native/libs/gui/ConsumerBase.cpp
void ConsumerBase::onFrameAvailable(const BufferItem& item) {
    CB_LOGV("onFrameAvailable");

    sp<FrameAvailableListener> listener;
    { // scope for the lock
        Mutex::Autolock lock(mFrameAvailableMutex);
        listener = mFrameAvailableListener.promote();
    }

    if (listener != nullptr) {
        CB_LOGV("actually calling onFrameAvailable");
        listener->onFrameAvailable(item);
    }
}

4. 获取 android.media.Image
   应用通过 ImageReader 设置 OnImageAvailableListener 回调拿到 android.media.ImageReader
   再 通过 ImageReader.acquireNextImage() 拿到 android.media.Image
   ImageReader.java 中相关代码如下

public Image acquireNextImage() {
    // Initialize with reader format, but can be overwritten by native if the image
    // format is different from the reader format.
    SurfaceImage si = new SurfaceImage(mFormat);
    int status = acquireNextSurfaceImage(si);
    return si;
}
private int acquireNextSurfaceImage(SurfaceImage si) {
    ....
    status = nativeImageSetup(si);
    ....
}

5. nativeImageSetup 实现如下

// frameworks.base\media\jni\android_media_ImageReader.cpp 中  
{"nativeImageSetup",       "(Landroid/media/Image;)I",   (void*)ImageReader_imageSetup }

static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz, jobject image){
    JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
    BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
    BufferItem* buffer = ctx->getBufferItem();
    status_t res = bufferConsumer->acquireBuffer(buffer, 0);
    ctx->returnBufferItem(buffer);
}

List<BufferItem*> mBuffers;
void JNIImageReaderContext::returnBufferItem(BufferItem* buffer) {
    buffer->mGraphicBuffer = nullptr;
    mBuffers.push_back(buffer);
}
frameworks/native/libs/gui/include/gui/BufferItem.h  

到此，就可以拿到 Image, 数据是封装在 Plane[] 中
6. Image#getPlanes() 分析

// android.media.ImageReader$SurfaceImage 实现了 Image#getPlanes() 方法
@Override
public Plane[] getPlanes() {
    throwISEIfImageIsInvalid();

    if (mPlanes == null) {
        // mNumPlanes = ImageUtils.getNumPlanesForFormat(mFormat);
        mPlanes = nativeCreatePlanes(ImageReader.this.mNumPlanes, ImageReader.this.mFormat);
    }
    // Shallow copy is fine.
    return mPlanes.clone();
}

// frameworks/base/media/jni/android_media_ImageReader.cpp 中
{"nativeCreatePlanes",      "(IIJ)[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;",  (void*)Image_createSurfacePlanes },

static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz,
        int numPlanes, int readerFormat, uint64_t ndkReaderUsage){
        
    jobjectArray surfacePlanes = env->NewObjectArray(numPlanes, gSurfacePlaneClassInfo.clazz, /*initial_element*/NULL);
    
    LockedImage lockedImg = LockedImage();    
    
    Image_getLockedImage(env, thiz, &lockedImg, ndkReaderUsage); // 把前面的 BufferItem 转为 LockedImage

    // 封装 Planes 
    for (int i = 0; i < numPlanes; i++) {
        if (!Image_getLockedImageInfo(env, &lockedImg, i, halReaderFormat,
                &pData, &dataSize, &pixelStride, &rowStride)) {
            return NULL;
        }
        byteBuffer = env->NewDirectByteBuffer(pData, dataSize);
        // Finally, create this SurfacePlane.
        jobject surfacePlane = env->NewObject(gSurfacePlaneClassInfo.clazz,
                    gSurfacePlaneClassInfo.ctor, thiz, rowStride, pixelStride, byteBuffer);
        env->SetObjectArrayElement(surfacePlanes, i, surfacePlane);
    }
 return surfacePlanes;
}

7. Image_getLockedImage 将 BufferItem 转换复制给 LockedImage，并拆分 y cb cr

static void Image_getLockedImage(JNIEnv* env, jobject thiz, LockedImage *image,
        uint64_t ndkReaderUsage) {
    .....
    BufferItem* buffer = Image_getBufferItem(env, thiz);
    status_t res = lockImageFromBuffer(buffer, lockUsage, buffer->mFence->dup(), image);
    ...
}

// lockImageFromBuffer 将 GraphicBuffer 转 LockedImage 
// frameworks/base/media/jni/android_media_Utils.cpp
status_t lockImageFromBuffer(BufferItem* bufferItem, uint32_t inUsage,
        int fenceFd, LockedImage* outputImage) {
   ...
    status_t res = lockImageFromBuffer(bufferItem->mGraphicBuffer, inUsage, bufferItem->mCrop,
            fenceFd, outputImage);
   ...
}
status_t lockImageFromBuffer(sp<GraphicBuffer> buffer, uint32_t inUsage,
        const Rect& rect, int fenceFd, LockedImage* outputImage) {
    ALOGV("%s: Try to lock the GraphicBuffer", __FUNCTION__);

    if (buffer == nullptr || outputImage == nullptr) {
        ALOGE("Input BufferItem or output LockedImage is NULL!");
        return BAD_VALUE;
    }
    if (isFormatOpaque(buffer->getPixelFormat())) {
        ALOGE("Opaque format buffer is not lockable!");
        return BAD_VALUE;
    }

    void* pData = NULL;
    android_ycbcr ycbcr = android_ycbcr();
    status_t res;
    int format = buffer->getPixelFormat();
    int flexFormat = format;

    if (isPossiblyYUV(format)) {
        res = buffer->lockAsyncYCbCr(inUsage, rect, &ycbcr, fenceFd);

        if (res != OK) {
            ALOGW("lockAsyncYCbCr failed with error %d (format = 0x%x)", res, format);
        }

        pData = ycbcr.y;
        flexFormat = HAL_PIXEL_FORMAT_YCbCr_420_888;
    }

    // lockAsyncYCbCr for YUV is unsuccessful.
    ...

    outputImage->data = reinterpret_cast<uint8_t*>(pData);
    outputImage->width = buffer->getWidth();
    outputImage->height = buffer->getHeight();
    outputImage->format = format;
    outputImage->flexFormat = flexFormat;
    outputImage->stride =
            (ycbcr.y != NULL) ? static_cast<uint32_t>(ycbcr.ystride) : buffer->getStride();
    // 对 yuv 数据拆分并赋值到 dataCb  dataCr 
    outputImage->dataCb = reinterpret_cast<uint8_t*>(ycbcr.cb);
    outputImage->dataCr = reinterpret_cast<uint8_t*>(ycbcr.cr);
    outputImage->chromaStride = static_cast<uint32_t>(ycbcr.cstride);
    outputImage->chromaStep = static_cast<uint32_t>(ycbcr.chroma_step);
    ALOGV("%s: Successfully locked the image from the GraphicBuffer", __FUNCTION__);
    // Crop, transform, scalingMode, timestamp, and frameNumber should be set by caller,
    // and cann't be set them here.
    return OK;
}

8. gSurfacePlaneClassInfo 初始化如下

static void ImageReader_classInit(JNIEnv* env, jclass clazz) {
    ....
    jclass planeClazz = env->FindClass("android/media/ImageReader$SurfaceImage$SurfacePlane");
    gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);
    gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>",
            "(Landroid/media/ImageReader$SurfaceImage;IILjava/nio/ByteBuffer;)V");
}

9. Image_getLockedImageInfo 调用 getLockedImageInfo 封装

// LockedImage 组装 
static bool Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx,
        int32_t writerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) {

    status_t res = getLockedImageInfo(buffer, idx, writerFormat, base, size,
            pixelStride, rowStride);
    if (res != OK) {
        jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
                             "Pixel format: 0x%x is unsupported", buffer->flexFormat);
        return false;
    }
    return true;
}

10. getLockedImageInfo 分析
    依据不同的需要的格式封装buffer中的数据到 *pData 也就是 *base 中

// frameworks/base/media/jni/android_media_Utils.cpp
status_t getLockedImageInfo(LockedImage* buffer, int idx,
        int32_t containerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) {

    uint32_t dataSize, ySize, cSize, cStride;
    uint32_t pStride = 0, rStride = 0;
    uint8_t *cb, *cr;
    uint8_t *pData = NULL;
    int bytesPerPixel = 0;

    dataSize = ySize = cSize = cStride = 0;
    int32_t fmt = buffer->flexFormat;

    bool usingRGBAOverride = usingRGBAToJpegOverride(fmt, containerFormat);
    fmt = applyFormatOverrides(fmt, containerFormat);
    switch (fmt) {
        case HAL_PIXEL_FORMAT_YCbCr_420_888:  //也就是  ImageFormat.YUV_420_888
            // Width and height should be multiple of 2. Wrong dataSize would be returned otherwise.
            if (buffer->width % 2 != 0) {
                ALOGE("YCbCr_420_888: width (%d) should be a multiple of 2", buffer->width);
                return BAD_VALUE;
            }

            if (buffer->height % 2 != 0) {
                ALOGE("YCbCr_420_888: height (%d) should be a multiple of 2", buffer->height);
                return BAD_VALUE;
            }

            if (buffer->width <= 0) {
                ALOGE("YCbCr_420_888: width (%d) should be a > 0", buffer->width);
                return BAD_VALUE;
            }

            if (buffer->height <= 0) {
                ALOGE("YCbCr_420_888: height (%d) should be a > 0", buffer->height);
                return BAD_VALUE;
            }
			// panles 中 每个数组对应的buffer
            pData =
                (idx == 0) ?
                    buffer->data :
                (idx == 1) ?
                    buffer->dataCb :
                buffer->dataCr;
                
            // only map until last pixel
            if (idx == 0) {
                pStride = 1;
                rStride = buffer->stride;
                dataSize = buffer->stride * (buffer->height - 1) + buffer->width;
            } else {
                pStride = buffer->chromaStep;
                rStride = buffer->chromaStride;
                dataSize = buffer->chromaStride * (buffer->height / 2 - 1) +
                        buffer->chromaStep * (buffer->width / 2 - 1) + 1;
            }
            break;
       
        case HAL_PIXEL_FORMAT_YCrCb_420_SP:     // 也就是 ImageFormat.NV21，
            // Width and height should be multiple of 2. Wrong dataSize would be returned otherwise.
            if (buffer->width % 2 != 0) {
                ALOGE("YCrCb_420_SP: width (%d) should be a multiple of 2", buffer->width);
                return BAD_VALUE;
            }

            if (buffer->height % 2 != 0) {
                ALOGE("YCrCb_420_SP: height (%d) should be a multiple of 2", buffer->height);
                return BAD_VALUE;
            }

            if (buffer->width <= 0) {
                ALOGE("YCrCb_420_SP: width (%d) should be a > 0", buffer->width);
                return BAD_VALUE;
            }

            if (buffer->height <= 0) {
                ALOGE("YCrCb_420_SP: height (%d) should be a > 0", buffer->height);
                return BAD_VALUE;
            }

            cr = buffer->data + (buffer->stride * buffer->height);
            cb = cr + 1;
            // only map until last pixel
            ySize = buffer->width * (buffer->height - 1) + buffer->width;
            cSize = buffer->width * (buffer->height / 2 - 1) + buffer->width - 1;

            pData =
                (idx == 0) ?
                    buffer->data :
                (idx == 1) ?
                    cb:
                cr;

            dataSize = (idx == 0) ? ySize : cSize;
            pStride = (idx == 0) ? 1 : 2;
            rStride = buffer->width;
            break;
        // 这里还有很多其它格式，详见源码
            ...
        default:
            ALOGV("%s: unrecognized format 0x%x", __FUNCTION__, fmt);
            return BAD_VALUE;
    }

    *base = pData;
    *size = dataSize;
    *pixelStride = pStride;
    *rowStride = rStride;

    return OK;
}

本文地址 https://blog.csdn.net/CSqingchen/article/details/128941420